arm64: vdso: Avoid ISB after reading from cntvct_el0

commit 77ec462536a13d4b428a1eead725c4818a49f0b1 upstream.

We can avoid the expensive ISB instruction after reading the counter in
the vDSO gettime functions by creating a fake address hazard against a
dummy stack read, just like we do inside the kernel.

Signed-off-by: Will Deacon <will@kernel.org>
Reviewed-by: Vincenzo Frascino <vincenzo.frascino@arm.com>
Link: https://lore.kernel.org/r/20210318170738.7756-5-will@kernel.org
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Signed-off-by: Chanho Park <chanho61.park@samsung.com>
This commit is contained in:
Will Deacon 2021-03-18 17:07:37 +00:00 committed by Greg Kroah-Hartman
parent 90e498ef3f
commit 07fd256d53
3 changed files with 20 additions and 26 deletions
arch/arm64/include/asm

View File

@ -165,25 +165,6 @@ static inline void arch_timer_set_cntkctl(u32 cntkctl)
isb(); isb();
} }
/*
* Ensure that reads of the counter are treated the same as memory reads
* for the purposes of ordering by subsequent memory barriers.
*
* This insanity brought to you by speculative system register reads,
* out-of-order memory accesses, sequence locks and Thomas Gleixner.
*
* http://lists.infradead.org/pipermail/linux-arm-kernel/2019-February/631195.html
*/
#define arch_counter_enforce_ordering(val) do { \
u64 tmp, _val = (val); \
\
asm volatile( \
" eor %0, %1, %1\n" \
" add %0, sp, %0\n" \
" ldr xzr, [%0]" \
: "=r" (tmp) : "r" (_val)); \
} while (0)
static __always_inline u64 __arch_counter_get_cntpct_stable(void) static __always_inline u64 __arch_counter_get_cntpct_stable(void)
{ {
u64 cnt; u64 cnt;
@ -224,8 +205,6 @@ static __always_inline u64 __arch_counter_get_cntvct(void)
return cnt; return cnt;
} }
#undef arch_counter_enforce_ordering
static inline int arch_timer_arch_init(void) static inline int arch_timer_arch_init(void)
{ {
return 0; return 0;

View File

@ -57,6 +57,25 @@ static inline unsigned long array_index_mask_nospec(unsigned long idx,
return mask; return mask;
} }
/*
* Ensure that reads of the counter are treated the same as memory reads
* for the purposes of ordering by subsequent memory barriers.
*
* This insanity brought to you by speculative system register reads,
* out-of-order memory accesses, sequence locks and Thomas Gleixner.
*
* http://lists.infradead.org/pipermail/linux-arm-kernel/2019-February/631195.html
*/
#define arch_counter_enforce_ordering(val) do { \
u64 tmp, _val = (val); \
\
asm volatile( \
" eor %0, %1, %1\n" \
" add %0, sp, %0\n" \
" ldr xzr, [%0]" \
: "=r" (tmp) : "r" (_val)); \
} while (0)
#define __smp_mb() dmb(ish) #define __smp_mb() dmb(ish)
#define __smp_rmb() dmb(ishld) #define __smp_rmb() dmb(ishld)
#define __smp_wmb() dmb(ishst) #define __smp_wmb() dmb(ishst)

View File

@ -85,11 +85,7 @@ static __always_inline u64 __arch_get_hw_counter(s32 clock_mode)
*/ */
isb(); isb();
asm volatile("mrs %0, cntvct_el0" : "=r" (res) :: "memory"); asm volatile("mrs %0, cntvct_el0" : "=r" (res) :: "memory");
/* arch_counter_enforce_ordering(res);
* This isb() is required to prevent that the seq lock is
* speculated.#
*/
isb();
return res; return res;
} }