From bb2eb3bf0dfe380bf4fe6807f081ddd46c1f7b70 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Mon, 27 May 2019 16:58:38 +0800 Subject: [PATCH] arm64: arch_timer: Ensure counter register reads occur with seqlock held mainline inclusion from mainline-5.2 commit 75a19a0202db21638a1c2b424afb867e1f9a2376 category: bugfix bugzilla: NA CVE: NA ------------------------------------------------- When executing clock_gettime(), either in the vDSO or via a system call, we need to ensure that the read of the counter register occurs within the seqlock reader critical section. This ensures that updates to the clocksource parameters (e.g. the multiplier) are consistent with the counter value and therefore avoids the situation where time appears to go backwards across multiple reads. Extend the vDSO logic so that the seqlock critical section covers the read of the counter register as well as accesses to the data page. Since reads of the counter system registers are not ordered by memory barrier instructions, introduce dependency ordering from the counter read to a subsequent memory access so that the seqlock memory barriers apply to the counter access in both the vDSO and the system call paths. Cc: Cc: Marc Zyngier Tested-by: Vincenzo Frascino Link: https://lore.kernel.org/linux-arm-kernel/alpine.DEB.2.21.1902081950260.1662@nanos.tec.linutronix.de/ Reported-by: Thomas Gleixner Signed-off-by: Will Deacon Signed-off-by: Xuefeng Wang Reviewed-by: Yang Yingliang Signed-off-by: Yang Yingliang --- arch/arm64/include/asm/arch_timer.h | 33 +++++++++++++++++-- arch/arm64/kernel/vdso/gettimeofday.c | 47 ++++++++++++++++++++------- 2 files changed, 66 insertions(+), 14 deletions(-) diff --git a/arch/arm64/include/asm/arch_timer.h b/arch/arm64/include/asm/arch_timer.h index f2a234d6516c..93e07512b4b6 100644 --- a/arch/arm64/include/asm/arch_timer.h +++ b/arch/arm64/include/asm/arch_timer.h @@ -148,18 +148,47 @@ static inline void arch_timer_set_cntkctl(u32 cntkctl) isb(); } +/* + * Ensure that reads of the counter are treated the same as memory reads + * for the purposes of ordering by subsequent memory barriers. + * + * This insanity brought to you by speculative system register reads, + * out-of-order memory accesses, sequence locks and Thomas Gleixner. + * + * http://lists.infradead.org/pipermail/linux-arm-kernel/2019-February/631195.html + */ +#define arch_counter_enforce_ordering(val) do { \ + u64 tmp, _val = (val); \ + \ + asm volatile( \ + " eor %0, %1, %1\n" \ + " add %0, sp, %0\n" \ + " ldr xzr, [%0]" \ + : "=r" (tmp) : "r" (_val)); \ +} while (0) + static inline u64 arch_counter_get_cntpct(void) { + u64 cnt; + isb(); - return arch_timer_reg_read_stable(cntpct_el0); + cnt = arch_timer_reg_read_stable(cntpct_el0); + arch_counter_enforce_ordering(cnt); + return cnt; } static inline u64 arch_counter_get_cntvct(void) { + u64 cnt; + isb(); - return arch_timer_reg_read_stable(cntvct_el0); + cnt = arch_timer_reg_read_stable(cntvct_el0); + arch_counter_enforce_ordering(cnt); + return cnt; } +#undef arch_counter_enforce_ordering + static inline int arch_timer_arch_init(void) { return 0; diff --git a/arch/arm64/kernel/vdso/gettimeofday.c b/arch/arm64/kernel/vdso/gettimeofday.c index 12937865bd95..53d40c0d92d0 100644 --- a/arch/arm64/kernel/vdso/gettimeofday.c +++ b/arch/arm64/kernel/vdso/gettimeofday.c @@ -81,7 +81,7 @@ static notrace int clock_getres_fallback(clockid_t _clkid, return ret; } -static notrace u32 vdso_read_begin(const struct vdso_data *vd) +static notrace u32 vdso_read_begin(struct vdso_data *vd) { u32 seq; @@ -98,7 +98,7 @@ static notrace u32 vdso_read_begin(const struct vdso_data *vd) return seq; } -static notrace u32 vdso_read_retry(const struct vdso_data *vd, u32 start) +static notrace u32 vdso_read_retry(struct vdso_data *vd, u32 start) { u32 seq; @@ -123,12 +123,32 @@ static notrace u64 get_clock_shifted_nsec(u64 cycle_last, u64 mult) res = res - cycle_last; /* We can only guarantee 56 bits of precision. */ res &= ~(0xff00ull<<48); + return res * mult; } +/* + * Fake address dependency from the value computed from the counter + * register to subsequent data page accesses so that the sequence + * locking also orders the read of the counter. + */ +static notrace struct vdso_data *arch_counter_vdso_data_ordering(struct vdso_data *vd, u64 res) +{ + struct vdso_data *vd_res = vd; + u64 tmp; + + asm volatile( + " and %0, %1, xzr\n" \ + " add %2, %2, %0\n" \ + : "=r" (tmp) \ + : "r"(res), "r"(vd_res)); + + return vd_res; +} /* Code size doesn't matter (vdso is 4k/16k/64k anyway) and this is faster. */ -static __always_inline notrace int do_realtime(const struct vdso_data *vd, + +static __always_inline notrace int do_realtime(struct vdso_data *vd, struct timespec *ts) { u32 seq, cs_mono_mult, cs_shift; @@ -148,9 +168,10 @@ static __always_inline notrace int do_realtime(const struct vdso_data *vd, sec = vd->xtime_clock_sec; ns = vd->xtime_clock_nsec; + ns += get_clock_shifted_nsec(cycle_last, cs_mono_mult); + vd = arch_counter_vdso_data_ordering(vd, ns); } while (unlikely(vdso_read_retry(vd, seq))); - ns += get_clock_shifted_nsec(cycle_last, cs_mono_mult); ns >>= cs_shift; ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); ts->tv_nsec = ns; @@ -158,7 +179,7 @@ static __always_inline notrace int do_realtime(const struct vdso_data *vd, return 0; } -static notrace int do_monotonic(const struct vdso_data *vd, +static notrace int do_monotonic(struct vdso_data *vd, struct timespec *ts) { u32 seq, cs_mono_mult, cs_shift; @@ -181,9 +202,10 @@ static notrace int do_monotonic(const struct vdso_data *vd, sec += vd->wtm_clock_sec; ns += vd->wtm_clock_nsec << cs_shift; + ns += get_clock_shifted_nsec(cycle_last, cs_mono_mult); + vd = arch_counter_vdso_data_ordering(vd, ns); } while (unlikely(vdso_read_retry(vd, seq))); - ns += get_clock_shifted_nsec(cycle_last, cs_mono_mult); ns >>= cs_shift; ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); @@ -192,7 +214,7 @@ static notrace int do_monotonic(const struct vdso_data *vd, return 0; } -static notrace int do_monotonic_raw(const struct vdso_data *vd, +static notrace int do_monotonic_raw(struct vdso_data *vd, struct timespec *ts) { u32 seq, cs_raw_mult, cs_shift; @@ -212,9 +234,10 @@ static notrace int do_monotonic_raw(const struct vdso_data *vd, sec = vd->raw_time_sec; ns = vd->raw_time_nsec; + ns += get_clock_shifted_nsec(cycle_last, cs_raw_mult); + vd = arch_counter_vdso_data_ordering(vd, ns); } while (unlikely(vdso_read_retry(vd, seq))); - ns += get_clock_shifted_nsec(cycle_last, cs_raw_mult); ns >>= cs_shift; ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); ts->tv_nsec = ns; @@ -223,7 +246,7 @@ static notrace int do_monotonic_raw(const struct vdso_data *vd, } -static notrace void do_realtime_coarse(const struct vdso_data *vd, +static notrace void do_realtime_coarse(struct vdso_data *vd, struct timespec *ts) { u32 seq; @@ -241,7 +264,7 @@ static notrace void do_realtime_coarse(const struct vdso_data *vd, ts->tv_nsec = ns; } -static notrace void do_monotonic_coarse(const struct vdso_data *vd, +static notrace void do_monotonic_coarse(struct vdso_data *vd, struct timespec *ts) { u32 seq; @@ -267,7 +290,7 @@ static notrace void do_monotonic_coarse(const struct vdso_data *vd, notrace int __kernel_clock_gettime(clockid_t clock, struct timespec *ts) { - const struct vdso_data *vd = &_vdso_data; + struct vdso_data *vd = &_vdso_data; switch (clock) { case CLOCK_REALTIME: @@ -301,7 +324,7 @@ notrace int __kernel_clock_gettime(clockid_t clock, struct timespec *ts) notrace int __kernel_gettimeofday(struct timeval *tv, struct timezone *tz) { - const struct vdso_data *vd = &_vdso_data; + struct vdso_data *vd = &_vdso_data; if (likely(tv != NULL)) { struct timespec ts; -- GitLab