arm64: arch_timer: Ensure counter register reads occur with seqlock held

mainline inclusion
from mainline-5.2
commit 75a19a0202db21638a1c2b424afb867e1f9a2376
category: bugfix
bugzilla: NA
CVE: NA

-------------------------------------------------

When executing clock_gettime(), either in the vDSO or via a system call,
we need to ensure that the read of the counter register occurs within
the seqlock reader critical section. This ensures that updates to the
clocksource parameters (e.g. the multiplier) are consistent with the
counter value and therefore avoids the situation where time appears to
go backwards across multiple reads.

Extend the vDSO logic so that the seqlock critical section covers the
read of the counter register as well as accesses to the data page. Since
reads of the counter system registers are not ordered by memory barrier
instructions, introduce dependency ordering from the counter read to a
subsequent memory access so that the seqlock memory barriers apply to
the counter access in both the vDSO and the system call paths.

Cc: <stable@vger.kernel.org>
Cc: Marc Zyngier <marc.zyngier@arm.com>
Tested-by: NVincenzo Frascino <vincenzo.frascino@arm.com>
Link: https://lore.kernel.org/linux-arm-kernel/alpine.DEB.2.21.1902081950260.1662@nanos.tec.linutronix.de/Reported-by: NThomas Gleixner <tglx@linutronix.de>
Signed-off-by: NWill Deacon <will.deacon@arm.com>
Signed-off-by: NXuefeng Wang <wxf.wang@hisilicon.com>
Reviewed-by: NYang Yingliang <yangyingliang@huawei.com>
Signed-off-by: NYang Yingliang <yangyingliang@huawei.com>

arch/arm64/include/asm/arch_timer.h

@@ -148,18 +148,47 @@ static inline void arch_timer_set_cntkctl(u32 cntkctl)
 	isb();
 }
 
+/*
+ * Ensure that reads of the counter are treated the same as memory reads
+ * for the purposes of ordering by subsequent memory barriers.
+ *
+ * This insanity brought to you by speculative system register reads,
+ * out-of-order memory accesses, sequence locks and Thomas Gleixner.
+ *
+ * http://lists.infradead.org/pipermail/linux-arm-kernel/2019-February/631195.html
+ */
+#define arch_counter_enforce_ordering(val) do {				\
+	u64 tmp, _val = (val);						\
+									\
+	asm volatile(							\
+	"	eor	%0, %1, %1\n"					\
+	"	add	%0, sp, %0\n"					\
+	"	ldr	xzr, [%0]"					\
+	: "=r" (tmp) : "r" (_val));					\
+} while (0)
+
 static inline u64 arch_counter_get_cntpct(void)
 {
+	u64 cnt;
+
 	isb();
-	return arch_timer_reg_read_stable(cntpct_el0);
+	cnt = arch_timer_reg_read_stable(cntpct_el0);
+	arch_counter_enforce_ordering(cnt);
+	return cnt;
 }
 
 static inline u64 arch_counter_get_cntvct(void)
 {
+	u64 cnt;
+
 	isb();
-	return arch_timer_reg_read_stable(cntvct_el0);
+	cnt = arch_timer_reg_read_stable(cntvct_el0);
+	arch_counter_enforce_ordering(cnt);
+	return cnt;
 }
 
+#undef arch_counter_enforce_ordering
+
 static inline int arch_timer_arch_init(void)
 {
 	return 0;

arch/arm64/kernel/vdso/gettimeofday.c

@@ -81,7 +81,7 @@ static notrace int clock_getres_fallback(clockid_t _clkid,
 	return ret;
 }
 
-static notrace u32 vdso_read_begin(const struct vdso_data *vd)
+static notrace u32 vdso_read_begin(struct vdso_data *vd)
 {
 	u32 seq;
 
@@ -98,7 +98,7 @@ static notrace u32 vdso_read_begin(const struct vdso_data *vd)
 	return seq;
 }
 
-static notrace u32 vdso_read_retry(const struct vdso_data *vd, u32 start)
+static notrace u32 vdso_read_retry(struct vdso_data *vd, u32 start)
 {
 	u32 seq;
 
@@ -123,12 +123,32 @@ static notrace u64 get_clock_shifted_nsec(u64 cycle_last, u64 mult)
 	res = res - cycle_last;
 	/* We can only guarantee 56 bits of precision. */
 	res &= ~(0xff00ull<<48);
+
 	return res * mult;
 }
 
+/*
+ * Fake address dependency from the value computed from the counter
+ * register to subsequent data page accesses so that the sequence
+ * locking also orders the read of the counter.
+ */
+static notrace struct vdso_data *arch_counter_vdso_data_ordering(struct vdso_data *vd, u64 res)
+{
+	struct vdso_data *vd_res = vd;
+	u64	tmp;
+
+	asm	volatile(
+	"	and		%0, %1, xzr\n"	\
+	"	add		%2, %2, %0\n" 	\
+	: "=r"	(tmp)				\
+	: "r"(res), "r"(vd_res));
+
+	return vd_res;
+}
 
 /* Code size doesn't matter (vdso is 4k/16k/64k anyway) and this is faster. */
-static __always_inline notrace int do_realtime(const struct vdso_data *vd,
+
+static __always_inline notrace int do_realtime(struct vdso_data *vd,
 					       struct timespec *ts)
 {
 	u32 seq, cs_mono_mult, cs_shift;
@@ -148,9 +168,10 @@ static __always_inline notrace int do_realtime(const struct vdso_data *vd,
 		sec = vd->xtime_clock_sec;
 		ns = vd->xtime_clock_nsec;
 
+		ns += get_clock_shifted_nsec(cycle_last, cs_mono_mult);
+		vd = arch_counter_vdso_data_ordering(vd, ns);
 	} while (unlikely(vdso_read_retry(vd, seq)));
 
-	ns += get_clock_shifted_nsec(cycle_last, cs_mono_mult);
 	ns >>= cs_shift;
 	ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
 	ts->tv_nsec = ns;
@@ -158,7 +179,7 @@ static __always_inline notrace int do_realtime(const struct vdso_data *vd,
 	return 0;
 }
 
-static notrace int do_monotonic(const struct vdso_data *vd,
+static notrace int do_monotonic(struct vdso_data *vd,
 				struct timespec *ts)
 {
 	u32 seq, cs_mono_mult, cs_shift;
@@ -181,9 +202,10 @@ static notrace int do_monotonic(const struct vdso_data *vd,
 		sec += vd->wtm_clock_sec;
 		ns += vd->wtm_clock_nsec << cs_shift;
 
+		ns += get_clock_shifted_nsec(cycle_last, cs_mono_mult);
+		vd = arch_counter_vdso_data_ordering(vd, ns);
 	} while (unlikely(vdso_read_retry(vd, seq)));
 
-	ns += get_clock_shifted_nsec(cycle_last, cs_mono_mult);
 	ns >>= cs_shift;
 
 	ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
@@ -192,7 +214,7 @@ static notrace int do_monotonic(const struct vdso_data *vd,
 	return 0;
 }
 
-static notrace int do_monotonic_raw(const struct vdso_data *vd,
+static notrace int do_monotonic_raw(struct vdso_data *vd,
 				    struct timespec *ts)
 {
 	u32 seq, cs_raw_mult, cs_shift;
@@ -212,9 +234,10 @@ static notrace int do_monotonic_raw(const struct vdso_data *vd,
 		sec = vd->raw_time_sec;
 		ns = vd->raw_time_nsec;
 
+		ns += get_clock_shifted_nsec(cycle_last, cs_raw_mult);
+		vd = arch_counter_vdso_data_ordering(vd, ns);
 	} while (unlikely(vdso_read_retry(vd, seq)));
 
-	ns += get_clock_shifted_nsec(cycle_last, cs_raw_mult);
 	ns >>= cs_shift;
 	ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
 	ts->tv_nsec = ns;
@@ -223,7 +246,7 @@ static notrace int do_monotonic_raw(const struct vdso_data *vd,
 }
 
 
-static notrace void do_realtime_coarse(const struct vdso_data *vd,
+static notrace void do_realtime_coarse(struct vdso_data *vd,
 				       struct timespec *ts)
 {
 	u32 seq;
@@ -241,7 +264,7 @@ static notrace void do_realtime_coarse(const struct vdso_data *vd,
 	ts->tv_nsec = ns;
 }
 
-static notrace void do_monotonic_coarse(const struct vdso_data *vd,
+static notrace void do_monotonic_coarse(struct vdso_data *vd,
 					struct timespec *ts)
 {
 	u32 seq;
@@ -267,7 +290,7 @@ static notrace void do_monotonic_coarse(const struct vdso_data *vd,
 
 notrace int __kernel_clock_gettime(clockid_t clock, struct timespec *ts)
 {
-	const struct vdso_data *vd = &_vdso_data;
+	struct vdso_data *vd = &_vdso_data;
 
 	switch (clock) {
 	case CLOCK_REALTIME:
@@ -301,7 +324,7 @@ notrace int __kernel_clock_gettime(clockid_t clock, struct timespec *ts)
 
 notrace int __kernel_gettimeofday(struct timeval *tv, struct timezone *tz)
 {
-	const struct vdso_data *vd = &_vdso_data;
+	struct vdso_data *vd = &_vdso_data;
 
 	if (likely(tv != NULL)) {
 		struct timespec ts;