timekeeping.c 64.7 KB
Newer Older
1 2 3 4 5 6 7 8 9 10
/*
 *  linux/kernel/time/timekeeping.c
 *
 *  Kernel timekeeping code and accessor functions
 *
 *  This code was moved from linux/kernel/timer.c.
 *  Please see that file for copyright and history logs.
 *
 */

11
#include <linux/timekeeper_internal.h>
12 13 14 15 16
#include <linux/module.h>
#include <linux/interrupt.h>
#include <linux/percpu.h>
#include <linux/init.h>
#include <linux/mm.h>
17
#include <linux/nmi.h>
18
#include <linux/sched.h>
19
#include <linux/sched/loadavg.h>
20
#include <linux/syscore_ops.h>
21 22 23 24
#include <linux/clocksource.h>
#include <linux/jiffies.h>
#include <linux/time.h>
#include <linux/tick.h>
25
#include <linux/stop_machine.h>
26
#include <linux/pvclock_gtod.h>
27
#include <linux/compiler.h>
28

29
#include "tick-internal.h"
30
#include "ntp_internal.h"
31
#include "timekeeping_internal.h"
32

33 34
#define TK_CLEAR_NTP		(1 << 0)
#define TK_MIRROR		(1 << 1)
35
#define TK_CLOCK_WAS_SET	(1 << 2)
36

37 38 39 40 41 42 43 44 45
/*
 * The most important data for readout fits into a single 64 byte
 * cache line.
 */
static struct {
	seqcount_t		seq;
	struct timekeeper	timekeeper;
} tk_core ____cacheline_aligned;

46
static DEFINE_RAW_SPINLOCK(timekeeper_lock);
47
static struct timekeeper shadow_timekeeper;
48

49 50 51 52 53 54 55 56 57 58 59 60 61 62 63
/**
 * struct tk_fast - NMI safe timekeeper
 * @seq:	Sequence counter for protecting updates. The lowest bit
 *		is the index for the tk_read_base array
 * @base:	tk_read_base array. Access is indexed by the lowest bit of
 *		@seq.
 *
 * See @update_fast_timekeeper() below.
 */
struct tk_fast {
	seqcount_t		seq;
	struct tk_read_base	base[2];
};

static struct tk_fast tk_fast_mono ____cacheline_aligned;
P
Peter Zijlstra 已提交
64
static struct tk_fast tk_fast_raw  ____cacheline_aligned;
65

66 67 68
/* flag for if timekeeping is suspended */
int __read_mostly timekeeping_suspended;

69 70
static inline void tk_normalize_xtime(struct timekeeper *tk)
{
71 72
	while (tk->tkr_mono.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_mono.shift)) {
		tk->tkr_mono.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_mono.shift;
73 74
		tk->xtime_sec++;
	}
75 76 77 78
	while (tk->tkr_raw.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_raw.shift)) {
		tk->tkr_raw.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_raw.shift;
		tk->raw_sec++;
	}
79 80
}

81 82 83 84 85
static inline struct timespec64 tk_xtime(struct timekeeper *tk)
{
	struct timespec64 ts;

	ts.tv_sec = tk->xtime_sec;
86
	ts.tv_nsec = (long)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift);
87 88 89
	return ts;
}

90
static void tk_set_xtime(struct timekeeper *tk, const struct timespec64 *ts)
91 92
{
	tk->xtime_sec = ts->tv_sec;
93
	tk->tkr_mono.xtime_nsec = (u64)ts->tv_nsec << tk->tkr_mono.shift;
94 95
}

96
static void tk_xtime_add(struct timekeeper *tk, const struct timespec64 *ts)
97 98
{
	tk->xtime_sec += ts->tv_sec;
99
	tk->tkr_mono.xtime_nsec += (u64)ts->tv_nsec << tk->tkr_mono.shift;
100
	tk_normalize_xtime(tk);
101
}
102

103
static void tk_set_wall_to_mono(struct timekeeper *tk, struct timespec64 wtm)
104
{
105
	struct timespec64 tmp;
106 107 108 109 110

	/*
	 * Verify consistency of: offset_real = -wall_to_monotonic
	 * before modifying anything
	 */
111
	set_normalized_timespec64(&tmp, -tk->wall_to_monotonic.tv_sec,
112
					-tk->wall_to_monotonic.tv_nsec);
T
Thomas Gleixner 已提交
113
	WARN_ON_ONCE(tk->offs_real != timespec64_to_ktime(tmp));
114
	tk->wall_to_monotonic = wtm;
115 116
	set_normalized_timespec64(&tmp, -wtm.tv_sec, -wtm.tv_nsec);
	tk->offs_real = timespec64_to_ktime(tmp);
117
	tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tk->tai_offset, 0));
118 119
}

120
static inline void tk_update_sleep_time(struct timekeeper *tk, ktime_t delta)
121
{
122
	tk->offs_boot = ktime_add(tk->offs_boot, delta);
123 124
}

125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144
/*
 * tk_clock_read - atomic clocksource read() helper
 *
 * This helper is necessary to use in the read paths because, while the
 * seqlock ensures we don't return a bad value while structures are updated,
 * it doesn't protect from potential crashes. There is the possibility that
 * the tkr's clocksource may change between the read reference, and the
 * clock reference passed to the read function.  This can cause crashes if
 * the wrong clocksource is passed to the wrong read function.
 * This isn't necessary to use when holding the timekeeper_lock or doing
 * a read of the fast-timekeeper tkrs (which is protected by its own locking
 * and update logic).
 */
static inline u64 tk_clock_read(struct tk_read_base *tkr)
{
	struct clocksource *clock = READ_ONCE(tkr->clock);

	return clock->read(clock);
}

145
#ifdef CONFIG_DEBUG_TIMEKEEPING
146 147
#define WARNING_FREQ (HZ*300) /* 5 minute rate-limiting */

148
static void timekeeping_check_update(struct timekeeper *tk, u64 offset)
149 150
{

151
	u64 max_cycles = tk->tkr_mono.clock->max_cycles;
152
	const char *name = tk->tkr_mono.clock->name;
153 154

	if (offset > max_cycles) {
155
		printk_deferred("WARNING: timekeeping: Cycle offset (%lld) is larger than allowed by the '%s' clock's max_cycles value (%lld): time overflow danger\n",
156
				offset, name, max_cycles);
157
		printk_deferred("         timekeeping: Your kernel is sick, but tries to cope by capping time updates\n");
158 159
	} else {
		if (offset > (max_cycles >> 1)) {
M
Masanari Iida 已提交
160
			printk_deferred("INFO: timekeeping: Cycle offset (%lld) is larger than the '%s' clock's 50%% safety margin (%lld)\n",
161 162 163 164
					offset, name, max_cycles >> 1);
			printk_deferred("      timekeeping: Your kernel is still fine, but is feeling a bit nervous\n");
		}
	}
165

166 167
	if (tk->underflow_seen) {
		if (jiffies - tk->last_warning > WARNING_FREQ) {
168 169 170
			printk_deferred("WARNING: Underflow in clocksource '%s' observed, time update ignored.\n", name);
			printk_deferred("         Please report this, consider using a different clocksource, if possible.\n");
			printk_deferred("         Your kernel is probably still fine.\n");
171
			tk->last_warning = jiffies;
172
		}
173
		tk->underflow_seen = 0;
174 175
	}

176 177
	if (tk->overflow_seen) {
		if (jiffies - tk->last_warning > WARNING_FREQ) {
178 179 180
			printk_deferred("WARNING: Overflow in clocksource '%s' observed, time update capped.\n", name);
			printk_deferred("         Please report this, consider using a different clocksource, if possible.\n");
			printk_deferred("         Your kernel is probably still fine.\n");
181
			tk->last_warning = jiffies;
182
		}
183
		tk->overflow_seen = 0;
184
	}
185
}
186

187
static inline u64 timekeeping_get_delta(struct tk_read_base *tkr)
188
{
189
	struct timekeeper *tk = &tk_core.timekeeper;
190
	u64 now, last, mask, max, delta;
191
	unsigned int seq;
192

193 194 195 196 197 198 199 200 201
	/*
	 * Since we're called holding a seqlock, the data may shift
	 * under us while we're doing the calculation. This can cause
	 * false positives, since we'd note a problem but throw the
	 * results away. So nest another seqlock here to atomically
	 * grab the points we are checking with.
	 */
	do {
		seq = read_seqcount_begin(&tk_core.seq);
202
		now = tk_clock_read(tkr);
203 204 205 206
		last = tkr->cycle_last;
		mask = tkr->mask;
		max = tkr->clock->max_cycles;
	} while (read_seqcount_retry(&tk_core.seq, seq));
207

208
	delta = clocksource_delta(now, last, mask);
209

210 211 212 213
	/*
	 * Try to catch underflows by checking if we are seeing small
	 * mask-relative negative values.
	 */
214
	if (unlikely((~delta & mask) < (mask >> 3))) {
215
		tk->underflow_seen = 1;
216
		delta = 0;
217
	}
218

219
	/* Cap delta value to the max_cycles values to avoid mult overflows */
220
	if (unlikely(delta > max)) {
221
		tk->overflow_seen = 1;
222
		delta = tkr->clock->max_cycles;
223
	}
224 225 226

	return delta;
}
227
#else
228
static inline void timekeeping_check_update(struct timekeeper *tk, u64 offset)
229 230
{
}
231
static inline u64 timekeeping_get_delta(struct tk_read_base *tkr)
232
{
233
	u64 cycle_now, delta;
234 235

	/* read clocksource */
236
	cycle_now = tk_clock_read(tkr);
237 238 239 240 241 242

	/* calculate the delta since the last update_wall_time */
	delta = clocksource_delta(cycle_now, tkr->cycle_last, tkr->mask);

	return delta;
}
243 244
#endif

245
/**
246
 * tk_setup_internals - Set up internals to use clocksource clock.
247
 *
248
 * @tk:		The target timekeeper to setup.
249 250 251 252 253 254 255
 * @clock:		Pointer to clocksource.
 *
 * Calculates a fixed cycle/nsec interval for a given clocksource/adjustment
 * pair and interval request.
 *
 * Unless you're the timekeeping code, you should not be using this!
 */
256
static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock)
257
{
258
	u64 interval;
259
	u64 tmp, ntpinterval;
260
	struct clocksource *old_clock;
261

262
	++tk->cs_was_changed_seq;
263 264 265
	old_clock = tk->tkr_mono.clock;
	tk->tkr_mono.clock = clock;
	tk->tkr_mono.mask = clock->mask;
266
	tk->tkr_mono.cycle_last = tk_clock_read(&tk->tkr_mono);
267

P
Peter Zijlstra 已提交
268 269 270 271
	tk->tkr_raw.clock = clock;
	tk->tkr_raw.mask = clock->mask;
	tk->tkr_raw.cycle_last = tk->tkr_mono.cycle_last;

272 273 274
	/* Do the ns -> cycle conversion first, using original mult */
	tmp = NTP_INTERVAL_LENGTH;
	tmp <<= clock->shift;
275
	ntpinterval = tmp;
276 277
	tmp += clock->mult/2;
	do_div(tmp, clock->mult);
278 279 280
	if (tmp == 0)
		tmp = 1;

281
	interval = (u64) tmp;
282
	tk->cycle_interval = interval;
283 284

	/* Go back from cycles -> shifted ns */
285
	tk->xtime_interval = interval * clock->mult;
286
	tk->xtime_remainder = ntpinterval - tk->xtime_interval;
287
	tk->raw_interval = interval * clock->mult;
288

289 290 291
	 /* if changing clocks, convert xtime_nsec shift units */
	if (old_clock) {
		int shift_change = clock->shift - old_clock->shift;
292
		if (shift_change < 0) {
293
			tk->tkr_mono.xtime_nsec >>= -shift_change;
294 295
			tk->tkr_raw.xtime_nsec >>= -shift_change;
		} else {
296
			tk->tkr_mono.xtime_nsec <<= shift_change;
297 298
			tk->tkr_raw.xtime_nsec <<= shift_change;
		}
299
	}
P
Peter Zijlstra 已提交
300

301
	tk->tkr_mono.shift = clock->shift;
P
Peter Zijlstra 已提交
302
	tk->tkr_raw.shift = clock->shift;
303

304 305
	tk->ntp_error = 0;
	tk->ntp_error_shift = NTP_SCALE_SHIFT - clock->shift;
306
	tk->ntp_tick = ntpinterval << tk->ntp_error_shift;
307 308 309 310 311 312

	/*
	 * The timekeeper keeps its own mult values for the currently
	 * active clocksource. These value will be adjusted via NTP
	 * to counteract clock drifting.
	 */
313
	tk->tkr_mono.mult = clock->mult;
P
Peter Zijlstra 已提交
314
	tk->tkr_raw.mult = clock->mult;
315
	tk->ntp_err_mult = 0;
316
}
317

318
/* Timekeeper helper functions. */
319 320

#ifdef CONFIG_ARCH_USES_GETTIMEOFFSET
321 322
static u32 default_arch_gettimeoffset(void) { return 0; }
u32 (*arch_gettimeoffset)(void) = default_arch_gettimeoffset;
323
#else
324
static inline u32 arch_gettimeoffset(void) { return 0; }
325 326
#endif

327
static inline u64 timekeeping_delta_to_ns(struct tk_read_base *tkr, u64 delta)
328
{
329
	u64 nsec;
330 331 332 333 334 335 336 337

	nsec = delta * tkr->mult + tkr->xtime_nsec;
	nsec >>= tkr->shift;

	/* If arch requires, add in get_arch_timeoffset() */
	return nsec + arch_gettimeoffset();
}

338
static inline u64 timekeeping_get_ns(struct tk_read_base *tkr)
339
{
340
	u64 delta;
341

342
	delta = timekeeping_get_delta(tkr);
343 344
	return timekeeping_delta_to_ns(tkr, delta);
}
345

346
static inline u64 timekeeping_cycles_to_ns(struct tk_read_base *tkr, u64 cycles)
347
{
348
	u64 delta;
349

350 351 352
	/* calculate the delta since the last update_wall_time */
	delta = clocksource_delta(cycles, tkr->cycle_last, tkr->mask);
	return timekeeping_delta_to_ns(tkr, delta);
353 354
}

355 356
/**
 * update_fast_timekeeper - Update the fast and NMI safe monotonic timekeeper.
357
 * @tkr: Timekeeping readout base from which we take the update
358 359 360 361
 *
 * We want to use this from any context including NMI and tracing /
 * instrumenting the timekeeping code itself.
 *
362
 * Employ the latch technique; see @raw_write_seqcount_latch.
363 364 365 366 367 368
 *
 * So if a NMI hits the update of base[0] then it will use base[1]
 * which is still consistent. In the worst case this can result is a
 * slightly wrong timestamp (a few nanoseconds). See
 * @ktime_get_mono_fast_ns.
 */
369
static void update_fast_timekeeper(struct tk_read_base *tkr, struct tk_fast *tkf)
370
{
371
	struct tk_read_base *base = tkf->base;
372 373

	/* Force readers off to base[1] */
374
	raw_write_seqcount_latch(&tkf->seq);
375 376

	/* Update base[0] */
377
	memcpy(base, tkr, sizeof(*base));
378 379

	/* Force readers back to base[0] */
380
	raw_write_seqcount_latch(&tkf->seq);
381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417

	/* Update base[1] */
	memcpy(base + 1, base, sizeof(*base));
}

/**
 * ktime_get_mono_fast_ns - Fast NMI safe access to clock monotonic
 *
 * This timestamp is not guaranteed to be monotonic across an update.
 * The timestamp is calculated by:
 *
 *	now = base_mono + clock_delta * slope
 *
 * So if the update lowers the slope, readers who are forced to the
 * not yet updated second array are still using the old steeper slope.
 *
 * tmono
 * ^
 * |    o  n
 * |   o n
 * |  u
 * | o
 * |o
 * |12345678---> reader order
 *
 * o = old slope
 * u = update
 * n = new slope
 *
 * So reader 6 will observe time going backwards versus reader 5.
 *
 * While other CPUs are likely to be able observe that, the only way
 * for a CPU local observation is when an NMI hits in the middle of
 * the update. Timestamps taken from that NMI context might be ahead
 * of the following timestamps. Callers need to be aware of that and
 * deal with it.
 */
418
static __always_inline u64 __ktime_get_fast_ns(struct tk_fast *tkf)
419 420 421 422 423 424
{
	struct tk_read_base *tkr;
	unsigned int seq;
	u64 now;

	do {
425
		seq = raw_read_seqcount_latch(&tkf->seq);
426
		tkr = tkf->base + (seq & 0x01);
427 428
		now = ktime_to_ns(tkr->base);

429 430
		now += timekeeping_delta_to_ns(tkr,
				clocksource_delta(
431
					tk_clock_read(tkr),
432 433
					tkr->cycle_last,
					tkr->mask));
434
	} while (read_seqcount_retry(&tkf->seq, seq));
435 436 437

	return now;
}
438 439 440 441 442

u64 ktime_get_mono_fast_ns(void)
{
	return __ktime_get_fast_ns(&tk_fast_mono);
}
443 444
EXPORT_SYMBOL_GPL(ktime_get_mono_fast_ns);

P
Peter Zijlstra 已提交
445 446 447 448 449 450
u64 ktime_get_raw_fast_ns(void)
{
	return __ktime_get_fast_ns(&tk_fast_raw);
}
EXPORT_SYMBOL_GPL(ktime_get_raw_fast_ns);

451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479
/**
 * ktime_get_boot_fast_ns - NMI safe and fast access to boot clock.
 *
 * To keep it NMI safe since we're accessing from tracing, we're not using a
 * separate timekeeper with updates to monotonic clock and boot offset
 * protected with seqlocks. This has the following minor side effects:
 *
 * (1) Its possible that a timestamp be taken after the boot offset is updated
 * but before the timekeeper is updated. If this happens, the new boot offset
 * is added to the old timekeeping making the clock appear to update slightly
 * earlier:
 *    CPU 0                                        CPU 1
 *    timekeeping_inject_sleeptime64()
 *    __timekeeping_inject_sleeptime(tk, delta);
 *                                                 timestamp();
 *    timekeeping_update(tk, TK_CLEAR_NTP...);
 *
 * (2) On 32-bit systems, the 64-bit boot offset (tk->offs_boot) may be
 * partially updated.  Since the tk->offs_boot update is a rare event, this
 * should be a rare occurrence which postprocessing should be able to handle.
 */
u64 notrace ktime_get_boot_fast_ns(void)
{
	struct timekeeper *tk = &tk_core.timekeeper;

	return (ktime_get_mono_fast_ns() + ktime_to_ns(tk->offs_boot));
}
EXPORT_SYMBOL_GPL(ktime_get_boot_fast_ns);

480
/* Suspend-time cycles value for halted fast timekeeper. */
481
static u64 cycles_at_suspend;
482

483
static u64 dummy_clock_read(struct clocksource *cs)
484 485 486 487
{
	return cycles_at_suspend;
}

488 489 490 491
static struct clocksource dummy_clock = {
	.read = dummy_clock_read,
};

492 493 494 495 496 497 498 499 500 501 502 503 504
/**
 * halt_fast_timekeeper - Prevent fast timekeeper from accessing clocksource.
 * @tk: Timekeeper to snapshot.
 *
 * It generally is unsafe to access the clocksource after timekeeping has been
 * suspended, so take a snapshot of the readout base of @tk and use it as the
 * fast timekeeper's readout base while suspended.  It will return the same
 * number of cycles every time until timekeeping is resumed at which time the
 * proper readout base for the fast timekeeper will be restored automatically.
 */
static void halt_fast_timekeeper(struct timekeeper *tk)
{
	static struct tk_read_base tkr_dummy;
505
	struct tk_read_base *tkr = &tk->tkr_mono;
506 507

	memcpy(&tkr_dummy, tkr, sizeof(tkr_dummy));
508 509
	cycles_at_suspend = tk_clock_read(tkr);
	tkr_dummy.clock = &dummy_clock;
510
	update_fast_timekeeper(&tkr_dummy, &tk_fast_mono);
P
Peter Zijlstra 已提交
511 512 513

	tkr = &tk->tkr_raw;
	memcpy(&tkr_dummy, tkr, sizeof(tkr_dummy));
514
	tkr_dummy.clock = &dummy_clock;
P
Peter Zijlstra 已提交
515
	update_fast_timekeeper(&tkr_dummy, &tk_fast_raw);
516 517
}

518
#ifdef CONFIG_GENERIC_TIME_VSYSCALL_OLD
519
#warning Please contact your maintainers, as GENERIC_TIME_VSYSCALL_OLD compatibity will disappear soon.
520 521 522

static inline void update_vsyscall(struct timekeeper *tk)
{
523
	struct timespec xt, wm;
524

525
	xt = timespec64_to_timespec(tk_xtime(tk));
526
	wm = timespec64_to_timespec(tk->wall_to_monotonic);
527 528
	update_vsyscall_old(&xt, &wm, tk->tkr_mono.clock, tk->tkr_mono.mult,
			    tk->tkr_mono.cycle_last);
529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544
}

static inline void old_vsyscall_fixup(struct timekeeper *tk)
{
	s64 remainder;

	/*
	* Store only full nanoseconds into xtime_nsec after rounding
	* it up and add the remainder to the error difference.
	* XXX - This is necessary to avoid small 1ns inconsistnecies caused
	* by truncating the remainder in vsyscalls. However, it causes
	* additional work to be done in timekeeping_adjust(). Once
	* the vsyscall implementations are converted to use xtime_nsec
	* (shifted nanoseconds), and CONFIG_GENERIC_TIME_VSYSCALL_OLD
	* users are removed, this can be killed.
	*/
545
	remainder = tk->tkr_mono.xtime_nsec & ((1ULL << tk->tkr_mono.shift) - 1);
546 547 548 549 550 551
	if (remainder != 0) {
		tk->tkr_mono.xtime_nsec -= remainder;
		tk->tkr_mono.xtime_nsec += 1ULL << tk->tkr_mono.shift;
		tk->ntp_error += remainder << tk->ntp_error_shift;
		tk->ntp_error -= (1ULL << tk->tkr_mono.shift) << tk->ntp_error_shift;
	}
552 553 554 555 556
}
#else
#define old_vsyscall_fixup(tk)
#endif

557 558
static RAW_NOTIFIER_HEAD(pvclock_gtod_chain);

559
static void update_pvclock_gtod(struct timekeeper *tk, bool was_set)
560
{
561
	raw_notifier_call_chain(&pvclock_gtod_chain, was_set, tk);
562 563 564 565 566 567 568
}

/**
 * pvclock_gtod_register_notifier - register a pvclock timedata update listener
 */
int pvclock_gtod_register_notifier(struct notifier_block *nb)
{
569
	struct timekeeper *tk = &tk_core.timekeeper;
570 571 572
	unsigned long flags;
	int ret;

573
	raw_spin_lock_irqsave(&timekeeper_lock, flags);
574
	ret = raw_notifier_chain_register(&pvclock_gtod_chain, nb);
575
	update_pvclock_gtod(tk, true);
576
	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
577 578 579 580 581 582 583 584 585 586 587 588 589 590

	return ret;
}
EXPORT_SYMBOL_GPL(pvclock_gtod_register_notifier);

/**
 * pvclock_gtod_unregister_notifier - unregister a pvclock
 * timedata update listener
 */
int pvclock_gtod_unregister_notifier(struct notifier_block *nb)
{
	unsigned long flags;
	int ret;

591
	raw_spin_lock_irqsave(&timekeeper_lock, flags);
592
	ret = raw_notifier_chain_unregister(&pvclock_gtod_chain, nb);
593
	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
594 595 596 597 598

	return ret;
}
EXPORT_SYMBOL_GPL(pvclock_gtod_unregister_notifier);

599 600 601 602 603 604
/*
 * tk_update_leap_state - helper to update the next_leap_ktime
 */
static inline void tk_update_leap_state(struct timekeeper *tk)
{
	tk->next_leap_ktime = ntp_get_next_leap();
T
Thomas Gleixner 已提交
605
	if (tk->next_leap_ktime != KTIME_MAX)
606 607 608 609
		/* Convert to monotonic time */
		tk->next_leap_ktime = ktime_sub(tk->next_leap_ktime, tk->offs_real);
}

610 611 612 613 614
/*
 * Update the ktime_t based scalar nsec members of the timekeeper
 */
static inline void tk_update_ktime_data(struct timekeeper *tk)
{
615 616
	u64 seconds;
	u32 nsec;
617 618 619 620 621 622 623 624

	/*
	 * The xtime based monotonic readout is:
	 *	nsec = (xtime_sec + wtm_sec) * 1e9 + wtm_nsec + now();
	 * The ktime based monotonic readout is:
	 *	nsec = base_mono + now();
	 * ==> base_mono = (xtime_sec + wtm_sec) * 1e9 + wtm_nsec
	 */
625 626
	seconds = (u64)(tk->xtime_sec + tk->wall_to_monotonic.tv_sec);
	nsec = (u32) tk->wall_to_monotonic.tv_nsec;
627
	tk->tkr_mono.base = ns_to_ktime(seconds * NSEC_PER_SEC + nsec);
628

629 630 631 632 633
	/*
	 * The sum of the nanoseconds portions of xtime and
	 * wall_to_monotonic can be greater/equal one second. Take
	 * this into account before updating tk->ktime_sec.
	 */
634
	nsec += (u32)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift);
635 636 637
	if (nsec >= NSEC_PER_SEC)
		seconds++;
	tk->ktime_sec = seconds;
638 639 640 641 642

	/* Update the monotonic raw base */
	seconds = tk->raw_sec;
	nsec = (u32)(tk->tkr_raw.xtime_nsec >> tk->tkr_raw.shift);
	tk->tkr_raw.base = ns_to_ktime(seconds * NSEC_PER_SEC + nsec);
643 644
}

645
/* must hold timekeeper_lock */
646
static void timekeeping_update(struct timekeeper *tk, unsigned int action)
647
{
648
	if (action & TK_CLEAR_NTP) {
649
		tk->ntp_error = 0;
650 651
		ntp_clear();
	}
652

653
	tk_update_leap_state(tk);
654 655
	tk_update_ktime_data(tk);

656 657 658
	update_vsyscall(tk);
	update_pvclock_gtod(tk, action & TK_CLOCK_WAS_SET);

659
	update_fast_timekeeper(&tk->tkr_mono, &tk_fast_mono);
P
Peter Zijlstra 已提交
660
	update_fast_timekeeper(&tk->tkr_raw,  &tk_fast_raw);
661 662 663

	if (action & TK_CLOCK_WAS_SET)
		tk->clock_was_set_seq++;
664 665 666 667 668 669 670 671
	/*
	 * The mirroring of the data to the shadow-timekeeper needs
	 * to happen last here to ensure we don't over-write the
	 * timekeeper structure on the next update with stale data
	 */
	if (action & TK_MIRROR)
		memcpy(&shadow_timekeeper, &tk_core.timekeeper,
		       sizeof(tk_core.timekeeper));
672 673
}

674
/**
675
 * timekeeping_forward_now - update clock to the current time
676
 *
677 678 679
 * Forward the current clock to update its state since the last call to
 * update_wall_time(). This is useful before significant clock changes,
 * as it avoids having to deal with this time offset explicitly.
680
 */
681
static void timekeeping_forward_now(struct timekeeper *tk)
682
{
683
	u64 cycle_now, delta;
684

685
	cycle_now = tk_clock_read(&tk->tkr_mono);
686 687
	delta = clocksource_delta(cycle_now, tk->tkr_mono.cycle_last, tk->tkr_mono.mask);
	tk->tkr_mono.cycle_last = cycle_now;
P
Peter Zijlstra 已提交
688
	tk->tkr_raw.cycle_last  = cycle_now;
689

690
	tk->tkr_mono.xtime_nsec += delta * tk->tkr_mono.mult;
691

692
	/* If arch requires, add in get_arch_timeoffset() */
693
	tk->tkr_mono.xtime_nsec += (u64)arch_gettimeoffset() << tk->tkr_mono.shift;
694

695

696 697 698 699 700 701
	tk->tkr_raw.xtime_nsec += delta * tk->tkr_raw.mult;

	/* If arch requires, add in get_arch_timeoffset() */
	tk->tkr_raw.xtime_nsec += (u64)arch_gettimeoffset() << tk->tkr_raw.shift;

	tk_normalize_xtime(tk);
702 703 704
}

/**
705
 * __getnstimeofday64 - Returns the time of day in a timespec64.
706 707
 * @ts:		pointer to the timespec to be set
 *
708 709
 * Updates the time of day in the timespec.
 * Returns 0 on success, or -ve when suspended (timespec will be undefined).
710
 */
711
int __getnstimeofday64(struct timespec64 *ts)
712
{
713
	struct timekeeper *tk = &tk_core.timekeeper;
714
	unsigned long seq;
715
	u64 nsecs;
716 717

	do {
718
		seq = read_seqcount_begin(&tk_core.seq);
719

720
		ts->tv_sec = tk->xtime_sec;
721
		nsecs = timekeeping_get_ns(&tk->tkr_mono);
722

723
	} while (read_seqcount_retry(&tk_core.seq, seq));
724

725
	ts->tv_nsec = 0;
726
	timespec64_add_ns(ts, nsecs);
727 728 729 730 731 732 733 734 735

	/*
	 * Do not bail out early, in case there were callers still using
	 * the value, even in the face of the WARN_ON.
	 */
	if (unlikely(timekeeping_suspended))
		return -EAGAIN;
	return 0;
}
736
EXPORT_SYMBOL(__getnstimeofday64);
737 738

/**
739
 * getnstimeofday64 - Returns the time of day in a timespec64.
740
 * @ts:		pointer to the timespec64 to be set
741
 *
742
 * Returns the time of day in a timespec64 (WARN if suspended).
743
 */
744
void getnstimeofday64(struct timespec64 *ts)
745
{
746
	WARN_ON(__getnstimeofday64(ts));
747
}
748
EXPORT_SYMBOL(getnstimeofday64);
749

750 751
ktime_t ktime_get(void)
{
752
	struct timekeeper *tk = &tk_core.timekeeper;
753
	unsigned int seq;
754
	ktime_t base;
755
	u64 nsecs;
756 757 758 759

	WARN_ON(timekeeping_suspended);

	do {
760
		seq = read_seqcount_begin(&tk_core.seq);
761 762
		base = tk->tkr_mono.base;
		nsecs = timekeeping_get_ns(&tk->tkr_mono);
763

764
	} while (read_seqcount_retry(&tk_core.seq, seq));
765

766
	return ktime_add_ns(base, nsecs);
767 768 769
}
EXPORT_SYMBOL_GPL(ktime_get);

770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786
u32 ktime_get_resolution_ns(void)
{
	struct timekeeper *tk = &tk_core.timekeeper;
	unsigned int seq;
	u32 nsecs;

	WARN_ON(timekeeping_suspended);

	do {
		seq = read_seqcount_begin(&tk_core.seq);
		nsecs = tk->tkr_mono.mult >> tk->tkr_mono.shift;
	} while (read_seqcount_retry(&tk_core.seq, seq));

	return nsecs;
}
EXPORT_SYMBOL_GPL(ktime_get_resolution_ns);

787 788 789 790 791 792 793 794 795 796 797
static ktime_t *offsets[TK_OFFS_MAX] = {
	[TK_OFFS_REAL]	= &tk_core.timekeeper.offs_real,
	[TK_OFFS_BOOT]	= &tk_core.timekeeper.offs_boot,
	[TK_OFFS_TAI]	= &tk_core.timekeeper.offs_tai,
};

ktime_t ktime_get_with_offset(enum tk_offsets offs)
{
	struct timekeeper *tk = &tk_core.timekeeper;
	unsigned int seq;
	ktime_t base, *offset = offsets[offs];
798
	u64 nsecs;
799 800 801 802 803

	WARN_ON(timekeeping_suspended);

	do {
		seq = read_seqcount_begin(&tk_core.seq);
804 805
		base = ktime_add(tk->tkr_mono.base, *offset);
		nsecs = timekeeping_get_ns(&tk->tkr_mono);
806 807 808 809 810 811 812 813

	} while (read_seqcount_retry(&tk_core.seq, seq));

	return ktime_add_ns(base, nsecs);

}
EXPORT_SYMBOL_GPL(ktime_get_with_offset);

814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833
/**
 * ktime_mono_to_any() - convert mononotic time to any other time
 * @tmono:	time to convert.
 * @offs:	which offset to use
 */
ktime_t ktime_mono_to_any(ktime_t tmono, enum tk_offsets offs)
{
	ktime_t *offset = offsets[offs];
	unsigned long seq;
	ktime_t tconv;

	do {
		seq = read_seqcount_begin(&tk_core.seq);
		tconv = ktime_add(tmono, *offset);
	} while (read_seqcount_retry(&tk_core.seq, seq));

	return tconv;
}
EXPORT_SYMBOL_GPL(ktime_mono_to_any);

834 835 836 837 838 839 840 841
/**
 * ktime_get_raw - Returns the raw monotonic time in ktime_t format
 */
ktime_t ktime_get_raw(void)
{
	struct timekeeper *tk = &tk_core.timekeeper;
	unsigned int seq;
	ktime_t base;
842
	u64 nsecs;
843 844 845

	do {
		seq = read_seqcount_begin(&tk_core.seq);
P
Peter Zijlstra 已提交
846 847
		base = tk->tkr_raw.base;
		nsecs = timekeeping_get_ns(&tk->tkr_raw);
848 849 850 851 852 853 854

	} while (read_seqcount_retry(&tk_core.seq, seq));

	return ktime_add_ns(base, nsecs);
}
EXPORT_SYMBOL_GPL(ktime_get_raw);

855
/**
856
 * ktime_get_ts64 - get the monotonic clock in timespec64 format
857 858 859 860
 * @ts:		pointer to timespec variable
 *
 * The function calculates the monotonic clock from the realtime
 * clock and the wall_to_monotonic offset and stores the result
861
 * in normalized timespec64 format in the variable pointed to by @ts.
862
 */
863
void ktime_get_ts64(struct timespec64 *ts)
864
{
865
	struct timekeeper *tk = &tk_core.timekeeper;
866
	struct timespec64 tomono;
867
	unsigned int seq;
868
	u64 nsec;
869 870 871 872

	WARN_ON(timekeeping_suspended);

	do {
873
		seq = read_seqcount_begin(&tk_core.seq);
874
		ts->tv_sec = tk->xtime_sec;
875
		nsec = timekeeping_get_ns(&tk->tkr_mono);
876
		tomono = tk->wall_to_monotonic;
877

878
	} while (read_seqcount_retry(&tk_core.seq, seq));
879

880 881 882
	ts->tv_sec += tomono.tv_sec;
	ts->tv_nsec = 0;
	timespec64_add_ns(ts, nsec + tomono.tv_nsec);
883
}
884
EXPORT_SYMBOL_GPL(ktime_get_ts64);
885

886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903
/**
 * ktime_get_seconds - Get the seconds portion of CLOCK_MONOTONIC
 *
 * Returns the seconds portion of CLOCK_MONOTONIC with a single non
 * serialized read. tk->ktime_sec is of type 'unsigned long' so this
 * works on both 32 and 64 bit systems. On 32 bit systems the readout
 * covers ~136 years of uptime which should be enough to prevent
 * premature wrap arounds.
 */
time64_t ktime_get_seconds(void)
{
	struct timekeeper *tk = &tk_core.timekeeper;

	WARN_ON(timekeeping_suspended);
	return tk->ktime_sec;
}
EXPORT_SYMBOL_GPL(ktime_get_seconds);

904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933
/**
 * ktime_get_real_seconds - Get the seconds portion of CLOCK_REALTIME
 *
 * Returns the wall clock seconds since 1970. This replaces the
 * get_seconds() interface which is not y2038 safe on 32bit systems.
 *
 * For 64bit systems the fast access to tk->xtime_sec is preserved. On
 * 32bit systems the access must be protected with the sequence
 * counter to provide "atomic" access to the 64bit tk->xtime_sec
 * value.
 */
time64_t ktime_get_real_seconds(void)
{
	struct timekeeper *tk = &tk_core.timekeeper;
	time64_t seconds;
	unsigned int seq;

	if (IS_ENABLED(CONFIG_64BIT))
		return tk->xtime_sec;

	do {
		seq = read_seqcount_begin(&tk_core.seq);
		seconds = tk->xtime_sec;

	} while (read_seqcount_retry(&tk_core.seq, seq));

	return seconds;
}
EXPORT_SYMBOL_GPL(ktime_get_real_seconds);

934 935 936 937 938 939 940 941 942 943 944 945
/**
 * __ktime_get_real_seconds - The same as ktime_get_real_seconds
 * but without the sequence counter protect. This internal function
 * is called just when timekeeping lock is already held.
 */
time64_t __ktime_get_real_seconds(void)
{
	struct timekeeper *tk = &tk_core.timekeeper;

	return tk->xtime_sec;
}

946 947 948 949 950 951 952 953 954 955
/**
 * ktime_get_snapshot - snapshots the realtime/monotonic raw clocks with counter
 * @systime_snapshot:	pointer to struct receiving the system time snapshot
 */
void ktime_get_snapshot(struct system_time_snapshot *systime_snapshot)
{
	struct timekeeper *tk = &tk_core.timekeeper;
	unsigned long seq;
	ktime_t base_raw;
	ktime_t base_real;
956 957
	u64 nsec_raw;
	u64 nsec_real;
958
	u64 now;
959

960 961
	WARN_ON_ONCE(timekeeping_suspended);

962 963
	do {
		seq = read_seqcount_begin(&tk_core.seq);
964
		now = tk_clock_read(&tk->tkr_mono);
965 966
		systime_snapshot->cs_was_changed_seq = tk->cs_was_changed_seq;
		systime_snapshot->clock_was_set_seq = tk->clock_was_set_seq;
967 968 969 970 971 972 973 974 975 976 977 978
		base_real = ktime_add(tk->tkr_mono.base,
				      tk_core.timekeeper.offs_real);
		base_raw = tk->tkr_raw.base;
		nsec_real = timekeeping_cycles_to_ns(&tk->tkr_mono, now);
		nsec_raw  = timekeeping_cycles_to_ns(&tk->tkr_raw, now);
	} while (read_seqcount_retry(&tk_core.seq, seq));

	systime_snapshot->cycles = now;
	systime_snapshot->real = ktime_add_ns(base_real, nsec_real);
	systime_snapshot->raw = ktime_add_ns(base_raw, nsec_raw);
}
EXPORT_SYMBOL_GPL(ktime_get_snapshot);
979

980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015
/* Scale base by mult/div checking for overflow */
static int scale64_check_overflow(u64 mult, u64 div, u64 *base)
{
	u64 tmp, rem;

	tmp = div64_u64_rem(*base, div, &rem);

	if (((int)sizeof(u64)*8 - fls64(mult) < fls64(tmp)) ||
	    ((int)sizeof(u64)*8 - fls64(mult) < fls64(rem)))
		return -EOVERFLOW;
	tmp *= mult;
	rem *= mult;

	do_div(rem, div);
	*base = tmp + rem;
	return 0;
}

/**
 * adjust_historical_crosststamp - adjust crosstimestamp previous to current interval
 * @history:			Snapshot representing start of history
 * @partial_history_cycles:	Cycle offset into history (fractional part)
 * @total_history_cycles:	Total history length in cycles
 * @discontinuity:		True indicates clock was set on history period
 * @ts:				Cross timestamp that should be adjusted using
 *	partial/total ratio
 *
 * Helper function used by get_device_system_crosststamp() to correct the
 * crosstimestamp corresponding to the start of the current interval to the
 * system counter value (timestamp point) provided by the driver. The
 * total_history_* quantities are the total history starting at the provided
 * reference point and ending at the start of the current interval. The cycle
 * count between the driver timestamp point and the start of the current
 * interval is partial_history_cycles.
 */
static int adjust_historical_crosststamp(struct system_time_snapshot *history,
1016 1017
					 u64 partial_history_cycles,
					 u64 total_history_cycles,
1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029
					 bool discontinuity,
					 struct system_device_crosststamp *ts)
{
	struct timekeeper *tk = &tk_core.timekeeper;
	u64 corr_raw, corr_real;
	bool interp_forward;
	int ret;

	if (total_history_cycles == 0 || partial_history_cycles == 0)
		return 0;

	/* Interpolate shortest distance from beginning or end of history */
1030
	interp_forward = partial_history_cycles > total_history_cycles / 2;
1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079
	partial_history_cycles = interp_forward ?
		total_history_cycles - partial_history_cycles :
		partial_history_cycles;

	/*
	 * Scale the monotonic raw time delta by:
	 *	partial_history_cycles / total_history_cycles
	 */
	corr_raw = (u64)ktime_to_ns(
		ktime_sub(ts->sys_monoraw, history->raw));
	ret = scale64_check_overflow(partial_history_cycles,
				     total_history_cycles, &corr_raw);
	if (ret)
		return ret;

	/*
	 * If there is a discontinuity in the history, scale monotonic raw
	 *	correction by:
	 *	mult(real)/mult(raw) yielding the realtime correction
	 * Otherwise, calculate the realtime correction similar to monotonic
	 *	raw calculation
	 */
	if (discontinuity) {
		corr_real = mul_u64_u32_div
			(corr_raw, tk->tkr_mono.mult, tk->tkr_raw.mult);
	} else {
		corr_real = (u64)ktime_to_ns(
			ktime_sub(ts->sys_realtime, history->real));
		ret = scale64_check_overflow(partial_history_cycles,
					     total_history_cycles, &corr_real);
		if (ret)
			return ret;
	}

	/* Fixup monotonic raw and real time time values */
	if (interp_forward) {
		ts->sys_monoraw = ktime_add_ns(history->raw, corr_raw);
		ts->sys_realtime = ktime_add_ns(history->real, corr_real);
	} else {
		ts->sys_monoraw = ktime_sub_ns(ts->sys_monoraw, corr_raw);
		ts->sys_realtime = ktime_sub_ns(ts->sys_realtime, corr_real);
	}

	return 0;
}

/*
 * cycle_between - true if test occurs chronologically between before and after
 */
1080
static bool cycle_between(u64 before, u64 test, u64 after)
1081 1082 1083 1084 1085 1086 1087 1088
{
	if (test > before && test < after)
		return true;
	if (test < before && before > after)
		return true;
	return false;
}

1089 1090
/**
 * get_device_system_crosststamp - Synchronously capture system/device timestamp
1091
 * @get_time_fn:	Callback to get simultaneous device time and
1092
 *	system counter from the device driver
1093 1094 1095
 * @ctx:		Context passed to get_time_fn()
 * @history_begin:	Historical reference point used to interpolate system
 *	time when counter provided by the driver is before the current interval
1096 1097 1098 1099 1100 1101 1102 1103 1104
 * @xtstamp:		Receives simultaneously captured system and device time
 *
 * Reads a timestamp from a device and correlates it to system time
 */
int get_device_system_crosststamp(int (*get_time_fn)
				  (ktime_t *device_time,
				   struct system_counterval_t *sys_counterval,
				   void *ctx),
				  void *ctx,
1105
				  struct system_time_snapshot *history_begin,
1106 1107 1108 1109
				  struct system_device_crosststamp *xtstamp)
{
	struct system_counterval_t system_counterval;
	struct timekeeper *tk = &tk_core.timekeeper;
1110
	u64 cycles, now, interval_start;
1111
	unsigned int clock_was_set_seq = 0;
1112
	ktime_t base_real, base_raw;
1113
	u64 nsec_real, nsec_raw;
1114
	u8 cs_was_changed_seq;
1115
	unsigned long seq;
1116
	bool do_interp;
1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135
	int ret;

	do {
		seq = read_seqcount_begin(&tk_core.seq);
		/*
		 * Try to synchronously capture device time and a system
		 * counter value calling back into the device driver
		 */
		ret = get_time_fn(&xtstamp->device, &system_counterval, ctx);
		if (ret)
			return ret;

		/*
		 * Verify that the clocksource associated with the captured
		 * system counter value is the same as the currently installed
		 * timekeeper clocksource
		 */
		if (tk->tkr_mono.clock != system_counterval.cs)
			return -ENODEV;
1136 1137 1138 1139 1140 1141
		cycles = system_counterval.cycles;

		/*
		 * Check whether the system counter value provided by the
		 * device driver is on the current timekeeping interval.
		 */
1142
		now = tk_clock_read(&tk->tkr_mono);
1143 1144 1145 1146 1147 1148 1149 1150 1151
		interval_start = tk->tkr_mono.cycle_last;
		if (!cycle_between(interval_start, cycles, now)) {
			clock_was_set_seq = tk->clock_was_set_seq;
			cs_was_changed_seq = tk->cs_was_changed_seq;
			cycles = interval_start;
			do_interp = true;
		} else {
			do_interp = false;
		}
1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164

		base_real = ktime_add(tk->tkr_mono.base,
				      tk_core.timekeeper.offs_real);
		base_raw = tk->tkr_raw.base;

		nsec_real = timekeeping_cycles_to_ns(&tk->tkr_mono,
						     system_counterval.cycles);
		nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw,
						    system_counterval.cycles);
	} while (read_seqcount_retry(&tk_core.seq, seq));

	xtstamp->sys_realtime = ktime_add_ns(base_real, nsec_real);
	xtstamp->sys_monoraw = ktime_add_ns(base_raw, nsec_raw);
1165 1166 1167 1168 1169 1170

	/*
	 * Interpolate if necessary, adjusting back from the start of the
	 * current interval
	 */
	if (do_interp) {
1171
		u64 partial_history_cycles, total_history_cycles;
1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196
		bool discontinuity;

		/*
		 * Check that the counter value occurs after the provided
		 * history reference and that the history doesn't cross a
		 * clocksource change
		 */
		if (!history_begin ||
		    !cycle_between(history_begin->cycles,
				   system_counterval.cycles, cycles) ||
		    history_begin->cs_was_changed_seq != cs_was_changed_seq)
			return -EINVAL;
		partial_history_cycles = cycles - system_counterval.cycles;
		total_history_cycles = cycles - history_begin->cycles;
		discontinuity =
			history_begin->clock_was_set_seq != clock_was_set_seq;

		ret = adjust_historical_crosststamp(history_begin,
						    partial_history_cycles,
						    total_history_cycles,
						    discontinuity, xtstamp);
		if (ret)
			return ret;
	}

1197 1198 1199 1200
	return 0;
}
EXPORT_SYMBOL_GPL(get_device_system_crosststamp);

1201 1202 1203 1204
/**
 * do_gettimeofday - Returns the time of day in a timeval
 * @tv:		pointer to the timeval to be set
 *
1205
 * NOTE: Users should be converted to using getnstimeofday()
1206 1207 1208
 */
void do_gettimeofday(struct timeval *tv)
{
1209
	struct timespec64 now;
1210

1211
	getnstimeofday64(&now);
1212 1213 1214 1215
	tv->tv_sec = now.tv_sec;
	tv->tv_usec = now.tv_nsec/1000;
}
EXPORT_SYMBOL(do_gettimeofday);
1216

1217
/**
1218 1219
 * do_settimeofday64 - Sets the time of day.
 * @ts:     pointer to the timespec64 variable containing the new time
1220 1221 1222
 *
 * Sets the time of day to the new time and update NTP and notify hrtimers
 */
1223
int do_settimeofday64(const struct timespec64 *ts)
1224
{
1225
	struct timekeeper *tk = &tk_core.timekeeper;
1226
	struct timespec64 ts_delta, xt;
1227
	unsigned long flags;
1228
	int ret = 0;
1229

1230
	if (!timespec64_valid_strict(ts))
1231 1232
		return -EINVAL;

1233
	raw_spin_lock_irqsave(&timekeeper_lock, flags);
1234
	write_seqcount_begin(&tk_core.seq);
1235

1236
	timekeeping_forward_now(tk);
1237

1238
	xt = tk_xtime(tk);
1239 1240
	ts_delta.tv_sec = ts->tv_sec - xt.tv_sec;
	ts_delta.tv_nsec = ts->tv_nsec - xt.tv_nsec;
1241

1242 1243 1244 1245 1246
	if (timespec64_compare(&tk->wall_to_monotonic, &ts_delta) > 0) {
		ret = -EINVAL;
		goto out;
	}

1247
	tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, ts_delta));
1248

1249
	tk_set_xtime(tk, ts);
1250
out:
1251
	timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
1252

1253
	write_seqcount_end(&tk_core.seq);
1254
	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1255 1256 1257 1258

	/* signal hrtimers about time change */
	clock_was_set();

1259
	return ret;
1260
}
1261
EXPORT_SYMBOL(do_settimeofday64);
1262

1263 1264 1265 1266 1267 1268 1269 1270
/**
 * timekeeping_inject_offset - Adds or subtracts from the current time.
 * @tv:		pointer to the timespec variable containing the offset
 *
 * Adds or subtracts an offset value from the current time.
 */
int timekeeping_inject_offset(struct timespec *ts)
{
1271
	struct timekeeper *tk = &tk_core.timekeeper;
1272
	unsigned long flags;
1273
	struct timespec64 ts64, tmp;
1274
	int ret = 0;
1275

1276
	if (!timespec_inject_offset_valid(ts))
1277 1278
		return -EINVAL;

1279 1280
	ts64 = timespec_to_timespec64(*ts);

1281
	raw_spin_lock_irqsave(&timekeeper_lock, flags);
1282
	write_seqcount_begin(&tk_core.seq);
1283

1284
	timekeeping_forward_now(tk);
1285

1286
	/* Make sure the proposed value is valid */
1287
	tmp = timespec64_add(tk_xtime(tk),  ts64);
1288 1289
	if (timespec64_compare(&tk->wall_to_monotonic, &ts64) > 0 ||
	    !timespec64_valid_strict(&tmp)) {
1290 1291 1292
		ret = -EINVAL;
		goto error;
	}
1293

1294 1295
	tk_xtime_add(tk, &ts64);
	tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, ts64));
1296

1297
error: /* even if we error out, we forwarded the time, so call update */
1298
	timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
1299

1300
	write_seqcount_end(&tk_core.seq);
1301
	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1302 1303 1304 1305

	/* signal hrtimers about time change */
	clock_was_set();

1306
	return ret;
1307 1308 1309
}
EXPORT_SYMBOL(timekeeping_inject_offset);

1310
/**
1311
 * __timekeeping_set_tai_offset - Sets the TAI offset from UTC and monotonic
1312 1313
 *
 */
1314
static void __timekeeping_set_tai_offset(struct timekeeper *tk, s32 tai_offset)
1315 1316
{
	tk->tai_offset = tai_offset;
1317
	tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tai_offset, 0));
1318 1319
}

1320 1321 1322 1323 1324
/**
 * change_clocksource - Swaps clocksources if a new one is available
 *
 * Accumulates current time interval and initializes new clocksource
 */
1325
static int change_clocksource(void *data)
1326
{
1327
	struct timekeeper *tk = &tk_core.timekeeper;
1328
	struct clocksource *new, *old;
1329
	unsigned long flags;
1330

1331
	new = (struct clocksource *) data;
1332

1333
	raw_spin_lock_irqsave(&timekeeper_lock, flags);
1334
	write_seqcount_begin(&tk_core.seq);
1335

1336
	timekeeping_forward_now(tk);
1337 1338 1339 1340 1341 1342
	/*
	 * If the cs is in module, get a module reference. Succeeds
	 * for built-in code (owner == NULL) as well.
	 */
	if (try_module_get(new->owner)) {
		if (!new->enable || new->enable(new) == 0) {
1343
			old = tk->tkr_mono.clock;
1344 1345 1346 1347 1348 1349 1350
			tk_setup_internals(tk, new);
			if (old->disable)
				old->disable(old);
			module_put(old->owner);
		} else {
			module_put(new->owner);
		}
1351
	}
1352
	timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
1353

1354
	write_seqcount_end(&tk_core.seq);
1355
	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1356

1357 1358
	return 0;
}
1359

1360 1361 1362 1363 1364 1365 1366
/**
 * timekeeping_notify - Install a new clock source
 * @clock:		pointer to the clock source
 *
 * This function is called from clocksource.c after a new, better clock
 * source has been registered. The caller holds the clocksource_mutex.
 */
1367
int timekeeping_notify(struct clocksource *clock)
1368
{
1369
	struct timekeeper *tk = &tk_core.timekeeper;
1370

1371
	if (tk->tkr_mono.clock == clock)
1372
		return 0;
1373
	stop_machine(change_clocksource, clock, NULL);
1374
	tick_clock_notify();
1375
	return tk->tkr_mono.clock == clock ? 0 : -1;
1376
}
1377

1378
/**
1379 1380
 * getrawmonotonic64 - Returns the raw monotonic time in a timespec
 * @ts:		pointer to the timespec64 to be set
1381 1382 1383
 *
 * Returns the raw monotonic time (completely un-modified by ntp)
 */
1384
void getrawmonotonic64(struct timespec64 *ts)
1385
{
1386
	struct timekeeper *tk = &tk_core.timekeeper;
1387
	unsigned long seq;
1388
	u64 nsecs;
1389 1390

	do {
1391
		seq = read_seqcount_begin(&tk_core.seq);
1392
		ts->tv_sec = tk->raw_sec;
P
Peter Zijlstra 已提交
1393
		nsecs = timekeeping_get_ns(&tk->tkr_raw);
1394

1395
	} while (read_seqcount_retry(&tk_core.seq, seq));
1396

1397 1398
	ts->tv_nsec = 0;
	timespec64_add_ns(ts, nsecs);
1399
}
1400 1401
EXPORT_SYMBOL(getrawmonotonic64);

1402

1403
/**
1404
 * timekeeping_valid_for_hres - Check if timekeeping is suitable for hres
1405
 */
1406
int timekeeping_valid_for_hres(void)
1407
{
1408
	struct timekeeper *tk = &tk_core.timekeeper;
1409 1410 1411 1412
	unsigned long seq;
	int ret;

	do {
1413
		seq = read_seqcount_begin(&tk_core.seq);
1414

1415
		ret = tk->tkr_mono.clock->flags & CLOCK_SOURCE_VALID_FOR_HRES;
1416

1417
	} while (read_seqcount_retry(&tk_core.seq, seq));
1418 1419 1420 1421

	return ret;
}

1422 1423 1424 1425 1426
/**
 * timekeeping_max_deferment - Returns max time the clocksource can be deferred
 */
u64 timekeeping_max_deferment(void)
{
1427
	struct timekeeper *tk = &tk_core.timekeeper;
J
John Stultz 已提交
1428 1429
	unsigned long seq;
	u64 ret;
1430

J
John Stultz 已提交
1431
	do {
1432
		seq = read_seqcount_begin(&tk_core.seq);
J
John Stultz 已提交
1433

1434
		ret = tk->tkr_mono.clock->max_idle_ns;
J
John Stultz 已提交
1435

1436
	} while (read_seqcount_retry(&tk_core.seq, seq));
J
John Stultz 已提交
1437 1438

	return ret;
1439 1440
}

1441
/**
1442
 * read_persistent_clock -  Return time from the persistent clock.
1443 1444
 *
 * Weak dummy function for arches that do not yet support it.
1445 1446
 * Reads the time from the battery backed persistent clock.
 * Returns a timespec with tv_sec=0 and tv_nsec=0 if unsupported.
1447 1448 1449
 *
 *  XXX - Do be sure to remove it once all arches implement it.
 */
1450
void __weak read_persistent_clock(struct timespec *ts)
1451
{
1452 1453
	ts->tv_sec = 0;
	ts->tv_nsec = 0;
1454 1455
}

1456 1457 1458 1459 1460 1461 1462 1463
void __weak read_persistent_clock64(struct timespec64 *ts64)
{
	struct timespec ts;

	read_persistent_clock(&ts);
	*ts64 = timespec_to_timespec64(ts);
}

1464
/**
X
Xunlei Pang 已提交
1465
 * read_boot_clock64 -  Return time of the system start.
1466 1467 1468
 *
 * Weak dummy function for arches that do not yet support it.
 * Function to read the exact time the system has been started.
X
Xunlei Pang 已提交
1469
 * Returns a timespec64 with tv_sec=0 and tv_nsec=0 if unsupported.
1470 1471 1472
 *
 *  XXX - Do be sure to remove it once all arches implement it.
 */
X
Xunlei Pang 已提交
1473
void __weak read_boot_clock64(struct timespec64 *ts)
1474 1475 1476 1477 1478
{
	ts->tv_sec = 0;
	ts->tv_nsec = 0;
}

1479 1480 1481 1482 1483 1484
/* Flag for if timekeeping_resume() has injected sleeptime */
static bool sleeptime_injected;

/* Flag for if there is a persistent clock on this platform */
static bool persistent_clock_exists;

1485 1486 1487 1488 1489
/*
 * timekeeping_init - Initializes the clocksource and common timekeeping values
 */
void __init timekeeping_init(void)
{
1490
	struct timekeeper *tk = &tk_core.timekeeper;
1491
	struct clocksource *clock;
1492
	unsigned long flags;
1493
	struct timespec64 now, boot, tmp;
1494

1495
	read_persistent_clock64(&now);
1496
	if (!timespec64_valid_strict(&now)) {
1497 1498 1499 1500
		pr_warn("WARNING: Persistent clock returned invalid value!\n"
			"         Check your CMOS/BIOS settings.\n");
		now.tv_sec = 0;
		now.tv_nsec = 0;
1501
	} else if (now.tv_sec || now.tv_nsec)
1502
		persistent_clock_exists = true;
1503

1504
	read_boot_clock64(&boot);
1505
	if (!timespec64_valid_strict(&boot)) {
1506 1507 1508 1509 1510
		pr_warn("WARNING: Boot clock returned invalid value!\n"
			"         Check your CMOS/BIOS settings.\n");
		boot.tv_sec = 0;
		boot.tv_nsec = 0;
	}
1511

1512
	raw_spin_lock_irqsave(&timekeeper_lock, flags);
1513
	write_seqcount_begin(&tk_core.seq);
1514 1515
	ntp_init();

1516
	clock = clocksource_default_clock();
1517 1518
	if (clock->enable)
		clock->enable(clock);
1519
	tk_setup_internals(tk, clock);
1520

1521
	tk_set_xtime(tk, &now);
1522
	tk->raw_sec = 0;
1523
	if (boot.tv_sec == 0 && boot.tv_nsec == 0)
1524
		boot = tk_xtime(tk);
1525

1526
	set_normalized_timespec64(&tmp, -boot.tv_sec, -boot.tv_nsec);
1527
	tk_set_wall_to_mono(tk, tmp);
1528

1529
	timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
1530

1531
	write_seqcount_end(&tk_core.seq);
1532
	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1533 1534
}

1535
/* time in seconds when suspend began for persistent clock */
1536
static struct timespec64 timekeeping_suspend_time;
1537

1538 1539 1540 1541 1542 1543 1544
/**
 * __timekeeping_inject_sleeptime - Internal function to add sleep interval
 * @delta: pointer to a timespec delta value
 *
 * Takes a timespec offset measuring a suspend interval and properly
 * adds the sleep offset to the timekeeping variables.
 */
1545
static void __timekeeping_inject_sleeptime(struct timekeeper *tk,
1546
					   struct timespec64 *delta)
1547
{
1548
	if (!timespec64_valid_strict(delta)) {
1549 1550 1551
		printk_deferred(KERN_WARNING
				"__timekeeping_inject_sleeptime: Invalid "
				"sleep delta value!\n");
1552 1553
		return;
	}
1554
	tk_xtime_add(tk, delta);
1555
	tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, *delta));
1556
	tk_update_sleep_time(tk, timespec64_to_ktime(*delta));
1557
	tk_debug_account_sleep_time(delta);
1558 1559
}

1560
#if defined(CONFIG_PM_SLEEP) && defined(CONFIG_RTC_HCTOSYS_DEVICE)
1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595
/**
 * We have three kinds of time sources to use for sleep time
 * injection, the preference order is:
 * 1) non-stop clocksource
 * 2) persistent clock (ie: RTC accessible when irqs are off)
 * 3) RTC
 *
 * 1) and 2) are used by timekeeping, 3) by RTC subsystem.
 * If system has neither 1) nor 2), 3) will be used finally.
 *
 *
 * If timekeeping has injected sleeptime via either 1) or 2),
 * 3) becomes needless, so in this case we don't need to call
 * rtc_resume(), and this is what timekeeping_rtc_skipresume()
 * means.
 */
bool timekeeping_rtc_skipresume(void)
{
	return sleeptime_injected;
}

/**
 * 1) can be determined whether to use or not only when doing
 * timekeeping_resume() which is invoked after rtc_suspend(),
 * so we can't skip rtc_suspend() surely if system has 1).
 *
 * But if system has 2), 2) will definitely be used, so in this
 * case we don't need to call rtc_suspend(), and this is what
 * timekeeping_rtc_skipsuspend() means.
 */
bool timekeeping_rtc_skipsuspend(void)
{
	return persistent_clock_exists;
}

1596
/**
1597 1598
 * timekeeping_inject_sleeptime64 - Adds suspend interval to timeekeeping values
 * @delta: pointer to a timespec64 delta value
1599
 *
1600
 * This hook is for architectures that cannot support read_persistent_clock64
1601
 * because their RTC/persistent clock is only accessible when irqs are enabled.
1602
 * and also don't have an effective nonstop clocksource.
1603 1604 1605 1606
 *
 * This function should only be called by rtc_resume(), and allows
 * a suspend offset to be injected into the timekeeping values.
 */
1607
void timekeeping_inject_sleeptime64(struct timespec64 *delta)
1608
{
1609
	struct timekeeper *tk = &tk_core.timekeeper;
1610
	unsigned long flags;
1611

1612
	raw_spin_lock_irqsave(&timekeeper_lock, flags);
1613
	write_seqcount_begin(&tk_core.seq);
J
John Stultz 已提交
1614

1615
	timekeeping_forward_now(tk);
1616

1617
	__timekeeping_inject_sleeptime(tk, delta);
1618

1619
	timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
1620

1621
	write_seqcount_end(&tk_core.seq);
1622
	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1623 1624 1625 1626

	/* signal hrtimers about time change */
	clock_was_set();
}
1627
#endif
1628

1629 1630 1631
/**
 * timekeeping_resume - Resumes the generic timekeeping subsystem.
 */
1632
void timekeeping_resume(void)
1633
{
1634
	struct timekeeper *tk = &tk_core.timekeeper;
1635
	struct clocksource *clock = tk->tkr_mono.clock;
1636
	unsigned long flags;
1637
	struct timespec64 ts_new, ts_delta;
1638
	u64 cycle_now;
1639

1640
	sleeptime_injected = false;
1641
	read_persistent_clock64(&ts_new);
1642

1643
	clockevents_resume();
1644 1645
	clocksource_resume();

1646
	raw_spin_lock_irqsave(&timekeeper_lock, flags);
1647
	write_seqcount_begin(&tk_core.seq);
1648

1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660
	/*
	 * After system resumes, we need to calculate the suspended time and
	 * compensate it for the OS time. There are 3 sources that could be
	 * used: Nonstop clocksource during suspend, persistent clock and rtc
	 * device.
	 *
	 * One specific platform may have 1 or 2 or all of them, and the
	 * preference will be:
	 *	suspend-nonstop clocksource -> persistent clock -> rtc
	 * The less preferred source will only be tried if there is no better
	 * usable source. The rtc part is handled separately in rtc core code.
	 */
1661
	cycle_now = tk_clock_read(&tk->tkr_mono);
1662
	if ((clock->flags & CLOCK_SOURCE_SUSPEND_NONSTOP) &&
1663
		cycle_now > tk->tkr_mono.cycle_last) {
1664
		u64 nsec, cyc_delta;
1665

1666 1667 1668
		cyc_delta = clocksource_delta(cycle_now, tk->tkr_mono.cycle_last,
					      tk->tkr_mono.mask);
		nsec = mul_u64_u32_shr(cyc_delta, clock->mult, clock->shift);
1669
		ts_delta = ns_to_timespec64(nsec);
1670
		sleeptime_injected = true;
1671 1672
	} else if (timespec64_compare(&ts_new, &timekeeping_suspend_time) > 0) {
		ts_delta = timespec64_sub(ts_new, timekeeping_suspend_time);
1673
		sleeptime_injected = true;
1674
	}
1675

1676
	if (sleeptime_injected)
1677 1678 1679
		__timekeeping_inject_sleeptime(tk, &ts_delta);

	/* Re-base the last cycle value */
1680
	tk->tkr_mono.cycle_last = cycle_now;
P
Peter Zijlstra 已提交
1681 1682
	tk->tkr_raw.cycle_last  = cycle_now;

1683
	tk->ntp_error = 0;
1684
	timekeeping_suspended = 0;
1685
	timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
1686
	write_seqcount_end(&tk_core.seq);
1687
	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1688 1689 1690

	touch_softlockup_watchdog();

1691
	tick_resume();
1692
	hrtimers_resume();
1693 1694
}

1695
int timekeeping_suspend(void)
1696
{
1697
	struct timekeeper *tk = &tk_core.timekeeper;
1698
	unsigned long flags;
1699 1700
	struct timespec64		delta, delta_delta;
	static struct timespec64	old_delta;
1701

1702
	read_persistent_clock64(&timekeeping_suspend_time);
1703

1704 1705 1706 1707 1708 1709
	/*
	 * On some systems the persistent_clock can not be detected at
	 * timekeeping_init by its return value, so if we see a valid
	 * value returned, update the persistent_clock_exists flag.
	 */
	if (timekeeping_suspend_time.tv_sec || timekeeping_suspend_time.tv_nsec)
1710
		persistent_clock_exists = true;
1711

1712
	raw_spin_lock_irqsave(&timekeeper_lock, flags);
1713
	write_seqcount_begin(&tk_core.seq);
1714
	timekeeping_forward_now(tk);
1715
	timekeeping_suspended = 1;
1716

1717
	if (persistent_clock_exists) {
1718
		/*
1719 1720 1721 1722
		 * To avoid drift caused by repeated suspend/resumes,
		 * which each can add ~1 second drift error,
		 * try to compensate so the difference in system time
		 * and persistent_clock time stays close to constant.
1723
		 */
1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736
		delta = timespec64_sub(tk_xtime(tk), timekeeping_suspend_time);
		delta_delta = timespec64_sub(delta, old_delta);
		if (abs(delta_delta.tv_sec) >= 2) {
			/*
			 * if delta_delta is too large, assume time correction
			 * has occurred and set old_delta to the current delta.
			 */
			old_delta = delta;
		} else {
			/* Otherwise try to adjust old_system to compensate */
			timekeeping_suspend_time =
				timespec64_add(timekeeping_suspend_time, delta_delta);
		}
1737
	}
1738 1739

	timekeeping_update(tk, TK_MIRROR);
1740
	halt_fast_timekeeper(tk);
1741
	write_seqcount_end(&tk_core.seq);
1742
	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1743

1744
	tick_suspend();
M
Magnus Damm 已提交
1745
	clocksource_suspend();
1746
	clockevents_suspend();
1747 1748 1749 1750 1751

	return 0;
}

/* sysfs resume/suspend bits for timekeeping */
1752
static struct syscore_ops timekeeping_syscore_ops = {
1753 1754 1755 1756
	.resume		= timekeeping_resume,
	.suspend	= timekeeping_suspend,
};

1757
static int __init timekeeping_init_ops(void)
1758
{
1759 1760
	register_syscore_ops(&timekeeping_syscore_ops);
	return 0;
1761
}
1762
device_initcall(timekeeping_init_ops);
1763 1764

/*
1765
 * Apply a multiplier adjustment to the timekeeper
1766
 */
1767 1768 1769 1770
static __always_inline void timekeeping_apply_adjustment(struct timekeeper *tk,
							 s64 offset,
							 bool negative,
							 int adj_scale)
1771
{
1772 1773
	s64 interval = tk->cycle_interval;
	s32 mult_adj = 1;
1774

1775 1776 1777 1778
	if (negative) {
		mult_adj = -mult_adj;
		interval = -interval;
		offset  = -offset;
1779
	}
1780 1781 1782
	mult_adj <<= adj_scale;
	interval <<= adj_scale;
	offset <<= adj_scale;
1783

1784 1785 1786
	/*
	 * So the following can be confusing.
	 *
1787
	 * To keep things simple, lets assume mult_adj == 1 for now.
1788
	 *
1789
	 * When mult_adj != 1, remember that the interval and offset values
1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832
	 * have been appropriately scaled so the math is the same.
	 *
	 * The basic idea here is that we're increasing the multiplier
	 * by one, this causes the xtime_interval to be incremented by
	 * one cycle_interval. This is because:
	 *	xtime_interval = cycle_interval * mult
	 * So if mult is being incremented by one:
	 *	xtime_interval = cycle_interval * (mult + 1)
	 * Its the same as:
	 *	xtime_interval = (cycle_interval * mult) + cycle_interval
	 * Which can be shortened to:
	 *	xtime_interval += cycle_interval
	 *
	 * So offset stores the non-accumulated cycles. Thus the current
	 * time (in shifted nanoseconds) is:
	 *	now = (offset * adj) + xtime_nsec
	 * Now, even though we're adjusting the clock frequency, we have
	 * to keep time consistent. In other words, we can't jump back
	 * in time, and we also want to avoid jumping forward in time.
	 *
	 * So given the same offset value, we need the time to be the same
	 * both before and after the freq adjustment.
	 *	now = (offset * adj_1) + xtime_nsec_1
	 *	now = (offset * adj_2) + xtime_nsec_2
	 * So:
	 *	(offset * adj_1) + xtime_nsec_1 =
	 *		(offset * adj_2) + xtime_nsec_2
	 * And we know:
	 *	adj_2 = adj_1 + 1
	 * So:
	 *	(offset * adj_1) + xtime_nsec_1 =
	 *		(offset * (adj_1+1)) + xtime_nsec_2
	 *	(offset * adj_1) + xtime_nsec_1 =
	 *		(offset * adj_1) + offset + xtime_nsec_2
	 * Canceling the sides:
	 *	xtime_nsec_1 = offset + xtime_nsec_2
	 * Which gives us:
	 *	xtime_nsec_2 = xtime_nsec_1 - offset
	 * Which simplfies to:
	 *	xtime_nsec -= offset
	 *
	 * XXX - TODO: Doc ntp_error calculation.
	 */
1833
	if ((mult_adj > 0) && (tk->tkr_mono.mult + mult_adj < mult_adj)) {
1834 1835 1836 1837 1838
		/* NTP adjustment caused clocksource mult overflow */
		WARN_ON_ONCE(1);
		return;
	}

1839
	tk->tkr_mono.mult += mult_adj;
1840
	tk->xtime_interval += interval;
1841
	tk->tkr_mono.xtime_nsec -= offset;
1842
	tk->ntp_error -= (interval - offset) << tk->ntp_error_shift;
1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853
}

/*
 * Calculate the multiplier adjustment needed to match the frequency
 * specified by NTP
 */
static __always_inline void timekeeping_freqadjust(struct timekeeper *tk,
							s64 offset)
{
	s64 interval = tk->cycle_interval;
	s64 xinterval = tk->xtime_interval;
1854 1855 1856
	u32 base = tk->tkr_mono.clock->mult;
	u32 max = tk->tkr_mono.clock->maxadj;
	u32 cur_adj = tk->tkr_mono.mult;
1857 1858
	s64 tick_error;
	bool negative;
1859
	u32 adj_scale;
1860 1861 1862 1863 1864

	/* Remove any current error adj from freq calculation */
	if (tk->ntp_err_mult)
		xinterval -= tk->cycle_interval;

1865 1866
	tk->ntp_tick = ntp_tick_length();

1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877
	/* Calculate current error per tick */
	tick_error = ntp_tick_length() >> tk->ntp_error_shift;
	tick_error -= (xinterval + tk->xtime_remainder);

	/* Don't worry about correcting it if its small */
	if (likely((tick_error >= 0) && (tick_error <= interval)))
		return;

	/* preserve the direction of correction */
	negative = (tick_error < 0);

1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888
	/* If any adjustment would pass the max, just return */
	if (negative && (cur_adj - 1) <= (base - max))
		return;
	if (!negative && (cur_adj + 1) >= (base + max))
		return;
	/*
	 * Sort out the magnitude of the correction, but
	 * avoid making so large a correction that we go
	 * over the max adjustment.
	 */
	adj_scale = 0;
A
Andrew Morton 已提交
1889
	tick_error = abs(tick_error);
1890 1891 1892 1893 1894 1895 1896 1897 1898 1899
	while (tick_error > interval) {
		u32 adj = 1 << (adj_scale + 1);

		/* Check if adjustment gets us within 1 unit from the max */
		if (negative && (cur_adj - adj) <= (base - max))
			break;
		if (!negative && (cur_adj + adj) >= (base + max))
			break;

		adj_scale++;
1900
		tick_error >>= 1;
1901
	}
1902 1903

	/* scale the corrections */
1904
	timekeeping_apply_adjustment(tk, offset, negative, adj_scale);
1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925
}

/*
 * Adjust the timekeeper's multiplier to the correct frequency
 * and also to reduce the accumulated error value.
 */
static void timekeeping_adjust(struct timekeeper *tk, s64 offset)
{
	/* Correct for the current frequency error */
	timekeeping_freqadjust(tk, offset);

	/* Next make a small adjustment to fix any cumulative error */
	if (!tk->ntp_err_mult && (tk->ntp_error > 0)) {
		tk->ntp_err_mult = 1;
		timekeeping_apply_adjustment(tk, offset, 0, 0);
	} else if (tk->ntp_err_mult && (tk->ntp_error <= 0)) {
		/* Undo any existing error adjustment */
		timekeeping_apply_adjustment(tk, offset, 1, 0);
		tk->ntp_err_mult = 0;
	}

1926 1927 1928
	if (unlikely(tk->tkr_mono.clock->maxadj &&
		(abs(tk->tkr_mono.mult - tk->tkr_mono.clock->mult)
			> tk->tkr_mono.clock->maxadj))) {
1929 1930
		printk_once(KERN_WARNING
			"Adjusting %s more than 11%% (%ld vs %ld)\n",
1931 1932
			tk->tkr_mono.clock->name, (long)tk->tkr_mono.mult,
			(long)tk->tkr_mono.clock->mult + tk->tkr_mono.clock->maxadj);
1933
	}
1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948

	/*
	 * It may be possible that when we entered this function, xtime_nsec
	 * was very small.  Further, if we're slightly speeding the clocksource
	 * in the code above, its possible the required corrective factor to
	 * xtime_nsec could cause it to underflow.
	 *
	 * Now, since we already accumulated the second, cannot simply roll
	 * the accumulated second back, since the NTP subsystem has been
	 * notified via second_overflow. So instead we push xtime_nsec forward
	 * by the amount we underflowed, and add that amount into the error.
	 *
	 * We'll correct this error next time through this function, when
	 * xtime_nsec is not as small.
	 */
1949 1950 1951
	if (unlikely((s64)tk->tkr_mono.xtime_nsec < 0)) {
		s64 neg = -(s64)tk->tkr_mono.xtime_nsec;
		tk->tkr_mono.xtime_nsec = 0;
1952
		tk->ntp_error += neg << tk->ntp_error_shift;
1953
	}
1954 1955
}

1956 1957 1958
/**
 * accumulate_nsecs_to_secs - Accumulates nsecs into secs
 *
Z
Zhen Lei 已提交
1959
 * Helper function that accumulates the nsecs greater than a second
1960 1961 1962 1963
 * from the xtime_nsec field to the xtime_secs field.
 * It also calls into the NTP code to handle leapsecond processing.
 *
 */
1964
static inline unsigned int accumulate_nsecs_to_secs(struct timekeeper *tk)
1965
{
1966
	u64 nsecps = (u64)NSEC_PER_SEC << tk->tkr_mono.shift;
1967
	unsigned int clock_set = 0;
1968

1969
	while (tk->tkr_mono.xtime_nsec >= nsecps) {
1970 1971
		int leap;

1972
		tk->tkr_mono.xtime_nsec -= nsecps;
1973 1974 1975 1976
		tk->xtime_sec++;

		/* Figure out if its a leap sec and apply if needed */
		leap = second_overflow(tk->xtime_sec);
1977
		if (unlikely(leap)) {
1978
			struct timespec64 ts;
1979 1980

			tk->xtime_sec += leap;
1981

1982 1983 1984
			ts.tv_sec = leap;
			ts.tv_nsec = 0;
			tk_set_wall_to_mono(tk,
1985
				timespec64_sub(tk->wall_to_monotonic, ts));
1986

1987 1988
			__timekeeping_set_tai_offset(tk, tk->tai_offset - leap);

1989
			clock_set = TK_CLOCK_WAS_SET;
1990
		}
1991
	}
1992
	return clock_set;
1993 1994
}

1995 1996 1997 1998 1999 2000 2001 2002 2003
/**
 * logarithmic_accumulation - shifted accumulation of cycles
 *
 * This functions accumulates a shifted interval of cycles into
 * into a shifted interval nanoseconds. Allows for O(log) accumulation
 * loop.
 *
 * Returns the unconsumed cycles.
 */
2004 2005
static u64 logarithmic_accumulation(struct timekeeper *tk, u64 offset,
				    u32 shift, unsigned int *clock_set)
2006
{
2007
	u64 interval = tk->cycle_interval << shift;
2008
	u64 snsec_per_sec;
2009

Z
Zhen Lei 已提交
2010
	/* If the offset is smaller than a shifted interval, do nothing */
T
Thomas Gleixner 已提交
2011
	if (offset < interval)
2012 2013 2014
		return offset;

	/* Accumulate one shifted interval */
T
Thomas Gleixner 已提交
2015
	offset -= interval;
2016
	tk->tkr_mono.cycle_last += interval;
P
Peter Zijlstra 已提交
2017
	tk->tkr_raw.cycle_last  += interval;
2018

2019
	tk->tkr_mono.xtime_nsec += tk->xtime_interval << shift;
2020
	*clock_set |= accumulate_nsecs_to_secs(tk);
2021

2022
	/* Accumulate raw time */
2023 2024 2025 2026
	tk->tkr_raw.xtime_nsec += tk->raw_interval << shift;
	snsec_per_sec = (u64)NSEC_PER_SEC << tk->tkr_raw.shift;
	while (tk->tkr_raw.xtime_nsec >= snsec_per_sec) {
		tk->tkr_raw.xtime_nsec -= snsec_per_sec;
2027
		tk->raw_sec++;
2028 2029 2030
	}

	/* Accumulate error between NTP and clock interval */
2031
	tk->ntp_error += tk->ntp_tick << shift;
2032 2033
	tk->ntp_error -= (tk->xtime_interval + tk->xtime_remainder) <<
						(tk->ntp_error_shift + shift);
2034 2035 2036 2037

	return offset;
}

2038 2039 2040 2041
/**
 * update_wall_time - Uses the current clocksource to increment the wall time
 *
 */
2042
void update_wall_time(void)
2043
{
2044
	struct timekeeper *real_tk = &tk_core.timekeeper;
2045
	struct timekeeper *tk = &shadow_timekeeper;
2046
	u64 offset;
2047
	int shift = 0, maxshift;
2048
	unsigned int clock_set = 0;
J
John Stultz 已提交
2049 2050
	unsigned long flags;

2051
	raw_spin_lock_irqsave(&timekeeper_lock, flags);
2052 2053 2054

	/* Make sure we're fully resumed: */
	if (unlikely(timekeeping_suspended))
J
John Stultz 已提交
2055
		goto out;
2056

J
John Stultz 已提交
2057
#ifdef CONFIG_ARCH_USES_GETTIMEOFFSET
2058
	offset = real_tk->cycle_interval;
J
John Stultz 已提交
2059
#else
2060
	offset = clocksource_delta(tk_clock_read(&tk->tkr_mono),
2061
				   tk->tkr_mono.cycle_last, tk->tkr_mono.mask);
2062 2063
#endif

2064
	/* Check if there's really nothing to do */
2065
	if (offset < real_tk->cycle_interval)
2066 2067
		goto out;

2068 2069 2070
	/* Do some additional sanity checking */
	timekeeping_check_update(real_tk, offset);

2071 2072 2073 2074
	/*
	 * With NO_HZ we may have to accumulate many cycle_intervals
	 * (think "ticks") worth of time at once. To do this efficiently,
	 * we calculate the largest doubling multiple of cycle_intervals
2075
	 * that is smaller than the offset.  We then accumulate that
2076 2077
	 * chunk in one go, and then try to consume the next smaller
	 * doubled multiple.
2078
	 */
2079
	shift = ilog2(offset) - ilog2(tk->cycle_interval);
2080
	shift = max(0, shift);
2081
	/* Bound shift to one less than what overflows tick_length */
2082
	maxshift = (64 - (ilog2(ntp_tick_length())+1)) - 1;
2083
	shift = min(shift, maxshift);
2084
	while (offset >= tk->cycle_interval) {
2085 2086
		offset = logarithmic_accumulation(tk, offset, shift,
							&clock_set);
2087
		if (offset < tk->cycle_interval<<shift)
2088
			shift--;
2089 2090 2091
	}

	/* correct the clock when NTP error is too big */
2092
	timekeeping_adjust(tk, offset);
2093

J
John Stultz 已提交
2094
	/*
2095 2096 2097 2098
	 * XXX This can be killed once everyone converts
	 * to the new update_vsyscall.
	 */
	old_vsyscall_fixup(tk);
2099

J
John Stultz 已提交
2100 2101
	/*
	 * Finally, make sure that after the rounding
2102
	 * xtime_nsec isn't larger than NSEC_PER_SEC
J
John Stultz 已提交
2103
	 */
2104
	clock_set |= accumulate_nsecs_to_secs(tk);
L
Linus Torvalds 已提交
2105

2106
	write_seqcount_begin(&tk_core.seq);
2107 2108 2109 2110 2111 2112 2113
	/*
	 * Update the real timekeeper.
	 *
	 * We could avoid this memcpy by switching pointers, but that
	 * requires changes to all other timekeeper usage sites as
	 * well, i.e. move the timekeeper pointer getter into the
	 * spinlocked/seqcount protected sections. And we trade this
2114
	 * memcpy under the tk_core.seq against one before we start
2115 2116
	 * updating.
	 */
2117
	timekeeping_update(tk, clock_set);
2118
	memcpy(real_tk, tk, sizeof(*tk));
2119
	/* The memcpy must come last. Do not put anything here! */
2120
	write_seqcount_end(&tk_core.seq);
2121
out:
2122
	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
2123
	if (clock_set)
2124 2125
		/* Have to call _delayed version, since in irq context*/
		clock_was_set_delayed();
2126
}
T
Tomas Janousek 已提交
2127 2128

/**
2129 2130
 * getboottime64 - Return the real time of system boot.
 * @ts:		pointer to the timespec64 to be set
T
Tomas Janousek 已提交
2131
 *
2132
 * Returns the wall-time of boot in a timespec64.
T
Tomas Janousek 已提交
2133 2134 2135 2136 2137 2138
 *
 * This is based on the wall_to_monotonic offset and the total suspend
 * time. Calls to settimeofday will affect the value returned (which
 * basically means that however wrong your real time clock is at boot time,
 * you get the right time here).
 */
2139
void getboottime64(struct timespec64 *ts)
T
Tomas Janousek 已提交
2140
{
2141
	struct timekeeper *tk = &tk_core.timekeeper;
2142 2143
	ktime_t t = ktime_sub(tk->offs_real, tk->offs_boot);

2144
	*ts = ktime_to_timespec64(t);
T
Tomas Janousek 已提交
2145
}
2146
EXPORT_SYMBOL_GPL(getboottime64);
T
Tomas Janousek 已提交
2147

2148 2149
unsigned long get_seconds(void)
{
2150
	struct timekeeper *tk = &tk_core.timekeeper;
2151 2152

	return tk->xtime_sec;
2153 2154 2155
}
EXPORT_SYMBOL(get_seconds);

2156 2157
struct timespec __current_kernel_time(void)
{
2158
	struct timekeeper *tk = &tk_core.timekeeper;
2159

2160
	return timespec64_to_timespec(tk_xtime(tk));
2161
}
2162

2163
struct timespec64 current_kernel_time64(void)
2164
{
2165
	struct timekeeper *tk = &tk_core.timekeeper;
2166
	struct timespec64 now;
2167 2168 2169
	unsigned long seq;

	do {
2170
		seq = read_seqcount_begin(&tk_core.seq);
L
Linus Torvalds 已提交
2171

2172
		now = tk_xtime(tk);
2173
	} while (read_seqcount_retry(&tk_core.seq, seq));
2174

2175
	return now;
2176
}
2177
EXPORT_SYMBOL(current_kernel_time64);
2178

2179
struct timespec64 get_monotonic_coarse64(void)
2180
{
2181
	struct timekeeper *tk = &tk_core.timekeeper;
2182
	struct timespec64 now, mono;
2183 2184 2185
	unsigned long seq;

	do {
2186
		seq = read_seqcount_begin(&tk_core.seq);
L
Linus Torvalds 已提交
2187

2188 2189
		now = tk_xtime(tk);
		mono = tk->wall_to_monotonic;
2190
	} while (read_seqcount_retry(&tk_core.seq, seq));
2191

2192
	set_normalized_timespec64(&now, now.tv_sec + mono.tv_sec,
2193
				now.tv_nsec + mono.tv_nsec);
2194

2195
	return now;
2196
}
2197
EXPORT_SYMBOL(get_monotonic_coarse64);
2198 2199

/*
2200
 * Must hold jiffies_lock
2201 2202 2203 2204 2205 2206
 */
void do_timer(unsigned long ticks)
{
	jiffies_64 += ticks;
	calc_global_load(ticks);
}
2207

2208
/**
2209
 * ktime_get_update_offsets_now - hrtimer helper
2210
 * @cwsseq:	pointer to check and store the clock was set sequence number
2211 2212
 * @offs_real:	pointer to storage for monotonic -> realtime offset
 * @offs_boot:	pointer to storage for monotonic -> boottime offset
2213
 * @offs_tai:	pointer to storage for monotonic -> clock tai offset
2214
 *
2215 2216 2217 2218
 * Returns current monotonic time and updates the offsets if the
 * sequence number in @cwsseq and timekeeper.clock_was_set_seq are
 * different.
 *
2219
 * Called from hrtimer_interrupt() or retrigger_next_event()
2220
 */
2221 2222
ktime_t ktime_get_update_offsets_now(unsigned int *cwsseq, ktime_t *offs_real,
				     ktime_t *offs_boot, ktime_t *offs_tai)
2223
{
2224
	struct timekeeper *tk = &tk_core.timekeeper;
2225
	unsigned int seq;
2226 2227
	ktime_t base;
	u64 nsecs;
2228 2229

	do {
2230
		seq = read_seqcount_begin(&tk_core.seq);
2231

2232 2233
		base = tk->tkr_mono.base;
		nsecs = timekeeping_get_ns(&tk->tkr_mono);
2234 2235
		base = ktime_add_ns(base, nsecs);

2236 2237 2238 2239 2240 2241
		if (*cwsseq != tk->clock_was_set_seq) {
			*cwsseq = tk->clock_was_set_seq;
			*offs_real = tk->offs_real;
			*offs_boot = tk->offs_boot;
			*offs_tai = tk->offs_tai;
		}
2242 2243

		/* Handle leapsecond insertion adjustments */
T
Thomas Gleixner 已提交
2244
		if (unlikely(base >= tk->next_leap_ktime))
2245 2246
			*offs_real = ktime_sub(tk->offs_real, ktime_set(1, 0));

2247
	} while (read_seqcount_retry(&tk_core.seq, seq));
2248

2249
	return base;
2250 2251
}

2252 2253 2254 2255 2256
/**
 * do_adjtimex() - Accessor function to NTP __do_adjtimex function
 */
int do_adjtimex(struct timex *txc)
{
2257
	struct timekeeper *tk = &tk_core.timekeeper;
2258
	unsigned long flags;
2259
	struct timespec64 ts;
2260
	s32 orig_tai, tai;
2261 2262 2263 2264 2265 2266 2267
	int ret;

	/* Validate the data before disabling interrupts */
	ret = ntp_validate_timex(txc);
	if (ret)
		return ret;

2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278
	if (txc->modes & ADJ_SETOFFSET) {
		struct timespec delta;
		delta.tv_sec  = txc->time.tv_sec;
		delta.tv_nsec = txc->time.tv_usec;
		if (!(txc->modes & ADJ_NANO))
			delta.tv_nsec *= 1000;
		ret = timekeeping_inject_offset(&delta);
		if (ret)
			return ret;
	}

2279
	getnstimeofday64(&ts);
2280

2281
	raw_spin_lock_irqsave(&timekeeper_lock, flags);
2282
	write_seqcount_begin(&tk_core.seq);
2283

2284
	orig_tai = tai = tk->tai_offset;
2285
	ret = __do_adjtimex(txc, &ts, &tai);
2286

2287 2288
	if (tai != orig_tai) {
		__timekeeping_set_tai_offset(tk, tai);
2289
		timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
2290
	}
2291 2292
	tk_update_leap_state(tk);

2293
	write_seqcount_end(&tk_core.seq);
2294 2295
	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);

2296 2297 2298
	if (tai != orig_tai)
		clock_was_set();

2299 2300
	ntp_notify_cmos_timer();

2301 2302
	return ret;
}
2303 2304 2305 2306 2307

#ifdef CONFIG_NTP_PPS
/**
 * hardpps() - Accessor function to NTP __hardpps function
 */
2308
void hardpps(const struct timespec64 *phase_ts, const struct timespec64 *raw_ts)
2309
{
2310 2311 2312
	unsigned long flags;

	raw_spin_lock_irqsave(&timekeeper_lock, flags);
2313
	write_seqcount_begin(&tk_core.seq);
2314

2315
	__hardpps(phase_ts, raw_ts);
2316

2317
	write_seqcount_end(&tk_core.seq);
2318
	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
2319 2320 2321 2322
}
EXPORT_SYMBOL(hardpps);
#endif

T
Torben Hohn 已提交
2323 2324 2325 2326 2327 2328 2329 2330
/**
 * xtime_update() - advances the timekeeping infrastructure
 * @ticks:	number of ticks, that have elapsed since the last call.
 *
 * Must be called with interrupts disabled.
 */
void xtime_update(unsigned long ticks)
{
2331
	write_seqlock(&jiffies_lock);
T
Torben Hohn 已提交
2332
	do_timer(ticks);
2333
	write_sequnlock(&jiffies_lock);
2334
	update_wall_time();
T
Torben Hohn 已提交
2335
}