timekeeping.c 51.9 KB
Newer Older
1 2 3 4 5 6 7 8 9 10
/*
 *  linux/kernel/time/timekeeping.c
 *
 *  Kernel timekeeping code and accessor functions
 *
 *  This code was moved from linux/kernel/timer.c.
 *  Please see that file for copyright and history logs.
 *
 */

11
#include <linux/timekeeper_internal.h>
12 13 14 15 16
#include <linux/module.h>
#include <linux/interrupt.h>
#include <linux/percpu.h>
#include <linux/init.h>
#include <linux/mm.h>
17
#include <linux/sched.h>
18
#include <linux/syscore_ops.h>
19 20 21 22
#include <linux/clocksource.h>
#include <linux/jiffies.h>
#include <linux/time.h>
#include <linux/tick.h>
23
#include <linux/stop_machine.h>
24
#include <linux/pvclock_gtod.h>
25
#include <linux/compiler.h>
26

27
#include "tick-internal.h"
28
#include "ntp_internal.h"
29
#include "timekeeping_internal.h"
30

31 32
#define TK_CLEAR_NTP		(1 << 0)
#define TK_MIRROR		(1 << 1)
33
#define TK_CLOCK_WAS_SET	(1 << 2)
34

35 36 37 38 39 40 41 42 43
/*
 * The most important data for readout fits into a single 64 byte
 * cache line.
 */
static struct {
	seqcount_t		seq;
	struct timekeeper	timekeeper;
} tk_core ____cacheline_aligned;

44
static DEFINE_RAW_SPINLOCK(timekeeper_lock);
45
static struct timekeeper shadow_timekeeper;
46

47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62
/**
 * struct tk_fast - NMI safe timekeeper
 * @seq:	Sequence counter for protecting updates. The lowest bit
 *		is the index for the tk_read_base array
 * @base:	tk_read_base array. Access is indexed by the lowest bit of
 *		@seq.
 *
 * See @update_fast_timekeeper() below.
 */
struct tk_fast {
	seqcount_t		seq;
	struct tk_read_base	base[2];
};

static struct tk_fast tk_fast_mono ____cacheline_aligned;

63 64 65
/* flag for if timekeeping is suspended */
int __read_mostly timekeeping_suspended;

66 67 68
/* Flag for if there is a persistent clock on this platform */
bool __read_mostly persistent_clock_exist = false;

69 70
static inline void tk_normalize_xtime(struct timekeeper *tk)
{
71 72
	while (tk->tkr.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr.shift)) {
		tk->tkr.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr.shift;
73 74 75 76
		tk->xtime_sec++;
	}
}

77 78 79 80 81
static inline struct timespec64 tk_xtime(struct timekeeper *tk)
{
	struct timespec64 ts;

	ts.tv_sec = tk->xtime_sec;
82
	ts.tv_nsec = (long)(tk->tkr.xtime_nsec >> tk->tkr.shift);
83 84 85
	return ts;
}

86
static void tk_set_xtime(struct timekeeper *tk, const struct timespec64 *ts)
87 88
{
	tk->xtime_sec = ts->tv_sec;
89
	tk->tkr.xtime_nsec = (u64)ts->tv_nsec << tk->tkr.shift;
90 91
}

92
static void tk_xtime_add(struct timekeeper *tk, const struct timespec64 *ts)
93 94
{
	tk->xtime_sec += ts->tv_sec;
95
	tk->tkr.xtime_nsec += (u64)ts->tv_nsec << tk->tkr.shift;
96
	tk_normalize_xtime(tk);
97
}
98

99
static void tk_set_wall_to_mono(struct timekeeper *tk, struct timespec64 wtm)
100
{
101
	struct timespec64 tmp;
102 103 104 105 106

	/*
	 * Verify consistency of: offset_real = -wall_to_monotonic
	 * before modifying anything
	 */
107
	set_normalized_timespec64(&tmp, -tk->wall_to_monotonic.tv_sec,
108
					-tk->wall_to_monotonic.tv_nsec);
109
	WARN_ON_ONCE(tk->offs_real.tv64 != timespec64_to_ktime(tmp).tv64);
110
	tk->wall_to_monotonic = wtm;
111 112
	set_normalized_timespec64(&tmp, -wtm.tv_sec, -wtm.tv_nsec);
	tk->offs_real = timespec64_to_ktime(tmp);
113
	tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tk->tai_offset, 0));
114 115
}

116
static inline void tk_update_sleep_time(struct timekeeper *tk, ktime_t delta)
117
{
118
	tk->offs_boot = ktime_add(tk->offs_boot, delta);
119 120
}

121 122 123 124 125 126 127 128
#ifdef CONFIG_DEBUG_TIMEKEEPING
static void timekeeping_check_update(struct timekeeper *tk, cycle_t offset)
{

	cycle_t max_cycles = tk->tkr.clock->max_cycles;
	const char *name = tk->tkr.clock->name;

	if (offset > max_cycles) {
129
		printk_deferred("WARNING: timekeeping: Cycle offset (%lld) is larger than allowed by the '%s' clock's max_cycles value (%lld): time overflow danger\n",
130
				offset, name, max_cycles);
131
		printk_deferred("         timekeeping: Your kernel is sick, but tries to cope by capping time updates\n");
132 133 134 135 136 137 138 139
	} else {
		if (offset > (max_cycles >> 1)) {
			printk_deferred("INFO: timekeeping: Cycle offset (%lld) is larger than the the '%s' clock's 50%% safety margin (%lld)\n",
					offset, name, max_cycles >> 1);
			printk_deferred("      timekeeping: Your kernel is still fine, but is feeling a bit nervous\n");
		}
	}
}
140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156

static inline cycle_t timekeeping_get_delta(struct tk_read_base *tkr)
{
	cycle_t cycle_now, delta;

	/* read clocksource */
	cycle_now = tkr->read(tkr->clock);

	/* calculate the delta since the last update_wall_time */
	delta = clocksource_delta(cycle_now, tkr->cycle_last, tkr->mask);

	/* Cap delta value to the max_cycles values to avoid mult overflows */
	if (unlikely(delta > tkr->clock->max_cycles))
		delta = tkr->clock->max_cycles;

	return delta;
}
157 158 159 160
#else
static inline void timekeeping_check_update(struct timekeeper *tk, cycle_t offset)
{
}
161 162 163 164 165 166 167 168 169 170 171 172
static inline cycle_t timekeeping_get_delta(struct tk_read_base *tkr)
{
	cycle_t cycle_now, delta;

	/* read clocksource */
	cycle_now = tkr->read(tkr->clock);

	/* calculate the delta since the last update_wall_time */
	delta = clocksource_delta(cycle_now, tkr->cycle_last, tkr->mask);

	return delta;
}
173 174
#endif

175
/**
176
 * tk_setup_internals - Set up internals to use clocksource clock.
177
 *
178
 * @tk:		The target timekeeper to setup.
179 180 181 182 183 184 185
 * @clock:		Pointer to clocksource.
 *
 * Calculates a fixed cycle/nsec interval for a given clocksource/adjustment
 * pair and interval request.
 *
 * Unless you're the timekeeping code, you should not be using this!
 */
186
static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock)
187 188
{
	cycle_t interval;
189
	u64 tmp, ntpinterval;
190
	struct clocksource *old_clock;
191

192 193 194 195 196
	old_clock = tk->tkr.clock;
	tk->tkr.clock = clock;
	tk->tkr.read = clock->read;
	tk->tkr.mask = clock->mask;
	tk->tkr.cycle_last = tk->tkr.read(clock);
197 198 199 200

	/* Do the ns -> cycle conversion first, using original mult */
	tmp = NTP_INTERVAL_LENGTH;
	tmp <<= clock->shift;
201
	ntpinterval = tmp;
202 203
	tmp += clock->mult/2;
	do_div(tmp, clock->mult);
204 205 206 207
	if (tmp == 0)
		tmp = 1;

	interval = (cycle_t) tmp;
208
	tk->cycle_interval = interval;
209 210

	/* Go back from cycles -> shifted ns */
211 212 213
	tk->xtime_interval = (u64) interval * clock->mult;
	tk->xtime_remainder = ntpinterval - tk->xtime_interval;
	tk->raw_interval =
214
		((u64) interval * clock->mult) >> clock->shift;
215

216 217 218 219
	 /* if changing clocks, convert xtime_nsec shift units */
	if (old_clock) {
		int shift_change = clock->shift - old_clock->shift;
		if (shift_change < 0)
220
			tk->tkr.xtime_nsec >>= -shift_change;
221
		else
222
			tk->tkr.xtime_nsec <<= shift_change;
223
	}
224
	tk->tkr.shift = clock->shift;
225

226 227
	tk->ntp_error = 0;
	tk->ntp_error_shift = NTP_SCALE_SHIFT - clock->shift;
228
	tk->ntp_tick = ntpinterval << tk->ntp_error_shift;
229 230 231 232 233 234

	/*
	 * The timekeeper keeps its own mult values for the currently
	 * active clocksource. These value will be adjusted via NTP
	 * to counteract clock drifting.
	 */
235
	tk->tkr.mult = clock->mult;
236
	tk->ntp_err_mult = 0;
237
}
238

239
/* Timekeeper helper functions. */
240 241

#ifdef CONFIG_ARCH_USES_GETTIMEOFFSET
242 243
static u32 default_arch_gettimeoffset(void) { return 0; }
u32 (*arch_gettimeoffset)(void) = default_arch_gettimeoffset;
244
#else
245
static inline u32 arch_gettimeoffset(void) { return 0; }
246 247
#endif

248
static inline s64 timekeeping_get_ns(struct tk_read_base *tkr)
249
{
250
	cycle_t delta;
251
	s64 nsec;
252

253
	delta = timekeeping_get_delta(tkr);
254

255 256
	nsec = delta * tkr->mult + tkr->xtime_nsec;
	nsec >>= tkr->shift;
257

258
	/* If arch requires, add in get_arch_timeoffset() */
259
	return nsec + arch_gettimeoffset();
260 261
}

262
static inline s64 timekeeping_get_ns_raw(struct timekeeper *tk)
263
{
264
	struct clocksource *clock = tk->tkr.clock;
265
	cycle_t delta;
266
	s64 nsec;
267

268
	delta = timekeeping_get_delta(&tk->tkr);
269

270
	/* convert delta to nanoseconds. */
271
	nsec = clocksource_cyc2ns(delta, clock->mult, clock->shift);
272

273
	/* If arch requires, add in get_arch_timeoffset() */
274
	return nsec + arch_gettimeoffset();
275 276
}

277 278
/**
 * update_fast_timekeeper - Update the fast and NMI safe monotonic timekeeper.
279
 * @tkr: Timekeeping readout base from which we take the update
280 281 282 283 284 285 286 287 288 289 290
 *
 * We want to use this from any context including NMI and tracing /
 * instrumenting the timekeeping code itself.
 *
 * So we handle this differently than the other timekeeping accessor
 * functions which retry when the sequence count has changed. The
 * update side does:
 *
 * smp_wmb();	<- Ensure that the last base[1] update is visible
 * tkf->seq++;
 * smp_wmb();	<- Ensure that the seqcount update is visible
291
 * update(tkf->base[0], tkr);
292 293 294
 * smp_wmb();	<- Ensure that the base[0] update is visible
 * tkf->seq++;
 * smp_wmb();	<- Ensure that the seqcount update is visible
295
 * update(tkf->base[1], tkr);
296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315
 *
 * The reader side does:
 *
 * do {
 *	seq = tkf->seq;
 *	smp_rmb();
 *	idx = seq & 0x01;
 *	now = now(tkf->base[idx]);
 *	smp_rmb();
 * } while (seq != tkf->seq)
 *
 * As long as we update base[0] readers are forced off to
 * base[1]. Once base[0] is updated readers are redirected to base[0]
 * and the base[1] update takes place.
 *
 * So if a NMI hits the update of base[0] then it will use base[1]
 * which is still consistent. In the worst case this can result is a
 * slightly wrong timestamp (a few nanoseconds). See
 * @ktime_get_mono_fast_ns.
 */
316
static void update_fast_timekeeper(struct tk_read_base *tkr)
317 318 319 320 321 322 323
{
	struct tk_read_base *base = tk_fast_mono.base;

	/* Force readers off to base[1] */
	raw_write_seqcount_latch(&tk_fast_mono.seq);

	/* Update base[0] */
324
	memcpy(base, tkr, sizeof(*base));
325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380

	/* Force readers back to base[0] */
	raw_write_seqcount_latch(&tk_fast_mono.seq);

	/* Update base[1] */
	memcpy(base + 1, base, sizeof(*base));
}

/**
 * ktime_get_mono_fast_ns - Fast NMI safe access to clock monotonic
 *
 * This timestamp is not guaranteed to be monotonic across an update.
 * The timestamp is calculated by:
 *
 *	now = base_mono + clock_delta * slope
 *
 * So if the update lowers the slope, readers who are forced to the
 * not yet updated second array are still using the old steeper slope.
 *
 * tmono
 * ^
 * |    o  n
 * |   o n
 * |  u
 * | o
 * |o
 * |12345678---> reader order
 *
 * o = old slope
 * u = update
 * n = new slope
 *
 * So reader 6 will observe time going backwards versus reader 5.
 *
 * While other CPUs are likely to be able observe that, the only way
 * for a CPU local observation is when an NMI hits in the middle of
 * the update. Timestamps taken from that NMI context might be ahead
 * of the following timestamps. Callers need to be aware of that and
 * deal with it.
 */
u64 notrace ktime_get_mono_fast_ns(void)
{
	struct tk_read_base *tkr;
	unsigned int seq;
	u64 now;

	do {
		seq = raw_read_seqcount(&tk_fast_mono.seq);
		tkr = tk_fast_mono.base + (seq & 0x01);
		now = ktime_to_ns(tkr->base_mono) + timekeeping_get_ns(tkr);

	} while (read_seqcount_retry(&tk_fast_mono.seq, seq));
	return now;
}
EXPORT_SYMBOL_GPL(ktime_get_mono_fast_ns);

381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409
/* Suspend-time cycles value for halted fast timekeeper. */
static cycle_t cycles_at_suspend;

static cycle_t dummy_clock_read(struct clocksource *cs)
{
	return cycles_at_suspend;
}

/**
 * halt_fast_timekeeper - Prevent fast timekeeper from accessing clocksource.
 * @tk: Timekeeper to snapshot.
 *
 * It generally is unsafe to access the clocksource after timekeeping has been
 * suspended, so take a snapshot of the readout base of @tk and use it as the
 * fast timekeeper's readout base while suspended.  It will return the same
 * number of cycles every time until timekeeping is resumed at which time the
 * proper readout base for the fast timekeeper will be restored automatically.
 */
static void halt_fast_timekeeper(struct timekeeper *tk)
{
	static struct tk_read_base tkr_dummy;
	struct tk_read_base *tkr = &tk->tkr;

	memcpy(&tkr_dummy, tkr, sizeof(tkr_dummy));
	cycles_at_suspend = tkr->read(tkr->clock);
	tkr_dummy.read = dummy_clock_read;
	update_fast_timekeeper(&tkr_dummy);
}

410 411 412 413
#ifdef CONFIG_GENERIC_TIME_VSYSCALL_OLD

static inline void update_vsyscall(struct timekeeper *tk)
{
414
	struct timespec xt, wm;
415

416
	xt = timespec64_to_timespec(tk_xtime(tk));
417 418
	wm = timespec64_to_timespec(tk->wall_to_monotonic);
	update_vsyscall_old(&xt, &wm, tk->tkr.clock, tk->tkr.mult,
419
			    tk->tkr.cycle_last);
420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435
}

static inline void old_vsyscall_fixup(struct timekeeper *tk)
{
	s64 remainder;

	/*
	* Store only full nanoseconds into xtime_nsec after rounding
	* it up and add the remainder to the error difference.
	* XXX - This is necessary to avoid small 1ns inconsistnecies caused
	* by truncating the remainder in vsyscalls. However, it causes
	* additional work to be done in timekeeping_adjust(). Once
	* the vsyscall implementations are converted to use xtime_nsec
	* (shifted nanoseconds), and CONFIG_GENERIC_TIME_VSYSCALL_OLD
	* users are removed, this can be killed.
	*/
436 437 438
	remainder = tk->tkr.xtime_nsec & ((1ULL << tk->tkr.shift) - 1);
	tk->tkr.xtime_nsec -= remainder;
	tk->tkr.xtime_nsec += 1ULL << tk->tkr.shift;
439
	tk->ntp_error += remainder << tk->ntp_error_shift;
440
	tk->ntp_error -= (1ULL << tk->tkr.shift) << tk->ntp_error_shift;
441 442 443 444 445
}
#else
#define old_vsyscall_fixup(tk)
#endif

446 447
static RAW_NOTIFIER_HEAD(pvclock_gtod_chain);

448
static void update_pvclock_gtod(struct timekeeper *tk, bool was_set)
449
{
450
	raw_notifier_call_chain(&pvclock_gtod_chain, was_set, tk);
451 452 453 454 455 456 457
}

/**
 * pvclock_gtod_register_notifier - register a pvclock timedata update listener
 */
int pvclock_gtod_register_notifier(struct notifier_block *nb)
{
458
	struct timekeeper *tk = &tk_core.timekeeper;
459 460 461
	unsigned long flags;
	int ret;

462
	raw_spin_lock_irqsave(&timekeeper_lock, flags);
463
	ret = raw_notifier_chain_register(&pvclock_gtod_chain, nb);
464
	update_pvclock_gtod(tk, true);
465
	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
466 467 468 469 470 471 472 473 474 475 476 477 478 479

	return ret;
}
EXPORT_SYMBOL_GPL(pvclock_gtod_register_notifier);

/**
 * pvclock_gtod_unregister_notifier - unregister a pvclock
 * timedata update listener
 */
int pvclock_gtod_unregister_notifier(struct notifier_block *nb)
{
	unsigned long flags;
	int ret;

480
	raw_spin_lock_irqsave(&timekeeper_lock, flags);
481
	ret = raw_notifier_chain_unregister(&pvclock_gtod_chain, nb);
482
	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
483 484 485 486 487

	return ret;
}
EXPORT_SYMBOL_GPL(pvclock_gtod_unregister_notifier);

488 489 490 491 492
/*
 * Update the ktime_t based scalar nsec members of the timekeeper
 */
static inline void tk_update_ktime_data(struct timekeeper *tk)
{
493 494
	u64 seconds;
	u32 nsec;
495 496 497 498 499 500 501 502

	/*
	 * The xtime based monotonic readout is:
	 *	nsec = (xtime_sec + wtm_sec) * 1e9 + wtm_nsec + now();
	 * The ktime based monotonic readout is:
	 *	nsec = base_mono + now();
	 * ==> base_mono = (xtime_sec + wtm_sec) * 1e9 + wtm_nsec
	 */
503 504 505
	seconds = (u64)(tk->xtime_sec + tk->wall_to_monotonic.tv_sec);
	nsec = (u32) tk->wall_to_monotonic.tv_nsec;
	tk->tkr.base_mono = ns_to_ktime(seconds * NSEC_PER_SEC + nsec);
506 507 508

	/* Update the monotonic raw base */
	tk->base_raw = timespec64_to_ktime(tk->raw_time);
509 510 511 512 513 514 515 516 517 518

	/*
	 * The sum of the nanoseconds portions of xtime and
	 * wall_to_monotonic can be greater/equal one second. Take
	 * this into account before updating tk->ktime_sec.
	 */
	nsec += (u32)(tk->tkr.xtime_nsec >> tk->tkr.shift);
	if (nsec >= NSEC_PER_SEC)
		seconds++;
	tk->ktime_sec = seconds;
519 520
}

521
/* must hold timekeeper_lock */
522
static void timekeeping_update(struct timekeeper *tk, unsigned int action)
523
{
524
	if (action & TK_CLEAR_NTP) {
525
		tk->ntp_error = 0;
526 527
		ntp_clear();
	}
528

529 530
	tk_update_ktime_data(tk);

531 532 533
	update_vsyscall(tk);
	update_pvclock_gtod(tk, action & TK_CLOCK_WAS_SET);

534
	if (action & TK_MIRROR)
535 536
		memcpy(&shadow_timekeeper, &tk_core.timekeeper,
		       sizeof(tk_core.timekeeper));
537

538
	update_fast_timekeeper(&tk->tkr);
539 540
}

541
/**
542
 * timekeeping_forward_now - update clock to the current time
543
 *
544 545 546
 * Forward the current clock to update its state since the last call to
 * update_wall_time(). This is useful before significant clock changes,
 * as it avoids having to deal with this time offset explicitly.
547
 */
548
static void timekeeping_forward_now(struct timekeeper *tk)
549
{
550
	struct clocksource *clock = tk->tkr.clock;
551
	cycle_t cycle_now, delta;
552
	s64 nsec;
553

554 555 556
	cycle_now = tk->tkr.read(clock);
	delta = clocksource_delta(cycle_now, tk->tkr.cycle_last, tk->tkr.mask);
	tk->tkr.cycle_last = cycle_now;
557

558
	tk->tkr.xtime_nsec += delta * tk->tkr.mult;
559

560
	/* If arch requires, add in get_arch_timeoffset() */
561
	tk->tkr.xtime_nsec += (u64)arch_gettimeoffset() << tk->tkr.shift;
562

563
	tk_normalize_xtime(tk);
564

565
	nsec = clocksource_cyc2ns(delta, clock->mult, clock->shift);
566
	timespec64_add_ns(&tk->raw_time, nsec);
567 568 569
}

/**
570
 * __getnstimeofday64 - Returns the time of day in a timespec64.
571 572
 * @ts:		pointer to the timespec to be set
 *
573 574
 * Updates the time of day in the timespec.
 * Returns 0 on success, or -ve when suspended (timespec will be undefined).
575
 */
576
int __getnstimeofday64(struct timespec64 *ts)
577
{
578
	struct timekeeper *tk = &tk_core.timekeeper;
579
	unsigned long seq;
580
	s64 nsecs = 0;
581 582

	do {
583
		seq = read_seqcount_begin(&tk_core.seq);
584

585
		ts->tv_sec = tk->xtime_sec;
586
		nsecs = timekeeping_get_ns(&tk->tkr);
587

588
	} while (read_seqcount_retry(&tk_core.seq, seq));
589

590
	ts->tv_nsec = 0;
591
	timespec64_add_ns(ts, nsecs);
592 593 594 595 596 597 598 599 600

	/*
	 * Do not bail out early, in case there were callers still using
	 * the value, even in the face of the WARN_ON.
	 */
	if (unlikely(timekeeping_suspended))
		return -EAGAIN;
	return 0;
}
601
EXPORT_SYMBOL(__getnstimeofday64);
602 603

/**
604
 * getnstimeofday64 - Returns the time of day in a timespec64.
605
 * @ts:		pointer to the timespec64 to be set
606
 *
607
 * Returns the time of day in a timespec64 (WARN if suspended).
608
 */
609
void getnstimeofday64(struct timespec64 *ts)
610
{
611
	WARN_ON(__getnstimeofday64(ts));
612
}
613
EXPORT_SYMBOL(getnstimeofday64);
614

615 616
ktime_t ktime_get(void)
{
617
	struct timekeeper *tk = &tk_core.timekeeper;
618
	unsigned int seq;
619 620
	ktime_t base;
	s64 nsecs;
621 622 623 624

	WARN_ON(timekeeping_suspended);

	do {
625
		seq = read_seqcount_begin(&tk_core.seq);
626
		base = tk->tkr.base_mono;
627
		nsecs = timekeeping_get_ns(&tk->tkr);
628

629
	} while (read_seqcount_retry(&tk_core.seq, seq));
630

631
	return ktime_add_ns(base, nsecs);
632 633 634
}
EXPORT_SYMBOL_GPL(ktime_get);

635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651
static ktime_t *offsets[TK_OFFS_MAX] = {
	[TK_OFFS_REAL]	= &tk_core.timekeeper.offs_real,
	[TK_OFFS_BOOT]	= &tk_core.timekeeper.offs_boot,
	[TK_OFFS_TAI]	= &tk_core.timekeeper.offs_tai,
};

ktime_t ktime_get_with_offset(enum tk_offsets offs)
{
	struct timekeeper *tk = &tk_core.timekeeper;
	unsigned int seq;
	ktime_t base, *offset = offsets[offs];
	s64 nsecs;

	WARN_ON(timekeeping_suspended);

	do {
		seq = read_seqcount_begin(&tk_core.seq);
652
		base = ktime_add(tk->tkr.base_mono, *offset);
653
		nsecs = timekeeping_get_ns(&tk->tkr);
654 655 656 657 658 659 660 661

	} while (read_seqcount_retry(&tk_core.seq, seq));

	return ktime_add_ns(base, nsecs);

}
EXPORT_SYMBOL_GPL(ktime_get_with_offset);

662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681
/**
 * ktime_mono_to_any() - convert mononotic time to any other time
 * @tmono:	time to convert.
 * @offs:	which offset to use
 */
ktime_t ktime_mono_to_any(ktime_t tmono, enum tk_offsets offs)
{
	ktime_t *offset = offsets[offs];
	unsigned long seq;
	ktime_t tconv;

	do {
		seq = read_seqcount_begin(&tk_core.seq);
		tconv = ktime_add(tmono, *offset);
	} while (read_seqcount_retry(&tk_core.seq, seq));

	return tconv;
}
EXPORT_SYMBOL_GPL(ktime_mono_to_any);

682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702
/**
 * ktime_get_raw - Returns the raw monotonic time in ktime_t format
 */
ktime_t ktime_get_raw(void)
{
	struct timekeeper *tk = &tk_core.timekeeper;
	unsigned int seq;
	ktime_t base;
	s64 nsecs;

	do {
		seq = read_seqcount_begin(&tk_core.seq);
		base = tk->base_raw;
		nsecs = timekeeping_get_ns_raw(tk);

	} while (read_seqcount_retry(&tk_core.seq, seq));

	return ktime_add_ns(base, nsecs);
}
EXPORT_SYMBOL_GPL(ktime_get_raw);

703
/**
704
 * ktime_get_ts64 - get the monotonic clock in timespec64 format
705 706 707 708
 * @ts:		pointer to timespec variable
 *
 * The function calculates the monotonic clock from the realtime
 * clock and the wall_to_monotonic offset and stores the result
709
 * in normalized timespec64 format in the variable pointed to by @ts.
710
 */
711
void ktime_get_ts64(struct timespec64 *ts)
712
{
713
	struct timekeeper *tk = &tk_core.timekeeper;
714
	struct timespec64 tomono;
715
	s64 nsec;
716 717 718 719 720
	unsigned int seq;

	WARN_ON(timekeeping_suspended);

	do {
721
		seq = read_seqcount_begin(&tk_core.seq);
722
		ts->tv_sec = tk->xtime_sec;
723
		nsec = timekeeping_get_ns(&tk->tkr);
724
		tomono = tk->wall_to_monotonic;
725

726
	} while (read_seqcount_retry(&tk_core.seq, seq));
727

728 729 730
	ts->tv_sec += tomono.tv_sec;
	ts->tv_nsec = 0;
	timespec64_add_ns(ts, nsec + tomono.tv_nsec);
731
}
732
EXPORT_SYMBOL_GPL(ktime_get_ts64);
733

734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751
/**
 * ktime_get_seconds - Get the seconds portion of CLOCK_MONOTONIC
 *
 * Returns the seconds portion of CLOCK_MONOTONIC with a single non
 * serialized read. tk->ktime_sec is of type 'unsigned long' so this
 * works on both 32 and 64 bit systems. On 32 bit systems the readout
 * covers ~136 years of uptime which should be enough to prevent
 * premature wrap arounds.
 */
time64_t ktime_get_seconds(void)
{
	struct timekeeper *tk = &tk_core.timekeeper;

	WARN_ON(timekeeping_suspended);
	return tk->ktime_sec;
}
EXPORT_SYMBOL_GPL(ktime_get_seconds);

752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781
/**
 * ktime_get_real_seconds - Get the seconds portion of CLOCK_REALTIME
 *
 * Returns the wall clock seconds since 1970. This replaces the
 * get_seconds() interface which is not y2038 safe on 32bit systems.
 *
 * For 64bit systems the fast access to tk->xtime_sec is preserved. On
 * 32bit systems the access must be protected with the sequence
 * counter to provide "atomic" access to the 64bit tk->xtime_sec
 * value.
 */
time64_t ktime_get_real_seconds(void)
{
	struct timekeeper *tk = &tk_core.timekeeper;
	time64_t seconds;
	unsigned int seq;

	if (IS_ENABLED(CONFIG_64BIT))
		return tk->xtime_sec;

	do {
		seq = read_seqcount_begin(&tk_core.seq);
		seconds = tk->xtime_sec;

	} while (read_seqcount_retry(&tk_core.seq, seq));

	return seconds;
}
EXPORT_SYMBOL_GPL(ktime_get_real_seconds);

782 783 784 785 786 787 788 789 790 791 792 793 794
#ifdef CONFIG_NTP_PPS

/**
 * getnstime_raw_and_real - get day and raw monotonic time in timespec format
 * @ts_raw:	pointer to the timespec to be set to raw monotonic time
 * @ts_real:	pointer to the timespec to be set to the time of day
 *
 * This function reads both the time of day and raw monotonic time at the
 * same time atomically and stores the resulting timestamps in timespec
 * format.
 */
void getnstime_raw_and_real(struct timespec *ts_raw, struct timespec *ts_real)
{
795
	struct timekeeper *tk = &tk_core.timekeeper;
796 797 798 799 800 801
	unsigned long seq;
	s64 nsecs_raw, nsecs_real;

	WARN_ON_ONCE(timekeeping_suspended);

	do {
802
		seq = read_seqcount_begin(&tk_core.seq);
803

804
		*ts_raw = timespec64_to_timespec(tk->raw_time);
805
		ts_real->tv_sec = tk->xtime_sec;
806
		ts_real->tv_nsec = 0;
807

808
		nsecs_raw = timekeeping_get_ns_raw(tk);
809
		nsecs_real = timekeeping_get_ns(&tk->tkr);
810

811
	} while (read_seqcount_retry(&tk_core.seq, seq));
812 813 814 815 816 817 818 819

	timespec_add_ns(ts_raw, nsecs_raw);
	timespec_add_ns(ts_real, nsecs_real);
}
EXPORT_SYMBOL(getnstime_raw_and_real);

#endif /* CONFIG_NTP_PPS */

820 821 822 823
/**
 * do_gettimeofday - Returns the time of day in a timeval
 * @tv:		pointer to the timeval to be set
 *
824
 * NOTE: Users should be converted to using getnstimeofday()
825 826 827
 */
void do_gettimeofday(struct timeval *tv)
{
828
	struct timespec64 now;
829

830
	getnstimeofday64(&now);
831 832 833 834
	tv->tv_sec = now.tv_sec;
	tv->tv_usec = now.tv_nsec/1000;
}
EXPORT_SYMBOL(do_gettimeofday);
835

836
/**
837 838
 * do_settimeofday64 - Sets the time of day.
 * @ts:     pointer to the timespec64 variable containing the new time
839 840 841
 *
 * Sets the time of day to the new time and update NTP and notify hrtimers
 */
842
int do_settimeofday64(const struct timespec64 *ts)
843
{
844
	struct timekeeper *tk = &tk_core.timekeeper;
845
	struct timespec64 ts_delta, xt;
846
	unsigned long flags;
847

848
	if (!timespec64_valid_strict(ts))
849 850
		return -EINVAL;

851
	raw_spin_lock_irqsave(&timekeeper_lock, flags);
852
	write_seqcount_begin(&tk_core.seq);
853

854
	timekeeping_forward_now(tk);
855

856
	xt = tk_xtime(tk);
857 858
	ts_delta.tv_sec = ts->tv_sec - xt.tv_sec;
	ts_delta.tv_nsec = ts->tv_nsec - xt.tv_nsec;
859

860
	tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, ts_delta));
861

862
	tk_set_xtime(tk, ts);
863

864
	timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
865

866
	write_seqcount_end(&tk_core.seq);
867
	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
868 869 870 871 872 873

	/* signal hrtimers about time change */
	clock_was_set();

	return 0;
}
874
EXPORT_SYMBOL(do_settimeofday64);
875

876 877 878 879 880 881 882 883
/**
 * timekeeping_inject_offset - Adds or subtracts from the current time.
 * @tv:		pointer to the timespec variable containing the offset
 *
 * Adds or subtracts an offset value from the current time.
 */
int timekeeping_inject_offset(struct timespec *ts)
{
884
	struct timekeeper *tk = &tk_core.timekeeper;
885
	unsigned long flags;
886
	struct timespec64 ts64, tmp;
887
	int ret = 0;
888 889 890 891

	if ((unsigned long)ts->tv_nsec >= NSEC_PER_SEC)
		return -EINVAL;

892 893
	ts64 = timespec_to_timespec64(*ts);

894
	raw_spin_lock_irqsave(&timekeeper_lock, flags);
895
	write_seqcount_begin(&tk_core.seq);
896

897
	timekeeping_forward_now(tk);
898

899
	/* Make sure the proposed value is valid */
900 901
	tmp = timespec64_add(tk_xtime(tk),  ts64);
	if (!timespec64_valid_strict(&tmp)) {
902 903 904
		ret = -EINVAL;
		goto error;
	}
905

906 907
	tk_xtime_add(tk, &ts64);
	tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, ts64));
908

909
error: /* even if we error out, we forwarded the time, so call update */
910
	timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
911

912
	write_seqcount_end(&tk_core.seq);
913
	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
914 915 916 917

	/* signal hrtimers about time change */
	clock_was_set();

918
	return ret;
919 920 921
}
EXPORT_SYMBOL(timekeeping_inject_offset);

922 923 924 925 926 927 928

/**
 * timekeeping_get_tai_offset - Returns current TAI offset from UTC
 *
 */
s32 timekeeping_get_tai_offset(void)
{
929
	struct timekeeper *tk = &tk_core.timekeeper;
930 931 932 933
	unsigned int seq;
	s32 ret;

	do {
934
		seq = read_seqcount_begin(&tk_core.seq);
935
		ret = tk->tai_offset;
936
	} while (read_seqcount_retry(&tk_core.seq, seq));
937 938 939 940 941 942 943 944

	return ret;
}

/**
 * __timekeeping_set_tai_offset - Lock free worker function
 *
 */
945
static void __timekeeping_set_tai_offset(struct timekeeper *tk, s32 tai_offset)
946 947
{
	tk->tai_offset = tai_offset;
948
	tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tai_offset, 0));
949 950 951 952 953 954 955 956
}

/**
 * timekeeping_set_tai_offset - Sets the current TAI offset from UTC
 *
 */
void timekeeping_set_tai_offset(s32 tai_offset)
{
957
	struct timekeeper *tk = &tk_core.timekeeper;
958 959
	unsigned long flags;

960
	raw_spin_lock_irqsave(&timekeeper_lock, flags);
961
	write_seqcount_begin(&tk_core.seq);
962
	__timekeeping_set_tai_offset(tk, tai_offset);
963
	timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
964
	write_seqcount_end(&tk_core.seq);
965
	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
966
	clock_was_set();
967 968
}

969 970 971 972 973
/**
 * change_clocksource - Swaps clocksources if a new one is available
 *
 * Accumulates current time interval and initializes new clocksource
 */
974
static int change_clocksource(void *data)
975
{
976
	struct timekeeper *tk = &tk_core.timekeeper;
977
	struct clocksource *new, *old;
978
	unsigned long flags;
979

980
	new = (struct clocksource *) data;
981

982
	raw_spin_lock_irqsave(&timekeeper_lock, flags);
983
	write_seqcount_begin(&tk_core.seq);
984

985
	timekeeping_forward_now(tk);
986 987 988 989 990 991
	/*
	 * If the cs is in module, get a module reference. Succeeds
	 * for built-in code (owner == NULL) as well.
	 */
	if (try_module_get(new->owner)) {
		if (!new->enable || new->enable(new) == 0) {
992
			old = tk->tkr.clock;
993 994 995 996 997 998 999
			tk_setup_internals(tk, new);
			if (old->disable)
				old->disable(old);
			module_put(old->owner);
		} else {
			module_put(new->owner);
		}
1000
	}
1001
	timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
1002

1003
	write_seqcount_end(&tk_core.seq);
1004
	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1005

1006 1007
	return 0;
}
1008

1009 1010 1011 1012 1013 1014 1015
/**
 * timekeeping_notify - Install a new clock source
 * @clock:		pointer to the clock source
 *
 * This function is called from clocksource.c after a new, better clock
 * source has been registered. The caller holds the clocksource_mutex.
 */
1016
int timekeeping_notify(struct clocksource *clock)
1017
{
1018
	struct timekeeper *tk = &tk_core.timekeeper;
1019

1020
	if (tk->tkr.clock == clock)
1021
		return 0;
1022
	stop_machine(change_clocksource, clock, NULL);
1023
	tick_clock_notify();
1024
	return tk->tkr.clock == clock ? 0 : -1;
1025
}
1026

1027
/**
1028 1029
 * getrawmonotonic64 - Returns the raw monotonic time in a timespec
 * @ts:		pointer to the timespec64 to be set
1030 1031 1032
 *
 * Returns the raw monotonic time (completely un-modified by ntp)
 */
1033
void getrawmonotonic64(struct timespec64 *ts)
1034
{
1035
	struct timekeeper *tk = &tk_core.timekeeper;
1036
	struct timespec64 ts64;
1037 1038 1039 1040
	unsigned long seq;
	s64 nsecs;

	do {
1041
		seq = read_seqcount_begin(&tk_core.seq);
1042
		nsecs = timekeeping_get_ns_raw(tk);
1043
		ts64 = tk->raw_time;
1044

1045
	} while (read_seqcount_retry(&tk_core.seq, seq));
1046

1047
	timespec64_add_ns(&ts64, nsecs);
1048
	*ts = ts64;
1049
}
1050 1051
EXPORT_SYMBOL(getrawmonotonic64);

1052

1053
/**
1054
 * timekeeping_valid_for_hres - Check if timekeeping is suitable for hres
1055
 */
1056
int timekeeping_valid_for_hres(void)
1057
{
1058
	struct timekeeper *tk = &tk_core.timekeeper;
1059 1060 1061 1062
	unsigned long seq;
	int ret;

	do {
1063
		seq = read_seqcount_begin(&tk_core.seq);
1064

1065
		ret = tk->tkr.clock->flags & CLOCK_SOURCE_VALID_FOR_HRES;
1066

1067
	} while (read_seqcount_retry(&tk_core.seq, seq));
1068 1069 1070 1071

	return ret;
}

1072 1073 1074 1075 1076
/**
 * timekeeping_max_deferment - Returns max time the clocksource can be deferred
 */
u64 timekeeping_max_deferment(void)
{
1077
	struct timekeeper *tk = &tk_core.timekeeper;
J
John Stultz 已提交
1078 1079
	unsigned long seq;
	u64 ret;
1080

J
John Stultz 已提交
1081
	do {
1082
		seq = read_seqcount_begin(&tk_core.seq);
J
John Stultz 已提交
1083

1084
		ret = tk->tkr.clock->max_idle_ns;
J
John Stultz 已提交
1085

1086
	} while (read_seqcount_retry(&tk_core.seq, seq));
J
John Stultz 已提交
1087 1088

	return ret;
1089 1090
}

1091
/**
1092
 * read_persistent_clock -  Return time from the persistent clock.
1093 1094
 *
 * Weak dummy function for arches that do not yet support it.
1095 1096
 * Reads the time from the battery backed persistent clock.
 * Returns a timespec with tv_sec=0 and tv_nsec=0 if unsupported.
1097 1098 1099
 *
 *  XXX - Do be sure to remove it once all arches implement it.
 */
1100
void __weak read_persistent_clock(struct timespec *ts)
1101
{
1102 1103
	ts->tv_sec = 0;
	ts->tv_nsec = 0;
1104 1105
}

1106 1107 1108 1109 1110 1111 1112 1113 1114
/**
 * read_boot_clock -  Return time of the system start.
 *
 * Weak dummy function for arches that do not yet support it.
 * Function to read the exact time the system has been started.
 * Returns a timespec with tv_sec=0 and tv_nsec=0 if unsupported.
 *
 *  XXX - Do be sure to remove it once all arches implement it.
 */
1115
void __weak read_boot_clock(struct timespec *ts)
1116 1117 1118 1119 1120
{
	ts->tv_sec = 0;
	ts->tv_nsec = 0;
}

1121 1122 1123 1124 1125
/*
 * timekeeping_init - Initializes the clocksource and common timekeeping values
 */
void __init timekeeping_init(void)
{
1126
	struct timekeeper *tk = &tk_core.timekeeper;
1127
	struct clocksource *clock;
1128
	unsigned long flags;
1129 1130
	struct timespec64 now, boot, tmp;
	struct timespec ts;
1131

1132 1133 1134
	read_persistent_clock(&ts);
	now = timespec_to_timespec64(ts);
	if (!timespec64_valid_strict(&now)) {
1135 1136 1137 1138
		pr_warn("WARNING: Persistent clock returned invalid value!\n"
			"         Check your CMOS/BIOS settings.\n");
		now.tv_sec = 0;
		now.tv_nsec = 0;
1139 1140
	} else if (now.tv_sec || now.tv_nsec)
		persistent_clock_exist = true;
1141

1142 1143 1144
	read_boot_clock(&ts);
	boot = timespec_to_timespec64(ts);
	if (!timespec64_valid_strict(&boot)) {
1145 1146 1147 1148 1149
		pr_warn("WARNING: Boot clock returned invalid value!\n"
			"         Check your CMOS/BIOS settings.\n");
		boot.tv_sec = 0;
		boot.tv_nsec = 0;
	}
1150

1151
	raw_spin_lock_irqsave(&timekeeper_lock, flags);
1152
	write_seqcount_begin(&tk_core.seq);
1153 1154
	ntp_init();

1155
	clock = clocksource_default_clock();
1156 1157
	if (clock->enable)
		clock->enable(clock);
1158
	tk_setup_internals(tk, clock);
1159

1160 1161 1162
	tk_set_xtime(tk, &now);
	tk->raw_time.tv_sec = 0;
	tk->raw_time.tv_nsec = 0;
1163
	tk->base_raw.tv64 = 0;
1164
	if (boot.tv_sec == 0 && boot.tv_nsec == 0)
1165
		boot = tk_xtime(tk);
1166

1167
	set_normalized_timespec64(&tmp, -boot.tv_sec, -boot.tv_nsec);
1168
	tk_set_wall_to_mono(tk, tmp);
1169

1170
	timekeeping_update(tk, TK_MIRROR);
1171

1172
	write_seqcount_end(&tk_core.seq);
1173
	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1174 1175 1176
}

/* time in seconds when suspend began */
1177
static struct timespec64 timekeeping_suspend_time;
1178

1179 1180 1181 1182 1183 1184 1185
/**
 * __timekeeping_inject_sleeptime - Internal function to add sleep interval
 * @delta: pointer to a timespec delta value
 *
 * Takes a timespec offset measuring a suspend interval and properly
 * adds the sleep offset to the timekeeping variables.
 */
1186
static void __timekeeping_inject_sleeptime(struct timekeeper *tk,
1187
					   struct timespec64 *delta)
1188
{
1189
	if (!timespec64_valid_strict(delta)) {
1190 1191 1192
		printk_deferred(KERN_WARNING
				"__timekeeping_inject_sleeptime: Invalid "
				"sleep delta value!\n");
1193 1194
		return;
	}
1195
	tk_xtime_add(tk, delta);
1196
	tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, *delta));
1197
	tk_update_sleep_time(tk, timespec64_to_ktime(*delta));
1198
	tk_debug_account_sleep_time(delta);
1199 1200 1201
}

/**
1202 1203
 * timekeeping_inject_sleeptime64 - Adds suspend interval to timeekeeping values
 * @delta: pointer to a timespec64 delta value
1204 1205 1206 1207 1208 1209 1210
 *
 * This hook is for architectures that cannot support read_persistent_clock
 * because their RTC/persistent clock is only accessible when irqs are enabled.
 *
 * This function should only be called by rtc_resume(), and allows
 * a suspend offset to be injected into the timekeeping values.
 */
1211
void timekeeping_inject_sleeptime64(struct timespec64 *delta)
1212
{
1213
	struct timekeeper *tk = &tk_core.timekeeper;
1214
	unsigned long flags;
1215

1216 1217 1218 1219 1220
	/*
	 * Make sure we don't set the clock twice, as timekeeping_resume()
	 * already did it
	 */
	if (has_persistent_clock())
1221 1222
		return;

1223
	raw_spin_lock_irqsave(&timekeeper_lock, flags);
1224
	write_seqcount_begin(&tk_core.seq);
J
John Stultz 已提交
1225

1226
	timekeeping_forward_now(tk);
1227

1228
	__timekeeping_inject_sleeptime(tk, delta);
1229

1230
	timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
1231

1232
	write_seqcount_end(&tk_core.seq);
1233
	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1234 1235 1236 1237 1238

	/* signal hrtimers about time change */
	clock_was_set();
}

1239 1240 1241 1242 1243 1244 1245
/**
 * timekeeping_resume - Resumes the generic timekeeping subsystem.
 *
 * This is for the generic clocksource timekeeping.
 * xtime/wall_to_monotonic/jiffies/etc are
 * still managed by arch specific suspend/resume code.
 */
1246
void timekeeping_resume(void)
1247
{
1248
	struct timekeeper *tk = &tk_core.timekeeper;
1249
	struct clocksource *clock = tk->tkr.clock;
1250
	unsigned long flags;
1251 1252
	struct timespec64 ts_new, ts_delta;
	struct timespec tmp;
1253 1254
	cycle_t cycle_now, cycle_delta;
	bool suspendtime_found = false;
1255

1256 1257
	read_persistent_clock(&tmp);
	ts_new = timespec_to_timespec64(tmp);
1258

1259
	clockevents_resume();
1260 1261
	clocksource_resume();

1262
	raw_spin_lock_irqsave(&timekeeper_lock, flags);
1263
	write_seqcount_begin(&tk_core.seq);
1264

1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276
	/*
	 * After system resumes, we need to calculate the suspended time and
	 * compensate it for the OS time. There are 3 sources that could be
	 * used: Nonstop clocksource during suspend, persistent clock and rtc
	 * device.
	 *
	 * One specific platform may have 1 or 2 or all of them, and the
	 * preference will be:
	 *	suspend-nonstop clocksource -> persistent clock -> rtc
	 * The less preferred source will only be tried if there is no better
	 * usable source. The rtc part is handled separately in rtc core code.
	 */
1277
	cycle_now = tk->tkr.read(clock);
1278
	if ((clock->flags & CLOCK_SOURCE_SUSPEND_NONSTOP) &&
1279
		cycle_now > tk->tkr.cycle_last) {
1280 1281 1282 1283 1284
		u64 num, max = ULLONG_MAX;
		u32 mult = clock->mult;
		u32 shift = clock->shift;
		s64 nsec = 0;

1285 1286
		cycle_delta = clocksource_delta(cycle_now, tk->tkr.cycle_last,
						tk->tkr.mask);
1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300

		/*
		 * "cycle_delta * mutl" may cause 64 bits overflow, if the
		 * suspended time is too long. In that case we need do the
		 * 64 bits math carefully
		 */
		do_div(max, mult);
		if (cycle_delta > max) {
			num = div64_u64(cycle_delta, max);
			nsec = (((u64) max * mult) >> shift) * num;
			cycle_delta -= num * max;
		}
		nsec += ((u64) cycle_delta * mult) >> shift;

1301
		ts_delta = ns_to_timespec64(nsec);
1302
		suspendtime_found = true;
1303 1304
	} else if (timespec64_compare(&ts_new, &timekeeping_suspend_time) > 0) {
		ts_delta = timespec64_sub(ts_new, timekeeping_suspend_time);
1305
		suspendtime_found = true;
1306
	}
1307 1308 1309 1310 1311

	if (suspendtime_found)
		__timekeeping_inject_sleeptime(tk, &ts_delta);

	/* Re-base the last cycle value */
1312
	tk->tkr.cycle_last = cycle_now;
1313
	tk->ntp_error = 0;
1314
	timekeeping_suspended = 0;
1315
	timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
1316
	write_seqcount_end(&tk_core.seq);
1317
	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1318 1319 1320 1321 1322 1323

	touch_softlockup_watchdog();

	clockevents_notify(CLOCK_EVT_NOTIFY_RESUME, NULL);

	/* Resume hrtimers */
1324
	hrtimers_resume();
1325 1326
}

1327
int timekeeping_suspend(void)
1328
{
1329
	struct timekeeper *tk = &tk_core.timekeeper;
1330
	unsigned long flags;
1331 1332 1333
	struct timespec64		delta, delta_delta;
	static struct timespec64	old_delta;
	struct timespec tmp;
1334

1335 1336
	read_persistent_clock(&tmp);
	timekeeping_suspend_time = timespec_to_timespec64(tmp);
1337

1338 1339 1340 1341 1342 1343 1344 1345
	/*
	 * On some systems the persistent_clock can not be detected at
	 * timekeeping_init by its return value, so if we see a valid
	 * value returned, update the persistent_clock_exists flag.
	 */
	if (timekeeping_suspend_time.tv_sec || timekeeping_suspend_time.tv_nsec)
		persistent_clock_exist = true;

1346
	raw_spin_lock_irqsave(&timekeeper_lock, flags);
1347
	write_seqcount_begin(&tk_core.seq);
1348
	timekeeping_forward_now(tk);
1349
	timekeeping_suspended = 1;
1350 1351 1352 1353 1354 1355 1356

	/*
	 * To avoid drift caused by repeated suspend/resumes,
	 * which each can add ~1 second drift error,
	 * try to compensate so the difference in system time
	 * and persistent_clock time stays close to constant.
	 */
1357 1358
	delta = timespec64_sub(tk_xtime(tk), timekeeping_suspend_time);
	delta_delta = timespec64_sub(delta, old_delta);
1359 1360 1361 1362 1363 1364 1365 1366 1367
	if (abs(delta_delta.tv_sec)  >= 2) {
		/*
		 * if delta_delta is too large, assume time correction
		 * has occured and set old_delta to the current delta.
		 */
		old_delta = delta;
	} else {
		/* Otherwise try to adjust old_system to compensate */
		timekeeping_suspend_time =
1368
			timespec64_add(timekeeping_suspend_time, delta_delta);
1369
	}
1370 1371

	timekeeping_update(tk, TK_MIRROR);
1372
	halt_fast_timekeeper(tk);
1373
	write_seqcount_end(&tk_core.seq);
1374
	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1375 1376

	clockevents_notify(CLOCK_EVT_NOTIFY_SUSPEND, NULL);
M
Magnus Damm 已提交
1377
	clocksource_suspend();
1378
	clockevents_suspend();
1379 1380 1381 1382 1383

	return 0;
}

/* sysfs resume/suspend bits for timekeeping */
1384
static struct syscore_ops timekeeping_syscore_ops = {
1385 1386 1387 1388
	.resume		= timekeeping_resume,
	.suspend	= timekeeping_suspend,
};

1389
static int __init timekeeping_init_ops(void)
1390
{
1391 1392
	register_syscore_ops(&timekeeping_syscore_ops);
	return 0;
1393
}
1394
device_initcall(timekeeping_init_ops);
1395 1396

/*
1397
 * Apply a multiplier adjustment to the timekeeper
1398
 */
1399 1400 1401 1402
static __always_inline void timekeeping_apply_adjustment(struct timekeeper *tk,
							 s64 offset,
							 bool negative,
							 int adj_scale)
1403
{
1404 1405
	s64 interval = tk->cycle_interval;
	s32 mult_adj = 1;
1406

1407 1408 1409 1410
	if (negative) {
		mult_adj = -mult_adj;
		interval = -interval;
		offset  = -offset;
1411
	}
1412 1413 1414
	mult_adj <<= adj_scale;
	interval <<= adj_scale;
	offset <<= adj_scale;
1415

1416 1417 1418
	/*
	 * So the following can be confusing.
	 *
1419
	 * To keep things simple, lets assume mult_adj == 1 for now.
1420
	 *
1421
	 * When mult_adj != 1, remember that the interval and offset values
1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464
	 * have been appropriately scaled so the math is the same.
	 *
	 * The basic idea here is that we're increasing the multiplier
	 * by one, this causes the xtime_interval to be incremented by
	 * one cycle_interval. This is because:
	 *	xtime_interval = cycle_interval * mult
	 * So if mult is being incremented by one:
	 *	xtime_interval = cycle_interval * (mult + 1)
	 * Its the same as:
	 *	xtime_interval = (cycle_interval * mult) + cycle_interval
	 * Which can be shortened to:
	 *	xtime_interval += cycle_interval
	 *
	 * So offset stores the non-accumulated cycles. Thus the current
	 * time (in shifted nanoseconds) is:
	 *	now = (offset * adj) + xtime_nsec
	 * Now, even though we're adjusting the clock frequency, we have
	 * to keep time consistent. In other words, we can't jump back
	 * in time, and we also want to avoid jumping forward in time.
	 *
	 * So given the same offset value, we need the time to be the same
	 * both before and after the freq adjustment.
	 *	now = (offset * adj_1) + xtime_nsec_1
	 *	now = (offset * adj_2) + xtime_nsec_2
	 * So:
	 *	(offset * adj_1) + xtime_nsec_1 =
	 *		(offset * adj_2) + xtime_nsec_2
	 * And we know:
	 *	adj_2 = adj_1 + 1
	 * So:
	 *	(offset * adj_1) + xtime_nsec_1 =
	 *		(offset * (adj_1+1)) + xtime_nsec_2
	 *	(offset * adj_1) + xtime_nsec_1 =
	 *		(offset * adj_1) + offset + xtime_nsec_2
	 * Canceling the sides:
	 *	xtime_nsec_1 = offset + xtime_nsec_2
	 * Which gives us:
	 *	xtime_nsec_2 = xtime_nsec_1 - offset
	 * Which simplfies to:
	 *	xtime_nsec -= offset
	 *
	 * XXX - TODO: Doc ntp_error calculation.
	 */
1465
	if ((mult_adj > 0) && (tk->tkr.mult + mult_adj < mult_adj)) {
1466 1467 1468 1469 1470
		/* NTP adjustment caused clocksource mult overflow */
		WARN_ON_ONCE(1);
		return;
	}

1471
	tk->tkr.mult += mult_adj;
1472
	tk->xtime_interval += interval;
1473
	tk->tkr.xtime_nsec -= offset;
1474
	tk->ntp_error -= (interval - offset) << tk->ntp_error_shift;
1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493
}

/*
 * Calculate the multiplier adjustment needed to match the frequency
 * specified by NTP
 */
static __always_inline void timekeeping_freqadjust(struct timekeeper *tk,
							s64 offset)
{
	s64 interval = tk->cycle_interval;
	s64 xinterval = tk->xtime_interval;
	s64 tick_error;
	bool negative;
	u32 adj;

	/* Remove any current error adj from freq calculation */
	if (tk->ntp_err_mult)
		xinterval -= tk->cycle_interval;

1494 1495
	tk->ntp_tick = ntp_tick_length();

1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535
	/* Calculate current error per tick */
	tick_error = ntp_tick_length() >> tk->ntp_error_shift;
	tick_error -= (xinterval + tk->xtime_remainder);

	/* Don't worry about correcting it if its small */
	if (likely((tick_error >= 0) && (tick_error <= interval)))
		return;

	/* preserve the direction of correction */
	negative = (tick_error < 0);

	/* Sort out the magnitude of the correction */
	tick_error = abs(tick_error);
	for (adj = 0; tick_error > interval; adj++)
		tick_error >>= 1;

	/* scale the corrections */
	timekeeping_apply_adjustment(tk, offset, negative, adj);
}

/*
 * Adjust the timekeeper's multiplier to the correct frequency
 * and also to reduce the accumulated error value.
 */
static void timekeeping_adjust(struct timekeeper *tk, s64 offset)
{
	/* Correct for the current frequency error */
	timekeeping_freqadjust(tk, offset);

	/* Next make a small adjustment to fix any cumulative error */
	if (!tk->ntp_err_mult && (tk->ntp_error > 0)) {
		tk->ntp_err_mult = 1;
		timekeeping_apply_adjustment(tk, offset, 0, 0);
	} else if (tk->ntp_err_mult && (tk->ntp_error <= 0)) {
		/* Undo any existing error adjustment */
		timekeeping_apply_adjustment(tk, offset, 1, 0);
		tk->ntp_err_mult = 0;
	}

	if (unlikely(tk->tkr.clock->maxadj &&
1536 1537
		(abs(tk->tkr.mult - tk->tkr.clock->mult)
			> tk->tkr.clock->maxadj))) {
1538 1539 1540 1541 1542
		printk_once(KERN_WARNING
			"Adjusting %s more than 11%% (%ld vs %ld)\n",
			tk->tkr.clock->name, (long)tk->tkr.mult,
			(long)tk->tkr.clock->mult + tk->tkr.clock->maxadj);
	}
1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557

	/*
	 * It may be possible that when we entered this function, xtime_nsec
	 * was very small.  Further, if we're slightly speeding the clocksource
	 * in the code above, its possible the required corrective factor to
	 * xtime_nsec could cause it to underflow.
	 *
	 * Now, since we already accumulated the second, cannot simply roll
	 * the accumulated second back, since the NTP subsystem has been
	 * notified via second_overflow. So instead we push xtime_nsec forward
	 * by the amount we underflowed, and add that amount into the error.
	 *
	 * We'll correct this error next time through this function, when
	 * xtime_nsec is not as small.
	 */
1558 1559 1560
	if (unlikely((s64)tk->tkr.xtime_nsec < 0)) {
		s64 neg = -(s64)tk->tkr.xtime_nsec;
		tk->tkr.xtime_nsec = 0;
1561
		tk->ntp_error += neg << tk->ntp_error_shift;
1562
	}
1563 1564
}

1565 1566 1567 1568 1569 1570 1571 1572
/**
 * accumulate_nsecs_to_secs - Accumulates nsecs into secs
 *
 * Helper function that accumulates a the nsecs greater then a second
 * from the xtime_nsec field to the xtime_secs field.
 * It also calls into the NTP code to handle leapsecond processing.
 *
 */
1573
static inline unsigned int accumulate_nsecs_to_secs(struct timekeeper *tk)
1574
{
1575
	u64 nsecps = (u64)NSEC_PER_SEC << tk->tkr.shift;
1576
	unsigned int clock_set = 0;
1577

1578
	while (tk->tkr.xtime_nsec >= nsecps) {
1579 1580
		int leap;

1581
		tk->tkr.xtime_nsec -= nsecps;
1582 1583 1584 1585
		tk->xtime_sec++;

		/* Figure out if its a leap sec and apply if needed */
		leap = second_overflow(tk->xtime_sec);
1586
		if (unlikely(leap)) {
1587
			struct timespec64 ts;
1588 1589

			tk->xtime_sec += leap;
1590

1591 1592 1593
			ts.tv_sec = leap;
			ts.tv_nsec = 0;
			tk_set_wall_to_mono(tk,
1594
				timespec64_sub(tk->wall_to_monotonic, ts));
1595

1596 1597
			__timekeeping_set_tai_offset(tk, tk->tai_offset - leap);

1598
			clock_set = TK_CLOCK_WAS_SET;
1599
		}
1600
	}
1601
	return clock_set;
1602 1603
}

1604 1605 1606 1607 1608 1609 1610 1611 1612
/**
 * logarithmic_accumulation - shifted accumulation of cycles
 *
 * This functions accumulates a shifted interval of cycles into
 * into a shifted interval nanoseconds. Allows for O(log) accumulation
 * loop.
 *
 * Returns the unconsumed cycles.
 */
1613
static cycle_t logarithmic_accumulation(struct timekeeper *tk, cycle_t offset,
1614 1615
						u32 shift,
						unsigned int *clock_set)
1616
{
T
Thomas Gleixner 已提交
1617
	cycle_t interval = tk->cycle_interval << shift;
1618
	u64 raw_nsecs;
1619

1620
	/* If the offset is smaller then a shifted interval, do nothing */
T
Thomas Gleixner 已提交
1621
	if (offset < interval)
1622 1623 1624
		return offset;

	/* Accumulate one shifted interval */
T
Thomas Gleixner 已提交
1625
	offset -= interval;
1626
	tk->tkr.cycle_last += interval;
1627

1628
	tk->tkr.xtime_nsec += tk->xtime_interval << shift;
1629
	*clock_set |= accumulate_nsecs_to_secs(tk);
1630

1631
	/* Accumulate raw time */
1632
	raw_nsecs = (u64)tk->raw_interval << shift;
1633
	raw_nsecs += tk->raw_time.tv_nsec;
1634 1635 1636
	if (raw_nsecs >= NSEC_PER_SEC) {
		u64 raw_secs = raw_nsecs;
		raw_nsecs = do_div(raw_secs, NSEC_PER_SEC);
1637
		tk->raw_time.tv_sec += raw_secs;
1638
	}
1639
	tk->raw_time.tv_nsec = raw_nsecs;
1640 1641

	/* Accumulate error between NTP and clock interval */
1642
	tk->ntp_error += tk->ntp_tick << shift;
1643 1644
	tk->ntp_error -= (tk->xtime_interval + tk->xtime_remainder) <<
						(tk->ntp_error_shift + shift);
1645 1646 1647 1648

	return offset;
}

1649 1650 1651 1652
/**
 * update_wall_time - Uses the current clocksource to increment the wall time
 *
 */
1653
void update_wall_time(void)
1654
{
1655
	struct timekeeper *real_tk = &tk_core.timekeeper;
1656
	struct timekeeper *tk = &shadow_timekeeper;
1657
	cycle_t offset;
1658
	int shift = 0, maxshift;
1659
	unsigned int clock_set = 0;
J
John Stultz 已提交
1660 1661
	unsigned long flags;

1662
	raw_spin_lock_irqsave(&timekeeper_lock, flags);
1663 1664 1665

	/* Make sure we're fully resumed: */
	if (unlikely(timekeeping_suspended))
J
John Stultz 已提交
1666
		goto out;
1667

J
John Stultz 已提交
1668
#ifdef CONFIG_ARCH_USES_GETTIMEOFFSET
1669
	offset = real_tk->cycle_interval;
J
John Stultz 已提交
1670
#else
1671 1672
	offset = clocksource_delta(tk->tkr.read(tk->tkr.clock),
				   tk->tkr.cycle_last, tk->tkr.mask);
1673 1674
#endif

1675
	/* Check if there's really nothing to do */
1676
	if (offset < real_tk->cycle_interval)
1677 1678
		goto out;

1679 1680 1681
	/* Do some additional sanity checking */
	timekeeping_check_update(real_tk, offset);

1682 1683 1684 1685
	/*
	 * With NO_HZ we may have to accumulate many cycle_intervals
	 * (think "ticks") worth of time at once. To do this efficiently,
	 * we calculate the largest doubling multiple of cycle_intervals
1686
	 * that is smaller than the offset.  We then accumulate that
1687 1688
	 * chunk in one go, and then try to consume the next smaller
	 * doubled multiple.
1689
	 */
1690
	shift = ilog2(offset) - ilog2(tk->cycle_interval);
1691
	shift = max(0, shift);
1692
	/* Bound shift to one less than what overflows tick_length */
1693
	maxshift = (64 - (ilog2(ntp_tick_length())+1)) - 1;
1694
	shift = min(shift, maxshift);
1695
	while (offset >= tk->cycle_interval) {
1696 1697
		offset = logarithmic_accumulation(tk, offset, shift,
							&clock_set);
1698
		if (offset < tk->cycle_interval<<shift)
1699
			shift--;
1700 1701 1702
	}

	/* correct the clock when NTP error is too big */
1703
	timekeeping_adjust(tk, offset);
1704

J
John Stultz 已提交
1705
	/*
1706 1707 1708 1709
	 * XXX This can be killed once everyone converts
	 * to the new update_vsyscall.
	 */
	old_vsyscall_fixup(tk);
1710

J
John Stultz 已提交
1711 1712
	/*
	 * Finally, make sure that after the rounding
1713
	 * xtime_nsec isn't larger than NSEC_PER_SEC
J
John Stultz 已提交
1714
	 */
1715
	clock_set |= accumulate_nsecs_to_secs(tk);
L
Linus Torvalds 已提交
1716

1717
	write_seqcount_begin(&tk_core.seq);
1718 1719 1720 1721 1722 1723 1724
	/*
	 * Update the real timekeeper.
	 *
	 * We could avoid this memcpy by switching pointers, but that
	 * requires changes to all other timekeeper usage sites as
	 * well, i.e. move the timekeeper pointer getter into the
	 * spinlocked/seqcount protected sections. And we trade this
1725
	 * memcpy under the tk_core.seq against one before we start
1726 1727 1728
	 * updating.
	 */
	memcpy(real_tk, tk, sizeof(*tk));
1729
	timekeeping_update(real_tk, clock_set);
1730
	write_seqcount_end(&tk_core.seq);
1731
out:
1732
	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1733
	if (clock_set)
1734 1735
		/* Have to call _delayed version, since in irq context*/
		clock_was_set_delayed();
1736
}
T
Tomas Janousek 已提交
1737 1738

/**
1739 1740
 * getboottime64 - Return the real time of system boot.
 * @ts:		pointer to the timespec64 to be set
T
Tomas Janousek 已提交
1741
 *
1742
 * Returns the wall-time of boot in a timespec64.
T
Tomas Janousek 已提交
1743 1744 1745 1746 1747 1748
 *
 * This is based on the wall_to_monotonic offset and the total suspend
 * time. Calls to settimeofday will affect the value returned (which
 * basically means that however wrong your real time clock is at boot time,
 * you get the right time here).
 */
1749
void getboottime64(struct timespec64 *ts)
T
Tomas Janousek 已提交
1750
{
1751
	struct timekeeper *tk = &tk_core.timekeeper;
1752 1753
	ktime_t t = ktime_sub(tk->offs_real, tk->offs_boot);

1754
	*ts = ktime_to_timespec64(t);
T
Tomas Janousek 已提交
1755
}
1756
EXPORT_SYMBOL_GPL(getboottime64);
T
Tomas Janousek 已提交
1757

1758 1759
unsigned long get_seconds(void)
{
1760
	struct timekeeper *tk = &tk_core.timekeeper;
1761 1762

	return tk->xtime_sec;
1763 1764 1765
}
EXPORT_SYMBOL(get_seconds);

1766 1767
struct timespec __current_kernel_time(void)
{
1768
	struct timekeeper *tk = &tk_core.timekeeper;
1769

1770
	return timespec64_to_timespec(tk_xtime(tk));
1771
}
1772

1773 1774
struct timespec current_kernel_time(void)
{
1775
	struct timekeeper *tk = &tk_core.timekeeper;
1776
	struct timespec64 now;
1777 1778 1779
	unsigned long seq;

	do {
1780
		seq = read_seqcount_begin(&tk_core.seq);
L
Linus Torvalds 已提交
1781

1782
		now = tk_xtime(tk);
1783
	} while (read_seqcount_retry(&tk_core.seq, seq));
1784

1785
	return timespec64_to_timespec(now);
1786 1787
}
EXPORT_SYMBOL(current_kernel_time);
1788

1789
struct timespec64 get_monotonic_coarse64(void)
1790
{
1791
	struct timekeeper *tk = &tk_core.timekeeper;
1792
	struct timespec64 now, mono;
1793 1794 1795
	unsigned long seq;

	do {
1796
		seq = read_seqcount_begin(&tk_core.seq);
L
Linus Torvalds 已提交
1797

1798 1799
		now = tk_xtime(tk);
		mono = tk->wall_to_monotonic;
1800
	} while (read_seqcount_retry(&tk_core.seq, seq));
1801

1802
	set_normalized_timespec64(&now, now.tv_sec + mono.tv_sec,
1803
				now.tv_nsec + mono.tv_nsec);
1804

1805
	return now;
1806
}
1807 1808

/*
1809
 * Must hold jiffies_lock
1810 1811 1812 1813 1814 1815
 */
void do_timer(unsigned long ticks)
{
	jiffies_64 += ticks;
	calc_global_load(ticks);
}
1816 1817

/**
1818 1819 1820 1821 1822 1823
 * ktime_get_update_offsets_tick - hrtimer helper
 * @offs_real:	pointer to storage for monotonic -> realtime offset
 * @offs_boot:	pointer to storage for monotonic -> boottime offset
 * @offs_tai:	pointer to storage for monotonic -> clock tai offset
 *
 * Returns monotonic time at last tick and various offsets
1824
 */
1825 1826
ktime_t ktime_get_update_offsets_tick(ktime_t *offs_real, ktime_t *offs_boot,
							ktime_t *offs_tai)
1827
{
1828
	struct timekeeper *tk = &tk_core.timekeeper;
1829
	unsigned int seq;
1830 1831
	ktime_t base;
	u64 nsecs;
1832 1833

	do {
1834
		seq = read_seqcount_begin(&tk_core.seq);
1835

1836 1837
		base = tk->tkr.base_mono;
		nsecs = tk->tkr.xtime_nsec >> tk->tkr.shift;
1838

1839 1840 1841
		*offs_real = tk->offs_real;
		*offs_boot = tk->offs_boot;
		*offs_tai = tk->offs_tai;
1842
	} while (read_seqcount_retry(&tk_core.seq, seq));
1843

1844
	return ktime_add_ns(base, nsecs);
1845
}
T
Torben Hohn 已提交
1846

1847 1848
#ifdef CONFIG_HIGH_RES_TIMERS
/**
1849
 * ktime_get_update_offsets_now - hrtimer helper
1850 1851
 * @offs_real:	pointer to storage for monotonic -> realtime offset
 * @offs_boot:	pointer to storage for monotonic -> boottime offset
1852
 * @offs_tai:	pointer to storage for monotonic -> clock tai offset
1853 1854
 *
 * Returns current monotonic time and updates the offsets
1855
 * Called from hrtimer_interrupt() or retrigger_next_event()
1856
 */
1857
ktime_t ktime_get_update_offsets_now(ktime_t *offs_real, ktime_t *offs_boot,
1858
							ktime_t *offs_tai)
1859
{
1860
	struct timekeeper *tk = &tk_core.timekeeper;
1861
	unsigned int seq;
1862 1863
	ktime_t base;
	u64 nsecs;
1864 1865

	do {
1866
		seq = read_seqcount_begin(&tk_core.seq);
1867

1868
		base = tk->tkr.base_mono;
1869
		nsecs = timekeeping_get_ns(&tk->tkr);
1870

1871 1872
		*offs_real = tk->offs_real;
		*offs_boot = tk->offs_boot;
1873
		*offs_tai = tk->offs_tai;
1874
	} while (read_seqcount_retry(&tk_core.seq, seq));
1875

1876
	return ktime_add_ns(base, nsecs);
1877 1878 1879
}
#endif

1880 1881 1882 1883 1884
/**
 * do_adjtimex() - Accessor function to NTP __do_adjtimex function
 */
int do_adjtimex(struct timex *txc)
{
1885
	struct timekeeper *tk = &tk_core.timekeeper;
1886
	unsigned long flags;
1887
	struct timespec64 ts;
1888
	s32 orig_tai, tai;
1889 1890 1891 1892 1893 1894 1895
	int ret;

	/* Validate the data before disabling interrupts */
	ret = ntp_validate_timex(txc);
	if (ret)
		return ret;

1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906
	if (txc->modes & ADJ_SETOFFSET) {
		struct timespec delta;
		delta.tv_sec  = txc->time.tv_sec;
		delta.tv_nsec = txc->time.tv_usec;
		if (!(txc->modes & ADJ_NANO))
			delta.tv_nsec *= 1000;
		ret = timekeeping_inject_offset(&delta);
		if (ret)
			return ret;
	}

1907
	getnstimeofday64(&ts);
1908

1909
	raw_spin_lock_irqsave(&timekeeper_lock, flags);
1910
	write_seqcount_begin(&tk_core.seq);
1911

1912
	orig_tai = tai = tk->tai_offset;
1913
	ret = __do_adjtimex(txc, &ts, &tai);
1914

1915 1916
	if (tai != orig_tai) {
		__timekeeping_set_tai_offset(tk, tai);
1917
		timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
1918
	}
1919
	write_seqcount_end(&tk_core.seq);
1920 1921
	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);

1922 1923 1924
	if (tai != orig_tai)
		clock_was_set();

1925 1926
	ntp_notify_cmos_timer();

1927 1928
	return ret;
}
1929 1930 1931 1932 1933 1934 1935

#ifdef CONFIG_NTP_PPS
/**
 * hardpps() - Accessor function to NTP __hardpps function
 */
void hardpps(const struct timespec *phase_ts, const struct timespec *raw_ts)
{
1936 1937 1938
	unsigned long flags;

	raw_spin_lock_irqsave(&timekeeper_lock, flags);
1939
	write_seqcount_begin(&tk_core.seq);
1940

1941
	__hardpps(phase_ts, raw_ts);
1942

1943
	write_seqcount_end(&tk_core.seq);
1944
	raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
1945 1946 1947 1948
}
EXPORT_SYMBOL(hardpps);
#endif

T
Torben Hohn 已提交
1949 1950 1951 1952 1953 1954 1955 1956
/**
 * xtime_update() - advances the timekeeping infrastructure
 * @ticks:	number of ticks, that have elapsed since the last call.
 *
 * Must be called with interrupts disabled.
 */
void xtime_update(unsigned long ticks)
{
1957
	write_seqlock(&jiffies_lock);
T
Torben Hohn 已提交
1958
	do_timer(ticks);
1959
	write_sequnlock(&jiffies_lock);
1960
	update_wall_time();
T
Torben Hohn 已提交
1961
}