time.c 30.3 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
/*
 * Common time routines among all ppc machines.
 *
 * Written by Cort Dougan (cort@cs.nmt.edu) to merge
 * Paul Mackerras' version and mine for PReP and Pmac.
 * MPC8xx/MBX changes by Dan Malek (dmalek@jlc.net).
 * Converted for 64-bit by Mike Corrigan (mikejc@us.ibm.com)
 *
 * First round of bugfixes by Gabriel Paubert (paubert@iram.es)
 * to make clock more stable (2.4.0-test5). The only thing
 * that this code assumes is that the timebases have been synchronized
 * by firmware on SMP and are never stopped (never do sleep
 * on SMP then, nap and doze are OK).
 * 
 * Speeded up do_gettimeofday by getting rid of references to
 * xtime (which required locks for consistency). (mikejc@us.ibm.com)
 *
 * TODO (not necessarily in this file):
 * - improve precision and reproducibility of timebase frequency
20
 * measurement at boot time.
L
Linus Torvalds 已提交
21 22 23 24 25 26 27 28 29 30 31 32 33 34
 * - for astronomical applications: add a new function to get
 * non ambiguous timestamps even around leap seconds. This needs
 * a new timestamp format and a good name.
 *
 * 1997-09-10  Updated NTP code according to technical memorandum Jan '96
 *             "A Kernel Model for Precision Timekeeping" by Dave Mills
 *
 *      This program is free software; you can redistribute it and/or
 *      modify it under the terms of the GNU General Public License
 *      as published by the Free Software Foundation; either version
 *      2 of the License, or (at your option) any later version.
 */

#include <linux/errno.h>
35
#include <linux/export.h>
L
Linus Torvalds 已提交
36 37 38 39 40 41 42 43 44
#include <linux/sched.h>
#include <linux/kernel.h>
#include <linux/param.h>
#include <linux/string.h>
#include <linux/mm.h>
#include <linux/interrupt.h>
#include <linux/timex.h>
#include <linux/kernel_stat.h>
#include <linux/time.h>
45
#include <linux/clockchips.h>
L
Linus Torvalds 已提交
46 47 48 49
#include <linux/init.h>
#include <linux/profile.h>
#include <linux/cpu.h>
#include <linux/security.h>
50 51
#include <linux/percpu.h>
#include <linux/rtc.h>
52
#include <linux/jiffies.h>
53
#include <linux/posix-timers.h>
54
#include <linux/irq.h>
55
#include <linux/delay.h>
56
#include <linux/irq_work.h>
57
#include <linux/clk-provider.h>
58
#include <linux/suspend.h>
59
#include <linux/rtc.h>
60
#include <asm/trace.h>
L
Linus Torvalds 已提交
61 62 63 64 65 66

#include <asm/io.h>
#include <asm/processor.h>
#include <asm/nvram.h>
#include <asm/cache.h>
#include <asm/machdep.h>
67
#include <linux/uaccess.h>
L
Linus Torvalds 已提交
68 69
#include <asm/time.h>
#include <asm/prom.h>
70 71
#include <asm/irq.h>
#include <asm/div64.h>
P
Paul Mackerras 已提交
72
#include <asm/smp.h>
73
#include <asm/vdso_datapage.h>
74
#include <asm/firmware.h>
M
Michael Neuling 已提交
75
#include <asm/cputime.h>
76
#include <asm/asm-prototypes.h>
L
Linus Torvalds 已提交
77

78 79
/* powerpc clocksource/clockevent code */

80
#include <linux/clockchips.h>
81
#include <linux/timekeeper_internal.h>
82

83
static u64 rtc_read(struct clocksource *);
84 85 86 87 88 89 90 91
static struct clocksource clocksource_rtc = {
	.name         = "rtc",
	.rating       = 400,
	.flags        = CLOCK_SOURCE_IS_CONTINUOUS,
	.mask         = CLOCKSOURCE_MASK(64),
	.read         = rtc_read,
};

92
static u64 timebase_read(struct clocksource *);
93 94 95 96 97 98 99 100
static struct clocksource clocksource_timebase = {
	.name         = "timebase",
	.rating       = 400,
	.flags        = CLOCK_SOURCE_IS_CONTINUOUS,
	.mask         = CLOCKSOURCE_MASK(64),
	.read         = timebase_read,
};

101 102
#define DECREMENTER_DEFAULT_MAX 0x7FFFFFFF
u64 decrementer_max = DECREMENTER_DEFAULT_MAX;
103 104 105

static int decrementer_set_next_event(unsigned long evt,
				      struct clock_event_device *dev);
106
static int decrementer_shutdown(struct clock_event_device *evt);
107

108
struct clock_event_device decrementer_clockevent = {
109 110 111 112 113 114 115 116
	.name			= "decrementer",
	.rating			= 200,
	.irq			= 0,
	.set_next_event		= decrementer_set_next_event,
	.set_state_shutdown	= decrementer_shutdown,
	.tick_resume		= decrementer_shutdown,
	.features		= CLOCK_EVT_FEAT_ONESHOT |
				  CLOCK_EVT_FEAT_C3STOP,
117
};
118
EXPORT_SYMBOL(decrementer_clockevent);
119

120 121
DEFINE_PER_CPU(u64, decrementers_next_tb);
static DEFINE_PER_CPU(struct clock_event_device, decrementers);
122

L
Linus Torvalds 已提交
123 124
#define XSEC_PER_SEC (1024*1024)

125 126 127 128 129 130 131
#ifdef CONFIG_PPC64
#define SCALE_XSEC(xsec, max)	(((xsec) * max) / XSEC_PER_SEC)
#else
/* compute ((xsec << 12) * max) >> 32 */
#define SCALE_XSEC(xsec, max)	mulhwu((xsec) << 12, max)
#endif

L
Linus Torvalds 已提交
132 133 134 135
unsigned long tb_ticks_per_jiffy;
unsigned long tb_ticks_per_usec = 100; /* sane default */
EXPORT_SYMBOL(tb_ticks_per_usec);
unsigned long tb_ticks_per_sec;
136
EXPORT_SYMBOL(tb_ticks_per_sec);	/* for cputime_t conversions */
137

L
Linus Torvalds 已提交
138
DEFINE_SPINLOCK(rtc_lock);
139
EXPORT_SYMBOL_GPL(rtc_lock);
L
Linus Torvalds 已提交
140

141 142
static u64 tb_to_ns_scale __read_mostly;
static unsigned tb_to_ns_shift __read_mostly;
143
static u64 boot_tb __read_mostly;
L
Linus Torvalds 已提交
144 145

extern struct timezone sys_tz;
146
static long timezone_offset;
L
Linus Torvalds 已提交
147

148
unsigned long ppc_proc_freq;
149
EXPORT_SYMBOL_GPL(ppc_proc_freq);
150
unsigned long ppc_tb_freq;
151
EXPORT_SYMBOL_GPL(ppc_tb_freq);
152

153
#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
154 155
/*
 * Factors for converting from cputime_t (timebase ticks) to
156
 * jiffies, microseconds, seconds, and clock_t (1/USER_HZ seconds).
157 158 159
 * These are all stored as 0.64 fixed-point binary fractions.
 */
u64 __cputime_jiffies_factor;
160
EXPORT_SYMBOL(__cputime_jiffies_factor);
161 162
u64 __cputime_usec_factor;
EXPORT_SYMBOL(__cputime_usec_factor);
163
u64 __cputime_sec_factor;
164
EXPORT_SYMBOL(__cputime_sec_factor);
165
u64 __cputime_clockt_factor;
166
EXPORT_SYMBOL(__cputime_clockt_factor);
167

168 169
cputime_t cputime_one_jiffy;

170
#ifdef CONFIG_PPC_SPLPAR
171
void (*dtl_consumer)(struct dtl_entry *, u64);
172 173 174 175 176 177 178
#endif

#ifdef CONFIG_PPC64
#define get_accounting(tsk)	(&get_paca()->accounting)
#else
#define get_accounting(tsk)	(&task_thread_info(tsk)->accounting)
#endif
179

180 181 182 183 184 185
static void calc_cputime_factors(void)
{
	struct div_result res;

	div128_by_32(HZ, 0, tb_ticks_per_sec, &res);
	__cputime_jiffies_factor = res.result_low;
186 187
	div128_by_32(1000000, 0, tb_ticks_per_sec, &res);
	__cputime_usec_factor = res.result_low;
188 189 190 191 192 193 194
	div128_by_32(1, 0, tb_ticks_per_sec, &res);
	__cputime_sec_factor = res.result_low;
	div128_by_32(USER_HZ, 0, tb_ticks_per_sec, &res);
	__cputime_clockt_factor = res.result_low;
}

/*
195 196
 * Read the SPURR on systems that have it, otherwise the PURR,
 * or if that doesn't exist return the timebase value passed in.
197
 */
198
static unsigned long read_spurr(unsigned long tb)
199
{
200 201
	if (cpu_has_feature(CPU_FTR_SPURR))
		return mfspr(SPRN_SPURR);
202 203
	if (cpu_has_feature(CPU_FTR_PURR))
		return mfspr(SPRN_PURR);
204
	return tb;
205 206
}

207 208
#ifdef CONFIG_PPC_SPLPAR

209
/*
210 211
 * Scan the dispatch trace log and count up the stolen time.
 * Should be called with interrupts disabled.
212
 */
213
static u64 scan_dispatch_log(u64 stop_tb)
214
{
215
	u64 i = local_paca->dtl_ridx;
216 217 218 219 220 221 222
	struct dtl_entry *dtl = local_paca->dtl_curr;
	struct dtl_entry *dtl_end = local_paca->dispatch_log_end;
	struct lppaca *vpa = local_paca->lppaca_ptr;
	u64 tb_delta;
	u64 stolen = 0;
	u64 dtb;

223 224 225
	if (!dtl)
		return 0;

226
	if (i == be64_to_cpu(vpa->dtl_idx))
227
		return 0;
228 229 230 231
	while (i < be64_to_cpu(vpa->dtl_idx)) {
		dtb = be64_to_cpu(dtl->timebase);
		tb_delta = be32_to_cpu(dtl->enqueue_to_dispatch_time) +
			be32_to_cpu(dtl->ready_to_enqueue_time);
232
		barrier();
233
		if (i + N_DISPATCH_LOG < be64_to_cpu(vpa->dtl_idx)) {
234
			/* buffer has overflowed */
235
			i = be64_to_cpu(vpa->dtl_idx) - N_DISPATCH_LOG;
236 237 238 239 240
			dtl = local_paca->dispatch_log + (i % N_DISPATCH_LOG);
			continue;
		}
		if (dtb > stop_tb)
			break;
241 242
		if (dtl_consumer)
			dtl_consumer(dtl, i);
243 244 245 246 247 248 249 250 251
		stolen += tb_delta;
		++i;
		++dtl;
		if (dtl == dtl_end)
			dtl = local_paca->dispatch_log;
	}
	local_paca->dtl_ridx = i;
	local_paca->dtl_curr = dtl;
	return stolen;
252 253
}

254 255 256 257 258 259 260
/*
 * Accumulate stolen time by scanning the dispatch trace log.
 * Called on entry from user mode.
 */
void accumulate_stolen_time(void)
{
	u64 sst, ust;
261
	u8 save_soft_enabled = local_paca->soft_enabled;
262
	struct cpu_accounting_data *acct = &local_paca->accounting;
263 264 265 266 267 268 269 270 271

	/* We are called early in the exception entry, before
	 * soft/hard_enabled are sync'ed to the expected state
	 * for the exception. We are hard disabled but the PACA
	 * needs to reflect that so various debug stuff doesn't
	 * complain
	 */
	local_paca->soft_enabled = 0;

272 273 274 275
	sst = scan_dispatch_log(acct->starttime_user);
	ust = scan_dispatch_log(acct->starttime);
	acct->system_time -= sst;
	acct->user_time -= ust;
276 277 278
	local_paca->stolen_time += ust + sst;

	local_paca->soft_enabled = save_soft_enabled;
279 280 281 282 283 284
}

static inline u64 calculate_stolen_time(u64 stop_tb)
{
	u64 stolen = 0;

285
	if (get_paca()->dtl_ridx != be64_to_cpu(get_lppaca()->dtl_idx)) {
286
		stolen = scan_dispatch_log(stop_tb);
287
		get_paca()->accounting.system_time -= stolen;
288 289 290 291 292
	}

	stolen += get_paca()->stolen_time;
	get_paca()->stolen_time = 0;
	return stolen;
293 294
}

295 296 297 298 299 300 301 302
#else /* CONFIG_PPC_SPLPAR */
static inline u64 calculate_stolen_time(u64 stop_tb)
{
	return 0;
}

#endif /* CONFIG_PPC_SPLPAR */

303 304 305 306
/*
 * Account time for a transition between system, hard irq
 * or soft irq state.
 */
307 308 309
static unsigned long vtime_delta(struct task_struct *tsk,
				 unsigned long *sys_scaled,
				 unsigned long *stolen)
310
{
311 312 313
	unsigned long now, nowscaled, deltascaled;
	unsigned long udelta, delta, user_scaled;
	struct cpu_accounting_data *acct = get_accounting(tsk);
314

315 316
	WARN_ON_ONCE(!irqs_disabled());

317
	now = mftb();
318
	nowscaled = read_spurr(now);
319 320 321 322
	acct->system_time += now - acct->starttime;
	acct->starttime = now;
	deltascaled = nowscaled - acct->startspurr;
	acct->startspurr = nowscaled;
323

324
	*stolen = calculate_stolen_time(now);
325

326 327 328 329
	delta = acct->system_time;
	acct->system_time = 0;
	udelta = acct->user_time - acct->utime_sspurr;
	acct->utime_sspurr = acct->user_time;
330 331 332 333 334 335 336 337 338 339 340

	/*
	 * Because we don't read the SPURR on every kernel entry/exit,
	 * deltascaled includes both user and system SPURR ticks.
	 * Apportion these ticks to system SPURR ticks and user
	 * SPURR ticks in the same ratio as the system time (delta)
	 * and user time (udelta) values obtained from the timebase
	 * over the same interval.  The system ticks get accounted here;
	 * the user ticks get saved up in paca->user_time_scaled to be
	 * used by account_process_tick.
	 */
341
	*sys_scaled = delta;
342 343 344
	user_scaled = udelta;
	if (deltascaled != delta + udelta) {
		if (udelta) {
345 346
			*sys_scaled = deltascaled * delta / (delta + udelta);
			user_scaled = deltascaled - *sys_scaled;
347
		} else {
348
			*sys_scaled = deltascaled;
349 350
		}
	}
351
	acct->user_time_scaled += user_scaled;
352

353 354 355
	return delta;
}

356
void vtime_account_system(struct task_struct *tsk)
357
{
358
	unsigned long delta, sys_scaled, stolen;
359 360

	delta = vtime_delta(tsk, &sys_scaled, &stolen);
361 362
	account_system_time(tsk, 0, delta);
	tsk->stimescaled += sys_scaled;
363 364 365
	if (stolen)
		account_steal_time(stolen);
}
366
EXPORT_SYMBOL_GPL(vtime_account_system);
367

368
void vtime_account_idle(struct task_struct *tsk)
369
{
370
	unsigned long delta, sys_scaled, stolen;
371 372 373

	delta = vtime_delta(tsk, &sys_scaled, &stolen);
	account_idle_time(delta + stolen);
374 375 376
}

/*
377 378 379
 * Transfer the user time accumulated in the paca
 * by the exception entry and exit code to the generic
 * process user time records.
380
 * Must be called with interrupts disabled.
381 382
 * Assumes that vtime_account_system/idle() has been called
 * recently (i.e. since the last entry from usermode) so that
383
 * get_paca()->user_time_scaled is up to date.
384
 */
385
void vtime_account_user(struct task_struct *tsk)
386
{
387
	cputime_t utime, utimescaled;
388
	struct cpu_accounting_data *acct = get_accounting(tsk);
389

390 391 392 393 394
	utime = acct->user_time;
	utimescaled = acct->user_time_scaled;
	acct->user_time = 0;
	acct->user_time_scaled = 0;
	acct->utime_sspurr = 0;
395 396
	account_user_time(tsk, utime);
	tsk->utimescaled += utimescaled;
397 398
}

399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414
#ifdef CONFIG_PPC32
/*
 * Called from the context switch with interrupts disabled, to charge all
 * accumulated times to the current process, and to prepare accounting on
 * the next process.
 */
void arch_vtime_task_switch(struct task_struct *prev)
{
	struct cpu_accounting_data *acct = get_accounting(current);

	acct->starttime = get_accounting(prev)->starttime;
	acct->system_time = 0;
	acct->user_time = 0;
}
#endif /* CONFIG_PPC32 */

415
#else /* ! CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
416 417 418
#define calc_cputime_factors()
#endif

419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446
void __delay(unsigned long loops)
{
	unsigned long start;
	int diff;

	if (__USE_RTC()) {
		start = get_rtcl();
		do {
			/* the RTCL register wraps at 1000000000 */
			diff = get_rtcl() - start;
			if (diff < 0)
				diff += 1000000000;
		} while (diff < loops);
	} else {
		start = get_tbl();
		while (get_tbl() - start < loops)
			HMT_low();
		HMT_medium();
	}
}
EXPORT_SYMBOL(__delay);

void udelay(unsigned long usecs)
{
	__delay(tb_ticks_per_usec * usecs);
}
EXPORT_SYMBOL(udelay);

L
Linus Torvalds 已提交
447 448 449 450 451 452 453 454 455 456 457 458 459
#ifdef CONFIG_SMP
unsigned long profile_pc(struct pt_regs *regs)
{
	unsigned long pc = instruction_pointer(regs);

	if (in_lock_functions(pc))
		return regs->link;

	return pc;
}
EXPORT_SYMBOL(profile_pc);
#endif

460
#ifdef CONFIG_IRQ_WORK
461

462 463 464 465
/*
 * 64-bit uses a byte in the PACA, 32-bit uses a per-cpu variable...
 */
#ifdef CONFIG_PPC64
466
static inline unsigned long test_irq_work_pending(void)
467
{
468 469 470 471
	unsigned long x;

	asm volatile("lbz %0,%1(13)"
		: "=r" (x)
472
		: "i" (offsetof(struct paca_struct, irq_work_pending)));
473 474 475
	return x;
}

476
static inline void set_irq_work_pending_flag(void)
477 478 479
{
	asm volatile("stb %0,%1(13)" : :
		"r" (1),
480
		"i" (offsetof(struct paca_struct, irq_work_pending)));
481 482
}

483
static inline void clear_irq_work_pending(void)
484 485 486
{
	asm volatile("stb %0,%1(13)" : :
		"r" (0),
487
		"i" (offsetof(struct paca_struct, irq_work_pending)));
488 489
}

490 491
#else /* 32-bit */

492
DEFINE_PER_CPU(u8, irq_work_pending);
493

494 495 496
#define set_irq_work_pending_flag()	__this_cpu_write(irq_work_pending, 1)
#define test_irq_work_pending()		__this_cpu_read(irq_work_pending)
#define clear_irq_work_pending()	__this_cpu_write(irq_work_pending, 0)
497

498 499
#endif /* 32 vs 64 bit */

500
void arch_irq_work_raise(void)
501 502
{
	preempt_disable();
503
	set_irq_work_pending_flag();
504 505 506 507
	set_dec(1);
	preempt_enable();
}

508
#else  /* CONFIG_IRQ_WORK */
509

510 511
#define test_irq_work_pending()	0
#define clear_irq_work_pending()
512

513
#endif /* CONFIG_IRQ_WORK */
514

515
static void __timer_interrupt(void)
516 517
{
	struct pt_regs *regs = get_irq_regs();
518 519
	u64 *next_tb = this_cpu_ptr(&decrementers_next_tb);
	struct clock_event_device *evt = this_cpu_ptr(&decrementers);
520 521 522 523 524 525 526 527 528 529 530 531 532 533
	u64 now;

	trace_timer_interrupt_entry(regs);

	if (test_irq_work_pending()) {
		clear_irq_work_pending();
		irq_work_run();
	}

	now = get_tb_or_rtc();
	if (now >= *next_tb) {
		*next_tb = ~(u64)0;
		if (evt->event_handler)
			evt->event_handler(evt);
534
		__this_cpu_inc(irq_stat.timer_irqs_event);
535 536
	} else {
		now = *next_tb - now;
537 538
		if (now <= decrementer_max)
			set_dec(now);
539 540 541
		/* We may have raced with new irq work */
		if (test_irq_work_pending())
			set_dec(1);
542
		__this_cpu_inc(irq_stat.timer_irqs_others);
543 544 545 546 547
	}

#ifdef CONFIG_PPC64
	/* collect purr register values often, for accurate calculations */
	if (firmware_has_feature(FW_FEATURE_SPLPAR)) {
548
		struct cpu_usage *cu = this_cpu_ptr(&cpu_usage_array);
549 550 551 552 553 554 555
		cu->current_tb = mfspr(SPRN_PURR);
	}
#endif

	trace_timer_interrupt_exit(regs);
}

L
Linus Torvalds 已提交
556 557 558 559
/*
 * timer_interrupt - gets called when the decrementer overflows,
 * with interrupts disabled.
 */
560
void timer_interrupt(struct pt_regs * regs)
L
Linus Torvalds 已提交
561
{
562
	struct pt_regs *old_regs;
563
	u64 *next_tb = this_cpu_ptr(&decrementers_next_tb);
564

565 566 567
	/* Ensure a positive value is written to the decrementer, or else
	 * some CPUs will continue to take decrementer exceptions.
	 */
568
	set_dec(decrementer_max);
569 570

	/* Some implementations of hotplug will get timer interrupts while
571 572 573 574
	 * offline, just ignore these and we also need to set
	 * decrementers_next_tb as MAX to make sure __check_irq_replay
	 * don't replay timer interrupt when return, otherwise we'll trap
	 * here infinitely :(
575
	 */
576 577
	if (!cpu_online(smp_processor_id())) {
		*next_tb = ~(u64)0;
578
		return;
579
	}
580

581 582 583 584 585
	/* Conditionally hard-enable interrupts now that the DEC has been
	 * bumped to its maximum value
	 */
	may_hard_irq_enable();

586

P
Paul Bolle 已提交
587
#if defined(CONFIG_PPC32) && defined(CONFIG_PPC_PMAC)
588 589 590
	if (atomic_read(&ppc_n_lost_interrupts) != 0)
		do_IRQ(regs);
#endif
L
Linus Torvalds 已提交
591

592
	old_regs = set_irq_regs(regs);
L
Linus Torvalds 已提交
593 594
	irq_enter();

595
	__timer_interrupt();
L
Linus Torvalds 已提交
596
	irq_exit();
597
	set_irq_regs(old_regs);
L
Linus Torvalds 已提交
598
}
A
Al Viro 已提交
599
EXPORT_SYMBOL(timer_interrupt);
L
Linus Torvalds 已提交
600

601 602 603 604 605 606 607 608 609
/*
 * Hypervisor decrementer interrupts shouldn't occur but are sometimes
 * left pending on exit from a KVM guest.  We don't need to do anything
 * to clear them, as they are edge-triggered.
 */
void hdec_interrupt(struct pt_regs *regs)
{
}

610
#ifdef CONFIG_SUSPEND
611
static void generic_suspend_disable_irqs(void)
612 613 614 615 616
{
	/* Disable the decrementer, so that it doesn't interfere
	 * with suspending.
	 */

617
	set_dec(decrementer_max);
618
	local_irq_disable();
619
	set_dec(decrementer_max);
620 621
}

622
static void generic_suspend_enable_irqs(void)
623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643
{
	local_irq_enable();
}

/* Overrides the weak version in kernel/power/main.c */
void arch_suspend_disable_irqs(void)
{
	if (ppc_md.suspend_disable_irqs)
		ppc_md.suspend_disable_irqs();
	generic_suspend_disable_irqs();
}

/* Overrides the weak version in kernel/power/main.c */
void arch_suspend_enable_irqs(void)
{
	generic_suspend_enable_irqs();
	if (ppc_md.suspend_enable_irqs)
		ppc_md.suspend_enable_irqs();
}
#endif

644 645 646 647 648 649
unsigned long long tb_to_ns(unsigned long long ticks)
{
	return mulhdu(ticks, tb_to_ns_scale) << tb_to_ns_shift;
}
EXPORT_SYMBOL_GPL(tb_to_ns);

L
Linus Torvalds 已提交
650 651 652 653 654 655 656 657 658
/*
 * Scheduler clock - returns current time in nanosec units.
 *
 * Note: mulhdu(a, b) (multiply high double unsigned) returns
 * the high 64 bits of a * b, i.e. (a * b) >> 64, where a and b
 * are 64-bit unsigned numbers.
 */
unsigned long long sched_clock(void)
{
659 660
	if (__USE_RTC())
		return get_rtc();
661
	return mulhdu(get_tb() - boot_tb, tb_to_ns_scale) << tb_to_ns_shift;
L
Linus Torvalds 已提交
662 663
}

664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695

#ifdef CONFIG_PPC_PSERIES

/*
 * Running clock - attempts to give a view of time passing for a virtualised
 * kernels.
 * Uses the VTB register if available otherwise a next best guess.
 */
unsigned long long running_clock(void)
{
	/*
	 * Don't read the VTB as a host since KVM does not switch in host
	 * timebase into the VTB when it takes a guest off the CPU, reading the
	 * VTB would result in reading 'last switched out' guest VTB.
	 *
	 * Host kernels are often compiled with CONFIG_PPC_PSERIES checked, it
	 * would be unsafe to rely only on the #ifdef above.
	 */
	if (firmware_has_feature(FW_FEATURE_LPAR) &&
	    cpu_has_feature(CPU_FTR_ARCH_207S))
		return mulhdu(get_vtb() - boot_tb, tb_to_ns_scale) << tb_to_ns_shift;

	/*
	 * This is a next best approximation without a VTB.
	 * On a host which is running bare metal there should never be any stolen
	 * time and on a host which doesn't do any virtualisation TB *should* equal
	 * VTB so it makes no difference anyway.
	 */
	return local_clock() - cputime_to_nsecs(kcpustat_this_cpu->cpustat[CPUTIME_STEAL]);
}
#endif

696
static int __init get_freq(char *name, int cells, unsigned long *val)
697 698
{
	struct device_node *cpu;
699
	const __be32 *fp;
700
	int found = 0;
701

702
	/* The cpu node should have timebase and clock frequency properties */
703 704
	cpu = of_find_node_by_type(NULL, "cpu");

705
	if (cpu) {
706
		fp = of_get_property(cpu, name, NULL);
707
		if (fp) {
708
			found = 1;
709
			*val = of_read_ulong(fp, cells);
710
		}
711 712

		of_node_put(cpu);
713
	}
714 715 716 717

	return found;
}

718
static void start_cpu_decrementer(void)
719 720 721 722 723 724 725 726 727 728
{
#if defined(CONFIG_BOOKE) || defined(CONFIG_40x)
	/* Clear any pending timer interrupts */
	mtspr(SPRN_TSR, TSR_ENW | TSR_WIS | TSR_DIS | TSR_FIS);

	/* Enable decrementer interrupt */
	mtspr(SPRN_TCR, TCR_DIE);
#endif /* defined(CONFIG_BOOKE) || defined(CONFIG_40x) */
}

729 730 731 732 733 734 735
void __init generic_calibrate_decr(void)
{
	ppc_tb_freq = DEFAULT_TB_FREQ;		/* hardcoded default */

	if (!get_freq("ibm,extended-timebase-frequency", 2, &ppc_tb_freq) &&
	    !get_freq("timebase-frequency", 1, &ppc_tb_freq)) {

736 737
		printk(KERN_ERR "WARNING: Estimating decrementer frequency "
				"(not found)\n");
738
	}
739

740 741 742 743 744 745 746
	ppc_proc_freq = DEFAULT_PROC_FREQ;	/* hardcoded default */

	if (!get_freq("ibm,extended-clock-frequency", 2, &ppc_proc_freq) &&
	    !get_freq("clock-frequency", 1, &ppc_proc_freq)) {

		printk(KERN_ERR "WARNING: Estimating processor frequency "
				"(not found)\n");
747 748 749
	}
}

750
int update_persistent_clock(struct timespec now)
751 752 753
{
	struct rtc_time tm;

754
	if (!ppc_md.set_rtc_time)
755
		return -ENODEV;
756 757 758 759 760 761 762 763

	to_tm(now.tv_sec + 1 + timezone_offset, &tm);
	tm.tm_year -= 1900;
	tm.tm_mon -= 1;

	return ppc_md.set_rtc_time(&tm);
}

764
static void __read_persistent_clock(struct timespec *ts)
765 766 767 768
{
	struct rtc_time tm;
	static int first = 1;

769
	ts->tv_nsec = 0;
770 771 772 773 774 775 776
	/* XXX this is a litle fragile but will work okay in the short term */
	if (first) {
		first = 0;
		if (ppc_md.time_init)
			timezone_offset = ppc_md.time_init();

		/* get_boot_time() isn't guaranteed to be safe to call late */
777 778 779 780 781 782 783 784
		if (ppc_md.get_boot_time) {
			ts->tv_sec = ppc_md.get_boot_time() - timezone_offset;
			return;
		}
	}
	if (!ppc_md.get_rtc_time) {
		ts->tv_sec = 0;
		return;
785
	}
786
	ppc_md.get_rtc_time(&tm);
787

788 789
	ts->tv_sec = mktime(tm.tm_year+1900, tm.tm_mon+1, tm.tm_mday,
			    tm.tm_hour, tm.tm_min, tm.tm_sec);
790 791
}

792 793 794 795 796 797 798 799 800 801 802 803
void read_persistent_clock(struct timespec *ts)
{
	__read_persistent_clock(ts);

	/* Sanitize it in case real time clock is set below EPOCH */
	if (ts->tv_sec < 0) {
		ts->tv_sec = 0;
		ts->tv_nsec = 0;
	}
		
}

804
/* clocksource code */
805
static u64 rtc_read(struct clocksource *cs)
806
{
807
	return (u64)get_rtc();
808 809
}

810
static u64 timebase_read(struct clocksource *cs)
811
{
812
	return (u64)get_tb();
813 814
}

815
void update_vsyscall_old(struct timespec *wall_time, struct timespec *wtm,
816
			 struct clocksource *clock, u32 mult, u64 cycle_last)
817
{
J
John Stultz 已提交
818
	u64 new_tb_to_xs, new_stamp_xsec;
819
	u32 frac_sec;
820 821 822 823 824 825 826 827

	if (clock != &clocksource_timebase)
		return;

	/* Make userspace gettimeofday spin until we're done. */
	++vdso_data->tb_update_count;
	smp_mb();

828 829
	/* 19342813113834067 ~= 2^(20+64) / 1e9 */
	new_tb_to_xs = (u64) mult * (19342813113834067ULL >> clock->shift);
J
John Stultz 已提交
830
	new_stamp_xsec = (u64) wall_time->tv_nsec * XSEC_PER_SEC;
J
John Stultz 已提交
831
	do_div(new_stamp_xsec, 1000000000);
J
John Stultz 已提交
832
	new_stamp_xsec += (u64) wall_time->tv_sec * XSEC_PER_SEC;
J
John Stultz 已提交
833

834 835 836 837
	BUG_ON(wall_time->tv_nsec >= NSEC_PER_SEC);
	/* this is tv_nsec / 1e9 as a 0.32 fraction */
	frac_sec = ((u64) wall_time->tv_nsec * 18446744073ULL) >> 32;

J
John Stultz 已提交
838 839 840 841 842 843 844 845 846 847 848
	/*
	 * tb_update_count is used to allow the userspace gettimeofday code
	 * to assure itself that it sees a consistent view of the tb_to_xs and
	 * stamp_xsec variables.  It reads the tb_update_count, then reads
	 * tb_to_xs and stamp_xsec and then reads tb_update_count again.  If
	 * the two values of tb_update_count match and are even then the
	 * tb_to_xs and stamp_xsec values are consistent.  If not, then it
	 * loops back and reads them again until this criteria is met.
	 * We expect the caller to have done the first increment of
	 * vdso_data->tb_update_count already.
	 */
849
	vdso_data->tb_orig_stamp = cycle_last;
J
John Stultz 已提交
850 851
	vdso_data->stamp_xsec = new_stamp_xsec;
	vdso_data->tb_to_xs = new_tb_to_xs;
852 853
	vdso_data->wtom_clock_sec = wtm->tv_sec;
	vdso_data->wtom_clock_nsec = wtm->tv_nsec;
J
John Stultz 已提交
854
	vdso_data->stamp_xtime = *wall_time;
855
	vdso_data->stamp_sec_fraction = frac_sec;
J
John Stultz 已提交
856 857
	smp_wmb();
	++(vdso_data->tb_update_count);
858 859 860 861 862 863 864 865
}

void update_vsyscall_tz(void)
{
	vdso_data->tz_minuteswest = sys_tz.tz_minuteswest;
	vdso_data->tz_dsttime = sys_tz.tz_dsttime;
}

866
static void __init clocksource_init(void)
867 868 869 870 871 872 873 874
{
	struct clocksource *clock;

	if (__USE_RTC())
		clock = &clocksource_rtc;
	else
		clock = &clocksource_timebase;

875
	if (clocksource_register_hz(clock, tb_ticks_per_sec)) {
876 877 878 879 880 881 882 883 884
		printk(KERN_ERR "clocksource: %s is already registered\n",
		       clock->name);
		return;
	}

	printk(KERN_INFO "clocksource: %s mult[%x] shift[%d] registered\n",
	       clock->name, clock->mult, clock->shift);
}

885 886 887
static int decrementer_set_next_event(unsigned long evt,
				      struct clock_event_device *dev)
{
888
	__this_cpu_write(decrementers_next_tb, get_tb_or_rtc() + evt);
889
	set_dec(evt);
890 891 892 893 894

	/* We may have raced with new irq work */
	if (test_irq_work_pending())
		set_dec(1);

895 896 897
	return 0;
}

898
static int decrementer_shutdown(struct clock_event_device *dev)
899
{
900
	decrementer_set_next_event(decrementer_max, dev);
901
	return 0;
902 903
}

904 905 906
/* Interrupt handler for the timer broadcast IPI */
void tick_broadcast_ipi_handler(void)
{
907
	u64 *next_tb = this_cpu_ptr(&decrementers_next_tb);
908 909 910

	*next_tb = get_tb_or_rtc();
	__timer_interrupt();
911 912
}

913 914
static void register_decrementer_clockevent(int cpu)
{
915
	struct clock_event_device *dec = &per_cpu(decrementers, cpu);
916 917

	*dec = decrementer_clockevent;
918
	dec->cpumask = cpumask_of(cpu);
919

920 921
	printk_once(KERN_DEBUG "clockevent: %s mult[%x] shift[%d] cpu[%d]\n",
		    dec->name, dec->mult, dec->shift, cpu);
922 923 924 925

	clockevents_register_device(dec);
}

926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968
static void enable_large_decrementer(void)
{
	if (!cpu_has_feature(CPU_FTR_ARCH_300))
		return;

	if (decrementer_max <= DECREMENTER_DEFAULT_MAX)
		return;

	/*
	 * If we're running as the hypervisor we need to enable the LD manually
	 * otherwise firmware should have done it for us.
	 */
	if (cpu_has_feature(CPU_FTR_HVMODE))
		mtspr(SPRN_LPCR, mfspr(SPRN_LPCR) | LPCR_LD);
}

static void __init set_decrementer_max(void)
{
	struct device_node *cpu;
	u32 bits = 32;

	/* Prior to ISAv3 the decrementer is always 32 bit */
	if (!cpu_has_feature(CPU_FTR_ARCH_300))
		return;

	cpu = of_find_node_by_type(NULL, "cpu");

	if (of_property_read_u32(cpu, "ibm,dec-bits", &bits) == 0) {
		if (bits > 64 || bits < 32) {
			pr_warn("time_init: firmware supplied invalid ibm,dec-bits");
			bits = 32;
		}

		/* calculate the signed maximum given this many bits */
		decrementer_max = (1ul << (bits - 1)) - 1;
	}

	of_node_put(cpu);

	pr_info("time_init: %u bit decrementer (max: %llx)\n",
		bits, decrementer_max);
}

969
static void __init init_decrementer_clockevent(void)
970 971 972
{
	int cpu = smp_processor_id();

973 974
	clockevents_calc_mult_shift(&decrementer_clockevent, ppc_tb_freq, 4);

975
	decrementer_clockevent.max_delta_ns =
976
		clockevent_delta2ns(decrementer_max, &decrementer_clockevent);
977 978
	decrementer_clockevent.min_delta_ns =
		clockevent_delta2ns(2, &decrementer_clockevent);
979 980 981 982 983 984

	register_decrementer_clockevent(cpu);
}

void secondary_cpu_time_init(void)
{
985 986 987
	/* Enable and test the large decrementer for this cpu */
	enable_large_decrementer();

988 989 990 991 992
	/* Start the decrementer on CPUs that have manual control
	 * such as BookE
	 */
	start_cpu_decrementer();

993 994 995 996 997
	/* FIME: Should make unrelatred change to move snapshot_timebase
	 * call here ! */
	register_decrementer_clockevent(smp_processor_id());
}

998
/* This function is only called on the boot processor */
L
Linus Torvalds 已提交
999 1000 1001
void __init time_init(void)
{
	struct div_result res;
1002
	u64 scale;
1003 1004
	unsigned shift;

1005 1006 1007 1008 1009 1010
	if (__USE_RTC()) {
		/* 601 processor: dec counts down by 128 every 128ns */
		ppc_tb_freq = 1000000000;
	} else {
		/* Normal PowerPC with timebase register */
		ppc_md.calibrate_decr();
1011
		printk(KERN_DEBUG "time_init: decrementer frequency = %lu.%.6lu MHz\n",
1012
		       ppc_tb_freq / 1000000, ppc_tb_freq % 1000000);
1013
		printk(KERN_DEBUG "time_init: processor frequency   = %lu.%.6lu MHz\n",
1014 1015
		       ppc_proc_freq / 1000000, ppc_proc_freq % 1000000);
	}
1016 1017

	tb_ticks_per_jiffy = ppc_tb_freq / HZ;
1018
	tb_ticks_per_sec = ppc_tb_freq;
1019
	tb_ticks_per_usec = ppc_tb_freq / 1000000;
1020
	calc_cputime_factors();
1021
	setup_cputime_one_jiffy();
1022

L
Linus Torvalds 已提交
1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040
	/*
	 * Compute scale factor for sched_clock.
	 * The calibrate_decr() function has set tb_ticks_per_sec,
	 * which is the timebase frequency.
	 * We compute 1e9 * 2^64 / tb_ticks_per_sec and interpret
	 * the 128-bit result as a 64.64 fixed-point number.
	 * We then shift that number right until it is less than 1.0,
	 * giving us the scale factor and shift count to use in
	 * sched_clock().
	 */
	div128_by_32(1000000000, 0, tb_ticks_per_sec, &res);
	scale = res.result_low;
	for (shift = 0; res.result_high != 0; ++shift) {
		scale = (scale >> 1) | (res.result_high << 63);
		res.result_high >>= 1;
	}
	tb_to_ns_scale = scale;
	tb_to_ns_shift = shift;
1041
	/* Save the current timebase to pretty up CONFIG_PRINTK_TIME */
1042
	boot_tb = get_tb_or_rtc();
L
Linus Torvalds 已提交
1043

1044
	/* If platform provided a timezone (pmac), we correct the time */
1045
	if (timezone_offset) {
1046 1047
		sys_tz.tz_minuteswest = -timezone_offset / 60;
		sys_tz.tz_dsttime = 0;
1048
	}
1049

1050 1051
	vdso_data->tb_update_count = 0;
	vdso_data->tb_ticks_per_sec = tb_ticks_per_sec;
L
Linus Torvalds 已提交
1052

1053 1054 1055 1056
	/* initialise and enable the large decrementer (if we have one) */
	set_decrementer_max();
	enable_large_decrementer();

1057 1058 1059 1060 1061
	/* Start the decrementer on CPUs that have manual control
	 * such as BookE
	 */
	start_cpu_decrementer();

1062 1063
	/* Register the clocksource */
	clocksource_init();
1064

1065
	init_decrementer_clockevent();
1066
	tick_setup_hrtimer_broadcast();
1067 1068 1069 1070

#ifdef CONFIG_COMMON_CLK
	of_clk_init(NULL);
#endif
L
Linus Torvalds 已提交
1071 1072 1073 1074 1075 1076 1077
}


#define FEBRUARY	2
#define	STARTOFTIME	1970
#define SECDAY		86400L
#define SECYR		(SECDAY * 365)
1078 1079
#define	leapyear(year)		((year) % 4 == 0 && \
				 ((year) % 100 != 0 || (year) % 400 == 0))
L
Linus Torvalds 已提交
1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116
#define	days_in_year(a) 	(leapyear(a) ? 366 : 365)
#define	days_in_month(a) 	(month_days[(a) - 1])

static int month_days[12] = {
	31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31
};

void to_tm(int tim, struct rtc_time * tm)
{
	register int    i;
	register long   hms, day;

	day = tim / SECDAY;
	hms = tim % SECDAY;

	/* Hours, minutes, seconds are easy */
	tm->tm_hour = hms / 3600;
	tm->tm_min = (hms % 3600) / 60;
	tm->tm_sec = (hms % 3600) % 60;

	/* Number of years in days */
	for (i = STARTOFTIME; day >= days_in_year(i); i++)
		day -= days_in_year(i);
	tm->tm_year = i;

	/* Number of months in days left */
	if (leapyear(tm->tm_year))
		days_in_month(FEBRUARY) = 29;
	for (i = 1; day >= days_in_month(i); i++)
		day -= days_in_month(i);
	days_in_month(FEBRUARY) = 28;
	tm->tm_mon = i;

	/* Days are what is left over (+1) from all that. */
	tm->tm_mday = day + 1;

	/*
1117
	 * No-one uses the day of the week.
L
Linus Torvalds 已提交
1118
	 */
1119
	tm->tm_wday = -1;
L
Linus Torvalds 已提交
1120
}
1121
EXPORT_SYMBOL(to_tm);
L
Linus Torvalds 已提交
1122 1123 1124 1125 1126

/*
 * Divide a 128-bit dividend by a 32-bit divisor, leaving a 128 bit
 * result.
 */
1127 1128
void div128_by_32(u64 dividend_high, u64 dividend_low,
		  unsigned divisor, struct div_result *dr)
L
Linus Torvalds 已提交
1129
{
1130 1131 1132
	unsigned long a, b, c, d;
	unsigned long w, x, y, z;
	u64 ra, rb, rc;
L
Linus Torvalds 已提交
1133 1134 1135 1136 1137 1138

	a = dividend_high >> 32;
	b = dividend_high & 0xffffffff;
	c = dividend_low >> 32;
	d = dividend_low & 0xffffffff;

1139 1140 1141 1142 1143
	w = a / divisor;
	ra = ((u64)(a - (w * divisor)) << 32) + b;

	rb = ((u64) do_div(ra, divisor) << 32) + c;
	x = ra;
L
Linus Torvalds 已提交
1144

1145 1146 1147 1148 1149
	rc = ((u64) do_div(rb, divisor) << 32) + d;
	y = rb;

	do_div(rc, divisor);
	z = rc;
L
Linus Torvalds 已提交
1150

1151 1152
	dr->result_high = ((u64)w << 32) + x;
	dr->result_low  = ((u64)y << 32) + z;
L
Linus Torvalds 已提交
1153 1154

}
1155

1156 1157 1158 1159 1160 1161 1162 1163 1164
/* We don't need to calibrate delay, we use the CPU timebase for that */
void calibrate_delay(void)
{
	/* Some generic code (such as spinlock debug) use loops_per_jiffy
	 * as the number of __delay(1) in a jiffy, so make it so
	 */
	loops_per_jiffy = tb_ticks_per_jiffy;
}

1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187
#if IS_ENABLED(CONFIG_RTC_DRV_GENERIC)
static int rtc_generic_get_time(struct device *dev, struct rtc_time *tm)
{
	ppc_md.get_rtc_time(tm);
	return rtc_valid_tm(tm);
}

static int rtc_generic_set_time(struct device *dev, struct rtc_time *tm)
{
	if (!ppc_md.set_rtc_time)
		return -EOPNOTSUPP;

	if (ppc_md.set_rtc_time(tm) < 0)
		return -EOPNOTSUPP;

	return 0;
}

static const struct rtc_class_ops rtc_generic_ops = {
	.read_time = rtc_generic_get_time,
	.set_time = rtc_generic_set_time,
};

1188 1189 1190 1191 1192 1193 1194
static int __init rtc_init(void)
{
	struct platform_device *pdev;

	if (!ppc_md.get_rtc_time)
		return -ENODEV;

1195 1196 1197
	pdev = platform_device_register_data(NULL, "rtc-generic", -1,
					     &rtc_generic_ops,
					     sizeof(rtc_generic_ops));
1198

1199
	return PTR_ERR_OR_ZERO(pdev);
1200 1201
}

1202
device_initcall(rtc_init);
1203
#endif