tick-sched.c 19.5 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11
/*
 *  linux/kernel/time/tick-sched.c
 *
 *  Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
 *  Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
 *  Copyright(C) 2006-2007  Timesys Corp., Thomas Gleixner
 *
 *  No idle tick implementation for low and high resolution timers
 *
 *  Started by: Thomas Gleixner and Ingo Molnar
 *
P
Pavel Machek 已提交
12
 *  Distribute under GPLv2.
13 14 15 16 17 18 19 20 21 22
 */
#include <linux/cpu.h>
#include <linux/err.h>
#include <linux/hrtimer.h>
#include <linux/interrupt.h>
#include <linux/kernel_stat.h>
#include <linux/percpu.h>
#include <linux/profile.h>
#include <linux/sched.h>
#include <linux/tick.h>
23
#include <linux/module.h>
24

25 26
#include <asm/irq_regs.h>

27 28 29 30 31 32 33 34 35 36 37 38
#include "tick-internal.h"

/*
 * Per cpu nohz control structure
 */
static DEFINE_PER_CPU(struct tick_sched, tick_cpu_sched);

/*
 * The time, when the last jiffy update happened. Protected by xtime_lock.
 */
static ktime_t last_jiffies_update;

39 40 41 42 43
struct tick_sched *tick_get_tick_sched(int cpu)
{
	return &per_cpu(tick_cpu_sched, cpu);
}

44 45 46 47 48 49 50 51
/*
 * Must be called with interrupts disabled !
 */
static void tick_do_update_jiffies64(ktime_t now)
{
	unsigned long ticks = 0;
	ktime_t delta;

52 53 54 55 56 57 58
	/*
	 * Do a quick check without holding xtime_lock:
	 */
	delta = ktime_sub(now, last_jiffies_update);
	if (delta.tv64 < tick_period.tv64)
		return;

59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78
	/* Reevalute with xtime_lock held */
	write_seqlock(&xtime_lock);

	delta = ktime_sub(now, last_jiffies_update);
	if (delta.tv64 >= tick_period.tv64) {

		delta = ktime_sub(delta, tick_period);
		last_jiffies_update = ktime_add(last_jiffies_update,
						tick_period);

		/* Slow path for long timeouts */
		if (unlikely(delta.tv64 >= tick_period.tv64)) {
			s64 incr = ktime_to_ns(tick_period);

			ticks = ktime_divns(delta, incr);

			last_jiffies_update = ktime_add_ns(last_jiffies_update,
							   incr * ticks);
		}
		do_timer(++ticks);
79 80 81

		/* Keep the tick_next_period variable up to date */
		tick_next_period = ktime_add(last_jiffies_update, tick_period);
82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136
	}
	write_sequnlock(&xtime_lock);
}

/*
 * Initialize and return retrieve the jiffies update.
 */
static ktime_t tick_init_jiffy_update(void)
{
	ktime_t period;

	write_seqlock(&xtime_lock);
	/* Did we start the jiffies update yet ? */
	if (last_jiffies_update.tv64 == 0)
		last_jiffies_update = tick_next_period;
	period = last_jiffies_update;
	write_sequnlock(&xtime_lock);
	return period;
}

/*
 * NOHZ - aka dynamic tick functionality
 */
#ifdef CONFIG_NO_HZ
/*
 * NO HZ enabled ?
 */
static int tick_nohz_enabled __read_mostly  = 1;

/*
 * Enable / Disable tickless mode
 */
static int __init setup_tick_nohz(char *str)
{
	if (!strcmp(str, "off"))
		tick_nohz_enabled = 0;
	else if (!strcmp(str, "on"))
		tick_nohz_enabled = 1;
	else
		return 0;
	return 1;
}

__setup("nohz=", setup_tick_nohz);

/**
 * tick_nohz_update_jiffies - update jiffies when idle was interrupted
 *
 * Called from interrupt entry when the CPU was idle
 *
 * In case the sched_tick was stopped on this CPU, we have to check if jiffies
 * must be updated. Otherwise an interrupt handler could use a stale jiffy
 * value. We do this unconditionally on any cpu, as we don't know whether the
 * cpu, which has the update task assigned is in a long sleep.
 */
137
static void tick_nohz_update_jiffies(ktime_t now)
138 139 140 141 142
{
	int cpu = smp_processor_id();
	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
	unsigned long flags;

143
	cpumask_clear_cpu(cpu, nohz_cpu_mask);
144
	ts->idle_waketime = now;
145 146 147 148

	local_irq_save(flags);
	tick_do_update_jiffies64(now);
	local_irq_restore(flags);
149 150

	touch_softlockup_watchdog();
151 152
}

153
static void tick_nohz_stop_idle(int cpu, ktime_t now)
154 155
{
	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
156
	ktime_t delta;
157

158 159 160 161
	delta = ktime_sub(now, ts->idle_entrytime);
	ts->idle_lastupdate = now;
	ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta);
	ts->idle_active = 0;
162

163
	sched_clock_idle_wakeup_event(0);
164 165
}

166
static ktime_t tick_nohz_start_idle(struct tick_sched *ts)
167 168 169 170 171 172 173 174 175 176 177
{
	ktime_t now, delta;

	now = ktime_get();
	if (ts->idle_active) {
		delta = ktime_sub(now, ts->idle_entrytime);
		ts->idle_lastupdate = now;
		ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta);
	}
	ts->idle_entrytime = now;
	ts->idle_active = 1;
178
	sched_clock_idle_sleep_event();
179 180 181 182 183 184 185
	return now;
}

u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time)
{
	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);

186 187 188 189 190 191 192 193
	if (!tick_nohz_enabled)
		return -1;

	if (ts->idle_active)
		*last_update_time = ktime_to_us(ts->idle_lastupdate);
	else
		*last_update_time = ktime_to_us(ktime_get());

194 195
	return ktime_to_us(ts->idle_sleeptime);
}
196
EXPORT_SYMBOL_GPL(get_cpu_idle_time_us);
197

198 199 200 201 202 203 204
/**
 * tick_nohz_stop_sched_tick - stop the idle tick from the idle task
 *
 * When the next event is more than a tick into the future, stop the idle tick
 * Called either from the idle loop or from irq_exit() when an idle period was
 * just interrupted by an interrupt which did not cause a reschedule.
 */
205
void tick_nohz_stop_sched_tick(int inidle)
206 207 208
{
	unsigned long seq, last_jiffies, next_jiffies, delta_jiffies, flags;
	struct tick_sched *ts;
209
	ktime_t last_update, expires, now;
210
	struct clock_event_device *dev = __get_cpu_var(tick_cpu_device).evtdev;
211 212 213 214 215 216
	int cpu;

	local_irq_save(flags);

	cpu = smp_processor_id();
	ts = &per_cpu(tick_cpu_sched, cpu);
217 218 219 220 221 222 223 224 225

	/*
	 * Call to tick_nohz_start_idle stops the last_update_time from being
	 * updated. Thus, it must not be called in the event we are called from
	 * irq_exit() with the prior state different than idle.
	 */
	if (!inidle && !ts->inidle)
		goto end;

226
	now = tick_nohz_start_idle(ts);
227

228 229 230 231 232 233 234 235 236
	/*
	 * If this cpu is offline and it is the one which updates
	 * jiffies, then give up the assignment and let it be taken by
	 * the cpu which runs the tick timer next. If we don't drop
	 * this here the jiffies might be stale and do_timer() never
	 * invoked.
	 */
	if (unlikely(!cpu_online(cpu))) {
		if (cpu == tick_do_timer_cpu)
237
			tick_do_timer_cpu = TICK_DO_TIMER_NONE;
238 239
	}

240 241 242
	if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE))
		goto end;

243 244
	ts->inidle = 1;

245 246 247
	if (need_resched())
		goto end;

248
	if (unlikely(local_softirq_pending() && cpu_online(cpu))) {
249 250 251 252 253 254 255
		static int ratelimit;

		if (ratelimit < 10) {
			printk(KERN_ERR "NOHZ: local_softirq_pending %02x\n",
			       local_softirq_pending());
			ratelimit++;
		}
256
		goto end;
257
	}
258 259 260 261 262 263 264 265 266 267 268 269 270

	ts->idle_calls++;
	/* Read jiffies and the time when jiffies were updated last */
	do {
		seq = read_seqbegin(&xtime_lock);
		last_update = last_jiffies_update;
		last_jiffies = jiffies;
	} while (read_seqretry(&xtime_lock, seq));

	/* Get the next timer wheel timer */
	next_jiffies = get_next_timer_interrupt(last_jiffies);
	delta_jiffies = next_jiffies - last_jiffies;

P
Peter Zijlstra 已提交
271
	if (rcu_needs_cpu(cpu) || printk_needs_cpu(cpu))
I
Ingo Molnar 已提交
272
		delta_jiffies = 1;
273 274 275 276
	/*
	 * Do not stop the tick, if we are only one off
	 * or if the cpu is required for rcu
	 */
I
Ingo Molnar 已提交
277
	if (!ts->tick_stopped && delta_jiffies == 1)
278 279 280 281 282
		goto out;

	/* Schedule the tick, if we are at least one jiffie off */
	if ((long)delta_jiffies >= 1) {

283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300
		/*
		* calculate the expiry time for the next timer wheel
		* timer
		*/
		expires = ktime_add_ns(last_update, tick_period.tv64 *
				   delta_jiffies);

		/*
		 * If this cpu is the one which updates jiffies, then
		 * give up the assignment and let it be taken by the
		 * cpu which runs the tick timer next, which might be
		 * this cpu as well. If we don't drop this here the
		 * jiffies might be stale and do_timer() never
		 * invoked.
		 */
		if (cpu == tick_do_timer_cpu)
			tick_do_timer_cpu = TICK_DO_TIMER_NONE;

I
Ingo Molnar 已提交
301
		if (delta_jiffies > 1)
302
			cpumask_set_cpu(cpu, nohz_cpu_mask);
303 304 305 306 307

		/* Skip reprogram of event if its not changed */
		if (ts->tick_stopped && ktime_equal(expires, dev->next_event))
			goto out;

308 309 310 311 312 313 314 315
		/*
		 * nohz_stop_sched_tick can be called several times before
		 * the nohz_restart_sched_tick is called. This happens when
		 * interrupts arrive which do not cause a reschedule. In the
		 * first call we save the current tick time, so we can restart
		 * the scheduler tick in nohz_restart_sched_tick.
		 */
		if (!ts->tick_stopped) {
316 317 318 319
			if (select_nohz_load_balancer(1)) {
				/*
				 * sched tick not stopped!
				 */
320
				cpumask_clear_cpu(cpu, nohz_cpu_mask);
321 322 323
				goto out;
			}

324
			ts->idle_tick = hrtimer_get_expires(&ts->sched_timer);
325 326
			ts->tick_stopped = 1;
			ts->idle_jiffies = last_jiffies;
327
			rcu_enter_nohz();
328
		}
329

330 331 332 333 334 335 336 337 338 339 340 341 342 343 344
		ts->idle_sleeps++;

		/*
		 * delta_jiffies >= NEXT_TIMER_MAX_DELTA signals that
		 * there is no timer pending or at least extremly far
		 * into the future (12 days for HZ=1000). In this case
		 * we simply stop the tick timer:
		 */
		if (unlikely(delta_jiffies >= NEXT_TIMER_MAX_DELTA)) {
			ts->idle_expires.tv64 = KTIME_MAX;
			if (ts->nohz_mode == NOHZ_MODE_HIGHRES)
				hrtimer_cancel(&ts->sched_timer);
			goto out;
		}

345
		/* Mark expiries */
346 347 348 349
		ts->idle_expires = expires;

		if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
			hrtimer_start(&ts->sched_timer, expires,
350
				      HRTIMER_MODE_ABS_PINNED);
351 352 353
			/* Check, if the timer was already in the past */
			if (hrtimer_active(&ts->sched_timer))
				goto out;
P
Pavel Machek 已提交
354
		} else if (!tick_program_event(expires, 0))
355 356 357 358 359 360 361
				goto out;
		/*
		 * We are past the event already. So we crossed a
		 * jiffie boundary. Update jiffies and raise the
		 * softirq.
		 */
		tick_do_update_jiffies64(ktime_get());
362
		cpumask_clear_cpu(cpu, nohz_cpu_mask);
363 364 365 366 367
	}
	raise_softirq_irqoff(TIMER_SOFTIRQ);
out:
	ts->next_jiffies = next_jiffies;
	ts->last_jiffies = last_jiffies;
368
	ts->sleep_length = ktime_sub(dev->next_event, now);
369 370 371 372
end:
	local_irq_restore(flags);
}

373 374 375 376 377 378 379 380 381 382 383 384
/**
 * tick_nohz_get_sleep_length - return the length of the current sleep
 *
 * Called from power state control code with interrupts disabled
 */
ktime_t tick_nohz_get_sleep_length(void)
{
	struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);

	return ts->sleep_length;
}

385 386 387
static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)
{
	hrtimer_cancel(&ts->sched_timer);
388
	hrtimer_set_expires(&ts->sched_timer, ts->idle_tick);
389 390 391 392 393 394

	while (1) {
		/* Forward the time to expire in the future */
		hrtimer_forward(&ts->sched_timer, now, tick_period);

		if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
395
			hrtimer_start_expires(&ts->sched_timer,
396
					      HRTIMER_MODE_ABS_PINNED);
397 398 399 400
			/* Check, if the timer was already in the past */
			if (hrtimer_active(&ts->sched_timer))
				break;
		} else {
401 402
			if (!tick_program_event(
				hrtimer_get_expires(&ts->sched_timer), 0))
403 404 405 406 407 408 409 410
				break;
		}
		/* Update jiffies and reread time */
		tick_do_update_jiffies64(now);
		now = ktime_get();
	}
}

411
/**
412
 * tick_nohz_restart_sched_tick - restart the idle tick from the idle task
413 414 415 416 417 418 419
 *
 * Restart the idle tick when the CPU is woken up from idle
 */
void tick_nohz_restart_sched_tick(void)
{
	int cpu = smp_processor_id();
	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
420
#ifndef CONFIG_VIRT_CPU_ACCOUNTING
421
	unsigned long ticks;
422
#endif
423
	ktime_t now;
424

425
	local_irq_disable();
426 427 428 429 430
	if (ts->idle_active || (ts->inidle && ts->tick_stopped))
		now = ktime_get();

	if (ts->idle_active)
		tick_nohz_stop_idle(cpu, now);
431

432 433
	if (!ts->inidle || !ts->tick_stopped) {
		ts->inidle = 0;
434
		local_irq_enable();
435
		return;
436
	}
437

438 439
	ts->inidle = 0;

440 441
	rcu_exit_nohz();

442
	/* Update jiffies first */
443
	select_nohz_load_balancer(0);
444
	tick_do_update_jiffies64(now);
445
	cpumask_clear_cpu(cpu, nohz_cpu_mask);
446

447
#ifndef CONFIG_VIRT_CPU_ACCOUNTING
448 449 450 451 452 453 454 455 456
	/*
	 * We stopped the tick in idle. Update process times would miss the
	 * time we slept as update_process_times does only a 1 tick
	 * accounting. Enforce that this is accounted to idle !
	 */
	ticks = jiffies - ts->idle_jiffies;
	/*
	 * We might be one off. Do not randomly account a huge number of ticks!
	 */
457 458 459
	if (ticks && ticks < LONG_MAX)
		account_idle_ticks(ticks);
#endif
460

I
Ingo Molnar 已提交
461
	touch_softlockup_watchdog();
462 463 464 465
	/*
	 * Cancel the scheduled timer and restore the tick
	 */
	ts->tick_stopped  = 0;
466
	ts->idle_exittime = now;
467

468
	tick_nohz_restart(ts, now);
469 470 471 472 473 474 475

	local_irq_enable();
}

static int tick_nohz_reprogram(struct tick_sched *ts, ktime_t now)
{
	hrtimer_forward(&ts->sched_timer, now, tick_period);
476
	return tick_program_event(hrtimer_get_expires(&ts->sched_timer), 0);
477 478 479 480 481 482 483 484 485
}

/*
 * The nohz low res interrupt handler
 */
static void tick_nohz_handler(struct clock_event_device *dev)
{
	struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
	struct pt_regs *regs = get_irq_regs();
486
	int cpu = smp_processor_id();
487 488 489 490
	ktime_t now = ktime_get();

	dev->next_event.tv64 = KTIME_MAX;

491 492 493 494 495 496 497
	/*
	 * Check if the do_timer duty was dropped. We don't care about
	 * concurrency: This happens only when the cpu in charge went
	 * into a long sleep. If two cpus happen to assign themself to
	 * this duty, then the jiffies update is still serialized by
	 * xtime_lock.
	 */
498
	if (unlikely(tick_do_timer_cpu == TICK_DO_TIMER_NONE))
499 500
		tick_do_timer_cpu = cpu;

501
	/* Check, if the jiffies need an update */
502 503
	if (tick_do_timer_cpu == cpu)
		tick_do_update_jiffies64(now);
504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554

	/*
	 * When we are idle and the tick is stopped, we have to touch
	 * the watchdog as we might not schedule for a really long
	 * time. This happens on complete idle SMP systems while
	 * waiting on the login prompt. We also increment the "start
	 * of idle" jiffy stamp so the idle accounting adjustment we
	 * do when we go busy again does not account too much ticks.
	 */
	if (ts->tick_stopped) {
		touch_softlockup_watchdog();
		ts->idle_jiffies++;
	}

	update_process_times(user_mode(regs));
	profile_tick(CPU_PROFILING);

	while (tick_nohz_reprogram(ts, now)) {
		now = ktime_get();
		tick_do_update_jiffies64(now);
	}
}

/**
 * tick_nohz_switch_to_nohz - switch to nohz mode
 */
static void tick_nohz_switch_to_nohz(void)
{
	struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
	ktime_t next;

	if (!tick_nohz_enabled)
		return;

	local_irq_disable();
	if (tick_switch_to_oneshot(tick_nohz_handler)) {
		local_irq_enable();
		return;
	}

	ts->nohz_mode = NOHZ_MODE_LOWRES;

	/*
	 * Recycle the hrtimer in ts, so we can share the
	 * hrtimer_forward with the highres code.
	 */
	hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
	/* Get the next period */
	next = tick_init_jiffy_update();

	for (;;) {
555
		hrtimer_set_expires(&ts->sched_timer, next);
556 557 558 559 560 561 562 563 564 565
		if (!tick_program_event(next, 0))
			break;
		next = ktime_add(next, tick_period);
	}
	local_irq_enable();

	printk(KERN_INFO "Switched to NOHz mode on CPU #%d\n",
	       smp_processor_id());
}

566 567 568 569 570 571 572 573 574 575 576
/*
 * When NOHZ is enabled and the tick is stopped, we need to kick the
 * tick timer from irq_enter() so that the jiffies update is kept
 * alive during long running softirqs. That's ugly as hell, but
 * correctness is key even if we need to fix the offending softirq in
 * the first place.
 *
 * Note, this is different to tick_nohz_restart. We just kick the
 * timer and do not touch the other magic bits which need to be done
 * when idle is left.
 */
577
static void tick_nohz_kick_tick(int cpu, ktime_t now)
578
{
579 580 581
#if 0
	/* Switch back to 2.6.27 behaviour */

582
	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
583
	ktime_t delta;
584

585 586 587 588
	/*
	 * Do not touch the tick device, when the next expiry is either
	 * already reached or less/equal than the tick period.
	 */
589
	delta =	ktime_sub(hrtimer_get_expires(&ts->sched_timer), now);
590 591 592 593
	if (delta.tv64 <= tick_period.tv64)
		return;

	tick_nohz_restart(ts, now);
594
#endif
595 596
}

597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612
static inline void tick_check_nohz(int cpu)
{
	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
	ktime_t now;

	if (!ts->idle_active && !ts->tick_stopped)
		return;
	now = ktime_get();
	if (ts->idle_active)
		tick_nohz_stop_idle(cpu, now);
	if (ts->tick_stopped) {
		tick_nohz_update_jiffies(now);
		tick_nohz_kick_tick(cpu, now);
	}
}

613 614 615
#else

static inline void tick_nohz_switch_to_nohz(void) { }
616
static inline void tick_check_nohz(int cpu) { }
617 618 619

#endif /* NO_HZ */

620 621 622 623 624
/*
 * Called from irq_enter to notify about the possible interruption of idle()
 */
void tick_check_idle(int cpu)
{
625
	tick_check_oneshot_broadcast(cpu);
626
	tick_check_nohz(cpu);
627 628
}

629 630 631 632 633
/*
 * High resolution timer specific code
 */
#ifdef CONFIG_HIGH_RES_TIMERS
/*
P
Pavel Machek 已提交
634
 * We rearm the timer until we get disabled by the idle code.
635 636 637 638 639 640 641 642
 * Called with interrupts disabled and timer->base->cpu_base->lock held.
 */
static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer)
{
	struct tick_sched *ts =
		container_of(timer, struct tick_sched, sched_timer);
	struct pt_regs *regs = get_irq_regs();
	ktime_t now = ktime_get();
643 644 645 646 647 648 649 650 651 652
	int cpu = smp_processor_id();

#ifdef CONFIG_NO_HZ
	/*
	 * Check if the do_timer duty was dropped. We don't care about
	 * concurrency: This happens only when the cpu in charge went
	 * into a long sleep. If two cpus happen to assign themself to
	 * this duty, then the jiffies update is still serialized by
	 * xtime_lock.
	 */
653
	if (unlikely(tick_do_timer_cpu == TICK_DO_TIMER_NONE))
654 655
		tick_do_timer_cpu = cpu;
#endif
656 657

	/* Check, if the jiffies need an update */
658 659
	if (tick_do_timer_cpu == cpu)
		tick_do_update_jiffies64(now);
660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693

	/*
	 * Do not call, when we are not in irq context and have
	 * no valid regs pointer
	 */
	if (regs) {
		/*
		 * When we are idle and the tick is stopped, we have to touch
		 * the watchdog as we might not schedule for a really long
		 * time. This happens on complete idle SMP systems while
		 * waiting on the login prompt. We also increment the "start of
		 * idle" jiffy stamp so the idle accounting adjustment we do
		 * when we go busy again does not account too much ticks.
		 */
		if (ts->tick_stopped) {
			touch_softlockup_watchdog();
			ts->idle_jiffies++;
		}
		update_process_times(user_mode(regs));
		profile_tick(CPU_PROFILING);
	}

	hrtimer_forward(timer, now, tick_period);

	return HRTIMER_RESTART;
}

/**
 * tick_setup_sched_timer - setup the tick emulation timer
 */
void tick_setup_sched_timer(void)
{
	struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
	ktime_t now = ktime_get();
694
	u64 offset;
695 696 697 698 699 700 701

	/*
	 * Emulate tick processing via per-CPU hrtimers:
	 */
	hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
	ts->sched_timer.function = tick_sched_timer;

702
	/* Get the next period (per cpu) */
703
	hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update());
704
	offset = ktime_to_ns(tick_period) >> 1;
705
	do_div(offset, num_possible_cpus());
706
	offset *= smp_processor_id();
707
	hrtimer_add_expires_ns(&ts->sched_timer, offset);
708 709 710

	for (;;) {
		hrtimer_forward(&ts->sched_timer, now, tick_period);
711 712
		hrtimer_start_expires(&ts->sched_timer,
				      HRTIMER_MODE_ABS_PINNED);
713 714 715 716 717 718 719 720 721 722 723
		/* Check, if the timer was already in the past */
		if (hrtimer_active(&ts->sched_timer))
			break;
		now = ktime_get();
	}

#ifdef CONFIG_NO_HZ
	if (tick_nohz_enabled)
		ts->nohz_mode = NOHZ_MODE_HIGHRES;
#endif
}
724
#endif /* HIGH_RES_TIMERS */
725

726
#if defined CONFIG_NO_HZ || defined CONFIG_HIGH_RES_TIMERS
727 728 729 730
void tick_cancel_sched_timer(int cpu)
{
	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);

731
# ifdef CONFIG_HIGH_RES_TIMERS
732 733
	if (ts->sched_timer.base)
		hrtimer_cancel(&ts->sched_timer);
734
# endif
735

736 737
	ts->nohz_mode = NOHZ_MODE_INACTIVE;
}
738
#endif
739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778

/**
 * Async notification about clocksource changes
 */
void tick_clock_notify(void)
{
	int cpu;

	for_each_possible_cpu(cpu)
		set_bit(0, &per_cpu(tick_cpu_sched, cpu).check_clocks);
}

/*
 * Async notification about clock event changes
 */
void tick_oneshot_notify(void)
{
	struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);

	set_bit(0, &ts->check_clocks);
}

/**
 * Check, if a change happened, which makes oneshot possible.
 *
 * Called cyclic from the hrtimer softirq (driven by the timer
 * softirq) allow_nohz signals, that we can switch into low-res nohz
 * mode, because high resolution timers are disabled (either compile
 * or runtime).
 */
int tick_check_oneshot_change(int allow_nohz)
{
	struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);

	if (!test_and_clear_bit(0, &ts->check_clocks))
		return 0;

	if (ts->nohz_mode != NOHZ_MODE_INACTIVE)
		return 0;

779
	if (!timekeeping_valid_for_hres() || !tick_is_oneshot_available())
780 781 782 783 784 785 786 787
		return 0;

	if (!allow_nohz)
		return 1;

	tick_nohz_switch_to_nohz();
	return 0;
}