tick-common.c 13.5 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16
/*
 * linux/kernel/time/tick-common.c
 *
 * This file contains the base functions to manage periodic tick
 * related events.
 *
 * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
 * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
 * Copyright(C) 2006-2007, Timesys Corp., Thomas Gleixner
 *
 * This code is licenced under the GPL version 2. For details see
 * kernel-base/COPYING.
 */
#include <linux/cpu.h>
#include <linux/err.h>
#include <linux/hrtimer.h>
17
#include <linux/interrupt.h>
18 19 20
#include <linux/percpu.h>
#include <linux/profile.h>
#include <linux/sched.h>
21
#include <linux/module.h>
22
#include <trace/events/power.h>
23

24 25
#include <asm/irq_regs.h>

26 27
#include "tick-internal.h"

28 29 30
/*
 * Tick devices
 */
31
DEFINE_PER_CPU(struct tick_device, tick_cpu_device);
32 33 34
/*
 * Tick next event: keeps track of the tick time
 */
35 36
ktime_t tick_next_period;
ktime_t tick_period;
A
Andrew Morton 已提交
37 38 39 40 41 42 43 44 45 46 47 48 49 50 51

/*
 * tick_do_timer_cpu is a timer core internal variable which holds the CPU NR
 * which is responsible for calling do_timer(), i.e. the timekeeping stuff. This
 * variable has two functions:
 *
 * 1) Prevent a thundering herd issue of a gazillion of CPUs trying to grab the
 *    timekeeping lock all at once. Only the CPU which is assigned to do the
 *    update is handling it.
 *
 * 2) Hand off the duty in the NOHZ idle case by setting the value to
 *    TICK_DO_TIMER_NONE, i.e. a non existing CPU. So the next cpu which looks
 *    at it will take over and keep the time keeping alive.  The handover
 *    procedure also covers cpu hotplug.
 */
52
int tick_do_timer_cpu __read_mostly = TICK_DO_TIMER_BOOT;
53

54 55 56 57 58 59 60 61
/*
 * Debugging: see timer_list.c
 */
struct tick_device *tick_get_device(int cpu)
{
	return &per_cpu(tick_cpu_device, cpu);
}

62 63 64 65 66
/**
 * tick_is_oneshot_available - check for a oneshot capable event device
 */
int tick_is_oneshot_available(void)
{
67
	struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
68

69 70 71 72 73
	if (!dev || !(dev->features & CLOCK_EVT_FEAT_ONESHOT))
		return 0;
	if (!(dev->features & CLOCK_EVT_FEAT_C3STOP))
		return 1;
	return tick_broadcast_oneshot_available();
74 75
}

76 77 78 79 80 81
/*
 * Periodic tick
 */
static void tick_periodic(int cpu)
{
	if (tick_do_timer_cpu == cpu) {
82
		write_seqlock(&jiffies_lock);
83 84 85 86 87

		/* Keep track of the next tick event */
		tick_next_period = ktime_add(tick_next_period, tick_period);

		do_timer(1);
88
		write_sequnlock(&jiffies_lock);
89
		update_wall_time();
90 91 92 93 94 95 96 97 98 99 100 101
	}

	update_process_times(user_mode(get_irq_regs()));
	profile_tick(CPU_PROFILING);
}

/*
 * Event handler for periodic ticks
 */
void tick_handle_periodic(struct clock_event_device *dev)
{
	int cpu = smp_processor_id();
102
	ktime_t next = dev->next_event;
103 104 105

	tick_periodic(cpu);

106 107 108 109 110 111 112 113 114 115
#if defined(CONFIG_HIGH_RES_TIMERS) || defined(CONFIG_NO_HZ_COMMON)
	/*
	 * The cpu might have transitioned to HIGHRES or NOHZ mode via
	 * update_process_times() -> run_local_timers() ->
	 * hrtimer_run_queues().
	 */
	if (dev->event_handler != tick_handle_periodic)
		return;
#endif

116
	if (!clockevent_state_oneshot(dev))
117 118
		return;
	for (;;) {
119 120 121 122 123 124
		/*
		 * Setup the next period for devices, which do not have
		 * periodic mode:
		 */
		next = ktime_add(next, tick_period);

125
		if (!clockevents_program_event(dev, next, false))
126
			return;
127 128 129 130 131 132
		/*
		 * Have to be careful here. If we're in oneshot mode,
		 * before we call tick_periodic() in a loop, we need
		 * to be sure we're using a real hardware clocksource.
		 * Otherwise we could get trapped in an infinite
		 * loop, as the tick_periodic() increments jiffies,
133
		 * which then will increment time, possibly causing
134 135 136 137
		 * the loop to trigger again and again.
		 */
		if (timekeeping_valid_for_hres())
			tick_periodic(cpu);
138 139 140 141 142 143
	}
}

/*
 * Setup the device for a periodic tick
 */
144
void tick_setup_periodic(struct clock_event_device *dev, int broadcast)
145
{
146 147 148 149 150
	tick_set_periodic_handler(dev, broadcast);

	/* Broadcast setup ? */
	if (!tick_device_is_functional(dev))
		return;
151

152 153
	if ((dev->features & CLOCK_EVT_FEAT_PERIODIC) &&
	    !tick_broadcast_oneshot_active()) {
154
		clockevents_switch_state(dev, CLOCK_EVT_STATE_PERIODIC);
155 156 157 158 159
	} else {
		unsigned long seq;
		ktime_t next;

		do {
160
			seq = read_seqbegin(&jiffies_lock);
161
			next = tick_next_period;
162
		} while (read_seqretry(&jiffies_lock, seq));
163

164
		clockevents_switch_state(dev, CLOCK_EVT_STATE_ONESHOT);
165 166

		for (;;) {
167
			if (!clockevents_program_event(dev, next, false))
168 169 170 171 172 173 174 175 176 177 178
				return;
			next = ktime_add(next, tick_period);
		}
	}
}

/*
 * Setup the tick device
 */
static void tick_setup_device(struct tick_device *td,
			      struct clock_event_device *newdev, int cpu,
179
			      const struct cpumask *cpumask)
180 181
{
	void (*handler)(struct clock_event_device *) = NULL;
T
Thomas Gleixner 已提交
182
	ktime_t next_event = 0;
183 184 185 186 187 188 189 190 191

	/*
	 * First device setup ?
	 */
	if (!td->evtdev) {
		/*
		 * If no cpu took the do_timer update, assign it to
		 * this cpu:
		 */
192
		if (tick_do_timer_cpu == TICK_DO_TIMER_BOOT) {
193
			if (!tick_nohz_full_cpu(cpu))
194 195 196
				tick_do_timer_cpu = cpu;
			else
				tick_do_timer_cpu = TICK_DO_TIMER_NONE;
197
			tick_next_period = ktime_get();
T
Thomas Gleixner 已提交
198
			tick_period = NSEC_PER_SEC / HZ;
199 200 201 202 203 204 205 206 207
		}

		/*
		 * Startup in periodic mode first.
		 */
		td->mode = TICKDEV_MODE_PERIODIC;
	} else {
		handler = td->evtdev->event_handler;
		next_event = td->evtdev->next_event;
208
		td->evtdev->event_handler = clockevents_handle_noop;
209 210 211 212 213 214 215 216
	}

	td->evtdev = newdev;

	/*
	 * When the device is not per cpu, pin the interrupt to the
	 * current cpu:
	 */
217
	if (!cpumask_equal(newdev->cpumask, cpumask))
218
		irq_set_affinity(newdev->irq, cpumask);
219

220 221 222 223
	/*
	 * When global broadcasting is active, check if the current
	 * device is registered as a placeholder for broadcast mode.
	 * This allows us to handle this x86 misfeature in a generic
224 225
	 * way. This function also returns !=0 when we keep the
	 * current active broadcast state for this CPU.
226 227 228 229
	 */
	if (tick_device_uses_broadcast(newdev, cpu))
		return;

230 231
	if (td->mode == TICKDEV_MODE_PERIODIC)
		tick_setup_periodic(newdev, 0);
232 233
	else
		tick_setup_oneshot(newdev, handler, next_event);
234 235
}

236 237
void tick_install_replacement(struct clock_event_device *newdev)
{
238
	struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
239 240 241 242 243 244 245 246
	int cpu = smp_processor_id();

	clockevents_exchange_device(td->evtdev, newdev);
	tick_setup_device(td, newdev, cpu, cpumask_of(cpu));
	if (newdev->features & CLOCK_EVT_FEAT_ONESHOT)
		tick_oneshot_notify();
}

247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273
static bool tick_check_percpu(struct clock_event_device *curdev,
			      struct clock_event_device *newdev, int cpu)
{
	if (!cpumask_test_cpu(cpu, newdev->cpumask))
		return false;
	if (cpumask_equal(newdev->cpumask, cpumask_of(cpu)))
		return true;
	/* Check if irq affinity can be set */
	if (newdev->irq >= 0 && !irq_can_set_affinity(newdev->irq))
		return false;
	/* Prefer an existing cpu local device */
	if (curdev && cpumask_equal(curdev->cpumask, cpumask_of(cpu)))
		return false;
	return true;
}

static bool tick_check_preferred(struct clock_event_device *curdev,
				 struct clock_event_device *newdev)
{
	/* Prefer oneshot capable device */
	if (!(newdev->features & CLOCK_EVT_FEAT_ONESHOT)) {
		if (curdev && (curdev->features & CLOCK_EVT_FEAT_ONESHOT))
			return false;
		if (tick_oneshot_mode_active())
			return false;
	}

274 275 276 277 278 279 280
	/*
	 * Use the higher rated one, but prefer a CPU local device with a lower
	 * rating than a non-CPU local device
	 */
	return !curdev ||
		newdev->rating > curdev->rating ||
	       !cpumask_equal(curdev->cpumask, newdev->cpumask);
281 282
}

283 284 285 286 287 288 289
/*
 * Check whether the new device is a better fit than curdev. curdev
 * can be NULL !
 */
bool tick_check_replacement(struct clock_event_device *curdev,
			    struct clock_event_device *newdev)
{
290
	if (!tick_check_percpu(curdev, newdev, smp_processor_id()))
291 292 293 294 295
		return false;

	return tick_check_preferred(curdev, newdev);
}

296
/*
T
Thomas Gleixner 已提交
297 298
 * Check, if the new registered device should be used. Called with
 * clockevents_lock held and interrupts disabled.
299
 */
300
void tick_check_new_device(struct clock_event_device *newdev)
301 302 303
{
	struct clock_event_device *curdev;
	struct tick_device *td;
304
	int cpu;
305 306 307 308 309 310

	cpu = smp_processor_id();
	td = &per_cpu(tick_cpu_device, cpu);
	curdev = td->evtdev;

	/* cpu local device ? */
311 312
	if (!tick_check_percpu(curdev, newdev, cpu))
		goto out_bc;
313

314 315 316
	/* Preference decision */
	if (!tick_check_preferred(curdev, newdev))
		goto out_bc;
317

318 319 320
	if (!try_module_get(newdev->owner))
		return;

321 322
	/*
	 * Replace the eventually existing device by the new
323 324
	 * device. If the current device is the broadcast device, do
	 * not give it back to the clockevents layer !
325
	 */
326
	if (tick_is_broadcast_device(curdev)) {
327
		clockevents_shutdown(curdev);
328 329
		curdev = NULL;
	}
330
	clockevents_exchange_device(curdev, newdev);
331
	tick_setup_device(td, newdev, cpu, cpumask_of(cpu));
332 333
	if (newdev->features & CLOCK_EVT_FEAT_ONESHOT)
		tick_oneshot_notify();
334
	return;
335 336 337 338 339

out_bc:
	/*
	 * Can the new device be used as a broadcast device ?
	 */
340
	tick_install_broadcast_device(newdev);
341 342
}

343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362
/**
 * tick_broadcast_oneshot_control - Enter/exit broadcast oneshot mode
 * @state:	The target state (enter/exit)
 *
 * The system enters/leaves a state, where affected devices might stop
 * Returns 0 on success, -EBUSY if the cpu is used to broadcast wakeups.
 *
 * Called with interrupts disabled, so clockevents_lock is not
 * required here because the local clock event device cannot go away
 * under us.
 */
int tick_broadcast_oneshot_control(enum tick_broadcast_state state)
{
	struct tick_device *td = this_cpu_ptr(&tick_cpu_device);

	if (!(td->evtdev->features & CLOCK_EVT_FEAT_C3STOP))
		return 0;

	return __tick_broadcast_oneshot_control(state);
}
363
EXPORT_SYMBOL_GPL(tick_broadcast_oneshot_control);
364

365
#ifdef CONFIG_HOTPLUG_CPU
366 367 368
/*
 * Transfer the do_timer job away from a dying cpu.
 *
369 370
 * Called with interrupts disabled. Not locking required. If
 * tick_do_timer_cpu is owned by this cpu, nothing can change it.
371
 */
372
void tick_handover_do_timer(void)
373
{
374
	if (tick_do_timer_cpu == smp_processor_id()) {
375 376 377 378 379 380 381
		int cpu = cpumask_first(cpu_online_mask);

		tick_do_timer_cpu = (cpu < nr_cpu_ids) ? cpu :
			TICK_DO_TIMER_NONE;
	}
}

382 383 384 385 386 387 388
/*
 * Shutdown an event device on a given cpu:
 *
 * This is called on a life CPU, when a CPU is dead. So we cannot
 * access the hardware device itself.
 * We just set the mode and remove it from the lists.
 */
389
void tick_shutdown(unsigned int cpu)
390
{
391
	struct tick_device *td = &per_cpu(tick_cpu_device, cpu);
392 393 394 395 396 397 398 399
	struct clock_event_device *dev = td->evtdev;

	td->mode = TICKDEV_MODE_PERIODIC;
	if (dev) {
		/*
		 * Prevent that the clock events layer tries to call
		 * the set mode function!
		 */
400
		clockevent_set_state(dev, CLOCK_EVT_STATE_DETACHED);
401
		clockevents_exchange_device(dev, NULL);
402
		dev->event_handler = clockevents_handle_noop;
403 404 405
		td->evtdev = NULL;
	}
}
406
#endif
407

408
/**
409
 * tick_suspend_local - Suspend the local tick device
410
 *
411
 * Called from the local cpu for freeze with interrupts disabled.
412 413 414
 *
 * No locks required. Nothing can change the per cpu device.
 */
415
void tick_suspend_local(void)
416
{
417
	struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
418

419
	clockevents_shutdown(td->evtdev);
420 421
}

422
/**
423
 * tick_resume_local - Resume the local tick device
424
 *
425
 * Called from the local CPU for unfreeze or XEN resume magic.
426 427 428
 *
 * No locks required. Nothing can change the per cpu device.
 */
429
void tick_resume_local(void)
430
{
431 432
	struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
	bool broadcast = tick_resume_check_broadcast();
433

434
	clockevents_tick_resume(td->evtdev);
T
Thomas Gleixner 已提交
435 436 437 438 439 440
	if (!broadcast) {
		if (td->mode == TICKDEV_MODE_PERIODIC)
			tick_setup_periodic(td->evtdev, 0);
		else
			tick_resume_oneshot();
	}
441 442
}

443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471
/**
 * tick_suspend - Suspend the tick and the broadcast device
 *
 * Called from syscore_suspend() via timekeeping_suspend with only one
 * CPU online and interrupts disabled or from tick_unfreeze() under
 * tick_freeze_lock.
 *
 * No locks required. Nothing can change the per cpu device.
 */
void tick_suspend(void)
{
	tick_suspend_local();
	tick_suspend_broadcast();
}

/**
 * tick_resume - Resume the tick and the broadcast device
 *
 * Called from syscore_resume() via timekeeping_resume with only one
 * CPU online and interrupts disabled.
 *
 * No locks required. Nothing can change the per cpu device.
 */
void tick_resume(void)
{
	tick_resume_broadcast();
	tick_resume_local();
}

472
#ifdef CONFIG_SUSPEND
473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489
static DEFINE_RAW_SPINLOCK(tick_freeze_lock);
static unsigned int tick_freeze_depth;

/**
 * tick_freeze - Suspend the local tick and (possibly) timekeeping.
 *
 * Check if this is the last online CPU executing the function and if so,
 * suspend timekeeping.  Otherwise suspend the local tick.
 *
 * Call with interrupts disabled.  Must be balanced with %tick_unfreeze().
 * Interrupts must not be enabled before the subsequent %tick_unfreeze().
 */
void tick_freeze(void)
{
	raw_spin_lock(&tick_freeze_lock);

	tick_freeze_depth++;
490 491 492
	if (tick_freeze_depth == num_online_cpus()) {
		trace_suspend_resume(TPS("timekeeping_freeze"),
				     smp_processor_id(), true);
493
		timekeeping_suspend();
494
	} else {
495
		tick_suspend_local();
496
	}
497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513

	raw_spin_unlock(&tick_freeze_lock);
}

/**
 * tick_unfreeze - Resume the local tick and (possibly) timekeeping.
 *
 * Check if this is the first CPU executing the function and if so, resume
 * timekeeping.  Otherwise resume the local tick.
 *
 * Call with interrupts disabled.  Must be balanced with %tick_freeze().
 * Interrupts must not be enabled after the preceding %tick_freeze().
 */
void tick_unfreeze(void)
{
	raw_spin_lock(&tick_freeze_lock);

514
	if (tick_freeze_depth == num_online_cpus()) {
515
		timekeeping_resume();
516 517 518
		trace_suspend_resume(TPS("timekeeping_freeze"),
				     smp_processor_id(), false);
	} else {
519
		tick_resume_local();
520
	}
521 522 523 524 525

	tick_freeze_depth--;

	raw_spin_unlock(&tick_freeze_lock);
}
526
#endif /* CONFIG_SUSPEND */
527

528 529 530 531 532
/**
 * tick_init - initialize the tick control
 */
void __init tick_init(void)
{
533
	tick_broadcast_init();
534
	tick_nohz_init();
535
}