workqueue.c 21.7 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14
/*
 * linux/kernel/workqueue.c
 *
 * Generic mechanism for defining kernel helper threads for running
 * arbitrary tasks in process context.
 *
 * Started by Ingo Molnar, Copyright (C) 2002
 *
 * Derived from the taskqueue/keventd code by:
 *
 *   David Woodhouse <dwmw2@infradead.org>
 *   Andrew Morton <andrewm@uow.edu.au>
 *   Kai Petzke <wpp@marie.physik.tu-berlin.de>
 *   Theodore Ts'o <tytso@mit.edu>
15 16
 *
 * Made to use alloc_percpu by Christoph Lameter <clameter@sgi.com>.
L
Linus Torvalds 已提交
17 18 19 20 21 22 23 24 25 26 27 28 29
 */

#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/init.h>
#include <linux/signal.h>
#include <linux/completion.h>
#include <linux/workqueue.h>
#include <linux/slab.h>
#include <linux/cpu.h>
#include <linux/notifier.h>
#include <linux/kthread.h>
30
#include <linux/hardirq.h>
31
#include <linux/mempolicy.h>
32
#include <linux/freezer.h>
33 34
#include <linux/kallsyms.h>
#include <linux/debug_locks.h>
35
#include <linux/lockdep.h>
L
Linus Torvalds 已提交
36 37

/*
38 39
 * The per-CPU workqueue (if single thread, we always use the first
 * possible cpu).
L
Linus Torvalds 已提交
40 41 42 43 44 45 46
 */
struct cpu_workqueue_struct {

	spinlock_t lock;

	struct list_head worklist;
	wait_queue_head_t more_work;
47
	struct work_struct *current_work;
L
Linus Torvalds 已提交
48 49

	struct workqueue_struct *wq;
50
	struct task_struct *thread;
L
Linus Torvalds 已提交
51 52 53 54 55 56 57 58 59

	int run_depth;		/* Detect run_workqueue() recursion depth */
} ____cacheline_aligned;

/*
 * The externally visible workqueue abstraction is an array of
 * per-CPU workqueues:
 */
struct workqueue_struct {
60
	struct cpu_workqueue_struct *cpu_wq;
61
	struct list_head list;
L
Linus Torvalds 已提交
62
	const char *name;
63
	int singlethread;
64
	int freezeable;		/* Freeze threads during suspend */
65 66 67
#ifdef CONFIG_LOCKDEP
	struct lockdep_map lockdep_map;
#endif
L
Linus Torvalds 已提交
68 69
};

70 71
/* Serializes the accesses to the list of workqueues. */
static DEFINE_SPINLOCK(workqueue_lock);
L
Linus Torvalds 已提交
72 73
static LIST_HEAD(workqueues);

74
static int singlethread_cpu __read_mostly;
75
static cpumask_t cpu_singlethread_map __read_mostly;
76 77 78 79 80 81 82
/*
 * _cpu_down() first removes CPU from cpu_online_map, then CPU_DEAD
 * flushes cwq->worklist. This means that flush_workqueue/wait_on_work
 * which comes in between can't use for_each_online_cpu(). We could
 * use cpu_possible_map, the cpumask below is more a documentation
 * than optimization.
 */
83
static cpumask_t cpu_populated_map __read_mostly;
84

L
Linus Torvalds 已提交
85 86 87
/* If it's single threaded, it isn't in the list of workqueues. */
static inline int is_single_threaded(struct workqueue_struct *wq)
{
88
	return wq->singlethread;
L
Linus Torvalds 已提交
89 90
}

91 92 93 94 95 96
static const cpumask_t *wq_cpu_map(struct workqueue_struct *wq)
{
	return is_single_threaded(wq)
		? &cpu_singlethread_map : &cpu_populated_map;
}

97 98 99 100 101 102 103 104
static
struct cpu_workqueue_struct *wq_per_cpu(struct workqueue_struct *wq, int cpu)
{
	if (unlikely(is_single_threaded(wq)))
		cpu = singlethread_cpu;
	return per_cpu_ptr(wq->cpu_wq, cpu);
}

105 106 107 108
/*
 * Set the workqueue on which a work item is to be run
 * - Must *only* be called if the pending flag is set
 */
109 110
static inline void set_wq_data(struct work_struct *work,
				struct cpu_workqueue_struct *cwq)
111
{
112 113 114
	unsigned long new;

	BUG_ON(!work_pending(work));
115

116
	new = (unsigned long) cwq | (1UL << WORK_STRUCT_PENDING);
117 118
	new |= WORK_STRUCT_FLAG_MASK & *work_data_bits(work);
	atomic_long_set(&work->data, new);
119 120
}

121 122
static inline
struct cpu_workqueue_struct *get_wq_data(struct work_struct *work)
123
{
124
	return (void *) (atomic_long_read(&work->data) & WORK_STRUCT_WQ_DATA_MASK);
125 126
}

O
Oleg Nesterov 已提交
127 128 129 130
static void insert_work(struct cpu_workqueue_struct *cwq,
				struct work_struct *work, int tail)
{
	set_wq_data(work, cwq);
131 132 133 134 135
	/*
	 * Ensure that we get the right work->data if we see the
	 * result of list_add() below, see try_to_grab_pending().
	 */
	smp_wmb();
O
Oleg Nesterov 已提交
136 137 138 139 140 141 142
	if (tail)
		list_add_tail(&work->entry, &cwq->worklist);
	else
		list_add(&work->entry, &cwq->worklist);
	wake_up(&cwq->more_work);
}

L
Linus Torvalds 已提交
143 144 145 146 147 148 149
/* Preempt must be disabled. */
static void __queue_work(struct cpu_workqueue_struct *cwq,
			 struct work_struct *work)
{
	unsigned long flags;

	spin_lock_irqsave(&cwq->lock, flags);
O
Oleg Nesterov 已提交
150
	insert_work(cwq, work, 1);
L
Linus Torvalds 已提交
151 152 153
	spin_unlock_irqrestore(&cwq->lock, flags);
}

154 155 156 157 158
/**
 * queue_work - queue work on a workqueue
 * @wq: workqueue to use
 * @work: work to queue
 *
A
Alan Stern 已提交
159
 * Returns 0 if @work was already on a queue, non-zero otherwise.
L
Linus Torvalds 已提交
160
 *
161 162
 * We queue the work to the CPU on which it was submitted, but if the CPU dies
 * it can be processed by another CPU.
L
Linus Torvalds 已提交
163
 */
164
int queue_work(struct workqueue_struct *wq, struct work_struct *work)
L
Linus Torvalds 已提交
165
{
166
	int ret = 0;
L
Linus Torvalds 已提交
167

168
	if (!test_and_set_bit(WORK_STRUCT_PENDING, work_data_bits(work))) {
L
Linus Torvalds 已提交
169
		BUG_ON(!list_empty(&work->entry));
170 171
		__queue_work(wq_per_cpu(wq, get_cpu()), work);
		put_cpu();
L
Linus Torvalds 已提交
172 173 174 175
		ret = 1;
	}
	return ret;
}
176
EXPORT_SYMBOL_GPL(queue_work);
L
Linus Torvalds 已提交
177

178
static void delayed_work_timer_fn(unsigned long __data)
L
Linus Torvalds 已提交
179
{
180
	struct delayed_work *dwork = (struct delayed_work *)__data;
181 182
	struct cpu_workqueue_struct *cwq = get_wq_data(&dwork->work);
	struct workqueue_struct *wq = cwq->wq;
L
Linus Torvalds 已提交
183

184
	__queue_work(wq_per_cpu(wq, smp_processor_id()), &dwork->work);
L
Linus Torvalds 已提交
185 186
}

187 188 189
/**
 * queue_delayed_work - queue work on a workqueue after delay
 * @wq: workqueue to use
190
 * @dwork: delayable work to queue
191 192
 * @delay: number of jiffies to wait before queueing
 *
A
Alan Stern 已提交
193
 * Returns 0 if @work was already on a queue, non-zero otherwise.
194
 */
195
int queue_delayed_work(struct workqueue_struct *wq,
196
			struct delayed_work *dwork, unsigned long delay)
L
Linus Torvalds 已提交
197
{
198
	timer_stats_timer_set_start_info(&dwork->timer);
199
	if (delay == 0)
200
		return queue_work(wq, &dwork->work);
L
Linus Torvalds 已提交
201

202
	return queue_delayed_work_on(-1, wq, dwork, delay);
L
Linus Torvalds 已提交
203
}
204
EXPORT_SYMBOL_GPL(queue_delayed_work);
L
Linus Torvalds 已提交
205

206 207 208 209
/**
 * queue_delayed_work_on - queue work on specific CPU after delay
 * @cpu: CPU number to execute work on
 * @wq: workqueue to use
210
 * @dwork: work to queue
211 212
 * @delay: number of jiffies to wait before queueing
 *
A
Alan Stern 已提交
213
 * Returns 0 if @work was already on a queue, non-zero otherwise.
214
 */
215
int queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
216
			struct delayed_work *dwork, unsigned long delay)
217 218
{
	int ret = 0;
219 220
	struct timer_list *timer = &dwork->timer;
	struct work_struct *work = &dwork->work;
221

222
	timer_stats_timer_set_start_info(&dwork->timer);
223
	if (!test_and_set_bit(WORK_STRUCT_PENDING, work_data_bits(work))) {
224 225 226
		BUG_ON(timer_pending(timer));
		BUG_ON(!list_empty(&work->entry));

227
		/* This stores cwq for the moment, for the timer_fn */
228
		set_wq_data(work, wq_per_cpu(wq, raw_smp_processor_id()));
229
		timer->expires = jiffies + delay;
230
		timer->data = (unsigned long)dwork;
231
		timer->function = delayed_work_timer_fn;
232 233 234 235 236

		if (unlikely(cpu >= 0))
			add_timer_on(timer, cpu);
		else
			add_timer(timer);
237 238 239 240
		ret = 1;
	}
	return ret;
}
241
EXPORT_SYMBOL_GPL(queue_delayed_work_on);
L
Linus Torvalds 已提交
242

243
static void run_workqueue(struct cpu_workqueue_struct *cwq)
L
Linus Torvalds 已提交
244
{
245
	spin_lock_irq(&cwq->lock);
L
Linus Torvalds 已提交
246 247 248 249
	cwq->run_depth++;
	if (cwq->run_depth > 3) {
		/* morton gets to eat his hat */
		printk("%s: recursion depth exceeded: %d\n",
250
			__func__, cwq->run_depth);
L
Linus Torvalds 已提交
251 252 253 254 255
		dump_stack();
	}
	while (!list_empty(&cwq->worklist)) {
		struct work_struct *work = list_entry(cwq->worklist.next,
						struct work_struct, entry);
256
		work_func_t f = work->func;
257 258 259 260 261 262 263 264 265 266 267
#ifdef CONFIG_LOCKDEP
		/*
		 * It is permissible to free the struct work_struct
		 * from inside the function that is called from it,
		 * this we need to take into account for lockdep too.
		 * To avoid bogus "held lock freed" warnings as well
		 * as problems when looking into work->lockdep_map,
		 * make a copy and use that here.
		 */
		struct lockdep_map lockdep_map = work->lockdep_map;
#endif
L
Linus Torvalds 已提交
268

O
Oleg Nesterov 已提交
269
		cwq->current_work = work;
L
Linus Torvalds 已提交
270
		list_del_init(cwq->worklist.next);
271
		spin_unlock_irq(&cwq->lock);
L
Linus Torvalds 已提交
272

273
		BUG_ON(get_wq_data(work) != cwq);
O
Oleg Nesterov 已提交
274
		work_clear_pending(work);
275 276
		lock_acquire(&cwq->wq->lockdep_map, 0, 0, 0, 2, _THIS_IP_);
		lock_acquire(&lockdep_map, 0, 0, 0, 2, _THIS_IP_);
277
		f(work);
278 279
		lock_release(&lockdep_map, 1, _THIS_IP_);
		lock_release(&cwq->wq->lockdep_map, 1, _THIS_IP_);
L
Linus Torvalds 已提交
280

281 282 283 284
		if (unlikely(in_atomic() || lockdep_depth(current) > 0)) {
			printk(KERN_ERR "BUG: workqueue leaked lock or atomic: "
					"%s/0x%08x/%d\n",
					current->comm, preempt_count(),
285
				       	task_pid_nr(current));
286 287 288 289 290 291
			printk(KERN_ERR "    last function: ");
			print_symbol("%s\n", (unsigned long)f);
			debug_show_held_locks(current);
			dump_stack();
		}

292
		spin_lock_irq(&cwq->lock);
O
Oleg Nesterov 已提交
293
		cwq->current_work = NULL;
L
Linus Torvalds 已提交
294 295
	}
	cwq->run_depth--;
296
	spin_unlock_irq(&cwq->lock);
L
Linus Torvalds 已提交
297 298 299 300 301
}

static int worker_thread(void *__cwq)
{
	struct cpu_workqueue_struct *cwq = __cwq;
302
	DEFINE_WAIT(wait);
L
Linus Torvalds 已提交
303

304 305
	if (cwq->wq->freezeable)
		set_freezable();
L
Linus Torvalds 已提交
306 307 308

	set_user_nice(current, -5);

309 310
	for (;;) {
		prepare_to_wait(&cwq->more_work, &wait, TASK_INTERRUPTIBLE);
311 312 313
		if (!freezing(current) &&
		    !kthread_should_stop() &&
		    list_empty(&cwq->worklist))
L
Linus Torvalds 已提交
314
			schedule();
315 316
		finish_wait(&cwq->more_work, &wait);

317 318
		try_to_freeze();

319
		if (kthread_should_stop())
320
			break;
L
Linus Torvalds 已提交
321

322
		run_workqueue(cwq);
L
Linus Torvalds 已提交
323
	}
324

L
Linus Torvalds 已提交
325 326 327
	return 0;
}

O
Oleg Nesterov 已提交
328 329 330 331 332 333 334 335 336 337 338
struct wq_barrier {
	struct work_struct	work;
	struct completion	done;
};

static void wq_barrier_func(struct work_struct *work)
{
	struct wq_barrier *barr = container_of(work, struct wq_barrier, work);
	complete(&barr->done);
}

339 340
static void insert_wq_barrier(struct cpu_workqueue_struct *cwq,
					struct wq_barrier *barr, int tail)
O
Oleg Nesterov 已提交
341 342 343 344 345
{
	INIT_WORK(&barr->work, wq_barrier_func);
	__set_bit(WORK_STRUCT_PENDING, work_data_bits(&barr->work));

	init_completion(&barr->done);
346 347

	insert_work(cwq, &barr->work, tail);
O
Oleg Nesterov 已提交
348 349
}

350
static int flush_cpu_workqueue(struct cpu_workqueue_struct *cwq)
L
Linus Torvalds 已提交
351
{
352 353
	int active;

L
Linus Torvalds 已提交
354 355 356 357 358 359
	if (cwq->thread == current) {
		/*
		 * Probably keventd trying to flush its own queue. So simply run
		 * it by hand rather than deadlocking.
		 */
		run_workqueue(cwq);
360
		active = 1;
L
Linus Torvalds 已提交
361
	} else {
O
Oleg Nesterov 已提交
362
		struct wq_barrier barr;
L
Linus Torvalds 已提交
363

364
		active = 0;
365 366 367 368 369 370
		spin_lock_irq(&cwq->lock);
		if (!list_empty(&cwq->worklist) || cwq->current_work != NULL) {
			insert_wq_barrier(cwq, &barr, 1);
			active = 1;
		}
		spin_unlock_irq(&cwq->lock);
L
Linus Torvalds 已提交
371

372
		if (active)
373
			wait_for_completion(&barr.done);
L
Linus Torvalds 已提交
374
	}
375 376

	return active;
L
Linus Torvalds 已提交
377 378
}

379
/**
L
Linus Torvalds 已提交
380
 * flush_workqueue - ensure that any scheduled work has run to completion.
381
 * @wq: workqueue to flush
L
Linus Torvalds 已提交
382 383 384 385
 *
 * Forces execution of the workqueue and blocks until its completion.
 * This is typically used in driver shutdown handlers.
 *
O
Oleg Nesterov 已提交
386 387
 * We sleep until all works which were queued on entry have been handled,
 * but we are not livelocked by new incoming ones.
L
Linus Torvalds 已提交
388 389 390 391
 *
 * This function used to run the workqueues itself.  Now we just wait for the
 * helper threads to do it.
 */
392
void flush_workqueue(struct workqueue_struct *wq)
L
Linus Torvalds 已提交
393
{
394
	const cpumask_t *cpu_map = wq_cpu_map(wq);
395
	int cpu;
L
Linus Torvalds 已提交
396

397
	might_sleep();
398 399
	lock_acquire(&wq->lockdep_map, 0, 0, 0, 2, _THIS_IP_);
	lock_release(&wq->lockdep_map, 1, _THIS_IP_);
400 401
	for_each_cpu_mask(cpu, *cpu_map)
		flush_cpu_workqueue(per_cpu_ptr(wq->cpu_wq, cpu));
L
Linus Torvalds 已提交
402
}
403
EXPORT_SYMBOL_GPL(flush_workqueue);
L
Linus Torvalds 已提交
404

405
/*
406
 * Upon a successful return (>= 0), the caller "owns" WORK_STRUCT_PENDING bit,
407 408 409 410 411
 * so this work can't be re-armed in any way.
 */
static int try_to_grab_pending(struct work_struct *work)
{
	struct cpu_workqueue_struct *cwq;
412
	int ret = -1;
413 414

	if (!test_and_set_bit(WORK_STRUCT_PENDING, work_data_bits(work)))
415
		return 0;
416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444

	/*
	 * The queueing is in progress, or it is already queued. Try to
	 * steal it from ->worklist without clearing WORK_STRUCT_PENDING.
	 */

	cwq = get_wq_data(work);
	if (!cwq)
		return ret;

	spin_lock_irq(&cwq->lock);
	if (!list_empty(&work->entry)) {
		/*
		 * This work is queued, but perhaps we locked the wrong cwq.
		 * In that case we must see the new value after rmb(), see
		 * insert_work()->wmb().
		 */
		smp_rmb();
		if (cwq == get_wq_data(work)) {
			list_del_init(&work->entry);
			ret = 1;
		}
	}
	spin_unlock_irq(&cwq->lock);

	return ret;
}

static void wait_on_cpu_work(struct cpu_workqueue_struct *cwq,
O
Oleg Nesterov 已提交
445 446 447 448 449 450 451
				struct work_struct *work)
{
	struct wq_barrier barr;
	int running = 0;

	spin_lock_irq(&cwq->lock);
	if (unlikely(cwq->current_work == work)) {
452
		insert_wq_barrier(cwq, &barr, 0);
O
Oleg Nesterov 已提交
453 454 455 456
		running = 1;
	}
	spin_unlock_irq(&cwq->lock);

457
	if (unlikely(running))
O
Oleg Nesterov 已提交
458 459 460
		wait_for_completion(&barr.done);
}

461
static void wait_on_work(struct work_struct *work)
O
Oleg Nesterov 已提交
462 463
{
	struct cpu_workqueue_struct *cwq;
464 465
	struct workqueue_struct *wq;
	const cpumask_t *cpu_map;
466
	int cpu;
O
Oleg Nesterov 已提交
467

468 469
	might_sleep();

470 471 472
	lock_acquire(&work->lockdep_map, 0, 0, 0, 2, _THIS_IP_);
	lock_release(&work->lockdep_map, 1, _THIS_IP_);

O
Oleg Nesterov 已提交
473 474
	cwq = get_wq_data(work);
	if (!cwq)
475
		return;
O
Oleg Nesterov 已提交
476

477 478 479
	wq = cwq->wq;
	cpu_map = wq_cpu_map(wq);

480
	for_each_cpu_mask(cpu, *cpu_map)
481 482 483
		wait_on_cpu_work(per_cpu_ptr(wq->cpu_wq, cpu), work);
}

484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499
static int __cancel_work_timer(struct work_struct *work,
				struct timer_list* timer)
{
	int ret;

	do {
		ret = (timer && likely(del_timer(timer)));
		if (!ret)
			ret = try_to_grab_pending(work);
		wait_on_work(work);
	} while (unlikely(ret < 0));

	work_clear_pending(work);
	return ret;
}

500 501 502 503
/**
 * cancel_work_sync - block until a work_struct's callback has terminated
 * @work: the work which is to be flushed
 *
504 505
 * Returns true if @work was pending.
 *
506 507 508 509 510 511 512 513 514 515 516 517 518 519 520
 * cancel_work_sync() will cancel the work if it is queued. If the work's
 * callback appears to be running, cancel_work_sync() will block until it
 * has completed.
 *
 * It is possible to use this function if the work re-queues itself. It can
 * cancel the work even if it migrates to another workqueue, however in that
 * case it only guarantees that work->func() has completed on the last queued
 * workqueue.
 *
 * cancel_work_sync(&delayed_work->work) should be used only if ->timer is not
 * pending, otherwise it goes into a busy-wait loop until the timer expires.
 *
 * The caller must ensure that workqueue_struct on which this work was last
 * queued can't be destroyed before this function returns.
 */
521
int cancel_work_sync(struct work_struct *work)
522
{
523
	return __cancel_work_timer(work, NULL);
O
Oleg Nesterov 已提交
524
}
525
EXPORT_SYMBOL_GPL(cancel_work_sync);
O
Oleg Nesterov 已提交
526

527
/**
528
 * cancel_delayed_work_sync - reliably kill off a delayed work.
529 530
 * @dwork: the delayed work struct
 *
531 532
 * Returns true if @dwork was pending.
 *
533 534 535
 * It is possible to use this function if @dwork rearms itself via queue_work()
 * or queue_delayed_work(). See also the comment for cancel_work_sync().
 */
536
int cancel_delayed_work_sync(struct delayed_work *dwork)
537
{
538
	return __cancel_work_timer(&dwork->work, &dwork->timer);
539
}
540
EXPORT_SYMBOL(cancel_delayed_work_sync);
L
Linus Torvalds 已提交
541

542
static struct workqueue_struct *keventd_wq __read_mostly;
L
Linus Torvalds 已提交
543

544 545 546 547 548 549
/**
 * schedule_work - put work task in global workqueue
 * @work: job to be done
 *
 * This puts a job in the kernel-global workqueue.
 */
550
int schedule_work(struct work_struct *work)
L
Linus Torvalds 已提交
551 552 553
{
	return queue_work(keventd_wq, work);
}
554
EXPORT_SYMBOL(schedule_work);
L
Linus Torvalds 已提交
555

556 557
/**
 * schedule_delayed_work - put work task in global workqueue after delay
558 559
 * @dwork: job to be done
 * @delay: number of jiffies to wait or 0 for immediate execution
560 561 562 563
 *
 * After waiting for a given time this puts a job in the kernel-global
 * workqueue.
 */
564
int schedule_delayed_work(struct delayed_work *dwork,
565
					unsigned long delay)
L
Linus Torvalds 已提交
566
{
567
	timer_stats_timer_set_start_info(&dwork->timer);
568
	return queue_delayed_work(keventd_wq, dwork, delay);
L
Linus Torvalds 已提交
569
}
570
EXPORT_SYMBOL(schedule_delayed_work);
L
Linus Torvalds 已提交
571

572 573 574
/**
 * schedule_delayed_work_on - queue work in global workqueue on CPU after delay
 * @cpu: cpu to use
575
 * @dwork: job to be done
576 577 578 579 580
 * @delay: number of jiffies to wait
 *
 * After waiting for a given time this puts a job in the kernel-global
 * workqueue on the specified CPU.
 */
L
Linus Torvalds 已提交
581
int schedule_delayed_work_on(int cpu,
582
			struct delayed_work *dwork, unsigned long delay)
L
Linus Torvalds 已提交
583
{
584
	timer_stats_timer_set_start_info(&dwork->timer);
585
	return queue_delayed_work_on(cpu, keventd_wq, dwork, delay);
L
Linus Torvalds 已提交
586
}
587
EXPORT_SYMBOL(schedule_delayed_work_on);
L
Linus Torvalds 已提交
588

589 590 591 592 593 594 595 596 597
/**
 * schedule_on_each_cpu - call a function on each online CPU from keventd
 * @func: the function to call
 *
 * Returns zero on success.
 * Returns -ve errno on failure.
 *
 * schedule_on_each_cpu() is very slow.
 */
598
int schedule_on_each_cpu(work_func_t func)
599 600
{
	int cpu;
601
	struct work_struct *works;
602

603 604
	works = alloc_percpu(struct work_struct);
	if (!works)
605
		return -ENOMEM;
606

607
	get_online_cpus();
608
	for_each_online_cpu(cpu) {
609 610 611 612 613
		struct work_struct *work = per_cpu_ptr(works, cpu);

		INIT_WORK(work, func);
		set_bit(WORK_STRUCT_PENDING, work_data_bits(work));
		__queue_work(per_cpu_ptr(keventd_wq->cpu_wq, cpu), work);
614 615
	}
	flush_workqueue(keventd_wq);
616
	put_online_cpus();
617
	free_percpu(works);
618 619 620
	return 0;
}

L
Linus Torvalds 已提交
621 622 623 624
void flush_scheduled_work(void)
{
	flush_workqueue(keventd_wq);
}
625
EXPORT_SYMBOL(flush_scheduled_work);
L
Linus Torvalds 已提交
626

627 628 629 630 631 632 633 634 635 636 637 638
/**
 * execute_in_process_context - reliably execute the routine with user context
 * @fn:		the function to execute
 * @ew:		guaranteed storage for the execute work structure (must
 *		be available when the work executes)
 *
 * Executes the function immediately if process context is available,
 * otherwise schedules the function for delayed execution.
 *
 * Returns:	0 - function was executed
 *		1 - function was scheduled for execution
 */
639
int execute_in_process_context(work_func_t fn, struct execute_work *ew)
640 641
{
	if (!in_interrupt()) {
642
		fn(&ew->work);
643 644 645
		return 0;
	}

646
	INIT_WORK(&ew->work, fn);
647 648 649 650 651 652
	schedule_work(&ew->work);

	return 1;
}
EXPORT_SYMBOL_GPL(execute_in_process_context);

L
Linus Torvalds 已提交
653 654 655 656 657 658 659 660
int keventd_up(void)
{
	return keventd_wq != NULL;
}

int current_is_keventd(void)
{
	struct cpu_workqueue_struct *cwq;
H
Hugh Dickins 已提交
661
	int cpu = raw_smp_processor_id(); /* preempt-safe: keventd is per-cpu */
L
Linus Torvalds 已提交
662 663 664 665
	int ret = 0;

	BUG_ON(!keventd_wq);

666
	cwq = per_cpu_ptr(keventd_wq->cpu_wq, cpu);
L
Linus Torvalds 已提交
667 668 669 670 671 672 673
	if (current == cwq->thread)
		ret = 1;

	return ret;

}

674 675
static struct cpu_workqueue_struct *
init_cpu_workqueue(struct workqueue_struct *wq, int cpu)
L
Linus Torvalds 已提交
676
{
677
	struct cpu_workqueue_struct *cwq = per_cpu_ptr(wq->cpu_wq, cpu);
L
Linus Torvalds 已提交
678

679 680 681 682 683 684
	cwq->wq = wq;
	spin_lock_init(&cwq->lock);
	INIT_LIST_HEAD(&cwq->worklist);
	init_waitqueue_head(&cwq->more_work);

	return cwq;
L
Linus Torvalds 已提交
685 686
}

687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709
static int create_workqueue_thread(struct cpu_workqueue_struct *cwq, int cpu)
{
	struct workqueue_struct *wq = cwq->wq;
	const char *fmt = is_single_threaded(wq) ? "%s" : "%s/%d";
	struct task_struct *p;

	p = kthread_create(worker_thread, cwq, fmt, wq->name, cpu);
	/*
	 * Nobody can add the work_struct to this cwq,
	 *	if (caller is __create_workqueue)
	 *		nobody should see this wq
	 *	else // caller is CPU_UP_PREPARE
	 *		cpu is not on cpu_online_map
	 * so we can abort safely.
	 */
	if (IS_ERR(p))
		return PTR_ERR(p);

	cwq->thread = p;

	return 0;
}

710 711 712 713 714 715 716 717 718 719 720
static void start_workqueue_thread(struct cpu_workqueue_struct *cwq, int cpu)
{
	struct task_struct *p = cwq->thread;

	if (p != NULL) {
		if (cpu >= 0)
			kthread_bind(p, cpu);
		wake_up_process(p);
	}
}

721 722 723
struct workqueue_struct *__create_workqueue_key(const char *name,
						int singlethread,
						int freezeable,
724 725
						struct lock_class_key *key,
						const char *lock_name)
L
Linus Torvalds 已提交
726 727
{
	struct workqueue_struct *wq;
728 729
	struct cpu_workqueue_struct *cwq;
	int err = 0, cpu;
L
Linus Torvalds 已提交
730

731 732 733 734 735 736 737 738 739 740 741
	wq = kzalloc(sizeof(*wq), GFP_KERNEL);
	if (!wq)
		return NULL;

	wq->cpu_wq = alloc_percpu(struct cpu_workqueue_struct);
	if (!wq->cpu_wq) {
		kfree(wq);
		return NULL;
	}

	wq->name = name;
742
	lockdep_init_map(&wq->lockdep_map, lock_name, key, 0);
743
	wq->singlethread = singlethread;
744
	wq->freezeable = freezeable;
745
	INIT_LIST_HEAD(&wq->list);
746 747 748 749

	if (singlethread) {
		cwq = init_cpu_workqueue(wq, singlethread_cpu);
		err = create_workqueue_thread(cwq, singlethread_cpu);
750
		start_workqueue_thread(cwq, -1);
751
	} else {
752 753
		get_online_cpus();
		spin_lock(&workqueue_lock);
754
		list_add(&wq->list, &workqueues);
755
		spin_unlock(&workqueue_lock);
756 757 758 759 760 761

		for_each_possible_cpu(cpu) {
			cwq = init_cpu_workqueue(wq, cpu);
			if (err || !cpu_online(cpu))
				continue;
			err = create_workqueue_thread(cwq, cpu);
762
			start_workqueue_thread(cwq, cpu);
L
Linus Torvalds 已提交
763
		}
764
		put_online_cpus();
765 766 767 768 769 770 771 772
	}

	if (err) {
		destroy_workqueue(wq);
		wq = NULL;
	}
	return wq;
}
773
EXPORT_SYMBOL_GPL(__create_workqueue_key);
L
Linus Torvalds 已提交
774

775
static void cleanup_workqueue_thread(struct cpu_workqueue_struct *cwq)
776
{
777 778
	/*
	 * Our caller is either destroy_workqueue() or CPU_DEAD,
779
	 * get_online_cpus() protects cwq->thread.
780 781 782
	 */
	if (cwq->thread == NULL)
		return;
783

784 785 786
	lock_acquire(&cwq->wq->lockdep_map, 0, 0, 0, 2, _THIS_IP_);
	lock_release(&cwq->wq->lockdep_map, 1, _THIS_IP_);

O
Oleg Nesterov 已提交
787
	flush_cpu_workqueue(cwq);
788
	/*
O
Oleg Nesterov 已提交
789 790 791 792
	 * If the caller is CPU_DEAD and cwq->worklist was not empty,
	 * a concurrent flush_workqueue() can insert a barrier after us.
	 * However, in that case run_workqueue() won't return and check
	 * kthread_should_stop() until it flushes all work_struct's.
793 794 795 796 797 798 799
	 * When ->worklist becomes empty it is safe to exit because no
	 * more work_structs can be queued on this cwq: flush_workqueue
	 * checks list_empty(), and a "normal" queue_work() can't use
	 * a dead CPU.
	 */
	kthread_stop(cwq->thread);
	cwq->thread = NULL;
800 801 802 803 804 805 806 807 808 809
}

/**
 * destroy_workqueue - safely terminate a workqueue
 * @wq: target workqueue
 *
 * Safely destroy a workqueue. All work currently pending will be done first.
 */
void destroy_workqueue(struct workqueue_struct *wq)
{
810 811
	const cpumask_t *cpu_map = wq_cpu_map(wq);
	int cpu;
812

813 814
	get_online_cpus();
	spin_lock(&workqueue_lock);
815
	list_del(&wq->list);
816
	spin_unlock(&workqueue_lock);
817

818 819
	for_each_cpu_mask(cpu, *cpu_map)
		cleanup_workqueue_thread(per_cpu_ptr(wq->cpu_wq, cpu));
820
	put_online_cpus();
821

822 823 824 825 826 827 828 829 830 831 832 833 834
	free_percpu(wq->cpu_wq);
	kfree(wq);
}
EXPORT_SYMBOL_GPL(destroy_workqueue);

static int __devinit workqueue_cpu_callback(struct notifier_block *nfb,
						unsigned long action,
						void *hcpu)
{
	unsigned int cpu = (unsigned long)hcpu;
	struct cpu_workqueue_struct *cwq;
	struct workqueue_struct *wq;

835 836
	action &= ~CPU_TASKS_FROZEN;

837 838 839 840 841 842 843 844 845 846 847 848
	switch (action) {
	case CPU_UP_PREPARE:
		cpu_set(cpu, cpu_populated_map);
	}

	list_for_each_entry(wq, &workqueues, list) {
		cwq = per_cpu_ptr(wq->cpu_wq, cpu);

		switch (action) {
		case CPU_UP_PREPARE:
			if (!create_workqueue_thread(cwq, cpu))
				break;
849 850
			printk(KERN_ERR "workqueue [%s] for %i failed\n",
				wq->name, cpu);
851 852 853
			return NOTIFY_BAD;

		case CPU_ONLINE:
854
			start_workqueue_thread(cwq, cpu);
855 856 857
			break;

		case CPU_UP_CANCELED:
858
			start_workqueue_thread(cwq, -1);
859
		case CPU_DEAD:
860
			cleanup_workqueue_thread(cwq);
861 862
			break;
		}
L
Linus Torvalds 已提交
863 864
	}

865 866 867 868 869 870
	switch (action) {
	case CPU_UP_CANCELED:
	case CPU_DEAD:
		cpu_clear(cpu, cpu_populated_map);
	}

L
Linus Torvalds 已提交
871 872 873
	return NOTIFY_OK;
}

874
void __init init_workqueues(void)
L
Linus Torvalds 已提交
875
{
876
	cpu_populated_map = cpu_online_map;
877
	singlethread_cpu = first_cpu(cpu_possible_map);
878
	cpu_singlethread_map = cpumask_of_cpu(singlethread_cpu);
L
Linus Torvalds 已提交
879 880 881 882
	hotcpu_notifier(workqueue_cpu_callback, 0);
	keventd_wq = create_workqueue("events");
	BUG_ON(!keventd_wq);
}