workqueue.c 100.0 KB
Newer Older
L
Linus Torvalds 已提交
1
/*
T
Tejun Heo 已提交
2
 * kernel/workqueue.c - generic async execution with shared worker pool
L
Linus Torvalds 已提交
3
 *
T
Tejun Heo 已提交
4
 * Copyright (C) 2002		Ingo Molnar
L
Linus Torvalds 已提交
5
 *
T
Tejun Heo 已提交
6 7 8 9 10
 *   Derived from the taskqueue/keventd code by:
 *     David Woodhouse <dwmw2@infradead.org>
 *     Andrew Morton
 *     Kai Petzke <wpp@marie.physik.tu-berlin.de>
 *     Theodore Ts'o <tytso@mit.edu>
L
Linus Torvalds 已提交
11
 *
T
Tejun Heo 已提交
12
 * Made to use alloc_percpu by Christoph Lameter.
L
Linus Torvalds 已提交
13
 *
T
Tejun Heo 已提交
14 15
 * Copyright (C) 2010		SUSE Linux Products GmbH
 * Copyright (C) 2010		Tejun Heo <tj@kernel.org>
16
 *
T
Tejun Heo 已提交
17 18 19 20 21 22 23
 * This is the generic async execution mechanism.  Work items as are
 * executed in process context.  The worker pool is shared and
 * automatically managed.  There is one worker pool for each CPU and
 * one extra for works which are better served by workers which are
 * not bound to any specific CPU.
 *
 * Please read Documentation/workqueue.txt for details.
L
Linus Torvalds 已提交
24 25 26 27 28 29 30 31 32 33 34 35 36
 */

#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/init.h>
#include <linux/signal.h>
#include <linux/completion.h>
#include <linux/workqueue.h>
#include <linux/slab.h>
#include <linux/cpu.h>
#include <linux/notifier.h>
#include <linux/kthread.h>
37
#include <linux/hardirq.h>
38
#include <linux/mempolicy.h>
39
#include <linux/freezer.h>
40 41
#include <linux/kallsyms.h>
#include <linux/debug_locks.h>
42
#include <linux/lockdep.h>
T
Tejun Heo 已提交
43
#include <linux/idr.h>
44 45

#include "workqueue_sched.h"
L
Linus Torvalds 已提交
46

T
Tejun Heo 已提交
47
enum {
48
	/* global_cwq flags */
49 50 51
	GCWQ_MANAGE_WORKERS	= 1 << 0,	/* need to manage workers */
	GCWQ_MANAGING_WORKERS	= 1 << 1,	/* managing workers */
	GCWQ_DISASSOCIATED	= 1 << 2,	/* cpu can't serve workers */
52
	GCWQ_FREEZING		= 1 << 3,	/* freeze in progress */
53
	GCWQ_HIGHPRI_PENDING	= 1 << 4,	/* highpri works on queue */
54

T
Tejun Heo 已提交
55 56 57 58
	/* worker flags */
	WORKER_STARTED		= 1 << 0,	/* started */
	WORKER_DIE		= 1 << 1,	/* die die die */
	WORKER_IDLE		= 1 << 2,	/* is idle */
59
	WORKER_PREP		= 1 << 3,	/* preparing to run works */
60
	WORKER_ROGUE		= 1 << 4,	/* not bound to any cpu */
61
	WORKER_REBIND		= 1 << 5,	/* mom is home, come back */
62
	WORKER_CPU_INTENSIVE	= 1 << 6,	/* cpu intensive */
63
	WORKER_UNBOUND		= 1 << 7,	/* worker is unbound */
64

65
	WORKER_NOT_RUNNING	= WORKER_PREP | WORKER_ROGUE | WORKER_REBIND |
66
				  WORKER_CPU_INTENSIVE | WORKER_UNBOUND,
67 68 69 70 71 72 73

	/* gcwq->trustee_state */
	TRUSTEE_START		= 0,		/* start */
	TRUSTEE_IN_CHARGE	= 1,		/* trustee in charge of gcwq */
	TRUSTEE_BUTCHER		= 2,		/* butcher workers */
	TRUSTEE_RELEASE		= 3,		/* release workers */
	TRUSTEE_DONE		= 4,		/* trustee is done */
T
Tejun Heo 已提交
74 75 76 77

	BUSY_WORKER_HASH_ORDER	= 6,		/* 64 pointers */
	BUSY_WORKER_HASH_SIZE	= 1 << BUSY_WORKER_HASH_ORDER,
	BUSY_WORKER_HASH_MASK	= BUSY_WORKER_HASH_SIZE - 1,
78

79 80 81 82 83 84
	MAX_IDLE_WORKERS_RATIO	= 4,		/* 1/4 of busy can be idle */
	IDLE_WORKER_TIMEOUT	= 300 * HZ,	/* keep idle ones for 5 mins */

	MAYDAY_INITIAL_TIMEOUT	= HZ / 100,	/* call for help after 10ms */
	MAYDAY_INTERVAL		= HZ / 10,	/* and then every 100ms */
	CREATE_COOLDOWN		= HZ,		/* time to breath after fail */
85
	TRUSTEE_COOLDOWN	= HZ / 10,	/* for trustee draining */
86 87 88 89 90 91

	/*
	 * Rescue workers are used only on emergencies and shared by
	 * all cpus.  Give -20.
	 */
	RESCUER_NICE_LEVEL	= -20,
T
Tejun Heo 已提交
92
};
L
Linus Torvalds 已提交
93 94

/*
T
Tejun Heo 已提交
95 96
 * Structure fields follow one of the following exclusion rules.
 *
97 98
 * I: Modifiable by initialization/destruction paths and read-only for
 *    everyone else.
T
Tejun Heo 已提交
99
 *
100 101 102
 * P: Preemption protected.  Disabling preemption is enough and should
 *    only be modified and accessed from the local cpu.
 *
103
 * L: gcwq->lock protected.  Access with gcwq->lock held.
T
Tejun Heo 已提交
104
 *
105 106 107
 * X: During normal operation, modification requires gcwq->lock and
 *    should be done only from local cpu.  Either disabling preemption
 *    on local cpu or grabbing gcwq->lock is enough for read access.
108
 *    If GCWQ_DISASSOCIATED is set, it's identical to L.
109
 *
110 111
 * F: wq->flush_mutex protected.
 *
T
Tejun Heo 已提交
112
 * W: workqueue_lock protected.
L
Linus Torvalds 已提交
113 114
 */

115
struct global_cwq;
L
Linus Torvalds 已提交
116

117 118 119 120
/*
 * The poor guys doing the actual heavy lifting.  All on-duty workers
 * are either serving the manager role, on idle list or on busy hash.
 */
T
Tejun Heo 已提交
121
struct worker {
T
Tejun Heo 已提交
122 123 124 125 126
	/* on idle list while idle, on busy hash table while busy */
	union {
		struct list_head	entry;	/* L: while idle */
		struct hlist_node	hentry;	/* L: while busy */
	};
L
Linus Torvalds 已提交
127

T
Tejun Heo 已提交
128
	struct work_struct	*current_work;	/* L: work being processed */
129
	struct cpu_workqueue_struct *current_cwq; /* L: current_work's cwq */
130
	struct list_head	scheduled;	/* L: scheduled works */
T
Tejun Heo 已提交
131
	struct task_struct	*task;		/* I: worker task */
132
	struct global_cwq	*gcwq;		/* I: the associated gcwq */
133 134 135
	/* 64 bytes boundary on 64bit, 32 on 32bit */
	unsigned long		last_active;	/* L: last active timestamp */
	unsigned int		flags;		/* X: flags */
T
Tejun Heo 已提交
136
	int			id;		/* I: worker id */
137
	struct work_struct	rebind_work;	/* L: rebind worker to cpu */
T
Tejun Heo 已提交
138 139
};

140
/*
141 142 143
 * Global per-cpu workqueue.  There's one and only one for each cpu
 * and all works are queued and processed here regardless of their
 * target workqueues.
144 145 146
 */
struct global_cwq {
	spinlock_t		lock;		/* the gcwq lock */
147
	struct list_head	worklist;	/* L: list of pending works */
148
	unsigned int		cpu;		/* I: the associated cpu */
149
	unsigned int		flags;		/* L: GCWQ_* flags */
T
Tejun Heo 已提交
150 151 152 153 154

	int			nr_workers;	/* L: total number of workers */
	int			nr_idle;	/* L: currently idle ones */

	/* workers are chained either in the idle_list or busy_hash */
155
	struct list_head	idle_list;	/* X: list of idle workers */
T
Tejun Heo 已提交
156 157 158
	struct hlist_head	busy_hash[BUSY_WORKER_HASH_SIZE];
						/* L: hash of busy workers */

159 160 161
	struct timer_list	idle_timer;	/* L: worker idle timeout */
	struct timer_list	mayday_timer;	/* L: SOS timer for dworkers */

162
	struct ida		worker_ida;	/* L: for worker IDs */
163 164 165 166

	struct task_struct	*trustee;	/* L: for gcwq shutdown */
	unsigned int		trustee_state;	/* L: trustee state */
	wait_queue_head_t	trustee_wait;	/* trustee wait */
167
	struct worker		*first_idle;	/* L: first idle worker */
168 169
} ____cacheline_aligned_in_smp;

L
Linus Torvalds 已提交
170
/*
171
 * The per-CPU workqueue.  The lower WORK_STRUCT_FLAG_BITS of
T
Tejun Heo 已提交
172 173
 * work_struct->data are used for flags and thus cwqs need to be
 * aligned at two's power of the number of flag bits.
L
Linus Torvalds 已提交
174 175
 */
struct cpu_workqueue_struct {
176
	struct global_cwq	*gcwq;		/* I: the associated gcwq */
T
Tejun Heo 已提交
177
	struct workqueue_struct *wq;		/* I: the owning workqueue */
178 179 180 181
	int			work_color;	/* L: current color */
	int			flush_color;	/* L: flushing color */
	int			nr_in_flight[WORK_NR_COLORS];
						/* L: nr of in_flight works */
182
	int			nr_active;	/* L: nr of active works */
183
	int			max_active;	/* L: max active works */
184
	struct list_head	delayed_works;	/* L: delayed works */
T
Tejun Heo 已提交
185
};
L
Linus Torvalds 已提交
186

187 188 189 190 191 192 193 194 195
/*
 * Structure used to wait for workqueue flush.
 */
struct wq_flusher {
	struct list_head	list;		/* F: list of flushers */
	int			flush_color;	/* F: flush color waiting for */
	struct completion	done;		/* flush completion */
};

196 197 198 199 200 201 202 203 204 205
/*
 * All cpumasks are assumed to be always set on UP and thus can't be
 * used to determine whether there's something to be done.
 */
#ifdef CONFIG_SMP
typedef cpumask_var_t mayday_mask_t;
#define mayday_test_and_set_cpu(cpu, mask)	\
	cpumask_test_and_set_cpu((cpu), (mask))
#define mayday_clear_cpu(cpu, mask)		cpumask_clear_cpu((cpu), (mask))
#define for_each_mayday_cpu(cpu, mask)		for_each_cpu((cpu), (mask))
206
#define alloc_mayday_mask(maskp, gfp)		zalloc_cpumask_var((maskp), (gfp))
207 208 209 210 211 212 213 214 215
#define free_mayday_mask(mask)			free_cpumask_var((mask))
#else
typedef unsigned long mayday_mask_t;
#define mayday_test_and_set_cpu(cpu, mask)	test_and_set_bit(0, &(mask))
#define mayday_clear_cpu(cpu, mask)		clear_bit(0, &(mask))
#define for_each_mayday_cpu(cpu, mask)		if ((cpu) = 0, (mask))
#define alloc_mayday_mask(maskp, gfp)		true
#define free_mayday_mask(mask)			do { } while (0)
#endif
L
Linus Torvalds 已提交
216 217 218 219 220 221

/*
 * The externally visible workqueue abstraction is an array of
 * per-CPU workqueues:
 */
struct workqueue_struct {
222
	unsigned int		flags;		/* I: WQ_* flags */
223 224 225 226 227
	union {
		struct cpu_workqueue_struct __percpu	*pcpu;
		struct cpu_workqueue_struct		*single;
		unsigned long				v;
	} cpu_wq;				/* I: cwq's */
T
Tejun Heo 已提交
228
	struct list_head	list;		/* W: list of all workqueues */
229 230 231 232 233 234 235 236 237

	struct mutex		flush_mutex;	/* protects wq flushing */
	int			work_color;	/* F: current work color */
	int			flush_color;	/* F: current flush color */
	atomic_t		nr_cwqs_to_flush; /* flush in progress */
	struct wq_flusher	*first_flusher;	/* F: first flusher */
	struct list_head	flusher_queue;	/* F: flush waiters */
	struct list_head	flusher_overflow; /* F: flush overflow list */

238
	mayday_mask_t		mayday_mask;	/* cpus requesting rescue */
239 240
	struct worker		*rescuer;	/* I: rescue worker */

241
	int			saved_max_active; /* W: saved cwq max_active */
T
Tejun Heo 已提交
242
	const char		*name;		/* I: workqueue name */
243
#ifdef CONFIG_LOCKDEP
T
Tejun Heo 已提交
244
	struct lockdep_map	lockdep_map;
245
#endif
L
Linus Torvalds 已提交
246 247
};

248 249 250
struct workqueue_struct *system_wq __read_mostly;
struct workqueue_struct *system_long_wq __read_mostly;
struct workqueue_struct *system_nrt_wq __read_mostly;
251
struct workqueue_struct *system_unbound_wq __read_mostly;
252 253 254
EXPORT_SYMBOL_GPL(system_wq);
EXPORT_SYMBOL_GPL(system_long_wq);
EXPORT_SYMBOL_GPL(system_nrt_wq);
255
EXPORT_SYMBOL_GPL(system_unbound_wq);
256

257 258 259
#define CREATE_TRACE_POINTS
#include <trace/events/workqueue.h>

260 261 262 263
#define for_each_busy_worker(worker, i, pos, gcwq)			\
	for (i = 0; i < BUSY_WORKER_HASH_SIZE; i++)			\
		hlist_for_each_entry(worker, pos, &gcwq->busy_hash[i], hentry)

264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284
static inline int __next_gcwq_cpu(int cpu, const struct cpumask *mask,
				  unsigned int sw)
{
	if (cpu < nr_cpu_ids) {
		if (sw & 1) {
			cpu = cpumask_next(cpu, mask);
			if (cpu < nr_cpu_ids)
				return cpu;
		}
		if (sw & 2)
			return WORK_CPU_UNBOUND;
	}
	return WORK_CPU_NONE;
}

static inline int __next_wq_cpu(int cpu, const struct cpumask *mask,
				struct workqueue_struct *wq)
{
	return __next_gcwq_cpu(cpu, mask, !(wq->flags & WQ_UNBOUND) ? 1 : 2);
}

285 286 287 288 289 290 291 292 293 294 295 296 297
/*
 * CPU iterators
 *
 * An extra gcwq is defined for an invalid cpu number
 * (WORK_CPU_UNBOUND) to host workqueues which are not bound to any
 * specific CPU.  The following iterators are similar to
 * for_each_*_cpu() iterators but also considers the unbound gcwq.
 *
 * for_each_gcwq_cpu()		: possible CPUs + WORK_CPU_UNBOUND
 * for_each_online_gcwq_cpu()	: online CPUs + WORK_CPU_UNBOUND
 * for_each_cwq_cpu()		: possible CPUs for bound workqueues,
 *				  WORK_CPU_UNBOUND for unbound workqueues
 */
298 299 300 301 302 303 304 305 306 307 308 309 310 311 312
#define for_each_gcwq_cpu(cpu)						\
	for ((cpu) = __next_gcwq_cpu(-1, cpu_possible_mask, 3);		\
	     (cpu) < WORK_CPU_NONE;					\
	     (cpu) = __next_gcwq_cpu((cpu), cpu_possible_mask, 3))

#define for_each_online_gcwq_cpu(cpu)					\
	for ((cpu) = __next_gcwq_cpu(-1, cpu_online_mask, 3);		\
	     (cpu) < WORK_CPU_NONE;					\
	     (cpu) = __next_gcwq_cpu((cpu), cpu_online_mask, 3))

#define for_each_cwq_cpu(cpu, wq)					\
	for ((cpu) = __next_wq_cpu(-1, cpu_possible_mask, (wq));	\
	     (cpu) < WORK_CPU_NONE;					\
	     (cpu) = __next_wq_cpu((cpu), cpu_possible_mask, (wq)))

313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351
#ifdef CONFIG_DEBUG_OBJECTS_WORK

static struct debug_obj_descr work_debug_descr;

/*
 * fixup_init is called when:
 * - an active object is initialized
 */
static int work_fixup_init(void *addr, enum debug_obj_state state)
{
	struct work_struct *work = addr;

	switch (state) {
	case ODEBUG_STATE_ACTIVE:
		cancel_work_sync(work);
		debug_object_init(work, &work_debug_descr);
		return 1;
	default:
		return 0;
	}
}

/*
 * fixup_activate is called when:
 * - an active object is activated
 * - an unknown object is activated (might be a statically initialized object)
 */
static int work_fixup_activate(void *addr, enum debug_obj_state state)
{
	struct work_struct *work = addr;

	switch (state) {

	case ODEBUG_STATE_NOTAVAILABLE:
		/*
		 * This is not really a fixup. The work struct was
		 * statically initialized. We just make sure that it
		 * is tracked in the object tracker.
		 */
352
		if (test_bit(WORK_STRUCT_STATIC_BIT, work_data_bits(work))) {
353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422
			debug_object_init(work, &work_debug_descr);
			debug_object_activate(work, &work_debug_descr);
			return 0;
		}
		WARN_ON_ONCE(1);
		return 0;

	case ODEBUG_STATE_ACTIVE:
		WARN_ON(1);

	default:
		return 0;
	}
}

/*
 * fixup_free is called when:
 * - an active object is freed
 */
static int work_fixup_free(void *addr, enum debug_obj_state state)
{
	struct work_struct *work = addr;

	switch (state) {
	case ODEBUG_STATE_ACTIVE:
		cancel_work_sync(work);
		debug_object_free(work, &work_debug_descr);
		return 1;
	default:
		return 0;
	}
}

static struct debug_obj_descr work_debug_descr = {
	.name		= "work_struct",
	.fixup_init	= work_fixup_init,
	.fixup_activate	= work_fixup_activate,
	.fixup_free	= work_fixup_free,
};

static inline void debug_work_activate(struct work_struct *work)
{
	debug_object_activate(work, &work_debug_descr);
}

static inline void debug_work_deactivate(struct work_struct *work)
{
	debug_object_deactivate(work, &work_debug_descr);
}

void __init_work(struct work_struct *work, int onstack)
{
	if (onstack)
		debug_object_init_on_stack(work, &work_debug_descr);
	else
		debug_object_init(work, &work_debug_descr);
}
EXPORT_SYMBOL_GPL(__init_work);

void destroy_work_on_stack(struct work_struct *work)
{
	debug_object_free(work, &work_debug_descr);
}
EXPORT_SYMBOL_GPL(destroy_work_on_stack);

#else
static inline void debug_work_activate(struct work_struct *work) { }
static inline void debug_work_deactivate(struct work_struct *work) { }
#endif

423 424
/* Serializes the accesses to the list of workqueues. */
static DEFINE_SPINLOCK(workqueue_lock);
L
Linus Torvalds 已提交
425
static LIST_HEAD(workqueues);
426
static bool workqueue_freezing;		/* W: have wqs started freezing? */
T
Tejun Heo 已提交
427

428 429 430 431 432
/*
 * The almighty global cpu workqueues.  nr_running is the only field
 * which is expected to be used frequently by other cpus via
 * try_to_wake_up().  Put it in a separate cacheline.
 */
433
static DEFINE_PER_CPU(struct global_cwq, global_cwq);
434
static DEFINE_PER_CPU_SHARED_ALIGNED(atomic_t, gcwq_nr_running);
435

436 437 438 439 440 441 442 443
/*
 * Global cpu workqueue and nr_running counter for unbound gcwq.  The
 * gcwq is always online, has GCWQ_DISASSOCIATED set, and all its
 * workers have WORKER_UNBOUND set.
 */
static struct global_cwq unbound_global_cwq;
static atomic_t unbound_gcwq_nr_running = ATOMIC_INIT(0);	/* always 0 */

T
Tejun Heo 已提交
444
static int worker_thread(void *__worker);
L
Linus Torvalds 已提交
445

446 447
static struct global_cwq *get_gcwq(unsigned int cpu)
{
448 449 450 451
	if (cpu != WORK_CPU_UNBOUND)
		return &per_cpu(global_cwq, cpu);
	else
		return &unbound_global_cwq;
452 453
}

454 455
static atomic_t *get_gcwq_nr_running(unsigned int cpu)
{
456 457 458 459
	if (cpu != WORK_CPU_UNBOUND)
		return &per_cpu(gcwq_nr_running, cpu);
	else
		return &unbound_gcwq_nr_running;
460 461
}

T
Tejun Heo 已提交
462 463
static struct cpu_workqueue_struct *get_cwq(unsigned int cpu,
					    struct workqueue_struct *wq)
464
{
465 466 467 468
	if (!(wq->flags & WQ_UNBOUND)) {
		if (likely(cpu < nr_cpu_ids)) {
#ifdef CONFIG_SMP
			return per_cpu_ptr(wq->cpu_wq.pcpu, cpu);
469
#else
470
			return wq->cpu_wq.single;
471
#endif
472 473 474 475
		}
	} else if (likely(cpu == WORK_CPU_UNBOUND))
		return wq->cpu_wq.single;
	return NULL;
476 477
}

478 479 480 481 482 483 484 485 486 487 488 489 490 491 492
static unsigned int work_color_to_flags(int color)
{
	return color << WORK_STRUCT_COLOR_SHIFT;
}

static int get_work_color(struct work_struct *work)
{
	return (*work_data_bits(work) >> WORK_STRUCT_COLOR_SHIFT) &
		((1 << WORK_STRUCT_COLOR_BITS) - 1);
}

static int work_next_color(int color)
{
	return (color + 1) % WORK_NR_COLORS;
}
L
Linus Torvalds 已提交
493

494
/*
495 496 497
 * A work's data points to the cwq with WORK_STRUCT_CWQ set while the
 * work is on queue.  Once execution starts, WORK_STRUCT_CWQ is
 * cleared and the work data contains the cpu number it was last on.
498 499 500 501 502 503 504 505 506
 *
 * set_work_{cwq|cpu}() and clear_work_data() can be used to set the
 * cwq, cpu or clear work->data.  These functions should only be
 * called while the work is owned - ie. while the PENDING bit is set.
 *
 * get_work_[g]cwq() can be used to obtain the gcwq or cwq
 * corresponding to a work.  gcwq is available once the work has been
 * queued anywhere after initialization.  cwq is available only from
 * queueing until execution starts.
507
 */
508 509
static inline void set_work_data(struct work_struct *work, unsigned long data,
				 unsigned long flags)
510
{
511
	BUG_ON(!work_pending(work));
512 513
	atomic_long_set(&work->data, data | flags | work_static(work));
}
514

515 516 517 518 519
static void set_work_cwq(struct work_struct *work,
			 struct cpu_workqueue_struct *cwq,
			 unsigned long extra_flags)
{
	set_work_data(work, (unsigned long)cwq,
520
		      WORK_STRUCT_PENDING | WORK_STRUCT_CWQ | extra_flags);
521 522
}

523 524 525 526
static void set_work_cpu(struct work_struct *work, unsigned int cpu)
{
	set_work_data(work, cpu << WORK_STRUCT_FLAG_BITS, WORK_STRUCT_PENDING);
}
527

528
static void clear_work_data(struct work_struct *work)
L
Linus Torvalds 已提交
529
{
530
	set_work_data(work, WORK_STRUCT_NO_CPU, 0);
L
Linus Torvalds 已提交
531 532
}

533
static struct cpu_workqueue_struct *get_work_cwq(struct work_struct *work)
534
{
535
	unsigned long data = atomic_long_read(&work->data);
536

537 538 539 540
	if (data & WORK_STRUCT_CWQ)
		return (void *)(data & WORK_STRUCT_WQ_DATA_MASK);
	else
		return NULL;
541 542
}

543
static struct global_cwq *get_work_gcwq(struct work_struct *work)
544
{
545
	unsigned long data = atomic_long_read(&work->data);
546 547
	unsigned int cpu;

548 549 550
	if (data & WORK_STRUCT_CWQ)
		return ((struct cpu_workqueue_struct *)
			(data & WORK_STRUCT_WQ_DATA_MASK))->gcwq;
551 552

	cpu = data >> WORK_STRUCT_FLAG_BITS;
553
	if (cpu == WORK_CPU_NONE)
554 555
		return NULL;

556
	BUG_ON(cpu >= nr_cpu_ids && cpu != WORK_CPU_UNBOUND);
557
	return get_gcwq(cpu);
558 559
}

560 561 562 563 564 565
/*
 * Policy functions.  These define the policies on how the global
 * worker pool is managed.  Unless noted otherwise, these functions
 * assume that they're being called with gcwq->lock held.
 */

566
static bool __need_more_worker(struct global_cwq *gcwq)
567
{
568 569
	return !atomic_read(get_gcwq_nr_running(gcwq->cpu)) ||
		gcwq->flags & GCWQ_HIGHPRI_PENDING;
570 571
}

572
/*
573 574
 * Need to wake up a worker?  Called from anything but currently
 * running workers.
575
 */
576
static bool need_more_worker(struct global_cwq *gcwq)
577
{
578
	return !list_empty(&gcwq->worklist) && __need_more_worker(gcwq);
579
}
580

581 582 583 584 585 586 587 588 589 590 591
/* Can I start working?  Called from busy but !running workers. */
static bool may_start_working(struct global_cwq *gcwq)
{
	return gcwq->nr_idle;
}

/* Do I need to keep working?  Called from currently running workers. */
static bool keep_working(struct global_cwq *gcwq)
{
	atomic_t *nr_running = get_gcwq_nr_running(gcwq->cpu);

592 593 594
	return !list_empty(&gcwq->worklist) &&
		(atomic_read(nr_running) <= 1 ||
		 gcwq->flags & GCWQ_HIGHPRI_PENDING);
595 596 597 598 599 600 601
}

/* Do we need a new worker?  Called from manager. */
static bool need_to_create_worker(struct global_cwq *gcwq)
{
	return need_more_worker(gcwq) && !may_start_working(gcwq);
}
602

603 604 605 606 607 608 609 610 611 612 613 614 615 616
/* Do I need to be the manager? */
static bool need_to_manage_workers(struct global_cwq *gcwq)
{
	return need_to_create_worker(gcwq) || gcwq->flags & GCWQ_MANAGE_WORKERS;
}

/* Do we have too many workers and should some go away? */
static bool too_many_workers(struct global_cwq *gcwq)
{
	bool managing = gcwq->flags & GCWQ_MANAGING_WORKERS;
	int nr_idle = gcwq->nr_idle + managing; /* manager is considered idle */
	int nr_busy = gcwq->nr_workers - nr_idle;

	return nr_idle > 2 && (nr_idle - 2) * MAX_IDLE_WORKERS_RATIO >= nr_busy;
617 618
}

619
/*
620 621 622
 * Wake up functions.
 */

623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648
/* Return the first worker.  Safe with preemption disabled */
static struct worker *first_worker(struct global_cwq *gcwq)
{
	if (unlikely(list_empty(&gcwq->idle_list)))
		return NULL;

	return list_first_entry(&gcwq->idle_list, struct worker, entry);
}

/**
 * wake_up_worker - wake up an idle worker
 * @gcwq: gcwq to wake worker for
 *
 * Wake up the first idle worker of @gcwq.
 *
 * CONTEXT:
 * spin_lock_irq(gcwq->lock).
 */
static void wake_up_worker(struct global_cwq *gcwq)
{
	struct worker *worker = first_worker(gcwq);

	if (likely(worker))
		wake_up_process(worker->task);
}

649
/**
650 651 652 653 654 655 656 657 658 659 660 661 662 663
 * wq_worker_waking_up - a worker is waking up
 * @task: task waking up
 * @cpu: CPU @task is waking up to
 *
 * This function is called during try_to_wake_up() when a worker is
 * being awoken.
 *
 * CONTEXT:
 * spin_lock_irq(rq->lock)
 */
void wq_worker_waking_up(struct task_struct *task, unsigned int cpu)
{
	struct worker *worker = kthread_data(task);

664
	if (!(worker->flags & WORKER_NOT_RUNNING))
665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689
		atomic_inc(get_gcwq_nr_running(cpu));
}

/**
 * wq_worker_sleeping - a worker is going to sleep
 * @task: task going to sleep
 * @cpu: CPU in question, must be the current CPU number
 *
 * This function is called during schedule() when a busy worker is
 * going to sleep.  Worker on the same cpu can be woken up by
 * returning pointer to its task.
 *
 * CONTEXT:
 * spin_lock_irq(rq->lock)
 *
 * RETURNS:
 * Worker task on @cpu to wake up, %NULL if none.
 */
struct task_struct *wq_worker_sleeping(struct task_struct *task,
				       unsigned int cpu)
{
	struct worker *worker = kthread_data(task), *to_wakeup = NULL;
	struct global_cwq *gcwq = get_gcwq(cpu);
	atomic_t *nr_running = get_gcwq_nr_running(cpu);

690
	if (worker->flags & WORKER_NOT_RUNNING)
691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713
		return NULL;

	/* this can only happen on the local cpu */
	BUG_ON(cpu != raw_smp_processor_id());

	/*
	 * The counterpart of the following dec_and_test, implied mb,
	 * worklist not empty test sequence is in insert_work().
	 * Please read comment there.
	 *
	 * NOT_RUNNING is clear.  This means that trustee is not in
	 * charge and we're running on the local cpu w/ rq lock held
	 * and preemption disabled, which in turn means that none else
	 * could be manipulating idle_list, so dereferencing idle_list
	 * without gcwq lock is safe.
	 */
	if (atomic_dec_and_test(nr_running) && !list_empty(&gcwq->worklist))
		to_wakeup = first_worker(gcwq);
	return to_wakeup ? to_wakeup->task : NULL;
}

/**
 * worker_set_flags - set worker flags and adjust nr_running accordingly
714
 * @worker: self
715 716 717
 * @flags: flags to set
 * @wakeup: wakeup an idle worker if necessary
 *
718 719 720
 * Set @flags in @worker->flags and adjust nr_running accordingly.  If
 * nr_running becomes zero and @wakeup is %true, an idle worker is
 * woken up.
721
 *
722 723
 * CONTEXT:
 * spin_lock_irq(gcwq->lock)
724 725 726 727
 */
static inline void worker_set_flags(struct worker *worker, unsigned int flags,
				    bool wakeup)
{
728 729
	struct global_cwq *gcwq = worker->gcwq;

730 731
	WARN_ON_ONCE(worker->task != current);

732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748
	/*
	 * If transitioning into NOT_RUNNING, adjust nr_running and
	 * wake up an idle worker as necessary if requested by
	 * @wakeup.
	 */
	if ((flags & WORKER_NOT_RUNNING) &&
	    !(worker->flags & WORKER_NOT_RUNNING)) {
		atomic_t *nr_running = get_gcwq_nr_running(gcwq->cpu);

		if (wakeup) {
			if (atomic_dec_and_test(nr_running) &&
			    !list_empty(&gcwq->worklist))
				wake_up_worker(gcwq);
		} else
			atomic_dec(nr_running);
	}

749 750 751 752
	worker->flags |= flags;
}

/**
753
 * worker_clr_flags - clear worker flags and adjust nr_running accordingly
754
 * @worker: self
755 756
 * @flags: flags to clear
 *
757
 * Clear @flags in @worker->flags and adjust nr_running accordingly.
758
 *
759 760
 * CONTEXT:
 * spin_lock_irq(gcwq->lock)
761 762 763
 */
static inline void worker_clr_flags(struct worker *worker, unsigned int flags)
{
764 765 766
	struct global_cwq *gcwq = worker->gcwq;
	unsigned int oflags = worker->flags;

767 768
	WARN_ON_ONCE(worker->task != current);

769
	worker->flags &= ~flags;
770 771 772 773 774

	/* if transitioning out of NOT_RUNNING, increment nr_running */
	if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING))
		if (!(worker->flags & WORKER_NOT_RUNNING))
			atomic_inc(get_gcwq_nr_running(gcwq->cpu));
775 776
}

T
Tejun Heo 已提交
777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803
/**
 * busy_worker_head - return the busy hash head for a work
 * @gcwq: gcwq of interest
 * @work: work to be hashed
 *
 * Return hash head of @gcwq for @work.
 *
 * CONTEXT:
 * spin_lock_irq(gcwq->lock).
 *
 * RETURNS:
 * Pointer to the hash head.
 */
static struct hlist_head *busy_worker_head(struct global_cwq *gcwq,
					   struct work_struct *work)
{
	const int base_shift = ilog2(sizeof(struct work_struct));
	unsigned long v = (unsigned long)work;

	/* simple shift and fold hash, do we need something better? */
	v >>= base_shift;
	v += v >> BUSY_WORKER_HASH_ORDER;
	v &= BUSY_WORKER_HASH_MASK;

	return &gcwq->busy_hash[v];
}

804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848
/**
 * __find_worker_executing_work - find worker which is executing a work
 * @gcwq: gcwq of interest
 * @bwh: hash head as returned by busy_worker_head()
 * @work: work to find worker for
 *
 * Find a worker which is executing @work on @gcwq.  @bwh should be
 * the hash head obtained by calling busy_worker_head() with the same
 * work.
 *
 * CONTEXT:
 * spin_lock_irq(gcwq->lock).
 *
 * RETURNS:
 * Pointer to worker which is executing @work if found, NULL
 * otherwise.
 */
static struct worker *__find_worker_executing_work(struct global_cwq *gcwq,
						   struct hlist_head *bwh,
						   struct work_struct *work)
{
	struct worker *worker;
	struct hlist_node *tmp;

	hlist_for_each_entry(worker, tmp, bwh, hentry)
		if (worker->current_work == work)
			return worker;
	return NULL;
}

/**
 * find_worker_executing_work - find worker which is executing a work
 * @gcwq: gcwq of interest
 * @work: work to find worker for
 *
 * Find a worker which is executing @work on @gcwq.  This function is
 * identical to __find_worker_executing_work() except that this
 * function calculates @bwh itself.
 *
 * CONTEXT:
 * spin_lock_irq(gcwq->lock).
 *
 * RETURNS:
 * Pointer to worker which is executing @work if found, NULL
 * otherwise.
849
 */
850 851
static struct worker *find_worker_executing_work(struct global_cwq *gcwq,
						 struct work_struct *work)
852
{
853 854
	return __find_worker_executing_work(gcwq, busy_worker_head(gcwq, work),
					    work);
855 856
}

857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876
/**
 * gcwq_determine_ins_pos - find insertion position
 * @gcwq: gcwq of interest
 * @cwq: cwq a work is being queued for
 *
 * A work for @cwq is about to be queued on @gcwq, determine insertion
 * position for the work.  If @cwq is for HIGHPRI wq, the work is
 * queued at the head of the queue but in FIFO order with respect to
 * other HIGHPRI works; otherwise, at the end of the queue.  This
 * function also sets GCWQ_HIGHPRI_PENDING flag to hint @gcwq that
 * there are HIGHPRI works pending.
 *
 * CONTEXT:
 * spin_lock_irq(gcwq->lock).
 *
 * RETURNS:
 * Pointer to inserstion position.
 */
static inline struct list_head *gcwq_determine_ins_pos(struct global_cwq *gcwq,
					       struct cpu_workqueue_struct *cwq)
877
{
878 879 880 881 882 883 884 885 886 887 888 889 890 891
	struct work_struct *twork;

	if (likely(!(cwq->wq->flags & WQ_HIGHPRI)))
		return &gcwq->worklist;

	list_for_each_entry(twork, &gcwq->worklist, entry) {
		struct cpu_workqueue_struct *tcwq = get_work_cwq(twork);

		if (!(tcwq->wq->flags & WQ_HIGHPRI))
			break;
	}

	gcwq->flags |= GCWQ_HIGHPRI_PENDING;
	return &twork->entry;
892 893
}

T
Tejun Heo 已提交
894
/**
895
 * insert_work - insert a work into gcwq
T
Tejun Heo 已提交
896 897 898 899 900
 * @cwq: cwq @work belongs to
 * @work: work to insert
 * @head: insertion point
 * @extra_flags: extra WORK_STRUCT_* flags to set
 *
901 902
 * Insert @work which belongs to @cwq into @gcwq after @head.
 * @extra_flags is or'd to work_struct flags.
T
Tejun Heo 已提交
903 904
 *
 * CONTEXT:
905
 * spin_lock_irq(gcwq->lock).
T
Tejun Heo 已提交
906
 */
O
Oleg Nesterov 已提交
907
static void insert_work(struct cpu_workqueue_struct *cwq,
T
Tejun Heo 已提交
908 909
			struct work_struct *work, struct list_head *head,
			unsigned int extra_flags)
O
Oleg Nesterov 已提交
910
{
911 912
	struct global_cwq *gcwq = cwq->gcwq;

T
Tejun Heo 已提交
913
	/* we own @work, set data and link */
914
	set_work_cwq(work, cwq, extra_flags);
915

916 917 918 919 920
	/*
	 * Ensure that we get the right work->data if we see the
	 * result of list_add() below, see try_to_grab_pending().
	 */
	smp_wmb();
T
Tejun Heo 已提交
921

922
	list_add_tail(&work->entry, head);
923 924 925 926 927 928 929 930

	/*
	 * Ensure either worker_sched_deactivated() sees the above
	 * list_add_tail() or we see zero nr_running to avoid workers
	 * lying around lazily while there are works to be processed.
	 */
	smp_mb();

931
	if (__need_more_worker(gcwq))
932
		wake_up_worker(gcwq);
O
Oleg Nesterov 已提交
933 934
}

T
Tejun Heo 已提交
935
static void __queue_work(unsigned int cpu, struct workqueue_struct *wq,
L
Linus Torvalds 已提交
936 937
			 struct work_struct *work)
{
938 939
	struct global_cwq *gcwq;
	struct cpu_workqueue_struct *cwq;
940
	struct list_head *worklist;
941
	unsigned int work_flags;
L
Linus Torvalds 已提交
942 943
	unsigned long flags;

944
	debug_work_activate(work);
945

946 947 948
	if (WARN_ON_ONCE(wq->flags & WQ_DYING))
		return;

949 950
	/* determine gcwq to use */
	if (!(wq->flags & WQ_UNBOUND)) {
951 952
		struct global_cwq *last_gcwq;

953 954 955
		if (unlikely(cpu == WORK_CPU_UNBOUND))
			cpu = raw_smp_processor_id();

956 957 958 959 960 961
		/*
		 * It's multi cpu.  If @wq is non-reentrant and @work
		 * was previously on a different cpu, it might still
		 * be running there, in which case the work needs to
		 * be queued on that cpu to guarantee non-reentrance.
		 */
962
		gcwq = get_gcwq(cpu);
963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979
		if (wq->flags & WQ_NON_REENTRANT &&
		    (last_gcwq = get_work_gcwq(work)) && last_gcwq != gcwq) {
			struct worker *worker;

			spin_lock_irqsave(&last_gcwq->lock, flags);

			worker = find_worker_executing_work(last_gcwq, work);

			if (worker && worker->current_cwq->wq == wq)
				gcwq = last_gcwq;
			else {
				/* meh... not running there, queue here */
				spin_unlock_irqrestore(&last_gcwq->lock, flags);
				spin_lock_irqsave(&gcwq->lock, flags);
			}
		} else
			spin_lock_irqsave(&gcwq->lock, flags);
980 981 982
	} else {
		gcwq = get_gcwq(WORK_CPU_UNBOUND);
		spin_lock_irqsave(&gcwq->lock, flags);
983 984 985 986
	}

	/* gcwq determined, get cwq and queue */
	cwq = get_cwq(gcwq->cpu, wq);
987
	trace_workqueue_queue_work(cpu, cwq, work);
988

T
Tejun Heo 已提交
989
	BUG_ON(!list_empty(&work->entry));
990

991
	cwq->nr_in_flight[cwq->work_color]++;
992
	work_flags = work_color_to_flags(cwq->work_color);
993 994

	if (likely(cwq->nr_active < cwq->max_active)) {
995
		trace_workqueue_activate_work(work);
996
		cwq->nr_active++;
997
		worklist = gcwq_determine_ins_pos(gcwq, cwq);
998 999
	} else {
		work_flags |= WORK_STRUCT_DELAYED;
1000
		worklist = &cwq->delayed_works;
1001
	}
1002

1003
	insert_work(cwq, work, worklist, work_flags);
1004

1005
	spin_unlock_irqrestore(&gcwq->lock, flags);
L
Linus Torvalds 已提交
1006 1007
}

1008 1009 1010 1011 1012
/**
 * queue_work - queue work on a workqueue
 * @wq: workqueue to use
 * @work: work to queue
 *
A
Alan Stern 已提交
1013
 * Returns 0 if @work was already on a queue, non-zero otherwise.
L
Linus Torvalds 已提交
1014
 *
1015 1016
 * We queue the work to the CPU on which it was submitted, but if the CPU dies
 * it can be processed by another CPU.
L
Linus Torvalds 已提交
1017
 */
1018
int queue_work(struct workqueue_struct *wq, struct work_struct *work)
L
Linus Torvalds 已提交
1019
{
1020 1021 1022 1023 1024
	int ret;

	ret = queue_work_on(get_cpu(), wq, work);
	put_cpu();

L
Linus Torvalds 已提交
1025 1026
	return ret;
}
1027
EXPORT_SYMBOL_GPL(queue_work);
L
Linus Torvalds 已提交
1028

1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044
/**
 * queue_work_on - queue work on specific cpu
 * @cpu: CPU number to execute work on
 * @wq: workqueue to use
 * @work: work to queue
 *
 * Returns 0 if @work was already on a queue, non-zero otherwise.
 *
 * We queue the work to a specific CPU, the caller must ensure it
 * can't go away.
 */
int
queue_work_on(int cpu, struct workqueue_struct *wq, struct work_struct *work)
{
	int ret = 0;

1045
	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
T
Tejun Heo 已提交
1046
		__queue_work(cpu, wq, work);
1047 1048 1049 1050 1051 1052
		ret = 1;
	}
	return ret;
}
EXPORT_SYMBOL_GPL(queue_work_on);

1053
static void delayed_work_timer_fn(unsigned long __data)
L
Linus Torvalds 已提交
1054
{
1055
	struct delayed_work *dwork = (struct delayed_work *)__data;
1056
	struct cpu_workqueue_struct *cwq = get_work_cwq(&dwork->work);
L
Linus Torvalds 已提交
1057

T
Tejun Heo 已提交
1058
	__queue_work(smp_processor_id(), cwq->wq, &dwork->work);
L
Linus Torvalds 已提交
1059 1060
}

1061 1062 1063
/**
 * queue_delayed_work - queue work on a workqueue after delay
 * @wq: workqueue to use
1064
 * @dwork: delayable work to queue
1065 1066
 * @delay: number of jiffies to wait before queueing
 *
A
Alan Stern 已提交
1067
 * Returns 0 if @work was already on a queue, non-zero otherwise.
1068
 */
1069
int queue_delayed_work(struct workqueue_struct *wq,
1070
			struct delayed_work *dwork, unsigned long delay)
L
Linus Torvalds 已提交
1071
{
1072
	if (delay == 0)
1073
		return queue_work(wq, &dwork->work);
L
Linus Torvalds 已提交
1074

1075
	return queue_delayed_work_on(-1, wq, dwork, delay);
L
Linus Torvalds 已提交
1076
}
1077
EXPORT_SYMBOL_GPL(queue_delayed_work);
L
Linus Torvalds 已提交
1078

1079 1080 1081 1082
/**
 * queue_delayed_work_on - queue work on specific CPU after delay
 * @cpu: CPU number to execute work on
 * @wq: workqueue to use
1083
 * @dwork: work to queue
1084 1085
 * @delay: number of jiffies to wait before queueing
 *
A
Alan Stern 已提交
1086
 * Returns 0 if @work was already on a queue, non-zero otherwise.
1087
 */
1088
int queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
1089
			struct delayed_work *dwork, unsigned long delay)
1090 1091
{
	int ret = 0;
1092 1093
	struct timer_list *timer = &dwork->timer;
	struct work_struct *work = &dwork->work;
1094

1095
	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
1096
		unsigned int lcpu;
1097

1098 1099 1100
		BUG_ON(timer_pending(timer));
		BUG_ON(!list_empty(&work->entry));

1101 1102
		timer_stats_timer_set_start_info(&dwork->timer);

1103 1104 1105 1106 1107
		/*
		 * This stores cwq for the moment, for the timer_fn.
		 * Note that the work's gcwq is preserved to allow
		 * reentrance detection for delayed works.
		 */
1108 1109 1110 1111 1112 1113 1114 1115 1116 1117
		if (!(wq->flags & WQ_UNBOUND)) {
			struct global_cwq *gcwq = get_work_gcwq(work);

			if (gcwq && gcwq->cpu != WORK_CPU_UNBOUND)
				lcpu = gcwq->cpu;
			else
				lcpu = raw_smp_processor_id();
		} else
			lcpu = WORK_CPU_UNBOUND;

1118
		set_work_cwq(work, get_cwq(lcpu, wq), 0);
1119

1120
		timer->expires = jiffies + delay;
1121
		timer->data = (unsigned long)dwork;
1122
		timer->function = delayed_work_timer_fn;
1123 1124 1125 1126 1127

		if (unlikely(cpu >= 0))
			add_timer_on(timer, cpu);
		else
			add_timer(timer);
1128 1129 1130 1131
		ret = 1;
	}
	return ret;
}
1132
EXPORT_SYMBOL_GPL(queue_delayed_work_on);
L
Linus Torvalds 已提交
1133

T
Tejun Heo 已提交
1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144
/**
 * worker_enter_idle - enter idle state
 * @worker: worker which is entering idle state
 *
 * @worker is entering idle state.  Update stats and idle timer if
 * necessary.
 *
 * LOCKING:
 * spin_lock_irq(gcwq->lock).
 */
static void worker_enter_idle(struct worker *worker)
L
Linus Torvalds 已提交
1145
{
T
Tejun Heo 已提交
1146 1147 1148 1149 1150 1151
	struct global_cwq *gcwq = worker->gcwq;

	BUG_ON(worker->flags & WORKER_IDLE);
	BUG_ON(!list_empty(&worker->entry) &&
	       (worker->hentry.next || worker->hentry.pprev));

1152 1153
	/* can't use worker_set_flags(), also called from start_worker() */
	worker->flags |= WORKER_IDLE;
T
Tejun Heo 已提交
1154
	gcwq->nr_idle++;
1155
	worker->last_active = jiffies;
T
Tejun Heo 已提交
1156 1157 1158

	/* idle_list is LIFO */
	list_add(&worker->entry, &gcwq->idle_list);
1159

1160 1161 1162 1163 1164
	if (likely(!(worker->flags & WORKER_ROGUE))) {
		if (too_many_workers(gcwq) && !timer_pending(&gcwq->idle_timer))
			mod_timer(&gcwq->idle_timer,
				  jiffies + IDLE_WORKER_TIMEOUT);
	} else
1165
		wake_up_all(&gcwq->trustee_wait);
1166 1167 1168 1169

	/* sanity check nr_running */
	WARN_ON_ONCE(gcwq->nr_workers == gcwq->nr_idle &&
		     atomic_read(get_gcwq_nr_running(gcwq->cpu)));
T
Tejun Heo 已提交
1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185
}

/**
 * worker_leave_idle - leave idle state
 * @worker: worker which is leaving idle state
 *
 * @worker is leaving idle state.  Update stats.
 *
 * LOCKING:
 * spin_lock_irq(gcwq->lock).
 */
static void worker_leave_idle(struct worker *worker)
{
	struct global_cwq *gcwq = worker->gcwq;

	BUG_ON(!(worker->flags & WORKER_IDLE));
1186
	worker_clr_flags(worker, WORKER_IDLE);
T
Tejun Heo 已提交
1187 1188 1189 1190
	gcwq->nr_idle--;
	list_del_init(&worker->entry);
}

1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221
/**
 * worker_maybe_bind_and_lock - bind worker to its cpu if possible and lock gcwq
 * @worker: self
 *
 * Works which are scheduled while the cpu is online must at least be
 * scheduled to a worker which is bound to the cpu so that if they are
 * flushed from cpu callbacks while cpu is going down, they are
 * guaranteed to execute on the cpu.
 *
 * This function is to be used by rogue workers and rescuers to bind
 * themselves to the target cpu and may race with cpu going down or
 * coming online.  kthread_bind() can't be used because it may put the
 * worker to already dead cpu and set_cpus_allowed_ptr() can't be used
 * verbatim as it's best effort and blocking and gcwq may be
 * [dis]associated in the meantime.
 *
 * This function tries set_cpus_allowed() and locks gcwq and verifies
 * the binding against GCWQ_DISASSOCIATED which is set during
 * CPU_DYING and cleared during CPU_ONLINE, so if the worker enters
 * idle state or fetches works without dropping lock, it can guarantee
 * the scheduling requirement described in the first paragraph.
 *
 * CONTEXT:
 * Might sleep.  Called without any lock but returns with gcwq->lock
 * held.
 *
 * RETURNS:
 * %true if the associated gcwq is online (@worker is successfully
 * bound), %false if offline.
 */
static bool worker_maybe_bind_and_lock(struct worker *worker)
1222
__acquires(&gcwq->lock)
1223 1224 1225 1226 1227
{
	struct global_cwq *gcwq = worker->gcwq;
	struct task_struct *task = worker->task;

	while (true) {
1228
		/*
1229 1230 1231 1232
		 * The following call may fail, succeed or succeed
		 * without actually migrating the task to the cpu if
		 * it races with cpu hotunplug operation.  Verify
		 * against GCWQ_DISASSOCIATED.
1233
		 */
1234 1235
		if (!(gcwq->flags & GCWQ_DISASSOCIATED))
			set_cpus_allowed_ptr(task, get_cpu_mask(gcwq->cpu));
1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267

		spin_lock_irq(&gcwq->lock);
		if (gcwq->flags & GCWQ_DISASSOCIATED)
			return false;
		if (task_cpu(task) == gcwq->cpu &&
		    cpumask_equal(&current->cpus_allowed,
				  get_cpu_mask(gcwq->cpu)))
			return true;
		spin_unlock_irq(&gcwq->lock);

		/* CPU has come up inbetween, retry migration */
		cpu_relax();
	}
}

/*
 * Function for worker->rebind_work used to rebind rogue busy workers
 * to the associated cpu which is coming back online.  This is
 * scheduled by cpu up but can race with other cpu hotplug operations
 * and may be executed twice without intervening cpu down.
 */
static void worker_rebind_fn(struct work_struct *work)
{
	struct worker *worker = container_of(work, struct worker, rebind_work);
	struct global_cwq *gcwq = worker->gcwq;

	if (worker_maybe_bind_and_lock(worker))
		worker_clr_flags(worker, WORKER_REBIND);

	spin_unlock_irq(&gcwq->lock);
}

T
Tejun Heo 已提交
1268 1269 1270 1271 1272
static struct worker *alloc_worker(void)
{
	struct worker *worker;

	worker = kzalloc(sizeof(*worker), GFP_KERNEL);
T
Tejun Heo 已提交
1273 1274
	if (worker) {
		INIT_LIST_HEAD(&worker->entry);
1275
		INIT_LIST_HEAD(&worker->scheduled);
1276 1277 1278
		INIT_WORK(&worker->rebind_work, worker_rebind_fn);
		/* on creation a worker is in !idle && prep state */
		worker->flags = WORKER_PREP;
T
Tejun Heo 已提交
1279
	}
T
Tejun Heo 已提交
1280 1281 1282 1283 1284
	return worker;
}

/**
 * create_worker - create a new workqueue worker
1285
 * @gcwq: gcwq the new worker will belong to
T
Tejun Heo 已提交
1286 1287
 * @bind: whether to set affinity to @cpu or not
 *
1288
 * Create a new worker which is bound to @gcwq.  The returned worker
T
Tejun Heo 已提交
1289 1290 1291 1292 1293 1294 1295 1296 1297
 * can be started by calling start_worker() or destroyed using
 * destroy_worker().
 *
 * CONTEXT:
 * Might sleep.  Does GFP_KERNEL allocations.
 *
 * RETURNS:
 * Pointer to the newly created worker.
 */
1298
static struct worker *create_worker(struct global_cwq *gcwq, bool bind)
T
Tejun Heo 已提交
1299
{
1300
	bool on_unbound_cpu = gcwq->cpu == WORK_CPU_UNBOUND;
T
Tejun Heo 已提交
1301
	struct worker *worker = NULL;
1302
	int id = -1;
T
Tejun Heo 已提交
1303

1304 1305 1306 1307
	spin_lock_irq(&gcwq->lock);
	while (ida_get_new(&gcwq->worker_ida, &id)) {
		spin_unlock_irq(&gcwq->lock);
		if (!ida_pre_get(&gcwq->worker_ida, GFP_KERNEL))
T
Tejun Heo 已提交
1308
			goto fail;
1309
		spin_lock_irq(&gcwq->lock);
T
Tejun Heo 已提交
1310
	}
1311
	spin_unlock_irq(&gcwq->lock);
T
Tejun Heo 已提交
1312 1313 1314 1315 1316

	worker = alloc_worker();
	if (!worker)
		goto fail;

1317
	worker->gcwq = gcwq;
T
Tejun Heo 已提交
1318 1319
	worker->id = id;

1320 1321 1322 1323 1324 1325
	if (!on_unbound_cpu)
		worker->task = kthread_create(worker_thread, worker,
					      "kworker/%u:%d", gcwq->cpu, id);
	else
		worker->task = kthread_create(worker_thread, worker,
					      "kworker/u:%d", id);
T
Tejun Heo 已提交
1326 1327 1328
	if (IS_ERR(worker->task))
		goto fail;

1329 1330 1331 1332 1333
	/*
	 * A rogue worker will become a regular one if CPU comes
	 * online later on.  Make sure every worker has
	 * PF_THREAD_BOUND set.
	 */
1334
	if (bind && !on_unbound_cpu)
1335
		kthread_bind(worker->task, gcwq->cpu);
1336
	else {
1337
		worker->task->flags |= PF_THREAD_BOUND;
1338 1339 1340
		if (on_unbound_cpu)
			worker->flags |= WORKER_UNBOUND;
	}
T
Tejun Heo 已提交
1341 1342 1343 1344

	return worker;
fail:
	if (id >= 0) {
1345 1346 1347
		spin_lock_irq(&gcwq->lock);
		ida_remove(&gcwq->worker_ida, id);
		spin_unlock_irq(&gcwq->lock);
T
Tejun Heo 已提交
1348 1349 1350 1351 1352 1353 1354 1355 1356
	}
	kfree(worker);
	return NULL;
}

/**
 * start_worker - start a newly created worker
 * @worker: worker to start
 *
T
Tejun Heo 已提交
1357
 * Make the gcwq aware of @worker and start it.
T
Tejun Heo 已提交
1358 1359
 *
 * CONTEXT:
1360
 * spin_lock_irq(gcwq->lock).
T
Tejun Heo 已提交
1361 1362 1363
 */
static void start_worker(struct worker *worker)
{
1364
	worker->flags |= WORKER_STARTED;
T
Tejun Heo 已提交
1365 1366
	worker->gcwq->nr_workers++;
	worker_enter_idle(worker);
T
Tejun Heo 已提交
1367 1368 1369 1370 1371 1372 1373
	wake_up_process(worker->task);
}

/**
 * destroy_worker - destroy a workqueue worker
 * @worker: worker to be destroyed
 *
T
Tejun Heo 已提交
1374 1375 1376 1377
 * Destroy @worker and adjust @gcwq stats accordingly.
 *
 * CONTEXT:
 * spin_lock_irq(gcwq->lock) which is released and regrabbed.
T
Tejun Heo 已提交
1378 1379 1380
 */
static void destroy_worker(struct worker *worker)
{
1381
	struct global_cwq *gcwq = worker->gcwq;
T
Tejun Heo 已提交
1382 1383 1384 1385
	int id = worker->id;

	/* sanity check frenzy */
	BUG_ON(worker->current_work);
1386
	BUG_ON(!list_empty(&worker->scheduled));
T
Tejun Heo 已提交
1387

T
Tejun Heo 已提交
1388 1389 1390 1391 1392 1393
	if (worker->flags & WORKER_STARTED)
		gcwq->nr_workers--;
	if (worker->flags & WORKER_IDLE)
		gcwq->nr_idle--;

	list_del_init(&worker->entry);
1394
	worker->flags |= WORKER_DIE;
T
Tejun Heo 已提交
1395 1396 1397

	spin_unlock_irq(&gcwq->lock);

T
Tejun Heo 已提交
1398 1399 1400
	kthread_stop(worker->task);
	kfree(worker);

1401 1402
	spin_lock_irq(&gcwq->lock);
	ida_remove(&gcwq->worker_ida, id);
T
Tejun Heo 已提交
1403 1404
}

1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424
static void idle_worker_timeout(unsigned long __gcwq)
{
	struct global_cwq *gcwq = (void *)__gcwq;

	spin_lock_irq(&gcwq->lock);

	if (too_many_workers(gcwq)) {
		struct worker *worker;
		unsigned long expires;

		/* idle_list is kept in LIFO order, check the last one */
		worker = list_entry(gcwq->idle_list.prev, struct worker, entry);
		expires = worker->last_active + IDLE_WORKER_TIMEOUT;

		if (time_before(jiffies, expires))
			mod_timer(&gcwq->idle_timer, expires);
		else {
			/* it's been idle for too long, wake up manager */
			gcwq->flags |= GCWQ_MANAGE_WORKERS;
			wake_up_worker(gcwq);
1425
		}
1426 1427 1428 1429
	}

	spin_unlock_irq(&gcwq->lock);
}
1430

1431 1432 1433 1434
static bool send_mayday(struct work_struct *work)
{
	struct cpu_workqueue_struct *cwq = get_work_cwq(work);
	struct workqueue_struct *wq = cwq->wq;
1435
	unsigned int cpu;
1436 1437 1438 1439 1440

	if (!(wq->flags & WQ_RESCUER))
		return false;

	/* mayday mayday mayday */
1441 1442 1443 1444
	cpu = cwq->gcwq->cpu;
	/* WORK_CPU_UNBOUND can't be set in cpumask, use cpu 0 instead */
	if (cpu == WORK_CPU_UNBOUND)
		cpu = 0;
1445
	if (!mayday_test_and_set_cpu(cpu, wq->mayday_mask))
1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465
		wake_up_process(wq->rescuer->task);
	return true;
}

static void gcwq_mayday_timeout(unsigned long __gcwq)
{
	struct global_cwq *gcwq = (void *)__gcwq;
	struct work_struct *work;

	spin_lock_irq(&gcwq->lock);

	if (need_to_create_worker(gcwq)) {
		/*
		 * We've been trying to create a new worker but
		 * haven't been successful.  We might be hitting an
		 * allocation deadlock.  Send distress signals to
		 * rescuers.
		 */
		list_for_each_entry(work, &gcwq->worklist, entry)
			send_mayday(work);
L
Linus Torvalds 已提交
1466
	}
1467 1468 1469 1470

	spin_unlock_irq(&gcwq->lock);

	mod_timer(&gcwq->mayday_timer, jiffies + MAYDAY_INTERVAL);
L
Linus Torvalds 已提交
1471 1472
}

1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495
/**
 * maybe_create_worker - create a new worker if necessary
 * @gcwq: gcwq to create a new worker for
 *
 * Create a new worker for @gcwq if necessary.  @gcwq is guaranteed to
 * have at least one idle worker on return from this function.  If
 * creating a new worker takes longer than MAYDAY_INTERVAL, mayday is
 * sent to all rescuers with works scheduled on @gcwq to resolve
 * possible allocation deadlock.
 *
 * On return, need_to_create_worker() is guaranteed to be false and
 * may_start_working() true.
 *
 * LOCKING:
 * spin_lock_irq(gcwq->lock) which may be released and regrabbed
 * multiple times.  Does GFP_KERNEL allocations.  Called only from
 * manager.
 *
 * RETURNS:
 * false if no action was taken and gcwq->lock stayed locked, true
 * otherwise.
 */
static bool maybe_create_worker(struct global_cwq *gcwq)
1496 1497
__releases(&gcwq->lock)
__acquires(&gcwq->lock)
L
Linus Torvalds 已提交
1498
{
1499 1500 1501
	if (!need_to_create_worker(gcwq))
		return false;
restart:
1502 1503
	spin_unlock_irq(&gcwq->lock);

1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520
	/* if we don't make progress in MAYDAY_INITIAL_TIMEOUT, call for help */
	mod_timer(&gcwq->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT);

	while (true) {
		struct worker *worker;

		worker = create_worker(gcwq, true);
		if (worker) {
			del_timer_sync(&gcwq->mayday_timer);
			spin_lock_irq(&gcwq->lock);
			start_worker(worker);
			BUG_ON(need_to_create_worker(gcwq));
			return true;
		}

		if (!need_to_create_worker(gcwq))
			break;
L
Linus Torvalds 已提交
1521

1522 1523
		__set_current_state(TASK_INTERRUPTIBLE);
		schedule_timeout(CREATE_COOLDOWN);
1524

1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553
		if (!need_to_create_worker(gcwq))
			break;
	}

	del_timer_sync(&gcwq->mayday_timer);
	spin_lock_irq(&gcwq->lock);
	if (need_to_create_worker(gcwq))
		goto restart;
	return true;
}

/**
 * maybe_destroy_worker - destroy workers which have been idle for a while
 * @gcwq: gcwq to destroy workers for
 *
 * Destroy @gcwq workers which have been idle for longer than
 * IDLE_WORKER_TIMEOUT.
 *
 * LOCKING:
 * spin_lock_irq(gcwq->lock) which may be released and regrabbed
 * multiple times.  Called only from manager.
 *
 * RETURNS:
 * false if no action was taken and gcwq->lock stayed locked, true
 * otherwise.
 */
static bool maybe_destroy_workers(struct global_cwq *gcwq)
{
	bool ret = false;
L
Linus Torvalds 已提交
1554

1555 1556 1557
	while (too_many_workers(gcwq)) {
		struct worker *worker;
		unsigned long expires;
1558

1559 1560
		worker = list_entry(gcwq->idle_list.prev, struct worker, entry);
		expires = worker->last_active + IDLE_WORKER_TIMEOUT;
1561

1562 1563
		if (time_before(jiffies, expires)) {
			mod_timer(&gcwq->idle_timer, expires);
1564
			break;
1565
		}
L
Linus Torvalds 已提交
1566

1567 1568
		destroy_worker(worker);
		ret = true;
L
Linus Torvalds 已提交
1569
	}
1570

1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623
	return ret;
}

/**
 * manage_workers - manage worker pool
 * @worker: self
 *
 * Assume the manager role and manage gcwq worker pool @worker belongs
 * to.  At any given time, there can be only zero or one manager per
 * gcwq.  The exclusion is handled automatically by this function.
 *
 * The caller can safely start processing works on false return.  On
 * true return, it's guaranteed that need_to_create_worker() is false
 * and may_start_working() is true.
 *
 * CONTEXT:
 * spin_lock_irq(gcwq->lock) which may be released and regrabbed
 * multiple times.  Does GFP_KERNEL allocations.
 *
 * RETURNS:
 * false if no action was taken and gcwq->lock stayed locked, true if
 * some action was taken.
 */
static bool manage_workers(struct worker *worker)
{
	struct global_cwq *gcwq = worker->gcwq;
	bool ret = false;

	if (gcwq->flags & GCWQ_MANAGING_WORKERS)
		return ret;

	gcwq->flags &= ~GCWQ_MANAGE_WORKERS;
	gcwq->flags |= GCWQ_MANAGING_WORKERS;

	/*
	 * Destroy and then create so that may_start_working() is true
	 * on return.
	 */
	ret |= maybe_destroy_workers(gcwq);
	ret |= maybe_create_worker(gcwq);

	gcwq->flags &= ~GCWQ_MANAGING_WORKERS;

	/*
	 * The trustee might be waiting to take over the manager
	 * position, tell it we're done.
	 */
	if (unlikely(gcwq->trustee))
		wake_up_all(&gcwq->trustee_wait);

	return ret;
}

1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638
/**
 * move_linked_works - move linked works to a list
 * @work: start of series of works to be scheduled
 * @head: target list to append @work to
 * @nextp: out paramter for nested worklist walking
 *
 * Schedule linked works starting from @work to @head.  Work series to
 * be scheduled starts at @work and includes any consecutive work with
 * WORK_STRUCT_LINKED set in its predecessor.
 *
 * If @nextp is not NULL, it's updated to point to the next work of
 * the last scheduled work.  This allows move_linked_works() to be
 * nested inside outer list_for_each_entry_safe().
 *
 * CONTEXT:
1639
 * spin_lock_irq(gcwq->lock).
1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664
 */
static void move_linked_works(struct work_struct *work, struct list_head *head,
			      struct work_struct **nextp)
{
	struct work_struct *n;

	/*
	 * Linked worklist will always end before the end of the list,
	 * use NULL for list head.
	 */
	list_for_each_entry_safe_from(work, n, NULL, entry) {
		list_move_tail(&work->entry, head);
		if (!(*work_data_bits(work) & WORK_STRUCT_LINKED))
			break;
	}

	/*
	 * If we're already inside safe list traversal and have moved
	 * multiple works to the scheduled queue, the next position
	 * needs to be updated.
	 */
	if (nextp)
		*nextp = n;
}

1665 1666 1667 1668
static void cwq_activate_first_delayed(struct cpu_workqueue_struct *cwq)
{
	struct work_struct *work = list_first_entry(&cwq->delayed_works,
						    struct work_struct, entry);
1669
	struct list_head *pos = gcwq_determine_ins_pos(cwq->gcwq, cwq);
1670

1671
	trace_workqueue_activate_work(work);
1672
	move_linked_works(work, pos, NULL);
1673
	__clear_bit(WORK_STRUCT_DELAYED_BIT, work_data_bits(work));
1674 1675 1676
	cwq->nr_active++;
}

1677 1678 1679 1680
/**
 * cwq_dec_nr_in_flight - decrement cwq's nr_in_flight
 * @cwq: cwq of interest
 * @color: color of work which left the queue
1681
 * @delayed: for a delayed work
1682 1683 1684 1685 1686
 *
 * A work either has completed or is removed from pending queue,
 * decrement nr_in_flight of its cwq and handle workqueue flushing.
 *
 * CONTEXT:
1687
 * spin_lock_irq(gcwq->lock).
1688
 */
1689 1690
static void cwq_dec_nr_in_flight(struct cpu_workqueue_struct *cwq, int color,
				 bool delayed)
1691 1692 1693 1694 1695 1696
{
	/* ignore uncolored works */
	if (color == WORK_NO_COLOR)
		return;

	cwq->nr_in_flight[color]--;
1697

1698 1699 1700 1701 1702 1703 1704
	if (!delayed) {
		cwq->nr_active--;
		if (!list_empty(&cwq->delayed_works)) {
			/* one down, submit a delayed one */
			if (cwq->nr_active < cwq->max_active)
				cwq_activate_first_delayed(cwq);
		}
1705
	}
1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725

	/* is flush in progress and are we at the flushing tip? */
	if (likely(cwq->flush_color != color))
		return;

	/* are there still in-flight works? */
	if (cwq->nr_in_flight[color])
		return;

	/* this cwq is done, clear flush_color */
	cwq->flush_color = -1;

	/*
	 * If this was the last cwq, wake up the first flusher.  It
	 * will handle the rest.
	 */
	if (atomic_dec_and_test(&cwq->wq->nr_cwqs_to_flush))
		complete(&cwq->wq->first_flusher->done);
}

1726 1727
/**
 * process_one_work - process single work
T
Tejun Heo 已提交
1728
 * @worker: self
1729 1730 1731 1732 1733 1734 1735 1736 1737
 * @work: work to process
 *
 * Process @work.  This function contains all the logics necessary to
 * process a single work including synchronization against and
 * interaction with other workers on the same cpu, queueing and
 * flushing.  As long as context requirement is met, any worker can
 * call this function to process a work.
 *
 * CONTEXT:
1738
 * spin_lock_irq(gcwq->lock) which is released and regrabbed.
1739
 */
T
Tejun Heo 已提交
1740
static void process_one_work(struct worker *worker, struct work_struct *work)
1741 1742
__releases(&gcwq->lock)
__acquires(&gcwq->lock)
1743
{
1744
	struct cpu_workqueue_struct *cwq = get_work_cwq(work);
1745
	struct global_cwq *gcwq = cwq->gcwq;
T
Tejun Heo 已提交
1746
	struct hlist_head *bwh = busy_worker_head(gcwq, work);
1747
	bool cpu_intensive = cwq->wq->flags & WQ_CPU_INTENSIVE;
1748
	work_func_t f = work->func;
1749
	int work_color;
1750
	struct worker *collision;
1751 1752 1753 1754 1755 1756 1757 1758 1759 1760
#ifdef CONFIG_LOCKDEP
	/*
	 * It is permissible to free the struct work_struct from
	 * inside the function that is called from it, this we need to
	 * take into account for lockdep too.  To avoid bogus "held
	 * lock freed" warnings as well as problems when looking into
	 * work->lockdep_map, make a copy and use that here.
	 */
	struct lockdep_map lockdep_map = work->lockdep_map;
#endif
1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772
	/*
	 * A single work shouldn't be executed concurrently by
	 * multiple workers on a single cpu.  Check whether anyone is
	 * already processing the work.  If so, defer the work to the
	 * currently executing one.
	 */
	collision = __find_worker_executing_work(gcwq, bwh, work);
	if (unlikely(collision)) {
		move_linked_works(work, &collision->scheduled, NULL);
		return;
	}

1773 1774
	/* claim and process */
	debug_work_deactivate(work);
T
Tejun Heo 已提交
1775
	hlist_add_head(&worker->hentry, bwh);
T
Tejun Heo 已提交
1776
	worker->current_work = work;
1777
	worker->current_cwq = cwq;
1778
	work_color = get_work_color(work);
1779 1780 1781

	/* record the current cpu number in the work data and dequeue */
	set_work_cpu(work, gcwq->cpu);
1782 1783
	list_del_init(&work->entry);

1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798
	/*
	 * If HIGHPRI_PENDING, check the next work, and, if HIGHPRI,
	 * wake up another worker; otherwise, clear HIGHPRI_PENDING.
	 */
	if (unlikely(gcwq->flags & GCWQ_HIGHPRI_PENDING)) {
		struct work_struct *nwork = list_first_entry(&gcwq->worklist,
						struct work_struct, entry);

		if (!list_empty(&gcwq->worklist) &&
		    get_work_cwq(nwork)->wq->flags & WQ_HIGHPRI)
			wake_up_worker(gcwq);
		else
			gcwq->flags &= ~GCWQ_HIGHPRI_PENDING;
	}

1799 1800 1801 1802 1803 1804 1805
	/*
	 * CPU intensive works don't participate in concurrency
	 * management.  They're the scheduler's responsibility.
	 */
	if (unlikely(cpu_intensive))
		worker_set_flags(worker, WORKER_CPU_INTENSIVE, true);

1806
	spin_unlock_irq(&gcwq->lock);
1807 1808 1809 1810

	work_clear_pending(work);
	lock_map_acquire(&cwq->wq->lockdep_map);
	lock_map_acquire(&lockdep_map);
1811
	trace_workqueue_execute_start(work);
1812
	f(work);
1813 1814 1815 1816 1817
	/*
	 * While we must be careful to not use "work" after this, the trace
	 * point will only record its address.
	 */
	trace_workqueue_execute_end(work);
1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830
	lock_map_release(&lockdep_map);
	lock_map_release(&cwq->wq->lockdep_map);

	if (unlikely(in_atomic() || lockdep_depth(current) > 0)) {
		printk(KERN_ERR "BUG: workqueue leaked lock or atomic: "
		       "%s/0x%08x/%d\n",
		       current->comm, preempt_count(), task_pid_nr(current));
		printk(KERN_ERR "    last function: ");
		print_symbol("%s\n", (unsigned long)f);
		debug_show_held_locks(current);
		dump_stack();
	}

1831
	spin_lock_irq(&gcwq->lock);
1832

1833 1834 1835 1836
	/* clear cpu intensive status */
	if (unlikely(cpu_intensive))
		worker_clr_flags(worker, WORKER_CPU_INTENSIVE);

1837
	/* we're done with it, release */
T
Tejun Heo 已提交
1838
	hlist_del_init(&worker->hentry);
T
Tejun Heo 已提交
1839
	worker->current_work = NULL;
1840
	worker->current_cwq = NULL;
1841
	cwq_dec_nr_in_flight(cwq, work_color, false);
1842 1843
}

1844 1845 1846 1847 1848 1849 1850 1851 1852
/**
 * process_scheduled_works - process scheduled works
 * @worker: self
 *
 * Process all scheduled works.  Please note that the scheduled list
 * may change while processing a work, so this function repeatedly
 * fetches a work from the top and executes it.
 *
 * CONTEXT:
1853
 * spin_lock_irq(gcwq->lock) which may be released and regrabbed
1854 1855 1856
 * multiple times.
 */
static void process_scheduled_works(struct worker *worker)
L
Linus Torvalds 已提交
1857
{
1858 1859
	while (!list_empty(&worker->scheduled)) {
		struct work_struct *work = list_first_entry(&worker->scheduled,
L
Linus Torvalds 已提交
1860
						struct work_struct, entry);
T
Tejun Heo 已提交
1861
		process_one_work(worker, work);
L
Linus Torvalds 已提交
1862 1863 1864
	}
}

T
Tejun Heo 已提交
1865 1866
/**
 * worker_thread - the worker thread function
T
Tejun Heo 已提交
1867
 * @__worker: self
T
Tejun Heo 已提交
1868
 *
1869 1870 1871 1872 1873
 * The gcwq worker thread function.  There's a single dynamic pool of
 * these per each cpu.  These workers process all works regardless of
 * their specific target workqueue.  The only exception is works which
 * belong to workqueues with a rescuer which will be explained in
 * rescuer_thread().
T
Tejun Heo 已提交
1874
 */
T
Tejun Heo 已提交
1875
static int worker_thread(void *__worker)
L
Linus Torvalds 已提交
1876
{
T
Tejun Heo 已提交
1877
	struct worker *worker = __worker;
1878
	struct global_cwq *gcwq = worker->gcwq;
L
Linus Torvalds 已提交
1879

1880 1881
	/* tell the scheduler that this is a workqueue worker */
	worker->task->flags |= PF_WQ_WORKER;
T
Tejun Heo 已提交
1882 1883
woke_up:
	spin_lock_irq(&gcwq->lock);
L
Linus Torvalds 已提交
1884

T
Tejun Heo 已提交
1885 1886 1887
	/* DIE can be set only while we're idle, checking here is enough */
	if (worker->flags & WORKER_DIE) {
		spin_unlock_irq(&gcwq->lock);
1888
		worker->task->flags &= ~PF_WQ_WORKER;
T
Tejun Heo 已提交
1889 1890
		return 0;
	}
1891

T
Tejun Heo 已提交
1892
	worker_leave_idle(worker);
1893
recheck:
1894 1895 1896 1897 1898 1899 1900 1901
	/* no more worker necessary? */
	if (!need_more_worker(gcwq))
		goto sleep;

	/* do we need to manage? */
	if (unlikely(!may_start_working(gcwq)) && manage_workers(worker))
		goto recheck;

T
Tejun Heo 已提交
1902 1903 1904 1905 1906 1907 1908
	/*
	 * ->scheduled list can only be filled while a worker is
	 * preparing to process a work or actually processing it.
	 * Make sure nobody diddled with it while I was sleeping.
	 */
	BUG_ON(!list_empty(&worker->scheduled));

1909 1910 1911 1912 1913 1914 1915 1916
	/*
	 * When control reaches this point, we're guaranteed to have
	 * at least one idle worker or that someone else has already
	 * assumed the manager role.
	 */
	worker_clr_flags(worker, WORKER_PREP);

	do {
T
Tejun Heo 已提交
1917
		struct work_struct *work =
1918
			list_first_entry(&gcwq->worklist,
T
Tejun Heo 已提交
1919 1920 1921 1922 1923 1924
					 struct work_struct, entry);

		if (likely(!(*work_data_bits(work) & WORK_STRUCT_LINKED))) {
			/* optimization path, not strictly necessary */
			process_one_work(worker, work);
			if (unlikely(!list_empty(&worker->scheduled)))
1925
				process_scheduled_works(worker);
T
Tejun Heo 已提交
1926 1927 1928
		} else {
			move_linked_works(work, &worker->scheduled, NULL);
			process_scheduled_works(worker);
1929
		}
1930 1931 1932
	} while (keep_working(gcwq));

	worker_set_flags(worker, WORKER_PREP, false);
1933
sleep:
1934 1935
	if (unlikely(need_to_manage_workers(gcwq)) && manage_workers(worker))
		goto recheck;
1936

T
Tejun Heo 已提交
1937
	/*
1938 1939 1940 1941 1942
	 * gcwq->lock is held and there's no work to process and no
	 * need to manage, sleep.  Workers are woken up only while
	 * holding gcwq->lock or from local cpu, so setting the
	 * current state before releasing gcwq->lock is enough to
	 * prevent losing any event.
T
Tejun Heo 已提交
1943 1944 1945 1946 1947 1948
	 */
	worker_enter_idle(worker);
	__set_current_state(TASK_INTERRUPTIBLE);
	spin_unlock_irq(&gcwq->lock);
	schedule();
	goto woke_up;
L
Linus Torvalds 已提交
1949 1950
}

1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974
/**
 * rescuer_thread - the rescuer thread function
 * @__wq: the associated workqueue
 *
 * Workqueue rescuer thread function.  There's one rescuer for each
 * workqueue which has WQ_RESCUER set.
 *
 * Regular work processing on a gcwq may block trying to create a new
 * worker which uses GFP_KERNEL allocation which has slight chance of
 * developing into deadlock if some works currently on the same queue
 * need to be processed to satisfy the GFP_KERNEL allocation.  This is
 * the problem rescuer solves.
 *
 * When such condition is possible, the gcwq summons rescuers of all
 * workqueues which have works queued on the gcwq and let them process
 * those works so that forward progress can be guaranteed.
 *
 * This should happen rarely.
 */
static int rescuer_thread(void *__wq)
{
	struct workqueue_struct *wq = __wq;
	struct worker *rescuer = wq->rescuer;
	struct list_head *scheduled = &rescuer->scheduled;
1975
	bool is_unbound = wq->flags & WQ_UNBOUND;
1976 1977 1978 1979 1980 1981 1982 1983 1984
	unsigned int cpu;

	set_user_nice(current, RESCUER_NICE_LEVEL);
repeat:
	set_current_state(TASK_INTERRUPTIBLE);

	if (kthread_should_stop())
		return 0;

1985 1986 1987 1988
	/*
	 * See whether any cpu is asking for help.  Unbounded
	 * workqueues use cpu 0 in mayday_mask for CPU_UNBOUND.
	 */
1989
	for_each_mayday_cpu(cpu, wq->mayday_mask) {
1990 1991
		unsigned int tcpu = is_unbound ? WORK_CPU_UNBOUND : cpu;
		struct cpu_workqueue_struct *cwq = get_cwq(tcpu, wq);
1992 1993 1994 1995
		struct global_cwq *gcwq = cwq->gcwq;
		struct work_struct *work, *n;

		__set_current_state(TASK_RUNNING);
1996
		mayday_clear_cpu(cpu, wq->mayday_mask);
1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016

		/* migrate to the target cpu if possible */
		rescuer->gcwq = gcwq;
		worker_maybe_bind_and_lock(rescuer);

		/*
		 * Slurp in all works issued via this workqueue and
		 * process'em.
		 */
		BUG_ON(!list_empty(&rescuer->scheduled));
		list_for_each_entry_safe(work, n, &gcwq->worklist, entry)
			if (get_work_cwq(work) == cwq)
				move_linked_works(work, scheduled, &n);

		process_scheduled_works(rescuer);
		spin_unlock_irq(&gcwq->lock);
	}

	schedule();
	goto repeat;
L
Linus Torvalds 已提交
2017 2018
}

O
Oleg Nesterov 已提交
2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029
struct wq_barrier {
	struct work_struct	work;
	struct completion	done;
};

static void wq_barrier_func(struct work_struct *work)
{
	struct wq_barrier *barr = container_of(work, struct wq_barrier, work);
	complete(&barr->done);
}

T
Tejun Heo 已提交
2030 2031 2032 2033
/**
 * insert_wq_barrier - insert a barrier work
 * @cwq: cwq to insert barrier into
 * @barr: wq_barrier to insert
2034 2035
 * @target: target work to attach @barr to
 * @worker: worker currently executing @target, NULL if @target is not executing
T
Tejun Heo 已提交
2036
 *
2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049
 * @barr is linked to @target such that @barr is completed only after
 * @target finishes execution.  Please note that the ordering
 * guarantee is observed only with respect to @target and on the local
 * cpu.
 *
 * Currently, a queued barrier can't be canceled.  This is because
 * try_to_grab_pending() can't determine whether the work to be
 * grabbed is at the head of the queue and thus can't clear LINKED
 * flag of the previous work while there must be a valid next work
 * after a work with LINKED flag set.
 *
 * Note that when @worker is non-NULL, @target may be modified
 * underneath us, so we can't reliably determine cwq from @target.
T
Tejun Heo 已提交
2050 2051
 *
 * CONTEXT:
2052
 * spin_lock_irq(gcwq->lock).
T
Tejun Heo 已提交
2053
 */
2054
static void insert_wq_barrier(struct cpu_workqueue_struct *cwq,
2055 2056
			      struct wq_barrier *barr,
			      struct work_struct *target, struct worker *worker)
O
Oleg Nesterov 已提交
2057
{
2058 2059 2060
	struct list_head *head;
	unsigned int linked = 0;

2061
	/*
2062
	 * debugobject calls are safe here even with gcwq->lock locked
2063 2064 2065 2066
	 * as we know for sure that this will not trigger any of the
	 * checks and call back into the fixup functions where we
	 * might deadlock.
	 */
A
Andrew Morton 已提交
2067
	INIT_WORK_ONSTACK(&barr->work, wq_barrier_func);
2068
	__set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&barr->work));
O
Oleg Nesterov 已提交
2069
	init_completion(&barr->done);
2070

2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085
	/*
	 * If @target is currently being executed, schedule the
	 * barrier to the worker; otherwise, put it after @target.
	 */
	if (worker)
		head = worker->scheduled.next;
	else {
		unsigned long *bits = work_data_bits(target);

		head = target->entry.next;
		/* there can already be other linked works, inherit and set */
		linked = *bits & WORK_STRUCT_LINKED;
		__set_bit(WORK_STRUCT_LINKED_BIT, bits);
	}

2086
	debug_work_activate(&barr->work);
2087 2088
	insert_work(cwq, &barr->work, head,
		    work_color_to_flags(WORK_NO_COLOR) | linked);
O
Oleg Nesterov 已提交
2089 2090
}

2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123
/**
 * flush_workqueue_prep_cwqs - prepare cwqs for workqueue flushing
 * @wq: workqueue being flushed
 * @flush_color: new flush color, < 0 for no-op
 * @work_color: new work color, < 0 for no-op
 *
 * Prepare cwqs for workqueue flushing.
 *
 * If @flush_color is non-negative, flush_color on all cwqs should be
 * -1.  If no cwq has in-flight commands at the specified color, all
 * cwq->flush_color's stay at -1 and %false is returned.  If any cwq
 * has in flight commands, its cwq->flush_color is set to
 * @flush_color, @wq->nr_cwqs_to_flush is updated accordingly, cwq
 * wakeup logic is armed and %true is returned.
 *
 * The caller should have initialized @wq->first_flusher prior to
 * calling this function with non-negative @flush_color.  If
 * @flush_color is negative, no flush color update is done and %false
 * is returned.
 *
 * If @work_color is non-negative, all cwqs should have the same
 * work_color which is previous to @work_color and all will be
 * advanced to @work_color.
 *
 * CONTEXT:
 * mutex_lock(wq->flush_mutex).
 *
 * RETURNS:
 * %true if @flush_color >= 0 and there's something to flush.  %false
 * otherwise.
 */
static bool flush_workqueue_prep_cwqs(struct workqueue_struct *wq,
				      int flush_color, int work_color)
L
Linus Torvalds 已提交
2124
{
2125 2126
	bool wait = false;
	unsigned int cpu;
L
Linus Torvalds 已提交
2127

2128 2129 2130
	if (flush_color >= 0) {
		BUG_ON(atomic_read(&wq->nr_cwqs_to_flush));
		atomic_set(&wq->nr_cwqs_to_flush, 1);
L
Linus Torvalds 已提交
2131
	}
2132

2133
	for_each_cwq_cpu(cpu, wq) {
2134
		struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
2135
		struct global_cwq *gcwq = cwq->gcwq;
O
Oleg Nesterov 已提交
2136

2137
		spin_lock_irq(&gcwq->lock);
2138

2139 2140
		if (flush_color >= 0) {
			BUG_ON(cwq->flush_color != -1);
O
Oleg Nesterov 已提交
2141

2142 2143 2144 2145 2146 2147
			if (cwq->nr_in_flight[flush_color]) {
				cwq->flush_color = flush_color;
				atomic_inc(&wq->nr_cwqs_to_flush);
				wait = true;
			}
		}
L
Linus Torvalds 已提交
2148

2149 2150 2151 2152
		if (work_color >= 0) {
			BUG_ON(work_color != work_next_color(cwq->work_color));
			cwq->work_color = work_color;
		}
L
Linus Torvalds 已提交
2153

2154
		spin_unlock_irq(&gcwq->lock);
L
Linus Torvalds 已提交
2155
	}
2156

2157 2158
	if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_cwqs_to_flush))
		complete(&wq->first_flusher->done);
2159

2160
	return wait;
L
Linus Torvalds 已提交
2161 2162
}

2163
/**
L
Linus Torvalds 已提交
2164
 * flush_workqueue - ensure that any scheduled work has run to completion.
2165
 * @wq: workqueue to flush
L
Linus Torvalds 已提交
2166 2167 2168 2169
 *
 * Forces execution of the workqueue and blocks until its completion.
 * This is typically used in driver shutdown handlers.
 *
O
Oleg Nesterov 已提交
2170 2171
 * We sleep until all works which were queued on entry have been handled,
 * but we are not livelocked by new incoming ones.
L
Linus Torvalds 已提交
2172
 */
2173
void flush_workqueue(struct workqueue_struct *wq)
L
Linus Torvalds 已提交
2174
{
2175 2176 2177 2178 2179 2180
	struct wq_flusher this_flusher = {
		.list = LIST_HEAD_INIT(this_flusher.list),
		.flush_color = -1,
		.done = COMPLETION_INITIALIZER_ONSTACK(this_flusher.done),
	};
	int next_color;
L
Linus Torvalds 已提交
2181

2182 2183
	lock_map_acquire(&wq->lockdep_map);
	lock_map_release(&wq->lockdep_map);
2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244

	mutex_lock(&wq->flush_mutex);

	/*
	 * Start-to-wait phase
	 */
	next_color = work_next_color(wq->work_color);

	if (next_color != wq->flush_color) {
		/*
		 * Color space is not full.  The current work_color
		 * becomes our flush_color and work_color is advanced
		 * by one.
		 */
		BUG_ON(!list_empty(&wq->flusher_overflow));
		this_flusher.flush_color = wq->work_color;
		wq->work_color = next_color;

		if (!wq->first_flusher) {
			/* no flush in progress, become the first flusher */
			BUG_ON(wq->flush_color != this_flusher.flush_color);

			wq->first_flusher = &this_flusher;

			if (!flush_workqueue_prep_cwqs(wq, wq->flush_color,
						       wq->work_color)) {
				/* nothing to flush, done */
				wq->flush_color = next_color;
				wq->first_flusher = NULL;
				goto out_unlock;
			}
		} else {
			/* wait in queue */
			BUG_ON(wq->flush_color == this_flusher.flush_color);
			list_add_tail(&this_flusher.list, &wq->flusher_queue);
			flush_workqueue_prep_cwqs(wq, -1, wq->work_color);
		}
	} else {
		/*
		 * Oops, color space is full, wait on overflow queue.
		 * The next flush completion will assign us
		 * flush_color and transfer to flusher_queue.
		 */
		list_add_tail(&this_flusher.list, &wq->flusher_overflow);
	}

	mutex_unlock(&wq->flush_mutex);

	wait_for_completion(&this_flusher.done);

	/*
	 * Wake-up-and-cascade phase
	 *
	 * First flushers are responsible for cascading flushes and
	 * handling overflow.  Non-first flushers can simply return.
	 */
	if (wq->first_flusher != &this_flusher)
		return;

	mutex_lock(&wq->flush_mutex);

2245 2246 2247 2248
	/* we might have raced, check again with mutex held */
	if (wq->first_flusher != &this_flusher)
		goto out_unlock;

2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315
	wq->first_flusher = NULL;

	BUG_ON(!list_empty(&this_flusher.list));
	BUG_ON(wq->flush_color != this_flusher.flush_color);

	while (true) {
		struct wq_flusher *next, *tmp;

		/* complete all the flushers sharing the current flush color */
		list_for_each_entry_safe(next, tmp, &wq->flusher_queue, list) {
			if (next->flush_color != wq->flush_color)
				break;
			list_del_init(&next->list);
			complete(&next->done);
		}

		BUG_ON(!list_empty(&wq->flusher_overflow) &&
		       wq->flush_color != work_next_color(wq->work_color));

		/* this flush_color is finished, advance by one */
		wq->flush_color = work_next_color(wq->flush_color);

		/* one color has been freed, handle overflow queue */
		if (!list_empty(&wq->flusher_overflow)) {
			/*
			 * Assign the same color to all overflowed
			 * flushers, advance work_color and append to
			 * flusher_queue.  This is the start-to-wait
			 * phase for these overflowed flushers.
			 */
			list_for_each_entry(tmp, &wq->flusher_overflow, list)
				tmp->flush_color = wq->work_color;

			wq->work_color = work_next_color(wq->work_color);

			list_splice_tail_init(&wq->flusher_overflow,
					      &wq->flusher_queue);
			flush_workqueue_prep_cwqs(wq, -1, wq->work_color);
		}

		if (list_empty(&wq->flusher_queue)) {
			BUG_ON(wq->flush_color != wq->work_color);
			break;
		}

		/*
		 * Need to flush more colors.  Make the next flusher
		 * the new first flusher and arm cwqs.
		 */
		BUG_ON(wq->flush_color == wq->work_color);
		BUG_ON(wq->flush_color != next->flush_color);

		list_del_init(&next->list);
		wq->first_flusher = next;

		if (flush_workqueue_prep_cwqs(wq, wq->flush_color, -1))
			break;

		/*
		 * Meh... this color is already done, clear first
		 * flusher and repeat cascading.
		 */
		wq->first_flusher = NULL;
	}

out_unlock:
	mutex_unlock(&wq->flush_mutex);
L
Linus Torvalds 已提交
2316
}
2317
EXPORT_SYMBOL_GPL(flush_workqueue);
L
Linus Torvalds 已提交
2318

2319 2320
static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr,
			     bool wait_executing)
2321
{
2322
	struct worker *worker = NULL;
2323
	struct global_cwq *gcwq;
2324 2325 2326
	struct cpu_workqueue_struct *cwq;

	might_sleep();
2327 2328
	gcwq = get_work_gcwq(work);
	if (!gcwq)
2329
		return false;
2330

2331
	spin_lock_irq(&gcwq->lock);
2332 2333 2334
	if (!list_empty(&work->entry)) {
		/*
		 * See the comment near try_to_grab_pending()->smp_rmb().
2335 2336
		 * If it was re-queued to a different gcwq under us, we
		 * are not going to wait.
2337 2338
		 */
		smp_rmb();
2339 2340
		cwq = get_work_cwq(work);
		if (unlikely(!cwq || gcwq != cwq->gcwq))
T
Tejun Heo 已提交
2341
			goto already_gone;
2342
	} else if (wait_executing) {
2343
		worker = find_worker_executing_work(gcwq, work);
2344
		if (!worker)
T
Tejun Heo 已提交
2345
			goto already_gone;
2346
		cwq = worker->current_cwq;
2347 2348
	} else
		goto already_gone;
2349

2350
	insert_wq_barrier(cwq, barr, work, worker);
2351
	spin_unlock_irq(&gcwq->lock);
2352 2353 2354

	lock_map_acquire(&cwq->wq->lockdep_map);
	lock_map_release(&cwq->wq->lockdep_map);
2355
	return true;
T
Tejun Heo 已提交
2356
already_gone:
2357
	spin_unlock_irq(&gcwq->lock);
2358
	return false;
2359
}
2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389

/**
 * flush_work - wait for a work to finish executing the last queueing instance
 * @work: the work to flush
 *
 * Wait until @work has finished execution.  This function considers
 * only the last queueing instance of @work.  If @work has been
 * enqueued across different CPUs on a non-reentrant workqueue or on
 * multiple workqueues, @work might still be executing on return on
 * some of the CPUs from earlier queueing.
 *
 * If @work was queued only on a non-reentrant, ordered or unbound
 * workqueue, @work is guaranteed to be idle on return if it hasn't
 * been requeued since flush started.
 *
 * RETURNS:
 * %true if flush_work() waited for the work to finish execution,
 * %false if it was already idle.
 */
bool flush_work(struct work_struct *work)
{
	struct wq_barrier barr;

	if (start_flush_work(work, &barr, true)) {
		wait_for_completion(&barr.done);
		destroy_work_on_stack(&barr.work);
		return true;
	} else
		return false;
}
2390 2391
EXPORT_SYMBOL_GPL(flush_work);

2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427
static bool wait_on_cpu_work(struct global_cwq *gcwq, struct work_struct *work)
{
	struct wq_barrier barr;
	struct worker *worker;

	spin_lock_irq(&gcwq->lock);

	worker = find_worker_executing_work(gcwq, work);
	if (unlikely(worker))
		insert_wq_barrier(worker->current_cwq, &barr, work, worker);

	spin_unlock_irq(&gcwq->lock);

	if (unlikely(worker)) {
		wait_for_completion(&barr.done);
		destroy_work_on_stack(&barr.work);
		return true;
	} else
		return false;
}

static bool wait_on_work(struct work_struct *work)
{
	bool ret = false;
	int cpu;

	might_sleep();

	lock_map_acquire(&work->lockdep_map);
	lock_map_release(&work->lockdep_map);

	for_each_gcwq_cpu(cpu)
		ret |= wait_on_cpu_work(get_gcwq(cpu), work);
	return ret;
}

2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462
/**
 * flush_work_sync - wait until a work has finished execution
 * @work: the work to flush
 *
 * Wait until @work has finished execution.  On return, it's
 * guaranteed that all queueing instances of @work which happened
 * before this function is called are finished.  In other words, if
 * @work hasn't been requeued since this function was called, @work is
 * guaranteed to be idle on return.
 *
 * RETURNS:
 * %true if flush_work_sync() waited for the work to finish execution,
 * %false if it was already idle.
 */
bool flush_work_sync(struct work_struct *work)
{
	struct wq_barrier barr;
	bool pending, waited;

	/* we'll wait for executions separately, queue barr only if pending */
	pending = start_flush_work(work, &barr, false);

	/* wait for executions to finish */
	waited = wait_on_work(work);

	/* wait for the pending one */
	if (pending) {
		wait_for_completion(&barr.done);
		destroy_work_on_stack(&barr.work);
	}

	return pending || waited;
}
EXPORT_SYMBOL_GPL(flush_work_sync);

2463
/*
2464
 * Upon a successful return (>= 0), the caller "owns" WORK_STRUCT_PENDING bit,
2465 2466 2467 2468
 * so this work can't be re-armed in any way.
 */
static int try_to_grab_pending(struct work_struct *work)
{
2469
	struct global_cwq *gcwq;
2470
	int ret = -1;
2471

2472
	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)))
2473
		return 0;
2474 2475 2476 2477 2478

	/*
	 * The queueing is in progress, or it is already queued. Try to
	 * steal it from ->worklist without clearing WORK_STRUCT_PENDING.
	 */
2479 2480
	gcwq = get_work_gcwq(work);
	if (!gcwq)
2481 2482
		return ret;

2483
	spin_lock_irq(&gcwq->lock);
2484 2485
	if (!list_empty(&work->entry)) {
		/*
2486
		 * This work is queued, but perhaps we locked the wrong gcwq.
2487 2488 2489 2490
		 * In that case we must see the new value after rmb(), see
		 * insert_work()->wmb().
		 */
		smp_rmb();
2491
		if (gcwq == get_work_gcwq(work)) {
2492
			debug_work_deactivate(work);
2493
			list_del_init(&work->entry);
2494
			cwq_dec_nr_in_flight(get_work_cwq(work),
2495 2496
				get_work_color(work),
				*work_data_bits(work) & WORK_STRUCT_DELAYED);
2497 2498 2499
			ret = 1;
		}
	}
2500
	spin_unlock_irq(&gcwq->lock);
2501 2502 2503 2504

	return ret;
}

2505
static bool __cancel_work_timer(struct work_struct *work,
2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516
				struct timer_list* timer)
{
	int ret;

	do {
		ret = (timer && likely(del_timer(timer)));
		if (!ret)
			ret = try_to_grab_pending(work);
		wait_on_work(work);
	} while (unlikely(ret < 0));

2517
	clear_work_data(work);
2518 2519 2520
	return ret;
}

2521
/**
2522 2523
 * cancel_work_sync - cancel a work and wait for it to finish
 * @work: the work to cancel
2524
 *
2525 2526 2527 2528
 * Cancel @work and wait for its execution to finish.  This function
 * can be used even if the work re-queues itself or migrates to
 * another workqueue.  On return from this function, @work is
 * guaranteed to be not pending or executing on any CPU.
2529
 *
2530 2531
 * cancel_work_sync(&delayed_work->work) must not be used for
 * delayed_work's.  Use cancel_delayed_work_sync() instead.
2532
 *
2533
 * The caller must ensure that the workqueue on which @work was last
2534
 * queued can't be destroyed before this function returns.
2535 2536 2537
 *
 * RETURNS:
 * %true if @work was pending, %false otherwise.
2538
 */
2539
bool cancel_work_sync(struct work_struct *work)
2540
{
2541
	return __cancel_work_timer(work, NULL);
O
Oleg Nesterov 已提交
2542
}
2543
EXPORT_SYMBOL_GPL(cancel_work_sync);
O
Oleg Nesterov 已提交
2544

2545
/**
2546 2547
 * flush_delayed_work - wait for a dwork to finish executing the last queueing
 * @dwork: the delayed work to flush
2548
 *
2549 2550 2551
 * Delayed timer is cancelled and the pending work is queued for
 * immediate execution.  Like flush_work(), this function only
 * considers the last queueing instance of @dwork.
2552
 *
2553 2554 2555
 * RETURNS:
 * %true if flush_work() waited for the work to finish execution,
 * %false if it was already idle.
2556
 */
2557 2558 2559 2560 2561 2562 2563 2564 2565
bool flush_delayed_work(struct delayed_work *dwork)
{
	if (del_timer_sync(&dwork->timer))
		__queue_work(raw_smp_processor_id(),
			     get_work_cwq(&dwork->work)->wq, &dwork->work);
	return flush_work(&dwork->work);
}
EXPORT_SYMBOL(flush_delayed_work);

2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586
/**
 * flush_delayed_work_sync - wait for a dwork to finish
 * @dwork: the delayed work to flush
 *
 * Delayed timer is cancelled and the pending work is queued for
 * execution immediately.  Other than timer handling, its behavior
 * is identical to flush_work_sync().
 *
 * RETURNS:
 * %true if flush_work_sync() waited for the work to finish execution,
 * %false if it was already idle.
 */
bool flush_delayed_work_sync(struct delayed_work *dwork)
{
	if (del_timer_sync(&dwork->timer))
		__queue_work(raw_smp_processor_id(),
			     get_work_cwq(&dwork->work)->wq, &dwork->work);
	return flush_work_sync(&dwork->work);
}
EXPORT_SYMBOL(flush_delayed_work_sync);

2587 2588 2589 2590 2591 2592 2593 2594 2595 2596
/**
 * cancel_delayed_work_sync - cancel a delayed work and wait for it to finish
 * @dwork: the delayed work cancel
 *
 * This is cancel_work_sync() for delayed works.
 *
 * RETURNS:
 * %true if @dwork was pending, %false otherwise.
 */
bool cancel_delayed_work_sync(struct delayed_work *dwork)
2597
{
2598
	return __cancel_work_timer(&dwork->work, &dwork->timer);
2599
}
2600
EXPORT_SYMBOL(cancel_delayed_work_sync);
L
Linus Torvalds 已提交
2601

2602 2603 2604 2605
/**
 * schedule_work - put work task in global workqueue
 * @work: job to be done
 *
2606 2607 2608 2609 2610 2611
 * Returns zero if @work was already on the kernel-global workqueue and
 * non-zero otherwise.
 *
 * This puts a job in the kernel-global workqueue if it was not already
 * queued and leaves it in the same position on the kernel-global
 * workqueue otherwise.
2612
 */
2613
int schedule_work(struct work_struct *work)
L
Linus Torvalds 已提交
2614
{
2615
	return queue_work(system_wq, work);
L
Linus Torvalds 已提交
2616
}
2617
EXPORT_SYMBOL(schedule_work);
L
Linus Torvalds 已提交
2618

2619 2620 2621 2622 2623 2624 2625 2626 2627
/*
 * schedule_work_on - put work task on a specific cpu
 * @cpu: cpu to put the work task on
 * @work: job to be done
 *
 * This puts a job on a specific cpu
 */
int schedule_work_on(int cpu, struct work_struct *work)
{
2628
	return queue_work_on(cpu, system_wq, work);
2629 2630 2631
}
EXPORT_SYMBOL(schedule_work_on);

2632 2633
/**
 * schedule_delayed_work - put work task in global workqueue after delay
2634 2635
 * @dwork: job to be done
 * @delay: number of jiffies to wait or 0 for immediate execution
2636 2637 2638 2639
 *
 * After waiting for a given time this puts a job in the kernel-global
 * workqueue.
 */
2640
int schedule_delayed_work(struct delayed_work *dwork,
2641
					unsigned long delay)
L
Linus Torvalds 已提交
2642
{
2643
	return queue_delayed_work(system_wq, dwork, delay);
L
Linus Torvalds 已提交
2644
}
2645
EXPORT_SYMBOL(schedule_delayed_work);
L
Linus Torvalds 已提交
2646

2647 2648 2649
/**
 * schedule_delayed_work_on - queue work in global workqueue on CPU after delay
 * @cpu: cpu to use
2650
 * @dwork: job to be done
2651 2652 2653 2654 2655
 * @delay: number of jiffies to wait
 *
 * After waiting for a given time this puts a job in the kernel-global
 * workqueue on the specified CPU.
 */
L
Linus Torvalds 已提交
2656
int schedule_delayed_work_on(int cpu,
2657
			struct delayed_work *dwork, unsigned long delay)
L
Linus Torvalds 已提交
2658
{
2659
	return queue_delayed_work_on(cpu, system_wq, dwork, delay);
L
Linus Torvalds 已提交
2660
}
2661
EXPORT_SYMBOL(schedule_delayed_work_on);
L
Linus Torvalds 已提交
2662

2663
/**
2664
 * schedule_on_each_cpu - execute a function synchronously on each online CPU
2665 2666
 * @func: the function to call
 *
2667 2668
 * schedule_on_each_cpu() executes @func on each online CPU using the
 * system workqueue and blocks until all CPUs have completed.
2669
 * schedule_on_each_cpu() is very slow.
2670 2671 2672
 *
 * RETURNS:
 * 0 on success, -errno on failure.
2673
 */
2674
int schedule_on_each_cpu(work_func_t func)
2675 2676
{
	int cpu;
2677
	struct work_struct __percpu *works;
2678

2679 2680
	works = alloc_percpu(struct work_struct);
	if (!works)
2681
		return -ENOMEM;
2682

2683 2684
	get_online_cpus();

2685
	for_each_online_cpu(cpu) {
2686 2687 2688
		struct work_struct *work = per_cpu_ptr(works, cpu);

		INIT_WORK(work, func);
2689
		schedule_work_on(cpu, work);
2690
	}
2691 2692 2693 2694

	for_each_online_cpu(cpu)
		flush_work(per_cpu_ptr(works, cpu));

2695
	put_online_cpus();
2696
	free_percpu(works);
2697 2698 2699
	return 0;
}

2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723
/**
 * flush_scheduled_work - ensure that any scheduled work has run to completion.
 *
 * Forces execution of the kernel-global workqueue and blocks until its
 * completion.
 *
 * Think twice before calling this function!  It's very easy to get into
 * trouble if you don't take great care.  Either of the following situations
 * will lead to deadlock:
 *
 *	One of the work items currently on the workqueue needs to acquire
 *	a lock held by your code or its caller.
 *
 *	Your code is running in the context of a work routine.
 *
 * They will be detected by lockdep when they occur, but the first might not
 * occur very often.  It depends on what work items are on the workqueue and
 * what locks they need, which you have no control over.
 *
 * In most situations flushing the entire workqueue is overkill; you merely
 * need to know that a particular work item isn't queued and isn't running.
 * In such cases you should use cancel_delayed_work_sync() or
 * cancel_work_sync() instead.
 */
L
Linus Torvalds 已提交
2724 2725
void flush_scheduled_work(void)
{
2726
	flush_workqueue(system_wq);
L
Linus Torvalds 已提交
2727
}
2728
EXPORT_SYMBOL(flush_scheduled_work);
L
Linus Torvalds 已提交
2729

2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741
/**
 * execute_in_process_context - reliably execute the routine with user context
 * @fn:		the function to execute
 * @ew:		guaranteed storage for the execute work structure (must
 *		be available when the work executes)
 *
 * Executes the function immediately if process context is available,
 * otherwise schedules the function for delayed execution.
 *
 * Returns:	0 - function was executed
 *		1 - function was scheduled for execution
 */
2742
int execute_in_process_context(work_func_t fn, struct execute_work *ew)
2743 2744
{
	if (!in_interrupt()) {
2745
		fn(&ew->work);
2746 2747 2748
		return 0;
	}

2749
	INIT_WORK(&ew->work, fn);
2750 2751 2752 2753 2754 2755
	schedule_work(&ew->work);

	return 1;
}
EXPORT_SYMBOL_GPL(execute_in_process_context);

L
Linus Torvalds 已提交
2756 2757
int keventd_up(void)
{
2758
	return system_wq != NULL;
L
Linus Torvalds 已提交
2759 2760
}

2761
static int alloc_cwqs(struct workqueue_struct *wq)
T
Tejun Heo 已提交
2762
{
2763
	/*
T
Tejun Heo 已提交
2764 2765 2766
	 * cwqs are forced aligned according to WORK_STRUCT_FLAG_BITS.
	 * Make sure that the alignment isn't lower than that of
	 * unsigned long long.
2767
	 */
T
Tejun Heo 已提交
2768 2769 2770
	const size_t size = sizeof(struct cpu_workqueue_struct);
	const size_t align = max_t(size_t, 1 << WORK_STRUCT_FLAG_BITS,
				   __alignof__(unsigned long long));
2771 2772 2773 2774 2775
#ifdef CONFIG_SMP
	bool percpu = !(wq->flags & WQ_UNBOUND);
#else
	bool percpu = false;
#endif
2776

2777
	if (percpu)
2778
		wq->cpu_wq.pcpu = __alloc_percpu(size, align);
2779
	else {
2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791
		void *ptr;

		/*
		 * Allocate enough room to align cwq and put an extra
		 * pointer at the end pointing back to the originally
		 * allocated pointer which will be used for free.
		 */
		ptr = kzalloc(size + align + sizeof(void *), GFP_KERNEL);
		if (ptr) {
			wq->cpu_wq.single = PTR_ALIGN(ptr, align);
			*(void **)(wq->cpu_wq.single + 1) = ptr;
		}
2792
	}
2793

2794 2795 2796
	/* just in case, make sure it's actually aligned
	 * - this is affected by PERCPU() alignment in vmlinux.lds.S
	 */
2797 2798
	BUG_ON(!IS_ALIGNED(wq->cpu_wq.v, align));
	return wq->cpu_wq.v ? 0 : -ENOMEM;
T
Tejun Heo 已提交
2799 2800
}

2801
static void free_cwqs(struct workqueue_struct *wq)
T
Tejun Heo 已提交
2802
{
2803 2804 2805 2806 2807 2808 2809
#ifdef CONFIG_SMP
	bool percpu = !(wq->flags & WQ_UNBOUND);
#else
	bool percpu = false;
#endif

	if (percpu)
2810 2811 2812
		free_percpu(wq->cpu_wq.pcpu);
	else if (wq->cpu_wq.single) {
		/* the pointer to free is stored right after the cwq */
2813
		kfree(*(void **)(wq->cpu_wq.single + 1));
2814
	}
T
Tejun Heo 已提交
2815 2816
}

2817 2818
static int wq_clamp_max_active(int max_active, unsigned int flags,
			       const char *name)
2819
{
2820 2821 2822
	int lim = flags & WQ_UNBOUND ? WQ_UNBOUND_MAX_ACTIVE : WQ_MAX_ACTIVE;

	if (max_active < 1 || max_active > lim)
2823 2824
		printk(KERN_WARNING "workqueue: max_active %d requested for %s "
		       "is out of range, clamping between %d and %d\n",
2825
		       max_active, name, 1, lim);
2826

2827
	return clamp_val(max_active, 1, lim);
2828 2829
}

2830 2831 2832 2833 2834
struct workqueue_struct *__alloc_workqueue_key(const char *name,
					       unsigned int flags,
					       int max_active,
					       struct lock_class_key *key,
					       const char *lock_name)
L
Linus Torvalds 已提交
2835 2836
{
	struct workqueue_struct *wq;
T
Tejun Heo 已提交
2837
	unsigned int cpu;
L
Linus Torvalds 已提交
2838

2839 2840 2841 2842 2843 2844 2845
	/*
	 * Workqueues which may be used during memory reclaim should
	 * have a rescuer to guarantee forward progress.
	 */
	if (flags & WQ_MEM_RECLAIM)
		flags |= WQ_RESCUER;

2846 2847 2848 2849 2850 2851 2852
	/*
	 * Unbound workqueues aren't concurrency managed and should be
	 * dispatched to workers immediately.
	 */
	if (flags & WQ_UNBOUND)
		flags |= WQ_HIGHPRI;

2853
	max_active = max_active ?: WQ_DFL_ACTIVE;
2854
	max_active = wq_clamp_max_active(max_active, flags, name);
2855

2856 2857
	wq = kzalloc(sizeof(*wq), GFP_KERNEL);
	if (!wq)
T
Tejun Heo 已提交
2858
		goto err;
2859

2860
	wq->flags = flags;
2861
	wq->saved_max_active = max_active;
2862 2863 2864 2865
	mutex_init(&wq->flush_mutex);
	atomic_set(&wq->nr_cwqs_to_flush, 0);
	INIT_LIST_HEAD(&wq->flusher_queue);
	INIT_LIST_HEAD(&wq->flusher_overflow);
2866

2867
	wq->name = name;
2868
	lockdep_init_map(&wq->lockdep_map, lock_name, key, 0);
2869
	INIT_LIST_HEAD(&wq->list);
2870

2871 2872 2873
	if (alloc_cwqs(wq) < 0)
		goto err;

2874
	for_each_cwq_cpu(cpu, wq) {
T
Tejun Heo 已提交
2875
		struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
2876
		struct global_cwq *gcwq = get_gcwq(cpu);
T
Tejun Heo 已提交
2877

T
Tejun Heo 已提交
2878
		BUG_ON((unsigned long)cwq & WORK_STRUCT_FLAG_MASK);
2879
		cwq->gcwq = gcwq;
T
Tejun Heo 已提交
2880
		cwq->wq = wq;
2881
		cwq->flush_color = -1;
2882 2883
		cwq->max_active = max_active;
		INIT_LIST_HEAD(&cwq->delayed_works);
2884
	}
T
Tejun Heo 已提交
2885

2886 2887 2888
	if (flags & WQ_RESCUER) {
		struct worker *rescuer;

2889
		if (!alloc_mayday_mask(&wq->mayday_mask, GFP_KERNEL))
2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901
			goto err;

		wq->rescuer = rescuer = alloc_worker();
		if (!rescuer)
			goto err;

		rescuer->task = kthread_create(rescuer_thread, wq, "%s", name);
		if (IS_ERR(rescuer->task))
			goto err;

		rescuer->task->flags |= PF_THREAD_BOUND;
		wake_up_process(rescuer->task);
2902 2903
	}

2904 2905 2906 2907 2908
	/*
	 * workqueue_lock protects global freeze state and workqueues
	 * list.  Grab it, set max_active accordingly and add the new
	 * workqueue to workqueues list.
	 */
T
Tejun Heo 已提交
2909
	spin_lock(&workqueue_lock);
2910 2911

	if (workqueue_freezing && wq->flags & WQ_FREEZEABLE)
2912
		for_each_cwq_cpu(cpu, wq)
2913 2914
			get_cwq(cpu, wq)->max_active = 0;

T
Tejun Heo 已提交
2915
	list_add(&wq->list, &workqueues);
2916

T
Tejun Heo 已提交
2917 2918
	spin_unlock(&workqueue_lock);

2919
	return wq;
T
Tejun Heo 已提交
2920 2921
err:
	if (wq) {
2922
		free_cwqs(wq);
2923
		free_mayday_mask(wq->mayday_mask);
2924
		kfree(wq->rescuer);
T
Tejun Heo 已提交
2925 2926 2927
		kfree(wq);
	}
	return NULL;
2928
}
2929
EXPORT_SYMBOL_GPL(__alloc_workqueue_key);
L
Linus Torvalds 已提交
2930

2931 2932 2933 2934 2935 2936 2937 2938
/**
 * destroy_workqueue - safely terminate a workqueue
 * @wq: target workqueue
 *
 * Safely destroy a workqueue. All work currently pending will be done first.
 */
void destroy_workqueue(struct workqueue_struct *wq)
{
T
Tejun Heo 已提交
2939
	unsigned int cpu;
2940

2941
	wq->flags |= WQ_DYING;
2942 2943 2944 2945 2946 2947
	flush_workqueue(wq);

	/*
	 * wq list is used to freeze wq, remove from list after
	 * flushing is complete in case freeze races us.
	 */
2948
	spin_lock(&workqueue_lock);
2949
	list_del(&wq->list);
2950
	spin_unlock(&workqueue_lock);
2951

2952
	/* sanity check */
2953
	for_each_cwq_cpu(cpu, wq) {
2954 2955 2956 2957 2958
		struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
		int i;

		for (i = 0; i < WORK_NR_COLORS; i++)
			BUG_ON(cwq->nr_in_flight[i]);
2959 2960
		BUG_ON(cwq->nr_active);
		BUG_ON(!list_empty(&cwq->delayed_works));
2961
	}
2962

2963 2964
	if (wq->flags & WQ_RESCUER) {
		kthread_stop(wq->rescuer->task);
2965
		free_mayday_mask(wq->mayday_mask);
2966
		kfree(wq->rescuer);
2967 2968
	}

2969
	free_cwqs(wq);
2970 2971 2972 2973
	kfree(wq);
}
EXPORT_SYMBOL_GPL(destroy_workqueue);

2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987
/**
 * workqueue_set_max_active - adjust max_active of a workqueue
 * @wq: target workqueue
 * @max_active: new max_active value.
 *
 * Set max_active of @wq to @max_active.
 *
 * CONTEXT:
 * Don't call from IRQ context.
 */
void workqueue_set_max_active(struct workqueue_struct *wq, int max_active)
{
	unsigned int cpu;

2988
	max_active = wq_clamp_max_active(max_active, wq->flags, wq->name);
2989 2990 2991 2992 2993

	spin_lock(&workqueue_lock);

	wq->saved_max_active = max_active;

2994
	for_each_cwq_cpu(cpu, wq) {
2995 2996 2997 2998 2999 3000 3001
		struct global_cwq *gcwq = get_gcwq(cpu);

		spin_lock_irq(&gcwq->lock);

		if (!(wq->flags & WQ_FREEZEABLE) ||
		    !(gcwq->flags & GCWQ_FREEZING))
			get_cwq(gcwq->cpu, wq)->max_active = max_active;
3002

3003
		spin_unlock_irq(&gcwq->lock);
3004
	}
3005

3006
	spin_unlock(&workqueue_lock);
3007
}
3008
EXPORT_SYMBOL_GPL(workqueue_set_max_active);
3009

3010
/**
3011 3012 3013
 * workqueue_congested - test whether a workqueue is congested
 * @cpu: CPU in question
 * @wq: target workqueue
3014
 *
3015 3016 3017
 * Test whether @wq's cpu workqueue for @cpu is congested.  There is
 * no synchronization around this function and the test result is
 * unreliable and only useful as advisory hints or for debugging.
3018
 *
3019 3020
 * RETURNS:
 * %true if congested, %false otherwise.
3021
 */
3022
bool workqueue_congested(unsigned int cpu, struct workqueue_struct *wq)
L
Linus Torvalds 已提交
3023
{
3024 3025 3026
	struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);

	return !list_empty(&cwq->delayed_works);
L
Linus Torvalds 已提交
3027
}
3028
EXPORT_SYMBOL_GPL(workqueue_congested);
L
Linus Torvalds 已提交
3029

3030
/**
3031 3032
 * work_cpu - return the last known associated cpu for @work
 * @work: the work of interest
3033
 *
3034
 * RETURNS:
3035
 * CPU number if @work was ever queued.  WORK_CPU_NONE otherwise.
3036
 */
3037
unsigned int work_cpu(struct work_struct *work)
3038
{
3039
	struct global_cwq *gcwq = get_work_gcwq(work);
3040

3041
	return gcwq ? gcwq->cpu : WORK_CPU_NONE;
3042
}
3043
EXPORT_SYMBOL_GPL(work_cpu);
3044

3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058
/**
 * work_busy - test whether a work is currently pending or running
 * @work: the work to be tested
 *
 * Test whether @work is currently pending or running.  There is no
 * synchronization around this function and the test result is
 * unreliable and only useful as advisory hints or for debugging.
 * Especially for reentrant wqs, the pending state might hide the
 * running state.
 *
 * RETURNS:
 * OR'd bitmask of WORK_BUSY_* bits.
 */
unsigned int work_busy(struct work_struct *work)
L
Linus Torvalds 已提交
3059
{
3060 3061 3062
	struct global_cwq *gcwq = get_work_gcwq(work);
	unsigned long flags;
	unsigned int ret = 0;
L
Linus Torvalds 已提交
3063

3064 3065
	if (!gcwq)
		return false;
L
Linus Torvalds 已提交
3066

3067
	spin_lock_irqsave(&gcwq->lock, flags);
L
Linus Torvalds 已提交
3068

3069 3070 3071 3072
	if (work_pending(work))
		ret |= WORK_BUSY_PENDING;
	if (find_worker_executing_work(gcwq, work))
		ret |= WORK_BUSY_RUNNING;
L
Linus Torvalds 已提交
3073

3074
	spin_unlock_irqrestore(&gcwq->lock, flags);
L
Linus Torvalds 已提交
3075

3076
	return ret;
L
Linus Torvalds 已提交
3077
}
3078
EXPORT_SYMBOL_GPL(work_busy);
L
Linus Torvalds 已提交
3079

3080 3081 3082
/*
 * CPU hotplug.
 *
3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094
 * There are two challenges in supporting CPU hotplug.  Firstly, there
 * are a lot of assumptions on strong associations among work, cwq and
 * gcwq which make migrating pending and scheduled works very
 * difficult to implement without impacting hot paths.  Secondly,
 * gcwqs serve mix of short, long and very long running works making
 * blocked draining impractical.
 *
 * This is solved by allowing a gcwq to be detached from CPU, running
 * it with unbound (rogue) workers and allowing it to be reattached
 * later if the cpu comes back online.  A separate thread is created
 * to govern a gcwq in such state and is called the trustee of the
 * gcwq.
3095 3096 3097 3098 3099 3100 3101
 *
 * Trustee states and their descriptions.
 *
 * START	Command state used on startup.  On CPU_DOWN_PREPARE, a
 *		new trustee is started with this state.
 *
 * IN_CHARGE	Once started, trustee will enter this state after
3102 3103 3104 3105 3106 3107
 *		assuming the manager role and making all existing
 *		workers rogue.  DOWN_PREPARE waits for trustee to
 *		enter this state.  After reaching IN_CHARGE, trustee
 *		tries to execute the pending worklist until it's empty
 *		and the state is set to BUTCHER, or the state is set
 *		to RELEASE.
3108 3109 3110 3111 3112 3113 3114 3115 3116 3117
 *
 * BUTCHER	Command state which is set by the cpu callback after
 *		the cpu has went down.  Once this state is set trustee
 *		knows that there will be no new works on the worklist
 *		and once the worklist is empty it can proceed to
 *		killing idle workers.
 *
 * RELEASE	Command state which is set by the cpu callback if the
 *		cpu down has been canceled or it has come online
 *		again.  After recognizing this state, trustee stops
3118 3119 3120
 *		trying to drain or butcher and clears ROGUE, rebinds
 *		all remaining workers back to the cpu and releases
 *		manager role.
3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131
 *
 * DONE		Trustee will enter this state after BUTCHER or RELEASE
 *		is complete.
 *
 *          trustee                 CPU                draining
 *         took over                down               complete
 * START -----------> IN_CHARGE -----------> BUTCHER -----------> DONE
 *                        |                     |                  ^
 *                        | CPU is back online  v   return workers |
 *                         ----------------> RELEASE --------------
 */
L
Linus Torvalds 已提交
3132

3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160
/**
 * trustee_wait_event_timeout - timed event wait for trustee
 * @cond: condition to wait for
 * @timeout: timeout in jiffies
 *
 * wait_event_timeout() for trustee to use.  Handles locking and
 * checks for RELEASE request.
 *
 * CONTEXT:
 * spin_lock_irq(gcwq->lock) which may be released and regrabbed
 * multiple times.  To be used by trustee.
 *
 * RETURNS:
 * Positive indicating left time if @cond is satisfied, 0 if timed
 * out, -1 if canceled.
 */
#define trustee_wait_event_timeout(cond, timeout) ({			\
	long __ret = (timeout);						\
	while (!((cond) || (gcwq->trustee_state == TRUSTEE_RELEASE)) &&	\
	       __ret) {							\
		spin_unlock_irq(&gcwq->lock);				\
		__wait_event_timeout(gcwq->trustee_wait, (cond) ||	\
			(gcwq->trustee_state == TRUSTEE_RELEASE),	\
			__ret);						\
		spin_lock_irq(&gcwq->lock);				\
	}								\
	gcwq->trustee_state == TRUSTEE_RELEASE ? -1 : (__ret);		\
})
3161

3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180
/**
 * trustee_wait_event - event wait for trustee
 * @cond: condition to wait for
 *
 * wait_event() for trustee to use.  Automatically handles locking and
 * checks for CANCEL request.
 *
 * CONTEXT:
 * spin_lock_irq(gcwq->lock) which may be released and regrabbed
 * multiple times.  To be used by trustee.
 *
 * RETURNS:
 * 0 if @cond is satisfied, -1 if canceled.
 */
#define trustee_wait_event(cond) ({					\
	long __ret1;							\
	__ret1 = trustee_wait_event_timeout(cond, MAX_SCHEDULE_TIMEOUT);\
	__ret1 < 0 ? -1 : 0;						\
})
L
Linus Torvalds 已提交
3181

3182
static int __cpuinit trustee_thread(void *__gcwq)
3183
{
3184 3185
	struct global_cwq *gcwq = __gcwq;
	struct worker *worker;
3186
	struct work_struct *work;
3187
	struct hlist_node *pos;
3188
	long rc;
3189
	int i;
3190

3191 3192 3193
	BUG_ON(gcwq->cpu != smp_processor_id());

	spin_lock_irq(&gcwq->lock);
3194
	/*
3195 3196 3197
	 * Claim the manager position and make all workers rogue.
	 * Trustee must be bound to the target cpu and can't be
	 * cancelled.
3198
	 */
3199
	BUG_ON(gcwq->cpu != smp_processor_id());
3200 3201
	rc = trustee_wait_event(!(gcwq->flags & GCWQ_MANAGING_WORKERS));
	BUG_ON(rc < 0);
3202

3203
	gcwq->flags |= GCWQ_MANAGING_WORKERS;
3204

3205
	list_for_each_entry(worker, &gcwq->idle_list, entry)
3206
		worker->flags |= WORKER_ROGUE;
3207

3208
	for_each_busy_worker(worker, i, pos, gcwq)
3209
		worker->flags |= WORKER_ROGUE;
3210

3211 3212 3213 3214 3215 3216 3217 3218 3219
	/*
	 * Call schedule() so that we cross rq->lock and thus can
	 * guarantee sched callbacks see the rogue flag.  This is
	 * necessary as scheduler callbacks may be invoked from other
	 * cpus.
	 */
	spin_unlock_irq(&gcwq->lock);
	schedule();
	spin_lock_irq(&gcwq->lock);
3220

3221
	/*
3222 3223 3224 3225
	 * Sched callbacks are disabled now.  Zap nr_running.  After
	 * this, nr_running stays zero and need_more_worker() and
	 * keep_working() are always true as long as the worklist is
	 * not empty.
3226
	 */
3227
	atomic_set(get_gcwq_nr_running(gcwq->cpu), 0);
L
Linus Torvalds 已提交
3228

3229 3230 3231
	spin_unlock_irq(&gcwq->lock);
	del_timer_sync(&gcwq->idle_timer);
	spin_lock_irq(&gcwq->lock);
3232

3233 3234 3235 3236 3237 3238 3239 3240
	/*
	 * We're now in charge.  Notify and proceed to drain.  We need
	 * to keep the gcwq running during the whole CPU down
	 * procedure as other cpu hotunplug callbacks may need to
	 * flush currently running tasks.
	 */
	gcwq->trustee_state = TRUSTEE_IN_CHARGE;
	wake_up_all(&gcwq->trustee_wait);
3241

3242 3243 3244
	/*
	 * The original cpu is in the process of dying and may go away
	 * anytime now.  When that happens, we and all workers would
3245 3246 3247 3248 3249 3250
	 * be migrated to other cpus.  Try draining any left work.  We
	 * want to get it over with ASAP - spam rescuers, wake up as
	 * many idlers as necessary and create new ones till the
	 * worklist is empty.  Note that if the gcwq is frozen, there
	 * may be frozen works in freezeable cwqs.  Don't declare
	 * completion while frozen.
3251 3252 3253 3254
	 */
	while (gcwq->nr_workers != gcwq->nr_idle ||
	       gcwq->flags & GCWQ_FREEZING ||
	       gcwq->trustee_state == TRUSTEE_IN_CHARGE) {
3255 3256 3257 3258 3259 3260
		int nr_works = 0;

		list_for_each_entry(work, &gcwq->worklist, entry) {
			send_mayday(work);
			nr_works++;
		}
3261

3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272
		list_for_each_entry(worker, &gcwq->idle_list, entry) {
			if (!nr_works--)
				break;
			wake_up_process(worker->task);
		}

		if (need_to_create_worker(gcwq)) {
			spin_unlock_irq(&gcwq->lock);
			worker = create_worker(gcwq, false);
			spin_lock_irq(&gcwq->lock);
			if (worker) {
3273
				worker->flags |= WORKER_ROGUE;
3274 3275
				start_worker(worker);
			}
L
Linus Torvalds 已提交
3276
		}
3277

3278 3279 3280
		/* give a breather */
		if (trustee_wait_event_timeout(false, TRUSTEE_COOLDOWN) < 0)
			break;
3281
	}
L
Linus Torvalds 已提交
3282

3283
	/*
3284 3285 3286
	 * Either all works have been scheduled and cpu is down, or
	 * cpu down has already been canceled.  Wait for and butcher
	 * all workers till we're canceled.
3287
	 */
3288 3289 3290 3291 3292 3293
	do {
		rc = trustee_wait_event(!list_empty(&gcwq->idle_list));
		while (!list_empty(&gcwq->idle_list))
			destroy_worker(list_first_entry(&gcwq->idle_list,
							struct worker, entry));
	} while (gcwq->nr_workers && rc >= 0);
3294

3295
	/*
3296 3297 3298 3299 3300
	 * At this point, either draining has completed and no worker
	 * is left, or cpu down has been canceled or the cpu is being
	 * brought back up.  There shouldn't be any idle one left.
	 * Tell the remaining busy ones to rebind once it finishes the
	 * currently scheduled works by scheduling the rebind_work.
3301
	 */
3302 3303 3304 3305 3306 3307 3308 3309 3310 3311
	WARN_ON(!list_empty(&gcwq->idle_list));

	for_each_busy_worker(worker, i, pos, gcwq) {
		struct work_struct *rebind_work = &worker->rebind_work;

		/*
		 * Rebind_work may race with future cpu hotplug
		 * operations.  Use a separate flag to mark that
		 * rebinding is scheduled.
		 */
3312 3313
		worker->flags |= WORKER_REBIND;
		worker->flags &= ~WORKER_ROGUE;
3314 3315 3316 3317 3318 3319 3320

		/* queue rebind_work, wq doesn't matter, use the default one */
		if (test_and_set_bit(WORK_STRUCT_PENDING_BIT,
				     work_data_bits(rebind_work)))
			continue;

		debug_work_activate(rebind_work);
3321
		insert_work(get_cwq(gcwq->cpu, system_wq), rebind_work,
3322 3323 3324 3325 3326 3327 3328
			    worker->scheduled.next,
			    work_color_to_flags(WORK_NO_COLOR));
	}

	/* relinquish manager role */
	gcwq->flags &= ~GCWQ_MANAGING_WORKERS;

3329 3330 3331 3332 3333 3334
	/* notify completion */
	gcwq->trustee = NULL;
	gcwq->trustee_state = TRUSTEE_DONE;
	wake_up_all(&gcwq->trustee_wait);
	spin_unlock_irq(&gcwq->lock);
	return 0;
3335 3336 3337
}

/**
3338 3339 3340
 * wait_trustee_state - wait for trustee to enter the specified state
 * @gcwq: gcwq the trustee of interest belongs to
 * @state: target state to wait for
3341
 *
3342 3343 3344 3345 3346
 * Wait for the trustee to reach @state.  DONE is already matched.
 *
 * CONTEXT:
 * spin_lock_irq(gcwq->lock) which may be released and regrabbed
 * multiple times.  To be used by cpu_callback.
3347
 */
3348
static void __cpuinit wait_trustee_state(struct global_cwq *gcwq, int state)
3349 3350
__releases(&gcwq->lock)
__acquires(&gcwq->lock)
3351
{
3352 3353 3354 3355 3356 3357 3358 3359
	if (!(gcwq->trustee_state == state ||
	      gcwq->trustee_state == TRUSTEE_DONE)) {
		spin_unlock_irq(&gcwq->lock);
		__wait_event(gcwq->trustee_wait,
			     gcwq->trustee_state == state ||
			     gcwq->trustee_state == TRUSTEE_DONE);
		spin_lock_irq(&gcwq->lock);
	}
3360 3361 3362 3363 3364 3365 3366
}

static int __devinit workqueue_cpu_callback(struct notifier_block *nfb,
						unsigned long action,
						void *hcpu)
{
	unsigned int cpu = (unsigned long)hcpu;
3367 3368
	struct global_cwq *gcwq = get_gcwq(cpu);
	struct task_struct *new_trustee = NULL;
3369
	struct worker *uninitialized_var(new_worker);
3370
	unsigned long flags;
3371

3372 3373
	action &= ~CPU_TASKS_FROZEN;

3374
	switch (action) {
3375 3376 3377 3378 3379 3380
	case CPU_DOWN_PREPARE:
		new_trustee = kthread_create(trustee_thread, gcwq,
					     "workqueue_trustee/%d\n", cpu);
		if (IS_ERR(new_trustee))
			return notifier_from_errno(PTR_ERR(new_trustee));
		kthread_bind(new_trustee, cpu);
3381
		/* fall through */
3382
	case CPU_UP_PREPARE:
3383 3384 3385 3386 3387 3388
		BUG_ON(gcwq->first_idle);
		new_worker = create_worker(gcwq, false);
		if (!new_worker) {
			if (new_trustee)
				kthread_stop(new_trustee);
			return NOTIFY_BAD;
3389
		}
L
Linus Torvalds 已提交
3390 3391
	}

3392 3393
	/* some are called w/ irq disabled, don't disturb irq status */
	spin_lock_irqsave(&gcwq->lock, flags);
3394

3395
	switch (action) {
3396 3397 3398 3399 3400 3401 3402
	case CPU_DOWN_PREPARE:
		/* initialize trustee and tell it to acquire the gcwq */
		BUG_ON(gcwq->trustee || gcwq->trustee_state != TRUSTEE_DONE);
		gcwq->trustee = new_trustee;
		gcwq->trustee_state = TRUSTEE_START;
		wake_up_process(gcwq->trustee);
		wait_trustee_state(gcwq, TRUSTEE_IN_CHARGE);
3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416
		/* fall through */
	case CPU_UP_PREPARE:
		BUG_ON(gcwq->first_idle);
		gcwq->first_idle = new_worker;
		break;

	case CPU_DYING:
		/*
		 * Before this, the trustee and all workers except for
		 * the ones which are still executing works from
		 * before the last CPU down must be on the cpu.  After
		 * this, they'll all be diasporas.
		 */
		gcwq->flags |= GCWQ_DISASSOCIATED;
3417 3418
		break;

3419
	case CPU_POST_DEAD:
3420
		gcwq->trustee_state = TRUSTEE_BUTCHER;
3421 3422 3423 3424
		/* fall through */
	case CPU_UP_CANCELED:
		destroy_worker(gcwq->first_idle);
		gcwq->first_idle = NULL;
3425 3426 3427 3428
		break;

	case CPU_DOWN_FAILED:
	case CPU_ONLINE:
3429
		gcwq->flags &= ~GCWQ_DISASSOCIATED;
3430 3431 3432 3433
		if (gcwq->trustee_state != TRUSTEE_DONE) {
			gcwq->trustee_state = TRUSTEE_RELEASE;
			wake_up_process(gcwq->trustee);
			wait_trustee_state(gcwq, TRUSTEE_DONE);
3434
		}
3435

3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446
		/*
		 * Trustee is done and there might be no worker left.
		 * Put the first_idle in and request a real manager to
		 * take a look.
		 */
		spin_unlock_irq(&gcwq->lock);
		kthread_bind(gcwq->first_idle->task, cpu);
		spin_lock_irq(&gcwq->lock);
		gcwq->flags |= GCWQ_MANAGE_WORKERS;
		start_worker(gcwq->first_idle);
		gcwq->first_idle = NULL;
3447
		break;
3448 3449
	}

3450 3451
	spin_unlock_irqrestore(&gcwq->lock, flags);

T
Tejun Heo 已提交
3452
	return notifier_from_errno(0);
L
Linus Torvalds 已提交
3453 3454
}

3455
#ifdef CONFIG_SMP
3456

3457
struct work_for_cpu {
3458
	struct completion completion;
3459 3460 3461 3462 3463
	long (*fn)(void *);
	void *arg;
	long ret;
};

3464
static int do_work_for_cpu(void *_wfc)
3465
{
3466
	struct work_for_cpu *wfc = _wfc;
3467
	wfc->ret = wfc->fn(wfc->arg);
3468 3469
	complete(&wfc->completion);
	return 0;
3470 3471 3472 3473 3474 3475 3476 3477
}

/**
 * work_on_cpu - run a function in user context on a particular cpu
 * @cpu: the cpu to run on
 * @fn: the function to run
 * @arg: the function arg
 *
3478 3479
 * This will return the value @fn returns.
 * It is up to the caller to ensure that the cpu doesn't go offline.
3480
 * The caller must not hold any locks which would prevent @fn from completing.
3481 3482 3483
 */
long work_on_cpu(unsigned int cpu, long (*fn)(void *), void *arg)
{
3484 3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496
	struct task_struct *sub_thread;
	struct work_for_cpu wfc = {
		.completion = COMPLETION_INITIALIZER_ONSTACK(wfc.completion),
		.fn = fn,
		.arg = arg,
	};

	sub_thread = kthread_create(do_work_for_cpu, &wfc, "work_for_cpu");
	if (IS_ERR(sub_thread))
		return PTR_ERR(sub_thread);
	kthread_bind(sub_thread, cpu);
	wake_up_process(sub_thread);
	wait_for_completion(&wfc.completion);
3497 3498 3499 3500 3501
	return wfc.ret;
}
EXPORT_SYMBOL_GPL(work_on_cpu);
#endif /* CONFIG_SMP */

3502 3503 3504 3505 3506 3507 3508
#ifdef CONFIG_FREEZER

/**
 * freeze_workqueues_begin - begin freezing workqueues
 *
 * Start freezing workqueues.  After this function returns, all
 * freezeable workqueues will queue new works to their frozen_works
3509
 * list instead of gcwq->worklist.
3510 3511
 *
 * CONTEXT:
3512
 * Grabs and releases workqueue_lock and gcwq->lock's.
3513 3514 3515 3516 3517 3518 3519 3520 3521 3522
 */
void freeze_workqueues_begin(void)
{
	unsigned int cpu;

	spin_lock(&workqueue_lock);

	BUG_ON(workqueue_freezing);
	workqueue_freezing = true;

3523
	for_each_gcwq_cpu(cpu) {
3524
		struct global_cwq *gcwq = get_gcwq(cpu);
3525
		struct workqueue_struct *wq;
3526 3527 3528

		spin_lock_irq(&gcwq->lock);

3529 3530 3531
		BUG_ON(gcwq->flags & GCWQ_FREEZING);
		gcwq->flags |= GCWQ_FREEZING;

3532 3533 3534
		list_for_each_entry(wq, &workqueues, list) {
			struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);

3535
			if (cwq && wq->flags & WQ_FREEZEABLE)
3536 3537
				cwq->max_active = 0;
		}
3538 3539

		spin_unlock_irq(&gcwq->lock);
3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566
	}

	spin_unlock(&workqueue_lock);
}

/**
 * freeze_workqueues_busy - are freezeable workqueues still busy?
 *
 * Check whether freezing is complete.  This function must be called
 * between freeze_workqueues_begin() and thaw_workqueues().
 *
 * CONTEXT:
 * Grabs and releases workqueue_lock.
 *
 * RETURNS:
 * %true if some freezeable workqueues are still busy.  %false if
 * freezing is complete.
 */
bool freeze_workqueues_busy(void)
{
	unsigned int cpu;
	bool busy = false;

	spin_lock(&workqueue_lock);

	BUG_ON(!workqueue_freezing);

3567
	for_each_gcwq_cpu(cpu) {
3568
		struct workqueue_struct *wq;
3569 3570 3571 3572 3573 3574 3575
		/*
		 * nr_active is monotonically decreasing.  It's safe
		 * to peek without lock.
		 */
		list_for_each_entry(wq, &workqueues, list) {
			struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);

3576
			if (!cwq || !(wq->flags & WQ_FREEZEABLE))
3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594
				continue;

			BUG_ON(cwq->nr_active < 0);
			if (cwq->nr_active) {
				busy = true;
				goto out_unlock;
			}
		}
	}
out_unlock:
	spin_unlock(&workqueue_lock);
	return busy;
}

/**
 * thaw_workqueues - thaw workqueues
 *
 * Thaw workqueues.  Normal queueing is restored and all collected
3595
 * frozen works are transferred to their respective gcwq worklists.
3596 3597
 *
 * CONTEXT:
3598
 * Grabs and releases workqueue_lock and gcwq->lock's.
3599 3600 3601 3602 3603 3604 3605 3606 3607 3608
 */
void thaw_workqueues(void)
{
	unsigned int cpu;

	spin_lock(&workqueue_lock);

	if (!workqueue_freezing)
		goto out_unlock;

3609
	for_each_gcwq_cpu(cpu) {
3610
		struct global_cwq *gcwq = get_gcwq(cpu);
3611
		struct workqueue_struct *wq;
3612 3613 3614

		spin_lock_irq(&gcwq->lock);

3615 3616 3617
		BUG_ON(!(gcwq->flags & GCWQ_FREEZING));
		gcwq->flags &= ~GCWQ_FREEZING;

3618 3619 3620
		list_for_each_entry(wq, &workqueues, list) {
			struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);

3621
			if (!cwq || !(wq->flags & WQ_FREEZEABLE))
3622 3623 3624 3625 3626 3627 3628 3629 3630
				continue;

			/* restore max_active and repopulate worklist */
			cwq->max_active = wq->saved_max_active;

			while (!list_empty(&cwq->delayed_works) &&
			       cwq->nr_active < cwq->max_active)
				cwq_activate_first_delayed(cwq);
		}
3631

3632 3633
		wake_up_worker(gcwq);

3634
		spin_unlock_irq(&gcwq->lock);
3635 3636 3637 3638 3639 3640 3641 3642
	}

	workqueue_freezing = false;
out_unlock:
	spin_unlock(&workqueue_lock);
}
#endif /* CONFIG_FREEZER */

3643
static int __init init_workqueues(void)
L
Linus Torvalds 已提交
3644
{
T
Tejun Heo 已提交
3645
	unsigned int cpu;
T
Tejun Heo 已提交
3646
	int i;
T
Tejun Heo 已提交
3647

3648
	cpu_notifier(workqueue_cpu_callback, CPU_PRI_WORKQUEUE);
3649 3650

	/* initialize gcwqs */
3651
	for_each_gcwq_cpu(cpu) {
3652 3653 3654
		struct global_cwq *gcwq = get_gcwq(cpu);

		spin_lock_init(&gcwq->lock);
3655
		INIT_LIST_HEAD(&gcwq->worklist);
3656
		gcwq->cpu = cpu;
3657
		gcwq->flags |= GCWQ_DISASSOCIATED;
3658

T
Tejun Heo 已提交
3659 3660 3661 3662
		INIT_LIST_HEAD(&gcwq->idle_list);
		for (i = 0; i < BUSY_WORKER_HASH_SIZE; i++)
			INIT_HLIST_HEAD(&gcwq->busy_hash[i]);

3663 3664 3665
		init_timer_deferrable(&gcwq->idle_timer);
		gcwq->idle_timer.function = idle_worker_timeout;
		gcwq->idle_timer.data = (unsigned long)gcwq;
3666

3667 3668 3669
		setup_timer(&gcwq->mayday_timer, gcwq_mayday_timeout,
			    (unsigned long)gcwq);

3670
		ida_init(&gcwq->worker_ida);
3671 3672 3673

		gcwq->trustee_state = TRUSTEE_DONE;
		init_waitqueue_head(&gcwq->trustee_wait);
3674 3675
	}

3676
	/* create the initial worker */
3677
	for_each_online_gcwq_cpu(cpu) {
3678 3679 3680
		struct global_cwq *gcwq = get_gcwq(cpu);
		struct worker *worker;

3681 3682
		if (cpu != WORK_CPU_UNBOUND)
			gcwq->flags &= ~GCWQ_DISASSOCIATED;
3683 3684 3685 3686 3687 3688 3689
		worker = create_worker(gcwq, true);
		BUG_ON(!worker);
		spin_lock_irq(&gcwq->lock);
		start_worker(worker);
		spin_unlock_irq(&gcwq->lock);
	}

3690 3691 3692
	system_wq = alloc_workqueue("events", 0, 0);
	system_long_wq = alloc_workqueue("events_long", 0, 0);
	system_nrt_wq = alloc_workqueue("events_nrt", WQ_NON_REENTRANT, 0);
3693 3694
	system_unbound_wq = alloc_workqueue("events_unbound", WQ_UNBOUND,
					    WQ_UNBOUND_MAX_ACTIVE);
3695 3696
	BUG_ON(!system_wq || !system_long_wq || !system_nrt_wq ||
	       !system_unbound_wq);
3697
	return 0;
L
Linus Torvalds 已提交
3698
}
3699
early_initcall(init_workqueues);