workqueue.c 141.6 KB
Newer Older
L
Linus Torvalds 已提交
1
/*
T
Tejun Heo 已提交
2
 * kernel/workqueue.c - generic async execution with shared worker pool
L
Linus Torvalds 已提交
3
 *
T
Tejun Heo 已提交
4
 * Copyright (C) 2002		Ingo Molnar
L
Linus Torvalds 已提交
5
 *
T
Tejun Heo 已提交
6 7 8 9 10
 *   Derived from the taskqueue/keventd code by:
 *     David Woodhouse <dwmw2@infradead.org>
 *     Andrew Morton
 *     Kai Petzke <wpp@marie.physik.tu-berlin.de>
 *     Theodore Ts'o <tytso@mit.edu>
L
Linus Torvalds 已提交
11
 *
T
Tejun Heo 已提交
12
 * Made to use alloc_percpu by Christoph Lameter.
L
Linus Torvalds 已提交
13
 *
T
Tejun Heo 已提交
14 15
 * Copyright (C) 2010		SUSE Linux Products GmbH
 * Copyright (C) 2010		Tejun Heo <tj@kernel.org>
16
 *
T
Tejun Heo 已提交
17 18
 * This is the generic async execution mechanism.  Work items as are
 * executed in process context.  The worker pool is shared and
19 20 21 22
 * automatically managed.  There are two worker pools for each CPU (one for
 * normal work items and the other for high priority ones) and some extra
 * pools for workqueues which are not bound to any specific CPU - the
 * number of these backing pools is dynamic.
T
Tejun Heo 已提交
23 24
 *
 * Please read Documentation/workqueue.txt for details.
L
Linus Torvalds 已提交
25 26
 */

27
#include <linux/export.h>
L
Linus Torvalds 已提交
28 29 30 31 32 33 34 35 36 37
#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/init.h>
#include <linux/signal.h>
#include <linux/completion.h>
#include <linux/workqueue.h>
#include <linux/slab.h>
#include <linux/cpu.h>
#include <linux/notifier.h>
#include <linux/kthread.h>
38
#include <linux/hardirq.h>
39
#include <linux/mempolicy.h>
40
#include <linux/freezer.h>
41 42
#include <linux/kallsyms.h>
#include <linux/debug_locks.h>
43
#include <linux/lockdep.h>
T
Tejun Heo 已提交
44
#include <linux/idr.h>
45
#include <linux/jhash.h>
46
#include <linux/hashtable.h>
47
#include <linux/rculist.h>
48
#include <linux/nodemask.h>
49
#include <linux/moduleparam.h>
50
#include <linux/uaccess.h>
51

52
#include "workqueue_internal.h"
L
Linus Torvalds 已提交
53

T
Tejun Heo 已提交
54
enum {
55 56
	/*
	 * worker_pool flags
57
	 *
58
	 * A bound pool is either associated or disassociated with its CPU.
59 60 61 62 63 64
	 * While associated (!DISASSOCIATED), all workers are bound to the
	 * CPU and none has %WORKER_UNBOUND set and concurrency management
	 * is in effect.
	 *
	 * While DISASSOCIATED, the cpu may be offline and all workers have
	 * %WORKER_UNBOUND set and concurrency management disabled, and may
65
	 * be executing on any CPU.  The pool behaves as an unbound one.
66
	 *
67
	 * Note that DISASSOCIATED should be flipped only while holding
68
	 * attach_mutex to avoid changing binding state while
69
	 * worker_attach_to_pool() is in progress.
70
	 */
71
	POOL_DISASSOCIATED	= 1 << 2,	/* cpu can't serve workers */
72

T
Tejun Heo 已提交
73 74 75
	/* worker flags */
	WORKER_DIE		= 1 << 1,	/* die die die */
	WORKER_IDLE		= 1 << 2,	/* is idle */
76
	WORKER_PREP		= 1 << 3,	/* preparing to run works */
77
	WORKER_CPU_INTENSIVE	= 1 << 6,	/* cpu intensive */
78
	WORKER_UNBOUND		= 1 << 7,	/* worker is unbound */
79
	WORKER_REBOUND		= 1 << 8,	/* worker was rebound */
80

81 82
	WORKER_NOT_RUNNING	= WORKER_PREP | WORKER_CPU_INTENSIVE |
				  WORKER_UNBOUND | WORKER_REBOUND,
83

84
	NR_STD_WORKER_POOLS	= 2,		/* # standard pools per cpu */
85

86
	UNBOUND_POOL_HASH_ORDER	= 6,		/* hashed by pool->attrs */
T
Tejun Heo 已提交
87
	BUSY_WORKER_HASH_ORDER	= 6,		/* 64 pointers */
88

89 90 91
	MAX_IDLE_WORKERS_RATIO	= 4,		/* 1/4 of busy can be idle */
	IDLE_WORKER_TIMEOUT	= 300 * HZ,	/* keep idle ones for 5 mins */

92 93 94
	MAYDAY_INITIAL_TIMEOUT  = HZ / 100 >= 2 ? HZ / 100 : 2,
						/* call for help after 10ms
						   (min two ticks) */
95 96 97 98 99
	MAYDAY_INTERVAL		= HZ / 10,	/* and then every 100ms */
	CREATE_COOLDOWN		= HZ,		/* time to breath after fail */

	/*
	 * Rescue workers are used only on emergencies and shared by
100
	 * all cpus.  Give MIN_NICE.
101
	 */
102 103
	RESCUER_NICE_LEVEL	= MIN_NICE,
	HIGHPRI_NICE_LEVEL	= MIN_NICE,
104 105

	WQ_NAME_LEN		= 24,
T
Tejun Heo 已提交
106
};
L
Linus Torvalds 已提交
107 108

/*
T
Tejun Heo 已提交
109 110
 * Structure fields follow one of the following exclusion rules.
 *
111 112
 * I: Modifiable by initialization/destruction paths and read-only for
 *    everyone else.
T
Tejun Heo 已提交
113
 *
114 115 116
 * P: Preemption protected.  Disabling preemption is enough and should
 *    only be modified and accessed from the local cpu.
 *
117
 * L: pool->lock protected.  Access with pool->lock held.
T
Tejun Heo 已提交
118
 *
119 120 121 122
 * X: During normal operation, modification requires pool->lock and should
 *    be done only from local cpu.  Either disabling preemption on local
 *    cpu or grabbing pool->lock is enough for read access.  If
 *    POOL_DISASSOCIATED is set, it's identical to L.
123
 *
124
 * A: pool->attach_mutex protected.
125
 *
126
 * PL: wq_pool_mutex protected.
127
 *
128
 * PR: wq_pool_mutex protected for writes.  Sched-RCU protected for reads.
129
 *
130 131
 * WQ: wq->mutex protected.
 *
132
 * WR: wq->mutex protected for writes.  Sched-RCU protected for reads.
133 134
 *
 * MD: wq_mayday_lock protected.
L
Linus Torvalds 已提交
135 136
 */

137
/* struct worker is defined in workqueue_internal.h */
T
Tejun Heo 已提交
138

139
struct worker_pool {
140
	spinlock_t		lock;		/* the pool lock */
141
	int			cpu;		/* I: the associated cpu */
142
	int			node;		/* I: the associated node ID */
T
Tejun Heo 已提交
143
	int			id;		/* I: pool ID */
144
	unsigned int		flags;		/* X: flags */
145 146 147

	struct list_head	worklist;	/* L: list of pending works */
	int			nr_workers;	/* L: total number of workers */
148 149

	/* nr_idle includes the ones off idle_list for rebinding */
150 151 152 153 154 155
	int			nr_idle;	/* L: currently idle ones */

	struct list_head	idle_list;	/* X: list of idle workers */
	struct timer_list	idle_timer;	/* L: worker idle timeout */
	struct timer_list	mayday_timer;	/* L: SOS timer for workers */

156
	/* a workers is either on busy_hash or idle_list, or the manager */
157 158 159
	DECLARE_HASHTABLE(busy_hash, BUSY_WORKER_HASH_ORDER);
						/* L: hash of busy workers */

160
	/* see manage_workers() for details on the two manager mutexes */
161
	struct mutex		manager_arb;	/* manager arbitration */
162
	struct worker		*manager;	/* L: purely informational */
163 164
	struct mutex		attach_mutex;	/* attach/detach exclusion */
	struct list_head	workers;	/* A: attached workers */
165
	struct completion	*detach_completion; /* all workers detached */
166

167
	struct ida		worker_ida;	/* worker IDs for task name */
168

T
Tejun Heo 已提交
169
	struct workqueue_attrs	*attrs;		/* I: worker attributes */
170 171
	struct hlist_node	hash_node;	/* PL: unbound_pool_hash node */
	int			refcnt;		/* PL: refcnt for unbound pools */
T
Tejun Heo 已提交
172

173 174 175 176 177 178
	/*
	 * The current concurrency level.  As it's likely to be accessed
	 * from other CPUs during try_to_wake_up(), put it in a separate
	 * cacheline.
	 */
	atomic_t		nr_running ____cacheline_aligned_in_smp;
179 180 181 182 183 184

	/*
	 * Destruction of pool is sched-RCU protected to allow dereferences
	 * from get_work_pool().
	 */
	struct rcu_head		rcu;
185 186
} ____cacheline_aligned_in_smp;

L
Linus Torvalds 已提交
187
/*
188 189 190 191
 * The per-pool workqueue.  While queued, the lower WORK_STRUCT_FLAG_BITS
 * of work_struct->data are used for flags and the remaining high bits
 * point to the pwq; thus, pwqs need to be aligned at two's power of the
 * number of flag bits.
L
Linus Torvalds 已提交
192
 */
193
struct pool_workqueue {
194
	struct worker_pool	*pool;		/* I: the associated pool */
T
Tejun Heo 已提交
195
	struct workqueue_struct *wq;		/* I: the owning workqueue */
196 197
	int			work_color;	/* L: current color */
	int			flush_color;	/* L: flushing color */
T
Tejun Heo 已提交
198
	int			refcnt;		/* L: reference count */
199 200
	int			nr_in_flight[WORK_NR_COLORS];
						/* L: nr of in_flight works */
201
	int			nr_active;	/* L: nr of active works */
202
	int			max_active;	/* L: max active works */
203
	struct list_head	delayed_works;	/* L: delayed works */
204
	struct list_head	pwqs_node;	/* WR: node on wq->pwqs */
205
	struct list_head	mayday_node;	/* MD: node on wq->maydays */
T
Tejun Heo 已提交
206 207 208 209 210

	/*
	 * Release of unbound pwq is punted to system_wq.  See put_pwq()
	 * and pwq_unbound_release_workfn() for details.  pool_workqueue
	 * itself is also sched-RCU protected so that the first pwq can be
211
	 * determined without grabbing wq->mutex.
T
Tejun Heo 已提交
212 213 214
	 */
	struct work_struct	unbound_release_work;
	struct rcu_head		rcu;
215
} __aligned(1 << WORK_STRUCT_FLAG_BITS);
L
Linus Torvalds 已提交
216

217 218 219 220
/*
 * Structure used to wait for workqueue flush.
 */
struct wq_flusher {
221 222
	struct list_head	list;		/* WQ: list of flushers */
	int			flush_color;	/* WQ: flush color waiting for */
223 224 225
	struct completion	done;		/* flush completion */
};

226 227
struct wq_device;

L
Linus Torvalds 已提交
228
/*
229 230
 * The externally visible workqueue.  It relays the issued work items to
 * the appropriate worker_pool through its pool_workqueues.
L
Linus Torvalds 已提交
231 232
 */
struct workqueue_struct {
233
	struct list_head	pwqs;		/* WR: all pwqs of this wq */
234
	struct list_head	list;		/* PR: list of all workqueues */
235

236 237 238
	struct mutex		mutex;		/* protects this wq */
	int			work_color;	/* WQ: current work color */
	int			flush_color;	/* WQ: current flush color */
239
	atomic_t		nr_pwqs_to_flush; /* flush in progress */
240 241 242
	struct wq_flusher	*first_flusher;	/* WQ: first flusher */
	struct list_head	flusher_queue;	/* WQ: flush waiters */
	struct list_head	flusher_overflow; /* WQ: flush overflow list */
243

244
	struct list_head	maydays;	/* MD: pwqs requesting rescue */
245 246
	struct worker		*rescuer;	/* I: rescue worker */

247
	int			nr_drainers;	/* WQ: drain in progress */
248
	int			saved_max_active; /* WQ: saved pwq max_active */
249

250
	struct workqueue_attrs	*unbound_attrs;	/* WQ: only for unbound wqs */
251
	struct pool_workqueue	*dfl_pwq;	/* WQ: only for unbound wqs */
252

253 254 255
#ifdef CONFIG_SYSFS
	struct wq_device	*wq_dev;	/* I: for sysfs interface */
#endif
256
#ifdef CONFIG_LOCKDEP
T
Tejun Heo 已提交
257
	struct lockdep_map	lockdep_map;
258
#endif
259
	char			name[WQ_NAME_LEN]; /* I: workqueue name */
260

261 262 263 264 265 266 267
	/*
	 * Destruction of workqueue_struct is sched-RCU protected to allow
	 * walking the workqueues list without grabbing wq_pool_mutex.
	 * This is used to dump all workqueues from sysrq.
	 */
	struct rcu_head		rcu;

268 269 270
	/* hot fields used during command issue, aligned to cacheline */
	unsigned int		flags ____cacheline_aligned; /* WQ: WQ_* flags */
	struct pool_workqueue __percpu *cpu_pwqs; /* I: per-cpu pwqs */
271
	struct pool_workqueue __rcu *numa_pwq_tbl[]; /* FR: unbound pwqs indexed by node */
L
Linus Torvalds 已提交
272 273
};

274 275
static struct kmem_cache *pwq_cache;

276 277 278
static cpumask_var_t *wq_numa_possible_cpumask;
					/* possible CPUs of each node */

279 280 281
static bool wq_disable_numa;
module_param_named(disable_numa, wq_disable_numa, bool, 0444);

282 283 284 285 286 287 288 289 290
/* see the comment above the definition of WQ_POWER_EFFICIENT */
#ifdef CONFIG_WQ_POWER_EFFICIENT_DEFAULT
static bool wq_power_efficient = true;
#else
static bool wq_power_efficient;
#endif

module_param_named(power_efficient, wq_power_efficient, bool, 0444);

291 292
static bool wq_numa_enabled;		/* unbound NUMA affinity enabled */

293 294 295
/* buf for wq_update_unbound_numa_attrs(), protected by CPU hotplug exclusion */
static struct workqueue_attrs *wq_update_unbound_numa_attrs_buf;

296
static DEFINE_MUTEX(wq_pool_mutex);	/* protects pools and workqueues list */
297
static DEFINE_SPINLOCK(wq_mayday_lock);	/* protects wq->maydays list */
298

299
static LIST_HEAD(workqueues);		/* PR: list of all workqueues */
300
static bool workqueue_freezing;		/* PL: have wqs started freezing? */
301 302 303 304 305

/* the per-cpu worker pools */
static DEFINE_PER_CPU_SHARED_ALIGNED(struct worker_pool [NR_STD_WORKER_POOLS],
				     cpu_worker_pools);

306
static DEFINE_IDR(worker_pool_idr);	/* PR: idr of all pools */
307

308
/* PL: hash of all unbound pools keyed by pool->attrs */
309 310
static DEFINE_HASHTABLE(unbound_pool_hash, UNBOUND_POOL_HASH_ORDER);

311
/* I: attributes used when instantiating standard unbound pools on demand */
312 313
static struct workqueue_attrs *unbound_std_wq_attrs[NR_STD_WORKER_POOLS];

314 315 316
/* I: attributes used when instantiating ordered pools on demand */
static struct workqueue_attrs *ordered_wq_attrs[NR_STD_WORKER_POOLS];

317
struct workqueue_struct *system_wq __read_mostly;
318
EXPORT_SYMBOL(system_wq);
V
Valentin Ilie 已提交
319
struct workqueue_struct *system_highpri_wq __read_mostly;
320
EXPORT_SYMBOL_GPL(system_highpri_wq);
V
Valentin Ilie 已提交
321
struct workqueue_struct *system_long_wq __read_mostly;
322
EXPORT_SYMBOL_GPL(system_long_wq);
V
Valentin Ilie 已提交
323
struct workqueue_struct *system_unbound_wq __read_mostly;
324
EXPORT_SYMBOL_GPL(system_unbound_wq);
V
Valentin Ilie 已提交
325
struct workqueue_struct *system_freezable_wq __read_mostly;
326
EXPORT_SYMBOL_GPL(system_freezable_wq);
327 328 329 330
struct workqueue_struct *system_power_efficient_wq __read_mostly;
EXPORT_SYMBOL_GPL(system_power_efficient_wq);
struct workqueue_struct *system_freezable_power_efficient_wq __read_mostly;
EXPORT_SYMBOL_GPL(system_freezable_power_efficient_wq);
331

332 333 334
static int worker_thread(void *__worker);
static void copy_workqueue_attrs(struct workqueue_attrs *to,
				 const struct workqueue_attrs *from);
335
static void workqueue_sysfs_unregister(struct workqueue_struct *wq);
336

337 338 339
#define CREATE_TRACE_POINTS
#include <trace/events/workqueue.h>

340
#define assert_rcu_or_pool_mutex()					\
341
	rcu_lockdep_assert(rcu_read_lock_sched_held() ||		\
342 343
			   lockdep_is_held(&wq_pool_mutex),		\
			   "sched RCU or wq_pool_mutex should be held")
344

345
#define assert_rcu_or_wq_mutex(wq)					\
346
	rcu_lockdep_assert(rcu_read_lock_sched_held() ||		\
347
			   lockdep_is_held(&wq->mutex),			\
348
			   "sched RCU or wq->mutex should be held")
349

350 351 352
#define for_each_cpu_worker_pool(pool, cpu)				\
	for ((pool) = &per_cpu(cpu_worker_pools, cpu)[0];		\
	     (pool) < &per_cpu(cpu_worker_pools, cpu)[NR_STD_WORKER_POOLS]; \
353
	     (pool)++)
354

T
Tejun Heo 已提交
355 356 357
/**
 * for_each_pool - iterate through all worker_pools in the system
 * @pool: iteration cursor
358
 * @pi: integer used for iteration
359
 *
360 361 362
 * This must be called either with wq_pool_mutex held or sched RCU read
 * locked.  If the pool needs to be used beyond the locking in effect, the
 * caller is responsible for guaranteeing that the pool stays online.
363 364 365
 *
 * The if/else clause exists only for the lockdep assertion and can be
 * ignored.
T
Tejun Heo 已提交
366
 */
367 368
#define for_each_pool(pool, pi)						\
	idr_for_each_entry(&worker_pool_idr, pool, pi)			\
369
		if (({ assert_rcu_or_pool_mutex(); false; })) { }	\
370
		else
T
Tejun Heo 已提交
371

372 373 374 375 376
/**
 * for_each_pool_worker - iterate through all workers of a worker_pool
 * @worker: iteration cursor
 * @pool: worker_pool to iterate workers of
 *
377
 * This must be called with @pool->attach_mutex.
378 379 380 381
 *
 * The if/else clause exists only for the lockdep assertion and can be
 * ignored.
 */
382 383
#define for_each_pool_worker(worker, pool)				\
	list_for_each_entry((worker), &(pool)->workers, node)		\
384
		if (({ lockdep_assert_held(&pool->attach_mutex); false; })) { } \
385 386
		else

387 388 389 390
/**
 * for_each_pwq - iterate through all pool_workqueues of the specified workqueue
 * @pwq: iteration cursor
 * @wq: the target workqueue
391
 *
392
 * This must be called either with wq->mutex held or sched RCU read locked.
393 394
 * If the pwq needs to be used beyond the locking in effect, the caller is
 * responsible for guaranteeing that the pwq stays online.
395 396 397
 *
 * The if/else clause exists only for the lockdep assertion and can be
 * ignored.
398 399
 */
#define for_each_pwq(pwq, wq)						\
400
	list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node)		\
401
		if (({ assert_rcu_or_wq_mutex(wq); false; })) { }	\
402
		else
403

404 405 406 407
#ifdef CONFIG_DEBUG_OBJECTS_WORK

static struct debug_obj_descr work_debug_descr;

408 409 410 411 412
static void *work_debug_hint(void *addr)
{
	return ((struct work_struct *) addr)->func;
}

413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447
/*
 * fixup_init is called when:
 * - an active object is initialized
 */
static int work_fixup_init(void *addr, enum debug_obj_state state)
{
	struct work_struct *work = addr;

	switch (state) {
	case ODEBUG_STATE_ACTIVE:
		cancel_work_sync(work);
		debug_object_init(work, &work_debug_descr);
		return 1;
	default:
		return 0;
	}
}

/*
 * fixup_activate is called when:
 * - an active object is activated
 * - an unknown object is activated (might be a statically initialized object)
 */
static int work_fixup_activate(void *addr, enum debug_obj_state state)
{
	struct work_struct *work = addr;

	switch (state) {

	case ODEBUG_STATE_NOTAVAILABLE:
		/*
		 * This is not really a fixup. The work struct was
		 * statically initialized. We just make sure that it
		 * is tracked in the object tracker.
		 */
448
		if (test_bit(WORK_STRUCT_STATIC_BIT, work_data_bits(work))) {
449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483
			debug_object_init(work, &work_debug_descr);
			debug_object_activate(work, &work_debug_descr);
			return 0;
		}
		WARN_ON_ONCE(1);
		return 0;

	case ODEBUG_STATE_ACTIVE:
		WARN_ON(1);

	default:
		return 0;
	}
}

/*
 * fixup_free is called when:
 * - an active object is freed
 */
static int work_fixup_free(void *addr, enum debug_obj_state state)
{
	struct work_struct *work = addr;

	switch (state) {
	case ODEBUG_STATE_ACTIVE:
		cancel_work_sync(work);
		debug_object_free(work, &work_debug_descr);
		return 1;
	default:
		return 0;
	}
}

static struct debug_obj_descr work_debug_descr = {
	.name		= "work_struct",
484
	.debug_hint	= work_debug_hint,
485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514
	.fixup_init	= work_fixup_init,
	.fixup_activate	= work_fixup_activate,
	.fixup_free	= work_fixup_free,
};

static inline void debug_work_activate(struct work_struct *work)
{
	debug_object_activate(work, &work_debug_descr);
}

static inline void debug_work_deactivate(struct work_struct *work)
{
	debug_object_deactivate(work, &work_debug_descr);
}

void __init_work(struct work_struct *work, int onstack)
{
	if (onstack)
		debug_object_init_on_stack(work, &work_debug_descr);
	else
		debug_object_init(work, &work_debug_descr);
}
EXPORT_SYMBOL_GPL(__init_work);

void destroy_work_on_stack(struct work_struct *work)
{
	debug_object_free(work, &work_debug_descr);
}
EXPORT_SYMBOL_GPL(destroy_work_on_stack);

515 516 517 518 519 520 521
void destroy_delayed_work_on_stack(struct delayed_work *work)
{
	destroy_timer_on_stack(&work->timer);
	debug_object_free(&work->work, &work_debug_descr);
}
EXPORT_SYMBOL_GPL(destroy_delayed_work_on_stack);

522 523 524 525 526
#else
static inline void debug_work_activate(struct work_struct *work) { }
static inline void debug_work_deactivate(struct work_struct *work) { }
#endif

527 528 529 530 531 532 533
/**
 * worker_pool_assign_id - allocate ID and assing it to @pool
 * @pool: the pool pointer of interest
 *
 * Returns 0 if ID in [0, WORK_OFFQ_POOL_NONE) is allocated and assigned
 * successfully, -errno on failure.
 */
T
Tejun Heo 已提交
534 535 536 537
static int worker_pool_assign_id(struct worker_pool *pool)
{
	int ret;

538
	lockdep_assert_held(&wq_pool_mutex);
539

540 541
	ret = idr_alloc(&worker_pool_idr, pool, 0, WORK_OFFQ_POOL_NONE,
			GFP_KERNEL);
542
	if (ret >= 0) {
T
Tejun Heo 已提交
543
		pool->id = ret;
544 545
		return 0;
	}
546
	return ret;
547 548
}

549 550 551 552 553 554 555 556
/**
 * unbound_pwq_by_node - return the unbound pool_workqueue for the given node
 * @wq: the target workqueue
 * @node: the node ID
 *
 * This must be called either with pwq_lock held or sched RCU read locked.
 * If the pwq needs to be used beyond the locking in effect, the caller is
 * responsible for guaranteeing that the pwq stays online.
557 558
 *
 * Return: The unbound pool_workqueue for @node.
559 560 561 562 563 564 565 566
 */
static struct pool_workqueue *unbound_pwq_by_node(struct workqueue_struct *wq,
						  int node)
{
	assert_rcu_or_wq_mutex(wq);
	return rcu_dereference_raw(wq->numa_pwq_tbl[node]);
}

567 568 569 570 571 572 573 574 575 576 577 578 579 580 581
static unsigned int work_color_to_flags(int color)
{
	return color << WORK_STRUCT_COLOR_SHIFT;
}

static int get_work_color(struct work_struct *work)
{
	return (*work_data_bits(work) >> WORK_STRUCT_COLOR_SHIFT) &
		((1 << WORK_STRUCT_COLOR_BITS) - 1);
}

static int work_next_color(int color)
{
	return (color + 1) % WORK_NR_COLORS;
}
L
Linus Torvalds 已提交
582

583
/*
584 585
 * While queued, %WORK_STRUCT_PWQ is set and non flag bits of a work's data
 * contain the pointer to the queued pwq.  Once execution starts, the flag
586
 * is cleared and the high bits contain OFFQ flags and pool ID.
587
 *
588 589
 * set_work_pwq(), set_work_pool_and_clear_pending(), mark_work_canceling()
 * and clear_work_data() can be used to set the pwq, pool or clear
590 591
 * work->data.  These functions should only be called while the work is
 * owned - ie. while the PENDING bit is set.
592
 *
593
 * get_work_pool() and get_work_pwq() can be used to obtain the pool or pwq
594
 * corresponding to a work.  Pool is available once the work has been
595
 * queued anywhere after initialization until it is sync canceled.  pwq is
596
 * available only while the work item is queued.
597
 *
598 599 600 601
 * %WORK_OFFQ_CANCELING is used to mark a work item which is being
 * canceled.  While being canceled, a work item may have its PENDING set
 * but stay off timer and worklist for arbitrarily long and nobody should
 * try to steal the PENDING bit.
602
 */
603 604
static inline void set_work_data(struct work_struct *work, unsigned long data,
				 unsigned long flags)
605
{
606
	WARN_ON_ONCE(!work_pending(work));
607 608
	atomic_long_set(&work->data, data | flags | work_static(work));
}
609

610
static void set_work_pwq(struct work_struct *work, struct pool_workqueue *pwq,
611 612
			 unsigned long extra_flags)
{
613 614
	set_work_data(work, (unsigned long)pwq,
		      WORK_STRUCT_PENDING | WORK_STRUCT_PWQ | extra_flags);
615 616
}

617 618 619 620 621 622 623
static void set_work_pool_and_keep_pending(struct work_struct *work,
					   int pool_id)
{
	set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT,
		      WORK_STRUCT_PENDING);
}

624 625
static void set_work_pool_and_clear_pending(struct work_struct *work,
					    int pool_id)
626
{
627 628 629 630 631 632 633
	/*
	 * The following wmb is paired with the implied mb in
	 * test_and_set_bit(PENDING) and ensures all updates to @work made
	 * here are visible to and precede any updates by the next PENDING
	 * owner.
	 */
	smp_wmb();
634
	set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT, 0);
635
}
636

637
static void clear_work_data(struct work_struct *work)
L
Linus Torvalds 已提交
638
{
639 640
	smp_wmb();	/* see set_work_pool_and_clear_pending() */
	set_work_data(work, WORK_STRUCT_NO_POOL, 0);
L
Linus Torvalds 已提交
641 642
}

643
static struct pool_workqueue *get_work_pwq(struct work_struct *work)
644
{
645
	unsigned long data = atomic_long_read(&work->data);
646

647
	if (data & WORK_STRUCT_PWQ)
648 649 650
		return (void *)(data & WORK_STRUCT_WQ_DATA_MASK);
	else
		return NULL;
651 652
}

653 654 655 656
/**
 * get_work_pool - return the worker_pool a given work was associated with
 * @work: the work item of interest
 *
657 658 659
 * Pools are created and destroyed under wq_pool_mutex, and allows read
 * access under sched-RCU read lock.  As such, this function should be
 * called under wq_pool_mutex or with preemption disabled.
660 661 662 663 664
 *
 * All fields of the returned pool are accessible as long as the above
 * mentioned locking is in effect.  If the returned pool needs to be used
 * beyond the critical section, the caller is responsible for ensuring the
 * returned pool is and stays online.
665 666
 *
 * Return: The worker_pool @work was last associated with.  %NULL if none.
667 668
 */
static struct worker_pool *get_work_pool(struct work_struct *work)
669
{
670
	unsigned long data = atomic_long_read(&work->data);
671
	int pool_id;
672

673
	assert_rcu_or_pool_mutex();
674

675 676
	if (data & WORK_STRUCT_PWQ)
		return ((struct pool_workqueue *)
677
			(data & WORK_STRUCT_WQ_DATA_MASK))->pool;
678

679 680
	pool_id = data >> WORK_OFFQ_POOL_SHIFT;
	if (pool_id == WORK_OFFQ_POOL_NONE)
681 682
		return NULL;

683
	return idr_find(&worker_pool_idr, pool_id);
684 685 686 687 688 689
}

/**
 * get_work_pool_id - return the worker pool ID a given work is associated with
 * @work: the work item of interest
 *
690
 * Return: The worker_pool ID @work was last associated with.
691 692 693 694
 * %WORK_OFFQ_POOL_NONE if none.
 */
static int get_work_pool_id(struct work_struct *work)
{
695 696
	unsigned long data = atomic_long_read(&work->data);

697 698
	if (data & WORK_STRUCT_PWQ)
		return ((struct pool_workqueue *)
699
			(data & WORK_STRUCT_WQ_DATA_MASK))->pool->id;
700

701
	return data >> WORK_OFFQ_POOL_SHIFT;
702 703
}

704 705
static void mark_work_canceling(struct work_struct *work)
{
706
	unsigned long pool_id = get_work_pool_id(work);
707

708 709
	pool_id <<= WORK_OFFQ_POOL_SHIFT;
	set_work_data(work, pool_id | WORK_OFFQ_CANCELING, WORK_STRUCT_PENDING);
710 711 712 713 714 715
}

static bool work_is_canceling(struct work_struct *work)
{
	unsigned long data = atomic_long_read(&work->data);

716
	return !(data & WORK_STRUCT_PWQ) && (data & WORK_OFFQ_CANCELING);
717 718
}

719
/*
720 721
 * Policy functions.  These define the policies on how the global worker
 * pools are managed.  Unless noted otherwise, these functions assume that
722
 * they're being called with pool->lock held.
723 724
 */

725
static bool __need_more_worker(struct worker_pool *pool)
726
{
727
	return !atomic_read(&pool->nr_running);
728 729
}

730
/*
731 732
 * Need to wake up a worker?  Called from anything but currently
 * running workers.
733 734
 *
 * Note that, because unbound workers never contribute to nr_running, this
735
 * function will always return %true for unbound pools as long as the
736
 * worklist isn't empty.
737
 */
738
static bool need_more_worker(struct worker_pool *pool)
739
{
740
	return !list_empty(&pool->worklist) && __need_more_worker(pool);
741
}
742

743
/* Can I start working?  Called from busy but !running workers. */
744
static bool may_start_working(struct worker_pool *pool)
745
{
746
	return pool->nr_idle;
747 748 749
}

/* Do I need to keep working?  Called from currently running workers. */
750
static bool keep_working(struct worker_pool *pool)
751
{
752 753
	return !list_empty(&pool->worklist) &&
		atomic_read(&pool->nr_running) <= 1;
754 755 756
}

/* Do we need a new worker?  Called from manager. */
757
static bool need_to_create_worker(struct worker_pool *pool)
758
{
759
	return need_more_worker(pool) && !may_start_working(pool);
760
}
761

762
/* Do we have too many workers and should some go away? */
763
static bool too_many_workers(struct worker_pool *pool)
764
{
765
	bool managing = mutex_is_locked(&pool->manager_arb);
766 767
	int nr_idle = pool->nr_idle + managing; /* manager is considered idle */
	int nr_busy = pool->nr_workers - nr_idle;
768 769

	return nr_idle > 2 && (nr_idle - 2) * MAX_IDLE_WORKERS_RATIO >= nr_busy;
770 771
}

772
/*
773 774 775
 * Wake up functions.
 */

776 777
/* Return the first idle worker.  Safe with preemption disabled */
static struct worker *first_idle_worker(struct worker_pool *pool)
778
{
779
	if (unlikely(list_empty(&pool->idle_list)))
780 781
		return NULL;

782
	return list_first_entry(&pool->idle_list, struct worker, entry);
783 784 785 786
}

/**
 * wake_up_worker - wake up an idle worker
787
 * @pool: worker pool to wake worker from
788
 *
789
 * Wake up the first idle worker of @pool.
790 791
 *
 * CONTEXT:
792
 * spin_lock_irq(pool->lock).
793
 */
794
static void wake_up_worker(struct worker_pool *pool)
795
{
796
	struct worker *worker = first_idle_worker(pool);
797 798 799 800 801

	if (likely(worker))
		wake_up_process(worker->task);
}

802
/**
803 804 805 806 807 808 809 810 811 812
 * wq_worker_waking_up - a worker is waking up
 * @task: task waking up
 * @cpu: CPU @task is waking up to
 *
 * This function is called during try_to_wake_up() when a worker is
 * being awoken.
 *
 * CONTEXT:
 * spin_lock_irq(rq->lock)
 */
813
void wq_worker_waking_up(struct task_struct *task, int cpu)
814 815 816
{
	struct worker *worker = kthread_data(task);

817
	if (!(worker->flags & WORKER_NOT_RUNNING)) {
818
		WARN_ON_ONCE(worker->pool->cpu != cpu);
819
		atomic_inc(&worker->pool->nr_running);
820
	}
821 822 823 824 825 826 827 828 829 830 831 832 833 834
}

/**
 * wq_worker_sleeping - a worker is going to sleep
 * @task: task going to sleep
 * @cpu: CPU in question, must be the current CPU number
 *
 * This function is called during schedule() when a busy worker is
 * going to sleep.  Worker on the same cpu can be woken up by
 * returning pointer to its task.
 *
 * CONTEXT:
 * spin_lock_irq(rq->lock)
 *
835
 * Return:
836 837
 * Worker task on @cpu to wake up, %NULL if none.
 */
838
struct task_struct *wq_worker_sleeping(struct task_struct *task, int cpu)
839 840
{
	struct worker *worker = kthread_data(task), *to_wakeup = NULL;
841
	struct worker_pool *pool;
842

843 844 845 846 847
	/*
	 * Rescuers, which may not have all the fields set up like normal
	 * workers, also reach here, let's not access anything before
	 * checking NOT_RUNNING.
	 */
848
	if (worker->flags & WORKER_NOT_RUNNING)
849 850
		return NULL;

851 852
	pool = worker->pool;

853
	/* this can only happen on the local cpu */
854
	if (WARN_ON_ONCE(cpu != raw_smp_processor_id() || pool->cpu != cpu))
855
		return NULL;
856 857 858 859 860 861

	/*
	 * The counterpart of the following dec_and_test, implied mb,
	 * worklist not empty test sequence is in insert_work().
	 * Please read comment there.
	 *
862 863 864
	 * NOT_RUNNING is clear.  This means that we're bound to and
	 * running on the local cpu w/ rq lock held and preemption
	 * disabled, which in turn means that none else could be
865
	 * manipulating idle_list, so dereferencing idle_list without pool
866
	 * lock is safe.
867
	 */
868 869
	if (atomic_dec_and_test(&pool->nr_running) &&
	    !list_empty(&pool->worklist))
870
		to_wakeup = first_idle_worker(pool);
871 872 873 874 875
	return to_wakeup ? to_wakeup->task : NULL;
}

/**
 * worker_set_flags - set worker flags and adjust nr_running accordingly
876
 * @worker: self
877 878
 * @flags: flags to set
 *
879
 * Set @flags in @worker->flags and adjust nr_running accordingly.
880
 *
881
 * CONTEXT:
882
 * spin_lock_irq(pool->lock)
883
 */
884
static inline void worker_set_flags(struct worker *worker, unsigned int flags)
885
{
886
	struct worker_pool *pool = worker->pool;
887

888 889
	WARN_ON_ONCE(worker->task != current);

890
	/* If transitioning into NOT_RUNNING, adjust nr_running. */
891 892
	if ((flags & WORKER_NOT_RUNNING) &&
	    !(worker->flags & WORKER_NOT_RUNNING)) {
893
		atomic_dec(&pool->nr_running);
894 895
	}

896 897 898 899
	worker->flags |= flags;
}

/**
900
 * worker_clr_flags - clear worker flags and adjust nr_running accordingly
901
 * @worker: self
902 903
 * @flags: flags to clear
 *
904
 * Clear @flags in @worker->flags and adjust nr_running accordingly.
905
 *
906
 * CONTEXT:
907
 * spin_lock_irq(pool->lock)
908 909 910
 */
static inline void worker_clr_flags(struct worker *worker, unsigned int flags)
{
911
	struct worker_pool *pool = worker->pool;
912 913
	unsigned int oflags = worker->flags;

914 915
	WARN_ON_ONCE(worker->task != current);

916
	worker->flags &= ~flags;
917

918 919 920 921 922
	/*
	 * If transitioning out of NOT_RUNNING, increment nr_running.  Note
	 * that the nested NOT_RUNNING is not a noop.  NOT_RUNNING is mask
	 * of multiple flags, not a single flag.
	 */
923 924
	if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING))
		if (!(worker->flags & WORKER_NOT_RUNNING))
925
			atomic_inc(&pool->nr_running);
926 927
}

928 929
/**
 * find_worker_executing_work - find worker which is executing a work
930
 * @pool: pool of interest
931 932
 * @work: work to find worker for
 *
933 934
 * Find a worker which is executing @work on @pool by searching
 * @pool->busy_hash which is keyed by the address of @work.  For a worker
935 936 937 938 939 940 941 942 943 944 945 946
 * to match, its current execution should match the address of @work and
 * its work function.  This is to avoid unwanted dependency between
 * unrelated work executions through a work item being recycled while still
 * being executed.
 *
 * This is a bit tricky.  A work item may be freed once its execution
 * starts and nothing prevents the freed area from being recycled for
 * another work item.  If the same work item address ends up being reused
 * before the original execution finishes, workqueue will identify the
 * recycled work item as currently executing and make it wait until the
 * current execution finishes, introducing an unwanted dependency.
 *
947 948 949 950 951 952
 * This function checks the work item address and work function to avoid
 * false positives.  Note that this isn't complete as one may construct a
 * work function which can introduce dependency onto itself through a
 * recycled work item.  Well, if somebody wants to shoot oneself in the
 * foot that badly, there's only so much we can do, and if such deadlock
 * actually occurs, it should be easy to locate the culprit work function.
953 954
 *
 * CONTEXT:
955
 * spin_lock_irq(pool->lock).
956
 *
957 958
 * Return:
 * Pointer to worker which is executing @work if found, %NULL
959
 * otherwise.
960
 */
961
static struct worker *find_worker_executing_work(struct worker_pool *pool,
962
						 struct work_struct *work)
963
{
964 965
	struct worker *worker;

966
	hash_for_each_possible(pool->busy_hash, worker, hentry,
967 968 969
			       (unsigned long)work)
		if (worker->current_work == work &&
		    worker->current_func == work->func)
970 971 972
			return worker;

	return NULL;
973 974
}

975 976 977 978 979 980 981 982 983 984 985 986 987 988 989
/**
 * move_linked_works - move linked works to a list
 * @work: start of series of works to be scheduled
 * @head: target list to append @work to
 * @nextp: out paramter for nested worklist walking
 *
 * Schedule linked works starting from @work to @head.  Work series to
 * be scheduled starts at @work and includes any consecutive work with
 * WORK_STRUCT_LINKED set in its predecessor.
 *
 * If @nextp is not NULL, it's updated to point to the next work of
 * the last scheduled work.  This allows move_linked_works() to be
 * nested inside outer list_for_each_entry_safe().
 *
 * CONTEXT:
990
 * spin_lock_irq(pool->lock).
991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015
 */
static void move_linked_works(struct work_struct *work, struct list_head *head,
			      struct work_struct **nextp)
{
	struct work_struct *n;

	/*
	 * Linked worklist will always end before the end of the list,
	 * use NULL for list head.
	 */
	list_for_each_entry_safe_from(work, n, NULL, entry) {
		list_move_tail(&work->entry, head);
		if (!(*work_data_bits(work) & WORK_STRUCT_LINKED))
			break;
	}

	/*
	 * If we're already inside safe list traversal and have moved
	 * multiple works to the scheduled queue, the next position
	 * needs to be updated.
	 */
	if (nextp)
		*nextp = n;
}

T
Tejun Heo 已提交
1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054
/**
 * get_pwq - get an extra reference on the specified pool_workqueue
 * @pwq: pool_workqueue to get
 *
 * Obtain an extra reference on @pwq.  The caller should guarantee that
 * @pwq has positive refcnt and be holding the matching pool->lock.
 */
static void get_pwq(struct pool_workqueue *pwq)
{
	lockdep_assert_held(&pwq->pool->lock);
	WARN_ON_ONCE(pwq->refcnt <= 0);
	pwq->refcnt++;
}

/**
 * put_pwq - put a pool_workqueue reference
 * @pwq: pool_workqueue to put
 *
 * Drop a reference of @pwq.  If its refcnt reaches zero, schedule its
 * destruction.  The caller should be holding the matching pool->lock.
 */
static void put_pwq(struct pool_workqueue *pwq)
{
	lockdep_assert_held(&pwq->pool->lock);
	if (likely(--pwq->refcnt))
		return;
	if (WARN_ON_ONCE(!(pwq->wq->flags & WQ_UNBOUND)))
		return;
	/*
	 * @pwq can't be released under pool->lock, bounce to
	 * pwq_unbound_release_workfn().  This never recurses on the same
	 * pool->lock as this path is taken only for unbound workqueues and
	 * the release work item is scheduled on a per-cpu workqueue.  To
	 * avoid lockdep warning, unbound pool->locks are given lockdep
	 * subclass of 1 in get_unbound_pool().
	 */
	schedule_work(&pwq->unbound_release_work);
}

1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073
/**
 * put_pwq_unlocked - put_pwq() with surrounding pool lock/unlock
 * @pwq: pool_workqueue to put (can be %NULL)
 *
 * put_pwq() with locking.  This function also allows %NULL @pwq.
 */
static void put_pwq_unlocked(struct pool_workqueue *pwq)
{
	if (pwq) {
		/*
		 * As both pwqs and pools are sched-RCU protected, the
		 * following lock operations are safe.
		 */
		spin_lock_irq(&pwq->pool->lock);
		put_pwq(pwq);
		spin_unlock_irq(&pwq->pool->lock);
	}
}

1074
static void pwq_activate_delayed_work(struct work_struct *work)
1075
{
1076
	struct pool_workqueue *pwq = get_work_pwq(work);
1077 1078

	trace_workqueue_activate_work(work);
1079
	move_linked_works(work, &pwq->pool->worklist, NULL);
1080
	__clear_bit(WORK_STRUCT_DELAYED_BIT, work_data_bits(work));
1081
	pwq->nr_active++;
1082 1083
}

1084
static void pwq_activate_first_delayed(struct pool_workqueue *pwq)
1085
{
1086
	struct work_struct *work = list_first_entry(&pwq->delayed_works,
1087 1088
						    struct work_struct, entry);

1089
	pwq_activate_delayed_work(work);
1090 1091
}

1092
/**
1093 1094
 * pwq_dec_nr_in_flight - decrement pwq's nr_in_flight
 * @pwq: pwq of interest
1095 1096 1097
 * @color: color of work which left the queue
 *
 * A work either has completed or is removed from pending queue,
1098
 * decrement nr_in_flight of its pwq and handle workqueue flushing.
1099 1100
 *
 * CONTEXT:
1101
 * spin_lock_irq(pool->lock).
1102
 */
1103
static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, int color)
1104
{
T
Tejun Heo 已提交
1105
	/* uncolored work items don't participate in flushing or nr_active */
1106
	if (color == WORK_NO_COLOR)
T
Tejun Heo 已提交
1107
		goto out_put;
1108

1109
	pwq->nr_in_flight[color]--;
1110

1111 1112
	pwq->nr_active--;
	if (!list_empty(&pwq->delayed_works)) {
1113
		/* one down, submit a delayed one */
1114 1115
		if (pwq->nr_active < pwq->max_active)
			pwq_activate_first_delayed(pwq);
1116 1117 1118
	}

	/* is flush in progress and are we at the flushing tip? */
1119
	if (likely(pwq->flush_color != color))
T
Tejun Heo 已提交
1120
		goto out_put;
1121 1122

	/* are there still in-flight works? */
1123
	if (pwq->nr_in_flight[color])
T
Tejun Heo 已提交
1124
		goto out_put;
1125

1126 1127
	/* this pwq is done, clear flush_color */
	pwq->flush_color = -1;
1128 1129

	/*
1130
	 * If this was the last pwq, wake up the first flusher.  It
1131 1132
	 * will handle the rest.
	 */
1133 1134
	if (atomic_dec_and_test(&pwq->wq->nr_pwqs_to_flush))
		complete(&pwq->wq->first_flusher->done);
T
Tejun Heo 已提交
1135 1136
out_put:
	put_pwq(pwq);
1137 1138
}

1139
/**
1140
 * try_to_grab_pending - steal work item from worklist and disable irq
1141 1142
 * @work: work item to steal
 * @is_dwork: @work is a delayed_work
1143
 * @flags: place to store irq state
1144 1145
 *
 * Try to grab PENDING bit of @work.  This function can handle @work in any
1146
 * stable state - idle, on timer or on worklist.
1147
 *
1148
 * Return:
1149 1150 1151
 *  1		if @work was pending and we successfully stole PENDING
 *  0		if @work was idle and we claimed PENDING
 *  -EAGAIN	if PENDING couldn't be grabbed at the moment, safe to busy-retry
1152 1153
 *  -ENOENT	if someone else is canceling @work, this state may persist
 *		for arbitrarily long
1154
 *
1155
 * Note:
1156
 * On >= 0 return, the caller owns @work's PENDING bit.  To avoid getting
1157 1158 1159
 * interrupted while holding PENDING and @work off queue, irq must be
 * disabled on entry.  This, combined with delayed_work->timer being
 * irqsafe, ensures that we return -EAGAIN for finite short period of time.
1160 1161 1162 1163
 *
 * On successful return, >= 0, irq is disabled and the caller is
 * responsible for releasing it using local_irq_restore(*@flags).
 *
1164
 * This function is safe to call from any context including IRQ handler.
1165
 */
1166 1167
static int try_to_grab_pending(struct work_struct *work, bool is_dwork,
			       unsigned long *flags)
1168
{
1169
	struct worker_pool *pool;
1170
	struct pool_workqueue *pwq;
1171

1172 1173
	local_irq_save(*flags);

1174 1175 1176 1177
	/* try to steal the timer if it exists */
	if (is_dwork) {
		struct delayed_work *dwork = to_delayed_work(work);

1178 1179 1180 1181 1182
		/*
		 * dwork->timer is irqsafe.  If del_timer() fails, it's
		 * guaranteed that the timer is not queued anywhere and not
		 * running on the local CPU.
		 */
1183 1184 1185 1186 1187
		if (likely(del_timer(&dwork->timer)))
			return 1;
	}

	/* try to claim PENDING the normal way */
1188 1189 1190 1191 1192 1193 1194
	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)))
		return 0;

	/*
	 * The queueing is in progress, or it is already queued. Try to
	 * steal it from ->worklist without clearing WORK_STRUCT_PENDING.
	 */
1195 1196
	pool = get_work_pool(work);
	if (!pool)
1197
		goto fail;
1198

1199
	spin_lock(&pool->lock);
1200
	/*
1201 1202 1203 1204 1205
	 * work->data is guaranteed to point to pwq only while the work
	 * item is queued on pwq->wq, and both updating work->data to point
	 * to pwq on queueing and to pool on dequeueing are done under
	 * pwq->pool->lock.  This in turn guarantees that, if work->data
	 * points to pwq which is associated with a locked pool, the work
1206 1207
	 * item is currently queued on that pool.
	 */
1208 1209
	pwq = get_work_pwq(work);
	if (pwq && pwq->pool == pool) {
1210 1211 1212 1213 1214
		debug_work_deactivate(work);

		/*
		 * A delayed work item cannot be grabbed directly because
		 * it might have linked NO_COLOR work items which, if left
1215
		 * on the delayed_list, will confuse pwq->nr_active
1216 1217 1218 1219
		 * management later on and cause stall.  Make sure the work
		 * item is activated before grabbing.
		 */
		if (*work_data_bits(work) & WORK_STRUCT_DELAYED)
1220
			pwq_activate_delayed_work(work);
1221 1222

		list_del_init(&work->entry);
1223
		pwq_dec_nr_in_flight(pwq, get_work_color(work));
1224

1225
		/* work->data points to pwq iff queued, point to pool */
1226 1227 1228 1229
		set_work_pool_and_keep_pending(work, pool->id);

		spin_unlock(&pool->lock);
		return 1;
1230
	}
1231
	spin_unlock(&pool->lock);
1232 1233 1234 1235 1236
fail:
	local_irq_restore(*flags);
	if (work_is_canceling(work))
		return -ENOENT;
	cpu_relax();
1237
	return -EAGAIN;
1238 1239
}

T
Tejun Heo 已提交
1240
/**
1241
 * insert_work - insert a work into a pool
1242
 * @pwq: pwq @work belongs to
T
Tejun Heo 已提交
1243 1244 1245 1246
 * @work: work to insert
 * @head: insertion point
 * @extra_flags: extra WORK_STRUCT_* flags to set
 *
1247
 * Insert @work which belongs to @pwq after @head.  @extra_flags is or'd to
1248
 * work_struct flags.
T
Tejun Heo 已提交
1249 1250
 *
 * CONTEXT:
1251
 * spin_lock_irq(pool->lock).
T
Tejun Heo 已提交
1252
 */
1253 1254
static void insert_work(struct pool_workqueue *pwq, struct work_struct *work,
			struct list_head *head, unsigned int extra_flags)
O
Oleg Nesterov 已提交
1255
{
1256
	struct worker_pool *pool = pwq->pool;
1257

T
Tejun Heo 已提交
1258
	/* we own @work, set data and link */
1259
	set_work_pwq(work, pwq, extra_flags);
1260
	list_add_tail(&work->entry, head);
T
Tejun Heo 已提交
1261
	get_pwq(pwq);
1262 1263

	/*
1264 1265 1266
	 * Ensure either wq_worker_sleeping() sees the above
	 * list_add_tail() or we see zero nr_running to avoid workers lying
	 * around lazily while there are works to be processed.
1267 1268 1269
	 */
	smp_mb();

1270 1271
	if (__need_more_worker(pool))
		wake_up_worker(pool);
O
Oleg Nesterov 已提交
1272 1273
}

1274 1275
/*
 * Test whether @work is being queued from another work executing on the
1276
 * same workqueue.
1277 1278 1279
 */
static bool is_chained_work(struct workqueue_struct *wq)
{
1280 1281 1282 1283 1284 1285 1286
	struct worker *worker;

	worker = current_wq_worker();
	/*
	 * Return %true iff I'm a worker execuing a work item on @wq.  If
	 * I'm @worker, it's safe to dereference it without locking.
	 */
1287
	return worker && worker->current_pwq->wq == wq;
1288 1289
}

1290
static void __queue_work(int cpu, struct workqueue_struct *wq,
L
Linus Torvalds 已提交
1291 1292
			 struct work_struct *work)
{
1293
	struct pool_workqueue *pwq;
1294
	struct worker_pool *last_pool;
1295
	struct list_head *worklist;
1296
	unsigned int work_flags;
1297
	unsigned int req_cpu = cpu;
1298 1299 1300 1301 1302 1303 1304 1305

	/*
	 * While a work item is PENDING && off queue, a task trying to
	 * steal the PENDING will busy-loop waiting for it to either get
	 * queued or lose PENDING.  Grabbing PENDING and queueing should
	 * happen with IRQ disabled.
	 */
	WARN_ON_ONCE(!irqs_disabled());
L
Linus Torvalds 已提交
1306

1307
	debug_work_activate(work);
1308

1309
	/* if draining, only works from the same workqueue are allowed */
1310
	if (unlikely(wq->flags & __WQ_DRAINING) &&
1311
	    WARN_ON_ONCE(!is_chained_work(wq)))
1312
		return;
1313
retry:
1314 1315 1316
	if (req_cpu == WORK_CPU_UNBOUND)
		cpu = raw_smp_processor_id();

1317
	/* pwq which will be used unless @work is executing elsewhere */
1318
	if (!(wq->flags & WQ_UNBOUND))
1319
		pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);
1320 1321
	else
		pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));
1322

1323 1324 1325 1326 1327 1328 1329 1330
	/*
	 * If @work was previously on a different pool, it might still be
	 * running there, in which case the work needs to be queued on that
	 * pool to guarantee non-reentrancy.
	 */
	last_pool = get_work_pool(work);
	if (last_pool && last_pool != pwq->pool) {
		struct worker *worker;
1331

1332
		spin_lock(&last_pool->lock);
1333

1334
		worker = find_worker_executing_work(last_pool, work);
1335

1336 1337
		if (worker && worker->current_pwq->wq == wq) {
			pwq = worker->current_pwq;
1338
		} else {
1339 1340
			/* meh... not running there, queue here */
			spin_unlock(&last_pool->lock);
1341
			spin_lock(&pwq->pool->lock);
1342
		}
1343
	} else {
1344
		spin_lock(&pwq->pool->lock);
1345 1346
	}

1347 1348 1349 1350
	/*
	 * pwq is determined and locked.  For unbound pools, we could have
	 * raced with pwq release and it could already be dead.  If its
	 * refcnt is zero, repeat pwq selection.  Note that pwqs never die
1351 1352
	 * without another pwq replacing it in the numa_pwq_tbl or while
	 * work items are executing on it, so the retrying is guaranteed to
1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365
	 * make forward-progress.
	 */
	if (unlikely(!pwq->refcnt)) {
		if (wq->flags & WQ_UNBOUND) {
			spin_unlock(&pwq->pool->lock);
			cpu_relax();
			goto retry;
		}
		/* oops */
		WARN_ONCE(true, "workqueue: per-cpu pwq for %s on cpu%d has 0 refcnt",
			  wq->name, cpu);
	}

1366 1367
	/* pwq determined, queue */
	trace_workqueue_queue_work(req_cpu, pwq, work);
1368

1369
	if (WARN_ON(!list_empty(&work->entry))) {
1370
		spin_unlock(&pwq->pool->lock);
1371 1372
		return;
	}
1373

1374 1375
	pwq->nr_in_flight[pwq->work_color]++;
	work_flags = work_color_to_flags(pwq->work_color);
1376

1377
	if (likely(pwq->nr_active < pwq->max_active)) {
1378
		trace_workqueue_activate_work(work);
1379 1380
		pwq->nr_active++;
		worklist = &pwq->pool->worklist;
1381 1382
	} else {
		work_flags |= WORK_STRUCT_DELAYED;
1383
		worklist = &pwq->delayed_works;
1384
	}
1385

1386
	insert_work(pwq, work, worklist, work_flags);
1387

1388
	spin_unlock(&pwq->pool->lock);
L
Linus Torvalds 已提交
1389 1390
}

1391
/**
1392 1393
 * queue_work_on - queue work on specific cpu
 * @cpu: CPU number to execute work on
1394 1395 1396
 * @wq: workqueue to use
 * @work: work to queue
 *
1397 1398
 * We queue the work to a specific CPU, the caller must ensure it
 * can't go away.
1399 1400
 *
 * Return: %false if @work was already on a queue, %true otherwise.
L
Linus Torvalds 已提交
1401
 */
1402 1403
bool queue_work_on(int cpu, struct workqueue_struct *wq,
		   struct work_struct *work)
L
Linus Torvalds 已提交
1404
{
1405
	bool ret = false;
1406
	unsigned long flags;
1407

1408
	local_irq_save(flags);
1409

1410
	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
T
Tejun Heo 已提交
1411
		__queue_work(cpu, wq, work);
1412
		ret = true;
1413
	}
1414

1415
	local_irq_restore(flags);
L
Linus Torvalds 已提交
1416 1417
	return ret;
}
1418
EXPORT_SYMBOL(queue_work_on);
L
Linus Torvalds 已提交
1419

1420
void delayed_work_timer_fn(unsigned long __data)
L
Linus Torvalds 已提交
1421
{
1422
	struct delayed_work *dwork = (struct delayed_work *)__data;
L
Linus Torvalds 已提交
1423

1424
	/* should have been called from irqsafe timer with irq already off */
1425
	__queue_work(dwork->cpu, dwork->wq, &dwork->work);
L
Linus Torvalds 已提交
1426
}
1427
EXPORT_SYMBOL(delayed_work_timer_fn);
L
Linus Torvalds 已提交
1428

1429 1430
static void __queue_delayed_work(int cpu, struct workqueue_struct *wq,
				struct delayed_work *dwork, unsigned long delay)
L
Linus Torvalds 已提交
1431
{
1432 1433 1434 1435 1436
	struct timer_list *timer = &dwork->timer;
	struct work_struct *work = &dwork->work;

	WARN_ON_ONCE(timer->function != delayed_work_timer_fn ||
		     timer->data != (unsigned long)dwork);
1437 1438
	WARN_ON_ONCE(timer_pending(timer));
	WARN_ON_ONCE(!list_empty(&work->entry));
1439

1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450
	/*
	 * If @delay is 0, queue @dwork->work immediately.  This is for
	 * both optimization and correctness.  The earliest @timer can
	 * expire is on the closest next tick and delayed_work users depend
	 * on that there's no such delay when @delay is 0.
	 */
	if (!delay) {
		__queue_work(cpu, wq, &dwork->work);
		return;
	}

1451
	timer_stats_timer_set_start_info(&dwork->timer);
L
Linus Torvalds 已提交
1452

1453
	dwork->wq = wq;
1454
	dwork->cpu = cpu;
1455 1456 1457 1458 1459 1460
	timer->expires = jiffies + delay;

	if (unlikely(cpu != WORK_CPU_UNBOUND))
		add_timer_on(timer, cpu);
	else
		add_timer(timer);
L
Linus Torvalds 已提交
1461 1462
}

1463 1464 1465 1466
/**
 * queue_delayed_work_on - queue work on specific CPU after delay
 * @cpu: CPU number to execute work on
 * @wq: workqueue to use
1467
 * @dwork: work to queue
1468 1469
 * @delay: number of jiffies to wait before queueing
 *
1470
 * Return: %false if @work was already on a queue, %true otherwise.  If
1471 1472
 * @delay is zero and @dwork is idle, it will be scheduled for immediate
 * execution.
1473
 */
1474 1475
bool queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
			   struct delayed_work *dwork, unsigned long delay)
1476
{
1477
	struct work_struct *work = &dwork->work;
1478
	bool ret = false;
1479
	unsigned long flags;
1480

1481 1482
	/* read the comment in __queue_work() */
	local_irq_save(flags);
1483

1484
	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
1485
		__queue_delayed_work(cpu, wq, dwork, delay);
1486
		ret = true;
1487
	}
1488

1489
	local_irq_restore(flags);
1490 1491
	return ret;
}
1492
EXPORT_SYMBOL(queue_delayed_work_on);
1493

1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505
/**
 * mod_delayed_work_on - modify delay of or queue a delayed work on specific CPU
 * @cpu: CPU number to execute work on
 * @wq: workqueue to use
 * @dwork: work to queue
 * @delay: number of jiffies to wait before queueing
 *
 * If @dwork is idle, equivalent to queue_delayed_work_on(); otherwise,
 * modify @dwork's timer so that it expires after @delay.  If @delay is
 * zero, @work is guaranteed to be scheduled immediately regardless of its
 * current state.
 *
1506
 * Return: %false if @dwork was idle and queued, %true if @dwork was
1507 1508
 * pending and its timer was modified.
 *
1509
 * This function is safe to call from any context including IRQ handler.
1510 1511 1512 1513 1514 1515 1516
 * See try_to_grab_pending() for details.
 */
bool mod_delayed_work_on(int cpu, struct workqueue_struct *wq,
			 struct delayed_work *dwork, unsigned long delay)
{
	unsigned long flags;
	int ret;
1517

1518 1519 1520
	do {
		ret = try_to_grab_pending(&dwork->work, true, &flags);
	} while (unlikely(ret == -EAGAIN));
1521

1522 1523 1524
	if (likely(ret >= 0)) {
		__queue_delayed_work(cpu, wq, dwork, delay);
		local_irq_restore(flags);
1525
	}
1526 1527

	/* -ENOENT from try_to_grab_pending() becomes %true */
1528 1529
	return ret;
}
1530 1531
EXPORT_SYMBOL_GPL(mod_delayed_work_on);

T
Tejun Heo 已提交
1532 1533 1534 1535 1536 1537 1538 1539
/**
 * worker_enter_idle - enter idle state
 * @worker: worker which is entering idle state
 *
 * @worker is entering idle state.  Update stats and idle timer if
 * necessary.
 *
 * LOCKING:
1540
 * spin_lock_irq(pool->lock).
T
Tejun Heo 已提交
1541 1542
 */
static void worker_enter_idle(struct worker *worker)
L
Linus Torvalds 已提交
1543
{
1544
	struct worker_pool *pool = worker->pool;
T
Tejun Heo 已提交
1545

1546 1547 1548 1549
	if (WARN_ON_ONCE(worker->flags & WORKER_IDLE) ||
	    WARN_ON_ONCE(!list_empty(&worker->entry) &&
			 (worker->hentry.next || worker->hentry.pprev)))
		return;
T
Tejun Heo 已提交
1550

1551
	/* can't use worker_set_flags(), also called from create_worker() */
1552
	worker->flags |= WORKER_IDLE;
1553
	pool->nr_idle++;
1554
	worker->last_active = jiffies;
T
Tejun Heo 已提交
1555 1556

	/* idle_list is LIFO */
1557
	list_add(&worker->entry, &pool->idle_list);
1558

1559 1560
	if (too_many_workers(pool) && !timer_pending(&pool->idle_timer))
		mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT);
1561

1562
	/*
1563
	 * Sanity check nr_running.  Because wq_unbind_fn() releases
1564
	 * pool->lock between setting %WORKER_UNBOUND and zapping
1565 1566
	 * nr_running, the warning may trigger spuriously.  Check iff
	 * unbind is not in progress.
1567
	 */
1568
	WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) &&
1569
		     pool->nr_workers == pool->nr_idle &&
1570
		     atomic_read(&pool->nr_running));
T
Tejun Heo 已提交
1571 1572 1573 1574 1575 1576 1577 1578 1579
}

/**
 * worker_leave_idle - leave idle state
 * @worker: worker which is leaving idle state
 *
 * @worker is leaving idle state.  Update stats.
 *
 * LOCKING:
1580
 * spin_lock_irq(pool->lock).
T
Tejun Heo 已提交
1581 1582 1583
 */
static void worker_leave_idle(struct worker *worker)
{
1584
	struct worker_pool *pool = worker->pool;
T
Tejun Heo 已提交
1585

1586 1587
	if (WARN_ON_ONCE(!(worker->flags & WORKER_IDLE)))
		return;
1588
	worker_clr_flags(worker, WORKER_IDLE);
1589
	pool->nr_idle--;
T
Tejun Heo 已提交
1590 1591 1592
	list_del_init(&worker->entry);
}

1593
static struct worker *alloc_worker(int node)
T
Tejun Heo 已提交
1594 1595 1596
{
	struct worker *worker;

1597
	worker = kzalloc_node(sizeof(*worker), GFP_KERNEL, node);
T
Tejun Heo 已提交
1598 1599
	if (worker) {
		INIT_LIST_HEAD(&worker->entry);
1600
		INIT_LIST_HEAD(&worker->scheduled);
1601
		INIT_LIST_HEAD(&worker->node);
1602 1603
		/* on creation a worker is in !idle && prep state */
		worker->flags = WORKER_PREP;
T
Tejun Heo 已提交
1604
	}
T
Tejun Heo 已提交
1605 1606 1607
	return worker;
}

1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640
/**
 * worker_attach_to_pool() - attach a worker to a pool
 * @worker: worker to be attached
 * @pool: the target pool
 *
 * Attach @worker to @pool.  Once attached, the %WORKER_UNBOUND flag and
 * cpu-binding of @worker are kept coordinated with the pool across
 * cpu-[un]hotplugs.
 */
static void worker_attach_to_pool(struct worker *worker,
				   struct worker_pool *pool)
{
	mutex_lock(&pool->attach_mutex);

	/*
	 * set_cpus_allowed_ptr() will fail if the cpumask doesn't have any
	 * online CPUs.  It'll be re-applied when any of the CPUs come up.
	 */
	set_cpus_allowed_ptr(worker->task, pool->attrs->cpumask);

	/*
	 * The pool->attach_mutex ensures %POOL_DISASSOCIATED remains
	 * stable across this function.  See the comments above the
	 * flag definition for details.
	 */
	if (pool->flags & POOL_DISASSOCIATED)
		worker->flags |= WORKER_UNBOUND;

	list_add_tail(&worker->node, &pool->workers);

	mutex_unlock(&pool->attach_mutex);
}

1641 1642 1643 1644 1645
/**
 * worker_detach_from_pool() - detach a worker from its pool
 * @worker: worker which is attached to its pool
 * @pool: the pool @worker is attached to
 *
1646 1647 1648
 * Undo the attaching which had been done in worker_attach_to_pool().  The
 * caller worker shouldn't access to the pool after detached except it has
 * other reference to the pool.
1649 1650 1651 1652 1653 1654
 */
static void worker_detach_from_pool(struct worker *worker,
				    struct worker_pool *pool)
{
	struct completion *detach_completion = NULL;

1655
	mutex_lock(&pool->attach_mutex);
1656 1657
	list_del(&worker->node);
	if (list_empty(&pool->workers))
1658
		detach_completion = pool->detach_completion;
1659
	mutex_unlock(&pool->attach_mutex);
1660

1661 1662 1663
	/* clear leftover flags without pool->lock after it is detached */
	worker->flags &= ~(WORKER_UNBOUND | WORKER_REBOUND);

1664 1665 1666 1667
	if (detach_completion)
		complete(detach_completion);
}

T
Tejun Heo 已提交
1668 1669
/**
 * create_worker - create a new workqueue worker
1670
 * @pool: pool the new worker will belong to
T
Tejun Heo 已提交
1671
 *
1672
 * Create and start a new worker which is attached to @pool.
T
Tejun Heo 已提交
1673 1674 1675 1676
 *
 * CONTEXT:
 * Might sleep.  Does GFP_KERNEL allocations.
 *
1677
 * Return:
T
Tejun Heo 已提交
1678 1679
 * Pointer to the newly created worker.
 */
1680
static struct worker *create_worker(struct worker_pool *pool)
T
Tejun Heo 已提交
1681 1682
{
	struct worker *worker = NULL;
1683
	int id = -1;
1684
	char id_buf[16];
T
Tejun Heo 已提交
1685

1686 1687
	/* ID is needed to determine kthread name */
	id = ida_simple_get(&pool->worker_ida, 0, 0, GFP_KERNEL);
1688 1689
	if (id < 0)
		goto fail;
T
Tejun Heo 已提交
1690

1691
	worker = alloc_worker(pool->node);
T
Tejun Heo 已提交
1692 1693 1694
	if (!worker)
		goto fail;

1695
	worker->pool = pool;
T
Tejun Heo 已提交
1696 1697
	worker->id = id;

1698
	if (pool->cpu >= 0)
1699 1700
		snprintf(id_buf, sizeof(id_buf), "%d:%d%s", pool->cpu, id,
			 pool->attrs->nice < 0  ? "H" : "");
1701
	else
1702 1703
		snprintf(id_buf, sizeof(id_buf), "u%d:%d", pool->id, id);

1704
	worker->task = kthread_create_on_node(worker_thread, worker, pool->node,
1705
					      "kworker/%s", id_buf);
T
Tejun Heo 已提交
1706 1707 1708
	if (IS_ERR(worker->task))
		goto fail;

1709 1710 1711 1712 1713
	set_user_nice(worker->task, pool->attrs->nice);

	/* prevent userland from meddling with cpumask of workqueue workers */
	worker->task->flags |= PF_NO_SETAFFINITY;

1714
	/* successful, attach the worker to the pool */
1715
	worker_attach_to_pool(worker, pool);
1716

1717 1718 1719 1720 1721 1722 1723
	/* start the newly created worker */
	spin_lock_irq(&pool->lock);
	worker->pool->nr_workers++;
	worker_enter_idle(worker);
	wake_up_process(worker->task);
	spin_unlock_irq(&pool->lock);

T
Tejun Heo 已提交
1724
	return worker;
1725

T
Tejun Heo 已提交
1726
fail:
1727
	if (id >= 0)
1728
		ida_simple_remove(&pool->worker_ida, id);
T
Tejun Heo 已提交
1729 1730 1731 1732 1733 1734 1735 1736
	kfree(worker);
	return NULL;
}

/**
 * destroy_worker - destroy a workqueue worker
 * @worker: worker to be destroyed
 *
1737 1738
 * Destroy @worker and adjust @pool stats accordingly.  The worker should
 * be idle.
T
Tejun Heo 已提交
1739 1740
 *
 * CONTEXT:
1741
 * spin_lock_irq(pool->lock).
T
Tejun Heo 已提交
1742 1743 1744
 */
static void destroy_worker(struct worker *worker)
{
1745
	struct worker_pool *pool = worker->pool;
T
Tejun Heo 已提交
1746

1747 1748
	lockdep_assert_held(&pool->lock);

T
Tejun Heo 已提交
1749
	/* sanity check frenzy */
1750
	if (WARN_ON(worker->current_work) ||
1751 1752
	    WARN_ON(!list_empty(&worker->scheduled)) ||
	    WARN_ON(!(worker->flags & WORKER_IDLE)))
1753
		return;
T
Tejun Heo 已提交
1754

1755 1756
	pool->nr_workers--;
	pool->nr_idle--;
1757

T
Tejun Heo 已提交
1758
	list_del_init(&worker->entry);
1759
	worker->flags |= WORKER_DIE;
1760
	wake_up_process(worker->task);
T
Tejun Heo 已提交
1761 1762
}

1763
static void idle_worker_timeout(unsigned long __pool)
1764
{
1765
	struct worker_pool *pool = (void *)__pool;
1766

1767
	spin_lock_irq(&pool->lock);
1768

1769
	while (too_many_workers(pool)) {
1770 1771 1772 1773
		struct worker *worker;
		unsigned long expires;

		/* idle_list is kept in LIFO order, check the last one */
1774
		worker = list_entry(pool->idle_list.prev, struct worker, entry);
1775 1776
		expires = worker->last_active + IDLE_WORKER_TIMEOUT;

1777
		if (time_before(jiffies, expires)) {
1778
			mod_timer(&pool->idle_timer, expires);
1779
			break;
1780
		}
1781 1782

		destroy_worker(worker);
1783 1784
	}

1785
	spin_unlock_irq(&pool->lock);
1786
}
1787

1788
static void send_mayday(struct work_struct *work)
1789
{
1790 1791
	struct pool_workqueue *pwq = get_work_pwq(work);
	struct workqueue_struct *wq = pwq->wq;
1792

1793
	lockdep_assert_held(&wq_mayday_lock);
1794

1795
	if (!wq->rescuer)
1796
		return;
1797 1798

	/* mayday mayday mayday */
1799
	if (list_empty(&pwq->mayday_node)) {
1800 1801 1802 1803 1804 1805
		/*
		 * If @pwq is for an unbound wq, its base ref may be put at
		 * any time due to an attribute change.  Pin @pwq until the
		 * rescuer is done with it.
		 */
		get_pwq(pwq);
1806
		list_add_tail(&pwq->mayday_node, &wq->maydays);
1807
		wake_up_process(wq->rescuer->task);
1808
	}
1809 1810
}

1811
static void pool_mayday_timeout(unsigned long __pool)
1812
{
1813
	struct worker_pool *pool = (void *)__pool;
1814 1815
	struct work_struct *work;

1816 1817
	spin_lock_irq(&pool->lock);
	spin_lock(&wq_mayday_lock);		/* for wq->maydays */
1818

1819
	if (need_to_create_worker(pool)) {
1820 1821 1822 1823 1824 1825
		/*
		 * We've been trying to create a new worker but
		 * haven't been successful.  We might be hitting an
		 * allocation deadlock.  Send distress signals to
		 * rescuers.
		 */
1826
		list_for_each_entry(work, &pool->worklist, entry)
1827
			send_mayday(work);
L
Linus Torvalds 已提交
1828
	}
1829

1830 1831
	spin_unlock(&wq_mayday_lock);
	spin_unlock_irq(&pool->lock);
1832

1833
	mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INTERVAL);
L
Linus Torvalds 已提交
1834 1835
}

1836 1837
/**
 * maybe_create_worker - create a new worker if necessary
1838
 * @pool: pool to create a new worker for
1839
 *
1840
 * Create a new worker for @pool if necessary.  @pool is guaranteed to
1841 1842
 * have at least one idle worker on return from this function.  If
 * creating a new worker takes longer than MAYDAY_INTERVAL, mayday is
1843
 * sent to all rescuers with works scheduled on @pool to resolve
1844 1845
 * possible allocation deadlock.
 *
1846 1847
 * On return, need_to_create_worker() is guaranteed to be %false and
 * may_start_working() %true.
1848 1849
 *
 * LOCKING:
1850
 * spin_lock_irq(pool->lock) which may be released and regrabbed
1851 1852 1853
 * multiple times.  Does GFP_KERNEL allocations.  Called only from
 * manager.
 */
1854
static void maybe_create_worker(struct worker_pool *pool)
1855 1856
__releases(&pool->lock)
__acquires(&pool->lock)
L
Linus Torvalds 已提交
1857
{
1858
restart:
1859
	spin_unlock_irq(&pool->lock);
1860

1861
	/* if we don't make progress in MAYDAY_INITIAL_TIMEOUT, call for help */
1862
	mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT);
1863 1864

	while (true) {
1865
		if (create_worker(pool) || !need_to_create_worker(pool))
1866
			break;
L
Linus Torvalds 已提交
1867

1868
		schedule_timeout_interruptible(CREATE_COOLDOWN);
1869

1870
		if (!need_to_create_worker(pool))
1871 1872 1873
			break;
	}

1874
	del_timer_sync(&pool->mayday_timer);
1875
	spin_lock_irq(&pool->lock);
1876 1877 1878 1879 1880
	/*
	 * This is necessary even after a new worker was just successfully
	 * created as @pool->lock was dropped and the new worker might have
	 * already become busy.
	 */
1881
	if (need_to_create_worker(pool))
1882 1883 1884
		goto restart;
}

1885
/**
1886 1887
 * manage_workers - manage worker pool
 * @worker: self
1888
 *
1889
 * Assume the manager role and manage the worker pool @worker belongs
1890
 * to.  At any given time, there can be only zero or one manager per
1891
 * pool.  The exclusion is handled automatically by this function.
1892 1893 1894 1895
 *
 * The caller can safely start processing works on false return.  On
 * true return, it's guaranteed that need_to_create_worker() is false
 * and may_start_working() is true.
1896 1897
 *
 * CONTEXT:
1898
 * spin_lock_irq(pool->lock) which may be released and regrabbed
1899 1900
 * multiple times.  Does GFP_KERNEL allocations.
 *
1901
 * Return:
1902 1903 1904 1905
 * %false if the pool doesn't need management and the caller can safely
 * start processing works, %true if management function was performed and
 * the conditions that the caller verified before calling the function may
 * no longer be true.
1906
 */
1907
static bool manage_workers(struct worker *worker)
1908
{
1909
	struct worker_pool *pool = worker->pool;
1910

1911 1912 1913 1914 1915 1916 1917 1918 1919 1920
	/*
	 * Anyone who successfully grabs manager_arb wins the arbitration
	 * and becomes the manager.  mutex_trylock() on pool->manager_arb
	 * failure while holding pool->lock reliably indicates that someone
	 * else is managing the pool and the worker which failed trylock
	 * can proceed to executing work items.  This means that anyone
	 * grabbing manager_arb is responsible for actually performing
	 * manager duties.  If manager_arb is grabbed and released without
	 * actual management, the pool may stall indefinitely.
	 */
1921
	if (!mutex_trylock(&pool->manager_arb))
1922
		return false;
1923
	pool->manager = worker;
1924

1925
	maybe_create_worker(pool);
1926

1927
	pool->manager = NULL;
1928
	mutex_unlock(&pool->manager_arb);
1929
	return true;
1930 1931
}

1932 1933
/**
 * process_one_work - process single work
T
Tejun Heo 已提交
1934
 * @worker: self
1935 1936 1937 1938 1939 1940 1941 1942 1943
 * @work: work to process
 *
 * Process @work.  This function contains all the logics necessary to
 * process a single work including synchronization against and
 * interaction with other workers on the same cpu, queueing and
 * flushing.  As long as context requirement is met, any worker can
 * call this function to process a work.
 *
 * CONTEXT:
1944
 * spin_lock_irq(pool->lock) which is released and regrabbed.
1945
 */
T
Tejun Heo 已提交
1946
static void process_one_work(struct worker *worker, struct work_struct *work)
1947 1948
__releases(&pool->lock)
__acquires(&pool->lock)
1949
{
1950
	struct pool_workqueue *pwq = get_work_pwq(work);
1951
	struct worker_pool *pool = worker->pool;
1952
	bool cpu_intensive = pwq->wq->flags & WQ_CPU_INTENSIVE;
1953
	int work_color;
1954
	struct worker *collision;
1955 1956 1957 1958 1959 1960 1961 1962
#ifdef CONFIG_LOCKDEP
	/*
	 * It is permissible to free the struct work_struct from
	 * inside the function that is called from it, this we need to
	 * take into account for lockdep too.  To avoid bogus "held
	 * lock freed" warnings as well as problems when looking into
	 * work->lockdep_map, make a copy and use that here.
	 */
1963 1964 1965
	struct lockdep_map lockdep_map;

	lockdep_copy_map(&lockdep_map, &work->lockdep_map);
1966
#endif
1967
	/* ensure we're on the correct CPU */
1968
	WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) &&
1969
		     raw_smp_processor_id() != pool->cpu);
1970

1971 1972 1973 1974 1975 1976
	/*
	 * A single work shouldn't be executed concurrently by
	 * multiple workers on a single cpu.  Check whether anyone is
	 * already processing the work.  If so, defer the work to the
	 * currently executing one.
	 */
1977
	collision = find_worker_executing_work(pool, work);
1978 1979 1980 1981 1982
	if (unlikely(collision)) {
		move_linked_works(work, &collision->scheduled, NULL);
		return;
	}

1983
	/* claim and dequeue */
1984
	debug_work_deactivate(work);
1985
	hash_add(pool->busy_hash, &worker->hentry, (unsigned long)work);
T
Tejun Heo 已提交
1986
	worker->current_work = work;
1987
	worker->current_func = work->func;
1988
	worker->current_pwq = pwq;
1989
	work_color = get_work_color(work);
1990

1991 1992
	list_del_init(&work->entry);

1993
	/*
1994 1995 1996 1997
	 * CPU intensive works don't participate in concurrency management.
	 * They're the scheduler's responsibility.  This takes @worker out
	 * of concurrency management and the next code block will chain
	 * execution of the pending work items.
1998 1999
	 */
	if (unlikely(cpu_intensive))
2000
		worker_set_flags(worker, WORKER_CPU_INTENSIVE);
2001

2002
	/*
2003 2004 2005 2006
	 * Wake up another worker if necessary.  The condition is always
	 * false for normal per-cpu workers since nr_running would always
	 * be >= 1 at this point.  This is used to chain execution of the
	 * pending work items for WORKER_NOT_RUNNING workers such as the
2007
	 * UNBOUND and CPU_INTENSIVE ones.
2008
	 */
2009
	if (need_more_worker(pool))
2010
		wake_up_worker(pool);
2011

2012
	/*
2013
	 * Record the last pool and clear PENDING which should be the last
2014
	 * update to @work.  Also, do this inside @pool->lock so that
2015 2016
	 * PENDING and queued state changes happen together while IRQ is
	 * disabled.
2017
	 */
2018
	set_work_pool_and_clear_pending(work, pool->id);
2019

2020
	spin_unlock_irq(&pool->lock);
2021

2022
	lock_map_acquire_read(&pwq->wq->lockdep_map);
2023
	lock_map_acquire(&lockdep_map);
2024
	trace_workqueue_execute_start(work);
2025
	worker->current_func(work);
2026 2027 2028 2029 2030
	/*
	 * While we must be careful to not use "work" after this, the trace
	 * point will only record its address.
	 */
	trace_workqueue_execute_end(work);
2031
	lock_map_release(&lockdep_map);
2032
	lock_map_release(&pwq->wq->lockdep_map);
2033 2034

	if (unlikely(in_atomic() || lockdep_depth(current) > 0)) {
V
Valentin Ilie 已提交
2035 2036
		pr_err("BUG: workqueue leaked lock or atomic: %s/0x%08x/%d\n"
		       "     last function: %pf\n",
2037 2038
		       current->comm, preempt_count(), task_pid_nr(current),
		       worker->current_func);
2039 2040 2041 2042
		debug_show_held_locks(current);
		dump_stack();
	}

2043 2044 2045 2046 2047
	/*
	 * The following prevents a kworker from hogging CPU on !PREEMPT
	 * kernels, where a requeueing work item waiting for something to
	 * happen could deadlock with stop_machine as such work item could
	 * indefinitely requeue itself while all other CPUs are trapped in
2048 2049
	 * stop_machine. At the same time, report a quiescent RCU state so
	 * the same condition doesn't freeze RCU.
2050
	 */
2051
	cond_resched_rcu_qs();
2052

2053
	spin_lock_irq(&pool->lock);
2054

2055 2056 2057 2058
	/* clear cpu intensive status */
	if (unlikely(cpu_intensive))
		worker_clr_flags(worker, WORKER_CPU_INTENSIVE);

2059
	/* we're done with it, release */
2060
	hash_del(&worker->hentry);
T
Tejun Heo 已提交
2061
	worker->current_work = NULL;
2062
	worker->current_func = NULL;
2063
	worker->current_pwq = NULL;
2064
	worker->desc_valid = false;
2065
	pwq_dec_nr_in_flight(pwq, work_color);
2066 2067
}

2068 2069 2070 2071 2072 2073 2074 2075 2076
/**
 * process_scheduled_works - process scheduled works
 * @worker: self
 *
 * Process all scheduled works.  Please note that the scheduled list
 * may change while processing a work, so this function repeatedly
 * fetches a work from the top and executes it.
 *
 * CONTEXT:
2077
 * spin_lock_irq(pool->lock) which may be released and regrabbed
2078 2079 2080
 * multiple times.
 */
static void process_scheduled_works(struct worker *worker)
L
Linus Torvalds 已提交
2081
{
2082 2083
	while (!list_empty(&worker->scheduled)) {
		struct work_struct *work = list_first_entry(&worker->scheduled,
L
Linus Torvalds 已提交
2084
						struct work_struct, entry);
T
Tejun Heo 已提交
2085
		process_one_work(worker, work);
L
Linus Torvalds 已提交
2086 2087 2088
	}
}

T
Tejun Heo 已提交
2089 2090
/**
 * worker_thread - the worker thread function
T
Tejun Heo 已提交
2091
 * @__worker: self
T
Tejun Heo 已提交
2092
 *
2093 2094 2095 2096 2097
 * The worker thread function.  All workers belong to a worker_pool -
 * either a per-cpu one or dynamic unbound one.  These workers process all
 * work items regardless of their specific target workqueue.  The only
 * exception is work items which belong to workqueues with a rescuer which
 * will be explained in rescuer_thread().
2098 2099
 *
 * Return: 0
T
Tejun Heo 已提交
2100
 */
T
Tejun Heo 已提交
2101
static int worker_thread(void *__worker)
L
Linus Torvalds 已提交
2102
{
T
Tejun Heo 已提交
2103
	struct worker *worker = __worker;
2104
	struct worker_pool *pool = worker->pool;
L
Linus Torvalds 已提交
2105

2106 2107
	/* tell the scheduler that this is a workqueue worker */
	worker->task->flags |= PF_WQ_WORKER;
T
Tejun Heo 已提交
2108
woke_up:
2109
	spin_lock_irq(&pool->lock);
L
Linus Torvalds 已提交
2110

2111 2112
	/* am I supposed to die? */
	if (unlikely(worker->flags & WORKER_DIE)) {
2113
		spin_unlock_irq(&pool->lock);
2114 2115
		WARN_ON_ONCE(!list_empty(&worker->entry));
		worker->task->flags &= ~PF_WQ_WORKER;
2116 2117

		set_task_comm(worker->task, "kworker/dying");
2118
		ida_simple_remove(&pool->worker_ida, worker->id);
2119 2120
		worker_detach_from_pool(worker, pool);
		kfree(worker);
2121
		return 0;
T
Tejun Heo 已提交
2122
	}
2123

T
Tejun Heo 已提交
2124
	worker_leave_idle(worker);
2125
recheck:
2126
	/* no more worker necessary? */
2127
	if (!need_more_worker(pool))
2128 2129 2130
		goto sleep;

	/* do we need to manage? */
2131
	if (unlikely(!may_start_working(pool)) && manage_workers(worker))
2132 2133
		goto recheck;

T
Tejun Heo 已提交
2134 2135 2136 2137 2138
	/*
	 * ->scheduled list can only be filled while a worker is
	 * preparing to process a work or actually processing it.
	 * Make sure nobody diddled with it while I was sleeping.
	 */
2139
	WARN_ON_ONCE(!list_empty(&worker->scheduled));
T
Tejun Heo 已提交
2140

2141
	/*
2142 2143 2144 2145 2146
	 * Finish PREP stage.  We're guaranteed to have at least one idle
	 * worker or that someone else has already assumed the manager
	 * role.  This is where @worker starts participating in concurrency
	 * management if applicable and concurrency management is restored
	 * after being rebound.  See rebind_workers() for details.
2147
	 */
2148
	worker_clr_flags(worker, WORKER_PREP | WORKER_REBOUND);
2149 2150

	do {
T
Tejun Heo 已提交
2151
		struct work_struct *work =
2152
			list_first_entry(&pool->worklist,
T
Tejun Heo 已提交
2153 2154 2155 2156 2157 2158
					 struct work_struct, entry);

		if (likely(!(*work_data_bits(work) & WORK_STRUCT_LINKED))) {
			/* optimization path, not strictly necessary */
			process_one_work(worker, work);
			if (unlikely(!list_empty(&worker->scheduled)))
2159
				process_scheduled_works(worker);
T
Tejun Heo 已提交
2160 2161 2162
		} else {
			move_linked_works(work, &worker->scheduled, NULL);
			process_scheduled_works(worker);
2163
		}
2164
	} while (keep_working(pool));
2165

2166
	worker_set_flags(worker, WORKER_PREP);
2167
sleep:
T
Tejun Heo 已提交
2168
	/*
2169 2170 2171 2172 2173
	 * pool->lock is held and there's no work to process and no need to
	 * manage, sleep.  Workers are woken up only while holding
	 * pool->lock or from local cpu, so setting the current state
	 * before releasing pool->lock is enough to prevent losing any
	 * event.
T
Tejun Heo 已提交
2174 2175 2176
	 */
	worker_enter_idle(worker);
	__set_current_state(TASK_INTERRUPTIBLE);
2177
	spin_unlock_irq(&pool->lock);
T
Tejun Heo 已提交
2178 2179
	schedule();
	goto woke_up;
L
Linus Torvalds 已提交
2180 2181
}

2182 2183
/**
 * rescuer_thread - the rescuer thread function
2184
 * @__rescuer: self
2185 2186
 *
 * Workqueue rescuer thread function.  There's one rescuer for each
2187
 * workqueue which has WQ_MEM_RECLAIM set.
2188
 *
2189
 * Regular work processing on a pool may block trying to create a new
2190 2191 2192 2193 2194
 * worker which uses GFP_KERNEL allocation which has slight chance of
 * developing into deadlock if some works currently on the same queue
 * need to be processed to satisfy the GFP_KERNEL allocation.  This is
 * the problem rescuer solves.
 *
2195 2196
 * When such condition is possible, the pool summons rescuers of all
 * workqueues which have works queued on the pool and let them process
2197 2198 2199
 * those works so that forward progress can be guaranteed.
 *
 * This should happen rarely.
2200 2201
 *
 * Return: 0
2202
 */
2203
static int rescuer_thread(void *__rescuer)
2204
{
2205 2206
	struct worker *rescuer = __rescuer;
	struct workqueue_struct *wq = rescuer->rescue_wq;
2207
	struct list_head *scheduled = &rescuer->scheduled;
2208
	bool should_stop;
2209 2210

	set_user_nice(current, RESCUER_NICE_LEVEL);
2211 2212 2213 2214 2215 2216

	/*
	 * Mark rescuer as worker too.  As WORKER_PREP is never cleared, it
	 * doesn't participate in concurrency management.
	 */
	rescuer->task->flags |= PF_WQ_WORKER;
2217 2218 2219
repeat:
	set_current_state(TASK_INTERRUPTIBLE);

2220 2221 2222 2223 2224 2225 2226 2227 2228
	/*
	 * By the time the rescuer is requested to stop, the workqueue
	 * shouldn't have any work pending, but @wq->maydays may still have
	 * pwq(s) queued.  This can happen by non-rescuer workers consuming
	 * all the work items before the rescuer got to them.  Go through
	 * @wq->maydays processing before acting on should_stop so that the
	 * list is always empty on exit.
	 */
	should_stop = kthread_should_stop();
2229

2230
	/* see whether any pwq is asking for help */
2231
	spin_lock_irq(&wq_mayday_lock);
2232 2233 2234 2235

	while (!list_empty(&wq->maydays)) {
		struct pool_workqueue *pwq = list_first_entry(&wq->maydays,
					struct pool_workqueue, mayday_node);
2236
		struct worker_pool *pool = pwq->pool;
2237 2238 2239
		struct work_struct *work, *n;

		__set_current_state(TASK_RUNNING);
2240 2241
		list_del_init(&pwq->mayday_node);

2242
		spin_unlock_irq(&wq_mayday_lock);
2243

2244 2245 2246
		worker_attach_to_pool(rescuer, pool);

		spin_lock_irq(&pool->lock);
2247
		rescuer->pool = pool;
2248 2249 2250 2251 2252

		/*
		 * Slurp in all works issued via this workqueue and
		 * process'em.
		 */
2253
		WARN_ON_ONCE(!list_empty(scheduled));
2254
		list_for_each_entry_safe(work, n, &pool->worklist, entry)
2255
			if (get_work_pwq(work) == pwq)
2256 2257
				move_linked_works(work, scheduled, &n);

2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276
		if (!list_empty(scheduled)) {
			process_scheduled_works(rescuer);

			/*
			 * The above execution of rescued work items could
			 * have created more to rescue through
			 * pwq_activate_first_delayed() or chained
			 * queueing.  Let's put @pwq back on mayday list so
			 * that such back-to-back work items, which may be
			 * being used to relieve memory pressure, don't
			 * incur MAYDAY_INTERVAL delay inbetween.
			 */
			if (need_to_create_worker(pool)) {
				spin_lock(&wq_mayday_lock);
				get_pwq(pwq);
				list_move_tail(&pwq->mayday_node, &wq->maydays);
				spin_unlock(&wq_mayday_lock);
			}
		}
2277

2278 2279
		/*
		 * Put the reference grabbed by send_mayday().  @pool won't
2280
		 * go away while we're still attached to it.
2281 2282 2283
		 */
		put_pwq(pwq);

2284
		/*
2285
		 * Leave this pool.  If need_more_worker() is %true, notify a
2286 2287 2288
		 * regular worker; otherwise, we end up with 0 concurrency
		 * and stalling the execution.
		 */
2289
		if (need_more_worker(pool))
2290
			wake_up_worker(pool);
2291

2292
		rescuer->pool = NULL;
2293 2294 2295 2296 2297
		spin_unlock_irq(&pool->lock);

		worker_detach_from_pool(rescuer, pool);

		spin_lock_irq(&wq_mayday_lock);
2298 2299
	}

2300
	spin_unlock_irq(&wq_mayday_lock);
2301

2302 2303 2304 2305 2306 2307
	if (should_stop) {
		__set_current_state(TASK_RUNNING);
		rescuer->task->flags &= ~PF_WQ_WORKER;
		return 0;
	}

2308 2309
	/* rescuers should never participate in concurrency management */
	WARN_ON_ONCE(!(rescuer->flags & WORKER_NOT_RUNNING));
2310 2311
	schedule();
	goto repeat;
L
Linus Torvalds 已提交
2312 2313
}

O
Oleg Nesterov 已提交
2314 2315 2316
struct wq_barrier {
	struct work_struct	work;
	struct completion	done;
2317
	struct task_struct	*task;	/* purely informational */
O
Oleg Nesterov 已提交
2318 2319 2320 2321 2322 2323 2324 2325
};

static void wq_barrier_func(struct work_struct *work)
{
	struct wq_barrier *barr = container_of(work, struct wq_barrier, work);
	complete(&barr->done);
}

T
Tejun Heo 已提交
2326 2327
/**
 * insert_wq_barrier - insert a barrier work
2328
 * @pwq: pwq to insert barrier into
T
Tejun Heo 已提交
2329
 * @barr: wq_barrier to insert
2330 2331
 * @target: target work to attach @barr to
 * @worker: worker currently executing @target, NULL if @target is not executing
T
Tejun Heo 已提交
2332
 *
2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344
 * @barr is linked to @target such that @barr is completed only after
 * @target finishes execution.  Please note that the ordering
 * guarantee is observed only with respect to @target and on the local
 * cpu.
 *
 * Currently, a queued barrier can't be canceled.  This is because
 * try_to_grab_pending() can't determine whether the work to be
 * grabbed is at the head of the queue and thus can't clear LINKED
 * flag of the previous work while there must be a valid next work
 * after a work with LINKED flag set.
 *
 * Note that when @worker is non-NULL, @target may be modified
2345
 * underneath us, so we can't reliably determine pwq from @target.
T
Tejun Heo 已提交
2346 2347
 *
 * CONTEXT:
2348
 * spin_lock_irq(pool->lock).
T
Tejun Heo 已提交
2349
 */
2350
static void insert_wq_barrier(struct pool_workqueue *pwq,
2351 2352
			      struct wq_barrier *barr,
			      struct work_struct *target, struct worker *worker)
O
Oleg Nesterov 已提交
2353
{
2354 2355 2356
	struct list_head *head;
	unsigned int linked = 0;

2357
	/*
2358
	 * debugobject calls are safe here even with pool->lock locked
2359 2360 2361 2362
	 * as we know for sure that this will not trigger any of the
	 * checks and call back into the fixup functions where we
	 * might deadlock.
	 */
A
Andrew Morton 已提交
2363
	INIT_WORK_ONSTACK(&barr->work, wq_barrier_func);
2364
	__set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&barr->work));
O
Oleg Nesterov 已提交
2365
	init_completion(&barr->done);
2366
	barr->task = current;
2367

2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382
	/*
	 * If @target is currently being executed, schedule the
	 * barrier to the worker; otherwise, put it after @target.
	 */
	if (worker)
		head = worker->scheduled.next;
	else {
		unsigned long *bits = work_data_bits(target);

		head = target->entry.next;
		/* there can already be other linked works, inherit and set */
		linked = *bits & WORK_STRUCT_LINKED;
		__set_bit(WORK_STRUCT_LINKED_BIT, bits);
	}

2383
	debug_work_activate(&barr->work);
2384
	insert_work(pwq, &barr->work, head,
2385
		    work_color_to_flags(WORK_NO_COLOR) | linked);
O
Oleg Nesterov 已提交
2386 2387
}

2388
/**
2389
 * flush_workqueue_prep_pwqs - prepare pwqs for workqueue flushing
2390 2391 2392 2393
 * @wq: workqueue being flushed
 * @flush_color: new flush color, < 0 for no-op
 * @work_color: new work color, < 0 for no-op
 *
2394
 * Prepare pwqs for workqueue flushing.
2395
 *
2396 2397 2398 2399 2400
 * If @flush_color is non-negative, flush_color on all pwqs should be
 * -1.  If no pwq has in-flight commands at the specified color, all
 * pwq->flush_color's stay at -1 and %false is returned.  If any pwq
 * has in flight commands, its pwq->flush_color is set to
 * @flush_color, @wq->nr_pwqs_to_flush is updated accordingly, pwq
2401 2402 2403 2404 2405 2406 2407
 * wakeup logic is armed and %true is returned.
 *
 * The caller should have initialized @wq->first_flusher prior to
 * calling this function with non-negative @flush_color.  If
 * @flush_color is negative, no flush color update is done and %false
 * is returned.
 *
2408
 * If @work_color is non-negative, all pwqs should have the same
2409 2410 2411 2412
 * work_color which is previous to @work_color and all will be
 * advanced to @work_color.
 *
 * CONTEXT:
2413
 * mutex_lock(wq->mutex).
2414
 *
2415
 * Return:
2416 2417 2418
 * %true if @flush_color >= 0 and there's something to flush.  %false
 * otherwise.
 */
2419
static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq,
2420
				      int flush_color, int work_color)
L
Linus Torvalds 已提交
2421
{
2422
	bool wait = false;
2423
	struct pool_workqueue *pwq;
L
Linus Torvalds 已提交
2424

2425
	if (flush_color >= 0) {
2426
		WARN_ON_ONCE(atomic_read(&wq->nr_pwqs_to_flush));
2427
		atomic_set(&wq->nr_pwqs_to_flush, 1);
L
Linus Torvalds 已提交
2428
	}
2429

2430
	for_each_pwq(pwq, wq) {
2431
		struct worker_pool *pool = pwq->pool;
O
Oleg Nesterov 已提交
2432

2433
		spin_lock_irq(&pool->lock);
2434

2435
		if (flush_color >= 0) {
2436
			WARN_ON_ONCE(pwq->flush_color != -1);
O
Oleg Nesterov 已提交
2437

2438 2439 2440
			if (pwq->nr_in_flight[flush_color]) {
				pwq->flush_color = flush_color;
				atomic_inc(&wq->nr_pwqs_to_flush);
2441 2442 2443
				wait = true;
			}
		}
L
Linus Torvalds 已提交
2444

2445
		if (work_color >= 0) {
2446
			WARN_ON_ONCE(work_color != work_next_color(pwq->work_color));
2447
			pwq->work_color = work_color;
2448
		}
L
Linus Torvalds 已提交
2449

2450
		spin_unlock_irq(&pool->lock);
L
Linus Torvalds 已提交
2451
	}
2452

2453
	if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush))
2454
		complete(&wq->first_flusher->done);
2455

2456
	return wait;
L
Linus Torvalds 已提交
2457 2458
}

2459
/**
L
Linus Torvalds 已提交
2460
 * flush_workqueue - ensure that any scheduled work has run to completion.
2461
 * @wq: workqueue to flush
L
Linus Torvalds 已提交
2462
 *
2463 2464
 * This function sleeps until all work items which were queued on entry
 * have finished execution, but it is not livelocked by new incoming ones.
L
Linus Torvalds 已提交
2465
 */
2466
void flush_workqueue(struct workqueue_struct *wq)
L
Linus Torvalds 已提交
2467
{
2468 2469 2470 2471 2472 2473
	struct wq_flusher this_flusher = {
		.list = LIST_HEAD_INIT(this_flusher.list),
		.flush_color = -1,
		.done = COMPLETION_INITIALIZER_ONSTACK(this_flusher.done),
	};
	int next_color;
L
Linus Torvalds 已提交
2474

2475 2476
	lock_map_acquire(&wq->lockdep_map);
	lock_map_release(&wq->lockdep_map);
2477

2478
	mutex_lock(&wq->mutex);
2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490

	/*
	 * Start-to-wait phase
	 */
	next_color = work_next_color(wq->work_color);

	if (next_color != wq->flush_color) {
		/*
		 * Color space is not full.  The current work_color
		 * becomes our flush_color and work_color is advanced
		 * by one.
		 */
2491
		WARN_ON_ONCE(!list_empty(&wq->flusher_overflow));
2492 2493 2494 2495 2496
		this_flusher.flush_color = wq->work_color;
		wq->work_color = next_color;

		if (!wq->first_flusher) {
			/* no flush in progress, become the first flusher */
2497
			WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
2498 2499 2500

			wq->first_flusher = &this_flusher;

2501
			if (!flush_workqueue_prep_pwqs(wq, wq->flush_color,
2502 2503 2504 2505 2506 2507 2508 2509
						       wq->work_color)) {
				/* nothing to flush, done */
				wq->flush_color = next_color;
				wq->first_flusher = NULL;
				goto out_unlock;
			}
		} else {
			/* wait in queue */
2510
			WARN_ON_ONCE(wq->flush_color == this_flusher.flush_color);
2511
			list_add_tail(&this_flusher.list, &wq->flusher_queue);
2512
			flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
2513 2514 2515 2516 2517 2518 2519 2520 2521 2522
		}
	} else {
		/*
		 * Oops, color space is full, wait on overflow queue.
		 * The next flush completion will assign us
		 * flush_color and transfer to flusher_queue.
		 */
		list_add_tail(&this_flusher.list, &wq->flusher_overflow);
	}

2523
	mutex_unlock(&wq->mutex);
2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535

	wait_for_completion(&this_flusher.done);

	/*
	 * Wake-up-and-cascade phase
	 *
	 * First flushers are responsible for cascading flushes and
	 * handling overflow.  Non-first flushers can simply return.
	 */
	if (wq->first_flusher != &this_flusher)
		return;

2536
	mutex_lock(&wq->mutex);
2537

2538 2539 2540 2541
	/* we might have raced, check again with mutex held */
	if (wq->first_flusher != &this_flusher)
		goto out_unlock;

2542 2543
	wq->first_flusher = NULL;

2544 2545
	WARN_ON_ONCE(!list_empty(&this_flusher.list));
	WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557

	while (true) {
		struct wq_flusher *next, *tmp;

		/* complete all the flushers sharing the current flush color */
		list_for_each_entry_safe(next, tmp, &wq->flusher_queue, list) {
			if (next->flush_color != wq->flush_color)
				break;
			list_del_init(&next->list);
			complete(&next->done);
		}

2558 2559
		WARN_ON_ONCE(!list_empty(&wq->flusher_overflow) &&
			     wq->flush_color != work_next_color(wq->work_color));
2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578

		/* this flush_color is finished, advance by one */
		wq->flush_color = work_next_color(wq->flush_color);

		/* one color has been freed, handle overflow queue */
		if (!list_empty(&wq->flusher_overflow)) {
			/*
			 * Assign the same color to all overflowed
			 * flushers, advance work_color and append to
			 * flusher_queue.  This is the start-to-wait
			 * phase for these overflowed flushers.
			 */
			list_for_each_entry(tmp, &wq->flusher_overflow, list)
				tmp->flush_color = wq->work_color;

			wq->work_color = work_next_color(wq->work_color);

			list_splice_tail_init(&wq->flusher_overflow,
					      &wq->flusher_queue);
2579
			flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
2580 2581 2582
		}

		if (list_empty(&wq->flusher_queue)) {
2583
			WARN_ON_ONCE(wq->flush_color != wq->work_color);
2584 2585 2586 2587 2588
			break;
		}

		/*
		 * Need to flush more colors.  Make the next flusher
2589
		 * the new first flusher and arm pwqs.
2590
		 */
2591 2592
		WARN_ON_ONCE(wq->flush_color == wq->work_color);
		WARN_ON_ONCE(wq->flush_color != next->flush_color);
2593 2594 2595 2596

		list_del_init(&next->list);
		wq->first_flusher = next;

2597
		if (flush_workqueue_prep_pwqs(wq, wq->flush_color, -1))
2598 2599 2600 2601 2602 2603 2604 2605 2606 2607
			break;

		/*
		 * Meh... this color is already done, clear first
		 * flusher and repeat cascading.
		 */
		wq->first_flusher = NULL;
	}

out_unlock:
2608
	mutex_unlock(&wq->mutex);
L
Linus Torvalds 已提交
2609
}
2610
EXPORT_SYMBOL_GPL(flush_workqueue);
L
Linus Torvalds 已提交
2611

2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625
/**
 * drain_workqueue - drain a workqueue
 * @wq: workqueue to drain
 *
 * Wait until the workqueue becomes empty.  While draining is in progress,
 * only chain queueing is allowed.  IOW, only currently pending or running
 * work items on @wq can queue further work items on it.  @wq is flushed
 * repeatedly until it becomes empty.  The number of flushing is detemined
 * by the depth of chaining and should be relatively short.  Whine if it
 * takes too long.
 */
void drain_workqueue(struct workqueue_struct *wq)
{
	unsigned int flush_cnt = 0;
2626
	struct pool_workqueue *pwq;
2627 2628 2629 2630

	/*
	 * __queue_work() needs to test whether there are drainers, is much
	 * hotter than drain_workqueue() and already looks at @wq->flags.
2631
	 * Use __WQ_DRAINING so that queue doesn't have to check nr_drainers.
2632
	 */
2633
	mutex_lock(&wq->mutex);
2634
	if (!wq->nr_drainers++)
2635
		wq->flags |= __WQ_DRAINING;
2636
	mutex_unlock(&wq->mutex);
2637 2638 2639
reflush:
	flush_workqueue(wq);

2640
	mutex_lock(&wq->mutex);
2641

2642
	for_each_pwq(pwq, wq) {
2643
		bool drained;
2644

2645
		spin_lock_irq(&pwq->pool->lock);
2646
		drained = !pwq->nr_active && list_empty(&pwq->delayed_works);
2647
		spin_unlock_irq(&pwq->pool->lock);
2648 2649

		if (drained)
2650 2651 2652 2653
			continue;

		if (++flush_cnt == 10 ||
		    (flush_cnt % 100 == 0 && flush_cnt <= 1000))
2654
			pr_warn("workqueue %s: drain_workqueue() isn't complete after %u tries\n",
V
Valentin Ilie 已提交
2655
				wq->name, flush_cnt);
2656

2657
		mutex_unlock(&wq->mutex);
2658 2659 2660 2661
		goto reflush;
	}

	if (!--wq->nr_drainers)
2662
		wq->flags &= ~__WQ_DRAINING;
2663
	mutex_unlock(&wq->mutex);
2664 2665 2666
}
EXPORT_SYMBOL_GPL(drain_workqueue);

2667
static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr)
2668
{
2669
	struct worker *worker = NULL;
2670
	struct worker_pool *pool;
2671
	struct pool_workqueue *pwq;
2672 2673

	might_sleep();
2674 2675

	local_irq_disable();
2676
	pool = get_work_pool(work);
2677 2678
	if (!pool) {
		local_irq_enable();
2679
		return false;
2680
	}
2681

2682
	spin_lock(&pool->lock);
2683
	/* see the comment in try_to_grab_pending() with the same code */
2684 2685 2686
	pwq = get_work_pwq(work);
	if (pwq) {
		if (unlikely(pwq->pool != pool))
T
Tejun Heo 已提交
2687
			goto already_gone;
2688
	} else {
2689
		worker = find_worker_executing_work(pool, work);
2690
		if (!worker)
T
Tejun Heo 已提交
2691
			goto already_gone;
2692
		pwq = worker->current_pwq;
2693
	}
2694

2695
	insert_wq_barrier(pwq, barr, work, worker);
2696
	spin_unlock_irq(&pool->lock);
2697

2698 2699 2700 2701 2702 2703
	/*
	 * If @max_active is 1 or rescuer is in use, flushing another work
	 * item on the same workqueue may lead to deadlock.  Make sure the
	 * flusher is not running on the same workqueue by verifying write
	 * access.
	 */
2704
	if (pwq->wq->saved_max_active == 1 || pwq->wq->rescuer)
2705
		lock_map_acquire(&pwq->wq->lockdep_map);
2706
	else
2707 2708
		lock_map_acquire_read(&pwq->wq->lockdep_map);
	lock_map_release(&pwq->wq->lockdep_map);
2709

2710
	return true;
T
Tejun Heo 已提交
2711
already_gone:
2712
	spin_unlock_irq(&pool->lock);
2713
	return false;
2714
}
2715 2716 2717 2718 2719

/**
 * flush_work - wait for a work to finish executing the last queueing instance
 * @work: the work to flush
 *
2720 2721
 * Wait until @work has finished execution.  @work is guaranteed to be idle
 * on return if it hasn't been requeued since flush started.
2722
 *
2723
 * Return:
2724 2725 2726 2727 2728
 * %true if flush_work() waited for the work to finish execution,
 * %false if it was already idle.
 */
bool flush_work(struct work_struct *work)
{
2729 2730
	struct wq_barrier barr;

2731 2732 2733
	lock_map_acquire(&work->lockdep_map);
	lock_map_release(&work->lockdep_map);

2734 2735 2736 2737 2738 2739 2740
	if (start_flush_work(work, &barr)) {
		wait_for_completion(&barr.done);
		destroy_work_on_stack(&barr.work);
		return true;
	} else {
		return false;
	}
2741
}
2742
EXPORT_SYMBOL_GPL(flush_work);
2743

2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757
struct cwt_wait {
	wait_queue_t		wait;
	struct work_struct	*work;
};

static int cwt_wakefn(wait_queue_t *wait, unsigned mode, int sync, void *key)
{
	struct cwt_wait *cwait = container_of(wait, struct cwt_wait, wait);

	if (cwait->work != key)
		return 0;
	return autoremove_wake_function(wait, mode, sync, key);
}

2758
static bool __cancel_work_timer(struct work_struct *work, bool is_dwork)
2759
{
2760
	static DECLARE_WAIT_QUEUE_HEAD(cancel_waitq);
2761
	unsigned long flags;
2762 2763 2764
	int ret;

	do {
2765 2766
		ret = try_to_grab_pending(work, is_dwork, &flags);
		/*
2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780
		 * If someone else is already canceling, wait for it to
		 * finish.  flush_work() doesn't work for PREEMPT_NONE
		 * because we may get scheduled between @work's completion
		 * and the other canceling task resuming and clearing
		 * CANCELING - flush_work() will return false immediately
		 * as @work is no longer busy, try_to_grab_pending() will
		 * return -ENOENT as @work is still being canceled and the
		 * other canceling task won't be able to clear CANCELING as
		 * we're hogging the CPU.
		 *
		 * Let's wait for completion using a waitqueue.  As this
		 * may lead to the thundering herd problem, use a custom
		 * wake function which matches @work along with exclusive
		 * wait and wakeup.
2781
		 */
2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794
		if (unlikely(ret == -ENOENT)) {
			struct cwt_wait cwait;

			init_wait(&cwait.wait);
			cwait.wait.func = cwt_wakefn;
			cwait.work = work;

			prepare_to_wait_exclusive(&cancel_waitq, &cwait.wait,
						  TASK_UNINTERRUPTIBLE);
			if (work_is_canceling(work))
				schedule();
			finish_wait(&cancel_waitq, &cwait.wait);
		}
2795 2796
	} while (unlikely(ret < 0));

2797 2798 2799 2800
	/* tell other tasks trying to grab @work to back off */
	mark_work_canceling(work);
	local_irq_restore(flags);

2801
	flush_work(work);
2802
	clear_work_data(work);
2803 2804 2805 2806 2807 2808 2809 2810 2811 2812

	/*
	 * Paired with prepare_to_wait() above so that either
	 * waitqueue_active() is visible here or !work_is_canceling() is
	 * visible there.
	 */
	smp_mb();
	if (waitqueue_active(&cancel_waitq))
		__wake_up(&cancel_waitq, TASK_NORMAL, 1, work);

2813 2814 2815
	return ret;
}

2816
/**
2817 2818
 * cancel_work_sync - cancel a work and wait for it to finish
 * @work: the work to cancel
2819
 *
2820 2821 2822 2823
 * Cancel @work and wait for its execution to finish.  This function
 * can be used even if the work re-queues itself or migrates to
 * another workqueue.  On return from this function, @work is
 * guaranteed to be not pending or executing on any CPU.
2824
 *
2825 2826
 * cancel_work_sync(&delayed_work->work) must not be used for
 * delayed_work's.  Use cancel_delayed_work_sync() instead.
2827
 *
2828
 * The caller must ensure that the workqueue on which @work was last
2829
 * queued can't be destroyed before this function returns.
2830
 *
2831
 * Return:
2832
 * %true if @work was pending, %false otherwise.
2833
 */
2834
bool cancel_work_sync(struct work_struct *work)
2835
{
2836
	return __cancel_work_timer(work, false);
O
Oleg Nesterov 已提交
2837
}
2838
EXPORT_SYMBOL_GPL(cancel_work_sync);
O
Oleg Nesterov 已提交
2839

2840
/**
2841 2842
 * flush_delayed_work - wait for a dwork to finish executing the last queueing
 * @dwork: the delayed work to flush
2843
 *
2844 2845 2846
 * Delayed timer is cancelled and the pending work is queued for
 * immediate execution.  Like flush_work(), this function only
 * considers the last queueing instance of @dwork.
2847
 *
2848
 * Return:
2849 2850
 * %true if flush_work() waited for the work to finish execution,
 * %false if it was already idle.
2851
 */
2852 2853
bool flush_delayed_work(struct delayed_work *dwork)
{
2854
	local_irq_disable();
2855
	if (del_timer_sync(&dwork->timer))
2856
		__queue_work(dwork->cpu, dwork->wq, &dwork->work);
2857
	local_irq_enable();
2858 2859 2860 2861
	return flush_work(&dwork->work);
}
EXPORT_SYMBOL(flush_delayed_work);

2862
/**
2863 2864
 * cancel_delayed_work - cancel a delayed work
 * @dwork: delayed_work to cancel
2865
 *
2866 2867 2868 2869 2870 2871 2872 2873 2874
 * Kill off a pending delayed_work.
 *
 * Return: %true if @dwork was pending and canceled; %false if it wasn't
 * pending.
 *
 * Note:
 * The work callback function may still be running on return, unless
 * it returns %true and the work doesn't re-arm itself.  Explicitly flush or
 * use cancel_delayed_work_sync() to wait on it.
2875
 *
2876
 * This function is safe to call from any context including IRQ handler.
2877
 */
2878
bool cancel_delayed_work(struct delayed_work *dwork)
2879
{
2880 2881 2882 2883 2884 2885 2886 2887 2888 2889
	unsigned long flags;
	int ret;

	do {
		ret = try_to_grab_pending(&dwork->work, true, &flags);
	} while (unlikely(ret == -EAGAIN));

	if (unlikely(ret < 0))
		return false;

2890 2891
	set_work_pool_and_clear_pending(&dwork->work,
					get_work_pool_id(&dwork->work));
2892
	local_irq_restore(flags);
2893
	return ret;
2894
}
2895
EXPORT_SYMBOL(cancel_delayed_work);
2896

2897 2898 2899 2900 2901 2902
/**
 * cancel_delayed_work_sync - cancel a delayed work and wait for it to finish
 * @dwork: the delayed work cancel
 *
 * This is cancel_work_sync() for delayed works.
 *
2903
 * Return:
2904 2905 2906
 * %true if @dwork was pending, %false otherwise.
 */
bool cancel_delayed_work_sync(struct delayed_work *dwork)
2907
{
2908
	return __cancel_work_timer(&dwork->work, true);
2909
}
2910
EXPORT_SYMBOL(cancel_delayed_work_sync);
L
Linus Torvalds 已提交
2911

2912
/**
2913
 * schedule_on_each_cpu - execute a function synchronously on each online CPU
2914 2915
 * @func: the function to call
 *
2916 2917
 * schedule_on_each_cpu() executes @func on each online CPU using the
 * system workqueue and blocks until all CPUs have completed.
2918
 * schedule_on_each_cpu() is very slow.
2919
 *
2920
 * Return:
2921
 * 0 on success, -errno on failure.
2922
 */
2923
int schedule_on_each_cpu(work_func_t func)
2924 2925
{
	int cpu;
2926
	struct work_struct __percpu *works;
2927

2928 2929
	works = alloc_percpu(struct work_struct);
	if (!works)
2930
		return -ENOMEM;
2931

2932 2933
	get_online_cpus();

2934
	for_each_online_cpu(cpu) {
2935 2936 2937
		struct work_struct *work = per_cpu_ptr(works, cpu);

		INIT_WORK(work, func);
2938
		schedule_work_on(cpu, work);
2939
	}
2940 2941 2942 2943

	for_each_online_cpu(cpu)
		flush_work(per_cpu_ptr(works, cpu));

2944
	put_online_cpus();
2945
	free_percpu(works);
2946 2947 2948
	return 0;
}

2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972
/**
 * flush_scheduled_work - ensure that any scheduled work has run to completion.
 *
 * Forces execution of the kernel-global workqueue and blocks until its
 * completion.
 *
 * Think twice before calling this function!  It's very easy to get into
 * trouble if you don't take great care.  Either of the following situations
 * will lead to deadlock:
 *
 *	One of the work items currently on the workqueue needs to acquire
 *	a lock held by your code or its caller.
 *
 *	Your code is running in the context of a work routine.
 *
 * They will be detected by lockdep when they occur, but the first might not
 * occur very often.  It depends on what work items are on the workqueue and
 * what locks they need, which you have no control over.
 *
 * In most situations flushing the entire workqueue is overkill; you merely
 * need to know that a particular work item isn't queued and isn't running.
 * In such cases you should use cancel_delayed_work_sync() or
 * cancel_work_sync() instead.
 */
L
Linus Torvalds 已提交
2973 2974
void flush_scheduled_work(void)
{
2975
	flush_workqueue(system_wq);
L
Linus Torvalds 已提交
2976
}
2977
EXPORT_SYMBOL(flush_scheduled_work);
L
Linus Torvalds 已提交
2978

2979 2980 2981 2982 2983 2984 2985 2986 2987
/**
 * execute_in_process_context - reliably execute the routine with user context
 * @fn:		the function to execute
 * @ew:		guaranteed storage for the execute work structure (must
 *		be available when the work executes)
 *
 * Executes the function immediately if process context is available,
 * otherwise schedules the function for delayed execution.
 *
2988
 * Return:	0 - function was executed
2989 2990
 *		1 - function was scheduled for execution
 */
2991
int execute_in_process_context(work_func_t fn, struct execute_work *ew)
2992 2993
{
	if (!in_interrupt()) {
2994
		fn(&ew->work);
2995 2996 2997
		return 0;
	}

2998
	INIT_WORK(&ew->work, fn);
2999 3000 3001 3002 3003 3004
	schedule_work(&ew->work);

	return 1;
}
EXPORT_SYMBOL_GPL(execute_in_process_context);

3005 3006 3007
/**
 * free_workqueue_attrs - free a workqueue_attrs
 * @attrs: workqueue_attrs to free
3008
 *
3009
 * Undo alloc_workqueue_attrs().
3010
 */
3011
void free_workqueue_attrs(struct workqueue_attrs *attrs)
3012
{
3013 3014 3015 3016
	if (attrs) {
		free_cpumask_var(attrs->cpumask);
		kfree(attrs);
	}
3017 3018
}

3019 3020 3021 3022 3023 3024 3025 3026 3027 3028
/**
 * alloc_workqueue_attrs - allocate a workqueue_attrs
 * @gfp_mask: allocation mask to use
 *
 * Allocate a new workqueue_attrs, initialize with default settings and
 * return it.
 *
 * Return: The allocated new workqueue_attr on success. %NULL on failure.
 */
struct workqueue_attrs *alloc_workqueue_attrs(gfp_t gfp_mask)
3029
{
3030
	struct workqueue_attrs *attrs;
3031

3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042
	attrs = kzalloc(sizeof(*attrs), gfp_mask);
	if (!attrs)
		goto fail;
	if (!alloc_cpumask_var(&attrs->cpumask, gfp_mask))
		goto fail;

	cpumask_copy(attrs->cpumask, cpu_possible_mask);
	return attrs;
fail:
	free_workqueue_attrs(attrs);
	return NULL;
3043 3044
}

3045 3046
static void copy_workqueue_attrs(struct workqueue_attrs *to,
				 const struct workqueue_attrs *from)
3047
{
3048 3049 3050 3051 3052 3053 3054 3055
	to->nice = from->nice;
	cpumask_copy(to->cpumask, from->cpumask);
	/*
	 * Unlike hash and equality test, this function doesn't ignore
	 * ->no_numa as it is used for both pool and wq attrs.  Instead,
	 * get_unbound_pool() explicitly clears ->no_numa after copying.
	 */
	to->no_numa = from->no_numa;
3056 3057
}

3058 3059
/* hash value of the content of @attr */
static u32 wqattrs_hash(const struct workqueue_attrs *attrs)
3060
{
3061
	u32 hash = 0;
3062

3063 3064 3065 3066
	hash = jhash_1word(attrs->nice, hash);
	hash = jhash(cpumask_bits(attrs->cpumask),
		     BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long), hash);
	return hash;
3067 3068
}

3069 3070 3071
/* content equality test */
static bool wqattrs_equal(const struct workqueue_attrs *a,
			  const struct workqueue_attrs *b)
3072
{
3073 3074 3075 3076 3077
	if (a->nice != b->nice)
		return false;
	if (!cpumask_equal(a->cpumask, b->cpumask))
		return false;
	return true;
3078 3079
}

3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090
/**
 * init_worker_pool - initialize a newly zalloc'd worker_pool
 * @pool: worker_pool to initialize
 *
 * Initiailize a newly zalloc'd @pool.  It also allocates @pool->attrs.
 *
 * Return: 0 on success, -errno on failure.  Even on failure, all fields
 * inside @pool proper are initialized and put_unbound_pool() can be called
 * on @pool safely to release it.
 */
static int init_worker_pool(struct worker_pool *pool)
3091
{
3092 3093 3094 3095 3096 3097 3098 3099
	spin_lock_init(&pool->lock);
	pool->id = -1;
	pool->cpu = -1;
	pool->node = NUMA_NO_NODE;
	pool->flags |= POOL_DISASSOCIATED;
	INIT_LIST_HEAD(&pool->worklist);
	INIT_LIST_HEAD(&pool->idle_list);
	hash_init(pool->busy_hash);
3100

3101 3102 3103
	init_timer_deferrable(&pool->idle_timer);
	pool->idle_timer.function = idle_worker_timeout;
	pool->idle_timer.data = (unsigned long)pool;
3104

3105 3106
	setup_timer(&pool->mayday_timer, pool_mayday_timeout,
		    (unsigned long)pool);
3107

3108 3109 3110
	mutex_init(&pool->manager_arb);
	mutex_init(&pool->attach_mutex);
	INIT_LIST_HEAD(&pool->workers);
3111

3112 3113 3114
	ida_init(&pool->worker_ida);
	INIT_HLIST_NODE(&pool->hash_node);
	pool->refcnt = 1;
3115

3116 3117 3118 3119 3120
	/* shouldn't fail above this point */
	pool->attrs = alloc_workqueue_attrs(GFP_KERNEL);
	if (!pool->attrs)
		return -ENOMEM;
	return 0;
3121 3122
}

3123
static void rcu_free_wq(struct rcu_head *rcu)
3124
{
3125 3126
	struct workqueue_struct *wq =
		container_of(rcu, struct workqueue_struct, rcu);
3127

3128 3129
	if (!(wq->flags & WQ_UNBOUND))
		free_percpu(wq->cpu_pwqs);
3130
	else
3131
		free_workqueue_attrs(wq->unbound_attrs);
3132

3133 3134
	kfree(wq->rescuer);
	kfree(wq);
3135 3136
}

3137
static void rcu_free_pool(struct rcu_head *rcu)
3138
{
3139
	struct worker_pool *pool = container_of(rcu, struct worker_pool, rcu);
3140

3141 3142 3143
	ida_destroy(&pool->worker_ida);
	free_workqueue_attrs(pool->attrs);
	kfree(pool);
3144 3145
}

3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157
/**
 * put_unbound_pool - put a worker_pool
 * @pool: worker_pool to put
 *
 * Put @pool.  If its refcnt reaches zero, it gets destroyed in sched-RCU
 * safe manner.  get_unbound_pool() calls this function on its failure path
 * and this function should be able to release pools which went through,
 * successfully or not, init_worker_pool().
 *
 * Should be called with wq_pool_mutex held.
 */
static void put_unbound_pool(struct worker_pool *pool)
3158
{
3159 3160
	DECLARE_COMPLETION_ONSTACK(detach_completion);
	struct worker *worker;
3161

3162
	lockdep_assert_held(&wq_pool_mutex);
3163

3164 3165
	if (--pool->refcnt)
		return;
3166

3167 3168 3169 3170
	/* sanity checks */
	if (WARN_ON(!(pool->cpu < 0)) ||
	    WARN_ON(!list_empty(&pool->worklist)))
		return;
3171

3172 3173 3174 3175
	/* release id and unhash */
	if (pool->id >= 0)
		idr_remove(&worker_pool_idr, pool->id);
	hash_del(&pool->hash_node);
3176

3177 3178 3179 3180 3181 3182
	/*
	 * Become the manager and destroy all workers.  Grabbing
	 * manager_arb prevents @pool's workers from blocking on
	 * attach_mutex.
	 */
	mutex_lock(&pool->manager_arb);
3183

3184 3185 3186 3187 3188
	spin_lock_irq(&pool->lock);
	while ((worker = first_idle_worker(pool)))
		destroy_worker(worker);
	WARN_ON(pool->nr_workers || pool->nr_idle);
	spin_unlock_irq(&pool->lock);
3189

3190 3191 3192 3193
	mutex_lock(&pool->attach_mutex);
	if (!list_empty(&pool->workers))
		pool->detach_completion = &detach_completion;
	mutex_unlock(&pool->attach_mutex);
3194

3195 3196
	if (pool->detach_completion)
		wait_for_completion(pool->detach_completion);
3197

3198
	mutex_unlock(&pool->manager_arb);
3199

3200 3201 3202
	/* shut down the timers */
	del_timer_sync(&pool->idle_timer);
	del_timer_sync(&pool->mayday_timer);
3203

3204 3205
	/* sched-RCU protected to allow dereferences from get_work_pool() */
	call_rcu_sched(&pool->rcu, rcu_free_pool);
3206 3207 3208
}

/**
3209 3210
 * get_unbound_pool - get a worker_pool with the specified attributes
 * @attrs: the attributes of the worker_pool to get
3211
 *
3212 3213 3214 3215
 * Obtain a worker_pool which has the same attributes as @attrs, bump the
 * reference count and return it.  If there already is a matching
 * worker_pool, it will be used; otherwise, this function attempts to
 * create a new one.
3216
 *
3217
 * Should be called with wq_pool_mutex held.
3218
 *
3219 3220
 * Return: On success, a worker_pool with the same attributes as @attrs.
 * On failure, %NULL.
3221
 */
3222
static struct worker_pool *get_unbound_pool(const struct workqueue_attrs *attrs)
3223
{
3224 3225 3226
	u32 hash = wqattrs_hash(attrs);
	struct worker_pool *pool;
	int node;
3227

3228
	lockdep_assert_held(&wq_pool_mutex);
3229

3230 3231 3232 3233 3234 3235 3236
	/* do we already have a matching pool? */
	hash_for_each_possible(unbound_pool_hash, pool, hash_node, hash) {
		if (wqattrs_equal(pool->attrs, attrs)) {
			pool->refcnt++;
			return pool;
		}
	}
3237

3238 3239 3240 3241 3242 3243 3244
	/* nope, create a new one */
	pool = kzalloc(sizeof(*pool), GFP_KERNEL);
	if (!pool || init_worker_pool(pool) < 0)
		goto fail;

	lockdep_set_subclass(&pool->lock, 1);	/* see put_pwq() */
	copy_workqueue_attrs(pool->attrs, attrs);
3245 3246

	/*
3247 3248
	 * no_numa isn't a worker_pool attribute, always clear it.  See
	 * 'struct workqueue_attrs' comments for detail.
3249
	 */
3250
	pool->attrs->no_numa = false;
3251

3252 3253 3254 3255 3256 3257 3258
	/* if cpumask is contained inside a NUMA node, we belong to that node */
	if (wq_numa_enabled) {
		for_each_node(node) {
			if (cpumask_subset(pool->attrs->cpumask,
					   wq_numa_possible_cpumask[node])) {
				pool->node = node;
				break;
3259 3260 3261 3262
			}
		}
	}

3263 3264
	if (worker_pool_assign_id(pool) < 0)
		goto fail;
3265

3266 3267 3268
	/* create and start the initial worker */
	if (!create_worker(pool))
		goto fail;
3269

3270 3271
	/* install */
	hash_add(unbound_pool_hash, &pool->hash_node, hash);
3272

3273 3274 3275 3276 3277
	return pool;
fail:
	if (pool)
		put_unbound_pool(pool);
	return NULL;
3278 3279
}

3280
static void rcu_free_pwq(struct rcu_head *rcu)
T
Tejun Heo 已提交
3281
{
3282 3283
	kmem_cache_free(pwq_cache,
			container_of(rcu, struct pool_workqueue, rcu));
T
Tejun Heo 已提交
3284 3285
}

3286 3287 3288
/*
 * Scheduled on system_wq by put_pwq() when an unbound pwq hits zero refcnt
 * and needs to be destroyed.
T
Tejun Heo 已提交
3289
 */
3290
static void pwq_unbound_release_workfn(struct work_struct *work)
T
Tejun Heo 已提交
3291
{
3292 3293 3294 3295 3296
	struct pool_workqueue *pwq = container_of(work, struct pool_workqueue,
						  unbound_release_work);
	struct workqueue_struct *wq = pwq->wq;
	struct worker_pool *pool = pwq->pool;
	bool is_last;
T
Tejun Heo 已提交
3297

3298 3299
	if (WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND)))
		return;
T
Tejun Heo 已提交
3300

3301 3302 3303 3304 3305 3306 3307 3308 3309 3310
	mutex_lock(&wq->mutex);
	list_del_rcu(&pwq->pwqs_node);
	is_last = list_empty(&wq->pwqs);
	mutex_unlock(&wq->mutex);

	mutex_lock(&wq_pool_mutex);
	put_unbound_pool(pool);
	mutex_unlock(&wq_pool_mutex);

	call_rcu_sched(&pwq->rcu, rcu_free_pwq);
T
Tejun Heo 已提交
3311

3312
	/*
3313 3314
	 * If we're the last pwq going away, @wq is already dead and no one
	 * is gonna access it anymore.  Schedule RCU free.
3315
	 */
3316 3317
	if (is_last)
		call_rcu_sched(&wq->rcu, rcu_free_wq);
3318 3319
}

T
Tejun Heo 已提交
3320
/**
3321 3322
 * pwq_adjust_max_active - update a pwq's max_active to the current setting
 * @pwq: target pool_workqueue
3323
 *
3324 3325 3326
 * If @pwq isn't freezing, set @pwq->max_active to the associated
 * workqueue's saved_max_active and activate delayed work items
 * accordingly.  If @pwq is freezing, clear @pwq->max_active to zero.
T
Tejun Heo 已提交
3327
 */
3328
static void pwq_adjust_max_active(struct pool_workqueue *pwq)
3329
{
3330 3331
	struct workqueue_struct *wq = pwq->wq;
	bool freezable = wq->flags & WQ_FREEZABLE;
3332

3333 3334
	/* for @wq->saved_max_active */
	lockdep_assert_held(&wq->mutex);
3335

3336 3337 3338
	/* fast exit for non-freezable wqs */
	if (!freezable && pwq->max_active == wq->saved_max_active)
		return;
T
Tejun Heo 已提交
3339

3340
	spin_lock_irq(&pwq->pool->lock);
3341

3342 3343 3344 3345 3346 3347 3348
	/*
	 * During [un]freezing, the caller is responsible for ensuring that
	 * this function is called at least once after @workqueue_freezing
	 * is updated and visible.
	 */
	if (!freezable || !workqueue_freezing) {
		pwq->max_active = wq->saved_max_active;
3349

3350 3351 3352
		while (!list_empty(&pwq->delayed_works) &&
		       pwq->nr_active < pwq->max_active)
			pwq_activate_first_delayed(pwq);
3353

3354 3355 3356 3357 3358 3359 3360 3361
		/*
		 * Need to kick a worker after thawed or an unbound wq's
		 * max_active is bumped.  It's a slow path.  Do it always.
		 */
		wake_up_worker(pwq->pool);
	} else {
		pwq->max_active = 0;
	}
3362

3363
	spin_unlock_irq(&pwq->pool->lock);
3364 3365
}

3366 3367 3368
/* initialize newly alloced @pwq which is associated with @wq and @pool */
static void init_pwq(struct pool_workqueue *pwq, struct workqueue_struct *wq,
		     struct worker_pool *pool)
3369
{
3370
	BUG_ON((unsigned long)pwq & WORK_STRUCT_FLAG_MASK);
3371

3372 3373 3374 3375 3376 3377 3378 3379 3380 3381
	memset(pwq, 0, sizeof(*pwq));

	pwq->pool = pool;
	pwq->wq = wq;
	pwq->flush_color = -1;
	pwq->refcnt = 1;
	INIT_LIST_HEAD(&pwq->delayed_works);
	INIT_LIST_HEAD(&pwq->pwqs_node);
	INIT_LIST_HEAD(&pwq->mayday_node);
	INIT_WORK(&pwq->unbound_release_work, pwq_unbound_release_workfn);
3382 3383
}

3384 3385
/* sync @pwq with the current state of its associated wq and link it */
static void link_pwq(struct pool_workqueue *pwq)
3386
{
3387
	struct workqueue_struct *wq = pwq->wq;
3388

3389
	lockdep_assert_held(&wq->mutex);
3390

3391 3392
	/* may be called multiple times, ignore if already linked */
	if (!list_empty(&pwq->pwqs_node))
3393 3394
		return;

3395 3396
	/* set the matching work_color */
	pwq->work_color = wq->work_color;
3397

3398 3399
	/* sync max_active to the current setting */
	pwq_adjust_max_active(pwq);
3400

3401 3402 3403
	/* link in @pwq */
	list_add_rcu(&pwq->pwqs_node, &wq->pwqs);
}
3404

3405 3406 3407 3408 3409 3410
/* obtain a pool matching @attr and create a pwq associating the pool and @wq */
static struct pool_workqueue *alloc_unbound_pwq(struct workqueue_struct *wq,
					const struct workqueue_attrs *attrs)
{
	struct worker_pool *pool;
	struct pool_workqueue *pwq;
3411

3412
	lockdep_assert_held(&wq_pool_mutex);
3413

3414 3415 3416
	pool = get_unbound_pool(attrs);
	if (!pool)
		return NULL;
3417

3418 3419 3420 3421 3422
	pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node);
	if (!pwq) {
		put_unbound_pool(pool);
		return NULL;
	}
3423

3424 3425 3426
	init_pwq(pwq, wq, pool);
	return pwq;
}
3427

3428 3429 3430 3431 3432 3433 3434 3435 3436
/* undo alloc_unbound_pwq(), used only in the error path */
static void free_unbound_pwq(struct pool_workqueue *pwq)
{
	lockdep_assert_held(&wq_pool_mutex);

	if (pwq) {
		put_unbound_pool(pwq->pool);
		kmem_cache_free(pwq_cache, pwq);
	}
3437 3438 3439
}

/**
3440 3441 3442 3443 3444
 * wq_calc_node_mask - calculate a wq_attrs' cpumask for the specified node
 * @attrs: the wq_attrs of interest
 * @node: the target NUMA node
 * @cpu_going_down: if >= 0, the CPU to consider as offline
 * @cpumask: outarg, the resulting cpumask
3445
 *
3446 3447 3448
 * Calculate the cpumask a workqueue with @attrs should use on @node.  If
 * @cpu_going_down is >= 0, that cpu is considered offline during
 * calculation.  The result is stored in @cpumask.
3449
 *
3450 3451 3452 3453
 * If NUMA affinity is not enabled, @attrs->cpumask is always used.  If
 * enabled and @node has online CPUs requested by @attrs, the returned
 * cpumask is the intersection of the possible CPUs of @node and
 * @attrs->cpumask.
3454
 *
3455 3456 3457 3458 3459
 * The caller is responsible for ensuring that the cpumask of @node stays
 * stable.
 *
 * Return: %true if the resulting @cpumask is different from @attrs->cpumask,
 * %false if equal.
3460
 */
3461 3462
static bool wq_calc_node_cpumask(const struct workqueue_attrs *attrs, int node,
				 int cpu_going_down, cpumask_t *cpumask)
3463
{
3464 3465
	if (!wq_numa_enabled || attrs->no_numa)
		goto use_dfl;
3466

3467 3468 3469 3470
	/* does @node have any online CPUs @attrs wants? */
	cpumask_and(cpumask, cpumask_of_node(node), attrs->cpumask);
	if (cpu_going_down >= 0)
		cpumask_clear_cpu(cpu_going_down, cpumask);
3471

3472 3473
	if (cpumask_empty(cpumask))
		goto use_dfl;
3474 3475 3476 3477 3478 3479 3480 3481 3482 3483

	/* yeap, return possible CPUs in @node that @attrs wants */
	cpumask_and(cpumask, attrs->cpumask, wq_numa_possible_cpumask[node]);
	return !cpumask_equal(cpumask, attrs->cpumask);

use_dfl:
	cpumask_copy(cpumask, attrs->cpumask);
	return false;
}

3484 3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498 3499 3500
/* install @pwq into @wq's numa_pwq_tbl[] for @node and return the old pwq */
static struct pool_workqueue *numa_pwq_tbl_install(struct workqueue_struct *wq,
						   int node,
						   struct pool_workqueue *pwq)
{
	struct pool_workqueue *old_pwq;

	lockdep_assert_held(&wq->mutex);

	/* link_pwq() can handle duplicate calls */
	link_pwq(pwq);

	old_pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]);
	rcu_assign_pointer(wq->numa_pwq_tbl[node], pwq);
	return old_pwq;
}

3501 3502 3503 3504 3505
/**
 * apply_workqueue_attrs - apply new workqueue_attrs to an unbound workqueue
 * @wq: the target workqueue
 * @attrs: the workqueue_attrs to apply, allocated with alloc_workqueue_attrs()
 *
3506 3507 3508 3509 3510 3511
 * Apply @attrs to an unbound workqueue @wq.  Unless disabled, on NUMA
 * machines, this function maps a separate pwq to each NUMA node with
 * possibles CPUs in @attrs->cpumask so that work items are affine to the
 * NUMA node it was issued on.  Older pwqs are released as in-flight work
 * items finish.  Note that a work item which repeatedly requeues itself
 * back-to-back will stay on its current pwq.
3512
 *
3513 3514 3515
 * Performs GFP_KERNEL allocations.
 *
 * Return: 0 on success and -errno on failure.
3516 3517 3518 3519
 */
int apply_workqueue_attrs(struct workqueue_struct *wq,
			  const struct workqueue_attrs *attrs)
{
3520 3521
	struct workqueue_attrs *new_attrs, *tmp_attrs;
	struct pool_workqueue **pwq_tbl, *dfl_pwq;
3522
	int node, ret;
3523

3524
	/* only unbound workqueues can change attributes */
3525 3526 3527
	if (WARN_ON(!(wq->flags & WQ_UNBOUND)))
		return -EINVAL;

3528 3529 3530 3531
	/* creating multiple pwqs breaks ordering guarantee */
	if (WARN_ON((wq->flags & __WQ_ORDERED) && !list_empty(&wq->pwqs)))
		return -EINVAL;

3532
	pwq_tbl = kzalloc(nr_node_ids * sizeof(pwq_tbl[0]), GFP_KERNEL);
3533
	new_attrs = alloc_workqueue_attrs(GFP_KERNEL);
3534 3535
	tmp_attrs = alloc_workqueue_attrs(GFP_KERNEL);
	if (!pwq_tbl || !new_attrs || !tmp_attrs)
3536 3537
		goto enomem;

3538
	/* make a copy of @attrs and sanitize it */
3539 3540 3541
	copy_workqueue_attrs(new_attrs, attrs);
	cpumask_and(new_attrs->cpumask, new_attrs->cpumask, cpu_possible_mask);

3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555
	/*
	 * We may create multiple pwqs with differing cpumasks.  Make a
	 * copy of @new_attrs which will be modified and used to obtain
	 * pools.
	 */
	copy_workqueue_attrs(tmp_attrs, new_attrs);

	/*
	 * CPUs should stay stable across pwq creations and installations.
	 * Pin CPUs, determine the target cpumask for each node and create
	 * pwqs accordingly.
	 */
	get_online_cpus();

3556
	mutex_lock(&wq_pool_mutex);
3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577

	/*
	 * If something goes wrong during CPU up/down, we'll fall back to
	 * the default pwq covering whole @attrs->cpumask.  Always create
	 * it even if we don't use it immediately.
	 */
	dfl_pwq = alloc_unbound_pwq(wq, new_attrs);
	if (!dfl_pwq)
		goto enomem_pwq;

	for_each_node(node) {
		if (wq_calc_node_cpumask(attrs, node, -1, tmp_attrs->cpumask)) {
			pwq_tbl[node] = alloc_unbound_pwq(wq, tmp_attrs);
			if (!pwq_tbl[node])
				goto enomem_pwq;
		} else {
			dfl_pwq->refcnt++;
			pwq_tbl[node] = dfl_pwq;
		}
	}

3578
	mutex_unlock(&wq_pool_mutex);
3579

3580
	/* all pwqs have been created successfully, let's install'em */
3581
	mutex_lock(&wq->mutex);
3582

3583
	copy_workqueue_attrs(wq->unbound_attrs, new_attrs);
3584 3585

	/* save the previous pwq and install the new one */
3586
	for_each_node(node)
3587 3588 3589 3590 3591
		pwq_tbl[node] = numa_pwq_tbl_install(wq, node, pwq_tbl[node]);

	/* @dfl_pwq might not have been used, ensure it's linked */
	link_pwq(dfl_pwq);
	swap(wq->dfl_pwq, dfl_pwq);
3592 3593

	mutex_unlock(&wq->mutex);
3594

3595 3596 3597 3598 3599 3600
	/* put the old pwqs */
	for_each_node(node)
		put_pwq_unlocked(pwq_tbl[node]);
	put_pwq_unlocked(dfl_pwq);

	put_online_cpus();
3601 3602 3603
	ret = 0;
	/* fall through */
out_free:
3604
	free_workqueue_attrs(tmp_attrs);
3605
	free_workqueue_attrs(new_attrs);
3606
	kfree(pwq_tbl);
3607
	return ret;
3608

3609 3610 3611 3612 3613 3614 3615
enomem_pwq:
	free_unbound_pwq(dfl_pwq);
	for_each_node(node)
		if (pwq_tbl && pwq_tbl[node] != dfl_pwq)
			free_unbound_pwq(pwq_tbl[node]);
	mutex_unlock(&wq_pool_mutex);
	put_online_cpus();
3616
enomem:
3617 3618
	ret = -ENOMEM;
	goto out_free;
3619 3620
}

3621 3622 3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665
/**
 * wq_update_unbound_numa - update NUMA affinity of a wq for CPU hot[un]plug
 * @wq: the target workqueue
 * @cpu: the CPU coming up or going down
 * @online: whether @cpu is coming up or going down
 *
 * This function is to be called from %CPU_DOWN_PREPARE, %CPU_ONLINE and
 * %CPU_DOWN_FAILED.  @cpu is being hot[un]plugged, update NUMA affinity of
 * @wq accordingly.
 *
 * If NUMA affinity can't be adjusted due to memory allocation failure, it
 * falls back to @wq->dfl_pwq which may not be optimal but is always
 * correct.
 *
 * Note that when the last allowed CPU of a NUMA node goes offline for a
 * workqueue with a cpumask spanning multiple nodes, the workers which were
 * already executing the work items for the workqueue will lose their CPU
 * affinity and may execute on any CPU.  This is similar to how per-cpu
 * workqueues behave on CPU_DOWN.  If a workqueue user wants strict
 * affinity, it's the user's responsibility to flush the work item from
 * CPU_DOWN_PREPARE.
 */
static void wq_update_unbound_numa(struct workqueue_struct *wq, int cpu,
				   bool online)
{
	int node = cpu_to_node(cpu);
	int cpu_off = online ? -1 : cpu;
	struct pool_workqueue *old_pwq = NULL, *pwq;
	struct workqueue_attrs *target_attrs;
	cpumask_t *cpumask;

	lockdep_assert_held(&wq_pool_mutex);

	if (!wq_numa_enabled || !(wq->flags & WQ_UNBOUND))
		return;

	/*
	 * We don't wanna alloc/free wq_attrs for each wq for each CPU.
	 * Let's use a preallocated one.  The following buf is protected by
	 * CPU hotplug exclusion.
	 */
	target_attrs = wq_update_unbound_numa_attrs_buf;
	cpumask = target_attrs->cpumask;

	mutex_lock(&wq->mutex);
3666 3667
	if (wq->unbound_attrs->no_numa)
		goto out_unlock;
3668 3669 3670 3671 3672 3673 3674 3675

	copy_workqueue_attrs(target_attrs, wq->unbound_attrs);
	pwq = unbound_pwq_by_node(wq, node);

	/*
	 * Let's determine what needs to be done.  If the target cpumask is
	 * different from wq's, we need to compare it to @pwq's and create
	 * a new one if they don't match.  If the target cpumask equals
3676
	 * wq's, the default pwq should be used.
3677 3678 3679 3680 3681
	 */
	if (wq_calc_node_cpumask(wq->unbound_attrs, node, cpu_off, cpumask)) {
		if (cpumask_equal(cpumask, pwq->pool->attrs->cpumask))
			goto out_unlock;
	} else {
3682
		goto use_dfl_pwq;
3683 3684 3685 3686 3687 3688 3689
	}

	mutex_unlock(&wq->mutex);

	/* create a new pwq */
	pwq = alloc_unbound_pwq(wq, target_attrs);
	if (!pwq) {
3690 3691
		pr_warn("workqueue: allocation failed while updating NUMA affinity of \"%s\"\n",
			wq->name);
3692 3693
		mutex_lock(&wq->mutex);
		goto use_dfl_pwq;
3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715
	}

	/*
	 * Install the new pwq.  As this function is called only from CPU
	 * hotplug callbacks and applying a new attrs is wrapped with
	 * get/put_online_cpus(), @wq->unbound_attrs couldn't have changed
	 * inbetween.
	 */
	mutex_lock(&wq->mutex);
	old_pwq = numa_pwq_tbl_install(wq, node, pwq);
	goto out_unlock;

use_dfl_pwq:
	spin_lock_irq(&wq->dfl_pwq->pool->lock);
	get_pwq(wq->dfl_pwq);
	spin_unlock_irq(&wq->dfl_pwq->pool->lock);
	old_pwq = numa_pwq_tbl_install(wq, node, wq->dfl_pwq);
out_unlock:
	mutex_unlock(&wq->mutex);
	put_pwq_unlocked(old_pwq);
}

3716
static int alloc_and_link_pwqs(struct workqueue_struct *wq)
T
Tejun Heo 已提交
3717
{
3718
	bool highpri = wq->flags & WQ_HIGHPRI;
3719
	int cpu, ret;
3720 3721

	if (!(wq->flags & WQ_UNBOUND)) {
3722 3723
		wq->cpu_pwqs = alloc_percpu(struct pool_workqueue);
		if (!wq->cpu_pwqs)
3724 3725 3726
			return -ENOMEM;

		for_each_possible_cpu(cpu) {
3727 3728
			struct pool_workqueue *pwq =
				per_cpu_ptr(wq->cpu_pwqs, cpu);
3729
			struct worker_pool *cpu_pools =
3730
				per_cpu(cpu_worker_pools, cpu);
3731

3732 3733 3734
			init_pwq(pwq, wq, &cpu_pools[highpri]);

			mutex_lock(&wq->mutex);
3735
			link_pwq(pwq);
3736
			mutex_unlock(&wq->mutex);
3737
		}
3738
		return 0;
3739 3740 3741 3742 3743 3744 3745
	} else if (wq->flags & __WQ_ORDERED) {
		ret = apply_workqueue_attrs(wq, ordered_wq_attrs[highpri]);
		/* there should only be single pwq for ordering guarantee */
		WARN(!ret && (wq->pwqs.next != &wq->dfl_pwq->pwqs_node ||
			      wq->pwqs.prev != &wq->dfl_pwq->pwqs_node),
		     "ordering guarantee broken for workqueue %s\n", wq->name);
		return ret;
3746
	} else {
3747
		return apply_workqueue_attrs(wq, unbound_std_wq_attrs[highpri]);
3748
	}
T
Tejun Heo 已提交
3749 3750
}

3751 3752
static int wq_clamp_max_active(int max_active, unsigned int flags,
			       const char *name)
3753
{
3754 3755 3756
	int lim = flags & WQ_UNBOUND ? WQ_UNBOUND_MAX_ACTIVE : WQ_MAX_ACTIVE;

	if (max_active < 1 || max_active > lim)
V
Valentin Ilie 已提交
3757 3758
		pr_warn("workqueue: max_active %d requested for %s is out of range, clamping between %d and %d\n",
			max_active, name, 1, lim);
3759

3760
	return clamp_val(max_active, 1, lim);
3761 3762
}

3763
struct workqueue_struct *__alloc_workqueue_key(const char *fmt,
3764 3765 3766
					       unsigned int flags,
					       int max_active,
					       struct lock_class_key *key,
3767
					       const char *lock_name, ...)
L
Linus Torvalds 已提交
3768
{
3769
	size_t tbl_size = 0;
3770
	va_list args;
L
Linus Torvalds 已提交
3771
	struct workqueue_struct *wq;
3772
	struct pool_workqueue *pwq;
3773

3774 3775 3776 3777
	/* see the comment above the definition of WQ_POWER_EFFICIENT */
	if ((flags & WQ_POWER_EFFICIENT) && wq_power_efficient)
		flags |= WQ_UNBOUND;

3778
	/* allocate wq and format name */
3779
	if (flags & WQ_UNBOUND)
3780
		tbl_size = nr_node_ids * sizeof(wq->numa_pwq_tbl[0]);
3781 3782

	wq = kzalloc(sizeof(*wq) + tbl_size, GFP_KERNEL);
3783
	if (!wq)
3784
		return NULL;
3785

3786 3787 3788 3789 3790 3791
	if (flags & WQ_UNBOUND) {
		wq->unbound_attrs = alloc_workqueue_attrs(GFP_KERNEL);
		if (!wq->unbound_attrs)
			goto err_free_wq;
	}

3792 3793
	va_start(args, lock_name);
	vsnprintf(wq->name, sizeof(wq->name), fmt, args);
3794
	va_end(args);
L
Linus Torvalds 已提交
3795

3796
	max_active = max_active ?: WQ_DFL_ACTIVE;
3797
	max_active = wq_clamp_max_active(max_active, flags, wq->name);
3798

3799
	/* init wq */
3800
	wq->flags = flags;
3801
	wq->saved_max_active = max_active;
3802
	mutex_init(&wq->mutex);
3803
	atomic_set(&wq->nr_pwqs_to_flush, 0);
3804
	INIT_LIST_HEAD(&wq->pwqs);
3805 3806
	INIT_LIST_HEAD(&wq->flusher_queue);
	INIT_LIST_HEAD(&wq->flusher_overflow);
3807
	INIT_LIST_HEAD(&wq->maydays);
3808

3809
	lockdep_init_map(&wq->lockdep_map, lock_name, key, 0);
3810
	INIT_LIST_HEAD(&wq->list);
3811

3812
	if (alloc_and_link_pwqs(wq) < 0)
3813
		goto err_free_wq;
T
Tejun Heo 已提交
3814

3815 3816 3817 3818 3819
	/*
	 * Workqueues which may be used during memory reclaim should
	 * have a rescuer to guarantee forward progress.
	 */
	if (flags & WQ_MEM_RECLAIM) {
3820 3821
		struct worker *rescuer;

3822
		rescuer = alloc_worker(NUMA_NO_NODE);
3823
		if (!rescuer)
3824
			goto err_destroy;
3825

3826 3827
		rescuer->rescue_wq = wq;
		rescuer->task = kthread_create(rescuer_thread, rescuer, "%s",
3828
					       wq->name);
3829 3830 3831 3832
		if (IS_ERR(rescuer->task)) {
			kfree(rescuer);
			goto err_destroy;
		}
3833

3834
		wq->rescuer = rescuer;
3835
		rescuer->task->flags |= PF_NO_SETAFFINITY;
3836
		wake_up_process(rescuer->task);
3837 3838
	}

3839 3840 3841
	if ((wq->flags & WQ_SYSFS) && workqueue_sysfs_register(wq))
		goto err_destroy;

3842
	/*
3843 3844 3845
	 * wq_pool_mutex protects global freeze state and workqueues list.
	 * Grab it, adjust max_active and add the new @wq to workqueues
	 * list.
3846
	 */
3847
	mutex_lock(&wq_pool_mutex);
3848

3849
	mutex_lock(&wq->mutex);
3850 3851
	for_each_pwq(pwq, wq)
		pwq_adjust_max_active(pwq);
3852
	mutex_unlock(&wq->mutex);
3853

3854
	list_add_tail_rcu(&wq->list, &workqueues);
3855

3856
	mutex_unlock(&wq_pool_mutex);
T
Tejun Heo 已提交
3857

3858
	return wq;
3859 3860

err_free_wq:
3861
	free_workqueue_attrs(wq->unbound_attrs);
3862 3863 3864 3865
	kfree(wq);
	return NULL;
err_destroy:
	destroy_workqueue(wq);
T
Tejun Heo 已提交
3866
	return NULL;
3867
}
3868
EXPORT_SYMBOL_GPL(__alloc_workqueue_key);
L
Linus Torvalds 已提交
3869

3870 3871 3872 3873 3874 3875 3876 3877
/**
 * destroy_workqueue - safely terminate a workqueue
 * @wq: target workqueue
 *
 * Safely destroy a workqueue. All work currently pending will be done first.
 */
void destroy_workqueue(struct workqueue_struct *wq)
{
3878
	struct pool_workqueue *pwq;
3879
	int node;
3880

3881 3882
	/* drain it before proceeding with destruction */
	drain_workqueue(wq);
3883

3884
	/* sanity checks */
3885
	mutex_lock(&wq->mutex);
3886
	for_each_pwq(pwq, wq) {
3887 3888
		int i;

3889 3890
		for (i = 0; i < WORK_NR_COLORS; i++) {
			if (WARN_ON(pwq->nr_in_flight[i])) {
3891
				mutex_unlock(&wq->mutex);
3892
				return;
3893 3894 3895
			}
		}

3896
		if (WARN_ON((pwq != wq->dfl_pwq) && (pwq->refcnt > 1)) ||
T
Tejun Heo 已提交
3897
		    WARN_ON(pwq->nr_active) ||
3898
		    WARN_ON(!list_empty(&pwq->delayed_works))) {
3899
			mutex_unlock(&wq->mutex);
3900
			return;
3901
		}
3902
	}
3903
	mutex_unlock(&wq->mutex);
3904

3905 3906 3907 3908
	/*
	 * wq list is used to freeze wq, remove from list after
	 * flushing is complete in case freeze races us.
	 */
3909
	mutex_lock(&wq_pool_mutex);
3910
	list_del_rcu(&wq->list);
3911
	mutex_unlock(&wq_pool_mutex);
3912

3913 3914
	workqueue_sysfs_unregister(wq);

3915
	if (wq->rescuer)
3916 3917
		kthread_stop(wq->rescuer->task);

T
Tejun Heo 已提交
3918 3919 3920
	if (!(wq->flags & WQ_UNBOUND)) {
		/*
		 * The base ref is never dropped on per-cpu pwqs.  Directly
3921
		 * schedule RCU free.
T
Tejun Heo 已提交
3922
		 */
3923
		call_rcu_sched(&wq->rcu, rcu_free_wq);
T
Tejun Heo 已提交
3924 3925 3926
	} else {
		/*
		 * We're the sole accessor of @wq at this point.  Directly
3927 3928
		 * access numa_pwq_tbl[] and dfl_pwq to put the base refs.
		 * @wq will be freed when the last pwq is released.
T
Tejun Heo 已提交
3929
		 */
3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941
		for_each_node(node) {
			pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]);
			RCU_INIT_POINTER(wq->numa_pwq_tbl[node], NULL);
			put_pwq_unlocked(pwq);
		}

		/*
		 * Put dfl_pwq.  @wq may be freed any time after dfl_pwq is
		 * put.  Don't access it afterwards.
		 */
		pwq = wq->dfl_pwq;
		wq->dfl_pwq = NULL;
3942
		put_pwq_unlocked(pwq);
3943
	}
3944 3945 3946
}
EXPORT_SYMBOL_GPL(destroy_workqueue);

3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958
/**
 * workqueue_set_max_active - adjust max_active of a workqueue
 * @wq: target workqueue
 * @max_active: new max_active value.
 *
 * Set max_active of @wq to @max_active.
 *
 * CONTEXT:
 * Don't call from IRQ context.
 */
void workqueue_set_max_active(struct workqueue_struct *wq, int max_active)
{
3959
	struct pool_workqueue *pwq;
3960

3961 3962 3963 3964
	/* disallow meddling with max_active for ordered workqueues */
	if (WARN_ON(wq->flags & __WQ_ORDERED))
		return;

3965
	max_active = wq_clamp_max_active(max_active, wq->flags, wq->name);
3966

3967
	mutex_lock(&wq->mutex);
3968 3969 3970

	wq->saved_max_active = max_active;

3971 3972
	for_each_pwq(pwq, wq)
		pwq_adjust_max_active(pwq);
3973

3974
	mutex_unlock(&wq->mutex);
3975
}
3976
EXPORT_SYMBOL_GPL(workqueue_set_max_active);
3977

3978 3979 3980 3981 3982
/**
 * current_is_workqueue_rescuer - is %current workqueue rescuer?
 *
 * Determine whether %current is a workqueue rescuer.  Can be used from
 * work functions to determine whether it's being run off the rescuer task.
3983 3984
 *
 * Return: %true if %current is a workqueue rescuer. %false otherwise.
3985 3986 3987 3988 3989
 */
bool current_is_workqueue_rescuer(void)
{
	struct worker *worker = current_wq_worker();

3990
	return worker && worker->rescue_wq;
3991 3992
}

3993
/**
3994 3995 3996
 * workqueue_congested - test whether a workqueue is congested
 * @cpu: CPU in question
 * @wq: target workqueue
3997
 *
3998 3999 4000
 * Test whether @wq's cpu workqueue for @cpu is congested.  There is
 * no synchronization around this function and the test result is
 * unreliable and only useful as advisory hints or for debugging.
4001
 *
4002 4003 4004 4005 4006 4007
 * If @cpu is WORK_CPU_UNBOUND, the test is performed on the local CPU.
 * Note that both per-cpu and unbound workqueues may be associated with
 * multiple pool_workqueues which have separate congested states.  A
 * workqueue being congested on one CPU doesn't mean the workqueue is also
 * contested on other CPUs / NUMA nodes.
 *
4008
 * Return:
4009
 * %true if congested, %false otherwise.
4010
 */
4011
bool workqueue_congested(int cpu, struct workqueue_struct *wq)
L
Linus Torvalds 已提交
4012
{
4013
	struct pool_workqueue *pwq;
4014 4015
	bool ret;

4016
	rcu_read_lock_sched();
4017

4018 4019 4020
	if (cpu == WORK_CPU_UNBOUND)
		cpu = smp_processor_id();

4021 4022 4023
	if (!(wq->flags & WQ_UNBOUND))
		pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);
	else
4024
		pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));
4025

4026
	ret = !list_empty(&pwq->delayed_works);
4027
	rcu_read_unlock_sched();
4028 4029

	return ret;
L
Linus Torvalds 已提交
4030
}
4031
EXPORT_SYMBOL_GPL(workqueue_congested);
L
Linus Torvalds 已提交
4032

4033 4034 4035 4036 4037 4038 4039 4040
/**
 * work_busy - test whether a work is currently pending or running
 * @work: the work to be tested
 *
 * Test whether @work is currently pending or running.  There is no
 * synchronization around this function and the test result is
 * unreliable and only useful as advisory hints or for debugging.
 *
4041
 * Return:
4042 4043 4044
 * OR'd bitmask of WORK_BUSY_* bits.
 */
unsigned int work_busy(struct work_struct *work)
L
Linus Torvalds 已提交
4045
{
4046
	struct worker_pool *pool;
4047 4048
	unsigned long flags;
	unsigned int ret = 0;
L
Linus Torvalds 已提交
4049

4050 4051
	if (work_pending(work))
		ret |= WORK_BUSY_PENDING;
L
Linus Torvalds 已提交
4052

4053 4054
	local_irq_save(flags);
	pool = get_work_pool(work);
4055
	if (pool) {
4056
		spin_lock(&pool->lock);
4057 4058
		if (find_worker_executing_work(pool, work))
			ret |= WORK_BUSY_RUNNING;
4059
		spin_unlock(&pool->lock);
4060
	}
4061
	local_irq_restore(flags);
L
Linus Torvalds 已提交
4062

4063
	return ret;
L
Linus Torvalds 已提交
4064
}
4065
EXPORT_SYMBOL_GPL(work_busy);
L
Linus Torvalds 已提交
4066

4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077 4078 4079 4080 4081 4082 4083 4084 4085 4086 4087 4088 4089 4090 4091 4092 4093 4094 4095 4096 4097 4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114 4115 4116 4117 4118 4119 4120 4121 4122 4123 4124 4125 4126 4127 4128 4129 4130 4131 4132 4133 4134 4135 4136 4137 4138 4139 4140 4141 4142 4143
/**
 * set_worker_desc - set description for the current work item
 * @fmt: printf-style format string
 * @...: arguments for the format string
 *
 * This function can be called by a running work function to describe what
 * the work item is about.  If the worker task gets dumped, this
 * information will be printed out together to help debugging.  The
 * description can be at most WORKER_DESC_LEN including the trailing '\0'.
 */
void set_worker_desc(const char *fmt, ...)
{
	struct worker *worker = current_wq_worker();
	va_list args;

	if (worker) {
		va_start(args, fmt);
		vsnprintf(worker->desc, sizeof(worker->desc), fmt, args);
		va_end(args);
		worker->desc_valid = true;
	}
}

/**
 * print_worker_info - print out worker information and description
 * @log_lvl: the log level to use when printing
 * @task: target task
 *
 * If @task is a worker and currently executing a work item, print out the
 * name of the workqueue being serviced and worker description set with
 * set_worker_desc() by the currently executing work item.
 *
 * This function can be safely called on any task as long as the
 * task_struct itself is accessible.  While safe, this function isn't
 * synchronized and may print out mixups or garbages of limited length.
 */
void print_worker_info(const char *log_lvl, struct task_struct *task)
{
	work_func_t *fn = NULL;
	char name[WQ_NAME_LEN] = { };
	char desc[WORKER_DESC_LEN] = { };
	struct pool_workqueue *pwq = NULL;
	struct workqueue_struct *wq = NULL;
	bool desc_valid = false;
	struct worker *worker;

	if (!(task->flags & PF_WQ_WORKER))
		return;

	/*
	 * This function is called without any synchronization and @task
	 * could be in any state.  Be careful with dereferences.
	 */
	worker = probe_kthread_data(task);

	/*
	 * Carefully copy the associated workqueue's workfn and name.  Keep
	 * the original last '\0' in case the original contains garbage.
	 */
	probe_kernel_read(&fn, &worker->current_func, sizeof(fn));
	probe_kernel_read(&pwq, &worker->current_pwq, sizeof(pwq));
	probe_kernel_read(&wq, &pwq->wq, sizeof(wq));
	probe_kernel_read(name, wq->name, sizeof(name) - 1);

	/* copy worker description */
	probe_kernel_read(&desc_valid, &worker->desc_valid, sizeof(desc_valid));
	if (desc_valid)
		probe_kernel_read(desc, worker->desc, sizeof(desc) - 1);

	if (fn || name[0] || desc[0]) {
		printk("%sWorkqueue: %s %pf", log_lvl, name, fn);
		if (desc[0])
			pr_cont(" (%s)", desc);
		pr_cont("\n");
	}
}

4144 4145 4146 4147 4148 4149 4150 4151 4152 4153 4154 4155 4156 4157 4158 4159 4160 4161 4162 4163 4164 4165 4166 4167 4168 4169 4170 4171 4172 4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183 4184 4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214 4215 4216 4217 4218 4219 4220 4221 4222 4223 4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235 4236 4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259 4260 4261 4262 4263 4264 4265 4266 4267 4268 4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303
static void pr_cont_pool_info(struct worker_pool *pool)
{
	pr_cont(" cpus=%*pbl", nr_cpumask_bits, pool->attrs->cpumask);
	if (pool->node != NUMA_NO_NODE)
		pr_cont(" node=%d", pool->node);
	pr_cont(" flags=0x%x nice=%d", pool->flags, pool->attrs->nice);
}

static void pr_cont_work(bool comma, struct work_struct *work)
{
	if (work->func == wq_barrier_func) {
		struct wq_barrier *barr;

		barr = container_of(work, struct wq_barrier, work);

		pr_cont("%s BAR(%d)", comma ? "," : "",
			task_pid_nr(barr->task));
	} else {
		pr_cont("%s %pf", comma ? "," : "", work->func);
	}
}

static void show_pwq(struct pool_workqueue *pwq)
{
	struct worker_pool *pool = pwq->pool;
	struct work_struct *work;
	struct worker *worker;
	bool has_in_flight = false, has_pending = false;
	int bkt;

	pr_info("  pwq %d:", pool->id);
	pr_cont_pool_info(pool);

	pr_cont(" active=%d/%d%s\n", pwq->nr_active, pwq->max_active,
		!list_empty(&pwq->mayday_node) ? " MAYDAY" : "");

	hash_for_each(pool->busy_hash, bkt, worker, hentry) {
		if (worker->current_pwq == pwq) {
			has_in_flight = true;
			break;
		}
	}
	if (has_in_flight) {
		bool comma = false;

		pr_info("    in-flight:");
		hash_for_each(pool->busy_hash, bkt, worker, hentry) {
			if (worker->current_pwq != pwq)
				continue;

			pr_cont("%s %d%s:%pf", comma ? "," : "",
				task_pid_nr(worker->task),
				worker == pwq->wq->rescuer ? "(RESCUER)" : "",
				worker->current_func);
			list_for_each_entry(work, &worker->scheduled, entry)
				pr_cont_work(false, work);
			comma = true;
		}
		pr_cont("\n");
	}

	list_for_each_entry(work, &pool->worklist, entry) {
		if (get_work_pwq(work) == pwq) {
			has_pending = true;
			break;
		}
	}
	if (has_pending) {
		bool comma = false;

		pr_info("    pending:");
		list_for_each_entry(work, &pool->worklist, entry) {
			if (get_work_pwq(work) != pwq)
				continue;

			pr_cont_work(comma, work);
			comma = !(*work_data_bits(work) & WORK_STRUCT_LINKED);
		}
		pr_cont("\n");
	}

	if (!list_empty(&pwq->delayed_works)) {
		bool comma = false;

		pr_info("    delayed:");
		list_for_each_entry(work, &pwq->delayed_works, entry) {
			pr_cont_work(comma, work);
			comma = !(*work_data_bits(work) & WORK_STRUCT_LINKED);
		}
		pr_cont("\n");
	}
}

/**
 * show_workqueue_state - dump workqueue state
 *
 * Called from a sysrq handler and prints out all busy workqueues and
 * pools.
 */
void show_workqueue_state(void)
{
	struct workqueue_struct *wq;
	struct worker_pool *pool;
	unsigned long flags;
	int pi;

	rcu_read_lock_sched();

	pr_info("Showing busy workqueues and worker pools:\n");

	list_for_each_entry_rcu(wq, &workqueues, list) {
		struct pool_workqueue *pwq;
		bool idle = true;

		for_each_pwq(pwq, wq) {
			if (pwq->nr_active || !list_empty(&pwq->delayed_works)) {
				idle = false;
				break;
			}
		}
		if (idle)
			continue;

		pr_info("workqueue %s: flags=0x%x\n", wq->name, wq->flags);

		for_each_pwq(pwq, wq) {
			spin_lock_irqsave(&pwq->pool->lock, flags);
			if (pwq->nr_active || !list_empty(&pwq->delayed_works))
				show_pwq(pwq);
			spin_unlock_irqrestore(&pwq->pool->lock, flags);
		}
	}

	for_each_pool(pool, pi) {
		struct worker *worker;
		bool first = true;

		spin_lock_irqsave(&pool->lock, flags);
		if (pool->nr_workers == pool->nr_idle)
			goto next_pool;

		pr_info("pool %d:", pool->id);
		pr_cont_pool_info(pool);
		pr_cont(" workers=%d", pool->nr_workers);
		if (pool->manager)
			pr_cont(" manager: %d",
				task_pid_nr(pool->manager->task));
		list_for_each_entry(worker, &pool->idle_list, entry) {
			pr_cont(" %s%d", first ? "idle: " : "",
				task_pid_nr(worker->task));
			first = false;
		}
		pr_cont("\n");
	next_pool:
		spin_unlock_irqrestore(&pool->lock, flags);
	}

	rcu_read_unlock_sched();
}

4304 4305 4306
/*
 * CPU hotplug.
 *
4307
 * There are two challenges in supporting CPU hotplug.  Firstly, there
4308
 * are a lot of assumptions on strong associations among work, pwq and
4309
 * pool which make migrating pending and scheduled works very
4310
 * difficult to implement without impacting hot paths.  Secondly,
4311
 * worker pools serve mix of short, long and very long running works making
4312 4313
 * blocked draining impractical.
 *
4314
 * This is solved by allowing the pools to be disassociated from the CPU
4315 4316
 * running as an unbound one and allowing it to be reattached later if the
 * cpu comes back online.
4317
 */
L
Linus Torvalds 已提交
4318

4319
static void wq_unbind_fn(struct work_struct *work)
4320
{
4321
	int cpu = smp_processor_id();
4322
	struct worker_pool *pool;
4323
	struct worker *worker;
4324

4325
	for_each_cpu_worker_pool(pool, cpu) {
4326
		mutex_lock(&pool->attach_mutex);
4327
		spin_lock_irq(&pool->lock);
4328

4329
		/*
4330
		 * We've blocked all attach/detach operations. Make all workers
4331 4332 4333 4334 4335
		 * unbound and set DISASSOCIATED.  Before this, all workers
		 * except for the ones which are still executing works from
		 * before the last CPU down must be on the cpu.  After
		 * this, they may become diasporas.
		 */
4336
		for_each_pool_worker(worker, pool)
4337
			worker->flags |= WORKER_UNBOUND;
4338

4339
		pool->flags |= POOL_DISASSOCIATED;
4340

4341
		spin_unlock_irq(&pool->lock);
4342
		mutex_unlock(&pool->attach_mutex);
4343

4344 4345 4346 4347 4348 4349 4350
		/*
		 * Call schedule() so that we cross rq->lock and thus can
		 * guarantee sched callbacks see the %WORKER_UNBOUND flag.
		 * This is necessary as scheduler callbacks may be invoked
		 * from other cpus.
		 */
		schedule();
4351

4352 4353 4354 4355 4356 4357 4358 4359
		/*
		 * Sched callbacks are disabled now.  Zap nr_running.
		 * After this, nr_running stays zero and need_more_worker()
		 * and keep_working() are always true as long as the
		 * worklist is not empty.  This pool now behaves as an
		 * unbound (in terms of concurrency management) pool which
		 * are served by workers tied to the pool.
		 */
4360
		atomic_set(&pool->nr_running, 0);
4361 4362 4363 4364 4365 4366 4367 4368 4369 4370

		/*
		 * With concurrency management just turned off, a busy
		 * worker blocking could lead to lengthy stalls.  Kick off
		 * unbound chain execution of currently pending work items.
		 */
		spin_lock_irq(&pool->lock);
		wake_up_worker(pool);
		spin_unlock_irq(&pool->lock);
	}
4371 4372
}

T
Tejun Heo 已提交
4373 4374 4375 4376
/**
 * rebind_workers - rebind all workers of a pool to the associated CPU
 * @pool: pool of interest
 *
4377
 * @pool->cpu is coming online.  Rebind all workers to the CPU.
T
Tejun Heo 已提交
4378 4379 4380
 */
static void rebind_workers(struct worker_pool *pool)
{
4381
	struct worker *worker;
T
Tejun Heo 已提交
4382

4383
	lockdep_assert_held(&pool->attach_mutex);
T
Tejun Heo 已提交
4384

4385 4386 4387 4388 4389 4390 4391
	/*
	 * Restore CPU affinity of all workers.  As all idle workers should
	 * be on the run-queue of the associated CPU before any local
	 * wake-ups for concurrency management happen, restore CPU affinty
	 * of all workers first and then clear UNBOUND.  As we're called
	 * from CPU_ONLINE, the following shouldn't fail.
	 */
4392
	for_each_pool_worker(worker, pool)
4393 4394
		WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,
						  pool->attrs->cpumask) < 0);
T
Tejun Heo 已提交
4395

4396
	spin_lock_irq(&pool->lock);
4397
	pool->flags &= ~POOL_DISASSOCIATED;
T
Tejun Heo 已提交
4398

4399
	for_each_pool_worker(worker, pool) {
4400
		unsigned int worker_flags = worker->flags;
T
Tejun Heo 已提交
4401 4402

		/*
4403 4404 4405 4406 4407 4408
		 * A bound idle worker should actually be on the runqueue
		 * of the associated CPU for local wake-ups targeting it to
		 * work.  Kick all idle workers so that they migrate to the
		 * associated CPU.  Doing this in the same loop as
		 * replacing UNBOUND with REBOUND is safe as no worker will
		 * be bound before @pool->lock is released.
T
Tejun Heo 已提交
4409
		 */
4410 4411
		if (worker_flags & WORKER_IDLE)
			wake_up_process(worker->task);
T
Tejun Heo 已提交
4412

4413 4414 4415 4416 4417 4418 4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430 4431
		/*
		 * We want to clear UNBOUND but can't directly call
		 * worker_clr_flags() or adjust nr_running.  Atomically
		 * replace UNBOUND with another NOT_RUNNING flag REBOUND.
		 * @worker will clear REBOUND using worker_clr_flags() when
		 * it initiates the next execution cycle thus restoring
		 * concurrency management.  Note that when or whether
		 * @worker clears REBOUND doesn't affect correctness.
		 *
		 * ACCESS_ONCE() is necessary because @worker->flags may be
		 * tested without holding any lock in
		 * wq_worker_waking_up().  Without it, NOT_RUNNING test may
		 * fail incorrectly leading to premature concurrency
		 * management operations.
		 */
		WARN_ON_ONCE(!(worker_flags & WORKER_UNBOUND));
		worker_flags |= WORKER_REBOUND;
		worker_flags &= ~WORKER_UNBOUND;
		ACCESS_ONCE(worker->flags) = worker_flags;
T
Tejun Heo 已提交
4432
	}
4433 4434

	spin_unlock_irq(&pool->lock);
T
Tejun Heo 已提交
4435 4436
}

4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451
/**
 * restore_unbound_workers_cpumask - restore cpumask of unbound workers
 * @pool: unbound pool of interest
 * @cpu: the CPU which is coming up
 *
 * An unbound pool may end up with a cpumask which doesn't have any online
 * CPUs.  When a worker of such pool get scheduled, the scheduler resets
 * its cpus_allowed.  If @cpu is in @pool's cpumask which didn't have any
 * online CPU before, cpus_allowed of all its workers should be restored.
 */
static void restore_unbound_workers_cpumask(struct worker_pool *pool, int cpu)
{
	static cpumask_t cpumask;
	struct worker *worker;

4452
	lockdep_assert_held(&pool->attach_mutex);
4453 4454 4455 4456 4457 4458 4459 4460 4461 4462 4463

	/* is @cpu allowed for @pool? */
	if (!cpumask_test_cpu(cpu, pool->attrs->cpumask))
		return;

	/* is @cpu the only online CPU? */
	cpumask_and(&cpumask, pool->attrs->cpumask, cpu_online_mask);
	if (cpumask_weight(&cpumask) != 1)
		return;

	/* as we're called from CPU_ONLINE, the following shouldn't fail */
4464
	for_each_pool_worker(worker, pool)
4465 4466 4467 4468
		WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,
						  pool->attrs->cpumask) < 0);
}

T
Tejun Heo 已提交
4469 4470 4471 4472
/*
 * Workqueues should be brought up before normal priority CPU notifiers.
 * This will be registered high priority CPU notifier.
 */
4473
static int workqueue_cpu_up_callback(struct notifier_block *nfb,
T
Tejun Heo 已提交
4474 4475
					       unsigned long action,
					       void *hcpu)
4476
{
4477
	int cpu = (unsigned long)hcpu;
4478
	struct worker_pool *pool;
4479
	struct workqueue_struct *wq;
4480
	int pi;
4481

T
Tejun Heo 已提交
4482
	switch (action & ~CPU_TASKS_FROZEN) {
4483
	case CPU_UP_PREPARE:
4484
		for_each_cpu_worker_pool(pool, cpu) {
4485 4486
			if (pool->nr_workers)
				continue;
4487
			if (!create_worker(pool))
4488
				return NOTIFY_BAD;
4489
		}
T
Tejun Heo 已提交
4490
		break;
4491

4492 4493
	case CPU_DOWN_FAILED:
	case CPU_ONLINE:
4494
		mutex_lock(&wq_pool_mutex);
4495 4496

		for_each_pool(pool, pi) {
4497
			mutex_lock(&pool->attach_mutex);
4498

4499
			if (pool->cpu == cpu)
4500
				rebind_workers(pool);
4501
			else if (pool->cpu < 0)
4502
				restore_unbound_workers_cpumask(pool, cpu);
4503

4504 4505 4506 4507 4508 4509 4510 4511 4512 4513 4514 4515 4516 4517 4518 4519 4520 4521 4522 4523 4524 4525 4526 4527 4528 4529 4530 4531 4532 4533 4534 4535 4536 4537 4538 4539 4540 4541 4542 4543 4544 4545 4546 4547 4548 4549 4550 4551 4552 4553 4554 4555 4556 4557 4558 4559 4560 4561 4562 4563 4564 4565 4566 4567 4568 4569 4570 4571 4572 4573 4574 4575 4576 4577 4578 4579 4580 4581 4582 4583 4584 4585 4586 4587 4588 4589 4590 4591 4592 4593 4594 4595 4596 4597 4598 4599 4600 4601 4602 4603 4604 4605 4606 4607 4608 4609 4610 4611 4612 4613 4614 4615 4616 4617 4618 4619 4620 4621 4622 4623 4624 4625 4626 4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637 4638 4639 4640 4641 4642 4643 4644 4645 4646 4647 4648 4649 4650 4651 4652 4653 4654 4655 4656 4657 4658 4659 4660 4661 4662 4663 4664 4665 4666 4667 4668 4669 4670 4671 4672 4673 4674 4675 4676 4677 4678 4679 4680 4681 4682 4683 4684 4685 4686 4687 4688 4689 4690 4691 4692 4693 4694 4695 4696 4697 4698 4699 4700 4701 4702 4703 4704 4705 4706 4707 4708 4709 4710 4711 4712 4713 4714 4715 4716 4717 4718 4719 4720 4721 4722 4723 4724 4725 4726 4727 4728 4729 4730 4731 4732 4733 4734 4735 4736 4737 4738 4739 4740 4741 4742 4743 4744 4745 4746 4747 4748 4749 4750 4751 4752 4753 4754 4755 4756 4757 4758 4759 4760 4761 4762 4763 4764 4765 4766 4767 4768 4769 4770 4771 4772 4773 4774 4775 4776 4777 4778 4779 4780 4781 4782 4783 4784 4785 4786 4787 4788 4789 4790 4791 4792 4793 4794 4795 4796 4797 4798 4799 4800 4801 4802 4803 4804 4805 4806 4807 4808 4809 4810 4811 4812 4813 4814 4815 4816 4817 4818 4819 4820 4821 4822 4823 4824 4825 4826 4827 4828 4829 4830 4831 4832 4833 4834 4835 4836 4837 4838 4839 4840 4841 4842 4843 4844 4845 4846 4847 4848 4849 4850 4851 4852 4853 4854 4855 4856 4857 4858 4859 4860 4861 4862 4863 4864 4865 4866 4867 4868 4869 4870 4871 4872 4873
			mutex_unlock(&pool->attach_mutex);
		}

		/* update NUMA affinity of unbound workqueues */
		list_for_each_entry(wq, &workqueues, list)
			wq_update_unbound_numa(wq, cpu, true);

		mutex_unlock(&wq_pool_mutex);
		break;
	}
	return NOTIFY_OK;
}

/*
 * Workqueues should be brought down after normal priority CPU notifiers.
 * This will be registered as low priority CPU notifier.
 */
static int workqueue_cpu_down_callback(struct notifier_block *nfb,
						 unsigned long action,
						 void *hcpu)
{
	int cpu = (unsigned long)hcpu;
	struct work_struct unbind_work;
	struct workqueue_struct *wq;

	switch (action & ~CPU_TASKS_FROZEN) {
	case CPU_DOWN_PREPARE:
		/* unbinding per-cpu workers should happen on the local CPU */
		INIT_WORK_ONSTACK(&unbind_work, wq_unbind_fn);
		queue_work_on(cpu, system_highpri_wq, &unbind_work);

		/* update NUMA affinity of unbound workqueues */
		mutex_lock(&wq_pool_mutex);
		list_for_each_entry(wq, &workqueues, list)
			wq_update_unbound_numa(wq, cpu, false);
		mutex_unlock(&wq_pool_mutex);

		/* wait for per-cpu unbinding to finish */
		flush_work(&unbind_work);
		destroy_work_on_stack(&unbind_work);
		break;
	}
	return NOTIFY_OK;
}

#ifdef CONFIG_SMP

struct work_for_cpu {
	struct work_struct work;
	long (*fn)(void *);
	void *arg;
	long ret;
};

static void work_for_cpu_fn(struct work_struct *work)
{
	struct work_for_cpu *wfc = container_of(work, struct work_for_cpu, work);

	wfc->ret = wfc->fn(wfc->arg);
}

/**
 * work_on_cpu - run a function in user context on a particular cpu
 * @cpu: the cpu to run on
 * @fn: the function to run
 * @arg: the function arg
 *
 * It is up to the caller to ensure that the cpu doesn't go offline.
 * The caller must not hold any locks which would prevent @fn from completing.
 *
 * Return: The value @fn returns.
 */
long work_on_cpu(int cpu, long (*fn)(void *), void *arg)
{
	struct work_for_cpu wfc = { .fn = fn, .arg = arg };

	INIT_WORK_ONSTACK(&wfc.work, work_for_cpu_fn);
	schedule_work_on(cpu, &wfc.work);
	flush_work(&wfc.work);
	destroy_work_on_stack(&wfc.work);
	return wfc.ret;
}
EXPORT_SYMBOL_GPL(work_on_cpu);
#endif /* CONFIG_SMP */

#ifdef CONFIG_FREEZER

/**
 * freeze_workqueues_begin - begin freezing workqueues
 *
 * Start freezing workqueues.  After this function returns, all freezable
 * workqueues will queue new works to their delayed_works list instead of
 * pool->worklist.
 *
 * CONTEXT:
 * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
 */
void freeze_workqueues_begin(void)
{
	struct workqueue_struct *wq;
	struct pool_workqueue *pwq;

	mutex_lock(&wq_pool_mutex);

	WARN_ON_ONCE(workqueue_freezing);
	workqueue_freezing = true;

	list_for_each_entry(wq, &workqueues, list) {
		mutex_lock(&wq->mutex);
		for_each_pwq(pwq, wq)
			pwq_adjust_max_active(pwq);
		mutex_unlock(&wq->mutex);
	}

	mutex_unlock(&wq_pool_mutex);
}

/**
 * freeze_workqueues_busy - are freezable workqueues still busy?
 *
 * Check whether freezing is complete.  This function must be called
 * between freeze_workqueues_begin() and thaw_workqueues().
 *
 * CONTEXT:
 * Grabs and releases wq_pool_mutex.
 *
 * Return:
 * %true if some freezable workqueues are still busy.  %false if freezing
 * is complete.
 */
bool freeze_workqueues_busy(void)
{
	bool busy = false;
	struct workqueue_struct *wq;
	struct pool_workqueue *pwq;

	mutex_lock(&wq_pool_mutex);

	WARN_ON_ONCE(!workqueue_freezing);

	list_for_each_entry(wq, &workqueues, list) {
		if (!(wq->flags & WQ_FREEZABLE))
			continue;
		/*
		 * nr_active is monotonically decreasing.  It's safe
		 * to peek without lock.
		 */
		rcu_read_lock_sched();
		for_each_pwq(pwq, wq) {
			WARN_ON_ONCE(pwq->nr_active < 0);
			if (pwq->nr_active) {
				busy = true;
				rcu_read_unlock_sched();
				goto out_unlock;
			}
		}
		rcu_read_unlock_sched();
	}
out_unlock:
	mutex_unlock(&wq_pool_mutex);
	return busy;
}

/**
 * thaw_workqueues - thaw workqueues
 *
 * Thaw workqueues.  Normal queueing is restored and all collected
 * frozen works are transferred to their respective pool worklists.
 *
 * CONTEXT:
 * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
 */
void thaw_workqueues(void)
{
	struct workqueue_struct *wq;
	struct pool_workqueue *pwq;

	mutex_lock(&wq_pool_mutex);

	if (!workqueue_freezing)
		goto out_unlock;

	workqueue_freezing = false;

	/* restore max_active and repopulate worklist */
	list_for_each_entry(wq, &workqueues, list) {
		mutex_lock(&wq->mutex);
		for_each_pwq(pwq, wq)
			pwq_adjust_max_active(pwq);
		mutex_unlock(&wq->mutex);
	}

out_unlock:
	mutex_unlock(&wq_pool_mutex);
}
#endif /* CONFIG_FREEZER */

#ifdef CONFIG_SYSFS
/*
 * Workqueues with WQ_SYSFS flag set is visible to userland via
 * /sys/bus/workqueue/devices/WQ_NAME.  All visible workqueues have the
 * following attributes.
 *
 *  per_cpu	RO bool	: whether the workqueue is per-cpu or unbound
 *  max_active	RW int	: maximum number of in-flight work items
 *
 * Unbound workqueues have the following extra attributes.
 *
 *  id		RO int	: the associated pool ID
 *  nice	RW int	: nice value of the workers
 *  cpumask	RW mask	: bitmask of allowed CPUs for the workers
 */
struct wq_device {
	struct workqueue_struct		*wq;
	struct device			dev;
};

static struct workqueue_struct *dev_to_wq(struct device *dev)
{
	struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);

	return wq_dev->wq;
}

static ssize_t per_cpu_show(struct device *dev, struct device_attribute *attr,
			    char *buf)
{
	struct workqueue_struct *wq = dev_to_wq(dev);

	return scnprintf(buf, PAGE_SIZE, "%d\n", (bool)!(wq->flags & WQ_UNBOUND));
}
static DEVICE_ATTR_RO(per_cpu);

static ssize_t max_active_show(struct device *dev,
			       struct device_attribute *attr, char *buf)
{
	struct workqueue_struct *wq = dev_to_wq(dev);

	return scnprintf(buf, PAGE_SIZE, "%d\n", wq->saved_max_active);
}

static ssize_t max_active_store(struct device *dev,
				struct device_attribute *attr, const char *buf,
				size_t count)
{
	struct workqueue_struct *wq = dev_to_wq(dev);
	int val;

	if (sscanf(buf, "%d", &val) != 1 || val <= 0)
		return -EINVAL;

	workqueue_set_max_active(wq, val);
	return count;
}
static DEVICE_ATTR_RW(max_active);

static struct attribute *wq_sysfs_attrs[] = {
	&dev_attr_per_cpu.attr,
	&dev_attr_max_active.attr,
	NULL,
};
ATTRIBUTE_GROUPS(wq_sysfs);

static ssize_t wq_pool_ids_show(struct device *dev,
				struct device_attribute *attr, char *buf)
{
	struct workqueue_struct *wq = dev_to_wq(dev);
	const char *delim = "";
	int node, written = 0;

	rcu_read_lock_sched();
	for_each_node(node) {
		written += scnprintf(buf + written, PAGE_SIZE - written,
				     "%s%d:%d", delim, node,
				     unbound_pwq_by_node(wq, node)->pool->id);
		delim = " ";
	}
	written += scnprintf(buf + written, PAGE_SIZE - written, "\n");
	rcu_read_unlock_sched();

	return written;
}

static ssize_t wq_nice_show(struct device *dev, struct device_attribute *attr,
			    char *buf)
{
	struct workqueue_struct *wq = dev_to_wq(dev);
	int written;

	mutex_lock(&wq->mutex);
	written = scnprintf(buf, PAGE_SIZE, "%d\n", wq->unbound_attrs->nice);
	mutex_unlock(&wq->mutex);

	return written;
}

/* prepare workqueue_attrs for sysfs store operations */
static struct workqueue_attrs *wq_sysfs_prep_attrs(struct workqueue_struct *wq)
{
	struct workqueue_attrs *attrs;

	attrs = alloc_workqueue_attrs(GFP_KERNEL);
	if (!attrs)
		return NULL;

	mutex_lock(&wq->mutex);
	copy_workqueue_attrs(attrs, wq->unbound_attrs);
	mutex_unlock(&wq->mutex);
	return attrs;
}

static ssize_t wq_nice_store(struct device *dev, struct device_attribute *attr,
			     const char *buf, size_t count)
{
	struct workqueue_struct *wq = dev_to_wq(dev);
	struct workqueue_attrs *attrs;
	int ret;

	attrs = wq_sysfs_prep_attrs(wq);
	if (!attrs)
		return -ENOMEM;

	if (sscanf(buf, "%d", &attrs->nice) == 1 &&
	    attrs->nice >= MIN_NICE && attrs->nice <= MAX_NICE)
		ret = apply_workqueue_attrs(wq, attrs);
	else
		ret = -EINVAL;

	free_workqueue_attrs(attrs);
	return ret ?: count;
}

static ssize_t wq_cpumask_show(struct device *dev,
			       struct device_attribute *attr, char *buf)
{
	struct workqueue_struct *wq = dev_to_wq(dev);
	int written;

	mutex_lock(&wq->mutex);
	written = scnprintf(buf, PAGE_SIZE, "%*pb\n",
			    cpumask_pr_args(wq->unbound_attrs->cpumask));
	mutex_unlock(&wq->mutex);
	return written;
}

static ssize_t wq_cpumask_store(struct device *dev,
				struct device_attribute *attr,
				const char *buf, size_t count)
{
	struct workqueue_struct *wq = dev_to_wq(dev);
	struct workqueue_attrs *attrs;
	int ret;

	attrs = wq_sysfs_prep_attrs(wq);
	if (!attrs)
		return -ENOMEM;

	ret = cpumask_parse(buf, attrs->cpumask);
	if (!ret)
		ret = apply_workqueue_attrs(wq, attrs);

	free_workqueue_attrs(attrs);
	return ret ?: count;
}

static ssize_t wq_numa_show(struct device *dev, struct device_attribute *attr,
			    char *buf)
{
	struct workqueue_struct *wq = dev_to_wq(dev);
	int written;
4874

4875 4876 4877 4878
	mutex_lock(&wq->mutex);
	written = scnprintf(buf, PAGE_SIZE, "%d\n",
			    !wq->unbound_attrs->no_numa);
	mutex_unlock(&wq->mutex);
4879

4880
	return written;
4881 4882
}

4883 4884
static ssize_t wq_numa_store(struct device *dev, struct device_attribute *attr,
			     const char *buf, size_t count)
4885
{
4886 4887 4888
	struct workqueue_struct *wq = dev_to_wq(dev);
	struct workqueue_attrs *attrs;
	int v, ret;
4889

4890 4891 4892
	attrs = wq_sysfs_prep_attrs(wq);
	if (!attrs)
		return -ENOMEM;
4893

4894 4895 4896 4897
	ret = -EINVAL;
	if (sscanf(buf, "%d", &v) == 1) {
		attrs->no_numa = !v;
		ret = apply_workqueue_attrs(wq, attrs);
4898
	}
4899 4900 4901

	free_workqueue_attrs(attrs);
	return ret ?: count;
4902 4903
}

4904 4905 4906 4907 4908 4909 4910
static struct device_attribute wq_sysfs_unbound_attrs[] = {
	__ATTR(pool_ids, 0444, wq_pool_ids_show, NULL),
	__ATTR(nice, 0644, wq_nice_show, wq_nice_store),
	__ATTR(cpumask, 0644, wq_cpumask_show, wq_cpumask_store),
	__ATTR(numa, 0644, wq_numa_show, wq_numa_store),
	__ATTR_NULL,
};
4911

4912 4913 4914
static struct bus_type wq_subsys = {
	.name				= "workqueue",
	.dev_groups			= wq_sysfs_groups,
4915 4916
};

4917
static int __init wq_sysfs_init(void)
4918
{
4919
	return subsys_virtual_register(&wq_subsys, NULL);
4920
}
4921
core_initcall(wq_sysfs_init);
4922

4923
static void wq_device_release(struct device *dev)
4924
{
4925
	struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);
4926

4927
	kfree(wq_dev);
4928
}
4929 4930

/**
4931 4932
 * workqueue_sysfs_register - make a workqueue visible in sysfs
 * @wq: the workqueue to register
4933
 *
4934 4935 4936
 * Expose @wq in sysfs under /sys/bus/workqueue/devices.
 * alloc_workqueue*() automatically calls this function if WQ_SYSFS is set
 * which is the preferred method.
4937
 *
4938 4939 4940 4941 4942 4943
 * Workqueue user should use this function directly iff it wants to apply
 * workqueue_attrs before making the workqueue visible in sysfs; otherwise,
 * apply_workqueue_attrs() may race against userland updating the
 * attributes.
 *
 * Return: 0 on success, -errno on failure.
4944
 */
4945
int workqueue_sysfs_register(struct workqueue_struct *wq)
4946
{
4947 4948
	struct wq_device *wq_dev;
	int ret;
4949

4950 4951 4952 4953 4954 4955 4956
	/*
	 * Adjusting max_active or creating new pwqs by applyting
	 * attributes breaks ordering guarantee.  Disallow exposing ordered
	 * workqueues.
	 */
	if (WARN_ON(wq->flags & __WQ_ORDERED))
		return -EINVAL;
4957

4958 4959 4960
	wq->wq_dev = wq_dev = kzalloc(sizeof(*wq_dev), GFP_KERNEL);
	if (!wq_dev)
		return -ENOMEM;
4961

4962 4963 4964 4965
	wq_dev->wq = wq;
	wq_dev->dev.bus = &wq_subsys;
	wq_dev->dev.init_name = wq->name;
	wq_dev->dev.release = wq_device_release;
4966

4967 4968 4969 4970 4971
	/*
	 * unbound_attrs are created separately.  Suppress uevent until
	 * everything is ready.
	 */
	dev_set_uevent_suppress(&wq_dev->dev, true);
4972

4973 4974 4975 4976 4977 4978
	ret = device_register(&wq_dev->dev);
	if (ret) {
		kfree(wq_dev);
		wq->wq_dev = NULL;
		return ret;
	}
4979

4980 4981
	if (wq->flags & WQ_UNBOUND) {
		struct device_attribute *attr;
4982

4983 4984 4985 4986 4987 4988
		for (attr = wq_sysfs_unbound_attrs; attr->attr.name; attr++) {
			ret = device_create_file(&wq_dev->dev, attr);
			if (ret) {
				device_unregister(&wq_dev->dev);
				wq->wq_dev = NULL;
				return ret;
4989 4990 4991
			}
		}
	}
4992 4993 4994 4995

	dev_set_uevent_suppress(&wq_dev->dev, false);
	kobject_uevent(&wq_dev->dev.kobj, KOBJ_ADD);
	return 0;
4996 4997 4998
}

/**
4999 5000
 * workqueue_sysfs_unregister - undo workqueue_sysfs_register()
 * @wq: the workqueue to unregister
5001
 *
5002
 * If @wq is registered to sysfs by workqueue_sysfs_register(), unregister.
5003
 */
5004
static void workqueue_sysfs_unregister(struct workqueue_struct *wq)
5005
{
5006
	struct wq_device *wq_dev = wq->wq_dev;
5007

5008 5009
	if (!wq->wq_dev)
		return;
5010

5011 5012
	wq->wq_dev = NULL;
	device_unregister(&wq_dev->dev);
5013
}
5014 5015 5016
#else	/* CONFIG_SYSFS */
static void workqueue_sysfs_unregister(struct workqueue_struct *wq)	{ }
#endif	/* CONFIG_SYSFS */
5017

5018 5019 5020 5021 5022 5023 5024 5025
static void __init wq_numa_init(void)
{
	cpumask_var_t *tbl;
	int node, cpu;

	if (num_possible_nodes() <= 1)
		return;

5026 5027 5028 5029 5030
	if (wq_disable_numa) {
		pr_info("workqueue: NUMA affinity support disabled\n");
		return;
	}

5031 5032 5033
	wq_update_unbound_numa_attrs_buf = alloc_workqueue_attrs(GFP_KERNEL);
	BUG_ON(!wq_update_unbound_numa_attrs_buf);

5034 5035 5036 5037 5038
	/*
	 * We want masks of possible CPUs of each node which isn't readily
	 * available.  Build one from cpu_to_node() which should have been
	 * fully initialized by now.
	 */
5039
	tbl = kzalloc(nr_node_ids * sizeof(tbl[0]), GFP_KERNEL);
5040 5041 5042
	BUG_ON(!tbl);

	for_each_node(node)
5043
		BUG_ON(!zalloc_cpumask_var_node(&tbl[node], GFP_KERNEL,
5044
				node_online(node) ? node : NUMA_NO_NODE));
5045 5046 5047 5048 5049 5050 5051 5052 5053 5054 5055 5056 5057 5058 5059

	for_each_possible_cpu(cpu) {
		node = cpu_to_node(cpu);
		if (WARN_ON(node == NUMA_NO_NODE)) {
			pr_warn("workqueue: NUMA node mapping not available for cpu%d, disabling NUMA support\n", cpu);
			/* happens iff arch is bonkers, let's just proceed */
			return;
		}
		cpumask_set_cpu(cpu, tbl[node]);
	}

	wq_numa_possible_cpumask = tbl;
	wq_numa_enabled = true;
}

5060
static int __init init_workqueues(void)
L
Linus Torvalds 已提交
5061
{
T
Tejun Heo 已提交
5062 5063
	int std_nice[NR_STD_WORKER_POOLS] = { 0, HIGHPRI_NICE_LEVEL };
	int i, cpu;
T
Tejun Heo 已提交
5064

5065 5066 5067 5068
	WARN_ON(__alignof__(struct pool_workqueue) < __alignof__(long long));

	pwq_cache = KMEM_CACHE(pool_workqueue, SLAB_PANIC);

5069
	cpu_notifier(workqueue_cpu_up_callback, CPU_PRI_WORKQUEUE_UP);
5070
	hotcpu_notifier(workqueue_cpu_down_callback, CPU_PRI_WORKQUEUE_DOWN);
5071

5072 5073
	wq_numa_init();

5074
	/* initialize CPU pools */
5075
	for_each_possible_cpu(cpu) {
5076
		struct worker_pool *pool;
5077

T
Tejun Heo 已提交
5078
		i = 0;
5079
		for_each_cpu_worker_pool(pool, cpu) {
T
Tejun Heo 已提交
5080
			BUG_ON(init_worker_pool(pool));
5081
			pool->cpu = cpu;
5082
			cpumask_copy(pool->attrs->cpumask, cpumask_of(cpu));
T
Tejun Heo 已提交
5083
			pool->attrs->nice = std_nice[i++];
5084
			pool->node = cpu_to_node(cpu);
T
Tejun Heo 已提交
5085

T
Tejun Heo 已提交
5086
			/* alloc pool ID */
5087
			mutex_lock(&wq_pool_mutex);
T
Tejun Heo 已提交
5088
			BUG_ON(worker_pool_assign_id(pool));
5089
			mutex_unlock(&wq_pool_mutex);
5090
		}
5091 5092
	}

5093
	/* create the initial worker */
5094
	for_each_online_cpu(cpu) {
5095
		struct worker_pool *pool;
5096

5097
		for_each_cpu_worker_pool(pool, cpu) {
5098
			pool->flags &= ~POOL_DISASSOCIATED;
5099
			BUG_ON(!create_worker(pool));
5100
		}
5101 5102
	}

5103
	/* create default unbound and ordered wq attrs */
5104 5105 5106 5107 5108 5109
	for (i = 0; i < NR_STD_WORKER_POOLS; i++) {
		struct workqueue_attrs *attrs;

		BUG_ON(!(attrs = alloc_workqueue_attrs(GFP_KERNEL)));
		attrs->nice = std_nice[i];
		unbound_std_wq_attrs[i] = attrs;
5110 5111 5112 5113 5114 5115 5116 5117 5118 5119

		/*
		 * An ordered wq should have only one pwq as ordering is
		 * guaranteed by max_active which is enforced by pwqs.
		 * Turn off NUMA so that dfl_pwq is used for all nodes.
		 */
		BUG_ON(!(attrs = alloc_workqueue_attrs(GFP_KERNEL)));
		attrs->nice = std_nice[i];
		attrs->no_numa = true;
		ordered_wq_attrs[i] = attrs;
5120 5121
	}

5122
	system_wq = alloc_workqueue("events", 0, 0);
5123
	system_highpri_wq = alloc_workqueue("events_highpri", WQ_HIGHPRI, 0);
5124
	system_long_wq = alloc_workqueue("events_long", 0, 0);
5125 5126
	system_unbound_wq = alloc_workqueue("events_unbound", WQ_UNBOUND,
					    WQ_UNBOUND_MAX_ACTIVE);
5127 5128
	system_freezable_wq = alloc_workqueue("events_freezable",
					      WQ_FREEZABLE, 0);
5129 5130 5131 5132 5133
	system_power_efficient_wq = alloc_workqueue("events_power_efficient",
					      WQ_POWER_EFFICIENT, 0);
	system_freezable_power_efficient_wq = alloc_workqueue("events_freezable_power_efficient",
					      WQ_FREEZABLE | WQ_POWER_EFFICIENT,
					      0);
5134
	BUG_ON(!system_wq || !system_highpri_wq || !system_long_wq ||
5135 5136 5137
	       !system_unbound_wq || !system_freezable_wq ||
	       !system_power_efficient_wq ||
	       !system_freezable_power_efficient_wq);
5138
	return 0;
L
Linus Torvalds 已提交
5139
}
5140
early_initcall(init_workqueues);