kthread.c 19.3 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3
/* Kernel thread helper functions.
 *   Copyright (C) 2004 IBM Corporation, Rusty Russell.
 *
4
 * Creation is done via kthreadd, so that we get a clean environment
L
Linus Torvalds 已提交
5 6 7 8 9 10 11
 * even if we're invoked from userspace (think modprobe, hotplug cpu,
 * etc.).
 */
#include <linux/sched.h>
#include <linux/kthread.h>
#include <linux/completion.h>
#include <linux/err.h>
12
#include <linux/cpuset.h>
L
Linus Torvalds 已提交
13 14
#include <linux/unistd.h>
#include <linux/file.h>
15
#include <linux/export.h>
A
Arjan van de Ven 已提交
16
#include <linux/mutex.h>
T
Tejun Heo 已提交
17 18
#include <linux/slab.h>
#include <linux/freezer.h>
19
#include <linux/ptrace.h>
20
#include <linux/uaccess.h>
21
#include <trace/events/sched.h>
L
Linus Torvalds 已提交
22

23 24 25
static DEFINE_SPINLOCK(kthread_create_lock);
static LIST_HEAD(kthread_create_list);
struct task_struct *kthreadd_task;
L
Linus Torvalds 已提交
26 27 28

struct kthread_create_info
{
29
	/* Information passed to kthread() from kthreadd. */
L
Linus Torvalds 已提交
30 31
	int (*threadfn)(void *data);
	void *data;
32
	int node;
L
Linus Torvalds 已提交
33

34
	/* Result passed back to kthread_create() from kthreadd. */
L
Linus Torvalds 已提交
35
	struct task_struct *result;
36
	struct completion *done;
37

38
	struct list_head list;
L
Linus Torvalds 已提交
39 40
};

O
Oleg Nesterov 已提交
41
struct kthread {
42 43
	unsigned long flags;
	unsigned int cpu;
T
Tejun Heo 已提交
44
	void *data;
45
	struct completion parked;
O
Oleg Nesterov 已提交
46
	struct completion exited;
L
Linus Torvalds 已提交
47 48
};

49 50 51 52 53 54 55
enum KTHREAD_BITS {
	KTHREAD_IS_PER_CPU = 0,
	KTHREAD_SHOULD_STOP,
	KTHREAD_SHOULD_PARK,
	KTHREAD_IS_PARKED,
};

56 57 58 59 60 61 62 63 64 65 66
#define __to_kthread(vfork)	\
	container_of(vfork, struct kthread, exited)

static inline struct kthread *to_kthread(struct task_struct *k)
{
	return __to_kthread(k->vfork_done);
}

static struct kthread *to_live_kthread(struct task_struct *k)
{
	struct completion *vfork = ACCESS_ONCE(k->vfork_done);
67
	if (likely(vfork) && try_get_task_stack(k))
68 69 70
		return __to_kthread(vfork);
	return NULL;
}
L
Linus Torvalds 已提交
71

72 73 74
/**
 * kthread_should_stop - should this kthread return now?
 *
75
 * When someone calls kthread_stop() on your kthread, it will be woken
76 77 78
 * and this will return true.  You should then return, and your return
 * value will be passed through to kthread_stop().
 */
79
bool kthread_should_stop(void)
L
Linus Torvalds 已提交
80
{
81
	return test_bit(KTHREAD_SHOULD_STOP, &to_kthread(current)->flags);
L
Linus Torvalds 已提交
82 83 84
}
EXPORT_SYMBOL(kthread_should_stop);

85 86 87 88 89 90 91 92 93 94 95 96 97 98 99
/**
 * kthread_should_park - should this kthread park now?
 *
 * When someone calls kthread_park() on your kthread, it will be woken
 * and this will return true.  You should then do the necessary
 * cleanup and call kthread_parkme()
 *
 * Similar to kthread_should_stop(), but this keeps the thread alive
 * and in a park position. kthread_unpark() "restarts" the thread and
 * calls the thread function again.
 */
bool kthread_should_park(void)
{
	return test_bit(KTHREAD_SHOULD_PARK, &to_kthread(current)->flags);
}
100
EXPORT_SYMBOL_GPL(kthread_should_park);
101

102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126
/**
 * kthread_freezable_should_stop - should this freezable kthread return now?
 * @was_frozen: optional out parameter, indicates whether %current was frozen
 *
 * kthread_should_stop() for freezable kthreads, which will enter
 * refrigerator if necessary.  This function is safe from kthread_stop() /
 * freezer deadlock and freezable kthreads should use this function instead
 * of calling try_to_freeze() directly.
 */
bool kthread_freezable_should_stop(bool *was_frozen)
{
	bool frozen = false;

	might_sleep();

	if (unlikely(freezing(current)))
		frozen = __refrigerator(true);

	if (was_frozen)
		*was_frozen = frozen;

	return kthread_should_stop();
}
EXPORT_SYMBOL_GPL(kthread_freezable_should_stop);

T
Tejun Heo 已提交
127 128 129 130 131 132 133 134 135 136 137 138 139
/**
 * kthread_data - return data value specified on kthread creation
 * @task: kthread task in question
 *
 * Return the data value specified when kthread @task was created.
 * The caller is responsible for ensuring the validity of @task when
 * calling this function.
 */
void *kthread_data(struct task_struct *task)
{
	return to_kthread(task)->data;
}

140
/**
141
 * kthread_probe_data - speculative version of kthread_data()
142 143 144 145 146 147 148
 * @task: possible kthread task in question
 *
 * @task could be a kthread task.  Return the data value specified when it
 * was created if accessible.  If @task isn't a kthread task or its data is
 * inaccessible for any reason, %NULL is returned.  This function requires
 * that @task itself is safe to dereference.
 */
149
void *kthread_probe_data(struct task_struct *task)
150 151 152 153 154 155 156 157
{
	struct kthread *kthread = to_kthread(task);
	void *data = NULL;

	probe_kernel_read(&data, &kthread->data, sizeof(data));
	return data;
}

158 159
static void __kthread_parkme(struct kthread *self)
{
160
	__set_current_state(TASK_PARKED);
161 162 163 164
	while (test_bit(KTHREAD_SHOULD_PARK, &self->flags)) {
		if (!test_and_set_bit(KTHREAD_IS_PARKED, &self->flags))
			complete(&self->parked);
		schedule();
165
		__set_current_state(TASK_PARKED);
166 167 168 169 170 171 172 173 174
	}
	clear_bit(KTHREAD_IS_PARKED, &self->flags);
	__set_current_state(TASK_RUNNING);
}

void kthread_parkme(void)
{
	__kthread_parkme(to_kthread(current));
}
175
EXPORT_SYMBOL_GPL(kthread_parkme);
176

L
Linus Torvalds 已提交
177 178
static int kthread(void *_create)
{
O
Oleg Nesterov 已提交
179
	/* Copy data: it's on kthread's stack */
L
Linus Torvalds 已提交
180
	struct kthread_create_info *create = _create;
O
Oleg Nesterov 已提交
181 182
	int (*threadfn)(void *data) = create->threadfn;
	void *data = create->data;
183
	struct completion *done;
O
Oleg Nesterov 已提交
184 185
	struct kthread self;
	int ret;
L
Linus Torvalds 已提交
186

187
	self.flags = 0;
T
Tejun Heo 已提交
188
	self.data = data;
O
Oleg Nesterov 已提交
189
	init_completion(&self.exited);
190
	init_completion(&self.parked);
O
Oleg Nesterov 已提交
191
	current->vfork_done = &self.exited;
L
Linus Torvalds 已提交
192

193 194 195 196 197 198
	/* If user was SIGKILLed, I release the structure. */
	done = xchg(&create->done, NULL);
	if (!done) {
		kfree(create);
		do_exit(-EINTR);
	}
L
Linus Torvalds 已提交
199
	/* OK, tell user we're spawned, wait for stop or wakeup */
200
	__set_current_state(TASK_UNINTERRUPTIBLE);
201
	create->result = current;
202
	complete(done);
L
Linus Torvalds 已提交
203 204
	schedule();

O
Oleg Nesterov 已提交
205
	ret = -EINTR;
L
Linus Torvalds 已提交
206

207 208 209 210
	if (!test_bit(KTHREAD_SHOULD_STOP, &self.flags)) {
		__kthread_parkme(&self);
		ret = threadfn(data);
	}
O
Oleg Nesterov 已提交
211 212
	/* we can't just return, we must preserve "self" on stack */
	do_exit(ret);
L
Linus Torvalds 已提交
213 214
}

215 216 217 218 219 220 221
/* called from do_fork() to get node information for about to be created task */
int tsk_fork_get_node(struct task_struct *tsk)
{
#ifdef CONFIG_NUMA
	if (tsk == kthreadd_task)
		return tsk->pref_node_fork;
#endif
222
	return NUMA_NO_NODE;
223 224
}

225
static void create_kthread(struct kthread_create_info *create)
L
Linus Torvalds 已提交
226 227 228
{
	int pid;

229 230 231
#ifdef CONFIG_NUMA
	current->pref_node_fork = create->node;
#endif
L
Linus Torvalds 已提交
232 233
	/* We want our own signal handler (we take no signals by default). */
	pid = kernel_thread(kthread, create, CLONE_FS | CLONE_FILES | SIGCHLD);
234
	if (pid < 0) {
235 236 237 238 239 240 241
		/* If user was SIGKILLed, I release the structure. */
		struct completion *done = xchg(&create->done, NULL);

		if (!done) {
			kfree(create);
			return;
		}
L
Linus Torvalds 已提交
242
		create->result = ERR_PTR(pid);
243
		complete(done);
244
	}
L
Linus Torvalds 已提交
245 246
}

247
/**
248
 * kthread_create_on_node - create a kthread.
249 250
 * @threadfn: the function to run until signal_pending(current).
 * @data: data ptr for @threadfn.
251
 * @node: task and thread structures for the thread are allocated on this node
252 253 254 255
 * @namefmt: printf-style name for the thread.
 *
 * Description: This helper function creates and names a kernel
 * thread.  The thread will be stopped: use wake_up_process() to start
256 257
 * it.  See also kthread_run().  The new thread has SCHED_NORMAL policy and
 * is affine to all CPUs.
258
 *
259
 * If thread is going to be bound on a particular cpu, give its node
260
 * in @node, to get NUMA affinity for kthread stack, or else give NUMA_NO_NODE.
261
 * When woken, the thread will run @threadfn() with @data as its
262
 * argument. @threadfn() can either call do_exit() directly if it is a
L
Lucas De Marchi 已提交
263
 * standalone thread for which no one will call kthread_stop(), or
264 265 266 267
 * return when 'kthread_should_stop()' is true (which means
 * kthread_stop() has been called).  The return value should be zero
 * or a negative error number; it will be passed to kthread_stop().
 *
268
 * Returns a task_struct or ERR_PTR(-ENOMEM) or ERR_PTR(-EINTR).
269
 */
270
struct task_struct *kthread_create_on_node(int (*threadfn)(void *data),
271
					   void *data, int node,
272 273
					   const char namefmt[],
					   ...)
L
Linus Torvalds 已提交
274
{
275 276 277 278 279 280 281 282 283 284 285
	DECLARE_COMPLETION_ONSTACK(done);
	struct task_struct *task;
	struct kthread_create_info *create = kmalloc(sizeof(*create),
						     GFP_KERNEL);

	if (!create)
		return ERR_PTR(-ENOMEM);
	create->threadfn = threadfn;
	create->data = data;
	create->node = node;
	create->done = &done;
286 287

	spin_lock(&kthread_create_lock);
288
	list_add_tail(&create->list, &kthread_create_list);
289 290
	spin_unlock(&kthread_create_lock);

291
	wake_up_process(kthreadd_task);
292 293 294 295 296 297 298 299 300 301 302 303
	/*
	 * Wait for completion in killable state, for I might be chosen by
	 * the OOM killer while kthreadd is trying to allocate memory for
	 * new kernel thread.
	 */
	if (unlikely(wait_for_completion_killable(&done))) {
		/*
		 * If I was SIGKILLed before kthreadd (or new kernel thread)
		 * calls complete(), leave the cleanup of this structure to
		 * that thread.
		 */
		if (xchg(&create->done, NULL))
304
			return ERR_PTR(-EINTR);
305 306 307 308 309 310 311 312
		/*
		 * kthreadd (or new kernel thread) will call complete()
		 * shortly.
		 */
		wait_for_completion(&done);
	}
	task = create->result;
	if (!IS_ERR(task)) {
313
		static const struct sched_param param = { .sched_priority = 0 };
L
Linus Torvalds 已提交
314
		va_list args;
315

L
Linus Torvalds 已提交
316
		va_start(args, namefmt);
317
		vsnprintf(task->comm, sizeof(task->comm), namefmt, args);
L
Linus Torvalds 已提交
318
		va_end(args);
319 320 321 322
		/*
		 * root may have changed our (kthreadd's) priority or CPU mask.
		 * The kernel thread should not inherit these properties.
		 */
323 324
		sched_setscheduler_nocheck(task, SCHED_NORMAL, &param);
		set_cpus_allowed_ptr(task, cpu_all_mask);
L
Linus Torvalds 已提交
325
	}
326 327
	kfree(create);
	return task;
L
Linus Torvalds 已提交
328
}
329
EXPORT_SYMBOL(kthread_create_on_node);
L
Linus Torvalds 已提交
330

331
static void __kthread_bind_mask(struct task_struct *p, const struct cpumask *mask, long state)
332
{
333 334
	unsigned long flags;

335 336 337 338
	if (!wait_task_inactive(p, state)) {
		WARN_ON(1);
		return;
	}
339

340
	/* It's safe because the task is inactive. */
341 342
	raw_spin_lock_irqsave(&p->pi_lock, flags);
	do_set_cpus_allowed(p, mask);
343
	p->flags |= PF_NO_SETAFFINITY;
344 345 346 347 348 349 350 351 352 353 354
	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
}

static void __kthread_bind(struct task_struct *p, unsigned int cpu, long state)
{
	__kthread_bind_mask(p, cpumask_of(cpu), state);
}

void kthread_bind_mask(struct task_struct *p, const struct cpumask *mask)
{
	__kthread_bind_mask(p, mask, TASK_UNINTERRUPTIBLE);
355 356
}

357 358 359 360 361 362 363 364 365 366 367
/**
 * kthread_bind - bind a just-created kthread to a cpu.
 * @p: thread created by kthread_create().
 * @cpu: cpu (might not be online, must be possible) for @k to run on.
 *
 * Description: This function is equivalent to set_cpus_allowed(),
 * except that @cpu doesn't need to be online, and the thread must be
 * stopped (i.e., just returned from kthread_create()).
 */
void kthread_bind(struct task_struct *p, unsigned int cpu)
{
368
	__kthread_bind(p, cpu, TASK_UNINTERRUPTIBLE);
369 370 371
}
EXPORT_SYMBOL(kthread_bind);

372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388
/**
 * kthread_create_on_cpu - Create a cpu bound kthread
 * @threadfn: the function to run until signal_pending(current).
 * @data: data ptr for @threadfn.
 * @cpu: The cpu on which the thread should be bound,
 * @namefmt: printf-style name for the thread. Format is restricted
 *	     to "name.*%u". Code fills in cpu number.
 *
 * Description: This helper function creates and names a kernel thread
 * The thread will be woken and put into park mode.
 */
struct task_struct *kthread_create_on_cpu(int (*threadfn)(void *data),
					  void *data, unsigned int cpu,
					  const char *namefmt)
{
	struct task_struct *p;

389
	p = kthread_create_on_node(threadfn, data, cpu_to_node(cpu), namefmt,
390 391 392 393 394 395 396 397 398 399
				   cpu);
	if (IS_ERR(p))
		return p;
	set_bit(KTHREAD_IS_PER_CPU, &to_kthread(p)->flags);
	to_kthread(p)->cpu = cpu;
	/* Park the thread to get it out of TASK_UNINTERRUPTIBLE state */
	kthread_park(p);
	return p;
}

400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415
static void __kthread_unpark(struct task_struct *k, struct kthread *kthread)
{
	clear_bit(KTHREAD_SHOULD_PARK, &kthread->flags);
	/*
	 * We clear the IS_PARKED bit here as we don't wait
	 * until the task has left the park code. So if we'd
	 * park before that happens we'd see the IS_PARKED bit
	 * which might be about to be cleared.
	 */
	if (test_and_clear_bit(KTHREAD_IS_PARKED, &kthread->flags)) {
		if (test_bit(KTHREAD_IS_PER_CPU, &kthread->flags))
			__kthread_bind(k, kthread->cpu, TASK_PARKED);
		wake_up_state(k, TASK_PARKED);
	}
}

416 417 418 419 420 421 422 423 424 425
/**
 * kthread_unpark - unpark a thread created by kthread_create().
 * @k:		thread created by kthread_create().
 *
 * Sets kthread_should_park() for @k to return false, wakes it, and
 * waits for it to return. If the thread is marked percpu then its
 * bound to the cpu again.
 */
void kthread_unpark(struct task_struct *k)
{
426
	struct kthread *kthread = to_live_kthread(k);
427

428
	if (kthread) {
429
		__kthread_unpark(k, kthread);
430 431
		put_task_stack(k);
	}
432
}
433
EXPORT_SYMBOL_GPL(kthread_unpark);
434 435 436 437 438 439 440 441 442 443 444 445 446 447 448

/**
 * kthread_park - park a thread created by kthread_create().
 * @k: thread created by kthread_create().
 *
 * Sets kthread_should_park() for @k to return true, wakes it, and
 * waits for it to return. This can also be called after kthread_create()
 * instead of calling wake_up_process(): the thread will park without
 * calling threadfn().
 *
 * Returns 0 if the thread is parked, -ENOSYS if the thread exited.
 * If called by the kthread itself just the park bit is set.
 */
int kthread_park(struct task_struct *k)
{
449
	struct kthread *kthread = to_live_kthread(k);
450 451 452 453 454 455 456 457 458 459
	int ret = -ENOSYS;

	if (kthread) {
		if (!test_bit(KTHREAD_IS_PARKED, &kthread->flags)) {
			set_bit(KTHREAD_SHOULD_PARK, &kthread->flags);
			if (k != current) {
				wake_up_process(k);
				wait_for_completion(&kthread->parked);
			}
		}
460
		put_task_stack(k);
461 462 463 464
		ret = 0;
	}
	return ret;
}
465
EXPORT_SYMBOL_GPL(kthread_park);
466

467 468 469 470 471
/**
 * kthread_stop - stop a thread created by kthread_create().
 * @k: thread created by kthread_create().
 *
 * Sets kthread_should_stop() for @k to return true, wakes it, and
472 473 474 475 476 477
 * waits for it to exit. This can also be called after kthread_create()
 * instead of calling wake_up_process(): the thread will exit without
 * calling threadfn().
 *
 * If threadfn() may call do_exit() itself, the caller must ensure
 * task_struct can't go away.
478 479 480 481
 *
 * Returns the result of threadfn(), or %-EINTR if wake_up_process()
 * was never called.
 */
L
Linus Torvalds 已提交
482 483
int kthread_stop(struct task_struct *k)
{
484
	struct kthread *kthread;
L
Linus Torvalds 已提交
485 486
	int ret;

487
	trace_sched_kthread_stop(k);
488 489 490

	get_task_struct(k);
	kthread = to_live_kthread(k);
491 492
	if (kthread) {
		set_bit(KTHREAD_SHOULD_STOP, &kthread->flags);
493
		__kthread_unpark(k, kthread);
O
Oleg Nesterov 已提交
494 495
		wake_up_process(k);
		wait_for_completion(&kthread->exited);
496
		put_task_stack(k);
O
Oleg Nesterov 已提交
497 498
	}
	ret = k->exit_code;
L
Linus Torvalds 已提交
499
	put_task_struct(k);
500

501
	trace_sched_kthread_stop_ret(ret);
L
Linus Torvalds 已提交
502 503
	return ret;
}
504
EXPORT_SYMBOL(kthread_stop);
L
Linus Torvalds 已提交
505

506
int kthreadd(void *unused)
L
Linus Torvalds 已提交
507
{
508
	struct task_struct *tsk = current;
L
Linus Torvalds 已提交
509

510
	/* Setup a clean context for our children to inherit. */
511
	set_task_comm(tsk, "kthreadd");
512
	ignore_signals(tsk);
513
	set_cpus_allowed_ptr(tsk, cpu_all_mask);
514
	set_mems_allowed(node_states[N_MEMORY]);
515

516
	current->flags |= PF_NOFREEZE;
517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541

	for (;;) {
		set_current_state(TASK_INTERRUPTIBLE);
		if (list_empty(&kthread_create_list))
			schedule();
		__set_current_state(TASK_RUNNING);

		spin_lock(&kthread_create_lock);
		while (!list_empty(&kthread_create_list)) {
			struct kthread_create_info *create;

			create = list_entry(kthread_create_list.next,
					    struct kthread_create_info, list);
			list_del_init(&create->list);
			spin_unlock(&kthread_create_lock);

			create_kthread(create);

			spin_lock(&kthread_create_lock);
		}
		spin_unlock(&kthread_create_lock);
	}

	return 0;
}
T
Tejun Heo 已提交
542

P
Petr Mladek 已提交
543
void __kthread_init_worker(struct kthread_worker *worker,
Y
Yong Zhang 已提交
544 545 546 547 548 549 550 551
				const char *name,
				struct lock_class_key *key)
{
	spin_lock_init(&worker->lock);
	lockdep_set_class_and_name(&worker->lock, key, name);
	INIT_LIST_HEAD(&worker->work_list);
	worker->task = NULL;
}
P
Petr Mladek 已提交
552
EXPORT_SYMBOL_GPL(__kthread_init_worker);
Y
Yong Zhang 已提交
553

T
Tejun Heo 已提交
554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593
/**
 * kthread_worker_fn - kthread function to process kthread_worker
 * @worker_ptr: pointer to initialized kthread_worker
 *
 * This function can be used as @threadfn to kthread_create() or
 * kthread_run() with @worker_ptr argument pointing to an initialized
 * kthread_worker.  The started kthread will process work_list until
 * the it is stopped with kthread_stop().  A kthread can also call
 * this function directly after extra initialization.
 *
 * Different kthreads can be used for the same kthread_worker as long
 * as there's only one kthread attached to it at any given time.  A
 * kthread_worker without an attached kthread simply collects queued
 * kthread_works.
 */
int kthread_worker_fn(void *worker_ptr)
{
	struct kthread_worker *worker = worker_ptr;
	struct kthread_work *work;

	WARN_ON(worker->task);
	worker->task = current;
repeat:
	set_current_state(TASK_INTERRUPTIBLE);	/* mb paired w/ kthread_stop */

	if (kthread_should_stop()) {
		__set_current_state(TASK_RUNNING);
		spin_lock_irq(&worker->lock);
		worker->task = NULL;
		spin_unlock_irq(&worker->lock);
		return 0;
	}

	work = NULL;
	spin_lock_irq(&worker->lock);
	if (!list_empty(&worker->work_list)) {
		work = list_first_entry(&worker->work_list,
					struct kthread_work, node);
		list_del_init(&work->node);
	}
594
	worker->current_work = work;
T
Tejun Heo 已提交
595 596 597 598 599 600 601 602 603 604 605 606 607
	spin_unlock_irq(&worker->lock);

	if (work) {
		__set_current_state(TASK_RUNNING);
		work->func(work);
	} else if (!freezing(current))
		schedule();

	try_to_freeze();
	goto repeat;
}
EXPORT_SYMBOL_GPL(kthread_worker_fn);

608
/* insert @work before @pos in @worker */
P
Petr Mladek 已提交
609
static void kthread_insert_work(struct kthread_worker *worker,
610 611 612 613 614 615
			       struct kthread_work *work,
			       struct list_head *pos)
{
	lockdep_assert_held(&worker->lock);

	list_add_tail(&work->node, pos);
616
	work->worker = worker;
617
	if (!worker->current_work && likely(worker->task))
618 619 620
		wake_up_process(worker->task);
}

T
Tejun Heo 已提交
621
/**
P
Petr Mladek 已提交
622
 * kthread_queue_work - queue a kthread_work
T
Tejun Heo 已提交
623 624 625 626 627 628 629
 * @worker: target kthread_worker
 * @work: kthread_work to queue
 *
 * Queue @work to work processor @task for async execution.  @task
 * must have been created with kthread_worker_create().  Returns %true
 * if @work was successfully queued, %false if it was already pending.
 */
P
Petr Mladek 已提交
630
bool kthread_queue_work(struct kthread_worker *worker,
T
Tejun Heo 已提交
631 632 633 634 635 636 637
			struct kthread_work *work)
{
	bool ret = false;
	unsigned long flags;

	spin_lock_irqsave(&worker->lock, flags);
	if (list_empty(&work->node)) {
P
Petr Mladek 已提交
638
		kthread_insert_work(worker, work, &worker->work_list);
T
Tejun Heo 已提交
639 640 641 642 643
		ret = true;
	}
	spin_unlock_irqrestore(&worker->lock, flags);
	return ret;
}
P
Petr Mladek 已提交
644
EXPORT_SYMBOL_GPL(kthread_queue_work);
T
Tejun Heo 已提交
645

646 647 648 649 650 651 652 653 654 655 656 657
struct kthread_flush_work {
	struct kthread_work	work;
	struct completion	done;
};

static void kthread_flush_work_fn(struct kthread_work *work)
{
	struct kthread_flush_work *fwork =
		container_of(work, struct kthread_flush_work, work);
	complete(&fwork->done);
}

T
Tejun Heo 已提交
658
/**
P
Petr Mladek 已提交
659
 * kthread_flush_work - flush a kthread_work
T
Tejun Heo 已提交
660 661 662 663
 * @work: work to flush
 *
 * If @work is queued or executing, wait for it to finish execution.
 */
P
Petr Mladek 已提交
664
void kthread_flush_work(struct kthread_work *work)
T
Tejun Heo 已提交
665
{
666 667 668 669 670 671 672 673 674 675 676
	struct kthread_flush_work fwork = {
		KTHREAD_WORK_INIT(fwork.work, kthread_flush_work_fn),
		COMPLETION_INITIALIZER_ONSTACK(fwork.done),
	};
	struct kthread_worker *worker;
	bool noop = false;

retry:
	worker = work->worker;
	if (!worker)
		return;
T
Tejun Heo 已提交
677

678 679 680 681 682
	spin_lock_irq(&worker->lock);
	if (work->worker != worker) {
		spin_unlock_irq(&worker->lock);
		goto retry;
	}
T
Tejun Heo 已提交
683

684
	if (!list_empty(&work->node))
P
Petr Mladek 已提交
685
		kthread_insert_work(worker, &fwork.work, work->node.next);
686
	else if (worker->current_work == work)
P
Petr Mladek 已提交
687 688
		kthread_insert_work(worker, &fwork.work,
				    worker->work_list.next);
689 690
	else
		noop = true;
T
Tejun Heo 已提交
691

692
	spin_unlock_irq(&worker->lock);
T
Tejun Heo 已提交
693

694 695
	if (!noop)
		wait_for_completion(&fwork.done);
T
Tejun Heo 已提交
696
}
P
Petr Mladek 已提交
697
EXPORT_SYMBOL_GPL(kthread_flush_work);
T
Tejun Heo 已提交
698 699

/**
P
Petr Mladek 已提交
700
 * kthread_flush_worker - flush all current works on a kthread_worker
T
Tejun Heo 已提交
701 702 703 704 705
 * @worker: worker to flush
 *
 * Wait until all currently executing or pending works on @worker are
 * finished.
 */
P
Petr Mladek 已提交
706
void kthread_flush_worker(struct kthread_worker *worker)
T
Tejun Heo 已提交
707 708 709 710 711 712
{
	struct kthread_flush_work fwork = {
		KTHREAD_WORK_INIT(fwork.work, kthread_flush_work_fn),
		COMPLETION_INITIALIZER_ONSTACK(fwork.done),
	};

P
Petr Mladek 已提交
713
	kthread_queue_work(worker, &fwork.work);
T
Tejun Heo 已提交
714 715
	wait_for_completion(&fwork.done);
}
P
Petr Mladek 已提交
716
EXPORT_SYMBOL_GPL(kthread_flush_worker);