dm-kcopyd.c 14.0 KB
Newer Older
L
Linus Torvalds 已提交
1 2
/*
 * Copyright (C) 2002 Sistina Software (UK) Limited.
M
Milan Broz 已提交
3
 * Copyright (C) 2006 Red Hat GmbH
L
Linus Torvalds 已提交
4 5 6 7 8 9 10 11
 *
 * This file is released under the GPL.
 *
 * Kcopyd provides a simple interface for copying an area of one
 * block-device to one or more other block-devices, with an asynchronous
 * completion notification.
 */

H
Heinz Mauelshagen 已提交
12
#include <linux/types.h>
L
Linus Torvalds 已提交
13 14 15 16 17 18 19 20 21 22 23
#include <asm/atomic.h>
#include <linux/blkdev.h>
#include <linux/fs.h>
#include <linux/init.h>
#include <linux/list.h>
#include <linux/mempool.h>
#include <linux/module.h>
#include <linux/pagemap.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/workqueue.h>
A
Arjan van de Ven 已提交
24
#include <linux/mutex.h>
25
#include <linux/device-mapper.h>
A
Alasdair G Kergon 已提交
26
#include <linux/dm-kcopyd.h>
L
Linus Torvalds 已提交
27

H
Heinz Mauelshagen 已提交
28
#include "dm.h"
L
Linus Torvalds 已提交
29 30 31 32 33

/*-----------------------------------------------------------------
 * Each kcopyd client has its own little pool of preallocated
 * pages for kcopyd io.
 *---------------------------------------------------------------*/
H
Heinz Mauelshagen 已提交
34
struct dm_kcopyd_client {
L
Linus Torvalds 已提交
35 36 37 38
	spinlock_t lock;
	struct page_list *pages;
	unsigned int nr_pages;
	unsigned int nr_free_pages;
39

M
Milan Broz 已提交
40 41
	struct dm_io_client *io_client;

42 43
	wait_queue_head_t destroyq;
	atomic_t nr_jobs;
M
Mikulas Patocka 已提交
44

M
Mikulas Patocka 已提交
45 46
	mempool_t *job_pool;

M
Mikulas Patocka 已提交
47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62
	struct workqueue_struct *kcopyd_wq;
	struct work_struct kcopyd_work;

/*
 * We maintain three lists of jobs:
 *
 * i)   jobs waiting for pages
 * ii)  jobs that have pages, and are waiting for the io to be issued.
 * iii) jobs that have completed.
 *
 * All three of these are protected by job_lock.
 */
	spinlock_t job_lock;
	struct list_head complete_jobs;
	struct list_head io_jobs;
	struct list_head pages_jobs;
L
Linus Torvalds 已提交
63 64
};

M
Mikulas Patocka 已提交
65 66 67 68 69
static void wake(struct dm_kcopyd_client *kc)
{
	queue_work(kc->kcopyd_wq, &kc->kcopyd_work);
}

L
Linus Torvalds 已提交
70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92
static struct page_list *alloc_pl(void)
{
	struct page_list *pl;

	pl = kmalloc(sizeof(*pl), GFP_KERNEL);
	if (!pl)
		return NULL;

	pl->page = alloc_page(GFP_KERNEL);
	if (!pl->page) {
		kfree(pl);
		return NULL;
	}

	return pl;
}

static void free_pl(struct page_list *pl)
{
	__free_page(pl->page);
	kfree(pl);
}

H
Heinz Mauelshagen 已提交
93
static int kcopyd_get_pages(struct dm_kcopyd_client *kc,
L
Linus Torvalds 已提交
94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115
			    unsigned int nr, struct page_list **pages)
{
	struct page_list *pl;

	spin_lock(&kc->lock);
	if (kc->nr_free_pages < nr) {
		spin_unlock(&kc->lock);
		return -ENOMEM;
	}

	kc->nr_free_pages -= nr;
	for (*pages = pl = kc->pages; --nr; pl = pl->next)
		;

	kc->pages = pl->next;
	pl->next = NULL;

	spin_unlock(&kc->lock);

	return 0;
}

H
Heinz Mauelshagen 已提交
116
static void kcopyd_put_pages(struct dm_kcopyd_client *kc, struct page_list *pl)
L
Linus Torvalds 已提交
117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143
{
	struct page_list *cursor;

	spin_lock(&kc->lock);
	for (cursor = pl; cursor->next; cursor = cursor->next)
		kc->nr_free_pages++;

	kc->nr_free_pages++;
	cursor->next = kc->pages;
	kc->pages = pl;
	spin_unlock(&kc->lock);
}

/*
 * These three functions resize the page pool.
 */
static void drop_pages(struct page_list *pl)
{
	struct page_list *next;

	while (pl) {
		next = pl->next;
		free_pl(pl);
		pl = next;
	}
}

H
Heinz Mauelshagen 已提交
144
static int client_alloc_pages(struct dm_kcopyd_client *kc, unsigned int nr)
L
Linus Torvalds 已提交
145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164
{
	unsigned int i;
	struct page_list *pl = NULL, *next;

	for (i = 0; i < nr; i++) {
		next = alloc_pl();
		if (!next) {
			if (pl)
				drop_pages(pl);
			return -ENOMEM;
		}
		next->next = pl;
		pl = next;
	}

	kcopyd_put_pages(kc, pl);
	kc->nr_pages += nr;
	return 0;
}

H
Heinz Mauelshagen 已提交
165
static void client_free_pages(struct dm_kcopyd_client *kc)
L
Linus Torvalds 已提交
166 167 168 169 170 171 172 173 174 175 176 177 178
{
	BUG_ON(kc->nr_free_pages != kc->nr_pages);
	drop_pages(kc->pages);
	kc->pages = NULL;
	kc->nr_free_pages = kc->nr_pages = 0;
}

/*-----------------------------------------------------------------
 * kcopyd_jobs need to be allocated by the *clients* of kcopyd,
 * for this reason we use a mempool to prevent the client from
 * ever having to do io (which could cause a deadlock).
 *---------------------------------------------------------------*/
struct kcopyd_job {
H
Heinz Mauelshagen 已提交
179
	struct dm_kcopyd_client *kc;
L
Linus Torvalds 已提交
180 181 182 183 184 185 186
	struct list_head list;
	unsigned long flags;

	/*
	 * Error state of the job.
	 */
	int read_err;
187
	unsigned long write_err;
L
Linus Torvalds 已提交
188 189 190 191 192

	/*
	 * Either READ or WRITE
	 */
	int rw;
H
Heinz Mauelshagen 已提交
193
	struct dm_io_region source;
L
Linus Torvalds 已提交
194 195 196 197 198

	/*
	 * The destinations for the transfer.
	 */
	unsigned int num_dests;
H
Heinz Mauelshagen 已提交
199
	struct dm_io_region dests[DM_KCOPYD_MAX_REGIONS];
L
Linus Torvalds 已提交
200 201 202 203 204 205 206 207 208

	sector_t offset;
	unsigned int nr_pages;
	struct page_list *pages;

	/*
	 * Set this to ensure you are notified when the job has
	 * completed.  'context' is for callback to use.
	 */
H
Heinz Mauelshagen 已提交
209
	dm_kcopyd_notify_fn fn;
L
Linus Torvalds 已提交
210 211 212 213 214 215
	void *context;

	/*
	 * These fields are only used if the job has been split
	 * into more manageable parts.
	 */
216
	struct mutex lock;
L
Linus Torvalds 已提交
217 218 219 220 221 222 223
	atomic_t sub_jobs;
	sector_t progress;
};

/* FIXME: this should scale with the number of pages */
#define MIN_JOBS 512

224
static struct kmem_cache *_job_cache;
L
Linus Torvalds 已提交
225

226
int __init dm_kcopyd_init(void)
L
Linus Torvalds 已提交
227
{
A
Alasdair G Kergon 已提交
228
	_job_cache = KMEM_CACHE(kcopyd_job, 0);
L
Linus Torvalds 已提交
229 230 231 232 233 234
	if (!_job_cache)
		return -ENOMEM;

	return 0;
}

235
void dm_kcopyd_exit(void)
L
Linus Torvalds 已提交
236 237 238 239 240 241 242 243 244
{
	kmem_cache_destroy(_job_cache);
	_job_cache = NULL;
}

/*
 * Functions to push and pop a job onto the head of a given job
 * list.
 */
M
Mikulas Patocka 已提交
245 246
static struct kcopyd_job *pop(struct list_head *jobs,
			      struct dm_kcopyd_client *kc)
L
Linus Torvalds 已提交
247 248 249 250
{
	struct kcopyd_job *job = NULL;
	unsigned long flags;

M
Mikulas Patocka 已提交
251
	spin_lock_irqsave(&kc->job_lock, flags);
L
Linus Torvalds 已提交
252 253 254 255 256

	if (!list_empty(jobs)) {
		job = list_entry(jobs->next, struct kcopyd_job, list);
		list_del(&job->list);
	}
M
Mikulas Patocka 已提交
257
	spin_unlock_irqrestore(&kc->job_lock, flags);
L
Linus Torvalds 已提交
258 259 260 261

	return job;
}

A
Alasdair G Kergon 已提交
262
static void push(struct list_head *jobs, struct kcopyd_job *job)
L
Linus Torvalds 已提交
263 264
{
	unsigned long flags;
M
Mikulas Patocka 已提交
265
	struct dm_kcopyd_client *kc = job->kc;
L
Linus Torvalds 已提交
266

M
Mikulas Patocka 已提交
267
	spin_lock_irqsave(&kc->job_lock, flags);
L
Linus Torvalds 已提交
268
	list_add_tail(&job->list, jobs);
M
Mikulas Patocka 已提交
269
	spin_unlock_irqrestore(&kc->job_lock, flags);
L
Linus Torvalds 已提交
270 271
}

K
Kazuo Ito 已提交
272 273 274 275 276 277 278 279 280 281 282

static void push_head(struct list_head *jobs, struct kcopyd_job *job)
{
	unsigned long flags;
	struct dm_kcopyd_client *kc = job->kc;

	spin_lock_irqsave(&kc->job_lock, flags);
	list_add(&job->list, jobs);
	spin_unlock_irqrestore(&kc->job_lock, flags);
}

L
Linus Torvalds 已提交
283 284 285 286 287 288 289 290 291 292 293 294 295
/*
 * These three functions process 1 item from the corresponding
 * job list.
 *
 * They return:
 * < 0: error
 *   0: success
 * > 0: can't process yet.
 */
static int run_complete_job(struct kcopyd_job *job)
{
	void *context = job->context;
	int read_err = job->read_err;
296
	unsigned long write_err = job->write_err;
H
Heinz Mauelshagen 已提交
297 298
	dm_kcopyd_notify_fn fn = job->fn;
	struct dm_kcopyd_client *kc = job->kc;
L
Linus Torvalds 已提交
299

300 301
	if (job->pages)
		kcopyd_put_pages(kc, job->pages);
M
Mikulas Patocka 已提交
302
	mempool_free(job, kc->job_pool);
L
Linus Torvalds 已提交
303
	fn(read_err, write_err, context);
304 305 306 307

	if (atomic_dec_and_test(&kc->nr_jobs))
		wake_up(&kc->destroyq);

L
Linus Torvalds 已提交
308 309 310 311 312 313
	return 0;
}

static void complete_io(unsigned long error, void *context)
{
	struct kcopyd_job *job = (struct kcopyd_job *) context;
M
Mikulas Patocka 已提交
314
	struct dm_kcopyd_client *kc = job->kc;
L
Linus Torvalds 已提交
315 316 317

	if (error) {
		if (job->rw == WRITE)
318
			job->write_err |= error;
L
Linus Torvalds 已提交
319 320 321
		else
			job->read_err = 1;

H
Heinz Mauelshagen 已提交
322
		if (!test_bit(DM_KCOPYD_IGNORE_ERROR, &job->flags)) {
M
Mikulas Patocka 已提交
323 324
			push(&kc->complete_jobs, job);
			wake(kc);
L
Linus Torvalds 已提交
325 326 327 328 329
			return;
		}
	}

	if (job->rw == WRITE)
M
Mikulas Patocka 已提交
330
		push(&kc->complete_jobs, job);
L
Linus Torvalds 已提交
331 332 333

	else {
		job->rw = WRITE;
M
Mikulas Patocka 已提交
334
		push(&kc->io_jobs, job);
L
Linus Torvalds 已提交
335 336
	}

M
Mikulas Patocka 已提交
337
	wake(kc);
L
Linus Torvalds 已提交
338 339 340 341 342 343 344 345 346
}

/*
 * Request io on as many buffer heads as we can currently get for
 * a particular job.
 */
static int run_io_job(struct kcopyd_job *job)
{
	int r;
M
Milan Broz 已提交
347
	struct dm_io_request io_req = {
348
		.bi_rw = job->rw | REQ_UNPLUG,
M
Milan Broz 已提交
349 350 351 352 353 354 355
		.mem.type = DM_IO_PAGE_LIST,
		.mem.ptr.pl = job->pages,
		.mem.offset = job->offset,
		.notify.fn = complete_io,
		.notify.context = job,
		.client = job->kc->io_client,
	};
L
Linus Torvalds 已提交
356 357

	if (job->rw == READ)
M
Milan Broz 已提交
358
		r = dm_io(&io_req, 1, &job->source, NULL);
L
Linus Torvalds 已提交
359
	else
M
Milan Broz 已提交
360
		r = dm_io(&io_req, job->num_dests, job->dests, NULL);
L
Linus Torvalds 已提交
361 362 363 364 365 366 367 368 369 370 371 372 373

	return r;
}

static int run_pages_job(struct kcopyd_job *job)
{
	int r;

	job->nr_pages = dm_div_up(job->dests[0].count + job->offset,
				  PAGE_SIZE >> 9);
	r = kcopyd_get_pages(job->kc, job->nr_pages, &job->pages);
	if (!r) {
		/* this job is ready for io */
M
Mikulas Patocka 已提交
374
		push(&job->kc->io_jobs, job);
L
Linus Torvalds 已提交
375 376 377 378 379 380 381 382 383 384 385 386 387 388
		return 0;
	}

	if (r == -ENOMEM)
		/* can't complete now */
		return 1;

	return r;
}

/*
 * Run through a list for as long as possible.  Returns the count
 * of successful jobs.
 */
M
Mikulas Patocka 已提交
389 390
static int process_jobs(struct list_head *jobs, struct dm_kcopyd_client *kc,
			int (*fn) (struct kcopyd_job *))
L
Linus Torvalds 已提交
391 392 393 394
{
	struct kcopyd_job *job;
	int r, count = 0;

M
Mikulas Patocka 已提交
395
	while ((job = pop(jobs, kc))) {
L
Linus Torvalds 已提交
396 397 398 399 400 401

		r = fn(job);

		if (r < 0) {
			/* error this rogue job */
			if (job->rw == WRITE)
402
				job->write_err = (unsigned long) -1L;
L
Linus Torvalds 已提交
403 404
			else
				job->read_err = 1;
M
Mikulas Patocka 已提交
405
			push(&kc->complete_jobs, job);
L
Linus Torvalds 已提交
406 407 408 409 410 411 412 413
			break;
		}

		if (r > 0) {
			/*
			 * We couldn't service this job ATM, so
			 * push this job back onto the list.
			 */
K
Kazuo Ito 已提交
414
			push_head(jobs, job);
L
Linus Torvalds 已提交
415 416 417 418 419 420 421 422 423 424 425 426
			break;
		}

		count++;
	}

	return count;
}

/*
 * kcopyd does this every time it's woken up.
 */
M
Mikulas Patocka 已提交
427
static void do_work(struct work_struct *work)
L
Linus Torvalds 已提交
428
{
M
Mikulas Patocka 已提交
429 430 431
	struct dm_kcopyd_client *kc = container_of(work,
					struct dm_kcopyd_client, kcopyd_work);

L
Linus Torvalds 已提交
432 433 434 435 436 437 438
	/*
	 * The order that these are called is *very* important.
	 * complete jobs can free some pages for pages jobs.
	 * Pages jobs when successful will jump onto the io jobs
	 * list.  io jobs call wake when they complete and it all
	 * starts again.
	 */
M
Mikulas Patocka 已提交
439 440 441
	process_jobs(&kc->complete_jobs, kc, run_complete_job);
	process_jobs(&kc->pages_jobs, kc, run_pages_job);
	process_jobs(&kc->io_jobs, kc, run_io_job);
L
Linus Torvalds 已提交
442 443 444 445 446 447 448 449 450
}

/*
 * If we are copying a small region we just dispatch a single job
 * to do the copy, otherwise the io has to be split up into many
 * jobs.
 */
static void dispatch_job(struct kcopyd_job *job)
{
M
Mikulas Patocka 已提交
451 452
	struct dm_kcopyd_client *kc = job->kc;
	atomic_inc(&kc->nr_jobs);
M
Mikulas Patocka 已提交
453 454 455 456
	if (unlikely(!job->source.count))
		push(&kc->complete_jobs, job);
	else
		push(&kc->pages_jobs, job);
M
Mikulas Patocka 已提交
457
	wake(kc);
L
Linus Torvalds 已提交
458 459 460
}

#define SUB_JOB_SIZE 128
461 462
static void segment_complete(int read_err, unsigned long write_err,
			     void *context)
L
Linus Torvalds 已提交
463 464 465 466 467
{
	/* FIXME: tidy this function */
	sector_t progress = 0;
	sector_t count = 0;
	struct kcopyd_job *job = (struct kcopyd_job *) context;
468
	struct dm_kcopyd_client *kc = job->kc;
L
Linus Torvalds 已提交
469

470
	mutex_lock(&job->lock);
L
Linus Torvalds 已提交
471 472 473 474 475 476

	/* update the error */
	if (read_err)
		job->read_err = 1;

	if (write_err)
477
		job->write_err |= write_err;
L
Linus Torvalds 已提交
478 479 480 481 482

	/*
	 * Only dispatch more work if there hasn't been an error.
	 */
	if ((!job->read_err && !job->write_err) ||
H
Heinz Mauelshagen 已提交
483
	    test_bit(DM_KCOPYD_IGNORE_ERROR, &job->flags)) {
L
Linus Torvalds 已提交
484 485 486 487 488 489 490 491 492 493
		/* get the next chunk of work */
		progress = job->progress;
		count = job->source.count - progress;
		if (count) {
			if (count > SUB_JOB_SIZE)
				count = SUB_JOB_SIZE;

			job->progress += count;
		}
	}
494
	mutex_unlock(&job->lock);
L
Linus Torvalds 已提交
495 496 497

	if (count) {
		int i;
498
		struct kcopyd_job *sub_job = mempool_alloc(kc->job_pool,
M
Mikulas Patocka 已提交
499
							   GFP_NOIO);
L
Linus Torvalds 已提交
500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516

		*sub_job = *job;
		sub_job->source.sector += progress;
		sub_job->source.count = count;

		for (i = 0; i < job->num_dests; i++) {
			sub_job->dests[i].sector += progress;
			sub_job->dests[i].count = count;
		}

		sub_job->fn = segment_complete;
		sub_job->context = job;
		dispatch_job(sub_job);

	} else if (atomic_dec_and_test(&job->sub_jobs)) {

		/*
M
Mikulas Patocka 已提交
517 518 519 520 521 522 523
		 * Queue the completion callback to the kcopyd thread.
		 *
		 * Some callers assume that all the completions are called
		 * from a single thread and don't race with each other.
		 *
		 * We must not call the callback directly here because this
		 * code may not be executing in the thread.
L
Linus Torvalds 已提交
524
		 */
M
Mikulas Patocka 已提交
525 526
		push(&kc->complete_jobs, job);
		wake(kc);
L
Linus Torvalds 已提交
527 528 529 530 531 532 533 534 535 536 537 538
	}
}

/*
 * Create some little jobs that will do the move between
 * them.
 */
#define SPLIT_COUNT 8
static void split_job(struct kcopyd_job *job)
{
	int i;

M
Mikulas Patocka 已提交
539 540
	atomic_inc(&job->kc->nr_jobs);

L
Linus Torvalds 已提交
541 542 543 544 545
	atomic_set(&job->sub_jobs, SPLIT_COUNT);
	for (i = 0; i < SPLIT_COUNT; i++)
		segment_complete(0, 0u, job);
}

H
Heinz Mauelshagen 已提交
546 547 548
int dm_kcopyd_copy(struct dm_kcopyd_client *kc, struct dm_io_region *from,
		   unsigned int num_dests, struct dm_io_region *dests,
		   unsigned int flags, dm_kcopyd_notify_fn fn, void *context)
L
Linus Torvalds 已提交
549 550 551 552 553 554
{
	struct kcopyd_job *job;

	/*
	 * Allocate a new job.
	 */
M
Mikulas Patocka 已提交
555
	job = mempool_alloc(kc->job_pool, GFP_NOIO);
L
Linus Torvalds 已提交
556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581

	/*
	 * set up for the read.
	 */
	job->kc = kc;
	job->flags = flags;
	job->read_err = 0;
	job->write_err = 0;
	job->rw = READ;

	job->source = *from;

	job->num_dests = num_dests;
	memcpy(&job->dests, dests, sizeof(*dests) * num_dests);

	job->offset = 0;
	job->nr_pages = 0;
	job->pages = NULL;

	job->fn = fn;
	job->context = context;

	if (job->source.count < SUB_JOB_SIZE)
		dispatch_job(job);

	else {
582
		mutex_init(&job->lock);
L
Linus Torvalds 已提交
583 584 585 586 587 588
		job->progress = 0;
		split_job(job);
	}

	return 0;
}
H
Heinz Mauelshagen 已提交
589
EXPORT_SYMBOL(dm_kcopyd_copy);
L
Linus Torvalds 已提交
590 591 592 593 594

/*
 * Cancels a kcopyd job, eg. someone might be deactivating a
 * mirror.
 */
595
#if 0
L
Linus Torvalds 已提交
596 597 598 599 600
int kcopyd_cancel(struct kcopyd_job *job, int block)
{
	/* FIXME: finish */
	return -1;
}
601
#endif  /*  0  */
L
Linus Torvalds 已提交
602 603

/*-----------------------------------------------------------------
604
 * Client setup
L
Linus Torvalds 已提交
605
 *---------------------------------------------------------------*/
H
Heinz Mauelshagen 已提交
606 607
int dm_kcopyd_client_create(unsigned int nr_pages,
			    struct dm_kcopyd_client **result)
L
Linus Torvalds 已提交
608
{
609
	int r = -ENOMEM;
H
Heinz Mauelshagen 已提交
610
	struct dm_kcopyd_client *kc;
L
Linus Torvalds 已提交
611 612

	kc = kmalloc(sizeof(*kc), GFP_KERNEL);
613 614
	if (!kc)
		return -ENOMEM;
L
Linus Torvalds 已提交
615 616

	spin_lock_init(&kc->lock);
M
Mikulas Patocka 已提交
617 618 619 620 621
	spin_lock_init(&kc->job_lock);
	INIT_LIST_HEAD(&kc->complete_jobs);
	INIT_LIST_HEAD(&kc->io_jobs);
	INIT_LIST_HEAD(&kc->pages_jobs);

M
Mikulas Patocka 已提交
622
	kc->job_pool = mempool_create_slab_pool(MIN_JOBS, _job_cache);
623 624
	if (!kc->job_pool)
		goto bad_slab;
M
Mikulas Patocka 已提交
625

M
Mikulas Patocka 已提交
626 627
	INIT_WORK(&kc->kcopyd_work, do_work);
	kc->kcopyd_wq = create_singlethread_workqueue("kcopyd");
628 629
	if (!kc->kcopyd_wq)
		goto bad_workqueue;
M
Mikulas Patocka 已提交
630

L
Linus Torvalds 已提交
631 632 633
	kc->pages = NULL;
	kc->nr_pages = kc->nr_free_pages = 0;
	r = client_alloc_pages(kc, nr_pages);
634 635
	if (r)
		goto bad_client_pages;
L
Linus Torvalds 已提交
636

M
Milan Broz 已提交
637 638 639
	kc->io_client = dm_io_client_create(nr_pages);
	if (IS_ERR(kc->io_client)) {
		r = PTR_ERR(kc->io_client);
640
		goto bad_io_client;
L
Linus Torvalds 已提交
641 642
	}

643 644 645
	init_waitqueue_head(&kc->destroyq);
	atomic_set(&kc->nr_jobs, 0);

L
Linus Torvalds 已提交
646 647
	*result = kc;
	return 0;
648 649 650 651 652 653 654 655 656 657 658

bad_io_client:
	client_free_pages(kc);
bad_client_pages:
	destroy_workqueue(kc->kcopyd_wq);
bad_workqueue:
	mempool_destroy(kc->job_pool);
bad_slab:
	kfree(kc);

	return r;
L
Linus Torvalds 已提交
659
}
H
Heinz Mauelshagen 已提交
660
EXPORT_SYMBOL(dm_kcopyd_client_create);
L
Linus Torvalds 已提交
661

H
Heinz Mauelshagen 已提交
662
void dm_kcopyd_client_destroy(struct dm_kcopyd_client *kc)
L
Linus Torvalds 已提交
663
{
664 665 666
	/* Wait for completion of all jobs submitted by this client. */
	wait_event(kc->destroyq, !atomic_read(&kc->nr_jobs));

M
Mikulas Patocka 已提交
667 668 669 670
	BUG_ON(!list_empty(&kc->complete_jobs));
	BUG_ON(!list_empty(&kc->io_jobs));
	BUG_ON(!list_empty(&kc->pages_jobs));
	destroy_workqueue(kc->kcopyd_wq);
M
Milan Broz 已提交
671
	dm_io_client_destroy(kc->io_client);
L
Linus Torvalds 已提交
672
	client_free_pages(kc);
M
Mikulas Patocka 已提交
673
	mempool_destroy(kc->job_pool);
L
Linus Torvalds 已提交
674 675
	kfree(kc);
}
H
Heinz Mauelshagen 已提交
676
EXPORT_SYMBOL(dm_kcopyd_client_destroy);