dm-kcopyd.c 14.2 KB
Newer Older
L
Linus Torvalds 已提交
1 2
/*
 * Copyright (C) 2002 Sistina Software (UK) Limited.
M
Milan Broz 已提交
3
 * Copyright (C) 2006 Red Hat GmbH
L
Linus Torvalds 已提交
4 5 6 7 8 9 10 11
 *
 * This file is released under the GPL.
 *
 * Kcopyd provides a simple interface for copying an area of one
 * block-device to one or more other block-devices, with an asynchronous
 * completion notification.
 */

H
Heinz Mauelshagen 已提交
12
#include <linux/types.h>
L
Linus Torvalds 已提交
13 14 15 16 17 18 19 20 21 22 23
#include <asm/atomic.h>
#include <linux/blkdev.h>
#include <linux/fs.h>
#include <linux/init.h>
#include <linux/list.h>
#include <linux/mempool.h>
#include <linux/module.h>
#include <linux/pagemap.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/workqueue.h>
A
Arjan van de Ven 已提交
24
#include <linux/mutex.h>
25
#include <linux/device-mapper.h>
A
Alasdair G Kergon 已提交
26
#include <linux/dm-kcopyd.h>
L
Linus Torvalds 已提交
27

H
Heinz Mauelshagen 已提交
28
#include "dm.h"
L
Linus Torvalds 已提交
29

30 31 32 33
#define SUB_JOB_SIZE	128
#define SPLIT_COUNT	8
#define MIN_JOBS	8

L
Linus Torvalds 已提交
34 35 36 37
/*-----------------------------------------------------------------
 * Each kcopyd client has its own little pool of preallocated
 * pages for kcopyd io.
 *---------------------------------------------------------------*/
H
Heinz Mauelshagen 已提交
38
struct dm_kcopyd_client {
L
Linus Torvalds 已提交
39 40 41
	struct page_list *pages;
	unsigned int nr_pages;
	unsigned int nr_free_pages;
42

M
Milan Broz 已提交
43 44
	struct dm_io_client *io_client;

45 46
	wait_queue_head_t destroyq;
	atomic_t nr_jobs;
M
Mikulas Patocka 已提交
47

M
Mikulas Patocka 已提交
48 49
	mempool_t *job_pool;

M
Mikulas Patocka 已提交
50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65
	struct workqueue_struct *kcopyd_wq;
	struct work_struct kcopyd_work;

/*
 * We maintain three lists of jobs:
 *
 * i)   jobs waiting for pages
 * ii)  jobs that have pages, and are waiting for the io to be issued.
 * iii) jobs that have completed.
 *
 * All three of these are protected by job_lock.
 */
	spinlock_t job_lock;
	struct list_head complete_jobs;
	struct list_head io_jobs;
	struct list_head pages_jobs;
L
Linus Torvalds 已提交
66 67
};

M
Mikulas Patocka 已提交
68 69 70 71 72
static void wake(struct dm_kcopyd_client *kc)
{
	queue_work(kc->kcopyd_wq, &kc->kcopyd_work);
}

73
static struct page_list *alloc_pl(gfp_t gfp)
L
Linus Torvalds 已提交
74 75 76
{
	struct page_list *pl;

77
	pl = kmalloc(sizeof(*pl), gfp);
L
Linus Torvalds 已提交
78 79 80
	if (!pl)
		return NULL;

81
	pl->page = alloc_page(gfp);
L
Linus Torvalds 已提交
82 83 84 85 86 87 88 89 90 91 92 93 94 95
	if (!pl->page) {
		kfree(pl);
		return NULL;
	}

	return pl;
}

static void free_pl(struct page_list *pl)
{
	__free_page(pl->page);
	kfree(pl);
}

H
Heinz Mauelshagen 已提交
96
static int kcopyd_get_pages(struct dm_kcopyd_client *kc,
L
Linus Torvalds 已提交
97 98 99 100
			    unsigned int nr, struct page_list **pages)
{
	struct page_list *pl;

101
	if (kc->nr_free_pages < nr)
L
Linus Torvalds 已提交
102 103 104 105 106 107 108 109 110 111 112 113
		return -ENOMEM;

	kc->nr_free_pages -= nr;
	for (*pages = pl = kc->pages; --nr; pl = pl->next)
		;

	kc->pages = pl->next;
	pl->next = NULL;

	return 0;
}

H
Heinz Mauelshagen 已提交
114
static void kcopyd_put_pages(struct dm_kcopyd_client *kc, struct page_list *pl)
L
Linus Torvalds 已提交
115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139
{
	struct page_list *cursor;

	for (cursor = pl; cursor->next; cursor = cursor->next)
		kc->nr_free_pages++;

	kc->nr_free_pages++;
	cursor->next = kc->pages;
	kc->pages = pl;
}

/*
 * These three functions resize the page pool.
 */
static void drop_pages(struct page_list *pl)
{
	struct page_list *next;

	while (pl) {
		next = pl->next;
		free_pl(pl);
		pl = next;
	}
}

H
Heinz Mauelshagen 已提交
140
static int client_alloc_pages(struct dm_kcopyd_client *kc, unsigned int nr)
L
Linus Torvalds 已提交
141 142 143 144 145
{
	unsigned int i;
	struct page_list *pl = NULL, *next;

	for (i = 0; i < nr; i++) {
146
		next = alloc_pl(GFP_KERNEL);
L
Linus Torvalds 已提交
147 148 149 150 151 152 153 154 155 156 157 158 159 160
		if (!next) {
			if (pl)
				drop_pages(pl);
			return -ENOMEM;
		}
		next->next = pl;
		pl = next;
	}

	kcopyd_put_pages(kc, pl);
	kc->nr_pages += nr;
	return 0;
}

H
Heinz Mauelshagen 已提交
161
static void client_free_pages(struct dm_kcopyd_client *kc)
L
Linus Torvalds 已提交
162 163 164 165 166 167 168 169 170 171 172 173 174
{
	BUG_ON(kc->nr_free_pages != kc->nr_pages);
	drop_pages(kc->pages);
	kc->pages = NULL;
	kc->nr_free_pages = kc->nr_pages = 0;
}

/*-----------------------------------------------------------------
 * kcopyd_jobs need to be allocated by the *clients* of kcopyd,
 * for this reason we use a mempool to prevent the client from
 * ever having to do io (which could cause a deadlock).
 *---------------------------------------------------------------*/
struct kcopyd_job {
H
Heinz Mauelshagen 已提交
175
	struct dm_kcopyd_client *kc;
L
Linus Torvalds 已提交
176 177 178 179 180 181 182
	struct list_head list;
	unsigned long flags;

	/*
	 * Error state of the job.
	 */
	int read_err;
183
	unsigned long write_err;
L
Linus Torvalds 已提交
184 185 186 187 188

	/*
	 * Either READ or WRITE
	 */
	int rw;
H
Heinz Mauelshagen 已提交
189
	struct dm_io_region source;
L
Linus Torvalds 已提交
190 191 192 193 194

	/*
	 * The destinations for the transfer.
	 */
	unsigned int num_dests;
H
Heinz Mauelshagen 已提交
195
	struct dm_io_region dests[DM_KCOPYD_MAX_REGIONS];
L
Linus Torvalds 已提交
196 197 198 199 200 201 202 203 204

	sector_t offset;
	unsigned int nr_pages;
	struct page_list *pages;

	/*
	 * Set this to ensure you are notified when the job has
	 * completed.  'context' is for callback to use.
	 */
H
Heinz Mauelshagen 已提交
205
	dm_kcopyd_notify_fn fn;
L
Linus Torvalds 已提交
206 207 208 209 210 211
	void *context;

	/*
	 * These fields are only used if the job has been split
	 * into more manageable parts.
	 */
212
	struct mutex lock;
L
Linus Torvalds 已提交
213 214 215
	atomic_t sub_jobs;
	sector_t progress;

216 217
	struct kcopyd_job *master_job;
};
L
Linus Torvalds 已提交
218

219
static struct kmem_cache *_job_cache;
L
Linus Torvalds 已提交
220

221
int __init dm_kcopyd_init(void)
L
Linus Torvalds 已提交
222
{
223 224 225
	_job_cache = kmem_cache_create("kcopyd_job",
				sizeof(struct kcopyd_job) * (SPLIT_COUNT + 1),
				__alignof__(struct kcopyd_job), 0, NULL);
L
Linus Torvalds 已提交
226 227 228 229 230 231
	if (!_job_cache)
		return -ENOMEM;

	return 0;
}

232
void dm_kcopyd_exit(void)
L
Linus Torvalds 已提交
233 234 235 236 237 238 239 240 241
{
	kmem_cache_destroy(_job_cache);
	_job_cache = NULL;
}

/*
 * Functions to push and pop a job onto the head of a given job
 * list.
 */
M
Mikulas Patocka 已提交
242 243
static struct kcopyd_job *pop(struct list_head *jobs,
			      struct dm_kcopyd_client *kc)
L
Linus Torvalds 已提交
244 245 246 247
{
	struct kcopyd_job *job = NULL;
	unsigned long flags;

M
Mikulas Patocka 已提交
248
	spin_lock_irqsave(&kc->job_lock, flags);
L
Linus Torvalds 已提交
249 250 251 252 253

	if (!list_empty(jobs)) {
		job = list_entry(jobs->next, struct kcopyd_job, list);
		list_del(&job->list);
	}
M
Mikulas Patocka 已提交
254
	spin_unlock_irqrestore(&kc->job_lock, flags);
L
Linus Torvalds 已提交
255 256 257 258

	return job;
}

A
Alasdair G Kergon 已提交
259
static void push(struct list_head *jobs, struct kcopyd_job *job)
L
Linus Torvalds 已提交
260 261
{
	unsigned long flags;
M
Mikulas Patocka 已提交
262
	struct dm_kcopyd_client *kc = job->kc;
L
Linus Torvalds 已提交
263

M
Mikulas Patocka 已提交
264
	spin_lock_irqsave(&kc->job_lock, flags);
L
Linus Torvalds 已提交
265
	list_add_tail(&job->list, jobs);
M
Mikulas Patocka 已提交
266
	spin_unlock_irqrestore(&kc->job_lock, flags);
L
Linus Torvalds 已提交
267 268
}

K
Kazuo Ito 已提交
269 270 271 272 273 274 275 276 277 278 279

static void push_head(struct list_head *jobs, struct kcopyd_job *job)
{
	unsigned long flags;
	struct dm_kcopyd_client *kc = job->kc;

	spin_lock_irqsave(&kc->job_lock, flags);
	list_add(&job->list, jobs);
	spin_unlock_irqrestore(&kc->job_lock, flags);
}

L
Linus Torvalds 已提交
280 281 282 283 284 285 286 287 288 289 290 291 292
/*
 * These three functions process 1 item from the corresponding
 * job list.
 *
 * They return:
 * < 0: error
 *   0: success
 * > 0: can't process yet.
 */
static int run_complete_job(struct kcopyd_job *job)
{
	void *context = job->context;
	int read_err = job->read_err;
293
	unsigned long write_err = job->write_err;
H
Heinz Mauelshagen 已提交
294 295
	dm_kcopyd_notify_fn fn = job->fn;
	struct dm_kcopyd_client *kc = job->kc;
L
Linus Torvalds 已提交
296

297 298
	if (job->pages)
		kcopyd_put_pages(kc, job->pages);
299 300 301 302 303 304
	/*
	 * If this is the master job, the sub jobs have already
	 * completed so we can free everything.
	 */
	if (job->master_job == job)
		mempool_free(job, kc->job_pool);
L
Linus Torvalds 已提交
305
	fn(read_err, write_err, context);
306 307 308 309

	if (atomic_dec_and_test(&kc->nr_jobs))
		wake_up(&kc->destroyq);

L
Linus Torvalds 已提交
310 311 312 313 314 315
	return 0;
}

static void complete_io(unsigned long error, void *context)
{
	struct kcopyd_job *job = (struct kcopyd_job *) context;
M
Mikulas Patocka 已提交
316
	struct dm_kcopyd_client *kc = job->kc;
L
Linus Torvalds 已提交
317 318 319

	if (error) {
		if (job->rw == WRITE)
320
			job->write_err |= error;
L
Linus Torvalds 已提交
321 322 323
		else
			job->read_err = 1;

H
Heinz Mauelshagen 已提交
324
		if (!test_bit(DM_KCOPYD_IGNORE_ERROR, &job->flags)) {
M
Mikulas Patocka 已提交
325 326
			push(&kc->complete_jobs, job);
			wake(kc);
L
Linus Torvalds 已提交
327 328 329 330 331
			return;
		}
	}

	if (job->rw == WRITE)
M
Mikulas Patocka 已提交
332
		push(&kc->complete_jobs, job);
L
Linus Torvalds 已提交
333 334 335

	else {
		job->rw = WRITE;
M
Mikulas Patocka 已提交
336
		push(&kc->io_jobs, job);
L
Linus Torvalds 已提交
337 338
	}

M
Mikulas Patocka 已提交
339
	wake(kc);
L
Linus Torvalds 已提交
340 341 342 343 344 345 346 347 348
}

/*
 * Request io on as many buffer heads as we can currently get for
 * a particular job.
 */
static int run_io_job(struct kcopyd_job *job)
{
	int r;
M
Milan Broz 已提交
349
	struct dm_io_request io_req = {
M
Mikulas Patocka 已提交
350
		.bi_rw = job->rw,
M
Milan Broz 已提交
351 352 353 354 355 356 357
		.mem.type = DM_IO_PAGE_LIST,
		.mem.ptr.pl = job->pages,
		.mem.offset = job->offset,
		.notify.fn = complete_io,
		.notify.context = job,
		.client = job->kc->io_client,
	};
L
Linus Torvalds 已提交
358

J
Jens Axboe 已提交
359
	if (job->rw == READ)
M
Milan Broz 已提交
360
		r = dm_io(&io_req, 1, &job->source, NULL);
J
Jens Axboe 已提交
361
	else
M
Milan Broz 已提交
362
		r = dm_io(&io_req, job->num_dests, job->dests, NULL);
L
Linus Torvalds 已提交
363 364 365 366 367 368 369 370 371 372 373 374 375

	return r;
}

static int run_pages_job(struct kcopyd_job *job)
{
	int r;

	job->nr_pages = dm_div_up(job->dests[0].count + job->offset,
				  PAGE_SIZE >> 9);
	r = kcopyd_get_pages(job->kc, job->nr_pages, &job->pages);
	if (!r) {
		/* this job is ready for io */
M
Mikulas Patocka 已提交
376
		push(&job->kc->io_jobs, job);
L
Linus Torvalds 已提交
377 378 379 380 381 382 383 384 385 386 387 388 389 390
		return 0;
	}

	if (r == -ENOMEM)
		/* can't complete now */
		return 1;

	return r;
}

/*
 * Run through a list for as long as possible.  Returns the count
 * of successful jobs.
 */
M
Mikulas Patocka 已提交
391 392
static int process_jobs(struct list_head *jobs, struct dm_kcopyd_client *kc,
			int (*fn) (struct kcopyd_job *))
L
Linus Torvalds 已提交
393 394 395 396
{
	struct kcopyd_job *job;
	int r, count = 0;

M
Mikulas Patocka 已提交
397
	while ((job = pop(jobs, kc))) {
L
Linus Torvalds 已提交
398 399 400 401 402 403

		r = fn(job);

		if (r < 0) {
			/* error this rogue job */
			if (job->rw == WRITE)
404
				job->write_err = (unsigned long) -1L;
L
Linus Torvalds 已提交
405 406
			else
				job->read_err = 1;
M
Mikulas Patocka 已提交
407
			push(&kc->complete_jobs, job);
L
Linus Torvalds 已提交
408 409 410 411 412 413 414 415
			break;
		}

		if (r > 0) {
			/*
			 * We couldn't service this job ATM, so
			 * push this job back onto the list.
			 */
K
Kazuo Ito 已提交
416
			push_head(jobs, job);
L
Linus Torvalds 已提交
417 418 419 420 421 422 423 424 425 426 427 428
			break;
		}

		count++;
	}

	return count;
}

/*
 * kcopyd does this every time it's woken up.
 */
M
Mikulas Patocka 已提交
429
static void do_work(struct work_struct *work)
L
Linus Torvalds 已提交
430
{
M
Mikulas Patocka 已提交
431 432
	struct dm_kcopyd_client *kc = container_of(work,
					struct dm_kcopyd_client, kcopyd_work);
J
Jens Axboe 已提交
433
	struct blk_plug plug;
M
Mikulas Patocka 已提交
434

L
Linus Torvalds 已提交
435 436 437 438 439 440 441
	/*
	 * The order that these are called is *very* important.
	 * complete jobs can free some pages for pages jobs.
	 * Pages jobs when successful will jump onto the io jobs
	 * list.  io jobs call wake when they complete and it all
	 * starts again.
	 */
J
Jens Axboe 已提交
442
	blk_start_plug(&plug);
M
Mikulas Patocka 已提交
443 444 445
	process_jobs(&kc->complete_jobs, kc, run_complete_job);
	process_jobs(&kc->pages_jobs, kc, run_pages_job);
	process_jobs(&kc->io_jobs, kc, run_io_job);
J
Jens Axboe 已提交
446
	blk_finish_plug(&plug);
L
Linus Torvalds 已提交
447 448 449 450 451 452 453 454 455
}

/*
 * If we are copying a small region we just dispatch a single job
 * to do the copy, otherwise the io has to be split up into many
 * jobs.
 */
static void dispatch_job(struct kcopyd_job *job)
{
M
Mikulas Patocka 已提交
456 457
	struct dm_kcopyd_client *kc = job->kc;
	atomic_inc(&kc->nr_jobs);
M
Mikulas Patocka 已提交
458 459 460 461
	if (unlikely(!job->source.count))
		push(&kc->complete_jobs, job);
	else
		push(&kc->pages_jobs, job);
M
Mikulas Patocka 已提交
462
	wake(kc);
L
Linus Torvalds 已提交
463 464
}

465 466
static void segment_complete(int read_err, unsigned long write_err,
			     void *context)
L
Linus Torvalds 已提交
467 468 469 470
{
	/* FIXME: tidy this function */
	sector_t progress = 0;
	sector_t count = 0;
471 472
	struct kcopyd_job *sub_job = (struct kcopyd_job *) context;
	struct kcopyd_job *job = sub_job->master_job;
473
	struct dm_kcopyd_client *kc = job->kc;
L
Linus Torvalds 已提交
474

475
	mutex_lock(&job->lock);
L
Linus Torvalds 已提交
476 477 478 479 480 481

	/* update the error */
	if (read_err)
		job->read_err = 1;

	if (write_err)
482
		job->write_err |= write_err;
L
Linus Torvalds 已提交
483 484 485 486 487

	/*
	 * Only dispatch more work if there hasn't been an error.
	 */
	if ((!job->read_err && !job->write_err) ||
H
Heinz Mauelshagen 已提交
488
	    test_bit(DM_KCOPYD_IGNORE_ERROR, &job->flags)) {
L
Linus Torvalds 已提交
489 490 491 492 493 494 495 496 497 498
		/* get the next chunk of work */
		progress = job->progress;
		count = job->source.count - progress;
		if (count) {
			if (count > SUB_JOB_SIZE)
				count = SUB_JOB_SIZE;

			job->progress += count;
		}
	}
499
	mutex_unlock(&job->lock);
L
Linus Torvalds 已提交
500 501 502 503 504 505 506 507 508 509 510 511 512 513

	if (count) {
		int i;

		*sub_job = *job;
		sub_job->source.sector += progress;
		sub_job->source.count = count;

		for (i = 0; i < job->num_dests; i++) {
			sub_job->dests[i].sector += progress;
			sub_job->dests[i].count = count;
		}

		sub_job->fn = segment_complete;
514
		sub_job->context = sub_job;
L
Linus Torvalds 已提交
515 516 517 518 519
		dispatch_job(sub_job);

	} else if (atomic_dec_and_test(&job->sub_jobs)) {

		/*
M
Mikulas Patocka 已提交
520 521 522 523 524 525 526
		 * Queue the completion callback to the kcopyd thread.
		 *
		 * Some callers assume that all the completions are called
		 * from a single thread and don't race with each other.
		 *
		 * We must not call the callback directly here because this
		 * code may not be executing in the thread.
L
Linus Torvalds 已提交
527
		 */
M
Mikulas Patocka 已提交
528 529
		push(&kc->complete_jobs, job);
		wake(kc);
L
Linus Torvalds 已提交
530 531 532 533
	}
}

/*
534
 * Create some sub jobs to share the work between them.
L
Linus Torvalds 已提交
535
 */
536
static void split_job(struct kcopyd_job *master_job)
L
Linus Torvalds 已提交
537 538 539
{
	int i;

540
	atomic_inc(&master_job->kc->nr_jobs);
M
Mikulas Patocka 已提交
541

542 543 544 545 546
	atomic_set(&master_job->sub_jobs, SPLIT_COUNT);
	for (i = 0; i < SPLIT_COUNT; i++) {
		master_job[i + 1].master_job = master_job;
		segment_complete(0, 0u, &master_job[i + 1]);
	}
L
Linus Torvalds 已提交
547 548
}

H
Heinz Mauelshagen 已提交
549 550 551
int dm_kcopyd_copy(struct dm_kcopyd_client *kc, struct dm_io_region *from,
		   unsigned int num_dests, struct dm_io_region *dests,
		   unsigned int flags, dm_kcopyd_notify_fn fn, void *context)
L
Linus Torvalds 已提交
552 553 554 555
{
	struct kcopyd_job *job;

	/*
556 557
	 * Allocate an array of jobs consisting of one master job
	 * followed by SPLIT_COUNT sub jobs.
L
Linus Torvalds 已提交
558
	 */
M
Mikulas Patocka 已提交
559
	job = mempool_alloc(kc->job_pool, GFP_NOIO);
L
Linus Torvalds 已提交
560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580

	/*
	 * set up for the read.
	 */
	job->kc = kc;
	job->flags = flags;
	job->read_err = 0;
	job->write_err = 0;
	job->rw = READ;

	job->source = *from;

	job->num_dests = num_dests;
	memcpy(&job->dests, dests, sizeof(*dests) * num_dests);

	job->offset = 0;
	job->nr_pages = 0;
	job->pages = NULL;

	job->fn = fn;
	job->context = context;
581
	job->master_job = job;
L
Linus Torvalds 已提交
582

583
	if (job->source.count <= SUB_JOB_SIZE)
L
Linus Torvalds 已提交
584 585
		dispatch_job(job);
	else {
586
		mutex_init(&job->lock);
L
Linus Torvalds 已提交
587 588 589 590 591 592
		job->progress = 0;
		split_job(job);
	}

	return 0;
}
H
Heinz Mauelshagen 已提交
593
EXPORT_SYMBOL(dm_kcopyd_copy);
L
Linus Torvalds 已提交
594 595 596 597 598

/*
 * Cancels a kcopyd job, eg. someone might be deactivating a
 * mirror.
 */
599
#if 0
L
Linus Torvalds 已提交
600 601 602 603 604
int kcopyd_cancel(struct kcopyd_job *job, int block)
{
	/* FIXME: finish */
	return -1;
}
605
#endif  /*  0  */
L
Linus Torvalds 已提交
606 607

/*-----------------------------------------------------------------
608
 * Client setup
L
Linus Torvalds 已提交
609
 *---------------------------------------------------------------*/
H
Heinz Mauelshagen 已提交
610 611
int dm_kcopyd_client_create(unsigned int nr_pages,
			    struct dm_kcopyd_client **result)
L
Linus Torvalds 已提交
612
{
613
	int r = -ENOMEM;
H
Heinz Mauelshagen 已提交
614
	struct dm_kcopyd_client *kc;
L
Linus Torvalds 已提交
615 616

	kc = kmalloc(sizeof(*kc), GFP_KERNEL);
617 618
	if (!kc)
		return -ENOMEM;
L
Linus Torvalds 已提交
619

M
Mikulas Patocka 已提交
620 621 622 623 624
	spin_lock_init(&kc->job_lock);
	INIT_LIST_HEAD(&kc->complete_jobs);
	INIT_LIST_HEAD(&kc->io_jobs);
	INIT_LIST_HEAD(&kc->pages_jobs);

M
Mikulas Patocka 已提交
625
	kc->job_pool = mempool_create_slab_pool(MIN_JOBS, _job_cache);
626 627
	if (!kc->job_pool)
		goto bad_slab;
M
Mikulas Patocka 已提交
628

M
Mikulas Patocka 已提交
629
	INIT_WORK(&kc->kcopyd_work, do_work);
630 631
	kc->kcopyd_wq = alloc_workqueue("kcopyd",
					WQ_NON_REENTRANT | WQ_MEM_RECLAIM, 0);
632 633
	if (!kc->kcopyd_wq)
		goto bad_workqueue;
M
Mikulas Patocka 已提交
634

L
Linus Torvalds 已提交
635 636 637
	kc->pages = NULL;
	kc->nr_pages = kc->nr_free_pages = 0;
	r = client_alloc_pages(kc, nr_pages);
638 639
	if (r)
		goto bad_client_pages;
L
Linus Torvalds 已提交
640

M
Milan Broz 已提交
641 642 643
	kc->io_client = dm_io_client_create(nr_pages);
	if (IS_ERR(kc->io_client)) {
		r = PTR_ERR(kc->io_client);
644
		goto bad_io_client;
L
Linus Torvalds 已提交
645 646
	}

647 648 649
	init_waitqueue_head(&kc->destroyq);
	atomic_set(&kc->nr_jobs, 0);

L
Linus Torvalds 已提交
650 651
	*result = kc;
	return 0;
652 653 654 655 656 657 658 659 660 661 662

bad_io_client:
	client_free_pages(kc);
bad_client_pages:
	destroy_workqueue(kc->kcopyd_wq);
bad_workqueue:
	mempool_destroy(kc->job_pool);
bad_slab:
	kfree(kc);

	return r;
L
Linus Torvalds 已提交
663
}
H
Heinz Mauelshagen 已提交
664
EXPORT_SYMBOL(dm_kcopyd_client_create);
L
Linus Torvalds 已提交
665

H
Heinz Mauelshagen 已提交
666
void dm_kcopyd_client_destroy(struct dm_kcopyd_client *kc)
L
Linus Torvalds 已提交
667
{
668 669 670
	/* Wait for completion of all jobs submitted by this client. */
	wait_event(kc->destroyq, !atomic_read(&kc->nr_jobs));

M
Mikulas Patocka 已提交
671 672 673 674
	BUG_ON(!list_empty(&kc->complete_jobs));
	BUG_ON(!list_empty(&kc->io_jobs));
	BUG_ON(!list_empty(&kc->pages_jobs));
	destroy_workqueue(kc->kcopyd_wq);
M
Milan Broz 已提交
675
	dm_io_client_destroy(kc->io_client);
L
Linus Torvalds 已提交
676
	client_free_pages(kc);
M
Mikulas Patocka 已提交
677
	mempool_destroy(kc->job_pool);
L
Linus Torvalds 已提交
678 679
	kfree(kc);
}
H
Heinz Mauelshagen 已提交
680
EXPORT_SYMBOL(dm_kcopyd_client_destroy);