dm-crypt.c 25.7 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3
/*
 * Copyright (C) 2003 Christophe Saout <christophe@saout.de>
 * Copyright (C) 2004 Clemens Fruhwirth <clemens@endorphin.org>
M
Milan Broz 已提交
4
 * Copyright (C) 2006 Red Hat, Inc. All rights reserved.
L
Linus Torvalds 已提交
5 6 7 8
 *
 * This file is released under the GPL.
 */

9
#include <linux/err.h>
L
Linus Torvalds 已提交
10 11 12 13 14 15 16 17 18
#include <linux/module.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
#include <linux/mempool.h>
#include <linux/slab.h>
#include <linux/crypto.h>
#include <linux/workqueue.h>
19
#include <linux/backing-dev.h>
L
Linus Torvalds 已提交
20
#include <asm/atomic.h>
21
#include <linux/scatterlist.h>
L
Linus Torvalds 已提交
22
#include <asm/page.h>
23
#include <asm/unaligned.h>
L
Linus Torvalds 已提交
24 25 26

#include "dm.h"

27
#define DM_MSG_PREFIX "crypt"
M
Milan Broz 已提交
28
#define MESG_STR(x) x, sizeof(x)
L
Linus Torvalds 已提交
29 30 31 32

/*
 * per bio private data
 */
A
Alasdair G Kergon 已提交
33
struct dm_crypt_io {
L
Linus Torvalds 已提交
34
	struct dm_target *target;
35
	struct bio *base_bio;
L
Linus Torvalds 已提交
36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58
	struct work_struct work;
	atomic_t pending;
	int error;
};

/*
 * context holding the current state of a multi-part conversion
 */
struct convert_context {
	struct bio *bio_in;
	struct bio *bio_out;
	unsigned int offset_in;
	unsigned int offset_out;
	unsigned int idx_in;
	unsigned int idx_out;
	sector_t sector;
	int write;
};

struct crypt_config;

struct crypt_iv_operations {
	int (*ctr)(struct crypt_config *cc, struct dm_target *ti,
M
Milan Broz 已提交
59
		   const char *opts);
L
Linus Torvalds 已提交
60 61 62 63 64 65 66 67 68
	void (*dtr)(struct crypt_config *cc);
	const char *(*status)(struct crypt_config *cc);
	int (*generator)(struct crypt_config *cc, u8 *iv, sector_t sector);
};

/*
 * Crypt: maps a linear range of a block device
 * and encrypts / decrypts at the same time.
 */
M
Milan Broz 已提交
69
enum flags { DM_CRYPT_SUSPENDED, DM_CRYPT_KEY_VALID };
L
Linus Torvalds 已提交
70 71 72 73 74 75 76 77 78 79
struct crypt_config {
	struct dm_dev *dev;
	sector_t start;

	/*
	 * pool for per bio private data and
	 * for encryption buffer pages
	 */
	mempool_t *io_pool;
	mempool_t *page_pool;
M
Milan Broz 已提交
80
	struct bio_set *bs;
L
Linus Torvalds 已提交
81

M
Milan Broz 已提交
82 83
	struct workqueue_struct *io_queue;
	struct workqueue_struct *crypt_queue;
L
Linus Torvalds 已提交
84 85 86 87 88
	/*
	 * crypto related data
	 */
	struct crypt_iv_operations *iv_gen_ops;
	char *iv_mode;
89 90 91 92
	union {
		struct crypto_cipher *essiv_tfm;
		int benbi_shift;
	} iv_gen_private;
L
Linus Torvalds 已提交
93 94 95
	sector_t iv_offset;
	unsigned int iv_size;

96 97 98
	char cipher[CRYPTO_MAX_ALG_NAME];
	char chainmode[CRYPTO_MAX_ALG_NAME];
	struct crypto_blkcipher *tfm;
M
Milan Broz 已提交
99
	unsigned long flags;
L
Linus Torvalds 已提交
100 101 102 103
	unsigned int key_size;
	u8 key[0];
};

M
Milan Broz 已提交
104
#define MIN_IOS        16
L
Linus Torvalds 已提交
105 106 107
#define MIN_POOL_PAGES 32
#define MIN_BIO_PAGES  8

108
static struct kmem_cache *_crypt_io_pool;
L
Linus Torvalds 已提交
109

A
Alasdair G Kergon 已提交
110
static void clone_init(struct dm_crypt_io *, struct bio *);
O
Olaf Kirch 已提交
111

L
Linus Torvalds 已提交
112 113 114
/*
 * Different IV generation algorithms:
 *
115
 * plain: the initial vector is the 32-bit little-endian version of the sector
116
 *        number, padded with zeros if necessary.
L
Linus Torvalds 已提交
117
 *
118 119 120
 * essiv: "encrypted sector|salt initial vector", the sector number is
 *        encrypted with the bulk cipher using a salt as key. The salt
 *        should be derived from the bulk cipher's key via hashing.
L
Linus Torvalds 已提交
121
 *
122 123 124
 * benbi: the 64-bit "big-endian 'narrow block'-count", starting at 1
 *        (needed for LRW-32-AES and possible other narrow block modes)
 *
L
Ludwig Nussel 已提交
125 126 127
 * null: the initial vector is always zero.  Provides compatibility with
 *       obsolete loop_fish2 devices.  Do not use for new devices.
 *
L
Linus Torvalds 已提交
128 129 130 131 132 133 134 135 136 137 138 139 140
 * plumb: unimplemented, see:
 * http://article.gmane.org/gmane.linux.kernel.device-mapper.dm-crypt/454
 */

static int crypt_iv_plain_gen(struct crypt_config *cc, u8 *iv, sector_t sector)
{
	memset(iv, 0, cc->iv_size);
	*(u32 *)iv = cpu_to_le32(sector & 0xffffffff);

	return 0;
}

static int crypt_iv_essiv_ctr(struct crypt_config *cc, struct dm_target *ti,
M
Milan Broz 已提交
141
			      const char *opts)
L
Linus Torvalds 已提交
142
{
143
	struct crypto_cipher *essiv_tfm;
144 145
	struct crypto_hash *hash_tfm;
	struct hash_desc desc;
L
Linus Torvalds 已提交
146 147 148
	struct scatterlist sg;
	unsigned int saltsize;
	u8 *salt;
149
	int err;
L
Linus Torvalds 已提交
150 151

	if (opts == NULL) {
152
		ti->error = "Digest algorithm missing for ESSIV mode";
L
Linus Torvalds 已提交
153 154 155 156
		return -EINVAL;
	}

	/* Hash the cipher key with the given hash algorithm */
157 158
	hash_tfm = crypto_alloc_hash(opts, 0, CRYPTO_ALG_ASYNC);
	if (IS_ERR(hash_tfm)) {
159
		ti->error = "Error initializing ESSIV hash";
160
		return PTR_ERR(hash_tfm);
L
Linus Torvalds 已提交
161 162
	}

163
	saltsize = crypto_hash_digestsize(hash_tfm);
L
Linus Torvalds 已提交
164 165
	salt = kmalloc(saltsize, GFP_KERNEL);
	if (salt == NULL) {
166
		ti->error = "Error kmallocing salt storage in ESSIV";
167
		crypto_free_hash(hash_tfm);
L
Linus Torvalds 已提交
168 169 170
		return -ENOMEM;
	}

171
	sg_set_buf(&sg, cc->key, cc->key_size);
172 173 174 175 176 177 178
	desc.tfm = hash_tfm;
	desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP;
	err = crypto_hash_digest(&desc, &sg, cc->key_size, salt);
	crypto_free_hash(hash_tfm);

	if (err) {
		ti->error = "Error calculating hash in ESSIV";
179
		kfree(salt);
180 181
		return err;
	}
L
Linus Torvalds 已提交
182 183

	/* Setup the essiv_tfm with the given salt */
184 185
	essiv_tfm = crypto_alloc_cipher(cc->cipher, 0, CRYPTO_ALG_ASYNC);
	if (IS_ERR(essiv_tfm)) {
186
		ti->error = "Error allocating crypto tfm for ESSIV";
L
Linus Torvalds 已提交
187
		kfree(salt);
188
		return PTR_ERR(essiv_tfm);
L
Linus Torvalds 已提交
189
	}
190 191
	if (crypto_cipher_blocksize(essiv_tfm) !=
	    crypto_blkcipher_ivsize(cc->tfm)) {
192
		ti->error = "Block size of ESSIV cipher does "
M
Milan Broz 已提交
193
			    "not match IV size of block cipher";
194
		crypto_free_cipher(essiv_tfm);
L
Linus Torvalds 已提交
195 196 197
		kfree(salt);
		return -EINVAL;
	}
198 199
	err = crypto_cipher_setkey(essiv_tfm, salt, saltsize);
	if (err) {
200
		ti->error = "Failed to set key for ESSIV cipher";
201
		crypto_free_cipher(essiv_tfm);
L
Linus Torvalds 已提交
202
		kfree(salt);
203
		return err;
L
Linus Torvalds 已提交
204 205 206
	}
	kfree(salt);

207
	cc->iv_gen_private.essiv_tfm = essiv_tfm;
L
Linus Torvalds 已提交
208 209 210 211 212
	return 0;
}

static void crypt_iv_essiv_dtr(struct crypt_config *cc)
{
213 214
	crypto_free_cipher(cc->iv_gen_private.essiv_tfm);
	cc->iv_gen_private.essiv_tfm = NULL;
L
Linus Torvalds 已提交
215 216 217 218 219 220
}

static int crypt_iv_essiv_gen(struct crypt_config *cc, u8 *iv, sector_t sector)
{
	memset(iv, 0, cc->iv_size);
	*(u64 *)iv = cpu_to_le64(sector);
221
	crypto_cipher_encrypt_one(cc->iv_gen_private.essiv_tfm, iv, iv);
L
Linus Torvalds 已提交
222 223 224
	return 0;
}

225 226 227 228
static int crypt_iv_benbi_ctr(struct crypt_config *cc, struct dm_target *ti,
			      const char *opts)
{
	unsigned int bs = crypto_blkcipher_blocksize(cc->tfm);
229
	int log = ilog2(bs);
230 231 232 233 234 235 236 237 238 239 240 241 242 243

	/* we need to calculate how far we must shift the sector count
	 * to get the cipher block count, we use this shift in _gen */

	if (1 << log != bs) {
		ti->error = "cypher blocksize is not a power of 2";
		return -EINVAL;
	}

	if (log > 9) {
		ti->error = "cypher blocksize is > 512";
		return -EINVAL;
	}

244
	cc->iv_gen_private.benbi_shift = 9 - log;
245 246 247 248 249 250 251 252 253 254

	return 0;
}

static void crypt_iv_benbi_dtr(struct crypt_config *cc)
{
}

static int crypt_iv_benbi_gen(struct crypt_config *cc, u8 *iv, sector_t sector)
{
255 256
	__be64 val;

257
	memset(iv, 0, cc->iv_size - sizeof(u64)); /* rest is cleared below */
258 259 260

	val = cpu_to_be64(((u64)sector << cc->iv_gen_private.benbi_shift) + 1);
	put_unaligned(val, (__be64 *)(iv + cc->iv_size - sizeof(u64)));
261

L
Linus Torvalds 已提交
262 263 264
	return 0;
}

L
Ludwig Nussel 已提交
265 266 267 268 269 270 271
static int crypt_iv_null_gen(struct crypt_config *cc, u8 *iv, sector_t sector)
{
	memset(iv, 0, cc->iv_size);

	return 0;
}

L
Linus Torvalds 已提交
272 273 274 275 276 277 278 279 280 281
static struct crypt_iv_operations crypt_iv_plain_ops = {
	.generator = crypt_iv_plain_gen
};

static struct crypt_iv_operations crypt_iv_essiv_ops = {
	.ctr       = crypt_iv_essiv_ctr,
	.dtr       = crypt_iv_essiv_dtr,
	.generator = crypt_iv_essiv_gen
};

282 283 284 285 286
static struct crypt_iv_operations crypt_iv_benbi_ops = {
	.ctr	   = crypt_iv_benbi_ctr,
	.dtr	   = crypt_iv_benbi_dtr,
	.generator = crypt_iv_benbi_gen
};
L
Linus Torvalds 已提交
287

L
Ludwig Nussel 已提交
288 289 290 291
static struct crypt_iv_operations crypt_iv_null_ops = {
	.generator = crypt_iv_null_gen
};

292
static int
L
Linus Torvalds 已提交
293 294 295 296
crypt_convert_scatterlist(struct crypt_config *cc, struct scatterlist *out,
                          struct scatterlist *in, unsigned int length,
                          int write, sector_t sector)
{
297
	u8 iv[cc->iv_size] __attribute__ ((aligned(__alignof__(u64))));
298 299 300 301 302
	struct blkcipher_desc desc = {
		.tfm = cc->tfm,
		.info = iv,
		.flags = CRYPTO_TFM_REQ_MAY_SLEEP,
	};
L
Linus Torvalds 已提交
303 304 305 306 307 308 309 310
	int r;

	if (cc->iv_gen_ops) {
		r = cc->iv_gen_ops->generator(cc, iv, sector);
		if (r < 0)
			return r;

		if (write)
311
			r = crypto_blkcipher_encrypt_iv(&desc, out, in, length);
L
Linus Torvalds 已提交
312
		else
313
			r = crypto_blkcipher_decrypt_iv(&desc, out, in, length);
L
Linus Torvalds 已提交
314 315
	} else {
		if (write)
316
			r = crypto_blkcipher_encrypt(&desc, out, in, length);
L
Linus Torvalds 已提交
317
		else
318
			r = crypto_blkcipher_decrypt(&desc, out, in, length);
L
Linus Torvalds 已提交
319 320 321 322 323
	}

	return r;
}

M
Milan Broz 已提交
324 325 326 327
static void crypt_convert_init(struct crypt_config *cc,
			       struct convert_context *ctx,
			       struct bio *bio_out, struct bio *bio_in,
			       sector_t sector, int write)
L
Linus Torvalds 已提交
328 329 330 331 332 333 334 335 336 337 338 339 340 341 342
{
	ctx->bio_in = bio_in;
	ctx->bio_out = bio_out;
	ctx->offset_in = 0;
	ctx->offset_out = 0;
	ctx->idx_in = bio_in ? bio_in->bi_idx : 0;
	ctx->idx_out = bio_out ? bio_out->bi_idx : 0;
	ctx->sector = sector + cc->iv_offset;
	ctx->write = write;
}

/*
 * Encrypt / decrypt data from one bio to another one (can be the same one)
 */
static int crypt_convert(struct crypt_config *cc,
M
Milan Broz 已提交
343
			 struct convert_context *ctx)
L
Linus Torvalds 已提交
344 345 346 347 348 349 350
{
	int r = 0;

	while(ctx->idx_in < ctx->bio_in->bi_vcnt &&
	      ctx->idx_out < ctx->bio_out->bi_vcnt) {
		struct bio_vec *bv_in = bio_iovec_idx(ctx->bio_in, ctx->idx_in);
		struct bio_vec *bv_out = bio_iovec_idx(ctx->bio_out, ctx->idx_out);
J
Jens Axboe 已提交
351 352 353
		struct scatterlist sg_in, sg_out;

		sg_init_table(&sg_in, 1);
354
		sg_set_page(&sg_in, bv_in->bv_page, 1 << SECTOR_SHIFT, bv_in->bv_offset + ctx->offset_in);
J
Jens Axboe 已提交
355 356

		sg_init_table(&sg_out, 1);
357
		sg_set_page(&sg_out, bv_out->bv_page, 1 << SECTOR_SHIFT, bv_out->bv_offset + ctx->offset_out);
L
Linus Torvalds 已提交
358 359 360 361 362 363 364 365 366 367 368 369 370 371

		ctx->offset_in += sg_in.length;
		if (ctx->offset_in >= bv_in->bv_len) {
			ctx->offset_in = 0;
			ctx->idx_in++;
		}

		ctx->offset_out += sg_out.length;
		if (ctx->offset_out >= bv_out->bv_len) {
			ctx->offset_out = 0;
			ctx->idx_out++;
		}

		r = crypt_convert_scatterlist(cc, &sg_out, &sg_in, sg_in.length,
M
Milan Broz 已提交
372
					      ctx->write, ctx->sector);
L
Linus Torvalds 已提交
373 374 375 376 377 378 379 380 381
		if (r < 0)
			break;

		ctx->sector++;
	}

	return r;
}

M
Milan Broz 已提交
382 383
static void dm_crypt_bio_destructor(struct bio *bio)
{
A
Alasdair G Kergon 已提交
384
	struct dm_crypt_io *io = bio->bi_private;
M
Milan Broz 已提交
385 386 387
	struct crypt_config *cc = io->target->private;

	bio_free(bio, cc->bs);
M
Milan Broz 已提交
388
}
M
Milan Broz 已提交
389

L
Linus Torvalds 已提交
390 391 392 393 394
/*
 * Generate a new unfragmented bio with the given size
 * This should never violate the device limitations
 * May return a smaller bio when running out of pages
 */
A
Alasdair G Kergon 已提交
395
static struct bio *crypt_alloc_buffer(struct dm_crypt_io *io, unsigned size)
L
Linus Torvalds 已提交
396
{
O
Olaf Kirch 已提交
397
	struct crypt_config *cc = io->target->private;
398
	struct bio *clone;
L
Linus Torvalds 已提交
399
	unsigned int nr_iovecs = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
400
	gfp_t gfp_mask = GFP_NOIO | __GFP_HIGHMEM;
L
Linus Torvalds 已提交
401 402
	unsigned int i;

O
Olaf Kirch 已提交
403
	clone = bio_alloc_bioset(GFP_NOIO, nr_iovecs, cc->bs);
404
	if (!clone)
L
Linus Torvalds 已提交
405 406
		return NULL;

O
Olaf Kirch 已提交
407
	clone_init(io, clone);
M
Milan Broz 已提交
408

409
	for (i = 0; i < nr_iovecs; i++) {
410
		struct bio_vec *bv = bio_iovec_idx(clone, i);
L
Linus Torvalds 已提交
411 412 413 414 415 416 417 418 419 420

		bv->bv_page = mempool_alloc(cc->page_pool, gfp_mask);
		if (!bv->bv_page)
			break;

		/*
		 * if additional pages cannot be allocated without waiting,
		 * return a partially allocated bio, the caller will then try
		 * to allocate additional bios while submitting this partial bio
		 */
421
		if (i == (MIN_BIO_PAGES - 1))
L
Linus Torvalds 已提交
422 423 424 425 426 427 428 429
			gfp_mask = (gfp_mask | __GFP_NOWARN) & ~__GFP_WAIT;

		bv->bv_offset = 0;
		if (size > PAGE_SIZE)
			bv->bv_len = PAGE_SIZE;
		else
			bv->bv_len = size;

430 431
		clone->bi_size += bv->bv_len;
		clone->bi_vcnt++;
L
Linus Torvalds 已提交
432 433 434
		size -= bv->bv_len;
	}

435 436
	if (!clone->bi_size) {
		bio_put(clone);
L
Linus Torvalds 已提交
437 438 439
		return NULL;
	}

440
	return clone;
L
Linus Torvalds 已提交
441 442
}

N
Neil Brown 已提交
443
static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone)
L
Linus Torvalds 已提交
444
{
N
Neil Brown 已提交
445
	unsigned int i;
L
Linus Torvalds 已提交
446 447
	struct bio_vec *bv;

N
Neil Brown 已提交
448
	for (i = 0; i < clone->bi_vcnt; i++) {
449
		bv = bio_iovec_idx(clone, i);
L
Linus Torvalds 已提交
450 451 452 453 454 455 456 457 458 459
		BUG_ON(!bv->bv_page);
		mempool_free(bv->bv_page, cc->page_pool);
		bv->bv_page = NULL;
	}
}

/*
 * One of the bios was finished. Check for completion of
 * the whole request and correctly clean up the buffer.
 */
M
Milan Broz 已提交
460
static void crypt_dec_pending(struct dm_crypt_io *io, int error)
L
Linus Torvalds 已提交
461 462 463 464 465 466 467 468 469
{
	struct crypt_config *cc = (struct crypt_config *) io->target->private;

	if (error < 0)
		io->error = error;

	if (!atomic_dec_and_test(&io->pending))
		return;

470
	bio_endio(io->base_bio, io->error);
L
Linus Torvalds 已提交
471 472 473 474 475

	mempool_free(io, cc->io_pool);
}

/*
M
Milan Broz 已提交
476
 * kcryptd/kcryptd_io:
L
Linus Torvalds 已提交
477 478
 *
 * Needed because it would be very unwise to do decryption in an
479
 * interrupt context.
M
Milan Broz 已提交
480 481 482 483 484 485 486 487
 *
 * kcryptd performs the actual encryption or decryption.
 *
 * kcryptd_io performs the IO submission.
 *
 * They must be separated as otherwise the final stages could be
 * starved by new requests which can block in the first stages due
 * to memory allocation.
L
Linus Torvalds 已提交
488
 */
D
David Howells 已提交
489
static void kcryptd_do_work(struct work_struct *work);
M
Milan Broz 已提交
490
static void kcryptd_do_crypt(struct work_struct *work);
L
Linus Torvalds 已提交
491

A
Alasdair G Kergon 已提交
492
static void kcryptd_queue_io(struct dm_crypt_io *io)
L
Linus Torvalds 已提交
493
{
494 495
	struct crypt_config *cc = io->target->private;

D
David Howells 已提交
496
	INIT_WORK(&io->work, kcryptd_do_work);
M
Milan Broz 已提交
497 498 499 500 501 502 503 504 505
	queue_work(cc->io_queue, &io->work);
}

static void kcryptd_queue_crypt(struct dm_crypt_io *io)
{
	struct crypt_config *cc = io->target->private;

	INIT_WORK(&io->work, kcryptd_do_crypt);
	queue_work(cc->crypt_queue, &io->work);
506 507
}

508
static void crypt_endio(struct bio *clone, int error)
509
{
A
Alasdair G Kergon 已提交
510
	struct dm_crypt_io *io = clone->bi_private;
511 512 513 514
	struct crypt_config *cc = io->target->private;
	unsigned read_io = bio_data_dir(clone) == READ;

	/*
515
	 * free the processed pages
516
	 */
517
	if (!read_io) {
N
Neil Brown 已提交
518
		crypt_free_buffer_pages(cc, clone);
519
		goto out;
520
	}
521 522 523 524 525 526 527

	if (unlikely(!bio_flagged(clone, BIO_UPTODATE))) {
		error = -EIO;
		goto out;
	}

	bio_put(clone);
M
Milan Broz 已提交
528
	kcryptd_queue_crypt(io);
529
	return;
530 531 532

out:
	bio_put(clone);
M
Milan Broz 已提交
533
	crypt_dec_pending(io, error);
534 535
}

A
Alasdair G Kergon 已提交
536
static void clone_init(struct dm_crypt_io *io, struct bio *clone)
537 538 539 540 541 542 543
{
	struct crypt_config *cc = io->target->private;

	clone->bi_private = io;
	clone->bi_end_io  = crypt_endio;
	clone->bi_bdev    = cc->dev->bdev;
	clone->bi_rw      = io->base_bio->bi_rw;
O
Olaf Kirch 已提交
544
	clone->bi_destructor = dm_crypt_bio_destructor;
545 546
}

A
Alasdair G Kergon 已提交
547
static void process_read(struct dm_crypt_io *io)
548 549 550 551
{
	struct crypt_config *cc = io->target->private;
	struct bio *base_bio = io->base_bio;
	struct bio *clone;
552 553 554
	sector_t sector = base_bio->bi_sector - io->target->begin;

	atomic_inc(&io->pending);
555 556 557 558 559 560

	/*
	 * The block layer might modify the bvec array, so always
	 * copy the required bvecs because we need the original
	 * one in order to decrypt the whole bio data *afterwards*.
	 */
M
Milan Broz 已提交
561
	clone = bio_alloc_bioset(GFP_NOIO, bio_segments(base_bio), cc->bs);
562
	if (unlikely(!clone)) {
M
Milan Broz 已提交
563
		crypt_dec_pending(io, -ENOMEM);
564
		return;
565
	}
566 567 568 569 570

	clone_init(io, clone);
	clone->bi_idx = 0;
	clone->bi_vcnt = bio_segments(base_bio);
	clone->bi_size = base_bio->bi_size;
571
	clone->bi_sector = cc->start + sector;
572 573 574
	memcpy(clone->bi_io_vec, bio_iovec(base_bio),
	       sizeof(struct bio_vec) * clone->bi_vcnt);

575
	generic_make_request(clone);
576 577
}

A
Alasdair G Kergon 已提交
578
static void process_write(struct dm_crypt_io *io)
579 580 581 582
{
	struct crypt_config *cc = io->target->private;
	struct bio *base_bio = io->base_bio;
	struct bio *clone;
583 584 585
	struct convert_context ctx;
	unsigned remaining = base_bio->bi_size;
	sector_t sector = base_bio->bi_sector - io->target->begin;
586

587
	atomic_inc(&io->pending);
588

589
	crypt_convert_init(cc, &ctx, NULL, base_bio, sector, 1);
590

591 592 593 594 595
	/*
	 * The allocated buffers can be smaller than the whole bio,
	 * so repeat the whole process until all the data can be handled.
	 */
	while (remaining) {
596
		clone = crypt_alloc_buffer(io, remaining);
597
		if (unlikely(!clone)) {
M
Milan Broz 已提交
598
			crypt_dec_pending(io, -ENOMEM);
599 600
			return;
		}
601 602

		ctx.bio_out = clone;
603
		ctx.idx_out = 0;
604 605

		if (unlikely(crypt_convert(cc, &ctx) < 0)) {
N
Neil Brown 已提交
606
			crypt_free_buffer_pages(cc, clone);
607
			bio_put(clone);
M
Milan Broz 已提交
608
			crypt_dec_pending(io, -EIO);
609
			return;
610 611
		}

612 613 614
		/* crypt_convert should have filled the clone bio */
		BUG_ON(ctx.idx_out < clone->bi_vcnt);

615 616 617 618
		clone->bi_sector = cc->start + sector;
		remaining -= clone->bi_size;
		sector += bio_sectors(clone);

O
Olaf Kirch 已提交
619 620
		/* Grab another reference to the io struct
		 * before we kick off the request */
621 622 623
		if (remaining)
			atomic_inc(&io->pending);

624 625
		generic_make_request(clone);

626 627 628
		/* Do not reference clone after this - it
		 * may be gone already. */

629 630
		/* out of memory -> run queues */
		if (remaining)
631
			congestion_wait(WRITE, HZ/100);
632
	}
633 634
}

A
Alasdair G Kergon 已提交
635
static void process_read_endio(struct dm_crypt_io *io)
636 637
{
	struct crypt_config *cc = io->target->private;
L
Linus Torvalds 已提交
638 639
	struct convert_context ctx;

640 641
	crypt_convert_init(cc, &ctx, io->base_bio, io->base_bio,
			   io->base_bio->bi_sector - io->target->begin, 0);
L
Linus Torvalds 已提交
642

M
Milan Broz 已提交
643
	crypt_dec_pending(io, crypt_convert(cc, &ctx));
L
Linus Torvalds 已提交
644 645
}

D
David Howells 已提交
646
static void kcryptd_do_work(struct work_struct *work)
L
Linus Torvalds 已提交
647
{
A
Alasdair G Kergon 已提交
648
	struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);
649

M
Milan Broz 已提交
650
	if (bio_data_dir(io->base_bio) == READ)
651
		process_read(io);
M
Milan Broz 已提交
652 653 654 655 656 657 658 659
}

static void kcryptd_do_crypt(struct work_struct *work)
{
	struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);

	if (bio_data_dir(io->base_bio) == READ)
		process_read_endio(io);
660 661
	else
		process_write(io);
L
Linus Torvalds 已提交
662 663 664 665 666 667 668 669 670 671 672 673 674
}

/*
 * Decode key from its hex representation
 */
static int crypt_decode_key(u8 *key, char *hex, unsigned int size)
{
	char buffer[3];
	char *endp;
	unsigned int i;

	buffer[2] = '\0';

675
	for (i = 0; i < size; i++) {
L
Linus Torvalds 已提交
676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697
		buffer[0] = *hex++;
		buffer[1] = *hex++;

		key[i] = (u8)simple_strtoul(buffer, &endp, 16);

		if (endp != &buffer[2])
			return -EINVAL;
	}

	if (*hex != '\0')
		return -EINVAL;

	return 0;
}

/*
 * Encode key into its hex representation
 */
static void crypt_encode_key(char *hex, u8 *key, unsigned int size)
{
	unsigned int i;

698
	for (i = 0; i < size; i++) {
L
Linus Torvalds 已提交
699 700 701 702 703 704
		sprintf(hex, "%02x", *key);
		hex += 2;
		key++;
	}
}

M
Milan Broz 已提交
705 706 707 708 709 710 711 712 713 714
static int crypt_set_key(struct crypt_config *cc, char *key)
{
	unsigned key_size = strlen(key) >> 1;

	if (cc->key_size && cc->key_size != key_size)
		return -EINVAL;

	cc->key_size = key_size; /* initial settings */

	if ((!key_size && strcmp(key, "-")) ||
M
Milan Broz 已提交
715
	   (key_size && crypt_decode_key(cc->key, key, key_size) < 0))
M
Milan Broz 已提交
716 717 718 719 720 721 722 723 724 725 726 727 728 729
		return -EINVAL;

	set_bit(DM_CRYPT_KEY_VALID, &cc->flags);

	return 0;
}

static int crypt_wipe_key(struct crypt_config *cc)
{
	clear_bit(DM_CRYPT_KEY_VALID, &cc->flags);
	memset(&cc->key, 0, cc->key_size * sizeof(u8));
	return 0;
}

L
Linus Torvalds 已提交
730 731 732 733 734 735 736
/*
 * Construct an encryption mapping:
 * <cipher> <key> <iv_offset> <dev_path> <start>
 */
static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv)
{
	struct crypt_config *cc;
737
	struct crypto_blkcipher *tfm;
L
Linus Torvalds 已提交
738 739 740 741 742 743
	char *tmp;
	char *cipher;
	char *chainmode;
	char *ivmode;
	char *ivopts;
	unsigned int key_size;
A
Andrew Morton 已提交
744
	unsigned long long tmpll;
L
Linus Torvalds 已提交
745 746

	if (argc != 5) {
747
		ti->error = "Not enough arguments";
L
Linus Torvalds 已提交
748 749 750 751 752 753 754 755 756 757
		return -EINVAL;
	}

	tmp = argv[0];
	cipher = strsep(&tmp, "-");
	chainmode = strsep(&tmp, "-");
	ivopts = strsep(&tmp, "-");
	ivmode = strsep(&ivopts, ":");

	if (tmp)
758
		DMWARN("Unexpected additional cipher options");
L
Linus Torvalds 已提交
759 760 761

	key_size = strlen(argv[1]) >> 1;

M
Milan Broz 已提交
762
 	cc = kzalloc(sizeof(*cc) + key_size * sizeof(u8), GFP_KERNEL);
L
Linus Torvalds 已提交
763 764
	if (cc == NULL) {
		ti->error =
765
			"Cannot allocate transparent encryption context";
L
Linus Torvalds 已提交
766 767 768
		return -ENOMEM;
	}

M
Milan Broz 已提交
769
 	if (crypt_set_key(cc, argv[1])) {
770
		ti->error = "Error decoding key";
M
Milan Broz 已提交
771
		goto bad_cipher;
L
Linus Torvalds 已提交
772 773 774 775 776 777 778 779
	}

	/* Compatiblity mode for old dm-crypt cipher strings */
	if (!chainmode || (strcmp(chainmode, "plain") == 0 && !ivmode)) {
		chainmode = "cbc";
		ivmode = "plain";
	}

780 781
	if (strcmp(chainmode, "ecb") && !ivmode) {
		ti->error = "This chaining mode requires an IV mechanism";
M
Milan Broz 已提交
782
		goto bad_cipher;
L
Linus Torvalds 已提交
783 784
	}

M
Milan Broz 已提交
785 786
	if (snprintf(cc->cipher, CRYPTO_MAX_ALG_NAME, "%s(%s)",
		     chainmode, cipher) >= CRYPTO_MAX_ALG_NAME) {
787
		ti->error = "Chain mode + cipher name is too long";
M
Milan Broz 已提交
788
		goto bad_cipher;
L
Linus Torvalds 已提交
789 790
	}

791 792
	tfm = crypto_alloc_blkcipher(cc->cipher, 0, CRYPTO_ALG_ASYNC);
	if (IS_ERR(tfm)) {
793
		ti->error = "Error allocating crypto tfm";
M
Milan Broz 已提交
794
		goto bad_cipher;
L
Linus Torvalds 已提交
795 796
	}

797 798
	strcpy(cc->cipher, cipher);
	strcpy(cc->chainmode, chainmode);
L
Linus Torvalds 已提交
799 800 801
	cc->tfm = tfm;

	/*
802
	 * Choose ivmode. Valid modes: "plain", "essiv:<esshash>", "benbi".
L
Linus Torvalds 已提交
803 804 805 806 807 808 809 810 811
	 * See comments at iv code
	 */

	if (ivmode == NULL)
		cc->iv_gen_ops = NULL;
	else if (strcmp(ivmode, "plain") == 0)
		cc->iv_gen_ops = &crypt_iv_plain_ops;
	else if (strcmp(ivmode, "essiv") == 0)
		cc->iv_gen_ops = &crypt_iv_essiv_ops;
812 813
	else if (strcmp(ivmode, "benbi") == 0)
		cc->iv_gen_ops = &crypt_iv_benbi_ops;
L
Ludwig Nussel 已提交
814 815
	else if (strcmp(ivmode, "null") == 0)
		cc->iv_gen_ops = &crypt_iv_null_ops;
L
Linus Torvalds 已提交
816
	else {
817
		ti->error = "Invalid IV mode";
M
Milan Broz 已提交
818
		goto bad_ivmode;
L
Linus Torvalds 已提交
819 820 821 822
	}

	if (cc->iv_gen_ops && cc->iv_gen_ops->ctr &&
	    cc->iv_gen_ops->ctr(cc, ti, ivopts) < 0)
M
Milan Broz 已提交
823
		goto bad_ivmode;
L
Linus Torvalds 已提交
824

825 826
	cc->iv_size = crypto_blkcipher_ivsize(tfm);
	if (cc->iv_size)
L
Linus Torvalds 已提交
827
		/* at least a 64 bit sector number should fit in our buffer */
828
		cc->iv_size = max(cc->iv_size,
M
Milan Broz 已提交
829
				  (unsigned int)(sizeof(u64) / sizeof(u8)));
L
Linus Torvalds 已提交
830 831
	else {
		if (cc->iv_gen_ops) {
832
			DMWARN("Selected cipher does not support IVs");
L
Linus Torvalds 已提交
833 834 835 836 837 838
			if (cc->iv_gen_ops->dtr)
				cc->iv_gen_ops->dtr(cc);
			cc->iv_gen_ops = NULL;
		}
	}

839
	cc->io_pool = mempool_create_slab_pool(MIN_IOS, _crypt_io_pool);
L
Linus Torvalds 已提交
840
	if (!cc->io_pool) {
841
		ti->error = "Cannot allocate crypt io mempool";
M
Milan Broz 已提交
842
		goto bad_slab_pool;
L
Linus Torvalds 已提交
843 844
	}

845
	cc->page_pool = mempool_create_page_pool(MIN_POOL_PAGES, 0);
L
Linus Torvalds 已提交
846
	if (!cc->page_pool) {
847
		ti->error = "Cannot allocate page mempool";
M
Milan Broz 已提交
848
		goto bad_page_pool;
L
Linus Torvalds 已提交
849 850
	}

851
	cc->bs = bioset_create(MIN_IOS, MIN_IOS);
M
Milan Broz 已提交
852 853 854 855 856
	if (!cc->bs) {
		ti->error = "Cannot allocate crypt bioset";
		goto bad_bs;
	}

857
	if (crypto_blkcipher_setkey(tfm, cc->key, key_size) < 0) {
858
		ti->error = "Error setting key";
M
Milan Broz 已提交
859
		goto bad_device;
L
Linus Torvalds 已提交
860 861
	}

A
Andrew Morton 已提交
862
	if (sscanf(argv[2], "%llu", &tmpll) != 1) {
863
		ti->error = "Invalid iv_offset sector";
M
Milan Broz 已提交
864
		goto bad_device;
L
Linus Torvalds 已提交
865
	}
A
Andrew Morton 已提交
866
	cc->iv_offset = tmpll;
L
Linus Torvalds 已提交
867

A
Andrew Morton 已提交
868
	if (sscanf(argv[4], "%llu", &tmpll) != 1) {
869
		ti->error = "Invalid device sector";
M
Milan Broz 已提交
870
		goto bad_device;
L
Linus Torvalds 已提交
871
	}
A
Andrew Morton 已提交
872
	cc->start = tmpll;
L
Linus Torvalds 已提交
873 874

	if (dm_get_device(ti, argv[3], cc->start, ti->len,
M
Milan Broz 已提交
875
			  dm_table_get_mode(ti->table), &cc->dev)) {
876
		ti->error = "Device lookup failed";
M
Milan Broz 已提交
877
		goto bad_device;
L
Linus Torvalds 已提交
878 879 880 881 882 883 884
	}

	if (ivmode && cc->iv_gen_ops) {
		if (ivopts)
			*(ivopts - 1) = ':';
		cc->iv_mode = kmalloc(strlen(ivmode) + 1, GFP_KERNEL);
		if (!cc->iv_mode) {
885
			ti->error = "Error kmallocing iv_mode string";
M
Milan Broz 已提交
886
			goto bad_ivmode_string;
L
Linus Torvalds 已提交
887 888 889 890 891
		}
		strcpy(cc->iv_mode, ivmode);
	} else
		cc->iv_mode = NULL;

M
Milan Broz 已提交
892 893 894 895 896 897 898 899
	cc->io_queue = create_singlethread_workqueue("kcryptd_io");
	if (!cc->io_queue) {
		ti->error = "Couldn't create kcryptd io queue";
		goto bad_io_queue;
	}

	cc->crypt_queue = create_singlethread_workqueue("kcryptd");
	if (!cc->crypt_queue) {
900
		ti->error = "Couldn't create kcryptd queue";
M
Milan Broz 已提交
901
		goto bad_crypt_queue;
902 903
	}

L
Linus Torvalds 已提交
904 905 906
	ti->private = cc;
	return 0;

M
Milan Broz 已提交
907 908 909
bad_crypt_queue:
	destroy_workqueue(cc->io_queue);
bad_io_queue:
910
	kfree(cc->iv_mode);
M
Milan Broz 已提交
911
bad_ivmode_string:
912
	dm_put_device(ti, cc->dev);
M
Milan Broz 已提交
913
bad_device:
M
Milan Broz 已提交
914 915
	bioset_free(cc->bs);
bad_bs:
L
Linus Torvalds 已提交
916
	mempool_destroy(cc->page_pool);
M
Milan Broz 已提交
917
bad_page_pool:
L
Linus Torvalds 已提交
918
	mempool_destroy(cc->io_pool);
M
Milan Broz 已提交
919
bad_slab_pool:
L
Linus Torvalds 已提交
920 921
	if (cc->iv_gen_ops && cc->iv_gen_ops->dtr)
		cc->iv_gen_ops->dtr(cc);
M
Milan Broz 已提交
922
bad_ivmode:
923
	crypto_free_blkcipher(tfm);
M
Milan Broz 已提交
924
bad_cipher:
925 926
	/* Must zero key material before freeing */
	memset(cc, 0, sizeof(*cc) + cc->key_size * sizeof(u8));
L
Linus Torvalds 已提交
927 928 929 930 931 932 933 934
	kfree(cc);
	return -EINVAL;
}

static void crypt_dtr(struct dm_target *ti)
{
	struct crypt_config *cc = (struct crypt_config *) ti->private;

M
Milan Broz 已提交
935 936
	destroy_workqueue(cc->io_queue);
	destroy_workqueue(cc->crypt_queue);
M
Milan Broz 已提交
937

M
Milan Broz 已提交
938
	bioset_free(cc->bs);
L
Linus Torvalds 已提交
939 940 941
	mempool_destroy(cc->page_pool);
	mempool_destroy(cc->io_pool);

942
	kfree(cc->iv_mode);
L
Linus Torvalds 已提交
943 944
	if (cc->iv_gen_ops && cc->iv_gen_ops->dtr)
		cc->iv_gen_ops->dtr(cc);
945
	crypto_free_blkcipher(cc->tfm);
L
Linus Torvalds 已提交
946
	dm_put_device(ti, cc->dev);
947 948 949

	/* Must zero key material before freeing */
	memset(cc, 0, sizeof(*cc) + cc->key_size * sizeof(u8));
L
Linus Torvalds 已提交
950 951 952 953 954 955
	kfree(cc);
}

static int crypt_map(struct dm_target *ti, struct bio *bio,
		     union map_info *map_context)
{
956
	struct crypt_config *cc = ti->private;
A
Alasdair G Kergon 已提交
957
	struct dm_crypt_io *io;
L
Linus Torvalds 已提交
958

M
Milan Broz 已提交
959
	io = mempool_alloc(cc->io_pool, GFP_NOIO);
L
Linus Torvalds 已提交
960
	io->target = ti;
961
	io->base_bio = bio;
M
Milan Broz 已提交
962
	io->error = 0;
963
	atomic_set(&io->pending, 0);
M
Milan Broz 已提交
964 965 966 967 968

	if (bio_data_dir(io->base_bio) == READ)
		kcryptd_queue_io(io);
	else
		kcryptd_queue_crypt(io);
L
Linus Torvalds 已提交
969

970
	return DM_MAPIO_SUBMITTED;
L
Linus Torvalds 已提交
971 972 973 974 975 976 977 978 979 980 981 982 983 984 985
}

static int crypt_status(struct dm_target *ti, status_type_t type,
			char *result, unsigned int maxlen)
{
	struct crypt_config *cc = (struct crypt_config *) ti->private;
	unsigned int sz = 0;

	switch (type) {
	case STATUSTYPE_INFO:
		result[0] = '\0';
		break;

	case STATUSTYPE_TABLE:
		if (cc->iv_mode)
986 987
			DMEMIT("%s-%s-%s ", cc->cipher, cc->chainmode,
			       cc->iv_mode);
L
Linus Torvalds 已提交
988
		else
989
			DMEMIT("%s-%s ", cc->cipher, cc->chainmode);
L
Linus Torvalds 已提交
990 991 992 993 994 995 996 997 998 999 1000 1001 1002

		if (cc->key_size > 0) {
			if ((maxlen - sz) < ((cc->key_size << 1) + 1))
				return -ENOMEM;

			crypt_encode_key(result + sz, cc->key, cc->key_size);
			sz += cc->key_size << 1;
		} else {
			if (sz >= maxlen)
				return -ENOMEM;
			result[sz++] = '-';
		}

A
Andrew Morton 已提交
1003 1004
		DMEMIT(" %llu %s %llu", (unsigned long long)cc->iv_offset,
				cc->dev->name, (unsigned long long)cc->start);
L
Linus Torvalds 已提交
1005 1006 1007 1008 1009
		break;
	}
	return 0;
}

M
Milan Broz 已提交
1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062
static void crypt_postsuspend(struct dm_target *ti)
{
	struct crypt_config *cc = ti->private;

	set_bit(DM_CRYPT_SUSPENDED, &cc->flags);
}

static int crypt_preresume(struct dm_target *ti)
{
	struct crypt_config *cc = ti->private;

	if (!test_bit(DM_CRYPT_KEY_VALID, &cc->flags)) {
		DMERR("aborting resume - crypt key is not set.");
		return -EAGAIN;
	}

	return 0;
}

static void crypt_resume(struct dm_target *ti)
{
	struct crypt_config *cc = ti->private;

	clear_bit(DM_CRYPT_SUSPENDED, &cc->flags);
}

/* Message interface
 *	key set <key>
 *	key wipe
 */
static int crypt_message(struct dm_target *ti, unsigned argc, char **argv)
{
	struct crypt_config *cc = ti->private;

	if (argc < 2)
		goto error;

	if (!strnicmp(argv[0], MESG_STR("key"))) {
		if (!test_bit(DM_CRYPT_SUSPENDED, &cc->flags)) {
			DMWARN("not suspended during key manipulation.");
			return -EINVAL;
		}
		if (argc == 3 && !strnicmp(argv[1], MESG_STR("set")))
			return crypt_set_key(cc, argv[2]);
		if (argc == 2 && !strnicmp(argv[1], MESG_STR("wipe")))
			return crypt_wipe_key(cc);
	}

error:
	DMWARN("unrecognised message received.");
	return -EINVAL;
}

L
Linus Torvalds 已提交
1063 1064
static struct target_type crypt_target = {
	.name   = "crypt",
L
Ludwig Nussel 已提交
1065
	.version= {1, 5, 0},
L
Linus Torvalds 已提交
1066 1067 1068 1069 1070
	.module = THIS_MODULE,
	.ctr    = crypt_ctr,
	.dtr    = crypt_dtr,
	.map    = crypt_map,
	.status = crypt_status,
M
Milan Broz 已提交
1071 1072 1073 1074
	.postsuspend = crypt_postsuspend,
	.preresume = crypt_preresume,
	.resume = crypt_resume,
	.message = crypt_message,
L
Linus Torvalds 已提交
1075 1076 1077 1078 1079 1080
};

static int __init dm_crypt_init(void)
{
	int r;

A
Alasdair G Kergon 已提交
1081
	_crypt_io_pool = KMEM_CACHE(dm_crypt_io, 0);
L
Linus Torvalds 已提交
1082 1083 1084 1085 1086
	if (!_crypt_io_pool)
		return -ENOMEM;

	r = dm_register_target(&crypt_target);
	if (r < 0) {
1087
		DMERR("register failed %d", r);
1088
		kmem_cache_destroy(_crypt_io_pool);
L
Linus Torvalds 已提交
1089 1090 1091 1092 1093 1094 1095 1096 1097 1098
	}

	return r;
}

static void __exit dm_crypt_exit(void)
{
	int r = dm_unregister_target(&crypt_target);

	if (r < 0)
1099
		DMERR("unregister failed %d", r);
L
Linus Torvalds 已提交
1100 1101 1102 1103 1104 1105 1106 1107 1108 1109

	kmem_cache_destroy(_crypt_io_pool);
}

module_init(dm_crypt_init);
module_exit(dm_crypt_exit);

MODULE_AUTHOR("Christophe Saout <christophe@saout.de>");
MODULE_DESCRIPTION(DM_NAME " target for transparent encryption / decryption");
MODULE_LICENSE("GPL");