dm-crypt.c 47.5 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3
/*
 * Copyright (C) 2003 Christophe Saout <christophe@saout.de>
 * Copyright (C) 2004 Clemens Fruhwirth <clemens@endorphin.org>
4
 * Copyright (C) 2006-2009 Red Hat, Inc. All rights reserved.
5
 * Copyright (C) 2013 Milan Broz <gmazyland@gmail.com>
L
Linus Torvalds 已提交
6 7 8 9
 *
 * This file is released under the GPL.
 */

10
#include <linux/completion.h>
11
#include <linux/err.h>
L
Linus Torvalds 已提交
12 13 14 15 16 17 18 19 20
#include <linux/module.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
#include <linux/mempool.h>
#include <linux/slab.h>
#include <linux/crypto.h>
#include <linux/workqueue.h>
21
#include <linux/backing-dev.h>
22
#include <linux/percpu.h>
23
#include <linux/atomic.h>
24
#include <linux/scatterlist.h>
L
Linus Torvalds 已提交
25
#include <asm/page.h>
26
#include <asm/unaligned.h>
27 28 29
#include <crypto/hash.h>
#include <crypto/md5.h>
#include <crypto/algapi.h>
L
Linus Torvalds 已提交
30

31
#include <linux/device-mapper.h>
L
Linus Torvalds 已提交
32

33
#define DM_MSG_PREFIX "crypt"
L
Linus Torvalds 已提交
34 35 36 37 38

/*
 * context holding the current state of a multi-part conversion
 */
struct convert_context {
39
	struct completion restart;
L
Linus Torvalds 已提交
40 41
	struct bio *bio_in;
	struct bio *bio_out;
42 43
	struct bvec_iter iter_in;
	struct bvec_iter iter_out;
44
	sector_t cc_sector;
45
	atomic_t cc_pending;
L
Linus Torvalds 已提交
46 47
};

48 49 50 51
/*
 * per bio private data
 */
struct dm_crypt_io {
52
	struct crypt_config *cc;
53 54 55 56 57
	struct bio *base_bio;
	struct work_struct work;

	struct convert_context ctx;

58
	atomic_t io_pending;
59
	int error;
60
	sector_t sector;
M
Milan Broz 已提交
61
	struct dm_crypt_io *base_io;
62 63
};

64
struct dm_crypt_request {
65
	struct convert_context *ctx;
66 67
	struct scatterlist sg_in;
	struct scatterlist sg_out;
68
	sector_t iv_sector;
69 70
};

L
Linus Torvalds 已提交
71 72 73 74
struct crypt_config;

struct crypt_iv_operations {
	int (*ctr)(struct crypt_config *cc, struct dm_target *ti,
M
Milan Broz 已提交
75
		   const char *opts);
L
Linus Torvalds 已提交
76
	void (*dtr)(struct crypt_config *cc);
77
	int (*init)(struct crypt_config *cc);
78
	int (*wipe)(struct crypt_config *cc);
79 80 81 82
	int (*generator)(struct crypt_config *cc, u8 *iv,
			 struct dm_crypt_request *dmreq);
	int (*post)(struct crypt_config *cc, u8 *iv,
		    struct dm_crypt_request *dmreq);
L
Linus Torvalds 已提交
83 84
};

85
struct iv_essiv_private {
86 87
	struct crypto_hash *hash_tfm;
	u8 *salt;
88 89 90 91 92 93
};

struct iv_benbi_private {
	int shift;
};

94 95 96 97 98 99
#define LMK_SEED_SIZE 64 /* hash + 0 */
struct iv_lmk_private {
	struct crypto_shash *hash_tfm;
	u8 *seed;
};

100 101 102 103 104 105 106
#define TCW_WHITENING_SIZE 16
struct iv_tcw_private {
	struct crypto_shash *crc32_tfm;
	u8 *iv_seed;
	u8 *whitening;
};

L
Linus Torvalds 已提交
107 108 109 110
/*
 * Crypt: maps a linear range of a block device
 * and encrypts / decrypts at the same time.
 */
111
enum flags { DM_CRYPT_SUSPENDED, DM_CRYPT_KEY_VALID };
112 113 114 115 116 117 118 119 120 121 122 123

/*
 * Duplicated per-CPU state for cipher.
 */
struct crypt_cpu {
	struct ablkcipher_request *req;
};

/*
 * The fields in here must be read only after initialization,
 * changing state should be in crypt_cpu.
 */
L
Linus Torvalds 已提交
124 125 126 127 128
struct crypt_config {
	struct dm_dev *dev;
	sector_t start;

	/*
129 130
	 * pool for per bio private data, crypto requests and
	 * encryption requeusts/buffer pages
L
Linus Torvalds 已提交
131 132
	 */
	mempool_t *io_pool;
133
	mempool_t *req_pool;
L
Linus Torvalds 已提交
134
	mempool_t *page_pool;
135
	struct bio_set *bs;
L
Linus Torvalds 已提交
136

137 138
	struct workqueue_struct *io_queue;
	struct workqueue_struct *crypt_queue;
M
Milan Broz 已提交
139

M
Milan Broz 已提交
140
	char *cipher;
141
	char *cipher_string;
M
Milan Broz 已提交
142

L
Linus Torvalds 已提交
143
	struct crypt_iv_operations *iv_gen_ops;
144
	union {
145 146
		struct iv_essiv_private essiv;
		struct iv_benbi_private benbi;
147
		struct iv_lmk_private lmk;
148
		struct iv_tcw_private tcw;
149
	} iv_gen_private;
L
Linus Torvalds 已提交
150 151 152
	sector_t iv_offset;
	unsigned int iv_size;

153 154 155 156 157
	/*
	 * Duplicated per cpu state. Access through
	 * per_cpu_ptr() only.
	 */
	struct crypt_cpu __percpu *cpu;
158 159 160 161

	/* ESSIV: struct crypto_cipher *essiv_tfm */
	void *iv_private;
	struct crypto_ablkcipher **tfms;
162
	unsigned tfms_count;
163

164 165 166 167 168 169 170 171 172 173 174 175 176 177 178
	/*
	 * Layout of each crypto request:
	 *
	 *   struct ablkcipher_request
	 *      context
	 *      padding
	 *   struct dm_crypt_request
	 *      padding
	 *   IV
	 *
	 * The padding is added so that dm_crypt_request and the IV are
	 * correctly aligned.
	 */
	unsigned int dmreq_start;

179
	unsigned long flags;
L
Linus Torvalds 已提交
180
	unsigned int key_size;
181 182
	unsigned int key_parts;      /* independent parts in key buffer */
	unsigned int key_extra_size; /* additional keys length */
L
Linus Torvalds 已提交
183 184 185
	u8 key[0];
};

186
#define MIN_IOS        16
L
Linus Torvalds 已提交
187 188
#define MIN_POOL_PAGES 32

189
static struct kmem_cache *_crypt_io_pool;
L
Linus Torvalds 已提交
190

191
static void clone_init(struct dm_crypt_io *, struct bio *);
192
static void kcryptd_queue_crypt(struct dm_crypt_io *io);
193
static u8 *iv_of_dmreq(struct crypt_config *cc, struct dm_crypt_request *dmreq);
194

195 196 197 198 199 200 201 202 203 204
static struct crypt_cpu *this_crypt_config(struct crypt_config *cc)
{
	return this_cpu_ptr(cc->cpu);
}

/*
 * Use this to access cipher attributes that are the same for each CPU.
 */
static struct crypto_ablkcipher *any_tfm(struct crypt_config *cc)
{
205
	return cc->tfms[0];
206 207
}

L
Linus Torvalds 已提交
208 209 210
/*
 * Different IV generation algorithms:
 *
211
 * plain: the initial vector is the 32-bit little-endian version of the sector
212
 *        number, padded with zeros if necessary.
L
Linus Torvalds 已提交
213
 *
M
Milan Broz 已提交
214 215 216
 * plain64: the initial vector is the 64-bit little-endian version of the sector
 *        number, padded with zeros if necessary.
 *
217 218 219
 * essiv: "encrypted sector|salt initial vector", the sector number is
 *        encrypted with the bulk cipher using a salt as key. The salt
 *        should be derived from the bulk cipher's key via hashing.
L
Linus Torvalds 已提交
220
 *
221 222 223
 * benbi: the 64-bit "big-endian 'narrow block'-count", starting at 1
 *        (needed for LRW-32-AES and possible other narrow block modes)
 *
L
Ludwig Nussel 已提交
224 225 226
 * null: the initial vector is always zero.  Provides compatibility with
 *       obsolete loop_fish2 devices.  Do not use for new devices.
 *
227 228 229 230 231 232 233 234 235 236 237 238 239 240
 * lmk:  Compatible implementation of the block chaining mode used
 *       by the Loop-AES block device encryption system
 *       designed by Jari Ruusu. See http://loop-aes.sourceforge.net/
 *       It operates on full 512 byte sectors and uses CBC
 *       with an IV derived from the sector number, the data and
 *       optionally extra IV seed.
 *       This means that after decryption the first block
 *       of sector must be tweaked according to decrypted data.
 *       Loop-AES can use three encryption schemes:
 *         version 1: is plain aes-cbc mode
 *         version 2: uses 64 multikey scheme with lmk IV generator
 *         version 3: the same as version 2 with additional IV seed
 *                   (it uses 65 keys, last key is used as IV seed)
 *
241 242 243 244 245 246 247 248 249 250
 * tcw:  Compatible implementation of the block chaining mode used
 *       by the TrueCrypt device encryption system (prior to version 4.1).
 *       For more info see: http://www.truecrypt.org
 *       It operates on full 512 byte sectors and uses CBC
 *       with an IV derived from initial key and the sector number.
 *       In addition, whitening value is applied on every sector, whitening
 *       is calculated from initial key, sector number and mixed using CRC32.
 *       Note that this encryption scheme is vulnerable to watermarking attacks
 *       and should be used for old compatible containers access only.
 *
L
Linus Torvalds 已提交
251 252 253 254
 * plumb: unimplemented, see:
 * http://article.gmane.org/gmane.linux.kernel.device-mapper.dm-crypt/454
 */

255 256
static int crypt_iv_plain_gen(struct crypt_config *cc, u8 *iv,
			      struct dm_crypt_request *dmreq)
L
Linus Torvalds 已提交
257 258
{
	memset(iv, 0, cc->iv_size);
259
	*(__le32 *)iv = cpu_to_le32(dmreq->iv_sector & 0xffffffff);
L
Linus Torvalds 已提交
260 261 262 263

	return 0;
}

M
Milan Broz 已提交
264
static int crypt_iv_plain64_gen(struct crypt_config *cc, u8 *iv,
265
				struct dm_crypt_request *dmreq)
M
Milan Broz 已提交
266 267
{
	memset(iv, 0, cc->iv_size);
268
	*(__le64 *)iv = cpu_to_le64(dmreq->iv_sector);
M
Milan Broz 已提交
269 270 271 272

	return 0;
}

273 274 275 276 277 278
/* Initialise ESSIV - compute salt but no local memory allocations */
static int crypt_iv_essiv_init(struct crypt_config *cc)
{
	struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv;
	struct hash_desc desc;
	struct scatterlist sg;
279
	struct crypto_cipher *essiv_tfm;
280
	int err;
281 282 283 284 285 286 287 288 289

	sg_init_one(&sg, cc->key, cc->key_size);
	desc.tfm = essiv->hash_tfm;
	desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP;

	err = crypto_hash_digest(&desc, &sg, cc->key_size, essiv->salt);
	if (err)
		return err;

290
	essiv_tfm = cc->iv_private;
291

292 293 294 295
	err = crypto_cipher_setkey(essiv_tfm, essiv->salt,
			    crypto_hash_digestsize(essiv->hash_tfm));
	if (err)
		return err;
296 297

	return 0;
298 299
}

300 301 302 303 304
/* Wipe salt and reset key derived from volume key */
static int crypt_iv_essiv_wipe(struct crypt_config *cc)
{
	struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv;
	unsigned salt_size = crypto_hash_digestsize(essiv->hash_tfm);
305
	struct crypto_cipher *essiv_tfm;
306
	int r, err = 0;
307 308 309

	memset(essiv->salt, 0, salt_size);

310 311 312 313
	essiv_tfm = cc->iv_private;
	r = crypto_cipher_setkey(essiv_tfm, essiv->salt, salt_size);
	if (r)
		err = r;
314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348

	return err;
}

/* Set up per cpu cipher state */
static struct crypto_cipher *setup_essiv_cpu(struct crypt_config *cc,
					     struct dm_target *ti,
					     u8 *salt, unsigned saltsize)
{
	struct crypto_cipher *essiv_tfm;
	int err;

	/* Setup the essiv_tfm with the given salt */
	essiv_tfm = crypto_alloc_cipher(cc->cipher, 0, CRYPTO_ALG_ASYNC);
	if (IS_ERR(essiv_tfm)) {
		ti->error = "Error allocating crypto tfm for ESSIV";
		return essiv_tfm;
	}

	if (crypto_cipher_blocksize(essiv_tfm) !=
	    crypto_ablkcipher_ivsize(any_tfm(cc))) {
		ti->error = "Block size of ESSIV cipher does "
			    "not match IV size of block cipher";
		crypto_free_cipher(essiv_tfm);
		return ERR_PTR(-EINVAL);
	}

	err = crypto_cipher_setkey(essiv_tfm, salt, saltsize);
	if (err) {
		ti->error = "Failed to set key for ESSIV cipher";
		crypto_free_cipher(essiv_tfm);
		return ERR_PTR(err);
	}

	return essiv_tfm;
349 350
}

351 352
static void crypt_iv_essiv_dtr(struct crypt_config *cc)
{
353
	struct crypto_cipher *essiv_tfm;
354 355
	struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv;

356 357 358 359 360
	crypto_free_hash(essiv->hash_tfm);
	essiv->hash_tfm = NULL;

	kzfree(essiv->salt);
	essiv->salt = NULL;
361

362
	essiv_tfm = cc->iv_private;
363

364 365
	if (essiv_tfm)
		crypto_free_cipher(essiv_tfm);
366

367
	cc->iv_private = NULL;
368 369
}

L
Linus Torvalds 已提交
370
static int crypt_iv_essiv_ctr(struct crypt_config *cc, struct dm_target *ti,
M
Milan Broz 已提交
371
			      const char *opts)
L
Linus Torvalds 已提交
372
{
373 374 375
	struct crypto_cipher *essiv_tfm = NULL;
	struct crypto_hash *hash_tfm = NULL;
	u8 *salt = NULL;
376
	int err;
L
Linus Torvalds 已提交
377

378
	if (!opts) {
379
		ti->error = "Digest algorithm missing for ESSIV mode";
L
Linus Torvalds 已提交
380 381 382
		return -EINVAL;
	}

383
	/* Allocate hash algorithm */
384 385
	hash_tfm = crypto_alloc_hash(opts, 0, CRYPTO_ALG_ASYNC);
	if (IS_ERR(hash_tfm)) {
386
		ti->error = "Error initializing ESSIV hash";
387 388
		err = PTR_ERR(hash_tfm);
		goto bad;
L
Linus Torvalds 已提交
389 390
	}

391
	salt = kzalloc(crypto_hash_digestsize(hash_tfm), GFP_KERNEL);
392
	if (!salt) {
393
		ti->error = "Error kmallocing salt storage in ESSIV";
394 395
		err = -ENOMEM;
		goto bad;
L
Linus Torvalds 已提交
396 397
	}

398 399 400
	cc->iv_gen_private.essiv.salt = salt;
	cc->iv_gen_private.essiv.hash_tfm = hash_tfm;

401 402 403 404 405
	essiv_tfm = setup_essiv_cpu(cc, ti, salt,
				crypto_hash_digestsize(hash_tfm));
	if (IS_ERR(essiv_tfm)) {
		crypt_iv_essiv_dtr(cc);
		return PTR_ERR(essiv_tfm);
406
	}
407
	cc->iv_private = essiv_tfm;
408

L
Linus Torvalds 已提交
409
	return 0;
410 411 412 413

bad:
	if (hash_tfm && !IS_ERR(hash_tfm))
		crypto_free_hash(hash_tfm);
414
	kfree(salt);
415
	return err;
L
Linus Torvalds 已提交
416 417
}

418 419
static int crypt_iv_essiv_gen(struct crypt_config *cc, u8 *iv,
			      struct dm_crypt_request *dmreq)
L
Linus Torvalds 已提交
420
{
421
	struct crypto_cipher *essiv_tfm = cc->iv_private;
422

L
Linus Torvalds 已提交
423
	memset(iv, 0, cc->iv_size);
424
	*(__le64 *)iv = cpu_to_le64(dmreq->iv_sector);
425 426
	crypto_cipher_encrypt_one(essiv_tfm, iv, iv);

L
Linus Torvalds 已提交
427 428 429
	return 0;
}

430 431 432
static int crypt_iv_benbi_ctr(struct crypt_config *cc, struct dm_target *ti,
			      const char *opts)
{
433
	unsigned bs = crypto_ablkcipher_blocksize(any_tfm(cc));
434
	int log = ilog2(bs);
435 436 437 438 439 440 441 442 443 444 445 446 447 448

	/* we need to calculate how far we must shift the sector count
	 * to get the cipher block count, we use this shift in _gen */

	if (1 << log != bs) {
		ti->error = "cypher blocksize is not a power of 2";
		return -EINVAL;
	}

	if (log > 9) {
		ti->error = "cypher blocksize is > 512";
		return -EINVAL;
	}

449
	cc->iv_gen_private.benbi.shift = 9 - log;
450 451 452 453 454 455 456 457

	return 0;
}

static void crypt_iv_benbi_dtr(struct crypt_config *cc)
{
}

458 459
static int crypt_iv_benbi_gen(struct crypt_config *cc, u8 *iv,
			      struct dm_crypt_request *dmreq)
460
{
461 462
	__be64 val;

463
	memset(iv, 0, cc->iv_size - sizeof(u64)); /* rest is cleared below */
464

465
	val = cpu_to_be64(((u64)dmreq->iv_sector << cc->iv_gen_private.benbi.shift) + 1);
466
	put_unaligned(val, (__be64 *)(iv + cc->iv_size - sizeof(u64)));
467

L
Linus Torvalds 已提交
468 469 470
	return 0;
}

471 472
static int crypt_iv_null_gen(struct crypt_config *cc, u8 *iv,
			     struct dm_crypt_request *dmreq)
L
Ludwig Nussel 已提交
473 474 475 476 477 478
{
	memset(iv, 0, cc->iv_size);

	return 0;
}

479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550
static void crypt_iv_lmk_dtr(struct crypt_config *cc)
{
	struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;

	if (lmk->hash_tfm && !IS_ERR(lmk->hash_tfm))
		crypto_free_shash(lmk->hash_tfm);
	lmk->hash_tfm = NULL;

	kzfree(lmk->seed);
	lmk->seed = NULL;
}

static int crypt_iv_lmk_ctr(struct crypt_config *cc, struct dm_target *ti,
			    const char *opts)
{
	struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;

	lmk->hash_tfm = crypto_alloc_shash("md5", 0, 0);
	if (IS_ERR(lmk->hash_tfm)) {
		ti->error = "Error initializing LMK hash";
		return PTR_ERR(lmk->hash_tfm);
	}

	/* No seed in LMK version 2 */
	if (cc->key_parts == cc->tfms_count) {
		lmk->seed = NULL;
		return 0;
	}

	lmk->seed = kzalloc(LMK_SEED_SIZE, GFP_KERNEL);
	if (!lmk->seed) {
		crypt_iv_lmk_dtr(cc);
		ti->error = "Error kmallocing seed storage in LMK";
		return -ENOMEM;
	}

	return 0;
}

static int crypt_iv_lmk_init(struct crypt_config *cc)
{
	struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
	int subkey_size = cc->key_size / cc->key_parts;

	/* LMK seed is on the position of LMK_KEYS + 1 key */
	if (lmk->seed)
		memcpy(lmk->seed, cc->key + (cc->tfms_count * subkey_size),
		       crypto_shash_digestsize(lmk->hash_tfm));

	return 0;
}

static int crypt_iv_lmk_wipe(struct crypt_config *cc)
{
	struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;

	if (lmk->seed)
		memset(lmk->seed, 0, LMK_SEED_SIZE);

	return 0;
}

static int crypt_iv_lmk_one(struct crypt_config *cc, u8 *iv,
			    struct dm_crypt_request *dmreq,
			    u8 *data)
{
	struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
	struct {
		struct shash_desc desc;
		char ctx[crypto_shash_descsize(lmk->hash_tfm)];
	} sdesc;
	struct md5_state md5state;
551
	__le32 buf[4];
552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599
	int i, r;

	sdesc.desc.tfm = lmk->hash_tfm;
	sdesc.desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP;

	r = crypto_shash_init(&sdesc.desc);
	if (r)
		return r;

	if (lmk->seed) {
		r = crypto_shash_update(&sdesc.desc, lmk->seed, LMK_SEED_SIZE);
		if (r)
			return r;
	}

	/* Sector is always 512B, block size 16, add data of blocks 1-31 */
	r = crypto_shash_update(&sdesc.desc, data + 16, 16 * 31);
	if (r)
		return r;

	/* Sector is cropped to 56 bits here */
	buf[0] = cpu_to_le32(dmreq->iv_sector & 0xFFFFFFFF);
	buf[1] = cpu_to_le32((((u64)dmreq->iv_sector >> 32) & 0x00FFFFFF) | 0x80000000);
	buf[2] = cpu_to_le32(4024);
	buf[3] = 0;
	r = crypto_shash_update(&sdesc.desc, (u8 *)buf, sizeof(buf));
	if (r)
		return r;

	/* No MD5 padding here */
	r = crypto_shash_export(&sdesc.desc, &md5state);
	if (r)
		return r;

	for (i = 0; i < MD5_HASH_WORDS; i++)
		__cpu_to_le32s(&md5state.hash[i]);
	memcpy(iv, &md5state.hash, cc->iv_size);

	return 0;
}

static int crypt_iv_lmk_gen(struct crypt_config *cc, u8 *iv,
			    struct dm_crypt_request *dmreq)
{
	u8 *src;
	int r = 0;

	if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) {
600
		src = kmap_atomic(sg_page(&dmreq->sg_in));
601
		r = crypt_iv_lmk_one(cc, iv, dmreq, src + dmreq->sg_in.offset);
602
		kunmap_atomic(src);
603 604 605 606 607 608 609 610 611 612 613 614 615 616 617
	} else
		memset(iv, 0, cc->iv_size);

	return r;
}

static int crypt_iv_lmk_post(struct crypt_config *cc, u8 *iv,
			     struct dm_crypt_request *dmreq)
{
	u8 *dst;
	int r;

	if (bio_data_dir(dmreq->ctx->bio_in) == WRITE)
		return 0;

618
	dst = kmap_atomic(sg_page(&dmreq->sg_out));
619 620 621 622 623 624
	r = crypt_iv_lmk_one(cc, iv, dmreq, dst + dmreq->sg_out.offset);

	/* Tweak the first block of plaintext sector */
	if (!r)
		crypto_xor(dst + dmreq->sg_out.offset, iv, cc->iv_size);

625
	kunmap_atomic(dst);
626 627 628
	return r;
}

629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775
static void crypt_iv_tcw_dtr(struct crypt_config *cc)
{
	struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;

	kzfree(tcw->iv_seed);
	tcw->iv_seed = NULL;
	kzfree(tcw->whitening);
	tcw->whitening = NULL;

	if (tcw->crc32_tfm && !IS_ERR(tcw->crc32_tfm))
		crypto_free_shash(tcw->crc32_tfm);
	tcw->crc32_tfm = NULL;
}

static int crypt_iv_tcw_ctr(struct crypt_config *cc, struct dm_target *ti,
			    const char *opts)
{
	struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;

	if (cc->key_size <= (cc->iv_size + TCW_WHITENING_SIZE)) {
		ti->error = "Wrong key size for TCW";
		return -EINVAL;
	}

	tcw->crc32_tfm = crypto_alloc_shash("crc32", 0, 0);
	if (IS_ERR(tcw->crc32_tfm)) {
		ti->error = "Error initializing CRC32 in TCW";
		return PTR_ERR(tcw->crc32_tfm);
	}

	tcw->iv_seed = kzalloc(cc->iv_size, GFP_KERNEL);
	tcw->whitening = kzalloc(TCW_WHITENING_SIZE, GFP_KERNEL);
	if (!tcw->iv_seed || !tcw->whitening) {
		crypt_iv_tcw_dtr(cc);
		ti->error = "Error allocating seed storage in TCW";
		return -ENOMEM;
	}

	return 0;
}

static int crypt_iv_tcw_init(struct crypt_config *cc)
{
	struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
	int key_offset = cc->key_size - cc->iv_size - TCW_WHITENING_SIZE;

	memcpy(tcw->iv_seed, &cc->key[key_offset], cc->iv_size);
	memcpy(tcw->whitening, &cc->key[key_offset + cc->iv_size],
	       TCW_WHITENING_SIZE);

	return 0;
}

static int crypt_iv_tcw_wipe(struct crypt_config *cc)
{
	struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;

	memset(tcw->iv_seed, 0, cc->iv_size);
	memset(tcw->whitening, 0, TCW_WHITENING_SIZE);

	return 0;
}

static int crypt_iv_tcw_whitening(struct crypt_config *cc,
				  struct dm_crypt_request *dmreq,
				  u8 *data)
{
	struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
	u64 sector = cpu_to_le64((u64)dmreq->iv_sector);
	u8 buf[TCW_WHITENING_SIZE];
	struct {
		struct shash_desc desc;
		char ctx[crypto_shash_descsize(tcw->crc32_tfm)];
	} sdesc;
	int i, r;

	/* xor whitening with sector number */
	memcpy(buf, tcw->whitening, TCW_WHITENING_SIZE);
	crypto_xor(buf, (u8 *)&sector, 8);
	crypto_xor(&buf[8], (u8 *)&sector, 8);

	/* calculate crc32 for every 32bit part and xor it */
	sdesc.desc.tfm = tcw->crc32_tfm;
	sdesc.desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP;
	for (i = 0; i < 4; i++) {
		r = crypto_shash_init(&sdesc.desc);
		if (r)
			goto out;
		r = crypto_shash_update(&sdesc.desc, &buf[i * 4], 4);
		if (r)
			goto out;
		r = crypto_shash_final(&sdesc.desc, &buf[i * 4]);
		if (r)
			goto out;
	}
	crypto_xor(&buf[0], &buf[12], 4);
	crypto_xor(&buf[4], &buf[8], 4);

	/* apply whitening (8 bytes) to whole sector */
	for (i = 0; i < ((1 << SECTOR_SHIFT) / 8); i++)
		crypto_xor(data + i * 8, buf, 8);
out:
	memset(buf, 0, sizeof(buf));
	return r;
}

static int crypt_iv_tcw_gen(struct crypt_config *cc, u8 *iv,
			    struct dm_crypt_request *dmreq)
{
	struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
	u64 sector = cpu_to_le64((u64)dmreq->iv_sector);
	u8 *src;
	int r = 0;

	/* Remove whitening from ciphertext */
	if (bio_data_dir(dmreq->ctx->bio_in) != WRITE) {
		src = kmap_atomic(sg_page(&dmreq->sg_in));
		r = crypt_iv_tcw_whitening(cc, dmreq, src + dmreq->sg_in.offset);
		kunmap_atomic(src);
	}

	/* Calculate IV */
	memcpy(iv, tcw->iv_seed, cc->iv_size);
	crypto_xor(iv, (u8 *)&sector, 8);
	if (cc->iv_size > 8)
		crypto_xor(&iv[8], (u8 *)&sector, cc->iv_size - 8);

	return r;
}

static int crypt_iv_tcw_post(struct crypt_config *cc, u8 *iv,
			     struct dm_crypt_request *dmreq)
{
	u8 *dst;
	int r;

	if (bio_data_dir(dmreq->ctx->bio_in) != WRITE)
		return 0;

	/* Apply whitening on ciphertext */
	dst = kmap_atomic(sg_page(&dmreq->sg_out));
	r = crypt_iv_tcw_whitening(cc, dmreq, dst + dmreq->sg_out.offset);
	kunmap_atomic(dst);

	return r;
}

L
Linus Torvalds 已提交
776 777 778 779
static struct crypt_iv_operations crypt_iv_plain_ops = {
	.generator = crypt_iv_plain_gen
};

M
Milan Broz 已提交
780 781 782 783
static struct crypt_iv_operations crypt_iv_plain64_ops = {
	.generator = crypt_iv_plain64_gen
};

L
Linus Torvalds 已提交
784 785 786
static struct crypt_iv_operations crypt_iv_essiv_ops = {
	.ctr       = crypt_iv_essiv_ctr,
	.dtr       = crypt_iv_essiv_dtr,
787
	.init      = crypt_iv_essiv_init,
788
	.wipe      = crypt_iv_essiv_wipe,
L
Linus Torvalds 已提交
789 790 791
	.generator = crypt_iv_essiv_gen
};

792 793 794 795 796
static struct crypt_iv_operations crypt_iv_benbi_ops = {
	.ctr	   = crypt_iv_benbi_ctr,
	.dtr	   = crypt_iv_benbi_dtr,
	.generator = crypt_iv_benbi_gen
};
L
Linus Torvalds 已提交
797

L
Ludwig Nussel 已提交
798 799 800 801
static struct crypt_iv_operations crypt_iv_null_ops = {
	.generator = crypt_iv_null_gen
};

802 803 804 805 806 807 808 809 810
static struct crypt_iv_operations crypt_iv_lmk_ops = {
	.ctr	   = crypt_iv_lmk_ctr,
	.dtr	   = crypt_iv_lmk_dtr,
	.init	   = crypt_iv_lmk_init,
	.wipe	   = crypt_iv_lmk_wipe,
	.generator = crypt_iv_lmk_gen,
	.post	   = crypt_iv_lmk_post
};

811 812 813 814 815 816 817 818 819
static struct crypt_iv_operations crypt_iv_tcw_ops = {
	.ctr	   = crypt_iv_tcw_ctr,
	.dtr	   = crypt_iv_tcw_dtr,
	.init	   = crypt_iv_tcw_init,
	.wipe	   = crypt_iv_tcw_wipe,
	.generator = crypt_iv_tcw_gen,
	.post	   = crypt_iv_tcw_post
};

M
Milan Broz 已提交
820 821 822
static void crypt_convert_init(struct crypt_config *cc,
			       struct convert_context *ctx,
			       struct bio *bio_out, struct bio *bio_in,
823
			       sector_t sector)
L
Linus Torvalds 已提交
824 825 826
{
	ctx->bio_in = bio_in;
	ctx->bio_out = bio_out;
827 828 829 830
	if (bio_in)
		ctx->iter_in = bio_in->bi_iter;
	if (bio_out)
		ctx->iter_out = bio_out->bi_iter;
831
	ctx->cc_sector = sector + cc->iv_offset;
832
	init_completion(&ctx->restart);
L
Linus Torvalds 已提交
833 834
}

835 836 837 838 839 840 841 842 843 844 845 846
static struct dm_crypt_request *dmreq_of_req(struct crypt_config *cc,
					     struct ablkcipher_request *req)
{
	return (struct dm_crypt_request *)((char *)req + cc->dmreq_start);
}

static struct ablkcipher_request *req_of_dmreq(struct crypt_config *cc,
					       struct dm_crypt_request *dmreq)
{
	return (struct ablkcipher_request *)((char *)dmreq - cc->dmreq_start);
}

847 848 849 850 851 852 853
static u8 *iv_of_dmreq(struct crypt_config *cc,
		       struct dm_crypt_request *dmreq)
{
	return (u8 *)ALIGN((unsigned long)(dmreq + 1),
		crypto_ablkcipher_alignmask(any_tfm(cc)) + 1);
}

854
static int crypt_convert_block(struct crypt_config *cc,
M
Milan Broz 已提交
855 856
			       struct convert_context *ctx,
			       struct ablkcipher_request *req)
857
{
858 859
	struct bio_vec bv_in = bio_iter_iovec(ctx->bio_in, ctx->iter_in);
	struct bio_vec bv_out = bio_iter_iovec(ctx->bio_out, ctx->iter_out);
M
Milan Broz 已提交
860 861
	struct dm_crypt_request *dmreq;
	u8 *iv;
862
	int r;
M
Milan Broz 已提交
863

864
	dmreq = dmreq_of_req(cc, req);
865
	iv = iv_of_dmreq(cc, dmreq);
866

867
	dmreq->iv_sector = ctx->cc_sector;
868
	dmreq->ctx = ctx;
M
Milan Broz 已提交
869
	sg_init_table(&dmreq->sg_in, 1);
870 871
	sg_set_page(&dmreq->sg_in, bv_in.bv_page, 1 << SECTOR_SHIFT,
		    bv_in.bv_offset);
872

M
Milan Broz 已提交
873
	sg_init_table(&dmreq->sg_out, 1);
874 875
	sg_set_page(&dmreq->sg_out, bv_out.bv_page, 1 << SECTOR_SHIFT,
		    bv_out.bv_offset);
876

877 878
	bio_advance_iter(ctx->bio_in, &ctx->iter_in, 1 << SECTOR_SHIFT);
	bio_advance_iter(ctx->bio_out, &ctx->iter_out, 1 << SECTOR_SHIFT);
879

M
Milan Broz 已提交
880
	if (cc->iv_gen_ops) {
881
		r = cc->iv_gen_ops->generator(cc, iv, dmreq);
M
Milan Broz 已提交
882 883 884 885 886 887 888 889 890 891 892 893
		if (r < 0)
			return r;
	}

	ablkcipher_request_set_crypt(req, &dmreq->sg_in, &dmreq->sg_out,
				     1 << SECTOR_SHIFT, iv);

	if (bio_data_dir(ctx->bio_in) == WRITE)
		r = crypto_ablkcipher_encrypt(req);
	else
		r = crypto_ablkcipher_decrypt(req);

894 895 896
	if (!r && cc->iv_gen_ops && cc->iv_gen_ops->post)
		r = cc->iv_gen_ops->post(cc, iv, dmreq);

M
Milan Broz 已提交
897
	return r;
898 899
}

900 901
static void kcryptd_async_done(struct crypto_async_request *async_req,
			       int error);
902

903 904 905
static void crypt_alloc_req(struct crypt_config *cc,
			    struct convert_context *ctx)
{
906
	struct crypt_cpu *this_cc = this_crypt_config(cc);
907
	unsigned key_index = ctx->cc_sector & (cc->tfms_count - 1);
908 909 910 911

	if (!this_cc->req)
		this_cc->req = mempool_alloc(cc->req_pool, GFP_NOIO);

912
	ablkcipher_request_set_tfm(this_cc->req, cc->tfms[key_index]);
913 914 915
	ablkcipher_request_set_callback(this_cc->req,
	    CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP,
	    kcryptd_async_done, dmreq_of_req(cc, this_cc->req));
916 917
}

L
Linus Torvalds 已提交
918 919 920 921
/*
 * Encrypt / decrypt data from one bio to another one (can be the same one)
 */
static int crypt_convert(struct crypt_config *cc,
M
Milan Broz 已提交
922
			 struct convert_context *ctx)
L
Linus Torvalds 已提交
923
{
924
	struct crypt_cpu *this_cc = this_crypt_config(cc);
M
Milan Broz 已提交
925
	int r;
L
Linus Torvalds 已提交
926

927
	atomic_set(&ctx->cc_pending, 1);
M
Milan Broz 已提交
928

929
	while (ctx->iter_in.bi_size && ctx->iter_out.bi_size) {
L
Linus Torvalds 已提交
930

M
Milan Broz 已提交
931 932
		crypt_alloc_req(cc, ctx);

933
		atomic_inc(&ctx->cc_pending);
M
Milan Broz 已提交
934

935
		r = crypt_convert_block(cc, ctx, this_cc->req);
M
Milan Broz 已提交
936 937

		switch (r) {
M
Milan Broz 已提交
938
		/* async */
M
Milan Broz 已提交
939 940
		case -EBUSY:
			wait_for_completion(&ctx->restart);
941
			reinit_completion(&ctx->restart);
M
Milan Broz 已提交
942 943
			/* fall through*/
		case -EINPROGRESS:
944
			this_cc->req = NULL;
945
			ctx->cc_sector++;
M
Milan Broz 已提交
946 947 948
			continue;

		/* sync */
M
Milan Broz 已提交
949
		case 0:
950
			atomic_dec(&ctx->cc_pending);
951
			ctx->cc_sector++;
M
Milan Broz 已提交
952
			cond_resched();
M
Milan Broz 已提交
953 954
			continue;

M
Milan Broz 已提交
955 956
		/* error */
		default:
957
			atomic_dec(&ctx->cc_pending);
M
Milan Broz 已提交
958 959
			return r;
		}
L
Linus Torvalds 已提交
960 961
	}

M
Milan Broz 已提交
962
	return 0;
L
Linus Torvalds 已提交
963 964 965 966 967
}

/*
 * Generate a new unfragmented bio with the given size
 * This should never violate the device limitations
968 969
 * May return a smaller bio when running out of pages, indicated by
 * *out_of_pages set to 1.
L
Linus Torvalds 已提交
970
 */
971 972
static struct bio *crypt_alloc_buffer(struct dm_crypt_io *io, unsigned size,
				      unsigned *out_of_pages)
L
Linus Torvalds 已提交
973
{
974
	struct crypt_config *cc = io->cc;
975
	struct bio *clone;
L
Linus Torvalds 已提交
976
	unsigned int nr_iovecs = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
977
	gfp_t gfp_mask = GFP_NOIO | __GFP_HIGHMEM;
M
Milan Broz 已提交
978 979
	unsigned i, len;
	struct page *page;
L
Linus Torvalds 已提交
980

981
	clone = bio_alloc_bioset(GFP_NOIO, nr_iovecs, cc->bs);
982
	if (!clone)
L
Linus Torvalds 已提交
983 984
		return NULL;

985
	clone_init(io, clone);
986
	*out_of_pages = 0;
987

988
	for (i = 0; i < nr_iovecs; i++) {
M
Milan Broz 已提交
989
		page = mempool_alloc(cc->page_pool, gfp_mask);
990 991
		if (!page) {
			*out_of_pages = 1;
L
Linus Torvalds 已提交
992
			break;
993
		}
L
Linus Torvalds 已提交
994 995

		/*
996 997 998
		 * If additional pages cannot be allocated without waiting,
		 * return a partially-allocated bio.  The caller will then try
		 * to allocate more bios while submitting this partial bio.
L
Linus Torvalds 已提交
999
		 */
1000
		gfp_mask = (gfp_mask | __GFP_NOWARN) & ~__GFP_WAIT;
L
Linus Torvalds 已提交
1001

M
Milan Broz 已提交
1002 1003 1004 1005 1006 1007
		len = (size > PAGE_SIZE) ? PAGE_SIZE : size;

		if (!bio_add_page(clone, page, len, 0)) {
			mempool_free(page, cc->page_pool);
			break;
		}
L
Linus Torvalds 已提交
1008

M
Milan Broz 已提交
1009
		size -= len;
L
Linus Torvalds 已提交
1010 1011
	}

1012
	if (!clone->bi_iter.bi_size) {
1013
		bio_put(clone);
L
Linus Torvalds 已提交
1014 1015 1016
		return NULL;
	}

1017
	return clone;
L
Linus Torvalds 已提交
1018 1019
}

N
Neil Brown 已提交
1020
static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone)
L
Linus Torvalds 已提交
1021
{
N
Neil Brown 已提交
1022
	unsigned int i;
L
Linus Torvalds 已提交
1023 1024
	struct bio_vec *bv;

1025
	bio_for_each_segment_all(bv, clone, i) {
L
Linus Torvalds 已提交
1026 1027 1028 1029 1030 1031
		BUG_ON(!bv->bv_page);
		mempool_free(bv->bv_page, cc->page_pool);
		bv->bv_page = NULL;
	}
}

1032
static struct dm_crypt_io *crypt_io_alloc(struct crypt_config *cc,
M
Milan Broz 已提交
1033 1034 1035 1036 1037
					  struct bio *bio, sector_t sector)
{
	struct dm_crypt_io *io;

	io = mempool_alloc(cc->io_pool, GFP_NOIO);
1038
	io->cc = cc;
M
Milan Broz 已提交
1039 1040 1041
	io->base_bio = bio;
	io->sector = sector;
	io->error = 0;
M
Milan Broz 已提交
1042
	io->base_io = NULL;
1043
	atomic_set(&io->io_pending, 0);
M
Milan Broz 已提交
1044 1045 1046 1047

	return io;
}

M
Milan Broz 已提交
1048 1049
static void crypt_inc_pending(struct dm_crypt_io *io)
{
1050
	atomic_inc(&io->io_pending);
M
Milan Broz 已提交
1051 1052
}

L
Linus Torvalds 已提交
1053 1054 1055
/*
 * One of the bios was finished. Check for completion of
 * the whole request and correctly clean up the buffer.
M
Milan Broz 已提交
1056
 * If base_io is set, wait for the last fragment to complete.
L
Linus Torvalds 已提交
1057
 */
1058
static void crypt_dec_pending(struct dm_crypt_io *io)
L
Linus Torvalds 已提交
1059
{
1060
	struct crypt_config *cc = io->cc;
1061 1062 1063
	struct bio *base_bio = io->base_bio;
	struct dm_crypt_io *base_io = io->base_io;
	int error = io->error;
L
Linus Torvalds 已提交
1064

1065
	if (!atomic_dec_and_test(&io->io_pending))
L
Linus Torvalds 已提交
1066 1067
		return;

1068 1069 1070 1071
	mempool_free(io, cc->io_pool);

	if (likely(!base_io))
		bio_endio(base_bio, error);
M
Milan Broz 已提交
1072
	else {
1073 1074 1075
		if (error && !base_io->error)
			base_io->error = error;
		crypt_dec_pending(base_io);
M
Milan Broz 已提交
1076
	}
L
Linus Torvalds 已提交
1077 1078 1079
}

/*
1080
 * kcryptd/kcryptd_io:
L
Linus Torvalds 已提交
1081 1082
 *
 * Needed because it would be very unwise to do decryption in an
1083
 * interrupt context.
1084 1085 1086 1087 1088 1089 1090 1091
 *
 * kcryptd performs the actual encryption or decryption.
 *
 * kcryptd_io performs the IO submission.
 *
 * They must be separated as otherwise the final stages could be
 * starved by new requests which can block in the first stages due
 * to memory allocation.
1092 1093 1094
 *
 * The work is done per CPU global for all dm-crypt instances.
 * They should not depend on each other and do not block.
L
Linus Torvalds 已提交
1095
 */
1096
static void crypt_endio(struct bio *clone, int error)
1097
{
1098
	struct dm_crypt_io *io = clone->bi_private;
1099
	struct crypt_config *cc = io->cc;
M
Milan Broz 已提交
1100
	unsigned rw = bio_data_dir(clone);
1101

M
Milan Broz 已提交
1102 1103 1104
	if (unlikely(!bio_flagged(clone, BIO_UPTODATE) && !error))
		error = -EIO;

1105
	/*
1106
	 * free the processed pages
1107
	 */
M
Milan Broz 已提交
1108
	if (rw == WRITE)
N
Neil Brown 已提交
1109
		crypt_free_buffer_pages(cc, clone);
1110 1111 1112

	bio_put(clone);

M
Milan Broz 已提交
1113 1114 1115 1116
	if (rw == READ && !error) {
		kcryptd_queue_crypt(io);
		return;
	}
1117 1118 1119 1120 1121

	if (unlikely(error))
		io->error = error;

	crypt_dec_pending(io);
1122 1123
}

1124
static void clone_init(struct dm_crypt_io *io, struct bio *clone)
1125
{
1126
	struct crypt_config *cc = io->cc;
1127 1128 1129 1130 1131 1132 1133

	clone->bi_private = io;
	clone->bi_end_io  = crypt_endio;
	clone->bi_bdev    = cc->dev->bdev;
	clone->bi_rw      = io->base_bio->bi_rw;
}

1134
static int kcryptd_io_read(struct dm_crypt_io *io, gfp_t gfp)
1135
{
1136
	struct crypt_config *cc = io->cc;
1137 1138
	struct bio *base_bio = io->base_bio;
	struct bio *clone;
1139

1140 1141 1142 1143 1144
	/*
	 * The block layer might modify the bvec array, so always
	 * copy the required bvecs because we need the original
	 * one in order to decrypt the whole bio data *afterwards*.
	 */
1145
	clone = bio_clone_bioset(base_bio, gfp, cc->bs);
1146
	if (!clone)
1147
		return 1;
1148

1149 1150
	crypt_inc_pending(io);

1151
	clone_init(io, clone);
1152
	clone->bi_iter.bi_sector = cc->start + io->sector;
1153

1154
	generic_make_request(clone);
1155
	return 0;
1156 1157
}

1158 1159
static void kcryptd_io_write(struct dm_crypt_io *io)
{
1160 1161
	struct bio *clone = io->ctx.bio_out;
	generic_make_request(clone);
1162 1163
}

1164 1165 1166 1167
static void kcryptd_io(struct work_struct *work)
{
	struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);

1168 1169 1170 1171 1172 1173
	if (bio_data_dir(io->base_bio) == READ) {
		crypt_inc_pending(io);
		if (kcryptd_io_read(io, GFP_NOIO))
			io->error = -ENOMEM;
		crypt_dec_pending(io);
	} else
1174 1175 1176 1177 1178
		kcryptd_io_write(io);
}

static void kcryptd_queue_io(struct dm_crypt_io *io)
{
1179
	struct crypt_config *cc = io->cc;
1180 1181 1182 1183 1184

	INIT_WORK(&io->work, kcryptd_io);
	queue_work(cc->io_queue, &io->work);
}

1185
static void kcryptd_crypt_write_io_submit(struct dm_crypt_io *io, int async)
1186
{
1187
	struct bio *clone = io->ctx.bio_out;
1188
	struct crypt_config *cc = io->cc;
1189

1190
	if (unlikely(io->error < 0)) {
1191 1192
		crypt_free_buffer_pages(cc, clone);
		bio_put(clone);
1193
		crypt_dec_pending(io);
1194 1195 1196 1197
		return;
	}

	/* crypt_convert should have filled the clone bio */
1198
	BUG_ON(io->ctx.iter_out.bi_size);
1199

1200
	clone->bi_iter.bi_sector = cc->start + io->sector;
1201

1202 1203
	if (async)
		kcryptd_queue_io(io);
1204
	else
1205
		generic_make_request(clone);
1206 1207
}

1208
static void kcryptd_crypt_write_convert(struct dm_crypt_io *io)
1209
{
1210
	struct crypt_config *cc = io->cc;
1211
	struct bio *clone;
M
Milan Broz 已提交
1212
	struct dm_crypt_io *new_io;
M
Milan Broz 已提交
1213
	int crypt_finished;
1214
	unsigned out_of_pages = 0;
1215
	unsigned remaining = io->base_bio->bi_iter.bi_size;
M
Milan Broz 已提交
1216
	sector_t sector = io->sector;
1217
	int r;
1218

1219 1220 1221 1222
	/*
	 * Prevent io from disappearing until this function completes.
	 */
	crypt_inc_pending(io);
M
Milan Broz 已提交
1223
	crypt_convert_init(cc, &io->ctx, NULL, io->base_bio, sector);
1224

1225 1226 1227 1228 1229
	/*
	 * The allocated buffers can be smaller than the whole bio,
	 * so repeat the whole process until all the data can be handled.
	 */
	while (remaining) {
1230
		clone = crypt_alloc_buffer(io, remaining, &out_of_pages);
1231
		if (unlikely(!clone)) {
1232
			io->error = -ENOMEM;
1233
			break;
1234
		}
1235

1236
		io->ctx.bio_out = clone;
1237
		io->ctx.iter_out = clone->bi_iter;
1238

1239
		remaining -= clone->bi_iter.bi_size;
M
Milan Broz 已提交
1240
		sector += bio_sectors(clone);
1241

1242
		crypt_inc_pending(io);
1243

1244
		r = crypt_convert(cc, &io->ctx);
1245 1246 1247
		if (r < 0)
			io->error = -EIO;

1248
		crypt_finished = atomic_dec_and_test(&io->ctx.cc_pending);
1249

M
Milan Broz 已提交
1250 1251
		/* Encryption was already finished, submit io now */
		if (crypt_finished) {
1252
			kcryptd_crypt_write_io_submit(io, 0);
M
Milan Broz 已提交
1253 1254 1255 1256 1257

			/*
			 * If there was an error, do not try next fragments.
			 * For async, error is processed in async handler.
			 */
1258
			if (unlikely(r < 0))
1259
				break;
M
Milan Broz 已提交
1260 1261

			io->sector = sector;
1262
		}
1263

1264 1265 1266 1267 1268
		/*
		 * Out of memory -> run queues
		 * But don't wait if split was due to the io size restriction
		 */
		if (unlikely(out_of_pages))
1269
			congestion_wait(BLK_RW_ASYNC, HZ/100);
1270

M
Milan Broz 已提交
1271 1272 1273 1274 1275
		/*
		 * With async crypto it is unsafe to share the crypto context
		 * between fragments, so switch to a new dm_crypt_io structure.
		 */
		if (unlikely(!crypt_finished && remaining)) {
1276
			new_io = crypt_io_alloc(io->cc, io->base_bio,
M
Milan Broz 已提交
1277 1278 1279 1280
						sector);
			crypt_inc_pending(new_io);
			crypt_convert_init(cc, &new_io->ctx, NULL,
					   io->base_bio, sector);
1281
			new_io->ctx.iter_in = io->ctx.iter_in;
M
Milan Broz 已提交
1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296

			/*
			 * Fragments after the first use the base_io
			 * pending count.
			 */
			if (!io->base_io)
				new_io->base_io = io;
			else {
				new_io->base_io = io->base_io;
				crypt_inc_pending(io->base_io);
				crypt_dec_pending(io);
			}

			io = new_io;
		}
1297
	}
1298 1299

	crypt_dec_pending(io);
1300 1301
}

1302
static void kcryptd_crypt_read_done(struct dm_crypt_io *io)
1303 1304 1305 1306
{
	crypt_dec_pending(io);
}

1307
static void kcryptd_crypt_read_convert(struct dm_crypt_io *io)
1308
{
1309
	struct crypt_config *cc = io->cc;
1310
	int r = 0;
L
Linus Torvalds 已提交
1311

M
Milan Broz 已提交
1312
	crypt_inc_pending(io);
M
Milan Broz 已提交
1313

1314
	crypt_convert_init(cc, &io->ctx, io->base_bio, io->base_bio,
1315
			   io->sector);
L
Linus Torvalds 已提交
1316

1317
	r = crypt_convert(cc, &io->ctx);
1318 1319
	if (r < 0)
		io->error = -EIO;
1320

1321
	if (atomic_dec_and_test(&io->ctx.cc_pending))
1322
		kcryptd_crypt_read_done(io);
M
Milan Broz 已提交
1323 1324

	crypt_dec_pending(io);
L
Linus Torvalds 已提交
1325 1326
}

1327 1328 1329
static void kcryptd_async_done(struct crypto_async_request *async_req,
			       int error)
{
1330 1331
	struct dm_crypt_request *dmreq = async_req->data;
	struct convert_context *ctx = dmreq->ctx;
1332
	struct dm_crypt_io *io = container_of(ctx, struct dm_crypt_io, ctx);
1333
	struct crypt_config *cc = io->cc;
1334 1335 1336 1337 1338 1339

	if (error == -EINPROGRESS) {
		complete(&ctx->restart);
		return;
	}

1340 1341 1342
	if (!error && cc->iv_gen_ops && cc->iv_gen_ops->post)
		error = cc->iv_gen_ops->post(cc, iv_of_dmreq(cc, dmreq), dmreq);

1343 1344 1345
	if (error < 0)
		io->error = -EIO;

1346
	mempool_free(req_of_dmreq(cc, dmreq), cc->req_pool);
1347

1348
	if (!atomic_dec_and_test(&ctx->cc_pending))
1349 1350 1351
		return;

	if (bio_data_dir(io->base_bio) == READ)
1352
		kcryptd_crypt_read_done(io);
1353
	else
1354
		kcryptd_crypt_write_io_submit(io, 1);
1355 1356
}

1357
static void kcryptd_crypt(struct work_struct *work)
L
Linus Torvalds 已提交
1358
{
1359
	struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);
1360

1361
	if (bio_data_dir(io->base_bio) == READ)
1362
		kcryptd_crypt_read_convert(io);
1363
	else
1364
		kcryptd_crypt_write_convert(io);
1365 1366
}

1367
static void kcryptd_queue_crypt(struct dm_crypt_io *io)
1368
{
1369
	struct crypt_config *cc = io->cc;
1370

1371 1372
	INIT_WORK(&io->work, kcryptd_crypt);
	queue_work(cc->crypt_queue, &io->work);
L
Linus Torvalds 已提交
1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384
}

/*
 * Decode key from its hex representation
 */
static int crypt_decode_key(u8 *key, char *hex, unsigned int size)
{
	char buffer[3];
	unsigned int i;

	buffer[2] = '\0';

1385
	for (i = 0; i < size; i++) {
L
Linus Torvalds 已提交
1386 1387 1388
		buffer[0] = *hex++;
		buffer[1] = *hex++;

M
majianpeng 已提交
1389
		if (kstrtou8(buffer, 16, &key[i]))
L
Linus Torvalds 已提交
1390 1391 1392 1393 1394 1395 1396 1397 1398
			return -EINVAL;
	}

	if (*hex != '\0')
		return -EINVAL;

	return 0;
}

1399
static void crypt_free_tfms(struct crypt_config *cc)
1400 1401 1402
{
	unsigned i;

1403 1404 1405
	if (!cc->tfms)
		return;

1406
	for (i = 0; i < cc->tfms_count; i++)
1407 1408 1409
		if (cc->tfms[i] && !IS_ERR(cc->tfms[i])) {
			crypto_free_ablkcipher(cc->tfms[i]);
			cc->tfms[i] = NULL;
1410
		}
1411 1412 1413

	kfree(cc->tfms);
	cc->tfms = NULL;
1414 1415
}

1416
static int crypt_alloc_tfms(struct crypt_config *cc, char *ciphermode)
1417 1418 1419 1420
{
	unsigned i;
	int err;

1421 1422 1423 1424 1425
	cc->tfms = kmalloc(cc->tfms_count * sizeof(struct crypto_ablkcipher *),
			   GFP_KERNEL);
	if (!cc->tfms)
		return -ENOMEM;

1426
	for (i = 0; i < cc->tfms_count; i++) {
1427 1428 1429 1430
		cc->tfms[i] = crypto_alloc_ablkcipher(ciphermode, 0, 0);
		if (IS_ERR(cc->tfms[i])) {
			err = PTR_ERR(cc->tfms[i]);
			crypt_free_tfms(cc);
1431 1432 1433 1434 1435 1436 1437
			return err;
		}
	}

	return 0;
}

1438 1439
static int crypt_setkey_allcpus(struct crypt_config *cc)
{
1440
	unsigned subkey_size;
1441 1442
	int err = 0, i, r;

1443 1444 1445
	/* Ignore extra keys (which are used for IV etc) */
	subkey_size = (cc->key_size - cc->key_extra_size) >> ilog2(cc->tfms_count);

1446 1447 1448 1449 1450 1451
	for (i = 0; i < cc->tfms_count; i++) {
		r = crypto_ablkcipher_setkey(cc->tfms[i],
					     cc->key + (i * subkey_size),
					     subkey_size);
		if (r)
			err = r;
1452 1453 1454 1455 1456
	}

	return err;
}

1457 1458
static int crypt_set_key(struct crypt_config *cc, char *key)
{
1459 1460 1461
	int r = -EINVAL;
	int key_string_len = strlen(key);

M
Milan Broz 已提交
1462
	/* The key size may not be changed. */
1463 1464
	if (cc->key_size != (key_string_len >> 1))
		goto out;
1465

M
Milan Broz 已提交
1466 1467
	/* Hyphen (which gives a key_size of zero) means there is no key. */
	if (!cc->key_size && strcmp(key, "-"))
1468
		goto out;
1469

M
Milan Broz 已提交
1470
	if (cc->key_size && crypt_decode_key(cc->key, key, cc->key_size) < 0)
1471
		goto out;
1472 1473 1474

	set_bit(DM_CRYPT_KEY_VALID, &cc->flags);

1475 1476 1477 1478 1479 1480 1481
	r = crypt_setkey_allcpus(cc);

out:
	/* Hex key string not needed after here, so wipe it. */
	memset(key, '0', key_string_len);

	return r;
1482 1483 1484 1485 1486 1487
}

static int crypt_wipe_key(struct crypt_config *cc)
{
	clear_bit(DM_CRYPT_KEY_VALID, &cc->flags);
	memset(&cc->key, 0, cc->key_size * sizeof(u8));
1488 1489

	return crypt_setkey_allcpus(cc);
1490 1491
}

1492 1493 1494
static void crypt_dtr(struct dm_target *ti)
{
	struct crypt_config *cc = ti->private;
1495 1496
	struct crypt_cpu *cpu_cc;
	int cpu;
1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507

	ti->private = NULL;

	if (!cc)
		return;

	if (cc->io_queue)
		destroy_workqueue(cc->io_queue);
	if (cc->crypt_queue)
		destroy_workqueue(cc->crypt_queue);

1508 1509 1510 1511 1512 1513 1514
	if (cc->cpu)
		for_each_possible_cpu(cpu) {
			cpu_cc = per_cpu_ptr(cc->cpu, cpu);
			if (cpu_cc->req)
				mempool_free(cpu_cc->req, cc->req_pool);
		}

1515 1516
	crypt_free_tfms(cc);

1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532
	if (cc->bs)
		bioset_free(cc->bs);

	if (cc->page_pool)
		mempool_destroy(cc->page_pool);
	if (cc->req_pool)
		mempool_destroy(cc->req_pool);
	if (cc->io_pool)
		mempool_destroy(cc->io_pool);

	if (cc->iv_gen_ops && cc->iv_gen_ops->dtr)
		cc->iv_gen_ops->dtr(cc);

	if (cc->dev)
		dm_put_device(ti, cc->dev);

1533 1534 1535
	if (cc->cpu)
		free_percpu(cc->cpu);

M
Milan Broz 已提交
1536
	kzfree(cc->cipher);
1537
	kzfree(cc->cipher_string);
1538 1539 1540 1541 1542

	/* Must zero key material before freeing */
	kzfree(cc);
}

M
Milan Broz 已提交
1543 1544
static int crypt_ctr_cipher(struct dm_target *ti,
			    char *cipher_in, char *key)
L
Linus Torvalds 已提交
1545
{
M
Milan Broz 已提交
1546
	struct crypt_config *cc = ti->private;
1547
	char *tmp, *cipher, *chainmode, *ivmode, *ivopts, *keycount;
M
Milan Broz 已提交
1548
	char *cipher_api = NULL;
1549
	int ret = -EINVAL;
1550
	char dummy;
L
Linus Torvalds 已提交
1551

M
Milan Broz 已提交
1552 1553 1554
	/* Convert to crypto api definition? */
	if (strchr(cipher_in, '(')) {
		ti->error = "Bad cipher specification";
L
Linus Torvalds 已提交
1555 1556 1557
		return -EINVAL;
	}

1558 1559 1560 1561
	cc->cipher_string = kstrdup(cipher_in, GFP_KERNEL);
	if (!cc->cipher_string)
		goto bad_mem;

M
Milan Broz 已提交
1562 1563
	/*
	 * Legacy dm-crypt cipher specification
1564
	 * cipher[:keycount]-mode-iv:ivopts
M
Milan Broz 已提交
1565 1566
	 */
	tmp = cipher_in;
1567 1568 1569 1570 1571
	keycount = strsep(&tmp, "-");
	cipher = strsep(&keycount, ":");

	if (!keycount)
		cc->tfms_count = 1;
1572
	else if (sscanf(keycount, "%u%c", &cc->tfms_count, &dummy) != 1 ||
1573 1574 1575 1576 1577
		 !is_power_of_2(cc->tfms_count)) {
		ti->error = "Bad cipher key count specification";
		return -EINVAL;
	}
	cc->key_parts = cc->tfms_count;
1578
	cc->key_extra_size = 0;
M
Milan Broz 已提交
1579 1580 1581 1582 1583

	cc->cipher = kstrdup(cipher, GFP_KERNEL);
	if (!cc->cipher)
		goto bad_mem;

L
Linus Torvalds 已提交
1584 1585 1586 1587 1588
	chainmode = strsep(&tmp, "-");
	ivopts = strsep(&tmp, "-");
	ivmode = strsep(&ivopts, ":");

	if (tmp)
M
Milan Broz 已提交
1589
		DMWARN("Ignoring unexpected additional cipher options");
L
Linus Torvalds 已提交
1590

1591
	cc->cpu = __alloc_percpu(sizeof(*(cc->cpu)),
1592
				 __alignof__(struct crypt_cpu));
1593 1594 1595 1596 1597
	if (!cc->cpu) {
		ti->error = "Cannot allocate per cpu state";
		goto bad_mem;
	}

1598 1599 1600 1601
	/*
	 * For compatibility with the original dm-crypt mapping format, if
	 * only the cipher name is supplied, use cbc-plain.
	 */
M
Milan Broz 已提交
1602
	if (!chainmode || (!strcmp(chainmode, "plain") && !ivmode)) {
L
Linus Torvalds 已提交
1603 1604 1605 1606
		chainmode = "cbc";
		ivmode = "plain";
	}

1607
	if (strcmp(chainmode, "ecb") && !ivmode) {
M
Milan Broz 已提交
1608 1609
		ti->error = "IV mechanism required";
		return -EINVAL;
L
Linus Torvalds 已提交
1610 1611
	}

M
Milan Broz 已提交
1612 1613 1614 1615 1616 1617 1618 1619 1620
	cipher_api = kmalloc(CRYPTO_MAX_ALG_NAME, GFP_KERNEL);
	if (!cipher_api)
		goto bad_mem;

	ret = snprintf(cipher_api, CRYPTO_MAX_ALG_NAME,
		       "%s(%s)", chainmode, cipher);
	if (ret < 0) {
		kfree(cipher_api);
		goto bad_mem;
L
Linus Torvalds 已提交
1621 1622
	}

M
Milan Broz 已提交
1623
	/* Allocate cipher */
1624 1625 1626 1627
	ret = crypt_alloc_tfms(cc, cipher_api);
	if (ret < 0) {
		ti->error = "Error allocating crypto tfm";
		goto bad;
L
Linus Torvalds 已提交
1628 1629
	}

M
Milan Broz 已提交
1630
	/* Initialize IV */
1631
	cc->iv_size = crypto_ablkcipher_ivsize(any_tfm(cc));
M
Milan Broz 已提交
1632 1633 1634 1635 1636 1637 1638 1639 1640 1641
	if (cc->iv_size)
		/* at least a 64 bit sector number should fit in our buffer */
		cc->iv_size = max(cc->iv_size,
				  (unsigned int)(sizeof(u64) / sizeof(u8)));
	else if (ivmode) {
		DMWARN("Selected cipher does not support IVs");
		ivmode = NULL;
	}

	/* Choose ivmode, see comments at iv code. */
L
Linus Torvalds 已提交
1642 1643 1644 1645
	if (ivmode == NULL)
		cc->iv_gen_ops = NULL;
	else if (strcmp(ivmode, "plain") == 0)
		cc->iv_gen_ops = &crypt_iv_plain_ops;
M
Milan Broz 已提交
1646 1647
	else if (strcmp(ivmode, "plain64") == 0)
		cc->iv_gen_ops = &crypt_iv_plain64_ops;
L
Linus Torvalds 已提交
1648 1649
	else if (strcmp(ivmode, "essiv") == 0)
		cc->iv_gen_ops = &crypt_iv_essiv_ops;
1650 1651
	else if (strcmp(ivmode, "benbi") == 0)
		cc->iv_gen_ops = &crypt_iv_benbi_ops;
L
Ludwig Nussel 已提交
1652 1653
	else if (strcmp(ivmode, "null") == 0)
		cc->iv_gen_ops = &crypt_iv_null_ops;
1654 1655
	else if (strcmp(ivmode, "lmk") == 0) {
		cc->iv_gen_ops = &crypt_iv_lmk_ops;
1656 1657
		/*
		 * Version 2 and 3 is recognised according
1658 1659
		 * to length of provided multi-key string.
		 * If present (version 3), last key is used as IV seed.
1660
		 * All keys (including IV seed) are always the same size.
1661
		 */
1662
		if (cc->key_size % cc->key_parts) {
1663
			cc->key_parts++;
1664 1665
			cc->key_extra_size = cc->key_size / cc->key_parts;
		}
1666 1667 1668 1669
	} else if (strcmp(ivmode, "tcw") == 0) {
		cc->iv_gen_ops = &crypt_iv_tcw_ops;
		cc->key_parts += 2; /* IV + whitening */
		cc->key_extra_size = cc->iv_size + TCW_WHITENING_SIZE;
1670
	} else {
M
Milan Broz 已提交
1671
		ret = -EINVAL;
1672
		ti->error = "Invalid IV mode";
1673
		goto bad;
L
Linus Torvalds 已提交
1674 1675
	}

1676 1677 1678 1679 1680 1681 1682
	/* Initialize and set key */
	ret = crypt_set_key(cc, key);
	if (ret < 0) {
		ti->error = "Error decoding and setting key";
		goto bad;
	}

1683 1684 1685 1686 1687 1688 1689 1690
	/* Allocate IV */
	if (cc->iv_gen_ops && cc->iv_gen_ops->ctr) {
		ret = cc->iv_gen_ops->ctr(cc, ti, ivopts);
		if (ret < 0) {
			ti->error = "Error creating IV";
			goto bad;
		}
	}
L
Linus Torvalds 已提交
1691

1692 1693 1694 1695 1696 1697 1698
	/* Initialize IV (set keys for ESSIV etc) */
	if (cc->iv_gen_ops && cc->iv_gen_ops->init) {
		ret = cc->iv_gen_ops->init(cc);
		if (ret < 0) {
			ti->error = "Error initialising IV";
			goto bad;
		}
1699 1700
	}

M
Milan Broz 已提交
1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717
	ret = 0;
bad:
	kfree(cipher_api);
	return ret;

bad_mem:
	ti->error = "Cannot allocate cipher strings";
	return -ENOMEM;
}

/*
 * Construct an encryption mapping:
 * <cipher> <key> <iv_offset> <dev_path> <start>
 */
static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv)
{
	struct crypt_config *cc;
1718
	unsigned int key_size, opt_params;
M
Milan Broz 已提交
1719 1720
	unsigned long long tmpll;
	int ret;
1721 1722
	struct dm_arg_set as;
	const char *opt_string;
1723
	char dummy;
1724 1725 1726 1727

	static struct dm_arg _args[] = {
		{0, 1, "Invalid number of feature args"},
	};
M
Milan Broz 已提交
1728

1729
	if (argc < 5) {
M
Milan Broz 已提交
1730 1731
		ti->error = "Not enough arguments";
		return -EINVAL;
L
Linus Torvalds 已提交
1732 1733
	}

M
Milan Broz 已提交
1734 1735 1736 1737 1738 1739 1740
	key_size = strlen(argv[1]) >> 1;

	cc = kzalloc(sizeof(*cc) + key_size * sizeof(u8), GFP_KERNEL);
	if (!cc) {
		ti->error = "Cannot allocate encryption context";
		return -ENOMEM;
	}
M
Milan Broz 已提交
1741
	cc->key_size = key_size;
M
Milan Broz 已提交
1742 1743 1744 1745 1746 1747

	ti->private = cc;
	ret = crypt_ctr_cipher(ti, argv[0], argv[1]);
	if (ret < 0)
		goto bad;

1748
	ret = -ENOMEM;
1749
	cc->io_pool = mempool_create_slab_pool(MIN_IOS, _crypt_io_pool);
L
Linus Torvalds 已提交
1750
	if (!cc->io_pool) {
1751
		ti->error = "Cannot allocate crypt io mempool";
1752
		goto bad;
L
Linus Torvalds 已提交
1753 1754
	}

1755
	cc->dmreq_start = sizeof(struct ablkcipher_request);
1756
	cc->dmreq_start += crypto_ablkcipher_reqsize(any_tfm(cc));
1757
	cc->dmreq_start = ALIGN(cc->dmreq_start, crypto_tfm_ctx_alignment());
1758
	cc->dmreq_start += crypto_ablkcipher_alignmask(any_tfm(cc)) &
M
Milan Broz 已提交
1759
			   ~(crypto_tfm_ctx_alignment() - 1);
1760 1761 1762 1763 1764

	cc->req_pool = mempool_create_kmalloc_pool(MIN_IOS, cc->dmreq_start +
			sizeof(struct dm_crypt_request) + cc->iv_size);
	if (!cc->req_pool) {
		ti->error = "Cannot allocate crypt request mempool";
1765
		goto bad;
1766 1767
	}

1768
	cc->page_pool = mempool_create_page_pool(MIN_POOL_PAGES, 0);
L
Linus Torvalds 已提交
1769
	if (!cc->page_pool) {
1770
		ti->error = "Cannot allocate page mempool";
1771
		goto bad;
L
Linus Torvalds 已提交
1772 1773
	}

1774
	cc->bs = bioset_create(MIN_IOS, 0);
1775 1776
	if (!cc->bs) {
		ti->error = "Cannot allocate crypt bioset";
1777
		goto bad;
1778 1779
	}

1780
	ret = -EINVAL;
1781
	if (sscanf(argv[2], "%llu%c", &tmpll, &dummy) != 1) {
1782
		ti->error = "Invalid iv_offset sector";
1783
		goto bad;
L
Linus Torvalds 已提交
1784
	}
1785
	cc->iv_offset = tmpll;
L
Linus Torvalds 已提交
1786

1787 1788 1789 1790 1791
	if (dm_get_device(ti, argv[3], dm_table_get_mode(ti->table), &cc->dev)) {
		ti->error = "Device lookup failed";
		goto bad;
	}

1792
	if (sscanf(argv[4], "%llu%c", &tmpll, &dummy) != 1) {
1793
		ti->error = "Invalid device sector";
1794
		goto bad;
L
Linus Torvalds 已提交
1795
	}
1796
	cc->start = tmpll;
L
Linus Torvalds 已提交
1797

1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813
	argv += 5;
	argc -= 5;

	/* Optional parameters */
	if (argc) {
		as.argc = argc;
		as.argv = argv;

		ret = dm_read_arg_group(_args, &as, &opt_params, &ti->error);
		if (ret)
			goto bad;

		opt_string = dm_shift_arg(&as);

		if (opt_params == 1 && opt_string &&
		    !strcasecmp(opt_string, "allow_discards"))
1814
			ti->num_discard_bios = 1;
1815 1816 1817 1818 1819 1820 1821
		else if (opt_params) {
			ret = -EINVAL;
			ti->error = "Invalid feature arguments";
			goto bad;
		}
	}

1822
	ret = -ENOMEM;
1823
	cc->io_queue = alloc_workqueue("kcryptd_io", WQ_MEM_RECLAIM, 1);
1824 1825
	if (!cc->io_queue) {
		ti->error = "Couldn't create kcryptd io queue";
1826
		goto bad;
1827 1828
	}

1829
	cc->crypt_queue = alloc_workqueue("kcryptd",
1830
					  WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 1);
1831
	if (!cc->crypt_queue) {
1832
		ti->error = "Couldn't create kcryptd queue";
1833
		goto bad;
1834 1835
	}

1836
	ti->num_flush_bios = 1;
1837
	ti->discard_zeroes_data_unsupported = true;
1838

L
Linus Torvalds 已提交
1839 1840
	return 0;

1841 1842 1843
bad:
	crypt_dtr(ti);
	return ret;
L
Linus Torvalds 已提交
1844 1845
}

M
Mikulas Patocka 已提交
1846
static int crypt_map(struct dm_target *ti, struct bio *bio)
L
Linus Torvalds 已提交
1847
{
1848
	struct dm_crypt_io *io;
1849
	struct crypt_config *cc = ti->private;
M
Mikulas Patocka 已提交
1850

1851 1852 1853 1854 1855 1856
	/*
	 * If bio is REQ_FLUSH or REQ_DISCARD, just bypass crypt queues.
	 * - for REQ_FLUSH device-mapper core ensures that no IO is in-flight
	 * - for REQ_DISCARD caller must use flush if IO ordering matters
	 */
	if (unlikely(bio->bi_rw & (REQ_FLUSH | REQ_DISCARD))) {
M
Mikulas Patocka 已提交
1857
		bio->bi_bdev = cc->dev->bdev;
1858
		if (bio_sectors(bio))
1859 1860
			bio->bi_iter.bi_sector = cc->start +
				dm_target_offset(ti, bio->bi_iter.bi_sector);
M
Mikulas Patocka 已提交
1861 1862
		return DM_MAPIO_REMAPPED;
	}
L
Linus Torvalds 已提交
1863

1864
	io = crypt_io_alloc(cc, bio, dm_target_offset(ti, bio->bi_iter.bi_sector));
1865

1866 1867 1868 1869
	if (bio_data_dir(io->base_bio) == READ) {
		if (kcryptd_io_read(io, GFP_NOWAIT))
			kcryptd_queue_io(io);
	} else
1870
		kcryptd_queue_crypt(io);
L
Linus Torvalds 已提交
1871

1872
	return DM_MAPIO_SUBMITTED;
L
Linus Torvalds 已提交
1873 1874
}

1875 1876
static void crypt_status(struct dm_target *ti, status_type_t type,
			 unsigned status_flags, char *result, unsigned maxlen)
L
Linus Torvalds 已提交
1877
{
M
Milan Broz 已提交
1878
	struct crypt_config *cc = ti->private;
1879
	unsigned i, sz = 0;
L
Linus Torvalds 已提交
1880 1881 1882 1883 1884 1885 1886

	switch (type) {
	case STATUSTYPE_INFO:
		result[0] = '\0';
		break;

	case STATUSTYPE_TABLE:
1887
		DMEMIT("%s ", cc->cipher_string);
L
Linus Torvalds 已提交
1888

1889 1890 1891 1892 1893
		if (cc->key_size > 0)
			for (i = 0; i < cc->key_size; i++)
				DMEMIT("%02x", cc->key[i]);
		else
			DMEMIT("-");
L
Linus Torvalds 已提交
1894

1895 1896
		DMEMIT(" %llu %s %llu", (unsigned long long)cc->iv_offset,
				cc->dev->name, (unsigned long long)cc->start);
1897

1898
		if (ti->num_discard_bios)
1899 1900
			DMEMIT(" 1 allow_discards");

L
Linus Torvalds 已提交
1901 1902 1903 1904
		break;
	}
}

1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937
static void crypt_postsuspend(struct dm_target *ti)
{
	struct crypt_config *cc = ti->private;

	set_bit(DM_CRYPT_SUSPENDED, &cc->flags);
}

static int crypt_preresume(struct dm_target *ti)
{
	struct crypt_config *cc = ti->private;

	if (!test_bit(DM_CRYPT_KEY_VALID, &cc->flags)) {
		DMERR("aborting resume - crypt key is not set.");
		return -EAGAIN;
	}

	return 0;
}

static void crypt_resume(struct dm_target *ti)
{
	struct crypt_config *cc = ti->private;

	clear_bit(DM_CRYPT_SUSPENDED, &cc->flags);
}

/* Message interface
 *	key set <key>
 *	key wipe
 */
static int crypt_message(struct dm_target *ti, unsigned argc, char **argv)
{
	struct crypt_config *cc = ti->private;
1938
	int ret = -EINVAL;
1939 1940 1941 1942

	if (argc < 2)
		goto error;

1943
	if (!strcasecmp(argv[0], "key")) {
1944 1945 1946 1947
		if (!test_bit(DM_CRYPT_SUSPENDED, &cc->flags)) {
			DMWARN("not suspended during key manipulation.");
			return -EINVAL;
		}
1948
		if (argc == 3 && !strcasecmp(argv[1], "set")) {
1949 1950 1951 1952 1953 1954 1955
			ret = crypt_set_key(cc, argv[2]);
			if (ret)
				return ret;
			if (cc->iv_gen_ops && cc->iv_gen_ops->init)
				ret = cc->iv_gen_ops->init(cc);
			return ret;
		}
1956
		if (argc == 2 && !strcasecmp(argv[1], "wipe")) {
1957 1958 1959 1960 1961
			if (cc->iv_gen_ops && cc->iv_gen_ops->wipe) {
				ret = cc->iv_gen_ops->wipe(cc);
				if (ret)
					return ret;
			}
1962
			return crypt_wipe_key(cc);
1963
		}
1964 1965 1966 1967 1968 1969 1970
	}

error:
	DMWARN("unrecognised message received.");
	return -EINVAL;
}

M
Milan Broz 已提交
1971 1972 1973 1974 1975 1976 1977 1978 1979 1980
static int crypt_merge(struct dm_target *ti, struct bvec_merge_data *bvm,
		       struct bio_vec *biovec, int max_size)
{
	struct crypt_config *cc = ti->private;
	struct request_queue *q = bdev_get_queue(cc->dev->bdev);

	if (!q->merge_bvec_fn)
		return max_size;

	bvm->bi_bdev = cc->dev->bdev;
1981
	bvm->bi_sector = cc->start + dm_target_offset(ti, bvm->bi_sector);
M
Milan Broz 已提交
1982 1983 1984 1985

	return min(max_size, q->merge_bvec_fn(q, bvm, biovec));
}

1986 1987 1988 1989 1990
static int crypt_iterate_devices(struct dm_target *ti,
				 iterate_devices_callout_fn fn, void *data)
{
	struct crypt_config *cc = ti->private;

1991
	return fn(ti, cc->dev, cc->start, ti->len, data);
1992 1993
}

L
Linus Torvalds 已提交
1994 1995
static struct target_type crypt_target = {
	.name   = "crypt",
1996
	.version = {1, 13, 0},
L
Linus Torvalds 已提交
1997 1998 1999 2000 2001
	.module = THIS_MODULE,
	.ctr    = crypt_ctr,
	.dtr    = crypt_dtr,
	.map    = crypt_map,
	.status = crypt_status,
2002 2003 2004 2005
	.postsuspend = crypt_postsuspend,
	.preresume = crypt_preresume,
	.resume = crypt_resume,
	.message = crypt_message,
M
Milan Broz 已提交
2006
	.merge  = crypt_merge,
2007
	.iterate_devices = crypt_iterate_devices,
L
Linus Torvalds 已提交
2008 2009 2010 2011 2012 2013
};

static int __init dm_crypt_init(void)
{
	int r;

2014
	_crypt_io_pool = KMEM_CACHE(dm_crypt_io, 0);
L
Linus Torvalds 已提交
2015 2016 2017 2018 2019
	if (!_crypt_io_pool)
		return -ENOMEM;

	r = dm_register_target(&crypt_target);
	if (r < 0) {
2020
		DMERR("register failed %d", r);
2021
		kmem_cache_destroy(_crypt_io_pool);
L
Linus Torvalds 已提交
2022 2023 2024 2025 2026 2027 2028
	}

	return r;
}

static void __exit dm_crypt_exit(void)
{
2029
	dm_unregister_target(&crypt_target);
L
Linus Torvalds 已提交
2030 2031 2032 2033 2034 2035 2036 2037 2038
	kmem_cache_destroy(_crypt_io_pool);
}

module_init(dm_crypt_init);
module_exit(dm_crypt_exit);

MODULE_AUTHOR("Christophe Saout <christophe@saout.de>");
MODULE_DESCRIPTION(DM_NAME " target for transparent encryption / decryption");
MODULE_LICENSE("GPL");
反馈
建议
客服 返回
顶部