safexcel_cipher.c 29.0 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14
/*
 * Copyright (C) 2017 Marvell
 *
 * Antoine Tenart <antoine.tenart@free-electrons.com>
 *
 * This file is licensed under the terms of the GNU General Public
 * License version 2. This program is licensed "as is" without any
 * warranty of any kind, whether express or implied.
 */

#include <linux/device.h>
#include <linux/dma-mapping.h>
#include <linux/dmapool.h>

15
#include <crypto/aead.h>
16
#include <crypto/aes.h>
17 18
#include <crypto/authenc.h>
#include <crypto/sha.h>
19
#include <crypto/skcipher.h>
20
#include <crypto/internal/aead.h>
21
#include <crypto/internal/skcipher.h>
22 23 24 25 26 27 28 29 30 31 32 33 34

#include "safexcel.h"

enum safexcel_cipher_direction {
	SAFEXCEL_ENCRYPT,
	SAFEXCEL_DECRYPT,
};

struct safexcel_cipher_ctx {
	struct safexcel_context base;
	struct safexcel_crypto_priv *priv;

	u32 mode;
35
	bool aead;
36 37 38

	__le32 key[8];
	unsigned int key_len;
39 40 41 42

	/* All the below is AEAD specific */
	u32 alg;
	u32 state_sz;
43 44
	u32 ipad[SHA512_DIGEST_SIZE / sizeof(u32)];
	u32 opad[SHA512_DIGEST_SIZE / sizeof(u32)];
45 46
};

47
struct safexcel_cipher_req {
48
	enum safexcel_cipher_direction direction;
49 50 51
	bool needs_inv;
};

52 53 54
static void safexcel_skcipher_token(struct safexcel_cipher_ctx *ctx, u8 *iv,
				    struct safexcel_command_desc *cdesc,
				    u32 length)
55 56 57 58 59 60
{
	struct safexcel_token *token;
	unsigned offset = 0;

	if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) {
		offset = AES_BLOCK_SIZE / sizeof(u32);
61
		memcpy(cdesc->control_data.token, iv, AES_BLOCK_SIZE);
62 63 64 65 66 67 68 69

		cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD;
	}

	token = (struct safexcel_token *)(cdesc->control_data.token + offset);

	token[0].opcode = EIP197_TOKEN_OPCODE_DIRECTION;
	token[0].packet_length = length;
70 71
	token[0].stat = EIP197_TOKEN_STAT_LAST_PACKET |
			EIP197_TOKEN_STAT_LAST_HASH;
72 73 74 75 76
	token[0].instructions = EIP197_TOKEN_INS_LAST |
				EIP197_TOKEN_INS_TYPE_CRYTO |
				EIP197_TOKEN_INS_TYPE_OUTPUT;
}

77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132
static void safexcel_aead_token(struct safexcel_cipher_ctx *ctx, u8 *iv,
				struct safexcel_command_desc *cdesc,
				enum safexcel_cipher_direction direction,
				u32 cryptlen, u32 assoclen, u32 digestsize)
{
	struct safexcel_token *token;
	unsigned offset = 0;

	if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) {
		offset = AES_BLOCK_SIZE / sizeof(u32);
		memcpy(cdesc->control_data.token, iv, AES_BLOCK_SIZE);

		cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD;
	}

	token = (struct safexcel_token *)(cdesc->control_data.token + offset);

	if (direction == SAFEXCEL_DECRYPT)
		cryptlen -= digestsize;

	token[0].opcode = EIP197_TOKEN_OPCODE_DIRECTION;
	token[0].packet_length = assoclen;
	token[0].instructions = EIP197_TOKEN_INS_TYPE_HASH |
				EIP197_TOKEN_INS_TYPE_OUTPUT;

	token[1].opcode = EIP197_TOKEN_OPCODE_DIRECTION;
	token[1].packet_length = cryptlen;
	token[1].stat = EIP197_TOKEN_STAT_LAST_HASH;
	token[1].instructions = EIP197_TOKEN_INS_LAST |
				EIP197_TOKEN_INS_TYPE_CRYTO |
				EIP197_TOKEN_INS_TYPE_HASH |
				EIP197_TOKEN_INS_TYPE_OUTPUT;

	if (direction == SAFEXCEL_ENCRYPT) {
		token[2].opcode = EIP197_TOKEN_OPCODE_INSERT;
		token[2].packet_length = digestsize;
		token[2].stat = EIP197_TOKEN_STAT_LAST_HASH |
				EIP197_TOKEN_STAT_LAST_PACKET;
		token[2].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT |
					EIP197_TOKEN_INS_INSERT_HASH_DIGEST;
	} else {
		token[2].opcode = EIP197_TOKEN_OPCODE_RETRIEVE;
		token[2].packet_length = digestsize;
		token[2].stat = EIP197_TOKEN_STAT_LAST_HASH |
				EIP197_TOKEN_STAT_LAST_PACKET;
		token[2].instructions = EIP197_TOKEN_INS_INSERT_HASH_DIGEST;

		token[3].opcode = EIP197_TOKEN_OPCODE_VERIFY;
		token[3].packet_length = digestsize |
					 EIP197_TOKEN_HASH_RESULT_VERIFY;
		token[3].stat = EIP197_TOKEN_STAT_LAST_HASH |
				EIP197_TOKEN_STAT_LAST_PACKET;
		token[3].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT;
	}
}

133 134
static int safexcel_skcipher_aes_setkey(struct crypto_skcipher *ctfm,
					const u8 *key, unsigned int len)
135 136 137
{
	struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm);
	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
138
	struct safexcel_crypto_priv *priv = ctx->priv;
139 140 141 142 143 144 145 146 147
	struct crypto_aes_ctx aes;
	int ret, i;

	ret = crypto_aes_expand_key(&aes, key, len);
	if (ret) {
		crypto_skcipher_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
		return ret;
	}

148
	if (priv->version == EIP197 && ctx->base.ctxr_dma) {
149 150 151 152 153
		for (i = 0; i < len / sizeof(u32); i++) {
			if (ctx->key[i] != cpu_to_le32(aes.key_enc[i])) {
				ctx->base.needs_inv = true;
				break;
			}
154 155 156 157 158 159 160 161 162 163 164 165
		}
	}

	for (i = 0; i < len / sizeof(u32); i++)
		ctx->key[i] = cpu_to_le32(aes.key_enc[i]);

	ctx->key_len = len;

	memzero_explicit(&aes, sizeof(aes));
	return 0;
}

166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186
static int safexcel_aead_aes_setkey(struct crypto_aead *ctfm, const u8 *key,
				    unsigned int len)
{
	struct crypto_tfm *tfm = crypto_aead_tfm(ctfm);
	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
	struct safexcel_ahash_export_state istate, ostate;
	struct safexcel_crypto_priv *priv = ctx->priv;
	struct crypto_authenc_keys keys;

	if (crypto_authenc_extractkeys(&keys, key, len) != 0)
		goto badkey;

	if (keys.enckeylen > sizeof(ctx->key))
		goto badkey;

	/* Encryption key */
	if (priv->version == EIP197 && ctx->base.ctxr_dma &&
	    memcmp(ctx->key, keys.enckey, keys.enckeylen))
		ctx->base.needs_inv = true;

	/* Auth key */
187
	switch (ctx->alg) {
188 189 190 191 192
	case CONTEXT_CONTROL_CRYPTO_ALG_SHA1:
		if (safexcel_hmac_setkey("safexcel-sha1", keys.authkey,
					 keys.authkeylen, &istate, &ostate))
			goto badkey;
		break;
193 194 195 196 197 198 199 200 201 202
	case CONTEXT_CONTROL_CRYPTO_ALG_SHA224:
		if (safexcel_hmac_setkey("safexcel-sha224", keys.authkey,
					 keys.authkeylen, &istate, &ostate))
			goto badkey;
		break;
	case CONTEXT_CONTROL_CRYPTO_ALG_SHA256:
		if (safexcel_hmac_setkey("safexcel-sha256", keys.authkey,
					 keys.authkeylen, &istate, &ostate))
			goto badkey;
		break;
203 204 205 206 207
	case CONTEXT_CONTROL_CRYPTO_ALG_SHA512:
		if (safexcel_hmac_setkey("safexcel-sha512", keys.authkey,
					 keys.authkeylen, &istate, &ostate))
			goto badkey;
		break;
208 209
	default:
		dev_err(priv->dev, "aead: unsupported hash algorithm\n");
210
		goto badkey;
211
	}
212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236

	crypto_aead_set_flags(ctfm, crypto_aead_get_flags(ctfm) &
				    CRYPTO_TFM_RES_MASK);

	if (priv->version == EIP197 && ctx->base.ctxr_dma &&
	    (memcmp(ctx->ipad, istate.state, ctx->state_sz) ||
	     memcmp(ctx->opad, ostate.state, ctx->state_sz)))
		ctx->base.needs_inv = true;

	/* Now copy the keys into the context */
	memcpy(ctx->key, keys.enckey, keys.enckeylen);
	ctx->key_len = keys.enckeylen;

	memcpy(ctx->ipad, &istate.state, ctx->state_sz);
	memcpy(ctx->opad, &ostate.state, ctx->state_sz);

	memzero_explicit(&keys, sizeof(keys));
	return 0;

badkey:
	crypto_aead_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
	memzero_explicit(&keys, sizeof(keys));
	return -EINVAL;
}

237
static int safexcel_context_control(struct safexcel_cipher_ctx *ctx,
238
				    struct crypto_async_request *async,
239
				    struct safexcel_cipher_req *sreq,
240 241 242 243 244
				    struct safexcel_command_desc *cdesc)
{
	struct safexcel_crypto_priv *priv = ctx->priv;
	int ctrl_size;

245 246 247 248 249 250 251 252 253 254 255 256 257 258 259
	if (ctx->aead) {
		if (sreq->direction == SAFEXCEL_ENCRYPT)
			cdesc->control_data.control0 |= CONTEXT_CONTROL_TYPE_ENCRYPT_HASH_OUT;
		else
			cdesc->control_data.control0 |= CONTEXT_CONTROL_TYPE_HASH_DECRYPT_IN;
	} else {
		cdesc->control_data.control0 |= CONTEXT_CONTROL_TYPE_CRYPTO_OUT;

		/* The decryption control type is a combination of the
		 * encryption type and CONTEXT_CONTROL_TYPE_NULL_IN, for all
		 * types.
		 */
		if (sreq->direction == SAFEXCEL_DECRYPT)
			cdesc->control_data.control0 |= CONTEXT_CONTROL_TYPE_NULL_IN;
	}
260 261 262 263

	cdesc->control_data.control0 |= CONTEXT_CONTROL_KEY_EN;
	cdesc->control_data.control1 |= ctx->mode;

264 265 266 267
	if (ctx->aead)
		cdesc->control_data.control0 |= CONTEXT_CONTROL_DIGEST_HMAC |
						ctx->alg;

268 269 270 271 272 273 274 275 276 277 278 279 280 281 282
	switch (ctx->key_len) {
	case AES_KEYSIZE_128:
		cdesc->control_data.control0 |= CONTEXT_CONTROL_CRYPTO_ALG_AES128;
		break;
	case AES_KEYSIZE_192:
		cdesc->control_data.control0 |= CONTEXT_CONTROL_CRYPTO_ALG_AES192;
		break;
	case AES_KEYSIZE_256:
		cdesc->control_data.control0 |= CONTEXT_CONTROL_CRYPTO_ALG_AES256;
		break;
	default:
		dev_err(priv->dev, "aes keysize not supported: %u\n",
			ctx->key_len);
		return -EINVAL;
	}
283 284

	ctrl_size = ctx->key_len / sizeof(u32);
285 286 287
	if (ctx->aead)
		/* Take in account the ipad+opad digests */
		ctrl_size += ctx->state_sz / sizeof(u32) * 2;
288 289 290 291 292
	cdesc->control_data.control0 |= CONTEXT_CONTROL_SIZE(ctrl_size);

	return 0;
}

293 294
static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int ring,
				      struct crypto_async_request *async,
295 296 297 298
				      struct scatterlist *src,
				      struct scatterlist *dst,
				      unsigned int cryptlen,
				      struct safexcel_cipher_req *sreq,
299
				      bool *should_complete, int *ret)
300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315
{
	struct safexcel_result_desc *rdesc;
	int ndesc = 0;

	*ret = 0;

	spin_lock_bh(&priv->ring[ring].egress_lock);
	do {
		rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
		if (IS_ERR(rdesc)) {
			dev_err(priv->dev,
				"cipher: result: could not retrieve the result descriptor\n");
			*ret = PTR_ERR(rdesc);
			break;
		}

316 317
		if (likely(!*ret))
			*ret = safexcel_rdesc_check_errors(priv, rdesc);
318 319 320 321 322 323 324

		ndesc++;
	} while (!rdesc->last_seg);

	safexcel_complete(priv, ring);
	spin_unlock_bh(&priv->ring[ring].egress_lock);

325 326 327
	if (src == dst) {
		dma_unmap_sg(priv->dev, src,
			     sg_nents_for_len(src, cryptlen),
328 329
			     DMA_BIDIRECTIONAL);
	} else {
330 331
		dma_unmap_sg(priv->dev, src,
			     sg_nents_for_len(src, cryptlen),
332
			     DMA_TO_DEVICE);
333 334
		dma_unmap_sg(priv->dev, dst,
			     sg_nents_for_len(dst, cryptlen),
335 336 337 338 339 340 341 342
			     DMA_FROM_DEVICE);
	}

	*should_complete = true;

	return ndesc;
}

343 344 345 346
static int safexcel_aes_send(struct crypto_async_request *base, int ring,
			     struct safexcel_request *request,
			     struct safexcel_cipher_req *sreq,
			     struct scatterlist *src, struct scatterlist *dst,
347 348
			     unsigned int cryptlen, unsigned int assoclen,
			     unsigned int digestsize, u8 *iv, int *commands,
349
			     int *results)
350
{
351
	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
352 353 354 355
	struct safexcel_crypto_priv *priv = ctx->priv;
	struct safexcel_command_desc *cdesc;
	struct safexcel_result_desc *rdesc;
	struct scatterlist *sg;
356 357
	unsigned int totlen = cryptlen + assoclen;
	int nr_src, nr_dst, n_cdesc = 0, n_rdesc = 0, queued = totlen;
358 359
	int i, ret = 0;

360 361
	if (src == dst) {
		nr_src = dma_map_sg(priv->dev, src,
362
				    sg_nents_for_len(src, totlen),
363 364 365 366 367
				    DMA_BIDIRECTIONAL);
		nr_dst = nr_src;
		if (!nr_src)
			return -EINVAL;
	} else {
368
		nr_src = dma_map_sg(priv->dev, src,
369
				    sg_nents_for_len(src, totlen),
370 371 372 373
				    DMA_TO_DEVICE);
		if (!nr_src)
			return -EINVAL;

374
		nr_dst = dma_map_sg(priv->dev, dst,
375
				    sg_nents_for_len(dst, totlen),
376 377
				    DMA_FROM_DEVICE);
		if (!nr_dst) {
378
			dma_unmap_sg(priv->dev, src,
379
				     sg_nents_for_len(src, totlen),
380 381 382 383 384 385 386
				     DMA_TO_DEVICE);
			return -EINVAL;
		}
	}

	memcpy(ctx->base.ctxr->data, ctx->key, ctx->key_len);

387 388 389 390 391 392 393
	if (ctx->aead) {
		memcpy(ctx->base.ctxr->data + ctx->key_len / sizeof(u32),
		       ctx->ipad, ctx->state_sz);
		memcpy(ctx->base.ctxr->data + (ctx->key_len + ctx->state_sz) / sizeof(u32),
		       ctx->opad, ctx->state_sz);
	}

394 395 396
	spin_lock_bh(&priv->ring[ring].egress_lock);

	/* command descriptors */
397
	for_each_sg(src, sg, nr_src, i) {
398 399 400 401 402 403 404
		int len = sg_dma_len(sg);

		/* Do not overflow the request */
		if (queued - len < 0)
			len = queued;

		cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc, !(queued - len),
405
					   sg_dma_address(sg), len, totlen,
406 407 408 409 410 411 412 413 414
					   ctx->base.ctxr_dma);
		if (IS_ERR(cdesc)) {
			/* No space left in the command descriptor ring */
			ret = PTR_ERR(cdesc);
			goto cdesc_rollback;
		}
		n_cdesc++;

		if (n_cdesc == 1) {
415
			safexcel_context_control(ctx, base, sreq, cdesc);
416 417 418 419 420 421 422
			if (ctx->aead)
				safexcel_aead_token(ctx, iv, cdesc,
						    sreq->direction, cryptlen,
						    assoclen, digestsize);
			else
				safexcel_skcipher_token(ctx, iv, cdesc,
							cryptlen);
423 424 425 426 427 428 429 430
		}

		queued -= len;
		if (!queued)
			break;
	}

	/* result descriptors */
431
	for_each_sg(dst, sg, nr_dst, i) {
432 433 434 435 436 437 438 439 440 441 442 443 444 445 446
		bool first = !i, last = (i == nr_dst - 1);
		u32 len = sg_dma_len(sg);

		rdesc = safexcel_add_rdesc(priv, ring, first, last,
					   sg_dma_address(sg), len);
		if (IS_ERR(rdesc)) {
			/* No space left in the result descriptor ring */
			ret = PTR_ERR(rdesc);
			goto rdesc_rollback;
		}
		n_rdesc++;
	}

	spin_unlock_bh(&priv->ring[ring].egress_lock);

447
	request->req = base;
448

449
	*commands = n_cdesc;
450
	*results = n_rdesc;
451 452 453 454 455 456 457 458 459 460 461
	return 0;

rdesc_rollback:
	for (i = 0; i < n_rdesc; i++)
		safexcel_ring_rollback_wptr(priv, &priv->ring[ring].rdr);
cdesc_rollback:
	for (i = 0; i < n_cdesc; i++)
		safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr);

	spin_unlock_bh(&priv->ring[ring].egress_lock);

462 463
	if (src == dst) {
		dma_unmap_sg(priv->dev, src,
464
			     sg_nents_for_len(src, totlen),
465 466
			     DMA_BIDIRECTIONAL);
	} else {
467
		dma_unmap_sg(priv->dev, src,
468
			     sg_nents_for_len(src, totlen),
469
			     DMA_TO_DEVICE);
470
		dma_unmap_sg(priv->dev, dst,
471
			     sg_nents_for_len(dst, totlen),
472 473 474 475 476 477 478 479
			     DMA_FROM_DEVICE);
	}

	return ret;
}

static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv,
				      int ring,
480
				      struct crypto_async_request *base,
481 482
				      bool *should_complete, int *ret)
{
483
	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
484 485 486 487 488 489 490 491 492 493 494 495 496 497 498
	struct safexcel_result_desc *rdesc;
	int ndesc = 0, enq_ret;

	*ret = 0;

	spin_lock_bh(&priv->ring[ring].egress_lock);
	do {
		rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
		if (IS_ERR(rdesc)) {
			dev_err(priv->dev,
				"cipher: invalidate: could not retrieve the result descriptor\n");
			*ret = PTR_ERR(rdesc);
			break;
		}

499 500
		if (likely(!*ret))
			*ret = safexcel_rdesc_check_errors(priv, rdesc);
501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516

		ndesc++;
	} while (!rdesc->last_seg);

	safexcel_complete(priv, ring);
	spin_unlock_bh(&priv->ring[ring].egress_lock);

	if (ctx->base.exit_inv) {
		dma_pool_free(priv->context_pool, ctx->base.ctxr,
			      ctx->base.ctxr_dma);

		*should_complete = true;

		return ndesc;
	}

517 518
	ring = safexcel_select_ring(priv);
	ctx->base.ring = ring;
519

520
	spin_lock_bh(&priv->ring[ring].queue_lock);
521
	enq_ret = crypto_enqueue_request(&priv->ring[ring].queue, base);
522
	spin_unlock_bh(&priv->ring[ring].queue_lock);
523 524 525 526

	if (enq_ret != -EINPROGRESS)
		*ret = enq_ret;

527 528
	queue_work(priv->ring[ring].workqueue,
		   &priv->ring[ring].work_data.work);
529

530 531 532 533 534
	*should_complete = false;

	return ndesc;
}

535 536 537 538
static int safexcel_skcipher_handle_result(struct safexcel_crypto_priv *priv,
					   int ring,
					   struct crypto_async_request *async,
					   bool *should_complete, int *ret)
539 540 541 542 543 544 545 546 547 548
{
	struct skcipher_request *req = skcipher_request_cast(async);
	struct safexcel_cipher_req *sreq = skcipher_request_ctx(req);
	int err;

	if (sreq->needs_inv) {
		sreq->needs_inv = false;
		err = safexcel_handle_inv_result(priv, ring, async,
						 should_complete, ret);
	} else {
549 550
		err = safexcel_handle_req_result(priv, ring, async, req->src,
						 req->dst, req->cryptlen, sreq,
551 552 553 554 555 556
						 should_complete, ret);
	}

	return err;
}

557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580
static int safexcel_aead_handle_result(struct safexcel_crypto_priv *priv,
				       int ring,
				       struct crypto_async_request *async,
				       bool *should_complete, int *ret)
{
	struct aead_request *req = aead_request_cast(async);
	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
	struct safexcel_cipher_req *sreq = aead_request_ctx(req);
	int err;

	if (sreq->needs_inv) {
		sreq->needs_inv = false;
		err = safexcel_handle_inv_result(priv, ring, async,
						 should_complete, ret);
	} else {
		err = safexcel_handle_req_result(priv, ring, async, req->src,
						 req->dst,
						 req->cryptlen + crypto_aead_authsize(tfm),
						 sreq, should_complete, ret);
	}

	return err;
}

581
static int safexcel_cipher_send_inv(struct crypto_async_request *base,
582 583 584
				    int ring, struct safexcel_request *request,
				    int *commands, int *results)
{
585
	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
586 587 588
	struct safexcel_crypto_priv *priv = ctx->priv;
	int ret;

589 590
	ret = safexcel_invalidate_cache(base, priv, ctx->base.ctxr_dma, ring,
					request);
591 592 593 594 595 596 597 598 599
	if (unlikely(ret))
		return ret;

	*commands = 1;
	*results = 1;

	return 0;
}

600 601 602
static int safexcel_skcipher_send(struct crypto_async_request *async, int ring,
				  struct safexcel_request *request,
				  int *commands, int *results)
603 604
{
	struct skcipher_request *req = skcipher_request_cast(async);
605
	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
606
	struct safexcel_cipher_req *sreq = skcipher_request_ctx(req);
607
	struct safexcel_crypto_priv *priv = ctx->priv;
608 609
	int ret;

610 611
	BUG_ON(priv->version == EIP97 && sreq->needs_inv);

612
	if (sreq->needs_inv)
613 614
		ret = safexcel_cipher_send_inv(async, ring, request, commands,
					       results);
615
	else
616
		ret = safexcel_aes_send(async, ring, request, sreq, req->src,
617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641
					req->dst, req->cryptlen, 0, 0, req->iv,
					commands, results);
	return ret;
}

static int safexcel_aead_send(struct crypto_async_request *async, int ring,
			      struct safexcel_request *request, int *commands,
			      int *results)
{
	struct aead_request *req = aead_request_cast(async);
	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
	struct safexcel_cipher_req *sreq = aead_request_ctx(req);
	struct safexcel_crypto_priv *priv = ctx->priv;
	int ret;

	BUG_ON(priv->version == EIP97 && sreq->needs_inv);

	if (sreq->needs_inv)
		ret = safexcel_cipher_send_inv(async, ring, request, commands,
					       results);
	else
		ret = safexcel_aes_send(async, ring, request, sreq, req->src,
					req->dst, req->cryptlen, req->assoclen,
					crypto_aead_authsize(tfm), req->iv,
642 643 644 645
					commands, results);
	return ret;
}

646 647 648 649
static int safexcel_cipher_exit_inv(struct crypto_tfm *tfm,
				    struct crypto_async_request *base,
				    struct safexcel_cipher_req *sreq,
				    struct safexcel_inv_result *result)
650 651 652
{
	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
	struct safexcel_crypto_priv *priv = ctx->priv;
653
	int ring = ctx->base.ring;
654

655
	init_completion(&result->completion);
656

657
	ctx = crypto_tfm_ctx(base->tfm);
658
	ctx->base.exit_inv = true;
659
	sreq->needs_inv = true;
660

661
	spin_lock_bh(&priv->ring[ring].queue_lock);
662
	crypto_enqueue_request(&priv->ring[ring].queue, base);
663
	spin_unlock_bh(&priv->ring[ring].queue_lock);
664

665 666
	queue_work(priv->ring[ring].workqueue,
		   &priv->ring[ring].work_data.work);
667

668
	wait_for_completion(&result->completion);
669

670
	if (result->error) {
671 672
		dev_warn(priv->dev,
			"cipher: sync: invalidate: completion error %d\n",
673 674
			 result->error);
		return result->error;
675 676 677 678 679
	}

	return 0;
}

680
static int safexcel_skcipher_exit_inv(struct crypto_tfm *tfm)
681
{
682
	EIP197_REQUEST_ON_STACK(req, skcipher, EIP197_SKCIPHER_REQ_SIZE);
683
	struct safexcel_cipher_req *sreq = skcipher_request_ctx(req);
684 685 686 687 688 689 690 691 692 693 694
	struct safexcel_inv_result result = {};

	memset(req, 0, sizeof(struct skcipher_request));

	skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
				      safexcel_inv_complete, &result);
	skcipher_request_set_tfm(req, __crypto_skcipher_cast(tfm));

	return safexcel_cipher_exit_inv(tfm, &req->base, sreq, &result);
}

695 696 697 698 699 700 701 702 703 704 705 706 707 708 709
static int safexcel_aead_exit_inv(struct crypto_tfm *tfm)
{
	EIP197_REQUEST_ON_STACK(req, aead, EIP197_AEAD_REQ_SIZE);
	struct safexcel_cipher_req *sreq = aead_request_ctx(req);
	struct safexcel_inv_result result = {};

	memset(req, 0, sizeof(struct aead_request));

	aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
				  safexcel_inv_complete, &result);
	aead_request_set_tfm(req, __crypto_aead_cast(tfm));

	return safexcel_cipher_exit_inv(tfm, &req->base, sreq, &result);
}

710 711 712 713 714
static int safexcel_aes(struct crypto_async_request *base,
			struct safexcel_cipher_req *sreq,
			enum safexcel_cipher_direction dir, u32 mode)
{
	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
715
	struct safexcel_crypto_priv *priv = ctx->priv;
716
	int ret, ring;
717

718
	sreq->needs_inv = false;
719
	sreq->direction = dir;
720 721 722
	ctx->mode = mode;

	if (ctx->base.ctxr) {
723
		if (priv->version == EIP197 && ctx->base.needs_inv) {
724 725 726
			sreq->needs_inv = true;
			ctx->base.needs_inv = false;
		}
727 728 729
	} else {
		ctx->base.ring = safexcel_select_ring(priv);
		ctx->base.ctxr = dma_pool_zalloc(priv->context_pool,
730
						 EIP197_GFP_FLAGS(*base),
731 732 733 734 735
						 &ctx->base.ctxr_dma);
		if (!ctx->base.ctxr)
			return -ENOMEM;
	}

736 737 738
	ring = ctx->base.ring;

	spin_lock_bh(&priv->ring[ring].queue_lock);
739
	ret = crypto_enqueue_request(&priv->ring[ring].queue, base);
740
	spin_unlock_bh(&priv->ring[ring].queue_lock);
741

742 743
	queue_work(priv->ring[ring].workqueue,
		   &priv->ring[ring].work_data.work);
744 745 746 747 748 749

	return ret;
}

static int safexcel_ecb_aes_encrypt(struct skcipher_request *req)
{
750 751
	return safexcel_aes(&req->base, skcipher_request_ctx(req),
			    SAFEXCEL_ENCRYPT, CONTEXT_CONTROL_CRYPTO_MODE_ECB);
752 753 754 755
}

static int safexcel_ecb_aes_decrypt(struct skcipher_request *req)
{
756 757
	return safexcel_aes(&req->base, skcipher_request_ctx(req),
			    SAFEXCEL_DECRYPT, CONTEXT_CONTROL_CRYPTO_MODE_ECB);
758 759 760 761 762 763 764 765 766
}

static int safexcel_skcipher_cra_init(struct crypto_tfm *tfm)
{
	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
	struct safexcel_alg_template *tmpl =
		container_of(tfm->__crt_alg, struct safexcel_alg_template,
			     alg.skcipher.base);

767 768
	crypto_skcipher_set_reqsize(__crypto_skcipher_cast(tfm),
				    sizeof(struct safexcel_cipher_req));
769

770 771 772 773
	ctx->priv = tmpl->priv;

	ctx->base.send = safexcel_skcipher_send;
	ctx->base.handle_result = safexcel_skcipher_handle_result;
774 775 776
	return 0;
}

777
static int safexcel_cipher_cra_exit(struct crypto_tfm *tfm)
778 779 780
{
	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);

781
	memzero_explicit(ctx->key, sizeof(ctx->key));
782 783 784

	/* context not allocated, skip invalidation */
	if (!ctx->base.ctxr)
785
		return -ENOMEM;
786

787
	memzero_explicit(ctx->base.ctxr->data, sizeof(ctx->base.ctxr->data));
788 789 790 791 792 793 794 795 796 797 798
	return 0;
}

static void safexcel_skcipher_cra_exit(struct crypto_tfm *tfm)
{
	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
	struct safexcel_crypto_priv *priv = ctx->priv;
	int ret;

	if (safexcel_cipher_cra_exit(tfm))
		return;
799

800
	if (priv->version == EIP197) {
801
		ret = safexcel_skcipher_exit_inv(tfm);
802
		if (ret)
803 804
			dev_warn(priv->dev, "skcipher: invalidation error %d\n",
				 ret);
805 806 807 808
	} else {
		dma_pool_free(priv->context_pool, ctx->base.ctxr,
			      ctx->base.ctxr_dma);
	}
809 810
}

811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830
static void safexcel_aead_cra_exit(struct crypto_tfm *tfm)
{
	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
	struct safexcel_crypto_priv *priv = ctx->priv;
	int ret;

	if (safexcel_cipher_cra_exit(tfm))
		return;

	if (priv->version == EIP197) {
		ret = safexcel_aead_exit_inv(tfm);
		if (ret)
			dev_warn(priv->dev, "aead: invalidation error %d\n",
				 ret);
	} else {
		dma_pool_free(priv->context_pool, ctx->base.ctxr,
			      ctx->base.ctxr_dma);
	}
}

831 832 833
struct safexcel_alg_template safexcel_alg_ecb_aes = {
	.type = SAFEXCEL_ALG_TYPE_SKCIPHER,
	.alg.skcipher = {
834
		.setkey = safexcel_skcipher_aes_setkey,
835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856
		.encrypt = safexcel_ecb_aes_encrypt,
		.decrypt = safexcel_ecb_aes_decrypt,
		.min_keysize = AES_MIN_KEY_SIZE,
		.max_keysize = AES_MAX_KEY_SIZE,
		.base = {
			.cra_name = "ecb(aes)",
			.cra_driver_name = "safexcel-ecb-aes",
			.cra_priority = 300,
			.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC |
				     CRYPTO_ALG_KERN_DRIVER_ONLY,
			.cra_blocksize = AES_BLOCK_SIZE,
			.cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
			.cra_alignmask = 0,
			.cra_init = safexcel_skcipher_cra_init,
			.cra_exit = safexcel_skcipher_cra_exit,
			.cra_module = THIS_MODULE,
		},
	},
};

static int safexcel_cbc_aes_encrypt(struct skcipher_request *req)
{
857 858
	return safexcel_aes(&req->base, skcipher_request_ctx(req),
			    SAFEXCEL_ENCRYPT, CONTEXT_CONTROL_CRYPTO_MODE_CBC);
859 860 861 862
}

static int safexcel_cbc_aes_decrypt(struct skcipher_request *req)
{
863 864
	return safexcel_aes(&req->base, skcipher_request_ctx(req),
			    SAFEXCEL_DECRYPT, CONTEXT_CONTROL_CRYPTO_MODE_CBC);
865 866 867 868 869
}

struct safexcel_alg_template safexcel_alg_cbc_aes = {
	.type = SAFEXCEL_ALG_TYPE_SKCIPHER,
	.alg.skcipher = {
870
		.setkey = safexcel_skcipher_aes_setkey,
871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890
		.encrypt = safexcel_cbc_aes_encrypt,
		.decrypt = safexcel_cbc_aes_decrypt,
		.min_keysize = AES_MIN_KEY_SIZE,
		.max_keysize = AES_MAX_KEY_SIZE,
		.ivsize = AES_BLOCK_SIZE,
		.base = {
			.cra_name = "cbc(aes)",
			.cra_driver_name = "safexcel-cbc-aes",
			.cra_priority = 300,
			.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC |
				     CRYPTO_ALG_KERN_DRIVER_ONLY,
			.cra_blocksize = AES_BLOCK_SIZE,
			.cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
			.cra_alignmask = 0,
			.cra_init = safexcel_skcipher_cra_init,
			.cra_exit = safexcel_skcipher_cra_exit,
			.cra_module = THIS_MODULE,
		},
	},
};
891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925

static int safexcel_aead_encrypt(struct aead_request *req)
{
	struct safexcel_cipher_req *creq = aead_request_ctx(req);

	return safexcel_aes(&req->base, creq, SAFEXCEL_ENCRYPT,
			    CONTEXT_CONTROL_CRYPTO_MODE_CBC);
}

static int safexcel_aead_decrypt(struct aead_request *req)
{
	struct safexcel_cipher_req *creq = aead_request_ctx(req);

	return safexcel_aes(&req->base, creq, SAFEXCEL_DECRYPT,
			    CONTEXT_CONTROL_CRYPTO_MODE_CBC);
}

static int safexcel_aead_cra_init(struct crypto_tfm *tfm)
{
	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
	struct safexcel_alg_template *tmpl =
		container_of(tfm->__crt_alg, struct safexcel_alg_template,
			     alg.aead.base);

	crypto_aead_set_reqsize(__crypto_aead_cast(tfm),
				sizeof(struct safexcel_cipher_req));

	ctx->priv = tmpl->priv;

	ctx->aead = true;
	ctx->base.send = safexcel_aead_send;
	ctx->base.handle_result = safexcel_aead_handle_result;
	return 0;
}

926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959
static int safexcel_aead_sha1_cra_init(struct crypto_tfm *tfm)
{
	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);

	safexcel_aead_cra_init(tfm);
	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1;
	ctx->state_sz = SHA1_DIGEST_SIZE;
	return 0;
}

struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_aes = {
	.type = SAFEXCEL_ALG_TYPE_AEAD,
	.alg.aead = {
		.setkey = safexcel_aead_aes_setkey,
		.encrypt = safexcel_aead_encrypt,
		.decrypt = safexcel_aead_decrypt,
		.ivsize = AES_BLOCK_SIZE,
		.maxauthsize = SHA1_DIGEST_SIZE,
		.base = {
			.cra_name = "authenc(hmac(sha1),cbc(aes))",
			.cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-aes",
			.cra_priority = 300,
			.cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC |
				     CRYPTO_ALG_KERN_DRIVER_ONLY,
			.cra_blocksize = AES_BLOCK_SIZE,
			.cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
			.cra_alignmask = 0,
			.cra_init = safexcel_aead_sha1_cra_init,
			.cra_exit = safexcel_aead_cra_exit,
			.cra_module = THIS_MODULE,
		},
	},
};

960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992
static int safexcel_aead_sha256_cra_init(struct crypto_tfm *tfm)
{
	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);

	safexcel_aead_cra_init(tfm);
	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256;
	ctx->state_sz = SHA256_DIGEST_SIZE;
	return 0;
}

struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_aes = {
	.type = SAFEXCEL_ALG_TYPE_AEAD,
	.alg.aead = {
		.setkey = safexcel_aead_aes_setkey,
		.encrypt = safexcel_aead_encrypt,
		.decrypt = safexcel_aead_decrypt,
		.ivsize = AES_BLOCK_SIZE,
		.maxauthsize = SHA256_DIGEST_SIZE,
		.base = {
			.cra_name = "authenc(hmac(sha256),cbc(aes))",
			.cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-aes",
			.cra_priority = 300,
			.cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC |
				     CRYPTO_ALG_KERN_DRIVER_ONLY,
			.cra_blocksize = AES_BLOCK_SIZE,
			.cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
			.cra_alignmask = 0,
			.cra_init = safexcel_aead_sha256_cra_init,
			.cra_exit = safexcel_aead_cra_exit,
			.cra_module = THIS_MODULE,
		},
	},
};
993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026

static int safexcel_aead_sha224_cra_init(struct crypto_tfm *tfm)
{
	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);

	safexcel_aead_cra_init(tfm);
	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224;
	ctx->state_sz = SHA256_DIGEST_SIZE;
	return 0;
}

struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_aes = {
	.type = SAFEXCEL_ALG_TYPE_AEAD,
	.alg.aead = {
		.setkey = safexcel_aead_aes_setkey,
		.encrypt = safexcel_aead_encrypt,
		.decrypt = safexcel_aead_decrypt,
		.ivsize = AES_BLOCK_SIZE,
		.maxauthsize = SHA224_DIGEST_SIZE,
		.base = {
			.cra_name = "authenc(hmac(sha224),cbc(aes))",
			.cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-aes",
			.cra_priority = 300,
			.cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC |
				     CRYPTO_ALG_KERN_DRIVER_ONLY,
			.cra_blocksize = AES_BLOCK_SIZE,
			.cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
			.cra_alignmask = 0,
			.cra_init = safexcel_aead_sha224_cra_init,
			.cra_exit = safexcel_aead_cra_exit,
			.cra_module = THIS_MODULE,
		},
	},
};
1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060

static int safexcel_aead_sha512_cra_init(struct crypto_tfm *tfm)
{
	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);

	safexcel_aead_cra_init(tfm);
	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512;
	ctx->state_sz = SHA512_DIGEST_SIZE;
	return 0;
}

struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_aes = {
	.type = SAFEXCEL_ALG_TYPE_AEAD,
	.alg.aead = {
		.setkey = safexcel_aead_aes_setkey,
		.encrypt = safexcel_aead_encrypt,
		.decrypt = safexcel_aead_decrypt,
		.ivsize = AES_BLOCK_SIZE,
		.maxauthsize = SHA512_DIGEST_SIZE,
		.base = {
			.cra_name = "authenc(hmac(sha512),cbc(aes))",
			.cra_driver_name = "safexcel-authenc-hmac-sha512-cbc-aes",
			.cra_priority = 300,
			.cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC |
				     CRYPTO_ALG_KERN_DRIVER_ONLY,
			.cra_blocksize = AES_BLOCK_SIZE,
			.cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
			.cra_alignmask = 0,
			.cra_init = safexcel_aead_sha512_cra_init,
			.cra_exit = safexcel_aead_cra_exit,
			.cra_module = THIS_MODULE,
		},
	},
};