27 	struct qce_aead_reqctx *rctx = aead_request_ctx(req);  in qce_aead_done()  local
49 		dma_unmap_sg(qce->dev, rctx->src_sg, rctx->src_nents, dir_src);  in qce_aead_done()
51 	dma_unmap_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst);  in qce_aead_done()
53 	if (IS_CCM(rctx->flags)) {  in qce_aead_done()
55 			sg_free_table(&rctx->src_tbl);  in qce_aead_done()
57 				sg_free_table(&rctx->dst_tbl);  in qce_aead_done()
59 			if (!(IS_DECRYPT(rctx->flags) && !diff_dst))  in qce_aead_done()
60 				sg_free_table(&rctx->dst_tbl);  in qce_aead_done()
63 		sg_free_table(&rctx->dst_tbl);  in qce_aead_done()
70 	if (IS_ENCRYPT(rctx->flags)) {  in qce_aead_done()
72 		if (IS_CCM(rctx->flags))  in qce_aead_done()
73 			scatterwalk_map_and_copy(rctx->ccmresult_buf, req->dst,  in qce_aead_done()
79 	} else if (!IS_CCM(rctx->flags)) {  in qce_aead_done()
95 	struct qce_aead_reqctx *rctx = aead_request_ctx(req);  in qce_aead_prepare_result_buf()  local
99 	sg_init_one(&rctx->result_sg, qce->dma.result_buf, QCE_RESULT_BUF_SZ);  in qce_aead_prepare_result_buf()
100 	return qce_sgtable_add(tbl, &rctx->result_sg, QCE_RESULT_BUF_SZ);  in qce_aead_prepare_result_buf()
106 	struct qce_aead_reqctx *rctx = aead_request_ctx(req);  in qce_aead_prepare_ccm_result_buf()  local
108 	sg_init_one(&rctx->result_sg, rctx->ccmresult_buf, QCE_BAM_BURST_SIZE);  in qce_aead_prepare_ccm_result_buf()
109 	return qce_sgtable_add(tbl, &rctx->result_sg, QCE_BAM_BURST_SIZE);  in qce_aead_prepare_ccm_result_buf()
115 	struct qce_aead_reqctx *rctx = aead_request_ctx(req);  in qce_aead_prepare_dst_buf()  local
124 	totallen = rctx->cryptlen + assoclen;  in qce_aead_prepare_dst_buf()
125 	rctx->dst_nents = sg_nents_for_len(req->dst, totallen);  in qce_aead_prepare_dst_buf()
126 	if (rctx->dst_nents < 0) {  in qce_aead_prepare_dst_buf()
130 	if (IS_CCM(rctx->flags))  in qce_aead_prepare_dst_buf()
131 		rctx->dst_nents += 2;  in qce_aead_prepare_dst_buf()
133 		rctx->dst_nents += 1;  in qce_aead_prepare_dst_buf()
137 	ret = sg_alloc_table(&rctx->dst_tbl, rctx->dst_nents, gfp);  in qce_aead_prepare_dst_buf()
141 	if (IS_CCM(rctx->flags) && assoclen) {  in qce_aead_prepare_dst_buf()
145 		sg = qce_sgtable_add(&rctx->dst_tbl, &rctx->adata_sg,  in qce_aead_prepare_dst_buf()
146 				     rctx->assoclen);  in qce_aead_prepare_dst_buf()
152 		sg = qce_sgtable_add(&rctx->dst_tbl, msg_sg, rctx->cryptlen);  in qce_aead_prepare_dst_buf()
157 		totallen = rctx->cryptlen + rctx->assoclen;  in qce_aead_prepare_dst_buf()
160 			sg = qce_sgtable_add(&rctx->dst_tbl, req->dst, totallen);  in qce_aead_prepare_dst_buf()
165 	if (IS_CCM(rctx->flags))  in qce_aead_prepare_dst_buf()
166 		sg = qce_aead_prepare_ccm_result_buf(&rctx->dst_tbl, req);  in qce_aead_prepare_dst_buf()
168 		sg = qce_aead_prepare_result_buf(&rctx->dst_tbl, req);  in qce_aead_prepare_dst_buf()
174 	rctx->dst_sg = rctx->dst_tbl.sgl;  in qce_aead_prepare_dst_buf()
175 	rctx->dst_nents = sg_nents_for_len(rctx->dst_sg, totallen) + 1;  in qce_aead_prepare_dst_buf()
180 	sg_free_table(&rctx->dst_tbl);  in qce_aead_prepare_dst_buf()
189 	struct qce_aead_reqctx *rctx = aead_request_ctx(req);  in qce_aead_ccm_prepare_buf_assoclen()  local
191 	unsigned int assoclen = rctx->assoclen;  in qce_aead_ccm_prepare_buf_assoclen()
197 	if (IS_DECRYPT(rctx->flags))  in qce_aead_ccm_prepare_buf_assoclen()
198 		cryptlen = rctx->cryptlen + ctx->authsize;  in qce_aead_ccm_prepare_buf_assoclen()
200 		cryptlen = rctx->cryptlen;  in qce_aead_ccm_prepare_buf_assoclen()
206 	rctx->adata = kzalloc((ALIGN(assoclen, 16) + MAX_CCM_ADATA_HEADER_LEN) *  in qce_aead_ccm_prepare_buf_assoclen()
208 	if (!rctx->adata)  in qce_aead_ccm_prepare_buf_assoclen()
220 		*(__be16 *)rctx->adata = cpu_to_be16(assoclen);  in qce_aead_ccm_prepare_buf_assoclen()
223 		*(__be16 *)rctx->adata = cpu_to_be16(0xfffe);  in qce_aead_ccm_prepare_buf_assoclen()
224 		*(__be32 *)(rctx->adata + 2) = cpu_to_be32(assoclen);  in qce_aead_ccm_prepare_buf_assoclen()
229 			      rctx->adata + adata_header_len,  in qce_aead_ccm_prepare_buf_assoclen()
234 	rctx->assoclen = ALIGN(assoclen + adata_header_len, 16);  in qce_aead_ccm_prepare_buf_assoclen()
239 		rctx->src_nents = sg_nents_for_len(req->src, totallen) + 1;  in qce_aead_ccm_prepare_buf_assoclen()
241 		rctx->src_nents = sg_nents_for_len(req->src, totallen) + 2;  in qce_aead_ccm_prepare_buf_assoclen()
244 	ret = sg_alloc_table(&rctx->src_tbl, rctx->src_nents, gfp);  in qce_aead_ccm_prepare_buf_assoclen()
249 	sg_init_one(&rctx->adata_sg, rctx->adata, rctx->assoclen);  in qce_aead_ccm_prepare_buf_assoclen()
250 	sg = qce_sgtable_add(&rctx->src_tbl, &rctx->adata_sg,  in qce_aead_ccm_prepare_buf_assoclen()
251 			     rctx->assoclen);  in qce_aead_ccm_prepare_buf_assoclen()
257 	sg = qce_sgtable_add(&rctx->src_tbl, msg_sg, cryptlen);  in qce_aead_ccm_prepare_buf_assoclen()
268 		if (!IS_DECRYPT(rctx->flags)) {  in qce_aead_ccm_prepare_buf_assoclen()
269 			sg = qce_aead_prepare_ccm_result_buf(&rctx->src_tbl, req);  in qce_aead_ccm_prepare_buf_assoclen()
277 	rctx->src_sg = rctx->src_tbl.sgl;  in qce_aead_ccm_prepare_buf_assoclen()
278 	totallen = cryptlen + rctx->assoclen;  in qce_aead_ccm_prepare_buf_assoclen()
279 	rctx->src_nents = sg_nents_for_len(rctx->src_sg, totallen);  in qce_aead_ccm_prepare_buf_assoclen()
288 		if (IS_ENCRYPT(rctx->flags))  in qce_aead_ccm_prepare_buf_assoclen()
289 			rctx->dst_nents = rctx->src_nents + 1;  in qce_aead_ccm_prepare_buf_assoclen()
291 			rctx->dst_nents = rctx->src_nents;  in qce_aead_ccm_prepare_buf_assoclen()
292 		rctx->dst_sg = rctx->src_sg;  in qce_aead_ccm_prepare_buf_assoclen()
297 	sg_free_table(&rctx->src_tbl);  in qce_aead_ccm_prepare_buf_assoclen()
303 	struct qce_aead_reqctx *rctx = aead_request_ctx(req);  in qce_aead_prepare_buf()  local
310 	totallen = rctx->cryptlen + rctx->assoclen;  in qce_aead_prepare_buf()
316 		rctx->src_nents = sg_nents_for_len(req->src, totallen);  in qce_aead_prepare_buf()
317 		if (rctx->src_nents < 0) {  in qce_aead_prepare_buf()
321 		rctx->src_sg = req->src;  in qce_aead_prepare_buf()
323 		rctx->src_nents = rctx->dst_nents - 1;  in qce_aead_prepare_buf()
324 		rctx->src_sg = rctx->dst_sg;  in qce_aead_prepare_buf()
331 	struct qce_aead_reqctx *rctx = aead_request_ctx(req);  in qce_aead_ccm_prepare_buf()  local
338 	if (rctx->assoclen)  in qce_aead_ccm_prepare_buf()
341 	if (IS_ENCRYPT(rctx->flags))  in qce_aead_ccm_prepare_buf()
344 	cryptlen = rctx->cryptlen + ctx->authsize;  in qce_aead_ccm_prepare_buf()
346 		rctx->src_nents = sg_nents_for_len(req->src, cryptlen);  in qce_aead_ccm_prepare_buf()
347 		rctx->src_sg = req->src;  in qce_aead_ccm_prepare_buf()
352 		rctx->src_nents = sg_nents_for_len(req->src, cryptlen);  in qce_aead_ccm_prepare_buf()
353 		rctx->src_sg = req->src;  in qce_aead_ccm_prepare_buf()
354 		rctx->dst_nents = rctx->src_nents;  in qce_aead_ccm_prepare_buf()
355 		rctx->dst_sg = rctx->src_sg;  in qce_aead_ccm_prepare_buf()
361 static int qce_aead_create_ccm_nonce(struct qce_aead_reqctx *rctx, struct qce_aead_ctx *ctx)  in qce_aead_create_ccm_nonce()  argument
367 	if (!rctx || !rctx->iv)  in qce_aead_create_ccm_nonce()
370 	msglen_size = rctx->iv[0] + 1;  in qce_aead_create_ccm_nonce()
376 	ivsize = rctx->ivsize;  in qce_aead_create_ccm_nonce()
382 	if (!IS_CCM_RFC4309(rctx->flags)) {  in qce_aead_create_ccm_nonce()
384 			rctx->iv[ivsize - i - 1] = 0;  in qce_aead_create_ccm_nonce()
394 	memcpy(&msg_len[0], &rctx->cryptlen, 4);  in qce_aead_create_ccm_nonce()
396 	memcpy(&rctx->ccm_nonce[0], rctx->iv, rctx->ivsize);  in qce_aead_create_ccm_nonce()
397 	if (rctx->assoclen)  in qce_aead_create_ccm_nonce()
398 		rctx->ccm_nonce[0] |= 1 << CCM_NONCE_ADATA_SHIFT;  in qce_aead_create_ccm_nonce()
399 	rctx->ccm_nonce[0] |= ((ctx->authsize - 2) / 2) <<  in qce_aead_create_ccm_nonce()
402 		rctx->ccm_nonce[QCE_MAX_NONCE - i - 1] = msg_len[i];  in qce_aead_create_ccm_nonce()
411 	struct qce_aead_reqctx *rctx = aead_request_ctx(req);  in qce_aead_async_req_handle()  local
420 	if (IS_CCM_RFC4309(rctx->flags)) {  in qce_aead_async_req_handle()
421 		memset(rctx->ccm_rfc4309_iv, 0, QCE_MAX_IV_SIZE);  in qce_aead_async_req_handle()
422 		rctx->ccm_rfc4309_iv[0] = 3;  in qce_aead_async_req_handle()
423 		memcpy(&rctx->ccm_rfc4309_iv[1], ctx->ccm4309_salt, QCE_CCM4309_SALT_SIZE);  in qce_aead_async_req_handle()
424 		memcpy(&rctx->ccm_rfc4309_iv[4], req->iv, 8);  in qce_aead_async_req_handle()
425 		rctx->iv = rctx->ccm_rfc4309_iv;  in qce_aead_async_req_handle()
426 		rctx->ivsize = AES_BLOCK_SIZE;  in qce_aead_async_req_handle()
428 		rctx->iv = req->iv;  in qce_aead_async_req_handle()
429 		rctx->ivsize = crypto_aead_ivsize(tfm);  in qce_aead_async_req_handle()
431 	if (IS_CCM_RFC4309(rctx->flags))  in qce_aead_async_req_handle()
432 		rctx->assoclen = req->assoclen - 8;  in qce_aead_async_req_handle()
434 		rctx->assoclen = req->assoclen;  in qce_aead_async_req_handle()
440 	if (IS_CCM(rctx->flags)) {  in qce_aead_async_req_handle()
441 		ret = qce_aead_create_ccm_nonce(rctx, ctx);  in qce_aead_async_req_handle()
445 	if (IS_CCM(rctx->flags))  in qce_aead_async_req_handle()
452 	dst_nents = dma_map_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst);  in qce_aead_async_req_handle()
459 		src_nents = dma_map_sg(qce->dev, rctx->src_sg, rctx->src_nents, dir_src);  in qce_aead_async_req_handle()
465 		if (IS_CCM(rctx->flags) && IS_DECRYPT(rctx->flags))  in qce_aead_async_req_handle()
471 	ret = qce_dma_prep_sgs(&qce->dma, rctx->src_sg, src_nents, rctx->dst_sg, dst_nents,  in qce_aead_async_req_handle()
488 		dma_unmap_sg(qce->dev, req->src, rctx->src_nents, dir_src);  in qce_aead_async_req_handle()
490 	dma_unmap_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst);  in qce_aead_async_req_handle()
492 	if (IS_CCM(rctx->flags) && rctx->assoclen) {  in qce_aead_async_req_handle()
493 		sg_free_table(&rctx->src_tbl);  in qce_aead_async_req_handle()
495 			sg_free_table(&rctx->dst_tbl);  in qce_aead_async_req_handle()
497 		sg_free_table(&rctx->dst_tbl);  in qce_aead_async_req_handle()
505 	struct qce_aead_reqctx *rctx = aead_request_ctx(req);  in qce_aead_crypt()  local
510 	rctx->flags  = tmpl->alg_flags;  in qce_aead_crypt()
511 	rctx->flags |= encrypt ? QCE_ENCRYPT : QCE_DECRYPT;  in qce_aead_crypt()
514 		rctx->cryptlen = req->cryptlen;  in qce_aead_crypt()
516 		rctx->cryptlen = req->cryptlen - ctx->authsize;  in qce_aead_crypt()
519 	if (!rctx->cryptlen) {  in qce_aead_crypt()
520 		if (!(IS_CCM(rctx->flags) && IS_DECRYPT(rctx->flags)))  in qce_aead_crypt()
529 		aead_request_set_tfm(&rctx->fallback_req, ctx->fallback);  in qce_aead_crypt()
530 		aead_request_set_callback(&rctx->fallback_req, req->base.flags,  in qce_aead_crypt()
532 		aead_request_set_crypt(&rctx->fallback_req, req->src,  in qce_aead_crypt()
534 		aead_request_set_ad(&rctx->fallback_req, req->assoclen);  in qce_aead_crypt()
536 		return encrypt ? crypto_aead_encrypt(&rctx->fallback_req) :  in qce_aead_crypt()
537 				 crypto_aead_decrypt(&rctx->fallback_req);  in qce_aead_crypt()
544 	if (IS_CBC(rctx->flags) && !IS_ALIGNED(rctx->cryptlen, blocksize))  in qce_aead_crypt()
548 	if (IS_CCM_RFC4309(rctx->flags))  in qce_aead_crypt()