Lines Matching refs:rctx
134 struct iproc_reqctx_s *rctx, in spu_ablkcipher_rx_sg_create() argument
140 struct iproc_ctx_s *ctx = rctx->ctx; in spu_ablkcipher_rx_sg_create()
144 rctx->gfp); in spu_ablkcipher_rx_sg_create()
151 sg_set_buf(sg++, rctx->msg_buf.spu_resp_hdr, ctx->spu_resp_hdr_len); in spu_ablkcipher_rx_sg_create()
156 sg_set_buf(sg++, rctx->msg_buf.c.supdt_tweak, in spu_ablkcipher_rx_sg_create()
160 datalen = spu_msg_sg_add(&sg, &rctx->dst_sg, &rctx->dst_skip, in spu_ablkcipher_rx_sg_create()
161 rctx->dst_nents, chunksize); in spu_ablkcipher_rx_sg_create()
170 sg_set_buf(sg++, rctx->msg_buf.c.supdt_tweak, SPU_SUPDT_LEN); in spu_ablkcipher_rx_sg_create()
173 sg_set_buf(sg++, rctx->msg_buf.rx_stat_pad, stat_pad_len); in spu_ablkcipher_rx_sg_create()
175 memset(rctx->msg_buf.rx_stat, 0, SPU_RX_STATUS_LEN); in spu_ablkcipher_rx_sg_create()
176 sg_set_buf(sg, rctx->msg_buf.rx_stat, spu->spu_rx_status_len()); in spu_ablkcipher_rx_sg_create()
202 struct iproc_reqctx_s *rctx, in spu_ablkcipher_tx_sg_create() argument
207 struct iproc_ctx_s *ctx = rctx->ctx; in spu_ablkcipher_tx_sg_create()
212 rctx->gfp); in spu_ablkcipher_tx_sg_create()
219 sg_set_buf(sg++, rctx->msg_buf.bcm_spu_req_hdr, in spu_ablkcipher_tx_sg_create()
225 sg_set_buf(sg++, rctx->msg_buf.iv_ctr, SPU_XTS_TWEAK_SIZE); in spu_ablkcipher_tx_sg_create()
228 datalen = spu_msg_sg_add(&sg, &rctx->src_sg, &rctx->src_skip, in spu_ablkcipher_tx_sg_create()
229 rctx->src_nents, chunksize); in spu_ablkcipher_tx_sg_create()
237 sg_set_buf(sg++, rctx->msg_buf.spu_req_pad, pad_len); in spu_ablkcipher_tx_sg_create()
241 memset(rctx->msg_buf.tx_stat, 0, stat_len); in spu_ablkcipher_tx_sg_create()
242 sg_set_buf(sg, rctx->msg_buf.tx_stat, stat_len); in spu_ablkcipher_tx_sg_create()
303 static int handle_ablkcipher_req(struct iproc_reqctx_s *rctx) in handle_ablkcipher_req() argument
306 struct crypto_async_request *areq = rctx->parent; in handle_ablkcipher_req()
309 struct iproc_ctx_s *ctx = rctx->ctx; in handle_ablkcipher_req()
335 cipher_parms.iv_len = rctx->iv_ctr_len; in handle_ablkcipher_req()
337 mssg = &rctx->mb_mssg; in handle_ablkcipher_req()
338 chunk_start = rctx->src_sent; in handle_ablkcipher_req()
339 remaining = rctx->total_todo - chunk_start; in handle_ablkcipher_req()
348 rctx->src_sent += chunksize; in handle_ablkcipher_req()
349 rctx->total_sent = rctx->src_sent; in handle_ablkcipher_req()
352 rctx->src_nents = spu_sg_count(rctx->src_sg, rctx->src_skip, chunksize); in handle_ablkcipher_req()
353 rctx->dst_nents = spu_sg_count(rctx->dst_sg, rctx->dst_skip, chunksize); in handle_ablkcipher_req()
356 rctx->is_encrypt && chunk_start) in handle_ablkcipher_req()
361 sg_copy_part_to_buf(req->dst, rctx->msg_buf.iv_ctr, in handle_ablkcipher_req()
362 rctx->iv_ctr_len, in handle_ablkcipher_req()
363 chunk_start - rctx->iv_ctr_len); in handle_ablkcipher_req()
365 if (rctx->iv_ctr_len) { in handle_ablkcipher_req()
367 __builtin_memcpy(local_iv_ctr, rctx->msg_buf.iv_ctr, in handle_ablkcipher_req()
368 rctx->iv_ctr_len); in handle_ablkcipher_req()
372 !rctx->is_encrypt) { in handle_ablkcipher_req()
377 sg_copy_part_to_buf(req->src, rctx->msg_buf.iv_ctr, in handle_ablkcipher_req()
378 rctx->iv_ctr_len, in handle_ablkcipher_req()
379 rctx->src_sent - rctx->iv_ctr_len); in handle_ablkcipher_req()
390 add_to_ctr(rctx->msg_buf.iv_ctr, chunksize >> 4); in handle_ablkcipher_req()
401 cipher_parms.key_buf = rctx->msg_buf.c.supdt_tweak; in handle_ablkcipher_req()
404 } else if (!rctx->is_encrypt) { in handle_ablkcipher_req()
422 rctx->src_sent, chunk_start, remaining, chunksize); in handle_ablkcipher_req()
425 memcpy(rctx->msg_buf.bcm_spu_req_hdr, ctx->bcm_spu_req_hdr, in handle_ablkcipher_req()
426 sizeof(rctx->msg_buf.bcm_spu_req_hdr)); in handle_ablkcipher_req()
433 spu->spu_cipher_req_finish(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN, in handle_ablkcipher_req()
434 ctx->spu_req_hdr_len, !(rctx->is_encrypt), in handle_ablkcipher_req()
445 spu->spu_request_pad(rctx->msg_buf.spu_req_pad, 0, in handle_ablkcipher_req()
447 rctx->total_sent, stat_pad_len); in handle_ablkcipher_req()
450 spu->spu_dump_msg_hdr(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN, in handle_ablkcipher_req()
453 dump_sg(rctx->src_sg, rctx->src_skip, chunksize); in handle_ablkcipher_req()
454 packet_dump(" pad: ", rctx->msg_buf.spu_req_pad, pad_len); in handle_ablkcipher_req()
462 mssg->ctx = rctx; /* Will be returned in response */ in handle_ablkcipher_req()
465 rx_frag_num += rctx->dst_nents; in handle_ablkcipher_req()
471 err = spu_ablkcipher_rx_sg_create(mssg, rctx, rx_frag_num, chunksize, in handle_ablkcipher_req()
477 tx_frag_num += rctx->src_nents; in handle_ablkcipher_req()
485 err = spu_ablkcipher_tx_sg_create(mssg, rctx, tx_frag_num, chunksize, in handle_ablkcipher_req()
490 err = mailbox_send_message(mssg, req->base.flags, rctx->chan_idx); in handle_ablkcipher_req()
502 static void handle_ablkcipher_resp(struct iproc_reqctx_s *rctx) in handle_ablkcipher_resp() argument
506 struct crypto_async_request *areq = rctx->parent; in handle_ablkcipher_resp()
509 struct iproc_ctx_s *ctx = rctx->ctx; in handle_ablkcipher_resp()
513 payload_len = spu->spu_payload_length(rctx->msg_buf.spu_resp_hdr); in handle_ablkcipher_resp()
527 __func__, rctx->total_received, payload_len); in handle_ablkcipher_resp()
529 dump_sg(req->dst, rctx->total_received, payload_len); in handle_ablkcipher_resp()
531 packet_dump(" supdt ", rctx->msg_buf.c.supdt_tweak, in handle_ablkcipher_resp()
534 rctx->total_received += payload_len; in handle_ablkcipher_resp()
535 if (rctx->total_received == rctx->total_todo) { in handle_ablkcipher_resp()
563 struct iproc_reqctx_s *rctx, in spu_ahash_rx_sg_create() argument
569 struct iproc_ctx_s *ctx = rctx->ctx; in spu_ahash_rx_sg_create()
572 rctx->gfp); in spu_ahash_rx_sg_create()
579 sg_set_buf(sg++, rctx->msg_buf.spu_resp_hdr, ctx->spu_resp_hdr_len); in spu_ahash_rx_sg_create()
582 sg_set_buf(sg++, rctx->msg_buf.digest, digestsize); in spu_ahash_rx_sg_create()
585 sg_set_buf(sg++, rctx->msg_buf.rx_stat_pad, stat_pad_len); in spu_ahash_rx_sg_create()
587 memset(rctx->msg_buf.rx_stat, 0, SPU_RX_STATUS_LEN); in spu_ahash_rx_sg_create()
588 sg_set_buf(sg, rctx->msg_buf.rx_stat, spu->spu_rx_status_len()); in spu_ahash_rx_sg_create()
615 struct iproc_reqctx_s *rctx, in spu_ahash_tx_sg_create() argument
627 rctx->gfp); in spu_ahash_tx_sg_create()
634 sg_set_buf(sg++, rctx->msg_buf.bcm_spu_req_hdr, in spu_ahash_tx_sg_create()
638 sg_set_buf(sg++, rctx->hash_carry, hash_carry_len); in spu_ahash_tx_sg_create()
642 datalen = spu_msg_sg_add(&sg, &rctx->src_sg, &rctx->src_skip, in spu_ahash_tx_sg_create()
643 rctx->src_nents, new_data_len); in spu_ahash_tx_sg_create()
652 sg_set_buf(sg++, rctx->msg_buf.spu_req_pad, pad_len); in spu_ahash_tx_sg_create()
656 memset(rctx->msg_buf.tx_stat, 0, stat_len); in spu_ahash_tx_sg_create()
657 sg_set_buf(sg, rctx->msg_buf.tx_stat, stat_len); in spu_ahash_tx_sg_create()
689 static int handle_ahash_req(struct iproc_reqctx_s *rctx) in handle_ahash_req() argument
692 struct crypto_async_request *areq = rctx->parent; in handle_ahash_req()
697 struct iproc_ctx_s *ctx = rctx->ctx; in handle_ahash_req()
732 rctx->total_todo, rctx->total_sent); in handle_ahash_req()
755 mssg = &rctx->mb_mssg; in handle_ahash_req()
756 chunk_start = rctx->src_sent; in handle_ahash_req()
762 nbytes_to_hash = rctx->total_todo - rctx->total_sent; in handle_ahash_req()
773 if (!rctx->is_final) { in handle_ahash_req()
774 u8 *dest = rctx->hash_carry + rctx->hash_carry_len; in handle_ahash_req()
783 new_len = rem - rctx->hash_carry_len; in handle_ahash_req()
785 rctx->src_sent); in handle_ahash_req()
786 rctx->hash_carry_len = rem; in handle_ahash_req()
788 rctx->hash_carry_len); in handle_ahash_req()
790 rctx->hash_carry, in handle_ahash_req()
791 rctx->hash_carry_len); in handle_ahash_req()
798 local_nbuf = rctx->hash_carry_len; in handle_ahash_req()
799 rctx->hash_carry_len = 0; in handle_ahash_req()
805 rctx->src_nents = spu_sg_count(rctx->src_sg, rctx->src_skip, in handle_ahash_req()
812 hash_parms.type = spu->spu_hash_type(rctx->total_sent); in handle_ahash_req()
819 rctx->total_sent += chunksize; in handle_ahash_req()
821 rctx->src_sent += new_data_len; in handle_ahash_req()
823 if ((rctx->total_sent == rctx->total_todo) && rctx->is_final) in handle_ahash_req()
835 hash_parms.key_buf = rctx->incr_hash; in handle_ahash_req()
842 __func__, rctx->is_final, local_nbuf); in handle_ahash_req()
852 memcpy(rctx->msg_buf.bcm_spu_req_hdr, BCMHEADER, BCM_HDR_LEN); in handle_ahash_req()
855 spu_hdr_len = spu->spu_create_request(rctx->msg_buf.bcm_spu_req_hdr + in handle_ahash_req()
880 spu->spu_request_pad(rctx->msg_buf.spu_req_pad, data_pad_len, in handle_ahash_req()
882 ctx->auth.mode, rctx->total_sent, in handle_ahash_req()
886 spu->spu_dump_msg_hdr(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN, in handle_ahash_req()
888 packet_dump(" prebuf: ", rctx->hash_carry, local_nbuf); in handle_ahash_req()
890 dump_sg(rctx->src_sg, rctx->src_skip, new_data_len); in handle_ahash_req()
891 packet_dump(" pad: ", rctx->msg_buf.spu_req_pad, pad_len); in handle_ahash_req()
899 mssg->ctx = rctx; /* Will be returned in response */ in handle_ahash_req()
902 err = spu_ahash_rx_sg_create(mssg, rctx, rx_frag_num, digestsize, in handle_ahash_req()
908 tx_frag_num += rctx->src_nents; in handle_ahash_req()
911 err = spu_ahash_tx_sg_create(mssg, rctx, tx_frag_num, spu_hdr_len, in handle_ahash_req()
916 err = mailbox_send_message(mssg, req->base.flags, rctx->chan_idx); in handle_ahash_req()
980 static int ahash_req_done(struct iproc_reqctx_s *rctx) in ahash_req_done() argument
983 struct crypto_async_request *areq = rctx->parent; in ahash_req_done()
985 struct iproc_ctx_s *ctx = rctx->ctx; in ahash_req_done()
988 memcpy(req->result, rctx->msg_buf.digest, ctx->digestsize); in ahash_req_done()
1006 if (rctx->is_sw_hmac) { in ahash_req_done()
1013 if (rctx->is_sw_hmac || ctx->auth.mode == HASH_MODE_HMAC) { in ahash_req_done()
1030 static void handle_ahash_resp(struct iproc_reqctx_s *rctx) in handle_ahash_resp() argument
1032 struct iproc_ctx_s *ctx = rctx->ctx; in handle_ahash_resp()
1034 struct crypto_async_request *areq = rctx->parent; in handle_ahash_resp()
1044 memcpy(rctx->incr_hash, rctx->msg_buf.digest, MAX_DIGEST_SIZE); in handle_ahash_resp()
1051 if (rctx->is_final && (rctx->total_sent == rctx->total_todo)) in handle_ahash_resp()
1052 ahash_req_done(rctx); in handle_ahash_resp()
1081 struct iproc_reqctx_s *rctx, in spu_aead_rx_sg_create() argument
1089 struct iproc_ctx_s *ctx = rctx->ctx; in spu_aead_rx_sg_create()
1104 rctx->is_encrypt); in spu_aead_rx_sg_create()
1118 rctx->gfp); in spu_aead_rx_sg_create()
1126 sg_set_buf(sg++, rctx->msg_buf.spu_resp_hdr, ctx->spu_resp_hdr_len); in spu_aead_rx_sg_create()
1133 memset(rctx->msg_buf.a.resp_aad, 0, assoc_buf_len); in spu_aead_rx_sg_create()
1134 sg_set_buf(sg++, rctx->msg_buf.a.resp_aad, assoc_buf_len); in spu_aead_rx_sg_create()
1142 datalen = spu_msg_sg_add(&sg, &rctx->dst_sg, &rctx->dst_skip, in spu_aead_rx_sg_create()
1143 rctx->dst_nents, resp_len); in spu_aead_rx_sg_create()
1153 memset(rctx->msg_buf.a.gcmpad, 0, data_padlen); in spu_aead_rx_sg_create()
1154 sg_set_buf(sg++, rctx->msg_buf.a.gcmpad, data_padlen); in spu_aead_rx_sg_create()
1158 sg_set_buf(sg++, rctx->msg_buf.digest, digestsize); in spu_aead_rx_sg_create()
1162 memset(rctx->msg_buf.rx_stat_pad, 0, stat_pad_len); in spu_aead_rx_sg_create()
1163 sg_set_buf(sg++, rctx->msg_buf.rx_stat_pad, stat_pad_len); in spu_aead_rx_sg_create()
1166 memset(rctx->msg_buf.rx_stat, 0, SPU_RX_STATUS_LEN); in spu_aead_rx_sg_create()
1167 sg_set_buf(sg, rctx->msg_buf.rx_stat, spu->spu_rx_status_len()); in spu_aead_rx_sg_create()
1200 struct iproc_reqctx_s *rctx, in spu_aead_tx_sg_create() argument
1213 struct iproc_ctx_s *ctx = rctx->ctx; in spu_aead_tx_sg_create()
1220 rctx->gfp); in spu_aead_tx_sg_create()
1227 sg_set_buf(sg++, rctx->msg_buf.bcm_spu_req_hdr, in spu_aead_tx_sg_create()
1242 sg_set_buf(sg++, rctx->msg_buf.iv_ctr, aead_iv_len); in spu_aead_tx_sg_create()
1245 memset(rctx->msg_buf.a.req_aad_pad, 0, aad_pad_len); in spu_aead_tx_sg_create()
1246 sg_set_buf(sg++, rctx->msg_buf.a.req_aad_pad, aad_pad_len); in spu_aead_tx_sg_create()
1254 written = spu_msg_sg_add(&sg, &rctx->src_sg, &rctx->src_skip, in spu_aead_tx_sg_create()
1255 rctx->src_nents, datalen); in spu_aead_tx_sg_create()
1264 memset(rctx->msg_buf.spu_req_pad, 0, pad_len); in spu_aead_tx_sg_create()
1265 sg_set_buf(sg++, rctx->msg_buf.spu_req_pad, pad_len); in spu_aead_tx_sg_create()
1269 sg_set_buf(sg++, rctx->msg_buf.digest, ctx->digestsize); in spu_aead_tx_sg_create()
1273 memset(rctx->msg_buf.tx_stat, 0, stat_len); in spu_aead_tx_sg_create()
1274 sg_set_buf(sg, rctx->msg_buf.tx_stat, stat_len); in spu_aead_tx_sg_create()
1296 static int handle_aead_req(struct iproc_reqctx_s *rctx) in handle_aead_req() argument
1299 struct crypto_async_request *areq = rctx->parent; in handle_aead_req()
1302 struct iproc_ctx_s *ctx = rctx->ctx; in handle_aead_req()
1325 chunksize = rctx->total_todo; in handle_aead_req()
1333 req_opts.is_inbound = !(rctx->is_encrypt); in handle_aead_req()
1343 cipher_parms.iv_buf = rctx->msg_buf.iv_ctr; in handle_aead_req()
1344 cipher_parms.iv_len = rctx->iv_ctr_len; in handle_aead_req()
1366 if (rctx->is_encrypt) { in handle_aead_req()
1381 rctx->src_nents = spu_sg_count(rctx->src_sg, rctx->src_skip, chunksize); in handle_aead_req()
1382 rctx->dst_nents = spu_sg_count(rctx->dst_sg, rctx->dst_skip, chunksize); in handle_aead_req()
1384 assoc_nents = spu_sg_count(rctx->assoc, 0, in handle_aead_req()
1387 mssg = &rctx->mb_mssg; in handle_aead_req()
1389 rctx->total_sent = chunksize; in handle_aead_req()
1390 rctx->src_sent = chunksize; in handle_aead_req()
1394 rctx->is_encrypt)) in handle_aead_req()
1398 rctx->iv_ctr_len); in handle_aead_req()
1424 if (!rctx->is_encrypt) in handle_aead_req()
1431 chunksize, rctx->is_encrypt, in handle_aead_req()
1441 if (!rctx->is_encrypt) in handle_aead_req()
1454 if (spu_req_incl_icv(ctx->cipher.mode, rctx->is_encrypt)) { in handle_aead_req()
1458 sg_copy_part_to_buf(req->src, rctx->msg_buf.digest, digestsize, in handle_aead_req()
1459 req->assoclen + rctx->total_sent - in handle_aead_req()
1468 memcpy(rctx->msg_buf.bcm_spu_req_hdr, BCMHEADER, BCM_HDR_LEN); in handle_aead_req()
1470 spu_hdr_len = spu->spu_create_request(rctx->msg_buf.bcm_spu_req_hdr + in handle_aead_req()
1487 spu->spu_request_pad(rctx->msg_buf.spu_req_pad, in handle_aead_req()
1490 rctx->total_sent, stat_pad_len); in handle_aead_req()
1493 spu->spu_dump_msg_hdr(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN, in handle_aead_req()
1495 dump_sg(rctx->assoc, 0, aead_parms.assoc_size); in handle_aead_req()
1496 packet_dump(" aead iv: ", rctx->msg_buf.iv_ctr, aead_parms.iv_len); in handle_aead_req()
1498 dump_sg(rctx->src_sg, rctx->src_skip, chunksize); in handle_aead_req()
1499 packet_dump(" pad: ", rctx->msg_buf.spu_req_pad, pad_len); in handle_aead_req()
1507 mssg->ctx = rctx; /* Will be returned in response */ in handle_aead_req()
1510 rx_frag_num += rctx->dst_nents; in handle_aead_req()
1521 (ctx->cipher.mode == CIPHER_MODE_CCM)) && !rctx->is_encrypt) { in handle_aead_req()
1529 rx_frag_num -= rctx->dst_nents; in handle_aead_req()
1532 err = spu_aead_rx_sg_create(mssg, req, rctx, rx_frag_num, in handle_aead_req()
1540 tx_frag_num += rctx->src_nents; in handle_aead_req()
1548 err = spu_aead_tx_sg_create(mssg, rctx, tx_frag_num, spu_hdr_len, in handle_aead_req()
1549 rctx->assoc, aead_parms.assoc_size, in handle_aead_req()
1555 err = mailbox_send_message(mssg, req->base.flags, rctx->chan_idx); in handle_aead_req()
1566 static void handle_aead_resp(struct iproc_reqctx_s *rctx) in handle_aead_resp() argument
1569 struct crypto_async_request *areq = rctx->parent; in handle_aead_resp()
1572 struct iproc_ctx_s *ctx = rctx->ctx; in handle_aead_resp()
1578 payload_len = spu->spu_payload_length(rctx->msg_buf.spu_resp_hdr); in handle_aead_resp()
1585 packet_dump(" assoc_data ", rctx->msg_buf.a.resp_aad, in handle_aead_resp()
1594 if (rctx->is_encrypt) { in handle_aead_resp()
1595 icv_offset = req->assoclen + rctx->total_sent; in handle_aead_resp()
1596 packet_dump(" ICV: ", rctx->msg_buf.digest, ctx->digestsize); in handle_aead_resp()
1598 sg_copy_part_from_buf(req->dst, rctx->msg_buf.digest, in handle_aead_resp()
1626 static void spu_chunk_cleanup(struct iproc_reqctx_s *rctx) in spu_chunk_cleanup() argument
1629 struct brcm_message *mssg = &rctx->mb_mssg; in spu_chunk_cleanup()
1644 static void finish_req(struct iproc_reqctx_s *rctx, int err) in finish_req() argument
1646 struct crypto_async_request *areq = rctx->parent; in finish_req()
1651 spu_chunk_cleanup(rctx); in finish_req()
1666 struct iproc_reqctx_s *rctx; in spu_rx_callback() local
1669 rctx = mssg->ctx; in spu_rx_callback()
1670 if (unlikely(!rctx)) { in spu_rx_callback()
1678 err = spu->spu_status_process(rctx->msg_buf.rx_stat); in spu_rx_callback()
1687 switch (rctx->ctx->alg->type) { in spu_rx_callback()
1689 handle_ablkcipher_resp(rctx); in spu_rx_callback()
1692 handle_ahash_resp(rctx); in spu_rx_callback()
1695 handle_aead_resp(rctx); in spu_rx_callback()
1706 if (rctx->total_sent < rctx->total_todo) { in spu_rx_callback()
1708 spu_chunk_cleanup(rctx); in spu_rx_callback()
1710 switch (rctx->ctx->alg->type) { in spu_rx_callback()
1712 err = handle_ablkcipher_req(rctx); in spu_rx_callback()
1715 err = handle_ahash_req(rctx); in spu_rx_callback()
1724 err = handle_aead_req(rctx); in spu_rx_callback()
1736 finish_req(rctx, err); in spu_rx_callback()
1752 struct iproc_reqctx_s *rctx = ablkcipher_request_ctx(req); in ablkcipher_enqueue() local
1759 rctx->gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | in ablkcipher_enqueue()
1761 rctx->parent = &req->base; in ablkcipher_enqueue()
1762 rctx->is_encrypt = encrypt; in ablkcipher_enqueue()
1763 rctx->bd_suppress = false; in ablkcipher_enqueue()
1764 rctx->total_todo = req->nbytes; in ablkcipher_enqueue()
1765 rctx->src_sent = 0; in ablkcipher_enqueue()
1766 rctx->total_sent = 0; in ablkcipher_enqueue()
1767 rctx->total_received = 0; in ablkcipher_enqueue()
1768 rctx->ctx = ctx; in ablkcipher_enqueue()
1771 rctx->src_sg = req->src; in ablkcipher_enqueue()
1772 rctx->src_nents = 0; in ablkcipher_enqueue()
1773 rctx->src_skip = 0; in ablkcipher_enqueue()
1774 rctx->dst_sg = req->dst; in ablkcipher_enqueue()
1775 rctx->dst_nents = 0; in ablkcipher_enqueue()
1776 rctx->dst_skip = 0; in ablkcipher_enqueue()
1784 rctx->iv_ctr_len = in ablkcipher_enqueue()
1786 memcpy(rctx->msg_buf.iv_ctr, req->info, rctx->iv_ctr_len); in ablkcipher_enqueue()
1788 rctx->iv_ctr_len = 0; in ablkcipher_enqueue()
1792 rctx->chan_idx = select_channel(); in ablkcipher_enqueue()
1793 err = handle_ablkcipher_req(rctx); in ablkcipher_enqueue()
1796 spu_chunk_cleanup(rctx); in ablkcipher_enqueue()
1968 struct iproc_reqctx_s *rctx = ahash_request_ctx(req); in ahash_enqueue() local
1976 rctx->gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | in ahash_enqueue()
1978 rctx->parent = &req->base; in ahash_enqueue()
1979 rctx->ctx = ctx; in ahash_enqueue()
1980 rctx->bd_suppress = true; in ahash_enqueue()
1981 memset(&rctx->mb_mssg, 0, sizeof(struct brcm_message)); in ahash_enqueue()
1984 rctx->src_sg = req->src; in ahash_enqueue()
1985 rctx->src_skip = 0; in ahash_enqueue()
1986 rctx->src_nents = 0; in ahash_enqueue()
1987 rctx->dst_sg = NULL; in ahash_enqueue()
1988 rctx->dst_skip = 0; in ahash_enqueue()
1989 rctx->dst_nents = 0; in ahash_enqueue()
1992 if ((rctx->is_final == 1) && (rctx->total_todo == 0) && in ahash_enqueue()
1996 rctx->is_final ? "" : "non-", alg_name); in ahash_enqueue()
2005 rctx->chan_idx = select_channel(); in ahash_enqueue()
2007 err = handle_ahash_req(rctx); in ahash_enqueue()
2010 spu_chunk_cleanup(rctx); in ahash_enqueue()
2025 struct iproc_reqctx_s *rctx = ahash_request_ctx(req); in __ahash_init() local
2032 rctx->hash_carry_len = 0; in __ahash_init()
2033 rctx->is_final = 0; in __ahash_init()
2035 rctx->total_todo = 0; in __ahash_init()
2036 rctx->src_sent = 0; in __ahash_init()
2037 rctx->total_sent = 0; in __ahash_init()
2038 rctx->total_received = 0; in __ahash_init()
2044 rctx->is_sw_hmac = false; in __ahash_init()
2141 struct iproc_reqctx_s *rctx = ahash_request_ctx(req); in __ahash_update() local
2147 rctx->total_todo += req->nbytes; in __ahash_update()
2148 rctx->src_sent = 0; in __ahash_update()
2199 struct iproc_reqctx_s *rctx = ahash_request_ctx(req); in __ahash_final() local
2203 rctx->is_final = 1; in __ahash_final()
2236 struct iproc_reqctx_s *rctx = ahash_request_ctx(req); in __ahash_finup() local
2240 rctx->total_todo += req->nbytes; in __ahash_finup()
2241 rctx->src_sent = 0; in __ahash_finup()
2242 rctx->is_final = 1; in __ahash_finup()
2351 const struct iproc_reqctx_s *rctx = ahash_request_ctx(req); in ahash_export() local
2354 spu_exp->total_todo = rctx->total_todo; in ahash_export()
2355 spu_exp->total_sent = rctx->total_sent; in ahash_export()
2356 spu_exp->is_sw_hmac = rctx->is_sw_hmac; in ahash_export()
2357 memcpy(spu_exp->hash_carry, rctx->hash_carry, sizeof(rctx->hash_carry)); in ahash_export()
2358 spu_exp->hash_carry_len = rctx->hash_carry_len; in ahash_export()
2359 memcpy(spu_exp->incr_hash, rctx->incr_hash, sizeof(rctx->incr_hash)); in ahash_export()
2366 struct iproc_reqctx_s *rctx = ahash_request_ctx(req); in ahash_import() local
2369 rctx->total_todo = spu_exp->total_todo; in ahash_import()
2370 rctx->total_sent = spu_exp->total_sent; in ahash_import()
2371 rctx->is_sw_hmac = spu_exp->is_sw_hmac; in ahash_import()
2372 memcpy(rctx->hash_carry, spu_exp->hash_carry, sizeof(rctx->hash_carry)); in ahash_import()
2373 rctx->hash_carry_len = spu_exp->hash_carry_len; in ahash_import()
2374 memcpy(rctx->incr_hash, spu_exp->incr_hash, sizeof(rctx->incr_hash)); in ahash_import()
2481 struct iproc_reqctx_s *rctx = ahash_request_ctx(req); in ahash_hmac_init() local
2494 rctx->is_sw_hmac = true; in ahash_hmac_init()
2497 memcpy(rctx->hash_carry, ctx->ipad, blocksize); in ahash_hmac_init()
2498 rctx->hash_carry_len = blocksize; in ahash_hmac_init()
2499 rctx->total_todo += blocksize; in ahash_hmac_init()
2531 struct iproc_reqctx_s *rctx = ahash_request_ctx(req); in ahash_hmac_digest() local
2551 rctx->is_sw_hmac = false; in ahash_hmac_digest()
2554 rctx->is_sw_hmac = true; in ahash_hmac_digest()
2557 memcpy(rctx->hash_carry, ctx->ipad, blocksize); in ahash_hmac_digest()
2558 rctx->hash_carry_len = blocksize; in ahash_hmac_digest()
2559 rctx->total_todo += blocksize; in ahash_hmac_digest()
2569 struct iproc_reqctx_s *rctx = aead_request_ctx(req); in aead_need_fallback() local
2582 if ((rctx->is_encrypt && (req->cryptlen == 0)) || in aead_need_fallback()
2583 (!rctx->is_encrypt && (req->cryptlen == ctx->digestsize))) { in aead_need_fallback()
2617 rctx->iv_ctr_len == GCM_RFC4106_IV_SIZE && in aead_need_fallback()
2640 struct iproc_reqctx_s *rctx = aead_request_ctx(req); in aead_complete() local
2647 areq->complete = rctx->old_complete; in aead_complete()
2648 areq->data = rctx->old_data; in aead_complete()
2657 struct iproc_reqctx_s *rctx = aead_request_ctx(req); in aead_do_fallback() local
2666 rctx->old_tfm = tfm; in aead_do_fallback()
2672 rctx->old_complete = req->base.complete; in aead_do_fallback()
2673 rctx->old_data = req->base.data; in aead_do_fallback()
2685 rctx->old_complete, req); in aead_do_fallback()
2686 req->base.data = rctx->old_data; in aead_do_fallback()
2700 struct iproc_reqctx_s *rctx = aead_request_ctx(req); in aead_enqueue() local
2714 rctx->gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | in aead_enqueue()
2716 rctx->parent = &req->base; in aead_enqueue()
2717 rctx->is_encrypt = is_encrypt; in aead_enqueue()
2718 rctx->bd_suppress = false; in aead_enqueue()
2719 rctx->total_todo = req->cryptlen; in aead_enqueue()
2720 rctx->src_sent = 0; in aead_enqueue()
2721 rctx->total_sent = 0; in aead_enqueue()
2722 rctx->total_received = 0; in aead_enqueue()
2723 rctx->is_sw_hmac = false; in aead_enqueue()
2724 rctx->ctx = ctx; in aead_enqueue()
2725 memset(&rctx->mb_mssg, 0, sizeof(struct brcm_message)); in aead_enqueue()
2728 rctx->assoc = req->src; in aead_enqueue()
2735 if (spu_sg_at_offset(req->src, req->assoclen, &rctx->src_sg, in aead_enqueue()
2736 &rctx->src_skip) < 0) { in aead_enqueue()
2742 rctx->src_nents = 0; in aead_enqueue()
2743 rctx->dst_nents = 0; in aead_enqueue()
2745 rctx->dst_sg = rctx->src_sg; in aead_enqueue()
2746 rctx->dst_skip = rctx->src_skip; in aead_enqueue()
2753 if (spu_sg_at_offset(req->dst, req->assoclen, &rctx->dst_sg, in aead_enqueue()
2754 &rctx->dst_skip) < 0) { in aead_enqueue()
2766 rctx->iv_ctr_len = in aead_enqueue()
2770 rctx->iv_ctr_len = CCM_AES_IV_SIZE; in aead_enqueue()
2772 rctx->iv_ctr_len = 0; in aead_enqueue()
2775 rctx->hash_carry_len = 0; in aead_enqueue()
2779 rctx->src_sg, rctx->src_skip); in aead_enqueue()
2780 flow_log(" assoc: %p, assoclen %u\n", rctx->assoc, req->assoclen); in aead_enqueue()
2783 rctx->dst_sg, rctx->dst_skip); in aead_enqueue()
2784 flow_log(" iv_ctr_len:%u\n", rctx->iv_ctr_len); in aead_enqueue()
2785 flow_dump(" iv: ", req->iv, rctx->iv_ctr_len); in aead_enqueue()
2801 if (rctx->iv_ctr_len) { in aead_enqueue()
2803 memcpy(rctx->msg_buf.iv_ctr + ctx->salt_offset, in aead_enqueue()
2805 memcpy(rctx->msg_buf.iv_ctr + ctx->salt_offset + ctx->salt_len, in aead_enqueue()
2807 rctx->iv_ctr_len - ctx->salt_len - ctx->salt_offset); in aead_enqueue()
2810 rctx->chan_idx = select_channel(); in aead_enqueue()
2811 err = handle_aead_req(rctx); in aead_enqueue()
2814 spu_chunk_cleanup(rctx); in aead_enqueue()