76 		struct chachapoly_req_ctx *rctx = aead_request_ctx(req);  in async_done_continue()  local
78 		rctx->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;  in async_done_continue()
99 	struct chachapoly_req_ctx *rctx = aead_request_ctx(req);  in poly_verify_tag()  local
100 	u8 tag[sizeof(rctx->tag)];  in poly_verify_tag()
103 				 req->assoclen + rctx->cryptlen,  in poly_verify_tag()
105 	if (crypto_memneq(tag, rctx->tag, sizeof(tag)))  in poly_verify_tag()
112 	struct chachapoly_req_ctx *rctx = aead_request_ctx(req);  in poly_copy_tag()  local
114 	scatterwalk_map_and_copy(rctx->tag, req->dst,  in poly_copy_tag()
115 				 req->assoclen + rctx->cryptlen,  in poly_copy_tag()
116 				 sizeof(rctx->tag), 1);  in poly_copy_tag()
128 	struct chachapoly_req_ctx *rctx = aead_request_ctx(req);  in chacha_decrypt()  local
129 	struct chacha_req *creq = &rctx->u.chacha;  in chacha_decrypt()
133 	if (rctx->cryptlen == 0)  in chacha_decrypt()
138 	src = scatterwalk_ffwd(rctx->src, req->src, req->assoclen);  in chacha_decrypt()
141 		dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen);  in chacha_decrypt()
143 	skcipher_request_set_callback(&creq->req, rctx->flags,  in chacha_decrypt()
147 				   rctx->cryptlen, creq->iv);  in chacha_decrypt()
158 	struct chachapoly_req_ctx *rctx = aead_request_ctx(req);  in poly_tail_continue()  local
160 	if (rctx->cryptlen == req->cryptlen) /* encrypting */  in poly_tail_continue()
175 	struct chachapoly_req_ctx *rctx = aead_request_ctx(req);  in poly_tail()  local
176 	struct poly_req *preq = &rctx->u.poly;  in poly_tail()
179 	preq->tail.assoclen = cpu_to_le64(rctx->assoclen);  in poly_tail()
180 	preq->tail.cryptlen = cpu_to_le64(rctx->cryptlen);  in poly_tail()
183 	ahash_request_set_callback(&preq->req, rctx->flags,  in poly_tail()
187 				rctx->tag, sizeof(preq->tail));  in poly_tail()
204 	struct chachapoly_req_ctx *rctx = aead_request_ctx(req);  in poly_cipherpad()  local
205 	struct poly_req *preq = &rctx->u.poly;  in poly_cipherpad()
209 	padlen = -rctx->cryptlen % POLY1305_BLOCK_SIZE;  in poly_cipherpad()
213 	ahash_request_set_callback(&preq->req, rctx->flags,  in poly_cipherpad()
233 	struct chachapoly_req_ctx *rctx = aead_request_ctx(req);  in poly_cipher()  local
234 	struct poly_req *preq = &rctx->u.poly;  in poly_cipher()
238 	if (rctx->cryptlen == req->cryptlen) /* encrypting */  in poly_cipher()
241 	crypt = scatterwalk_ffwd(rctx->src, crypt, req->assoclen);  in poly_cipher()
243 	ahash_request_set_callback(&preq->req, rctx->flags,  in poly_cipher()
246 	ahash_request_set_crypt(&preq->req, crypt, NULL, rctx->cryptlen);  in poly_cipher()
263 	struct chachapoly_req_ctx *rctx = aead_request_ctx(req);  in poly_adpad()  local
264 	struct poly_req *preq = &rctx->u.poly;  in poly_adpad()
268 	padlen = -rctx->assoclen % POLY1305_BLOCK_SIZE;  in poly_adpad()
272 	ahash_request_set_callback(&preq->req, rctx->flags,  in poly_adpad()
292 	struct chachapoly_req_ctx *rctx = aead_request_ctx(req);  in poly_ad()  local
293 	struct poly_req *preq = &rctx->u.poly;  in poly_ad()
296 	ahash_request_set_callback(&preq->req, rctx->flags,  in poly_ad()
299 	ahash_request_set_crypt(&preq->req, req->src, NULL, rctx->assoclen);  in poly_ad()
316 	struct chachapoly_req_ctx *rctx = aead_request_ctx(req);  in poly_setkey()  local
317 	struct poly_req *preq = &rctx->u.poly;  in poly_setkey()
320 	sg_init_one(preq->src, rctx->key, sizeof(rctx->key));  in poly_setkey()
322 	ahash_request_set_callback(&preq->req, rctx->flags,  in poly_setkey()
325 	ahash_request_set_crypt(&preq->req, preq->src, NULL, sizeof(rctx->key));  in poly_setkey()
342 	struct chachapoly_req_ctx *rctx = aead_request_ctx(req);  in poly_init()  local
343 	struct poly_req *preq = &rctx->u.poly;  in poly_init()
346 	ahash_request_set_callback(&preq->req, rctx->flags,  in poly_init()
366 	struct chachapoly_req_ctx *rctx = aead_request_ctx(req);  in poly_genkey()  local
367 	struct chacha_req *creq = &rctx->u.chacha;  in poly_genkey()
370 	rctx->assoclen = req->assoclen;  in poly_genkey()
373 		if (rctx->assoclen < 8)  in poly_genkey()
375 		rctx->assoclen -= 8;  in poly_genkey()
378 	memset(rctx->key, 0, sizeof(rctx->key));  in poly_genkey()
379 	sg_init_one(creq->src, rctx->key, sizeof(rctx->key));  in poly_genkey()
383 	skcipher_request_set_callback(&creq->req, rctx->flags,  in poly_genkey()
404 	struct chachapoly_req_ctx *rctx = aead_request_ctx(req);  in chacha_encrypt()  local
405 	struct chacha_req *creq = &rctx->u.chacha;  in chacha_encrypt()
414 	src = scatterwalk_ffwd(rctx->src, req->src, req->assoclen);  in chacha_encrypt()
417 		dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen);  in chacha_encrypt()
419 	skcipher_request_set_callback(&creq->req, rctx->flags,  in chacha_encrypt()
434 	struct chachapoly_req_ctx *rctx = aead_request_ctx(req);  in chachapoly_encrypt()  local
436 	rctx->cryptlen = req->cryptlen;  in chachapoly_encrypt()
437 	rctx->flags = aead_request_flags(req);  in chachapoly_encrypt()
456 	struct chachapoly_req_ctx *rctx = aead_request_ctx(req);  in chachapoly_decrypt()  local
458 	rctx->cryptlen = req->cryptlen - POLY1305_DIGEST_SIZE;  in chachapoly_decrypt()
459 	rctx->flags = aead_request_flags(req);  in chachapoly_decrypt()