100 			 struct dm_crypt_request *dmreq);
102 		    struct dm_crypt_request *dmreq);
311 			      struct dm_crypt_request *dmreq)  in crypt_iv_plain_gen()  argument
314 	*(__le32 *)iv = cpu_to_le32(dmreq->iv_sector & 0xffffffff);  in crypt_iv_plain_gen()
320 				struct dm_crypt_request *dmreq)  in crypt_iv_plain64_gen()  argument
323 	*(__le64 *)iv = cpu_to_le64(dmreq->iv_sector);  in crypt_iv_plain64_gen()
329 				  struct dm_crypt_request *dmreq)  in crypt_iv_plain64be_gen()  argument
333 	*(__be64 *)&iv[cc->iv_size - sizeof(u64)] = cpu_to_be64(dmreq->iv_sector);  in crypt_iv_plain64be_gen()
339 			      struct dm_crypt_request *dmreq)  in crypt_iv_essiv_gen()  argument
346 	*(__le64 *)iv = cpu_to_le64(dmreq->iv_sector);  in crypt_iv_essiv_gen()
386 			      struct dm_crypt_request *dmreq)  in crypt_iv_benbi_gen()  argument
392 	val = cpu_to_be64(((u64)dmreq->iv_sector << cc->iv_gen_private.benbi.shift) + 1);  in crypt_iv_benbi_gen()
399 			     struct dm_crypt_request *dmreq)  in crypt_iv_null_gen()  argument
475 			    struct dm_crypt_request *dmreq,  in crypt_iv_lmk_one()  argument
502 	buf[0] = cpu_to_le32(dmreq->iv_sector & 0xFFFFFFFF);  in crypt_iv_lmk_one()
503 	buf[1] = cpu_to_le32((((u64)dmreq->iv_sector >> 32) & 0x00FFFFFF) | 0x80000000);  in crypt_iv_lmk_one()
523 			    struct dm_crypt_request *dmreq)  in crypt_iv_lmk_gen()  argument
529 	if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) {  in crypt_iv_lmk_gen()
530 		sg = crypt_get_sg_data(cc, dmreq->sg_in);  in crypt_iv_lmk_gen()
532 		r = crypt_iv_lmk_one(cc, iv, dmreq, src + sg->offset);  in crypt_iv_lmk_gen()
541 			     struct dm_crypt_request *dmreq)  in crypt_iv_lmk_post()  argument
547 	if (bio_data_dir(dmreq->ctx->bio_in) == WRITE)  in crypt_iv_lmk_post()
550 	sg = crypt_get_sg_data(cc, dmreq->sg_out);  in crypt_iv_lmk_post()
552 	r = crypt_iv_lmk_one(cc, iv, dmreq, dst + sg->offset);  in crypt_iv_lmk_post()
632 				  struct dm_crypt_request *dmreq,  in crypt_iv_tcw_whitening()  argument
636 	__le64 sector = cpu_to_le64(dmreq->iv_sector);  in crypt_iv_tcw_whitening()
670 			    struct dm_crypt_request *dmreq)  in crypt_iv_tcw_gen()  argument
674 	__le64 sector = cpu_to_le64(dmreq->iv_sector);  in crypt_iv_tcw_gen()
679 	if (bio_data_dir(dmreq->ctx->bio_in) != WRITE) {  in crypt_iv_tcw_gen()
680 		sg = crypt_get_sg_data(cc, dmreq->sg_in);  in crypt_iv_tcw_gen()
682 		r = crypt_iv_tcw_whitening(cc, dmreq, src + sg->offset);  in crypt_iv_tcw_gen()
696 			     struct dm_crypt_request *dmreq)  in crypt_iv_tcw_post()  argument
702 	if (bio_data_dir(dmreq->ctx->bio_in) != WRITE)  in crypt_iv_tcw_post()
706 	sg = crypt_get_sg_data(cc, dmreq->sg_out);  in crypt_iv_tcw_post()
708 	r = crypt_iv_tcw_whitening(cc, dmreq, dst + sg->offset);  in crypt_iv_tcw_post()
715 				struct dm_crypt_request *dmreq)  in crypt_iv_random_gen()  argument
740 			    struct dm_crypt_request *dmreq)  in crypt_iv_eboiv_gen()  argument
753 	*(__le64 *)buf = cpu_to_le64(dmreq->iv_sector * cc->sector_size);  in crypt_iv_eboiv_gen()
937 static int crypt_iv_elephant(struct crypt_config *cc, struct dm_crypt_request *dmreq)  in crypt_iv_elephant()  argument
955 	*(__le64 *)es = cpu_to_le64(dmreq->iv_sector * cc->sector_size);  in crypt_iv_elephant()
973 	sg = crypt_get_sg_data(cc, dmreq->sg_out);  in crypt_iv_elephant()
978 	if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) {  in crypt_iv_elephant()
979 		sg2 = crypt_get_sg_data(cc, dmreq->sg_in);  in crypt_iv_elephant()
985 	if (bio_data_dir(dmreq->ctx->bio_in) != WRITE) {  in crypt_iv_elephant()
995 	if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) {  in crypt_iv_elephant()
1011 			    struct dm_crypt_request *dmreq)  in crypt_iv_elephant_gen()  argument
1015 	if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) {  in crypt_iv_elephant_gen()
1016 		r = crypt_iv_elephant(cc, dmreq);  in crypt_iv_elephant_gen()
1021 	return crypt_iv_eboiv_gen(cc, iv, dmreq);  in crypt_iv_elephant_gen()
1025 				  struct dm_crypt_request *dmreq)  in crypt_iv_elephant_post()  argument
1027 	if (bio_data_dir(dmreq->ctx->bio_in) != WRITE)  in crypt_iv_elephant_post()
1028 		return crypt_iv_elephant(cc, dmreq);  in crypt_iv_elephant_post()
1229 static void *req_of_dmreq(struct crypt_config *cc, struct dm_crypt_request *dmreq)  in req_of_dmreq()  argument
1231 	return (void *)((char *)dmreq - cc->dmreq_start);  in req_of_dmreq()
1235 		       struct dm_crypt_request *dmreq)  in iv_of_dmreq()  argument
1238 		return (u8 *)ALIGN((unsigned long)(dmreq + 1),  in iv_of_dmreq()
1241 		return (u8 *)ALIGN((unsigned long)(dmreq + 1),  in iv_of_dmreq()
1246 		       struct dm_crypt_request *dmreq)  in org_iv_of_dmreq()  argument
1248 	return iv_of_dmreq(cc, dmreq) + cc->iv_size;  in org_iv_of_dmreq()
1252 		       struct dm_crypt_request *dmreq)  in org_sector_of_dmreq()  argument
1254 	u8 *ptr = iv_of_dmreq(cc, dmreq) + cc->iv_size + cc->iv_size;  in org_sector_of_dmreq()
1259 		       struct dm_crypt_request *dmreq)  in org_tag_of_dmreq()  argument
1261 	u8 *ptr = iv_of_dmreq(cc, dmreq) + cc->iv_size +  in org_tag_of_dmreq()
1267 				struct dm_crypt_request *dmreq)  in tag_from_dmreq()  argument
1269 	struct convert_context *ctx = dmreq->ctx;  in tag_from_dmreq()
1272 	return &io->integrity_metadata[*org_tag_of_dmreq(cc, dmreq) *  in tag_from_dmreq()
1277 			       struct dm_crypt_request *dmreq)  in iv_tag_from_dmreq()  argument
1279 	return tag_from_dmreq(cc, dmreq) + cc->integrity_tag_size;  in iv_tag_from_dmreq()
1289 	struct dm_crypt_request *dmreq;  in crypt_convert_block_aead()  local
1300 	dmreq = dmreq_of_req(cc, req);  in crypt_convert_block_aead()
1301 	dmreq->iv_sector = ctx->cc_sector;  in crypt_convert_block_aead()
1303 		dmreq->iv_sector >>= cc->sector_shift;  in crypt_convert_block_aead()
1304 	dmreq->ctx = ctx;  in crypt_convert_block_aead()
1306 	*org_tag_of_dmreq(cc, dmreq) = tag_offset;  in crypt_convert_block_aead()
1308 	sector = org_sector_of_dmreq(cc, dmreq);  in crypt_convert_block_aead()
1311 	iv = iv_of_dmreq(cc, dmreq);  in crypt_convert_block_aead()
1312 	org_iv = org_iv_of_dmreq(cc, dmreq);  in crypt_convert_block_aead()
1313 	tag = tag_from_dmreq(cc, dmreq);  in crypt_convert_block_aead()
1314 	tag_iv = iv_tag_from_dmreq(cc, dmreq);  in crypt_convert_block_aead()
1321 	sg_init_table(dmreq->sg_in, 4);  in crypt_convert_block_aead()
1322 	sg_set_buf(&dmreq->sg_in[0], sector, sizeof(uint64_t));  in crypt_convert_block_aead()
1323 	sg_set_buf(&dmreq->sg_in[1], org_iv, cc->iv_size);  in crypt_convert_block_aead()
1324 	sg_set_page(&dmreq->sg_in[2], bv_in.bv_page, cc->sector_size, bv_in.bv_offset);  in crypt_convert_block_aead()
1325 	sg_set_buf(&dmreq->sg_in[3], tag, cc->integrity_tag_size);  in crypt_convert_block_aead()
1327 	sg_init_table(dmreq->sg_out, 4);  in crypt_convert_block_aead()
1328 	sg_set_buf(&dmreq->sg_out[0], sector, sizeof(uint64_t));  in crypt_convert_block_aead()
1329 	sg_set_buf(&dmreq->sg_out[1], org_iv, cc->iv_size);  in crypt_convert_block_aead()
1330 	sg_set_page(&dmreq->sg_out[2], bv_out.bv_page, cc->sector_size, bv_out.bv_offset);  in crypt_convert_block_aead()
1331 	sg_set_buf(&dmreq->sg_out[3], tag, cc->integrity_tag_size);  in crypt_convert_block_aead()
1338 			r = cc->iv_gen_ops->generator(cc, org_iv, dmreq);  in crypt_convert_block_aead()
1351 		aead_request_set_crypt(req, dmreq->sg_in, dmreq->sg_out,  in crypt_convert_block_aead()
1358 		aead_request_set_crypt(req, dmreq->sg_in, dmreq->sg_out,  in crypt_convert_block_aead()
1370 		r = cc->iv_gen_ops->post(cc, org_iv, dmreq);  in crypt_convert_block_aead()
1386 	struct dm_crypt_request *dmreq;  in crypt_convert_block_skcipher()  local
1395 	dmreq = dmreq_of_req(cc, req);  in crypt_convert_block_skcipher()
1396 	dmreq->iv_sector = ctx->cc_sector;  in crypt_convert_block_skcipher()
1398 		dmreq->iv_sector >>= cc->sector_shift;  in crypt_convert_block_skcipher()
1399 	dmreq->ctx = ctx;  in crypt_convert_block_skcipher()
1401 	*org_tag_of_dmreq(cc, dmreq) = tag_offset;  in crypt_convert_block_skcipher()
1403 	iv = iv_of_dmreq(cc, dmreq);  in crypt_convert_block_skcipher()
1404 	org_iv = org_iv_of_dmreq(cc, dmreq);  in crypt_convert_block_skcipher()
1405 	tag_iv = iv_tag_from_dmreq(cc, dmreq);  in crypt_convert_block_skcipher()
1407 	sector = org_sector_of_dmreq(cc, dmreq);  in crypt_convert_block_skcipher()
1411 	sg_in  = &dmreq->sg_in[0];  in crypt_convert_block_skcipher()
1412 	sg_out = &dmreq->sg_out[0];  in crypt_convert_block_skcipher()
1425 			r = cc->iv_gen_ops->generator(cc, org_iv, dmreq);  in crypt_convert_block_skcipher()
1447 		r = cc->iv_gen_ops->post(cc, org_iv, dmreq);  in crypt_convert_block_skcipher()
2156 	struct dm_crypt_request *dmreq = async_req->data;  in kcryptd_async_done()  local
2157 	struct convert_context *ctx = dmreq->ctx;  in kcryptd_async_done()
2172 		error = cc->iv_gen_ops->post(cc, org_iv_of_dmreq(cc, dmreq), dmreq);  in kcryptd_async_done()
2177 			    (unsigned long long)le64_to_cpu(*org_sector_of_dmreq(cc, dmreq)));  in kcryptd_async_done()
2182 	crypt_free_req(cc, req_of_dmreq(cc, dmreq), io->base_bio);  in kcryptd_async_done()