240 static void crypt_endio(struct bio *clone);
1653 static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone);
1678 	struct bio *clone;  in crypt_alloc_buffer()  local
1688 	clone = bio_alloc_bioset(cc->dev->bdev, nr_iovecs, io->base_bio->bi_opf,  in crypt_alloc_buffer()
1690 	clone->bi_private = io;  in crypt_alloc_buffer()
1691 	clone->bi_end_io = crypt_endio;  in crypt_alloc_buffer()
1712 			crypt_free_buffer_pages(cc, clone);  in crypt_alloc_buffer()
1713 			bio_put(clone);  in crypt_alloc_buffer()
1721 		__bio_add_page(clone, pages, size_to_add, 0);  in crypt_alloc_buffer()
1726 	if (dm_crypt_integrity_io_alloc(io, clone)) {  in crypt_alloc_buffer()
1727 		crypt_free_buffer_pages(cc, clone);  in crypt_alloc_buffer()
1728 		bio_put(clone);  in crypt_alloc_buffer()
1729 		clone = NULL;  in crypt_alloc_buffer()
1735 	return clone;  in crypt_alloc_buffer()
1738 static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone)  in crypt_free_buffer_pages()  argument
1742 	if (clone->bi_vcnt > 0) { /* bio_for_each_folio_all crashes with an empty bio */  in crypt_free_buffer_pages()
1743 		bio_for_each_folio_all(fi, clone) {  in crypt_free_buffer_pages()
1835 static void crypt_endio(struct bio *clone)  in crypt_endio()  argument
1837 	struct dm_crypt_io *io = clone->bi_private;  in crypt_endio()
1839 	unsigned int rw = bio_data_dir(clone);  in crypt_endio()
1846 		crypt_free_buffer_pages(cc, clone);  in crypt_endio()
1848 	error = clone->bi_status;  in crypt_endio()
1849 	bio_put(clone);  in crypt_endio()
1867 	struct bio *clone;  in kcryptd_io_read()  local
1875 	clone = bio_alloc_clone(cc->dev->bdev, io->base_bio, gfp, &cc->bs);  in kcryptd_io_read()
1876 	if (!clone)  in kcryptd_io_read()
1878 	clone->bi_private = io;  in kcryptd_io_read()
1879 	clone->bi_end_io = crypt_endio;  in kcryptd_io_read()
1883 	clone->bi_iter.bi_sector = cc->start + io->sector;  in kcryptd_io_read()
1885 	if (dm_crypt_integrity_io_alloc(io, clone)) {  in kcryptd_io_read()
1887 		bio_put(clone);  in kcryptd_io_read()
1891 	dm_submit_bio_remap(io->base_bio, clone);  in kcryptd_io_read()
1915 	struct bio *clone = io->ctx.bio_out;  in kcryptd_io_write()  local
1917 	dm_submit_bio_remap(io->base_bio, clone);  in kcryptd_io_write()
1977 	struct bio *clone = io->ctx.bio_out;  in kcryptd_crypt_write_io_submit()  local
1984 		crypt_free_buffer_pages(cc, clone);  in kcryptd_crypt_write_io_submit()
1985 		bio_put(clone);  in kcryptd_crypt_write_io_submit()
1993 	clone->bi_iter.bi_sector = cc->start + io->sector;  in kcryptd_crypt_write_io_submit()
1997 		dm_submit_bio_remap(io->base_bio, clone);  in kcryptd_crypt_write_io_submit()
2075 	struct bio *clone;  in kcryptd_crypt_write_convert()  local
2086 	clone = crypt_alloc_buffer(io, io->base_bio->bi_iter.bi_size);  in kcryptd_crypt_write_convert()
2087 	if (unlikely(!clone)) {  in kcryptd_crypt_write_convert()
2092 	io->ctx.bio_out = clone;  in kcryptd_crypt_write_convert()
2093 	io->ctx.iter_out = clone->bi_iter;  in kcryptd_crypt_write_convert()
2095 	sector += bio_sectors(clone);  in kcryptd_crypt_write_convert()