370 static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map);
1568 static void submit_flush_bio(struct dm_integrity_c *ic, struct dm_integrity_io *dio)  in submit_flush_bio()  argument
1574 	bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));  in submit_flush_bio()
1600 static void do_endio_flush(struct dm_integrity_c *ic, struct dm_integrity_io *dio)  in do_endio_flush()  argument
1602 	struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));  in do_endio_flush()
1604 	if (unlikely(dio->fua) && likely(!bio->bi_status) && likely(!dm_integrity_failed(ic)))  in do_endio_flush()
1605 		submit_flush_bio(ic, dio);  in do_endio_flush()
1610 static void dec_in_flight(struct dm_integrity_io *dio)  in dec_in_flight()  argument
1612 	if (atomic_dec_and_test(&dio->in_flight)) {  in dec_in_flight()
1613 		struct dm_integrity_c *ic = dio->ic;  in dec_in_flight()
1616 		remove_range(ic, &dio->range);  in dec_in_flight()
1618 		if (dio->op == REQ_OP_WRITE || unlikely(dio->op == REQ_OP_DISCARD))  in dec_in_flight()
1621 		bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));  in dec_in_flight()
1622 		if (unlikely(dio->bi_status) && !bio->bi_status)  in dec_in_flight()
1623 			bio->bi_status = dio->bi_status;  in dec_in_flight()
1624 		if (likely(!bio->bi_status) && unlikely(bio_sectors(bio) != dio->range.n_sectors)) {  in dec_in_flight()
1625 			dio->range.logical_sector += dio->range.n_sectors;  in dec_in_flight()
1626 			bio_advance(bio, dio->range.n_sectors << SECTOR_SHIFT);  in dec_in_flight()
1627 			INIT_WORK(&dio->work, integrity_bio_wait);  in dec_in_flight()
1628 			queue_work(ic->offload_wq, &dio->work);  in dec_in_flight()
1631 		do_endio_flush(ic, dio);  in dec_in_flight()
1637 	struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));  in integrity_end_io()  local
1639 	dm_bio_restore(&dio->bio_details, bio);  in integrity_end_io()
1643 	if (dio->completion)  in integrity_end_io()
1644 		complete(dio->completion);  in integrity_end_io()
1646 	dec_in_flight(dio);  in integrity_end_io()
1704 	struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work);  in integrity_metadata()  local
1705 	struct dm_integrity_c *ic = dio->ic;  in integrity_metadata()
1713 		struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));  in integrity_metadata()
1723 		if (likely(dio->op != REQ_OP_DISCARD))  in integrity_metadata()
1737 		if (unlikely(dio->op == REQ_OP_DISCARD)) {  in integrity_metadata()
1738 			unsigned int bi_size = dio->bio_details.bi_iter.bi_size;  in integrity_metadata()
1748 				r = dm_integrity_rw_tag(ic, checksums, &dio->metadata_block, &dio->metadata_offset,  in integrity_metadata()
1764 		sector = dio->range.logical_sector;  in integrity_metadata()
1765 		sectors_to_process = dio->range.n_sectors;  in integrity_metadata()
1767 		__bio_for_each_segment(bv, bio, iter, dio->bio_details.bi_iter) {  in integrity_metadata()
1784 			r = dm_integrity_rw_tag(ic, checksums, &dio->metadata_block, &dio->metadata_offset,  in integrity_metadata()
1785 						checksums_ptr - checksums, dio->op == REQ_OP_READ ? TAG_CMP : TAG_WRITE);  in integrity_metadata()
1816 		struct bio_integrity_payload *bip = dio->bio_details.bi_integrity;  in integrity_metadata()
1821 			unsigned int data_to_process = dio->range.n_sectors;  in integrity_metadata()
1833 				r = dm_integrity_rw_tag(ic, tag, &dio->metadata_block, &dio->metadata_offset,  in integrity_metadata()
1834 							this_len, dio->op == REQ_OP_READ ? TAG_READ : TAG_WRITE);  in integrity_metadata()
1844 	dec_in_flight(dio);  in integrity_metadata()
1847 	dio->bi_status = errno_to_blk_status(r);  in integrity_metadata()
1848 	dec_in_flight(dio);  in integrity_metadata()
1854 	struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));  in dm_integrity_map()  local
1859 	dio->ic = ic;  in dm_integrity_map()
1860 	dio->bi_status = 0;  in dm_integrity_map()
1861 	dio->op = bio_op(bio);  in dm_integrity_map()
1863 	if (unlikely(dio->op == REQ_OP_DISCARD)) {  in dm_integrity_map()
1879 		submit_flush_bio(ic, dio);  in dm_integrity_map()
1883 	dio->range.logical_sector = dm_target_offset(ti, bio->bi_iter.bi_sector);  in dm_integrity_map()
1884 	dio->fua = dio->op == REQ_OP_WRITE && bio->bi_opf & REQ_FUA;  in dm_integrity_map()
1885 	if (unlikely(dio->fua)) {  in dm_integrity_map()
1892 	if (unlikely(dio->range.logical_sector + bio_sectors(bio) > ic->provided_data_sectors)) {  in dm_integrity_map()
1894 		      dio->range.logical_sector, bio_sectors(bio),  in dm_integrity_map()
1898 …if (unlikely((dio->range.logical_sector | bio_sectors(bio)) & (unsigned int)(ic->sectors_per_block…  in dm_integrity_map()
1901 		      dio->range.logical_sector, bio_sectors(bio));  in dm_integrity_map()
1905 	if (ic->sectors_per_block > 1 && likely(dio->op != REQ_OP_DISCARD)) {  in dm_integrity_map()
1940 	if (unlikely(ic->mode == 'R') && unlikely(dio->op != REQ_OP_READ))  in dm_integrity_map()
1943 	get_area_and_offset(ic, dio->range.logical_sector, &area, &offset);  in dm_integrity_map()
1944 	dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset, &dio->metadata_offset);  in dm_integrity_map()
1947 	dm_integrity_map_continue(dio, true);  in dm_integrity_map()
1951 static bool __journal_read_write(struct dm_integrity_io *dio, struct bio *bio,  in __journal_read_write()  argument
1954 	struct dm_integrity_c *ic = dio->ic;  in __journal_read_write()
1958 	logical_sector = dio->range.logical_sector;  in __journal_read_write()
1959 	n_sectors = dio->range.n_sectors;  in __journal_read_write()
1970 		if (likely(dio->op == REQ_OP_WRITE))  in __journal_read_write()
1976 			if (unlikely(dio->op == REQ_OP_READ)) {  in __journal_read_write()
2027 						if (likely(dio->op == REQ_OP_WRITE))  in __journal_read_write()
2035 				} else if (likely(dio->op == REQ_OP_WRITE))  in __journal_read_write()
2039 			if (likely(dio->op == REQ_OP_WRITE)) {  in __journal_read_write()
2077 		if (unlikely(dio->op == REQ_OP_READ))  in __journal_read_write()
2082 	if (likely(dio->op == REQ_OP_WRITE)) {  in __journal_read_write()
2091 		remove_range(ic, &dio->range);  in __journal_read_write()
2096 		dio->range.logical_sector = logical_sector;  in __journal_read_write()
2097 		get_area_and_offset(ic, dio->range.logical_sector, &area, &offset);  in __journal_read_write()
2098 		dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset, &dio->metadata_offset);  in __journal_read_write()
2105 static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map)  in dm_integrity_map_continue()  argument
2107 	struct dm_integrity_c *ic = dio->ic;  in dm_integrity_map_continue()
2108 	struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));  in dm_integrity_map_continue()
2113 	bool need_sync_io = ic->internal_hash && dio->op == REQ_OP_READ;  in dm_integrity_map_continue()
2115 	if (unlikely(dio->op == REQ_OP_DISCARD) && ic->mode != 'D')  in dm_integrity_map_continue()
2119 		INIT_WORK(&dio->work, integrity_bio_wait);  in dm_integrity_map_continue()
2120 		queue_work(ic->offload_wq, &dio->work);  in dm_integrity_map_continue()
2132 	dio->range.n_sectors = bio_sectors(bio);  in dm_integrity_map_continue()
2134 	if (ic->mode == 'J' && likely(dio->op != REQ_OP_DISCARD)) {  in dm_integrity_map_continue()
2135 		if (dio->op == REQ_OP_WRITE) {  in dm_integrity_map_continue()
2139 			dio->range.n_sectors = min(dio->range.n_sectors,  in dm_integrity_map_continue()
2141 			if (unlikely(!dio->range.n_sectors)) {  in dm_integrity_map_continue()
2147 			range_sectors = dio->range.n_sectors >> ic->sb->log2_sectors_per_block;  in dm_integrity_map_continue()
2165 				add_journal_node(ic, &ic->journal_tree[pos], dio->range.logical_sector + i);  in dm_integrity_map_continue()
2179 			} while ((i += ic->sectors_per_block) < dio->range.n_sectors);  in dm_integrity_map_continue()
2186 			journal_read_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector);  in dm_integrity_map_continue()
2188 				if (unlikely(dio->range.n_sectors > next_sector - dio->range.logical_sector))  in dm_integrity_map_continue()
2189 					dio->range.n_sectors = next_sector - dio->range.logical_sector;  in dm_integrity_map_continue()
2194 				for (i = ic->sectors_per_block; i < dio->range.n_sectors; i += ic->sectors_per_block, jp++) {  in dm_integrity_map_continue()
2195 					if (!test_journal_node(ic, jp, dio->range.logical_sector + i))  in dm_integrity_map_continue()
2198 				dio->range.n_sectors = i;  in dm_integrity_map_continue()
2202 	if (unlikely(!add_new_range(ic, &dio->range, true))) {  in dm_integrity_map_continue()
2211 			INIT_WORK(&dio->work, integrity_bio_wait);  in dm_integrity_map_continue()
2212 			queue_work(ic->wait_wq, &dio->work);  in dm_integrity_map_continue()
2216 			dio->range.n_sectors = ic->sectors_per_block;  in dm_integrity_map_continue()
2217 		wait_and_add_new_range(ic, &dio->range);  in dm_integrity_map_continue()
2227 			new_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector);  in dm_integrity_map_continue()
2229 				remove_range_unlocked(ic, &dio->range);  in dm_integrity_map_continue()
2234 	if (ic->mode == 'J' && likely(dio->op == REQ_OP_DISCARD) && !discard_retried) {  in dm_integrity_map_continue()
2238 		new_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector);  in dm_integrity_map_continue()
2240 		    unlikely(next_sector < dio->range.logical_sector - dio->range.n_sectors)) {  in dm_integrity_map_continue()
2241 			remove_range_unlocked(ic, &dio->range);  in dm_integrity_map_continue()
2259 	if (ic->mode == 'B' && (dio->op == REQ_OP_WRITE || unlikely(dio->op == REQ_OP_DISCARD))) {  in dm_integrity_map_continue()
2260 		if (!block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector,  in dm_integrity_map_continue()
2261 				     dio->range.n_sectors, BITMAP_OP_TEST_ALL_SET)) {  in dm_integrity_map_continue()
2264 			bbs = sector_to_bitmap_block(ic, dio->range.logical_sector);  in dm_integrity_map_continue()
2273 	dio->in_flight = (atomic_t)ATOMIC_INIT(2);  in dm_integrity_map_continue()
2277 		dio->completion = &read_comp;  in dm_integrity_map_continue()
2279 		dio->completion = NULL;  in dm_integrity_map_continue()
2281 	dm_bio_record(&dio->bio_details, bio);  in dm_integrity_map_continue()
2286 	bio->bi_iter.bi_size = dio->range.n_sectors << SECTOR_SHIFT;  in dm_integrity_map_continue()
2288 	if (unlikely(dio->op == REQ_OP_DISCARD) && likely(ic->mode != 'D')) {  in dm_integrity_map_continue()
2289 		integrity_metadata(&dio->work);  in dm_integrity_map_continue()
2292 		dio->in_flight = (atomic_t)ATOMIC_INIT(1);  in dm_integrity_map_continue()
2293 		dio->completion = NULL;  in dm_integrity_map_continue()
2305 		    dio->range.logical_sector + dio->range.n_sectors > le64_to_cpu(ic->sb->recalc_sector))  in dm_integrity_map_continue()
2308 			if (!block_bitmap_op(ic, ic->recalc_bitmap, dio->range.logical_sector,  in dm_integrity_map_continue()
2309 					     dio->range.n_sectors, BITMAP_OP_TEST_ALL_CLEAR))  in dm_integrity_map_continue()
2314 			integrity_metadata(&dio->work);  in dm_integrity_map_continue()
2317 			dec_in_flight(dio);  in dm_integrity_map_continue()
2319 		INIT_WORK(&dio->work, integrity_metadata);  in dm_integrity_map_continue()
2320 		queue_work(ic->metadata_wq, &dio->work);  in dm_integrity_map_continue()
2326 	if (unlikely(__journal_read_write(dio, bio, journal_section, journal_entry)))  in dm_integrity_map_continue()
2329 	do_endio_flush(ic, dio);  in dm_integrity_map_continue()
2335 	struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work);  in integrity_bio_wait()  local
2337 	dm_integrity_map_continue(dio, false);  in integrity_bio_wait()
2822 		struct dm_integrity_io *dio;  in bitmap_block_work()  local
2824 		dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));  in bitmap_block_work()
2826 		if (block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector,  in bitmap_block_work()
2827 				    dio->range.n_sectors, BITMAP_OP_TEST_ALL_SET)) {  in bitmap_block_work()
2828 			remove_range(ic, &dio->range);  in bitmap_block_work()
2829 			INIT_WORK(&dio->work, integrity_bio_wait);  in bitmap_block_work()
2830 			queue_work(ic->offload_wq, &dio->work);  in bitmap_block_work()
2832 			block_bitmap_op(ic, ic->journal, dio->range.logical_sector,  in bitmap_block_work()
2833 					dio->range.n_sectors, BITMAP_OP_SET);  in bitmap_block_work()
2846 		struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));  in bitmap_block_work()  local
2848 		block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector,  in bitmap_block_work()
2849 				dio->range.n_sectors, BITMAP_OP_SET);  in bitmap_block_work()
2851 		remove_range(ic, &dio->range);  in bitmap_block_work()
2852 		INIT_WORK(&dio->work, integrity_bio_wait);  in bitmap_block_work()
2853 		queue_work(ic->offload_wq, &dio->work);  in bitmap_block_work()