380 static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map);
1561 static void submit_flush_bio(struct dm_integrity_c *ic, struct dm_integrity_io *dio)  in submit_flush_bio()  argument
1567 	bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));  in submit_flush_bio()
1590 static void do_endio_flush(struct dm_integrity_c *ic, struct dm_integrity_io *dio)  in do_endio_flush()  argument
1592 	struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));  in do_endio_flush()
1594 	if (unlikely(dio->fua) && likely(!bio->bi_status) && likely(!dm_integrity_failed(ic)))  in do_endio_flush()
1595 		submit_flush_bio(ic, dio);  in do_endio_flush()
1600 static void dec_in_flight(struct dm_integrity_io *dio)  in dec_in_flight()  argument
1602 	if (atomic_dec_and_test(&dio->in_flight)) {  in dec_in_flight()
1603 		struct dm_integrity_c *ic = dio->ic;  in dec_in_flight()
1606 		remove_range(ic, &dio->range);  in dec_in_flight()
1608 		if (dio->op == REQ_OP_WRITE || unlikely(dio->op == REQ_OP_DISCARD))  in dec_in_flight()
1611 		bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));  in dec_in_flight()
1613 		if (unlikely(dio->bi_status) && !bio->bi_status)  in dec_in_flight()
1614 			bio->bi_status = dio->bi_status;  in dec_in_flight()
1615 		if (likely(!bio->bi_status) && unlikely(bio_sectors(bio) != dio->range.n_sectors)) {  in dec_in_flight()
1616 			dio->range.logical_sector += dio->range.n_sectors;  in dec_in_flight()
1617 			bio_advance(bio, dio->range.n_sectors << SECTOR_SHIFT);  in dec_in_flight()
1618 			INIT_WORK(&dio->work, integrity_bio_wait);  in dec_in_flight()
1619 			queue_work(ic->offload_wq, &dio->work);  in dec_in_flight()
1622 		do_endio_flush(ic, dio);  in dec_in_flight()
1628 	struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));  in integrity_end_io()  local
1630 	dm_bio_restore(&dio->bio_details, bio);  in integrity_end_io()
1634 	if (dio->completion)  in integrity_end_io()
1635 		complete(dio->completion);  in integrity_end_io()
1637 	dec_in_flight(dio);  in integrity_end_io()
1695 	struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work);  in integrity_metadata()  local
1696 	struct dm_integrity_c *ic = dio->ic;  in integrity_metadata()
1704 		struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));  in integrity_metadata()
1714 		if (likely(dio->op != REQ_OP_DISCARD))  in integrity_metadata()
1728 		if (unlikely(dio->op == REQ_OP_DISCARD)) {  in integrity_metadata()
1729 			sector_t bi_sector = dio->bio_details.bi_iter.bi_sector;  in integrity_metadata()
1730 			unsigned bi_size = dio->bio_details.bi_iter.bi_size;  in integrity_metadata()
1738 				r = dm_integrity_rw_tag(ic, checksums, &dio->metadata_block, &dio->metadata_offset,  in integrity_metadata()
1760 		sector = dio->range.logical_sector;  in integrity_metadata()
1761 		sectors_to_process = dio->range.n_sectors;  in integrity_metadata()
1763 		__bio_for_each_segment(bv, bio, iter, dio->bio_details.bi_iter) {  in integrity_metadata()
1780 			r = dm_integrity_rw_tag(ic, checksums, &dio->metadata_block, &dio->metadata_offset,  in integrity_metadata()
1781 						checksums_ptr - checksums, dio->op == REQ_OP_READ ? TAG_CMP : TAG_WRITE);  in integrity_metadata()
1808 		struct bio_integrity_payload *bip = dio->bio_details.bi_integrity;  in integrity_metadata()
1813 			unsigned data_to_process = dio->range.n_sectors;  in integrity_metadata()
1824 				r = dm_integrity_rw_tag(ic, tag, &dio->metadata_block, &dio->metadata_offset,  in integrity_metadata()
1825 							this_len, dio->op == REQ_OP_READ ? TAG_READ : TAG_WRITE);  in integrity_metadata()
1835 	dec_in_flight(dio);  in integrity_metadata()
1838 	dio->bi_status = errno_to_blk_status(r);  in integrity_metadata()
1839 	dec_in_flight(dio);  in integrity_metadata()
1845 	struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));  in dm_integrity_map()  local
1850 	dio->ic = ic;  in dm_integrity_map()
1851 	dio->bi_status = 0;  in dm_integrity_map()
1852 	dio->op = bio_op(bio);  in dm_integrity_map()
1854 	if (unlikely(dio->op == REQ_OP_DISCARD)) {  in dm_integrity_map()
1868 		submit_flush_bio(ic, dio);  in dm_integrity_map()
1872 	dio->range.logical_sector = dm_target_offset(ti, bio->bi_iter.bi_sector);  in dm_integrity_map()
1873 	dio->fua = dio->op == REQ_OP_WRITE && bio->bi_opf & REQ_FUA;  in dm_integrity_map()
1874 	if (unlikely(dio->fua)) {  in dm_integrity_map()
1881 	if (unlikely(dio->range.logical_sector + bio_sectors(bio) > ic->provided_data_sectors)) {  in dm_integrity_map()
1883 		      dio->range.logical_sector, bio_sectors(bio),  in dm_integrity_map()
1887 …if (unlikely((dio->range.logical_sector | bio_sectors(bio)) & (unsigned)(ic->sectors_per_block - 1…  in dm_integrity_map()
1890 		      dio->range.logical_sector, bio_sectors(bio));  in dm_integrity_map()
1894 	if (ic->sectors_per_block > 1 && likely(dio->op != REQ_OP_DISCARD)) {  in dm_integrity_map()
1927 	if (unlikely(ic->mode == 'R') && unlikely(dio->op != REQ_OP_READ))  in dm_integrity_map()
1930 	get_area_and_offset(ic, dio->range.logical_sector, &area, &offset);  in dm_integrity_map()
1931 	dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset, &dio->metadata_offset);  in dm_integrity_map()
1934 	dm_integrity_map_continue(dio, true);  in dm_integrity_map()
1938 static bool __journal_read_write(struct dm_integrity_io *dio, struct bio *bio,  in __journal_read_write()  argument
1941 	struct dm_integrity_c *ic = dio->ic;  in __journal_read_write()
1945 	logical_sector = dio->range.logical_sector;  in __journal_read_write()
1946 	n_sectors = dio->range.n_sectors;  in __journal_read_write()
1957 		if (likely(dio->op == REQ_OP_WRITE))  in __journal_read_write()
1963 			if (unlikely(dio->op == REQ_OP_READ)) {  in __journal_read_write()
2010 					if (likely(dio->op == REQ_OP_WRITE))  in __journal_read_write()
2018 					if (likely(dio->op == REQ_OP_WRITE))  in __journal_read_write()
2023 			if (likely(dio->op == REQ_OP_WRITE)) {  in __journal_read_write()
2059 		if (unlikely(dio->op == REQ_OP_READ))  in __journal_read_write()
2064 	if (likely(dio->op == REQ_OP_WRITE)) {  in __journal_read_write()
2074 		remove_range(ic, &dio->range);  in __journal_read_write()
2080 		dio->range.logical_sector = logical_sector;  in __journal_read_write()
2081 		get_area_and_offset(ic, dio->range.logical_sector, &area, &offset);  in __journal_read_write()
2082 		dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset, &dio->metadata_offset);  in __journal_read_write()
2089 static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map)  in dm_integrity_map_continue()  argument
2091 	struct dm_integrity_c *ic = dio->ic;  in dm_integrity_map_continue()
2092 	struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));  in dm_integrity_map_continue()
2097 	bool need_sync_io = ic->internal_hash && dio->op == REQ_OP_READ;  in dm_integrity_map_continue()
2098 	if (unlikely(dio->op == REQ_OP_DISCARD) && ic->mode != 'D')  in dm_integrity_map_continue()
2102 		INIT_WORK(&dio->work, integrity_bio_wait);  in dm_integrity_map_continue()
2103 		queue_work(ic->offload_wq, &dio->work);  in dm_integrity_map_continue()
2115 	dio->range.n_sectors = bio_sectors(bio);  in dm_integrity_map_continue()
2117 	if (ic->mode == 'J' && likely(dio->op != REQ_OP_DISCARD)) {  in dm_integrity_map_continue()
2118 		if (dio->op == REQ_OP_WRITE) {  in dm_integrity_map_continue()
2122 			dio->range.n_sectors = min(dio->range.n_sectors,  in dm_integrity_map_continue()
2124 			if (unlikely(!dio->range.n_sectors)) {  in dm_integrity_map_continue()
2130 			range_sectors = dio->range.n_sectors >> ic->sb->log2_sectors_per_block;  in dm_integrity_map_continue()
2148 				add_journal_node(ic, &ic->journal_tree[pos], dio->range.logical_sector + i);  in dm_integrity_map_continue()
2162 			} while ((i += ic->sectors_per_block) < dio->range.n_sectors);  in dm_integrity_map_continue()
2168 			journal_read_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector);  in dm_integrity_map_continue()
2170 				if (unlikely(dio->range.n_sectors > next_sector - dio->range.logical_sector))  in dm_integrity_map_continue()
2171 					dio->range.n_sectors = next_sector - dio->range.logical_sector;  in dm_integrity_map_continue()
2175 				for (i = ic->sectors_per_block; i < dio->range.n_sectors; i += ic->sectors_per_block, jp++) {  in dm_integrity_map_continue()
2176 					if (!test_journal_node(ic, jp, dio->range.logical_sector + i))  in dm_integrity_map_continue()
2179 				dio->range.n_sectors = i;  in dm_integrity_map_continue()
2183 	if (unlikely(!add_new_range(ic, &dio->range, true))) {  in dm_integrity_map_continue()
2192 			INIT_WORK(&dio->work, integrity_bio_wait);  in dm_integrity_map_continue()
2193 			queue_work(ic->wait_wq, &dio->work);  in dm_integrity_map_continue()
2197 			dio->range.n_sectors = ic->sectors_per_block;  in dm_integrity_map_continue()
2198 		wait_and_add_new_range(ic, &dio->range);  in dm_integrity_map_continue()
2206 			unsigned new_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector);  in dm_integrity_map_continue()
2208 				remove_range_unlocked(ic, &dio->range);  in dm_integrity_map_continue()
2213 	if (ic->mode == 'J' && likely(dio->op == REQ_OP_DISCARD) && !discard_retried) {  in dm_integrity_map_continue()
2215 		unsigned new_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector);  in dm_integrity_map_continue()
2217 		    unlikely(next_sector < dio->range.logical_sector - dio->range.n_sectors)) {  in dm_integrity_map_continue()
2218 			remove_range_unlocked(ic, &dio->range);  in dm_integrity_map_continue()
2236 	if (ic->mode == 'B' && (dio->op == REQ_OP_WRITE || unlikely(dio->op == REQ_OP_DISCARD))) {  in dm_integrity_map_continue()
2237 		if (!block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector,  in dm_integrity_map_continue()
2238 				     dio->range.n_sectors, BITMAP_OP_TEST_ALL_SET)) {  in dm_integrity_map_continue()
2241 			bbs = sector_to_bitmap_block(ic, dio->range.logical_sector);  in dm_integrity_map_continue()
2250 	dio->in_flight = (atomic_t)ATOMIC_INIT(2);  in dm_integrity_map_continue()
2254 		dio->completion = &read_comp;  in dm_integrity_map_continue()
2256 		dio->completion = NULL;  in dm_integrity_map_continue()
2258 	dm_bio_record(&dio->bio_details, bio);  in dm_integrity_map_continue()
2263 	bio->bi_iter.bi_size = dio->range.n_sectors << SECTOR_SHIFT;  in dm_integrity_map_continue()
2265 	if (unlikely(dio->op == REQ_OP_DISCARD) && likely(ic->mode != 'D')) {  in dm_integrity_map_continue()
2266 		integrity_metadata(&dio->work);  in dm_integrity_map_continue()
2269 		dio->in_flight = (atomic_t)ATOMIC_INIT(1);  in dm_integrity_map_continue()
2270 		dio->completion = NULL;  in dm_integrity_map_continue()
2282 		    dio->range.logical_sector + dio->range.n_sectors > le64_to_cpu(ic->sb->recalc_sector))  in dm_integrity_map_continue()
2285 			if (!block_bitmap_op(ic, ic->recalc_bitmap, dio->range.logical_sector,  in dm_integrity_map_continue()
2286 					     dio->range.n_sectors, BITMAP_OP_TEST_ALL_CLEAR))  in dm_integrity_map_continue()
2291 			integrity_metadata(&dio->work);  in dm_integrity_map_continue()
2294 			dec_in_flight(dio);  in dm_integrity_map_continue()
2297 		INIT_WORK(&dio->work, integrity_metadata);  in dm_integrity_map_continue()
2298 		queue_work(ic->metadata_wq, &dio->work);  in dm_integrity_map_continue()
2304 	if (unlikely(__journal_read_write(dio, bio, journal_section, journal_entry)))  in dm_integrity_map_continue()
2307 	do_endio_flush(ic, dio);  in dm_integrity_map_continue()
2313 	struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work);  in integrity_bio_wait()  local
2315 	dm_integrity_map_continue(dio, false);  in integrity_bio_wait()
2767 		struct dm_integrity_io *dio;  in bitmap_block_work()  local
2769 		dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));  in bitmap_block_work()
2771 		if (block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector,  in bitmap_block_work()
2772 				    dio->range.n_sectors, BITMAP_OP_TEST_ALL_SET)) {  in bitmap_block_work()
2773 			remove_range(ic, &dio->range);  in bitmap_block_work()
2774 			INIT_WORK(&dio->work, integrity_bio_wait);  in bitmap_block_work()
2775 			queue_work(ic->offload_wq, &dio->work);  in bitmap_block_work()
2777 			block_bitmap_op(ic, ic->journal, dio->range.logical_sector,  in bitmap_block_work()
2778 					dio->range.n_sectors, BITMAP_OP_SET);  in bitmap_block_work()
2791 		struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));  in bitmap_block_work()  local
2793 		block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector,  in bitmap_block_work()
2794 				dio->range.n_sectors, BITMAP_OP_SET);  in bitmap_block_work()
2796 		remove_range(ic, &dio->range);  in bitmap_block_work()
2797 		INIT_WORK(&dio->work, integrity_bio_wait);  in bitmap_block_work()
2798 		queue_work(ic->offload_wq, &dio->work);  in bitmap_block_work()