372 static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map);
1415 static void submit_flush_bio(struct dm_integrity_c *ic, struct dm_integrity_io *dio)  in submit_flush_bio()  argument
1421 	bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));  in submit_flush_bio()
1444 static void do_endio_flush(struct dm_integrity_c *ic, struct dm_integrity_io *dio)  in do_endio_flush()  argument
1446 	struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));  in do_endio_flush()
1448 	if (unlikely(dio->fua) && likely(!bio->bi_status) && likely(!dm_integrity_failed(ic)))  in do_endio_flush()
1449 		submit_flush_bio(ic, dio);  in do_endio_flush()
1454 static void dec_in_flight(struct dm_integrity_io *dio)  in dec_in_flight()  argument
1456 	if (atomic_dec_and_test(&dio->in_flight)) {  in dec_in_flight()
1457 		struct dm_integrity_c *ic = dio->ic;  in dec_in_flight()
1460 		remove_range(ic, &dio->range);  in dec_in_flight()
1462 		if (dio->op == REQ_OP_WRITE || unlikely(dio->op == REQ_OP_DISCARD))  in dec_in_flight()
1465 		bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));  in dec_in_flight()
1467 		if (unlikely(dio->bi_status) && !bio->bi_status)  in dec_in_flight()
1468 			bio->bi_status = dio->bi_status;  in dec_in_flight()
1469 		if (likely(!bio->bi_status) && unlikely(bio_sectors(bio) != dio->range.n_sectors)) {  in dec_in_flight()
1470 			dio->range.logical_sector += dio->range.n_sectors;  in dec_in_flight()
1471 			bio_advance(bio, dio->range.n_sectors << SECTOR_SHIFT);  in dec_in_flight()
1472 			INIT_WORK(&dio->work, integrity_bio_wait);  in dec_in_flight()
1473 			queue_work(ic->offload_wq, &dio->work);  in dec_in_flight()
1476 		do_endio_flush(ic, dio);  in dec_in_flight()
1482 	struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));  in integrity_end_io()  local
1484 	dm_bio_restore(&dio->bio_details, bio);  in integrity_end_io()
1488 	if (dio->completion)  in integrity_end_io()
1489 		complete(dio->completion);  in integrity_end_io()
1491 	dec_in_flight(dio);  in integrity_end_io()
1541 	struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work);  in integrity_metadata()  local
1542 	struct dm_integrity_c *ic = dio->ic;  in integrity_metadata()
1550 		struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));  in integrity_metadata()
1560 		if (likely(dio->op != REQ_OP_DISCARD))  in integrity_metadata()
1574 		if (unlikely(dio->op == REQ_OP_DISCARD)) {  in integrity_metadata()
1575 			sector_t bi_sector = dio->bio_details.bi_iter.bi_sector;  in integrity_metadata()
1576 			unsigned bi_size = dio->bio_details.bi_iter.bi_size;  in integrity_metadata()
1584 				r = dm_integrity_rw_tag(ic, checksums, &dio->metadata_block, &dio->metadata_offset,  in integrity_metadata()
1606 		sector = dio->range.logical_sector;  in integrity_metadata()
1607 		sectors_to_process = dio->range.n_sectors;  in integrity_metadata()
1609 		__bio_for_each_segment(bv, bio, iter, dio->bio_details.bi_iter) {  in integrity_metadata()
1626 			r = dm_integrity_rw_tag(ic, checksums, &dio->metadata_block, &dio->metadata_offset,  in integrity_metadata()
1627 						checksums_ptr - checksums, dio->op == REQ_OP_READ ? TAG_CMP : TAG_WRITE);  in integrity_metadata()
1654 		struct bio_integrity_payload *bip = dio->bio_details.bi_integrity;  in integrity_metadata()
1659 			unsigned data_to_process = dio->range.n_sectors;  in integrity_metadata()
1670 				r = dm_integrity_rw_tag(ic, tag, &dio->metadata_block, &dio->metadata_offset,  in integrity_metadata()
1671 							this_len, dio->op == REQ_OP_READ ? TAG_READ : TAG_WRITE);  in integrity_metadata()
1681 	dec_in_flight(dio);  in integrity_metadata()
1684 	dio->bi_status = errno_to_blk_status(r);  in integrity_metadata()
1685 	dec_in_flight(dio);  in integrity_metadata()
1691 	struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));  in dm_integrity_map()  local
1696 	dio->ic = ic;  in dm_integrity_map()
1697 	dio->bi_status = 0;  in dm_integrity_map()
1698 	dio->op = bio_op(bio);  in dm_integrity_map()
1700 	if (unlikely(dio->op == REQ_OP_DISCARD)) {  in dm_integrity_map()
1714 		submit_flush_bio(ic, dio);  in dm_integrity_map()
1718 	dio->range.logical_sector = dm_target_offset(ti, bio->bi_iter.bi_sector);  in dm_integrity_map()
1719 	dio->fua = dio->op == REQ_OP_WRITE && bio->bi_opf & REQ_FUA;  in dm_integrity_map()
1720 	if (unlikely(dio->fua)) {  in dm_integrity_map()
1727 	if (unlikely(dio->range.logical_sector + bio_sectors(bio) > ic->provided_data_sectors)) {  in dm_integrity_map()
1729 		      dio->range.logical_sector, bio_sectors(bio),  in dm_integrity_map()
1733 …if (unlikely((dio->range.logical_sector | bio_sectors(bio)) & (unsigned)(ic->sectors_per_block - 1…  in dm_integrity_map()
1736 		      dio->range.logical_sector, bio_sectors(bio));  in dm_integrity_map()
1740 	if (ic->sectors_per_block > 1 && likely(dio->op != REQ_OP_DISCARD)) {  in dm_integrity_map()
1773 	if (unlikely(ic->mode == 'R') && unlikely(dio->op != REQ_OP_READ))  in dm_integrity_map()
1776 	get_area_and_offset(ic, dio->range.logical_sector, &area, &offset);  in dm_integrity_map()
1777 	dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset, &dio->metadata_offset);  in dm_integrity_map()
1780 	dm_integrity_map_continue(dio, true);  in dm_integrity_map()
1784 static bool __journal_read_write(struct dm_integrity_io *dio, struct bio *bio,  in __journal_read_write()  argument
1787 	struct dm_integrity_c *ic = dio->ic;  in __journal_read_write()
1791 	logical_sector = dio->range.logical_sector;  in __journal_read_write()
1792 	n_sectors = dio->range.n_sectors;  in __journal_read_write()
1803 		if (likely(dio->op == REQ_OP_WRITE))  in __journal_read_write()
1809 			if (unlikely(dio->op == REQ_OP_READ)) {  in __journal_read_write()
1856 					if (likely(dio->op == REQ_OP_WRITE))  in __journal_read_write()
1864 					if (likely(dio->op == REQ_OP_WRITE))  in __journal_read_write()
1869 			if (likely(dio->op == REQ_OP_WRITE)) {  in __journal_read_write()
1905 		if (unlikely(dio->op == REQ_OP_READ))  in __journal_read_write()
1910 	if (likely(dio->op == REQ_OP_WRITE)) {  in __journal_read_write()
1920 		remove_range(ic, &dio->range);  in __journal_read_write()
1926 		dio->range.logical_sector = logical_sector;  in __journal_read_write()
1927 		get_area_and_offset(ic, dio->range.logical_sector, &area, &offset);  in __journal_read_write()
1928 		dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset, &dio->metadata_offset);  in __journal_read_write()
1935 static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map)  in dm_integrity_map_continue()  argument
1937 	struct dm_integrity_c *ic = dio->ic;  in dm_integrity_map_continue()
1938 	struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));  in dm_integrity_map_continue()
1943 	bool need_sync_io = ic->internal_hash && dio->op == REQ_OP_READ;  in dm_integrity_map_continue()
1944 	if (unlikely(dio->op == REQ_OP_DISCARD) && ic->mode != 'D')  in dm_integrity_map_continue()
1948 		INIT_WORK(&dio->work, integrity_bio_wait);  in dm_integrity_map_continue()
1949 		queue_work(ic->offload_wq, &dio->work);  in dm_integrity_map_continue()
1961 	dio->range.n_sectors = bio_sectors(bio);  in dm_integrity_map_continue()
1963 	if (ic->mode == 'J' && likely(dio->op != REQ_OP_DISCARD)) {  in dm_integrity_map_continue()
1964 		if (dio->op == REQ_OP_WRITE) {  in dm_integrity_map_continue()
1968 			dio->range.n_sectors = min(dio->range.n_sectors,  in dm_integrity_map_continue()
1970 			if (unlikely(!dio->range.n_sectors)) {  in dm_integrity_map_continue()
1976 			range_sectors = dio->range.n_sectors >> ic->sb->log2_sectors_per_block;  in dm_integrity_map_continue()
1994 				add_journal_node(ic, &ic->journal_tree[pos], dio->range.logical_sector + i);  in dm_integrity_map_continue()
2008 			} while ((i += ic->sectors_per_block) < dio->range.n_sectors);  in dm_integrity_map_continue()
2014 			journal_read_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector);  in dm_integrity_map_continue()
2016 				if (unlikely(dio->range.n_sectors > next_sector - dio->range.logical_sector))  in dm_integrity_map_continue()
2017 					dio->range.n_sectors = next_sector - dio->range.logical_sector;  in dm_integrity_map_continue()
2021 				for (i = ic->sectors_per_block; i < dio->range.n_sectors; i += ic->sectors_per_block, jp++) {  in dm_integrity_map_continue()
2022 					if (!test_journal_node(ic, jp, dio->range.logical_sector + i))  in dm_integrity_map_continue()
2025 				dio->range.n_sectors = i;  in dm_integrity_map_continue()
2029 	if (unlikely(!add_new_range(ic, &dio->range, true))) {  in dm_integrity_map_continue()
2038 			INIT_WORK(&dio->work, integrity_bio_wait);  in dm_integrity_map_continue()
2039 			queue_work(ic->wait_wq, &dio->work);  in dm_integrity_map_continue()
2043 			dio->range.n_sectors = ic->sectors_per_block;  in dm_integrity_map_continue()
2044 		wait_and_add_new_range(ic, &dio->range);  in dm_integrity_map_continue()
2052 			unsigned new_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector);  in dm_integrity_map_continue()
2054 				remove_range_unlocked(ic, &dio->range);  in dm_integrity_map_continue()
2059 	if (ic->mode == 'J' && likely(dio->op == REQ_OP_DISCARD) && !discard_retried) {  in dm_integrity_map_continue()
2061 		unsigned new_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector);  in dm_integrity_map_continue()
2063 		    unlikely(next_sector < dio->range.logical_sector - dio->range.n_sectors)) {  in dm_integrity_map_continue()
2064 			remove_range_unlocked(ic, &dio->range);  in dm_integrity_map_continue()
2082 	if (ic->mode == 'B' && (dio->op == REQ_OP_WRITE || unlikely(dio->op == REQ_OP_DISCARD))) {  in dm_integrity_map_continue()
2083 		if (!block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector,  in dm_integrity_map_continue()
2084 				     dio->range.n_sectors, BITMAP_OP_TEST_ALL_SET)) {  in dm_integrity_map_continue()
2087 			bbs = sector_to_bitmap_block(ic, dio->range.logical_sector);  in dm_integrity_map_continue()
2096 	dio->in_flight = (atomic_t)ATOMIC_INIT(2);  in dm_integrity_map_continue()
2100 		dio->completion = &read_comp;  in dm_integrity_map_continue()
2102 		dio->completion = NULL;  in dm_integrity_map_continue()
2104 	dm_bio_record(&dio->bio_details, bio);  in dm_integrity_map_continue()
2109 	bio->bi_iter.bi_size = dio->range.n_sectors << SECTOR_SHIFT;  in dm_integrity_map_continue()
2111 	if (unlikely(dio->op == REQ_OP_DISCARD) && likely(ic->mode != 'D')) {  in dm_integrity_map_continue()
2112 		integrity_metadata(&dio->work);  in dm_integrity_map_continue()
2115 		dio->in_flight = (atomic_t)ATOMIC_INIT(1);  in dm_integrity_map_continue()
2116 		dio->completion = NULL;  in dm_integrity_map_continue()
2128 		    dio->range.logical_sector + dio->range.n_sectors > le64_to_cpu(ic->sb->recalc_sector))  in dm_integrity_map_continue()
2131 			if (!block_bitmap_op(ic, ic->recalc_bitmap, dio->range.logical_sector,  in dm_integrity_map_continue()
2132 					     dio->range.n_sectors, BITMAP_OP_TEST_ALL_CLEAR))  in dm_integrity_map_continue()
2137 			integrity_metadata(&dio->work);  in dm_integrity_map_continue()
2140 			dec_in_flight(dio);  in dm_integrity_map_continue()
2143 		INIT_WORK(&dio->work, integrity_metadata);  in dm_integrity_map_continue()
2144 		queue_work(ic->metadata_wq, &dio->work);  in dm_integrity_map_continue()
2150 	if (unlikely(__journal_read_write(dio, bio, journal_section, journal_entry)))  in dm_integrity_map_continue()
2153 	do_endio_flush(ic, dio);  in dm_integrity_map_continue()
2159 	struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work);  in integrity_bio_wait()  local
2161 	dm_integrity_map_continue(dio, false);  in integrity_bio_wait()
2613 		struct dm_integrity_io *dio;  in bitmap_block_work()  local
2615 		dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));  in bitmap_block_work()
2617 		if (block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector,  in bitmap_block_work()
2618 				    dio->range.n_sectors, BITMAP_OP_TEST_ALL_SET)) {  in bitmap_block_work()
2619 			remove_range(ic, &dio->range);  in bitmap_block_work()
2620 			INIT_WORK(&dio->work, integrity_bio_wait);  in bitmap_block_work()
2621 			queue_work(ic->offload_wq, &dio->work);  in bitmap_block_work()
2623 			block_bitmap_op(ic, ic->journal, dio->range.logical_sector,  in bitmap_block_work()
2624 					dio->range.n_sectors, BITMAP_OP_SET);  in bitmap_block_work()
2637 		struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));  in bitmap_block_work()  local
2639 		block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector,  in bitmap_block_work()
2640 				dio->range.n_sectors, BITMAP_OP_SET);  in bitmap_block_work()
2642 		remove_range(ic, &dio->range);  in bitmap_block_work()
2643 		INIT_WORK(&dio->work, integrity_bio_wait);  in bitmap_block_work()
2644 		queue_work(ic->offload_wq, &dio->work);  in bitmap_block_work()