615  * Search hash table for a hydration with hd->region_nr == region_nr
622 	struct dm_clone_region_hydration *hd;  in __hash_find()  local
624 	hlist_for_each_entry(hd, &bucket->head, h) {  in __hash_find()
625 		if (hd->region_nr == region_nr)  in __hash_find()
626 			return hd;  in __hash_find()
638 					     struct dm_clone_region_hydration *hd)  in __insert_region_hydration()  argument
640 	hlist_add_head(&hd->h, &bucket->head);  in __insert_region_hydration()
652 				  struct dm_clone_region_hydration *hd)  in __find_or_insert_region_hydration()  argument
656 	hd2 = __hash_find(bucket, hd->region_nr);  in __find_or_insert_region_hydration()
660 	__insert_region_hydration(bucket, hd);  in __find_or_insert_region_hydration()
662 	return hd;  in __find_or_insert_region_hydration()
670 	struct dm_clone_region_hydration *hd;  in alloc_hydration()  local
676 	hd = mempool_alloc(&clone->hydration_pool, GFP_NOIO);  in alloc_hydration()
677 	hd->clone = clone;  in alloc_hydration()
679 	return hd;  in alloc_hydration()
682 static inline void free_hydration(struct dm_clone_region_hydration *hd)  in free_hydration()  argument
684 	mempool_free(hd, &hd->clone->hydration_pool);  in free_hydration()
688 static void hydration_init(struct dm_clone_region_hydration *hd, unsigned long region_nr)  in hydration_init()  argument
690 	hd->region_nr = region_nr;  in hydration_init()
691 	hd->overwrite_bio = NULL;  in hydration_init()
692 	bio_list_init(&hd->deferred_bios);  in hydration_init()
693 	hd->status = 0;  in hydration_init()
695 	INIT_LIST_HEAD(&hd->list);  in hydration_init()
696 	INIT_HLIST_NODE(&hd->h);  in hydration_init()
705 static int hydration_update_metadata(struct dm_clone_region_hydration *hd)  in hydration_update_metadata()  argument
710 	struct clone *clone = hd->clone;  in hydration_update_metadata()
716 	if (likely(!r) && hd->status == BLK_STS_OK)  in hydration_update_metadata()
717 		r = dm_clone_set_region_hydrated(clone->cmd, hd->region_nr);  in hydration_update_metadata()
719 	bucket = get_hash_table_bucket(clone, hd->region_nr);  in hydration_update_metadata()
723 	hlist_del(&hd->h);  in hydration_update_metadata()
739 static void hydration_complete(struct dm_clone_region_hydration *hd)  in hydration_complete()  argument
743 	struct clone *clone = hd->clone;  in hydration_complete()
745 	r = hydration_update_metadata(hd);  in hydration_complete()
747 	if (hd->status == BLK_STS_OK && likely(!r)) {  in hydration_complete()
748 		if (hd->overwrite_bio)  in hydration_complete()
749 			complete_overwrite_bio(clone, hd->overwrite_bio);  in hydration_complete()
751 		issue_deferred_bios(clone, &hd->deferred_bios);  in hydration_complete()
753 		status = r ? BLK_STS_IOERR : hd->status;  in hydration_complete()
755 		if (hd->overwrite_bio)  in hydration_complete()
756 			bio_list_add(&hd->deferred_bios, hd->overwrite_bio);  in hydration_complete()
758 		fail_bios(&hd->deferred_bios, status);  in hydration_complete()
761 	free_hydration(hd);  in hydration_complete()
771 	struct dm_clone_region_hydration *tmp, *hd = context;  in hydration_kcopyd_callback()  local
772 	struct clone *clone = hd->clone;  in hydration_kcopyd_callback()
782 	list_splice_tail(&hd->list, &batched_hydrations);  in hydration_kcopyd_callback()
784 	hd->status = status;  in hydration_kcopyd_callback()
785 	hydration_complete(hd);  in hydration_kcopyd_callback()
788 	list_for_each_entry_safe(hd, tmp, &batched_hydrations, list) {  in hydration_kcopyd_callback()
789 		hd->status = status;  in hydration_kcopyd_callback()
790 		hydration_complete(hd);  in hydration_kcopyd_callback()
799 static void hydration_copy(struct dm_clone_region_hydration *hd, unsigned int nr_regions)  in hydration_copy()  argument
804 	struct clone *clone = hd->clone;  in hydration_copy()
810 	region_start = hd->region_nr;  in hydration_copy()
840 		       hydration_kcopyd_callback, hd);  in hydration_copy()
845 	struct dm_clone_region_hydration *hd = bio->bi_private;  in overwrite_endio()  local
847 	bio->bi_end_io = hd->overwrite_bio_end_io;  in overwrite_endio()
848 	hd->status = bio->bi_status;  in overwrite_endio()
850 	hydration_complete(hd);  in overwrite_endio()
853 static void hydration_overwrite(struct dm_clone_region_hydration *hd, struct bio *bio)  in hydration_overwrite()  argument
860 	hd->overwrite_bio = bio;  in hydration_overwrite()
861 	hd->overwrite_bio_end_io = bio->bi_end_io;  in hydration_overwrite()
864 	bio->bi_private = hd;  in hydration_overwrite()
866 	atomic_inc(&hd->clone->hydrations_in_flight);  in hydration_overwrite()
884 	struct dm_clone_region_hydration *hd, *hd2;  in hydrate_bio_region()  local
891 	hd = __hash_find(bucket, region_nr);  in hydrate_bio_region()
892 	if (hd) {  in hydrate_bio_region()
894 		bio_list_add(&hd->deferred_bios, bio);  in hydrate_bio_region()
912 	hd = alloc_hydration(clone);  in hydrate_bio_region()
913 	hydration_init(hd, region_nr);  in hydrate_bio_region()
920 		free_hydration(hd);  in hydrate_bio_region()
925 	hd2 = __find_or_insert_region_hydration(bucket, hd);  in hydrate_bio_region()
926 	if (hd2 != hd) {  in hydrate_bio_region()
930 		free_hydration(hd);  in hydrate_bio_region()
940 		hlist_del(&hd->h);  in hydrate_bio_region()
942 		free_hydration(hd);  in hydrate_bio_region()
956 		hydration_overwrite(hd, bio);  in hydrate_bio_region()
958 		bio_list_add(&hd->deferred_bios, bio);  in hydrate_bio_region()
960 		hydration_copy(hd, 1);  in hydrate_bio_region()
985 			      struct dm_clone_region_hydration *hd)  in __batch_hydration()  argument
987 	struct clone *clone = hd->clone;  in __batch_hydration()
993 		    (batch->head->region_nr + batch->nr_batched_regions) == hd->region_nr) {  in __batch_hydration()
994 			list_add_tail(&hd->list, &batch->head->list);  in __batch_hydration()
996 			hd = NULL;  in __batch_hydration()
1000 		if (batch->nr_batched_regions >= max_batch_size || hd) {  in __batch_hydration()
1007 	if (!hd)  in __batch_hydration()
1012 		hydration_copy(hd, 1);  in __batch_hydration()
1017 	BUG_ON(!list_empty(&hd->list));  in __batch_hydration()
1018 	batch->head = hd;  in __batch_hydration()
1027 	struct dm_clone_region_hydration *hd;  in __start_next_hydration()  local
1030 	hd = alloc_hydration(clone);  in __start_next_hydration()
1043 			hydration_init(hd, offset);  in __start_next_hydration()
1044 			__insert_region_hydration(bucket, hd);  in __start_next_hydration()
1048 			__batch_hydration(batch, hd);  in __start_next_hydration()
1057 	if (hd)  in __start_next_hydration()
1058 		free_hydration(hd);  in __start_next_hydration()