67 static int btrfs_dev_replace_finishing(struct btrfs_fs_info *fs_info,
71 int btrfs_init_dev_replace(struct btrfs_fs_info *fs_info)  in btrfs_init_dev_replace()  argument
74 	struct btrfs_root *dev_root = fs_info->dev_root;  in btrfs_init_dev_replace()
75 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_init_dev_replace()
103 		if (btrfs_find_device(fs_info->fs_devices,  in btrfs_init_dev_replace()
105 			btrfs_err(fs_info,  in btrfs_init_dev_replace()
135 		btrfs_warn(fs_info,  in btrfs_init_dev_replace()
166 		if (btrfs_find_device(fs_info->fs_devices,  in btrfs_init_dev_replace()
168 			btrfs_err(fs_info,  in btrfs_init_dev_replace()
178 		dev_replace->srcdev = btrfs_find_device(fs_info->fs_devices,  in btrfs_init_dev_replace()
180 		dev_replace->tgtdev = btrfs_find_device(fs_info->fs_devices,  in btrfs_init_dev_replace()
188 		    !btrfs_test_opt(fs_info, DEGRADED)) {  in btrfs_init_dev_replace()
190 			btrfs_warn(fs_info,  in btrfs_init_dev_replace()
192 			btrfs_warn(fs_info,  in btrfs_init_dev_replace()
197 		    !btrfs_test_opt(fs_info, DEGRADED)) {  in btrfs_init_dev_replace()
199 			btrfs_warn(fs_info,  in btrfs_init_dev_replace()
201 			btrfs_warn(fs_info,  in btrfs_init_dev_replace()
221 			WARN_ON(fs_info->fs_devices->rw_devices == 0);  in btrfs_init_dev_replace()
222 			dev_replace->tgtdev->io_width = fs_info->sectorsize;  in btrfs_init_dev_replace()
223 			dev_replace->tgtdev->io_align = fs_info->sectorsize;  in btrfs_init_dev_replace()
224 			dev_replace->tgtdev->sector_size = fs_info->sectorsize;  in btrfs_init_dev_replace()
225 			dev_replace->tgtdev->fs_info = fs_info;  in btrfs_init_dev_replace()
243 static int btrfs_init_dev_replace_tgtdev(struct btrfs_fs_info *fs_info,  in btrfs_init_dev_replace_tgtdev()  argument
256 		btrfs_err(fs_info, "the filesystem is a seed filesystem!");  in btrfs_init_dev_replace_tgtdev()
261 				  fs_info->bdev_holder);  in btrfs_init_dev_replace_tgtdev()
263 		btrfs_err(fs_info, "target device %s is invalid!", device_path);  in btrfs_init_dev_replace_tgtdev()
267 	if (!btrfs_check_device_zone_type(fs_info, bdev)) {  in btrfs_init_dev_replace_tgtdev()
268 		btrfs_err(fs_info,  in btrfs_init_dev_replace_tgtdev()
276 	list_for_each_entry(device, &fs_info->fs_devices->devices, dev_list) {  in btrfs_init_dev_replace_tgtdev()
278 			btrfs_err(fs_info,  in btrfs_init_dev_replace_tgtdev()
288 		btrfs_err(fs_info,  in btrfs_init_dev_replace_tgtdev()
311 	device->io_width = fs_info->sectorsize;  in btrfs_init_dev_replace_tgtdev()
312 	device->io_align = fs_info->sectorsize;  in btrfs_init_dev_replace_tgtdev()
313 	device->sector_size = fs_info->sectorsize;  in btrfs_init_dev_replace_tgtdev()
319 	device->fs_info = fs_info;  in btrfs_init_dev_replace_tgtdev()
326 	device->fs_devices = fs_info->fs_devices;  in btrfs_init_dev_replace_tgtdev()
332 	mutex_lock(&fs_info->fs_devices->device_list_mutex);  in btrfs_init_dev_replace_tgtdev()
333 	list_add(&device->dev_list, &fs_info->fs_devices->devices);  in btrfs_init_dev_replace_tgtdev()
334 	fs_info->fs_devices->num_devices++;  in btrfs_init_dev_replace_tgtdev()
335 	fs_info->fs_devices->open_devices++;  in btrfs_init_dev_replace_tgtdev()
336 	mutex_unlock(&fs_info->fs_devices->device_list_mutex);  in btrfs_init_dev_replace_tgtdev()
352 	struct btrfs_fs_info *fs_info = trans->fs_info;  in btrfs_run_dev_replace()  local
354 	struct btrfs_root *dev_root = fs_info->dev_root;  in btrfs_run_dev_replace()
359 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_run_dev_replace()
380 		btrfs_warn(fs_info,  in btrfs_run_dev_replace()
401 			btrfs_warn(fs_info,  in btrfs_run_dev_replace()
415 			btrfs_warn(fs_info,  in btrfs_run_dev_replace()
466 static int mark_block_group_to_copy(struct btrfs_fs_info *fs_info,  in mark_block_group_to_copy()  argument
472 	struct btrfs_root *root = fs_info->dev_root;  in mark_block_group_to_copy()
480 	if (!btrfs_is_zoned(fs_info))  in mark_block_group_to_copy()
483 	mutex_lock(&fs_info->chunk_mutex);  in mark_block_group_to_copy()
486 	spin_lock(&fs_info->trans_lock);  in mark_block_group_to_copy()
487 	while (fs_info->running_transaction &&  in mark_block_group_to_copy()
488 	       !list_empty(&fs_info->running_transaction->dev_update_list)) {  in mark_block_group_to_copy()
489 		spin_unlock(&fs_info->trans_lock);  in mark_block_group_to_copy()
490 		mutex_unlock(&fs_info->chunk_mutex);  in mark_block_group_to_copy()
494 			mutex_lock(&fs_info->chunk_mutex);  in mark_block_group_to_copy()
496 				spin_lock(&fs_info->trans_lock);  in mark_block_group_to_copy()
504 		mutex_lock(&fs_info->chunk_mutex);  in mark_block_group_to_copy()
508 		spin_lock(&fs_info->trans_lock);  in mark_block_group_to_copy()
510 	spin_unlock(&fs_info->trans_lock);  in mark_block_group_to_copy()
563 		cache = btrfs_lookup_block_group(fs_info, chunk_offset);  in mark_block_group_to_copy()
585 	mutex_unlock(&fs_info->chunk_mutex);  in mark_block_group_to_copy()
594 	struct btrfs_fs_info *fs_info = cache->fs_info;  in btrfs_finish_block_group_to_copy()  local
602 	if (!btrfs_is_zoned(fs_info))  in btrfs_finish_block_group_to_copy()
612 	em = btrfs_get_chunk_map(fs_info, chunk_offset, 1);  in btrfs_finish_block_group_to_copy()
645 static int btrfs_dev_replace_start(struct btrfs_fs_info *fs_info,  in btrfs_dev_replace_start()  argument
649 	struct btrfs_root *root = fs_info->dev_root;  in btrfs_dev_replace_start()
651 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_start()
656 	src_device = btrfs_find_device_by_devspec(fs_info, srcdevid,  in btrfs_dev_replace_start()
661 	if (btrfs_pinned_by_swapfile(fs_info, src_device)) {  in btrfs_dev_replace_start()
662 		btrfs_warn_in_rcu(fs_info,  in btrfs_dev_replace_start()
681 	ret = btrfs_init_dev_replace_tgtdev(fs_info, tgtdev_name,  in btrfs_dev_replace_start()
686 	ret = mark_block_group_to_copy(fs_info, src_device);  in btrfs_dev_replace_start()
708 	btrfs_info_in_rcu(fs_info,  in btrfs_dev_replace_start()
732 		btrfs_err(fs_info, "kobj add dev failed %d", ret);  in btrfs_dev_replace_start()
734 	btrfs_wait_ordered_roots(fs_info, U64_MAX, 0, (u64)-1);  in btrfs_dev_replace_start()
753 	ret = btrfs_scrub_dev(fs_info, src_device->devid, 0,  in btrfs_dev_replace_start()
757 	ret = btrfs_dev_replace_finishing(fs_info, ret);  in btrfs_dev_replace_start()
768 int btrfs_dev_replace_by_ioctl(struct btrfs_fs_info *fs_info,  in btrfs_dev_replace_by_ioctl()  argument
785 	ret = btrfs_dev_replace_start(fs_info, args->start.tgtdev_name,  in btrfs_dev_replace_by_ioctl()
801 static void btrfs_rm_dev_replace_blocked(struct btrfs_fs_info *fs_info)  in btrfs_rm_dev_replace_blocked()  argument
803 	set_bit(BTRFS_FS_STATE_DEV_REPLACING, &fs_info->fs_state);  in btrfs_rm_dev_replace_blocked()
804 	wait_event(fs_info->dev_replace.replace_wait, !percpu_counter_sum(  in btrfs_rm_dev_replace_blocked()
805 		   &fs_info->dev_replace.bio_counter));  in btrfs_rm_dev_replace_blocked()
811 static void btrfs_rm_dev_replace_unblocked(struct btrfs_fs_info *fs_info)  in btrfs_rm_dev_replace_unblocked()  argument
813 	clear_bit(BTRFS_FS_STATE_DEV_REPLACING, &fs_info->fs_state);  in btrfs_rm_dev_replace_unblocked()
814 	wake_up(&fs_info->dev_replace.replace_wait);  in btrfs_rm_dev_replace_unblocked()
832 	lockdep_assert_held(&srcdev->fs_info->chunk_mutex);  in btrfs_set_target_alloc_state()
849 						struct btrfs_fs_info *fs_info,  in btrfs_dev_replace_update_device_in_mapping_tree()  argument
853 	struct extent_map_tree *em_tree = &fs_info->mapping_tree;  in btrfs_dev_replace_update_device_in_mapping_tree()
874 static int btrfs_dev_replace_finishing(struct btrfs_fs_info *fs_info,  in btrfs_dev_replace_finishing()  argument
877 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_finishing()
880 	struct btrfs_root *root = fs_info->tree_root;  in btrfs_dev_replace_finishing()
905 	ret = btrfs_start_delalloc_roots(fs_info, LONG_MAX, false);  in btrfs_dev_replace_finishing()
910 	btrfs_wait_ordered_roots(fs_info, U64_MAX, 0, (u64)-1);  in btrfs_dev_replace_finishing()
931 		mutex_lock(&fs_info->fs_devices->device_list_mutex);  in btrfs_dev_replace_finishing()
933 		mutex_lock(&fs_info->chunk_mutex);  in btrfs_dev_replace_finishing()
936 			mutex_unlock(&fs_info->fs_devices->device_list_mutex);  in btrfs_dev_replace_finishing()
937 			mutex_unlock(&fs_info->chunk_mutex);  in btrfs_dev_replace_finishing()
960 		btrfs_dev_replace_update_device_in_mapping_tree(fs_info,  in btrfs_dev_replace_finishing()
965 			btrfs_err_in_rcu(fs_info,  in btrfs_dev_replace_finishing()
972 		mutex_unlock(&fs_info->chunk_mutex);  in btrfs_dev_replace_finishing()
973 		mutex_unlock(&fs_info->fs_devices->device_list_mutex);  in btrfs_dev_replace_finishing()
975 		btrfs_rm_dev_replace_blocked(fs_info);  in btrfs_dev_replace_finishing()
978 		btrfs_rm_dev_replace_unblocked(fs_info);  in btrfs_dev_replace_finishing()
984 	btrfs_info_in_rcu(fs_info,  in btrfs_dev_replace_finishing()
1003 	list_add(&tgt_device->dev_alloc_list, &fs_info->fs_devices->alloc_list);  in btrfs_dev_replace_finishing()
1004 	fs_info->fs_devices->rw_devices++;  in btrfs_dev_replace_finishing()
1007 	btrfs_rm_dev_replace_blocked(fs_info);  in btrfs_dev_replace_finishing()
1011 	btrfs_rm_dev_replace_unblocked(fs_info);  in btrfs_dev_replace_finishing()
1026 	mutex_unlock(&fs_info->chunk_mutex);  in btrfs_dev_replace_finishing()
1027 	mutex_unlock(&fs_info->fs_devices->device_list_mutex);  in btrfs_dev_replace_finishing()
1033 		btrfs_scratch_superblocks(fs_info, src_device->bdev,  in btrfs_dev_replace_finishing()
1053 static u64 btrfs_dev_replace_progress(struct btrfs_fs_info *fs_info)  in btrfs_dev_replace_progress()  argument
1055 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_progress()
1077 void btrfs_dev_replace_status(struct btrfs_fs_info *fs_info,  in btrfs_dev_replace_status()  argument
1080 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_status()
1093 	args->status.progress_1000 = btrfs_dev_replace_progress(fs_info);  in btrfs_dev_replace_status()
1097 int btrfs_dev_replace_cancel(struct btrfs_fs_info *fs_info)  in btrfs_dev_replace_cancel()  argument
1099 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_cancel()
1103 	struct btrfs_root *root = fs_info->tree_root;  in btrfs_dev_replace_cancel()
1107 	if (sb_rdonly(fs_info->sb))  in btrfs_dev_replace_cancel()
1123 		ret = btrfs_scrub_cancel(fs_info);  in btrfs_dev_replace_cancel()
1132 			btrfs_info_in_rcu(fs_info,  in btrfs_dev_replace_cancel()
1156 		ret = btrfs_scrub_cancel(fs_info);  in btrfs_dev_replace_cancel()
1167 		btrfs_info_in_rcu(fs_info,  in btrfs_dev_replace_cancel()
1184 void btrfs_dev_replace_suspend_for_unmount(struct btrfs_fs_info *fs_info)  in btrfs_dev_replace_suspend_for_unmount()  argument
1186 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_suspend_for_unmount()
1202 		btrfs_info(fs_info, "suspending dev_replace for unmount");  in btrfs_dev_replace_suspend_for_unmount()
1211 int btrfs_resume_dev_replace_async(struct btrfs_fs_info *fs_info)  in btrfs_resume_dev_replace_async()  argument
1214 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_resume_dev_replace_async()
1232 		btrfs_info(fs_info,  in btrfs_resume_dev_replace_async()
1234 		btrfs_info(fs_info,  in btrfs_resume_dev_replace_async()
1248 	if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_DEV_REPLACE)) {  in btrfs_resume_dev_replace_async()
1253 		btrfs_info(fs_info,  in btrfs_resume_dev_replace_async()
1258 	task = kthread_run(btrfs_dev_replace_kthread, fs_info, "btrfs-devrepl");  in btrfs_resume_dev_replace_async()
1264 	struct btrfs_fs_info *fs_info = data;  in btrfs_dev_replace_kthread()  local
1265 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_kthread()
1269 	progress = btrfs_dev_replace_progress(fs_info);  in btrfs_dev_replace_kthread()
1271 	btrfs_info_in_rcu(fs_info,  in btrfs_dev_replace_kthread()
1278 	ret = btrfs_scrub_dev(fs_info, dev_replace->srcdev->devid,  in btrfs_dev_replace_kthread()
1282 	ret = btrfs_dev_replace_finishing(fs_info, ret);  in btrfs_dev_replace_kthread()
1285 	btrfs_exclop_finish(fs_info);  in btrfs_dev_replace_kthread()
1316 void btrfs_bio_counter_inc_noblocked(struct btrfs_fs_info *fs_info)  in btrfs_bio_counter_inc_noblocked()  argument
1318 	percpu_counter_inc(&fs_info->dev_replace.bio_counter);  in btrfs_bio_counter_inc_noblocked()
1321 void btrfs_bio_counter_sub(struct btrfs_fs_info *fs_info, s64 amount)  in btrfs_bio_counter_sub()  argument
1323 	percpu_counter_sub(&fs_info->dev_replace.bio_counter, amount);  in btrfs_bio_counter_sub()
1324 	cond_wake_up_nomb(&fs_info->dev_replace.replace_wait);  in btrfs_bio_counter_sub()
1327 void btrfs_bio_counter_inc_blocked(struct btrfs_fs_info *fs_info)  in btrfs_bio_counter_inc_blocked()  argument
1330 		percpu_counter_inc(&fs_info->dev_replace.bio_counter);  in btrfs_bio_counter_inc_blocked()
1332 				     &fs_info->fs_state)))  in btrfs_bio_counter_inc_blocked()
1335 		btrfs_bio_counter_dec(fs_info);  in btrfs_bio_counter_inc_blocked()
1336 		wait_event(fs_info->dev_replace.replace_wait,  in btrfs_bio_counter_inc_blocked()
1338 				     &fs_info->fs_state));  in btrfs_bio_counter_inc_blocked()