75 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_init_dev_replace()  local
111 		dev_replace->replace_state =  in btrfs_init_dev_replace()
113 		dev_replace->cont_reading_from_srcdev_mode =  in btrfs_init_dev_replace()
115 		dev_replace->time_started = 0;  in btrfs_init_dev_replace()
116 		dev_replace->time_stopped = 0;  in btrfs_init_dev_replace()
117 		atomic64_set(&dev_replace->num_write_errors, 0);  in btrfs_init_dev_replace()
118 		atomic64_set(&dev_replace->num_uncorrectable_read_errors, 0);  in btrfs_init_dev_replace()
119 		dev_replace->cursor_left = 0;  in btrfs_init_dev_replace()
120 		dev_replace->committed_cursor_left = 0;  in btrfs_init_dev_replace()
121 		dev_replace->cursor_left_last_write_of_item = 0;  in btrfs_init_dev_replace()
122 		dev_replace->cursor_right = 0;  in btrfs_init_dev_replace()
123 		dev_replace->srcdev = NULL;  in btrfs_init_dev_replace()
124 		dev_replace->tgtdev = NULL;  in btrfs_init_dev_replace()
125 		dev_replace->is_valid = 0;  in btrfs_init_dev_replace()
126 		dev_replace->item_needs_writeback = 0;  in btrfs_init_dev_replace()
141 	dev_replace->cont_reading_from_srcdev_mode =  in btrfs_init_dev_replace()
143 	dev_replace->replace_state = btrfs_dev_replace_replace_state(eb, ptr);  in btrfs_init_dev_replace()
144 	dev_replace->time_started = btrfs_dev_replace_time_started(eb, ptr);  in btrfs_init_dev_replace()
145 	dev_replace->time_stopped =  in btrfs_init_dev_replace()
147 	atomic64_set(&dev_replace->num_write_errors,  in btrfs_init_dev_replace()
149 	atomic64_set(&dev_replace->num_uncorrectable_read_errors,  in btrfs_init_dev_replace()
151 	dev_replace->cursor_left = btrfs_dev_replace_cursor_left(eb, ptr);  in btrfs_init_dev_replace()
152 	dev_replace->committed_cursor_left = dev_replace->cursor_left;  in btrfs_init_dev_replace()
153 	dev_replace->cursor_left_last_write_of_item = dev_replace->cursor_left;  in btrfs_init_dev_replace()
154 	dev_replace->cursor_right = btrfs_dev_replace_cursor_right(eb, ptr);  in btrfs_init_dev_replace()
155 	dev_replace->is_valid = 1;  in btrfs_init_dev_replace()
157 	dev_replace->item_needs_writeback = 0;  in btrfs_init_dev_replace()
158 	switch (dev_replace->replace_state) {  in btrfs_init_dev_replace()
172 			dev_replace->srcdev = NULL;  in btrfs_init_dev_replace()
173 			dev_replace->tgtdev = NULL;  in btrfs_init_dev_replace()
178 		dev_replace->srcdev = btrfs_find_device(fs_info->fs_devices,  in btrfs_init_dev_replace()
180 		dev_replace->tgtdev = btrfs_find_device(fs_info->fs_devices,  in btrfs_init_dev_replace()
187 		if (!dev_replace->srcdev &&  in btrfs_init_dev_replace()
196 		if (!dev_replace->tgtdev &&  in btrfs_init_dev_replace()
205 		if (dev_replace->tgtdev) {  in btrfs_init_dev_replace()
206 			if (dev_replace->srcdev) {  in btrfs_init_dev_replace()
207 				dev_replace->tgtdev->total_bytes =  in btrfs_init_dev_replace()
208 					dev_replace->srcdev->total_bytes;  in btrfs_init_dev_replace()
209 				dev_replace->tgtdev->disk_total_bytes =  in btrfs_init_dev_replace()
210 					dev_replace->srcdev->disk_total_bytes;  in btrfs_init_dev_replace()
211 				dev_replace->tgtdev->commit_total_bytes =  in btrfs_init_dev_replace()
212 					dev_replace->srcdev->commit_total_bytes;  in btrfs_init_dev_replace()
213 				dev_replace->tgtdev->bytes_used =  in btrfs_init_dev_replace()
214 					dev_replace->srcdev->bytes_used;  in btrfs_init_dev_replace()
215 				dev_replace->tgtdev->commit_bytes_used =  in btrfs_init_dev_replace()
216 					dev_replace->srcdev->commit_bytes_used;  in btrfs_init_dev_replace()
219 				&dev_replace->tgtdev->dev_state);  in btrfs_init_dev_replace()
222 			dev_replace->tgtdev->io_width = fs_info->sectorsize;  in btrfs_init_dev_replace()
223 			dev_replace->tgtdev->io_align = fs_info->sectorsize;  in btrfs_init_dev_replace()
224 			dev_replace->tgtdev->sector_size = fs_info->sectorsize;  in btrfs_init_dev_replace()
225 			dev_replace->tgtdev->fs_info = fs_info;  in btrfs_init_dev_replace()
227 				&dev_replace->tgtdev->dev_state);  in btrfs_init_dev_replace()
359 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_run_dev_replace()  local
361 	down_read(&dev_replace->rwsem);  in btrfs_run_dev_replace()
362 	if (!dev_replace->is_valid ||  in btrfs_run_dev_replace()
363 	    !dev_replace->item_needs_writeback) {  in btrfs_run_dev_replace()
364 		up_read(&dev_replace->rwsem);  in btrfs_run_dev_replace()
367 	up_read(&dev_replace->rwsem);  in btrfs_run_dev_replace()
425 	down_write(&dev_replace->rwsem);  in btrfs_run_dev_replace()
426 	if (dev_replace->srcdev)  in btrfs_run_dev_replace()
428 			dev_replace->srcdev->devid);  in btrfs_run_dev_replace()
432 		dev_replace->cont_reading_from_srcdev_mode);  in btrfs_run_dev_replace()
434 		dev_replace->replace_state);  in btrfs_run_dev_replace()
435 	btrfs_set_dev_replace_time_started(eb, ptr, dev_replace->time_started);  in btrfs_run_dev_replace()
436 	btrfs_set_dev_replace_time_stopped(eb, ptr, dev_replace->time_stopped);  in btrfs_run_dev_replace()
438 		atomic64_read(&dev_replace->num_write_errors));  in btrfs_run_dev_replace()
440 		atomic64_read(&dev_replace->num_uncorrectable_read_errors));  in btrfs_run_dev_replace()
441 	dev_replace->cursor_left_last_write_of_item =  in btrfs_run_dev_replace()
442 		dev_replace->cursor_left;  in btrfs_run_dev_replace()
444 		dev_replace->cursor_left_last_write_of_item);  in btrfs_run_dev_replace()
446 		dev_replace->cursor_right);  in btrfs_run_dev_replace()
447 	dev_replace->item_needs_writeback = 0;  in btrfs_run_dev_replace()
448 	up_write(&dev_replace->rwsem);  in btrfs_run_dev_replace()
651 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_start()  local
690 	down_write(&dev_replace->rwsem);  in btrfs_dev_replace_start()
691 	switch (dev_replace->replace_state) {  in btrfs_dev_replace_start()
700 		up_write(&dev_replace->rwsem);  in btrfs_dev_replace_start()
704 	dev_replace->cont_reading_from_srcdev_mode = read_src;  in btrfs_dev_replace_start()
705 	dev_replace->srcdev = src_device;  in btrfs_dev_replace_start()
706 	dev_replace->tgtdev = tgt_device;  in btrfs_dev_replace_start()
718 	dev_replace->replace_state = BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED;  in btrfs_dev_replace_start()
719 	dev_replace->time_started = ktime_get_real_seconds();  in btrfs_dev_replace_start()
720 	dev_replace->cursor_left = 0;  in btrfs_dev_replace_start()
721 	dev_replace->committed_cursor_left = 0;  in btrfs_dev_replace_start()
722 	dev_replace->cursor_left_last_write_of_item = 0;  in btrfs_dev_replace_start()
723 	dev_replace->cursor_right = 0;  in btrfs_dev_replace_start()
724 	dev_replace->is_valid = 1;  in btrfs_dev_replace_start()
725 	dev_replace->item_needs_writeback = 1;  in btrfs_dev_replace_start()
726 	atomic64_set(&dev_replace->num_write_errors, 0);  in btrfs_dev_replace_start()
727 	atomic64_set(&dev_replace->num_uncorrectable_read_errors, 0);  in btrfs_dev_replace_start()
728 	up_write(&dev_replace->rwsem);  in btrfs_dev_replace_start()
740 		down_write(&dev_replace->rwsem);  in btrfs_dev_replace_start()
741 		dev_replace->replace_state =  in btrfs_dev_replace_start()
743 		dev_replace->srcdev = NULL;  in btrfs_dev_replace_start()
744 		dev_replace->tgtdev = NULL;  in btrfs_dev_replace_start()
745 		up_write(&dev_replace->rwsem);  in btrfs_dev_replace_start()
755 			      &dev_replace->scrub_progress, 0, 1);  in btrfs_dev_replace_start()
804 	wait_event(fs_info->dev_replace.replace_wait, !percpu_counter_sum(  in btrfs_rm_dev_replace_blocked()
805 		   &fs_info->dev_replace.bio_counter));  in btrfs_rm_dev_replace_blocked()
814 	wake_up(&fs_info->dev_replace.replace_wait);  in btrfs_rm_dev_replace_unblocked()
877 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_finishing()  local
886 	mutex_lock(&dev_replace->lock_finishing_cancel_unmount);  in btrfs_dev_replace_finishing()
888 	down_read(&dev_replace->rwsem);  in btrfs_dev_replace_finishing()
890 	if (dev_replace->replace_state !=  in btrfs_dev_replace_finishing()
892 		up_read(&dev_replace->rwsem);  in btrfs_dev_replace_finishing()
893 		mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);  in btrfs_dev_replace_finishing()
897 	tgt_device = dev_replace->tgtdev;  in btrfs_dev_replace_finishing()
898 	src_device = dev_replace->srcdev;  in btrfs_dev_replace_finishing()
899 	up_read(&dev_replace->rwsem);  in btrfs_dev_replace_finishing()
907 		mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);  in btrfs_dev_replace_finishing()
924 			mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);  in btrfs_dev_replace_finishing()
943 	down_write(&dev_replace->rwsem);  in btrfs_dev_replace_finishing()
944 	dev_replace->replace_state =  in btrfs_dev_replace_finishing()
947 	dev_replace->tgtdev = NULL;  in btrfs_dev_replace_finishing()
948 	dev_replace->srcdev = NULL;  in btrfs_dev_replace_finishing()
949 	dev_replace->time_stopped = ktime_get_real_seconds();  in btrfs_dev_replace_finishing()
950 	dev_replace->item_needs_writeback = 1;  in btrfs_dev_replace_finishing()
971 		up_write(&dev_replace->rwsem);  in btrfs_dev_replace_finishing()
979 		mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);  in btrfs_dev_replace_finishing()
1006 	up_write(&dev_replace->rwsem);  in btrfs_dev_replace_finishing()
1041 	mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);  in btrfs_dev_replace_finishing()
1055 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_progress()  local
1058 	switch (dev_replace->replace_state) {  in btrfs_dev_replace_progress()
1068 		ret = div64_u64(dev_replace->cursor_left,  in btrfs_dev_replace_progress()
1070 						dev_replace->srcdev), 1000));  in btrfs_dev_replace_progress()
1080 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_status()  local
1082 	down_read(&dev_replace->rwsem);  in btrfs_dev_replace_status()
1086 	args->status.replace_state = dev_replace->replace_state;  in btrfs_dev_replace_status()
1087 	args->status.time_started = dev_replace->time_started;  in btrfs_dev_replace_status()
1088 	args->status.time_stopped = dev_replace->time_stopped;  in btrfs_dev_replace_status()
1090 		atomic64_read(&dev_replace->num_write_errors);  in btrfs_dev_replace_status()
1092 		atomic64_read(&dev_replace->num_uncorrectable_read_errors);  in btrfs_dev_replace_status()
1094 	up_read(&dev_replace->rwsem);  in btrfs_dev_replace_status()
1099 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_cancel()  local
1110 	mutex_lock(&dev_replace->lock_finishing_cancel_unmount);  in btrfs_dev_replace_cancel()
1111 	down_write(&dev_replace->rwsem);  in btrfs_dev_replace_cancel()
1112 	switch (dev_replace->replace_state) {  in btrfs_dev_replace_cancel()
1117 		up_write(&dev_replace->rwsem);  in btrfs_dev_replace_cancel()
1120 		tgt_device = dev_replace->tgtdev;  in btrfs_dev_replace_cancel()
1121 		src_device = dev_replace->srcdev;  in btrfs_dev_replace_cancel()
1122 		up_write(&dev_replace->rwsem);  in btrfs_dev_replace_cancel()
1144 		tgt_device = dev_replace->tgtdev;  in btrfs_dev_replace_cancel()
1145 		src_device = dev_replace->srcdev;  in btrfs_dev_replace_cancel()
1146 		dev_replace->tgtdev = NULL;  in btrfs_dev_replace_cancel()
1147 		dev_replace->srcdev = NULL;  in btrfs_dev_replace_cancel()
1148 		dev_replace->replace_state =  in btrfs_dev_replace_cancel()
1150 		dev_replace->time_stopped = ktime_get_real_seconds();  in btrfs_dev_replace_cancel()
1151 		dev_replace->item_needs_writeback = 1;  in btrfs_dev_replace_cancel()
1153 		up_write(&dev_replace->rwsem);  in btrfs_dev_replace_cancel()
1161 			mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);  in btrfs_dev_replace_cancel()
1176 		up_write(&dev_replace->rwsem);  in btrfs_dev_replace_cancel()
1180 	mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);  in btrfs_dev_replace_cancel()
1186 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_suspend_for_unmount()  local
1188 	mutex_lock(&dev_replace->lock_finishing_cancel_unmount);  in btrfs_dev_replace_suspend_for_unmount()
1189 	down_write(&dev_replace->rwsem);  in btrfs_dev_replace_suspend_for_unmount()
1191 	switch (dev_replace->replace_state) {  in btrfs_dev_replace_suspend_for_unmount()
1198 		dev_replace->replace_state =  in btrfs_dev_replace_suspend_for_unmount()
1200 		dev_replace->time_stopped = ktime_get_real_seconds();  in btrfs_dev_replace_suspend_for_unmount()
1201 		dev_replace->item_needs_writeback = 1;  in btrfs_dev_replace_suspend_for_unmount()
1206 	up_write(&dev_replace->rwsem);  in btrfs_dev_replace_suspend_for_unmount()
1207 	mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);  in btrfs_dev_replace_suspend_for_unmount()
1214 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_resume_dev_replace_async()  local
1216 	down_write(&dev_replace->rwsem);  in btrfs_resume_dev_replace_async()
1218 	switch (dev_replace->replace_state) {  in btrfs_resume_dev_replace_async()
1222 		up_write(&dev_replace->rwsem);  in btrfs_resume_dev_replace_async()
1227 		dev_replace->replace_state =  in btrfs_resume_dev_replace_async()
1231 	if (!dev_replace->tgtdev || !dev_replace->tgtdev->bdev) {  in btrfs_resume_dev_replace_async()
1236 		dev_replace->replace_state =  in btrfs_resume_dev_replace_async()
1238 		up_write(&dev_replace->rwsem);  in btrfs_resume_dev_replace_async()
1241 	up_write(&dev_replace->rwsem);  in btrfs_resume_dev_replace_async()
1249 		down_write(&dev_replace->rwsem);  in btrfs_resume_dev_replace_async()
1250 		dev_replace->replace_state =  in btrfs_resume_dev_replace_async()
1252 		up_write(&dev_replace->rwsem);  in btrfs_resume_dev_replace_async()
1265 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_kthread()  local
1273 		btrfs_dev_name(dev_replace->srcdev),  in btrfs_dev_replace_kthread()
1274 		dev_replace->srcdev->devid,  in btrfs_dev_replace_kthread()
1275 		btrfs_dev_name(dev_replace->tgtdev),  in btrfs_dev_replace_kthread()
1278 	ret = btrfs_scrub_dev(fs_info, dev_replace->srcdev->devid,  in btrfs_dev_replace_kthread()
1279 			      dev_replace->committed_cursor_left,  in btrfs_dev_replace_kthread()
1280 			      btrfs_device_get_total_bytes(dev_replace->srcdev),  in btrfs_dev_replace_kthread()
1281 			      &dev_replace->scrub_progress, 0, 1);  in btrfs_dev_replace_kthread()
1289 int __pure btrfs_dev_replace_is_ongoing(struct btrfs_dev_replace *dev_replace)  in btrfs_dev_replace_is_ongoing()  argument
1291 	if (!dev_replace->is_valid)  in btrfs_dev_replace_is_ongoing()
1294 	switch (dev_replace->replace_state) {  in btrfs_dev_replace_is_ongoing()
1318 	percpu_counter_inc(&fs_info->dev_replace.bio_counter);  in btrfs_bio_counter_inc_noblocked()
1323 	percpu_counter_sub(&fs_info->dev_replace.bio_counter, amount);  in btrfs_bio_counter_sub()
1324 	cond_wake_up_nomb(&fs_info->dev_replace.replace_wait);  in btrfs_bio_counter_sub()
1330 		percpu_counter_inc(&fs_info->dev_replace.bio_counter);  in btrfs_bio_counter_inc_blocked()
1336 		wait_event(fs_info->dev_replace.replace_wait,  in btrfs_bio_counter_inc_blocked()