36 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_init_dev_replace()  local
58 		dev_replace->replace_state =  in btrfs_init_dev_replace()
60 		dev_replace->cont_reading_from_srcdev_mode =  in btrfs_init_dev_replace()
62 		dev_replace->replace_state = 0;  in btrfs_init_dev_replace()
63 		dev_replace->time_started = 0;  in btrfs_init_dev_replace()
64 		dev_replace->time_stopped = 0;  in btrfs_init_dev_replace()
65 		atomic64_set(&dev_replace->num_write_errors, 0);  in btrfs_init_dev_replace()
66 		atomic64_set(&dev_replace->num_uncorrectable_read_errors, 0);  in btrfs_init_dev_replace()
67 		dev_replace->cursor_left = 0;  in btrfs_init_dev_replace()
68 		dev_replace->committed_cursor_left = 0;  in btrfs_init_dev_replace()
69 		dev_replace->cursor_left_last_write_of_item = 0;  in btrfs_init_dev_replace()
70 		dev_replace->cursor_right = 0;  in btrfs_init_dev_replace()
71 		dev_replace->srcdev = NULL;  in btrfs_init_dev_replace()
72 		dev_replace->tgtdev = NULL;  in btrfs_init_dev_replace()
73 		dev_replace->is_valid = 0;  in btrfs_init_dev_replace()
74 		dev_replace->item_needs_writeback = 0;  in btrfs_init_dev_replace()
89 	dev_replace->cont_reading_from_srcdev_mode =  in btrfs_init_dev_replace()
91 	dev_replace->replace_state = btrfs_dev_replace_replace_state(eb, ptr);  in btrfs_init_dev_replace()
92 	dev_replace->time_started = btrfs_dev_replace_time_started(eb, ptr);  in btrfs_init_dev_replace()
93 	dev_replace->time_stopped =  in btrfs_init_dev_replace()
95 	atomic64_set(&dev_replace->num_write_errors,  in btrfs_init_dev_replace()
97 	atomic64_set(&dev_replace->num_uncorrectable_read_errors,  in btrfs_init_dev_replace()
99 	dev_replace->cursor_left = btrfs_dev_replace_cursor_left(eb, ptr);  in btrfs_init_dev_replace()
100 	dev_replace->committed_cursor_left = dev_replace->cursor_left;  in btrfs_init_dev_replace()
101 	dev_replace->cursor_left_last_write_of_item = dev_replace->cursor_left;  in btrfs_init_dev_replace()
102 	dev_replace->cursor_right = btrfs_dev_replace_cursor_right(eb, ptr);  in btrfs_init_dev_replace()
103 	dev_replace->is_valid = 1;  in btrfs_init_dev_replace()
105 	dev_replace->item_needs_writeback = 0;  in btrfs_init_dev_replace()
106 	switch (dev_replace->replace_state) {  in btrfs_init_dev_replace()
110 		dev_replace->srcdev = NULL;  in btrfs_init_dev_replace()
111 		dev_replace->tgtdev = NULL;  in btrfs_init_dev_replace()
115 		dev_replace->srcdev = btrfs_find_device(fs_info, src_devid,  in btrfs_init_dev_replace()
117 		dev_replace->tgtdev = btrfs_find_device(fs_info,  in btrfs_init_dev_replace()
124 		if (!dev_replace->srcdev &&  in btrfs_init_dev_replace()
133 		if (!dev_replace->tgtdev &&  in btrfs_init_dev_replace()
142 		if (dev_replace->tgtdev) {  in btrfs_init_dev_replace()
143 			if (dev_replace->srcdev) {  in btrfs_init_dev_replace()
144 				dev_replace->tgtdev->total_bytes =  in btrfs_init_dev_replace()
145 					dev_replace->srcdev->total_bytes;  in btrfs_init_dev_replace()
146 				dev_replace->tgtdev->disk_total_bytes =  in btrfs_init_dev_replace()
147 					dev_replace->srcdev->disk_total_bytes;  in btrfs_init_dev_replace()
148 				dev_replace->tgtdev->commit_total_bytes =  in btrfs_init_dev_replace()
149 					dev_replace->srcdev->commit_total_bytes;  in btrfs_init_dev_replace()
150 				dev_replace->tgtdev->bytes_used =  in btrfs_init_dev_replace()
151 					dev_replace->srcdev->bytes_used;  in btrfs_init_dev_replace()
152 				dev_replace->tgtdev->commit_bytes_used =  in btrfs_init_dev_replace()
153 					dev_replace->srcdev->commit_bytes_used;  in btrfs_init_dev_replace()
156 				&dev_replace->tgtdev->dev_state);  in btrfs_init_dev_replace()
159 			dev_replace->tgtdev->io_width = fs_info->sectorsize;  in btrfs_init_dev_replace()
160 			dev_replace->tgtdev->io_align = fs_info->sectorsize;  in btrfs_init_dev_replace()
161 			dev_replace->tgtdev->sector_size = fs_info->sectorsize;  in btrfs_init_dev_replace()
162 			dev_replace->tgtdev->fs_info = fs_info;  in btrfs_init_dev_replace()
164 				&dev_replace->tgtdev->dev_state);  in btrfs_init_dev_replace()
286 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_run_dev_replace()  local
288 	btrfs_dev_replace_read_lock(dev_replace);  in btrfs_run_dev_replace()
289 	if (!dev_replace->is_valid ||  in btrfs_run_dev_replace()
290 	    !dev_replace->item_needs_writeback) {  in btrfs_run_dev_replace()
291 		btrfs_dev_replace_read_unlock(dev_replace);  in btrfs_run_dev_replace()
294 	btrfs_dev_replace_read_unlock(dev_replace);  in btrfs_run_dev_replace()
352 	btrfs_dev_replace_write_lock(dev_replace);  in btrfs_run_dev_replace()
353 	if (dev_replace->srcdev)  in btrfs_run_dev_replace()
355 			dev_replace->srcdev->devid);  in btrfs_run_dev_replace()
359 		dev_replace->cont_reading_from_srcdev_mode);  in btrfs_run_dev_replace()
361 		dev_replace->replace_state);  in btrfs_run_dev_replace()
362 	btrfs_set_dev_replace_time_started(eb, ptr, dev_replace->time_started);  in btrfs_run_dev_replace()
363 	btrfs_set_dev_replace_time_stopped(eb, ptr, dev_replace->time_stopped);  in btrfs_run_dev_replace()
365 		atomic64_read(&dev_replace->num_write_errors));  in btrfs_run_dev_replace()
367 		atomic64_read(&dev_replace->num_uncorrectable_read_errors));  in btrfs_run_dev_replace()
368 	dev_replace->cursor_left_last_write_of_item =  in btrfs_run_dev_replace()
369 		dev_replace->cursor_left;  in btrfs_run_dev_replace()
371 		dev_replace->cursor_left_last_write_of_item);  in btrfs_run_dev_replace()
373 		dev_replace->cursor_right);  in btrfs_run_dev_replace()
374 	dev_replace->item_needs_writeback = 0;  in btrfs_run_dev_replace()
375 	btrfs_dev_replace_write_unlock(dev_replace);  in btrfs_run_dev_replace()
387 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_after_dev_replace_commit()  local
389 	dev_replace->committed_cursor_left =  in btrfs_after_dev_replace_commit()
390 		dev_replace->cursor_left_last_write_of_item;  in btrfs_after_dev_replace_commit()
407 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_start()  local
435 	btrfs_dev_replace_write_lock(dev_replace);  in btrfs_dev_replace_start()
436 	switch (dev_replace->replace_state) {  in btrfs_dev_replace_start()
447 	dev_replace->cont_reading_from_srcdev_mode = read_src;  in btrfs_dev_replace_start()
449 	dev_replace->srcdev = src_device;  in btrfs_dev_replace_start()
450 	dev_replace->tgtdev = tgt_device;  in btrfs_dev_replace_start()
462 	dev_replace->replace_state = BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED;  in btrfs_dev_replace_start()
463 	dev_replace->time_started = ktime_get_real_seconds();  in btrfs_dev_replace_start()
464 	dev_replace->cursor_left = 0;  in btrfs_dev_replace_start()
465 	dev_replace->committed_cursor_left = 0;  in btrfs_dev_replace_start()
466 	dev_replace->cursor_left_last_write_of_item = 0;  in btrfs_dev_replace_start()
467 	dev_replace->cursor_right = 0;  in btrfs_dev_replace_start()
468 	dev_replace->is_valid = 1;  in btrfs_dev_replace_start()
469 	dev_replace->item_needs_writeback = 1;  in btrfs_dev_replace_start()
470 	atomic64_set(&dev_replace->num_write_errors, 0);  in btrfs_dev_replace_start()
471 	atomic64_set(&dev_replace->num_uncorrectable_read_errors, 0);  in btrfs_dev_replace_start()
472 	btrfs_dev_replace_write_unlock(dev_replace);  in btrfs_dev_replace_start()
484 		btrfs_dev_replace_write_lock(dev_replace);  in btrfs_dev_replace_start()
494 			      &dev_replace->scrub_progress, 0, 1);  in btrfs_dev_replace_start()
506 	dev_replace->srcdev = NULL;  in btrfs_dev_replace_start()
507 	dev_replace->tgtdev = NULL;  in btrfs_dev_replace_start()
508 	btrfs_dev_replace_write_unlock(dev_replace);  in btrfs_dev_replace_start()
564 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_finishing()  local
573 	mutex_lock(&dev_replace->lock_finishing_cancel_unmount);  in btrfs_dev_replace_finishing()
575 	btrfs_dev_replace_read_lock(dev_replace);  in btrfs_dev_replace_finishing()
577 	if (dev_replace->replace_state !=  in btrfs_dev_replace_finishing()
579 		btrfs_dev_replace_read_unlock(dev_replace);  in btrfs_dev_replace_finishing()
580 		mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);  in btrfs_dev_replace_finishing()
584 	tgt_device = dev_replace->tgtdev;  in btrfs_dev_replace_finishing()
585 	src_device = dev_replace->srcdev;  in btrfs_dev_replace_finishing()
586 	btrfs_dev_replace_read_unlock(dev_replace);  in btrfs_dev_replace_finishing()
594 		mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);  in btrfs_dev_replace_finishing()
601 		mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);  in btrfs_dev_replace_finishing()
610 	btrfs_dev_replace_write_lock(dev_replace);  in btrfs_dev_replace_finishing()
611 	dev_replace->replace_state =  in btrfs_dev_replace_finishing()
614 	dev_replace->tgtdev = NULL;  in btrfs_dev_replace_finishing()
615 	dev_replace->srcdev = NULL;  in btrfs_dev_replace_finishing()
616 	dev_replace->time_stopped = ktime_get_real_seconds();  in btrfs_dev_replace_finishing()
617 	dev_replace->item_needs_writeback = 1;  in btrfs_dev_replace_finishing()
630 		btrfs_dev_replace_write_unlock(dev_replace);  in btrfs_dev_replace_finishing()
637 		mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);  in btrfs_dev_replace_finishing()
666 	btrfs_dev_replace_write_unlock(dev_replace);  in btrfs_dev_replace_finishing()
699 	mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);  in btrfs_dev_replace_finishing()
737 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_progress()  local
740 	switch (dev_replace->replace_state) {  in btrfs_dev_replace_progress()
750 		ret = div64_u64(dev_replace->cursor_left,  in btrfs_dev_replace_progress()
752 						dev_replace->srcdev), 1000));  in btrfs_dev_replace_progress()
762 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_status()  local
764 	btrfs_dev_replace_read_lock(dev_replace);  in btrfs_dev_replace_status()
768 	args->status.replace_state = dev_replace->replace_state;  in btrfs_dev_replace_status()
769 	args->status.time_started = dev_replace->time_started;  in btrfs_dev_replace_status()
770 	args->status.time_stopped = dev_replace->time_stopped;  in btrfs_dev_replace_status()
772 		atomic64_read(&dev_replace->num_write_errors);  in btrfs_dev_replace_status()
774 		atomic64_read(&dev_replace->num_uncorrectable_read_errors);  in btrfs_dev_replace_status()
776 	btrfs_dev_replace_read_unlock(dev_replace);  in btrfs_dev_replace_status()
781 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_cancel()  local
792 	mutex_lock(&dev_replace->lock_finishing_cancel_unmount);  in btrfs_dev_replace_cancel()
793 	btrfs_dev_replace_write_lock(dev_replace);  in btrfs_dev_replace_cancel()
794 	switch (dev_replace->replace_state) {  in btrfs_dev_replace_cancel()
799 		btrfs_dev_replace_write_unlock(dev_replace);  in btrfs_dev_replace_cancel()
804 		tgt_device = dev_replace->tgtdev;  in btrfs_dev_replace_cancel()
805 		src_device = dev_replace->srcdev;  in btrfs_dev_replace_cancel()
806 		dev_replace->tgtdev = NULL;  in btrfs_dev_replace_cancel()
807 		dev_replace->srcdev = NULL;  in btrfs_dev_replace_cancel()
810 	dev_replace->replace_state = BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED;  in btrfs_dev_replace_cancel()
811 	dev_replace->time_stopped = ktime_get_real_seconds();  in btrfs_dev_replace_cancel()
812 	dev_replace->item_needs_writeback = 1;  in btrfs_dev_replace_cancel()
813 	btrfs_dev_replace_write_unlock(dev_replace);  in btrfs_dev_replace_cancel()
818 		mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);  in btrfs_dev_replace_cancel()
833 	mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);  in btrfs_dev_replace_cancel()
839 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_suspend_for_unmount()  local
841 	mutex_lock(&dev_replace->lock_finishing_cancel_unmount);  in btrfs_dev_replace_suspend_for_unmount()
842 	btrfs_dev_replace_write_lock(dev_replace);  in btrfs_dev_replace_suspend_for_unmount()
843 	switch (dev_replace->replace_state) {  in btrfs_dev_replace_suspend_for_unmount()
850 		dev_replace->replace_state =  in btrfs_dev_replace_suspend_for_unmount()
852 		dev_replace->time_stopped = ktime_get_real_seconds();  in btrfs_dev_replace_suspend_for_unmount()
853 		dev_replace->item_needs_writeback = 1;  in btrfs_dev_replace_suspend_for_unmount()
858 	btrfs_dev_replace_write_unlock(dev_replace);  in btrfs_dev_replace_suspend_for_unmount()
859 	mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);  in btrfs_dev_replace_suspend_for_unmount()
866 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_resume_dev_replace_async()  local
868 	btrfs_dev_replace_write_lock(dev_replace);  in btrfs_resume_dev_replace_async()
869 	switch (dev_replace->replace_state) {  in btrfs_resume_dev_replace_async()
873 		btrfs_dev_replace_write_unlock(dev_replace);  in btrfs_resume_dev_replace_async()
878 		dev_replace->replace_state =  in btrfs_resume_dev_replace_async()
882 	if (!dev_replace->tgtdev || !dev_replace->tgtdev->bdev) {  in btrfs_resume_dev_replace_async()
887 		btrfs_dev_replace_write_unlock(dev_replace);  in btrfs_resume_dev_replace_async()
890 	btrfs_dev_replace_write_unlock(dev_replace);  in btrfs_resume_dev_replace_async()
910 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;  in btrfs_dev_replace_kthread()  local
918 		btrfs_dev_name(dev_replace->srcdev),  in btrfs_dev_replace_kthread()
919 		dev_replace->srcdev->devid,  in btrfs_dev_replace_kthread()
920 		btrfs_dev_name(dev_replace->tgtdev),  in btrfs_dev_replace_kthread()
923 	ret = btrfs_scrub_dev(fs_info, dev_replace->srcdev->devid,  in btrfs_dev_replace_kthread()
924 			      dev_replace->committed_cursor_left,  in btrfs_dev_replace_kthread()
925 			      btrfs_device_get_total_bytes(dev_replace->srcdev),  in btrfs_dev_replace_kthread()
926 			      &dev_replace->scrub_progress, 0, 1);  in btrfs_dev_replace_kthread()
934 int btrfs_dev_replace_is_ongoing(struct btrfs_dev_replace *dev_replace)  in btrfs_dev_replace_is_ongoing()  argument
936 	if (!dev_replace->is_valid)  in btrfs_dev_replace_is_ongoing()
939 	switch (dev_replace->replace_state) {  in btrfs_dev_replace_is_ongoing()
961 void btrfs_dev_replace_read_lock(struct btrfs_dev_replace *dev_replace)  in btrfs_dev_replace_read_lock()  argument
963 	read_lock(&dev_replace->lock);  in btrfs_dev_replace_read_lock()
964 	atomic_inc(&dev_replace->read_locks);  in btrfs_dev_replace_read_lock()
967 void btrfs_dev_replace_read_unlock(struct btrfs_dev_replace *dev_replace)  in btrfs_dev_replace_read_unlock()  argument
969 	ASSERT(atomic_read(&dev_replace->read_locks) > 0);  in btrfs_dev_replace_read_unlock()
970 	atomic_dec(&dev_replace->read_locks);  in btrfs_dev_replace_read_unlock()
971 	read_unlock(&dev_replace->lock);  in btrfs_dev_replace_read_unlock()
974 void btrfs_dev_replace_write_lock(struct btrfs_dev_replace *dev_replace)  in btrfs_dev_replace_write_lock()  argument
977 	wait_event(dev_replace->read_lock_wq,  in btrfs_dev_replace_write_lock()
978 		   atomic_read(&dev_replace->blocking_readers) == 0);  in btrfs_dev_replace_write_lock()
979 	write_lock(&dev_replace->lock);  in btrfs_dev_replace_write_lock()
980 	if (atomic_read(&dev_replace->blocking_readers)) {  in btrfs_dev_replace_write_lock()
981 		write_unlock(&dev_replace->lock);  in btrfs_dev_replace_write_lock()
986 void btrfs_dev_replace_write_unlock(struct btrfs_dev_replace *dev_replace)  in btrfs_dev_replace_write_unlock()  argument
988 	ASSERT(atomic_read(&dev_replace->blocking_readers) == 0);  in btrfs_dev_replace_write_unlock()
989 	write_unlock(&dev_replace->lock);  in btrfs_dev_replace_write_unlock()
994 					struct btrfs_dev_replace *dev_replace)  in btrfs_dev_replace_set_lock_blocking()  argument
997 	ASSERT(atomic_read(&dev_replace->read_locks) > 0);  in btrfs_dev_replace_set_lock_blocking()
998 	atomic_inc(&dev_replace->blocking_readers);  in btrfs_dev_replace_set_lock_blocking()
999 	read_unlock(&dev_replace->lock);  in btrfs_dev_replace_set_lock_blocking()
1004 					struct btrfs_dev_replace *dev_replace)  in btrfs_dev_replace_clear_lock_blocking()  argument
1007 	ASSERT(atomic_read(&dev_replace->read_locks) > 0);  in btrfs_dev_replace_clear_lock_blocking()
1008 	ASSERT(atomic_read(&dev_replace->blocking_readers) > 0);  in btrfs_dev_replace_clear_lock_blocking()
1009 	read_lock(&dev_replace->lock);  in btrfs_dev_replace_clear_lock_blocking()
1011 	if (atomic_dec_and_test(&dev_replace->blocking_readers))  in btrfs_dev_replace_clear_lock_blocking()
1012 		cond_wake_up_nomb(&dev_replace->read_lock_wq);  in btrfs_dev_replace_clear_lock_blocking()