Lines Matching refs:rbio
71 static void index_rbio_pages(struct btrfs_raid_bio *rbio);
72 static int alloc_rbio_pages(struct btrfs_raid_bio *rbio);
74 static int finish_parity_scrub(struct btrfs_raid_bio *rbio);
77 static void free_raid_bio_pointers(struct btrfs_raid_bio *rbio) in free_raid_bio_pointers() argument
79 bitmap_free(rbio->error_bitmap); in free_raid_bio_pointers()
80 kfree(rbio->stripe_pages); in free_raid_bio_pointers()
81 kfree(rbio->bio_sectors); in free_raid_bio_pointers()
82 kfree(rbio->stripe_sectors); in free_raid_bio_pointers()
83 kfree(rbio->finish_pointers); in free_raid_bio_pointers()
86 static void free_raid_bio(struct btrfs_raid_bio *rbio) in free_raid_bio() argument
90 if (!refcount_dec_and_test(&rbio->refs)) in free_raid_bio()
93 WARN_ON(!list_empty(&rbio->stripe_cache)); in free_raid_bio()
94 WARN_ON(!list_empty(&rbio->hash_list)); in free_raid_bio()
95 WARN_ON(!bio_list_empty(&rbio->bio_list)); in free_raid_bio()
97 for (i = 0; i < rbio->nr_pages; i++) { in free_raid_bio()
98 if (rbio->stripe_pages[i]) { in free_raid_bio()
99 __free_page(rbio->stripe_pages[i]); in free_raid_bio()
100 rbio->stripe_pages[i] = NULL; in free_raid_bio()
104 btrfs_put_bioc(rbio->bioc); in free_raid_bio()
105 free_raid_bio_pointers(rbio); in free_raid_bio()
106 kfree(rbio); in free_raid_bio()
109 static void start_async_work(struct btrfs_raid_bio *rbio, work_func_t work_func) in start_async_work() argument
111 INIT_WORK(&rbio->work, work_func); in start_async_work()
112 queue_work(rbio->bioc->fs_info->rmw_workers, &rbio->work); in start_async_work()
167 static void cache_rbio_pages(struct btrfs_raid_bio *rbio) in cache_rbio_pages() argument
172 ret = alloc_rbio_pages(rbio); in cache_rbio_pages()
176 for (i = 0; i < rbio->nr_sectors; i++) { in cache_rbio_pages()
178 if (!rbio->bio_sectors[i].page) { in cache_rbio_pages()
184 if (i < rbio->nr_data * rbio->stripe_nsectors) in cache_rbio_pages()
185 ASSERT(rbio->stripe_sectors[i].uptodate); in cache_rbio_pages()
189 ASSERT(rbio->stripe_sectors[i].page); in cache_rbio_pages()
190 memcpy_page(rbio->stripe_sectors[i].page, in cache_rbio_pages()
191 rbio->stripe_sectors[i].pgoff, in cache_rbio_pages()
192 rbio->bio_sectors[i].page, in cache_rbio_pages()
193 rbio->bio_sectors[i].pgoff, in cache_rbio_pages()
194 rbio->bioc->fs_info->sectorsize); in cache_rbio_pages()
195 rbio->stripe_sectors[i].uptodate = 1; in cache_rbio_pages()
197 set_bit(RBIO_CACHE_READY_BIT, &rbio->flags); in cache_rbio_pages()
203 static int rbio_bucket(struct btrfs_raid_bio *rbio) in rbio_bucket() argument
205 u64 num = rbio->bioc->full_stripe_logical; in rbio_bucket()
218 static bool full_page_sectors_uptodate(struct btrfs_raid_bio *rbio, in full_page_sectors_uptodate() argument
221 const u32 sectorsize = rbio->bioc->fs_info->sectorsize; in full_page_sectors_uptodate()
225 ASSERT(page_nr < rbio->nr_pages); in full_page_sectors_uptodate()
230 if (!rbio->stripe_sectors[i].uptodate) in full_page_sectors_uptodate()
241 static void index_stripe_sectors(struct btrfs_raid_bio *rbio) in index_stripe_sectors() argument
243 const u32 sectorsize = rbio->bioc->fs_info->sectorsize; in index_stripe_sectors()
247 for (i = 0, offset = 0; i < rbio->nr_sectors; i++, offset += sectorsize) { in index_stripe_sectors()
250 ASSERT(page_index < rbio->nr_pages); in index_stripe_sectors()
251 rbio->stripe_sectors[i].page = rbio->stripe_pages[page_index]; in index_stripe_sectors()
252 rbio->stripe_sectors[i].pgoff = offset_in_page(offset); in index_stripe_sectors()
274 static bool is_data_stripe_page(struct btrfs_raid_bio *rbio, int page_nr) in is_data_stripe_page() argument
277 rbio->bioc->fs_info->sectorsize_bits; in is_data_stripe_page()
286 return (sector_nr < rbio->nr_data * rbio->stripe_nsectors); in is_data_stripe_page()
347 static void __remove_rbio_from_cache(struct btrfs_raid_bio *rbio) in __remove_rbio_from_cache() argument
349 int bucket = rbio_bucket(rbio); in __remove_rbio_from_cache()
357 if (!test_bit(RBIO_CACHE_BIT, &rbio->flags)) in __remove_rbio_from_cache()
360 table = rbio->bioc->fs_info->stripe_hash_table; in __remove_rbio_from_cache()
372 spin_lock(&rbio->bio_list_lock); in __remove_rbio_from_cache()
374 if (test_and_clear_bit(RBIO_CACHE_BIT, &rbio->flags)) { in __remove_rbio_from_cache()
375 list_del_init(&rbio->stripe_cache); in __remove_rbio_from_cache()
388 if (bio_list_empty(&rbio->bio_list)) { in __remove_rbio_from_cache()
389 if (!list_empty(&rbio->hash_list)) { in __remove_rbio_from_cache()
390 list_del_init(&rbio->hash_list); in __remove_rbio_from_cache()
391 refcount_dec(&rbio->refs); in __remove_rbio_from_cache()
392 BUG_ON(!list_empty(&rbio->plug_list)); in __remove_rbio_from_cache()
397 spin_unlock(&rbio->bio_list_lock); in __remove_rbio_from_cache()
401 free_raid_bio(rbio); in __remove_rbio_from_cache()
407 static void remove_rbio_from_cache(struct btrfs_raid_bio *rbio) in remove_rbio_from_cache() argument
411 if (!test_bit(RBIO_CACHE_BIT, &rbio->flags)) in remove_rbio_from_cache()
414 table = rbio->bioc->fs_info->stripe_hash_table; in remove_rbio_from_cache()
417 __remove_rbio_from_cache(rbio); in remove_rbio_from_cache()
427 struct btrfs_raid_bio *rbio; in btrfs_clear_rbio_cache() local
433 rbio = list_entry(table->stripe_cache.next, in btrfs_clear_rbio_cache()
436 __remove_rbio_from_cache(rbio); in btrfs_clear_rbio_cache()
465 static void cache_rbio(struct btrfs_raid_bio *rbio) in cache_rbio() argument
469 if (!test_bit(RBIO_CACHE_READY_BIT, &rbio->flags)) in cache_rbio()
472 table = rbio->bioc->fs_info->stripe_hash_table; in cache_rbio()
475 spin_lock(&rbio->bio_list_lock); in cache_rbio()
478 if (!test_and_set_bit(RBIO_CACHE_BIT, &rbio->flags)) in cache_rbio()
479 refcount_inc(&rbio->refs); in cache_rbio()
481 if (!list_empty(&rbio->stripe_cache)){ in cache_rbio()
482 list_move(&rbio->stripe_cache, &table->stripe_cache); in cache_rbio()
484 list_add(&rbio->stripe_cache, &table->stripe_cache); in cache_rbio()
488 spin_unlock(&rbio->bio_list_lock); in cache_rbio()
497 if (found != rbio) in cache_rbio()
528 static int rbio_is_full(struct btrfs_raid_bio *rbio) in rbio_is_full() argument
530 unsigned long size = rbio->bio_list_bytes; in rbio_is_full()
533 spin_lock(&rbio->bio_list_lock); in rbio_is_full()
534 if (size != rbio->nr_data * BTRFS_STRIPE_LEN) in rbio_is_full()
536 BUG_ON(size > rbio->nr_data * BTRFS_STRIPE_LEN); in rbio_is_full()
537 spin_unlock(&rbio->bio_list_lock); in rbio_is_full()
593 static unsigned int rbio_stripe_sector_index(const struct btrfs_raid_bio *rbio, in rbio_stripe_sector_index() argument
597 ASSERT(stripe_nr < rbio->real_stripes); in rbio_stripe_sector_index()
598 ASSERT(sector_nr < rbio->stripe_nsectors); in rbio_stripe_sector_index()
600 return stripe_nr * rbio->stripe_nsectors + sector_nr; in rbio_stripe_sector_index()
604 static struct sector_ptr *rbio_stripe_sector(const struct btrfs_raid_bio *rbio, in rbio_stripe_sector() argument
608 return &rbio->stripe_sectors[rbio_stripe_sector_index(rbio, stripe_nr, in rbio_stripe_sector()
613 static struct sector_ptr *rbio_pstripe_sector(const struct btrfs_raid_bio *rbio, in rbio_pstripe_sector() argument
616 return rbio_stripe_sector(rbio, rbio->nr_data, sector_nr); in rbio_pstripe_sector()
620 static struct sector_ptr *rbio_qstripe_sector(const struct btrfs_raid_bio *rbio, in rbio_qstripe_sector() argument
623 if (rbio->nr_data + 1 == rbio->real_stripes) in rbio_qstripe_sector()
625 return rbio_stripe_sector(rbio, rbio->nr_data + 1, sector_nr); in rbio_qstripe_sector()
650 static noinline int lock_stripe_add(struct btrfs_raid_bio *rbio) in lock_stripe_add() argument
659 h = rbio->bioc->fs_info->stripe_hash_table->table + rbio_bucket(rbio); in lock_stripe_add()
663 if (cur->bioc->full_stripe_logical != rbio->bioc->full_stripe_logical) in lock_stripe_add()
676 steal_rbio(cur, rbio); in lock_stripe_add()
684 if (rbio_can_merge(cur, rbio)) { in lock_stripe_add()
685 merge_rbio(cur, rbio); in lock_stripe_add()
687 freeit = rbio; in lock_stripe_add()
699 if (rbio_can_merge(pending, rbio)) { in lock_stripe_add()
700 merge_rbio(pending, rbio); in lock_stripe_add()
702 freeit = rbio; in lock_stripe_add()
712 list_add_tail(&rbio->plug_list, &cur->plug_list); in lock_stripe_add()
718 refcount_inc(&rbio->refs); in lock_stripe_add()
719 list_add(&rbio->hash_list, &h->hash_list); in lock_stripe_add()
735 static noinline void unlock_stripe(struct btrfs_raid_bio *rbio) in unlock_stripe() argument
741 bucket = rbio_bucket(rbio); in unlock_stripe()
742 h = rbio->bioc->fs_info->stripe_hash_table->table + bucket; in unlock_stripe()
744 if (list_empty(&rbio->plug_list)) in unlock_stripe()
745 cache_rbio(rbio); in unlock_stripe()
748 spin_lock(&rbio->bio_list_lock); in unlock_stripe()
750 if (!list_empty(&rbio->hash_list)) { in unlock_stripe()
756 if (list_empty(&rbio->plug_list) && in unlock_stripe()
757 test_bit(RBIO_CACHE_BIT, &rbio->flags)) { in unlock_stripe()
759 clear_bit(RBIO_RMW_LOCKED_BIT, &rbio->flags); in unlock_stripe()
760 BUG_ON(!bio_list_empty(&rbio->bio_list)); in unlock_stripe()
764 list_del_init(&rbio->hash_list); in unlock_stripe()
765 refcount_dec(&rbio->refs); in unlock_stripe()
772 if (!list_empty(&rbio->plug_list)) { in unlock_stripe()
774 struct list_head *head = rbio->plug_list.next; in unlock_stripe()
779 list_del_init(&rbio->plug_list); in unlock_stripe()
783 spin_unlock(&rbio->bio_list_lock); in unlock_stripe()
789 steal_rbio(rbio, next); in unlock_stripe()
792 steal_rbio(rbio, next); in unlock_stripe()
800 spin_unlock(&rbio->bio_list_lock); in unlock_stripe()
805 remove_rbio_from_cache(rbio); in unlock_stripe()
825 static void rbio_orig_end_io(struct btrfs_raid_bio *rbio, blk_status_t err) in rbio_orig_end_io() argument
827 struct bio *cur = bio_list_get(&rbio->bio_list); in rbio_orig_end_io()
830 kfree(rbio->csum_buf); in rbio_orig_end_io()
831 bitmap_free(rbio->csum_bitmap); in rbio_orig_end_io()
832 rbio->csum_buf = NULL; in rbio_orig_end_io()
833 rbio->csum_bitmap = NULL; in rbio_orig_end_io()
840 bitmap_clear(&rbio->dbitmap, 0, rbio->stripe_nsectors); in rbio_orig_end_io()
850 unlock_stripe(rbio); in rbio_orig_end_io()
851 extra = bio_list_get(&rbio->bio_list); in rbio_orig_end_io()
852 free_raid_bio(rbio); in rbio_orig_end_io()
871 static struct sector_ptr *sector_in_rbio(struct btrfs_raid_bio *rbio, in sector_in_rbio() argument
878 ASSERT(stripe_nr >= 0 && stripe_nr < rbio->real_stripes); in sector_in_rbio()
879 ASSERT(sector_nr >= 0 && sector_nr < rbio->stripe_nsectors); in sector_in_rbio()
881 index = stripe_nr * rbio->stripe_nsectors + sector_nr; in sector_in_rbio()
882 ASSERT(index >= 0 && index < rbio->nr_sectors); in sector_in_rbio()
884 spin_lock(&rbio->bio_list_lock); in sector_in_rbio()
885 sector = &rbio->bio_sectors[index]; in sector_in_rbio()
890 spin_unlock(&rbio->bio_list_lock); in sector_in_rbio()
893 spin_unlock(&rbio->bio_list_lock); in sector_in_rbio()
895 return &rbio->stripe_sectors[index]; in sector_in_rbio()
911 struct btrfs_raid_bio *rbio; in alloc_rbio() local
921 rbio = kzalloc(sizeof(*rbio), GFP_NOFS); in alloc_rbio()
922 if (!rbio) in alloc_rbio()
924 rbio->stripe_pages = kcalloc(num_pages, sizeof(struct page *), in alloc_rbio()
926 rbio->bio_sectors = kcalloc(num_sectors, sizeof(struct sector_ptr), in alloc_rbio()
928 rbio->stripe_sectors = kcalloc(num_sectors, sizeof(struct sector_ptr), in alloc_rbio()
930 rbio->finish_pointers = kcalloc(real_stripes, sizeof(void *), GFP_NOFS); in alloc_rbio()
931 rbio->error_bitmap = bitmap_zalloc(num_sectors, GFP_NOFS); in alloc_rbio()
933 if (!rbio->stripe_pages || !rbio->bio_sectors || !rbio->stripe_sectors || in alloc_rbio()
934 !rbio->finish_pointers || !rbio->error_bitmap) { in alloc_rbio()
935 free_raid_bio_pointers(rbio); in alloc_rbio()
936 kfree(rbio); in alloc_rbio()
940 bio_list_init(&rbio->bio_list); in alloc_rbio()
941 init_waitqueue_head(&rbio->io_wait); in alloc_rbio()
942 INIT_LIST_HEAD(&rbio->plug_list); in alloc_rbio()
943 spin_lock_init(&rbio->bio_list_lock); in alloc_rbio()
944 INIT_LIST_HEAD(&rbio->stripe_cache); in alloc_rbio()
945 INIT_LIST_HEAD(&rbio->hash_list); in alloc_rbio()
947 rbio->bioc = bioc; in alloc_rbio()
948 rbio->nr_pages = num_pages; in alloc_rbio()
949 rbio->nr_sectors = num_sectors; in alloc_rbio()
950 rbio->real_stripes = real_stripes; in alloc_rbio()
951 rbio->stripe_npages = stripe_npages; in alloc_rbio()
952 rbio->stripe_nsectors = stripe_nsectors; in alloc_rbio()
953 refcount_set(&rbio->refs, 1); in alloc_rbio()
954 atomic_set(&rbio->stripes_pending, 0); in alloc_rbio()
957 rbio->nr_data = real_stripes - btrfs_nr_parity_stripes(bioc->map_type); in alloc_rbio()
959 return rbio; in alloc_rbio()
963 static int alloc_rbio_pages(struct btrfs_raid_bio *rbio) in alloc_rbio_pages() argument
967 ret = btrfs_alloc_page_array(rbio->nr_pages, rbio->stripe_pages); in alloc_rbio_pages()
971 index_stripe_sectors(rbio); in alloc_rbio_pages()
976 static int alloc_rbio_parity_pages(struct btrfs_raid_bio *rbio) in alloc_rbio_parity_pages() argument
978 const int data_pages = rbio->nr_data * rbio->stripe_npages; in alloc_rbio_parity_pages()
981 ret = btrfs_alloc_page_array(rbio->nr_pages - data_pages, in alloc_rbio_parity_pages()
982 rbio->stripe_pages + data_pages); in alloc_rbio_parity_pages()
986 index_stripe_sectors(rbio); in alloc_rbio_parity_pages()
996 static int get_rbio_veritical_errors(struct btrfs_raid_bio *rbio, int sector_nr, in get_rbio_veritical_errors() argument
1012 for (stripe_nr = 0; stripe_nr < rbio->real_stripes; stripe_nr++) { in get_rbio_veritical_errors()
1013 int total_sector_nr = stripe_nr * rbio->stripe_nsectors + sector_nr; in get_rbio_veritical_errors()
1015 if (test_bit(total_sector_nr, rbio->error_bitmap)) { in get_rbio_veritical_errors()
1035 static int rbio_add_io_sector(struct btrfs_raid_bio *rbio, in rbio_add_io_sector() argument
1042 const u32 sectorsize = rbio->bioc->fs_info->sectorsize; in rbio_add_io_sector()
1054 ASSERT(stripe_nr >= 0 && stripe_nr < rbio->bioc->num_stripes); in rbio_add_io_sector()
1055 ASSERT(sector_nr >= 0 && sector_nr < rbio->stripe_nsectors); in rbio_add_io_sector()
1058 stripe = &rbio->bioc->stripes[stripe_nr]; in rbio_add_io_sector()
1065 set_bit(stripe_nr * rbio->stripe_nsectors + sector_nr, in rbio_add_io_sector()
1066 rbio->error_bitmap); in rbio_add_io_sector()
1069 found_errors = get_rbio_veritical_errors(rbio, sector_nr, in rbio_add_io_sector()
1071 if (found_errors > rbio->bioc->max_errors) in rbio_add_io_sector()
1099 bio->bi_private = rbio; in rbio_add_io_sector()
1106 static void index_one_bio(struct btrfs_raid_bio *rbio, struct bio *bio) in index_one_bio() argument
1108 const u32 sectorsize = rbio->bioc->fs_info->sectorsize; in index_one_bio()
1112 rbio->bioc->full_stripe_logical; in index_one_bio()
1120 struct sector_ptr *sector = &rbio->bio_sectors[index]; in index_one_bio()
1137 static void index_rbio_pages(struct btrfs_raid_bio *rbio) in index_rbio_pages() argument
1141 spin_lock(&rbio->bio_list_lock); in index_rbio_pages()
1142 bio_list_for_each(bio, &rbio->bio_list) in index_rbio_pages()
1143 index_one_bio(rbio, bio); in index_rbio_pages()
1145 spin_unlock(&rbio->bio_list_lock); in index_rbio_pages()
1148 static void bio_get_trace_info(struct btrfs_raid_bio *rbio, struct bio *bio, in bio_get_trace_info() argument
1151 const struct btrfs_io_context *bioc = rbio->bioc; in bio_get_trace_info()
1185 static void generate_pq_vertical(struct btrfs_raid_bio *rbio, int sectornr) in generate_pq_vertical() argument
1187 void **pointers = rbio->finish_pointers; in generate_pq_vertical()
1188 const u32 sectorsize = rbio->bioc->fs_info->sectorsize; in generate_pq_vertical()
1191 const bool has_qstripe = rbio->bioc->map_type & BTRFS_BLOCK_GROUP_RAID6; in generate_pq_vertical()
1194 for (stripe = 0; stripe < rbio->nr_data; stripe++) { in generate_pq_vertical()
1195 sector = sector_in_rbio(rbio, stripe, sectornr, 0); in generate_pq_vertical()
1201 sector = rbio_pstripe_sector(rbio, sectornr); in generate_pq_vertical()
1210 sector = rbio_qstripe_sector(rbio, sectornr); in generate_pq_vertical()
1215 raid6_call.gen_syndrome(rbio->real_stripes, sectorsize, in generate_pq_vertical()
1219 memcpy(pointers[rbio->nr_data], pointers[0], sectorsize); in generate_pq_vertical()
1220 run_xor(pointers + 1, rbio->nr_data - 1, sectorsize); in generate_pq_vertical()
1226 static int rmw_assemble_write_bios(struct btrfs_raid_bio *rbio, in rmw_assemble_write_bios() argument
1238 ASSERT(bitmap_weight(&rbio->dbitmap, rbio->stripe_nsectors)); in rmw_assemble_write_bios()
1244 bitmap_clear(rbio->error_bitmap, 0, rbio->nr_sectors); in rmw_assemble_write_bios()
1250 for (total_sector_nr = 0; total_sector_nr < rbio->nr_sectors; in rmw_assemble_write_bios()
1254 stripe = total_sector_nr / rbio->stripe_nsectors; in rmw_assemble_write_bios()
1255 sectornr = total_sector_nr % rbio->stripe_nsectors; in rmw_assemble_write_bios()
1258 if (!test_bit(sectornr, &rbio->dbitmap)) in rmw_assemble_write_bios()
1261 if (stripe < rbio->nr_data) { in rmw_assemble_write_bios()
1262 sector = sector_in_rbio(rbio, stripe, sectornr, 1); in rmw_assemble_write_bios()
1266 sector = rbio_stripe_sector(rbio, stripe, sectornr); in rmw_assemble_write_bios()
1269 ret = rbio_add_io_sector(rbio, bio_list, sector, stripe, in rmw_assemble_write_bios()
1275 if (likely(!rbio->bioc->replace_nr_stripes)) in rmw_assemble_write_bios()
1283 ASSERT(rbio->bioc->replace_stripe_src >= 0); in rmw_assemble_write_bios()
1285 for (total_sector_nr = 0; total_sector_nr < rbio->nr_sectors; in rmw_assemble_write_bios()
1289 stripe = total_sector_nr / rbio->stripe_nsectors; in rmw_assemble_write_bios()
1290 sectornr = total_sector_nr % rbio->stripe_nsectors; in rmw_assemble_write_bios()
1297 if (stripe != rbio->bioc->replace_stripe_src) { in rmw_assemble_write_bios()
1303 total_sector_nr += rbio->stripe_nsectors - 1; in rmw_assemble_write_bios()
1308 if (!test_bit(sectornr, &rbio->dbitmap)) in rmw_assemble_write_bios()
1311 if (stripe < rbio->nr_data) { in rmw_assemble_write_bios()
1312 sector = sector_in_rbio(rbio, stripe, sectornr, 1); in rmw_assemble_write_bios()
1316 sector = rbio_stripe_sector(rbio, stripe, sectornr); in rmw_assemble_write_bios()
1319 ret = rbio_add_io_sector(rbio, bio_list, sector, in rmw_assemble_write_bios()
1320 rbio->real_stripes, in rmw_assemble_write_bios()
1332 static void set_rbio_range_error(struct btrfs_raid_bio *rbio, struct bio *bio) in set_rbio_range_error() argument
1334 struct btrfs_fs_info *fs_info = rbio->bioc->fs_info; in set_rbio_range_error()
1336 rbio->bioc->full_stripe_logical; in set_rbio_range_error()
1339 ASSERT(total_nr_sector < rbio->nr_data * rbio->stripe_nsectors); in set_rbio_range_error()
1341 bitmap_set(rbio->error_bitmap, total_nr_sector, in set_rbio_range_error()
1354 for (stripe_nr = 0; stripe_nr < rbio->real_stripes; stripe_nr++) { in set_rbio_range_error()
1355 if (!rbio->bioc->stripes[stripe_nr].dev->bdev) { in set_rbio_range_error()
1357 bitmap_set(rbio->error_bitmap, in set_rbio_range_error()
1358 stripe_nr * rbio->stripe_nsectors, in set_rbio_range_error()
1359 rbio->stripe_nsectors); in set_rbio_range_error()
1370 static struct sector_ptr *find_stripe_sector(struct btrfs_raid_bio *rbio, in find_stripe_sector() argument
1376 for (i = 0; i < rbio->nr_sectors; i++) { in find_stripe_sector()
1377 struct sector_ptr *sector = &rbio->stripe_sectors[i]; in find_stripe_sector()
1389 static void set_bio_pages_uptodate(struct btrfs_raid_bio *rbio, struct bio *bio) in set_bio_pages_uptodate() argument
1391 const u32 sectorsize = rbio->bioc->fs_info->sectorsize; in set_bio_pages_uptodate()
1403 sector = find_stripe_sector(rbio, bvec->bv_page, pgoff); in set_bio_pages_uptodate()
1411 static int get_bio_sector_nr(struct btrfs_raid_bio *rbio, struct bio *bio) in get_bio_sector_nr() argument
1416 for (i = 0; i < rbio->nr_sectors; i++) { in get_bio_sector_nr()
1419 sector = &rbio->stripe_sectors[i]; in get_bio_sector_nr()
1422 sector = &rbio->bio_sectors[i]; in get_bio_sector_nr()
1426 ASSERT(i < rbio->nr_sectors); in get_bio_sector_nr()
1430 static void rbio_update_error_bitmap(struct btrfs_raid_bio *rbio, struct bio *bio) in rbio_update_error_bitmap() argument
1432 int total_sector_nr = get_bio_sector_nr(rbio, bio); in rbio_update_error_bitmap()
1447 (bio_size >> rbio->bioc->fs_info->sectorsize_bits); i++) in rbio_update_error_bitmap()
1448 set_bit(i, rbio->error_bitmap); in rbio_update_error_bitmap()
1452 static void verify_bio_data_sectors(struct btrfs_raid_bio *rbio, in verify_bio_data_sectors() argument
1455 struct btrfs_fs_info *fs_info = rbio->bioc->fs_info; in verify_bio_data_sectors()
1456 int total_sector_nr = get_bio_sector_nr(rbio, bio); in verify_bio_data_sectors()
1461 if (!rbio->csum_bitmap || !rbio->csum_buf) in verify_bio_data_sectors()
1465 if (total_sector_nr >= rbio->nr_data * rbio->stripe_nsectors) in verify_bio_data_sectors()
1475 u8 *expected_csum = rbio->csum_buf + in verify_bio_data_sectors()
1480 if (!test_bit(total_sector_nr, rbio->csum_bitmap)) in verify_bio_data_sectors()
1486 set_bit(total_sector_nr, rbio->error_bitmap); in verify_bio_data_sectors()
1493 struct btrfs_raid_bio *rbio = bio->bi_private; in raid_wait_read_end_io() local
1496 rbio_update_error_bitmap(rbio, bio); in raid_wait_read_end_io()
1498 set_bio_pages_uptodate(rbio, bio); in raid_wait_read_end_io()
1499 verify_bio_data_sectors(rbio, bio); in raid_wait_read_end_io()
1503 if (atomic_dec_and_test(&rbio->stripes_pending)) in raid_wait_read_end_io()
1504 wake_up(&rbio->io_wait); in raid_wait_read_end_io()
1507 static void submit_read_wait_bio_list(struct btrfs_raid_bio *rbio, in submit_read_wait_bio_list() argument
1512 atomic_set(&rbio->stripes_pending, bio_list_size(bio_list)); in submit_read_wait_bio_list()
1519 bio_get_trace_info(rbio, bio, &trace_info); in submit_read_wait_bio_list()
1520 trace_raid56_read(rbio, bio, &trace_info); in submit_read_wait_bio_list()
1525 wait_event(rbio->io_wait, atomic_read(&rbio->stripes_pending) == 0); in submit_read_wait_bio_list()
1528 static int alloc_rbio_data_pages(struct btrfs_raid_bio *rbio) in alloc_rbio_data_pages() argument
1530 const int data_pages = rbio->nr_data * rbio->stripe_npages; in alloc_rbio_data_pages()
1533 ret = btrfs_alloc_page_array(data_pages, rbio->stripe_pages); in alloc_rbio_data_pages()
1537 index_stripe_sectors(rbio); in alloc_rbio_data_pages()
1609 static void rbio_add_bio(struct btrfs_raid_bio *rbio, struct bio *orig_bio) in rbio_add_bio() argument
1611 const struct btrfs_fs_info *fs_info = rbio->bioc->fs_info; in rbio_add_bio()
1613 const u64 full_stripe_start = rbio->bioc->full_stripe_logical; in rbio_add_bio()
1620 rbio->nr_data * BTRFS_STRIPE_LEN); in rbio_add_bio()
1622 bio_list_add(&rbio->bio_list, orig_bio); in rbio_add_bio()
1623 rbio->bio_list_bytes += orig_bio->bi_iter.bi_size; in rbio_add_bio()
1629 fs_info->sectorsize_bits) % rbio->stripe_nsectors; in rbio_add_bio()
1631 set_bit(bit, &rbio->dbitmap); in rbio_add_bio()
1641 struct btrfs_raid_bio *rbio; in raid56_parity_write() local
1645 rbio = alloc_rbio(fs_info, bioc); in raid56_parity_write()
1646 if (IS_ERR(rbio)) { in raid56_parity_write()
1647 bio->bi_status = errno_to_blk_status(PTR_ERR(rbio)); in raid56_parity_write()
1651 rbio->operation = BTRFS_RBIO_WRITE; in raid56_parity_write()
1652 rbio_add_bio(rbio, bio); in raid56_parity_write()
1658 if (!rbio_is_full(rbio)) { in raid56_parity_write()
1666 list_add_tail(&rbio->plug_list, &plug->rbio_list); in raid56_parity_write()
1675 start_async_work(rbio, rmw_rbio_work); in raid56_parity_write()
1678 static int verify_one_sector(struct btrfs_raid_bio *rbio, in verify_one_sector() argument
1681 struct btrfs_fs_info *fs_info = rbio->bioc->fs_info; in verify_one_sector()
1687 if (!rbio->csum_bitmap || !rbio->csum_buf) in verify_one_sector()
1691 if (stripe_nr >= rbio->nr_data) in verify_one_sector()
1697 if (rbio->operation == BTRFS_RBIO_READ_REBUILD) { in verify_one_sector()
1698 sector = sector_in_rbio(rbio, stripe_nr, sector_nr, 0); in verify_one_sector()
1700 sector = rbio_stripe_sector(rbio, stripe_nr, sector_nr); in verify_one_sector()
1705 csum_expected = rbio->csum_buf + in verify_one_sector()
1706 (stripe_nr * rbio->stripe_nsectors + sector_nr) * in verify_one_sector()
1718 static int recover_vertical(struct btrfs_raid_bio *rbio, int sector_nr, in recover_vertical() argument
1721 struct btrfs_fs_info *fs_info = rbio->bioc->fs_info; in recover_vertical()
1734 if (rbio->operation == BTRFS_RBIO_PARITY_SCRUB && in recover_vertical()
1735 !test_bit(sector_nr, &rbio->dbitmap)) in recover_vertical()
1738 found_errors = get_rbio_veritical_errors(rbio, sector_nr, &faila, in recover_vertical()
1747 if (found_errors > rbio->bioc->max_errors) in recover_vertical()
1756 for (stripe_nr = 0; stripe_nr < rbio->real_stripes; stripe_nr++) { in recover_vertical()
1761 if (rbio->operation == BTRFS_RBIO_READ_REBUILD) { in recover_vertical()
1762 sector = sector_in_rbio(rbio, stripe_nr, sector_nr, 0); in recover_vertical()
1764 sector = rbio_stripe_sector(rbio, stripe_nr, sector_nr); in recover_vertical()
1773 if (rbio->bioc->map_type & BTRFS_BLOCK_GROUP_RAID6) { in recover_vertical()
1776 if (faila == rbio->nr_data) in recover_vertical()
1798 if (failb == rbio->real_stripes - 1) { in recover_vertical()
1799 if (faila == rbio->real_stripes - 2) in recover_vertical()
1813 if (failb == rbio->real_stripes - 2) { in recover_vertical()
1814 raid6_datap_recov(rbio->real_stripes, sectorsize, in recover_vertical()
1817 raid6_2data_recov(rbio->real_stripes, sectorsize, in recover_vertical()
1827 memcpy(pointers[faila], pointers[rbio->nr_data], sectorsize); in recover_vertical()
1831 for (stripe_nr = faila; stripe_nr < rbio->nr_data - 1; in recover_vertical()
1834 pointers[rbio->nr_data - 1] = p; in recover_vertical()
1837 run_xor(pointers, rbio->nr_data - 1, sectorsize); in recover_vertical()
1852 ret = verify_one_sector(rbio, faila, sector_nr); in recover_vertical()
1856 sector = rbio_stripe_sector(rbio, faila, sector_nr); in recover_vertical()
1860 ret = verify_one_sector(rbio, failb, sector_nr); in recover_vertical()
1864 sector = rbio_stripe_sector(rbio, failb, sector_nr); in recover_vertical()
1869 for (stripe_nr = rbio->real_stripes - 1; stripe_nr >= 0; stripe_nr--) in recover_vertical()
1874 static int recover_sectors(struct btrfs_raid_bio *rbio) in recover_sectors() argument
1887 pointers = kcalloc(rbio->real_stripes, sizeof(void *), GFP_NOFS); in recover_sectors()
1888 unmap_array = kcalloc(rbio->real_stripes, sizeof(void *), GFP_NOFS); in recover_sectors()
1894 if (rbio->operation == BTRFS_RBIO_READ_REBUILD) { in recover_sectors()
1895 spin_lock(&rbio->bio_list_lock); in recover_sectors()
1896 set_bit(RBIO_RMW_LOCKED_BIT, &rbio->flags); in recover_sectors()
1897 spin_unlock(&rbio->bio_list_lock); in recover_sectors()
1900 index_rbio_pages(rbio); in recover_sectors()
1902 for (sectornr = 0; sectornr < rbio->stripe_nsectors; sectornr++) { in recover_sectors()
1903 ret = recover_vertical(rbio, sectornr, pointers, unmap_array); in recover_sectors()
1914 static void recover_rbio(struct btrfs_raid_bio *rbio) in recover_rbio() argument
1924 ASSERT(bitmap_weight(rbio->error_bitmap, rbio->nr_sectors)); in recover_rbio()
1927 ret = alloc_rbio_pages(rbio); in recover_rbio()
1931 index_rbio_pages(rbio); in recover_rbio()
1941 for (total_sector_nr = 0; total_sector_nr < rbio->nr_sectors; in recover_rbio()
1943 int stripe = total_sector_nr / rbio->stripe_nsectors; in recover_rbio()
1944 int sectornr = total_sector_nr % rbio->stripe_nsectors; in recover_rbio()
1952 if (!rbio->bioc->stripes[stripe].dev->bdev || in recover_rbio()
1953 test_bit(total_sector_nr, rbio->error_bitmap)) { in recover_rbio()
1958 set_bit(total_sector_nr, rbio->error_bitmap); in recover_rbio()
1962 sector = rbio_stripe_sector(rbio, stripe, sectornr); in recover_rbio()
1963 ret = rbio_add_io_sector(rbio, &bio_list, sector, stripe, in recover_rbio()
1971 submit_read_wait_bio_list(rbio, &bio_list); in recover_rbio()
1972 ret = recover_sectors(rbio); in recover_rbio()
1974 rbio_orig_end_io(rbio, errno_to_blk_status(ret)); in recover_rbio()
1979 struct btrfs_raid_bio *rbio; in recover_rbio_work() local
1981 rbio = container_of(work, struct btrfs_raid_bio, work); in recover_rbio_work()
1982 if (!lock_stripe_add(rbio)) in recover_rbio_work()
1983 recover_rbio(rbio); in recover_rbio_work()
1991 static void set_rbio_raid6_extra_error(struct btrfs_raid_bio *rbio, int mirror_num) in set_rbio_raid6_extra_error() argument
2003 for (sector_nr = 0; sector_nr < rbio->stripe_nsectors; sector_nr++) { in set_rbio_raid6_extra_error()
2008 found_errors = get_rbio_veritical_errors(rbio, sector_nr, in set_rbio_raid6_extra_error()
2022 failb = rbio->real_stripes - (mirror_num - 1); in set_rbio_raid6_extra_error()
2028 set_bit(failb * rbio->stripe_nsectors + sector_nr, in set_rbio_raid6_extra_error()
2029 rbio->error_bitmap); in set_rbio_raid6_extra_error()
2046 struct btrfs_raid_bio *rbio; in raid56_parity_recover() local
2048 rbio = alloc_rbio(fs_info, bioc); in raid56_parity_recover()
2049 if (IS_ERR(rbio)) { in raid56_parity_recover()
2050 bio->bi_status = errno_to_blk_status(PTR_ERR(rbio)); in raid56_parity_recover()
2055 rbio->operation = BTRFS_RBIO_READ_REBUILD; in raid56_parity_recover()
2056 rbio_add_bio(rbio, bio); in raid56_parity_recover()
2058 set_rbio_range_error(rbio, bio); in raid56_parity_recover()
2066 set_rbio_raid6_extra_error(rbio, mirror_num); in raid56_parity_recover()
2068 start_async_work(rbio, recover_rbio_work); in raid56_parity_recover()
2071 static void fill_data_csums(struct btrfs_raid_bio *rbio) in fill_data_csums() argument
2073 struct btrfs_fs_info *fs_info = rbio->bioc->fs_info; in fill_data_csums()
2075 rbio->bioc->full_stripe_logical); in fill_data_csums()
2076 const u64 start = rbio->bioc->full_stripe_logical; in fill_data_csums()
2077 const u32 len = (rbio->nr_data * rbio->stripe_nsectors) << in fill_data_csums()
2082 ASSERT(!rbio->csum_buf && !rbio->csum_bitmap); in fill_data_csums()
2095 if (!(rbio->bioc->map_type & BTRFS_BLOCK_GROUP_DATA) || in fill_data_csums()
2096 rbio->bioc->map_type & BTRFS_BLOCK_GROUP_METADATA) in fill_data_csums()
2099 rbio->csum_buf = kzalloc(rbio->nr_data * rbio->stripe_nsectors * in fill_data_csums()
2101 rbio->csum_bitmap = bitmap_zalloc(rbio->nr_data * rbio->stripe_nsectors, in fill_data_csums()
2103 if (!rbio->csum_buf || !rbio->csum_bitmap) { in fill_data_csums()
2109 rbio->csum_buf, rbio->csum_bitmap); in fill_data_csums()
2112 if (bitmap_empty(rbio->csum_bitmap, len >> fs_info->sectorsize_bits)) in fill_data_csums()
2124 rbio->bioc->full_stripe_logical, ret); in fill_data_csums()
2126 kfree(rbio->csum_buf); in fill_data_csums()
2127 bitmap_free(rbio->csum_bitmap); in fill_data_csums()
2128 rbio->csum_buf = NULL; in fill_data_csums()
2129 rbio->csum_bitmap = NULL; in fill_data_csums()
2132 static int rmw_read_wait_recover(struct btrfs_raid_bio *rbio) in rmw_read_wait_recover() argument
2143 fill_data_csums(rbio); in rmw_read_wait_recover()
2150 for (total_sector_nr = 0; total_sector_nr < rbio->nr_sectors; in rmw_read_wait_recover()
2153 int stripe = total_sector_nr / rbio->stripe_nsectors; in rmw_read_wait_recover()
2154 int sectornr = total_sector_nr % rbio->stripe_nsectors; in rmw_read_wait_recover()
2156 sector = rbio_stripe_sector(rbio, stripe, sectornr); in rmw_read_wait_recover()
2157 ret = rbio_add_io_sector(rbio, &bio_list, sector, in rmw_read_wait_recover()
2169 submit_read_wait_bio_list(rbio, &bio_list); in rmw_read_wait_recover()
2170 return recover_sectors(rbio); in rmw_read_wait_recover()
2175 struct btrfs_raid_bio *rbio = bio->bi_private; in raid_wait_write_end_io() local
2179 rbio_update_error_bitmap(rbio, bio); in raid_wait_write_end_io()
2181 if (atomic_dec_and_test(&rbio->stripes_pending)) in raid_wait_write_end_io()
2182 wake_up(&rbio->io_wait); in raid_wait_write_end_io()
2185 static void submit_write_bios(struct btrfs_raid_bio *rbio, in submit_write_bios() argument
2190 atomic_set(&rbio->stripes_pending, bio_list_size(bio_list)); in submit_write_bios()
2197 bio_get_trace_info(rbio, bio, &trace_info); in submit_write_bios()
2198 trace_raid56_write(rbio, bio, &trace_info); in submit_write_bios()
2208 static bool need_read_stripe_sectors(struct btrfs_raid_bio *rbio) in need_read_stripe_sectors() argument
2212 for (i = 0; i < rbio->nr_data * rbio->stripe_nsectors; i++) { in need_read_stripe_sectors()
2213 struct sector_ptr *sector = &rbio->stripe_sectors[i]; in need_read_stripe_sectors()
2226 static void rmw_rbio(struct btrfs_raid_bio *rbio) in rmw_rbio() argument
2236 ret = alloc_rbio_parity_pages(rbio); in rmw_rbio()
2244 if (!rbio_is_full(rbio) && need_read_stripe_sectors(rbio)) { in rmw_rbio()
2249 ret = alloc_rbio_data_pages(rbio); in rmw_rbio()
2253 index_rbio_pages(rbio); in rmw_rbio()
2255 ret = rmw_read_wait_recover(rbio); in rmw_rbio()
2265 spin_lock(&rbio->bio_list_lock); in rmw_rbio()
2266 set_bit(RBIO_RMW_LOCKED_BIT, &rbio->flags); in rmw_rbio()
2267 spin_unlock(&rbio->bio_list_lock); in rmw_rbio()
2269 bitmap_clear(rbio->error_bitmap, 0, rbio->nr_sectors); in rmw_rbio()
2271 index_rbio_pages(rbio); in rmw_rbio()
2279 if (!rbio_is_full(rbio)) in rmw_rbio()
2280 cache_rbio_pages(rbio); in rmw_rbio()
2282 clear_bit(RBIO_CACHE_READY_BIT, &rbio->flags); in rmw_rbio()
2284 for (sectornr = 0; sectornr < rbio->stripe_nsectors; sectornr++) in rmw_rbio()
2285 generate_pq_vertical(rbio, sectornr); in rmw_rbio()
2288 ret = rmw_assemble_write_bios(rbio, &bio_list); in rmw_rbio()
2294 submit_write_bios(rbio, &bio_list); in rmw_rbio()
2295 wait_event(rbio->io_wait, atomic_read(&rbio->stripes_pending) == 0); in rmw_rbio()
2298 for (sectornr = 0; sectornr < rbio->stripe_nsectors; sectornr++) { in rmw_rbio()
2301 found_errors = get_rbio_veritical_errors(rbio, sectornr, NULL, NULL); in rmw_rbio()
2302 if (found_errors > rbio->bioc->max_errors) { in rmw_rbio()
2308 rbio_orig_end_io(rbio, errno_to_blk_status(ret)); in rmw_rbio()
2313 struct btrfs_raid_bio *rbio; in rmw_rbio_work() local
2315 rbio = container_of(work, struct btrfs_raid_bio, work); in rmw_rbio_work()
2316 if (lock_stripe_add(rbio) == 0) in rmw_rbio_work()
2317 rmw_rbio(rbio); in rmw_rbio_work()
2341 struct btrfs_raid_bio *rbio; in raid56_parity_alloc_scrub_rbio() local
2344 rbio = alloc_rbio(fs_info, bioc); in raid56_parity_alloc_scrub_rbio()
2345 if (IS_ERR(rbio)) in raid56_parity_alloc_scrub_rbio()
2347 bio_list_add(&rbio->bio_list, bio); in raid56_parity_alloc_scrub_rbio()
2353 rbio->operation = BTRFS_RBIO_PARITY_SCRUB; in raid56_parity_alloc_scrub_rbio()
2360 for (i = rbio->nr_data; i < rbio->real_stripes; i++) { in raid56_parity_alloc_scrub_rbio()
2362 rbio->scrubp = i; in raid56_parity_alloc_scrub_rbio()
2366 ASSERT(i < rbio->real_stripes); in raid56_parity_alloc_scrub_rbio()
2368 bitmap_copy(&rbio->dbitmap, dbitmap, stripe_nsectors); in raid56_parity_alloc_scrub_rbio()
2369 return rbio; in raid56_parity_alloc_scrub_rbio()
2376 static int alloc_rbio_essential_pages(struct btrfs_raid_bio *rbio) in alloc_rbio_essential_pages() argument
2378 const u32 sectorsize = rbio->bioc->fs_info->sectorsize; in alloc_rbio_essential_pages()
2381 for (total_sector_nr = 0; total_sector_nr < rbio->nr_sectors; in alloc_rbio_essential_pages()
2384 int sectornr = total_sector_nr % rbio->stripe_nsectors; in alloc_rbio_essential_pages()
2387 if (!test_bit(sectornr, &rbio->dbitmap)) in alloc_rbio_essential_pages()
2389 if (rbio->stripe_pages[index]) in alloc_rbio_essential_pages()
2394 rbio->stripe_pages[index] = page; in alloc_rbio_essential_pages()
2396 index_stripe_sectors(rbio); in alloc_rbio_essential_pages()
2400 static int finish_parity_scrub(struct btrfs_raid_bio *rbio) in finish_parity_scrub() argument
2402 struct btrfs_io_context *bioc = rbio->bioc; in finish_parity_scrub()
2404 void **pointers = rbio->finish_pointers; in finish_parity_scrub()
2405 unsigned long *pbitmap = &rbio->finish_pbitmap; in finish_parity_scrub()
2406 int nr_data = rbio->nr_data; in finish_parity_scrub()
2418 if (rbio->real_stripes - rbio->nr_data == 1) in finish_parity_scrub()
2420 else if (rbio->real_stripes - rbio->nr_data == 2) in finish_parity_scrub()
2429 if (bioc->replace_nr_stripes && bioc->replace_stripe_src == rbio->scrubp) { in finish_parity_scrub()
2431 bitmap_copy(pbitmap, &rbio->dbitmap, rbio->stripe_nsectors); in finish_parity_scrub()
2439 clear_bit(RBIO_CACHE_READY_BIT, &rbio->flags); in finish_parity_scrub()
2457 pointers[rbio->real_stripes - 1] = kmap_local_page(q_sector.page); in finish_parity_scrub()
2460 bitmap_clear(rbio->error_bitmap, 0, rbio->nr_sectors); in finish_parity_scrub()
2465 for_each_set_bit(sectornr, &rbio->dbitmap, rbio->stripe_nsectors) { in finish_parity_scrub()
2471 sector = sector_in_rbio(rbio, stripe, sectornr, 0); in finish_parity_scrub()
2478 raid6_call.gen_syndrome(rbio->real_stripes, sectorsize, in finish_parity_scrub()
2487 sector = rbio_stripe_sector(rbio, rbio->scrubp, sectornr); in finish_parity_scrub()
2489 if (memcmp(parity, pointers[rbio->scrubp], sectorsize) != 0) in finish_parity_scrub()
2490 memcpy(parity, pointers[rbio->scrubp], sectorsize); in finish_parity_scrub()
2493 bitmap_clear(&rbio->dbitmap, sectornr, 1); in finish_parity_scrub()
2504 kunmap_local(pointers[rbio->real_stripes - 1]); in finish_parity_scrub()
2514 for_each_set_bit(sectornr, &rbio->dbitmap, rbio->stripe_nsectors) { in finish_parity_scrub()
2517 sector = rbio_stripe_sector(rbio, rbio->scrubp, sectornr); in finish_parity_scrub()
2518 ret = rbio_add_io_sector(rbio, &bio_list, sector, rbio->scrubp, in finish_parity_scrub()
2531 ASSERT(rbio->bioc->replace_stripe_src >= 0); in finish_parity_scrub()
2532 for_each_set_bit(sectornr, pbitmap, rbio->stripe_nsectors) { in finish_parity_scrub()
2535 sector = rbio_stripe_sector(rbio, rbio->scrubp, sectornr); in finish_parity_scrub()
2536 ret = rbio_add_io_sector(rbio, &bio_list, sector, in finish_parity_scrub()
2537 rbio->real_stripes, in finish_parity_scrub()
2544 submit_write_bios(rbio, &bio_list); in finish_parity_scrub()
2552 static inline int is_data_stripe(struct btrfs_raid_bio *rbio, int stripe) in is_data_stripe() argument
2554 if (stripe >= 0 && stripe < rbio->nr_data) in is_data_stripe()
2559 static int recover_scrub_rbio(struct btrfs_raid_bio *rbio) in recover_scrub_rbio() argument
2572 pointers = kcalloc(rbio->real_stripes, sizeof(void *), GFP_NOFS); in recover_scrub_rbio()
2573 unmap_array = kcalloc(rbio->real_stripes, sizeof(void *), GFP_NOFS); in recover_scrub_rbio()
2579 for (sector_nr = 0; sector_nr < rbio->stripe_nsectors; sector_nr++) { in recover_scrub_rbio()
2585 found_errors = get_rbio_veritical_errors(rbio, sector_nr, in recover_scrub_rbio()
2587 if (found_errors > rbio->bioc->max_errors) { in recover_scrub_rbio()
2597 if (is_data_stripe(rbio, faila)) in recover_scrub_rbio()
2602 if (is_data_stripe(rbio, failb)) in recover_scrub_rbio()
2611 if (dfail > rbio->bioc->max_errors - 1) { in recover_scrub_rbio()
2628 if (failp != rbio->scrubp) { in recover_scrub_rbio()
2633 ret = recover_vertical(rbio, sector_nr, pointers, unmap_array); in recover_scrub_rbio()
2643 static int scrub_assemble_read_bios(struct btrfs_raid_bio *rbio) in scrub_assemble_read_bios() argument
2650 for (total_sector_nr = 0; total_sector_nr < rbio->nr_sectors; in scrub_assemble_read_bios()
2652 int sectornr = total_sector_nr % rbio->stripe_nsectors; in scrub_assemble_read_bios()
2653 int stripe = total_sector_nr / rbio->stripe_nsectors; in scrub_assemble_read_bios()
2657 if (!test_bit(sectornr, &rbio->dbitmap)) in scrub_assemble_read_bios()
2665 sector = sector_in_rbio(rbio, stripe, sectornr, 1); in scrub_assemble_read_bios()
2669 sector = rbio_stripe_sector(rbio, stripe, sectornr); in scrub_assemble_read_bios()
2677 ret = rbio_add_io_sector(rbio, &bio_list, sector, stripe, in scrub_assemble_read_bios()
2685 submit_read_wait_bio_list(rbio, &bio_list); in scrub_assemble_read_bios()
2689 static void scrub_rbio(struct btrfs_raid_bio *rbio) in scrub_rbio() argument
2694 ret = alloc_rbio_essential_pages(rbio); in scrub_rbio()
2698 bitmap_clear(rbio->error_bitmap, 0, rbio->nr_sectors); in scrub_rbio()
2700 ret = scrub_assemble_read_bios(rbio); in scrub_rbio()
2705 ret = recover_scrub_rbio(rbio); in scrub_rbio()
2713 ret = finish_parity_scrub(rbio); in scrub_rbio()
2714 wait_event(rbio->io_wait, atomic_read(&rbio->stripes_pending) == 0); in scrub_rbio()
2715 for (sector_nr = 0; sector_nr < rbio->stripe_nsectors; sector_nr++) { in scrub_rbio()
2718 found_errors = get_rbio_veritical_errors(rbio, sector_nr, NULL, NULL); in scrub_rbio()
2719 if (found_errors > rbio->bioc->max_errors) { in scrub_rbio()
2725 rbio_orig_end_io(rbio, errno_to_blk_status(ret)); in scrub_rbio()
2733 void raid56_parity_submit_scrub_rbio(struct btrfs_raid_bio *rbio) in raid56_parity_submit_scrub_rbio() argument
2735 if (!lock_stripe_add(rbio)) in raid56_parity_submit_scrub_rbio()
2736 start_async_work(rbio, scrub_rbio_work_locked); in raid56_parity_submit_scrub_rbio()
2746 void raid56_parity_cache_data_pages(struct btrfs_raid_bio *rbio, in raid56_parity_cache_data_pages() argument
2750 rbio->bioc->full_stripe_logical; in raid56_parity_cache_data_pages()
2752 const u32 sectorsize = rbio->bioc->fs_info->sectorsize; in raid56_parity_cache_data_pages()
2764 ret = alloc_rbio_data_pages(rbio); in raid56_parity_cache_data_pages()
2770 ASSERT(offset_in_full_stripe < (rbio->nr_data << BTRFS_STRIPE_LEN_SHIFT)); in raid56_parity_cache_data_pages()
2773 struct page *dst = rbio->stripe_pages[page_nr + page_index]; in raid56_parity_cache_data_pages()
2780 rbio->stripe_sectors[sector_nr].uptodate = true; in raid56_parity_cache_data_pages()