Lines Matching refs:sctx
85 struct scrub_ctx *sctx; member
106 struct scrub_ctx *sctx; member
123 struct scrub_ctx *sctx; member
235 static int scrub_pages(struct scrub_ctx *sctx, u64 logical, u32 len,
247 static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx,
249 static void scrub_wr_submit(struct scrub_ctx *sctx);
252 static void scrub_put_ctx(struct scrub_ctx *sctx);
260 static void scrub_pending_bio_inc(struct scrub_ctx *sctx) in scrub_pending_bio_inc() argument
262 refcount_inc(&sctx->refs); in scrub_pending_bio_inc()
263 atomic_inc(&sctx->bios_in_flight); in scrub_pending_bio_inc()
266 static void scrub_pending_bio_dec(struct scrub_ctx *sctx) in scrub_pending_bio_dec() argument
268 atomic_dec(&sctx->bios_in_flight); in scrub_pending_bio_dec()
269 wake_up(&sctx->list_wait); in scrub_pending_bio_dec()
270 scrub_put_ctx(sctx); in scrub_pending_bio_dec()
522 static void scrub_free_csums(struct scrub_ctx *sctx) in scrub_free_csums() argument
524 while (!list_empty(&sctx->csum_list)) { in scrub_free_csums()
526 sum = list_first_entry(&sctx->csum_list, in scrub_free_csums()
533 static noinline_for_stack void scrub_free_ctx(struct scrub_ctx *sctx) in scrub_free_ctx() argument
537 if (!sctx) in scrub_free_ctx()
541 if (sctx->curr != -1) { in scrub_free_ctx()
542 struct scrub_bio *sbio = sctx->bios[sctx->curr]; in scrub_free_ctx()
552 struct scrub_bio *sbio = sctx->bios[i]; in scrub_free_ctx()
559 kfree(sctx->wr_curr_bio); in scrub_free_ctx()
560 scrub_free_csums(sctx); in scrub_free_ctx()
561 kfree(sctx); in scrub_free_ctx()
564 static void scrub_put_ctx(struct scrub_ctx *sctx) in scrub_put_ctx() argument
566 if (refcount_dec_and_test(&sctx->refs)) in scrub_put_ctx()
567 scrub_free_ctx(sctx); in scrub_put_ctx()
573 struct scrub_ctx *sctx; in scrub_setup_ctx() local
576 sctx = kzalloc(sizeof(*sctx), GFP_KERNEL); in scrub_setup_ctx()
577 if (!sctx) in scrub_setup_ctx()
579 refcount_set(&sctx->refs, 1); in scrub_setup_ctx()
580 sctx->is_dev_replace = is_dev_replace; in scrub_setup_ctx()
581 sctx->pages_per_rd_bio = SCRUB_PAGES_PER_RD_BIO; in scrub_setup_ctx()
582 sctx->curr = -1; in scrub_setup_ctx()
583 sctx->fs_info = fs_info; in scrub_setup_ctx()
584 INIT_LIST_HEAD(&sctx->csum_list); in scrub_setup_ctx()
591 sctx->bios[i] = sbio; in scrub_setup_ctx()
594 sbio->sctx = sctx; in scrub_setup_ctx()
600 sctx->bios[i]->next_free = i + 1; in scrub_setup_ctx()
602 sctx->bios[i]->next_free = -1; in scrub_setup_ctx()
604 sctx->first_free = 0; in scrub_setup_ctx()
605 atomic_set(&sctx->bios_in_flight, 0); in scrub_setup_ctx()
606 atomic_set(&sctx->workers_pending, 0); in scrub_setup_ctx()
607 atomic_set(&sctx->cancel_req, 0); in scrub_setup_ctx()
609 spin_lock_init(&sctx->list_lock); in scrub_setup_ctx()
610 spin_lock_init(&sctx->stat_lock); in scrub_setup_ctx()
611 init_waitqueue_head(&sctx->list_wait); in scrub_setup_ctx()
612 sctx->throttle_deadline = 0; in scrub_setup_ctx()
614 WARN_ON(sctx->wr_curr_bio != NULL); in scrub_setup_ctx()
615 mutex_init(&sctx->wr_lock); in scrub_setup_ctx()
616 sctx->wr_curr_bio = NULL; in scrub_setup_ctx()
619 sctx->pages_per_wr_bio = SCRUB_PAGES_PER_WR_BIO; in scrub_setup_ctx()
620 sctx->wr_tgtdev = fs_info->dev_replace.tgtdev; in scrub_setup_ctx()
621 sctx->flush_all_writes = false; in scrub_setup_ctx()
624 return sctx; in scrub_setup_ctx()
627 scrub_free_ctx(sctx); in scrub_setup_ctx()
740 fs_info = sblock->sctx->fs_info; in scrub_print_warning()
816 struct scrub_ctx *sctx = sblock_to_check->sctx; in scrub_handle_errored_block() local
835 fs_info = sctx->fs_info; in scrub_handle_errored_block()
842 spin_lock(&sctx->stat_lock); in scrub_handle_errored_block()
843 ++sctx->stat.super_errors; in scrub_handle_errored_block()
844 spin_unlock(&sctx->stat_lock); in scrub_handle_errored_block()
855 if (btrfs_is_zoned(fs_info) && !sctx->is_dev_replace) in scrub_handle_errored_block()
878 spin_lock(&sctx->stat_lock); in scrub_handle_errored_block()
880 sctx->stat.malloc_errors++; in scrub_handle_errored_block()
881 sctx->stat.read_errors++; in scrub_handle_errored_block()
882 sctx->stat.uncorrectable_errors++; in scrub_handle_errored_block()
883 spin_unlock(&sctx->stat_lock); in scrub_handle_errored_block()
919 spin_lock(&sctx->stat_lock); in scrub_handle_errored_block()
920 sctx->stat.malloc_errors++; in scrub_handle_errored_block()
921 sctx->stat.read_errors++; in scrub_handle_errored_block()
922 sctx->stat.uncorrectable_errors++; in scrub_handle_errored_block()
923 spin_unlock(&sctx->stat_lock); in scrub_handle_errored_block()
931 spin_lock(&sctx->stat_lock); in scrub_handle_errored_block()
932 sctx->stat.read_errors++; in scrub_handle_errored_block()
933 sctx->stat.uncorrectable_errors++; in scrub_handle_errored_block()
934 spin_unlock(&sctx->stat_lock); in scrub_handle_errored_block()
954 spin_lock(&sctx->stat_lock); in scrub_handle_errored_block()
955 sctx->stat.unverified_errors++; in scrub_handle_errored_block()
957 spin_unlock(&sctx->stat_lock); in scrub_handle_errored_block()
959 if (sctx->is_dev_replace) in scrub_handle_errored_block()
965 spin_lock(&sctx->stat_lock); in scrub_handle_errored_block()
966 sctx->stat.read_errors++; in scrub_handle_errored_block()
967 spin_unlock(&sctx->stat_lock); in scrub_handle_errored_block()
972 spin_lock(&sctx->stat_lock); in scrub_handle_errored_block()
973 sctx->stat.csum_errors++; in scrub_handle_errored_block()
974 spin_unlock(&sctx->stat_lock); in scrub_handle_errored_block()
980 spin_lock(&sctx->stat_lock); in scrub_handle_errored_block()
981 sctx->stat.verify_errors++; in scrub_handle_errored_block()
982 spin_unlock(&sctx->stat_lock); in scrub_handle_errored_block()
994 if (sctx->readonly) { in scrub_handle_errored_block()
995 ASSERT(!sctx->is_dev_replace); in scrub_handle_errored_block()
1049 if (sctx->is_dev_replace) { in scrub_handle_errored_block()
1061 if (sblock_bad->no_io_error_seen && !sctx->is_dev_replace) in scrub_handle_errored_block()
1095 if (!spage_bad->io_error && !sctx->is_dev_replace) in scrub_handle_errored_block()
1124 if (sctx->is_dev_replace) { in scrub_handle_errored_block()
1152 if (success && !sctx->is_dev_replace) { in scrub_handle_errored_block()
1172 spin_lock(&sctx->stat_lock); in scrub_handle_errored_block()
1173 sctx->stat.corrected_errors++; in scrub_handle_errored_block()
1175 spin_unlock(&sctx->stat_lock); in scrub_handle_errored_block()
1182 spin_lock(&sctx->stat_lock); in scrub_handle_errored_block()
1183 sctx->stat.uncorrectable_errors++; in scrub_handle_errored_block()
1184 spin_unlock(&sctx->stat_lock); in scrub_handle_errored_block()
1264 struct scrub_ctx *sctx = original_sblock->sctx; in scrub_setup_recheck_block() local
1265 struct btrfs_fs_info *fs_info = sctx->fs_info; in scrub_setup_recheck_block()
1327 sblock->sctx = sctx; in scrub_setup_recheck_block()
1332 spin_lock(&sctx->stat_lock); in scrub_setup_recheck_block()
1333 sctx->stat.malloc_errors++; in scrub_setup_recheck_block()
1334 spin_unlock(&sctx->stat_lock); in scrub_setup_recheck_block()
1348 sctx->fs_info->csum_size); in scrub_setup_recheck_block()
1549 struct btrfs_fs_info *fs_info = sblock_bad->sctx->fs_info; in scrub_repair_page_from_good_copy()
1591 struct btrfs_fs_info *fs_info = sblock->sctx->fs_info; in scrub_write_block_to_dev_replace()
1619 return scrub_add_page_to_wr_bio(sblock->sctx, spage); in scrub_write_page_to_dev_replace()
1622 static int fill_writer_pointer_gap(struct scrub_ctx *sctx, u64 physical) in fill_writer_pointer_gap() argument
1627 if (!btrfs_is_zoned(sctx->fs_info)) in fill_writer_pointer_gap()
1630 if (!btrfs_dev_is_sequential(sctx->wr_tgtdev, physical)) in fill_writer_pointer_gap()
1633 if (sctx->write_pointer < physical) { in fill_writer_pointer_gap()
1634 length = physical - sctx->write_pointer; in fill_writer_pointer_gap()
1636 ret = btrfs_zoned_issue_zeroout(sctx->wr_tgtdev, in fill_writer_pointer_gap()
1637 sctx->write_pointer, length); in fill_writer_pointer_gap()
1639 sctx->write_pointer = physical; in fill_writer_pointer_gap()
1644 static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx, in scrub_add_page_to_wr_bio() argument
1649 const u32 sectorsize = sctx->fs_info->sectorsize; in scrub_add_page_to_wr_bio()
1651 mutex_lock(&sctx->wr_lock); in scrub_add_page_to_wr_bio()
1653 if (!sctx->wr_curr_bio) { in scrub_add_page_to_wr_bio()
1654 sctx->wr_curr_bio = kzalloc(sizeof(*sctx->wr_curr_bio), in scrub_add_page_to_wr_bio()
1656 if (!sctx->wr_curr_bio) { in scrub_add_page_to_wr_bio()
1657 mutex_unlock(&sctx->wr_lock); in scrub_add_page_to_wr_bio()
1660 sctx->wr_curr_bio->sctx = sctx; in scrub_add_page_to_wr_bio()
1661 sctx->wr_curr_bio->page_count = 0; in scrub_add_page_to_wr_bio()
1663 sbio = sctx->wr_curr_bio; in scrub_add_page_to_wr_bio()
1667 ret = fill_writer_pointer_gap(sctx, in scrub_add_page_to_wr_bio()
1670 mutex_unlock(&sctx->wr_lock); in scrub_add_page_to_wr_bio()
1676 sbio->dev = sctx->wr_tgtdev; in scrub_add_page_to_wr_bio()
1679 bio = btrfs_io_bio_alloc(sctx->pages_per_wr_bio); in scrub_add_page_to_wr_bio()
1693 scrub_wr_submit(sctx); in scrub_add_page_to_wr_bio()
1702 mutex_unlock(&sctx->wr_lock); in scrub_add_page_to_wr_bio()
1705 scrub_wr_submit(sctx); in scrub_add_page_to_wr_bio()
1712 if (sbio->page_count == sctx->pages_per_wr_bio) in scrub_add_page_to_wr_bio()
1713 scrub_wr_submit(sctx); in scrub_add_page_to_wr_bio()
1714 mutex_unlock(&sctx->wr_lock); in scrub_add_page_to_wr_bio()
1719 static void scrub_wr_submit(struct scrub_ctx *sctx) in scrub_wr_submit() argument
1723 if (!sctx->wr_curr_bio) in scrub_wr_submit()
1726 sbio = sctx->wr_curr_bio; in scrub_wr_submit()
1727 sctx->wr_curr_bio = NULL; in scrub_wr_submit()
1729 scrub_pending_bio_inc(sctx); in scrub_wr_submit()
1736 if (btrfs_is_zoned(sctx->fs_info)) in scrub_wr_submit()
1737 sctx->write_pointer = sbio->physical + sbio->page_count * in scrub_wr_submit()
1738 sctx->fs_info->sectorsize; in scrub_wr_submit()
1756 struct scrub_ctx *sctx = sbio->sctx; in scrub_wr_bio_end_io_worker() local
1762 &sbio->sctx->fs_info->dev_replace; in scrub_wr_bio_end_io_worker()
1777 scrub_pending_bio_dec(sctx); in scrub_wr_bio_end_io_worker()
1816 struct scrub_ctx *sctx = sblock->sctx; in scrub_checksum_data() local
1817 struct btrfs_fs_info *fs_info = sctx->fs_info; in scrub_checksum_data()
1846 struct scrub_ctx *sctx = sblock->sctx; in scrub_checksum_tree_block() local
1848 struct btrfs_fs_info *fs_info = sctx->fs_info; in scrub_checksum_tree_block()
1857 const u32 sectorsize = sctx->fs_info->sectorsize; in scrub_checksum_tree_block()
1871 memcpy(on_disk_csum, h->csum, sctx->fs_info->csum_size); in scrub_checksum_tree_block()
1904 if (memcmp(calculated_csum, on_disk_csum, sctx->fs_info->csum_size)) in scrub_checksum_tree_block()
1913 struct scrub_ctx *sctx = sblock->sctx; in scrub_checksum_super() local
1914 struct btrfs_fs_info *fs_info = sctx->fs_info; in scrub_checksum_super()
1941 if (memcmp(calculated_csum, s->csum, sctx->fs_info->csum_size)) in scrub_checksum_super()
1950 spin_lock(&sctx->stat_lock); in scrub_checksum_super()
1951 ++sctx->stat.super_errors; in scrub_checksum_super()
1952 spin_unlock(&sctx->stat_lock); in scrub_checksum_super()
2001 static void scrub_throttle(struct scrub_ctx *sctx) in scrub_throttle() argument
2011 sbio = sctx->bios[sctx->curr]; in scrub_throttle()
2026 if (sctx->throttle_deadline == 0) { in scrub_throttle()
2027 sctx->throttle_deadline = ktime_add_ms(now, time_slice / div); in scrub_throttle()
2028 sctx->throttle_sent = 0; in scrub_throttle()
2032 if (ktime_before(now, sctx->throttle_deadline)) { in scrub_throttle()
2034 sctx->throttle_sent += sbio->bio->bi_iter.bi_size; in scrub_throttle()
2035 if (sctx->throttle_sent <= div_u64(bwlimit, div)) in scrub_throttle()
2039 delta = ktime_ms_delta(sctx->throttle_deadline, now); in scrub_throttle()
2053 sctx->throttle_deadline = 0; in scrub_throttle()
2056 static void scrub_submit(struct scrub_ctx *sctx) in scrub_submit() argument
2060 if (sctx->curr == -1) in scrub_submit()
2063 scrub_throttle(sctx); in scrub_submit()
2065 sbio = sctx->bios[sctx->curr]; in scrub_submit()
2066 sctx->curr = -1; in scrub_submit()
2067 scrub_pending_bio_inc(sctx); in scrub_submit()
2071 static int scrub_add_page_to_rd_bio(struct scrub_ctx *sctx, in scrub_add_page_to_rd_bio() argument
2076 const u32 sectorsize = sctx->fs_info->sectorsize; in scrub_add_page_to_rd_bio()
2083 while (sctx->curr == -1) { in scrub_add_page_to_rd_bio()
2084 spin_lock(&sctx->list_lock); in scrub_add_page_to_rd_bio()
2085 sctx->curr = sctx->first_free; in scrub_add_page_to_rd_bio()
2086 if (sctx->curr != -1) { in scrub_add_page_to_rd_bio()
2087 sctx->first_free = sctx->bios[sctx->curr]->next_free; in scrub_add_page_to_rd_bio()
2088 sctx->bios[sctx->curr]->next_free = -1; in scrub_add_page_to_rd_bio()
2089 sctx->bios[sctx->curr]->page_count = 0; in scrub_add_page_to_rd_bio()
2090 spin_unlock(&sctx->list_lock); in scrub_add_page_to_rd_bio()
2092 spin_unlock(&sctx->list_lock); in scrub_add_page_to_rd_bio()
2093 wait_event(sctx->list_wait, sctx->first_free != -1); in scrub_add_page_to_rd_bio()
2096 sbio = sctx->bios[sctx->curr]; in scrub_add_page_to_rd_bio()
2105 bio = btrfs_io_bio_alloc(sctx->pages_per_rd_bio); in scrub_add_page_to_rd_bio()
2120 scrub_submit(sctx); in scrub_add_page_to_rd_bio()
2132 scrub_submit(sctx); in scrub_add_page_to_rd_bio()
2139 if (sbio->page_count == sctx->pages_per_rd_bio) in scrub_add_page_to_rd_bio()
2140 scrub_submit(sctx); in scrub_add_page_to_rd_bio()
2148 struct btrfs_fs_info *fs_info = sblock->sctx->fs_info; in scrub_missing_raid56_end_io()
2161 struct scrub_ctx *sctx = sblock->sctx; in scrub_missing_raid56_worker() local
2162 struct btrfs_fs_info *fs_info = sctx->fs_info; in scrub_missing_raid56_worker()
2173 spin_lock(&sctx->stat_lock); in scrub_missing_raid56_worker()
2174 sctx->stat.read_errors++; in scrub_missing_raid56_worker()
2175 spin_unlock(&sctx->stat_lock); in scrub_missing_raid56_worker()
2180 spin_lock(&sctx->stat_lock); in scrub_missing_raid56_worker()
2181 sctx->stat.uncorrectable_errors++; in scrub_missing_raid56_worker()
2182 spin_unlock(&sctx->stat_lock); in scrub_missing_raid56_worker()
2190 if (sctx->is_dev_replace && sctx->flush_all_writes) { in scrub_missing_raid56_worker()
2191 mutex_lock(&sctx->wr_lock); in scrub_missing_raid56_worker()
2192 scrub_wr_submit(sctx); in scrub_missing_raid56_worker()
2193 mutex_unlock(&sctx->wr_lock); in scrub_missing_raid56_worker()
2197 scrub_pending_bio_dec(sctx); in scrub_missing_raid56_worker()
2202 struct scrub_ctx *sctx = sblock->sctx; in scrub_missing_raid56_pages() local
2203 struct btrfs_fs_info *fs_info = sctx->fs_info; in scrub_missing_raid56_pages()
2218 if (WARN_ON(!sctx->is_dev_replace || in scrub_missing_raid56_pages()
2246 scrub_pending_bio_inc(sctx); in scrub_missing_raid56_pages()
2255 spin_lock(&sctx->stat_lock); in scrub_missing_raid56_pages()
2256 sctx->stat.malloc_errors++; in scrub_missing_raid56_pages()
2257 spin_unlock(&sctx->stat_lock); in scrub_missing_raid56_pages()
2260 static int scrub_pages(struct scrub_ctx *sctx, u64 logical, u32 len, in scrub_pages() argument
2266 const u32 sectorsize = sctx->fs_info->sectorsize; in scrub_pages()
2271 spin_lock(&sctx->stat_lock); in scrub_pages()
2272 sctx->stat.malloc_errors++; in scrub_pages()
2273 spin_unlock(&sctx->stat_lock); in scrub_pages()
2280 sblock->sctx = sctx; in scrub_pages()
2295 spin_lock(&sctx->stat_lock); in scrub_pages()
2296 sctx->stat.malloc_errors++; in scrub_pages()
2297 spin_unlock(&sctx->stat_lock); in scrub_pages()
2314 memcpy(spage->csum, csum, sctx->fs_info->csum_size); in scrub_pages()
2340 ret = scrub_add_page_to_rd_bio(sctx, spage); in scrub_pages()
2348 scrub_submit(sctx); in scrub_pages()
2370 struct scrub_ctx *sctx = sbio->sctx; in scrub_bio_end_io_worker() local
2395 spin_lock(&sctx->list_lock); in scrub_bio_end_io_worker()
2396 sbio->next_free = sctx->first_free; in scrub_bio_end_io_worker()
2397 sctx->first_free = sbio->index; in scrub_bio_end_io_worker()
2398 spin_unlock(&sctx->list_lock); in scrub_bio_end_io_worker()
2400 if (sctx->is_dev_replace && sctx->flush_all_writes) { in scrub_bio_end_io_worker()
2401 mutex_lock(&sctx->wr_lock); in scrub_bio_end_io_worker()
2402 scrub_wr_submit(sctx); in scrub_bio_end_io_worker()
2403 mutex_unlock(&sctx->wr_lock); in scrub_bio_end_io_worker()
2406 scrub_pending_bio_dec(sctx); in scrub_bio_end_io_worker()
2415 u32 sectorsize_bits = sparity->sctx->fs_info->sectorsize_bits; in __scrub_mark_bitmap()
2462 if (!corrupted && sblock->sctx->is_dev_replace) in scrub_block_complete()
2469 sblock->sctx->fs_info->sectorsize; in scrub_block_complete()
2477 static void drop_csum_range(struct scrub_ctx *sctx, struct btrfs_ordered_sum *sum) in drop_csum_range() argument
2479 sctx->stat.csum_discards += sum->len >> sctx->fs_info->sectorsize_bits; in drop_csum_range()
2495 static int scrub_find_csum(struct scrub_ctx *sctx, u64 logical, u8 *csum) in scrub_find_csum() argument
2499 while (!list_empty(&sctx->csum_list)) { in scrub_find_csum()
2504 sum = list_first_entry(&sctx->csum_list, in scrub_find_csum()
2517 drop_csum_range(sctx, sum); in scrub_find_csum()
2523 index = (logical - sum->bytenr) >> sctx->fs_info->sectorsize_bits; in scrub_find_csum()
2524 num_sectors = sum->len >> sctx->fs_info->sectorsize_bits; in scrub_find_csum()
2526 memcpy(csum, sum->sums + index * sctx->fs_info->csum_size, in scrub_find_csum()
2527 sctx->fs_info->csum_size); in scrub_find_csum()
2531 drop_csum_range(sctx, sum); in scrub_find_csum()
2540 static int scrub_extent(struct scrub_ctx *sctx, struct map_lookup *map, in scrub_extent() argument
2553 blocksize = sctx->fs_info->sectorsize; in scrub_extent()
2554 spin_lock(&sctx->stat_lock); in scrub_extent()
2555 sctx->stat.data_extents_scrubbed++; in scrub_extent()
2556 sctx->stat.data_bytes_scrubbed += len; in scrub_extent()
2557 spin_unlock(&sctx->stat_lock); in scrub_extent()
2562 blocksize = sctx->fs_info->nodesize; in scrub_extent()
2563 spin_lock(&sctx->stat_lock); in scrub_extent()
2564 sctx->stat.tree_extents_scrubbed++; in scrub_extent()
2565 sctx->stat.tree_bytes_scrubbed += len; in scrub_extent()
2566 spin_unlock(&sctx->stat_lock); in scrub_extent()
2568 blocksize = sctx->fs_info->sectorsize; in scrub_extent()
2578 have_csum = scrub_find_csum(sctx, logical, csum); in scrub_extent()
2580 ++sctx->stat.no_csum; in scrub_extent()
2582 ret = scrub_pages(sctx, logical, l, physical, dev, flags, gen, in scrub_extent()
2600 struct scrub_ctx *sctx = sparity->sctx; in scrub_pages_for_parity() local
2602 const u32 sectorsize = sctx->fs_info->sectorsize; in scrub_pages_for_parity()
2609 spin_lock(&sctx->stat_lock); in scrub_pages_for_parity()
2610 sctx->stat.malloc_errors++; in scrub_pages_for_parity()
2611 spin_unlock(&sctx->stat_lock); in scrub_pages_for_parity()
2618 sblock->sctx = sctx; in scrub_pages_for_parity()
2629 spin_lock(&sctx->stat_lock); in scrub_pages_for_parity()
2630 sctx->stat.malloc_errors++; in scrub_pages_for_parity()
2631 spin_unlock(&sctx->stat_lock); in scrub_pages_for_parity()
2651 memcpy(spage->csum, csum, sctx->fs_info->csum_size); in scrub_pages_for_parity()
2672 ret = scrub_add_page_to_rd_bio(sctx, spage); in scrub_pages_for_parity()
2689 struct scrub_ctx *sctx = sparity->sctx; in scrub_extent_for_parity() local
2704 blocksize = sctx->fs_info->sectorsize; in scrub_extent_for_parity()
2714 have_csum = scrub_find_csum(sctx, logical, csum); in scrub_extent_for_parity()
2777 struct scrub_ctx *sctx = sparity->sctx; in scrub_free_parity() local
2783 spin_lock(&sctx->stat_lock); in scrub_free_parity()
2784 sctx->stat.read_errors += nbits; in scrub_free_parity()
2785 sctx->stat.uncorrectable_errors += nbits; in scrub_free_parity()
2786 spin_unlock(&sctx->stat_lock); in scrub_free_parity()
2801 struct scrub_ctx *sctx = sparity->sctx; in scrub_parity_bio_endio_worker() local
2804 scrub_pending_bio_dec(sctx); in scrub_parity_bio_endio_worker()
2810 struct btrfs_fs_info *fs_info = sparity->sctx->fs_info; in scrub_parity_bio_endio()
2825 struct scrub_ctx *sctx = sparity->sctx; in scrub_parity_check_and_repair() local
2826 struct btrfs_fs_info *fs_info = sctx->fs_info; in scrub_parity_check_and_repair()
2857 scrub_pending_bio_inc(sctx); in scrub_parity_check_and_repair()
2868 spin_lock(&sctx->stat_lock); in scrub_parity_check_and_repair()
2869 sctx->stat.malloc_errors++; in scrub_parity_check_and_repair()
2870 spin_unlock(&sctx->stat_lock); in scrub_parity_check_and_repair()
2893 static noinline_for_stack int scrub_raid56_parity(struct scrub_ctx *sctx, in scrub_raid56_parity() argument
2900 struct btrfs_fs_info *fs_info = sctx->fs_info; in scrub_raid56_parity()
2929 spin_lock(&sctx->stat_lock); in scrub_raid56_parity()
2930 sctx->stat.malloc_errors++; in scrub_raid56_parity()
2931 spin_unlock(&sctx->stat_lock); in scrub_raid56_parity()
2938 sparity->sctx = sctx; in scrub_raid56_parity()
3023 spin_lock(&sctx->stat_lock); in scrub_raid56_parity()
3024 sctx->stat.uncorrectable_errors++; in scrub_raid56_parity()
3025 spin_unlock(&sctx->stat_lock); in scrub_raid56_parity()
3067 &sctx->csum_list, 1); in scrub_raid56_parity()
3078 scrub_free_csums(sctx); in scrub_raid56_parity()
3115 scrub_submit(sctx); in scrub_raid56_parity()
3116 mutex_lock(&sctx->wr_lock); in scrub_raid56_parity()
3117 scrub_wr_submit(sctx); in scrub_raid56_parity()
3118 mutex_unlock(&sctx->wr_lock); in scrub_raid56_parity()
3124 static void sync_replace_for_zoned(struct scrub_ctx *sctx) in sync_replace_for_zoned() argument
3126 if (!btrfs_is_zoned(sctx->fs_info)) in sync_replace_for_zoned()
3129 sctx->flush_all_writes = true; in sync_replace_for_zoned()
3130 scrub_submit(sctx); in sync_replace_for_zoned()
3131 mutex_lock(&sctx->wr_lock); in sync_replace_for_zoned()
3132 scrub_wr_submit(sctx); in sync_replace_for_zoned()
3133 mutex_unlock(&sctx->wr_lock); in sync_replace_for_zoned()
3135 wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0); in sync_replace_for_zoned()
3138 static int sync_write_pointer_for_zoned(struct scrub_ctx *sctx, u64 logical, in sync_write_pointer_for_zoned() argument
3141 struct btrfs_fs_info *fs_info = sctx->fs_info; in sync_write_pointer_for_zoned()
3147 wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0); in sync_write_pointer_for_zoned()
3149 mutex_lock(&sctx->wr_lock); in sync_write_pointer_for_zoned()
3150 if (sctx->write_pointer < physical_end) { in sync_write_pointer_for_zoned()
3151 ret = btrfs_sync_zone_write_pointer(sctx->wr_tgtdev, logical, in sync_write_pointer_for_zoned()
3153 sctx->write_pointer); in sync_write_pointer_for_zoned()
3158 mutex_unlock(&sctx->wr_lock); in sync_write_pointer_for_zoned()
3159 btrfs_dev_clear_zone_empty(sctx->wr_tgtdev, physical); in sync_write_pointer_for_zoned()
3164 static noinline_for_stack int scrub_stripe(struct scrub_ctx *sctx, in scrub_stripe() argument
3171 struct btrfs_fs_info *fs_info = sctx->fs_info; in scrub_stripe()
3262 wait_event(sctx->list_wait, in scrub_stripe()
3263 atomic_read(&sctx->bios_in_flight) == 0); in scrub_stripe()
3299 if (sctx->is_dev_replace && in scrub_stripe()
3300 btrfs_dev_is_sequential(sctx->wr_tgtdev, physical)) { in scrub_stripe()
3301 mutex_lock(&sctx->wr_lock); in scrub_stripe()
3302 sctx->write_pointer = physical; in scrub_stripe()
3303 mutex_unlock(&sctx->wr_lock); in scrub_stripe()
3304 sctx->flush_all_writes = true; in scrub_stripe()
3316 atomic_read(&sctx->cancel_req)) { in scrub_stripe()
3325 sctx->flush_all_writes = true; in scrub_stripe()
3326 scrub_submit(sctx); in scrub_stripe()
3327 mutex_lock(&sctx->wr_lock); in scrub_stripe()
3328 scrub_wr_submit(sctx); in scrub_stripe()
3329 mutex_unlock(&sctx->wr_lock); in scrub_stripe()
3330 wait_event(sctx->list_wait, in scrub_stripe()
3331 atomic_read(&sctx->bios_in_flight) == 0); in scrub_stripe()
3332 sctx->flush_all_writes = false; in scrub_stripe()
3345 ret = scrub_raid56_parity(sctx, map, scrub_dev, in scrub_stripe()
3443 spin_lock(&sctx->stat_lock); in scrub_stripe()
3444 sctx->stat.uncorrectable_errors++; in scrub_stripe()
3445 spin_unlock(&sctx->stat_lock); in scrub_stripe()
3470 if (sctx->is_dev_replace) in scrub_stripe()
3480 &sctx->csum_list, 1); in scrub_stripe()
3485 ret = scrub_extent(sctx, map, extent_logical, extent_len, in scrub_stripe()
3490 scrub_free_csums(sctx); in scrub_stripe()
3495 if (sctx->is_dev_replace) in scrub_stripe()
3496 sync_replace_for_zoned(sctx); in scrub_stripe()
3516 ret = scrub_raid56_parity(sctx, in scrub_stripe()
3545 spin_lock(&sctx->stat_lock); in scrub_stripe()
3547 sctx->stat.last_physical = map->stripes[num].physical + in scrub_stripe()
3550 sctx->stat.last_physical = physical; in scrub_stripe()
3551 spin_unlock(&sctx->stat_lock); in scrub_stripe()
3557 scrub_submit(sctx); in scrub_stripe()
3558 mutex_lock(&sctx->wr_lock); in scrub_stripe()
3559 scrub_wr_submit(sctx); in scrub_stripe()
3560 mutex_unlock(&sctx->wr_lock); in scrub_stripe()
3566 if (sctx->is_dev_replace && ret >= 0) { in scrub_stripe()
3569 ret2 = sync_write_pointer_for_zoned(sctx, base + offset, in scrub_stripe()
3579 static noinline_for_stack int scrub_chunk(struct scrub_ctx *sctx, in scrub_chunk() argument
3585 struct btrfs_fs_info *fs_info = sctx->fs_info; in scrub_chunk()
3619 ret = scrub_stripe(sctx, map, scrub_dev, i, in scrub_chunk()
3651 int scrub_enumerate_chunks(struct scrub_ctx *sctx, in scrub_enumerate_chunks() argument
3656 struct btrfs_fs_info *fs_info = sctx->fs_info; in scrub_enumerate_chunks()
3736 if (sctx->is_dev_replace && btrfs_is_zoned(fs_info)) { in scrub_enumerate_chunks()
3803 ret = btrfs_inc_block_group_ro(cache, sctx->is_dev_replace); in scrub_enumerate_chunks()
3804 if (!ret && sctx->is_dev_replace) { in scrub_enumerate_chunks()
3816 } else if (ret == -ENOSPC && !sctx->is_dev_replace) { in scrub_enumerate_chunks()
3846 if (sctx->is_dev_replace) { in scrub_enumerate_chunks()
3859 ret = scrub_chunk(sctx, scrub_dev, chunk_offset, length, in scrub_enumerate_chunks()
3872 sctx->flush_all_writes = true; in scrub_enumerate_chunks()
3873 scrub_submit(sctx); in scrub_enumerate_chunks()
3874 mutex_lock(&sctx->wr_lock); in scrub_enumerate_chunks()
3875 scrub_wr_submit(sctx); in scrub_enumerate_chunks()
3876 mutex_unlock(&sctx->wr_lock); in scrub_enumerate_chunks()
3878 wait_event(sctx->list_wait, in scrub_enumerate_chunks()
3879 atomic_read(&sctx->bios_in_flight) == 0); in scrub_enumerate_chunks()
3888 wait_event(sctx->list_wait, in scrub_enumerate_chunks()
3889 atomic_read(&sctx->workers_pending) == 0); in scrub_enumerate_chunks()
3890 sctx->flush_all_writes = false; in scrub_enumerate_chunks()
3894 if (sctx->is_dev_replace && in scrub_enumerate_chunks()
3931 if (sctx->is_dev_replace && in scrub_enumerate_chunks()
3936 if (sctx->stat.malloc_errors > 0) { in scrub_enumerate_chunks()
3950 static noinline_for_stack int scrub_supers(struct scrub_ctx *sctx, in scrub_supers() argument
3957 struct btrfs_fs_info *fs_info = sctx->fs_info; in scrub_supers()
3976 ret = scrub_pages(sctx, bytenr, BTRFS_SUPER_INFO_SIZE, bytenr, in scrub_supers()
3982 wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0); in scrub_supers()
4071 struct scrub_ctx *sctx; in btrfs_scrub_dev() local
4109 sctx = scrub_setup_ctx(fs_info, is_dev_replace); in btrfs_scrub_dev()
4110 if (IS_ERR(sctx)) in btrfs_scrub_dev()
4111 return PTR_ERR(sctx); in btrfs_scrub_dev()
4157 sctx->readonly = readonly; in btrfs_scrub_dev()
4158 dev->scrub_ctx = sctx; in btrfs_scrub_dev()
4186 ret = scrub_supers(sctx, dev); in btrfs_scrub_dev()
4191 ret = scrub_enumerate_chunks(sctx, dev, start, end); in btrfs_scrub_dev()
4194 wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0); in btrfs_scrub_dev()
4198 wait_event(sctx->list_wait, atomic_read(&sctx->workers_pending) == 0); in btrfs_scrub_dev()
4201 memcpy(progress, &sctx->stat, sizeof(*progress)); in btrfs_scrub_dev()
4212 scrub_put_ctx(sctx); in btrfs_scrub_dev()
4218 scrub_free_ctx(sctx); in btrfs_scrub_dev()
4268 struct scrub_ctx *sctx; in btrfs_scrub_cancel_dev() local
4271 sctx = dev->scrub_ctx; in btrfs_scrub_cancel_dev()
4272 if (!sctx) { in btrfs_scrub_cancel_dev()
4276 atomic_inc(&sctx->cancel_req); in btrfs_scrub_cancel_dev()
4292 struct scrub_ctx *sctx = NULL; in btrfs_scrub_progress() local
4297 sctx = dev->scrub_ctx; in btrfs_scrub_progress()
4298 if (sctx) in btrfs_scrub_progress()
4299 memcpy(progress, &sctx->stat, sizeof(*progress)); in btrfs_scrub_progress()
4302 return dev ? (sctx ? 0 : -ENOTCONN) : -ENODEV; in btrfs_scrub_progress()