Lines Matching refs:sctx
85 struct scrub_ctx *sctx; member
106 struct scrub_ctx *sctx; member
123 struct scrub_ctx *sctx; member
209 static void scrub_pending_bio_inc(struct scrub_ctx *sctx);
210 static void scrub_pending_bio_dec(struct scrub_ctx *sctx);
235 static int scrub_add_page_to_rd_bio(struct scrub_ctx *sctx,
237 static int scrub_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
249 static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx,
251 static void scrub_wr_submit(struct scrub_ctx *sctx);
256 static void scrub_put_ctx(struct scrub_ctx *sctx);
264 static void scrub_pending_bio_inc(struct scrub_ctx *sctx) in scrub_pending_bio_inc() argument
266 refcount_inc(&sctx->refs); in scrub_pending_bio_inc()
267 atomic_inc(&sctx->bios_in_flight); in scrub_pending_bio_inc()
270 static void scrub_pending_bio_dec(struct scrub_ctx *sctx) in scrub_pending_bio_dec() argument
272 atomic_dec(&sctx->bios_in_flight); in scrub_pending_bio_dec()
273 wake_up(&sctx->list_wait); in scrub_pending_bio_dec()
274 scrub_put_ctx(sctx); in scrub_pending_bio_dec()
527 static void scrub_free_csums(struct scrub_ctx *sctx) in scrub_free_csums() argument
529 while (!list_empty(&sctx->csum_list)) { in scrub_free_csums()
531 sum = list_first_entry(&sctx->csum_list, in scrub_free_csums()
538 static noinline_for_stack void scrub_free_ctx(struct scrub_ctx *sctx) in scrub_free_ctx() argument
542 if (!sctx) in scrub_free_ctx()
546 if (sctx->curr != -1) { in scrub_free_ctx()
547 struct scrub_bio *sbio = sctx->bios[sctx->curr]; in scrub_free_ctx()
557 struct scrub_bio *sbio = sctx->bios[i]; in scrub_free_ctx()
564 kfree(sctx->wr_curr_bio); in scrub_free_ctx()
565 scrub_free_csums(sctx); in scrub_free_ctx()
566 kfree(sctx); in scrub_free_ctx()
569 static void scrub_put_ctx(struct scrub_ctx *sctx) in scrub_put_ctx() argument
571 if (refcount_dec_and_test(&sctx->refs)) in scrub_put_ctx()
572 scrub_free_ctx(sctx); in scrub_put_ctx()
578 struct scrub_ctx *sctx; in scrub_setup_ctx() local
581 sctx = kzalloc(sizeof(*sctx), GFP_KERNEL); in scrub_setup_ctx()
582 if (!sctx) in scrub_setup_ctx()
584 refcount_set(&sctx->refs, 1); in scrub_setup_ctx()
585 sctx->is_dev_replace = is_dev_replace; in scrub_setup_ctx()
586 sctx->pages_per_rd_bio = SCRUB_PAGES_PER_RD_BIO; in scrub_setup_ctx()
587 sctx->curr = -1; in scrub_setup_ctx()
588 sctx->fs_info = fs_info; in scrub_setup_ctx()
589 INIT_LIST_HEAD(&sctx->csum_list); in scrub_setup_ctx()
596 sctx->bios[i] = sbio; in scrub_setup_ctx()
599 sbio->sctx = sctx; in scrub_setup_ctx()
605 sctx->bios[i]->next_free = i + 1; in scrub_setup_ctx()
607 sctx->bios[i]->next_free = -1; in scrub_setup_ctx()
609 sctx->first_free = 0; in scrub_setup_ctx()
610 atomic_set(&sctx->bios_in_flight, 0); in scrub_setup_ctx()
611 atomic_set(&sctx->workers_pending, 0); in scrub_setup_ctx()
612 atomic_set(&sctx->cancel_req, 0); in scrub_setup_ctx()
613 sctx->csum_size = btrfs_super_csum_size(fs_info->super_copy); in scrub_setup_ctx()
615 spin_lock_init(&sctx->list_lock); in scrub_setup_ctx()
616 spin_lock_init(&sctx->stat_lock); in scrub_setup_ctx()
617 init_waitqueue_head(&sctx->list_wait); in scrub_setup_ctx()
619 WARN_ON(sctx->wr_curr_bio != NULL); in scrub_setup_ctx()
620 mutex_init(&sctx->wr_lock); in scrub_setup_ctx()
621 sctx->wr_curr_bio = NULL; in scrub_setup_ctx()
624 sctx->pages_per_wr_bio = SCRUB_PAGES_PER_WR_BIO; in scrub_setup_ctx()
625 sctx->wr_tgtdev = fs_info->dev_replace.tgtdev; in scrub_setup_ctx()
626 sctx->flush_all_writes = false; in scrub_setup_ctx()
629 return sctx; in scrub_setup_ctx()
632 scrub_free_ctx(sctx); in scrub_setup_ctx()
748 fs_info = sblock->sctx->fs_info; in scrub_print_warning()
824 struct scrub_ctx *sctx = sblock_to_check->sctx; in scrub_handle_errored_block() local
843 fs_info = sctx->fs_info; in scrub_handle_errored_block()
850 spin_lock(&sctx->stat_lock); in scrub_handle_errored_block()
851 ++sctx->stat.super_errors; in scrub_handle_errored_block()
852 spin_unlock(&sctx->stat_lock); in scrub_handle_errored_block()
883 spin_lock(&sctx->stat_lock); in scrub_handle_errored_block()
885 sctx->stat.malloc_errors++; in scrub_handle_errored_block()
886 sctx->stat.read_errors++; in scrub_handle_errored_block()
887 sctx->stat.uncorrectable_errors++; in scrub_handle_errored_block()
888 spin_unlock(&sctx->stat_lock); in scrub_handle_errored_block()
924 spin_lock(&sctx->stat_lock); in scrub_handle_errored_block()
925 sctx->stat.malloc_errors++; in scrub_handle_errored_block()
926 sctx->stat.read_errors++; in scrub_handle_errored_block()
927 sctx->stat.uncorrectable_errors++; in scrub_handle_errored_block()
928 spin_unlock(&sctx->stat_lock); in scrub_handle_errored_block()
936 spin_lock(&sctx->stat_lock); in scrub_handle_errored_block()
937 sctx->stat.read_errors++; in scrub_handle_errored_block()
938 sctx->stat.uncorrectable_errors++; in scrub_handle_errored_block()
939 spin_unlock(&sctx->stat_lock); in scrub_handle_errored_block()
959 spin_lock(&sctx->stat_lock); in scrub_handle_errored_block()
960 sctx->stat.unverified_errors++; in scrub_handle_errored_block()
962 spin_unlock(&sctx->stat_lock); in scrub_handle_errored_block()
964 if (sctx->is_dev_replace) in scrub_handle_errored_block()
970 spin_lock(&sctx->stat_lock); in scrub_handle_errored_block()
971 sctx->stat.read_errors++; in scrub_handle_errored_block()
972 spin_unlock(&sctx->stat_lock); in scrub_handle_errored_block()
977 spin_lock(&sctx->stat_lock); in scrub_handle_errored_block()
978 sctx->stat.csum_errors++; in scrub_handle_errored_block()
979 spin_unlock(&sctx->stat_lock); in scrub_handle_errored_block()
985 spin_lock(&sctx->stat_lock); in scrub_handle_errored_block()
986 sctx->stat.verify_errors++; in scrub_handle_errored_block()
987 spin_unlock(&sctx->stat_lock); in scrub_handle_errored_block()
999 if (sctx->readonly) { in scrub_handle_errored_block()
1000 ASSERT(!sctx->is_dev_replace); in scrub_handle_errored_block()
1054 if (sctx->is_dev_replace) { in scrub_handle_errored_block()
1066 if (sblock_bad->no_io_error_seen && !sctx->is_dev_replace) in scrub_handle_errored_block()
1100 if (!page_bad->io_error && !sctx->is_dev_replace) in scrub_handle_errored_block()
1129 if (sctx->is_dev_replace) { in scrub_handle_errored_block()
1157 if (success && !sctx->is_dev_replace) { in scrub_handle_errored_block()
1177 spin_lock(&sctx->stat_lock); in scrub_handle_errored_block()
1178 sctx->stat.corrected_errors++; in scrub_handle_errored_block()
1180 spin_unlock(&sctx->stat_lock); in scrub_handle_errored_block()
1187 spin_lock(&sctx->stat_lock); in scrub_handle_errored_block()
1188 sctx->stat.uncorrectable_errors++; in scrub_handle_errored_block()
1189 spin_unlock(&sctx->stat_lock); in scrub_handle_errored_block()
1269 struct scrub_ctx *sctx = original_sblock->sctx; in scrub_setup_recheck_block() local
1270 struct btrfs_fs_info *fs_info = sctx->fs_info; in scrub_setup_recheck_block()
1332 sblock->sctx = sctx; in scrub_setup_recheck_block()
1337 spin_lock(&sctx->stat_lock); in scrub_setup_recheck_block()
1338 sctx->stat.malloc_errors++; in scrub_setup_recheck_block()
1339 spin_unlock(&sctx->stat_lock); in scrub_setup_recheck_block()
1353 sctx->csum_size); in scrub_setup_recheck_block()
1554 struct btrfs_fs_info *fs_info = sblock_bad->sctx->fs_info; in scrub_repair_page_from_good_copy()
1595 struct btrfs_fs_info *fs_info = sblock->sctx->fs_info; in scrub_write_block_to_dev_replace()
1627 return scrub_add_page_to_wr_bio(sblock->sctx, spage); in scrub_write_page_to_dev_replace()
1630 static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx, in scrub_add_page_to_wr_bio() argument
1636 mutex_lock(&sctx->wr_lock); in scrub_add_page_to_wr_bio()
1638 if (!sctx->wr_curr_bio) { in scrub_add_page_to_wr_bio()
1639 sctx->wr_curr_bio = kzalloc(sizeof(*sctx->wr_curr_bio), in scrub_add_page_to_wr_bio()
1641 if (!sctx->wr_curr_bio) { in scrub_add_page_to_wr_bio()
1642 mutex_unlock(&sctx->wr_lock); in scrub_add_page_to_wr_bio()
1645 sctx->wr_curr_bio->sctx = sctx; in scrub_add_page_to_wr_bio()
1646 sctx->wr_curr_bio->page_count = 0; in scrub_add_page_to_wr_bio()
1648 sbio = sctx->wr_curr_bio; in scrub_add_page_to_wr_bio()
1654 sbio->dev = sctx->wr_tgtdev; in scrub_add_page_to_wr_bio()
1657 bio = btrfs_io_bio_alloc(sctx->pages_per_wr_bio); in scrub_add_page_to_wr_bio()
1671 scrub_wr_submit(sctx); in scrub_add_page_to_wr_bio()
1680 mutex_unlock(&sctx->wr_lock); in scrub_add_page_to_wr_bio()
1683 scrub_wr_submit(sctx); in scrub_add_page_to_wr_bio()
1690 if (sbio->page_count == sctx->pages_per_wr_bio) in scrub_add_page_to_wr_bio()
1691 scrub_wr_submit(sctx); in scrub_add_page_to_wr_bio()
1692 mutex_unlock(&sctx->wr_lock); in scrub_add_page_to_wr_bio()
1697 static void scrub_wr_submit(struct scrub_ctx *sctx) in scrub_wr_submit() argument
1701 if (!sctx->wr_curr_bio) in scrub_wr_submit()
1704 sbio = sctx->wr_curr_bio; in scrub_wr_submit()
1705 sctx->wr_curr_bio = NULL; in scrub_wr_submit()
1707 scrub_pending_bio_inc(sctx); in scrub_wr_submit()
1731 struct scrub_ctx *sctx = sbio->sctx; in scrub_wr_bio_end_io_worker() local
1737 &sbio->sctx->fs_info->dev_replace; in scrub_wr_bio_end_io_worker()
1752 scrub_pending_bio_dec(sctx); in scrub_wr_bio_end_io_worker()
1791 struct scrub_ctx *sctx = sblock->sctx; in scrub_checksum_data() local
1792 struct btrfs_fs_info *fs_info = sctx->fs_info; in scrub_checksum_data()
1812 len = sctx->fs_info->sectorsize; in scrub_checksum_data()
1830 if (memcmp(csum, on_disk_csum, sctx->csum_size)) in scrub_checksum_data()
1838 struct scrub_ctx *sctx = sblock->sctx; in scrub_checksum_tree_block() local
1840 struct btrfs_fs_info *fs_info = sctx->fs_info; in scrub_checksum_tree_block()
1858 memcpy(on_disk_csum, h->csum, sctx->csum_size); in scrub_checksum_tree_block()
1880 len = sctx->fs_info->nodesize - BTRFS_CSUM_SIZE; in scrub_checksum_tree_block()
1902 if (memcmp(calculated_csum, on_disk_csum, sctx->csum_size)) in scrub_checksum_tree_block()
1911 struct scrub_ctx *sctx = sblock->sctx; in scrub_checksum_super() local
1912 struct btrfs_fs_info *fs_info = sctx->fs_info; in scrub_checksum_super()
1932 memcpy(on_disk_csum, s->csum, sctx->csum_size); in scrub_checksum_super()
1965 if (memcmp(calculated_csum, on_disk_csum, sctx->csum_size)) in scrub_checksum_super()
1974 spin_lock(&sctx->stat_lock); in scrub_checksum_super()
1975 ++sctx->stat.super_errors; in scrub_checksum_super()
1976 spin_unlock(&sctx->stat_lock); in scrub_checksum_super()
2021 static void scrub_submit(struct scrub_ctx *sctx) in scrub_submit() argument
2025 if (sctx->curr == -1) in scrub_submit()
2028 sbio = sctx->bios[sctx->curr]; in scrub_submit()
2029 sctx->curr = -1; in scrub_submit()
2030 scrub_pending_bio_inc(sctx); in scrub_submit()
2034 static int scrub_add_page_to_rd_bio(struct scrub_ctx *sctx, in scrub_add_page_to_rd_bio() argument
2045 while (sctx->curr == -1) { in scrub_add_page_to_rd_bio()
2046 spin_lock(&sctx->list_lock); in scrub_add_page_to_rd_bio()
2047 sctx->curr = sctx->first_free; in scrub_add_page_to_rd_bio()
2048 if (sctx->curr != -1) { in scrub_add_page_to_rd_bio()
2049 sctx->first_free = sctx->bios[sctx->curr]->next_free; in scrub_add_page_to_rd_bio()
2050 sctx->bios[sctx->curr]->next_free = -1; in scrub_add_page_to_rd_bio()
2051 sctx->bios[sctx->curr]->page_count = 0; in scrub_add_page_to_rd_bio()
2052 spin_unlock(&sctx->list_lock); in scrub_add_page_to_rd_bio()
2054 spin_unlock(&sctx->list_lock); in scrub_add_page_to_rd_bio()
2055 wait_event(sctx->list_wait, sctx->first_free != -1); in scrub_add_page_to_rd_bio()
2058 sbio = sctx->bios[sctx->curr]; in scrub_add_page_to_rd_bio()
2067 bio = btrfs_io_bio_alloc(sctx->pages_per_rd_bio); in scrub_add_page_to_rd_bio()
2082 scrub_submit(sctx); in scrub_add_page_to_rd_bio()
2094 scrub_submit(sctx); in scrub_add_page_to_rd_bio()
2101 if (sbio->page_count == sctx->pages_per_rd_bio) in scrub_add_page_to_rd_bio()
2102 scrub_submit(sctx); in scrub_add_page_to_rd_bio()
2110 struct btrfs_fs_info *fs_info = sblock->sctx->fs_info; in scrub_missing_raid56_end_io()
2123 struct scrub_ctx *sctx = sblock->sctx; in scrub_missing_raid56_worker() local
2124 struct btrfs_fs_info *fs_info = sctx->fs_info; in scrub_missing_raid56_worker()
2135 spin_lock(&sctx->stat_lock); in scrub_missing_raid56_worker()
2136 sctx->stat.read_errors++; in scrub_missing_raid56_worker()
2137 spin_unlock(&sctx->stat_lock); in scrub_missing_raid56_worker()
2142 spin_lock(&sctx->stat_lock); in scrub_missing_raid56_worker()
2143 sctx->stat.uncorrectable_errors++; in scrub_missing_raid56_worker()
2144 spin_unlock(&sctx->stat_lock); in scrub_missing_raid56_worker()
2154 if (sctx->is_dev_replace && sctx->flush_all_writes) { in scrub_missing_raid56_worker()
2155 mutex_lock(&sctx->wr_lock); in scrub_missing_raid56_worker()
2156 scrub_wr_submit(sctx); in scrub_missing_raid56_worker()
2157 mutex_unlock(&sctx->wr_lock); in scrub_missing_raid56_worker()
2160 scrub_pending_bio_dec(sctx); in scrub_missing_raid56_worker()
2165 struct scrub_ctx *sctx = sblock->sctx; in scrub_missing_raid56_pages() local
2166 struct btrfs_fs_info *fs_info = sctx->fs_info; in scrub_missing_raid56_pages()
2181 if (WARN_ON(!sctx->is_dev_replace || in scrub_missing_raid56_pages()
2210 scrub_pending_bio_inc(sctx); in scrub_missing_raid56_pages()
2219 spin_lock(&sctx->stat_lock); in scrub_missing_raid56_pages()
2220 sctx->stat.malloc_errors++; in scrub_missing_raid56_pages()
2221 spin_unlock(&sctx->stat_lock); in scrub_missing_raid56_pages()
2224 static int scrub_pages(struct scrub_ctx *sctx, u64 logical, u64 len, in scrub_pages() argument
2234 spin_lock(&sctx->stat_lock); in scrub_pages()
2235 sctx->stat.malloc_errors++; in scrub_pages()
2236 spin_unlock(&sctx->stat_lock); in scrub_pages()
2243 sblock->sctx = sctx; in scrub_pages()
2253 spin_lock(&sctx->stat_lock); in scrub_pages()
2254 sctx->stat.malloc_errors++; in scrub_pages()
2255 spin_unlock(&sctx->stat_lock); in scrub_pages()
2272 memcpy(spage->csum, csum, sctx->csum_size); in scrub_pages()
2298 ret = scrub_add_page_to_rd_bio(sctx, spage); in scrub_pages()
2306 scrub_submit(sctx); in scrub_pages()
2328 struct scrub_ctx *sctx = sbio->sctx; in scrub_bio_end_io_worker() local
2353 spin_lock(&sctx->list_lock); in scrub_bio_end_io_worker()
2354 sbio->next_free = sctx->first_free; in scrub_bio_end_io_worker()
2355 sctx->first_free = sbio->index; in scrub_bio_end_io_worker()
2356 spin_unlock(&sctx->list_lock); in scrub_bio_end_io_worker()
2358 if (sctx->is_dev_replace && sctx->flush_all_writes) { in scrub_bio_end_io_worker()
2359 mutex_lock(&sctx->wr_lock); in scrub_bio_end_io_worker()
2360 scrub_wr_submit(sctx); in scrub_bio_end_io_worker()
2361 mutex_unlock(&sctx->wr_lock); in scrub_bio_end_io_worker()
2364 scrub_pending_bio_dec(sctx); in scrub_bio_end_io_worker()
2374 int sectorsize = sparity->sctx->fs_info->sectorsize; in __scrub_mark_bitmap()
2424 if (!corrupted && sblock->sctx->is_dev_replace) in scrub_block_complete()
2438 static int scrub_find_csum(struct scrub_ctx *sctx, u64 logical, u8 *csum) in scrub_find_csum() argument
2444 while (!list_empty(&sctx->csum_list)) { in scrub_find_csum()
2445 sum = list_first_entry(&sctx->csum_list, in scrub_find_csum()
2452 ++sctx->stat.csum_discards; in scrub_find_csum()
2460 index = div_u64(logical - sum->bytenr, sctx->fs_info->sectorsize); in scrub_find_csum()
2463 num_sectors = sum->len / sctx->fs_info->sectorsize; in scrub_find_csum()
2464 memcpy(csum, sum->sums + index * sctx->csum_size, sctx->csum_size); in scrub_find_csum()
2473 static int scrub_extent(struct scrub_ctx *sctx, struct map_lookup *map, in scrub_extent() argument
2486 blocksize = sctx->fs_info->sectorsize; in scrub_extent()
2487 spin_lock(&sctx->stat_lock); in scrub_extent()
2488 sctx->stat.data_extents_scrubbed++; in scrub_extent()
2489 sctx->stat.data_bytes_scrubbed += len; in scrub_extent()
2490 spin_unlock(&sctx->stat_lock); in scrub_extent()
2495 blocksize = sctx->fs_info->nodesize; in scrub_extent()
2496 spin_lock(&sctx->stat_lock); in scrub_extent()
2497 sctx->stat.tree_extents_scrubbed++; in scrub_extent()
2498 sctx->stat.tree_bytes_scrubbed += len; in scrub_extent()
2499 spin_unlock(&sctx->stat_lock); in scrub_extent()
2501 blocksize = sctx->fs_info->sectorsize; in scrub_extent()
2511 have_csum = scrub_find_csum(sctx, logical, csum); in scrub_extent()
2513 ++sctx->stat.no_csum; in scrub_extent()
2515 ret = scrub_pages(sctx, logical, l, physical, dev, flags, gen, in scrub_extent()
2533 struct scrub_ctx *sctx = sparity->sctx; in scrub_pages_for_parity() local
2539 spin_lock(&sctx->stat_lock); in scrub_pages_for_parity()
2540 sctx->stat.malloc_errors++; in scrub_pages_for_parity()
2541 spin_unlock(&sctx->stat_lock); in scrub_pages_for_parity()
2548 sblock->sctx = sctx; in scrub_pages_for_parity()
2560 spin_lock(&sctx->stat_lock); in scrub_pages_for_parity()
2561 sctx->stat.malloc_errors++; in scrub_pages_for_parity()
2562 spin_unlock(&sctx->stat_lock); in scrub_pages_for_parity()
2582 memcpy(spage->csum, csum, sctx->csum_size); in scrub_pages_for_parity()
2600 ret = scrub_add_page_to_rd_bio(sctx, spage); in scrub_pages_for_parity()
2617 struct scrub_ctx *sctx = sparity->sctx; in scrub_extent_for_parity() local
2632 blocksize = sctx->fs_info->sectorsize; in scrub_extent_for_parity()
2642 have_csum = scrub_find_csum(sctx, logical, csum); in scrub_extent_for_parity()
2705 struct scrub_ctx *sctx = sparity->sctx; in scrub_free_parity() local
2711 spin_lock(&sctx->stat_lock); in scrub_free_parity()
2712 sctx->stat.read_errors += nbits; in scrub_free_parity()
2713 sctx->stat.uncorrectable_errors += nbits; in scrub_free_parity()
2714 spin_unlock(&sctx->stat_lock); in scrub_free_parity()
2729 struct scrub_ctx *sctx = sparity->sctx; in scrub_parity_bio_endio_worker() local
2732 scrub_pending_bio_dec(sctx); in scrub_parity_bio_endio_worker()
2738 struct btrfs_fs_info *fs_info = sparity->sctx->fs_info; in scrub_parity_bio_endio()
2753 struct scrub_ctx *sctx = sparity->sctx; in scrub_parity_check_and_repair() local
2754 struct btrfs_fs_info *fs_info = sctx->fs_info; in scrub_parity_check_and_repair()
2785 scrub_pending_bio_inc(sctx); in scrub_parity_check_and_repair()
2796 spin_lock(&sctx->stat_lock); in scrub_parity_check_and_repair()
2797 sctx->stat.malloc_errors++; in scrub_parity_check_and_repair()
2798 spin_unlock(&sctx->stat_lock); in scrub_parity_check_and_repair()
2821 static noinline_for_stack int scrub_raid56_parity(struct scrub_ctx *sctx, in scrub_raid56_parity() argument
2828 struct btrfs_fs_info *fs_info = sctx->fs_info; in scrub_raid56_parity()
2855 spin_lock(&sctx->stat_lock); in scrub_raid56_parity()
2856 sctx->stat.malloc_errors++; in scrub_raid56_parity()
2857 spin_unlock(&sctx->stat_lock); in scrub_raid56_parity()
2863 sparity->sctx = sctx; in scrub_raid56_parity()
2948 spin_lock(&sctx->stat_lock); in scrub_raid56_parity()
2949 sctx->stat.uncorrectable_errors++; in scrub_raid56_parity()
2950 spin_unlock(&sctx->stat_lock); in scrub_raid56_parity()
2991 &sctx->csum_list, 1); in scrub_raid56_parity()
3002 scrub_free_csums(sctx); in scrub_raid56_parity()
3037 scrub_submit(sctx); in scrub_raid56_parity()
3038 mutex_lock(&sctx->wr_lock); in scrub_raid56_parity()
3039 scrub_wr_submit(sctx); in scrub_raid56_parity()
3040 mutex_unlock(&sctx->wr_lock); in scrub_raid56_parity()
3046 static noinline_for_stack int scrub_stripe(struct scrub_ctx *sctx, in scrub_stripe() argument
3052 struct btrfs_fs_info *fs_info = sctx->fs_info; in scrub_stripe()
3144 wait_event(sctx->list_wait, in scrub_stripe()
3145 atomic_read(&sctx->bios_in_flight) == 0); in scrub_stripe()
3186 atomic_read(&sctx->cancel_req)) { in scrub_stripe()
3195 sctx->flush_all_writes = true; in scrub_stripe()
3196 scrub_submit(sctx); in scrub_stripe()
3197 mutex_lock(&sctx->wr_lock); in scrub_stripe()
3198 scrub_wr_submit(sctx); in scrub_stripe()
3199 mutex_unlock(&sctx->wr_lock); in scrub_stripe()
3200 wait_event(sctx->list_wait, in scrub_stripe()
3201 atomic_read(&sctx->bios_in_flight) == 0); in scrub_stripe()
3202 sctx->flush_all_writes = false; in scrub_stripe()
3215 ret = scrub_raid56_parity(sctx, map, scrub_dev, in scrub_stripe()
3299 spin_lock(&sctx->stat_lock); in scrub_stripe()
3300 sctx->stat.uncorrectable_errors++; in scrub_stripe()
3301 spin_unlock(&sctx->stat_lock); in scrub_stripe()
3325 if (sctx->is_dev_replace) in scrub_stripe()
3335 &sctx->csum_list, 1); in scrub_stripe()
3339 ret = scrub_extent(sctx, map, extent_logical, extent_len, in scrub_stripe()
3344 scrub_free_csums(sctx); in scrub_stripe()
3367 ret = scrub_raid56_parity(sctx, in scrub_stripe()
3396 spin_lock(&sctx->stat_lock); in scrub_stripe()
3398 sctx->stat.last_physical = map->stripes[num].physical + in scrub_stripe()
3401 sctx->stat.last_physical = physical; in scrub_stripe()
3402 spin_unlock(&sctx->stat_lock); in scrub_stripe()
3408 scrub_submit(sctx); in scrub_stripe()
3409 mutex_lock(&sctx->wr_lock); in scrub_stripe()
3410 scrub_wr_submit(sctx); in scrub_stripe()
3411 mutex_unlock(&sctx->wr_lock); in scrub_stripe()
3419 static noinline_for_stack int scrub_chunk(struct scrub_ctx *sctx, in scrub_chunk() argument
3425 struct btrfs_fs_info *fs_info = sctx->fs_info; in scrub_chunk()
3459 ret = scrub_stripe(sctx, map, scrub_dev, i, in scrub_chunk()
3472 int scrub_enumerate_chunks(struct scrub_ctx *sctx, in scrub_enumerate_chunks() argument
3477 struct btrfs_fs_info *fs_info = sctx->fs_info; in scrub_enumerate_chunks()
3567 if (!ret && sctx->is_dev_replace) { in scrub_enumerate_chunks()
3632 ret = scrub_chunk(sctx, scrub_dev, chunk_offset, length, in scrub_enumerate_chunks()
3645 sctx->flush_all_writes = true; in scrub_enumerate_chunks()
3646 scrub_submit(sctx); in scrub_enumerate_chunks()
3647 mutex_lock(&sctx->wr_lock); in scrub_enumerate_chunks()
3648 scrub_wr_submit(sctx); in scrub_enumerate_chunks()
3649 mutex_unlock(&sctx->wr_lock); in scrub_enumerate_chunks()
3651 wait_event(sctx->list_wait, in scrub_enumerate_chunks()
3652 atomic_read(&sctx->bios_in_flight) == 0); in scrub_enumerate_chunks()
3661 wait_event(sctx->list_wait, in scrub_enumerate_chunks()
3662 atomic_read(&sctx->workers_pending) == 0); in scrub_enumerate_chunks()
3663 sctx->flush_all_writes = false; in scrub_enumerate_chunks()
3694 if (sctx->is_dev_replace && in scrub_enumerate_chunks()
3699 if (sctx->stat.malloc_errors > 0) { in scrub_enumerate_chunks()
3713 static noinline_for_stack int scrub_supers(struct scrub_ctx *sctx, in scrub_supers() argument
3720 struct btrfs_fs_info *fs_info = sctx->fs_info; in scrub_supers()
3737 ret = scrub_pages(sctx, bytenr, BTRFS_SUPER_INFO_SIZE, bytenr, in scrub_supers()
3743 wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0); in scrub_supers()
3798 struct scrub_ctx *sctx; in btrfs_scrub_dev() local
3847 sctx = scrub_setup_ctx(fs_info, is_dev_replace); in btrfs_scrub_dev()
3848 if (IS_ERR(sctx)) in btrfs_scrub_dev()
3849 return PTR_ERR(sctx); in btrfs_scrub_dev()
3897 sctx->readonly = readonly; in btrfs_scrub_dev()
3898 dev->scrub_ctx = sctx; in btrfs_scrub_dev()
3926 ret = scrub_supers(sctx, dev); in btrfs_scrub_dev()
3931 ret = scrub_enumerate_chunks(sctx, dev, start, end); in btrfs_scrub_dev()
3934 wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0); in btrfs_scrub_dev()
3938 wait_event(sctx->list_wait, atomic_read(&sctx->workers_pending) == 0); in btrfs_scrub_dev()
3941 memcpy(progress, &sctx->stat, sizeof(*progress)); in btrfs_scrub_dev()
3963 scrub_put_ctx(sctx); in btrfs_scrub_dev()
3968 scrub_free_ctx(sctx); in btrfs_scrub_dev()
4018 struct scrub_ctx *sctx; in btrfs_scrub_cancel_dev() local
4021 sctx = dev->scrub_ctx; in btrfs_scrub_cancel_dev()
4022 if (!sctx) { in btrfs_scrub_cancel_dev()
4026 atomic_inc(&sctx->cancel_req); in btrfs_scrub_cancel_dev()
4042 struct scrub_ctx *sctx = NULL; in btrfs_scrub_progress() local
4047 sctx = dev->scrub_ctx; in btrfs_scrub_progress()
4048 if (sctx) in btrfs_scrub_progress()
4049 memcpy(progress, &sctx->stat, sizeof(*progress)); in btrfs_scrub_progress()
4052 return dev ? (sctx ? 0 : -ENOTCONN) : -ENODEV; in btrfs_scrub_progress()