Lines Matching refs:sblock
66 struct scrub_block *sblock; member
261 struct scrub_block *sblock; in alloc_scrub_block() local
263 sblock = kzalloc(sizeof(*sblock), GFP_KERNEL); in alloc_scrub_block()
264 if (!sblock) in alloc_scrub_block()
266 refcount_set(&sblock->refs, 1); in alloc_scrub_block()
267 sblock->sctx = sctx; in alloc_scrub_block()
268 sblock->logical = logical; in alloc_scrub_block()
269 sblock->physical = physical; in alloc_scrub_block()
270 sblock->physical_for_dev_replace = physical_for_dev_replace; in alloc_scrub_block()
271 sblock->dev = dev; in alloc_scrub_block()
272 sblock->mirror_num = mirror_num; in alloc_scrub_block()
273 sblock->no_io_error_seen = 1; in alloc_scrub_block()
278 return sblock; in alloc_scrub_block()
286 static struct scrub_sector *alloc_scrub_sector(struct scrub_block *sblock, in alloc_scrub_sector() argument
289 const pgoff_t page_index = (logical - sblock->logical) >> PAGE_SHIFT; in alloc_scrub_sector()
293 ASSERT(logical - sblock->logical < U32_MAX); in alloc_scrub_sector()
300 if (!sblock->pages[page_index]) { in alloc_scrub_sector()
303 sblock->pages[page_index] = alloc_page(gfp); in alloc_scrub_sector()
304 if (!sblock->pages[page_index]) { in alloc_scrub_sector()
308 ret = attach_scrub_page_private(sblock->pages[page_index], in alloc_scrub_sector()
309 sblock->logical + (page_index << PAGE_SHIFT)); in alloc_scrub_sector()
312 __free_page(sblock->pages[page_index]); in alloc_scrub_sector()
313 sblock->pages[page_index] = NULL; in alloc_scrub_sector()
319 ssector->sblock = sblock; in alloc_scrub_sector()
321 ASSERT(sblock->sectors[sblock->sector_count] == NULL); in alloc_scrub_sector()
322 ssector->offset = logical - sblock->logical; in alloc_scrub_sector()
325 ASSERT(sblock->sector_count < SCRUB_MAX_SECTORS_PER_BLOCK); in alloc_scrub_sector()
327 sblock->sectors[sblock->sector_count] = ssector; in alloc_scrub_sector()
328 sblock->sector_count++; in alloc_scrub_sector()
329 sblock->len += sblock->sctx->fs_info->sectorsize; in alloc_scrub_sector()
336 struct scrub_block *sblock = ssector->sblock; in scrub_sector_get_page() local
342 ASSERT(sblock); in scrub_sector_get_page()
345 ASSERT(ssector->offset < sblock->len); in scrub_sector_get_page()
349 ASSERT(sblock->pages[index]); in scrub_sector_get_page()
350 ASSERT(PagePrivate(sblock->pages[index])); in scrub_sector_get_page()
351 return sblock->pages[index]; in scrub_sector_get_page()
356 struct scrub_block *sblock = ssector->sblock; in scrub_sector_get_page_offset() local
362 ASSERT(sblock); in scrub_sector_get_page_offset()
365 ASSERT(ssector->offset < sblock->len); in scrub_sector_get_page_offset()
386 struct scrub_block *sblock,
388 static void scrub_recheck_block_checksum(struct scrub_block *sblock);
394 static void scrub_write_block_to_dev_replace(struct scrub_block *sblock);
395 static int scrub_write_sector_to_dev_replace(struct scrub_block *sblock,
397 static int scrub_checksum_data(struct scrub_block *sblock);
398 static int scrub_checksum_tree_block(struct scrub_block *sblock);
399 static int scrub_checksum_super(struct scrub_block *sblock);
400 static void scrub_block_put(struct scrub_block *sblock);
411 static void scrub_block_complete(struct scrub_block *sblock);
715 scrub_block_put(sbio->sectors[i]->sblock); in scrub_free_ctx()
887 static void scrub_print_warning(const char *errstr, struct scrub_block *sblock) in scrub_print_warning() argument
904 WARN_ON(sblock->sector_count < 1); in scrub_print_warning()
905 dev = sblock->dev; in scrub_print_warning()
906 fs_info = sblock->sctx->fs_info; in scrub_print_warning()
909 if (sblock->sectors[0]->flags & BTRFS_EXTENT_FLAG_SUPER) { in scrub_print_warning()
912 sblock->physical); in scrub_print_warning()
919 swarn.physical = sblock->physical; in scrub_print_warning()
920 swarn.logical = sblock->logical; in scrub_print_warning()
1375 struct scrub_block *sblock = sblocks_for_recheck[mirror_index]; in scrub_handle_errored_block() local
1380 if (!sblock) in scrub_handle_errored_block()
1383 for (sector_index = 0; sector_index < sblock->sector_count; in scrub_handle_errored_block()
1389 recover = sblock->sectors[sector_index]->recover; in scrub_handle_errored_block()
1392 sblock->sectors[sector_index]->recover = NULL; in scrub_handle_errored_block()
1395 scrub_block_put(sblock); in scrub_handle_errored_block()
1500 struct scrub_block *sblock; in scrub_setup_recheck_block() local
1503 sblock = sblocks_for_recheck[mirror_index]; in scrub_setup_recheck_block()
1504 sblock->sctx = sctx; in scrub_setup_recheck_block()
1506 sector = alloc_scrub_sector(sblock, logical, GFP_NOFS); in scrub_setup_recheck_block()
1535 sblock->physical = in scrub_setup_recheck_block()
1538 sblock->dev = bioc->stripes[stripe_index].dev; in scrub_setup_recheck_block()
1539 sblock->physical_for_dev_replace = in scrub_setup_recheck_block()
1567 bio->bi_iter.bi_sector = (sector->offset + sector->sblock->logical) >> in scrub_submit_raid56_bio_wait()
1571 raid56_parity_recover(bio, sector->recover->bioc, sector->sblock->mirror_num); in scrub_submit_raid56_bio_wait()
1578 struct scrub_block *sblock) in scrub_recheck_block_on_raid56() argument
1580 struct scrub_sector *first_sector = sblock->sectors[0]; in scrub_recheck_block_on_raid56()
1585 ASSERT(sblock->dev); in scrub_recheck_block_on_raid56()
1586 if (!sblock->dev->bdev) in scrub_recheck_block_on_raid56()
1589 bio = bio_alloc(sblock->dev->bdev, BIO_MAX_VECS, REQ_OP_READ, GFP_NOFS); in scrub_recheck_block_on_raid56()
1591 for (i = 0; i < sblock->sector_count; i++) { in scrub_recheck_block_on_raid56()
1592 struct scrub_sector *sector = sblock->sectors[i]; in scrub_recheck_block_on_raid56()
1604 scrub_recheck_block_checksum(sblock); in scrub_recheck_block_on_raid56()
1608 for (i = 0; i < sblock->sector_count; i++) in scrub_recheck_block_on_raid56()
1609 sblock->sectors[i]->io_error = 1; in scrub_recheck_block_on_raid56()
1611 sblock->no_io_error_seen = 0; in scrub_recheck_block_on_raid56()
1622 struct scrub_block *sblock, in scrub_recheck_block() argument
1627 sblock->no_io_error_seen = 1; in scrub_recheck_block()
1630 if (!retry_failed_mirror && scrub_is_page_on_raid56(sblock->sectors[0])) in scrub_recheck_block()
1631 return scrub_recheck_block_on_raid56(fs_info, sblock); in scrub_recheck_block()
1633 for (i = 0; i < sblock->sector_count; i++) { in scrub_recheck_block()
1634 struct scrub_sector *sector = sblock->sectors[i]; in scrub_recheck_block()
1638 if (sblock->dev->bdev == NULL) { in scrub_recheck_block()
1640 sblock->no_io_error_seen = 0; in scrub_recheck_block()
1644 bio_init(&bio, sblock->dev->bdev, &bvec, 1, REQ_OP_READ); in scrub_recheck_block()
1646 bio.bi_iter.bi_sector = (sblock->physical + sector->offset) >> in scrub_recheck_block()
1652 sblock->no_io_error_seen = 0; in scrub_recheck_block()
1658 if (sblock->no_io_error_seen) in scrub_recheck_block()
1659 scrub_recheck_block_checksum(sblock); in scrub_recheck_block()
1664 struct btrfs_fs_devices *fs_devices = sector->sblock->dev->fs_devices; in scrub_check_fsid()
1671 static void scrub_recheck_block_checksum(struct scrub_block *sblock) in scrub_recheck_block_checksum() argument
1673 sblock->header_error = 0; in scrub_recheck_block_checksum()
1674 sblock->checksum_error = 0; in scrub_recheck_block_checksum()
1675 sblock->generation_error = 0; in scrub_recheck_block_checksum()
1677 if (sblock->sectors[0]->flags & BTRFS_EXTENT_FLAG_DATA) in scrub_recheck_block_checksum()
1678 scrub_checksum_data(sblock); in scrub_recheck_block_checksum()
1680 scrub_checksum_tree_block(sblock); in scrub_recheck_block_checksum()
1742 static void scrub_write_block_to_dev_replace(struct scrub_block *sblock) in scrub_write_block_to_dev_replace() argument
1744 struct btrfs_fs_info *fs_info = sblock->sctx->fs_info; in scrub_write_block_to_dev_replace()
1751 if (sblock->sparity) in scrub_write_block_to_dev_replace()
1754 for (i = 0; i < sblock->sector_count; i++) { in scrub_write_block_to_dev_replace()
1757 ret = scrub_write_sector_to_dev_replace(sblock, i); in scrub_write_block_to_dev_replace()
1763 static int scrub_write_sector_to_dev_replace(struct scrub_block *sblock, int sector_num) in scrub_write_sector_to_dev_replace() argument
1765 const u32 sectorsize = sblock->sctx->fs_info->sectorsize; in scrub_write_sector_to_dev_replace()
1766 struct scrub_sector *sector = sblock->sectors[sector_num]; in scrub_write_sector_to_dev_replace()
1771 return scrub_add_sector_to_wr_bio(sblock->sctx, sector); in scrub_write_sector_to_dev_replace()
1796 static void scrub_block_get(struct scrub_block *sblock) in scrub_block_get() argument
1798 refcount_inc(&sblock->refs); in scrub_block_get()
1804 struct scrub_block *sblock = sector->sblock; in scrub_add_sector_to_wr_bio() local
1824 sblock->physical_for_dev_replace); in scrub_add_sector_to_wr_bio()
1830 sbio->physical = sblock->physical_for_dev_replace + sector->offset; in scrub_add_sector_to_wr_bio()
1831 sbio->logical = sblock->logical + sector->offset; in scrub_add_sector_to_wr_bio()
1842 sblock->physical_for_dev_replace + sector->offset || in scrub_add_sector_to_wr_bio()
1844 sblock->logical + sector->offset) { in scrub_add_sector_to_wr_bio()
1868 scrub_block_get(sector->sblock); in scrub_add_sector_to_wr_bio()
1936 scrub_block_put(sbio->sectors[i]->sblock); in scrub_wr_bio_end_io_worker()
1945 static int scrub_checksum(struct scrub_block *sblock) in scrub_checksum() argument
1958 sblock->header_error = 0; in scrub_checksum()
1959 sblock->generation_error = 0; in scrub_checksum()
1960 sblock->checksum_error = 0; in scrub_checksum()
1962 WARN_ON(sblock->sector_count < 1); in scrub_checksum()
1963 flags = sblock->sectors[0]->flags; in scrub_checksum()
1966 ret = scrub_checksum_data(sblock); in scrub_checksum()
1968 ret = scrub_checksum_tree_block(sblock); in scrub_checksum()
1970 ret = scrub_checksum_super(sblock); in scrub_checksum()
1974 scrub_handle_errored_block(sblock); in scrub_checksum()
1979 static int scrub_checksum_data(struct scrub_block *sblock) in scrub_checksum_data() argument
1981 struct scrub_ctx *sctx = sblock->sctx; in scrub_checksum_data()
1988 BUG_ON(sblock->sector_count < 1); in scrub_checksum_data()
1989 sector = sblock->sectors[0]; in scrub_checksum_data()
2001 sblock->checksum_error = 1; in scrub_checksum_data()
2002 return sblock->checksum_error; in scrub_checksum_data()
2005 static int scrub_checksum_tree_block(struct scrub_block *sblock) in scrub_checksum_tree_block() argument
2007 struct scrub_ctx *sctx = sblock->sctx; in scrub_checksum_tree_block()
2024 BUG_ON(sblock->sector_count < 1); in scrub_checksum_tree_block()
2027 ASSERT(sblock->sector_count == num_sectors); in scrub_checksum_tree_block()
2029 sector = sblock->sectors[0]; in scrub_checksum_tree_block()
2039 if (sblock->logical != btrfs_stack_header_bytenr(h)) in scrub_checksum_tree_block()
2040 sblock->header_error = 1; in scrub_checksum_tree_block()
2043 sblock->header_error = 1; in scrub_checksum_tree_block()
2044 sblock->generation_error = 1; in scrub_checksum_tree_block()
2048 sblock->header_error = 1; in scrub_checksum_tree_block()
2052 sblock->header_error = 1; in scrub_checksum_tree_block()
2060 kaddr = scrub_sector_get_kaddr(sblock->sectors[i]); in scrub_checksum_tree_block()
2066 sblock->checksum_error = 1; in scrub_checksum_tree_block()
2068 return sblock->header_error || sblock->checksum_error; in scrub_checksum_tree_block()
2071 static int scrub_checksum_super(struct scrub_block *sblock) in scrub_checksum_super() argument
2074 struct scrub_ctx *sctx = sblock->sctx; in scrub_checksum_super()
2083 BUG_ON(sblock->sector_count < 1); in scrub_checksum_super()
2084 sector = sblock->sectors[0]; in scrub_checksum_super()
2088 if (sblock->logical != btrfs_super_bytenr(s)) in scrub_checksum_super()
2108 static void scrub_block_put(struct scrub_block *sblock) in scrub_block_put() argument
2110 if (refcount_dec_and_test(&sblock->refs)) { in scrub_block_put()
2113 if (sblock->sparity) in scrub_block_put()
2114 scrub_parity_put(sblock->sparity); in scrub_block_put()
2116 for (i = 0; i < sblock->sector_count; i++) in scrub_block_put()
2117 scrub_sector_put(sblock->sectors[i]); in scrub_block_put()
2118 for (i = 0; i < DIV_ROUND_UP(sblock->len, PAGE_SIZE); i++) { in scrub_block_put()
2119 if (sblock->pages[i]) { in scrub_block_put()
2120 detach_scrub_page_private(sblock->pages[i]); in scrub_block_put()
2121 __free_page(sblock->pages[i]); in scrub_block_put()
2124 kfree(sblock); in scrub_block_put()
2217 struct scrub_block *sblock = sector->sblock; in scrub_add_sector_to_rd_bio() local
2241 sbio->physical = sblock->physical + sector->offset; in scrub_add_sector_to_rd_bio()
2242 sbio->logical = sblock->logical + sector->offset; in scrub_add_sector_to_rd_bio()
2243 sbio->dev = sblock->dev; in scrub_add_sector_to_rd_bio()
2253 sblock->physical + sector->offset || in scrub_add_sector_to_rd_bio()
2255 sblock->logical + sector->offset || in scrub_add_sector_to_rd_bio()
2256 sbio->dev != sblock->dev) { in scrub_add_sector_to_rd_bio()
2273 scrub_block_get(sblock); /* one for the page added to the bio */ in scrub_add_sector_to_rd_bio()
2274 atomic_inc(&sblock->outstanding_sectors); in scrub_add_sector_to_rd_bio()
2284 struct scrub_block *sblock = bio->bi_private; in scrub_missing_raid56_end_io() local
2285 struct btrfs_fs_info *fs_info = sblock->sctx->fs_info; in scrub_missing_raid56_end_io()
2289 sblock->no_io_error_seen = 0; in scrub_missing_raid56_end_io()
2293 queue_work(fs_info->scrub_workers, &sblock->work); in scrub_missing_raid56_end_io()
2298 struct scrub_block *sblock = container_of(work, struct scrub_block, work); in scrub_missing_raid56_worker() local
2299 struct scrub_ctx *sctx = sblock->sctx; in scrub_missing_raid56_worker()
2304 logical = sblock->logical; in scrub_missing_raid56_worker()
2305 dev = sblock->dev; in scrub_missing_raid56_worker()
2307 if (sblock->no_io_error_seen) in scrub_missing_raid56_worker()
2308 scrub_recheck_block_checksum(sblock); in scrub_missing_raid56_worker()
2310 if (!sblock->no_io_error_seen) { in scrub_missing_raid56_worker()
2317 } else if (sblock->header_error || sblock->checksum_error) { in scrub_missing_raid56_worker()
2325 scrub_write_block_to_dev_replace(sblock); in scrub_missing_raid56_worker()
2334 scrub_block_put(sblock); in scrub_missing_raid56_worker()
2338 static void scrub_missing_raid56_pages(struct scrub_block *sblock) in scrub_missing_raid56_pages() argument
2340 struct scrub_ctx *sctx = sblock->sctx; in scrub_missing_raid56_pages()
2342 u64 length = sblock->sector_count << fs_info->sectorsize_bits; in scrub_missing_raid56_pages()
2343 u64 logical = sblock->logical; in scrub_missing_raid56_pages()
2369 bio->bi_private = sblock; in scrub_missing_raid56_pages()
2376 for (i = 0; i < sblock->sector_count; i++) { in scrub_missing_raid56_pages()
2377 struct scrub_sector *sector = sblock->sectors[i]; in scrub_missing_raid56_pages()
2381 sector->offset + sector->sblock->logical); in scrub_missing_raid56_pages()
2384 INIT_WORK(&sblock->work, scrub_missing_raid56_worker); in scrub_missing_raid56_pages()
2385 scrub_block_get(sblock); in scrub_missing_raid56_pages()
2406 struct scrub_block *sblock; in scrub_sectors() local
2410 sblock = alloc_scrub_block(sctx, dev, logical, physical, in scrub_sectors()
2412 if (!sblock) { in scrub_sectors()
2428 sector = alloc_scrub_sector(sblock, logical, GFP_KERNEL); in scrub_sectors()
2433 scrub_block_put(sblock); in scrub_sectors()
2450 WARN_ON(sblock->sector_count == 0); in scrub_sectors()
2456 scrub_missing_raid56_pages(sblock); in scrub_sectors()
2458 for (index = 0; index < sblock->sector_count; index++) { in scrub_sectors()
2459 struct scrub_sector *sector = sblock->sectors[index]; in scrub_sectors()
2464 scrub_block_put(sblock); in scrub_sectors()
2474 scrub_block_put(sblock); in scrub_sectors()
2501 sector->sblock->no_io_error_seen = 0; in scrub_bio_end_io_worker()
2508 struct scrub_block *sblock = sector->sblock; in scrub_bio_end_io_worker() local
2510 if (atomic_dec_and_test(&sblock->outstanding_sectors)) in scrub_bio_end_io_worker()
2511 scrub_block_complete(sblock); in scrub_bio_end_io_worker()
2512 scrub_block_put(sblock); in scrub_bio_end_io_worker()
2570 static void scrub_block_complete(struct scrub_block *sblock) in scrub_block_complete() argument
2574 if (!sblock->no_io_error_seen) { in scrub_block_complete()
2576 scrub_handle_errored_block(sblock); in scrub_block_complete()
2583 corrupted = scrub_checksum(sblock); in scrub_block_complete()
2584 if (!corrupted && sblock->sctx->is_dev_replace) in scrub_block_complete()
2585 scrub_write_block_to_dev_replace(sblock); in scrub_block_complete()
2588 if (sblock->sparity && corrupted && !sblock->data_corrected) { in scrub_block_complete()
2589 u64 start = sblock->logical; in scrub_block_complete()
2590 u64 end = sblock->logical + in scrub_block_complete()
2591 sblock->sectors[sblock->sector_count - 1]->offset + in scrub_block_complete()
2592 sblock->sctx->fs_info->sectorsize; in scrub_block_complete()
2595 scrub_parity_mark_sectors_error(sblock->sparity, in scrub_block_complete()
2739 struct scrub_block *sblock; in scrub_sectors_for_parity() local
2745 sblock = alloc_scrub_block(sctx, dev, logical, physical, physical, mirror_num); in scrub_sectors_for_parity()
2746 if (!sblock) { in scrub_sectors_for_parity()
2753 sblock->sparity = sparity; in scrub_sectors_for_parity()
2759 sector = alloc_scrub_sector(sblock, logical, GFP_KERNEL); in scrub_sectors_for_parity()
2764 scrub_block_put(sblock); in scrub_sectors_for_parity()
2767 sblock->sectors[index] = sector; in scrub_sectors_for_parity()
2786 WARN_ON(sblock->sector_count == 0); in scrub_sectors_for_parity()
2787 for (index = 0; index < sblock->sector_count; index++) { in scrub_sectors_for_parity()
2788 struct scrub_sector *sector = sblock->sectors[index]; in scrub_sectors_for_parity()
2793 scrub_block_put(sblock); in scrub_sectors_for_parity()
2799 scrub_block_put(sblock); in scrub_sectors_for_parity()