63 	struct scrub_block	*sblock;  member
215 				struct scrub_block *sblock,
217 static void scrub_recheck_block_checksum(struct scrub_block *sblock);
223 static void scrub_write_block_to_dev_replace(struct scrub_block *sblock);
224 static int scrub_write_page_to_dev_replace(struct scrub_block *sblock,
226 static int scrub_checksum_data(struct scrub_block *sblock);
227 static int scrub_checksum_tree_block(struct scrub_block *sblock);
228 static int scrub_checksum_super(struct scrub_block *sblock);
229 static void scrub_block_get(struct scrub_block *sblock);
230 static void scrub_block_put(struct scrub_block *sblock);
243 static void scrub_block_complete(struct scrub_block *sblock);
551 			scrub_block_put(sbio->pagev[i]->sblock);  in scrub_free_ctx()
729 static void scrub_print_warning(const char *errstr, struct scrub_block *sblock)  in scrub_print_warning()  argument
746 	WARN_ON(sblock->page_count < 1);  in scrub_print_warning()
747 	dev = sblock->pagev[0]->dev;  in scrub_print_warning()
748 	fs_info = sblock->sctx->fs_info;  in scrub_print_warning()
754 	swarn.physical = sblock->pagev[0]->physical;  in scrub_print_warning()
755 	swarn.logical = sblock->pagev[0]->logical;  in scrub_print_warning()
1199 			struct scrub_block *sblock = sblocks_for_recheck +  in scrub_handle_errored_block()  local
1204 			for (page_index = 0; page_index < sblock->page_count;  in scrub_handle_errored_block()
1206 				sblock->pagev[page_index]->sblock = NULL;  in scrub_handle_errored_block()
1207 				recover = sblock->pagev[page_index]->recover;  in scrub_handle_errored_block()
1210 					sblock->pagev[page_index]->recover =  in scrub_handle_errored_block()
1213 				scrub_page_put(sblock->pagev[page_index]);  in scrub_handle_errored_block()
1328 			struct scrub_block *sblock;  in scrub_setup_recheck_block()  local
1331 			sblock = sblocks_for_recheck + mirror_index;  in scrub_setup_recheck_block()
1332 			sblock->sctx = sctx;  in scrub_setup_recheck_block()
1344 			sblock->pagev[page_index] = page;  in scrub_setup_recheck_block()
1345 			page->sblock = sblock;  in scrub_setup_recheck_block()
1374 			sblock->page_count++;  in scrub_setup_recheck_block()
1408 	mirror_num = page->sblock->pagev[0]->mirror_num;  in scrub_submit_raid56_bio_wait()
1420 					  struct scrub_block *sblock)  in scrub_recheck_block_on_raid56()  argument
1422 	struct scrub_page *first_page = sblock->pagev[0];  in scrub_recheck_block_on_raid56()
1434 	for (page_num = 0; page_num < sblock->page_count; page_num++) {  in scrub_recheck_block_on_raid56()
1435 		struct scrub_page *page = sblock->pagev[page_num];  in scrub_recheck_block_on_raid56()
1448 	scrub_recheck_block_checksum(sblock);  in scrub_recheck_block_on_raid56()
1452 	for (page_num = 0; page_num < sblock->page_count; page_num++)  in scrub_recheck_block_on_raid56()
1453 		sblock->pagev[page_num]->io_error = 1;  in scrub_recheck_block_on_raid56()
1455 	sblock->no_io_error_seen = 0;  in scrub_recheck_block_on_raid56()
1466 				struct scrub_block *sblock,  in scrub_recheck_block()  argument
1471 	sblock->no_io_error_seen = 1;  in scrub_recheck_block()
1474 	if (!retry_failed_mirror && scrub_is_page_on_raid56(sblock->pagev[0]))  in scrub_recheck_block()
1475 		return scrub_recheck_block_on_raid56(fs_info, sblock);  in scrub_recheck_block()
1477 	for (page_num = 0; page_num < sblock->page_count; page_num++) {  in scrub_recheck_block()
1479 		struct scrub_page *page = sblock->pagev[page_num];  in scrub_recheck_block()
1483 			sblock->no_io_error_seen = 0;  in scrub_recheck_block()
1497 			sblock->no_io_error_seen = 0;  in scrub_recheck_block()
1503 	if (sblock->no_io_error_seen)  in scrub_recheck_block()
1504 		scrub_recheck_block_checksum(sblock);  in scrub_recheck_block()
1517 static void scrub_recheck_block_checksum(struct scrub_block *sblock)  in scrub_recheck_block_checksum()  argument
1519 	sblock->header_error = 0;  in scrub_recheck_block_checksum()
1520 	sblock->checksum_error = 0;  in scrub_recheck_block_checksum()
1521 	sblock->generation_error = 0;  in scrub_recheck_block_checksum()
1523 	if (sblock->pagev[0]->flags & BTRFS_EXTENT_FLAG_DATA)  in scrub_recheck_block_checksum()
1524 		scrub_checksum_data(sblock);  in scrub_recheck_block_checksum()
1526 		scrub_checksum_tree_block(sblock);  in scrub_recheck_block_checksum()
1593 static void scrub_write_block_to_dev_replace(struct scrub_block *sblock)  in scrub_write_block_to_dev_replace()  argument
1595 	struct btrfs_fs_info *fs_info = sblock->sctx->fs_info;  in scrub_write_block_to_dev_replace()
1602 	if (sblock->sparity)  in scrub_write_block_to_dev_replace()
1605 	for (page_num = 0; page_num < sblock->page_count; page_num++) {  in scrub_write_block_to_dev_replace()
1608 		ret = scrub_write_page_to_dev_replace(sblock, page_num);  in scrub_write_block_to_dev_replace()
1614 static int scrub_write_page_to_dev_replace(struct scrub_block *sblock,  in scrub_write_page_to_dev_replace()  argument
1617 	struct scrub_page *spage = sblock->pagev[page_num];  in scrub_write_page_to_dev_replace()
1627 	return scrub_add_page_to_wr_bio(sblock->sctx, spage);  in scrub_write_page_to_dev_replace()
1755 static int scrub_checksum(struct scrub_block *sblock)  in scrub_checksum()  argument
1768 	sblock->header_error = 0;  in scrub_checksum()
1769 	sblock->generation_error = 0;  in scrub_checksum()
1770 	sblock->checksum_error = 0;  in scrub_checksum()
1772 	WARN_ON(sblock->page_count < 1);  in scrub_checksum()
1773 	flags = sblock->pagev[0]->flags;  in scrub_checksum()
1776 		ret = scrub_checksum_data(sblock);  in scrub_checksum()
1778 		ret = scrub_checksum_tree_block(sblock);  in scrub_checksum()
1780 		(void)scrub_checksum_super(sblock);  in scrub_checksum()
1784 		scrub_handle_errored_block(sblock);  in scrub_checksum()
1789 static int scrub_checksum_data(struct scrub_block *sblock)  in scrub_checksum_data()  argument
1791 	struct scrub_ctx *sctx = sblock->sctx;  in scrub_checksum_data()
1801 	BUG_ON(sblock->page_count < 1);  in scrub_checksum_data()
1802 	if (!sblock->pagev[0]->have_csum)  in scrub_checksum_data()
1808 	on_disk_csum = sblock->pagev[0]->csum;  in scrub_checksum_data()
1809 	page = sblock->pagev[0]->page;  in scrub_checksum_data()
1823 		BUG_ON(index >= sblock->page_count);  in scrub_checksum_data()
1824 		BUG_ON(!sblock->pagev[index]->page);  in scrub_checksum_data()
1825 		page = sblock->pagev[index]->page;  in scrub_checksum_data()
1831 		sblock->checksum_error = 1;  in scrub_checksum_data()
1833 	return sblock->checksum_error;  in scrub_checksum_data()
1836 static int scrub_checksum_tree_block(struct scrub_block *sblock)  in scrub_checksum_tree_block()  argument
1838 	struct scrub_ctx *sctx = sblock->sctx;  in scrub_checksum_tree_block()
1854 	BUG_ON(sblock->page_count < 1);  in scrub_checksum_tree_block()
1855 	page = sblock->pagev[0]->page;  in scrub_checksum_tree_block()
1865 	if (sblock->pagev[0]->logical != btrfs_stack_header_bytenr(h))  in scrub_checksum_tree_block()
1866 		sblock->header_error = 1;  in scrub_checksum_tree_block()
1868 	if (sblock->pagev[0]->generation != btrfs_stack_header_generation(h)) {  in scrub_checksum_tree_block()
1869 		sblock->header_error = 1;  in scrub_checksum_tree_block()
1870 		sblock->generation_error = 1;  in scrub_checksum_tree_block()
1873 	if (!scrub_check_fsid(h->fsid, sblock->pagev[0]))  in scrub_checksum_tree_block()
1874 		sblock->header_error = 1;  in scrub_checksum_tree_block()
1878 		sblock->header_error = 1;  in scrub_checksum_tree_block()
1893 		BUG_ON(index >= sblock->page_count);  in scrub_checksum_tree_block()
1894 		BUG_ON(!sblock->pagev[index]->page);  in scrub_checksum_tree_block()
1895 		page = sblock->pagev[index]->page;  in scrub_checksum_tree_block()
1903 		sblock->checksum_error = 1;  in scrub_checksum_tree_block()
1905 	return sblock->header_error || sblock->checksum_error;  in scrub_checksum_tree_block()
1908 static int scrub_checksum_super(struct scrub_block *sblock)  in scrub_checksum_super()  argument
1911 	struct scrub_ctx *sctx = sblock->sctx;  in scrub_checksum_super()
1928 	BUG_ON(sblock->page_count < 1);  in scrub_checksum_super()
1929 	page = sblock->pagev[0]->page;  in scrub_checksum_super()
1934 	if (sblock->pagev[0]->logical != btrfs_super_bytenr(s))  in scrub_checksum_super()
1937 	if (sblock->pagev[0]->generation != btrfs_super_generation(s))  in scrub_checksum_super()
1940 	if (!scrub_check_fsid(s->fsid, sblock->pagev[0]))  in scrub_checksum_super()
1956 		BUG_ON(index >= sblock->page_count);  in scrub_checksum_super()
1957 		BUG_ON(!sblock->pagev[index]->page);  in scrub_checksum_super()
1958 		page = sblock->pagev[index]->page;  in scrub_checksum_super()
1978 			btrfs_dev_stat_inc_and_print(sblock->pagev[0]->dev,  in scrub_checksum_super()
1981 			btrfs_dev_stat_inc_and_print(sblock->pagev[0]->dev,  in scrub_checksum_super()
1988 static void scrub_block_get(struct scrub_block *sblock)  in scrub_block_get()  argument
1990 	refcount_inc(&sblock->refs);  in scrub_block_get()
1993 static void scrub_block_put(struct scrub_block *sblock)  in scrub_block_put()  argument
1995 	if (refcount_dec_and_test(&sblock->refs)) {  in scrub_block_put()
1998 		if (sblock->sparity)  in scrub_block_put()
1999 			scrub_parity_put(sblock->sparity);  in scrub_block_put()
2001 		for (i = 0; i < sblock->page_count; i++)  in scrub_block_put()
2002 			scrub_page_put(sblock->pagev[i]);  in scrub_block_put()
2003 		kfree(sblock);  in scrub_block_put()
2037 	struct scrub_block *sblock = spage->sblock;  in scrub_add_page_to_rd_bio()  local
2098 	scrub_block_get(sblock); /* one for the page added to the bio */  in scrub_add_page_to_rd_bio()
2099 	atomic_inc(&sblock->outstanding_pages);  in scrub_add_page_to_rd_bio()
2109 	struct scrub_block *sblock = bio->bi_private;  in scrub_missing_raid56_end_io()  local
2110 	struct btrfs_fs_info *fs_info = sblock->sctx->fs_info;  in scrub_missing_raid56_end_io()
2113 		sblock->no_io_error_seen = 0;  in scrub_missing_raid56_end_io()
2117 	btrfs_queue_work(fs_info->scrub_workers, &sblock->work);  in scrub_missing_raid56_end_io()
2122 	struct scrub_block *sblock = container_of(work, struct scrub_block, work);  in scrub_missing_raid56_worker()  local
2123 	struct scrub_ctx *sctx = sblock->sctx;  in scrub_missing_raid56_worker()
2128 	logical = sblock->pagev[0]->logical;  in scrub_missing_raid56_worker()
2129 	dev = sblock->pagev[0]->dev;  in scrub_missing_raid56_worker()
2131 	if (sblock->no_io_error_seen)  in scrub_missing_raid56_worker()
2132 		scrub_recheck_block_checksum(sblock);  in scrub_missing_raid56_worker()
2134 	if (!sblock->no_io_error_seen) {  in scrub_missing_raid56_worker()
2141 	} else if (sblock->header_error || sblock->checksum_error) {  in scrub_missing_raid56_worker()
2149 		scrub_write_block_to_dev_replace(sblock);  in scrub_missing_raid56_worker()
2152 	scrub_block_put(sblock);  in scrub_missing_raid56_worker()
2163 static void scrub_missing_raid56_pages(struct scrub_block *sblock)  in scrub_missing_raid56_pages()  argument
2165 	struct scrub_ctx *sctx = sblock->sctx;  in scrub_missing_raid56_pages()
2167 	u64 length = sblock->page_count * PAGE_SIZE;  in scrub_missing_raid56_pages()
2168 	u64 logical = sblock->pagev[0]->logical;  in scrub_missing_raid56_pages()
2194 	bio->bi_private = sblock;  in scrub_missing_raid56_pages()
2201 	for (i = 0; i < sblock->page_count; i++) {  in scrub_missing_raid56_pages()
2202 		struct scrub_page *spage = sblock->pagev[i];  in scrub_missing_raid56_pages()
2207 	btrfs_init_work(&sblock->work, btrfs_scrub_helper,  in scrub_missing_raid56_pages()
2209 	scrub_block_get(sblock);  in scrub_missing_raid56_pages()
2229 	struct scrub_block *sblock;  in scrub_pages()  local
2232 	sblock = kzalloc(sizeof(*sblock), GFP_KERNEL);  in scrub_pages()
2233 	if (!sblock) {  in scrub_pages()
2242 	refcount_set(&sblock->refs, 1);  in scrub_pages()
2243 	sblock->sctx = sctx;  in scrub_pages()
2244 	sblock->no_io_error_seen = 1;  in scrub_pages()
2256 			scrub_block_put(sblock);  in scrub_pages()
2261 		sblock->pagev[index] = spage;  in scrub_pages()
2262 		spage->sblock = sblock;  in scrub_pages()
2276 		sblock->page_count++;  in scrub_pages()
2286 	WARN_ON(sblock->page_count == 0);  in scrub_pages()
2292 		scrub_missing_raid56_pages(sblock);  in scrub_pages()
2294 		for (index = 0; index < sblock->page_count; index++) {  in scrub_pages()
2295 			struct scrub_page *spage = sblock->pagev[index];  in scrub_pages()
2300 				scrub_block_put(sblock);  in scrub_pages()
2310 	scrub_block_put(sblock);  in scrub_pages()
2337 			spage->sblock->no_io_error_seen = 0;  in scrub_bio_end_io_worker()
2344 		struct scrub_block *sblock = spage->sblock;  in scrub_bio_end_io_worker()  local
2346 		if (atomic_dec_and_test(&sblock->outstanding_pages))  in scrub_bio_end_io_worker()
2347 			scrub_block_complete(sblock);  in scrub_bio_end_io_worker()
2348 		scrub_block_put(sblock);  in scrub_bio_end_io_worker()
2410 static void scrub_block_complete(struct scrub_block *sblock)  in scrub_block_complete()  argument
2414 	if (!sblock->no_io_error_seen) {  in scrub_block_complete()
2416 		scrub_handle_errored_block(sblock);  in scrub_block_complete()
2423 		corrupted = scrub_checksum(sblock);  in scrub_block_complete()
2424 		if (!corrupted && sblock->sctx->is_dev_replace)  in scrub_block_complete()
2425 			scrub_write_block_to_dev_replace(sblock);  in scrub_block_complete()
2428 	if (sblock->sparity && corrupted && !sblock->data_corrected) {  in scrub_block_complete()
2429 		u64 start = sblock->pagev[0]->logical;  in scrub_block_complete()
2430 		u64 end = sblock->pagev[sblock->page_count - 1]->logical +  in scrub_block_complete()
2433 		scrub_parity_mark_sectors_error(sblock->sparity,  in scrub_block_complete()
2534 	struct scrub_block *sblock;  in scrub_pages_for_parity()  local
2537 	sblock = kzalloc(sizeof(*sblock), GFP_KERNEL);  in scrub_pages_for_parity()
2538 	if (!sblock) {  in scrub_pages_for_parity()
2547 	refcount_set(&sblock->refs, 1);  in scrub_pages_for_parity()
2548 	sblock->sctx = sctx;  in scrub_pages_for_parity()
2549 	sblock->no_io_error_seen = 1;  in scrub_pages_for_parity()
2550 	sblock->sparity = sparity;  in scrub_pages_for_parity()
2563 			scrub_block_put(sblock);  in scrub_pages_for_parity()
2569 		sblock->pagev[index] = spage;  in scrub_pages_for_parity()
2573 		spage->sblock = sblock;  in scrub_pages_for_parity()
2586 		sblock->page_count++;  in scrub_pages_for_parity()
2595 	WARN_ON(sblock->page_count == 0);  in scrub_pages_for_parity()
2596 	for (index = 0; index < sblock->page_count; index++) {  in scrub_pages_for_parity()
2597 		struct scrub_page *spage = sblock->pagev[index];  in scrub_pages_for_parity()
2602 			scrub_block_put(sblock);  in scrub_pages_for_parity()
2608 	scrub_block_put(sblock);  in scrub_pages_for_parity()