61 	struct scrub_block	*sblock;  member
213 				struct scrub_block *sblock,
215 static void scrub_recheck_block_checksum(struct scrub_block *sblock);
221 static void scrub_write_block_to_dev_replace(struct scrub_block *sblock);
222 static int scrub_write_page_to_dev_replace(struct scrub_block *sblock,
224 static int scrub_checksum_data(struct scrub_block *sblock);
225 static int scrub_checksum_tree_block(struct scrub_block *sblock);
226 static int scrub_checksum_super(struct scrub_block *sblock);
227 static void scrub_block_get(struct scrub_block *sblock);
228 static void scrub_block_put(struct scrub_block *sblock);
241 static void scrub_block_complete(struct scrub_block *sblock);
547 			scrub_block_put(sbio->pagev[i]->sblock);  in scrub_free_ctx()
726 static void scrub_print_warning(const char *errstr, struct scrub_block *sblock)  in scrub_print_warning()  argument
743 	WARN_ON(sblock->page_count < 1);  in scrub_print_warning()
744 	dev = sblock->pagev[0]->dev;  in scrub_print_warning()
745 	fs_info = sblock->sctx->fs_info;  in scrub_print_warning()
751 	swarn.physical = sblock->pagev[0]->physical;  in scrub_print_warning()
752 	swarn.logical = sblock->pagev[0]->logical;  in scrub_print_warning()
1184 			struct scrub_block *sblock = sblocks_for_recheck +  in scrub_handle_errored_block()  local
1189 			for (page_index = 0; page_index < sblock->page_count;  in scrub_handle_errored_block()
1191 				sblock->pagev[page_index]->sblock = NULL;  in scrub_handle_errored_block()
1192 				recover = sblock->pagev[page_index]->recover;  in scrub_handle_errored_block()
1195 					sblock->pagev[page_index]->recover =  in scrub_handle_errored_block()
1198 				scrub_page_put(sblock->pagev[page_index]);  in scrub_handle_errored_block()
1312 			struct scrub_block *sblock;  in scrub_setup_recheck_block()  local
1315 			sblock = sblocks_for_recheck + mirror_index;  in scrub_setup_recheck_block()
1316 			sblock->sctx = sctx;  in scrub_setup_recheck_block()
1328 			sblock->pagev[page_index] = page;  in scrub_setup_recheck_block()
1329 			page->sblock = sblock;  in scrub_setup_recheck_block()
1358 			sblock->page_count++;  in scrub_setup_recheck_block()
1392 	mirror_num = page->sblock->pagev[0]->mirror_num;  in scrub_submit_raid56_bio_wait()
1404 					  struct scrub_block *sblock)  in scrub_recheck_block_on_raid56()  argument
1406 	struct scrub_page *first_page = sblock->pagev[0];  in scrub_recheck_block_on_raid56()
1418 	for (page_num = 0; page_num < sblock->page_count; page_num++) {  in scrub_recheck_block_on_raid56()
1419 		struct scrub_page *page = sblock->pagev[page_num];  in scrub_recheck_block_on_raid56()
1432 	scrub_recheck_block_checksum(sblock);  in scrub_recheck_block_on_raid56()
1436 	for (page_num = 0; page_num < sblock->page_count; page_num++)  in scrub_recheck_block_on_raid56()
1437 		sblock->pagev[page_num]->io_error = 1;  in scrub_recheck_block_on_raid56()
1439 	sblock->no_io_error_seen = 0;  in scrub_recheck_block_on_raid56()
1450 				struct scrub_block *sblock,  in scrub_recheck_block()  argument
1455 	sblock->no_io_error_seen = 1;  in scrub_recheck_block()
1458 	if (!retry_failed_mirror && scrub_is_page_on_raid56(sblock->pagev[0]))  in scrub_recheck_block()
1459 		return scrub_recheck_block_on_raid56(fs_info, sblock);  in scrub_recheck_block()
1461 	for (page_num = 0; page_num < sblock->page_count; page_num++) {  in scrub_recheck_block()
1463 		struct scrub_page *page = sblock->pagev[page_num];  in scrub_recheck_block()
1467 			sblock->no_io_error_seen = 0;  in scrub_recheck_block()
1481 			sblock->no_io_error_seen = 0;  in scrub_recheck_block()
1487 	if (sblock->no_io_error_seen)  in scrub_recheck_block()
1488 		scrub_recheck_block_checksum(sblock);  in scrub_recheck_block()
1501 static void scrub_recheck_block_checksum(struct scrub_block *sblock)  in scrub_recheck_block_checksum()  argument
1503 	sblock->header_error = 0;  in scrub_recheck_block_checksum()
1504 	sblock->checksum_error = 0;  in scrub_recheck_block_checksum()
1505 	sblock->generation_error = 0;  in scrub_recheck_block_checksum()
1507 	if (sblock->pagev[0]->flags & BTRFS_EXTENT_FLAG_DATA)  in scrub_recheck_block_checksum()
1508 		scrub_checksum_data(sblock);  in scrub_recheck_block_checksum()
1510 		scrub_checksum_tree_block(sblock);  in scrub_recheck_block_checksum()
1578 static void scrub_write_block_to_dev_replace(struct scrub_block *sblock)  in scrub_write_block_to_dev_replace()  argument
1580 	struct btrfs_fs_info *fs_info = sblock->sctx->fs_info;  in scrub_write_block_to_dev_replace()
1587 	if (sblock->sparity)  in scrub_write_block_to_dev_replace()
1590 	for (page_num = 0; page_num < sblock->page_count; page_num++) {  in scrub_write_block_to_dev_replace()
1593 		ret = scrub_write_page_to_dev_replace(sblock, page_num);  in scrub_write_block_to_dev_replace()
1600 static int scrub_write_page_to_dev_replace(struct scrub_block *sblock,  in scrub_write_page_to_dev_replace()  argument
1603 	struct scrub_page *spage = sblock->pagev[page_num];  in scrub_write_page_to_dev_replace()
1613 	return scrub_add_page_to_wr_bio(sblock->sctx, spage);  in scrub_write_page_to_dev_replace()
1742 static int scrub_checksum(struct scrub_block *sblock)  in scrub_checksum()  argument
1755 	sblock->header_error = 0;  in scrub_checksum()
1756 	sblock->generation_error = 0;  in scrub_checksum()
1757 	sblock->checksum_error = 0;  in scrub_checksum()
1759 	WARN_ON(sblock->page_count < 1);  in scrub_checksum()
1760 	flags = sblock->pagev[0]->flags;  in scrub_checksum()
1763 		ret = scrub_checksum_data(sblock);  in scrub_checksum()
1765 		ret = scrub_checksum_tree_block(sblock);  in scrub_checksum()
1767 		(void)scrub_checksum_super(sblock);  in scrub_checksum()
1771 		scrub_handle_errored_block(sblock);  in scrub_checksum()
1776 static int scrub_checksum_data(struct scrub_block *sblock)  in scrub_checksum_data()  argument
1778 	struct scrub_ctx *sctx = sblock->sctx;  in scrub_checksum_data()
1787 	BUG_ON(sblock->page_count < 1);  in scrub_checksum_data()
1788 	if (!sblock->pagev[0]->have_csum)  in scrub_checksum_data()
1791 	on_disk_csum = sblock->pagev[0]->csum;  in scrub_checksum_data()
1792 	page = sblock->pagev[0]->page;  in scrub_checksum_data()
1806 		BUG_ON(index >= sblock->page_count);  in scrub_checksum_data()
1807 		BUG_ON(!sblock->pagev[index]->page);  in scrub_checksum_data()
1808 		page = sblock->pagev[index]->page;  in scrub_checksum_data()
1814 		sblock->checksum_error = 1;  in scrub_checksum_data()
1816 	return sblock->checksum_error;  in scrub_checksum_data()
1819 static int scrub_checksum_tree_block(struct scrub_block *sblock)  in scrub_checksum_tree_block()  argument
1821 	struct scrub_ctx *sctx = sblock->sctx;  in scrub_checksum_tree_block()
1834 	BUG_ON(sblock->page_count < 1);  in scrub_checksum_tree_block()
1835 	page = sblock->pagev[0]->page;  in scrub_checksum_tree_block()
1845 	if (sblock->pagev[0]->logical != btrfs_stack_header_bytenr(h))  in scrub_checksum_tree_block()
1846 		sblock->header_error = 1;  in scrub_checksum_tree_block()
1848 	if (sblock->pagev[0]->generation != btrfs_stack_header_generation(h)) {  in scrub_checksum_tree_block()
1849 		sblock->header_error = 1;  in scrub_checksum_tree_block()
1850 		sblock->generation_error = 1;  in scrub_checksum_tree_block()
1853 	if (!scrub_check_fsid(h->fsid, sblock->pagev[0]))  in scrub_checksum_tree_block()
1854 		sblock->header_error = 1;  in scrub_checksum_tree_block()
1858 		sblock->header_error = 1;  in scrub_checksum_tree_block()
1873 		BUG_ON(index >= sblock->page_count);  in scrub_checksum_tree_block()
1874 		BUG_ON(!sblock->pagev[index]->page);  in scrub_checksum_tree_block()
1875 		page = sblock->pagev[index]->page;  in scrub_checksum_tree_block()
1883 		sblock->checksum_error = 1;  in scrub_checksum_tree_block()
1885 	return sblock->header_error || sblock->checksum_error;  in scrub_checksum_tree_block()
1888 static int scrub_checksum_super(struct scrub_block *sblock)  in scrub_checksum_super()  argument
1891 	struct scrub_ctx *sctx = sblock->sctx;  in scrub_checksum_super()
1904 	BUG_ON(sblock->page_count < 1);  in scrub_checksum_super()
1905 	page = sblock->pagev[0]->page;  in scrub_checksum_super()
1910 	if (sblock->pagev[0]->logical != btrfs_super_bytenr(s))  in scrub_checksum_super()
1913 	if (sblock->pagev[0]->generation != btrfs_super_generation(s))  in scrub_checksum_super()
1916 	if (!scrub_check_fsid(s->fsid, sblock->pagev[0]))  in scrub_checksum_super()
1932 		BUG_ON(index >= sblock->page_count);  in scrub_checksum_super()
1933 		BUG_ON(!sblock->pagev[index]->page);  in scrub_checksum_super()
1934 		page = sblock->pagev[index]->page;  in scrub_checksum_super()
1954 			btrfs_dev_stat_inc_and_print(sblock->pagev[0]->dev,  in scrub_checksum_super()
1957 			btrfs_dev_stat_inc_and_print(sblock->pagev[0]->dev,  in scrub_checksum_super()
1964 static void scrub_block_get(struct scrub_block *sblock)  in scrub_block_get()  argument
1966 	refcount_inc(&sblock->refs);  in scrub_block_get()
1969 static void scrub_block_put(struct scrub_block *sblock)  in scrub_block_put()  argument
1971 	if (refcount_dec_and_test(&sblock->refs)) {  in scrub_block_put()
1974 		if (sblock->sparity)  in scrub_block_put()
1975 			scrub_parity_put(sblock->sparity);  in scrub_block_put()
1977 		for (i = 0; i < sblock->page_count; i++)  in scrub_block_put()
1978 			scrub_page_put(sblock->pagev[i]);  in scrub_block_put()
1979 		kfree(sblock);  in scrub_block_put()
2013 	struct scrub_block *sblock = spage->sblock;  in scrub_add_page_to_rd_bio()  local
2074 	scrub_block_get(sblock); /* one for the page added to the bio */  in scrub_add_page_to_rd_bio()
2075 	atomic_inc(&sblock->outstanding_pages);  in scrub_add_page_to_rd_bio()
2085 	struct scrub_block *sblock = bio->bi_private;  in scrub_missing_raid56_end_io()  local
2086 	struct btrfs_fs_info *fs_info = sblock->sctx->fs_info;  in scrub_missing_raid56_end_io()
2089 		sblock->no_io_error_seen = 0;  in scrub_missing_raid56_end_io()
2093 	btrfs_queue_work(fs_info->scrub_workers, &sblock->work);  in scrub_missing_raid56_end_io()
2098 	struct scrub_block *sblock = container_of(work, struct scrub_block, work);  in scrub_missing_raid56_worker()  local
2099 	struct scrub_ctx *sctx = sblock->sctx;  in scrub_missing_raid56_worker()
2104 	logical = sblock->pagev[0]->logical;  in scrub_missing_raid56_worker()
2105 	dev = sblock->pagev[0]->dev;  in scrub_missing_raid56_worker()
2107 	if (sblock->no_io_error_seen)  in scrub_missing_raid56_worker()
2108 		scrub_recheck_block_checksum(sblock);  in scrub_missing_raid56_worker()
2110 	if (!sblock->no_io_error_seen) {  in scrub_missing_raid56_worker()
2117 	} else if (sblock->header_error || sblock->checksum_error) {  in scrub_missing_raid56_worker()
2125 		scrub_write_block_to_dev_replace(sblock);  in scrub_missing_raid56_worker()
2128 	scrub_block_put(sblock);  in scrub_missing_raid56_worker()
2139 static void scrub_missing_raid56_pages(struct scrub_block *sblock)  in scrub_missing_raid56_pages()  argument
2141 	struct scrub_ctx *sctx = sblock->sctx;  in scrub_missing_raid56_pages()
2143 	u64 length = sblock->page_count * PAGE_SIZE;  in scrub_missing_raid56_pages()
2144 	u64 logical = sblock->pagev[0]->logical;  in scrub_missing_raid56_pages()
2170 	bio->bi_private = sblock;  in scrub_missing_raid56_pages()
2177 	for (i = 0; i < sblock->page_count; i++) {  in scrub_missing_raid56_pages()
2178 		struct scrub_page *spage = sblock->pagev[i];  in scrub_missing_raid56_pages()
2183 	btrfs_init_work(&sblock->work, btrfs_scrub_helper,  in scrub_missing_raid56_pages()
2185 	scrub_block_get(sblock);  in scrub_missing_raid56_pages()
2205 	struct scrub_block *sblock;  in scrub_pages()  local
2208 	sblock = kzalloc(sizeof(*sblock), GFP_KERNEL);  in scrub_pages()
2209 	if (!sblock) {  in scrub_pages()
2218 	refcount_set(&sblock->refs, 1);  in scrub_pages()
2219 	sblock->sctx = sctx;  in scrub_pages()
2220 	sblock->no_io_error_seen = 1;  in scrub_pages()
2232 			scrub_block_put(sblock);  in scrub_pages()
2237 		sblock->pagev[index] = spage;  in scrub_pages()
2238 		spage->sblock = sblock;  in scrub_pages()
2252 		sblock->page_count++;  in scrub_pages()
2262 	WARN_ON(sblock->page_count == 0);  in scrub_pages()
2268 		scrub_missing_raid56_pages(sblock);  in scrub_pages()
2270 		for (index = 0; index < sblock->page_count; index++) {  in scrub_pages()
2271 			struct scrub_page *spage = sblock->pagev[index];  in scrub_pages()
2276 				scrub_block_put(sblock);  in scrub_pages()
2286 	scrub_block_put(sblock);  in scrub_pages()
2313 			spage->sblock->no_io_error_seen = 0;  in scrub_bio_end_io_worker()
2320 		struct scrub_block *sblock = spage->sblock;  in scrub_bio_end_io_worker()  local
2322 		if (atomic_dec_and_test(&sblock->outstanding_pages))  in scrub_bio_end_io_worker()
2323 			scrub_block_complete(sblock);  in scrub_bio_end_io_worker()
2324 		scrub_block_put(sblock);  in scrub_bio_end_io_worker()
2386 static void scrub_block_complete(struct scrub_block *sblock)  in scrub_block_complete()  argument
2390 	if (!sblock->no_io_error_seen) {  in scrub_block_complete()
2392 		scrub_handle_errored_block(sblock);  in scrub_block_complete()
2399 		corrupted = scrub_checksum(sblock);  in scrub_block_complete()
2400 		if (!corrupted && sblock->sctx->is_dev_replace)  in scrub_block_complete()
2401 			scrub_write_block_to_dev_replace(sblock);  in scrub_block_complete()
2404 	if (sblock->sparity && corrupted && !sblock->data_corrected) {  in scrub_block_complete()
2405 		u64 start = sblock->pagev[0]->logical;  in scrub_block_complete()
2406 		u64 end = sblock->pagev[sblock->page_count - 1]->logical +  in scrub_block_complete()
2409 		scrub_parity_mark_sectors_error(sblock->sparity,  in scrub_block_complete()
2510 	struct scrub_block *sblock;  in scrub_pages_for_parity()  local
2513 	sblock = kzalloc(sizeof(*sblock), GFP_KERNEL);  in scrub_pages_for_parity()
2514 	if (!sblock) {  in scrub_pages_for_parity()
2523 	refcount_set(&sblock->refs, 1);  in scrub_pages_for_parity()
2524 	sblock->sctx = sctx;  in scrub_pages_for_parity()
2525 	sblock->no_io_error_seen = 1;  in scrub_pages_for_parity()
2526 	sblock->sparity = sparity;  in scrub_pages_for_parity()
2539 			scrub_block_put(sblock);  in scrub_pages_for_parity()
2545 		sblock->pagev[index] = spage;  in scrub_pages_for_parity()
2549 		spage->sblock = sblock;  in scrub_pages_for_parity()
2562 		sblock->page_count++;  in scrub_pages_for_parity()
2571 	WARN_ON(sblock->page_count == 0);  in scrub_pages_for_parity()
2572 	for (index = 0; index < sblock->page_count; index++) {  in scrub_pages_for_parity()
2573 		struct scrub_page *spage = sblock->pagev[index];  in scrub_pages_for_parity()
2578 			scrub_block_put(sblock);  in scrub_pages_for_parity()
2584 	scrub_block_put(sblock);  in scrub_pages_for_parity()