543 		struct scrub_bio *sbio = sctx->bios[sctx->curr];  in scrub_free_ctx()  local
545 		for (i = 0; i < sbio->page_count; i++) {  in scrub_free_ctx()
546 			WARN_ON(!sbio->pagev[i]->page);  in scrub_free_ctx()
547 			scrub_block_put(sbio->pagev[i]->sblock);  in scrub_free_ctx()
549 		bio_put(sbio->bio);  in scrub_free_ctx()
553 		struct scrub_bio *sbio = sctx->bios[i];  in scrub_free_ctx()  local
555 		if (!sbio)  in scrub_free_ctx()
557 		kfree(sbio);  in scrub_free_ctx()
587 		struct scrub_bio *sbio;  in scrub_setup_ctx()  local
589 		sbio = kzalloc(sizeof(*sbio), GFP_KERNEL);  in scrub_setup_ctx()
590 		if (!sbio)  in scrub_setup_ctx()
592 		sctx->bios[i] = sbio;  in scrub_setup_ctx()
594 		sbio->index = i;  in scrub_setup_ctx()
595 		sbio->sctx = sctx;  in scrub_setup_ctx()
596 		sbio->page_count = 0;  in scrub_setup_ctx()
597 		btrfs_init_work(&sbio->work, btrfs_scrub_helper,  in scrub_setup_ctx()
1619 	struct scrub_bio *sbio;  in scrub_add_page_to_wr_bio()  local
1634 	sbio = sctx->wr_curr_bio;  in scrub_add_page_to_wr_bio()
1635 	if (sbio->page_count == 0) {  in scrub_add_page_to_wr_bio()
1638 		sbio->physical = spage->physical_for_dev_replace;  in scrub_add_page_to_wr_bio()
1639 		sbio->logical = spage->logical;  in scrub_add_page_to_wr_bio()
1640 		sbio->dev = sctx->wr_tgtdev;  in scrub_add_page_to_wr_bio()
1641 		bio = sbio->bio;  in scrub_add_page_to_wr_bio()
1644 			sbio->bio = bio;  in scrub_add_page_to_wr_bio()
1647 		bio->bi_private = sbio;  in scrub_add_page_to_wr_bio()
1649 		bio_set_dev(bio, sbio->dev->bdev);  in scrub_add_page_to_wr_bio()
1650 		bio->bi_iter.bi_sector = sbio->physical >> 9;  in scrub_add_page_to_wr_bio()
1652 		sbio->status = 0;  in scrub_add_page_to_wr_bio()
1653 	} else if (sbio->physical + sbio->page_count * PAGE_SIZE !=  in scrub_add_page_to_wr_bio()
1655 		   sbio->logical + sbio->page_count * PAGE_SIZE !=  in scrub_add_page_to_wr_bio()
1661 	ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0);  in scrub_add_page_to_wr_bio()
1663 		if (sbio->page_count < 1) {  in scrub_add_page_to_wr_bio()
1664 			bio_put(sbio->bio);  in scrub_add_page_to_wr_bio()
1665 			sbio->bio = NULL;  in scrub_add_page_to_wr_bio()
1673 	sbio->pagev[sbio->page_count] = spage;  in scrub_add_page_to_wr_bio()
1675 	sbio->page_count++;  in scrub_add_page_to_wr_bio()
1676 	if (sbio->page_count == sctx->pages_per_wr_bio)  in scrub_add_page_to_wr_bio()
1685 	struct scrub_bio *sbio;  in scrub_wr_submit()  local
1690 	sbio = sctx->wr_curr_bio;  in scrub_wr_submit()
1692 	WARN_ON(!sbio->bio->bi_disk);  in scrub_wr_submit()
1698 	btrfsic_submit_bio(sbio->bio);  in scrub_wr_submit()
1703 	struct scrub_bio *sbio = bio->bi_private;  in scrub_wr_bio_end_io()  local
1704 	struct btrfs_fs_info *fs_info = sbio->dev->fs_info;  in scrub_wr_bio_end_io()
1706 	sbio->status = bio->bi_status;  in scrub_wr_bio_end_io()
1707 	sbio->bio = bio;  in scrub_wr_bio_end_io()
1709 	btrfs_init_work(&sbio->work, btrfs_scrubwrc_helper,  in scrub_wr_bio_end_io()
1711 	btrfs_queue_work(fs_info->scrub_wr_completion_workers, &sbio->work);  in scrub_wr_bio_end_io()
1716 	struct scrub_bio *sbio = container_of(work, struct scrub_bio, work);  in scrub_wr_bio_end_io_worker()  local
1717 	struct scrub_ctx *sctx = sbio->sctx;  in scrub_wr_bio_end_io_worker()
1720 	WARN_ON(sbio->page_count > SCRUB_PAGES_PER_WR_BIO);  in scrub_wr_bio_end_io_worker()
1721 	if (sbio->status) {  in scrub_wr_bio_end_io_worker()
1723 			&sbio->sctx->fs_info->dev_replace;  in scrub_wr_bio_end_io_worker()
1725 		for (i = 0; i < sbio->page_count; i++) {  in scrub_wr_bio_end_io_worker()
1726 			struct scrub_page *spage = sbio->pagev[i];  in scrub_wr_bio_end_io_worker()
1734 	for (i = 0; i < sbio->page_count; i++)  in scrub_wr_bio_end_io_worker()
1735 		scrub_page_put(sbio->pagev[i]);  in scrub_wr_bio_end_io_worker()
1737 	bio_put(sbio->bio);  in scrub_wr_bio_end_io_worker()
1738 	kfree(sbio);  in scrub_wr_bio_end_io_worker()
1999 	struct scrub_bio *sbio;  in scrub_submit()  local
2004 	sbio = sctx->bios[sctx->curr];  in scrub_submit()
2007 	btrfsic_submit_bio(sbio->bio);  in scrub_submit()
2014 	struct scrub_bio *sbio;  in scrub_add_page_to_rd_bio()  local
2034 	sbio = sctx->bios[sctx->curr];  in scrub_add_page_to_rd_bio()
2035 	if (sbio->page_count == 0) {  in scrub_add_page_to_rd_bio()
2038 		sbio->physical = spage->physical;  in scrub_add_page_to_rd_bio()
2039 		sbio->logical = spage->logical;  in scrub_add_page_to_rd_bio()
2040 		sbio->dev = spage->dev;  in scrub_add_page_to_rd_bio()
2041 		bio = sbio->bio;  in scrub_add_page_to_rd_bio()
2044 			sbio->bio = bio;  in scrub_add_page_to_rd_bio()
2047 		bio->bi_private = sbio;  in scrub_add_page_to_rd_bio()
2049 		bio_set_dev(bio, sbio->dev->bdev);  in scrub_add_page_to_rd_bio()
2050 		bio->bi_iter.bi_sector = sbio->physical >> 9;  in scrub_add_page_to_rd_bio()
2052 		sbio->status = 0;  in scrub_add_page_to_rd_bio()
2053 	} else if (sbio->physical + sbio->page_count * PAGE_SIZE !=  in scrub_add_page_to_rd_bio()
2055 		   sbio->logical + sbio->page_count * PAGE_SIZE !=  in scrub_add_page_to_rd_bio()
2057 		   sbio->dev != spage->dev) {  in scrub_add_page_to_rd_bio()
2062 	sbio->pagev[sbio->page_count] = spage;  in scrub_add_page_to_rd_bio()
2063 	ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0);  in scrub_add_page_to_rd_bio()
2065 		if (sbio->page_count < 1) {  in scrub_add_page_to_rd_bio()
2066 			bio_put(sbio->bio);  in scrub_add_page_to_rd_bio()
2067 			sbio->bio = NULL;  in scrub_add_page_to_rd_bio()
2076 	sbio->page_count++;  in scrub_add_page_to_rd_bio()
2077 	if (sbio->page_count == sctx->pages_per_rd_bio)  in scrub_add_page_to_rd_bio()
2292 	struct scrub_bio *sbio = bio->bi_private;  in scrub_bio_end_io()  local
2293 	struct btrfs_fs_info *fs_info = sbio->dev->fs_info;  in scrub_bio_end_io()
2295 	sbio->status = bio->bi_status;  in scrub_bio_end_io()
2296 	sbio->bio = bio;  in scrub_bio_end_io()
2298 	btrfs_queue_work(fs_info->scrub_workers, &sbio->work);  in scrub_bio_end_io()
2303 	struct scrub_bio *sbio = container_of(work, struct scrub_bio, work);  in scrub_bio_end_io_worker()  local
2304 	struct scrub_ctx *sctx = sbio->sctx;  in scrub_bio_end_io_worker()
2307 	BUG_ON(sbio->page_count > SCRUB_PAGES_PER_RD_BIO);  in scrub_bio_end_io_worker()
2308 	if (sbio->status) {  in scrub_bio_end_io_worker()
2309 		for (i = 0; i < sbio->page_count; i++) {  in scrub_bio_end_io_worker()
2310 			struct scrub_page *spage = sbio->pagev[i];  in scrub_bio_end_io_worker()
2318 	for (i = 0; i < sbio->page_count; i++) {  in scrub_bio_end_io_worker()
2319 		struct scrub_page *spage = sbio->pagev[i];  in scrub_bio_end_io_worker()
2327 	bio_put(sbio->bio);  in scrub_bio_end_io_worker()
2328 	sbio->bio = NULL;  in scrub_bio_end_io_worker()
2330 	sbio->next_free = sctx->first_free;  in scrub_bio_end_io_worker()
2331 	sctx->first_free = sbio->index;  in scrub_bio_end_io_worker()