542 		struct scrub_bio *sbio = sctx->bios[sctx->curr];  in scrub_free_ctx()  local
544 		for (i = 0; i < sbio->page_count; i++) {  in scrub_free_ctx()
545 			WARN_ON(!sbio->pagev[i]->page);  in scrub_free_ctx()
546 			scrub_block_put(sbio->pagev[i]->sblock);  in scrub_free_ctx()
548 		bio_put(sbio->bio);  in scrub_free_ctx()
552 		struct scrub_bio *sbio = sctx->bios[i];  in scrub_free_ctx()  local
554 		if (!sbio)  in scrub_free_ctx()
556 		kfree(sbio);  in scrub_free_ctx()
586 		struct scrub_bio *sbio;  in scrub_setup_ctx()  local
588 		sbio = kzalloc(sizeof(*sbio), GFP_KERNEL);  in scrub_setup_ctx()
589 		if (!sbio)  in scrub_setup_ctx()
591 		sctx->bios[i] = sbio;  in scrub_setup_ctx()
593 		sbio->index = i;  in scrub_setup_ctx()
594 		sbio->sctx = sctx;  in scrub_setup_ctx()
595 		sbio->page_count = 0;  in scrub_setup_ctx()
596 		btrfs_init_work(&sbio->work, scrub_bio_end_io_worker, NULL,  in scrub_setup_ctx()
1647 	struct scrub_bio *sbio;  in scrub_add_page_to_wr_bio()  local
1663 	sbio = sctx->wr_curr_bio;  in scrub_add_page_to_wr_bio()
1664 	if (sbio->page_count == 0) {  in scrub_add_page_to_wr_bio()
1674 		sbio->physical = spage->physical_for_dev_replace;  in scrub_add_page_to_wr_bio()
1675 		sbio->logical = spage->logical;  in scrub_add_page_to_wr_bio()
1676 		sbio->dev = sctx->wr_tgtdev;  in scrub_add_page_to_wr_bio()
1677 		bio = sbio->bio;  in scrub_add_page_to_wr_bio()
1680 			sbio->bio = bio;  in scrub_add_page_to_wr_bio()
1683 		bio->bi_private = sbio;  in scrub_add_page_to_wr_bio()
1685 		bio_set_dev(bio, sbio->dev->bdev);  in scrub_add_page_to_wr_bio()
1686 		bio->bi_iter.bi_sector = sbio->physical >> 9;  in scrub_add_page_to_wr_bio()
1688 		sbio->status = 0;  in scrub_add_page_to_wr_bio()
1689 	} else if (sbio->physical + sbio->page_count * sectorsize !=  in scrub_add_page_to_wr_bio()
1691 		   sbio->logical + sbio->page_count * sectorsize !=  in scrub_add_page_to_wr_bio()
1697 	ret = bio_add_page(sbio->bio, spage->page, sectorsize, 0);  in scrub_add_page_to_wr_bio()
1699 		if (sbio->page_count < 1) {  in scrub_add_page_to_wr_bio()
1700 			bio_put(sbio->bio);  in scrub_add_page_to_wr_bio()
1701 			sbio->bio = NULL;  in scrub_add_page_to_wr_bio()
1709 	sbio->pagev[sbio->page_count] = spage;  in scrub_add_page_to_wr_bio()
1711 	sbio->page_count++;  in scrub_add_page_to_wr_bio()
1712 	if (sbio->page_count == sctx->pages_per_wr_bio)  in scrub_add_page_to_wr_bio()
1721 	struct scrub_bio *sbio;  in scrub_wr_submit()  local
1726 	sbio = sctx->wr_curr_bio;  in scrub_wr_submit()
1728 	WARN_ON(!sbio->bio->bi_bdev);  in scrub_wr_submit()
1734 	btrfsic_submit_bio(sbio->bio);  in scrub_wr_submit()
1737 		sctx->write_pointer = sbio->physical + sbio->page_count *  in scrub_wr_submit()
1743 	struct scrub_bio *sbio = bio->bi_private;  in scrub_wr_bio_end_io()  local
1744 	struct btrfs_fs_info *fs_info = sbio->dev->fs_info;  in scrub_wr_bio_end_io()
1746 	sbio->status = bio->bi_status;  in scrub_wr_bio_end_io()
1747 	sbio->bio = bio;  in scrub_wr_bio_end_io()
1749 	btrfs_init_work(&sbio->work, scrub_wr_bio_end_io_worker, NULL, NULL);  in scrub_wr_bio_end_io()
1750 	btrfs_queue_work(fs_info->scrub_wr_completion_workers, &sbio->work);  in scrub_wr_bio_end_io()
1755 	struct scrub_bio *sbio = container_of(work, struct scrub_bio, work);  in scrub_wr_bio_end_io_worker()  local
1756 	struct scrub_ctx *sctx = sbio->sctx;  in scrub_wr_bio_end_io_worker()
1759 	WARN_ON(sbio->page_count > SCRUB_PAGES_PER_WR_BIO);  in scrub_wr_bio_end_io_worker()
1760 	if (sbio->status) {  in scrub_wr_bio_end_io_worker()
1762 			&sbio->sctx->fs_info->dev_replace;  in scrub_wr_bio_end_io_worker()
1764 		for (i = 0; i < sbio->page_count; i++) {  in scrub_wr_bio_end_io_worker()
1765 			struct scrub_page *spage = sbio->pagev[i];  in scrub_wr_bio_end_io_worker()
1772 	for (i = 0; i < sbio->page_count; i++)  in scrub_wr_bio_end_io_worker()
1773 		scrub_page_put(sbio->pagev[i]);  in scrub_wr_bio_end_io_worker()
1775 	bio_put(sbio->bio);  in scrub_wr_bio_end_io_worker()
1776 	kfree(sbio);  in scrub_wr_bio_end_io_worker()
2004 	struct scrub_bio *sbio;  in scrub_throttle()  local
2011 	sbio = sctx->bios[sctx->curr];  in scrub_throttle()
2012 	device = sbio->dev;  in scrub_throttle()
2034 		sctx->throttle_sent += sbio->bio->bi_iter.bi_size;  in scrub_throttle()
2058 	struct scrub_bio *sbio;  in scrub_submit()  local
2065 	sbio = sctx->bios[sctx->curr];  in scrub_submit()
2068 	btrfsic_submit_bio(sbio->bio);  in scrub_submit()
2075 	struct scrub_bio *sbio;  in scrub_add_page_to_rd_bio()  local
2096 	sbio = sctx->bios[sctx->curr];  in scrub_add_page_to_rd_bio()
2097 	if (sbio->page_count == 0) {  in scrub_add_page_to_rd_bio()
2100 		sbio->physical = spage->physical;  in scrub_add_page_to_rd_bio()
2101 		sbio->logical = spage->logical;  in scrub_add_page_to_rd_bio()
2102 		sbio->dev = spage->dev;  in scrub_add_page_to_rd_bio()
2103 		bio = sbio->bio;  in scrub_add_page_to_rd_bio()
2106 			sbio->bio = bio;  in scrub_add_page_to_rd_bio()
2109 		bio->bi_private = sbio;  in scrub_add_page_to_rd_bio()
2111 		bio_set_dev(bio, sbio->dev->bdev);  in scrub_add_page_to_rd_bio()
2112 		bio->bi_iter.bi_sector = sbio->physical >> 9;  in scrub_add_page_to_rd_bio()
2114 		sbio->status = 0;  in scrub_add_page_to_rd_bio()
2115 	} else if (sbio->physical + sbio->page_count * sectorsize !=  in scrub_add_page_to_rd_bio()
2117 		   sbio->logical + sbio->page_count * sectorsize !=  in scrub_add_page_to_rd_bio()
2119 		   sbio->dev != spage->dev) {  in scrub_add_page_to_rd_bio()
2124 	sbio->pagev[sbio->page_count] = spage;  in scrub_add_page_to_rd_bio()
2125 	ret = bio_add_page(sbio->bio, spage->page, sectorsize, 0);  in scrub_add_page_to_rd_bio()
2127 		if (sbio->page_count < 1) {  in scrub_add_page_to_rd_bio()
2128 			bio_put(sbio->bio);  in scrub_add_page_to_rd_bio()
2129 			sbio->bio = NULL;  in scrub_add_page_to_rd_bio()
2138 	sbio->page_count++;  in scrub_add_page_to_rd_bio()
2139 	if (sbio->page_count == sctx->pages_per_rd_bio)  in scrub_add_page_to_rd_bio()
2358 	struct scrub_bio *sbio = bio->bi_private;  in scrub_bio_end_io()  local
2359 	struct btrfs_fs_info *fs_info = sbio->dev->fs_info;  in scrub_bio_end_io()
2361 	sbio->status = bio->bi_status;  in scrub_bio_end_io()
2362 	sbio->bio = bio;  in scrub_bio_end_io()
2364 	btrfs_queue_work(fs_info->scrub_workers, &sbio->work);  in scrub_bio_end_io()
2369 	struct scrub_bio *sbio = container_of(work, struct scrub_bio, work);  in scrub_bio_end_io_worker()  local
2370 	struct scrub_ctx *sctx = sbio->sctx;  in scrub_bio_end_io_worker()
2373 	BUG_ON(sbio->page_count > SCRUB_PAGES_PER_RD_BIO);  in scrub_bio_end_io_worker()
2374 	if (sbio->status) {  in scrub_bio_end_io_worker()
2375 		for (i = 0; i < sbio->page_count; i++) {  in scrub_bio_end_io_worker()
2376 			struct scrub_page *spage = sbio->pagev[i];  in scrub_bio_end_io_worker()
2384 	for (i = 0; i < sbio->page_count; i++) {  in scrub_bio_end_io_worker()
2385 		struct scrub_page *spage = sbio->pagev[i];  in scrub_bio_end_io_worker()
2393 	bio_put(sbio->bio);  in scrub_bio_end_io_worker()
2394 	sbio->bio = NULL;  in scrub_bio_end_io_worker()
2396 	sbio->next_free = sctx->first_free;  in scrub_bio_end_io_worker()
2397 	sctx->first_free = sbio->index;  in scrub_bio_end_io_worker()