105 	struct scrub_parity	*sparity;  member
231 static void scrub_parity_get(struct scrub_parity *sparity);
232 static void scrub_parity_put(struct scrub_parity *sparity);
1587 	if (sblock->sparity)  in scrub_write_block_to_dev_replace()
1974 		if (sblock->sparity)  in scrub_block_put()
1975 			scrub_parity_put(sblock->sparity);  in scrub_block_put()
2343 static inline void __scrub_mark_bitmap(struct scrub_parity *sparity,  in __scrub_mark_bitmap()  argument
2350 	int sectorsize = sparity->sctx->fs_info->sectorsize;  in __scrub_mark_bitmap()
2352 	if (len >= sparity->stripe_len) {  in __scrub_mark_bitmap()
2353 		bitmap_set(bitmap, 0, sparity->nsectors);  in __scrub_mark_bitmap()
2357 	start -= sparity->logic_start;  in __scrub_mark_bitmap()
2358 	start = div64_u64_rem(start, sparity->stripe_len, &offset);  in __scrub_mark_bitmap()
2365 	if (offset + nsectors <= sparity->nsectors) {  in __scrub_mark_bitmap()
2370 	bitmap_set(bitmap, offset, sparity->nsectors - offset);  in __scrub_mark_bitmap()
2371 	bitmap_set(bitmap, 0, nsectors - (sparity->nsectors - offset));  in __scrub_mark_bitmap()
2374 static inline void scrub_parity_mark_sectors_error(struct scrub_parity *sparity,  in scrub_parity_mark_sectors_error()  argument
2377 	__scrub_mark_bitmap(sparity, sparity->ebitmap, start, len);  in scrub_parity_mark_sectors_error()
2380 static inline void scrub_parity_mark_sectors_data(struct scrub_parity *sparity,  in scrub_parity_mark_sectors_data()  argument
2383 	__scrub_mark_bitmap(sparity, sparity->dbitmap, start, len);  in scrub_parity_mark_sectors_data()
2404 	if (sblock->sparity && corrupted && !sblock->data_corrected) {  in scrub_block_complete()
2409 		scrub_parity_mark_sectors_error(sblock->sparity,  in scrub_block_complete()
2504 static int scrub_pages_for_parity(struct scrub_parity *sparity,  in scrub_pages_for_parity()  argument
2509 	struct scrub_ctx *sctx = sparity->sctx;  in scrub_pages_for_parity()
2526 	sblock->sparity = sparity;  in scrub_pages_for_parity()
2527 	scrub_parity_get(sparity);  in scrub_pages_for_parity()
2548 		list_add_tail(&spage->list, &sparity->spages);  in scrub_pages_for_parity()
2588 static int scrub_extent_for_parity(struct scrub_parity *sparity,  in scrub_extent_for_parity()  argument
2593 	struct scrub_ctx *sctx = sparity->sctx;  in scrub_extent_for_parity()
2599 		scrub_parity_mark_sectors_error(sparity, logical, len);  in scrub_extent_for_parity()
2604 		blocksize = sparity->stripe_len;  in scrub_extent_for_parity()
2606 		blocksize = sparity->stripe_len;  in scrub_extent_for_parity()
2622 		ret = scrub_pages_for_parity(sparity, logical, l, physical, dev,  in scrub_extent_for_parity()
2679 static void scrub_free_parity(struct scrub_parity *sparity)  in scrub_free_parity()  argument
2681 	struct scrub_ctx *sctx = sparity->sctx;  in scrub_free_parity()
2685 	nbits = bitmap_weight(sparity->ebitmap, sparity->nsectors);  in scrub_free_parity()
2693 	list_for_each_entry_safe(curr, next, &sparity->spages, list) {  in scrub_free_parity()
2698 	kfree(sparity);  in scrub_free_parity()
2703 	struct scrub_parity *sparity = container_of(work, struct scrub_parity,  in scrub_parity_bio_endio_worker()  local
2705 	struct scrub_ctx *sctx = sparity->sctx;  in scrub_parity_bio_endio_worker()
2707 	scrub_free_parity(sparity);  in scrub_parity_bio_endio_worker()
2713 	struct scrub_parity *sparity = (struct scrub_parity *)bio->bi_private;  in scrub_parity_bio_endio()  local
2714 	struct btrfs_fs_info *fs_info = sparity->sctx->fs_info;  in scrub_parity_bio_endio()
2717 		bitmap_or(sparity->ebitmap, sparity->ebitmap, sparity->dbitmap,  in scrub_parity_bio_endio()
2718 			  sparity->nsectors);  in scrub_parity_bio_endio()
2722 	btrfs_init_work(&sparity->work, btrfs_scrubparity_helper,  in scrub_parity_bio_endio()
2724 	btrfs_queue_work(fs_info->scrub_parity_workers, &sparity->work);  in scrub_parity_bio_endio()
2727 static void scrub_parity_check_and_repair(struct scrub_parity *sparity)  in scrub_parity_check_and_repair()  argument
2729 	struct scrub_ctx *sctx = sparity->sctx;  in scrub_parity_check_and_repair()
2737 	if (!bitmap_andnot(sparity->dbitmap, sparity->dbitmap, sparity->ebitmap,  in scrub_parity_check_and_repair()
2738 			   sparity->nsectors))  in scrub_parity_check_and_repair()
2741 	length = sparity->logic_end - sparity->logic_start;  in scrub_parity_check_and_repair()
2744 	ret = btrfs_map_sblock(fs_info, BTRFS_MAP_WRITE, sparity->logic_start,  in scrub_parity_check_and_repair()
2750 	bio->bi_iter.bi_sector = sparity->logic_start >> 9;  in scrub_parity_check_and_repair()
2751 	bio->bi_private = sparity;  in scrub_parity_check_and_repair()
2755 					      length, sparity->scrub_dev,  in scrub_parity_check_and_repair()
2756 					      sparity->dbitmap,  in scrub_parity_check_and_repair()
2757 					      sparity->nsectors);  in scrub_parity_check_and_repair()
2770 	bitmap_or(sparity->ebitmap, sparity->ebitmap, sparity->dbitmap,  in scrub_parity_check_and_repair()
2771 		  sparity->nsectors);  in scrub_parity_check_and_repair()
2776 	scrub_free_parity(sparity);  in scrub_parity_check_and_repair()
2784 static void scrub_parity_get(struct scrub_parity *sparity)  in scrub_parity_get()  argument
2786 	refcount_inc(&sparity->refs);  in scrub_parity_get()
2789 static void scrub_parity_put(struct scrub_parity *sparity)  in scrub_parity_put()  argument
2791 	if (!refcount_dec_and_test(&sparity->refs))  in scrub_parity_put()
2794 	scrub_parity_check_and_repair(sparity);  in scrub_parity_put()
2820 	struct scrub_parity *sparity;  in scrub_raid56_parity()  local
2828 	sparity = kzalloc(sizeof(struct scrub_parity) + 2 * bitmap_len,  in scrub_raid56_parity()
2830 	if (!sparity) {  in scrub_raid56_parity()
2837 	sparity->stripe_len = map->stripe_len;  in scrub_raid56_parity()
2838 	sparity->nsectors = nsectors;  in scrub_raid56_parity()
2839 	sparity->sctx = sctx;  in scrub_raid56_parity()
2840 	sparity->scrub_dev = sdev;  in scrub_raid56_parity()
2841 	sparity->logic_start = logic_start;  in scrub_raid56_parity()
2842 	sparity->logic_end = logic_end;  in scrub_raid56_parity()
2843 	refcount_set(&sparity->refs, 1);  in scrub_raid56_parity()
2844 	INIT_LIST_HEAD(&sparity->spages);  in scrub_raid56_parity()
2845 	sparity->dbitmap = sparity->bitmap;  in scrub_raid56_parity()
2846 	sparity->ebitmap = (void *)sparity->bitmap + bitmap_len;  in scrub_raid56_parity()
2943 			scrub_parity_mark_sectors_data(sparity, extent_logical,  in scrub_raid56_parity()
2971 			ret = scrub_extent_for_parity(sparity, extent_logical,  in scrub_raid56_parity()
3010 		scrub_parity_mark_sectors_error(sparity, logic_start,  in scrub_raid56_parity()
3012 	scrub_parity_put(sparity);  in scrub_raid56_parity()