66 static int __raid56_parity_recover(struct btrfs_raid_bio *rbio);
67 static noinline void finish_rmw(struct btrfs_raid_bio *rbio);
70 static int fail_bio_stripe(struct btrfs_raid_bio *rbio, struct bio *bio);
71 static int fail_rbio_index(struct btrfs_raid_bio *rbio, int failed);
72 static void __free_raid_bio(struct btrfs_raid_bio *rbio);
73 static void index_rbio_pages(struct btrfs_raid_bio *rbio);
74 static int alloc_rbio_pages(struct btrfs_raid_bio *rbio);
76 static noinline void finish_parity_scrub(struct btrfs_raid_bio *rbio,
80 static void start_async_work(struct btrfs_raid_bio *rbio, work_func_t work_func)  in start_async_work()
138 static void cache_rbio_pages(struct btrfs_raid_bio *rbio)  in cache_rbio_pages()
166 static int rbio_bucket(struct btrfs_raid_bio *rbio)  in rbio_bucket()
181 static bool full_page_sectors_uptodate(struct btrfs_raid_bio *rbio,  in full_page_sectors_uptodate()
204 static void index_stripe_sectors(struct btrfs_raid_bio *rbio)  in index_stripe_sectors()
219 static void steal_rbio_page(struct btrfs_raid_bio *src,  in steal_rbio_page()
220 			    struct btrfs_raid_bio *dest, int page_nr)  in steal_rbio_page()
244 static void steal_rbio(struct btrfs_raid_bio *src, struct btrfs_raid_bio *dest)  in steal_rbio()
270 static void merge_rbio(struct btrfs_raid_bio *dest,  in merge_rbio()
271 		       struct btrfs_raid_bio *victim)  in merge_rbio()
285 static void __remove_rbio_from_cache(struct btrfs_raid_bio *rbio)  in __remove_rbio_from_cache()
345 static void remove_rbio_from_cache(struct btrfs_raid_bio *rbio)  in remove_rbio_from_cache()
367 	struct btrfs_raid_bio *rbio;  in btrfs_clear_rbio_cache()
374 				  struct btrfs_raid_bio,  in btrfs_clear_rbio_cache()
405 static void cache_rbio(struct btrfs_raid_bio *rbio)  in cache_rbio()
432 		struct btrfs_raid_bio *found;  in cache_rbio()
435 				  struct btrfs_raid_bio,  in cache_rbio()
469 static int rbio_is_full(struct btrfs_raid_bio *rbio)  in rbio_is_full()
494 static int rbio_can_merge(struct btrfs_raid_bio *last,  in rbio_can_merge()
495 			  struct btrfs_raid_bio *cur)  in rbio_can_merge()
554 static unsigned int rbio_stripe_sector_index(const struct btrfs_raid_bio *rbio,  in rbio_stripe_sector_index()
565 static struct sector_ptr *rbio_stripe_sector(const struct btrfs_raid_bio *rbio,  in rbio_stripe_sector()
574 static struct sector_ptr *rbio_pstripe_sector(const struct btrfs_raid_bio *rbio,  in rbio_pstripe_sector()
581 static struct sector_ptr *rbio_qstripe_sector(const struct btrfs_raid_bio *rbio,  in rbio_qstripe_sector()
611 static noinline int lock_stripe_add(struct btrfs_raid_bio *rbio)  in lock_stripe_add()
614 	struct btrfs_raid_bio *cur;  in lock_stripe_add()
615 	struct btrfs_raid_bio *pending;  in lock_stripe_add()
617 	struct btrfs_raid_bio *freeit = NULL;  in lock_stripe_add()
618 	struct btrfs_raid_bio *cache_drop = NULL;  in lock_stripe_add()
695 static noinline void unlock_stripe(struct btrfs_raid_bio *rbio)  in unlock_stripe()
734 			struct btrfs_raid_bio *next;  in unlock_stripe()
737 			next = list_entry(head, struct btrfs_raid_bio,  in unlock_stripe()
772 static void __free_raid_bio(struct btrfs_raid_bio *rbio)  in __free_raid_bio()
811 static void rbio_orig_end_io(struct btrfs_raid_bio *rbio, blk_status_t err)  in rbio_orig_end_io()
846 	struct btrfs_raid_bio *rbio = bio->bi_private;  in raid_write_end_io()
881 static struct sector_ptr *sector_in_rbio(struct btrfs_raid_bio *rbio,  in sector_in_rbio()
912 static struct btrfs_raid_bio *alloc_rbio(struct btrfs_fs_info *fs_info,  in alloc_rbio()
921 	struct btrfs_raid_bio *rbio;  in alloc_rbio()
981 static int alloc_rbio_pages(struct btrfs_raid_bio *rbio)  in alloc_rbio_pages()
994 static int alloc_rbio_parity_pages(struct btrfs_raid_bio *rbio)  in alloc_rbio_parity_pages()
1014 static int rbio_add_io_sector(struct btrfs_raid_bio *rbio,  in rbio_add_io_sector()
1081 static void validate_rbio_for_rmw(struct btrfs_raid_bio *rbio)  in validate_rbio_for_rmw()
1091 static void index_one_bio(struct btrfs_raid_bio *rbio, struct bio *bio)  in index_one_bio()
1122 static void index_rbio_pages(struct btrfs_raid_bio *rbio)  in index_rbio_pages()
1133 static void bio_get_trace_info(struct btrfs_raid_bio *rbio, struct bio *bio,  in bio_get_trace_info()
1169 static noinline void finish_rmw(struct btrfs_raid_bio *rbio)  in finish_rmw()
1359 static int find_bio_stripe(struct btrfs_raid_bio *rbio,  in find_bio_stripe()
1383 static int find_logical_bio_stripe(struct btrfs_raid_bio *rbio,  in find_logical_bio_stripe()
1401 static int fail_rbio_index(struct btrfs_raid_bio *rbio, int failed)  in fail_rbio_index()
1433 static int fail_bio_stripe(struct btrfs_raid_bio *rbio,  in fail_bio_stripe()
1448 static struct sector_ptr *find_stripe_sector(struct btrfs_raid_bio *rbio,  in find_stripe_sector()
1467 static void set_bio_pages_uptodate(struct btrfs_raid_bio *rbio, struct bio *bio)  in set_bio_pages_uptodate()
1491 	struct btrfs_raid_bio *rbio = bio->bi_private;  in raid56_bio_end_io()
1515 	struct btrfs_raid_bio *rbio =  in raid56_rmw_end_io_work()
1516 		container_of(work, struct btrfs_raid_bio, end_io_work);  in raid56_rmw_end_io_work()
1534 static int raid56_rmw_stripe(struct btrfs_raid_bio *rbio)  in raid56_rmw_stripe()
1630 static int full_stripe_write(struct btrfs_raid_bio *rbio)  in full_stripe_write()
1649 static int partial_stripe_write(struct btrfs_raid_bio *rbio)  in partial_stripe_write()
1665 static int __raid56_parity_write(struct btrfs_raid_bio *rbio)  in __raid56_parity_write()
1693 	const struct btrfs_raid_bio *ra = container_of(a, struct btrfs_raid_bio,  in plug_cmp()
1695 	const struct btrfs_raid_bio *rb = container_of(b, struct btrfs_raid_bio,  in plug_cmp()
1709 	struct btrfs_raid_bio *cur;  in run_plug()
1710 	struct btrfs_raid_bio *last = NULL;  in run_plug()
1720 				 struct btrfs_raid_bio, plug_list);  in run_plug()
1773 static void rbio_add_bio(struct btrfs_raid_bio *rbio, struct bio *orig_bio)  in rbio_add_bio()
1805 	struct btrfs_raid_bio *rbio;  in raid56_parity_write()
1859 static void __raid_recover_end_io(struct btrfs_raid_bio *rbio)  in __raid_recover_end_io()
2083 	struct btrfs_raid_bio *rbio =  in raid_recover_end_io_work()
2084 		container_of(work, struct btrfs_raid_bio, end_io_work);  in raid_recover_end_io_work()
2100 static int __raid56_parity_recover(struct btrfs_raid_bio *rbio)  in __raid56_parity_recover()
2200 	struct btrfs_raid_bio *rbio;  in raid56_parity_recover()
2255 	struct btrfs_raid_bio *rbio;  in rmw_work()
2257 	rbio = container_of(work, struct btrfs_raid_bio, work);  in rmw_work()
2263 	struct btrfs_raid_bio *rbio;  in read_rebuild_work()
2265 	rbio = container_of(work, struct btrfs_raid_bio, work);  in read_rebuild_work()
2279 struct btrfs_raid_bio *raid56_parity_alloc_scrub_rbio(struct bio *bio,  in raid56_parity_alloc_scrub_rbio()
2285 	struct btrfs_raid_bio *rbio;  in raid56_parity_alloc_scrub_rbio()
2317 void raid56_add_scrub_pages(struct btrfs_raid_bio *rbio, struct page *page,  in raid56_add_scrub_pages()
2337 static int alloc_rbio_essential_pages(struct btrfs_raid_bio *rbio)  in alloc_rbio_essential_pages()
2361 static noinline void finish_parity_scrub(struct btrfs_raid_bio *rbio,  in finish_parity_scrub()
2531 static inline int is_data_stripe(struct btrfs_raid_bio *rbio, int stripe)  in is_data_stripe()
2545 static void validate_rbio_for_parity_scrub(struct btrfs_raid_bio *rbio)  in validate_rbio_for_parity_scrub()
2609 	struct btrfs_raid_bio *rbio =  in raid56_parity_scrub_end_io_work()
2610 		container_of(work, struct btrfs_raid_bio, end_io_work);  in raid56_parity_scrub_end_io_work()
2619 static void raid56_parity_scrub_stripe(struct btrfs_raid_bio *rbio)  in raid56_parity_scrub_stripe()
2713 	struct btrfs_raid_bio *rbio;  in scrub_parity_work()
2715 	rbio = container_of(work, struct btrfs_raid_bio, work);  in scrub_parity_work()
2719 void raid56_parity_submit_scrub_rbio(struct btrfs_raid_bio *rbio)  in raid56_parity_submit_scrub_rbio()
2727 struct btrfs_raid_bio *
2731 	struct btrfs_raid_bio *rbio;  in raid56_alloc_missing_rbio()
2757 void raid56_submit_missing_rbio(struct btrfs_raid_bio *rbio)  in raid56_submit_missing_rbio()