73 static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio);
101 static inline struct r10bio *get_resync_r10bio(struct bio *bio)  in get_resync_r10bio()
109 	int size = offsetof(struct r10bio, devs[conf->geo.raid_disks]);  in r10bio_pool_alloc()
134 	struct r10bio *r10_bio;  in r10buf_pool_alloc()
232 	struct r10bio *r10bio = __r10_bio;  in r10buf_pool_free()  local
237 		struct bio *bio = r10bio->devs[j].bio;  in r10buf_pool_free()
246 		bio = r10bio->devs[j].repl_bio;  in r10buf_pool_free()
256 	rbio_pool_free(r10bio, conf);  in r10buf_pool_free()
259 static void put_all_bios(struct r10conf *conf, struct r10bio *r10_bio)  in put_all_bios()
275 static void free_r10bio(struct r10bio *r10_bio)  in free_r10bio()
283 static void put_buf(struct r10bio *r10_bio)  in put_buf()
298 static void reschedule_retry(struct r10bio *r10_bio)  in reschedule_retry()
320 static void raid_end_bio_io(struct r10bio *r10_bio)  in raid_end_bio_io()
343 static inline void update_head_pos(int slot, struct r10bio *r10_bio)  in update_head_pos()
354 static int find_bio_disk(struct r10conf *conf, struct r10bio *r10_bio,  in find_bio_disk()
381 	struct r10bio *r10_bio = bio->bi_private;  in raid10_end_read_request()
430 static void close_write(struct r10bio *r10_bio)  in close_write()
440 static void one_write_done(struct r10bio *r10_bio)  in one_write_done()
457 	struct r10bio *r10_bio = bio->bi_private;  in raid10_end_write_request()
589 static void __raid10_find_phys(struct geom *geo, struct r10bio *r10bio)  in __raid10_find_phys()  argument
606 	chunk = r10bio->sector >> geo->chunk_shift;  in __raid10_find_phys()
607 	sector = r10bio->sector & geo->chunk_mask;  in __raid10_find_phys()
622 		r10bio->devs[slot].devnum = d;  in __raid10_find_phys()
623 		r10bio->devs[slot].addr = s;  in __raid10_find_phys()
640 			r10bio->devs[slot].devnum = d;  in __raid10_find_phys()
641 			r10bio->devs[slot].addr = s;  in __raid10_find_phys()
652 static void raid10_find_phys(struct r10conf *conf, struct r10bio *r10bio)  in raid10_find_phys()  argument
657 	    ((r10bio->sector >= conf->reshape_progress) !=  in raid10_find_phys()
659 		set_bit(R10BIO_Previous, &r10bio->state);  in raid10_find_phys()
662 		clear_bit(R10BIO_Previous, &r10bio->state);  in raid10_find_phys()
664 	__raid10_find_phys(geo, r10bio);  in raid10_find_phys()
732 				    struct r10bio *r10_bio,  in read_balance()
1093 static sector_t choose_data_offset(struct r10bio *r10_bio,  in choose_data_offset()
1176 				struct r10bio *r10_bio)  in raid10_read_request()
1270 static void raid10_write_one_disk(struct mddev *mddev, struct r10bio *r10_bio,  in raid10_write_one_disk()
1334 static void wait_blocked_dev(struct mddev *mddev, struct r10bio *r10_bio)  in wait_blocked_dev()
1401 				 struct r10bio *r10_bio)  in raid10_write_request()
1566 	struct r10bio *r10_bio;  in __make_request()
1586 static void raid_end_discard_bio(struct r10bio *r10bio)  in raid_end_discard_bio()  argument
1588 	struct r10conf *conf = r10bio->mddev->private;  in raid_end_discard_bio()
1589 	struct r10bio *first_r10bio;  in raid_end_discard_bio()
1591 	while (atomic_dec_and_test(&r10bio->remaining)) {  in raid_end_discard_bio()
1595 		if (!test_bit(R10BIO_Discard, &r10bio->state)) {  in raid_end_discard_bio()
1596 			first_r10bio = (struct r10bio *)r10bio->master_bio;  in raid_end_discard_bio()
1597 			free_r10bio(r10bio);  in raid_end_discard_bio()
1598 			r10bio = first_r10bio;  in raid_end_discard_bio()
1600 			md_write_end(r10bio->mddev);  in raid_end_discard_bio()
1601 			bio_endio(r10bio->master_bio);  in raid_end_discard_bio()
1602 			free_r10bio(r10bio);  in raid_end_discard_bio()
1610 	struct r10bio *r10_bio = bio->bi_private;  in raid10_end_discard_request()
1651 	struct r10bio *r10_bio, *first_r10bio;  in raid10_handle_discard()
2269 static void __end_sync_read(struct r10bio *r10_bio, struct bio *bio, int d)  in __end_sync_read()
2297 	struct r10bio *r10_bio = get_resync_r10bio(bio);  in end_sync_read()
2307 	struct r10bio *r10_bio = bio->bi_private;  in end_reshape_read()
2312 static void end_sync_request(struct r10bio *r10_bio)  in end_sync_request()
2328 			struct r10bio *r10_bio2 = (struct r10bio *)r10_bio->master_bio;  in end_sync_request()
2341 	struct r10bio *r10_bio = get_resync_r10bio(bio);  in end_sync_write()
2394 static void sync_request_write(struct mddev *mddev, struct r10bio *r10_bio)  in sync_request_write()
2525 static void fix_recovery_read_error(struct r10bio *r10_bio)  in fix_recovery_read_error()
2608 static void recovery_request_write(struct mddev *mddev, struct r10bio *r10_bio)  in recovery_request_write()
2714 static void fix_read_error(struct r10conf *conf, struct mddev *mddev, struct r10bio *r10_bio)  in fix_read_error()
2891 static int narrow_write_error(struct r10bio *r10_bio, int i)  in narrow_write_error()
2952 static void handle_read_error(struct mddev *mddev, struct r10bio *r10_bio)  in handle_read_error()
2986 static void handle_write_completed(struct r10conf *conf, struct r10bio *r10_bio)  in handle_write_completed()
3090 	struct r10bio *r10_bio;  in raid10d()
3110 			r10_bio = list_first_entry(&tmp, struct r10bio,  in raid10d()
3133 		r10_bio = list_entry(head->prev, struct r10bio, retry_list);  in raid10d()
3179 static struct r10bio *raid10_alloc_init_r10buf(struct r10conf *conf)  in raid10_alloc_init_r10buf()
3181 	struct r10bio *r10bio = mempool_alloc(&conf->r10buf_pool, GFP_NOIO);  in raid10_alloc_init_r10buf()  local
3194 		bio = r10bio->devs[i].bio;  in raid10_alloc_init_r10buf()
3198 		bio = r10bio->devs[i].repl_bio;  in raid10_alloc_init_r10buf()
3205 	return r10bio;  in raid10_alloc_init_r10buf()
3281 	struct r10bio *r10_bio;  in raid10_sync_request()
3421 			struct r10bio *rb2;  in raid10_sync_request()
3664 				struct r10bio *rb2 = r10_bio;  in raid10_sync_request()
3665 				r10_bio = (struct r10bio*) rb2->master_bio;  in raid10_sync_request()
4789 	struct r10bio *r10_bio;  in reshape_request()
5024 static void end_reshape_request(struct r10bio *r10_bio);
5026 				     struct r10bio *r10_bio);
5027 static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio)  in reshape_request_write()
5106 				     struct r10bio *r10_bio)  in handle_reshape_read_error()
5111 	struct r10bio *r10b;  in handle_reshape_read_error()
5182 	struct r10bio *r10_bio = get_resync_r10bio(bio);  in end_reshape_write()
5207 static void end_reshape_request(struct r10bio *r10_bio)  in end_reshape_request()