5798 static void sort_parity_stripes(struct btrfs_bio *bbio, int num_stripes)  in sort_parity_stripes()  argument
5807 			if (bbio->raid_map[i] > bbio->raid_map[i + 1]) {  in sort_parity_stripes()
5808 				swap(bbio->stripes[i], bbio->stripes[i + 1]);  in sort_parity_stripes()
5809 				swap(bbio->raid_map[i], bbio->raid_map[i + 1]);  in sort_parity_stripes()
5818 	struct btrfs_bio *bbio = kzalloc(  in alloc_btrfs_bio()  local
5832 	atomic_set(&bbio->error, 0);  in alloc_btrfs_bio()
5833 	refcount_set(&bbio->refs, 1);  in alloc_btrfs_bio()
5835 	bbio->tgtdev_map = (int *)(bbio->stripes + total_stripes);  in alloc_btrfs_bio()
5836 	bbio->raid_map = (u64 *)(bbio->tgtdev_map + real_stripes);  in alloc_btrfs_bio()
5838 	return bbio;  in alloc_btrfs_bio()
5841 void btrfs_get_bbio(struct btrfs_bio *bbio)  in btrfs_get_bbio()  argument
5843 	WARN_ON(!refcount_read(&bbio->refs));  in btrfs_get_bbio()
5844 	refcount_inc(&bbio->refs);  in btrfs_get_bbio()
5847 void btrfs_put_bbio(struct btrfs_bio *bbio)  in btrfs_put_bbio()  argument
5849 	if (!bbio)  in btrfs_put_bbio()
5851 	if (refcount_dec_and_test(&bbio->refs))  in btrfs_put_bbio()
5852 		kfree(bbio);  in btrfs_put_bbio()
5866 	struct btrfs_bio *bbio;  in __btrfs_map_block_for_discard()  local
5949 	bbio = alloc_btrfs_bio(num_stripes, 0);  in __btrfs_map_block_for_discard()
5950 	if (!bbio) {  in __btrfs_map_block_for_discard()
5956 		bbio->stripes[i].physical =  in __btrfs_map_block_for_discard()
5959 		bbio->stripes[i].dev = map->stripes[stripe_index].dev;  in __btrfs_map_block_for_discard()
5963 			bbio->stripes[i].length = stripes_per_dev *  in __btrfs_map_block_for_discard()
5967 				bbio->stripes[i].length +=  in __btrfs_map_block_for_discard()
5979 				bbio->stripes[i].length -=  in __btrfs_map_block_for_discard()
5985 				bbio->stripes[i].length -=  in __btrfs_map_block_for_discard()
5991 			bbio->stripes[i].length = length;  in __btrfs_map_block_for_discard()
6001 	*bbio_ret = bbio;  in __btrfs_map_block_for_discard()
6002 	bbio->map_type = map->type;  in __btrfs_map_block_for_discard()
6003 	bbio->num_stripes = num_stripes;  in __btrfs_map_block_for_discard()
6027 	struct btrfs_bio *bbio = NULL;  in get_extra_mirror_from_replace()  local
6036 				logical, &length, &bbio, 0, 0);  in get_extra_mirror_from_replace()
6038 		ASSERT(bbio == NULL);  in get_extra_mirror_from_replace()
6042 	num_stripes = bbio->num_stripes;  in get_extra_mirror_from_replace()
6049 		btrfs_put_bbio(bbio);  in get_extra_mirror_from_replace()
6059 		if (bbio->stripes[i].dev->devid != srcdev_devid)  in get_extra_mirror_from_replace()
6067 		    physical_of_found <= bbio->stripes[i].physical)  in get_extra_mirror_from_replace()
6072 		physical_of_found = bbio->stripes[i].physical;  in get_extra_mirror_from_replace()
6075 	btrfs_put_bbio(bbio);  in get_extra_mirror_from_replace()
6111 	struct btrfs_bio *bbio = *bbio_ret;  in handle_ops_on_dev_replace()  local
6141 			if (bbio->stripes[i].dev->devid == srcdev_devid) {  in handle_ops_on_dev_replace()
6144 					bbio->stripes + index_where_to_add;  in handle_ops_on_dev_replace()
6146 					bbio->stripes + i;  in handle_ops_on_dev_replace()
6151 				bbio->tgtdev_map[i] = index_where_to_add;  in handle_ops_on_dev_replace()
6171 			if (bbio->stripes[i].dev->devid == srcdev_devid) {  in handle_ops_on_dev_replace()
6179 				     bbio->stripes[i].physical)  in handle_ops_on_dev_replace()
6183 				physical_of_found = bbio->stripes[i].physical;  in handle_ops_on_dev_replace()
6188 				bbio->stripes + num_stripes;  in handle_ops_on_dev_replace()
6192 				bbio->stripes[index_srcdev].length;  in handle_ops_on_dev_replace()
6194 			bbio->tgtdev_map[index_srcdev] = num_stripes;  in handle_ops_on_dev_replace()
6203 	bbio->num_tgtdevs = tgtdev_indexes;  in handle_ops_on_dev_replace()
6204 	*bbio_ret = bbio;  in handle_ops_on_dev_replace()
6322 	struct btrfs_bio *bbio = NULL;  in __btrfs_map_block()  local
6475 	bbio = alloc_btrfs_bio(num_alloc_stripes, tgtdev_indexes);  in __btrfs_map_block()
6476 	if (!bbio) {  in __btrfs_map_block()
6482 		bbio->stripes[i].physical = map->stripes[stripe_index].physical +  in __btrfs_map_block()
6484 		bbio->stripes[i].dev = map->stripes[stripe_index].dev;  in __btrfs_map_block()
6500 			bbio->raid_map[(i+rot) % num_stripes] =  in __btrfs_map_block()
6503 		bbio->raid_map[(i+rot) % map->num_stripes] = RAID5_P_STRIPE;  in __btrfs_map_block()
6505 			bbio->raid_map[(i+rot+1) % num_stripes] =  in __btrfs_map_block()
6508 		sort_parity_stripes(bbio, num_stripes);  in __btrfs_map_block()
6516 		handle_ops_on_dev_replace(op, &bbio, dev_replace, logical,  in __btrfs_map_block()
6520 	*bbio_ret = bbio;  in __btrfs_map_block()
6521 	bbio->map_type = map->type;  in __btrfs_map_block()
6522 	bbio->num_stripes = num_stripes;  in __btrfs_map_block()
6523 	bbio->max_errors = max_errors;  in __btrfs_map_block()
6524 	bbio->mirror_num = mirror_num;  in __btrfs_map_block()
6533 		bbio->stripes[0].dev = dev_replace->tgtdev;  in __btrfs_map_block()
6534 		bbio->stripes[0].physical = physical_to_patch_in_first_stripe;  in __btrfs_map_block()
6535 		bbio->mirror_num = map->num_stripes + 1;  in __btrfs_map_block()
6567 static inline void btrfs_end_bbio(struct btrfs_bio *bbio, struct bio *bio)  in btrfs_end_bbio()  argument
6569 	bio->bi_private = bbio->private;  in btrfs_end_bbio()
6570 	bio->bi_end_io = bbio->end_io;  in btrfs_end_bbio()
6573 	btrfs_put_bbio(bbio);  in btrfs_end_bbio()
6578 	struct btrfs_bio *bbio = bio->bi_private;  in btrfs_end_bio()  local
6582 		atomic_inc(&bbio->error);  in btrfs_end_bio()
6600 	if (bio == bbio->orig_bio)  in btrfs_end_bio()
6603 	btrfs_bio_counter_dec(bbio->fs_info);  in btrfs_end_bio()
6605 	if (atomic_dec_and_test(&bbio->stripes_pending)) {  in btrfs_end_bio()
6608 			bio = bbio->orig_bio;  in btrfs_end_bio()
6611 		btrfs_io_bio(bio)->mirror_num = bbio->mirror_num;  in btrfs_end_bio()
6615 		if (atomic_read(&bbio->error) > bbio->max_errors) {  in btrfs_end_bio()
6625 		btrfs_end_bbio(bbio, bio);  in btrfs_end_bio()
6631 static void submit_stripe_bio(struct btrfs_bio *bbio, struct bio *bio,  in submit_stripe_bio()  argument
6634 	struct btrfs_fs_info *fs_info = bbio->fs_info;  in submit_stripe_bio()
6636 	bio->bi_private = bbio;  in submit_stripe_bio()
6666 static void bbio_error(struct btrfs_bio *bbio, struct bio *bio, u64 logical)  in bbio_error()  argument
6668 	atomic_inc(&bbio->error);  in bbio_error()
6669 	if (atomic_dec_and_test(&bbio->stripes_pending)) {  in bbio_error()
6671 		WARN_ON(bio != bbio->orig_bio);  in bbio_error()
6673 		btrfs_io_bio(bio)->mirror_num = bbio->mirror_num;  in bbio_error()
6675 		if (atomic_read(&bbio->error) > bbio->max_errors)  in bbio_error()
6679 		btrfs_end_bbio(bbio, bio);  in bbio_error()
6694 	struct btrfs_bio *bbio = NULL;  in btrfs_map_bio()  local
6701 				&map_length, &bbio, mirror_num, 1);  in btrfs_map_bio()
6707 	total_devs = bbio->num_stripes;  in btrfs_map_bio()
6708 	bbio->orig_bio = first_bio;  in btrfs_map_bio()
6709 	bbio->private = first_bio->bi_private;  in btrfs_map_bio()
6710 	bbio->end_io = first_bio->bi_end_io;  in btrfs_map_bio()
6711 	bbio->fs_info = fs_info;  in btrfs_map_bio()
6712 	atomic_set(&bbio->stripes_pending, bbio->num_stripes);  in btrfs_map_bio()
6714 	if ((bbio->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) &&  in btrfs_map_bio()
6719 			ret = raid56_parity_write(fs_info, bio, bbio,  in btrfs_map_bio()
6722 			ret = raid56_parity_recover(fs_info, bio, bbio,  in btrfs_map_bio()
6738 		dev = bbio->stripes[dev_nr].dev;  in btrfs_map_bio()
6743 			bbio_error(bbio, first_bio, logical);  in btrfs_map_bio()
6752 		submit_stripe_bio(bbio, bio, bbio->stripes[dev_nr].physical, dev);  in btrfs_map_bio()