3547 				      struct find_free_extent_ctl *ffe_ctl,  in find_free_extent_clustered()  argument
3551 	struct btrfs_free_cluster *last_ptr = ffe_ctl->last_ptr;  in find_free_extent_clustered()
3556 	cluster_bg = btrfs_lock_cluster(bg, last_ptr, ffe_ctl->delalloc);  in find_free_extent_clustered()
3560 	    !block_group_bits(cluster_bg, ffe_ctl->flags)))  in find_free_extent_clustered()
3564 			ffe_ctl->num_bytes, cluster_bg->start,  in find_free_extent_clustered()
3565 			&ffe_ctl->max_extent_size);  in find_free_extent_clustered()
3570 				ffe_ctl->search_start, ffe_ctl->num_bytes);  in find_free_extent_clustered()
3572 		ffe_ctl->found_offset = offset;  in find_free_extent_clustered()
3589 	if (ffe_ctl->loop >= LOOP_NO_EMPTY_SIZE && cluster_bg != bg) {  in find_free_extent_clustered()
3591 		btrfs_release_block_group(cluster_bg, ffe_ctl->delalloc);  in find_free_extent_clustered()
3599 		btrfs_release_block_group(cluster_bg, ffe_ctl->delalloc);  in find_free_extent_clustered()
3602 	if (ffe_ctl->loop >= LOOP_NO_EMPTY_SIZE) {  in find_free_extent_clustered()
3608 			ffe_ctl->empty_cluster + ffe_ctl->empty_size,  in find_free_extent_clustered()
3610 	ret = btrfs_find_space_cluster(bg, last_ptr, ffe_ctl->search_start,  in find_free_extent_clustered()
3611 			ffe_ctl->num_bytes, aligned_cluster);  in find_free_extent_clustered()
3615 				ffe_ctl->num_bytes, ffe_ctl->search_start,  in find_free_extent_clustered()
3616 				&ffe_ctl->max_extent_size);  in find_free_extent_clustered()
3621 					ffe_ctl->search_start,  in find_free_extent_clustered()
3622 					ffe_ctl->num_bytes);  in find_free_extent_clustered()
3623 			ffe_ctl->found_offset = offset;  in find_free_extent_clustered()
3626 	} else if (!ffe_ctl->cached && ffe_ctl->loop > LOOP_CACHING_NOWAIT &&  in find_free_extent_clustered()
3627 		   !ffe_ctl->retry_clustered) {  in find_free_extent_clustered()
3630 		ffe_ctl->retry_clustered = true;  in find_free_extent_clustered()
3631 		btrfs_wait_block_group_cache_progress(bg, ffe_ctl->num_bytes +  in find_free_extent_clustered()
3632 				ffe_ctl->empty_cluster + ffe_ctl->empty_size);  in find_free_extent_clustered()
3651 					struct find_free_extent_ctl *ffe_ctl)  in find_free_extent_unclustered()  argument
3653 	struct btrfs_free_cluster *last_ptr = ffe_ctl->last_ptr;  in find_free_extent_unclustered()
3666 	if (ffe_ctl->cached) {  in find_free_extent_unclustered()
3672 		    ffe_ctl->num_bytes + ffe_ctl->empty_cluster +  in find_free_extent_unclustered()
3673 		    ffe_ctl->empty_size) {  in find_free_extent_unclustered()
3674 			ffe_ctl->total_free_space = max_t(u64,  in find_free_extent_unclustered()
3675 					ffe_ctl->total_free_space,  in find_free_extent_unclustered()
3683 	offset = btrfs_find_space_for_alloc(bg, ffe_ctl->search_start,  in find_free_extent_unclustered()
3684 			ffe_ctl->num_bytes, ffe_ctl->empty_size,  in find_free_extent_unclustered()
3685 			&ffe_ctl->max_extent_size);  in find_free_extent_unclustered()
3696 	if (!offset && !ffe_ctl->retry_unclustered && !ffe_ctl->cached &&  in find_free_extent_unclustered()
3697 	    ffe_ctl->loop > LOOP_CACHING_NOWAIT) {  in find_free_extent_unclustered()
3698 		btrfs_wait_block_group_cache_progress(bg, ffe_ctl->num_bytes +  in find_free_extent_unclustered()
3699 						      ffe_ctl->empty_size);  in find_free_extent_unclustered()
3700 		ffe_ctl->retry_unclustered = true;  in find_free_extent_unclustered()
3705 	ffe_ctl->found_offset = offset;  in find_free_extent_unclustered()
3710 				   struct find_free_extent_ctl *ffe_ctl,  in do_allocation_clustered()  argument
3716 	if (ffe_ctl->last_ptr && ffe_ctl->use_cluster) {  in do_allocation_clustered()
3717 		ret = find_free_extent_clustered(block_group, ffe_ctl, bg_ret);  in do_allocation_clustered()
3723 	return find_free_extent_unclustered(block_group, ffe_ctl);  in do_allocation_clustered()
3748 			       struct find_free_extent_ctl *ffe_ctl,  in do_allocation_zoned()  argument
3755 	u64 num_bytes = ffe_ctl->num_bytes;  in do_allocation_zoned()
3770 	skip = log_bytenr && ((ffe_ctl->for_treelog && bytenr != log_bytenr) ||  in do_allocation_zoned()
3771 			      (!ffe_ctl->for_treelog && bytenr == log_bytenr));  in do_allocation_zoned()
3780 	ASSERT(!ffe_ctl->for_treelog ||  in do_allocation_zoned()
3793 	if (ffe_ctl->for_treelog && !fs_info->treelog_bg &&  in do_allocation_zoned()
3801 		if (ffe_ctl->max_extent_size < avail) {  in do_allocation_zoned()
3806 			ffe_ctl->max_extent_size = avail;  in do_allocation_zoned()
3807 			ffe_ctl->total_free_space = avail;  in do_allocation_zoned()
3813 	if (ffe_ctl->for_treelog && !fs_info->treelog_bg)  in do_allocation_zoned()
3816 	ffe_ctl->found_offset = start + block_group->alloc_offset;  in do_allocation_zoned()
3827 	ffe_ctl->search_start = ffe_ctl->found_offset;  in do_allocation_zoned()
3830 	if (ret && ffe_ctl->for_treelog)  in do_allocation_zoned()
3839 			 struct find_free_extent_ctl *ffe_ctl,  in do_allocation()  argument
3842 	switch (ffe_ctl->policy) {  in do_allocation()
3844 		return do_allocation_clustered(block_group, ffe_ctl, bg_ret);  in do_allocation()
3846 		return do_allocation_zoned(block_group, ffe_ctl, bg_ret);  in do_allocation()
3853 				struct find_free_extent_ctl *ffe_ctl,  in release_block_group()  argument
3856 	switch (ffe_ctl->policy) {  in release_block_group()
3858 		ffe_ctl->retry_clustered = false;  in release_block_group()
3859 		ffe_ctl->retry_unclustered = false;  in release_block_group()
3869 	       ffe_ctl->index);  in release_block_group()
3873 static void found_extent_clustered(struct find_free_extent_ctl *ffe_ctl,  in found_extent_clustered()  argument
3876 	struct btrfs_free_cluster *last_ptr = ffe_ctl->last_ptr;  in found_extent_clustered()
3878 	if (!ffe_ctl->use_cluster && last_ptr) {  in found_extent_clustered()
3885 static void found_extent(struct find_free_extent_ctl *ffe_ctl,  in found_extent()  argument
3888 	switch (ffe_ctl->policy) {  in found_extent()
3890 		found_extent_clustered(ffe_ctl, ins);  in found_extent()
3900 static int chunk_allocation_failed(struct find_free_extent_ctl *ffe_ctl)  in chunk_allocation_failed()  argument
3902 	switch (ffe_ctl->policy) {  in chunk_allocation_failed()
3908 		ffe_ctl->loop = LOOP_NO_EMPTY_SIZE;  in chunk_allocation_failed()
3925 					struct find_free_extent_ctl *ffe_ctl,  in find_free_extent_update_loop()  argument
3931 	if ((ffe_ctl->loop == LOOP_CACHING_NOWAIT) &&  in find_free_extent_update_loop()
3932 	    ffe_ctl->have_caching_bg && !ffe_ctl->orig_have_caching_bg)  in find_free_extent_update_loop()
3933 		ffe_ctl->orig_have_caching_bg = true;  in find_free_extent_update_loop()
3935 	if (!ins->objectid && ffe_ctl->loop >= LOOP_CACHING_WAIT &&  in find_free_extent_update_loop()
3936 	    ffe_ctl->have_caching_bg)  in find_free_extent_update_loop()
3939 	if (!ins->objectid && ++(ffe_ctl->index) < BTRFS_NR_RAID_TYPES)  in find_free_extent_update_loop()
3943 		found_extent(ffe_ctl, ins);  in find_free_extent_update_loop()
3955 	if (ffe_ctl->loop < LOOP_NO_EMPTY_SIZE) {  in find_free_extent_update_loop()
3956 		ffe_ctl->index = 0;  in find_free_extent_update_loop()
3957 		if (ffe_ctl->loop == LOOP_CACHING_NOWAIT) {  in find_free_extent_update_loop()
3963 			if (ffe_ctl->orig_have_caching_bg || !full_search)  in find_free_extent_update_loop()
3964 				ffe_ctl->loop = LOOP_CACHING_WAIT;  in find_free_extent_update_loop()
3966 				ffe_ctl->loop = LOOP_ALLOC_CHUNK;  in find_free_extent_update_loop()
3968 			ffe_ctl->loop++;  in find_free_extent_update_loop()
3971 		if (ffe_ctl->loop == LOOP_ALLOC_CHUNK) {  in find_free_extent_update_loop()
3986 			ret = btrfs_chunk_alloc(trans, ffe_ctl->flags,  in find_free_extent_update_loop()
3991 				ret = chunk_allocation_failed(ffe_ctl);  in find_free_extent_update_loop()
4002 		if (ffe_ctl->loop == LOOP_NO_EMPTY_SIZE) {  in find_free_extent_update_loop()
4003 			if (ffe_ctl->policy != BTRFS_EXTENT_ALLOC_CLUSTERED)  in find_free_extent_update_loop()
4010 			if (ffe_ctl->empty_size == 0 &&  in find_free_extent_update_loop()
4011 			    ffe_ctl->empty_cluster == 0)  in find_free_extent_update_loop()
4013 			ffe_ctl->empty_size = 0;  in find_free_extent_update_loop()
4014 			ffe_ctl->empty_cluster = 0;  in find_free_extent_update_loop()
4022 					struct find_free_extent_ctl *ffe_ctl,  in prepare_allocation_clustered()  argument
4039 		    ffe_ctl->num_bytes > space_info->max_extent_size) {  in prepare_allocation_clustered()
4044 			ffe_ctl->use_cluster = false;  in prepare_allocation_clustered()
4049 	ffe_ctl->last_ptr = fetch_cluster_info(fs_info, space_info,  in prepare_allocation_clustered()
4050 					       &ffe_ctl->empty_cluster);  in prepare_allocation_clustered()
4051 	if (ffe_ctl->last_ptr) {  in prepare_allocation_clustered()
4052 		struct btrfs_free_cluster *last_ptr = ffe_ctl->last_ptr;  in prepare_allocation_clustered()
4056 			ffe_ctl->hint_byte = last_ptr->window_start;  in prepare_allocation_clustered()
4063 			ffe_ctl->hint_byte = last_ptr->window_start;  in prepare_allocation_clustered()
4064 			ffe_ctl->use_cluster = false;  in prepare_allocation_clustered()
4073 			      struct find_free_extent_ctl *ffe_ctl,  in prepare_allocation()  argument
4077 	switch (ffe_ctl->policy) {  in prepare_allocation()
4079 		return prepare_allocation_clustered(fs_info, ffe_ctl,  in prepare_allocation()
4082 		if (ffe_ctl->for_treelog) {  in prepare_allocation()
4085 				ffe_ctl->hint_byte = fs_info->treelog_bg;  in prepare_allocation()
4128 	struct find_free_extent_ctl ffe_ctl = {0};  in find_free_extent()  local
4135 	ffe_ctl.num_bytes = num_bytes;  in find_free_extent()
4136 	ffe_ctl.empty_size = empty_size;  in find_free_extent()
4137 	ffe_ctl.flags = flags;  in find_free_extent()
4138 	ffe_ctl.search_start = 0;  in find_free_extent()
4139 	ffe_ctl.delalloc = delalloc;  in find_free_extent()
4140 	ffe_ctl.index = btrfs_bg_flags_to_raid_index(flags);  in find_free_extent()
4141 	ffe_ctl.have_caching_bg = false;  in find_free_extent()
4142 	ffe_ctl.orig_have_caching_bg = false;  in find_free_extent()
4143 	ffe_ctl.found_offset = 0;  in find_free_extent()
4144 	ffe_ctl.hint_byte = hint_byte_orig;  in find_free_extent()
4145 	ffe_ctl.for_treelog = for_treelog;  in find_free_extent()
4146 	ffe_ctl.policy = BTRFS_EXTENT_ALLOC_CLUSTERED;  in find_free_extent()
4149 	ffe_ctl.retry_clustered = false;  in find_free_extent()
4150 	ffe_ctl.retry_unclustered = false;  in find_free_extent()
4151 	ffe_ctl.last_ptr = NULL;  in find_free_extent()
4152 	ffe_ctl.use_cluster = true;  in find_free_extent()
4155 		ffe_ctl.policy = BTRFS_EXTENT_ALLOC_ZONED;  in find_free_extent()
4169 	ret = prepare_allocation(fs_info, &ffe_ctl, space_info, ins);  in find_free_extent()
4173 	ffe_ctl.search_start = max(ffe_ctl.search_start,  in find_free_extent()
4175 	ffe_ctl.search_start = max(ffe_ctl.search_start, ffe_ctl.hint_byte);  in find_free_extent()
4176 	if (ffe_ctl.search_start == ffe_ctl.hint_byte) {  in find_free_extent()
4178 						       ffe_ctl.search_start);  in find_free_extent()
4200 				ffe_ctl.index = btrfs_bg_flags_to_raid_index(  in find_free_extent()
4210 	ffe_ctl.have_caching_bg = false;  in find_free_extent()
4211 	if (ffe_ctl.index == btrfs_bg_flags_to_raid_index(flags) ||  in find_free_extent()
4212 	    ffe_ctl.index == 0)  in find_free_extent()
4216 			    &space_info->block_groups[ffe_ctl.index], list) {  in find_free_extent()
4227 		ffe_ctl.search_start = block_group->start;  in find_free_extent()
4258 		ffe_ctl.cached = btrfs_block_group_done(block_group);  in find_free_extent()
4259 		if (unlikely(!ffe_ctl.cached)) {  in find_free_extent()
4260 			ffe_ctl.have_caching_bg = true;  in find_free_extent()
4283 		ret = do_allocation(block_group, &ffe_ctl, &bg_ret);  in find_free_extent()
4296 		ffe_ctl.search_start = round_up(ffe_ctl.found_offset,  in find_free_extent()
4300 		if (ffe_ctl.search_start + num_bytes >  in find_free_extent()
4303 					    ffe_ctl.found_offset, num_bytes);  in find_free_extent()
4307 		if (ffe_ctl.found_offset < ffe_ctl.search_start)  in find_free_extent()
4309 					ffe_ctl.found_offset,  in find_free_extent()
4310 					ffe_ctl.search_start - ffe_ctl.found_offset);  in find_free_extent()
4316 					ffe_ctl.found_offset, num_bytes);  in find_free_extent()
4322 		ins->objectid = ffe_ctl.search_start;  in find_free_extent()
4325 		trace_btrfs_reserve_extent(block_group, ffe_ctl.search_start,  in find_free_extent()
4330 		release_block_group(block_group, &ffe_ctl, delalloc);  in find_free_extent()
4335 	ret = find_free_extent_update_loop(fs_info, ins, &ffe_ctl, full_search);  in find_free_extent()
4344 		if (!ffe_ctl.max_extent_size)  in find_free_extent()
4345 			ffe_ctl.max_extent_size = ffe_ctl.total_free_space;  in find_free_extent()
4347 		space_info->max_extent_size = ffe_ctl.max_extent_size;  in find_free_extent()
4349 		ins->offset = ffe_ctl.max_extent_size;  in find_free_extent()