3502 				      struct find_free_extent_ctl *ffe_ctl,  in find_free_extent_clustered()  argument
3506 	struct btrfs_free_cluster *last_ptr = ffe_ctl->last_ptr;  in find_free_extent_clustered()
3511 	cluster_bg = btrfs_lock_cluster(bg, last_ptr, ffe_ctl->delalloc);  in find_free_extent_clustered()
3515 	    !block_group_bits(cluster_bg, ffe_ctl->flags)))  in find_free_extent_clustered()
3519 			ffe_ctl->num_bytes, cluster_bg->start,  in find_free_extent_clustered()
3520 			&ffe_ctl->max_extent_size);  in find_free_extent_clustered()
3524 		trace_btrfs_reserve_extent_cluster(cluster_bg, ffe_ctl);  in find_free_extent_clustered()
3526 		ffe_ctl->found_offset = offset;  in find_free_extent_clustered()
3543 	if (ffe_ctl->loop >= LOOP_NO_EMPTY_SIZE && cluster_bg != bg) {  in find_free_extent_clustered()
3545 		btrfs_release_block_group(cluster_bg, ffe_ctl->delalloc);  in find_free_extent_clustered()
3553 		btrfs_release_block_group(cluster_bg, ffe_ctl->delalloc);  in find_free_extent_clustered()
3556 	if (ffe_ctl->loop >= LOOP_NO_EMPTY_SIZE) {  in find_free_extent_clustered()
3562 			ffe_ctl->empty_cluster + ffe_ctl->empty_size,  in find_free_extent_clustered()
3564 	ret = btrfs_find_space_cluster(bg, last_ptr, ffe_ctl->search_start,  in find_free_extent_clustered()
3565 			ffe_ctl->num_bytes, aligned_cluster);  in find_free_extent_clustered()
3569 				ffe_ctl->num_bytes, ffe_ctl->search_start,  in find_free_extent_clustered()
3570 				&ffe_ctl->max_extent_size);  in find_free_extent_clustered()
3574 			ffe_ctl->found_offset = offset;  in find_free_extent_clustered()
3575 			trace_btrfs_reserve_extent_cluster(bg, ffe_ctl);  in find_free_extent_clustered()
3594 					struct find_free_extent_ctl *ffe_ctl)  in find_free_extent_unclustered()  argument
3596 	struct btrfs_free_cluster *last_ptr = ffe_ctl->last_ptr;  in find_free_extent_unclustered()
3609 	if (ffe_ctl->cached) {  in find_free_extent_unclustered()
3615 		    ffe_ctl->num_bytes + ffe_ctl->empty_cluster +  in find_free_extent_unclustered()
3616 		    ffe_ctl->empty_size) {  in find_free_extent_unclustered()
3617 			ffe_ctl->total_free_space = max_t(u64,  in find_free_extent_unclustered()
3618 					ffe_ctl->total_free_space,  in find_free_extent_unclustered()
3626 	offset = btrfs_find_space_for_alloc(bg, ffe_ctl->search_start,  in find_free_extent_unclustered()
3627 			ffe_ctl->num_bytes, ffe_ctl->empty_size,  in find_free_extent_unclustered()
3628 			&ffe_ctl->max_extent_size);  in find_free_extent_unclustered()
3631 	ffe_ctl->found_offset = offset;  in find_free_extent_unclustered()
3636 				   struct find_free_extent_ctl *ffe_ctl,  in do_allocation_clustered()  argument
3642 	if (ffe_ctl->last_ptr && ffe_ctl->use_cluster) {  in do_allocation_clustered()
3643 		ret = find_free_extent_clustered(block_group, ffe_ctl, bg_ret);  in do_allocation_clustered()
3649 	return find_free_extent_unclustered(block_group, ffe_ctl);  in do_allocation_clustered()
3674 			       struct find_free_extent_ctl *ffe_ctl,  in do_allocation_zoned()  argument
3681 	u64 num_bytes = ffe_ctl->num_bytes;  in do_allocation_zoned()
3697 	if (log_bytenr && ((ffe_ctl->for_treelog && bytenr != log_bytenr) ||  in do_allocation_zoned()
3698 			   (!ffe_ctl->for_treelog && bytenr == log_bytenr)))  in do_allocation_zoned()
3711 	    ((ffe_ctl->for_data_reloc && bytenr != data_reloc_bytenr) ||  in do_allocation_zoned()
3712 	     (!ffe_ctl->for_data_reloc && bytenr == data_reloc_bytenr)))  in do_allocation_zoned()
3747 	ASSERT(!ffe_ctl->for_treelog ||  in do_allocation_zoned()
3750 	ASSERT(!ffe_ctl->for_data_reloc ||  in do_allocation_zoned()
3755 	    (!ffe_ctl->for_data_reloc &&  in do_allocation_zoned()
3765 	if (ffe_ctl->for_treelog && !fs_info->treelog_bg &&  in do_allocation_zoned()
3775 	if (ffe_ctl->for_data_reloc && !fs_info->data_reloc_bg &&  in do_allocation_zoned()
3784 		if (ffe_ctl->max_extent_size < avail) {  in do_allocation_zoned()
3789 			ffe_ctl->max_extent_size = avail;  in do_allocation_zoned()
3790 			ffe_ctl->total_free_space = avail;  in do_allocation_zoned()
3796 	if (ffe_ctl->for_treelog && !fs_info->treelog_bg)  in do_allocation_zoned()
3799 	if (ffe_ctl->for_data_reloc) {  in do_allocation_zoned()
3820 	ffe_ctl->found_offset = start + block_group->alloc_offset;  in do_allocation_zoned()
3831 	ffe_ctl->search_start = ffe_ctl->found_offset;  in do_allocation_zoned()
3834 	if (ret && ffe_ctl->for_treelog)  in do_allocation_zoned()
3836 	if (ret && ffe_ctl->for_data_reloc)  in do_allocation_zoned()
3846 			 struct find_free_extent_ctl *ffe_ctl,  in do_allocation()  argument
3849 	switch (ffe_ctl->policy) {  in do_allocation()
3851 		return do_allocation_clustered(block_group, ffe_ctl, bg_ret);  in do_allocation()
3853 		return do_allocation_zoned(block_group, ffe_ctl, bg_ret);  in do_allocation()
3860 				struct find_free_extent_ctl *ffe_ctl,  in release_block_group()  argument
3863 	switch (ffe_ctl->policy) {  in release_block_group()
3865 		ffe_ctl->retry_uncached = false;  in release_block_group()
3875 	       ffe_ctl->index);  in release_block_group()
3879 static void found_extent_clustered(struct find_free_extent_ctl *ffe_ctl,  in found_extent_clustered()  argument
3882 	struct btrfs_free_cluster *last_ptr = ffe_ctl->last_ptr;  in found_extent_clustered()
3884 	if (!ffe_ctl->use_cluster && last_ptr) {  in found_extent_clustered()
3891 static void found_extent(struct find_free_extent_ctl *ffe_ctl,  in found_extent()  argument
3894 	switch (ffe_ctl->policy) {  in found_extent()
3896 		found_extent_clustered(ffe_ctl, ins);  in found_extent()
3907 				    struct find_free_extent_ctl *ffe_ctl)  in can_allocate_chunk_zoned()  argument
3910 	if (!(ffe_ctl->flags & BTRFS_BLOCK_GROUP_DATA))  in can_allocate_chunk_zoned()
3914 	if (btrfs_can_activate_zone(fs_info->fs_devices, ffe_ctl->flags))  in can_allocate_chunk_zoned()
3924 	if (ffe_ctl->flags & BTRFS_BLOCK_GROUP_DATA) {  in can_allocate_chunk_zoned()
3938 	if (ffe_ctl->max_extent_size >= ffe_ctl->min_alloc_size)  in can_allocate_chunk_zoned()
3948 	if (ffe_ctl->flags & BTRFS_BLOCK_GROUP_DATA)  in can_allocate_chunk_zoned()
3960 			      struct find_free_extent_ctl *ffe_ctl)  in can_allocate_chunk()  argument
3962 	switch (ffe_ctl->policy) {  in can_allocate_chunk()
3966 		return can_allocate_chunk_zoned(fs_info, ffe_ctl);  in can_allocate_chunk()
3979 					struct find_free_extent_ctl *ffe_ctl,  in find_free_extent_update_loop()  argument
3985 	if ((ffe_ctl->loop == LOOP_CACHING_NOWAIT) &&  in find_free_extent_update_loop()
3986 	    ffe_ctl->have_caching_bg && !ffe_ctl->orig_have_caching_bg)  in find_free_extent_update_loop()
3987 		ffe_ctl->orig_have_caching_bg = true;  in find_free_extent_update_loop()
3990 		found_extent(ffe_ctl, ins);  in find_free_extent_update_loop()
3994 	if (ffe_ctl->loop >= LOOP_CACHING_WAIT && ffe_ctl->have_caching_bg)  in find_free_extent_update_loop()
3997 	ffe_ctl->index++;  in find_free_extent_update_loop()
3998 	if (ffe_ctl->index < BTRFS_NR_RAID_TYPES)  in find_free_extent_update_loop()
4002 	if (ffe_ctl->loop < LOOP_NO_EMPTY_SIZE) {  in find_free_extent_update_loop()
4003 		ffe_ctl->index = 0;  in find_free_extent_update_loop()
4009 		if (ffe_ctl->loop == LOOP_CACHING_NOWAIT &&  in find_free_extent_update_loop()
4010 		    (!ffe_ctl->orig_have_caching_bg && full_search))  in find_free_extent_update_loop()
4011 			ffe_ctl->loop++;  in find_free_extent_update_loop()
4012 		ffe_ctl->loop++;  in find_free_extent_update_loop()
4014 		if (ffe_ctl->loop == LOOP_ALLOC_CHUNK) {  in find_free_extent_update_loop()
4019 			ret = can_allocate_chunk(fs_info, ffe_ctl);  in find_free_extent_update_loop()
4034 			ret = btrfs_chunk_alloc(trans, ffe_ctl->flags,  in find_free_extent_update_loop()
4040 				ffe_ctl->loop++;  in find_free_extent_update_loop()
4052 		if (ffe_ctl->loop == LOOP_NO_EMPTY_SIZE) {  in find_free_extent_update_loop()
4053 			if (ffe_ctl->policy != BTRFS_EXTENT_ALLOC_CLUSTERED)  in find_free_extent_update_loop()
4060 			if (ffe_ctl->empty_size == 0 &&  in find_free_extent_update_loop()
4061 			    ffe_ctl->empty_cluster == 0)  in find_free_extent_update_loop()
4063 			ffe_ctl->empty_size = 0;  in find_free_extent_update_loop()
4064 			ffe_ctl->empty_cluster = 0;  in find_free_extent_update_loop()
4071 static bool find_free_extent_check_size_class(struct find_free_extent_ctl *ffe_ctl,  in find_free_extent_check_size_class()  argument
4074 	if (ffe_ctl->policy == BTRFS_EXTENT_ALLOC_ZONED)  in find_free_extent_check_size_class()
4078 	if (ffe_ctl->loop >= LOOP_WRONG_SIZE_CLASS)  in find_free_extent_check_size_class()
4080 	if (ffe_ctl->loop >= LOOP_UNSET_SIZE_CLASS &&  in find_free_extent_check_size_class()
4083 	return ffe_ctl->size_class == bg->size_class;  in find_free_extent_check_size_class()
4087 					struct find_free_extent_ctl *ffe_ctl,  in prepare_allocation_clustered()  argument
4104 		    ffe_ctl->num_bytes > space_info->max_extent_size) {  in prepare_allocation_clustered()
4109 			ffe_ctl->use_cluster = false;  in prepare_allocation_clustered()
4114 	ffe_ctl->last_ptr = fetch_cluster_info(fs_info, space_info,  in prepare_allocation_clustered()
4115 					       &ffe_ctl->empty_cluster);  in prepare_allocation_clustered()
4116 	if (ffe_ctl->last_ptr) {  in prepare_allocation_clustered()
4117 		struct btrfs_free_cluster *last_ptr = ffe_ctl->last_ptr;  in prepare_allocation_clustered()
4121 			ffe_ctl->hint_byte = last_ptr->window_start;  in prepare_allocation_clustered()
4128 			ffe_ctl->hint_byte = last_ptr->window_start;  in prepare_allocation_clustered()
4129 			ffe_ctl->use_cluster = false;  in prepare_allocation_clustered()
4138 			      struct find_free_extent_ctl *ffe_ctl,  in prepare_allocation()  argument
4142 	switch (ffe_ctl->policy) {  in prepare_allocation()
4144 		return prepare_allocation_clustered(fs_info, ffe_ctl,  in prepare_allocation()
4147 		if (ffe_ctl->for_treelog) {  in prepare_allocation()
4150 				ffe_ctl->hint_byte = fs_info->treelog_bg;  in prepare_allocation()
4153 		if (ffe_ctl->for_data_reloc) {  in prepare_allocation()
4156 				ffe_ctl->hint_byte = fs_info->data_reloc_bg;  in prepare_allocation()
4192 				     struct find_free_extent_ctl *ffe_ctl)  in find_free_extent()  argument
4201 	WARN_ON(ffe_ctl->num_bytes < fs_info->sectorsize);  in find_free_extent()
4203 	ffe_ctl->search_start = 0;  in find_free_extent()
4205 	ffe_ctl->empty_cluster = 0;  in find_free_extent()
4206 	ffe_ctl->last_ptr = NULL;  in find_free_extent()
4207 	ffe_ctl->use_cluster = true;  in find_free_extent()
4208 	ffe_ctl->have_caching_bg = false;  in find_free_extent()
4209 	ffe_ctl->orig_have_caching_bg = false;  in find_free_extent()
4210 	ffe_ctl->index = btrfs_bg_flags_to_raid_index(ffe_ctl->flags);  in find_free_extent()
4211 	ffe_ctl->loop = 0;  in find_free_extent()
4212 	ffe_ctl->retry_uncached = false;  in find_free_extent()
4213 	ffe_ctl->cached = 0;  in find_free_extent()
4214 	ffe_ctl->max_extent_size = 0;  in find_free_extent()
4215 	ffe_ctl->total_free_space = 0;  in find_free_extent()
4216 	ffe_ctl->found_offset = 0;  in find_free_extent()
4217 	ffe_ctl->policy = BTRFS_EXTENT_ALLOC_CLUSTERED;  in find_free_extent()
4218 	ffe_ctl->size_class = btrfs_calc_block_group_size_class(ffe_ctl->num_bytes);  in find_free_extent()
4221 		ffe_ctl->policy = BTRFS_EXTENT_ALLOC_ZONED;  in find_free_extent()
4227 	trace_find_free_extent(root, ffe_ctl);  in find_free_extent()
4229 	space_info = btrfs_find_space_info(fs_info, ffe_ctl->flags);  in find_free_extent()
4231 		btrfs_err(fs_info, "No space info for %llu", ffe_ctl->flags);  in find_free_extent()
4235 	ret = prepare_allocation(fs_info, ffe_ctl, space_info, ins);  in find_free_extent()
4239 	ffe_ctl->search_start = max(ffe_ctl->search_start,  in find_free_extent()
4241 	ffe_ctl->search_start = max(ffe_ctl->search_start, ffe_ctl->hint_byte);  in find_free_extent()
4242 	if (ffe_ctl->search_start == ffe_ctl->hint_byte) {  in find_free_extent()
4244 						       ffe_ctl->search_start);  in find_free_extent()
4252 		if (block_group && block_group_bits(block_group, ffe_ctl->flags) &&  in find_free_extent()
4266 				ffe_ctl->index = btrfs_bg_flags_to_raid_index(  in find_free_extent()
4269 						       ffe_ctl->delalloc);  in find_free_extent()
4270 				ffe_ctl->hinted = true;  in find_free_extent()
4278 	trace_find_free_extent_search_loop(root, ffe_ctl);  in find_free_extent()
4279 	ffe_ctl->have_caching_bg = false;  in find_free_extent()
4280 	if (ffe_ctl->index == btrfs_bg_flags_to_raid_index(ffe_ctl->flags) ||  in find_free_extent()
4281 	    ffe_ctl->index == 0)  in find_free_extent()
4285 			    &space_info->block_groups[ffe_ctl->index], list) {  in find_free_extent()
4288 		ffe_ctl->hinted = false;  in find_free_extent()
4291 			if (ffe_ctl->for_treelog)  in find_free_extent()
4293 			if (ffe_ctl->for_data_reloc)  in find_free_extent()
4298 		btrfs_grab_block_group(block_group, ffe_ctl->delalloc);  in find_free_extent()
4299 		ffe_ctl->search_start = block_group->start;  in find_free_extent()
4306 		if (!block_group_bits(block_group, ffe_ctl->flags)) {  in find_free_extent()
4317 			if ((ffe_ctl->flags & extra) && !(block_group->flags & extra))  in find_free_extent()
4325 			btrfs_release_block_group(block_group, ffe_ctl->delalloc);  in find_free_extent()
4330 		trace_find_free_extent_have_block_group(root, ffe_ctl, block_group);  in find_free_extent()
4331 		ffe_ctl->cached = btrfs_block_group_done(block_group);  in find_free_extent()
4332 		if (unlikely(!ffe_ctl->cached)) {  in find_free_extent()
4333 			ffe_ctl->have_caching_bg = true;  in find_free_extent()
4358 		if (!find_free_extent_check_size_class(ffe_ctl, block_group))  in find_free_extent()
4362 		ret = do_allocation(block_group, ffe_ctl, &bg_ret);  in find_free_extent()
4367 			btrfs_release_block_group(block_group, ffe_ctl->delalloc);  in find_free_extent()
4372 		ffe_ctl->search_start = round_up(ffe_ctl->found_offset,  in find_free_extent()
4376 		if (ffe_ctl->search_start + ffe_ctl->num_bytes >  in find_free_extent()
4379 					    ffe_ctl->found_offset,  in find_free_extent()
4380 					    ffe_ctl->num_bytes);  in find_free_extent()
4384 		if (ffe_ctl->found_offset < ffe_ctl->search_start)  in find_free_extent()
4386 					ffe_ctl->found_offset,  in find_free_extent()
4387 					ffe_ctl->search_start - ffe_ctl->found_offset);  in find_free_extent()
4389 		ret = btrfs_add_reserved_bytes(block_group, ffe_ctl->ram_bytes,  in find_free_extent()
4390 					       ffe_ctl->num_bytes,  in find_free_extent()
4391 					       ffe_ctl->delalloc,  in find_free_extent()
4392 					       ffe_ctl->loop >= LOOP_WRONG_SIZE_CLASS);  in find_free_extent()
4395 					ffe_ctl->found_offset,  in find_free_extent()
4396 					ffe_ctl->num_bytes);  in find_free_extent()
4402 		ins->objectid = ffe_ctl->search_start;  in find_free_extent()
4403 		ins->offset = ffe_ctl->num_bytes;  in find_free_extent()
4405 		trace_btrfs_reserve_extent(block_group, ffe_ctl);  in find_free_extent()
4406 		btrfs_release_block_group(block_group, ffe_ctl->delalloc);  in find_free_extent()
4409 		if (!ffe_ctl->cached && ffe_ctl->loop > LOOP_CACHING_NOWAIT &&  in find_free_extent()
4410 		    !ffe_ctl->retry_uncached) {  in find_free_extent()
4411 			ffe_ctl->retry_uncached = true;  in find_free_extent()
4413 						ffe_ctl->num_bytes +  in find_free_extent()
4414 						ffe_ctl->empty_cluster +  in find_free_extent()
4415 						ffe_ctl->empty_size);  in find_free_extent()
4418 		release_block_group(block_group, ffe_ctl, ffe_ctl->delalloc);  in find_free_extent()
4423 	ret = find_free_extent_update_loop(fs_info, ins, ffe_ctl, full_search);  in find_free_extent()
4432 		if (!ffe_ctl->max_extent_size)  in find_free_extent()
4433 			ffe_ctl->max_extent_size = ffe_ctl->total_free_space;  in find_free_extent()
4435 		space_info->max_extent_size = ffe_ctl->max_extent_size;  in find_free_extent()
4437 		ins->offset = ffe_ctl->max_extent_size;  in find_free_extent()
4495 	struct find_free_extent_ctl ffe_ctl = {};  in btrfs_reserve_extent()  local
4506 	ffe_ctl.ram_bytes = ram_bytes;  in btrfs_reserve_extent()
4507 	ffe_ctl.num_bytes = num_bytes;  in btrfs_reserve_extent()
4508 	ffe_ctl.min_alloc_size = min_alloc_size;  in btrfs_reserve_extent()
4509 	ffe_ctl.empty_size = empty_size;  in btrfs_reserve_extent()
4510 	ffe_ctl.flags = flags;  in btrfs_reserve_extent()
4511 	ffe_ctl.delalloc = delalloc;  in btrfs_reserve_extent()
4512 	ffe_ctl.hint_byte = hint_byte;  in btrfs_reserve_extent()
4513 	ffe_ctl.for_treelog = for_treelog;  in btrfs_reserve_extent()
4514 	ffe_ctl.for_data_reloc = for_data_reloc;  in btrfs_reserve_extent()
4516 	ret = find_free_extent(root, ins, &ffe_ctl);  in btrfs_reserve_extent()