3530 				      struct find_free_extent_ctl *ffe_ctl,  in find_free_extent_clustered()  argument
3534 	struct btrfs_free_cluster *last_ptr = ffe_ctl->last_ptr;  in find_free_extent_clustered()
3539 	cluster_bg = btrfs_lock_cluster(bg, last_ptr, ffe_ctl->delalloc);  in find_free_extent_clustered()
3543 	    !block_group_bits(cluster_bg, ffe_ctl->flags)))  in find_free_extent_clustered()
3547 			ffe_ctl->num_bytes, cluster_bg->start,  in find_free_extent_clustered()
3548 			&ffe_ctl->max_extent_size);  in find_free_extent_clustered()
3553 				ffe_ctl->search_start, ffe_ctl->num_bytes);  in find_free_extent_clustered()
3555 		ffe_ctl->found_offset = offset;  in find_free_extent_clustered()
3572 	if (ffe_ctl->loop >= LOOP_NO_EMPTY_SIZE && cluster_bg != bg) {  in find_free_extent_clustered()
3574 		btrfs_release_block_group(cluster_bg, ffe_ctl->delalloc);  in find_free_extent_clustered()
3582 		btrfs_release_block_group(cluster_bg, ffe_ctl->delalloc);  in find_free_extent_clustered()
3585 	if (ffe_ctl->loop >= LOOP_NO_EMPTY_SIZE) {  in find_free_extent_clustered()
3591 			ffe_ctl->empty_cluster + ffe_ctl->empty_size,  in find_free_extent_clustered()
3593 	ret = btrfs_find_space_cluster(bg, last_ptr, ffe_ctl->search_start,  in find_free_extent_clustered()
3594 			ffe_ctl->num_bytes, aligned_cluster);  in find_free_extent_clustered()
3598 				ffe_ctl->num_bytes, ffe_ctl->search_start,  in find_free_extent_clustered()
3599 				&ffe_ctl->max_extent_size);  in find_free_extent_clustered()
3604 					ffe_ctl->search_start,  in find_free_extent_clustered()
3605 					ffe_ctl->num_bytes);  in find_free_extent_clustered()
3606 			ffe_ctl->found_offset = offset;  in find_free_extent_clustered()
3609 	} else if (!ffe_ctl->cached && ffe_ctl->loop > LOOP_CACHING_NOWAIT &&  in find_free_extent_clustered()
3610 		   !ffe_ctl->retry_clustered) {  in find_free_extent_clustered()
3613 		ffe_ctl->retry_clustered = true;  in find_free_extent_clustered()
3614 		btrfs_wait_block_group_cache_progress(bg, ffe_ctl->num_bytes +  in find_free_extent_clustered()
3615 				ffe_ctl->empty_cluster + ffe_ctl->empty_size);  in find_free_extent_clustered()
3634 					struct find_free_extent_ctl *ffe_ctl)  in find_free_extent_unclustered()  argument
3636 	struct btrfs_free_cluster *last_ptr = ffe_ctl->last_ptr;  in find_free_extent_unclustered()
3649 	if (ffe_ctl->cached) {  in find_free_extent_unclustered()
3655 		    ffe_ctl->num_bytes + ffe_ctl->empty_cluster +  in find_free_extent_unclustered()
3656 		    ffe_ctl->empty_size) {  in find_free_extent_unclustered()
3657 			ffe_ctl->total_free_space = max_t(u64,  in find_free_extent_unclustered()
3658 					ffe_ctl->total_free_space,  in find_free_extent_unclustered()
3666 	offset = btrfs_find_space_for_alloc(bg, ffe_ctl->search_start,  in find_free_extent_unclustered()
3667 			ffe_ctl->num_bytes, ffe_ctl->empty_size,  in find_free_extent_unclustered()
3668 			&ffe_ctl->max_extent_size);  in find_free_extent_unclustered()
3679 	if (!offset && !ffe_ctl->retry_unclustered && !ffe_ctl->cached &&  in find_free_extent_unclustered()
3680 	    ffe_ctl->loop > LOOP_CACHING_NOWAIT) {  in find_free_extent_unclustered()
3681 		btrfs_wait_block_group_cache_progress(bg, ffe_ctl->num_bytes +  in find_free_extent_unclustered()
3682 						      ffe_ctl->empty_size);  in find_free_extent_unclustered()
3683 		ffe_ctl->retry_unclustered = true;  in find_free_extent_unclustered()
3688 	ffe_ctl->found_offset = offset;  in find_free_extent_unclustered()
3693 				   struct find_free_extent_ctl *ffe_ctl,  in do_allocation_clustered()  argument
3699 	if (ffe_ctl->last_ptr && ffe_ctl->use_cluster) {  in do_allocation_clustered()
3700 		ret = find_free_extent_clustered(block_group, ffe_ctl, bg_ret);  in do_allocation_clustered()
3706 	return find_free_extent_unclustered(block_group, ffe_ctl);  in do_allocation_clustered()
3731 			       struct find_free_extent_ctl *ffe_ctl,  in do_allocation_zoned()  argument
3738 	u64 num_bytes = ffe_ctl->num_bytes;  in do_allocation_zoned()
3754 	if (log_bytenr && ((ffe_ctl->for_treelog && bytenr != log_bytenr) ||  in do_allocation_zoned()
3755 			   (!ffe_ctl->for_treelog && bytenr == log_bytenr)))  in do_allocation_zoned()
3768 	    ((ffe_ctl->for_data_reloc && bytenr != data_reloc_bytenr) ||  in do_allocation_zoned()
3769 	     (!ffe_ctl->for_data_reloc && bytenr == data_reloc_bytenr)))  in do_allocation_zoned()
3802 	ASSERT(!ffe_ctl->for_treelog ||  in do_allocation_zoned()
3805 	ASSERT(!ffe_ctl->for_data_reloc ||  in do_allocation_zoned()
3819 	if (ffe_ctl->for_treelog && !fs_info->treelog_bg &&  in do_allocation_zoned()
3829 	if (ffe_ctl->for_data_reloc && !fs_info->data_reloc_bg &&  in do_allocation_zoned()
3838 		if (ffe_ctl->max_extent_size < avail) {  in do_allocation_zoned()
3843 			ffe_ctl->max_extent_size = avail;  in do_allocation_zoned()
3844 			ffe_ctl->total_free_space = avail;  in do_allocation_zoned()
3850 	if (ffe_ctl->for_treelog && !fs_info->treelog_bg)  in do_allocation_zoned()
3853 	if (ffe_ctl->for_data_reloc && !fs_info->data_reloc_bg)  in do_allocation_zoned()
3856 	ffe_ctl->found_offset = start + block_group->alloc_offset;  in do_allocation_zoned()
3867 	ffe_ctl->search_start = ffe_ctl->found_offset;  in do_allocation_zoned()
3870 	if (ret && ffe_ctl->for_treelog)  in do_allocation_zoned()
3872 	if (ret && ffe_ctl->for_data_reloc &&  in do_allocation_zoned()
3898 			 struct find_free_extent_ctl *ffe_ctl,  in do_allocation()  argument
3901 	switch (ffe_ctl->policy) {  in do_allocation()
3903 		return do_allocation_clustered(block_group, ffe_ctl, bg_ret);  in do_allocation()
3905 		return do_allocation_zoned(block_group, ffe_ctl, bg_ret);  in do_allocation()
3912 				struct find_free_extent_ctl *ffe_ctl,  in release_block_group()  argument
3915 	switch (ffe_ctl->policy) {  in release_block_group()
3917 		ffe_ctl->retry_clustered = false;  in release_block_group()
3918 		ffe_ctl->retry_unclustered = false;  in release_block_group()
3928 	       ffe_ctl->index);  in release_block_group()
3932 static void found_extent_clustered(struct find_free_extent_ctl *ffe_ctl,  in found_extent_clustered()  argument
3935 	struct btrfs_free_cluster *last_ptr = ffe_ctl->last_ptr;  in found_extent_clustered()
3937 	if (!ffe_ctl->use_cluster && last_ptr) {  in found_extent_clustered()
3944 static void found_extent(struct find_free_extent_ctl *ffe_ctl,  in found_extent()  argument
3947 	switch (ffe_ctl->policy) {  in found_extent()
3949 		found_extent_clustered(ffe_ctl, ins);  in found_extent()
3960 				    struct find_free_extent_ctl *ffe_ctl)  in can_allocate_chunk_zoned()  argument
3963 	if (btrfs_can_activate_zone(fs_info->fs_devices, ffe_ctl->flags))  in can_allocate_chunk_zoned()
3973 	if (ffe_ctl->flags & BTRFS_BLOCK_GROUP_DATA) {  in can_allocate_chunk_zoned()
3987 	if (ffe_ctl->max_extent_size >= ffe_ctl->min_alloc_size)  in can_allocate_chunk_zoned()
3997 	if (ffe_ctl->flags & BTRFS_BLOCK_GROUP_DATA)  in can_allocate_chunk_zoned()
4009 			      struct find_free_extent_ctl *ffe_ctl)  in can_allocate_chunk()  argument
4011 	switch (ffe_ctl->policy) {  in can_allocate_chunk()
4015 		return can_allocate_chunk_zoned(fs_info, ffe_ctl);  in can_allocate_chunk()
4021 static int chunk_allocation_failed(struct find_free_extent_ctl *ffe_ctl)  in chunk_allocation_failed()  argument
4023 	switch (ffe_ctl->policy) {  in chunk_allocation_failed()
4029 		ffe_ctl->loop = LOOP_NO_EMPTY_SIZE;  in chunk_allocation_failed()
4046 					struct find_free_extent_ctl *ffe_ctl,  in find_free_extent_update_loop()  argument
4052 	if ((ffe_ctl->loop == LOOP_CACHING_NOWAIT) &&  in find_free_extent_update_loop()
4053 	    ffe_ctl->have_caching_bg && !ffe_ctl->orig_have_caching_bg)  in find_free_extent_update_loop()
4054 		ffe_ctl->orig_have_caching_bg = true;  in find_free_extent_update_loop()
4057 		found_extent(ffe_ctl, ins);  in find_free_extent_update_loop()
4061 	if (ffe_ctl->loop >= LOOP_CACHING_WAIT && ffe_ctl->have_caching_bg)  in find_free_extent_update_loop()
4064 	ffe_ctl->index++;  in find_free_extent_update_loop()
4065 	if (ffe_ctl->index < BTRFS_NR_RAID_TYPES)  in find_free_extent_update_loop()
4076 	if (ffe_ctl->loop < LOOP_NO_EMPTY_SIZE) {  in find_free_extent_update_loop()
4077 		ffe_ctl->index = 0;  in find_free_extent_update_loop()
4078 		if (ffe_ctl->loop == LOOP_CACHING_NOWAIT) {  in find_free_extent_update_loop()
4084 			if (ffe_ctl->orig_have_caching_bg || !full_search)  in find_free_extent_update_loop()
4085 				ffe_ctl->loop = LOOP_CACHING_WAIT;  in find_free_extent_update_loop()
4087 				ffe_ctl->loop = LOOP_ALLOC_CHUNK;  in find_free_extent_update_loop()
4089 			ffe_ctl->loop++;  in find_free_extent_update_loop()
4092 		if (ffe_ctl->loop == LOOP_ALLOC_CHUNK) {  in find_free_extent_update_loop()
4097 			ret = can_allocate_chunk(fs_info, ffe_ctl);  in find_free_extent_update_loop()
4112 			ret = btrfs_chunk_alloc(trans, ffe_ctl->flags,  in find_free_extent_update_loop()
4117 				ret = chunk_allocation_failed(ffe_ctl);  in find_free_extent_update_loop()
4128 		if (ffe_ctl->loop == LOOP_NO_EMPTY_SIZE) {  in find_free_extent_update_loop()
4129 			if (ffe_ctl->policy != BTRFS_EXTENT_ALLOC_CLUSTERED)  in find_free_extent_update_loop()
4136 			if (ffe_ctl->empty_size == 0 &&  in find_free_extent_update_loop()
4137 			    ffe_ctl->empty_cluster == 0)  in find_free_extent_update_loop()
4139 			ffe_ctl->empty_size = 0;  in find_free_extent_update_loop()
4140 			ffe_ctl->empty_cluster = 0;  in find_free_extent_update_loop()
4148 					struct find_free_extent_ctl *ffe_ctl,  in prepare_allocation_clustered()  argument
4165 		    ffe_ctl->num_bytes > space_info->max_extent_size) {  in prepare_allocation_clustered()
4170 			ffe_ctl->use_cluster = false;  in prepare_allocation_clustered()
4175 	ffe_ctl->last_ptr = fetch_cluster_info(fs_info, space_info,  in prepare_allocation_clustered()
4176 					       &ffe_ctl->empty_cluster);  in prepare_allocation_clustered()
4177 	if (ffe_ctl->last_ptr) {  in prepare_allocation_clustered()
4178 		struct btrfs_free_cluster *last_ptr = ffe_ctl->last_ptr;  in prepare_allocation_clustered()
4182 			ffe_ctl->hint_byte = last_ptr->window_start;  in prepare_allocation_clustered()
4189 			ffe_ctl->hint_byte = last_ptr->window_start;  in prepare_allocation_clustered()
4190 			ffe_ctl->use_cluster = false;  in prepare_allocation_clustered()
4199 			      struct find_free_extent_ctl *ffe_ctl,  in prepare_allocation()  argument
4203 	switch (ffe_ctl->policy) {  in prepare_allocation()
4205 		return prepare_allocation_clustered(fs_info, ffe_ctl,  in prepare_allocation()
4208 		if (ffe_ctl->for_treelog) {  in prepare_allocation()
4211 				ffe_ctl->hint_byte = fs_info->treelog_bg;  in prepare_allocation()
4214 		if (ffe_ctl->for_data_reloc) {  in prepare_allocation()
4217 				ffe_ctl->hint_byte = fs_info->data_reloc_bg;  in prepare_allocation()
4253 				     struct find_free_extent_ctl *ffe_ctl)  in find_free_extent()  argument
4262 	WARN_ON(ffe_ctl->num_bytes < fs_info->sectorsize);  in find_free_extent()
4264 	ffe_ctl->search_start = 0;  in find_free_extent()
4266 	ffe_ctl->empty_cluster = 0;  in find_free_extent()
4267 	ffe_ctl->last_ptr = NULL;  in find_free_extent()
4268 	ffe_ctl->use_cluster = true;  in find_free_extent()
4269 	ffe_ctl->have_caching_bg = false;  in find_free_extent()
4270 	ffe_ctl->orig_have_caching_bg = false;  in find_free_extent()
4271 	ffe_ctl->index = btrfs_bg_flags_to_raid_index(ffe_ctl->flags);  in find_free_extent()
4272 	ffe_ctl->loop = 0;  in find_free_extent()
4274 	ffe_ctl->retry_clustered = false;  in find_free_extent()
4275 	ffe_ctl->retry_unclustered = false;  in find_free_extent()
4276 	ffe_ctl->cached = 0;  in find_free_extent()
4277 	ffe_ctl->max_extent_size = 0;  in find_free_extent()
4278 	ffe_ctl->total_free_space = 0;  in find_free_extent()
4279 	ffe_ctl->found_offset = 0;  in find_free_extent()
4280 	ffe_ctl->policy = BTRFS_EXTENT_ALLOC_CLUSTERED;  in find_free_extent()
4283 		ffe_ctl->policy = BTRFS_EXTENT_ALLOC_ZONED;  in find_free_extent()
4289 	trace_find_free_extent(root, ffe_ctl->num_bytes, ffe_ctl->empty_size,  in find_free_extent()
4290 			       ffe_ctl->flags);  in find_free_extent()
4292 	space_info = btrfs_find_space_info(fs_info, ffe_ctl->flags);  in find_free_extent()
4294 		btrfs_err(fs_info, "No space info for %llu", ffe_ctl->flags);  in find_free_extent()
4298 	ret = prepare_allocation(fs_info, ffe_ctl, space_info, ins);  in find_free_extent()
4302 	ffe_ctl->search_start = max(ffe_ctl->search_start,  in find_free_extent()
4304 	ffe_ctl->search_start = max(ffe_ctl->search_start, ffe_ctl->hint_byte);  in find_free_extent()
4305 	if (ffe_ctl->search_start == ffe_ctl->hint_byte) {  in find_free_extent()
4307 						       ffe_ctl->search_start);  in find_free_extent()
4315 		if (block_group && block_group_bits(block_group, ffe_ctl->flags) &&  in find_free_extent()
4329 				ffe_ctl->index = btrfs_bg_flags_to_raid_index(  in find_free_extent()
4332 						       ffe_ctl->delalloc);  in find_free_extent()
4340 	ffe_ctl->have_caching_bg = false;  in find_free_extent()
4341 	if (ffe_ctl->index == btrfs_bg_flags_to_raid_index(ffe_ctl->flags) ||  in find_free_extent()
4342 	    ffe_ctl->index == 0)  in find_free_extent()
4346 			    &space_info->block_groups[ffe_ctl->index], list) {  in find_free_extent()
4351 			if (ffe_ctl->for_treelog)  in find_free_extent()
4353 			if (ffe_ctl->for_data_reloc)  in find_free_extent()
4358 		btrfs_grab_block_group(block_group, ffe_ctl->delalloc);  in find_free_extent()
4359 		ffe_ctl->search_start = block_group->start;  in find_free_extent()
4366 		if (!block_group_bits(block_group, ffe_ctl->flags)) {  in find_free_extent()
4377 			if ((ffe_ctl->flags & extra) && !(block_group->flags & extra))  in find_free_extent()
4385 			btrfs_release_block_group(block_group, ffe_ctl->delalloc);  in find_free_extent()
4390 		ffe_ctl->cached = btrfs_block_group_done(block_group);  in find_free_extent()
4391 		if (unlikely(!ffe_ctl->cached)) {  in find_free_extent()
4392 			ffe_ctl->have_caching_bg = true;  in find_free_extent()
4415 		ret = do_allocation(block_group, ffe_ctl, &bg_ret);  in find_free_extent()
4419 							  ffe_ctl->delalloc);  in find_free_extent()
4429 		ffe_ctl->search_start = round_up(ffe_ctl->found_offset,  in find_free_extent()
4433 		if (ffe_ctl->search_start + ffe_ctl->num_bytes >  in find_free_extent()
4436 					    ffe_ctl->found_offset,  in find_free_extent()
4437 					    ffe_ctl->num_bytes);  in find_free_extent()
4441 		if (ffe_ctl->found_offset < ffe_ctl->search_start)  in find_free_extent()
4443 					ffe_ctl->found_offset,  in find_free_extent()
4444 					ffe_ctl->search_start - ffe_ctl->found_offset);  in find_free_extent()
4446 		ret = btrfs_add_reserved_bytes(block_group, ffe_ctl->ram_bytes,  in find_free_extent()
4447 					       ffe_ctl->num_bytes,  in find_free_extent()
4448 					       ffe_ctl->delalloc);  in find_free_extent()
4451 					ffe_ctl->found_offset,  in find_free_extent()
4452 					ffe_ctl->num_bytes);  in find_free_extent()
4458 		ins->objectid = ffe_ctl->search_start;  in find_free_extent()
4459 		ins->offset = ffe_ctl->num_bytes;  in find_free_extent()
4461 		trace_btrfs_reserve_extent(block_group, ffe_ctl->search_start,  in find_free_extent()
4462 					   ffe_ctl->num_bytes);  in find_free_extent()
4463 		btrfs_release_block_group(block_group, ffe_ctl->delalloc);  in find_free_extent()
4466 		release_block_group(block_group, ffe_ctl, ffe_ctl->delalloc);  in find_free_extent()
4471 	ret = find_free_extent_update_loop(fs_info, ins, ffe_ctl, full_search);  in find_free_extent()
4480 		if (!ffe_ctl->max_extent_size)  in find_free_extent()
4481 			ffe_ctl->max_extent_size = ffe_ctl->total_free_space;  in find_free_extent()
4483 		space_info->max_extent_size = ffe_ctl->max_extent_size;  in find_free_extent()
4485 		ins->offset = ffe_ctl->max_extent_size;  in find_free_extent()
4543 	struct find_free_extent_ctl ffe_ctl = {};  in btrfs_reserve_extent()  local
4554 	ffe_ctl.ram_bytes = ram_bytes;  in btrfs_reserve_extent()
4555 	ffe_ctl.num_bytes = num_bytes;  in btrfs_reserve_extent()
4556 	ffe_ctl.min_alloc_size = min_alloc_size;  in btrfs_reserve_extent()
4557 	ffe_ctl.empty_size = empty_size;  in btrfs_reserve_extent()
4558 	ffe_ctl.flags = flags;  in btrfs_reserve_extent()
4559 	ffe_ctl.delalloc = delalloc;  in btrfs_reserve_extent()
4560 	ffe_ctl.hint_byte = hint_byte;  in btrfs_reserve_extent()
4561 	ffe_ctl.for_treelog = for_treelog;  in btrfs_reserve_extent()
4562 	ffe_ctl.for_data_reloc = for_data_reloc;  in btrfs_reserve_extent()
4564 	ret = find_free_extent(root, ins, &ffe_ctl);  in btrfs_reserve_extent()