84 				      *block_group, struct btrfs_path *path)  in lookup_free_space_inode()
89 	spin_lock(&block_group->lock);  in lookup_free_space_inode()
90 	if (block_group->inode)  in lookup_free_space_inode()
91 		inode = igrab(block_group->inode);  in lookup_free_space_inode()
92 	spin_unlock(&block_group->lock);  in lookup_free_space_inode()
97 					  block_group->key.objectid);  in lookup_free_space_inode()
101 	spin_lock(&block_group->lock);  in lookup_free_space_inode()
106 		block_group->disk_cache_state = BTRFS_DC_CLEAR;  in lookup_free_space_inode()
109 	if (!block_group->iref) {  in lookup_free_space_inode()
110 		block_group->inode = igrab(inode);  in lookup_free_space_inode()
111 		block_group->iref = 1;  in lookup_free_space_inode()
113 	spin_unlock(&block_group->lock);  in lookup_free_space_inode()
181 			    struct btrfs_block_group_cache *block_group,  in create_free_space_inode()  argument
192 					 block_group->key.objectid);  in create_free_space_inode()
215 				    struct btrfs_block_group_cache *block_group,  in btrfs_truncate_free_space_cache()  argument
222 	if (block_group) {  in btrfs_truncate_free_space_cache()
231 		if (!list_empty(&block_group->io_list)) {  in btrfs_truncate_free_space_cache()
232 			list_del_init(&block_group->io_list);  in btrfs_truncate_free_space_cache()
234 			btrfs_wait_cache_io(trans, block_group, path);  in btrfs_truncate_free_space_cache()
235 			btrfs_put_block_group(block_group);  in btrfs_truncate_free_space_cache()
242 		spin_lock(&block_group->lock);  in btrfs_truncate_free_space_cache()
243 		block_group->disk_cache_state = BTRFS_DC_CLEAR;  in btrfs_truncate_free_space_cache()
244 		spin_unlock(&block_group->lock);  in btrfs_truncate_free_space_cache()
807 			  struct btrfs_block_group_cache *block_group)  in load_free_space_cache()  argument
809 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in load_free_space_cache()
814 	u64 used = btrfs_block_group_used(&block_group->item);  in load_free_space_cache()
820 	spin_lock(&block_group->lock);  in load_free_space_cache()
821 	if (block_group->disk_cache_state != BTRFS_DC_WRITTEN) {  in load_free_space_cache()
822 		spin_unlock(&block_group->lock);  in load_free_space_cache()
825 	spin_unlock(&block_group->lock);  in load_free_space_cache()
833 	inode = lookup_free_space_inode(fs_info, block_group, path);  in load_free_space_cache()
840 	spin_lock(&block_group->lock);  in load_free_space_cache()
841 	if (block_group->disk_cache_state != BTRFS_DC_WRITTEN) {  in load_free_space_cache()
842 		spin_unlock(&block_group->lock);  in load_free_space_cache()
846 	spin_unlock(&block_group->lock);  in load_free_space_cache()
849 				      path, block_group->key.objectid);  in load_free_space_cache()
855 	matched = (ctl->free_space == (block_group->key.offset - used -  in load_free_space_cache()
856 				       block_group->bytes_super));  in load_free_space_cache()
863 			   block_group->key.objectid);  in load_free_space_cache()
869 		spin_lock(&block_group->lock);  in load_free_space_cache()
870 		block_group->disk_cache_state = BTRFS_DC_CLEAR;  in load_free_space_cache()
871 		spin_unlock(&block_group->lock);  in load_free_space_cache()
876 			   block_group->key.objectid);  in load_free_space_cache()
886 			      struct btrfs_block_group_cache *block_group,  in write_cache_extent_entries()  argument
897 	if (block_group && !list_empty(&block_group->cluster_list)) {  in write_cache_extent_entries()
898 		cluster = list_entry(block_group->cluster_list.next,  in write_cache_extent_entries()
1016 			    struct btrfs_block_group_cache *block_group,  in write_pinned_extent_entries()  argument
1024 	if (!block_group)  in write_pinned_extent_entries()
1036 	start = block_group->key.objectid;  in write_pinned_extent_entries()
1038 	while (start < block_group->key.objectid + block_group->key.offset) {  in write_pinned_extent_entries()
1046 		if (extent_start >= block_group->key.objectid +  in write_pinned_extent_entries()
1047 		    block_group->key.offset)  in write_pinned_extent_entries()
1051 		extent_end = min(block_group->key.objectid +  in write_pinned_extent_entries()
1052 				 block_group->key.offset, extent_end + 1);  in write_pinned_extent_entries()
1116 				 struct btrfs_block_group_cache *block_group,  in __btrfs_wait_cache_io()  argument
1139 		if (block_group) {  in __btrfs_wait_cache_io()
1143 				  block_group->key.objectid);  in __btrfs_wait_cache_io()
1149 	if (block_group) {  in __btrfs_wait_cache_io()
1154 		spin_lock(&block_group->lock);  in __btrfs_wait_cache_io()
1161 		if (!ret && list_empty(&block_group->dirty_list))  in __btrfs_wait_cache_io()
1162 			block_group->disk_cache_state = BTRFS_DC_WRITTEN;  in __btrfs_wait_cache_io()
1164 			block_group->disk_cache_state = BTRFS_DC_ERROR;  in __btrfs_wait_cache_io()
1166 		spin_unlock(&block_group->lock);  in __btrfs_wait_cache_io()
1185 			struct btrfs_block_group_cache *block_group,  in btrfs_wait_cache_io()  argument
1188 	return __btrfs_wait_cache_io(block_group->fs_info->tree_root, trans,  in btrfs_wait_cache_io()
1189 				     block_group, &block_group->io_ctl,  in btrfs_wait_cache_io()
1190 				     path, block_group->key.objectid);  in btrfs_wait_cache_io()
1206 				   struct btrfs_block_group_cache *block_group,  in __btrfs_write_out_cache()  argument
1226 	if (block_group && (block_group->flags & BTRFS_BLOCK_GROUP_DATA)) {  in __btrfs_write_out_cache()
1227 		down_write(&block_group->data_rwsem);  in __btrfs_write_out_cache()
1228 		spin_lock(&block_group->lock);  in __btrfs_write_out_cache()
1229 		if (block_group->delalloc_bytes) {  in __btrfs_write_out_cache()
1230 			block_group->disk_cache_state = BTRFS_DC_WRITTEN;  in __btrfs_write_out_cache()
1231 			spin_unlock(&block_group->lock);  in __btrfs_write_out_cache()
1232 			up_write(&block_group->data_rwsem);  in __btrfs_write_out_cache()
1238 		spin_unlock(&block_group->lock);  in __btrfs_write_out_cache()
1255 					 block_group, &entries, &bitmaps,  in __btrfs_write_out_cache()
1268 	ret = write_pinned_extent_entries(fs_info, block_group,  in __btrfs_write_out_cache()
1293 	if (block_group && (block_group->flags & BTRFS_BLOCK_GROUP_DATA))  in __btrfs_write_out_cache()
1294 		up_write(&block_group->data_rwsem);  in __btrfs_write_out_cache()
1339 	if (block_group && (block_group->flags & BTRFS_BLOCK_GROUP_DATA))  in __btrfs_write_out_cache()
1340 		up_write(&block_group->data_rwsem);  in __btrfs_write_out_cache()
1347 			  struct btrfs_block_group_cache *block_group,  in btrfs_write_out_cache()  argument
1350 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in btrfs_write_out_cache()
1354 	spin_lock(&block_group->lock);  in btrfs_write_out_cache()
1355 	if (block_group->disk_cache_state < BTRFS_DC_SETUP) {  in btrfs_write_out_cache()
1356 		spin_unlock(&block_group->lock);  in btrfs_write_out_cache()
1359 	spin_unlock(&block_group->lock);  in btrfs_write_out_cache()
1361 	inode = lookup_free_space_inode(fs_info, block_group, path);  in btrfs_write_out_cache()
1366 				block_group, &block_group->io_ctl, trans);  in btrfs_write_out_cache()
1371 			  block_group->key.objectid);  in btrfs_write_out_cache()
1373 		spin_lock(&block_group->lock);  in btrfs_write_out_cache()
1374 		block_group->disk_cache_state = BTRFS_DC_ERROR;  in btrfs_write_out_cache()
1375 		spin_unlock(&block_group->lock);  in btrfs_write_out_cache()
1377 		block_group->io_ctl.inode = NULL;  in btrfs_write_out_cache()
1624 	struct btrfs_block_group_cache *block_group = ctl->private;  in recalculate_thresholds()  local
1628 	u64 size = block_group->key.offset;  in recalculate_thresholds()
1957 	struct btrfs_block_group_cache *block_group = ctl->private;  in use_bitmap()  local
1958 	struct btrfs_fs_info *fs_info = block_group->fs_info;  in use_bitmap()
1962 	if (btrfs_should_fragment_free_space(block_group))  in use_bitmap()
1994 	if (((BITS_PER_BITMAP * ctl->unit) >> 1) > block_group->key.offset)  in use_bitmap()
2009 	struct btrfs_block_group_cache *block_group = NULL;  in insert_into_bitmap()  local
2021 		block_group = ctl->private;  in insert_into_bitmap()
2028 	if (block_group && !list_empty(&block_group->cluster_list)) {  in insert_into_bitmap()
2033 		cluster = list_entry(block_group->cluster_list.next,  in insert_into_bitmap()
2344 int btrfs_remove_free_space(struct btrfs_block_group_cache *block_group,  in btrfs_remove_free_space()  argument
2347 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in btrfs_remove_free_space()
2416 			ret = btrfs_add_free_space(block_group, offset + bytes,  in btrfs_remove_free_space()
2434 void btrfs_dump_free_space(struct btrfs_block_group_cache *block_group,  in btrfs_dump_free_space()  argument
2437 	struct btrfs_fs_info *fs_info = block_group->fs_info;  in btrfs_dump_free_space()
2438 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in btrfs_dump_free_space()
2445 		if (info->bytes >= bytes && !block_group->ro)  in btrfs_dump_free_space()
2452 	       list_empty(&block_group->cluster_list) ? "no" : "yes");  in btrfs_dump_free_space()
2457 void btrfs_init_free_space_ctl(struct btrfs_block_group_cache *block_group)  in btrfs_init_free_space_ctl()  argument
2459 	struct btrfs_fs_info *fs_info = block_group->fs_info;  in btrfs_init_free_space_ctl()
2460 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in btrfs_init_free_space_ctl()
2464 	ctl->start = block_group->key.objectid;  in btrfs_init_free_space_ctl()
2465 	ctl->private = block_group;  in btrfs_init_free_space_ctl()
2486 			     struct btrfs_block_group_cache *block_group,  in __btrfs_return_cluster_to_free_space()  argument
2489 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in __btrfs_return_cluster_to_free_space()
2494 	if (cluster->block_group != block_group)  in __btrfs_return_cluster_to_free_space()
2497 	cluster->block_group = NULL;  in __btrfs_return_cluster_to_free_space()
2522 	btrfs_put_block_group(block_group);  in __btrfs_return_cluster_to_free_space()
2552 void btrfs_remove_free_space_cache(struct btrfs_block_group_cache *block_group)  in btrfs_remove_free_space_cache()  argument
2554 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in btrfs_remove_free_space_cache()
2559 	while ((head = block_group->cluster_list.next) !=  in btrfs_remove_free_space_cache()
2560 	       &block_group->cluster_list) {  in btrfs_remove_free_space_cache()
2564 		WARN_ON(cluster->block_group != block_group);  in btrfs_remove_free_space_cache()
2565 		__btrfs_return_cluster_to_free_space(block_group, cluster);  in btrfs_remove_free_space_cache()
2574 u64 btrfs_find_space_for_alloc(struct btrfs_block_group_cache *block_group,  in btrfs_find_space_for_alloc()  argument
2578 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in btrfs_find_space_for_alloc()
2587 				block_group->full_stripe_len, max_extent_size);  in btrfs_find_space_for_alloc()
2614 		__btrfs_add_free_space(block_group->fs_info, ctl,  in btrfs_find_space_for_alloc()
2628 			       struct btrfs_block_group_cache *block_group,  in btrfs_return_cluster_to_free_space()  argument
2636 	if (!block_group) {  in btrfs_return_cluster_to_free_space()
2637 		block_group = cluster->block_group;  in btrfs_return_cluster_to_free_space()
2638 		if (!block_group) {  in btrfs_return_cluster_to_free_space()
2642 	} else if (cluster->block_group != block_group) {  in btrfs_return_cluster_to_free_space()
2647 	atomic_inc(&block_group->count);  in btrfs_return_cluster_to_free_space()
2650 	ctl = block_group->free_space_ctl;  in btrfs_return_cluster_to_free_space()
2654 	ret = __btrfs_return_cluster_to_free_space(block_group, cluster);  in btrfs_return_cluster_to_free_space()
2658 	btrfs_put_block_group(block_group);  in btrfs_return_cluster_to_free_space()
2662 static u64 btrfs_alloc_from_bitmap(struct btrfs_block_group_cache *block_group,  in btrfs_alloc_from_bitmap()  argument
2668 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in btrfs_alloc_from_bitmap()
2695 u64 btrfs_alloc_from_cluster(struct btrfs_block_group_cache *block_group,  in btrfs_alloc_from_cluster()  argument
2699 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in btrfs_alloc_from_cluster()
2708 	if (cluster->block_group != block_group)  in btrfs_alloc_from_cluster()
2731 			ret = btrfs_alloc_from_bitmap(block_group,  in btrfs_alloc_from_cluster()
2779 static int btrfs_bitmap_cluster(struct btrfs_block_group_cache *block_group,  in btrfs_bitmap_cluster()  argument
2785 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in btrfs_bitmap_cluster()
2850 	trace_btrfs_setup_cluster(block_group, cluster,  in btrfs_bitmap_cluster()
2861 setup_cluster_no_bitmap(struct btrfs_block_group_cache *block_group,  in setup_cluster_no_bitmap()  argument
2866 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in setup_cluster_no_bitmap()
2943 	trace_btrfs_setup_cluster(block_group, cluster, total_size, 0);  in setup_cluster_no_bitmap()
2952 setup_cluster_bitmap(struct btrfs_block_group_cache *block_group,  in setup_cluster_bitmap()  argument
2957 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in setup_cluster_bitmap()
2981 		ret = btrfs_bitmap_cluster(block_group, entry, cluster, offset,  in setup_cluster_bitmap()
3003 			     struct btrfs_block_group_cache *block_group,  in btrfs_find_space_cluster()  argument
3007 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in btrfs_find_space_cluster()
3022 	} else if (block_group->flags & BTRFS_BLOCK_GROUP_METADATA) {  in btrfs_find_space_cluster()
3044 	if (cluster->block_group) {  in btrfs_find_space_cluster()
3049 	trace_btrfs_find_cluster(block_group, offset, bytes, empty_size,  in btrfs_find_space_cluster()
3052 	ret = setup_cluster_no_bitmap(block_group, cluster, &bitmaps, offset,  in btrfs_find_space_cluster()
3056 		ret = setup_cluster_bitmap(block_group, cluster, &bitmaps,  in btrfs_find_space_cluster()
3065 		atomic_inc(&block_group->count);  in btrfs_find_space_cluster()
3067 			      &block_group->cluster_list);  in btrfs_find_space_cluster()
3068 		cluster->block_group = block_group;  in btrfs_find_space_cluster()
3070 		trace_btrfs_failed_cluster_setup(block_group);  in btrfs_find_space_cluster()
3090 	cluster->block_group = NULL;  in btrfs_init_free_cluster()
3093 static int do_trimming(struct btrfs_block_group_cache *block_group,  in do_trimming()  argument
3098 	struct btrfs_space_info *space_info = block_group->space_info;  in do_trimming()
3099 	struct btrfs_fs_info *fs_info = block_group->fs_info;  in do_trimming()
3100 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in do_trimming()
3106 	spin_lock(&block_group->lock);  in do_trimming()
3107 	if (!block_group->ro) {  in do_trimming()
3108 		block_group->reserved += reserved_bytes;  in do_trimming()
3112 	spin_unlock(&block_group->lock);  in do_trimming()
3120 	btrfs_add_free_space(block_group, reserved_start, reserved_bytes);  in do_trimming()
3126 		spin_lock(&block_group->lock);  in do_trimming()
3127 		if (block_group->ro)  in do_trimming()
3129 		block_group->reserved -= reserved_bytes;  in do_trimming()
3132 		spin_unlock(&block_group->lock);  in do_trimming()
3138 static int trim_no_bitmap(struct btrfs_block_group_cache *block_group,  in trim_no_bitmap()  argument
3141 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in trim_no_bitmap()
3205 		ret = do_trimming(block_group, total_trimmed, start, bytes,  in trim_no_bitmap()
3223 static int trim_bitmaps(struct btrfs_block_group_cache *block_group,  in trim_bitmaps()  argument
3226 	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;  in trim_bitmaps()
3280 		ret = do_trimming(block_group, total_trimmed, start, bytes,  in trim_bitmaps()
3309 void btrfs_put_block_group_trimming(struct btrfs_block_group_cache *block_group)  in btrfs_put_block_group_trimming()  argument
3311 	struct btrfs_fs_info *fs_info = block_group->fs_info;  in btrfs_put_block_group_trimming()
3316 	spin_lock(&block_group->lock);  in btrfs_put_block_group_trimming()
3317 	cleanup = (atomic_dec_and_test(&block_group->trimming) &&  in btrfs_put_block_group_trimming()
3318 		   block_group->removed);  in btrfs_put_block_group_trimming()
3319 	spin_unlock(&block_group->lock);  in btrfs_put_block_group_trimming()
3325 		em = lookup_extent_mapping(em_tree, block_group->key.objectid,  in btrfs_put_block_group_trimming()
3344 		__btrfs_remove_free_space_cache(block_group->free_space_ctl);  in btrfs_put_block_group_trimming()
3348 int btrfs_trim_block_group(struct btrfs_block_group_cache *block_group,  in btrfs_trim_block_group()  argument
3355 	spin_lock(&block_group->lock);  in btrfs_trim_block_group()
3356 	if (block_group->removed) {  in btrfs_trim_block_group()
3357 		spin_unlock(&block_group->lock);  in btrfs_trim_block_group()
3360 	btrfs_get_block_group_trimming(block_group);  in btrfs_trim_block_group()
3361 	spin_unlock(&block_group->lock);  in btrfs_trim_block_group()
3363 	ret = trim_no_bitmap(block_group, trimmed, start, end, minlen);  in btrfs_trim_block_group()
3367 	ret = trim_bitmaps(block_group, trimmed, start, end, minlen);  in btrfs_trim_block_group()
3369 	btrfs_put_block_group_trimming(block_group);  in btrfs_trim_block_group()