1634 static void ext4_mb_use_best_found(struct ext4_allocation_context *ac,  in ext4_mb_use_best_found()  argument
1637 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_use_best_found()
1640 	BUG_ON(ac->ac_b_ex.fe_group != e4b->bd_group);  in ext4_mb_use_best_found()
1641 	BUG_ON(ac->ac_status == AC_STATUS_FOUND);  in ext4_mb_use_best_found()
1643 	ac->ac_b_ex.fe_len = min(ac->ac_b_ex.fe_len, ac->ac_g_ex.fe_len);  in ext4_mb_use_best_found()
1644 	ac->ac_b_ex.fe_logical = ac->ac_g_ex.fe_logical;  in ext4_mb_use_best_found()
1645 	ret = mb_mark_used(e4b, &ac->ac_b_ex);  in ext4_mb_use_best_found()
1649 	ac->ac_f_ex = ac->ac_b_ex;  in ext4_mb_use_best_found()
1651 	ac->ac_status = AC_STATUS_FOUND;  in ext4_mb_use_best_found()
1652 	ac->ac_tail = ret & 0xffff;  in ext4_mb_use_best_found()
1653 	ac->ac_buddy = ret >> 16;  in ext4_mb_use_best_found()
1662 	ac->ac_bitmap_page = e4b->bd_bitmap_page;  in ext4_mb_use_best_found()
1663 	get_page(ac->ac_bitmap_page);  in ext4_mb_use_best_found()
1664 	ac->ac_buddy_page = e4b->bd_buddy_page;  in ext4_mb_use_best_found()
1665 	get_page(ac->ac_buddy_page);  in ext4_mb_use_best_found()
1667 	if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) {  in ext4_mb_use_best_found()
1669 		sbi->s_mb_last_group = ac->ac_f_ex.fe_group;  in ext4_mb_use_best_found()
1670 		sbi->s_mb_last_start = ac->ac_f_ex.fe_start;  in ext4_mb_use_best_found()
1679 static void ext4_mb_check_limits(struct ext4_allocation_context *ac,  in ext4_mb_check_limits()  argument
1683 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_check_limits()
1684 	struct ext4_free_extent *bex = &ac->ac_b_ex;  in ext4_mb_check_limits()
1685 	struct ext4_free_extent *gex = &ac->ac_g_ex;  in ext4_mb_check_limits()
1689 	if (ac->ac_status == AC_STATUS_FOUND)  in ext4_mb_check_limits()
1694 	if (ac->ac_found > sbi->s_mb_max_to_scan &&  in ext4_mb_check_limits()
1695 			!(ac->ac_flags & EXT4_MB_HINT_FIRST)) {  in ext4_mb_check_limits()
1696 		ac->ac_status = AC_STATUS_BREAK;  in ext4_mb_check_limits()
1706 	if ((finish_group || ac->ac_found > sbi->s_mb_min_to_scan)  in ext4_mb_check_limits()
1713 			ext4_mb_use_best_found(ac, e4b);  in ext4_mb_check_limits()
1729 static void ext4_mb_measure_extent(struct ext4_allocation_context *ac,  in ext4_mb_measure_extent()  argument
1733 	struct ext4_free_extent *bex = &ac->ac_b_ex;  in ext4_mb_measure_extent()
1734 	struct ext4_free_extent *gex = &ac->ac_g_ex;  in ext4_mb_measure_extent()
1737 	BUG_ON(ex->fe_len > EXT4_CLUSTERS_PER_GROUP(ac->ac_sb));  in ext4_mb_measure_extent()
1738 	BUG_ON(ex->fe_start >= EXT4_CLUSTERS_PER_GROUP(ac->ac_sb));  in ext4_mb_measure_extent()
1739 	BUG_ON(ac->ac_status != AC_STATUS_CONTINUE);  in ext4_mb_measure_extent()
1741 	ac->ac_found++;  in ext4_mb_measure_extent()
1746 	if (unlikely(ac->ac_flags & EXT4_MB_HINT_FIRST)) {  in ext4_mb_measure_extent()
1748 		ext4_mb_use_best_found(ac, e4b);  in ext4_mb_measure_extent()
1757 		ext4_mb_use_best_found(ac, e4b);  in ext4_mb_measure_extent()
1785 	ext4_mb_check_limits(ac, e4b, 0);  in ext4_mb_measure_extent()
1789 int ext4_mb_try_best_found(struct ext4_allocation_context *ac,  in ext4_mb_try_best_found()  argument
1792 	struct ext4_free_extent ex = ac->ac_b_ex;  in ext4_mb_try_best_found()
1798 	err = ext4_mb_load_buddy(ac->ac_sb, group, e4b);  in ext4_mb_try_best_found()
1802 	ext4_lock_group(ac->ac_sb, group);  in ext4_mb_try_best_found()
1806 		ac->ac_b_ex = ex;  in ext4_mb_try_best_found()
1807 		ext4_mb_use_best_found(ac, e4b);  in ext4_mb_try_best_found()
1810 	ext4_unlock_group(ac->ac_sb, group);  in ext4_mb_try_best_found()
1817 int ext4_mb_find_by_goal(struct ext4_allocation_context *ac,  in ext4_mb_find_by_goal()  argument
1820 	ext4_group_t group = ac->ac_g_ex.fe_group;  in ext4_mb_find_by_goal()
1823 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_find_by_goal()
1824 	struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group);  in ext4_mb_find_by_goal()
1827 	if (!(ac->ac_flags & EXT4_MB_HINT_TRY_GOAL))  in ext4_mb_find_by_goal()
1832 	err = ext4_mb_load_buddy(ac->ac_sb, group, e4b);  in ext4_mb_find_by_goal()
1841 	ext4_lock_group(ac->ac_sb, group);  in ext4_mb_find_by_goal()
1842 	max = mb_find_extent(e4b, ac->ac_g_ex.fe_start,  in ext4_mb_find_by_goal()
1843 			     ac->ac_g_ex.fe_len, &ex);  in ext4_mb_find_by_goal()
1846 	if (max >= ac->ac_g_ex.fe_len && ac->ac_g_ex.fe_len == sbi->s_stripe) {  in ext4_mb_find_by_goal()
1849 		start = ext4_group_first_block_no(ac->ac_sb, e4b->bd_group) +  in ext4_mb_find_by_goal()
1853 			ac->ac_found++;  in ext4_mb_find_by_goal()
1854 			ac->ac_b_ex = ex;  in ext4_mb_find_by_goal()
1855 			ext4_mb_use_best_found(ac, e4b);  in ext4_mb_find_by_goal()
1857 	} else if (max >= ac->ac_g_ex.fe_len) {  in ext4_mb_find_by_goal()
1859 		BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group);  in ext4_mb_find_by_goal()
1860 		BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start);  in ext4_mb_find_by_goal()
1861 		ac->ac_found++;  in ext4_mb_find_by_goal()
1862 		ac->ac_b_ex = ex;  in ext4_mb_find_by_goal()
1863 		ext4_mb_use_best_found(ac, e4b);  in ext4_mb_find_by_goal()
1864 	} else if (max > 0 && (ac->ac_flags & EXT4_MB_HINT_MERGE)) {  in ext4_mb_find_by_goal()
1868 		BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group);  in ext4_mb_find_by_goal()
1869 		BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start);  in ext4_mb_find_by_goal()
1870 		ac->ac_found++;  in ext4_mb_find_by_goal()
1871 		ac->ac_b_ex = ex;  in ext4_mb_find_by_goal()
1872 		ext4_mb_use_best_found(ac, e4b);  in ext4_mb_find_by_goal()
1874 	ext4_unlock_group(ac->ac_sb, group);  in ext4_mb_find_by_goal()
1885 void ext4_mb_simple_scan_group(struct ext4_allocation_context *ac,  in ext4_mb_simple_scan_group()  argument
1888 	struct super_block *sb = ac->ac_sb;  in ext4_mb_simple_scan_group()
1895 	BUG_ON(ac->ac_2order <= 0);  in ext4_mb_simple_scan_group()
1896 	for (i = ac->ac_2order; i <= sb->s_blocksize_bits + 1; i++) {  in ext4_mb_simple_scan_group()
1906 		ac->ac_found++;  in ext4_mb_simple_scan_group()
1908 		ac->ac_b_ex.fe_len = 1 << i;  in ext4_mb_simple_scan_group()
1909 		ac->ac_b_ex.fe_start = k << i;  in ext4_mb_simple_scan_group()
1910 		ac->ac_b_ex.fe_group = e4b->bd_group;  in ext4_mb_simple_scan_group()
1912 		ext4_mb_use_best_found(ac, e4b);  in ext4_mb_simple_scan_group()
1914 		BUG_ON(ac->ac_b_ex.fe_len != ac->ac_g_ex.fe_len);  in ext4_mb_simple_scan_group()
1929 void ext4_mb_complex_scan_group(struct ext4_allocation_context *ac,  in ext4_mb_complex_scan_group()  argument
1932 	struct super_block *sb = ac->ac_sb;  in ext4_mb_complex_scan_group()
1943 	while (free && ac->ac_status == AC_STATUS_CONTINUE) {  in ext4_mb_complex_scan_group()
1961 		mb_find_extent(e4b, i, ac->ac_g_ex.fe_len, &ex);  in ext4_mb_complex_scan_group()
1978 		ext4_mb_measure_extent(ac, &ex, e4b);  in ext4_mb_complex_scan_group()
1984 	ext4_mb_check_limits(ac, e4b, 1);  in ext4_mb_complex_scan_group()
1992 void ext4_mb_scan_aligned(struct ext4_allocation_context *ac,  in ext4_mb_scan_aligned()  argument
1995 	struct super_block *sb = ac->ac_sb;  in ext4_mb_scan_aligned()
2017 				ac->ac_found++;  in ext4_mb_scan_aligned()
2019 				ac->ac_b_ex = ex;  in ext4_mb_scan_aligned()
2020 				ext4_mb_use_best_found(ac, e4b);  in ext4_mb_scan_aligned()
2034 static int ext4_mb_good_group(struct ext4_allocation_context *ac,  in ext4_mb_good_group()  argument
2038 	int flex_size = ext4_flex_bg_size(EXT4_SB(ac->ac_sb));  in ext4_mb_good_group()
2039 	struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group);  in ext4_mb_good_group()
2046 	if (cr <= 2 && free < ac->ac_g_ex.fe_len)  in ext4_mb_good_group()
2054 		int ret = ext4_mb_init_group(ac->ac_sb, group, GFP_NOFS);  in ext4_mb_good_group()
2065 		BUG_ON(ac->ac_2order == 0);  in ext4_mb_good_group()
2068 		if ((ac->ac_flags & EXT4_MB_HINT_DATA) &&  in ext4_mb_good_group()
2073 		if ((ac->ac_2order > ac->ac_sb->s_blocksize_bits+1) ||  in ext4_mb_good_group()
2074 		    (free / fragments) >= ac->ac_g_ex.fe_len)  in ext4_mb_good_group()
2077 		if (grp->bb_largest_free_order < ac->ac_2order)  in ext4_mb_good_group()
2082 		if ((free / fragments) >= ac->ac_g_ex.fe_len)  in ext4_mb_good_group()
2086 		if (free >= ac->ac_g_ex.fe_len)  in ext4_mb_good_group()
2099 ext4_mb_regular_allocator(struct ext4_allocation_context *ac)  in ext4_mb_regular_allocator()  argument
2108 	sb = ac->ac_sb;  in ext4_mb_regular_allocator()
2112 	if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)))  in ext4_mb_regular_allocator()
2115 	BUG_ON(ac->ac_status == AC_STATUS_FOUND);  in ext4_mb_regular_allocator()
2118 	err = ext4_mb_find_by_goal(ac, &e4b);  in ext4_mb_regular_allocator()
2119 	if (err || ac->ac_status == AC_STATUS_FOUND)  in ext4_mb_regular_allocator()
2122 	if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY))  in ext4_mb_regular_allocator()
2130 	i = fls(ac->ac_g_ex.fe_len);  in ext4_mb_regular_allocator()
2131 	ac->ac_2order = 0;  in ext4_mb_regular_allocator()
2143 		if ((ac->ac_g_ex.fe_len & (~(1 << (i - 1)))) == 0)  in ext4_mb_regular_allocator()
2144 			ac->ac_2order = array_index_nospec(i - 1,  in ext4_mb_regular_allocator()
2149 	if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) {  in ext4_mb_regular_allocator()
2152 		ac->ac_g_ex.fe_group = sbi->s_mb_last_group;  in ext4_mb_regular_allocator()
2153 		ac->ac_g_ex.fe_start = sbi->s_mb_last_start;  in ext4_mb_regular_allocator()
2158 	cr = ac->ac_2order ? 0 : 1;  in ext4_mb_regular_allocator()
2164 	for (; cr < 4 && ac->ac_status == AC_STATUS_CONTINUE; cr++) {  in ext4_mb_regular_allocator()
2165 		ac->ac_criteria = cr;  in ext4_mb_regular_allocator()
2170 		group = ac->ac_g_ex.fe_group;  in ext4_mb_regular_allocator()
2183 			ret = ext4_mb_good_group(ac, group, cr);  in ext4_mb_regular_allocator()
2200 			ret = ext4_mb_good_group(ac, group, cr);  in ext4_mb_regular_allocator()
2209 			ac->ac_groups_scanned++;  in ext4_mb_regular_allocator()
2211 				ext4_mb_simple_scan_group(ac, &e4b);  in ext4_mb_regular_allocator()
2213 					!(ac->ac_g_ex.fe_len % sbi->s_stripe))  in ext4_mb_regular_allocator()
2214 				ext4_mb_scan_aligned(ac, &e4b);  in ext4_mb_regular_allocator()
2216 				ext4_mb_complex_scan_group(ac, &e4b);  in ext4_mb_regular_allocator()
2221 			if (ac->ac_status != AC_STATUS_CONTINUE)  in ext4_mb_regular_allocator()
2226 	if (ac->ac_b_ex.fe_len > 0 && ac->ac_status != AC_STATUS_FOUND &&  in ext4_mb_regular_allocator()
2227 	    !(ac->ac_flags & EXT4_MB_HINT_FIRST)) {  in ext4_mb_regular_allocator()
2233 		ext4_mb_try_best_found(ac, &e4b);  in ext4_mb_regular_allocator()
2234 		if (ac->ac_status != AC_STATUS_FOUND) {  in ext4_mb_regular_allocator()
2241 			ac->ac_b_ex.fe_group = 0;  in ext4_mb_regular_allocator()
2242 			ac->ac_b_ex.fe_start = 0;  in ext4_mb_regular_allocator()
2243 			ac->ac_b_ex.fe_len = 0;  in ext4_mb_regular_allocator()
2244 			ac->ac_status = AC_STATUS_CONTINUE;  in ext4_mb_regular_allocator()
2245 			ac->ac_flags |= EXT4_MB_HINT_FIRST;  in ext4_mb_regular_allocator()
2252 	if (!err && ac->ac_status != AC_STATUS_FOUND && first_err)  in ext4_mb_regular_allocator()
2923 ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac,  in ext4_mb_mark_diskspace_used()  argument
2934 	BUG_ON(ac->ac_status != AC_STATUS_FOUND);  in ext4_mb_mark_diskspace_used()
2935 	BUG_ON(ac->ac_b_ex.fe_len <= 0);  in ext4_mb_mark_diskspace_used()
2937 	sb = ac->ac_sb;  in ext4_mb_mark_diskspace_used()
2940 	bitmap_bh = ext4_read_block_bitmap(sb, ac->ac_b_ex.fe_group);  in ext4_mb_mark_diskspace_used()
2953 	gdp = ext4_get_group_desc(sb, ac->ac_b_ex.fe_group, &gdp_bh);  in ext4_mb_mark_diskspace_used()
2957 	ext4_debug("using block group %u(%d)\n", ac->ac_b_ex.fe_group,  in ext4_mb_mark_diskspace_used()
2965 	block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex);  in ext4_mb_mark_diskspace_used()
2967 	len = EXT4_C2B(sbi, ac->ac_b_ex.fe_len);  in ext4_mb_mark_diskspace_used()
2975 		ext4_lock_group(sb, ac->ac_b_ex.fe_group);  in ext4_mb_mark_diskspace_used()
2976 		ext4_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start,  in ext4_mb_mark_diskspace_used()
2977 			      ac->ac_b_ex.fe_len);  in ext4_mb_mark_diskspace_used()
2978 		ext4_unlock_group(sb, ac->ac_b_ex.fe_group);  in ext4_mb_mark_diskspace_used()
2985 	ext4_lock_group(sb, ac->ac_b_ex.fe_group);  in ext4_mb_mark_diskspace_used()
2989 		for (i = 0; i < ac->ac_b_ex.fe_len; i++) {  in ext4_mb_mark_diskspace_used()
2990 			BUG_ON(mb_test_bit(ac->ac_b_ex.fe_start + i,  in ext4_mb_mark_diskspace_used()
2995 	ext4_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start,  in ext4_mb_mark_diskspace_used()
2996 		      ac->ac_b_ex.fe_len);  in ext4_mb_mark_diskspace_used()
3002 						ac->ac_b_ex.fe_group, gdp));  in ext4_mb_mark_diskspace_used()
3004 	len = ext4_free_group_clusters(sb, gdp) - ac->ac_b_ex.fe_len;  in ext4_mb_mark_diskspace_used()
3006 	ext4_block_bitmap_csum_set(sb, ac->ac_b_ex.fe_group, gdp, bitmap_bh);  in ext4_mb_mark_diskspace_used()
3007 	ext4_group_desc_csum_set(sb, ac->ac_b_ex.fe_group, gdp);  in ext4_mb_mark_diskspace_used()
3009 	ext4_unlock_group(sb, ac->ac_b_ex.fe_group);  in ext4_mb_mark_diskspace_used()
3010 	percpu_counter_sub(&sbi->s_freeclusters_counter, ac->ac_b_ex.fe_len);  in ext4_mb_mark_diskspace_used()
3014 	if (!(ac->ac_flags & EXT4_MB_DELALLOC_RESERVED))  in ext4_mb_mark_diskspace_used()
3021 							  ac->ac_b_ex.fe_group);  in ext4_mb_mark_diskspace_used()
3022 		atomic64_sub(ac->ac_b_ex.fe_len,  in ext4_mb_mark_diskspace_used()
3045 static void ext4_mb_normalize_group_request(struct ext4_allocation_context *ac)  in ext4_mb_normalize_group_request()  argument
3047 	struct super_block *sb = ac->ac_sb;  in ext4_mb_normalize_group_request()
3048 	struct ext4_locality_group *lg = ac->ac_lg;  in ext4_mb_normalize_group_request()
3051 	ac->ac_g_ex.fe_len = EXT4_SB(sb)->s_mb_group_prealloc;  in ext4_mb_normalize_group_request()
3053 		current->pid, ac->ac_g_ex.fe_len);  in ext4_mb_normalize_group_request()
3061 ext4_mb_normalize_request(struct ext4_allocation_context *ac,  in ext4_mb_normalize_request()  argument
3064 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_normalize_request()
3070 	struct ext4_inode_info *ei = EXT4_I(ac->ac_inode);  in ext4_mb_normalize_request()
3075 	if (!(ac->ac_flags & EXT4_MB_HINT_DATA))  in ext4_mb_normalize_request()
3079 	if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY))  in ext4_mb_normalize_request()
3084 	if (ac->ac_flags & EXT4_MB_HINT_NOPREALLOC)  in ext4_mb_normalize_request()
3087 	if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) {  in ext4_mb_normalize_request()
3088 		ext4_mb_normalize_group_request(ac);  in ext4_mb_normalize_request()
3092 	bsbits = ac->ac_sb->s_blocksize_bits;  in ext4_mb_normalize_request()
3096 	size = ac->ac_o_ex.fe_logical + EXT4_C2B(sbi, ac->ac_o_ex.fe_len);  in ext4_mb_normalize_request()
3098 	if (size < i_size_read(ac->ac_inode))  in ext4_mb_normalize_request()
3099 		size = i_size_read(ac->ac_inode);  in ext4_mb_normalize_request()
3126 		start_off = ((loff_t)ac->ac_o_ex.fe_logical >>  in ext4_mb_normalize_request()
3130 		start_off = ((loff_t)ac->ac_o_ex.fe_logical >>  in ext4_mb_normalize_request()
3133 	} else if (NRL_CHECK_SIZE(ac->ac_o_ex.fe_len,  in ext4_mb_normalize_request()
3135 		start_off = ((loff_t)ac->ac_o_ex.fe_logical >>  in ext4_mb_normalize_request()
3139 		start_off = (loff_t) ac->ac_o_ex.fe_logical << bsbits;  in ext4_mb_normalize_request()
3140 		size	  = (loff_t) EXT4_C2B(EXT4_SB(ac->ac_sb),  in ext4_mb_normalize_request()
3141 					      ac->ac_o_ex.fe_len) << bsbits;  in ext4_mb_normalize_request()
3158 	if (size > EXT4_BLOCKS_PER_GROUP(ac->ac_sb))  in ext4_mb_normalize_request()
3159 		size = EXT4_BLOCKS_PER_GROUP(ac->ac_sb);  in ext4_mb_normalize_request()
3176 		pa_end = pa->pa_lstart + EXT4_C2B(EXT4_SB(ac->ac_sb),  in ext4_mb_normalize_request()
3180 		BUG_ON(!(ac->ac_o_ex.fe_logical >= pa_end ||  in ext4_mb_normalize_request()
3181 			ac->ac_o_ex.fe_logical < pa->pa_lstart));  in ext4_mb_normalize_request()
3191 		if (pa_end <= ac->ac_o_ex.fe_logical) {  in ext4_mb_normalize_request()
3194 		} else if (pa->pa_lstart > ac->ac_o_ex.fe_logical) {  in ext4_mb_normalize_request()
3210 			pa_end = pa->pa_lstart + EXT4_C2B(EXT4_SB(ac->ac_sb),  in ext4_mb_normalize_request()
3218 	if (start + size <= ac->ac_o_ex.fe_logical &&  in ext4_mb_normalize_request()
3219 			start > ac->ac_o_ex.fe_logical) {  in ext4_mb_normalize_request()
3220 		ext4_msg(ac->ac_sb, KERN_ERR,  in ext4_mb_normalize_request()
3223 			 (unsigned long) ac->ac_o_ex.fe_logical);  in ext4_mb_normalize_request()
3226 	BUG_ON(size <= 0 || size > EXT4_BLOCKS_PER_GROUP(ac->ac_sb));  in ext4_mb_normalize_request()
3232 	ac->ac_g_ex.fe_logical = start;  in ext4_mb_normalize_request()
3233 	ac->ac_g_ex.fe_len = EXT4_NUM_B2C(sbi, size);  in ext4_mb_normalize_request()
3238 		ext4_get_group_no_and_offset(ac->ac_sb, ar->pright - size,  in ext4_mb_normalize_request()
3239 						&ac->ac_f_ex.fe_group,  in ext4_mb_normalize_request()
3240 						&ac->ac_f_ex.fe_start);  in ext4_mb_normalize_request()
3241 		ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL;  in ext4_mb_normalize_request()
3245 		ext4_get_group_no_and_offset(ac->ac_sb, ar->pleft + 1,  in ext4_mb_normalize_request()
3246 						&ac->ac_f_ex.fe_group,  in ext4_mb_normalize_request()
3247 						&ac->ac_f_ex.fe_start);  in ext4_mb_normalize_request()
3248 		ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL;  in ext4_mb_normalize_request()
3255 static void ext4_mb_collect_stats(struct ext4_allocation_context *ac)  in ext4_mb_collect_stats()  argument
3257 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_collect_stats()
3259 	if (sbi->s_mb_stats && ac->ac_g_ex.fe_len > 1) {  in ext4_mb_collect_stats()
3261 		atomic_add(ac->ac_b_ex.fe_len, &sbi->s_bal_allocated);  in ext4_mb_collect_stats()
3262 		if (ac->ac_b_ex.fe_len >= ac->ac_o_ex.fe_len)  in ext4_mb_collect_stats()
3264 		atomic_add(ac->ac_found, &sbi->s_bal_ex_scanned);  in ext4_mb_collect_stats()
3265 		if (ac->ac_g_ex.fe_start == ac->ac_b_ex.fe_start &&  in ext4_mb_collect_stats()
3266 				ac->ac_g_ex.fe_group == ac->ac_b_ex.fe_group)  in ext4_mb_collect_stats()
3268 		if (ac->ac_found > sbi->s_mb_max_to_scan)  in ext4_mb_collect_stats()
3272 	if (ac->ac_op == EXT4_MB_HISTORY_ALLOC)  in ext4_mb_collect_stats()
3273 		trace_ext4_mballoc_alloc(ac);  in ext4_mb_collect_stats()
3275 		trace_ext4_mballoc_prealloc(ac);  in ext4_mb_collect_stats()
3284 static void ext4_discard_allocated_blocks(struct ext4_allocation_context *ac)  in ext4_discard_allocated_blocks()  argument
3286 	struct ext4_prealloc_space *pa = ac->ac_pa;  in ext4_discard_allocated_blocks()
3291 		if (ac->ac_f_ex.fe_len == 0)  in ext4_discard_allocated_blocks()
3293 		err = ext4_mb_load_buddy(ac->ac_sb, ac->ac_f_ex.fe_group, &e4b);  in ext4_discard_allocated_blocks()
3303 		ext4_lock_group(ac->ac_sb, ac->ac_f_ex.fe_group);  in ext4_discard_allocated_blocks()
3304 		mb_free_blocks(ac->ac_inode, &e4b, ac->ac_f_ex.fe_start,  in ext4_discard_allocated_blocks()
3305 			       ac->ac_f_ex.fe_len);  in ext4_discard_allocated_blocks()
3306 		ext4_unlock_group(ac->ac_sb, ac->ac_f_ex.fe_group);  in ext4_discard_allocated_blocks()
3311 		pa->pa_free += ac->ac_b_ex.fe_len;  in ext4_discard_allocated_blocks()
3317 static void ext4_mb_use_inode_pa(struct ext4_allocation_context *ac,  in ext4_mb_use_inode_pa()  argument
3320 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_use_inode_pa()
3326 	start = pa->pa_pstart + (ac->ac_o_ex.fe_logical - pa->pa_lstart);  in ext4_mb_use_inode_pa()
3328 		  start + EXT4_C2B(sbi, ac->ac_o_ex.fe_len));  in ext4_mb_use_inode_pa()
3330 	ext4_get_group_no_and_offset(ac->ac_sb, start, &ac->ac_b_ex.fe_group,  in ext4_mb_use_inode_pa()
3331 					&ac->ac_b_ex.fe_start);  in ext4_mb_use_inode_pa()
3332 	ac->ac_b_ex.fe_len = len;  in ext4_mb_use_inode_pa()
3333 	ac->ac_status = AC_STATUS_FOUND;  in ext4_mb_use_inode_pa()
3334 	ac->ac_pa = pa;  in ext4_mb_use_inode_pa()
3347 static void ext4_mb_use_group_pa(struct ext4_allocation_context *ac,  in ext4_mb_use_group_pa()  argument
3350 	unsigned int len = ac->ac_o_ex.fe_len;  in ext4_mb_use_group_pa()
3352 	ext4_get_group_no_and_offset(ac->ac_sb, pa->pa_pstart,  in ext4_mb_use_group_pa()
3353 					&ac->ac_b_ex.fe_group,  in ext4_mb_use_group_pa()
3354 					&ac->ac_b_ex.fe_start);  in ext4_mb_use_group_pa()
3355 	ac->ac_b_ex.fe_len = len;  in ext4_mb_use_group_pa()
3356 	ac->ac_status = AC_STATUS_FOUND;  in ext4_mb_use_group_pa()
3357 	ac->ac_pa = pa;  in ext4_mb_use_group_pa()
3401 ext4_mb_use_preallocated(struct ext4_allocation_context *ac)  in ext4_mb_use_preallocated()  argument
3403 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_use_preallocated()
3405 	struct ext4_inode_info *ei = EXT4_I(ac->ac_inode);  in ext4_mb_use_preallocated()
3411 	if (!(ac->ac_flags & EXT4_MB_HINT_DATA))  in ext4_mb_use_preallocated()
3420 		if (ac->ac_o_ex.fe_logical < pa->pa_lstart ||  in ext4_mb_use_preallocated()
3421 		    ac->ac_o_ex.fe_logical >= (pa->pa_lstart +  in ext4_mb_use_preallocated()
3426 		if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)) &&  in ext4_mb_use_preallocated()
3435 			ext4_mb_use_inode_pa(ac, pa);  in ext4_mb_use_preallocated()
3437 			ac->ac_criteria = 10;  in ext4_mb_use_preallocated()
3446 	if (!(ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC))  in ext4_mb_use_preallocated()
3450 	lg = ac->ac_lg;  in ext4_mb_use_preallocated()
3453 	order  = fls(ac->ac_o_ex.fe_len) - 1;  in ext4_mb_use_preallocated()
3458 	goal_block = ext4_grp_offs_to_block(ac->ac_sb, &ac->ac_g_ex);  in ext4_mb_use_preallocated()
3469 					pa->pa_free >= ac->ac_o_ex.fe_len) {  in ext4_mb_use_preallocated()
3479 		ext4_mb_use_group_pa(ac, cpa);  in ext4_mb_use_preallocated()
3480 		ac->ac_criteria = 20;  in ext4_mb_use_preallocated()
3565 static void ext4_mb_put_pa(struct ext4_allocation_context *ac,  in ext4_mb_put_pa()  argument
3625 ext4_mb_new_inode_pa(struct ext4_allocation_context *ac)  in ext4_mb_new_inode_pa()  argument
3627 	struct super_block *sb = ac->ac_sb;  in ext4_mb_new_inode_pa()
3634 	BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len);  in ext4_mb_new_inode_pa()
3635 	BUG_ON(ac->ac_status != AC_STATUS_FOUND);  in ext4_mb_new_inode_pa()
3636 	BUG_ON(!S_ISREG(ac->ac_inode->i_mode));  in ext4_mb_new_inode_pa()
3642 	if (ac->ac_b_ex.fe_len < ac->ac_g_ex.fe_len) {  in ext4_mb_new_inode_pa()
3651 		BUG_ON(ac->ac_g_ex.fe_logical > ac->ac_o_ex.fe_logical);  in ext4_mb_new_inode_pa()
3652 		BUG_ON(ac->ac_g_ex.fe_len < ac->ac_o_ex.fe_len);  in ext4_mb_new_inode_pa()
3657 		winl = ac->ac_o_ex.fe_logical - ac->ac_g_ex.fe_logical;  in ext4_mb_new_inode_pa()
3660 		wins = EXT4_C2B(sbi, ac->ac_b_ex.fe_len - ac->ac_o_ex.fe_len);  in ext4_mb_new_inode_pa()
3665 		offs = ac->ac_o_ex.fe_logical %  in ext4_mb_new_inode_pa()
3666 			EXT4_C2B(sbi, ac->ac_b_ex.fe_len);  in ext4_mb_new_inode_pa()
3670 		ac->ac_b_ex.fe_logical = ac->ac_o_ex.fe_logical -  in ext4_mb_new_inode_pa()
3672 		BUG_ON(ac->ac_o_ex.fe_logical < ac->ac_b_ex.fe_logical);  in ext4_mb_new_inode_pa()
3673 		BUG_ON(ac->ac_o_ex.fe_len > ac->ac_b_ex.fe_len);  in ext4_mb_new_inode_pa()
3678 	ac->ac_f_ex = ac->ac_b_ex;  in ext4_mb_new_inode_pa()
3680 	pa->pa_lstart = ac->ac_b_ex.fe_logical;  in ext4_mb_new_inode_pa()
3681 	pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex);  in ext4_mb_new_inode_pa()
3682 	pa->pa_len = ac->ac_b_ex.fe_len;  in ext4_mb_new_inode_pa()
3693 	trace_ext4_mb_new_inode_pa(ac, pa);  in ext4_mb_new_inode_pa()
3695 	ext4_mb_use_inode_pa(ac, pa);  in ext4_mb_new_inode_pa()
3698 	ei = EXT4_I(ac->ac_inode);  in ext4_mb_new_inode_pa()
3699 	grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group);  in ext4_mb_new_inode_pa()
3702 	pa->pa_inode = ac->ac_inode;  in ext4_mb_new_inode_pa()
3704 	ext4_lock_group(sb, ac->ac_b_ex.fe_group);  in ext4_mb_new_inode_pa()
3706 	ext4_unlock_group(sb, ac->ac_b_ex.fe_group);  in ext4_mb_new_inode_pa()
3719 ext4_mb_new_group_pa(struct ext4_allocation_context *ac)  in ext4_mb_new_group_pa()  argument
3721 	struct super_block *sb = ac->ac_sb;  in ext4_mb_new_group_pa()
3727 	BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len);  in ext4_mb_new_group_pa()
3728 	BUG_ON(ac->ac_status != AC_STATUS_FOUND);  in ext4_mb_new_group_pa()
3729 	BUG_ON(!S_ISREG(ac->ac_inode->i_mode));  in ext4_mb_new_group_pa()
3738 	ac->ac_f_ex = ac->ac_b_ex;  in ext4_mb_new_group_pa()
3740 	pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex);  in ext4_mb_new_group_pa()
3742 	pa->pa_len = ac->ac_b_ex.fe_len;  in ext4_mb_new_group_pa()
3753 	trace_ext4_mb_new_group_pa(ac, pa);  in ext4_mb_new_group_pa()
3755 	ext4_mb_use_group_pa(ac, pa);  in ext4_mb_new_group_pa()
3758 	grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group);  in ext4_mb_new_group_pa()
3759 	lg = ac->ac_lg;  in ext4_mb_new_group_pa()
3765 	ext4_lock_group(sb, ac->ac_b_ex.fe_group);  in ext4_mb_new_group_pa()
3767 	ext4_unlock_group(sb, ac->ac_b_ex.fe_group);  in ext4_mb_new_group_pa()
3776 static int ext4_mb_new_preallocation(struct ext4_allocation_context *ac)  in ext4_mb_new_preallocation()  argument
3780 	if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC)  in ext4_mb_new_preallocation()
3781 		err = ext4_mb_new_group_pa(ac);  in ext4_mb_new_preallocation()
3783 		err = ext4_mb_new_inode_pa(ac);  in ext4_mb_new_preallocation()
4094 static void ext4_mb_show_ac(struct ext4_allocation_context *ac)  in ext4_mb_show_ac()  argument
4096 	struct super_block *sb = ac->ac_sb;  in ext4_mb_show_ac()
4103 	ext4_msg(ac->ac_sb, KERN_ERR, "Can't allocate:"  in ext4_mb_show_ac()
4105 	ext4_msg(ac->ac_sb, KERN_ERR, "status %d flags %d",  in ext4_mb_show_ac()
4106 			ac->ac_status, ac->ac_flags);  in ext4_mb_show_ac()
4107 	ext4_msg(ac->ac_sb, KERN_ERR, "orig %lu/%lu/%lu@%lu, "  in ext4_mb_show_ac()
4110 			(unsigned long)ac->ac_o_ex.fe_group,  in ext4_mb_show_ac()
4111 			(unsigned long)ac->ac_o_ex.fe_start,  in ext4_mb_show_ac()
4112 			(unsigned long)ac->ac_o_ex.fe_len,  in ext4_mb_show_ac()
4113 			(unsigned long)ac->ac_o_ex.fe_logical,  in ext4_mb_show_ac()
4114 			(unsigned long)ac->ac_g_ex.fe_group,  in ext4_mb_show_ac()
4115 			(unsigned long)ac->ac_g_ex.fe_start,  in ext4_mb_show_ac()
4116 			(unsigned long)ac->ac_g_ex.fe_len,  in ext4_mb_show_ac()
4117 			(unsigned long)ac->ac_g_ex.fe_logical,  in ext4_mb_show_ac()
4118 			(unsigned long)ac->ac_b_ex.fe_group,  in ext4_mb_show_ac()
4119 			(unsigned long)ac->ac_b_ex.fe_start,  in ext4_mb_show_ac()
4120 			(unsigned long)ac->ac_b_ex.fe_len,  in ext4_mb_show_ac()
4121 			(unsigned long)ac->ac_b_ex.fe_logical,  in ext4_mb_show_ac()
4122 			(int)ac->ac_criteria);  in ext4_mb_show_ac()
4123 	ext4_msg(ac->ac_sb, KERN_ERR, "%d found", ac->ac_found);  in ext4_mb_show_ac()
4124 	ext4_msg(ac->ac_sb, KERN_ERR, "groups: ");  in ext4_mb_show_ac()
4152 static inline void ext4_mb_show_ac(struct ext4_allocation_context *ac)  in ext4_mb_show_ac()  argument
4165 static void ext4_mb_group_or_file(struct ext4_allocation_context *ac)  in ext4_mb_group_or_file()  argument
4167 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_group_or_file()
4168 	int bsbits = ac->ac_sb->s_blocksize_bits;  in ext4_mb_group_or_file()
4171 	if (!(ac->ac_flags & EXT4_MB_HINT_DATA))  in ext4_mb_group_or_file()
4174 	if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY))  in ext4_mb_group_or_file()
4177 	size = ac->ac_o_ex.fe_logical + EXT4_C2B(sbi, ac->ac_o_ex.fe_len);  in ext4_mb_group_or_file()
4178 	isize = (i_size_read(ac->ac_inode) + ac->ac_sb->s_blocksize - 1)  in ext4_mb_group_or_file()
4182 	    !inode_is_open_for_write(ac->ac_inode)) {  in ext4_mb_group_or_file()
4183 		ac->ac_flags |= EXT4_MB_HINT_NOPREALLOC;  in ext4_mb_group_or_file()
4188 		ac->ac_flags |= EXT4_MB_STREAM_ALLOC;  in ext4_mb_group_or_file()
4195 		ac->ac_flags |= EXT4_MB_STREAM_ALLOC;  in ext4_mb_group_or_file()
4199 	BUG_ON(ac->ac_lg != NULL);  in ext4_mb_group_or_file()
4205 	ac->ac_lg = raw_cpu_ptr(sbi->s_locality_groups);  in ext4_mb_group_or_file()
4208 	ac->ac_flags |= EXT4_MB_HINT_GROUP_ALLOC;  in ext4_mb_group_or_file()
4211 	mutex_lock(&ac->ac_lg->lg_mutex);  in ext4_mb_group_or_file()
4215 ext4_mb_initialize_context(struct ext4_allocation_context *ac,  in ext4_mb_initialize_context()  argument
4241 	ac->ac_b_ex.fe_logical = EXT4_LBLK_CMASK(sbi, ar->logical);  in ext4_mb_initialize_context()
4242 	ac->ac_status = AC_STATUS_CONTINUE;  in ext4_mb_initialize_context()
4243 	ac->ac_sb = sb;  in ext4_mb_initialize_context()
4244 	ac->ac_inode = ar->inode;  in ext4_mb_initialize_context()
4245 	ac->ac_o_ex.fe_logical = ac->ac_b_ex.fe_logical;  in ext4_mb_initialize_context()
4246 	ac->ac_o_ex.fe_group = group;  in ext4_mb_initialize_context()
4247 	ac->ac_o_ex.fe_start = block;  in ext4_mb_initialize_context()
4248 	ac->ac_o_ex.fe_len = len;  in ext4_mb_initialize_context()
4249 	ac->ac_g_ex = ac->ac_o_ex;  in ext4_mb_initialize_context()
4250 	ac->ac_flags = ar->flags;  in ext4_mb_initialize_context()
4254 	ext4_mb_group_or_file(ac);  in ext4_mb_initialize_context()
4259 			(unsigned) ar->goal, ac->ac_flags, ac->ac_2order,  in ext4_mb_initialize_context()
4352 static void ext4_mb_add_n_trim(struct ext4_allocation_context *ac)  in ext4_mb_add_n_trim()  argument
4355 	struct super_block *sb = ac->ac_sb;  in ext4_mb_add_n_trim()
4356 	struct ext4_locality_group *lg = ac->ac_lg;  in ext4_mb_add_n_trim()
4357 	struct ext4_prealloc_space *tmp_pa, *pa = ac->ac_pa;  in ext4_mb_add_n_trim()
4402 static int ext4_mb_release_context(struct ext4_allocation_context *ac)  in ext4_mb_release_context()  argument
4404 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);  in ext4_mb_release_context()
4405 	struct ext4_prealloc_space *pa = ac->ac_pa;  in ext4_mb_release_context()
4410 			pa->pa_pstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len);  in ext4_mb_release_context()
4411 			pa->pa_lstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len);  in ext4_mb_release_context()
4412 			pa->pa_free -= ac->ac_b_ex.fe_len;  in ext4_mb_release_context()
4413 			pa->pa_len -= ac->ac_b_ex.fe_len;  in ext4_mb_release_context()
4428 			ext4_mb_add_n_trim(ac);  in ext4_mb_release_context()
4430 		ext4_mb_put_pa(ac, ac->ac_sb, pa);  in ext4_mb_release_context()
4432 	if (ac->ac_bitmap_page)  in ext4_mb_release_context()
4433 		put_page(ac->ac_bitmap_page);  in ext4_mb_release_context()
4434 	if (ac->ac_buddy_page)  in ext4_mb_release_context()
4435 		put_page(ac->ac_buddy_page);  in ext4_mb_release_context()
4436 	if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC)  in ext4_mb_release_context()
4437 		mutex_unlock(&ac->ac_lg->lg_mutex);  in ext4_mb_release_context()
4438 	ext4_mb_collect_stats(ac);  in ext4_mb_release_context()
4467 	struct ext4_allocation_context *ac = NULL;  in ext4_mb_new_blocks()  local
4520 	ac = kmem_cache_zalloc(ext4_ac_cachep, GFP_NOFS);  in ext4_mb_new_blocks()
4521 	if (!ac) {  in ext4_mb_new_blocks()
4527 	*errp = ext4_mb_initialize_context(ac, ar);  in ext4_mb_new_blocks()
4533 	ac->ac_op = EXT4_MB_HISTORY_PREALLOC;  in ext4_mb_new_blocks()
4534 	if (!ext4_mb_use_preallocated(ac)) {  in ext4_mb_new_blocks()
4535 		ac->ac_op = EXT4_MB_HISTORY_ALLOC;  in ext4_mb_new_blocks()
4536 		ext4_mb_normalize_request(ac, ar);  in ext4_mb_new_blocks()
4539 		*errp = ext4_mb_regular_allocator(ac);  in ext4_mb_new_blocks()
4546 		if (ac->ac_status == AC_STATUS_FOUND &&  in ext4_mb_new_blocks()
4547 		    ac->ac_o_ex.fe_len < ac->ac_b_ex.fe_len)  in ext4_mb_new_blocks()
4548 			*errp = ext4_mb_new_preallocation(ac);  in ext4_mb_new_blocks()
4551 			ext4_discard_allocated_blocks(ac);  in ext4_mb_new_blocks()
4555 	if (likely(ac->ac_status == AC_STATUS_FOUND)) {  in ext4_mb_new_blocks()
4556 		*errp = ext4_mb_mark_diskspace_used(ac, handle, reserv_clstrs);  in ext4_mb_new_blocks()
4558 			ext4_discard_allocated_blocks(ac);  in ext4_mb_new_blocks()
4561 			block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex);  in ext4_mb_new_blocks()
4562 			ar->len = ac->ac_b_ex.fe_len;  in ext4_mb_new_blocks()
4565 		freed  = ext4_mb_discard_preallocations(sb, ac->ac_o_ex.fe_len);  in ext4_mb_new_blocks()
4573 		ac->ac_b_ex.fe_len = 0;  in ext4_mb_new_blocks()
4575 		ext4_mb_show_ac(ac);  in ext4_mb_new_blocks()
4577 	ext4_mb_release_context(ac);  in ext4_mb_new_blocks()
4579 	if (ac)  in ext4_mb_new_blocks()
4580 		kmem_cache_free(ext4_ac_cachep, ac);  in ext4_mb_new_blocks()