Lines Matching +full:1 +full:ac
100 * [ group 0 bitmap][ group 0 buddy] [group 1][ group 1]...
133 * 1) Array of largest free order lists (sbi->s_mb_largest_free_orders)
148 * average fragment size >= 2^i and < 2^(i+1). The average fragment size
159 * structure (1) above where largest_free_order = order of the request. If that
161 * largest_free_order. This allows us to perform CR = 0 lookup in O(1) time.
163 * At CR = 1, we only consider groups where average fragment size > request
166 * structure 2) in O(1) time.
169 * linear order which requires O(N) search time for each CR 0 and CR 1 phase.
261 * 1) if buddy is referenced, it's already initialized
409 static void ext4_mb_new_preallocation(struct ext4_allocation_context *ac);
411 static bool ext4_mb_good_group(struct ext4_allocation_context *ac,
420 * 1. We sample the percpu discard_pa_seq counter before trying for block
522 if (order > e4b->bd_blkbits + 1) { in mb_find_buddy()
529 *max = 1 << (e4b->bd_blkbits + 3); in mb_find_buddy()
672 int order = e4b->bd_blkbits + 1; in __mb_check_buddy()
689 while (order > 1) { in __mb_check_buddy()
692 buddy2 = mb_find_buddy(e4b, order - 1, &max2); in __mb_check_buddy()
702 if (!mb_test_bit(i << 1, buddy2)) { in __mb_check_buddy()
704 mb_test_bit((i<<1)+1, buddy2)); in __mb_check_buddy()
709 /* both bits in buddy2 must be 1 */ in __mb_check_buddy()
710 MB_CHECK_ASSERT(mb_test_bit(i << 1, buddy2)); in __mb_check_buddy()
711 MB_CHECK_ASSERT(mb_test_bit((i << 1) + 1, buddy2)); in __mb_check_buddy()
713 for (j = 0; j < (1 << order); j++) { in __mb_check_buddy()
714 k = (i * (1 << order)) + j; in __mb_check_buddy()
724 fstart = -1; in __mb_check_buddy()
729 if (fstart == -1) { in __mb_check_buddy()
735 fstart = -1; in __mb_check_buddy()
737 for (j = 0; j < e4b->bd_blkbits + 1; j++) { in __mb_check_buddy()
788 max = ffs(first | border) - 1; in ext4_mb_mark_free_simple()
791 min = fls(len) - 1; in ext4_mb_mark_free_simple()
795 chunk = 1 << min; in ext4_mb_mark_free_simple()
813 * We don't bother with a special lists groups with only 1 block free in mb_avg_fragment_size_order()
839 if (grp->bb_avg_fragment_size_order != -1) { in mb_update_avg_fragment_size()
859 static void ext4_mb_choose_next_group_cr0(struct ext4_allocation_context *ac, in ext4_mb_choose_next_group_cr0() argument
862 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_choose_next_group_cr0()
866 if (ac->ac_status == AC_STATUS_FOUND) in ext4_mb_choose_next_group_cr0()
869 if (unlikely(sbi->s_mb_stats && ac->ac_flags & EXT4_MB_CR0_OPTIMIZED)) in ext4_mb_choose_next_group_cr0()
873 for (i = ac->ac_2order; i < MB_NUM_ORDERS(ac->ac_sb); i++) { in ext4_mb_choose_next_group_cr0()
886 if (likely(ext4_mb_good_group(ac, iter->bb_group, 0))) { in ext4_mb_choose_next_group_cr0()
898 *new_cr = 1; in ext4_mb_choose_next_group_cr0()
901 ac->ac_flags |= EXT4_MB_CR0_OPTIMIZED; in ext4_mb_choose_next_group_cr0()
909 static void ext4_mb_choose_next_group_cr1(struct ext4_allocation_context *ac, in ext4_mb_choose_next_group_cr1() argument
912 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_choose_next_group_cr1()
916 if (unlikely(ac->ac_flags & EXT4_MB_CR1_OPTIMIZED)) { in ext4_mb_choose_next_group_cr1()
921 for (i = mb_avg_fragment_size_order(ac->ac_sb, ac->ac_g_ex.fe_len); in ext4_mb_choose_next_group_cr1()
922 i < MB_NUM_ORDERS(ac->ac_sb); i++) { in ext4_mb_choose_next_group_cr1()
933 atomic64_inc(&sbi->s_bal_cX_groups_considered[1]); in ext4_mb_choose_next_group_cr1()
934 if (likely(ext4_mb_good_group(ac, iter->bb_group, 1))) { in ext4_mb_choose_next_group_cr1()
946 ac->ac_flags |= EXT4_MB_CR1_OPTIMIZED; in ext4_mb_choose_next_group_cr1()
952 static inline int should_optimize_scan(struct ext4_allocation_context *ac) in should_optimize_scan() argument
954 if (unlikely(!test_opt2(ac->ac_sb, MB_OPTIMIZE_SCAN))) in should_optimize_scan()
956 if (ac->ac_criteria >= 2) in should_optimize_scan()
958 if (!ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)) in should_optimize_scan()
960 return 1; in should_optimize_scan()
968 next_linear_group(struct ext4_allocation_context *ac, int group, int ngroups) in next_linear_group() argument
970 if (!should_optimize_scan(ac)) in next_linear_group()
973 if (ac->ac_groups_linear_remaining) { in next_linear_group()
974 ac->ac_groups_linear_remaining--; in next_linear_group()
984 return group + 1 >= ngroups ? 0 : group + 1; in next_linear_group()
990 * @ac Allocation Context
1000 static void ext4_mb_choose_next_group(struct ext4_allocation_context *ac, in ext4_mb_choose_next_group() argument
1003 *new_cr = ac->ac_criteria; in ext4_mb_choose_next_group()
1005 if (!should_optimize_scan(ac) || ac->ac_groups_linear_remaining) { in ext4_mb_choose_next_group()
1006 *group = next_linear_group(ac, *group, ngroups); in ext4_mb_choose_next_group()
1011 ext4_mb_choose_next_group_cr0(ac, new_cr, group, ngroups); in ext4_mb_choose_next_group()
1012 } else if (*new_cr == 1) { in ext4_mb_choose_next_group()
1013 ext4_mb_choose_next_group_cr1(ac, new_cr, group, ngroups); in ext4_mb_choose_next_group()
1019 WARN_ON(1); in ext4_mb_choose_next_group()
1033 for (i = MB_NUM_ORDERS(sb) - 1; i >= 0; i--) in mb_set_largest_free_order()
1085 if (len > 1) in ext4_mb_generate_buddy()
1124 * [ group 0 bitmap][ group 0 buddy] [group 1][ group 1]...
1163 groups_per_page = blocks_per_page >> 1; in ext4_mb_init_cache()
1165 groups_per_page = 1; in ext4_mb_init_cache()
1168 if (groups_per_page > 1) { in ext4_mb_init_cache()
1218 group = (first_block + i) >> 1; in ext4_mb_init_cache()
1244 if ((first_block + i) & 1) { in ext4_mb_init_cache()
1584 int order = 1, max; in mb_find_order_for_block()
1588 BUG_ON(block >= (1 << (e4b->bd_blkbits + 3))); in mb_find_order_for_block()
1590 while (order <= e4b->bd_blkbits + 1) { in mb_find_order_for_block()
1620 * will return first found zero bit if any, -1 otherwise
1625 int zero_bit = -1; in mb_test_and_clear_bits()
1632 if (*addr != (__u32)(-1) && zero_bit == -1) in mb_test_and_clear_bits()
1638 if (!mb_test_and_clear_bit(cur, bm) && zero_bit == -1) in mb_test_and_clear_bits()
1669 return 1; in mb_buddy_adjust_border()
1674 return -1; in mb_buddy_adjust_border()
1681 int order = 1; in mb_buddy_mark_free()
1698 * | 1 | 1 | 1 | 1 | in mb_buddy_mark_free()
1700 * | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | in mb_buddy_mark_free()
1702 * 0 1 2 3 4 5 6 7 in mb_buddy_mark_free()
1705 * Neither [1] nor [6] is aligned to above layer. in mb_buddy_mark_free()
1716 if (first & 1) in mb_buddy_mark_free()
1717 e4b->bd_info->bb_counters[order] += mb_buddy_adjust_border(&first, buddy, -1); in mb_buddy_mark_free()
1718 if (!(last & 1)) in mb_buddy_mark_free()
1719 e4b->bd_info->bb_counters[order] += mb_buddy_adjust_border(&last, buddy, 1); in mb_buddy_mark_free()
1725 mb_clear_bits(buddy, first, last - first + 1); in mb_buddy_mark_free()
1726 e4b->bd_info->bb_counters[order - 1] += last - first + 1; in mb_buddy_mark_free()
1729 first >>= 1; in mb_buddy_mark_free()
1730 last >>= 1; in mb_buddy_mark_free()
1741 int last = first + count - 1; in mb_free_blocks()
1764 left_is_free = !mb_test_bit(first - 1, e4b->bd_bitmap); in mb_free_blocks()
1766 if (last + 1 < EXT4_SB(sb)->s_mb_maxs[0]) in mb_free_blocks()
1767 right_is_free = !mb_test_bit(last + 1, e4b->bd_bitmap); in mb_free_blocks()
1769 if (unlikely(block != -1)) { in mb_free_blocks()
1800 if (first & 1) { in mb_free_blocks()
1802 e4b->bd_info->bb_counters[0] += left_is_free ? -1 : 1; in mb_free_blocks()
1804 if (!(last & 1)) { in mb_free_blocks()
1806 e4b->bd_info->bb_counters[0] += right_is_free ? -1 : 1; in mb_free_blocks()
1810 mb_buddy_mark_free(e4b, first >> 1, last >> 1); in mb_free_blocks()
1842 ex->fe_len = 1 << order; in mb_find_extent()
1854 if (block + 1 >= max) in mb_find_extent()
1857 next = (block + 1) * (1 << order); in mb_find_extent()
1864 ex->fe_len += 1 << order; in mb_find_extent()
1869 WARN_ON(1); in mb_find_extent()
1908 mlen = !mb_test_bit(start - 1, e4b->bd_bitmap); in mb_mark_used()
1921 if (((start >> ord) << ord) == start && len >= (1 << ord)) { in mb_mark_used()
1923 mlen = 1 << ord; in mb_mark_used()
1948 cur = (start >> ord) & ~1U; in mb_mark_used()
1951 mb_clear_bit(cur + 1, buddy); in mb_mark_used()
1968 static void ext4_mb_use_best_found(struct ext4_allocation_context *ac, in ext4_mb_use_best_found() argument
1971 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_use_best_found()
1974 BUG_ON(ac->ac_b_ex.fe_group != e4b->bd_group); in ext4_mb_use_best_found()
1975 BUG_ON(ac->ac_status == AC_STATUS_FOUND); in ext4_mb_use_best_found()
1977 ac->ac_b_ex.fe_len = min(ac->ac_b_ex.fe_len, ac->ac_g_ex.fe_len); in ext4_mb_use_best_found()
1978 ac->ac_b_ex.fe_logical = ac->ac_g_ex.fe_logical; in ext4_mb_use_best_found()
1979 ret = mb_mark_used(e4b, &ac->ac_b_ex); in ext4_mb_use_best_found()
1983 ac->ac_f_ex = ac->ac_b_ex; in ext4_mb_use_best_found()
1985 ac->ac_status = AC_STATUS_FOUND; in ext4_mb_use_best_found()
1986 ac->ac_tail = ret & 0xffff; in ext4_mb_use_best_found()
1987 ac->ac_buddy = ret >> 16; in ext4_mb_use_best_found()
1996 ac->ac_bitmap_page = e4b->bd_bitmap_page; in ext4_mb_use_best_found()
1997 get_page(ac->ac_bitmap_page); in ext4_mb_use_best_found()
1998 ac->ac_buddy_page = e4b->bd_buddy_page; in ext4_mb_use_best_found()
1999 get_page(ac->ac_buddy_page); in ext4_mb_use_best_found()
2001 if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) { in ext4_mb_use_best_found()
2003 sbi->s_mb_last_group = ac->ac_f_ex.fe_group; in ext4_mb_use_best_found()
2004 sbi->s_mb_last_start = ac->ac_f_ex.fe_start; in ext4_mb_use_best_found()
2012 if (ac->ac_o_ex.fe_len < ac->ac_b_ex.fe_len) in ext4_mb_use_best_found()
2013 ext4_mb_new_preallocation(ac); in ext4_mb_use_best_found()
2017 static void ext4_mb_check_limits(struct ext4_allocation_context *ac, in ext4_mb_check_limits() argument
2021 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_check_limits()
2022 struct ext4_free_extent *bex = &ac->ac_b_ex; in ext4_mb_check_limits()
2023 struct ext4_free_extent *gex = &ac->ac_g_ex; in ext4_mb_check_limits()
2027 if (ac->ac_status == AC_STATUS_FOUND) in ext4_mb_check_limits()
2032 if (ac->ac_found > sbi->s_mb_max_to_scan && in ext4_mb_check_limits()
2033 !(ac->ac_flags & EXT4_MB_HINT_FIRST)) { in ext4_mb_check_limits()
2034 ac->ac_status = AC_STATUS_BREAK; in ext4_mb_check_limits()
2044 if ((finish_group || ac->ac_found > sbi->s_mb_min_to_scan) in ext4_mb_check_limits()
2051 ext4_mb_use_best_found(ac, e4b); in ext4_mb_check_limits()
2067 static void ext4_mb_measure_extent(struct ext4_allocation_context *ac, in ext4_mb_measure_extent() argument
2071 struct ext4_free_extent *bex = &ac->ac_b_ex; in ext4_mb_measure_extent()
2072 struct ext4_free_extent *gex = &ac->ac_g_ex; in ext4_mb_measure_extent()
2075 BUG_ON(ex->fe_len > EXT4_CLUSTERS_PER_GROUP(ac->ac_sb)); in ext4_mb_measure_extent()
2076 BUG_ON(ex->fe_start >= EXT4_CLUSTERS_PER_GROUP(ac->ac_sb)); in ext4_mb_measure_extent()
2077 BUG_ON(ac->ac_status != AC_STATUS_CONTINUE); in ext4_mb_measure_extent()
2079 ac->ac_found++; in ext4_mb_measure_extent()
2084 if (unlikely(ac->ac_flags & EXT4_MB_HINT_FIRST)) { in ext4_mb_measure_extent()
2086 ext4_mb_use_best_found(ac, e4b); in ext4_mb_measure_extent()
2095 ext4_mb_use_best_found(ac, e4b); in ext4_mb_measure_extent()
2123 ext4_mb_check_limits(ac, e4b, 0); in ext4_mb_measure_extent()
2127 int ext4_mb_try_best_found(struct ext4_allocation_context *ac, in ext4_mb_try_best_found() argument
2130 struct ext4_free_extent ex = ac->ac_b_ex; in ext4_mb_try_best_found()
2136 err = ext4_mb_load_buddy(ac->ac_sb, group, e4b); in ext4_mb_try_best_found()
2140 ext4_lock_group(ac->ac_sb, group); in ext4_mb_try_best_found()
2144 ac->ac_b_ex = ex; in ext4_mb_try_best_found()
2145 ext4_mb_use_best_found(ac, e4b); in ext4_mb_try_best_found()
2148 ext4_unlock_group(ac->ac_sb, group); in ext4_mb_try_best_found()
2155 int ext4_mb_find_by_goal(struct ext4_allocation_context *ac, in ext4_mb_find_by_goal() argument
2158 ext4_group_t group = ac->ac_g_ex.fe_group; in ext4_mb_find_by_goal()
2161 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_find_by_goal()
2162 struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group); in ext4_mb_find_by_goal()
2165 if (!(ac->ac_flags & EXT4_MB_HINT_TRY_GOAL)) in ext4_mb_find_by_goal()
2170 err = ext4_mb_load_buddy(ac->ac_sb, group, e4b); in ext4_mb_find_by_goal()
2179 ext4_lock_group(ac->ac_sb, group); in ext4_mb_find_by_goal()
2180 max = mb_find_extent(e4b, ac->ac_g_ex.fe_start, in ext4_mb_find_by_goal()
2181 ac->ac_g_ex.fe_len, &ex); in ext4_mb_find_by_goal()
2184 if (max >= ac->ac_g_ex.fe_len && ac->ac_g_ex.fe_len == sbi->s_stripe) { in ext4_mb_find_by_goal()
2187 start = ext4_group_first_block_no(ac->ac_sb, e4b->bd_group) + in ext4_mb_find_by_goal()
2191 ac->ac_found++; in ext4_mb_find_by_goal()
2192 ac->ac_b_ex = ex; in ext4_mb_find_by_goal()
2193 ext4_mb_use_best_found(ac, e4b); in ext4_mb_find_by_goal()
2195 } else if (max >= ac->ac_g_ex.fe_len) { in ext4_mb_find_by_goal()
2197 BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group); in ext4_mb_find_by_goal()
2198 BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start); in ext4_mb_find_by_goal()
2199 ac->ac_found++; in ext4_mb_find_by_goal()
2200 ac->ac_b_ex = ex; in ext4_mb_find_by_goal()
2201 ext4_mb_use_best_found(ac, e4b); in ext4_mb_find_by_goal()
2202 } else if (max > 0 && (ac->ac_flags & EXT4_MB_HINT_MERGE)) { in ext4_mb_find_by_goal()
2206 BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group); in ext4_mb_find_by_goal()
2207 BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start); in ext4_mb_find_by_goal()
2208 ac->ac_found++; in ext4_mb_find_by_goal()
2209 ac->ac_b_ex = ex; in ext4_mb_find_by_goal()
2210 ext4_mb_use_best_found(ac, e4b); in ext4_mb_find_by_goal()
2212 ext4_unlock_group(ac->ac_sb, group); in ext4_mb_find_by_goal()
2223 void ext4_mb_simple_scan_group(struct ext4_allocation_context *ac, in ext4_mb_simple_scan_group() argument
2226 struct super_block *sb = ac->ac_sb; in ext4_mb_simple_scan_group()
2233 BUG_ON(ac->ac_2order <= 0); in ext4_mb_simple_scan_group()
2234 for (i = ac->ac_2order; i < MB_NUM_ORDERS(sb); i++) { in ext4_mb_simple_scan_group()
2243 ext4_grp_locked_error(ac->ac_sb, e4b->bd_group, 0, 0, in ext4_mb_simple_scan_group()
2246 ext4_mark_group_bitmap_corrupted(ac->ac_sb, in ext4_mb_simple_scan_group()
2251 ac->ac_found++; in ext4_mb_simple_scan_group()
2253 ac->ac_b_ex.fe_len = 1 << i; in ext4_mb_simple_scan_group()
2254 ac->ac_b_ex.fe_start = k << i; in ext4_mb_simple_scan_group()
2255 ac->ac_b_ex.fe_group = e4b->bd_group; in ext4_mb_simple_scan_group()
2257 ext4_mb_use_best_found(ac, e4b); in ext4_mb_simple_scan_group()
2259 BUG_ON(ac->ac_f_ex.fe_len != ac->ac_g_ex.fe_len); in ext4_mb_simple_scan_group()
2274 void ext4_mb_complex_scan_group(struct ext4_allocation_context *ac, in ext4_mb_complex_scan_group() argument
2277 struct super_block *sb = ac->ac_sb; in ext4_mb_complex_scan_group()
2289 while (free && ac->ac_status == AC_STATUS_CONTINUE) { in ext4_mb_complex_scan_group()
2307 mb_find_extent(e4b, i, ac->ac_g_ex.fe_len, &ex); in ext4_mb_complex_scan_group()
2325 ext4_mb_measure_extent(ac, &ex, e4b); in ext4_mb_complex_scan_group()
2331 ext4_mb_check_limits(ac, e4b, 1); in ext4_mb_complex_scan_group()
2339 void ext4_mb_scan_aligned(struct ext4_allocation_context *ac, in ext4_mb_scan_aligned() argument
2342 struct super_block *sb = ac->ac_sb; in ext4_mb_scan_aligned()
2356 a = first_group_block + sbi->s_stripe - 1; in ext4_mb_scan_aligned()
2364 ac->ac_found++; in ext4_mb_scan_aligned()
2366 ac->ac_b_ex = ex; in ext4_mb_scan_aligned()
2367 ext4_mb_use_best_found(ac, e4b); in ext4_mb_scan_aligned()
2377 * Returns either 1 or 0 indicating that the group is either suitable
2380 static bool ext4_mb_good_group(struct ext4_allocation_context *ac, in ext4_mb_good_group() argument
2384 int flex_size = ext4_flex_bg_size(EXT4_SB(ac->ac_sb)); in ext4_mb_good_group()
2385 struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group); in ext4_mb_good_group()
2402 BUG_ON(ac->ac_2order == 0); in ext4_mb_good_group()
2405 if ((ac->ac_flags & EXT4_MB_HINT_DATA) && in ext4_mb_good_group()
2410 if (free < ac->ac_g_ex.fe_len) in ext4_mb_good_group()
2413 if (ac->ac_2order >= MB_NUM_ORDERS(ac->ac_sb)) in ext4_mb_good_group()
2416 if (grp->bb_largest_free_order < ac->ac_2order) in ext4_mb_good_group()
2420 case 1: in ext4_mb_good_group()
2421 if ((free / fragments) >= ac->ac_g_ex.fe_len) in ext4_mb_good_group()
2425 if (free >= ac->ac_g_ex.fe_len) in ext4_mb_good_group()
2448 static int ext4_mb_good_group_nolock(struct ext4_allocation_context *ac, in ext4_mb_good_group_nolock() argument
2451 struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group); in ext4_mb_good_group_nolock()
2452 struct super_block *sb = ac->ac_sb; in ext4_mb_good_group_nolock()
2454 bool should_lock = ac->ac_flags & EXT4_MB_STRICT_CHECK; in ext4_mb_good_group_nolock()
2459 atomic64_inc(&sbi->s_bal_cX_groups_considered[ac->ac_criteria]); in ext4_mb_good_group_nolock()
2467 if (cr <= 2 && free < ac->ac_g_ex.fe_len) in ext4_mb_good_group_nolock()
2482 /* cr=0/1 is a very optimistic search to find large in ext4_mb_good_group_nolock()
2492 ((group & ((1 << sbi->s_log_groups_per_flex) - 1)) != 0)) && in ext4_mb_good_group_nolock()
2505 ret = ext4_mb_good_group(ac, group, cr); in ext4_mb_good_group_nolock()
2593 ext4_mb_regular_allocator(struct ext4_allocation_context *ac) in ext4_mb_regular_allocator() argument
2596 int cr = -1, new_cr; in ext4_mb_regular_allocator()
2604 sb = ac->ac_sb; in ext4_mb_regular_allocator()
2608 if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS))) in ext4_mb_regular_allocator()
2611 BUG_ON(ac->ac_status == AC_STATUS_FOUND); in ext4_mb_regular_allocator()
2614 err = ext4_mb_find_by_goal(ac, &e4b); in ext4_mb_regular_allocator()
2615 if (err || ac->ac_status == AC_STATUS_FOUND) in ext4_mb_regular_allocator()
2618 if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) in ext4_mb_regular_allocator()
2622 * ac->ac_2order is set only if the fe_len is a power of 2 in ext4_mb_regular_allocator()
2623 * if ac->ac_2order is set we also set criteria to 0 so that we in ext4_mb_regular_allocator()
2626 i = fls(ac->ac_g_ex.fe_len); in ext4_mb_regular_allocator()
2627 ac->ac_2order = 0; in ext4_mb_regular_allocator()
2639 if ((ac->ac_g_ex.fe_len & (~(1 << (i - 1)))) == 0) in ext4_mb_regular_allocator()
2640 ac->ac_2order = array_index_nospec(i - 1, in ext4_mb_regular_allocator()
2645 if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) { in ext4_mb_regular_allocator()
2648 ac->ac_g_ex.fe_group = sbi->s_mb_last_group; in ext4_mb_regular_allocator()
2649 ac->ac_g_ex.fe_start = sbi->s_mb_last_start; in ext4_mb_regular_allocator()
2654 cr = ac->ac_2order ? 0 : 1; in ext4_mb_regular_allocator()
2660 for (; cr < 4 && ac->ac_status == AC_STATUS_CONTINUE; cr++) { in ext4_mb_regular_allocator()
2661 ac->ac_criteria = cr; in ext4_mb_regular_allocator()
2666 group = ac->ac_g_ex.fe_group; in ext4_mb_regular_allocator()
2667 ac->ac_groups_linear_remaining = sbi->s_mb_max_linear_groups; in ext4_mb_regular_allocator()
2671 ext4_mb_choose_next_group(ac, &new_cr, &group, ngroups)) { in ext4_mb_regular_allocator()
2683 * prefetching at cr=0/1, otherwise mballoc can in ext4_mb_regular_allocator()
2687 (cr > 1 || in ext4_mb_regular_allocator()
2693 nr = 1 << sbi->s_log_groups_per_flex; in ext4_mb_regular_allocator()
2694 nr -= group & (nr - 1); in ext4_mb_regular_allocator()
2704 ret = ext4_mb_good_group_nolock(ac, group, cr); in ext4_mb_regular_allocator()
2721 ret = ext4_mb_good_group(ac, group, cr); in ext4_mb_regular_allocator()
2728 ac->ac_groups_scanned++; in ext4_mb_regular_allocator()
2730 ext4_mb_simple_scan_group(ac, &e4b); in ext4_mb_regular_allocator()
2731 else if (cr == 1 && sbi->s_stripe && in ext4_mb_regular_allocator()
2732 !(ac->ac_g_ex.fe_len % sbi->s_stripe)) in ext4_mb_regular_allocator()
2733 ext4_mb_scan_aligned(ac, &e4b); in ext4_mb_regular_allocator()
2735 ext4_mb_complex_scan_group(ac, &e4b); in ext4_mb_regular_allocator()
2740 if (ac->ac_status != AC_STATUS_CONTINUE) in ext4_mb_regular_allocator()
2748 if (ac->ac_b_ex.fe_len > 0 && ac->ac_status != AC_STATUS_FOUND && in ext4_mb_regular_allocator()
2749 !(ac->ac_flags & EXT4_MB_HINT_FIRST)) { in ext4_mb_regular_allocator()
2754 ext4_mb_try_best_found(ac, &e4b); in ext4_mb_regular_allocator()
2755 if (ac->ac_status != AC_STATUS_FOUND) { in ext4_mb_regular_allocator()
2763 ac->ac_b_ex.fe_group, ac->ac_b_ex.fe_start, in ext4_mb_regular_allocator()
2764 ac->ac_b_ex.fe_len, lost); in ext4_mb_regular_allocator()
2766 ac->ac_b_ex.fe_group = 0; in ext4_mb_regular_allocator()
2767 ac->ac_b_ex.fe_start = 0; in ext4_mb_regular_allocator()
2768 ac->ac_b_ex.fe_len = 0; in ext4_mb_regular_allocator()
2769 ac->ac_status = AC_STATUS_CONTINUE; in ext4_mb_regular_allocator()
2770 ac->ac_flags |= EXT4_MB_HINT_FIRST; in ext4_mb_regular_allocator()
2776 if (sbi->s_mb_stats && ac->ac_status == AC_STATUS_FOUND) in ext4_mb_regular_allocator()
2777 atomic64_inc(&sbi->s_bal_cX_hits[ac->ac_criteria]); in ext4_mb_regular_allocator()
2779 if (!err && ac->ac_status != AC_STATUS_FOUND && first_err) in ext4_mb_regular_allocator()
2783 ac->ac_b_ex.fe_len, ac->ac_o_ex.fe_len, ac->ac_status, in ext4_mb_regular_allocator()
2784 ac->ac_flags, cr, err); in ext4_mb_regular_allocator()
2799 group = *pos + 1; in ext4_mb_seq_groups_start()
2811 group = *pos + 1; in ext4_mb_seq_groups_next()
2834 " 2^0 2^1 2^2 2^3 2^4 2^5 2^6 " in ext4_mb_seq_groups_show()
2848 buddy_loaded = 1; in ext4_mb_seq_groups_show()
2859 seq_printf(seq, " %-5u", i <= blocksize_bits + 1 ? in ext4_mb_seq_groups_show()
2885 seq_puts(seq, "\tTo enable, please write \"1\" to sysfs file mb_stats.\n"); in ext4_seq_mb_stats_show()
2903 seq_printf(seq, "\t\thits: %llu\n", atomic64_read(&sbi->s_bal_cX_hits[1])); in ext4_seq_mb_stats_show()
2905 atomic64_read(&sbi->s_bal_cX_groups_considered[1])); in ext4_seq_mb_stats_show()
2907 atomic64_read(&sbi->s_bal_cX_failed[1])); in ext4_seq_mb_stats_show()
2950 position = *pos + 1; in ext4_mb_seq_structs_summary_start()
2962 position = *pos + 1; in ext4_mb_seq_structs_summary_next()
2993 test_opt2(sb, MB_OPTIMIZE_SCAN) ? 1 : 0); in ext4_mb_seq_structs_summary_show()
3038 size = (ngroups + EXT4_DESC_PER_BLOCK(sb) - 1) >> in ext4_mb_alloc_groupinfo()
3095 i = group & (EXT4_DESC_PER_BLOCK(sb) - 1); in ext4_mb_add_groupinfo()
3123 meta_group_info[i]->bb_largest_free_order = -1; /* uninit */ in ext4_mb_add_groupinfo()
3124 meta_group_info[i]->bb_avg_fragment_size_order = -1; /* uninit */ in ext4_mb_add_groupinfo()
3190 sbi->s_mb_prefetch = min_t(uint, 1 << sbi->s_es->s_log_groups_per_flex, in ext4_mb_init_backend()
3319 fd->efd_start_cluster + fd->efd_count - 1, 1); in ext4_discard_work()
3360 i = 1; in ext4_mb_init()
3362 offset_incr = 1 << (sb->s_blocksize_bits - 1); in ext4_mb_init()
3368 offset_incr = offset_incr >> 1; in ext4_mb_init()
3369 max = max >> 1; in ext4_mb_init()
3427 * is 1 megabyte, then group preallocation size becomes half a in ext4_mb_init()
3432 * size is 256k, and 32 megs when the cluster size is 1 meg, in ext4_mb_init()
3438 * If there is a s_stripe > 1, then we set the s_mb_group_prealloc in ext4_mb_init()
3445 if (sbi->s_stripe > 1) { in ext4_mb_init()
3538 EXT4_DESC_PER_BLOCK(sb) - 1) >> in ext4_mb_release()
3737 * Check quota and mark chosen space (ac->ac_b_ex) non-free in bitmaps
3741 ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac, in ext4_mb_mark_diskspace_used() argument
3752 BUG_ON(ac->ac_status != AC_STATUS_FOUND); in ext4_mb_mark_diskspace_used()
3753 BUG_ON(ac->ac_b_ex.fe_len <= 0); in ext4_mb_mark_diskspace_used()
3755 sb = ac->ac_sb; in ext4_mb_mark_diskspace_used()
3758 bitmap_bh = ext4_read_block_bitmap(sb, ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
3772 gdp = ext4_get_group_desc(sb, ac->ac_b_ex.fe_group, &gdp_bh); in ext4_mb_mark_diskspace_used()
3776 ext4_debug("using block group %u(%d)\n", ac->ac_b_ex.fe_group, in ext4_mb_mark_diskspace_used()
3784 block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); in ext4_mb_mark_diskspace_used()
3786 len = EXT4_C2B(sbi, ac->ac_b_ex.fe_len); in ext4_mb_mark_diskspace_used()
3787 if (!ext4_inode_block_valid(ac->ac_inode, block, len)) { in ext4_mb_mark_diskspace_used()
3794 ext4_lock_group(sb, ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
3795 mb_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start, in ext4_mb_mark_diskspace_used()
3796 ac->ac_b_ex.fe_len); in ext4_mb_mark_diskspace_used()
3797 ext4_unlock_group(sb, ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
3804 ext4_lock_group(sb, ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
3808 for (i = 0; i < ac->ac_b_ex.fe_len; i++) { in ext4_mb_mark_diskspace_used()
3809 BUG_ON(mb_test_bit(ac->ac_b_ex.fe_start + i, in ext4_mb_mark_diskspace_used()
3814 mb_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start, in ext4_mb_mark_diskspace_used()
3815 ac->ac_b_ex.fe_len); in ext4_mb_mark_diskspace_used()
3821 ac->ac_b_ex.fe_group, gdp)); in ext4_mb_mark_diskspace_used()
3823 len = ext4_free_group_clusters(sb, gdp) - ac->ac_b_ex.fe_len; in ext4_mb_mark_diskspace_used()
3825 ext4_block_bitmap_csum_set(sb, ac->ac_b_ex.fe_group, gdp, bitmap_bh); in ext4_mb_mark_diskspace_used()
3826 ext4_group_desc_csum_set(sb, ac->ac_b_ex.fe_group, gdp); in ext4_mb_mark_diskspace_used()
3828 ext4_unlock_group(sb, ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
3829 percpu_counter_sub(&sbi->s_freeclusters_counter, ac->ac_b_ex.fe_len); in ext4_mb_mark_diskspace_used()
3833 if (!(ac->ac_flags & EXT4_MB_DELALLOC_RESERVED)) in ext4_mb_mark_diskspace_used()
3840 ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
3841 atomic64_sub(ac->ac_b_ex.fe_len, in ext4_mb_mark_diskspace_used()
3977 static void ext4_mb_normalize_group_request(struct ext4_allocation_context *ac) in ext4_mb_normalize_group_request() argument
3979 struct super_block *sb = ac->ac_sb; in ext4_mb_normalize_group_request()
3980 struct ext4_locality_group *lg = ac->ac_lg; in ext4_mb_normalize_group_request()
3983 ac->ac_g_ex.fe_len = EXT4_SB(sb)->s_mb_group_prealloc; in ext4_mb_normalize_group_request()
3984 mb_debug(sb, "goal %u blocks for locality group\n", ac->ac_g_ex.fe_len); in ext4_mb_normalize_group_request()
3992 ext4_mb_normalize_request(struct ext4_allocation_context *ac, in ext4_mb_normalize_request() argument
3995 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_normalize_request()
4001 struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); in ext4_mb_normalize_request()
4006 if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) in ext4_mb_normalize_request()
4010 if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) in ext4_mb_normalize_request()
4015 if (ac->ac_flags & EXT4_MB_HINT_NOPREALLOC) in ext4_mb_normalize_request()
4018 if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) { in ext4_mb_normalize_request()
4019 ext4_mb_normalize_group_request(ac); in ext4_mb_normalize_request()
4023 bsbits = ac->ac_sb->s_blocksize_bits; in ext4_mb_normalize_request()
4027 size = ac->ac_o_ex.fe_logical + EXT4_C2B(sbi, ac->ac_o_ex.fe_len); in ext4_mb_normalize_request()
4029 if (size < i_size_read(ac->ac_inode)) in ext4_mb_normalize_request()
4030 size = i_size_read(ac->ac_inode); in ext4_mb_normalize_request()
4057 start_off = ((loff_t)ac->ac_o_ex.fe_logical >> in ext4_mb_normalize_request()
4061 start_off = ((loff_t)ac->ac_o_ex.fe_logical >> in ext4_mb_normalize_request()
4064 } else if (NRL_CHECK_SIZE(ac->ac_o_ex.fe_len, in ext4_mb_normalize_request()
4066 start_off = ((loff_t)ac->ac_o_ex.fe_logical >> in ext4_mb_normalize_request()
4070 start_off = (loff_t) ac->ac_o_ex.fe_logical << bsbits; in ext4_mb_normalize_request()
4071 size = (loff_t) EXT4_C2B(EXT4_SB(ac->ac_sb), in ext4_mb_normalize_request()
4072 ac->ac_o_ex.fe_len) << bsbits; in ext4_mb_normalize_request()
4083 start = max(start, rounddown(ac->ac_o_ex.fe_logical, in ext4_mb_normalize_request()
4084 (ext4_lblk_t)EXT4_BLOCKS_PER_GROUP(ac->ac_sb))); in ext4_mb_normalize_request()
4088 size -= ar->lleft + 1 - start; in ext4_mb_normalize_request()
4089 start = ar->lleft + 1; in ext4_mb_normalize_request()
4091 if (ar->pright && start + size - 1 >= ar->lright) in ext4_mb_normalize_request()
4098 if (size > EXT4_BLOCKS_PER_GROUP(ac->ac_sb)) in ext4_mb_normalize_request()
4099 size = EXT4_BLOCKS_PER_GROUP(ac->ac_sb); in ext4_mb_normalize_request()
4116 pa_end = pa->pa_lstart + EXT4_C2B(EXT4_SB(ac->ac_sb), in ext4_mb_normalize_request()
4120 BUG_ON(!(ac->ac_o_ex.fe_logical >= pa_end || in ext4_mb_normalize_request()
4121 ac->ac_o_ex.fe_logical < pa->pa_lstart)); in ext4_mb_normalize_request()
4131 if (pa_end <= ac->ac_o_ex.fe_logical) { in ext4_mb_normalize_request()
4134 } else if (pa->pa_lstart > ac->ac_o_ex.fe_logical) { in ext4_mb_normalize_request()
4150 pa_end = pa->pa_lstart + EXT4_C2B(EXT4_SB(ac->ac_sb), in ext4_mb_normalize_request()
4162 * ac->ac_o_ex.fe_logical & fe_len should always lie within "start" and in ext4_mb_normalize_request()
4173 if (start + size <= ac->ac_o_ex.fe_logical || in ext4_mb_normalize_request()
4174 start > ac->ac_o_ex.fe_logical) { in ext4_mb_normalize_request()
4175 ext4_msg(ac->ac_sb, KERN_ERR, in ext4_mb_normalize_request()
4178 (unsigned long) ac->ac_o_ex.fe_logical); in ext4_mb_normalize_request()
4181 BUG_ON(size <= 0 || size > EXT4_BLOCKS_PER_GROUP(ac->ac_sb)); in ext4_mb_normalize_request()
4187 ac->ac_g_ex.fe_logical = start; in ext4_mb_normalize_request()
4188 ac->ac_g_ex.fe_len = EXT4_NUM_B2C(sbi, size); in ext4_mb_normalize_request()
4193 ext4_get_group_no_and_offset(ac->ac_sb, ar->pright - size, in ext4_mb_normalize_request()
4194 &ac->ac_f_ex.fe_group, in ext4_mb_normalize_request()
4195 &ac->ac_f_ex.fe_start); in ext4_mb_normalize_request()
4196 ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL; in ext4_mb_normalize_request()
4198 if (ar->pleft && (ar->lleft + 1 == start)) { in ext4_mb_normalize_request()
4200 ext4_get_group_no_and_offset(ac->ac_sb, ar->pleft + 1, in ext4_mb_normalize_request()
4201 &ac->ac_f_ex.fe_group, in ext4_mb_normalize_request()
4202 &ac->ac_f_ex.fe_start); in ext4_mb_normalize_request()
4203 ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL; in ext4_mb_normalize_request()
4206 mb_debug(ac->ac_sb, "goal: %lld(was %lld) blocks at %u\n", size, in ext4_mb_normalize_request()
4210 static void ext4_mb_collect_stats(struct ext4_allocation_context *ac) in ext4_mb_collect_stats() argument
4212 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_collect_stats()
4214 if (sbi->s_mb_stats && ac->ac_g_ex.fe_len >= 1) { in ext4_mb_collect_stats()
4216 atomic_add(ac->ac_b_ex.fe_len, &sbi->s_bal_allocated); in ext4_mb_collect_stats()
4217 if (ac->ac_b_ex.fe_len >= ac->ac_o_ex.fe_len) in ext4_mb_collect_stats()
4219 atomic_add(ac->ac_found, &sbi->s_bal_ex_scanned); in ext4_mb_collect_stats()
4220 atomic_add(ac->ac_groups_scanned, &sbi->s_bal_groups_scanned); in ext4_mb_collect_stats()
4221 if (ac->ac_g_ex.fe_start == ac->ac_b_ex.fe_start && in ext4_mb_collect_stats()
4222 ac->ac_g_ex.fe_group == ac->ac_b_ex.fe_group) in ext4_mb_collect_stats()
4224 if (ac->ac_found > sbi->s_mb_max_to_scan) in ext4_mb_collect_stats()
4228 if (ac->ac_op == EXT4_MB_HISTORY_ALLOC) in ext4_mb_collect_stats()
4229 trace_ext4_mballoc_alloc(ac); in ext4_mb_collect_stats()
4231 trace_ext4_mballoc_prealloc(ac); in ext4_mb_collect_stats()
4238 * zeroed out ac->ac_b_ex.fe_len, so group_pa->pa_free is not changed.
4240 static void ext4_discard_allocated_blocks(struct ext4_allocation_context *ac) in ext4_discard_allocated_blocks() argument
4242 struct ext4_prealloc_space *pa = ac->ac_pa; in ext4_discard_allocated_blocks()
4247 if (ac->ac_f_ex.fe_len == 0) in ext4_discard_allocated_blocks()
4249 err = ext4_mb_load_buddy(ac->ac_sb, ac->ac_f_ex.fe_group, &e4b); in ext4_discard_allocated_blocks()
4256 WARN(1, "mb_load_buddy failed (%d)", err); in ext4_discard_allocated_blocks()
4259 ext4_lock_group(ac->ac_sb, ac->ac_f_ex.fe_group); in ext4_discard_allocated_blocks()
4260 mb_free_blocks(ac->ac_inode, &e4b, ac->ac_f_ex.fe_start, in ext4_discard_allocated_blocks()
4261 ac->ac_f_ex.fe_len); in ext4_discard_allocated_blocks()
4262 ext4_unlock_group(ac->ac_sb, ac->ac_f_ex.fe_group); in ext4_discard_allocated_blocks()
4267 pa->pa_free += ac->ac_b_ex.fe_len; in ext4_discard_allocated_blocks()
4273 static void ext4_mb_use_inode_pa(struct ext4_allocation_context *ac, in ext4_mb_use_inode_pa() argument
4276 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_use_inode_pa()
4282 start = pa->pa_pstart + (ac->ac_o_ex.fe_logical - pa->pa_lstart); in ext4_mb_use_inode_pa()
4284 start + EXT4_C2B(sbi, ac->ac_o_ex.fe_len)); in ext4_mb_use_inode_pa()
4286 ext4_get_group_no_and_offset(ac->ac_sb, start, &ac->ac_b_ex.fe_group, in ext4_mb_use_inode_pa()
4287 &ac->ac_b_ex.fe_start); in ext4_mb_use_inode_pa()
4288 ac->ac_b_ex.fe_len = len; in ext4_mb_use_inode_pa()
4289 ac->ac_status = AC_STATUS_FOUND; in ext4_mb_use_inode_pa()
4290 ac->ac_pa = pa; in ext4_mb_use_inode_pa()
4297 mb_debug(ac->ac_sb, "use %llu/%d from inode pa %p\n", start, len, pa); in ext4_mb_use_inode_pa()
4303 static void ext4_mb_use_group_pa(struct ext4_allocation_context *ac, in ext4_mb_use_group_pa() argument
4306 unsigned int len = ac->ac_o_ex.fe_len; in ext4_mb_use_group_pa()
4308 ext4_get_group_no_and_offset(ac->ac_sb, pa->pa_pstart, in ext4_mb_use_group_pa()
4309 &ac->ac_b_ex.fe_group, in ext4_mb_use_group_pa()
4310 &ac->ac_b_ex.fe_start); in ext4_mb_use_group_pa()
4311 ac->ac_b_ex.fe_len = len; in ext4_mb_use_group_pa()
4312 ac->ac_status = AC_STATUS_FOUND; in ext4_mb_use_group_pa()
4313 ac->ac_pa = pa; in ext4_mb_use_group_pa()
4321 mb_debug(ac->ac_sb, "use %u/%u from group pa %p\n", in ext4_mb_use_group_pa()
4358 ext4_mb_use_preallocated(struct ext4_allocation_context *ac) in ext4_mb_use_preallocated() argument
4360 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_use_preallocated()
4362 struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); in ext4_mb_use_preallocated()
4368 if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) in ext4_mb_use_preallocated()
4377 if (ac->ac_o_ex.fe_logical < pa->pa_lstart || in ext4_mb_use_preallocated()
4378 ac->ac_o_ex.fe_logical >= (pa->pa_lstart + in ext4_mb_use_preallocated()
4383 if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)) && in ext4_mb_use_preallocated()
4392 ext4_mb_use_inode_pa(ac, pa); in ext4_mb_use_preallocated()
4394 ac->ac_criteria = 10; in ext4_mb_use_preallocated()
4403 if (!(ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC)) in ext4_mb_use_preallocated()
4407 lg = ac->ac_lg; in ext4_mb_use_preallocated()
4410 order = fls(ac->ac_o_ex.fe_len) - 1; in ext4_mb_use_preallocated()
4411 if (order > PREALLOC_TB_SIZE - 1) in ext4_mb_use_preallocated()
4413 order = PREALLOC_TB_SIZE - 1; in ext4_mb_use_preallocated()
4415 goal_block = ext4_grp_offs_to_block(ac->ac_sb, &ac->ac_g_ex); in ext4_mb_use_preallocated()
4426 pa->pa_free >= ac->ac_o_ex.fe_len) { in ext4_mb_use_preallocated()
4436 ext4_mb_use_group_pa(ac, cpa); in ext4_mb_use_preallocated()
4437 ac->ac_criteria = 20; in ext4_mb_use_preallocated()
4520 pa->pa_deleted = 1; in ext4_mb_mark_pa_deleted()
4542 static void ext4_mb_put_pa(struct ext4_allocation_context *ac, in ext4_mb_put_pa() argument
4555 if (pa->pa_deleted == 1) { in ext4_mb_put_pa()
4602 ext4_mb_new_inode_pa(struct ext4_allocation_context *ac) in ext4_mb_new_inode_pa() argument
4604 struct super_block *sb = ac->ac_sb; in ext4_mb_new_inode_pa()
4611 BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len); in ext4_mb_new_inode_pa()
4612 BUG_ON(ac->ac_status != AC_STATUS_FOUND); in ext4_mb_new_inode_pa()
4613 BUG_ON(!S_ISREG(ac->ac_inode->i_mode)); in ext4_mb_new_inode_pa()
4614 BUG_ON(ac->ac_pa == NULL); in ext4_mb_new_inode_pa()
4616 pa = ac->ac_pa; in ext4_mb_new_inode_pa()
4618 if (ac->ac_b_ex.fe_len < ac->ac_g_ex.fe_len) { in ext4_mb_new_inode_pa()
4627 BUG_ON(ac->ac_g_ex.fe_logical > ac->ac_o_ex.fe_logical); in ext4_mb_new_inode_pa()
4628 BUG_ON(ac->ac_g_ex.fe_len < ac->ac_o_ex.fe_len); in ext4_mb_new_inode_pa()
4633 winl = ac->ac_o_ex.fe_logical - ac->ac_g_ex.fe_logical; in ext4_mb_new_inode_pa()
4636 wins = EXT4_C2B(sbi, ac->ac_b_ex.fe_len - ac->ac_o_ex.fe_len); in ext4_mb_new_inode_pa()
4641 offs = ac->ac_o_ex.fe_logical % in ext4_mb_new_inode_pa()
4642 EXT4_C2B(sbi, ac->ac_b_ex.fe_len); in ext4_mb_new_inode_pa()
4646 ac->ac_b_ex.fe_logical = ac->ac_o_ex.fe_logical - in ext4_mb_new_inode_pa()
4648 BUG_ON(ac->ac_o_ex.fe_logical < ac->ac_b_ex.fe_logical); in ext4_mb_new_inode_pa()
4649 BUG_ON(ac->ac_o_ex.fe_len > ac->ac_b_ex.fe_len); in ext4_mb_new_inode_pa()
4654 ac->ac_f_ex = ac->ac_b_ex; in ext4_mb_new_inode_pa()
4656 pa->pa_lstart = ac->ac_b_ex.fe_logical; in ext4_mb_new_inode_pa()
4657 pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); in ext4_mb_new_inode_pa()
4658 pa->pa_len = ac->ac_b_ex.fe_len; in ext4_mb_new_inode_pa()
4668 trace_ext4_mb_new_inode_pa(ac, pa); in ext4_mb_new_inode_pa()
4670 ext4_mb_use_inode_pa(ac, pa); in ext4_mb_new_inode_pa()
4673 ei = EXT4_I(ac->ac_inode); in ext4_mb_new_inode_pa()
4674 grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group); in ext4_mb_new_inode_pa()
4677 pa->pa_inode = ac->ac_inode; in ext4_mb_new_inode_pa()
4691 ext4_mb_new_group_pa(struct ext4_allocation_context *ac) in ext4_mb_new_group_pa() argument
4693 struct super_block *sb = ac->ac_sb; in ext4_mb_new_group_pa()
4699 BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len); in ext4_mb_new_group_pa()
4700 BUG_ON(ac->ac_status != AC_STATUS_FOUND); in ext4_mb_new_group_pa()
4701 BUG_ON(!S_ISREG(ac->ac_inode->i_mode)); in ext4_mb_new_group_pa()
4702 BUG_ON(ac->ac_pa == NULL); in ext4_mb_new_group_pa()
4704 pa = ac->ac_pa; in ext4_mb_new_group_pa()
4708 ac->ac_f_ex = ac->ac_b_ex; in ext4_mb_new_group_pa()
4710 pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); in ext4_mb_new_group_pa()
4712 pa->pa_len = ac->ac_b_ex.fe_len; in ext4_mb_new_group_pa()
4722 trace_ext4_mb_new_group_pa(ac, pa); in ext4_mb_new_group_pa()
4724 ext4_mb_use_group_pa(ac, pa); in ext4_mb_new_group_pa()
4727 grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group); in ext4_mb_new_group_pa()
4728 lg = ac->ac_lg; in ext4_mb_new_group_pa()
4742 static void ext4_mb_new_preallocation(struct ext4_allocation_context *ac) in ext4_mb_new_preallocation() argument
4744 if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) in ext4_mb_new_preallocation()
4745 ext4_mb_new_group_pa(ac); in ext4_mb_new_preallocation()
4747 ext4_mb_new_inode_pa(ac); in ext4_mb_new_preallocation()
4792 bit = next + 1; in ext4_mb_release_inode_pa()
4836 * 1) ENOSPC
4838 * 1) how many requested
4880 *busy = 1; in ext4_mb_discard_group_preallocations()
4982 WARN_ON(1); in ext4_discard_preallocations()
5051 static int ext4_mb_pa_alloc(struct ext4_allocation_context *ac) in ext4_mb_pa_alloc() argument
5059 atomic_set(&pa->pa_count, 1); in ext4_mb_pa_alloc()
5060 ac->ac_pa = pa; in ext4_mb_pa_alloc()
5064 static void ext4_mb_pa_free(struct ext4_allocation_context *ac) in ext4_mb_pa_free() argument
5066 struct ext4_prealloc_space *pa = ac->ac_pa; in ext4_mb_pa_free()
5069 ac->ac_pa = NULL; in ext4_mb_pa_free()
5106 static void ext4_mb_show_ac(struct ext4_allocation_context *ac) in ext4_mb_show_ac() argument
5108 struct super_block *sb = ac->ac_sb; in ext4_mb_show_ac()
5116 ac->ac_status, ac->ac_flags); in ext4_mb_show_ac()
5120 (unsigned long)ac->ac_o_ex.fe_group, in ext4_mb_show_ac()
5121 (unsigned long)ac->ac_o_ex.fe_start, in ext4_mb_show_ac()
5122 (unsigned long)ac->ac_o_ex.fe_len, in ext4_mb_show_ac()
5123 (unsigned long)ac->ac_o_ex.fe_logical, in ext4_mb_show_ac()
5124 (unsigned long)ac->ac_g_ex.fe_group, in ext4_mb_show_ac()
5125 (unsigned long)ac->ac_g_ex.fe_start, in ext4_mb_show_ac()
5126 (unsigned long)ac->ac_g_ex.fe_len, in ext4_mb_show_ac()
5127 (unsigned long)ac->ac_g_ex.fe_logical, in ext4_mb_show_ac()
5128 (unsigned long)ac->ac_b_ex.fe_group, in ext4_mb_show_ac()
5129 (unsigned long)ac->ac_b_ex.fe_start, in ext4_mb_show_ac()
5130 (unsigned long)ac->ac_b_ex.fe_len, in ext4_mb_show_ac()
5131 (unsigned long)ac->ac_b_ex.fe_logical, in ext4_mb_show_ac()
5132 (int)ac->ac_criteria); in ext4_mb_show_ac()
5133 mb_debug(sb, "%u found", ac->ac_found); in ext4_mb_show_ac()
5141 static inline void ext4_mb_show_ac(struct ext4_allocation_context *ac) in ext4_mb_show_ac() argument
5143 ext4_mb_show_pa(ac->ac_sb); in ext4_mb_show_ac()
5155 static void ext4_mb_group_or_file(struct ext4_allocation_context *ac) in ext4_mb_group_or_file() argument
5157 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_group_or_file()
5158 int bsbits = ac->ac_sb->s_blocksize_bits; in ext4_mb_group_or_file()
5162 if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) in ext4_mb_group_or_file()
5165 if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) in ext4_mb_group_or_file()
5170 size = ac->ac_o_ex.fe_logical + EXT4_C2B(sbi, ac->ac_o_ex.fe_len); in ext4_mb_group_or_file()
5171 isize = (i_size_read(ac->ac_inode) + ac->ac_sb->s_blocksize - 1) in ext4_mb_group_or_file()
5176 !inode_is_open_for_write(ac->ac_inode)) in ext4_mb_group_or_file()
5186 ac->ac_flags |= EXT4_MB_STREAM_ALLOC; in ext4_mb_group_or_file()
5188 ac->ac_flags |= EXT4_MB_HINT_NOPREALLOC; in ext4_mb_group_or_file()
5192 BUG_ON(ac->ac_lg != NULL); in ext4_mb_group_or_file()
5198 ac->ac_lg = raw_cpu_ptr(sbi->s_locality_groups); in ext4_mb_group_or_file()
5201 ac->ac_flags |= EXT4_MB_HINT_GROUP_ALLOC; in ext4_mb_group_or_file()
5204 mutex_lock(&ac->ac_lg->lg_mutex); in ext4_mb_group_or_file()
5208 ext4_mb_initialize_context(struct ext4_allocation_context *ac, in ext4_mb_initialize_context() argument
5234 ac->ac_b_ex.fe_logical = EXT4_LBLK_CMASK(sbi, ar->logical); in ext4_mb_initialize_context()
5235 ac->ac_status = AC_STATUS_CONTINUE; in ext4_mb_initialize_context()
5236 ac->ac_sb = sb; in ext4_mb_initialize_context()
5237 ac->ac_inode = ar->inode; in ext4_mb_initialize_context()
5238 ac->ac_o_ex.fe_logical = ac->ac_b_ex.fe_logical; in ext4_mb_initialize_context()
5239 ac->ac_o_ex.fe_group = group; in ext4_mb_initialize_context()
5240 ac->ac_o_ex.fe_start = block; in ext4_mb_initialize_context()
5241 ac->ac_o_ex.fe_len = len; in ext4_mb_initialize_context()
5242 ac->ac_g_ex = ac->ac_o_ex; in ext4_mb_initialize_context()
5243 ac->ac_flags = ar->flags; in ext4_mb_initialize_context()
5247 ext4_mb_group_or_file(ac); in ext4_mb_initialize_context()
5249 mb_debug(sb, "init ac: %u blocks @ %u, goal %u, flags 0x%x, 2^%d, " in ext4_mb_initialize_context()
5252 (unsigned) ar->goal, ac->ac_flags, ac->ac_2order, in ext4_mb_initialize_context()
5346 static void ext4_mb_add_n_trim(struct ext4_allocation_context *ac) in ext4_mb_add_n_trim() argument
5348 int order, added = 0, lg_prealloc_count = 1; in ext4_mb_add_n_trim()
5349 struct super_block *sb = ac->ac_sb; in ext4_mb_add_n_trim()
5350 struct ext4_locality_group *lg = ac->ac_lg; in ext4_mb_add_n_trim()
5351 struct ext4_prealloc_space *tmp_pa, *pa = ac->ac_pa; in ext4_mb_add_n_trim()
5353 order = fls(pa->pa_free) - 1; in ext4_mb_add_n_trim()
5354 if (order > PREALLOC_TB_SIZE - 1) in ext4_mb_add_n_trim()
5356 order = PREALLOC_TB_SIZE - 1; in ext4_mb_add_n_trim()
5371 added = 1; in ext4_mb_add_n_trim()
5404 delta = (sbi->s_mb_max_inode_prealloc >> 2) + 1; in ext4_mb_trim_inode_pa()
5414 static int ext4_mb_release_context(struct ext4_allocation_context *ac) in ext4_mb_release_context() argument
5416 struct inode *inode = ac->ac_inode; in ext4_mb_release_context()
5418 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_release_context()
5419 struct ext4_prealloc_space *pa = ac->ac_pa; in ext4_mb_release_context()
5424 pa->pa_pstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len); in ext4_mb_release_context()
5425 pa->pa_lstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len); in ext4_mb_release_context()
5426 pa->pa_free -= ac->ac_b_ex.fe_len; in ext4_mb_release_context()
5427 pa->pa_len -= ac->ac_b_ex.fe_len; in ext4_mb_release_context()
5440 ext4_mb_add_n_trim(ac); in ext4_mb_release_context()
5454 ext4_mb_put_pa(ac, ac->ac_sb, pa); in ext4_mb_release_context()
5456 if (ac->ac_bitmap_page) in ext4_mb_release_context()
5457 put_page(ac->ac_bitmap_page); in ext4_mb_release_context()
5458 if (ac->ac_buddy_page) in ext4_mb_release_context()
5459 put_page(ac->ac_buddy_page); in ext4_mb_release_context()
5460 if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) in ext4_mb_release_context()
5461 mutex_unlock(&ac->ac_lg->lg_mutex); in ext4_mb_release_context()
5462 ext4_mb_collect_stats(ac); in ext4_mb_release_context()
5477 needed = EXT4_CLUSTERS_PER_GROUP(sb) + 1; in ext4_mb_discard_preallocations()
5495 struct ext4_allocation_context *ac, u64 *seq) in ext4_mb_discard_preallocations_should_retry() argument
5501 freed = ext4_mb_discard_preallocations(sb, ac->ac_o_ex.fe_len); in ext4_mb_discard_preallocations_should_retry()
5507 if (!(ac->ac_flags & EXT4_MB_STRICT_CHECK) || seq_retry != *seq) { in ext4_mb_discard_preallocations_should_retry()
5508 ac->ac_flags |= EXT4_MB_STRICT_CHECK; in ext4_mb_discard_preallocations_should_retry()
5529 struct ext4_allocation_context *ac = NULL; in ext4_mb_new_blocks() local
5560 ar->len = ar->len >> 1; in ext4_mb_new_blocks()
5587 ac = kmem_cache_zalloc(ext4_ac_cachep, GFP_NOFS); in ext4_mb_new_blocks()
5588 if (!ac) { in ext4_mb_new_blocks()
5594 *errp = ext4_mb_initialize_context(ac, ar); in ext4_mb_new_blocks()
5600 ac->ac_op = EXT4_MB_HISTORY_PREALLOC; in ext4_mb_new_blocks()
5602 if (!ext4_mb_use_preallocated(ac)) { in ext4_mb_new_blocks()
5603 ac->ac_op = EXT4_MB_HISTORY_ALLOC; in ext4_mb_new_blocks()
5604 ext4_mb_normalize_request(ac, ar); in ext4_mb_new_blocks()
5606 *errp = ext4_mb_pa_alloc(ac); in ext4_mb_new_blocks()
5611 *errp = ext4_mb_regular_allocator(ac); in ext4_mb_new_blocks()
5615 * ac->ac_status == AC_STATUS_FOUND. in ext4_mb_new_blocks()
5616 * And error from above mean ac->ac_status != AC_STATUS_FOUND in ext4_mb_new_blocks()
5620 ext4_mb_pa_free(ac); in ext4_mb_new_blocks()
5621 ext4_discard_allocated_blocks(ac); in ext4_mb_new_blocks()
5624 if (ac->ac_status == AC_STATUS_FOUND && in ext4_mb_new_blocks()
5625 ac->ac_o_ex.fe_len >= ac->ac_f_ex.fe_len) in ext4_mb_new_blocks()
5626 ext4_mb_pa_free(ac); in ext4_mb_new_blocks()
5628 if (likely(ac->ac_status == AC_STATUS_FOUND)) { in ext4_mb_new_blocks()
5629 *errp = ext4_mb_mark_diskspace_used(ac, handle, reserv_clstrs); in ext4_mb_new_blocks()
5631 ext4_discard_allocated_blocks(ac); in ext4_mb_new_blocks()
5634 block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); in ext4_mb_new_blocks()
5635 ar->len = ac->ac_b_ex.fe_len; in ext4_mb_new_blocks()
5639 ext4_mb_discard_preallocations_should_retry(sb, ac, &seq)) in ext4_mb_new_blocks()
5645 ext4_mb_pa_free(ac); in ext4_mb_new_blocks()
5651 ac->ac_b_ex.fe_len = 0; in ext4_mb_new_blocks()
5653 ext4_mb_show_ac(ac); in ext4_mb_new_blocks()
5655 ext4_mb_release_context(ac); in ext4_mb_new_blocks()
5657 if (ac) in ext4_mb_new_blocks()
5658 kmem_cache_free(ext4_ac_cachep, ac); in ext4_mb_new_blocks()
5809 while (1) { in ext4_mb_new_blocks_simple()
5816 blkoff = i + 1; in ext4_mb_new_blocks_simple()
5831 ext4_mb_mark_bb(sb, block, 1, 1); in ext4_mb_new_blocks_simple()
5832 ar->len = 1; in ext4_mb_new_blocks_simple()
6125 BUG_ON(count > 1); in ext4_free_blocks()
6206 ext4_fsblk_t last_cluster = EXT4_B2C(sbi, block + count - 1); in ext4_group_add_blocks()
6207 unsigned long cluster_count = last_cluster - first_cluster + 1; in ext4_group_add_blocks()
6209 ext4_debug("Adding block(s) %llu-%llu\n", block, block + count - 1); in ext4_group_add_blocks()
6376 start = mb_find_next_zero_bit(bitmap, max + 1, start); in ext4_try_to_trim_range()
6379 next = mb_find_next_bit(bitmap, max + 1, start); in ext4_try_to_trim_range()
6389 start = next + 1; in ext4_try_to_trim_range()
6486 end = start + (range->len >> sb->s_blocksize_bits) - 1; in ext4_trim_fs()
6501 if (end >= max_blks - 1) { in ext4_trim_fs()
6502 end = max_blks - 1; in ext4_trim_fs()
6517 end = EXT4_CLUSTERS_PER_GROUP(sb) - 1; in ext4_trim_fs()
6531 * always be EXT4_CLUSTERS_PER_GROUP(sb)-1, so we only need to in ext4_trim_fs()
6537 whole_group = eof ? true : end == EXT4_CLUSTERS_PER_GROUP(sb) - 1; in ext4_trim_fs()
6589 end = EXT4_CLUSTERS_PER_GROUP(sb) - 1; in ext4_mballoc_query_range()
6592 start = mb_find_next_zero_bit(bitmap, end + 1, start); in ext4_mballoc_query_range()
6595 next = mb_find_next_bit(bitmap, end + 1, start); in ext4_mballoc_query_range()
6603 start = next + 1; in ext4_mballoc_query_range()