Lines Matching full:ac
422 static void ext4_mb_new_preallocation(struct ext4_allocation_context *ac);
424 static bool ext4_mb_good_group(struct ext4_allocation_context *ac,
874 static void ext4_mb_choose_next_group_p2_aligned(struct ext4_allocation_context *ac, in ext4_mb_choose_next_group_p2_aligned() argument
877 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_choose_next_group_p2_aligned()
881 if (ac->ac_status == AC_STATUS_FOUND) in ext4_mb_choose_next_group_p2_aligned()
884 if (unlikely(sbi->s_mb_stats && ac->ac_flags & EXT4_MB_CR_POWER2_ALIGNED_OPTIMIZED)) in ext4_mb_choose_next_group_p2_aligned()
887 for (i = ac->ac_2order; i < MB_NUM_ORDERS(ac->ac_sb); i++) { in ext4_mb_choose_next_group_p2_aligned()
899 if (likely(ext4_mb_good_group(ac, iter->bb_group, CR_POWER2_ALIGNED))) { in ext4_mb_choose_next_group_p2_aligned()
901 ac->ac_flags |= EXT4_MB_CR_POWER2_ALIGNED_OPTIMIZED; in ext4_mb_choose_next_group_p2_aligned()
917 ext4_mb_find_good_group_avg_frag_lists(struct ext4_allocation_context *ac, int order) in ext4_mb_find_good_group_avg_frag_lists() argument
919 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_find_good_group_avg_frag_lists()
923 enum criteria cr = ac->ac_criteria; in ext4_mb_find_good_group_avg_frag_lists()
935 if (likely(ext4_mb_good_group(ac, iter->bb_group, cr))) { in ext4_mb_find_good_group_avg_frag_lists()
948 static void ext4_mb_choose_next_group_goal_fast(struct ext4_allocation_context *ac, in ext4_mb_choose_next_group_goal_fast() argument
951 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_choose_next_group_goal_fast()
955 if (unlikely(ac->ac_flags & EXT4_MB_CR_GOAL_LEN_FAST_OPTIMIZED)) { in ext4_mb_choose_next_group_goal_fast()
960 for (i = mb_avg_fragment_size_order(ac->ac_sb, ac->ac_g_ex.fe_len); in ext4_mb_choose_next_group_goal_fast()
961 i < MB_NUM_ORDERS(ac->ac_sb); i++) { in ext4_mb_choose_next_group_goal_fast()
962 grp = ext4_mb_find_good_group_avg_frag_lists(ac, i); in ext4_mb_choose_next_group_goal_fast()
965 ac->ac_flags |= EXT4_MB_CR_GOAL_LEN_FAST_OPTIMIZED; in ext4_mb_choose_next_group_goal_fast()
978 if (ac->ac_flags & EXT4_MB_HINT_DATA) in ext4_mb_choose_next_group_goal_fast()
993 static void ext4_mb_choose_next_group_best_avail(struct ext4_allocation_context *ac, in ext4_mb_choose_next_group_best_avail() argument
996 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_choose_next_group_best_avail()
1001 if (unlikely(ac->ac_flags & EXT4_MB_CR_BEST_AVAIL_LEN_OPTIMIZED)) { in ext4_mb_choose_next_group_best_avail()
1012 order = fls(ac->ac_g_ex.fe_len) - 1; in ext4_mb_choose_next_group_best_avail()
1031 if (1 << min_order < ac->ac_o_ex.fe_len) in ext4_mb_choose_next_group_best_avail()
1032 min_order = fls(ac->ac_o_ex.fe_len); in ext4_mb_choose_next_group_best_avail()
1041 ac->ac_g_ex.fe_len = 1 << i; in ext4_mb_choose_next_group_best_avail()
1049 ac->ac_g_ex.fe_len = roundup(ac->ac_g_ex.fe_len, in ext4_mb_choose_next_group_best_avail()
1053 frag_order = mb_avg_fragment_size_order(ac->ac_sb, in ext4_mb_choose_next_group_best_avail()
1054 ac->ac_g_ex.fe_len); in ext4_mb_choose_next_group_best_avail()
1056 grp = ext4_mb_find_good_group_avg_frag_lists(ac, frag_order); in ext4_mb_choose_next_group_best_avail()
1059 ac->ac_flags |= EXT4_MB_CR_BEST_AVAIL_LEN_OPTIMIZED; in ext4_mb_choose_next_group_best_avail()
1065 ac->ac_g_ex.fe_len = ac->ac_orig_goal_len; in ext4_mb_choose_next_group_best_avail()
1069 static inline int should_optimize_scan(struct ext4_allocation_context *ac) in should_optimize_scan() argument
1071 if (unlikely(!test_opt2(ac->ac_sb, MB_OPTIMIZE_SCAN))) in should_optimize_scan()
1073 if (ac->ac_criteria >= CR_GOAL_LEN_SLOW) in should_optimize_scan()
1075 if (!ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)) in should_optimize_scan()
1085 next_linear_group(struct ext4_allocation_context *ac, ext4_group_t group, in next_linear_group() argument
1088 if (!should_optimize_scan(ac)) in next_linear_group()
1091 if (ac->ac_groups_linear_remaining) { in next_linear_group()
1092 ac->ac_groups_linear_remaining--; in next_linear_group()
1108 * @ac Allocation Context
1118 static void ext4_mb_choose_next_group(struct ext4_allocation_context *ac, in ext4_mb_choose_next_group() argument
1121 *new_cr = ac->ac_criteria; in ext4_mb_choose_next_group()
1123 if (!should_optimize_scan(ac) || ac->ac_groups_linear_remaining) { in ext4_mb_choose_next_group()
1124 *group = next_linear_group(ac, *group, ngroups); in ext4_mb_choose_next_group()
1129 ext4_mb_choose_next_group_p2_aligned(ac, new_cr, group, ngroups); in ext4_mb_choose_next_group()
1131 ext4_mb_choose_next_group_goal_fast(ac, new_cr, group, ngroups); in ext4_mb_choose_next_group()
1133 ext4_mb_choose_next_group_best_avail(ac, new_cr, group, ngroups); in ext4_mb_choose_next_group()
2109 static void ext4_mb_use_best_found(struct ext4_allocation_context *ac, in ext4_mb_use_best_found() argument
2112 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_use_best_found()
2115 BUG_ON(ac->ac_b_ex.fe_group != e4b->bd_group); in ext4_mb_use_best_found()
2116 BUG_ON(ac->ac_status == AC_STATUS_FOUND); in ext4_mb_use_best_found()
2118 ac->ac_b_ex.fe_len = min(ac->ac_b_ex.fe_len, ac->ac_g_ex.fe_len); in ext4_mb_use_best_found()
2119 ac->ac_b_ex.fe_logical = ac->ac_g_ex.fe_logical; in ext4_mb_use_best_found()
2120 ret = mb_mark_used(e4b, &ac->ac_b_ex); in ext4_mb_use_best_found()
2124 ac->ac_f_ex = ac->ac_b_ex; in ext4_mb_use_best_found()
2126 ac->ac_status = AC_STATUS_FOUND; in ext4_mb_use_best_found()
2127 ac->ac_tail = ret & 0xffff; in ext4_mb_use_best_found()
2128 ac->ac_buddy = ret >> 16; in ext4_mb_use_best_found()
2137 ac->ac_bitmap_page = e4b->bd_bitmap_page; in ext4_mb_use_best_found()
2138 get_page(ac->ac_bitmap_page); in ext4_mb_use_best_found()
2139 ac->ac_buddy_page = e4b->bd_buddy_page; in ext4_mb_use_best_found()
2140 get_page(ac->ac_buddy_page); in ext4_mb_use_best_found()
2142 if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) { in ext4_mb_use_best_found()
2144 sbi->s_mb_last_group = ac->ac_f_ex.fe_group; in ext4_mb_use_best_found()
2145 sbi->s_mb_last_start = ac->ac_f_ex.fe_start; in ext4_mb_use_best_found()
2153 if (ac->ac_o_ex.fe_len < ac->ac_b_ex.fe_len) in ext4_mb_use_best_found()
2154 ext4_mb_new_preallocation(ac); in ext4_mb_use_best_found()
2158 static void ext4_mb_check_limits(struct ext4_allocation_context *ac, in ext4_mb_check_limits() argument
2162 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_check_limits()
2163 struct ext4_free_extent *bex = &ac->ac_b_ex; in ext4_mb_check_limits()
2164 struct ext4_free_extent *gex = &ac->ac_g_ex; in ext4_mb_check_limits()
2166 if (ac->ac_status == AC_STATUS_FOUND) in ext4_mb_check_limits()
2171 if (ac->ac_found > sbi->s_mb_max_to_scan && in ext4_mb_check_limits()
2172 !(ac->ac_flags & EXT4_MB_HINT_FIRST)) { in ext4_mb_check_limits()
2173 ac->ac_status = AC_STATUS_BREAK; in ext4_mb_check_limits()
2183 if (finish_group || ac->ac_found > sbi->s_mb_min_to_scan) in ext4_mb_check_limits()
2184 ext4_mb_use_best_found(ac, e4b); in ext4_mb_check_limits()
2211 static void ext4_mb_measure_extent(struct ext4_allocation_context *ac, in ext4_mb_measure_extent() argument
2215 struct ext4_free_extent *bex = &ac->ac_b_ex; in ext4_mb_measure_extent()
2216 struct ext4_free_extent *gex = &ac->ac_g_ex; in ext4_mb_measure_extent()
2219 BUG_ON(ex->fe_len > EXT4_CLUSTERS_PER_GROUP(ac->ac_sb)); in ext4_mb_measure_extent()
2220 BUG_ON(ex->fe_start >= EXT4_CLUSTERS_PER_GROUP(ac->ac_sb)); in ext4_mb_measure_extent()
2221 BUG_ON(ac->ac_status != AC_STATUS_CONTINUE); in ext4_mb_measure_extent()
2223 ac->ac_found++; in ext4_mb_measure_extent()
2224 ac->ac_cX_found[ac->ac_criteria]++; in ext4_mb_measure_extent()
2229 if (unlikely(ac->ac_flags & EXT4_MB_HINT_FIRST)) { in ext4_mb_measure_extent()
2231 ext4_mb_use_best_found(ac, e4b); in ext4_mb_measure_extent()
2240 ext4_mb_use_best_found(ac, e4b); in ext4_mb_measure_extent()
2268 ext4_mb_check_limits(ac, e4b, 0); in ext4_mb_measure_extent()
2272 void ext4_mb_try_best_found(struct ext4_allocation_context *ac, in ext4_mb_try_best_found() argument
2275 struct ext4_free_extent ex = ac->ac_b_ex; in ext4_mb_try_best_found()
2281 err = ext4_mb_load_buddy(ac->ac_sb, group, e4b); in ext4_mb_try_best_found()
2285 ext4_lock_group(ac->ac_sb, group); in ext4_mb_try_best_found()
2289 ac->ac_b_ex = ex; in ext4_mb_try_best_found()
2290 ext4_mb_use_best_found(ac, e4b); in ext4_mb_try_best_found()
2293 ext4_unlock_group(ac->ac_sb, group); in ext4_mb_try_best_found()
2298 int ext4_mb_find_by_goal(struct ext4_allocation_context *ac, in ext4_mb_find_by_goal() argument
2301 ext4_group_t group = ac->ac_g_ex.fe_group; in ext4_mb_find_by_goal()
2304 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_find_by_goal()
2305 struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group); in ext4_mb_find_by_goal()
2310 if (!(ac->ac_flags & (EXT4_MB_HINT_TRY_GOAL | EXT4_MB_HINT_GOAL_ONLY))) in ext4_mb_find_by_goal()
2315 err = ext4_mb_load_buddy(ac->ac_sb, group, e4b); in ext4_mb_find_by_goal()
2324 ext4_lock_group(ac->ac_sb, group); in ext4_mb_find_by_goal()
2325 max = mb_find_extent(e4b, ac->ac_g_ex.fe_start, in ext4_mb_find_by_goal()
2326 ac->ac_g_ex.fe_len, &ex); in ext4_mb_find_by_goal()
2329 if (max >= ac->ac_g_ex.fe_len && in ext4_mb_find_by_goal()
2330 ac->ac_g_ex.fe_len == EXT4_B2C(sbi, sbi->s_stripe)) { in ext4_mb_find_by_goal()
2333 start = ext4_grp_offs_to_block(ac->ac_sb, &ex); in ext4_mb_find_by_goal()
2336 ac->ac_found++; in ext4_mb_find_by_goal()
2337 ac->ac_b_ex = ex; in ext4_mb_find_by_goal()
2338 ext4_mb_use_best_found(ac, e4b); in ext4_mb_find_by_goal()
2340 } else if (max >= ac->ac_g_ex.fe_len) { in ext4_mb_find_by_goal()
2342 BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group); in ext4_mb_find_by_goal()
2343 BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start); in ext4_mb_find_by_goal()
2344 ac->ac_found++; in ext4_mb_find_by_goal()
2345 ac->ac_b_ex = ex; in ext4_mb_find_by_goal()
2346 ext4_mb_use_best_found(ac, e4b); in ext4_mb_find_by_goal()
2347 } else if (max > 0 && (ac->ac_flags & EXT4_MB_HINT_MERGE)) { in ext4_mb_find_by_goal()
2351 BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group); in ext4_mb_find_by_goal()
2352 BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start); in ext4_mb_find_by_goal()
2353 ac->ac_found++; in ext4_mb_find_by_goal()
2354 ac->ac_b_ex = ex; in ext4_mb_find_by_goal()
2355 ext4_mb_use_best_found(ac, e4b); in ext4_mb_find_by_goal()
2357 ext4_unlock_group(ac->ac_sb, group); in ext4_mb_find_by_goal()
2368 void ext4_mb_simple_scan_group(struct ext4_allocation_context *ac, in ext4_mb_simple_scan_group() argument
2371 struct super_block *sb = ac->ac_sb; in ext4_mb_simple_scan_group()
2378 BUG_ON(ac->ac_2order <= 0); in ext4_mb_simple_scan_group()
2379 for (i = ac->ac_2order; i < MB_NUM_ORDERS(sb); i++) { in ext4_mb_simple_scan_group()
2390 ext4_grp_locked_error(ac->ac_sb, e4b->bd_group, 0, 0, in ext4_mb_simple_scan_group()
2393 ext4_mark_group_bitmap_corrupted(ac->ac_sb, in ext4_mb_simple_scan_group()
2398 ac->ac_found++; in ext4_mb_simple_scan_group()
2399 ac->ac_cX_found[ac->ac_criteria]++; in ext4_mb_simple_scan_group()
2401 ac->ac_b_ex.fe_len = 1 << i; in ext4_mb_simple_scan_group()
2402 ac->ac_b_ex.fe_start = k << i; in ext4_mb_simple_scan_group()
2403 ac->ac_b_ex.fe_group = e4b->bd_group; in ext4_mb_simple_scan_group()
2405 ext4_mb_use_best_found(ac, e4b); in ext4_mb_simple_scan_group()
2407 BUG_ON(ac->ac_f_ex.fe_len != ac->ac_g_ex.fe_len); in ext4_mb_simple_scan_group()
2422 void ext4_mb_complex_scan_group(struct ext4_allocation_context *ac, in ext4_mb_complex_scan_group() argument
2425 struct super_block *sb = ac->ac_sb; in ext4_mb_complex_scan_group()
2437 while (free && ac->ac_status == AC_STATUS_CONTINUE) { in ext4_mb_complex_scan_group()
2455 if (!ext4_mb_cr_expensive(ac->ac_criteria)) { in ext4_mb_complex_scan_group()
2466 if (freelen < ac->ac_g_ex.fe_len) { in ext4_mb_complex_scan_group()
2473 mb_find_extent(e4b, i, ac->ac_g_ex.fe_len, &ex); in ext4_mb_complex_scan_group()
2491 ext4_mb_measure_extent(ac, &ex, e4b); in ext4_mb_complex_scan_group()
2497 ext4_mb_check_limits(ac, e4b, 1); in ext4_mb_complex_scan_group()
2505 void ext4_mb_scan_aligned(struct ext4_allocation_context *ac, in ext4_mb_scan_aligned() argument
2508 struct super_block *sb = ac->ac_sb; in ext4_mb_scan_aligned()
2532 ac->ac_found++; in ext4_mb_scan_aligned()
2533 ac->ac_cX_found[ac->ac_criteria]++; in ext4_mb_scan_aligned()
2535 ac->ac_b_ex = ex; in ext4_mb_scan_aligned()
2536 ext4_mb_use_best_found(ac, e4b); in ext4_mb_scan_aligned()
2549 static bool ext4_mb_good_group(struct ext4_allocation_context *ac, in ext4_mb_good_group() argument
2553 int flex_size = ext4_flex_bg_size(EXT4_SB(ac->ac_sb)); in ext4_mb_good_group()
2554 struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group); in ext4_mb_good_group()
2571 BUG_ON(ac->ac_2order == 0); in ext4_mb_good_group()
2574 if ((ac->ac_flags & EXT4_MB_HINT_DATA) && in ext4_mb_good_group()
2579 if (free < ac->ac_g_ex.fe_len) in ext4_mb_good_group()
2582 if (ac->ac_2order >= MB_NUM_ORDERS(ac->ac_sb)) in ext4_mb_good_group()
2585 if (grp->bb_largest_free_order < ac->ac_2order) in ext4_mb_good_group()
2591 if ((free / fragments) >= ac->ac_g_ex.fe_len) in ext4_mb_good_group()
2595 if (free >= ac->ac_g_ex.fe_len) in ext4_mb_good_group()
2618 static int ext4_mb_good_group_nolock(struct ext4_allocation_context *ac, in ext4_mb_good_group_nolock() argument
2621 struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group); in ext4_mb_good_group_nolock()
2622 struct super_block *sb = ac->ac_sb; in ext4_mb_good_group_nolock()
2624 bool should_lock = ac->ac_flags & EXT4_MB_STRICT_CHECK; in ext4_mb_good_group_nolock()
2631 atomic64_inc(&sbi->s_bal_cX_groups_considered[ac->ac_criteria]); in ext4_mb_good_group_nolock()
2644 if (cr < CR_ANY_FREE && free < ac->ac_g_ex.fe_len) in ext4_mb_good_group_nolock()
2683 ret = ext4_mb_good_group(ac, group, cr); in ext4_mb_good_group_nolock()
2767 ext4_mb_regular_allocator(struct ext4_allocation_context *ac) in ext4_mb_regular_allocator() argument
2778 sb = ac->ac_sb; in ext4_mb_regular_allocator()
2782 if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS))) in ext4_mb_regular_allocator()
2785 BUG_ON(ac->ac_status == AC_STATUS_FOUND); in ext4_mb_regular_allocator()
2788 err = ext4_mb_find_by_goal(ac, &e4b); in ext4_mb_regular_allocator()
2789 if (err || ac->ac_status == AC_STATUS_FOUND) in ext4_mb_regular_allocator()
2792 if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) in ext4_mb_regular_allocator()
2796 * ac->ac_2order is set only if the fe_len is a power of 2 in ext4_mb_regular_allocator()
2797 * if ac->ac_2order is set we also set criteria to CR_POWER2_ALIGNED in ext4_mb_regular_allocator()
2800 i = fls(ac->ac_g_ex.fe_len); in ext4_mb_regular_allocator()
2801 ac->ac_2order = 0; in ext4_mb_regular_allocator()
2810 if (is_power_of_2(ac->ac_g_ex.fe_len)) in ext4_mb_regular_allocator()
2811 ac->ac_2order = array_index_nospec(i - 1, in ext4_mb_regular_allocator()
2816 if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) { in ext4_mb_regular_allocator()
2819 ac->ac_g_ex.fe_group = sbi->s_mb_last_group; in ext4_mb_regular_allocator()
2820 ac->ac_g_ex.fe_start = sbi->s_mb_last_start; in ext4_mb_regular_allocator()
2829 if (ac->ac_2order) in ext4_mb_regular_allocator()
2832 for (; cr < EXT4_MB_NUM_CRS && ac->ac_status == AC_STATUS_CONTINUE; cr++) { in ext4_mb_regular_allocator()
2833 ac->ac_criteria = cr; in ext4_mb_regular_allocator()
2838 group = ac->ac_g_ex.fe_group; in ext4_mb_regular_allocator()
2839 ac->ac_groups_linear_remaining = sbi->s_mb_max_linear_groups; in ext4_mb_regular_allocator()
2843 ext4_mb_choose_next_group(ac, &new_cr, &group, ngroups)) { in ext4_mb_regular_allocator()
2872 ret = ext4_mb_good_group_nolock(ac, group, cr); in ext4_mb_regular_allocator()
2889 ret = ext4_mb_good_group(ac, group, cr); in ext4_mb_regular_allocator()
2896 ac->ac_groups_scanned++; in ext4_mb_regular_allocator()
2898 ext4_mb_simple_scan_group(ac, &e4b); in ext4_mb_regular_allocator()
2902 !(ac->ac_g_ex.fe_len % in ext4_mb_regular_allocator()
2904 ext4_mb_scan_aligned(ac, &e4b); in ext4_mb_regular_allocator()
2906 ext4_mb_complex_scan_group(ac, &e4b); in ext4_mb_regular_allocator()
2911 if (ac->ac_status != AC_STATUS_CONTINUE) in ext4_mb_regular_allocator()
2918 if (i == ngroups && ac->ac_criteria == CR_BEST_AVAIL_LEN) in ext4_mb_regular_allocator()
2921 ac->ac_g_ex.fe_len = ac->ac_orig_goal_len; in ext4_mb_regular_allocator()
2924 if (ac->ac_b_ex.fe_len > 0 && ac->ac_status != AC_STATUS_FOUND && in ext4_mb_regular_allocator()
2925 !(ac->ac_flags & EXT4_MB_HINT_FIRST)) { in ext4_mb_regular_allocator()
2930 ext4_mb_try_best_found(ac, &e4b); in ext4_mb_regular_allocator()
2931 if (ac->ac_status != AC_STATUS_FOUND) { in ext4_mb_regular_allocator()
2939 ac->ac_b_ex.fe_group, ac->ac_b_ex.fe_start, in ext4_mb_regular_allocator()
2940 ac->ac_b_ex.fe_len, lost); in ext4_mb_regular_allocator()
2942 ac->ac_b_ex.fe_group = 0; in ext4_mb_regular_allocator()
2943 ac->ac_b_ex.fe_start = 0; in ext4_mb_regular_allocator()
2944 ac->ac_b_ex.fe_len = 0; in ext4_mb_regular_allocator()
2945 ac->ac_status = AC_STATUS_CONTINUE; in ext4_mb_regular_allocator()
2946 ac->ac_flags |= EXT4_MB_HINT_FIRST; in ext4_mb_regular_allocator()
2952 if (sbi->s_mb_stats && ac->ac_status == AC_STATUS_FOUND) in ext4_mb_regular_allocator()
2953 atomic64_inc(&sbi->s_bal_cX_hits[ac->ac_criteria]); in ext4_mb_regular_allocator()
2955 if (!err && ac->ac_status != AC_STATUS_FOUND && first_err) in ext4_mb_regular_allocator()
2959 ac->ac_b_ex.fe_len, ac->ac_o_ex.fe_len, ac->ac_status, in ext4_mb_regular_allocator()
2960 ac->ac_flags, cr, err); in ext4_mb_regular_allocator()
3958 * Check quota and mark chosen space (ac->ac_b_ex) non-free in bitmaps
3962 ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac, in ext4_mb_mark_diskspace_used() argument
3973 BUG_ON(ac->ac_status != AC_STATUS_FOUND); in ext4_mb_mark_diskspace_used()
3974 BUG_ON(ac->ac_b_ex.fe_len <= 0); in ext4_mb_mark_diskspace_used()
3976 sb = ac->ac_sb; in ext4_mb_mark_diskspace_used()
3979 bitmap_bh = ext4_read_block_bitmap(sb, ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
3991 gdp = ext4_get_group_desc(sb, ac->ac_b_ex.fe_group, &gdp_bh); in ext4_mb_mark_diskspace_used()
3995 ext4_debug("using block group %u(%d)\n", ac->ac_b_ex.fe_group, in ext4_mb_mark_diskspace_used()
4003 block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); in ext4_mb_mark_diskspace_used()
4005 len = EXT4_C2B(sbi, ac->ac_b_ex.fe_len); in ext4_mb_mark_diskspace_used()
4006 if (!ext4_inode_block_valid(ac->ac_inode, block, len)) { in ext4_mb_mark_diskspace_used()
4013 ext4_lock_group(sb, ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
4014 mb_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start, in ext4_mb_mark_diskspace_used()
4015 ac->ac_b_ex.fe_len); in ext4_mb_mark_diskspace_used()
4016 ext4_unlock_group(sb, ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
4023 ext4_lock_group(sb, ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
4027 for (i = 0; i < ac->ac_b_ex.fe_len; i++) { in ext4_mb_mark_diskspace_used()
4028 BUG_ON(mb_test_bit(ac->ac_b_ex.fe_start + i, in ext4_mb_mark_diskspace_used()
4033 mb_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start, in ext4_mb_mark_diskspace_used()
4034 ac->ac_b_ex.fe_len); in ext4_mb_mark_diskspace_used()
4040 ac->ac_b_ex.fe_group, gdp)); in ext4_mb_mark_diskspace_used()
4042 len = ext4_free_group_clusters(sb, gdp) - ac->ac_b_ex.fe_len; in ext4_mb_mark_diskspace_used()
4045 ext4_group_desc_csum_set(sb, ac->ac_b_ex.fe_group, gdp); in ext4_mb_mark_diskspace_used()
4047 ext4_unlock_group(sb, ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
4048 percpu_counter_sub(&sbi->s_freeclusters_counter, ac->ac_b_ex.fe_len); in ext4_mb_mark_diskspace_used()
4052 if (!(ac->ac_flags & EXT4_MB_DELALLOC_RESERVED)) in ext4_mb_mark_diskspace_used()
4059 ac->ac_b_ex.fe_group); in ext4_mb_mark_diskspace_used()
4060 atomic64_sub(ac->ac_b_ex.fe_len, in ext4_mb_mark_diskspace_used()
4196 static void ext4_mb_normalize_group_request(struct ext4_allocation_context *ac) in ext4_mb_normalize_group_request() argument
4198 struct super_block *sb = ac->ac_sb; in ext4_mb_normalize_group_request()
4199 struct ext4_locality_group *lg = ac->ac_lg; in ext4_mb_normalize_group_request()
4202 ac->ac_g_ex.fe_len = EXT4_SB(sb)->s_mb_group_prealloc; in ext4_mb_normalize_group_request()
4203 mb_debug(sb, "goal %u blocks for locality group\n", ac->ac_g_ex.fe_len); in ext4_mb_normalize_group_request()
4225 ext4_mb_pa_assert_overlap(struct ext4_allocation_context *ac, in ext4_mb_pa_assert_overlap() argument
4228 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_pa_assert_overlap()
4229 struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); in ext4_mb_pa_assert_overlap()
4252 * Given an allocation context "ac" and a range "start", "end", check
4257 * ac allocation context
4262 ext4_mb_pa_adjust_overlap(struct ext4_allocation_context *ac, in ext4_mb_pa_adjust_overlap() argument
4265 struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); in ext4_mb_pa_adjust_overlap()
4266 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_pa_adjust_overlap()
4284 iter = ext4_mb_pa_rb_next_iter(ac->ac_o_ex.fe_logical, in ext4_mb_pa_adjust_overlap()
4294 BUG_ON(!(ac->ac_o_ex.fe_logical >= tmp_pa_end || in ext4_mb_pa_adjust_overlap()
4295 ac->ac_o_ex.fe_logical < tmp_pa_start)); in ext4_mb_pa_adjust_overlap()
4304 if (tmp_pa->pa_lstart < ac->ac_o_ex.fe_logical) { in ext4_mb_pa_adjust_overlap()
4370 BUG_ON(left_pa_end > ac->ac_o_ex.fe_logical); in ext4_mb_pa_adjust_overlap()
4375 BUG_ON(right_pa_start <= ac->ac_o_ex.fe_logical); in ext4_mb_pa_adjust_overlap()
4391 ext4_mb_pa_assert_overlap(ac, new_start, new_end); in ext4_mb_pa_adjust_overlap()
4402 ext4_mb_normalize_request(struct ext4_allocation_context *ac, in ext4_mb_normalize_request() argument
4405 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_normalize_request()
4414 if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) in ext4_mb_normalize_request()
4418 if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) in ext4_mb_normalize_request()
4423 if (ac->ac_flags & EXT4_MB_HINT_NOPREALLOC) in ext4_mb_normalize_request()
4426 if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) { in ext4_mb_normalize_request()
4427 ext4_mb_normalize_group_request(ac); in ext4_mb_normalize_request()
4431 bsbits = ac->ac_sb->s_blocksize_bits; in ext4_mb_normalize_request()
4435 size = extent_logical_end(sbi, &ac->ac_o_ex); in ext4_mb_normalize_request()
4437 if (size < i_size_read(ac->ac_inode)) in ext4_mb_normalize_request()
4438 size = i_size_read(ac->ac_inode); in ext4_mb_normalize_request()
4465 start_off = ((loff_t)ac->ac_o_ex.fe_logical >> in ext4_mb_normalize_request()
4469 start_off = ((loff_t)ac->ac_o_ex.fe_logical >> in ext4_mb_normalize_request()
4472 } else if (NRL_CHECK_SIZE(EXT4_C2B(sbi, ac->ac_o_ex.fe_len), in ext4_mb_normalize_request()
4474 start_off = ((loff_t)ac->ac_o_ex.fe_logical >> in ext4_mb_normalize_request()
4478 start_off = (loff_t) ac->ac_o_ex.fe_logical << bsbits; in ext4_mb_normalize_request()
4480 ac->ac_o_ex.fe_len) << bsbits; in ext4_mb_normalize_request()
4491 start = max(start, rounddown(ac->ac_o_ex.fe_logical, in ext4_mb_normalize_request()
4492 (ext4_lblk_t)EXT4_BLOCKS_PER_GROUP(ac->ac_sb))); in ext4_mb_normalize_request()
4506 if (size > EXT4_BLOCKS_PER_GROUP(ac->ac_sb)) in ext4_mb_normalize_request()
4507 size = EXT4_BLOCKS_PER_GROUP(ac->ac_sb); in ext4_mb_normalize_request()
4511 ext4_mb_pa_adjust_overlap(ac, &start, &end); in ext4_mb_normalize_request()
4519 * ac->ac_o_ex.fe_logical & fe_len should always lie within "start" and in ext4_mb_normalize_request()
4530 if (start + size <= ac->ac_o_ex.fe_logical || in ext4_mb_normalize_request()
4531 start > ac->ac_o_ex.fe_logical) { in ext4_mb_normalize_request()
4532 ext4_msg(ac->ac_sb, KERN_ERR, in ext4_mb_normalize_request()
4535 (unsigned long) ac->ac_o_ex.fe_logical); in ext4_mb_normalize_request()
4538 BUG_ON(size <= 0 || size > EXT4_BLOCKS_PER_GROUP(ac->ac_sb)); in ext4_mb_normalize_request()
4544 ac->ac_g_ex.fe_logical = start; in ext4_mb_normalize_request()
4545 ac->ac_g_ex.fe_len = EXT4_NUM_B2C(sbi, size); in ext4_mb_normalize_request()
4546 ac->ac_orig_goal_len = ac->ac_g_ex.fe_len; in ext4_mb_normalize_request()
4553 ext4_get_group_no_and_offset(ac->ac_sb, ar->pright - size, in ext4_mb_normalize_request()
4554 &ac->ac_g_ex.fe_group, in ext4_mb_normalize_request()
4555 &ac->ac_g_ex.fe_start); in ext4_mb_normalize_request()
4556 ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL; in ext4_mb_normalize_request()
4561 ext4_get_group_no_and_offset(ac->ac_sb, ar->pleft + 1, in ext4_mb_normalize_request()
4562 &ac->ac_g_ex.fe_group, in ext4_mb_normalize_request()
4563 &ac->ac_g_ex.fe_start); in ext4_mb_normalize_request()
4564 ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL; in ext4_mb_normalize_request()
4567 mb_debug(ac->ac_sb, "goal: %lld(was %lld) blocks at %u\n", size, in ext4_mb_normalize_request()
4571 static void ext4_mb_collect_stats(struct ext4_allocation_context *ac) in ext4_mb_collect_stats() argument
4573 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_collect_stats()
4575 if (sbi->s_mb_stats && ac->ac_g_ex.fe_len >= 1) { in ext4_mb_collect_stats()
4577 atomic_add(ac->ac_b_ex.fe_len, &sbi->s_bal_allocated); in ext4_mb_collect_stats()
4578 if (ac->ac_b_ex.fe_len >= ac->ac_o_ex.fe_len) in ext4_mb_collect_stats()
4581 atomic_add(ac->ac_found, &sbi->s_bal_ex_scanned); in ext4_mb_collect_stats()
4583 atomic_add(ac->ac_cX_found[i], &sbi->s_bal_cX_ex_scanned[i]); in ext4_mb_collect_stats()
4586 atomic_add(ac->ac_groups_scanned, &sbi->s_bal_groups_scanned); in ext4_mb_collect_stats()
4587 if (ac->ac_g_ex.fe_start == ac->ac_b_ex.fe_start && in ext4_mb_collect_stats()
4588 ac->ac_g_ex.fe_group == ac->ac_b_ex.fe_group) in ext4_mb_collect_stats()
4591 if (ac->ac_f_ex.fe_len == ac->ac_orig_goal_len) in ext4_mb_collect_stats()
4594 if (ac->ac_found > sbi->s_mb_max_to_scan) in ext4_mb_collect_stats()
4598 if (ac->ac_op == EXT4_MB_HISTORY_ALLOC) in ext4_mb_collect_stats()
4599 trace_ext4_mballoc_alloc(ac); in ext4_mb_collect_stats()
4601 trace_ext4_mballoc_prealloc(ac); in ext4_mb_collect_stats()
4608 * zeroed out ac->ac_b_ex.fe_len, so group_pa->pa_free is not changed.
4610 static void ext4_discard_allocated_blocks(struct ext4_allocation_context *ac) in ext4_discard_allocated_blocks() argument
4612 struct ext4_prealloc_space *pa = ac->ac_pa; in ext4_discard_allocated_blocks()
4617 if (ac->ac_f_ex.fe_len == 0) in ext4_discard_allocated_blocks()
4619 err = ext4_mb_load_buddy(ac->ac_sb, ac->ac_f_ex.fe_group, &e4b); in ext4_discard_allocated_blocks()
4628 ext4_lock_group(ac->ac_sb, ac->ac_f_ex.fe_group); in ext4_discard_allocated_blocks()
4629 mb_free_blocks(ac->ac_inode, &e4b, ac->ac_f_ex.fe_start, in ext4_discard_allocated_blocks()
4630 ac->ac_f_ex.fe_len); in ext4_discard_allocated_blocks()
4631 ext4_unlock_group(ac->ac_sb, ac->ac_f_ex.fe_group); in ext4_discard_allocated_blocks()
4637 pa->pa_free += ac->ac_b_ex.fe_len; in ext4_discard_allocated_blocks()
4645 static void ext4_mb_use_inode_pa(struct ext4_allocation_context *ac, in ext4_mb_use_inode_pa() argument
4648 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_use_inode_pa()
4654 start = pa->pa_pstart + (ac->ac_o_ex.fe_logical - pa->pa_lstart); in ext4_mb_use_inode_pa()
4656 start + EXT4_C2B(sbi, ac->ac_o_ex.fe_len)); in ext4_mb_use_inode_pa()
4658 ext4_get_group_no_and_offset(ac->ac_sb, start, &ac->ac_b_ex.fe_group, in ext4_mb_use_inode_pa()
4659 &ac->ac_b_ex.fe_start); in ext4_mb_use_inode_pa()
4660 ac->ac_b_ex.fe_len = len; in ext4_mb_use_inode_pa()
4661 ac->ac_status = AC_STATUS_FOUND; in ext4_mb_use_inode_pa()
4662 ac->ac_pa = pa; in ext4_mb_use_inode_pa()
4667 BUG_ON(ac->ac_b_ex.fe_len <= 0); in ext4_mb_use_inode_pa()
4670 mb_debug(ac->ac_sb, "use %llu/%d from inode pa %p\n", start, len, pa); in ext4_mb_use_inode_pa()
4676 static void ext4_mb_use_group_pa(struct ext4_allocation_context *ac, in ext4_mb_use_group_pa() argument
4679 unsigned int len = ac->ac_o_ex.fe_len; in ext4_mb_use_group_pa()
4681 ext4_get_group_no_and_offset(ac->ac_sb, pa->pa_pstart, in ext4_mb_use_group_pa()
4682 &ac->ac_b_ex.fe_group, in ext4_mb_use_group_pa()
4683 &ac->ac_b_ex.fe_start); in ext4_mb_use_group_pa()
4684 ac->ac_b_ex.fe_len = len; in ext4_mb_use_group_pa()
4685 ac->ac_status = AC_STATUS_FOUND; in ext4_mb_use_group_pa()
4686 ac->ac_pa = pa; in ext4_mb_use_group_pa()
4694 mb_debug(ac->ac_sb, "use %u/%u from group pa %p\n", in ext4_mb_use_group_pa()
4731 ext4_mb_pa_goal_check(struct ext4_allocation_context *ac, in ext4_mb_pa_goal_check() argument
4734 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_pa_goal_check()
4737 if (likely(!(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY))) in ext4_mb_pa_goal_check()
4747 (ac->ac_g_ex.fe_logical - pa->pa_lstart); in ext4_mb_pa_goal_check()
4748 if (ext4_grp_offs_to_block(ac->ac_sb, &ac->ac_g_ex) != start) in ext4_mb_pa_goal_check()
4751 if (ac->ac_g_ex.fe_len > pa->pa_len - in ext4_mb_pa_goal_check()
4752 EXT4_B2C(sbi, ac->ac_g_ex.fe_logical - pa->pa_lstart)) in ext4_mb_pa_goal_check()
4762 ext4_mb_use_preallocated(struct ext4_allocation_context *ac) in ext4_mb_use_preallocated() argument
4764 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_use_preallocated()
4766 struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); in ext4_mb_use_preallocated()
4773 if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) in ext4_mb_use_preallocated()
4796 iter = ext4_mb_pa_rb_next_iter(ac->ac_o_ex.fe_logical, in ext4_mb_use_preallocated()
4807 if (tmp_pa->pa_lstart > ac->ac_o_ex.fe_logical) { in ext4_mb_use_preallocated()
4824 BUG_ON(!(tmp_pa && tmp_pa->pa_lstart <= ac->ac_o_ex.fe_logical)); in ext4_mb_use_preallocated()
4856 BUG_ON(!(tmp_pa && tmp_pa->pa_lstart <= ac->ac_o_ex.fe_logical)); in ext4_mb_use_preallocated()
4864 if (ac->ac_o_ex.fe_logical >= pa_logical_end(sbi, tmp_pa)) { in ext4_mb_use_preallocated()
4870 if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)) && in ext4_mb_use_preallocated()
4881 if (tmp_pa->pa_free && likely(ext4_mb_pa_goal_check(ac, tmp_pa))) { in ext4_mb_use_preallocated()
4883 ext4_mb_use_inode_pa(ac, tmp_pa); in ext4_mb_use_preallocated()
4922 if (!(ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC)) in ext4_mb_use_preallocated()
4926 lg = ac->ac_lg; in ext4_mb_use_preallocated()
4929 order = fls(ac->ac_o_ex.fe_len) - 1; in ext4_mb_use_preallocated()
4934 goal_block = ext4_grp_offs_to_block(ac->ac_sb, &ac->ac_g_ex); in ext4_mb_use_preallocated()
4945 tmp_pa->pa_free >= ac->ac_o_ex.fe_len) { in ext4_mb_use_preallocated()
4955 ext4_mb_use_group_pa(ac, cpa); in ext4_mb_use_preallocated()
5070 static void ext4_mb_put_pa(struct ext4_allocation_context *ac, in ext4_mb_put_pa() argument
5075 struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); in ext4_mb_put_pa()
5162 ext4_mb_new_inode_pa(struct ext4_allocation_context *ac) in ext4_mb_new_inode_pa() argument
5164 struct super_block *sb = ac->ac_sb; in ext4_mb_new_inode_pa()
5171 BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len); in ext4_mb_new_inode_pa()
5172 BUG_ON(ac->ac_status != AC_STATUS_FOUND); in ext4_mb_new_inode_pa()
5173 BUG_ON(!S_ISREG(ac->ac_inode->i_mode)); in ext4_mb_new_inode_pa()
5174 BUG_ON(ac->ac_pa == NULL); in ext4_mb_new_inode_pa()
5176 pa = ac->ac_pa; in ext4_mb_new_inode_pa()
5178 if (ac->ac_b_ex.fe_len < ac->ac_orig_goal_len) { in ext4_mb_new_inode_pa()
5180 .fe_logical = ac->ac_g_ex.fe_logical, in ext4_mb_new_inode_pa()
5181 .fe_len = ac->ac_orig_goal_len, in ext4_mb_new_inode_pa()
5188 BUG_ON(ac->ac_g_ex.fe_logical > ac->ac_o_ex.fe_logical); in ext4_mb_new_inode_pa()
5189 BUG_ON(ac->ac_g_ex.fe_len < ac->ac_o_ex.fe_len); in ext4_mb_new_inode_pa()
5202 ex.fe_len = ac->ac_b_ex.fe_len; in ext4_mb_new_inode_pa()
5205 if (ac->ac_o_ex.fe_logical >= ex.fe_logical) in ext4_mb_new_inode_pa()
5208 ex.fe_logical = ac->ac_g_ex.fe_logical; in ext4_mb_new_inode_pa()
5209 if (ac->ac_o_ex.fe_logical < extent_logical_end(sbi, &ex)) in ext4_mb_new_inode_pa()
5212 ex.fe_logical = ac->ac_o_ex.fe_logical; in ext4_mb_new_inode_pa()
5214 ac->ac_b_ex.fe_logical = ex.fe_logical; in ext4_mb_new_inode_pa()
5216 BUG_ON(ac->ac_o_ex.fe_logical < ac->ac_b_ex.fe_logical); in ext4_mb_new_inode_pa()
5217 BUG_ON(ac->ac_o_ex.fe_len > ac->ac_b_ex.fe_len); in ext4_mb_new_inode_pa()
5221 pa->pa_lstart = ac->ac_b_ex.fe_logical; in ext4_mb_new_inode_pa()
5222 pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); in ext4_mb_new_inode_pa()
5223 pa->pa_len = ac->ac_b_ex.fe_len; in ext4_mb_new_inode_pa()
5232 trace_ext4_mb_new_inode_pa(ac, pa); in ext4_mb_new_inode_pa()
5235 ext4_mb_use_inode_pa(ac, pa); in ext4_mb_new_inode_pa()
5237 ei = EXT4_I(ac->ac_inode); in ext4_mb_new_inode_pa()
5238 grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group); in ext4_mb_new_inode_pa()
5243 pa->pa_inode = ac->ac_inode; in ext4_mb_new_inode_pa()
5257 ext4_mb_new_group_pa(struct ext4_allocation_context *ac) in ext4_mb_new_group_pa() argument
5259 struct super_block *sb = ac->ac_sb; in ext4_mb_new_group_pa()
5265 BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len); in ext4_mb_new_group_pa()
5266 BUG_ON(ac->ac_status != AC_STATUS_FOUND); in ext4_mb_new_group_pa()
5267 BUG_ON(!S_ISREG(ac->ac_inode->i_mode)); in ext4_mb_new_group_pa()
5268 BUG_ON(ac->ac_pa == NULL); in ext4_mb_new_group_pa()
5270 pa = ac->ac_pa; in ext4_mb_new_group_pa()
5272 pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); in ext4_mb_new_group_pa()
5274 pa->pa_len = ac->ac_b_ex.fe_len; in ext4_mb_new_group_pa()
5284 trace_ext4_mb_new_group_pa(ac, pa); in ext4_mb_new_group_pa()
5286 ext4_mb_use_group_pa(ac, pa); in ext4_mb_new_group_pa()
5289 grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group); in ext4_mb_new_group_pa()
5292 lg = ac->ac_lg; in ext4_mb_new_group_pa()
5306 static void ext4_mb_new_preallocation(struct ext4_allocation_context *ac) in ext4_mb_new_preallocation() argument
5308 if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) in ext4_mb_new_preallocation()
5309 ext4_mb_new_group_pa(ac); in ext4_mb_new_preallocation()
5311 ext4_mb_new_inode_pa(ac); in ext4_mb_new_preallocation()
5630 static int ext4_mb_pa_alloc(struct ext4_allocation_context *ac) in ext4_mb_pa_alloc() argument
5639 ac->ac_pa = pa; in ext4_mb_pa_alloc()
5643 static void ext4_mb_pa_put_free(struct ext4_allocation_context *ac) in ext4_mb_pa_put_free() argument
5645 struct ext4_prealloc_space *pa = ac->ac_pa; in ext4_mb_pa_put_free()
5648 ac->ac_pa = NULL; in ext4_mb_pa_put_free()
5694 static void ext4_mb_show_ac(struct ext4_allocation_context *ac) in ext4_mb_show_ac() argument
5696 struct super_block *sb = ac->ac_sb; in ext4_mb_show_ac()
5704 ac->ac_status, ac->ac_flags); in ext4_mb_show_ac()
5708 (unsigned long)ac->ac_o_ex.fe_group, in ext4_mb_show_ac()
5709 (unsigned long)ac->ac_o_ex.fe_start, in ext4_mb_show_ac()
5710 (unsigned long)ac->ac_o_ex.fe_len, in ext4_mb_show_ac()
5711 (unsigned long)ac->ac_o_ex.fe_logical, in ext4_mb_show_ac()
5712 (unsigned long)ac->ac_g_ex.fe_group, in ext4_mb_show_ac()
5713 (unsigned long)ac->ac_g_ex.fe_start, in ext4_mb_show_ac()
5714 (unsigned long)ac->ac_g_ex.fe_len, in ext4_mb_show_ac()
5715 (unsigned long)ac->ac_g_ex.fe_logical, in ext4_mb_show_ac()
5716 (unsigned long)ac->ac_b_ex.fe_group, in ext4_mb_show_ac()
5717 (unsigned long)ac->ac_b_ex.fe_start, in ext4_mb_show_ac()
5718 (unsigned long)ac->ac_b_ex.fe_len, in ext4_mb_show_ac()
5719 (unsigned long)ac->ac_b_ex.fe_logical, in ext4_mb_show_ac()
5720 (int)ac->ac_criteria); in ext4_mb_show_ac()
5721 mb_debug(sb, "%u found", ac->ac_found); in ext4_mb_show_ac()
5722 mb_debug(sb, "used pa: %s, ", ac->ac_pa ? "yes" : "no"); in ext4_mb_show_ac()
5723 if (ac->ac_pa) in ext4_mb_show_ac()
5724 mb_debug(sb, "pa_type %s\n", ac->ac_pa->pa_type == MB_GROUP_PA ? in ext4_mb_show_ac()
5732 static inline void ext4_mb_show_ac(struct ext4_allocation_context *ac) in ext4_mb_show_ac() argument
5734 ext4_mb_show_pa(ac->ac_sb); in ext4_mb_show_ac()
5745 static void ext4_mb_group_or_file(struct ext4_allocation_context *ac) in ext4_mb_group_or_file() argument
5747 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_group_or_file()
5748 int bsbits = ac->ac_sb->s_blocksize_bits; in ext4_mb_group_or_file()
5752 if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) in ext4_mb_group_or_file()
5755 if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) in ext4_mb_group_or_file()
5760 size = extent_logical_end(sbi, &ac->ac_o_ex); in ext4_mb_group_or_file()
5761 isize = (i_size_read(ac->ac_inode) + ac->ac_sb->s_blocksize - 1) in ext4_mb_group_or_file()
5766 !inode_is_open_for_write(ac->ac_inode)) in ext4_mb_group_or_file()
5776 ac->ac_flags |= EXT4_MB_STREAM_ALLOC; in ext4_mb_group_or_file()
5778 ac->ac_flags |= EXT4_MB_HINT_NOPREALLOC; in ext4_mb_group_or_file()
5782 BUG_ON(ac->ac_lg != NULL); in ext4_mb_group_or_file()
5788 ac->ac_lg = raw_cpu_ptr(sbi->s_locality_groups); in ext4_mb_group_or_file()
5791 ac->ac_flags |= EXT4_MB_HINT_GROUP_ALLOC; in ext4_mb_group_or_file()
5794 mutex_lock(&ac->ac_lg->lg_mutex); in ext4_mb_group_or_file()
5798 ext4_mb_initialize_context(struct ext4_allocation_context *ac, in ext4_mb_initialize_context() argument
5824 ac->ac_b_ex.fe_logical = EXT4_LBLK_CMASK(sbi, ar->logical); in ext4_mb_initialize_context()
5825 ac->ac_status = AC_STATUS_CONTINUE; in ext4_mb_initialize_context()
5826 ac->ac_sb = sb; in ext4_mb_initialize_context()
5827 ac->ac_inode = ar->inode; in ext4_mb_initialize_context()
5828 ac->ac_o_ex.fe_logical = ac->ac_b_ex.fe_logical; in ext4_mb_initialize_context()
5829 ac->ac_o_ex.fe_group = group; in ext4_mb_initialize_context()
5830 ac->ac_o_ex.fe_start = block; in ext4_mb_initialize_context()
5831 ac->ac_o_ex.fe_len = len; in ext4_mb_initialize_context()
5832 ac->ac_g_ex = ac->ac_o_ex; in ext4_mb_initialize_context()
5833 ac->ac_orig_goal_len = ac->ac_g_ex.fe_len; in ext4_mb_initialize_context()
5834 ac->ac_flags = ar->flags; in ext4_mb_initialize_context()
5838 ext4_mb_group_or_file(ac); in ext4_mb_initialize_context()
5840 mb_debug(sb, "init ac: %u blocks @ %u, goal %u, flags 0x%x, 2^%d, " in ext4_mb_initialize_context()
5843 (unsigned) ar->goal, ac->ac_flags, ac->ac_2order, in ext4_mb_initialize_context()
5933 static void ext4_mb_add_n_trim(struct ext4_allocation_context *ac) in ext4_mb_add_n_trim() argument
5936 struct super_block *sb = ac->ac_sb; in ext4_mb_add_n_trim()
5937 struct ext4_locality_group *lg = ac->ac_lg; in ext4_mb_add_n_trim()
5938 struct ext4_prealloc_space *tmp_pa, *pa = ac->ac_pa; in ext4_mb_add_n_trim()
5981 static int ext4_mb_release_context(struct ext4_allocation_context *ac) in ext4_mb_release_context() argument
5983 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); in ext4_mb_release_context()
5984 struct ext4_prealloc_space *pa = ac->ac_pa; in ext4_mb_release_context()
5989 pa->pa_pstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len); in ext4_mb_release_context()
5990 pa->pa_lstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len); in ext4_mb_release_context()
5991 pa->pa_free -= ac->ac_b_ex.fe_len; in ext4_mb_release_context()
5992 pa->pa_len -= ac->ac_b_ex.fe_len; in ext4_mb_release_context()
6005 ext4_mb_add_n_trim(ac); in ext4_mb_release_context()
6009 ext4_mb_put_pa(ac, ac->ac_sb, pa); in ext4_mb_release_context()
6011 if (ac->ac_bitmap_page) in ext4_mb_release_context()
6012 put_page(ac->ac_bitmap_page); in ext4_mb_release_context()
6013 if (ac->ac_buddy_page) in ext4_mb_release_context()
6014 put_page(ac->ac_buddy_page); in ext4_mb_release_context()
6015 if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) in ext4_mb_release_context()
6016 mutex_unlock(&ac->ac_lg->lg_mutex); in ext4_mb_release_context()
6017 ext4_mb_collect_stats(ac); in ext4_mb_release_context()
6049 struct ext4_allocation_context *ac, u64 *seq) in ext4_mb_discard_preallocations_should_retry() argument
6055 freed = ext4_mb_discard_preallocations(sb, ac->ac_o_ex.fe_len); in ext4_mb_discard_preallocations_should_retry()
6061 if (!(ac->ac_flags & EXT4_MB_STRICT_CHECK) || seq_retry != *seq) { in ext4_mb_discard_preallocations_should_retry()
6062 ac->ac_flags |= EXT4_MB_STRICT_CHECK; in ext4_mb_discard_preallocations_should_retry()
6147 struct ext4_allocation_context *ac = NULL; in ext4_mb_new_blocks() local
6205 ac = kmem_cache_zalloc(ext4_ac_cachep, GFP_NOFS); in ext4_mb_new_blocks()
6206 if (!ac) { in ext4_mb_new_blocks()
6212 ext4_mb_initialize_context(ac, ar); in ext4_mb_new_blocks()
6214 ac->ac_op = EXT4_MB_HISTORY_PREALLOC; in ext4_mb_new_blocks()
6216 if (!ext4_mb_use_preallocated(ac)) { in ext4_mb_new_blocks()
6217 ac->ac_op = EXT4_MB_HISTORY_ALLOC; in ext4_mb_new_blocks()
6218 ext4_mb_normalize_request(ac, ar); in ext4_mb_new_blocks()
6220 *errp = ext4_mb_pa_alloc(ac); in ext4_mb_new_blocks()
6225 *errp = ext4_mb_regular_allocator(ac); in ext4_mb_new_blocks()
6229 * ac->ac_status == AC_STATUS_FOUND. in ext4_mb_new_blocks()
6230 * And error from above mean ac->ac_status != AC_STATUS_FOUND in ext4_mb_new_blocks()
6234 ext4_mb_pa_put_free(ac); in ext4_mb_new_blocks()
6235 ext4_discard_allocated_blocks(ac); in ext4_mb_new_blocks()
6238 if (ac->ac_status == AC_STATUS_FOUND && in ext4_mb_new_blocks()
6239 ac->ac_o_ex.fe_len >= ac->ac_f_ex.fe_len) in ext4_mb_new_blocks()
6240 ext4_mb_pa_put_free(ac); in ext4_mb_new_blocks()
6242 if (likely(ac->ac_status == AC_STATUS_FOUND)) { in ext4_mb_new_blocks()
6243 *errp = ext4_mb_mark_diskspace_used(ac, handle, reserv_clstrs); in ext4_mb_new_blocks()
6245 ext4_discard_allocated_blocks(ac); in ext4_mb_new_blocks()
6248 block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); in ext4_mb_new_blocks()
6249 ar->len = ac->ac_b_ex.fe_len; in ext4_mb_new_blocks()
6253 ext4_mb_discard_preallocations_should_retry(sb, ac, &seq)) in ext4_mb_new_blocks()
6259 ext4_mb_pa_put_free(ac); in ext4_mb_new_blocks()
6265 ac->ac_b_ex.fe_len = 0; in ext4_mb_new_blocks()
6267 ext4_mb_show_ac(ac); in ext4_mb_new_blocks()
6269 ext4_mb_release_context(ac); in ext4_mb_new_blocks()
6270 kmem_cache_free(ext4_ac_cachep, ac); in ext4_mb_new_blocks()