Lines Matching +full:edge +full:- +full:offset
1 // SPDX-License-Identifier: GPL-2.0
12 #include <linux/error-injection.h>
14 #include "disk-io.h"
19 #include "async-thread.h"
20 #include "free-space-cache.h"
22 #include "print-tree.h"
23 #include "delalloc-space.h"
24 #include "block-group.h"
29 #include "inode-item.h"
42 * ------------------------------------------------------------------
49 * 1. Mark the target block group read-only
164 if (node->level == 0 || in mark_block_processed()
165 in_range(node->bytenr, rc->block_group->start, in mark_block_processed()
166 rc->block_group->length)) { in mark_block_processed()
167 blocksize = rc->extent_root->fs_info->nodesize; in mark_block_processed()
168 set_extent_bits(&rc->processed_blocks, node->bytenr, in mark_block_processed()
169 node->bytenr + blocksize - 1, EXTENT_DIRTY); in mark_block_processed()
171 node->processed = 1; in mark_block_processed()
177 tree->rb_root = RB_ROOT; in mapping_tree_init()
178 spin_lock_init(&tree->lock); in mapping_tree_init()
188 struct btrfs_backref_edge *edge; in walk_up_backref() local
191 while (!list_empty(&node->upper)) { in walk_up_backref()
192 edge = list_entry(node->upper.next, in walk_up_backref()
194 edges[idx++] = edge; in walk_up_backref()
195 node = edge->node[UPPER]; in walk_up_backref()
197 BUG_ON(node->detached); in walk_up_backref()
208 struct btrfs_backref_edge *edge; in walk_down_backref() local
213 edge = edges[idx - 1]; in walk_down_backref()
214 lower = edge->node[LOWER]; in walk_down_backref()
215 if (list_is_last(&edge->list[LOWER], &lower->upper)) { in walk_down_backref()
216 idx--; in walk_down_backref()
219 edge = list_entry(edge->list[LOWER].next, in walk_down_backref()
221 edges[idx - 1] = edge; in walk_down_backref()
223 return edge->node[UPPER]; in walk_down_backref()
233 rb_erase(&node->rb_node, &cache->rb_root); in update_backref_node()
234 node->bytenr = bytenr; in update_backref_node()
235 rb_node = rb_simple_insert(&cache->rb_root, node->bytenr, &node->rb_node); in update_backref_node()
237 btrfs_backref_panic(cache->fs_info, bytenr, -EEXIST); in update_backref_node()
249 if (cache->last_trans == 0) { in update_backref_cache()
250 cache->last_trans = trans->transid; in update_backref_cache()
254 if (cache->last_trans == trans->transid) in update_backref_cache()
262 while (!list_empty(&cache->detached)) { in update_backref_cache()
263 node = list_entry(cache->detached.next, in update_backref_cache()
268 while (!list_empty(&cache->changed)) { in update_backref_cache()
269 node = list_entry(cache->changed.next, in update_backref_cache()
271 list_del_init(&node->list); in update_backref_cache()
272 BUG_ON(node->pending); in update_backref_cache()
273 update_backref_node(cache, node, node->new_bytenr); in update_backref_cache()
281 list_for_each_entry(node, &cache->pending[level], list) { in update_backref_cache()
282 BUG_ON(!node->pending); in update_backref_cache()
283 if (node->bytenr == node->new_bytenr) in update_backref_cache()
285 update_backref_node(cache, node, node->new_bytenr); in update_backref_cache()
289 cache->last_trans = 0; in update_backref_cache()
301 if (test_bit(BTRFS_ROOT_DEAD_RELOC_TREE, &root->state)) in reloc_root_is_dead()
318 if (!root->reloc_root) in have_reloc_root()
327 if (!test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) in btrfs_should_ignore_reloc_root()
334 reloc_root = root->reloc_root; in btrfs_should_ignore_reloc_root()
338 if (btrfs_header_generation(reloc_root->commit_root) == in btrfs_should_ignore_reloc_root()
339 root->fs_info->running_transaction->transid) in btrfs_should_ignore_reloc_root()
355 struct reloc_control *rc = fs_info->reloc_ctl; in find_reloc_root()
361 spin_lock(&rc->reloc_root_tree.lock); in find_reloc_root()
362 rb_node = rb_simple_search(&rc->reloc_root_tree.rb_root, bytenr); in find_reloc_root()
365 root = node->data; in find_reloc_root()
367 spin_unlock(&rc->reloc_root_tree.lock); in find_reloc_root()
374 * - Cleanup the children edges and nodes
378 * - Freeing up leaves (level 0), keeps nodes detached
387 struct btrfs_backref_cache *cache = &rc->backref_cache; in handle_useless_nodes()
388 struct list_head *useless_node = &cache->useless_node; in handle_useless_nodes()
396 list_del_init(&cur->list); in handle_useless_nodes()
399 ASSERT(list_empty(&cur->upper)); in handle_useless_nodes()
405 if (cur->lowest) { in handle_useless_nodes()
406 list_del_init(&cur->lower); in handle_useless_nodes()
407 cur->lowest = 0; in handle_useless_nodes()
411 while (!list_empty(&cur->lower)) { in handle_useless_nodes()
412 struct btrfs_backref_edge *edge; in handle_useless_nodes() local
415 edge = list_entry(cur->lower.next, in handle_useless_nodes()
417 list_del(&edge->list[UPPER]); in handle_useless_nodes()
418 list_del(&edge->list[LOWER]); in handle_useless_nodes()
419 lower = edge->node[LOWER]; in handle_useless_nodes()
420 btrfs_backref_free_edge(cache, edge); in handle_useless_nodes()
423 if (list_empty(&lower->upper)) in handle_useless_nodes()
424 list_add(&lower->list, useless_node); in handle_useless_nodes()
434 if (cur->level > 0) { in handle_useless_nodes()
435 list_add(&cur->list, &cache->detached); in handle_useless_nodes()
436 cur->detached = 1; in handle_useless_nodes()
438 rb_erase(&cur->rb_node, &cache->rb_root); in handle_useless_nodes()
448 * b-trees that reference the tree block.
464 struct btrfs_backref_cache *cache = &rc->backref_cache; in build_backref_tree()
469 struct btrfs_backref_edge *edge; in build_backref_tree() local
473 iter = btrfs_backref_iter_alloc(rc->extent_root->fs_info, GFP_NOFS); in build_backref_tree()
475 return ERR_PTR(-ENOMEM); in build_backref_tree()
478 err = -ENOMEM; in build_backref_tree()
484 err = -ENOMEM; in build_backref_tree()
488 node->lowest = 1; in build_backref_tree()
491 /* Breadth-first search to build backref cache */ in build_backref_tree()
499 edge = list_first_entry_or_null(&cache->pending_edge, in build_backref_tree()
505 if (edge) { in build_backref_tree()
506 list_del_init(&edge->list[UPPER]); in build_backref_tree()
507 cur = edge->node[UPPER]; in build_backref_tree()
509 } while (edge); in build_backref_tree()
527 ASSERT(!node || !node->detached); in build_backref_tree()
528 ASSERT(list_empty(&cache->useless_node) && in build_backref_tree()
529 list_empty(&cache->pending_edge)); in build_backref_tree()
543 struct btrfs_root *reloc_root = src->reloc_root; in clone_backref_node()
544 struct btrfs_backref_cache *cache = &rc->backref_cache; in clone_backref_node()
547 struct btrfs_backref_edge *edge; in clone_backref_node() local
551 if (cache->last_trans > 0) in clone_backref_node()
554 rb_node = rb_simple_search(&cache->rb_root, src->commit_root->start); in clone_backref_node()
557 if (node->detached) in clone_backref_node()
560 BUG_ON(node->new_bytenr != reloc_root->node->start); in clone_backref_node()
564 rb_node = rb_simple_search(&cache->rb_root, in clone_backref_node()
565 reloc_root->commit_root->start); in clone_backref_node()
569 BUG_ON(node->detached); in clone_backref_node()
576 new_node = btrfs_backref_alloc_node(cache, dest->node->start, in clone_backref_node()
577 node->level); in clone_backref_node()
579 return -ENOMEM; in clone_backref_node()
581 new_node->lowest = node->lowest; in clone_backref_node()
582 new_node->checked = 1; in clone_backref_node()
583 new_node->root = btrfs_grab_root(dest); in clone_backref_node()
584 ASSERT(new_node->root); in clone_backref_node()
586 if (!node->lowest) { in clone_backref_node()
587 list_for_each_entry(edge, &node->lower, list[UPPER]) { in clone_backref_node()
592 btrfs_backref_link_edge(new_edge, edge->node[LOWER], in clone_backref_node()
596 list_add_tail(&new_node->lower, &cache->leaves); in clone_backref_node()
599 rb_node = rb_simple_insert(&cache->rb_root, new_node->bytenr, in clone_backref_node()
600 &new_node->rb_node); in clone_backref_node()
602 btrfs_backref_panic(trans->fs_info, new_node->bytenr, -EEXIST); in clone_backref_node()
604 if (!new_node->lowest) { in clone_backref_node()
605 list_for_each_entry(new_edge, &new_node->lower, list[UPPER]) { in clone_backref_node()
606 list_add_tail(&new_edge->list[LOWER], in clone_backref_node()
607 &new_edge->node[LOWER]->upper); in clone_backref_node()
612 while (!list_empty(&new_node->lower)) { in clone_backref_node()
613 new_edge = list_entry(new_node->lower.next, in clone_backref_node()
615 list_del(&new_edge->list[UPPER]); in clone_backref_node()
619 return -ENOMEM; in clone_backref_node()
623 * helper to add 'address of tree root -> reloc tree' mapping
627 struct btrfs_fs_info *fs_info = root->fs_info; in __add_reloc_root()
630 struct reloc_control *rc = fs_info->reloc_ctl; in __add_reloc_root()
634 return -ENOMEM; in __add_reloc_root()
636 node->bytenr = root->commit_root->start; in __add_reloc_root()
637 node->data = root; in __add_reloc_root()
639 spin_lock(&rc->reloc_root_tree.lock); in __add_reloc_root()
640 rb_node = rb_simple_insert(&rc->reloc_root_tree.rb_root, in __add_reloc_root()
641 node->bytenr, &node->rb_node); in __add_reloc_root()
642 spin_unlock(&rc->reloc_root_tree.lock); in __add_reloc_root()
646 node->bytenr); in __add_reloc_root()
647 return -EEXIST; in __add_reloc_root()
650 list_add_tail(&root->root_list, &rc->reloc_roots); in __add_reloc_root()
655 * helper to delete the 'address of tree root -> reloc tree'
660 struct btrfs_fs_info *fs_info = root->fs_info; in __del_reloc_root()
663 struct reloc_control *rc = fs_info->reloc_ctl; in __del_reloc_root()
666 if (rc && root->node) { in __del_reloc_root()
667 spin_lock(&rc->reloc_root_tree.lock); in __del_reloc_root()
668 rb_node = rb_simple_search(&rc->reloc_root_tree.rb_root, in __del_reloc_root()
669 root->commit_root->start); in __del_reloc_root()
672 rb_erase(&node->rb_node, &rc->reloc_root_tree.rb_root); in __del_reloc_root()
673 RB_CLEAR_NODE(&node->rb_node); in __del_reloc_root()
675 spin_unlock(&rc->reloc_root_tree.lock); in __del_reloc_root()
676 ASSERT(!node || (struct btrfs_root *)node->data == root); in __del_reloc_root()
681 * of places where the pattern is to splice the rc->reloc_roots, process in __del_reloc_root()
683 * rc->reloc_roots. If we call __del_reloc_root while it's off of the in __del_reloc_root()
687 spin_lock(&fs_info->trans_lock); in __del_reloc_root()
688 if (!list_empty(&root->root_list)) { in __del_reloc_root()
690 list_del_init(&root->root_list); in __del_reloc_root()
692 spin_unlock(&fs_info->trans_lock); in __del_reloc_root()
699 * helper to update the 'address of tree root -> reloc tree'
704 struct btrfs_fs_info *fs_info = root->fs_info; in __update_reloc_root()
707 struct reloc_control *rc = fs_info->reloc_ctl; in __update_reloc_root()
709 spin_lock(&rc->reloc_root_tree.lock); in __update_reloc_root()
710 rb_node = rb_simple_search(&rc->reloc_root_tree.rb_root, in __update_reloc_root()
711 root->commit_root->start); in __update_reloc_root()
714 rb_erase(&node->rb_node, &rc->reloc_root_tree.rb_root); in __update_reloc_root()
716 spin_unlock(&rc->reloc_root_tree.lock); in __update_reloc_root()
720 BUG_ON((struct btrfs_root *)node->data != root); in __update_reloc_root()
722 spin_lock(&rc->reloc_root_tree.lock); in __update_reloc_root()
723 node->bytenr = root->node->start; in __update_reloc_root()
724 rb_node = rb_simple_insert(&rc->reloc_root_tree.rb_root, in __update_reloc_root()
725 node->bytenr, &node->rb_node); in __update_reloc_root()
726 spin_unlock(&rc->reloc_root_tree.lock); in __update_reloc_root()
728 btrfs_backref_panic(fs_info, node->bytenr, -EEXIST); in __update_reloc_root()
735 struct btrfs_fs_info *fs_info = root->fs_info; in create_reloc_root()
745 return ERR_PTR(-ENOMEM); in create_reloc_root()
749 root_key.offset = objectid; in create_reloc_root()
751 if (root->root_key.objectid == objectid) { in create_reloc_root()
755 ret = btrfs_copy_root(trans, root, root->commit_root, &eb, in create_reloc_root()
762 * root - like this ctree.c:btrfs_block_can_be_shared() behaves in create_reloc_root()
768 commit_root_gen = btrfs_header_generation(root->commit_root); in create_reloc_root()
769 btrfs_set_root_last_snapshot(&root->root_item, commit_root_gen); in create_reloc_root()
778 ret = btrfs_copy_root(trans, root, root->node, &eb, in create_reloc_root()
790 memcpy(root_item, &root->root_item, sizeof(*root_item)); in create_reloc_root()
791 btrfs_set_root_bytenr(root_item, eb->start); in create_reloc_root()
793 btrfs_set_root_generation(root_item, trans->transid); in create_reloc_root()
795 if (root->root_key.objectid == objectid) { in create_reloc_root()
797 memset(&root_item->drop_progress, 0, in create_reloc_root()
805 ret = btrfs_insert_root(trans, fs_info->tree_root, in create_reloc_root()
812 reloc_root = btrfs_read_tree_root(fs_info->tree_root, &root_key); in create_reloc_root()
817 set_bit(BTRFS_ROOT_SHAREABLE, &reloc_root->state); in create_reloc_root()
818 reloc_root->last_trans = trans->transid; in create_reloc_root()
833 * root->reloc_root, and another for being on the rc->reloc_roots list.
838 struct btrfs_fs_info *fs_info = root->fs_info; in btrfs_init_reloc_root()
840 struct reloc_control *rc = fs_info->reloc_ctl; in btrfs_init_reloc_root()
863 if (root->reloc_root) { in btrfs_init_reloc_root()
864 reloc_root = root->reloc_root; in btrfs_init_reloc_root()
865 reloc_root->last_trans = trans->transid; in btrfs_init_reloc_root()
873 if (!rc->create_reloc_tree || in btrfs_init_reloc_root()
874 root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) in btrfs_init_reloc_root()
877 if (!trans->reloc_reserved) { in btrfs_init_reloc_root()
878 rsv = trans->block_rsv; in btrfs_init_reloc_root()
879 trans->block_rsv = rc->block_rsv; in btrfs_init_reloc_root()
882 reloc_root = create_reloc_root(trans, root, root->root_key.objectid); in btrfs_init_reloc_root()
884 trans->block_rsv = rsv; in btrfs_init_reloc_root()
889 ASSERT(ret != -EEXIST); in btrfs_init_reloc_root()
895 root->reloc_root = btrfs_grab_root(reloc_root); in btrfs_init_reloc_root()
905 struct btrfs_fs_info *fs_info = root->fs_info; in btrfs_update_reloc_root()
913 reloc_root = root->reloc_root; in btrfs_update_reloc_root()
914 root_item = &reloc_root->root_item; in btrfs_update_reloc_root()
918 * the root. We have the ref for root->reloc_root, but just in case in btrfs_update_reloc_root()
923 /* root->reloc_root will stay until current relocation finished */ in btrfs_update_reloc_root()
924 if (fs_info->reloc_ctl->merge_reloc_tree && in btrfs_update_reloc_root()
926 set_bit(BTRFS_ROOT_DEAD_RELOC_TREE, &root->state); in btrfs_update_reloc_root()
935 if (reloc_root->commit_root != reloc_root->node) { in btrfs_update_reloc_root()
937 btrfs_set_root_node(root_item, reloc_root->node); in btrfs_update_reloc_root()
938 free_extent_buffer(reloc_root->commit_root); in btrfs_update_reloc_root()
939 reloc_root->commit_root = btrfs_root_node(reloc_root); in btrfs_update_reloc_root()
942 ret = btrfs_update_root(trans, fs_info->tree_root, in btrfs_update_reloc_root()
943 &reloc_root->root_key, root_item); in btrfs_update_reloc_root()
959 spin_lock(&root->inode_lock); in find_next_inode()
961 node = root->inode_tree.rb_node; in find_next_inode()
968 node = node->rb_left; in find_next_inode()
970 node = node->rb_right; in find_next_inode()
986 inode = igrab(&entry->vfs_inode); in find_next_inode()
988 spin_unlock(&root->inode_lock); in find_next_inode()
993 if (cond_resched_lock(&root->inode_lock)) in find_next_inode()
998 spin_unlock(&root->inode_lock); in find_next_inode()
1008 struct btrfs_root *root = BTRFS_I(reloc_inode)->root; in get_new_location()
1016 return -ENOMEM; in get_new_location()
1018 bytenr -= BTRFS_I(reloc_inode)->index_cnt; in get_new_location()
1024 ret = -ENOENT; in get_new_location()
1028 leaf = path->nodes[0]; in get_new_location()
1029 fi = btrfs_item_ptr(leaf, path->slots[0], in get_new_location()
1038 ret = -EINVAL; in get_new_location()
1059 struct btrfs_fs_info *fs_info = root->fs_info; in replace_file_extents()
1074 if (rc->stage != UPDATE_DATA_PTRS) in replace_file_extents()
1078 if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) in replace_file_extents()
1079 parent = leaf->start; in replace_file_extents()
1099 if (!in_range(bytenr, rc->block_group->start, in replace_file_extents()
1100 rc->block_group->length)) in replace_file_extents()
1107 if (root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID) { in replace_file_extents()
1116 end = key.offset + in replace_file_extents()
1118 WARN_ON(!IS_ALIGNED(key.offset, in replace_file_extents()
1119 fs_info->sectorsize)); in replace_file_extents()
1120 WARN_ON(!IS_ALIGNED(end, fs_info->sectorsize)); in replace_file_extents()
1121 end--; in replace_file_extents()
1122 ret = try_lock_extent(&BTRFS_I(inode)->io_tree, in replace_file_extents()
1123 key.offset, end); in replace_file_extents()
1128 key.offset, end, true); in replace_file_extents()
1129 unlock_extent(&BTRFS_I(inode)->io_tree, in replace_file_extents()
1130 key.offset, end, NULL); in replace_file_extents()
1134 ret = get_new_location(rc->data_inode, &new_bytenr, in replace_file_extents()
1147 key.offset -= btrfs_file_extent_offset(leaf, fi); in replace_file_extents()
1151 key.objectid, key.offset, in replace_file_extents()
1152 root->root_key.objectid, false); in replace_file_extents()
1162 key.objectid, key.offset, in replace_file_extents()
1163 root->root_key.objectid, false); in replace_file_extents()
1184 btrfs_node_key(path->nodes[level], &key2, path->slots[level]); in memcmp_node_keys()
1203 struct btrfs_fs_info *fs_info = dest->fs_info; in replace_path()
1219 ASSERT(src->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID); in replace_path()
1220 ASSERT(dest->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID); in replace_path()
1222 last_snapshot = btrfs_root_last_snapshot(&src->root_item); in replace_path()
1224 slot = path->slots[lowest_level]; in replace_path()
1225 btrfs_node_key_to_cpu(path->nodes[lowest_level], &key, slot); in replace_path()
1247 next_key->objectid = (u64)-1; in replace_path()
1248 next_key->type = (u8)-1; in replace_path()
1249 next_key->offset = (u64)-1; in replace_path()
1261 slot--; in replace_path()
1267 blocksize = fs_info->nodesize; in replace_path()
1271 eb = path->nodes[level]; in replace_path()
1273 path->slots[level]); in replace_path()
1275 path->slots[level]); in replace_path()
1324 btrfs_node_key_to_cpu(path->nodes[level], &key, in replace_path()
1325 path->slots[level]); in replace_path()
1328 path->lowest_level = level; in replace_path()
1329 set_bit(BTRFS_ROOT_RESET_LOCKDEP_CLASS, &src->state); in replace_path()
1331 clear_bit(BTRFS_ROOT_RESET_LOCKDEP_CLASS, &src->state); in replace_path()
1332 path->lowest_level = 0; in replace_path()
1335 ret = -ENOENT; in replace_path()
1354 rc->block_group, parent, slot, in replace_path()
1355 path->nodes[level], path->slots[level], in replace_path()
1366 btrfs_set_node_blockptr(path->nodes[level], in replace_path()
1367 path->slots[level], old_bytenr); in replace_path()
1368 btrfs_set_node_ptr_generation(path->nodes[level], in replace_path()
1369 path->slots[level], old_ptr_gen); in replace_path()
1370 btrfs_mark_buffer_dirty(path->nodes[level]); in replace_path()
1373 blocksize, path->nodes[level]->start); in replace_path()
1374 btrfs_init_tree_ref(&ref, level - 1, src->root_key.objectid, in replace_path()
1383 btrfs_init_tree_ref(&ref, level - 1, dest->root_key.objectid, 0, in replace_path()
1392 blocksize, path->nodes[level]->start); in replace_path()
1393 btrfs_init_tree_ref(&ref, level - 1, src->root_key.objectid, in replace_path()
1403 btrfs_init_tree_ref(&ref, level - 1, dest->root_key.objectid, in replace_path()
1433 last_snapshot = btrfs_root_last_snapshot(&root->root_item); in walk_up_reloc_tree()
1436 free_extent_buffer(path->nodes[i]); in walk_up_reloc_tree()
1437 path->nodes[i] = NULL; in walk_up_reloc_tree()
1440 for (i = *level; i < BTRFS_MAX_LEVEL && path->nodes[i]; i++) { in walk_up_reloc_tree()
1441 eb = path->nodes[i]; in walk_up_reloc_tree()
1443 while (path->slots[i] + 1 < nritems) { in walk_up_reloc_tree()
1444 path->slots[i]++; in walk_up_reloc_tree()
1445 if (btrfs_node_ptr_generation(eb, path->slots[i]) <= in walk_up_reloc_tree()
1452 free_extent_buffer(path->nodes[i]); in walk_up_reloc_tree()
1453 path->nodes[i] = NULL; in walk_up_reloc_tree()
1471 last_snapshot = btrfs_root_last_snapshot(&root->root_item); in walk_down_reloc_tree()
1473 for (i = *level; i > 0; i--) { in walk_down_reloc_tree()
1474 eb = path->nodes[i]; in walk_down_reloc_tree()
1476 while (path->slots[i] < nritems) { in walk_down_reloc_tree()
1477 ptr_gen = btrfs_node_ptr_generation(eb, path->slots[i]); in walk_down_reloc_tree()
1480 path->slots[i]++; in walk_down_reloc_tree()
1482 if (path->slots[i] >= nritems) { in walk_down_reloc_tree()
1493 eb = btrfs_read_node_slot(eb, path->slots[i]); in walk_down_reloc_tree()
1496 BUG_ON(btrfs_header_level(eb) != i - 1); in walk_down_reloc_tree()
1497 path->nodes[i - 1] = eb; in walk_down_reloc_tree()
1498 path->slots[i - 1] = 0; in walk_down_reloc_tree()
1511 struct btrfs_fs_info *fs_info = root->fs_info; in invalidate_extent_cache()
1517 objectid = min_key->objectid; in invalidate_extent_cache()
1522 if (objectid > max_key->objectid) in invalidate_extent_cache()
1530 if (ino > max_key->objectid) { in invalidate_extent_cache()
1536 if (!S_ISREG(inode->i_mode)) in invalidate_extent_cache()
1539 if (unlikely(min_key->objectid == ino)) { in invalidate_extent_cache()
1540 if (min_key->type > BTRFS_EXTENT_DATA_KEY) in invalidate_extent_cache()
1542 if (min_key->type < BTRFS_EXTENT_DATA_KEY) in invalidate_extent_cache()
1545 start = min_key->offset; in invalidate_extent_cache()
1546 WARN_ON(!IS_ALIGNED(start, fs_info->sectorsize)); in invalidate_extent_cache()
1552 if (unlikely(max_key->objectid == ino)) { in invalidate_extent_cache()
1553 if (max_key->type < BTRFS_EXTENT_DATA_KEY) in invalidate_extent_cache()
1555 if (max_key->type > BTRFS_EXTENT_DATA_KEY) { in invalidate_extent_cache()
1556 end = (u64)-1; in invalidate_extent_cache()
1558 if (max_key->offset == 0) in invalidate_extent_cache()
1560 end = max_key->offset; in invalidate_extent_cache()
1561 WARN_ON(!IS_ALIGNED(end, fs_info->sectorsize)); in invalidate_extent_cache()
1562 end--; in invalidate_extent_cache()
1565 end = (u64)-1; in invalidate_extent_cache()
1569 lock_extent(&BTRFS_I(inode)->io_tree, start, end, NULL); in invalidate_extent_cache()
1571 unlock_extent(&BTRFS_I(inode)->io_tree, start, end, NULL); in invalidate_extent_cache()
1581 if (!path->nodes[level]) in find_next_key()
1583 if (path->slots[level] + 1 < in find_next_key()
1584 btrfs_header_nritems(path->nodes[level])) { in find_next_key()
1585 btrfs_node_key_to_cpu(path->nodes[level], key, in find_next_key()
1586 path->slots[level] + 1); in find_next_key()
1601 struct btrfs_root *reloc_root = root->reloc_root; in insert_dirty_subvol()
1606 ASSERT(root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID); in insert_dirty_subvol()
1609 reloc_root_item = &reloc_root->root_item; in insert_dirty_subvol()
1610 memset(&reloc_root_item->drop_progress, 0, in insert_dirty_subvol()
1611 sizeof(reloc_root_item->drop_progress)); in insert_dirty_subvol()
1618 if (list_empty(&root->reloc_dirty_list)) { in insert_dirty_subvol()
1620 list_add_tail(&root->reloc_dirty_list, &rc->dirty_subvol_roots); in insert_dirty_subvol()
1633 list_for_each_entry_safe(root, next, &rc->dirty_subvol_roots, in clean_dirty_subvols()
1635 if (root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID) { in clean_dirty_subvols()
1637 struct btrfs_root *reloc_root = root->reloc_root; in clean_dirty_subvols()
1639 list_del_init(&root->reloc_dirty_list); in clean_dirty_subvols()
1640 root->reloc_root = NULL; in clean_dirty_subvols()
1643 * root->reloc_root = NULL. Pairs with have_reloc_root. in clean_dirty_subvols()
1646 clear_bit(BTRFS_ROOT_DEAD_RELOC_TREE, &root->state); in clean_dirty_subvols()
1650 * ->reloc_root. If it fails however we must in clean_dirty_subvols()
1681 struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; in merge_reloc_root()
1698 return -ENOMEM; in merge_reloc_root()
1699 path->reada = READA_FORWARD; in merge_reloc_root()
1701 reloc_root = root->reloc_root; in merge_reloc_root()
1702 root_item = &reloc_root->root_item; in merge_reloc_root()
1704 if (btrfs_disk_key_objectid(&root_item->drop_progress) == 0) { in merge_reloc_root()
1706 atomic_inc(&reloc_root->node->refs); in merge_reloc_root()
1707 path->nodes[level] = reloc_root->node; in merge_reloc_root()
1708 path->slots[level] = 0; in merge_reloc_root()
1710 btrfs_disk_key_to_cpu(&key, &root_item->drop_progress); in merge_reloc_root()
1714 path->lowest_level = level; in merge_reloc_root()
1716 path->lowest_level = 0; in merge_reloc_root()
1722 btrfs_node_key_to_cpu(path->nodes[level], &next_key, in merge_reloc_root()
1723 path->slots[level]); in merge_reloc_root()
1738 min_reserved = fs_info->nodesize * reserve_level * 2; in merge_reloc_root()
1742 ret = btrfs_block_rsv_refill(fs_info, rc->block_rsv, in merge_reloc_root()
1764 reloc_root->last_trans = trans->transid; in merge_reloc_root()
1765 trans->block_rsv = rc->block_rsv; in merge_reloc_root()
1787 btrfs_node_key_to_cpu(path->nodes[level], &key, in merge_reloc_root()
1788 path->slots[level]); in merge_reloc_root()
1801 btrfs_node_key(path->nodes[level], &root_item->drop_progress, in merge_reloc_root()
1802 path->slots[level]); in merge_reloc_root()
1810 if (replaced && rc->stage == UPDATE_DATA_PTRS) in merge_reloc_root()
1837 if (replaced && rc->stage == UPDATE_DATA_PTRS) in merge_reloc_root()
1846 struct btrfs_root *root = rc->extent_root; in prepare_to_merge()
1847 struct btrfs_fs_info *fs_info = root->fs_info; in prepare_to_merge()
1854 mutex_lock(&fs_info->reloc_mutex); in prepare_to_merge()
1855 rc->merging_rsv_size += fs_info->nodesize * (BTRFS_MAX_LEVEL - 1) * 2; in prepare_to_merge()
1856 rc->merging_rsv_size += rc->nodes_relocated * 2; in prepare_to_merge()
1857 mutex_unlock(&fs_info->reloc_mutex); in prepare_to_merge()
1861 num_bytes = rc->merging_rsv_size; in prepare_to_merge()
1862 ret = btrfs_block_rsv_add(fs_info, rc->block_rsv, num_bytes, in prepare_to_merge()
1868 trans = btrfs_join_transaction(rc->extent_root); in prepare_to_merge()
1871 btrfs_block_rsv_release(fs_info, rc->block_rsv, in prepare_to_merge()
1877 if (num_bytes != rc->merging_rsv_size) { in prepare_to_merge()
1879 btrfs_block_rsv_release(fs_info, rc->block_rsv, in prepare_to_merge()
1885 rc->merge_reloc_tree = 1; in prepare_to_merge()
1887 while (!list_empty(&rc->reloc_roots)) { in prepare_to_merge()
1888 reloc_root = list_entry(rc->reloc_roots.next, in prepare_to_merge()
1890 list_del_init(&reloc_root->root_list); in prepare_to_merge()
1892 root = btrfs_get_fs_root(fs_info, reloc_root->root_key.offset, in prepare_to_merge()
1899 list_add(&reloc_root->root_list, &reloc_roots); in prepare_to_merge()
1905 ASSERT(root->reloc_root == reloc_root); in prepare_to_merge()
1912 btrfs_set_root_refs(&reloc_root->root_item, 1); in prepare_to_merge()
1919 list_add(&reloc_root->root_list, &reloc_roots); in prepare_to_merge()
1930 list_splice(&reloc_roots, &rc->reloc_roots); in prepare_to_merge()
1951 struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; in merge_reloc_roots()
1958 root = rc->extent_root; in merge_reloc_roots()
1966 mutex_lock(&fs_info->reloc_mutex); in merge_reloc_roots()
1967 list_splice_init(&rc->reloc_roots, &reloc_roots); in merge_reloc_roots()
1968 mutex_unlock(&fs_info->reloc_mutex); in merge_reloc_roots()
1975 root = btrfs_get_fs_root(fs_info, reloc_root->root_key.offset, in merge_reloc_roots()
1977 if (btrfs_root_refs(&reloc_root->root_item) > 0) { in merge_reloc_roots()
1991 if (root->reloc_root != reloc_root) { in merge_reloc_roots()
1998 ret = -EINVAL; in merge_reloc_roots()
2004 if (list_empty(&reloc_root->root_list)) in merge_reloc_roots()
2005 list_add_tail(&reloc_root->root_list, in merge_reloc_roots()
2011 if (root->reloc_root == reloc_root) { in merge_reloc_roots()
2012 root->reloc_root = NULL; in merge_reloc_roots()
2016 &root->state); in merge_reloc_roots()
2020 list_del_init(&reloc_root->root_list); in merge_reloc_roots()
2022 list_add_tail(&reloc_root->reloc_dirty_list, in merge_reloc_roots()
2023 &rc->dirty_subvol_roots); in merge_reloc_roots()
2037 mutex_lock(&fs_info->reloc_mutex); in merge_reloc_roots()
2038 list_splice_init(&rc->reloc_roots, &reloc_roots); in merge_reloc_roots()
2039 mutex_unlock(&fs_info->reloc_mutex); in merge_reloc_roots()
2046 * BUG_ON(!RB_EMPTY_ROOT(&rc->reloc_root_tree.rb_root)); in merge_reloc_roots()
2053 * do the reloc_dirty_list afterwards. Meanwhile the root->reloc_root in merge_reloc_roots()
2074 struct btrfs_fs_info *fs_info = reloc_root->fs_info; in record_reloc_root_in_trans()
2078 if (reloc_root->last_trans == trans->transid) in record_reloc_root_in_trans()
2081 root = btrfs_get_fs_root(fs_info, reloc_root->root_key.offset, false); in record_reloc_root_in_trans()
2095 if (root->reloc_root != reloc_root) { in record_reloc_root_in_trans()
2099 reloc_root->root_key.offset); in record_reloc_root_in_trans()
2101 return -EUCLEAN; in record_reloc_root_in_trans()
2124 root = next->root; in select_reloc_root()
2132 * non-SHAREABLE root then we have backrefs that resolve in select_reloc_root()
2140 btrfs_err(trans->fs_info, in select_reloc_root()
2142 node->bytenr); in select_reloc_root()
2143 return ERR_PTR(-EUCLEAN); in select_reloc_root()
2145 if (!test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) { in select_reloc_root()
2147 btrfs_err(trans->fs_info, in select_reloc_root()
2148 "bytenr %llu has multiple refs with one ending in a non-shareable root", in select_reloc_root()
2149 node->bytenr); in select_reloc_root()
2150 return ERR_PTR(-EUCLEAN); in select_reloc_root()
2153 if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) { in select_reloc_root()
2163 root = root->reloc_root; in select_reloc_root()
2167 * root->reloc_root may not be set, return ENOENT in this case. in select_reloc_root()
2170 return ERR_PTR(-ENOENT); in select_reloc_root()
2172 if (next->new_bytenr != root->node->start) { in select_reloc_root()
2175 * ->new_bytenr set and this shouldn't be in the changed in select_reloc_root()
2180 ASSERT(next->new_bytenr == 0); in select_reloc_root()
2181 ASSERT(list_empty(&next->list)); in select_reloc_root()
2182 if (next->new_bytenr || !list_empty(&next->list)) { in select_reloc_root()
2183 btrfs_err(trans->fs_info, in select_reloc_root()
2185 node->bytenr, next->bytenr); in select_reloc_root()
2186 return ERR_PTR(-EUCLEAN); in select_reloc_root()
2189 next->new_bytenr = root->node->start; in select_reloc_root()
2190 btrfs_put_root(next->root); in select_reloc_root()
2191 next->root = btrfs_grab_root(root); in select_reloc_root()
2192 ASSERT(next->root); in select_reloc_root()
2193 list_add_tail(&next->list, in select_reloc_root()
2194 &rc->backref_cache.changed); in select_reloc_root()
2202 if (!next || next->level <= node->level) in select_reloc_root()
2211 return ERR_PTR(-ENOENT); in select_reloc_root()
2217 rc->backref_cache.path[next->level] = next; in select_reloc_root()
2218 if (--index < 0) in select_reloc_root()
2220 next = edges[index]->node[UPPER]; in select_reloc_root()
2232 * Return -ENOENT if the block is root of reloc tree.
2240 struct btrfs_backref_edge *edges[BTRFS_MAX_LEVEL - 1]; in select_one_root()
2247 root = next->root; in select_one_root()
2251 * the way up a particular path, in this case return -EUCLEAN. in select_one_root()
2254 return ERR_PTR(-EUCLEAN); in select_one_root()
2256 /* No other choice for non-shareable tree */ in select_one_root()
2257 if (!test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) in select_one_root()
2260 if (root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID) in select_one_root()
2267 if (!next || next->level <= node->level) in select_one_root()
2272 return ERR_PTR(-ENOENT); in select_one_root()
2280 struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; in calcu_metadata_size()
2282 struct btrfs_backref_edge *edge; in calcu_metadata_size() local
2283 struct btrfs_backref_edge *edges[BTRFS_MAX_LEVEL - 1]; in calcu_metadata_size()
2287 BUG_ON(reserve && node->processed); in calcu_metadata_size()
2292 if (next->processed && (reserve || next != node)) in calcu_metadata_size()
2295 num_bytes += fs_info->nodesize; in calcu_metadata_size()
2297 if (list_empty(&next->upper)) in calcu_metadata_size()
2300 edge = list_entry(next->upper.next, in calcu_metadata_size()
2302 edges[index++] = edge; in calcu_metadata_size()
2303 next = edge->node[UPPER]; in calcu_metadata_size()
2314 struct btrfs_root *root = rc->extent_root; in reserve_metadata_space()
2315 struct btrfs_fs_info *fs_info = root->fs_info; in reserve_metadata_space()
2322 trans->block_rsv = rc->block_rsv; in reserve_metadata_space()
2323 rc->reserved_bytes += num_bytes; in reserve_metadata_space()
2327 * If we get an enospc just kick back -EAGAIN so we know to drop the in reserve_metadata_space()
2330 ret = btrfs_block_rsv_refill(fs_info, rc->block_rsv, num_bytes, in reserve_metadata_space()
2333 tmp = fs_info->nodesize * RELOCATION_RESERVED_NODES; in reserve_metadata_space()
2334 while (tmp <= rc->reserved_bytes) in reserve_metadata_space()
2343 rc->block_rsv->size = tmp + fs_info->nodesize * in reserve_metadata_space()
2345 return -EAGAIN; in reserve_metadata_space()
2365 struct btrfs_backref_edge *edge; in do_relocation() local
2366 struct btrfs_backref_edge *edges[BTRFS_MAX_LEVEL - 1]; in do_relocation()
2378 ASSERT(!lowest || !node->eb); in do_relocation()
2380 path->lowest_level = node->level + 1; in do_relocation()
2381 rc->backref_cache.path[node->level] = node; in do_relocation()
2382 list_for_each_entry(edge, &node->upper, list[LOWER]) { in do_relocation()
2387 upper = edge->node[UPPER]; in do_relocation()
2394 if (upper->eb && !upper->locked) { in do_relocation()
2396 ret = btrfs_bin_search(upper->eb, key, &slot); in do_relocation()
2400 bytenr = btrfs_node_blockptr(upper->eb, slot); in do_relocation()
2401 if (node->eb->start == bytenr) in do_relocation()
2407 if (!upper->eb) { in do_relocation()
2411 ret = -ENOENT; in do_relocation()
2417 if (!upper->eb) { in do_relocation()
2418 upper->eb = path->nodes[upper->level]; in do_relocation()
2419 path->nodes[upper->level] = NULL; in do_relocation()
2421 BUG_ON(upper->eb != path->nodes[upper->level]); in do_relocation()
2424 upper->locked = 1; in do_relocation()
2425 path->locks[upper->level] = 0; in do_relocation()
2427 slot = path->slots[upper->level]; in do_relocation()
2430 ret = btrfs_bin_search(upper->eb, key, &slot); in do_relocation()
2436 bytenr = btrfs_node_blockptr(upper->eb, slot); in do_relocation()
2438 if (bytenr != node->bytenr) { in do_relocation()
2439 btrfs_err(root->fs_info, in do_relocation()
2440 "lowest leaf/node mismatch: bytenr %llu node->bytenr %llu slot %d upper %llu", in do_relocation()
2441 bytenr, node->bytenr, slot, in do_relocation()
2442 upper->eb->start); in do_relocation()
2443 ret = -EIO; in do_relocation()
2447 if (node->eb->start == bytenr) in do_relocation()
2451 blocksize = root->fs_info->nodesize; in do_relocation()
2452 eb = btrfs_read_node_slot(upper->eb, slot); in do_relocation()
2459 if (!node->eb) { in do_relocation()
2460 ret = btrfs_cow_block(trans, root, eb, upper->eb, in do_relocation()
2470 ASSERT(node->eb == eb); in do_relocation()
2472 btrfs_set_node_blockptr(upper->eb, slot, in do_relocation()
2473 node->eb->start); in do_relocation()
2474 btrfs_set_node_ptr_generation(upper->eb, slot, in do_relocation()
2475 trans->transid); in do_relocation()
2476 btrfs_mark_buffer_dirty(upper->eb); in do_relocation()
2479 node->eb->start, blocksize, in do_relocation()
2480 upper->eb->start); in do_relocation()
2481 btrfs_init_tree_ref(&ref, node->level, in do_relocation()
2482 btrfs_header_owner(upper->eb), in do_relocation()
2483 root->root_key.objectid, false); in do_relocation()
2487 upper->eb); in do_relocation()
2492 if (!upper->pending) in do_relocation()
2500 if (!ret && node->pending) { in do_relocation()
2502 list_move_tail(&node->list, &rc->backref_cache.changed); in do_relocation()
2503 node->pending = 0; in do_relocation()
2506 path->lowest_level = 0; in do_relocation()
2512 ASSERT(ret != -ENOSPC); in do_relocation()
2523 btrfs_node_key_to_cpu(node->eb, &key, 0); in link_to_upper()
2532 struct btrfs_backref_cache *cache = &rc->backref_cache; in finish_pending_nodes()
2538 while (!list_empty(&cache->pending[level])) { in finish_pending_nodes()
2539 node = list_entry(cache->pending[level].next, in finish_pending_nodes()
2541 list_move_tail(&node->list, &list); in finish_pending_nodes()
2542 BUG_ON(!node->pending); in finish_pending_nodes()
2550 list_splice_init(&list, &cache->pending[level]); in finish_pending_nodes()
2563 struct btrfs_backref_edge *edge; in update_processed_blocks() local
2564 struct btrfs_backref_edge *edges[BTRFS_MAX_LEVEL - 1]; in update_processed_blocks()
2570 if (next->processed) in update_processed_blocks()
2575 if (list_empty(&next->upper)) in update_processed_blocks()
2578 edge = list_entry(next->upper.next, in update_processed_blocks()
2580 edges[index++] = edge; in update_processed_blocks()
2581 next = edge->node[UPPER]; in update_processed_blocks()
2589 u32 blocksize = rc->extent_root->fs_info->nodesize; in tree_block_processed()
2591 if (test_range_bit(&rc->processed_blocks, bytenr, in tree_block_processed()
2592 bytenr + blocksize - 1, EXTENT_DIRTY, 1, NULL)) in tree_block_processed()
2602 eb = read_tree_block(fs_info, block->bytenr, block->owner, in get_tree_block_key()
2603 block->key.offset, block->level, NULL); in get_tree_block_key()
2608 return -EIO; in get_tree_block_key()
2610 if (block->level == 0) in get_tree_block_key()
2611 btrfs_item_key_to_cpu(eb, &block->key, 0); in get_tree_block_key()
2613 btrfs_node_key_to_cpu(eb, &block->key, 0); in get_tree_block_key()
2615 block->key_ready = 1; in get_tree_block_key()
2642 BUG_ON(node->processed); in relocate_tree_block()
2647 /* See explanation in select_one_root for the -EUCLEAN case. */ in relocate_tree_block()
2648 ASSERT(ret == -ENOENT); in relocate_tree_block()
2649 if (ret == -ENOENT) { in relocate_tree_block()
2657 if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) { in relocate_tree_block()
2661 * should not have had the ->new_bytenr modified and in relocate_tree_block()
2671 ASSERT(node->new_bytenr == 0); in relocate_tree_block()
2672 ASSERT(list_empty(&node->list)); in relocate_tree_block()
2673 if (node->new_bytenr || !list_empty(&node->list)) { in relocate_tree_block()
2674 btrfs_err(root->fs_info, in relocate_tree_block()
2676 node->bytenr); in relocate_tree_block()
2677 ret = -EUCLEAN; in relocate_tree_block()
2687 if (!root->reloc_root) { in relocate_tree_block()
2688 ret = -ENOENT; in relocate_tree_block()
2691 root = root->reloc_root; in relocate_tree_block()
2692 node->new_bytenr = root->node->start; in relocate_tree_block()
2693 btrfs_put_root(node->root); in relocate_tree_block()
2694 node->root = btrfs_grab_root(root); in relocate_tree_block()
2695 ASSERT(node->root); in relocate_tree_block()
2696 list_add_tail(&node->list, &rc->backref_cache.changed); in relocate_tree_block()
2698 path->lowest_level = node->level; in relocate_tree_block()
2699 if (root == root->fs_info->chunk_root) in relocate_tree_block()
2703 if (root == root->fs_info->chunk_root) in relocate_tree_block()
2714 if (ret || node->level == 0 || node->cowonly) in relocate_tree_block()
2715 btrfs_backref_cleanup_node(&rc->backref_cache, node); in relocate_tree_block()
2726 struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; in relocate_tree_blocks()
2736 err = -ENOMEM; in relocate_tree_blocks()
2742 if (!block->key_ready) in relocate_tree_blocks()
2743 btrfs_readahead_tree_block(fs_info, block->bytenr, in relocate_tree_blocks()
2744 block->owner, 0, in relocate_tree_blocks()
2745 block->level); in relocate_tree_blocks()
2750 if (!block->key_ready) { in relocate_tree_blocks()
2759 node = build_backref_tree(rc, &block->key, in relocate_tree_blocks()
2760 block->level, block->bytenr); in relocate_tree_blocks()
2766 ret = relocate_tree_block(trans, rc, node, &block->key, in relocate_tree_blocks()
2790 u64 offset = inode->index_cnt; in prealloc_file_extent_cluster() local
2794 u64 i_size = i_size_read(&inode->vfs_inode); in prealloc_file_extent_cluster()
2795 u64 prealloc_start = cluster->start - offset; in prealloc_file_extent_cluster()
2796 u64 prealloc_end = cluster->end - offset; in prealloc_file_extent_cluster()
2811 struct address_space *mapping = inode->vfs_inode.i_mapping; in prealloc_file_extent_cluster()
2812 struct btrfs_fs_info *fs_info = inode->root->fs_info; in prealloc_file_extent_cluster()
2813 const u32 sectorsize = fs_info->sectorsize; in prealloc_file_extent_cluster()
2825 * |- btrfs_lock_and_flush_ordered_range() in prealloc_file_extent_cluster()
2826 * |- btrfs_start_ordered_extent() in prealloc_file_extent_cluster()
2827 * |- extent_write_cache_pages() in prealloc_file_extent_cluster()
2828 * |- lock_page() in prealloc_file_extent_cluster()
2842 clear_extent_bits(&inode->io_tree, i_size, in prealloc_file_extent_cluster()
2843 round_up(i_size, PAGE_SIZE) - 1, in prealloc_file_extent_cluster()
2848 * will re-read the whole page anyway. in prealloc_file_extent_cluster()
2852 round_up(i_size, PAGE_SIZE) - i_size); in prealloc_file_extent_cluster()
2858 BUG_ON(cluster->start != cluster->boundary[0]); in prealloc_file_extent_cluster()
2860 prealloc_end + 1 - prealloc_start); in prealloc_file_extent_cluster()
2864 btrfs_inode_lock(&inode->vfs_inode, 0); in prealloc_file_extent_cluster()
2865 for (nr = 0; nr < cluster->nr; nr++) { in prealloc_file_extent_cluster()
2866 start = cluster->boundary[nr] - offset; in prealloc_file_extent_cluster()
2867 if (nr + 1 < cluster->nr) in prealloc_file_extent_cluster()
2868 end = cluster->boundary[nr + 1] - 1 - offset; in prealloc_file_extent_cluster()
2870 end = cluster->end - offset; in prealloc_file_extent_cluster()
2872 lock_extent(&inode->io_tree, start, end, NULL); in prealloc_file_extent_cluster()
2873 num_bytes = end + 1 - start; in prealloc_file_extent_cluster()
2874 ret = btrfs_prealloc_file_range(&inode->vfs_inode, 0, start, in prealloc_file_extent_cluster()
2878 unlock_extent(&inode->io_tree, start, end, NULL); in prealloc_file_extent_cluster()
2882 btrfs_inode_unlock(&inode->vfs_inode, 0); in prealloc_file_extent_cluster()
2885 btrfs_free_reserved_data_space_noquota(inode->root->fs_info, in prealloc_file_extent_cluster()
2886 prealloc_end + 1 - cur_offset); in prealloc_file_extent_cluster()
2898 return -ENOMEM; in setup_relocation_extent_mapping()
2900 em->start = start; in setup_relocation_extent_mapping()
2901 em->len = end + 1 - start; in setup_relocation_extent_mapping()
2902 em->block_len = em->len; in setup_relocation_extent_mapping()
2903 em->block_start = block_start; in setup_relocation_extent_mapping()
2904 set_bit(EXTENT_FLAG_PINNED, &em->flags); in setup_relocation_extent_mapping()
2906 lock_extent(&BTRFS_I(inode)->io_tree, start, end, NULL); in setup_relocation_extent_mapping()
2908 unlock_extent(&BTRFS_I(inode)->io_tree, start, end, NULL); in setup_relocation_extent_mapping()
2919 return atomic_read(&fs_info->balance_cancel_req) || in btrfs_should_cancel_balance()
2920 atomic_read(&fs_info->reloc_cancel_req) || in btrfs_should_cancel_balance()
2929 if (cluster_nr >= cluster->nr - 1) in get_cluster_boundary_end()
2930 return cluster->end; in get_cluster_boundary_end()
2933 return cluster->boundary[cluster_nr + 1] - 1; in get_cluster_boundary_end()
2940 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); in relocate_one_page()
2941 u64 offset = BTRFS_I(inode)->index_cnt; in relocate_one_page() local
2942 const unsigned long last_index = (cluster->end - offset) >> PAGE_SHIFT; in relocate_one_page()
2943 gfp_t mask = btrfs_alloc_write_mask(inode->i_mapping); in relocate_one_page()
2951 page = find_lock_page(inode->i_mapping, page_index); in relocate_one_page()
2953 page_cache_sync_readahead(inode->i_mapping, ra, NULL, in relocate_one_page()
2954 page_index, last_index + 1 - page_index); in relocate_one_page()
2955 page = find_or_create_page(inode->i_mapping, page_index, mask); in relocate_one_page()
2957 return -ENOMEM; in relocate_one_page()
2964 page_cache_async_readahead(inode->i_mapping, ra, NULL, in relocate_one_page()
2966 last_index + 1 - page_index); in relocate_one_page()
2972 ret = -EIO; in relocate_one_page()
2978 page_end = page_start + PAGE_SIZE - 1; in relocate_one_page()
2984 cur = max(page_start, cluster->boundary[*cluster_nr] - offset); in relocate_one_page()
2986 u64 extent_start = cluster->boundary[*cluster_nr] - offset; in relocate_one_page()
2988 *cluster_nr) - offset; in relocate_one_page()
2991 u32 clamped_len = clamped_end + 1 - clamped_start; in relocate_one_page()
3001 lock_extent(&BTRFS_I(inode)->io_tree, clamped_start, clamped_end, NULL); in relocate_one_page()
3005 clear_extent_bits(&BTRFS_I(inode)->io_tree, in relocate_one_page()
3023 if (in_range(cluster->boundary[*cluster_nr] - offset, in relocate_one_page()
3025 u64 boundary_start = cluster->boundary[*cluster_nr] - in relocate_one_page()
3026 offset; in relocate_one_page()
3028 fs_info->sectorsize - 1; in relocate_one_page()
3030 set_extent_bits(&BTRFS_I(inode)->io_tree, in relocate_one_page()
3034 unlock_extent(&BTRFS_I(inode)->io_tree, clamped_start, clamped_end, NULL); in relocate_one_page()
3042 if (*cluster_nr >= cluster->nr) in relocate_one_page()
3049 balance_dirty_pages_ratelimited(inode->i_mapping); in relocate_one_page()
3052 ret = -ECANCELED; in relocate_one_page()
3064 u64 offset = BTRFS_I(inode)->index_cnt; in relocate_file_extent_cluster() local
3071 if (!cluster->nr) in relocate_file_extent_cluster()
3076 return -ENOMEM; in relocate_file_extent_cluster()
3082 file_ra_state_init(ra, inode->i_mapping); in relocate_file_extent_cluster()
3084 ret = setup_relocation_extent_mapping(inode, cluster->start - offset, in relocate_file_extent_cluster()
3085 cluster->end - offset, cluster->start); in relocate_file_extent_cluster()
3089 last_index = (cluster->end - offset) >> PAGE_SHIFT; in relocate_file_extent_cluster()
3090 for (index = (cluster->start - offset) >> PAGE_SHIFT; in relocate_file_extent_cluster()
3094 WARN_ON(cluster_nr != cluster->nr); in relocate_file_extent_cluster()
3106 if (cluster->nr > 0 && extent_key->objectid != cluster->end + 1) { in relocate_data_extent()
3110 cluster->nr = 0; in relocate_data_extent()
3113 if (!cluster->nr) in relocate_data_extent()
3114 cluster->start = extent_key->objectid; in relocate_data_extent()
3116 BUG_ON(cluster->nr >= MAX_EXTENTS); in relocate_data_extent()
3117 cluster->end = extent_key->objectid + extent_key->offset - 1; in relocate_data_extent()
3118 cluster->boundary[cluster->nr] = extent_key->objectid; in relocate_data_extent()
3119 cluster->nr++; in relocate_data_extent()
3121 if (cluster->nr >= MAX_EXTENTS) { in relocate_data_extent()
3125 cluster->nr = 0; in relocate_data_extent()
3145 int level = -1; in add_tree_block()
3149 eb = path->nodes[0]; in add_tree_block()
3150 item_size = btrfs_item_size(eb, path->slots[0]); in add_tree_block()
3152 if (extent_key->type == BTRFS_METADATA_ITEM_KEY || in add_tree_block()
3156 ei = btrfs_item_ptr(eb, path->slots[0], in add_tree_block()
3159 if (extent_key->type == BTRFS_EXTENT_ITEM_KEY) { in add_tree_block()
3164 level = (int)extent_key->offset; in add_tree_block()
3174 * inline ref offset. We know it's an fs root if in add_tree_block()
3180 * Otherwise it's safe to assume that the ref offset == the in add_tree_block()
3195 return -EINVAL; in add_tree_block()
3200 btrfs_print_v0_err(eb->fs_info); in add_tree_block()
3201 btrfs_handle_fs_error(eb->fs_info, -EINVAL, NULL); in add_tree_block()
3202 return -EINVAL; in add_tree_block()
3209 BUG_ON(level == -1); in add_tree_block()
3213 return -ENOMEM; in add_tree_block()
3215 block->bytenr = extent_key->objectid; in add_tree_block()
3216 block->key.objectid = rc->extent_root->fs_info->nodesize; in add_tree_block()
3217 block->key.offset = generation; in add_tree_block()
3218 block->level = level; in add_tree_block()
3219 block->key_ready = 0; in add_tree_block()
3220 block->owner = owner; in add_tree_block()
3222 rb_node = rb_simple_insert(blocks, block->bytenr, &block->rb_node); in add_tree_block()
3224 btrfs_backref_panic(rc->extent_root->fs_info, block->bytenr, in add_tree_block()
3225 -EEXIST); in add_tree_block()
3237 struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; in __add_tree_block()
3251 return -ENOMEM; in __add_tree_block()
3256 key.offset = (u64)-1; in __add_tree_block()
3259 key.offset = blocksize; in __add_tree_block()
3262 path->search_commit_root = 1; in __add_tree_block()
3263 path->skip_locking = 1; in __add_tree_block()
3264 ret = btrfs_search_slot(NULL, rc->extent_root, &key, path, 0, 0); in __add_tree_block()
3269 if (path->slots[0]) { in __add_tree_block()
3270 path->slots[0]--; in __add_tree_block()
3271 btrfs_item_key_to_cpu(path->nodes[0], &key, in __add_tree_block()
3272 path->slots[0]); in __add_tree_block()
3276 key.offset == blocksize))) in __add_tree_block()
3288 btrfs_print_leaf(path->nodes[0]); in __add_tree_block()
3293 ret = -EINVAL; in __add_tree_block()
3308 struct btrfs_root *root = fs_info->tree_root; in delete_block_group_cache()
3315 inode = btrfs_iget(fs_info->sb, ino, root); in delete_block_group_cache()
3317 return -ENOENT; in delete_block_group_cache()
3321 &fs_info->global_block_rsv); in delete_block_group_cache()
3376 return -ENOENT; in delete_v1_space_cache()
3377 ret = delete_block_group_cache(leaf->fs_info, block_group, NULL, in delete_v1_space_cache()
3391 struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; in add_data_references()
3395 const u32 blocksize = fs_info->nodesize; in add_data_references()
3399 ret = btrfs_find_all_leafs(NULL, fs_info, extent_key->objectid, in add_data_references()
3408 eb = read_tree_block(fs_info, ref_node->val, 0, 0, 0, NULL); in add_data_references()
3413 ret = delete_v1_space_cache(eb, rc->block_group, in add_data_references()
3414 extent_key->objectid); in add_data_references()
3418 ret = __add_tree_block(rc, ref_node->val, blocksize, blocks); in add_data_references()
3435 struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; in find_next_extent()
3441 last = rc->block_group->start + rc->block_group->length; in find_next_extent()
3444 if (rc->search_start >= last) { in find_next_extent()
3449 key.objectid = rc->search_start; in find_next_extent()
3451 key.offset = 0; in find_next_extent()
3453 path->search_commit_root = 1; in find_next_extent()
3454 path->skip_locking = 1; in find_next_extent()
3455 ret = btrfs_search_slot(NULL, rc->extent_root, &key, path, in find_next_extent()
3460 leaf = path->nodes[0]; in find_next_extent()
3461 if (path->slots[0] >= btrfs_header_nritems(leaf)) { in find_next_extent()
3462 ret = btrfs_next_leaf(rc->extent_root, path); in find_next_extent()
3465 leaf = path->nodes[0]; in find_next_extent()
3468 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); in find_next_extent()
3476 path->slots[0]++; in find_next_extent()
3481 key.objectid + key.offset <= rc->search_start) { in find_next_extent()
3482 path->slots[0]++; in find_next_extent()
3487 key.objectid + fs_info->nodesize <= in find_next_extent()
3488 rc->search_start) { in find_next_extent()
3489 path->slots[0]++; in find_next_extent()
3493 ret = find_first_extent_bit(&rc->processed_blocks, in find_next_extent()
3499 rc->search_start = end + 1; in find_next_extent()
3502 rc->search_start = key.objectid + key.offset; in find_next_extent()
3504 rc->search_start = key.objectid + in find_next_extent()
3505 fs_info->nodesize; in find_next_extent()
3516 struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; in set_reloc_control()
3518 mutex_lock(&fs_info->reloc_mutex); in set_reloc_control()
3519 fs_info->reloc_ctl = rc; in set_reloc_control()
3520 mutex_unlock(&fs_info->reloc_mutex); in set_reloc_control()
3525 struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; in unset_reloc_control()
3527 mutex_lock(&fs_info->reloc_mutex); in unset_reloc_control()
3528 fs_info->reloc_ctl = NULL; in unset_reloc_control()
3529 mutex_unlock(&fs_info->reloc_mutex); in unset_reloc_control()
3538 rc->block_rsv = btrfs_alloc_block_rsv(rc->extent_root->fs_info, in prepare_to_relocate()
3540 if (!rc->block_rsv) in prepare_to_relocate()
3541 return -ENOMEM; in prepare_to_relocate()
3543 memset(&rc->cluster, 0, sizeof(rc->cluster)); in prepare_to_relocate()
3544 rc->search_start = rc->block_group->start; in prepare_to_relocate()
3545 rc->extents_found = 0; in prepare_to_relocate()
3546 rc->nodes_relocated = 0; in prepare_to_relocate()
3547 rc->merging_rsv_size = 0; in prepare_to_relocate()
3548 rc->reserved_bytes = 0; in prepare_to_relocate()
3549 rc->block_rsv->size = rc->extent_root->fs_info->nodesize * in prepare_to_relocate()
3551 ret = btrfs_block_rsv_refill(rc->extent_root->fs_info, in prepare_to_relocate()
3552 rc->block_rsv, rc->block_rsv->size, in prepare_to_relocate()
3557 rc->create_reloc_tree = 1; in prepare_to_relocate()
3560 trans = btrfs_join_transaction(rc->extent_root); in prepare_to_relocate()
3580 struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; in relocate_block_group()
3593 return -ENOMEM; in relocate_block_group()
3594 path->reada = READA_FORWARD; in relocate_block_group()
3603 rc->reserved_bytes = 0; in relocate_block_group()
3604 ret = btrfs_block_rsv_refill(fs_info, rc->block_rsv, in relocate_block_group()
3605 rc->block_rsv->size, in relocate_block_group()
3612 trans = btrfs_start_transaction(rc->extent_root, 0); in relocate_block_group()
3619 if (update_backref_cache(trans, &rc->backref_cache)) { in relocate_block_group()
3631 rc->extents_found++; in relocate_block_group()
3633 ei = btrfs_item_ptr(path->nodes[0], path->slots[0], in relocate_block_group()
3635 flags = btrfs_extent_flags(path->nodes[0], ei); in relocate_block_group()
3639 } else if (rc->stage == UPDATE_DATA_PTRS && in relocate_block_group()
3654 if (ret != -EAGAIN) { in relocate_block_group()
3658 rc->extents_found--; in relocate_block_group()
3659 rc->search_start = key.objectid; in relocate_block_group()
3667 if (rc->stage == MOVE_DATA_EXTENTS && in relocate_block_group()
3669 rc->found_file_extent = 1; in relocate_block_group()
3670 ret = relocate_data_extent(rc->data_inode, in relocate_block_group()
3671 &key, &rc->cluster); in relocate_block_group()
3678 err = -ECANCELED; in relocate_block_group()
3682 if (trans && progress && err == -ENOSPC) { in relocate_block_group()
3683 ret = btrfs_force_chunk_alloc(trans, rc->block_group->flags); in relocate_block_group()
3692 clear_extent_bits(&rc->processed_blocks, 0, (u64)-1, EXTENT_DIRTY); in relocate_block_group()
3700 ret = relocate_file_extent_cluster(rc->data_inode, in relocate_block_group()
3701 &rc->cluster); in relocate_block_group()
3706 rc->create_reloc_tree = 0; in relocate_block_group()
3709 btrfs_backref_release_cache(&rc->backref_cache); in relocate_block_group()
3710 btrfs_block_rsv_release(fs_info, rc->block_rsv, (u64)-1, NULL); in relocate_block_group()
3724 rc->merge_reloc_tree = 0; in relocate_block_group()
3726 btrfs_block_rsv_release(fs_info, rc->block_rsv, (u64)-1, NULL); in relocate_block_group()
3729 trans = btrfs_join_transaction(rc->extent_root); in relocate_block_group()
3741 btrfs_free_block_rsv(fs_info, rc->block_rsv); in relocate_block_group()
3756 return -ENOMEM; in __insert_orphan_inode()
3762 leaf = path->nodes[0]; in __insert_orphan_inode()
3763 item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_inode_item); in __insert_orphan_inode()
3785 ret = -ENOMEM; in delete_orphan_inode()
3791 key.offset = 0; in delete_orphan_inode()
3792 ret = btrfs_search_slot(trans, root, &key, path, -1, 1); in delete_orphan_inode()
3795 ret = -ENOENT; in delete_orphan_inode()
3819 root = btrfs_grab_root(fs_info->data_reloc_root); in create_reloc_inode()
3834 inode = btrfs_iget(fs_info->sb, objectid, root); in create_reloc_inode()
3841 BTRFS_I(inode)->index_cnt = group->start; in create_reloc_inode()
3861 * -EINPROGRESS operation is already in progress, that's probably a bug
3862 * -ECANCELED cancellation request was set before the operation started
3866 if (test_and_set_bit(BTRFS_FS_RELOC_RUNNING, &fs_info->flags)) { in reloc_chunk_start()
3869 return -EINPROGRESS; in reloc_chunk_start()
3872 if (atomic_read(&fs_info->reloc_cancel_req) > 0) { in reloc_chunk_start()
3878 atomic_set(&fs_info->reloc_cancel_req, 0); in reloc_chunk_start()
3879 return -ECANCELED; in reloc_chunk_start()
3890 if (atomic_read(&fs_info->reloc_cancel_req) > 0) in reloc_chunk_end()
3892 clear_and_wake_up_bit(BTRFS_FS_RELOC_RUNNING, &fs_info->flags); in reloc_chunk_end()
3893 atomic_set(&fs_info->reloc_cancel_req, 0); in reloc_chunk_end()
3904 INIT_LIST_HEAD(&rc->reloc_roots); in alloc_reloc_control()
3905 INIT_LIST_HEAD(&rc->dirty_subvol_roots); in alloc_reloc_control()
3906 btrfs_backref_init_cache(fs_info, &rc->backref_cache, 1); in alloc_reloc_control()
3907 mapping_tree_init(&rc->reloc_root_tree); in alloc_reloc_control()
3908 extent_io_tree_init(fs_info, &rc->processed_blocks, in alloc_reloc_control()
3917 free_reloc_roots(&rc->reloc_roots); in free_reloc_control()
3919 &rc->reloc_root_tree.rb_root, rb_node) in free_reloc_control()
3933 btrfs_describe_block_groups(block_group->flags, buf, sizeof(buf)); in describe_relocation()
3937 block_group->start, buf); in describe_relocation()
3964 * This only gets set if we had a half-deleted snapshot on mount. We in btrfs_relocate_block_group()
3968 ret = wait_on_bit(&fs_info->flags, BTRFS_FS_UNFINISHED_DROPS, TASK_INTERRUPTIBLE); in btrfs_relocate_block_group()
3974 return -EINTR; in btrfs_relocate_block_group()
3978 return -ENOENT; in btrfs_relocate_block_group()
3988 if (bg->flags & BTRFS_BLOCK_GROUP_DATA) in btrfs_relocate_block_group()
3989 ASSERT(sb_write_started(fs_info->sb)); in btrfs_relocate_block_group()
3993 return -ETXTBSY; in btrfs_relocate_block_group()
3999 return -ENOMEM; in btrfs_relocate_block_group()
4008 rc->extent_root = extent_root; in btrfs_relocate_block_group()
4009 rc->block_group = bg; in btrfs_relocate_block_group()
4011 ret = btrfs_inc_block_group_ro(rc->block_group, true); in btrfs_relocate_block_group()
4020 err = -ENOMEM; in btrfs_relocate_block_group()
4024 inode = lookup_free_space_inode(rc->block_group, path); in btrfs_relocate_block_group()
4028 ret = delete_block_group_cache(fs_info, rc->block_group, inode, 0); in btrfs_relocate_block_group()
4032 if (ret && ret != -ENOENT) { in btrfs_relocate_block_group()
4037 rc->data_inode = create_reloc_inode(fs_info, rc->block_group); in btrfs_relocate_block_group()
4038 if (IS_ERR(rc->data_inode)) { in btrfs_relocate_block_group()
4039 err = PTR_ERR(rc->data_inode); in btrfs_relocate_block_group()
4040 rc->data_inode = NULL; in btrfs_relocate_block_group()
4044 describe_relocation(fs_info, rc->block_group); in btrfs_relocate_block_group()
4046 btrfs_wait_block_group_reservations(rc->block_group); in btrfs_relocate_block_group()
4047 btrfs_wait_nocow_writers(rc->block_group); in btrfs_relocate_block_group()
4049 rc->block_group->start, in btrfs_relocate_block_group()
4050 rc->block_group->length); in btrfs_relocate_block_group()
4052 ret = btrfs_zone_finish(rc->block_group); in btrfs_relocate_block_group()
4053 WARN_ON(ret && ret != -EAGAIN); in btrfs_relocate_block_group()
4058 mutex_lock(&fs_info->cleaner_mutex); in btrfs_relocate_block_group()
4060 mutex_unlock(&fs_info->cleaner_mutex); in btrfs_relocate_block_group()
4064 finishes_stage = rc->stage; in btrfs_relocate_block_group()
4069 * BUG_ON(rc->stage == UPDATE_DATA_PTRS) in in btrfs_relocate_block_group()
4074 if (rc->stage == MOVE_DATA_EXTENTS && rc->found_file_extent) { in btrfs_relocate_block_group()
4075 ret = btrfs_wait_ordered_range(rc->data_inode, 0, in btrfs_relocate_block_group()
4076 (u64)-1); in btrfs_relocate_block_group()
4079 invalidate_mapping_pages(rc->data_inode->i_mapping, in btrfs_relocate_block_group()
4080 0, -1); in btrfs_relocate_block_group()
4081 rc->stage = UPDATE_DATA_PTRS; in btrfs_relocate_block_group()
4087 if (rc->extents_found == 0) in btrfs_relocate_block_group()
4091 rc->extents_found, stage_to_string(finishes_stage)); in btrfs_relocate_block_group()
4094 WARN_ON(rc->block_group->pinned > 0); in btrfs_relocate_block_group()
4095 WARN_ON(rc->block_group->reserved > 0); in btrfs_relocate_block_group()
4096 WARN_ON(rc->block_group->used > 0); in btrfs_relocate_block_group()
4099 btrfs_dec_block_group_ro(rc->block_group); in btrfs_relocate_block_group()
4100 iput(rc->data_inode); in btrfs_relocate_block_group()
4110 struct btrfs_fs_info *fs_info = root->fs_info; in mark_garbage_root()
4114 trans = btrfs_start_transaction(fs_info->tree_root, 0); in mark_garbage_root()
4118 memset(&root->root_item.drop_progress, 0, in mark_garbage_root()
4119 sizeof(root->root_item.drop_progress)); in mark_garbage_root()
4120 btrfs_set_root_drop_level(&root->root_item, 0); in mark_garbage_root()
4121 btrfs_set_root_refs(&root->root_item, 0); in mark_garbage_root()
4122 ret = btrfs_update_root(trans, fs_info->tree_root, in mark_garbage_root()
4123 &root->root_key, &root->root_item); in mark_garbage_root()
4152 return -ENOMEM; in btrfs_recover_relocation()
4153 path->reada = READA_BACK; in btrfs_recover_relocation()
4157 key.offset = (u64)-1; in btrfs_recover_relocation()
4160 ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, in btrfs_recover_relocation()
4167 if (path->slots[0] == 0) in btrfs_recover_relocation()
4169 path->slots[0]--; in btrfs_recover_relocation()
4171 leaf = path->nodes[0]; in btrfs_recover_relocation()
4172 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); in btrfs_recover_relocation()
4179 reloc_root = btrfs_read_tree_root(fs_info->tree_root, &key); in btrfs_recover_relocation()
4185 set_bit(BTRFS_ROOT_SHAREABLE, &reloc_root->state); in btrfs_recover_relocation()
4186 list_add(&reloc_root->root_list, &reloc_roots); in btrfs_recover_relocation()
4188 if (btrfs_root_refs(&reloc_root->root_item) > 0) { in btrfs_recover_relocation()
4190 reloc_root->root_key.offset, false); in btrfs_recover_relocation()
4193 if (ret != -ENOENT) { in btrfs_recover_relocation()
4207 if (key.offset == 0) in btrfs_recover_relocation()
4210 key.offset--; in btrfs_recover_relocation()
4219 err = -ENOMEM; in btrfs_recover_relocation()
4229 rc->extent_root = btrfs_extent_root(fs_info, 0); in btrfs_recover_relocation()
4233 trans = btrfs_join_transaction(rc->extent_root); in btrfs_recover_relocation()
4239 rc->merge_reloc_tree = 1; in btrfs_recover_relocation()
4244 list_del(&reloc_root->root_list); in btrfs_recover_relocation()
4246 if (btrfs_root_refs(&reloc_root->root_item) == 0) { in btrfs_recover_relocation()
4247 list_add_tail(&reloc_root->root_list, in btrfs_recover_relocation()
4248 &rc->reloc_roots); in btrfs_recover_relocation()
4252 fs_root = btrfs_get_fs_root(fs_info, reloc_root->root_key.offset, in btrfs_recover_relocation()
4256 list_add_tail(&reloc_root->root_list, &reloc_roots); in btrfs_recover_relocation()
4262 ASSERT(err != -EEXIST); in btrfs_recover_relocation()
4264 list_add_tail(&reloc_root->root_list, &reloc_roots); in btrfs_recover_relocation()
4269 fs_root->reloc_root = btrfs_grab_root(reloc_root); in btrfs_recover_relocation()
4281 trans = btrfs_join_transaction(rc->extent_root); in btrfs_recover_relocation()
4303 fs_root = btrfs_grab_root(fs_info->data_reloc_root); in btrfs_recover_relocation()
4315 * it also saves CPU time to re-calculate the checksum.
4319 struct btrfs_fs_info *fs_info = inode->root->fs_info; in btrfs_reloc_clone_csums()
4329 BUG_ON(ordered->file_offset != file_pos || ordered->num_bytes != len); in btrfs_reloc_clone_csums()
4331 disk_bytenr = file_pos + inode->index_cnt; in btrfs_reloc_clone_csums()
4334 disk_bytenr + len - 1, &list, 0, false); in btrfs_reloc_clone_csums()
4340 list_del_init(&sums->list); in btrfs_reloc_clone_csums()
4343 * We need to offset the new_bytenr based on where the csum is. in btrfs_reloc_clone_csums()
4347 * the right disk offset. in btrfs_reloc_clone_csums()
4354 new_bytenr = ordered->disk_bytenr + sums->bytenr - disk_bytenr; in btrfs_reloc_clone_csums()
4355 sums->bytenr = new_bytenr; in btrfs_reloc_clone_csums()
4368 struct btrfs_fs_info *fs_info = root->fs_info; in btrfs_reloc_cow_block()
4375 rc = fs_info->reloc_ctl; in btrfs_reloc_cow_block()
4379 BUG_ON(rc->stage == UPDATE_DATA_PTRS && btrfs_is_data_reloc_root(root)); in btrfs_reloc_cow_block()
4383 btrfs_root_last_snapshot(&root->root_item)) in btrfs_reloc_cow_block()
4386 if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID && in btrfs_reloc_cow_block()
4387 rc->create_reloc_tree) { in btrfs_reloc_cow_block()
4390 node = rc->backref_cache.path[level]; in btrfs_reloc_cow_block()
4391 BUG_ON(node->bytenr != buf->start && in btrfs_reloc_cow_block()
4392 node->new_bytenr != buf->start); in btrfs_reloc_cow_block()
4395 atomic_inc(&cow->refs); in btrfs_reloc_cow_block()
4396 node->eb = cow; in btrfs_reloc_cow_block()
4397 node->new_bytenr = cow->start; in btrfs_reloc_cow_block()
4399 if (!node->pending) { in btrfs_reloc_cow_block()
4400 list_move_tail(&node->list, in btrfs_reloc_cow_block()
4401 &rc->backref_cache.pending[level]); in btrfs_reloc_cow_block()
4402 node->pending = 1; in btrfs_reloc_cow_block()
4409 rc->nodes_relocated += buf->len; in btrfs_reloc_cow_block()
4412 if (level == 0 && first_cow && rc->stage == UPDATE_DATA_PTRS) in btrfs_reloc_cow_block()
4424 struct btrfs_root *root = pending->root; in btrfs_reloc_pre_snapshot()
4425 struct reloc_control *rc = root->fs_info->reloc_ctl; in btrfs_reloc_pre_snapshot()
4430 if (!rc->merge_reloc_tree) in btrfs_reloc_pre_snapshot()
4433 root = root->reloc_root; in btrfs_reloc_pre_snapshot()
4434 BUG_ON(btrfs_root_refs(&root->root_item) == 0); in btrfs_reloc_pre_snapshot()
4445 *bytes_to_reserve += rc->nodes_relocated; in btrfs_reloc_pre_snapshot()
4453 * references held on the reloc_root, one for root->reloc_root and one for
4454 * rc->reloc_roots.
4459 struct btrfs_root *root = pending->root; in btrfs_reloc_post_snapshot()
4462 struct reloc_control *rc = root->fs_info->reloc_ctl; in btrfs_reloc_post_snapshot()
4468 rc = root->fs_info->reloc_ctl; in btrfs_reloc_post_snapshot()
4469 rc->merging_rsv_size += rc->nodes_relocated; in btrfs_reloc_post_snapshot()
4471 if (rc->merge_reloc_tree) { in btrfs_reloc_post_snapshot()
4472 ret = btrfs_block_rsv_migrate(&pending->block_rsv, in btrfs_reloc_post_snapshot()
4473 rc->block_rsv, in btrfs_reloc_post_snapshot()
4474 rc->nodes_relocated, true); in btrfs_reloc_post_snapshot()
4479 new_root = pending->snap; in btrfs_reloc_post_snapshot()
4480 reloc_root = create_reloc_root(trans, root->reloc_root, in btrfs_reloc_post_snapshot()
4481 new_root->root_key.objectid); in btrfs_reloc_post_snapshot()
4486 ASSERT(ret != -EEXIST); in btrfs_reloc_post_snapshot()
4492 new_root->reloc_root = btrfs_grab_root(reloc_root); in btrfs_reloc_post_snapshot()
4494 if (rc->create_reloc_tree) in btrfs_reloc_post_snapshot()