Lines Matching refs:sctx
368 static void inconsistent_snapshot_error(struct send_ctx *sctx, in inconsistent_snapshot_error() argument
393 btrfs_err(sctx->send_root->fs_info, in inconsistent_snapshot_error()
395 result_string, what, sctx->cmp_key->objectid, in inconsistent_snapshot_error()
396 sctx->send_root->root_key.objectid, in inconsistent_snapshot_error()
397 (sctx->parent_root ? in inconsistent_snapshot_error()
398 sctx->parent_root->root_key.objectid : 0)); in inconsistent_snapshot_error()
402 static bool proto_cmd_ok(const struct send_ctx *sctx, int cmd) in proto_cmd_ok() argument
404 switch (sctx->proto) { in proto_cmd_ok()
412 static int is_waiting_for_move(struct send_ctx *sctx, u64 ino);
415 get_waiting_dir_move(struct send_ctx *sctx, u64 ino);
417 static int is_waiting_for_rm(struct send_ctx *sctx, u64 dir_ino, u64 gen);
419 static int need_send_hole(struct send_ctx *sctx) in need_send_hole() argument
421 return (sctx->parent_root && !sctx->cur_inode_new && in need_send_hole()
422 !sctx->cur_inode_new_gen && !sctx->cur_inode_deleted && in need_send_hole()
423 S_ISREG(sctx->cur_inode_mode)); in need_send_hole()
659 static int tlv_put(struct send_ctx *sctx, u16 attr, const void *data, int len) in tlv_put() argument
663 int left = sctx->send_max_size - sctx->send_size; in tlv_put()
665 if (WARN_ON_ONCE(sctx->put_data)) in tlv_put()
671 hdr = (struct btrfs_tlv_header *) (sctx->send_buf + sctx->send_size); in tlv_put()
675 sctx->send_size += total_len; in tlv_put()
681 static int tlv_put_u##bits(struct send_ctx *sctx, \
685 return tlv_put(sctx, attr, &__tmp, sizeof(__tmp)); \
692 static int tlv_put_string(struct send_ctx *sctx, u16 attr, in tlv_put_string() argument
697 return tlv_put(sctx, attr, str, len); in tlv_put_string()
700 static int tlv_put_uuid(struct send_ctx *sctx, u16 attr, in tlv_put_uuid() argument
703 return tlv_put(sctx, attr, uuid, BTRFS_UUID_SIZE); in tlv_put_uuid()
706 static int tlv_put_btrfs_timespec(struct send_ctx *sctx, u16 attr, in tlv_put_btrfs_timespec() argument
712 return tlv_put(sctx, attr, &bts, sizeof(bts)); in tlv_put_btrfs_timespec()
716 #define TLV_PUT(sctx, attrtype, data, attrlen) \ argument
718 ret = tlv_put(sctx, attrtype, data, attrlen); \
723 #define TLV_PUT_INT(sctx, attrtype, bits, value) \ argument
725 ret = tlv_put_u##bits(sctx, attrtype, value); \
730 #define TLV_PUT_U8(sctx, attrtype, data) TLV_PUT_INT(sctx, attrtype, 8, data) argument
731 #define TLV_PUT_U16(sctx, attrtype, data) TLV_PUT_INT(sctx, attrtype, 16, data) argument
732 #define TLV_PUT_U32(sctx, attrtype, data) TLV_PUT_INT(sctx, attrtype, 32, data) argument
733 #define TLV_PUT_U64(sctx, attrtype, data) TLV_PUT_INT(sctx, attrtype, 64, data) argument
734 #define TLV_PUT_STRING(sctx, attrtype, str, len) \ argument
736 ret = tlv_put_string(sctx, attrtype, str, len); \
740 #define TLV_PUT_PATH(sctx, attrtype, p) \ argument
742 ret = tlv_put_string(sctx, attrtype, p->start, \
747 #define TLV_PUT_UUID(sctx, attrtype, uuid) \ argument
749 ret = tlv_put_uuid(sctx, attrtype, uuid); \
753 #define TLV_PUT_BTRFS_TIMESPEC(sctx, attrtype, eb, ts) \ argument
755 ret = tlv_put_btrfs_timespec(sctx, attrtype, eb, ts); \
760 static int send_header(struct send_ctx *sctx) in send_header() argument
765 hdr.version = cpu_to_le32(sctx->proto); in send_header()
766 return write_buf(sctx->send_filp, &hdr, sizeof(hdr), in send_header()
767 &sctx->send_off); in send_header()
773 static int begin_cmd(struct send_ctx *sctx, int cmd) in begin_cmd() argument
777 if (WARN_ON(!sctx->send_buf)) in begin_cmd()
780 BUG_ON(sctx->send_size); in begin_cmd()
782 sctx->send_size += sizeof(*hdr); in begin_cmd()
783 hdr = (struct btrfs_cmd_header *)sctx->send_buf; in begin_cmd()
789 static int send_cmd(struct send_ctx *sctx) in send_cmd() argument
795 hdr = (struct btrfs_cmd_header *)sctx->send_buf; in send_cmd()
796 put_unaligned_le32(sctx->send_size - sizeof(*hdr), &hdr->len); in send_cmd()
799 crc = btrfs_crc32c(0, (unsigned char *)sctx->send_buf, sctx->send_size); in send_cmd()
802 ret = write_buf(sctx->send_filp, sctx->send_buf, sctx->send_size, in send_cmd()
803 &sctx->send_off); in send_cmd()
805 sctx->send_size = 0; in send_cmd()
806 sctx->put_data = false; in send_cmd()
814 static int send_rename(struct send_ctx *sctx, in send_rename() argument
817 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info; in send_rename()
822 ret = begin_cmd(sctx, BTRFS_SEND_C_RENAME); in send_rename()
826 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, from); in send_rename()
827 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH_TO, to); in send_rename()
829 ret = send_cmd(sctx); in send_rename()
839 static int send_link(struct send_ctx *sctx, in send_link() argument
842 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info; in send_link()
847 ret = begin_cmd(sctx, BTRFS_SEND_C_LINK); in send_link()
851 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path); in send_link()
852 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH_LINK, lnk); in send_link()
854 ret = send_cmd(sctx); in send_link()
864 static int send_unlink(struct send_ctx *sctx, struct fs_path *path) in send_unlink() argument
866 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info; in send_unlink()
871 ret = begin_cmd(sctx, BTRFS_SEND_C_UNLINK); in send_unlink()
875 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path); in send_unlink()
877 ret = send_cmd(sctx); in send_unlink()
887 static int send_rmdir(struct send_ctx *sctx, struct fs_path *path) in send_rmdir() argument
889 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info; in send_rmdir()
894 ret = begin_cmd(sctx, BTRFS_SEND_C_RMDIR); in send_rmdir()
898 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path); in send_rmdir()
900 ret = send_cmd(sctx); in send_rmdir()
1279 struct send_ctx *sctx; member
1337 clone_root = bsearch((void *)(uintptr_t)root_id, bctx->sctx->clone_roots, in iterate_backrefs()
1338 bctx->sctx->clone_roots_cnt, in iterate_backrefs()
1345 if (clone_root->root == bctx->sctx->send_root && in iterate_backrefs()
1354 if (clone_root->root == bctx->sctx->send_root) { in iterate_backrefs()
1370 bctx->sctx->cur_inode_next_write_offset) in iterate_backrefs()
1403 struct send_ctx *sctx = bctx->sctx; in lookup_backref_cache() local
1404 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info; in lookup_backref_cache()
1409 if (btrfs_lru_cache_size(&sctx->backref_cache) == 0) in lookup_backref_cache()
1423 if (fs_info->last_reloc_trans > sctx->backref_cache_last_reloc_trans) { in lookup_backref_cache()
1424 btrfs_lru_cache_clear(&sctx->backref_cache); in lookup_backref_cache()
1428 raw_entry = btrfs_lru_cache_lookup(&sctx->backref_cache, key, 0); in lookup_backref_cache()
1443 struct send_ctx *sctx = bctx->sctx; in store_backref_cache() local
1444 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info; in store_backref_cache()
1468 root = bsearch((void *)(uintptr_t)root_id, sctx->clone_roots, in store_backref_cache()
1469 sctx->clone_roots_cnt, sizeof(struct clone_root), in store_backref_cache()
1493 ret = btrfs_lru_cache_store(&sctx->backref_cache, &new_entry->entry, in store_backref_cache()
1507 if (btrfs_lru_cache_size(&sctx->backref_cache) == 1) in store_backref_cache()
1508 sctx->backref_cache_last_reloc_trans = fs_info->last_reloc_trans; in store_backref_cache()
1516 const struct send_ctx *sctx = bctx->sctx; in check_extent_item() local
1531 if (refs == 1 && sctx->clone_roots_cnt == 1) in check_extent_item()
1569 static int find_extent_clone(struct send_ctx *sctx, in find_extent_clone() argument
1575 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info; in find_extent_clone()
1614 for (i = 0; i < sctx->clone_roots_cnt; i++) { in find_extent_clone()
1615 cur_clone_root = sctx->clone_roots + i; in find_extent_clone()
1622 backref_ctx.sctx = sctx; in find_extent_clone()
1664 if (sctx->clone_roots_cnt == 1) in find_extent_clone()
1673 if (fs_info->last_reloc_trans > sctx->last_reloc_trans) { in find_extent_clone()
1701 for (i = 0; i < sctx->clone_roots_cnt; i++) { in find_extent_clone()
1702 struct clone_root *clone_root = &sctx->clone_roots[i]; in find_extent_clone()
1807 static int gen_unique_name(struct send_ctx *sctx, in gen_unique_name() argument
1831 di = btrfs_lookup_dir_item(NULL, sctx->send_root, in gen_unique_name()
1845 if (!sctx->parent_root) { in gen_unique_name()
1851 di = btrfs_lookup_dir_item(NULL, sctx->parent_root, in gen_unique_name()
1883 static int get_cur_inode_state(struct send_ctx *sctx, u64 ino, u64 gen, in get_cur_inode_state() argument
1893 ret = get_inode_info(sctx->send_root, ino, &info); in get_cur_inode_state()
1901 if (!sctx->parent_root) { in get_cur_inode_state()
1904 ret = get_inode_info(sctx->parent_root, ino, &info); in get_cur_inode_state()
1917 if (ino < sctx->send_progress) in get_cur_inode_state()
1922 if (ino < sctx->send_progress) in get_cur_inode_state()
1931 if (ino < sctx->send_progress) in get_cur_inode_state()
1940 if (ino < sctx->send_progress) in get_cur_inode_state()
1955 static int is_inode_existent(struct send_ctx *sctx, u64 ino, u64 gen, in is_inode_existent() argument
1963 ret = get_cur_inode_state(sctx, ino, gen, send_gen, parent_gen); in is_inode_existent()
2120 static int will_overwrite_ref(struct send_ctx *sctx, u64 dir, u64 dir_gen, in will_overwrite_ref() argument
2129 if (!sctx->parent_root) in will_overwrite_ref()
2132 ret = is_inode_existent(sctx, dir, dir_gen, NULL, &parent_root_dir_gen); in will_overwrite_ref()
2144 if (sctx->parent_root && dir != BTRFS_FIRST_FREE_OBJECTID && in will_overwrite_ref()
2148 ret = lookup_dir_item_inode(sctx->parent_root, dir, name, name_len, in will_overwrite_ref()
2160 if (other_inode > sctx->send_progress || in will_overwrite_ref()
2161 is_waiting_for_move(sctx, other_inode)) { in will_overwrite_ref()
2162 ret = get_inode_info(sctx->parent_root, other_inode, &info); in will_overwrite_ref()
2182 static int did_overwrite_ref(struct send_ctx *sctx, in did_overwrite_ref() argument
2192 if (!sctx->parent_root) in did_overwrite_ref()
2195 ret = is_inode_existent(sctx, dir, dir_gen, &send_root_dir_gen, NULL); in did_overwrite_ref()
2207 ret = lookup_dir_item_inode(sctx->send_root, dir, name, name_len, in did_overwrite_ref()
2217 ret = get_inode_gen(sctx->send_root, ow_inode, &ow_gen); in did_overwrite_ref()
2232 if (ow_inode < sctx->send_progress) in did_overwrite_ref()
2235 if (ino != sctx->cur_ino && ow_inode == sctx->cur_ino) { in did_overwrite_ref()
2237 ret = get_inode_gen(sctx->send_root, ow_inode, &ow_gen); in did_overwrite_ref()
2241 if (ow_gen == sctx->cur_inode_gen) in did_overwrite_ref()
2253 static int did_overwrite_first_ref(struct send_ctx *sctx, u64 ino, u64 gen) in did_overwrite_first_ref() argument
2260 if (!sctx->parent_root) in did_overwrite_first_ref()
2267 ret = get_first_ref(sctx->parent_root, ino, &dir, &dir_gen, name); in did_overwrite_first_ref()
2271 ret = did_overwrite_ref(sctx, dir, dir_gen, ino, gen, in did_overwrite_first_ref()
2279 static inline struct name_cache_entry *name_cache_search(struct send_ctx *sctx, in name_cache_search() argument
2284 entry = btrfs_lru_cache_lookup(&sctx->name_cache, ino, gen); in name_cache_search()
2299 static int __get_cur_name_and_parent(struct send_ctx *sctx, in __get_cur_name_and_parent() argument
2314 nce = name_cache_search(sctx, ino, gen); in __get_cur_name_and_parent()
2316 if (ino < sctx->send_progress && nce->need_later_update) { in __get_cur_name_and_parent()
2317 btrfs_lru_cache_remove(&sctx->name_cache, &nce->entry); in __get_cur_name_and_parent()
2335 ret = is_inode_existent(sctx, ino, gen, NULL, NULL); in __get_cur_name_and_parent()
2340 ret = gen_unique_name(sctx, ino, gen, dest); in __get_cur_name_and_parent()
2351 if (ino < sctx->send_progress) in __get_cur_name_and_parent()
2352 ret = get_first_ref(sctx->send_root, ino, in __get_cur_name_and_parent()
2355 ret = get_first_ref(sctx->parent_root, ino, in __get_cur_name_and_parent()
2364 ret = did_overwrite_ref(sctx, *parent_ino, *parent_gen, ino, gen, in __get_cur_name_and_parent()
2370 ret = gen_unique_name(sctx, ino, gen, dest); in __get_cur_name_and_parent()
2394 if (ino < sctx->send_progress) in __get_cur_name_and_parent()
2399 nce_ret = btrfs_lru_cache_store(&sctx->name_cache, &nce->entry, GFP_KERNEL); in __get_cur_name_and_parent()
2434 static int get_cur_path(struct send_ctx *sctx, u64 ino, u64 gen, in get_cur_path() argument
2457 if (is_waiting_for_rm(sctx, ino, gen)) { in get_cur_path()
2458 ret = gen_unique_name(sctx, ino, gen, name); in get_cur_path()
2465 wdm = get_waiting_dir_move(sctx, ino); in get_cur_path()
2467 ret = gen_unique_name(sctx, ino, gen, name); in get_cur_path()
2470 ret = get_first_ref(sctx->parent_root, ino, in get_cur_path()
2473 ret = __get_cur_name_and_parent(sctx, ino, gen, in get_cur_path()
2501 static int send_subvol_begin(struct send_ctx *sctx) in send_subvol_begin() argument
2504 struct btrfs_root *send_root = sctx->send_root; in send_subvol_begin()
2505 struct btrfs_root *parent_root = sctx->parent_root; in send_subvol_begin()
2549 ret = begin_cmd(sctx, BTRFS_SEND_C_SNAPSHOT); in send_subvol_begin()
2553 ret = begin_cmd(sctx, BTRFS_SEND_C_SUBVOL); in send_subvol_begin()
2558 TLV_PUT_STRING(sctx, BTRFS_SEND_A_PATH, name, namelen); in send_subvol_begin()
2560 if (!btrfs_is_empty_uuid(sctx->send_root->root_item.received_uuid)) in send_subvol_begin()
2561 TLV_PUT_UUID(sctx, BTRFS_SEND_A_UUID, in send_subvol_begin()
2562 sctx->send_root->root_item.received_uuid); in send_subvol_begin()
2564 TLV_PUT_UUID(sctx, BTRFS_SEND_A_UUID, in send_subvol_begin()
2565 sctx->send_root->root_item.uuid); in send_subvol_begin()
2567 TLV_PUT_U64(sctx, BTRFS_SEND_A_CTRANSID, in send_subvol_begin()
2568 btrfs_root_ctransid(&sctx->send_root->root_item)); in send_subvol_begin()
2571 TLV_PUT_UUID(sctx, BTRFS_SEND_A_CLONE_UUID, in send_subvol_begin()
2574 TLV_PUT_UUID(sctx, BTRFS_SEND_A_CLONE_UUID, in send_subvol_begin()
2576 TLV_PUT_U64(sctx, BTRFS_SEND_A_CLONE_CTRANSID, in send_subvol_begin()
2577 btrfs_root_ctransid(&sctx->parent_root->root_item)); in send_subvol_begin()
2580 ret = send_cmd(sctx); in send_subvol_begin()
2589 static int send_truncate(struct send_ctx *sctx, u64 ino, u64 gen, u64 size) in send_truncate() argument
2591 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info; in send_truncate()
2601 ret = begin_cmd(sctx, BTRFS_SEND_C_TRUNCATE); in send_truncate()
2605 ret = get_cur_path(sctx, ino, gen, p); in send_truncate()
2608 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p); in send_truncate()
2609 TLV_PUT_U64(sctx, BTRFS_SEND_A_SIZE, size); in send_truncate()
2611 ret = send_cmd(sctx); in send_truncate()
2619 static int send_chmod(struct send_ctx *sctx, u64 ino, u64 gen, u64 mode) in send_chmod() argument
2621 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info; in send_chmod()
2631 ret = begin_cmd(sctx, BTRFS_SEND_C_CHMOD); in send_chmod()
2635 ret = get_cur_path(sctx, ino, gen, p); in send_chmod()
2638 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p); in send_chmod()
2639 TLV_PUT_U64(sctx, BTRFS_SEND_A_MODE, mode & 07777); in send_chmod()
2641 ret = send_cmd(sctx); in send_chmod()
2649 static int send_fileattr(struct send_ctx *sctx, u64 ino, u64 gen, u64 fileattr) in send_fileattr() argument
2651 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info; in send_fileattr()
2655 if (sctx->proto < 2) in send_fileattr()
2664 ret = begin_cmd(sctx, BTRFS_SEND_C_FILEATTR); in send_fileattr()
2668 ret = get_cur_path(sctx, ino, gen, p); in send_fileattr()
2671 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p); in send_fileattr()
2672 TLV_PUT_U64(sctx, BTRFS_SEND_A_FILEATTR, fileattr); in send_fileattr()
2674 ret = send_cmd(sctx); in send_fileattr()
2682 static int send_chown(struct send_ctx *sctx, u64 ino, u64 gen, u64 uid, u64 gid) in send_chown() argument
2684 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info; in send_chown()
2695 ret = begin_cmd(sctx, BTRFS_SEND_C_CHOWN); in send_chown()
2699 ret = get_cur_path(sctx, ino, gen, p); in send_chown()
2702 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p); in send_chown()
2703 TLV_PUT_U64(sctx, BTRFS_SEND_A_UID, uid); in send_chown()
2704 TLV_PUT_U64(sctx, BTRFS_SEND_A_GID, gid); in send_chown()
2706 ret = send_cmd(sctx); in send_chown()
2714 static int send_utimes(struct send_ctx *sctx, u64 ino, u64 gen) in send_utimes() argument
2716 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info; in send_utimes()
2740 ret = btrfs_search_slot(NULL, sctx->send_root, &key, path, 0, 0); in send_utimes()
2750 ret = begin_cmd(sctx, BTRFS_SEND_C_UTIMES); in send_utimes()
2754 ret = get_cur_path(sctx, ino, gen, p); in send_utimes()
2757 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p); in send_utimes()
2758 TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_ATIME, eb, &ii->atime); in send_utimes()
2759 TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_MTIME, eb, &ii->mtime); in send_utimes()
2760 TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_CTIME, eb, &ii->ctime); in send_utimes()
2761 if (sctx->proto >= 2) in send_utimes()
2762 TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_OTIME, eb, &ii->otime); in send_utimes()
2764 ret = send_cmd(sctx); in send_utimes()
2783 static int cache_dir_utimes(struct send_ctx *sctx, u64 dir, u64 gen) in cache_dir_utimes() argument
2788 entry = btrfs_lru_cache_lookup(&sctx->dir_utimes_cache, dir, gen); in cache_dir_utimes()
2795 return send_utimes(sctx, dir, gen); in cache_dir_utimes()
2800 ret = btrfs_lru_cache_store(&sctx->dir_utimes_cache, entry, GFP_KERNEL); in cache_dir_utimes()
2804 return send_utimes(sctx, dir, gen); in cache_dir_utimes()
2810 static int trim_dir_utimes_cache(struct send_ctx *sctx) in trim_dir_utimes_cache() argument
2812 while (btrfs_lru_cache_size(&sctx->dir_utimes_cache) > in trim_dir_utimes_cache()
2817 lru = btrfs_lru_cache_lru_entry(&sctx->dir_utimes_cache); in trim_dir_utimes_cache()
2820 ret = send_utimes(sctx, lru->key, lru->gen); in trim_dir_utimes_cache()
2824 btrfs_lru_cache_remove(&sctx->dir_utimes_cache, lru); in trim_dir_utimes_cache()
2835 static int send_create_inode(struct send_ctx *sctx, u64 ino) in send_create_inode() argument
2837 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info; in send_create_inode()
2852 if (ino != sctx->cur_ino) { in send_create_inode()
2853 ret = get_inode_info(sctx->send_root, ino, &info); in send_create_inode()
2860 gen = sctx->cur_inode_gen; in send_create_inode()
2861 mode = sctx->cur_inode_mode; in send_create_inode()
2862 rdev = sctx->cur_inode_rdev; in send_create_inode()
2878 btrfs_warn(sctx->send_root->fs_info, "unexpected inode type %o", in send_create_inode()
2884 ret = begin_cmd(sctx, cmd); in send_create_inode()
2888 ret = gen_unique_name(sctx, ino, gen, p); in send_create_inode()
2892 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p); in send_create_inode()
2893 TLV_PUT_U64(sctx, BTRFS_SEND_A_INO, ino); in send_create_inode()
2897 ret = read_symlink(sctx->send_root, ino, p); in send_create_inode()
2900 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH_LINK, p); in send_create_inode()
2903 TLV_PUT_U64(sctx, BTRFS_SEND_A_RDEV, new_encode_dev(rdev)); in send_create_inode()
2904 TLV_PUT_U64(sctx, BTRFS_SEND_A_MODE, mode); in send_create_inode()
2907 ret = send_cmd(sctx); in send_create_inode()
2918 static void cache_dir_created(struct send_ctx *sctx, u64 dir) in cache_dir_created() argument
2930 ret = btrfs_lru_cache_store(&sctx->dir_created_cache, entry, GFP_KERNEL); in cache_dir_created()
2940 static int did_create_dir(struct send_ctx *sctx, u64 dir) in did_create_dir() argument
2950 if (btrfs_lru_cache_lookup(&sctx->dir_created_cache, dir, 0)) in did_create_dir()
2961 btrfs_for_each_slot(sctx->send_root, &key, &found_key, path, iter_ret) { in did_create_dir()
2974 di_key.objectid < sctx->send_progress) { in did_create_dir()
2976 cache_dir_created(sctx, dir); in did_create_dir()
2994 static int send_create_inode_if_needed(struct send_ctx *sctx) in send_create_inode_if_needed() argument
2998 if (S_ISDIR(sctx->cur_inode_mode)) { in send_create_inode_if_needed()
2999 ret = did_create_dir(sctx, sctx->cur_ino); in send_create_inode_if_needed()
3006 ret = send_create_inode(sctx, sctx->cur_ino); in send_create_inode_if_needed()
3008 if (ret == 0 && S_ISDIR(sctx->cur_inode_mode)) in send_create_inode_if_needed()
3009 cache_dir_created(sctx, sctx->cur_ino); in send_create_inode_if_needed()
3079 static void free_recorded_refs(struct send_ctx *sctx) in free_recorded_refs() argument
3081 __free_recorded_refs(&sctx->new_refs); in free_recorded_refs()
3082 __free_recorded_refs(&sctx->deleted_refs); in free_recorded_refs()
3090 static int orphanize_inode(struct send_ctx *sctx, u64 ino, u64 gen, in orphanize_inode() argument
3100 ret = gen_unique_name(sctx, ino, gen, orphan); in orphanize_inode()
3104 ret = send_rename(sctx, path, orphan); in orphanize_inode()
3111 static struct orphan_dir_info *add_orphan_dir_info(struct send_ctx *sctx, in add_orphan_dir_info() argument
3114 struct rb_node **p = &sctx->orphan_dirs.rb_node; in add_orphan_dir_info()
3142 rb_insert_color(&odi->node, &sctx->orphan_dirs); in add_orphan_dir_info()
3146 static struct orphan_dir_info *get_orphan_dir_info(struct send_ctx *sctx, in get_orphan_dir_info() argument
3149 struct rb_node *n = sctx->orphan_dirs.rb_node; in get_orphan_dir_info()
3168 static int is_waiting_for_rm(struct send_ctx *sctx, u64 dir_ino, u64 gen) in is_waiting_for_rm() argument
3170 struct orphan_dir_info *odi = get_orphan_dir_info(sctx, dir_ino, gen); in is_waiting_for_rm()
3175 static void free_orphan_dir_info(struct send_ctx *sctx, in free_orphan_dir_info() argument
3180 rb_erase(&odi->node, &sctx->orphan_dirs); in free_orphan_dir_info()
3189 static int can_rmdir(struct send_ctx *sctx, u64 dir, u64 dir_gen) in can_rmdir() argument
3193 struct btrfs_root *root = sctx->parent_root; in can_rmdir()
3209 odi = get_orphan_dir_info(sctx, dir, dir_gen); in can_rmdir()
3210 if (odi && sctx->cur_ino < odi->dir_high_seq_ino) in can_rmdir()
3254 if (sctx->cur_ino < dir_high_seq_ino) { in can_rmdir()
3280 dm = get_waiting_dir_move(sctx, loc.objectid); in can_rmdir()
3288 if (loc.objectid > sctx->cur_ino) { in can_rmdir()
3297 free_orphan_dir_info(sctx, odi); in can_rmdir()
3308 odi = add_orphan_dir_info(sctx, dir, dir_gen); in can_rmdir()
3321 static int is_waiting_for_move(struct send_ctx *sctx, u64 ino) in is_waiting_for_move() argument
3323 struct waiting_dir_move *entry = get_waiting_dir_move(sctx, ino); in is_waiting_for_move()
3328 static int add_waiting_dir_move(struct send_ctx *sctx, u64 ino, bool orphanized) in add_waiting_dir_move() argument
3330 struct rb_node **p = &sctx->waiting_dir_moves.rb_node; in add_waiting_dir_move()
3356 rb_insert_color(&dm->node, &sctx->waiting_dir_moves); in add_waiting_dir_move()
3361 get_waiting_dir_move(struct send_ctx *sctx, u64 ino) in get_waiting_dir_move() argument
3363 struct rb_node *n = sctx->waiting_dir_moves.rb_node; in get_waiting_dir_move()
3378 static void free_waiting_dir_move(struct send_ctx *sctx, in free_waiting_dir_move() argument
3383 rb_erase(&dm->node, &sctx->waiting_dir_moves); in free_waiting_dir_move()
3387 static int add_pending_dir_move(struct send_ctx *sctx, in add_pending_dir_move() argument
3395 struct rb_node **p = &sctx->pending_dir_moves.rb_node; in add_pending_dir_move()
3436 ret = add_waiting_dir_move(sctx, pm->ino, is_orphan); in add_pending_dir_move()
3444 rb_insert_color(&pm->node, &sctx->pending_dir_moves); in add_pending_dir_move()
3455 static struct pending_dir_move *get_pending_dir_moves(struct send_ctx *sctx, in get_pending_dir_moves() argument
3458 struct rb_node *n = sctx->pending_dir_moves.rb_node; in get_pending_dir_moves()
3473 static int path_loop(struct send_ctx *sctx, struct fs_path *name, in path_loop() argument
3485 if (is_waiting_for_rm(sctx, ino, gen)) in path_loop()
3487 if (is_waiting_for_move(sctx, ino)) { in path_loop()
3490 ret = get_first_ref(sctx->parent_root, ino, in path_loop()
3493 ret = __get_cur_name_and_parent(sctx, ino, gen, in path_loop()
3515 static int apply_dir_move(struct send_ctx *sctx, struct pending_dir_move *pm) in apply_dir_move() argument
3520 u64 orig_progress = sctx->send_progress; in apply_dir_move()
3537 dm = get_waiting_dir_move(sctx, pm->ino); in apply_dir_move()
3542 free_waiting_dir_move(sctx, dm); in apply_dir_move()
3545 ret = gen_unique_name(sctx, pm->ino, in apply_dir_move()
3548 ret = get_first_ref(sctx->parent_root, pm->ino, in apply_dir_move()
3552 ret = get_cur_path(sctx, parent_ino, parent_gen, in apply_dir_move()
3561 sctx->send_progress = sctx->cur_ino + 1; in apply_dir_move()
3562 ret = path_loop(sctx, name, pm->ino, pm->gen, &ancestor); in apply_dir_move()
3568 ret = add_pending_dir_move(sctx, pm->ino, pm->gen, ancestor, in apply_dir_move()
3574 dm = get_waiting_dir_move(sctx, pm->ino); in apply_dir_move()
3584 ret = get_cur_path(sctx, pm->ino, pm->gen, to_path); in apply_dir_move()
3588 ret = send_rename(sctx, from_path, to_path); in apply_dir_move()
3596 odi = get_orphan_dir_info(sctx, rmdir_ino, rmdir_gen); in apply_dir_move()
3603 ret = can_rmdir(sctx, rmdir_ino, gen); in apply_dir_move()
3614 ret = get_cur_path(sctx, rmdir_ino, gen, name); in apply_dir_move()
3617 ret = send_rmdir(sctx, name); in apply_dir_move()
3623 ret = cache_dir_utimes(sctx, pm->ino, pm->gen); in apply_dir_move()
3635 ret = get_inode_info(sctx->send_root, cur->dir, NULL); in apply_dir_move()
3643 ret = cache_dir_utimes(sctx, cur->dir, cur->dir_gen); in apply_dir_move()
3652 sctx->send_progress = orig_progress; in apply_dir_move()
3657 static void free_pending_move(struct send_ctx *sctx, struct pending_dir_move *m) in free_pending_move() argument
3662 rb_erase(&m->node, &sctx->pending_dir_moves); in free_pending_move()
3667 static void tail_append_pending_moves(struct send_ctx *sctx, in tail_append_pending_moves() argument
3680 rb_erase(&moves->node, &sctx->pending_dir_moves); in tail_append_pending_moves()
3685 static int apply_children_dir_moves(struct send_ctx *sctx) in apply_children_dir_moves() argument
3689 u64 parent_ino = sctx->cur_ino; in apply_children_dir_moves()
3692 pm = get_pending_dir_moves(sctx, parent_ino); in apply_children_dir_moves()
3696 tail_append_pending_moves(sctx, pm, &stack); in apply_children_dir_moves()
3701 ret = apply_dir_move(sctx, pm); in apply_children_dir_moves()
3702 free_pending_move(sctx, pm); in apply_children_dir_moves()
3705 pm = get_pending_dir_moves(sctx, parent_ino); in apply_children_dir_moves()
3707 tail_append_pending_moves(sctx, pm, &stack); in apply_children_dir_moves()
3714 free_pending_move(sctx, pm); in apply_children_dir_moves()
3755 static int wait_for_dest_dir_move(struct send_ctx *sctx, in wait_for_dest_dir_move() argument
3759 struct btrfs_fs_info *fs_info = sctx->parent_root->fs_info; in wait_for_dest_dir_move()
3769 if (RB_EMPTY_ROOT(&sctx->waiting_dir_moves)) in wait_for_dest_dir_move()
3780 ret = btrfs_search_slot(NULL, sctx->parent_root, &key, path, 0, 0); in wait_for_dest_dir_move()
3808 ret = get_inode_gen(sctx->parent_root, di_key.objectid, &left_gen); in wait_for_dest_dir_move()
3811 ret = get_inode_gen(sctx->send_root, di_key.objectid, &right_gen); in wait_for_dest_dir_move()
3824 wdm = get_waiting_dir_move(sctx, di_key.objectid); in wait_for_dest_dir_move()
3826 ret = add_pending_dir_move(sctx, in wait_for_dest_dir_move()
3827 sctx->cur_ino, in wait_for_dest_dir_move()
3828 sctx->cur_inode_gen, in wait_for_dest_dir_move()
3830 &sctx->new_refs, in wait_for_dest_dir_move()
3831 &sctx->deleted_refs, in wait_for_dest_dir_move()
3961 static int wait_for_parent_move(struct send_ctx *sctx, in wait_for_parent_move() argument
3990 if (is_waiting_for_move(sctx, ino)) { in wait_for_parent_move()
4001 ret = is_ancestor(sctx->parent_root, in wait_for_parent_move()
4002 sctx->cur_ino, sctx->cur_inode_gen, in wait_for_parent_move()
4011 ret = get_first_ref(sctx->send_root, ino, &parent_ino_after, in wait_for_parent_move()
4015 ret = get_first_ref(sctx->parent_root, ino, &parent_ino_before, in wait_for_parent_move()
4026 if (ino > sctx->cur_ino && in wait_for_parent_move()
4031 ret = get_inode_gen(sctx->parent_root, ino, &parent_ino_gen); in wait_for_parent_move()
4048 ret = add_pending_dir_move(sctx, in wait_for_parent_move()
4049 sctx->cur_ino, in wait_for_parent_move()
4050 sctx->cur_inode_gen, in wait_for_parent_move()
4052 &sctx->new_refs, in wait_for_parent_move()
4053 &sctx->deleted_refs, in wait_for_parent_move()
4062 static int update_ref_path(struct send_ctx *sctx, struct recorded_ref *ref) in update_ref_path() argument
4075 ret = get_cur_path(sctx, ref->dir, ref->dir_gen, new_path); in update_ref_path()
4133 static int refresh_ref_path(struct send_ctx *sctx, struct recorded_ref *ref) in refresh_ref_path() argument
4143 ret = get_cur_path(sctx, ref->dir, ref->dir_gen, ref->full_path); in refresh_ref_path()
4161 static int process_recorded_refs(struct send_ctx *sctx, int *pending_move) in process_recorded_refs() argument
4163 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info; in process_recorded_refs()
4179 btrfs_debug(fs_info, "process_recorded_refs %llu", sctx->cur_ino); in process_recorded_refs()
4185 BUG_ON(sctx->cur_ino <= BTRFS_FIRST_FREE_OBJECTID); in process_recorded_refs()
4204 if (!sctx->cur_inode_new) { in process_recorded_refs()
4205 ret = did_overwrite_first_ref(sctx, sctx->cur_ino, in process_recorded_refs()
4206 sctx->cur_inode_gen); in process_recorded_refs()
4212 if (sctx->cur_inode_new || did_overwrite) { in process_recorded_refs()
4213 ret = gen_unique_name(sctx, sctx->cur_ino, in process_recorded_refs()
4214 sctx->cur_inode_gen, valid_path); in process_recorded_refs()
4219 ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, in process_recorded_refs()
4263 list_for_each_entry(cur, &sctx->new_refs, list) { in process_recorded_refs()
4264 ret = get_cur_inode_state(sctx, cur->dir, cur->dir_gen, NULL, NULL); in process_recorded_refs()
4276 ret = will_overwrite_ref(sctx, cur->dir, cur->dir_gen, in process_recorded_refs()
4282 ret = is_first_ref(sctx->parent_root, in process_recorded_refs()
4292 ret = refresh_ref_path(sctx, cur); in process_recorded_refs()
4297 ret = orphanize_inode(sctx, ow_inode, ow_gen, in process_recorded_refs()
4310 wdm = get_waiting_dir_move(sctx, ow_inode); in process_recorded_refs()
4324 nce = name_cache_search(sctx, ow_inode, ow_gen); in process_recorded_refs()
4326 btrfs_lru_cache_remove(&sctx->name_cache, in process_recorded_refs()
4336 ret = is_ancestor(sctx->parent_root, in process_recorded_refs()
4338 sctx->cur_ino, NULL); in process_recorded_refs()
4342 ret = get_cur_path(sctx, sctx->cur_ino, in process_recorded_refs()
4343 sctx->cur_inode_gen, in process_recorded_refs()
4356 ret = refresh_ref_path(sctx, cur); in process_recorded_refs()
4360 ret = send_unlink(sctx, cur->full_path); in process_recorded_refs()
4368 list_for_each_entry(cur, &sctx->new_refs, list) { in process_recorded_refs()
4376 ret = get_cur_inode_state(sctx, cur->dir, cur->dir_gen, NULL, NULL); in process_recorded_refs()
4385 list_for_each_entry(cur2, &sctx->new_refs, list) { in process_recorded_refs()
4399 ret = did_create_dir(sctx, cur->dir); in process_recorded_refs()
4403 ret = send_create_inode(sctx, cur->dir); in process_recorded_refs()
4406 cache_dir_created(sctx, cur->dir); in process_recorded_refs()
4410 if (S_ISDIR(sctx->cur_inode_mode) && sctx->parent_root) { in process_recorded_refs()
4411 ret = wait_for_dest_dir_move(sctx, cur, is_orphan); in process_recorded_refs()
4420 if (S_ISDIR(sctx->cur_inode_mode) && sctx->parent_root && in process_recorded_refs()
4422 ret = wait_for_parent_move(sctx, cur, is_orphan); in process_recorded_refs()
4437 ret = send_rename(sctx, valid_path, cur->full_path); in process_recorded_refs()
4445 if (S_ISDIR(sctx->cur_inode_mode)) { in process_recorded_refs()
4451 ret = send_rename(sctx, valid_path, in process_recorded_refs()
4467 ret = update_ref_path(sctx, cur); in process_recorded_refs()
4471 ret = send_link(sctx, cur->full_path, in process_recorded_refs()
4482 if (S_ISDIR(sctx->cur_inode_mode) && sctx->cur_inode_deleted) { in process_recorded_refs()
4489 ret = can_rmdir(sctx, sctx->cur_ino, sctx->cur_inode_gen); in process_recorded_refs()
4493 ret = send_rmdir(sctx, valid_path); in process_recorded_refs()
4497 ret = orphanize_inode(sctx, sctx->cur_ino, in process_recorded_refs()
4498 sctx->cur_inode_gen, valid_path); in process_recorded_refs()
4504 list_for_each_entry(cur, &sctx->deleted_refs, list) { in process_recorded_refs()
4509 } else if (S_ISDIR(sctx->cur_inode_mode) && in process_recorded_refs()
4510 !list_empty(&sctx->deleted_refs)) { in process_recorded_refs()
4514 cur = list_entry(sctx->deleted_refs.next, struct recorded_ref, in process_recorded_refs()
4519 } else if (!S_ISDIR(sctx->cur_inode_mode)) { in process_recorded_refs()
4525 list_for_each_entry(cur, &sctx->deleted_refs, list) { in process_recorded_refs()
4526 ret = did_overwrite_ref(sctx, cur->dir, cur->dir_gen, in process_recorded_refs()
4527 sctx->cur_ino, sctx->cur_inode_gen, in process_recorded_refs()
4540 ret = update_ref_path(sctx, cur); in process_recorded_refs()
4544 ret = send_unlink(sctx, cur->full_path); in process_recorded_refs()
4561 ret = send_unlink(sctx, valid_path); in process_recorded_refs()
4579 if (cur->dir > sctx->cur_ino) in process_recorded_refs()
4582 ret = get_cur_inode_state(sctx, cur->dir, cur->dir_gen, NULL, NULL); in process_recorded_refs()
4588 ret = cache_dir_utimes(sctx, cur->dir, cur->dir_gen); in process_recorded_refs()
4593 ret = can_rmdir(sctx, cur->dir, cur->dir_gen); in process_recorded_refs()
4597 ret = get_cur_path(sctx, cur->dir, in process_recorded_refs()
4601 ret = send_rmdir(sctx, valid_path); in process_recorded_refs()
4613 free_recorded_refs(sctx); in process_recorded_refs()
4653 struct send_ctx *sctx) in record_ref_in_tree() argument
4671 ret = get_cur_path(sctx, dir, dir_gen, path); in record_ref_in_tree()
4697 struct send_ctx *sctx = ctx; in record_new_ref_if_needed() local
4703 ret = get_inode_gen(sctx->send_root, dir, &dir_gen); in record_new_ref_if_needed()
4710 node = rb_find(&data, &sctx->rbtree_deleted_refs, rbtree_ref_comp); in record_new_ref_if_needed()
4715 ret = record_ref_in_tree(&sctx->rbtree_new_refs, in record_new_ref_if_needed()
4716 &sctx->new_refs, name, dir, dir_gen, in record_new_ref_if_needed()
4717 sctx); in record_new_ref_if_needed()
4727 struct send_ctx *sctx = ctx; in record_deleted_ref_if_needed() local
4733 ret = get_inode_gen(sctx->parent_root, dir, &dir_gen); in record_deleted_ref_if_needed()
4740 node = rb_find(&data, &sctx->rbtree_new_refs, rbtree_ref_comp); in record_deleted_ref_if_needed()
4745 ret = record_ref_in_tree(&sctx->rbtree_deleted_refs, in record_deleted_ref_if_needed()
4746 &sctx->deleted_refs, name, dir, in record_deleted_ref_if_needed()
4747 dir_gen, sctx); in record_deleted_ref_if_needed()
4753 static int record_new_ref(struct send_ctx *sctx) in record_new_ref() argument
4757 ret = iterate_inode_ref(sctx->send_root, sctx->left_path, in record_new_ref()
4758 sctx->cmp_key, 0, record_new_ref_if_needed, sctx); in record_new_ref()
4767 static int record_deleted_ref(struct send_ctx *sctx) in record_deleted_ref() argument
4771 ret = iterate_inode_ref(sctx->parent_root, sctx->right_path, in record_deleted_ref()
4772 sctx->cmp_key, 0, record_deleted_ref_if_needed, in record_deleted_ref()
4773 sctx); in record_deleted_ref()
4782 static int record_changed_ref(struct send_ctx *sctx) in record_changed_ref() argument
4786 ret = iterate_inode_ref(sctx->send_root, sctx->left_path, in record_changed_ref()
4787 sctx->cmp_key, 0, record_new_ref_if_needed, sctx); in record_changed_ref()
4790 ret = iterate_inode_ref(sctx->parent_root, sctx->right_path, in record_changed_ref()
4791 sctx->cmp_key, 0, record_deleted_ref_if_needed, sctx); in record_changed_ref()
4804 static int process_all_refs(struct send_ctx *sctx, in process_all_refs() argument
4821 root = sctx->send_root; in process_all_refs()
4824 root = sctx->parent_root; in process_all_refs()
4827 btrfs_err(sctx->send_root->fs_info, in process_all_refs()
4833 key.objectid = sctx->cmp_key->objectid; in process_all_refs()
4842 ret = iterate_inode_ref(root, path, &found_key, 0, cb, sctx); in process_all_refs()
4858 ret = process_recorded_refs(sctx, &pending_move); in process_all_refs()
4864 static int send_set_xattr(struct send_ctx *sctx, in send_set_xattr() argument
4871 ret = begin_cmd(sctx, BTRFS_SEND_C_SET_XATTR); in send_set_xattr()
4875 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path); in send_set_xattr()
4876 TLV_PUT_STRING(sctx, BTRFS_SEND_A_XATTR_NAME, name, name_len); in send_set_xattr()
4877 TLV_PUT(sctx, BTRFS_SEND_A_XATTR_DATA, data, data_len); in send_set_xattr()
4879 ret = send_cmd(sctx); in send_set_xattr()
4886 static int send_remove_xattr(struct send_ctx *sctx, in send_remove_xattr() argument
4892 ret = begin_cmd(sctx, BTRFS_SEND_C_REMOVE_XATTR); in send_remove_xattr()
4896 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path); in send_remove_xattr()
4897 TLV_PUT_STRING(sctx, BTRFS_SEND_A_XATTR_NAME, name, name_len); in send_remove_xattr()
4899 ret = send_cmd(sctx); in send_remove_xattr()
4911 struct send_ctx *sctx = ctx; in __process_new_xattr() local
4939 ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p); in __process_new_xattr()
4943 ret = send_set_xattr(sctx, p, name, name_len, data, data_len); in __process_new_xattr()
4955 struct send_ctx *sctx = ctx; in __process_deleted_xattr() local
4962 ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p); in __process_deleted_xattr()
4966 ret = send_remove_xattr(sctx, p, name, name_len); in __process_deleted_xattr()
4973 static int process_new_xattr(struct send_ctx *sctx) in process_new_xattr() argument
4977 ret = iterate_dir_item(sctx->send_root, sctx->left_path, in process_new_xattr()
4978 __process_new_xattr, sctx); in process_new_xattr()
4983 static int process_deleted_xattr(struct send_ctx *sctx) in process_deleted_xattr() argument
4985 return iterate_dir_item(sctx->parent_root, sctx->right_path, in process_deleted_xattr()
4986 __process_deleted_xattr, sctx); in process_deleted_xattr()
5051 struct send_ctx *sctx = ctx; in __process_changed_new_xattr() local
5055 ret = find_xattr(sctx->parent_root, sctx->right_path, in __process_changed_new_xattr()
5056 sctx->cmp_key, name, name_len, &found_data, in __process_changed_new_xattr()
5081 struct send_ctx *sctx = ctx; in __process_changed_deleted_xattr() local
5083 ret = find_xattr(sctx->send_root, sctx->left_path, sctx->cmp_key, in __process_changed_deleted_xattr()
5094 static int process_changed_xattr(struct send_ctx *sctx) in process_changed_xattr() argument
5098 ret = iterate_dir_item(sctx->send_root, sctx->left_path, in process_changed_xattr()
5099 __process_changed_new_xattr, sctx); in process_changed_xattr()
5102 ret = iterate_dir_item(sctx->parent_root, sctx->right_path, in process_changed_xattr()
5103 __process_changed_deleted_xattr, sctx); in process_changed_xattr()
5109 static int process_all_new_xattrs(struct send_ctx *sctx) in process_all_new_xattrs() argument
5122 root = sctx->send_root; in process_all_new_xattrs()
5124 key.objectid = sctx->cmp_key->objectid; in process_all_new_xattrs()
5134 ret = iterate_dir_item(root, path, __process_new_xattr, sctx); in process_all_new_xattrs()
5146 static int send_verity(struct send_ctx *sctx, struct fs_path *path, in send_verity() argument
5151 ret = begin_cmd(sctx, BTRFS_SEND_C_ENABLE_VERITY); in send_verity()
5155 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path); in send_verity()
5156 TLV_PUT_U8(sctx, BTRFS_SEND_A_VERITY_ALGORITHM, in send_verity()
5158 TLV_PUT_U32(sctx, BTRFS_SEND_A_VERITY_BLOCK_SIZE, in send_verity()
5160 TLV_PUT(sctx, BTRFS_SEND_A_VERITY_SALT_DATA, desc->salt, in send_verity()
5162 TLV_PUT(sctx, BTRFS_SEND_A_VERITY_SIG_DATA, desc->signature, in send_verity()
5165 ret = send_cmd(sctx); in send_verity()
5172 static int process_verity(struct send_ctx *sctx) in process_verity() argument
5175 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info; in process_verity()
5179 inode = btrfs_iget(fs_info->sb, sctx->cur_ino, sctx->send_root); in process_verity()
5191 if (!sctx->verity_descriptor) { in process_verity()
5192 sctx->verity_descriptor = kvmalloc(FS_VERITY_MAX_DESCRIPTOR_SIZE, in process_verity()
5194 if (!sctx->verity_descriptor) { in process_verity()
5200 ret = btrfs_get_verity_descriptor(inode, sctx->verity_descriptor, ret); in process_verity()
5209 ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p); in process_verity()
5213 ret = send_verity(sctx, p, sctx->verity_descriptor); in process_verity()
5224 static inline u64 max_send_read_size(const struct send_ctx *sctx) in max_send_read_size() argument
5226 return sctx->send_max_size - SZ_16K; in max_send_read_size()
5229 static int put_data_header(struct send_ctx *sctx, u32 len) in put_data_header() argument
5231 if (WARN_ON_ONCE(sctx->put_data)) in put_data_header()
5233 sctx->put_data = true; in put_data_header()
5234 if (sctx->proto >= 2) { in put_data_header()
5239 if (sctx->send_max_size - sctx->send_size < sizeof(__le16) + len) in put_data_header()
5241 put_unaligned_le16(BTRFS_SEND_A_DATA, sctx->send_buf + sctx->send_size); in put_data_header()
5242 sctx->send_size += sizeof(__le16); in put_data_header()
5246 if (sctx->send_max_size - sctx->send_size < sizeof(*hdr) + len) in put_data_header()
5248 hdr = (struct btrfs_tlv_header *)(sctx->send_buf + sctx->send_size); in put_data_header()
5251 sctx->send_size += sizeof(*hdr); in put_data_header()
5256 static int put_file_data(struct send_ctx *sctx, u64 offset, u32 len) in put_file_data() argument
5258 struct btrfs_root *root = sctx->send_root; in put_file_data()
5266 ret = put_data_header(sctx, len); in put_file_data()
5276 page = find_lock_page(sctx->cur_inode->i_mapping, index); in put_file_data()
5278 page_cache_sync_readahead(sctx->cur_inode->i_mapping, in put_file_data()
5279 &sctx->ra, NULL, index, in put_file_data()
5282 page = find_or_create_page(sctx->cur_inode->i_mapping, in put_file_data()
5291 page_cache_async_readahead(sctx->cur_inode->i_mapping, in put_file_data()
5292 &sctx->ra, NULL, page_folio(page), in put_file_data()
5302 page_offset(page), sctx->cur_ino, in put_file_data()
5303 sctx->send_root->root_key.objectid); in put_file_data()
5310 memcpy_from_page(sctx->send_buf + sctx->send_size, page, in put_file_data()
5317 sctx->send_size += cur_len; in put_file_data()
5327 static int send_write(struct send_ctx *sctx, u64 offset, u32 len) in send_write() argument
5329 struct btrfs_fs_info *fs_info = sctx->send_root->fs_info; in send_write()
5339 ret = begin_cmd(sctx, BTRFS_SEND_C_WRITE); in send_write()
5343 ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p); in send_write()
5347 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p); in send_write()
5348 TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset); in send_write()
5349 ret = put_file_data(sctx, offset, len); in send_write()
5353 ret = send_cmd(sctx); in send_write()
5364 static int send_clone(struct send_ctx *sctx, in send_clone() argument
5372 btrfs_debug(sctx->send_root->fs_info, in send_clone()
5381 ret = begin_cmd(sctx, BTRFS_SEND_C_CLONE); in send_clone()
5385 ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p); in send_clone()
5389 TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset); in send_clone()
5390 TLV_PUT_U64(sctx, BTRFS_SEND_A_CLONE_LEN, len); in send_clone()
5391 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p); in send_clone()
5393 if (clone_root->root == sctx->send_root) { in send_clone()
5394 ret = get_inode_gen(sctx->send_root, clone_root->ino, &gen); in send_clone()
5397 ret = get_cur_path(sctx, clone_root->ino, gen, p); in send_clone()
5414 TLV_PUT_UUID(sctx, BTRFS_SEND_A_CLONE_UUID, in send_clone()
5417 TLV_PUT_UUID(sctx, BTRFS_SEND_A_CLONE_UUID, in send_clone()
5419 TLV_PUT_U64(sctx, BTRFS_SEND_A_CLONE_CTRANSID, in send_clone()
5421 TLV_PUT_PATH(sctx, BTRFS_SEND_A_CLONE_PATH, p); in send_clone()
5422 TLV_PUT_U64(sctx, BTRFS_SEND_A_CLONE_OFFSET, in send_clone()
5425 ret = send_cmd(sctx); in send_clone()
5436 static int send_update_extent(struct send_ctx *sctx, in send_update_extent() argument
5446 ret = begin_cmd(sctx, BTRFS_SEND_C_UPDATE_EXTENT); in send_update_extent()
5450 ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p); in send_update_extent()
5454 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p); in send_update_extent()
5455 TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset); in send_update_extent()
5456 TLV_PUT_U64(sctx, BTRFS_SEND_A_SIZE, len); in send_update_extent()
5458 ret = send_cmd(sctx); in send_update_extent()
5466 static int send_hole(struct send_ctx *sctx, u64 end) in send_hole() argument
5469 u64 read_size = max_send_read_size(sctx); in send_hole()
5470 u64 offset = sctx->cur_inode_last_extent; in send_hole()
5479 if (offset >= sctx->cur_inode_size) in send_hole()
5486 end = min_t(u64, end, sctx->cur_inode_size); in send_hole()
5488 if (sctx->flags & BTRFS_SEND_FLAG_NO_FILE_DATA) in send_hole()
5489 return send_update_extent(sctx, offset, end - offset); in send_hole()
5494 ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p); in send_hole()
5500 ret = begin_cmd(sctx, BTRFS_SEND_C_WRITE); in send_hole()
5503 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p); in send_hole()
5504 TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset); in send_hole()
5505 ret = put_data_header(sctx, len); in send_hole()
5508 memset(sctx->send_buf + sctx->send_size, 0, len); in send_hole()
5509 sctx->send_size += len; in send_hole()
5510 ret = send_cmd(sctx); in send_hole()
5515 sctx->cur_inode_next_write_offset = offset; in send_hole()
5521 static int send_encoded_inline_extent(struct send_ctx *sctx, in send_encoded_inline_extent() argument
5525 struct btrfs_root *root = sctx->send_root; in send_encoded_inline_extent()
5536 inode = btrfs_iget(fs_info->sb, sctx->cur_ino, root); in send_encoded_inline_extent()
5546 ret = begin_cmd(sctx, BTRFS_SEND_C_ENCODED_WRITE); in send_encoded_inline_extent()
5550 ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, fspath); in send_encoded_inline_extent()
5559 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, fspath); in send_encoded_inline_extent()
5560 TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset); in send_encoded_inline_extent()
5561 TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_FILE_LEN, in send_encoded_inline_extent()
5563 TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_LEN, ram_bytes); in send_encoded_inline_extent()
5564 TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_OFFSET, offset - key.offset); in send_encoded_inline_extent()
5569 TLV_PUT_U32(sctx, BTRFS_SEND_A_COMPRESSION, ret); in send_encoded_inline_extent()
5571 ret = put_data_header(sctx, inline_size); in send_encoded_inline_extent()
5574 read_extent_buffer(leaf, sctx->send_buf + sctx->send_size, in send_encoded_inline_extent()
5576 sctx->send_size += inline_size; in send_encoded_inline_extent()
5578 ret = send_cmd(sctx); in send_encoded_inline_extent()
5587 static int send_encoded_extent(struct send_ctx *sctx, struct btrfs_path *path, in send_encoded_extent() argument
5590 struct btrfs_root *root = sctx->send_root; in send_encoded_extent()
5603 inode = btrfs_iget(fs_info->sb, sctx->cur_ino, root); in send_encoded_extent()
5613 ret = begin_cmd(sctx, BTRFS_SEND_C_ENCODED_WRITE); in send_encoded_extent()
5617 ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, fspath); in send_encoded_extent()
5626 TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, fspath); in send_encoded_extent()
5627 TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset); in send_encoded_extent()
5628 TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_FILE_LEN, in send_encoded_extent()
5631 TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_LEN, in send_encoded_extent()
5633 TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_OFFSET, in send_encoded_extent()
5639 TLV_PUT_U32(sctx, BTRFS_SEND_A_COMPRESSION, ret); in send_encoded_extent()
5640 TLV_PUT_U32(sctx, BTRFS_SEND_A_ENCRYPTION, 0); in send_encoded_extent()
5642 ret = put_data_header(sctx, disk_num_bytes); in send_encoded_extent()
5651 data_offset = PAGE_ALIGN(sctx->send_size); in send_encoded_extent()
5652 if (data_offset > sctx->send_max_size || in send_encoded_extent()
5653 sctx->send_max_size - data_offset < disk_num_bytes) { in send_encoded_extent()
5664 sctx->send_buf_pages + in send_encoded_extent()
5669 hdr = (struct btrfs_cmd_header *)sctx->send_buf; in send_encoded_extent()
5670 hdr->len = cpu_to_le32(sctx->send_size + disk_num_bytes - sizeof(*hdr)); in send_encoded_extent()
5672 crc = btrfs_crc32c(0, sctx->send_buf, sctx->send_size); in send_encoded_extent()
5673 crc = btrfs_crc32c(crc, sctx->send_buf + data_offset, disk_num_bytes); in send_encoded_extent()
5676 ret = write_buf(sctx->send_filp, sctx->send_buf, sctx->send_size, in send_encoded_extent()
5677 &sctx->send_off); in send_encoded_extent()
5679 ret = write_buf(sctx->send_filp, sctx->send_buf + data_offset, in send_encoded_extent()
5680 disk_num_bytes, &sctx->send_off); in send_encoded_extent()
5682 sctx->send_size = 0; in send_encoded_extent()
5683 sctx->put_data = false; in send_encoded_extent()
5692 static int send_extent_data(struct send_ctx *sctx, struct btrfs_path *path, in send_extent_data() argument
5698 u64 read_size = max_send_read_size(sctx); in send_extent_data()
5701 if (sctx->flags & BTRFS_SEND_FLAG_NO_FILE_DATA) in send_extent_data()
5702 return send_update_extent(sctx, offset, len); in send_extent_data()
5706 if ((sctx->flags & BTRFS_SEND_FLAG_COMPRESSED) && in send_extent_data()
5721 return send_encoded_inline_extent(sctx, path, offset, in send_extent_data()
5725 return send_encoded_extent(sctx, path, offset, len); in send_extent_data()
5729 if (sctx->cur_inode == NULL) { in send_extent_data()
5730 struct btrfs_root *root = sctx->send_root; in send_extent_data()
5732 sctx->cur_inode = btrfs_iget(root->fs_info->sb, sctx->cur_ino, root); in send_extent_data()
5733 if (IS_ERR(sctx->cur_inode)) { in send_extent_data()
5734 int err = PTR_ERR(sctx->cur_inode); in send_extent_data()
5736 sctx->cur_inode = NULL; in send_extent_data()
5739 memset(&sctx->ra, 0, sizeof(struct file_ra_state)); in send_extent_data()
5740 file_ra_state_init(&sctx->ra, sctx->cur_inode->i_mapping); in send_extent_data()
5761 sctx->clean_page_cache = (sctx->cur_inode->i_mapping->nrpages == 0); in send_extent_data()
5762 sctx->page_cache_clear_start = round_down(offset, PAGE_SIZE); in send_extent_data()
5769 ret = send_write(sctx, offset + sent, size); in send_extent_data()
5775 if (sctx->clean_page_cache && PAGE_ALIGNED(end)) { in send_extent_data()
5799 truncate_inode_pages_range(&sctx->cur_inode->i_data, in send_extent_data()
5800 sctx->page_cache_clear_start, in send_extent_data()
5802 sctx->page_cache_clear_start = end; in send_extent_data()
5815 static int send_capabilities(struct send_ctx *sctx) in send_capabilities() argument
5830 di = btrfs_lookup_xattr(NULL, sctx->send_root, path, sctx->cur_ino, in send_capabilities()
5850 ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, fspath); in send_capabilities()
5857 ret = send_set_xattr(sctx, fspath, XATTR_NAME_CAPS, in send_capabilities()
5866 static int clone_range(struct send_ctx *sctx, struct btrfs_path *dst_path, in clone_range() argument
5892 len == sctx->send_root->fs_info->sectorsize) in clone_range()
5893 return send_extent_data(sctx, dst_path, offset, len); in clone_range()
5991 ret = send_extent_data(sctx, dst_path, offset, in clone_range()
6056 offset + clone_len < sctx->cur_inode_size) { in clone_range()
6062 ret = send_clone(sctx, offset, slen, in clone_range()
6067 ret = send_extent_data(sctx, dst_path, in clone_range()
6071 ret = send_clone(sctx, offset, clone_len, in clone_range()
6094 ret = send_extent_data(sctx, dst_path, offset, in clone_range()
6116 if (clone_root->root == sctx->send_root && in clone_range()
6117 clone_root->ino == sctx->cur_ino && in clone_range()
6118 clone_root->offset >= sctx->cur_inode_next_write_offset) in clone_range()
6127 ret = send_extent_data(sctx, dst_path, offset, len); in clone_range()
6135 static int send_write_or_clone(struct send_ctx *sctx, in send_write_or_clone() argument
6143 u64 bs = sctx->send_root->fs_info->sb->s_blocksize; in send_write_or_clone()
6145 end = min_t(u64, btrfs_file_extent_end(path), sctx->cur_inode_size); in send_write_or_clone()
6158 ret = clone_range(sctx, path, clone_root, disk_byte, in send_write_or_clone()
6161 ret = send_extent_data(sctx, path, offset, end - offset); in send_write_or_clone()
6163 sctx->cur_inode_next_write_offset = end; in send_write_or_clone()
6167 static int is_extent_unchanged(struct send_ctx *sctx, in is_extent_unchanged() argument
6232 ret = btrfs_search_slot_for_read(sctx->parent_root, &key, path, 0, 0); in is_extent_unchanged()
6322 ret = btrfs_next_item(sctx->parent_root, path); in is_extent_unchanged()
6357 static int get_last_extent(struct send_ctx *sctx, u64 offset) in get_last_extent() argument
6360 struct btrfs_root *root = sctx->send_root; in get_last_extent()
6368 sctx->cur_inode_last_extent = 0; in get_last_extent()
6370 key.objectid = sctx->cur_ino; in get_last_extent()
6378 if (key.objectid != sctx->cur_ino || key.type != BTRFS_EXTENT_DATA_KEY) in get_last_extent()
6381 sctx->cur_inode_last_extent = btrfs_file_extent_end(path); in get_last_extent()
6387 static int range_is_hole_in_parent(struct send_ctx *sctx, in range_is_hole_in_parent() argument
6393 struct btrfs_root *root = sctx->parent_root; in range_is_hole_in_parent()
6401 key.objectid = sctx->cur_ino; in range_is_hole_in_parent()
6426 if (key.objectid < sctx->cur_ino || in range_is_hole_in_parent()
6429 if (key.objectid > sctx->cur_ino || in range_is_hole_in_parent()
6453 static int maybe_send_hole(struct send_ctx *sctx, struct btrfs_path *path, in maybe_send_hole() argument
6458 if (sctx->cur_ino != key->objectid || !need_send_hole(sctx)) in maybe_send_hole()
6461 if (sctx->cur_inode_last_extent == (u64)-1) { in maybe_send_hole()
6462 ret = get_last_extent(sctx, key->offset - 1); in maybe_send_hole()
6468 sctx->cur_inode_last_extent < key->offset) { in maybe_send_hole()
6476 ret = get_last_extent(sctx, key->offset - 1); in maybe_send_hole()
6481 if (sctx->cur_inode_last_extent < key->offset) { in maybe_send_hole()
6482 ret = range_is_hole_in_parent(sctx, in maybe_send_hole()
6483 sctx->cur_inode_last_extent, in maybe_send_hole()
6488 ret = send_hole(sctx, key->offset); in maybe_send_hole()
6492 sctx->cur_inode_last_extent = btrfs_file_extent_end(path); in maybe_send_hole()
6496 static int process_extent(struct send_ctx *sctx, in process_extent() argument
6503 if (S_ISLNK(sctx->cur_inode_mode)) in process_extent()
6506 if (sctx->parent_root && !sctx->cur_inode_new) { in process_extent()
6507 ret = is_extent_unchanged(sctx, path, key); in process_extent()
6542 ret = find_extent_clone(sctx, path, key->objectid, key->offset, in process_extent()
6543 sctx->cur_inode_size, &found_clone); in process_extent()
6547 ret = send_write_or_clone(sctx, path, key, found_clone); in process_extent()
6551 ret = maybe_send_hole(sctx, path, key); in process_extent()
6556 static int process_all_extents(struct send_ctx *sctx) in process_all_extents() argument
6565 root = sctx->send_root; in process_all_extents()
6570 key.objectid = sctx->cmp_key->objectid; in process_all_extents()
6580 ret = process_extent(sctx, path, &found_key); in process_all_extents()
6592 static int process_recorded_refs_if_needed(struct send_ctx *sctx, int at_end, in process_recorded_refs_if_needed() argument
6598 if (sctx->cur_ino == 0) in process_recorded_refs_if_needed()
6600 if (!at_end && sctx->cur_ino == sctx->cmp_key->objectid && in process_recorded_refs_if_needed()
6601 sctx->cmp_key->type <= BTRFS_INODE_EXTREF_KEY) in process_recorded_refs_if_needed()
6603 if (list_empty(&sctx->new_refs) && list_empty(&sctx->deleted_refs)) in process_recorded_refs_if_needed()
6606 ret = process_recorded_refs(sctx, pending_move); in process_recorded_refs_if_needed()
6615 static int finish_inode_if_needed(struct send_ctx *sctx, int at_end) in finish_inode_if_needed() argument
6634 if (sctx->ignore_cur_inode) in finish_inode_if_needed()
6637 ret = process_recorded_refs_if_needed(sctx, at_end, &pending_move, in finish_inode_if_needed()
6655 sctx->send_progress = sctx->cur_ino + 1; in finish_inode_if_needed()
6657 if (sctx->cur_ino == 0 || sctx->cur_inode_deleted) in finish_inode_if_needed()
6659 if (!at_end && sctx->cmp_key->objectid == sctx->cur_ino) in finish_inode_if_needed()
6661 ret = get_inode_info(sctx->send_root, sctx->cur_ino, &info); in finish_inode_if_needed()
6669 if (!sctx->parent_root || sctx->cur_inode_new) { in finish_inode_if_needed()
6671 if (!S_ISLNK(sctx->cur_inode_mode)) in finish_inode_if_needed()
6673 if (sctx->cur_inode_next_write_offset == sctx->cur_inode_size) in finish_inode_if_needed()
6678 ret = get_inode_info(sctx->parent_root, sctx->cur_ino, &info); in finish_inode_if_needed()
6689 if (!S_ISLNK(sctx->cur_inode_mode) && left_mode != right_mode) in finish_inode_if_needed()
6691 if (!S_ISLNK(sctx->cur_inode_mode) && left_fileattr != right_fileattr) in finish_inode_if_needed()
6693 if ((old_size == sctx->cur_inode_size) || in finish_inode_if_needed()
6694 (sctx->cur_inode_size > old_size && in finish_inode_if_needed()
6695 sctx->cur_inode_next_write_offset == sctx->cur_inode_size)) in finish_inode_if_needed()
6699 if (S_ISREG(sctx->cur_inode_mode)) { in finish_inode_if_needed()
6700 if (need_send_hole(sctx)) { in finish_inode_if_needed()
6701 if (sctx->cur_inode_last_extent == (u64)-1 || in finish_inode_if_needed()
6702 sctx->cur_inode_last_extent < in finish_inode_if_needed()
6703 sctx->cur_inode_size) { in finish_inode_if_needed()
6704 ret = get_last_extent(sctx, (u64)-1); in finish_inode_if_needed()
6708 if (sctx->cur_inode_last_extent < in finish_inode_if_needed()
6709 sctx->cur_inode_size) { in finish_inode_if_needed()
6710 ret = send_hole(sctx, sctx->cur_inode_size); in finish_inode_if_needed()
6716 ret = send_truncate(sctx, sctx->cur_ino, in finish_inode_if_needed()
6717 sctx->cur_inode_gen, in finish_inode_if_needed()
6718 sctx->cur_inode_size); in finish_inode_if_needed()
6725 ret = send_chown(sctx, sctx->cur_ino, sctx->cur_inode_gen, in finish_inode_if_needed()
6731 ret = send_chmod(sctx, sctx->cur_ino, sctx->cur_inode_gen, in finish_inode_if_needed()
6737 ret = send_fileattr(sctx, sctx->cur_ino, sctx->cur_inode_gen, in finish_inode_if_needed()
6743 if (proto_cmd_ok(sctx, BTRFS_SEND_C_ENABLE_VERITY) in finish_inode_if_needed()
6744 && sctx->cur_inode_needs_verity) { in finish_inode_if_needed()
6745 ret = process_verity(sctx); in finish_inode_if_needed()
6750 ret = send_capabilities(sctx); in finish_inode_if_needed()
6758 if (!is_waiting_for_move(sctx, sctx->cur_ino)) { in finish_inode_if_needed()
6759 ret = apply_children_dir_moves(sctx); in finish_inode_if_needed()
6769 sctx->send_progress = sctx->cur_ino + 1; in finish_inode_if_needed()
6777 if (S_ISDIR(sctx->cur_inode_mode) && sctx->cur_inode_size > 0) in finish_inode_if_needed()
6778 ret = cache_dir_utimes(sctx, sctx->cur_ino, sctx->cur_inode_gen); in finish_inode_if_needed()
6780 ret = send_utimes(sctx, sctx->cur_ino, sctx->cur_inode_gen); in finish_inode_if_needed()
6788 ret = trim_dir_utimes_cache(sctx); in finish_inode_if_needed()
6793 static void close_current_inode(struct send_ctx *sctx) in close_current_inode() argument
6797 if (sctx->cur_inode == NULL) in close_current_inode()
6800 i_size = i_size_read(sctx->cur_inode); in close_current_inode()
6808 if (sctx->clean_page_cache && sctx->page_cache_clear_start < i_size) in close_current_inode()
6809 truncate_inode_pages_range(&sctx->cur_inode->i_data, in close_current_inode()
6810 sctx->page_cache_clear_start, in close_current_inode()
6813 iput(sctx->cur_inode); in close_current_inode()
6814 sctx->cur_inode = NULL; in close_current_inode()
6817 static int changed_inode(struct send_ctx *sctx, in changed_inode() argument
6821 struct btrfs_key *key = sctx->cmp_key; in changed_inode()
6827 close_current_inode(sctx); in changed_inode()
6829 sctx->cur_ino = key->objectid; in changed_inode()
6830 sctx->cur_inode_new_gen = false; in changed_inode()
6831 sctx->cur_inode_last_extent = (u64)-1; in changed_inode()
6832 sctx->cur_inode_next_write_offset = 0; in changed_inode()
6833 sctx->ignore_cur_inode = false; in changed_inode()
6840 sctx->send_progress = sctx->cur_ino; in changed_inode()
6844 left_ii = btrfs_item_ptr(sctx->left_path->nodes[0], in changed_inode()
6845 sctx->left_path->slots[0], in changed_inode()
6847 left_gen = btrfs_inode_generation(sctx->left_path->nodes[0], in changed_inode()
6850 right_ii = btrfs_item_ptr(sctx->right_path->nodes[0], in changed_inode()
6851 sctx->right_path->slots[0], in changed_inode()
6853 right_gen = btrfs_inode_generation(sctx->right_path->nodes[0], in changed_inode()
6857 right_ii = btrfs_item_ptr(sctx->right_path->nodes[0], in changed_inode()
6858 sctx->right_path->slots[0], in changed_inode()
6861 right_gen = btrfs_inode_generation(sctx->right_path->nodes[0], in changed_inode()
6870 sctx->cur_ino != BTRFS_FIRST_FREE_OBJECTID) in changed_inode()
6871 sctx->cur_inode_new_gen = true; in changed_inode()
6909 if (btrfs_inode_nlink(sctx->left_path->nodes[0], left_ii) == 0) { in changed_inode()
6910 sctx->ignore_cur_inode = true; in changed_inode()
6913 sctx->cur_inode_gen = left_gen; in changed_inode()
6914 sctx->cur_inode_new = true; in changed_inode()
6915 sctx->cur_inode_deleted = false; in changed_inode()
6916 sctx->cur_inode_size = btrfs_inode_size( in changed_inode()
6917 sctx->left_path->nodes[0], left_ii); in changed_inode()
6918 sctx->cur_inode_mode = btrfs_inode_mode( in changed_inode()
6919 sctx->left_path->nodes[0], left_ii); in changed_inode()
6920 sctx->cur_inode_rdev = btrfs_inode_rdev( in changed_inode()
6921 sctx->left_path->nodes[0], left_ii); in changed_inode()
6922 if (sctx->cur_ino != BTRFS_FIRST_FREE_OBJECTID) in changed_inode()
6923 ret = send_create_inode_if_needed(sctx); in changed_inode()
6925 sctx->cur_inode_gen = right_gen; in changed_inode()
6926 sctx->cur_inode_new = false; in changed_inode()
6927 sctx->cur_inode_deleted = true; in changed_inode()
6928 sctx->cur_inode_size = btrfs_inode_size( in changed_inode()
6929 sctx->right_path->nodes[0], right_ii); in changed_inode()
6930 sctx->cur_inode_mode = btrfs_inode_mode( in changed_inode()
6931 sctx->right_path->nodes[0], right_ii); in changed_inode()
6935 new_nlinks = btrfs_inode_nlink(sctx->left_path->nodes[0], left_ii); in changed_inode()
6936 old_nlinks = btrfs_inode_nlink(sctx->right_path->nodes[0], right_ii); in changed_inode()
6938 sctx->ignore_cur_inode = true; in changed_inode()
6941 sctx->cur_inode_new_gen = 1; in changed_inode()
6950 if (sctx->cur_inode_new_gen) { in changed_inode()
6955 sctx->cur_inode_gen = right_gen; in changed_inode()
6956 sctx->cur_inode_new = false; in changed_inode()
6957 sctx->cur_inode_deleted = true; in changed_inode()
6958 sctx->cur_inode_size = btrfs_inode_size( in changed_inode()
6959 sctx->right_path->nodes[0], right_ii); in changed_inode()
6960 sctx->cur_inode_mode = btrfs_inode_mode( in changed_inode()
6961 sctx->right_path->nodes[0], right_ii); in changed_inode()
6962 ret = process_all_refs(sctx, in changed_inode()
6972 sctx->cur_inode_gen = left_gen; in changed_inode()
6973 sctx->cur_inode_new = true; in changed_inode()
6974 sctx->cur_inode_deleted = false; in changed_inode()
6975 sctx->cur_inode_size = btrfs_inode_size( in changed_inode()
6976 sctx->left_path->nodes[0], in changed_inode()
6978 sctx->cur_inode_mode = btrfs_inode_mode( in changed_inode()
6979 sctx->left_path->nodes[0], in changed_inode()
6981 sctx->cur_inode_rdev = btrfs_inode_rdev( in changed_inode()
6982 sctx->left_path->nodes[0], in changed_inode()
6984 ret = send_create_inode_if_needed(sctx); in changed_inode()
6988 ret = process_all_refs(sctx, BTRFS_COMPARE_TREE_NEW); in changed_inode()
6996 sctx->send_progress = sctx->cur_ino + 1; in changed_inode()
7002 ret = process_all_extents(sctx); in changed_inode()
7005 ret = process_all_new_xattrs(sctx); in changed_inode()
7010 sctx->cur_inode_gen = left_gen; in changed_inode()
7011 sctx->cur_inode_new = false; in changed_inode()
7012 sctx->cur_inode_new_gen = false; in changed_inode()
7013 sctx->cur_inode_deleted = false; in changed_inode()
7014 sctx->cur_inode_size = btrfs_inode_size( in changed_inode()
7015 sctx->left_path->nodes[0], left_ii); in changed_inode()
7016 sctx->cur_inode_mode = btrfs_inode_mode( in changed_inode()
7017 sctx->left_path->nodes[0], left_ii); in changed_inode()
7035 static int changed_ref(struct send_ctx *sctx, in changed_ref() argument
7040 if (sctx->cur_ino != sctx->cmp_key->objectid) { in changed_ref()
7041 inconsistent_snapshot_error(sctx, result, "reference"); in changed_ref()
7045 if (!sctx->cur_inode_new_gen && in changed_ref()
7046 sctx->cur_ino != BTRFS_FIRST_FREE_OBJECTID) { in changed_ref()
7048 ret = record_new_ref(sctx); in changed_ref()
7050 ret = record_deleted_ref(sctx); in changed_ref()
7052 ret = record_changed_ref(sctx); in changed_ref()
7063 static int changed_xattr(struct send_ctx *sctx, in changed_xattr() argument
7068 if (sctx->cur_ino != sctx->cmp_key->objectid) { in changed_xattr()
7069 inconsistent_snapshot_error(sctx, result, "xattr"); in changed_xattr()
7073 if (!sctx->cur_inode_new_gen && !sctx->cur_inode_deleted) { in changed_xattr()
7075 ret = process_new_xattr(sctx); in changed_xattr()
7077 ret = process_deleted_xattr(sctx); in changed_xattr()
7079 ret = process_changed_xattr(sctx); in changed_xattr()
7090 static int changed_extent(struct send_ctx *sctx, in changed_extent() argument
7108 if (sctx->cur_ino != sctx->cmp_key->objectid) in changed_extent()
7111 if (!sctx->cur_inode_new_gen && !sctx->cur_inode_deleted) { in changed_extent()
7113 ret = process_extent(sctx, sctx->left_path, in changed_extent()
7114 sctx->cmp_key); in changed_extent()
7120 static int changed_verity(struct send_ctx *sctx, enum btrfs_compare_tree_result result) in changed_verity() argument
7124 if (!sctx->cur_inode_new_gen && !sctx->cur_inode_deleted) { in changed_verity()
7126 sctx->cur_inode_needs_verity = true; in changed_verity()
7131 static int dir_changed(struct send_ctx *sctx, u64 dir) in dir_changed() argument
7136 ret = get_inode_gen(sctx->send_root, dir, &new_gen); in dir_changed()
7140 ret = get_inode_gen(sctx->parent_root, dir, &orig_gen); in dir_changed()
7147 static int compare_refs(struct send_ctx *sctx, struct btrfs_path *path, in compare_refs() argument
7163 ret = dir_changed(sctx, dirid); in compare_refs()
7178 ret = dir_changed(sctx, dirid); in compare_refs()
7195 struct send_ctx *sctx) in changed_cb() argument
7224 lockdep_assert_not_held(&sctx->send_root->fs_info->commit_root_sem); in changed_cb()
7246 ret = compare_refs(sctx, left_path, key); in changed_cb()
7252 return maybe_send_hole(sctx, left_path, key); in changed_cb()
7260 sctx->left_path = left_path; in changed_cb()
7261 sctx->right_path = right_path; in changed_cb()
7262 sctx->cmp_key = key; in changed_cb()
7264 ret = finish_inode_if_needed(sctx, 0); in changed_cb()
7274 ret = changed_inode(sctx, result); in changed_cb()
7275 } else if (!sctx->ignore_cur_inode) { in changed_cb()
7278 ret = changed_ref(sctx, result); in changed_cb()
7280 ret = changed_xattr(sctx, result); in changed_cb()
7282 ret = changed_extent(sctx, result); in changed_cb()
7285 ret = changed_verity(sctx, result); in changed_cb()
7292 static int search_key_again(const struct send_ctx *sctx, in search_key_again() argument
7316 (root == sctx->parent_root ? "parent" : "send"), in search_key_again()
7325 static int full_send_tree(struct send_ctx *sctx) in full_send_tree() argument
7328 struct btrfs_root *send_root = sctx->send_root; in full_send_tree()
7343 sctx->last_reloc_trans = fs_info->last_reloc_trans; in full_send_tree()
7356 BTRFS_COMPARE_TREE_NEW, sctx); in full_send_tree()
7361 if (fs_info->last_reloc_trans > sctx->last_reloc_trans) { in full_send_tree()
7362 sctx->last_reloc_trans = fs_info->last_reloc_trans; in full_send_tree()
7375 ret = search_key_again(sctx, send_root, path, &key); in full_send_tree()
7392 ret = finish_inode_if_needed(sctx, 1); in full_send_tree()
7566 const struct send_ctx *sctx) in restart_after_relocation() argument
7571 lockdep_assert_held_read(&sctx->send_root->fs_info->commit_root_sem); in restart_after_relocation()
7583 ret = search_key_again(sctx, sctx->send_root, left_path, left_key); in restart_after_relocation()
7588 ret = search_key_again(sctx, sctx->parent_root, right_path, right_key); in restart_after_relocation()
7615 root_level = btrfs_header_level(sctx->send_root->commit_root); in restart_after_relocation()
7622 root_level = btrfs_header_level(sctx->parent_root->commit_root); in restart_after_relocation()
7646 struct btrfs_root *right_root, struct send_ctx *sctx) in btrfs_compare_trees() argument
7775 sctx->last_reloc_trans = fs_info->last_reloc_trans; in btrfs_compare_trees()
7785 if (fs_info->last_reloc_trans > sctx->last_reloc_trans) { in btrfs_compare_trees()
7789 sctx); in btrfs_compare_trees()
7792 sctx->last_reloc_trans = fs_info->last_reloc_trans; in btrfs_compare_trees()
7827 sctx); in btrfs_compare_trees()
7840 sctx); in btrfs_compare_trees()
7856 sctx); in btrfs_compare_trees()
7862 sctx); in btrfs_compare_trees()
7875 &left_key, result, sctx); in btrfs_compare_trees()
7931 static int send_subvol(struct send_ctx *sctx) in send_subvol() argument
7935 if (!(sctx->flags & BTRFS_SEND_FLAG_OMIT_STREAM_HEADER)) { in send_subvol()
7936 ret = send_header(sctx); in send_subvol()
7941 ret = send_subvol_begin(sctx); in send_subvol()
7945 if (sctx->parent_root) { in send_subvol()
7946 ret = btrfs_compare_trees(sctx->send_root, sctx->parent_root, sctx); in send_subvol()
7949 ret = finish_inode_if_needed(sctx, 1); in send_subvol()
7953 ret = full_send_tree(sctx); in send_subvol()
7959 free_recorded_refs(sctx); in send_subvol()
7976 static int ensure_commit_roots_uptodate(struct send_ctx *sctx) in ensure_commit_roots_uptodate() argument
7982 if (sctx->parent_root && in ensure_commit_roots_uptodate()
7983 sctx->parent_root->node != sctx->parent_root->commit_root) in ensure_commit_roots_uptodate()
7986 for (i = 0; i < sctx->clone_roots_cnt; i++) in ensure_commit_roots_uptodate()
7987 if (sctx->clone_roots[i].root->node != in ensure_commit_roots_uptodate()
7988 sctx->clone_roots[i].root->commit_root) in ensure_commit_roots_uptodate()
7999 trans = btrfs_join_transaction(sctx->send_root); in ensure_commit_roots_uptodate()
8016 static int flush_delalloc_roots(struct send_ctx *sctx) in flush_delalloc_roots() argument
8018 struct btrfs_root *root = sctx->parent_root; in flush_delalloc_roots()
8029 for (i = 0; i < sctx->clone_roots_cnt; i++) { in flush_delalloc_roots()
8030 root = sctx->clone_roots[i].root; in flush_delalloc_roots()
8068 struct send_ctx *sctx = NULL; in btrfs_ioctl_send() local
8118 sctx = kzalloc(sizeof(struct send_ctx), GFP_KERNEL); in btrfs_ioctl_send()
8119 if (!sctx) { in btrfs_ioctl_send()
8124 INIT_LIST_HEAD(&sctx->new_refs); in btrfs_ioctl_send()
8125 INIT_LIST_HEAD(&sctx->deleted_refs); in btrfs_ioctl_send()
8127 btrfs_lru_cache_init(&sctx->name_cache, SEND_MAX_NAME_CACHE_SIZE); in btrfs_ioctl_send()
8128 btrfs_lru_cache_init(&sctx->backref_cache, SEND_MAX_BACKREF_CACHE_SIZE); in btrfs_ioctl_send()
8129 btrfs_lru_cache_init(&sctx->dir_created_cache, in btrfs_ioctl_send()
8135 btrfs_lru_cache_init(&sctx->dir_utimes_cache, 0); in btrfs_ioctl_send()
8137 sctx->pending_dir_moves = RB_ROOT; in btrfs_ioctl_send()
8138 sctx->waiting_dir_moves = RB_ROOT; in btrfs_ioctl_send()
8139 sctx->orphan_dirs = RB_ROOT; in btrfs_ioctl_send()
8140 sctx->rbtree_new_refs = RB_ROOT; in btrfs_ioctl_send()
8141 sctx->rbtree_deleted_refs = RB_ROOT; in btrfs_ioctl_send()
8143 sctx->flags = arg->flags; in btrfs_ioctl_send()
8151 sctx->proto = arg->version ?: BTRFS_SEND_STREAM_VERSION; in btrfs_ioctl_send()
8153 sctx->proto = 1; in btrfs_ioctl_send()
8155 if ((arg->flags & BTRFS_SEND_FLAG_COMPRESSED) && sctx->proto < 2) { in btrfs_ioctl_send()
8160 sctx->send_filp = fget(arg->send_fd); in btrfs_ioctl_send()
8161 if (!sctx->send_filp) { in btrfs_ioctl_send()
8166 sctx->send_root = send_root; in btrfs_ioctl_send()
8171 if (btrfs_root_dead(sctx->send_root)) { in btrfs_ioctl_send()
8176 sctx->clone_roots_cnt = arg->clone_sources_count; in btrfs_ioctl_send()
8178 if (sctx->proto >= 2) { in btrfs_ioctl_send()
8181 sctx->send_max_size = BTRFS_SEND_BUF_SIZE_V2; in btrfs_ioctl_send()
8182 sctx->send_buf = vmalloc(sctx->send_max_size); in btrfs_ioctl_send()
8183 if (!sctx->send_buf) { in btrfs_ioctl_send()
8187 send_buf_num_pages = sctx->send_max_size >> PAGE_SHIFT; in btrfs_ioctl_send()
8188 sctx->send_buf_pages = kcalloc(send_buf_num_pages, in btrfs_ioctl_send()
8189 sizeof(*sctx->send_buf_pages), in btrfs_ioctl_send()
8191 if (!sctx->send_buf_pages) { in btrfs_ioctl_send()
8196 sctx->send_buf_pages[i] = in btrfs_ioctl_send()
8197 vmalloc_to_page(sctx->send_buf + (i << PAGE_SHIFT)); in btrfs_ioctl_send()
8200 sctx->send_max_size = BTRFS_SEND_BUF_SIZE_V1; in btrfs_ioctl_send()
8201 sctx->send_buf = kvmalloc(sctx->send_max_size, GFP_KERNEL); in btrfs_ioctl_send()
8203 if (!sctx->send_buf) { in btrfs_ioctl_send()
8208 sctx->clone_roots = kvcalloc(sizeof(*sctx->clone_roots), in btrfs_ioctl_send()
8211 if (!sctx->clone_roots) { in btrfs_ioctl_send()
8258 sctx->clone_roots[i].root = clone_root; in btrfs_ioctl_send()
8266 sctx->parent_root = btrfs_get_fs_root(fs_info, arg->parent_root, in btrfs_ioctl_send()
8268 if (IS_ERR(sctx->parent_root)) { in btrfs_ioctl_send()
8269 ret = PTR_ERR(sctx->parent_root); in btrfs_ioctl_send()
8273 spin_lock(&sctx->parent_root->root_item_lock); in btrfs_ioctl_send()
8274 sctx->parent_root->send_in_progress++; in btrfs_ioctl_send()
8275 if (!btrfs_root_readonly(sctx->parent_root) || in btrfs_ioctl_send()
8276 btrfs_root_dead(sctx->parent_root)) { in btrfs_ioctl_send()
8277 spin_unlock(&sctx->parent_root->root_item_lock); in btrfs_ioctl_send()
8281 if (sctx->parent_root->dedupe_in_progress) { in btrfs_ioctl_send()
8282 dedupe_in_progress_warn(sctx->parent_root); in btrfs_ioctl_send()
8283 spin_unlock(&sctx->parent_root->root_item_lock); in btrfs_ioctl_send()
8287 spin_unlock(&sctx->parent_root->root_item_lock); in btrfs_ioctl_send()
8295 sctx->clone_roots[sctx->clone_roots_cnt++].root = in btrfs_ioctl_send()
8296 btrfs_grab_root(sctx->send_root); in btrfs_ioctl_send()
8299 sort(sctx->clone_roots, sctx->clone_roots_cnt, in btrfs_ioctl_send()
8300 sizeof(*sctx->clone_roots), __clone_root_cmp_sort, in btrfs_ioctl_send()
8304 ret = flush_delalloc_roots(sctx); in btrfs_ioctl_send()
8308 ret = ensure_commit_roots_uptodate(sctx); in btrfs_ioctl_send()
8312 ret = send_subvol(sctx); in btrfs_ioctl_send()
8316 btrfs_lru_cache_for_each_entry_safe(&sctx->dir_utimes_cache, entry, tmp) { in btrfs_ioctl_send()
8317 ret = send_utimes(sctx, entry->key, entry->gen); in btrfs_ioctl_send()
8320 btrfs_lru_cache_remove(&sctx->dir_utimes_cache, entry); in btrfs_ioctl_send()
8323 if (!(sctx->flags & BTRFS_SEND_FLAG_OMIT_END_CMD)) { in btrfs_ioctl_send()
8324 ret = begin_cmd(sctx, BTRFS_SEND_C_END); in btrfs_ioctl_send()
8327 ret = send_cmd(sctx); in btrfs_ioctl_send()
8333 WARN_ON(sctx && !ret && !RB_EMPTY_ROOT(&sctx->pending_dir_moves)); in btrfs_ioctl_send()
8334 while (sctx && !RB_EMPTY_ROOT(&sctx->pending_dir_moves)) { in btrfs_ioctl_send()
8338 n = rb_first(&sctx->pending_dir_moves); in btrfs_ioctl_send()
8345 free_pending_move(sctx, pm2); in btrfs_ioctl_send()
8347 free_pending_move(sctx, pm); in btrfs_ioctl_send()
8350 WARN_ON(sctx && !ret && !RB_EMPTY_ROOT(&sctx->waiting_dir_moves)); in btrfs_ioctl_send()
8351 while (sctx && !RB_EMPTY_ROOT(&sctx->waiting_dir_moves)) { in btrfs_ioctl_send()
8355 n = rb_first(&sctx->waiting_dir_moves); in btrfs_ioctl_send()
8357 rb_erase(&dm->node, &sctx->waiting_dir_moves); in btrfs_ioctl_send()
8361 WARN_ON(sctx && !ret && !RB_EMPTY_ROOT(&sctx->orphan_dirs)); in btrfs_ioctl_send()
8362 while (sctx && !RB_EMPTY_ROOT(&sctx->orphan_dirs)) { in btrfs_ioctl_send()
8366 n = rb_first(&sctx->orphan_dirs); in btrfs_ioctl_send()
8368 free_orphan_dir_info(sctx, odi); in btrfs_ioctl_send()
8372 for (i = 0; i < sctx->clone_roots_cnt; i++) { in btrfs_ioctl_send()
8374 sctx->clone_roots[i].root); in btrfs_ioctl_send()
8375 btrfs_put_root(sctx->clone_roots[i].root); in btrfs_ioctl_send()
8378 for (i = 0; sctx && i < clone_sources_to_rollback; i++) { in btrfs_ioctl_send()
8380 sctx->clone_roots[i].root); in btrfs_ioctl_send()
8381 btrfs_put_root(sctx->clone_roots[i].root); in btrfs_ioctl_send()
8386 if (sctx && !IS_ERR_OR_NULL(sctx->parent_root)) { in btrfs_ioctl_send()
8387 btrfs_root_dec_send_in_progress(sctx->parent_root); in btrfs_ioctl_send()
8388 btrfs_put_root(sctx->parent_root); in btrfs_ioctl_send()
8393 if (sctx) { in btrfs_ioctl_send()
8394 if (sctx->send_filp) in btrfs_ioctl_send()
8395 fput(sctx->send_filp); in btrfs_ioctl_send()
8397 kvfree(sctx->clone_roots); in btrfs_ioctl_send()
8398 kfree(sctx->send_buf_pages); in btrfs_ioctl_send()
8399 kvfree(sctx->send_buf); in btrfs_ioctl_send()
8400 kvfree(sctx->verity_descriptor); in btrfs_ioctl_send()
8402 close_current_inode(sctx); in btrfs_ioctl_send()
8404 btrfs_lru_cache_clear(&sctx->name_cache); in btrfs_ioctl_send()
8405 btrfs_lru_cache_clear(&sctx->backref_cache); in btrfs_ioctl_send()
8406 btrfs_lru_cache_clear(&sctx->dir_created_cache); in btrfs_ioctl_send()
8407 btrfs_lru_cache_clear(&sctx->dir_utimes_cache); in btrfs_ioctl_send()
8409 kfree(sctx); in btrfs_ioctl_send()