87 #define btrfs_debug_check_extent_io_range(tree, start, end)		\  argument
88 	__btrfs_debug_check_extent_io_range(__func__, (tree), (start), (end))
90 		struct extent_io_tree *tree, u64 start, u64 end)  in __btrfs_debug_check_extent_io_range()  argument
92 	if (tree->ops && tree->ops->check_extent_io_range)  in __btrfs_debug_check_extent_io_range()
93 		tree->ops->check_extent_io_range(tree->private_data, caller,  in __btrfs_debug_check_extent_io_range()
113 	struct extent_io_tree *tree;  member
194 void extent_io_tree_init(struct extent_io_tree *tree,  in extent_io_tree_init()  argument
197 	tree->state = RB_ROOT;  in extent_io_tree_init()
198 	tree->ops = NULL;  in extent_io_tree_init()
199 	tree->dirty_bytes = 0;  in extent_io_tree_init()
200 	spin_lock_init(&tree->lock);  in extent_io_tree_init()
201 	tree->private_data = private_data;  in extent_io_tree_init()
274 static struct rb_node *__etree_search(struct extent_io_tree *tree, u64 offset,  in __etree_search()  argument
280 	struct rb_root *root = &tree->state;  in __etree_search()
327 tree_search_for_insert(struct extent_io_tree *tree,  in tree_search_for_insert()  argument
335 	ret = __etree_search(tree, offset, &prev, NULL, p_ret, parent_ret);  in tree_search_for_insert()
341 static inline struct rb_node *tree_search(struct extent_io_tree *tree,  in tree_search()  argument
344 	return tree_search_for_insert(tree, offset, NULL, NULL);  in tree_search()
347 static void merge_cb(struct extent_io_tree *tree, struct extent_state *new,  in merge_cb()  argument
350 	if (tree->ops && tree->ops->merge_extent_hook)  in merge_cb()
351 		tree->ops->merge_extent_hook(tree->private_data, new, other);  in merge_cb()
363 static void merge_state(struct extent_io_tree *tree,  in merge_state()  argument
377 			merge_cb(tree, state, other);  in merge_state()
379 			rb_erase(&other->rb_node, &tree->state);  in merge_state()
389 			merge_cb(tree, state, other);  in merge_state()
391 			rb_erase(&other->rb_node, &tree->state);  in merge_state()
398 static void set_state_cb(struct extent_io_tree *tree,  in set_state_cb()  argument
401 	if (tree->ops && tree->ops->set_bit_hook)  in set_state_cb()
402 		tree->ops->set_bit_hook(tree->private_data, state, bits);  in set_state_cb()
405 static void clear_state_cb(struct extent_io_tree *tree,  in clear_state_cb()  argument
408 	if (tree->ops && tree->ops->clear_bit_hook)  in clear_state_cb()
409 		tree->ops->clear_bit_hook(tree->private_data, state, bits);  in clear_state_cb()
412 static void set_state_bits(struct extent_io_tree *tree,
426 static int insert_state(struct extent_io_tree *tree,  in insert_state()  argument
440 	set_state_bits(tree, state, bits, changeset);  in insert_state()
442 	node = tree_insert(&tree->state, NULL, end, &state->rb_node, p, parent);  in insert_state()
450 	merge_state(tree, state);  in insert_state()
454 static void split_cb(struct extent_io_tree *tree, struct extent_state *orig,  in split_cb()  argument
457 	if (tree->ops && tree->ops->split_extent_hook)  in split_cb()
458 		tree->ops->split_extent_hook(tree->private_data, orig, split);  in split_cb()
475 static int split_state(struct extent_io_tree *tree, struct extent_state *orig,  in split_state()  argument
480 	split_cb(tree, orig, split);  in split_state()
487 	node = tree_insert(&tree->state, &orig->rb_node, prealloc->end,  in split_state()
512 static struct extent_state *clear_state_bit(struct extent_io_tree *tree,  in clear_state_bit()  argument
523 		WARN_ON(range > tree->dirty_bytes);  in clear_state_bit()
524 		tree->dirty_bytes -= range;  in clear_state_bit()
526 	clear_state_cb(tree, state, bits);  in clear_state_bit()
535 			rb_erase(&state->rb_node, &tree->state);  in clear_state_bit()
542 		merge_state(tree, state);  in clear_state_bit()
557 static void extent_io_tree_panic(struct extent_io_tree *tree, int err)  in extent_io_tree_panic()  argument
559 	struct inode *inode = tree->private_data;  in extent_io_tree_panic()
577 int __clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,  in __clear_extent_bit()  argument
590 	btrfs_debug_check_extent_io_range(tree, start, end);  in __clear_extent_bit()
613 	spin_lock(&tree->lock);  in __clear_extent_bit()
636 	node = tree_search(tree, start);  in __clear_extent_bit()
671 		err = split_state(tree, state, prealloc, start);  in __clear_extent_bit()
673 			extent_io_tree_panic(tree, err);  in __clear_extent_bit()
679 			state = clear_state_bit(tree, state, &bits, wake,  in __clear_extent_bit()
694 		err = split_state(tree, state, prealloc, end + 1);  in __clear_extent_bit()
696 			extent_io_tree_panic(tree, err);  in __clear_extent_bit()
701 		clear_state_bit(tree, prealloc, &bits, wake, changeset);  in __clear_extent_bit()
707 	state = clear_state_bit(tree, state, &bits, wake, changeset);  in __clear_extent_bit()
718 	spin_unlock(&tree->lock);  in __clear_extent_bit()
724 	spin_unlock(&tree->lock);  in __clear_extent_bit()
732 static void wait_on_state(struct extent_io_tree *tree,  in wait_on_state()  argument
734 		__releases(tree->lock)  in wait_on_state()
735 		__acquires(tree->lock)  in wait_on_state()
739 	spin_unlock(&tree->lock);  in wait_on_state()
741 	spin_lock(&tree->lock);  in wait_on_state()
750 static void wait_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,  in wait_extent_bit()  argument
756 	btrfs_debug_check_extent_io_range(tree, start, end);  in wait_extent_bit()
758 	spin_lock(&tree->lock);  in wait_extent_bit()
765 		node = tree_search(tree, start);  in wait_extent_bit()
778 			wait_on_state(tree, state);  in wait_extent_bit()
787 		if (!cond_resched_lock(&tree->lock)) {  in wait_extent_bit()
793 	spin_unlock(&tree->lock);  in wait_extent_bit()
796 static void set_state_bits(struct extent_io_tree *tree,  in set_state_bits()  argument
803 	set_state_cb(tree, state, bits);  in set_state_bits()
806 		tree->dirty_bytes += range;  in set_state_bits()
844 __set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,  in __set_extent_bit()  argument
858 	btrfs_debug_check_extent_io_range(tree, start, end);  in __set_extent_bit()
873 	spin_lock(&tree->lock);  in __set_extent_bit()
886 	node = tree_search_for_insert(tree, start, &p, &parent);  in __set_extent_bit()
890 		err = insert_state(tree, prealloc, start, end,  in __set_extent_bit()
893 			extent_io_tree_panic(tree, err);  in __set_extent_bit()
917 		set_state_bits(tree, state, &bits, changeset);  in __set_extent_bit()
919 		merge_state(tree, state);  in __set_extent_bit()
955 		err = split_state(tree, state, prealloc, start);  in __set_extent_bit()
957 			extent_io_tree_panic(tree, err);  in __set_extent_bit()
963 			set_state_bits(tree, state, &bits, changeset);  in __set_extent_bit()
965 			merge_state(tree, state);  in __set_extent_bit()
997 		err = insert_state(tree, prealloc, start, this_end,  in __set_extent_bit()
1000 			extent_io_tree_panic(tree, err);  in __set_extent_bit()
1022 		err = split_state(tree, state, prealloc, end + 1);  in __set_extent_bit()
1024 			extent_io_tree_panic(tree, err);  in __set_extent_bit()
1026 		set_state_bits(tree, prealloc, &bits, changeset);  in __set_extent_bit()
1028 		merge_state(tree, prealloc);  in __set_extent_bit()
1036 	spin_unlock(&tree->lock);  in __set_extent_bit()
1042 	spin_unlock(&tree->lock);  in __set_extent_bit()
1050 int set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,  in set_extent_bit()  argument
1054 	return __set_extent_bit(tree, start, end, bits, 0, failed_start,  in set_extent_bit()
1077 int convert_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,  in convert_extent_bit()  argument
1091 	btrfs_debug_check_extent_io_range(tree, start, end);  in convert_extent_bit()
1107 	spin_lock(&tree->lock);  in convert_extent_bit()
1121 	node = tree_search_for_insert(tree, start, &p, &parent);  in convert_extent_bit()
1128 		err = insert_state(tree, prealloc, start, end,  in convert_extent_bit()
1131 			extent_io_tree_panic(tree, err);  in convert_extent_bit()
1148 		set_state_bits(tree, state, &bits, NULL);  in convert_extent_bit()
1150 		state = clear_state_bit(tree, state, &clear_bits, 0, NULL);  in convert_extent_bit()
1182 		err = split_state(tree, state, prealloc, start);  in convert_extent_bit()
1184 			extent_io_tree_panic(tree, err);  in convert_extent_bit()
1189 			set_state_bits(tree, state, &bits, NULL);  in convert_extent_bit()
1191 			state = clear_state_bit(tree, state, &clear_bits, 0,  in convert_extent_bit()
1226 		err = insert_state(tree, prealloc, start, this_end,  in convert_extent_bit()
1229 			extent_io_tree_panic(tree, err);  in convert_extent_bit()
1248 		err = split_state(tree, state, prealloc, end + 1);  in convert_extent_bit()
1250 			extent_io_tree_panic(tree, err);  in convert_extent_bit()
1252 		set_state_bits(tree, prealloc, &bits, NULL);  in convert_extent_bit()
1254 		clear_state_bit(tree, prealloc, &clear_bits, 0, NULL);  in convert_extent_bit()
1262 	spin_unlock(&tree->lock);  in convert_extent_bit()
1268 	spin_unlock(&tree->lock);  in convert_extent_bit()
1276 int set_record_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,  in set_record_extent_bits()  argument
1287 	return __set_extent_bit(tree, start, end, bits, 0, NULL, NULL, GFP_NOFS,  in set_record_extent_bits()
1291 int clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,  in clear_extent_bit()  argument
1295 	return __clear_extent_bit(tree, start, end, bits, wake, delete,  in clear_extent_bit()
1299 int clear_record_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,  in clear_record_extent_bits()  argument
1308 	return __clear_extent_bit(tree, start, end, bits, 0, 0, NULL, GFP_NOFS,  in clear_record_extent_bits()
1316 int lock_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,  in lock_extent_bits()  argument
1323 		err = __set_extent_bit(tree, start, end, EXTENT_LOCKED,  in lock_extent_bits()
1327 			wait_extent_bit(tree, failed_start, end, EXTENT_LOCKED);  in lock_extent_bits()
1336 int try_lock_extent(struct extent_io_tree *tree, u64 start, u64 end)  in try_lock_extent()  argument
1341 	err = __set_extent_bit(tree, start, end, EXTENT_LOCKED, EXTENT_LOCKED,  in try_lock_extent()
1345 			clear_extent_bit(tree, start, failed_start - 1,  in try_lock_extent()
1388 find_first_extent_bit_state(struct extent_io_tree *tree,  in find_first_extent_bit_state()  argument
1398 	node = tree_search(tree, start);  in find_first_extent_bit_state()
1422 int find_first_extent_bit(struct extent_io_tree *tree, u64 start,  in find_first_extent_bit()  argument
1430 	spin_lock(&tree->lock);  in find_first_extent_bit()
1450 	state = find_first_extent_bit_state(tree, start, bits);  in find_first_extent_bit()
1459 	spin_unlock(&tree->lock);  in find_first_extent_bit()
1469 static noinline u64 find_delalloc_range(struct extent_io_tree *tree,  in find_delalloc_range()  argument
1479 	spin_lock(&tree->lock);  in find_delalloc_range()
1485 	node = tree_search(tree, cur_start);  in find_delalloc_range()
1519 	spin_unlock(&tree->lock);  in find_delalloc_range()
1572 				    struct extent_io_tree *tree,  in find_lock_delalloc_range()  argument
1587 	found = find_delalloc_range(tree, &delalloc_start, &delalloc_end,  in find_lock_delalloc_range()
1631 	lock_extent_bits(tree, delalloc_start, delalloc_end, &cached_state);  in find_lock_delalloc_range()
1634 	ret = test_range_bit(tree, delalloc_start, delalloc_end,  in find_lock_delalloc_range()
1637 		unlock_extent_cached(tree, delalloc_start, delalloc_end,  in find_lock_delalloc_range()
1746 u64 count_range_bits(struct extent_io_tree *tree,  in count_range_bits()  argument
1760 	spin_lock(&tree->lock);  in count_range_bits()
1762 		total_bytes = tree->dirty_bytes;  in count_range_bits()
1769 	node = tree_search(tree, cur_start);  in count_range_bits()
1797 	spin_unlock(&tree->lock);  in count_range_bits()
1805 static noinline int set_state_failrec(struct extent_io_tree *tree, u64 start,  in set_state_failrec()  argument
1812 	spin_lock(&tree->lock);  in set_state_failrec()
1817 	node = tree_search(tree, start);  in set_state_failrec()
1829 	spin_unlock(&tree->lock);  in set_state_failrec()
1833 static noinline int get_state_failrec(struct extent_io_tree *tree, u64 start,  in get_state_failrec()  argument
1840 	spin_lock(&tree->lock);  in get_state_failrec()
1845 	node = tree_search(tree, start);  in get_state_failrec()
1857 	spin_unlock(&tree->lock);  in get_state_failrec()
1867 int test_range_bit(struct extent_io_tree *tree, u64 start, u64 end,  in test_range_bit()  argument
1874 	spin_lock(&tree->lock);  in test_range_bit()
1879 		node = tree_search(tree, start);  in test_range_bit()
1913 	spin_unlock(&tree->lock);  in test_range_bit()
1921 static void check_page_uptodate(struct extent_io_tree *tree, struct page *page)  in check_page_uptodate()  argument
1925 	if (test_range_bit(tree, start, end, EXTENT_UPTODATE, 1, NULL))  in check_page_uptodate()
2168 	struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;  in btrfs_get_io_failure_record()  local
2226 			ret = set_extent_bits(tree, start, end, EXTENT_DAMAGED);  in btrfs_get_io_failure_record()
2359 	struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;  in bio_readpage_error()  local
2375 		free_io_failure(failure_tree, tree, failrec);  in bio_readpage_error()
2393 	status = tree->ops->submit_bio_hook(tree->private_data, bio, failrec->this_mirror,  in bio_readpage_error()
2396 		free_io_failure(failure_tree, tree, failrec);  in bio_readpage_error()
2409 	struct extent_io_tree *tree;  in end_extent_writepage()  local
2412 	tree = &BTRFS_I(page->mapping->host)->io_tree;  in end_extent_writepage()
2414 	if (tree->ops && tree->ops->writepage_end_io_hook)  in end_extent_writepage()
2415 		tree->ops->writepage_end_io_hook(page, start, end, NULL,  in end_extent_writepage()
2476 endio_readpage_release_extent(struct extent_io_tree *tree, u64 start, u64 len,  in endio_readpage_release_extent()  argument
2482 	if (uptodate && tree->track_uptodate)  in endio_readpage_release_extent()
2483 		set_extent_uptodate(tree, start, end, &cached, GFP_ATOMIC);  in endio_readpage_release_extent()
2484 	unlock_extent_cached_atomic(tree, start, end, &cached);  in endio_readpage_release_extent()
2503 	struct extent_io_tree *tree, *failure_tree;  in end_bio_extent_readpage()  local
2524 		tree = &BTRFS_I(inode)->io_tree;  in end_bio_extent_readpage()
2548 		if (likely(uptodate && tree->ops)) {  in end_bio_extent_readpage()
2549 			ret = tree->ops->readpage_end_io_hook(io_bio, offset,  in end_bio_extent_readpage()
2556 						 failure_tree, tree, start,  in end_bio_extent_readpage()
2564 		if (tree->ops) {  in end_bio_extent_readpage()
2565 			ret = tree->ops->readpage_io_failed_hook(page, mirror);  in end_bio_extent_readpage()
2617 				endio_readpage_release_extent(tree,  in end_bio_extent_readpage()
2623 			endio_readpage_release_extent(tree, start,  in end_bio_extent_readpage()
2631 			endio_readpage_release_extent(tree, extent_start,  in end_bio_extent_readpage()
2639 		endio_readpage_release_extent(tree, extent_start, extent_len,  in end_bio_extent_readpage()
2718 	struct extent_io_tree *tree = bio->bi_private;  in submit_one_bio()  local
2725 	if (tree->ops)  in submit_one_bio()
2726 		ret = tree->ops->submit_bio_hook(tree->private_data, bio,  in submit_one_bio()
2750 static int submit_extent_page(unsigned int opf, struct extent_io_tree *tree,  in submit_extent_page()  argument
2779 		if (tree->ops && btrfs_merge_bio_hook(page, offset, page_size,  in submit_extent_page()
2802 	bio->bi_private = tree;  in submit_extent_page()
2870 static int __do_readpage(struct extent_io_tree *tree,  in __do_readpage()  argument
2901 			unlock_extent(tree, start, end);  in __do_readpage()
2931 			set_extent_uptodate(tree, cur, cur + iosize - 1,  in __do_readpage()
2933 			unlock_extent_cached(tree, cur,  in __do_readpage()
2941 			unlock_extent(tree, cur, end);  in __do_readpage()
3024 			set_extent_uptodate(tree, cur, cur + iosize - 1,  in __do_readpage()
3026 			unlock_extent_cached(tree, cur,  in __do_readpage()
3033 		if (test_range_bit(tree, cur, cur_end,  in __do_readpage()
3035 			check_page_uptodate(tree, page);  in __do_readpage()
3036 			unlock_extent(tree, cur, cur + iosize - 1);  in __do_readpage()
3046 			unlock_extent(tree, cur, cur + iosize - 1);  in __do_readpage()
3052 		ret = submit_extent_page(REQ_OP_READ | read_flags, tree, NULL,  in __do_readpage()
3064 			unlock_extent(tree, cur, cur + iosize - 1);  in __do_readpage()
3079 static inline void __do_contiguous_readpages(struct extent_io_tree *tree,  in __do_contiguous_readpages()  argument
3093 		lock_extent(tree, start, end);  in __do_contiguous_readpages()
3098 		unlock_extent(tree, start, end);  in __do_contiguous_readpages()
3104 		__do_readpage(tree, pages[index], btrfs_get_extent, em_cached,  in __do_contiguous_readpages()
3110 static void __extent_readpages(struct extent_io_tree *tree,  in __extent_readpages()  argument
3132 			__do_contiguous_readpages(tree, &pages[first_index],  in __extent_readpages()
3144 		__do_contiguous_readpages(tree, &pages[first_index],  in __extent_readpages()
3150 static int __extent_read_full_page(struct extent_io_tree *tree,  in __extent_read_full_page()  argument
3164 		lock_extent(tree, start, end);  in __extent_read_full_page()
3169 		unlock_extent(tree, start, end);  in __extent_read_full_page()
3174 	ret = __do_readpage(tree, page, get_extent, NULL, bio, mirror_num,  in __extent_read_full_page()
3179 int extent_read_full_page(struct extent_io_tree *tree, struct page *page,  in extent_read_full_page()  argument
3186 	ret = __extent_read_full_page(tree, page, get_extent, &bio, mirror_num,  in extent_read_full_page()
3215 	struct extent_io_tree *tree = epd->tree;  in writepage_delalloc()  local
3223 	if (epd->extent_locked || !tree->ops || !tree->ops->fill_delalloc)  in writepage_delalloc()
3227 		nr_delalloc = find_lock_delalloc_range(inode, tree,  in writepage_delalloc()
3236 		ret = tree->ops->fill_delalloc(inode, page,  in writepage_delalloc()
3304 	struct extent_io_tree *tree = epd->tree;  in __extent_writepage_io()  local
3320 	if (tree->ops && tree->ops->writepage_start_hook) {  in __extent_writepage_io()
3321 		ret = tree->ops->writepage_start_hook(page, start,  in __extent_writepage_io()
3344 		if (tree->ops && tree->ops->writepage_end_io_hook)  in __extent_writepage_io()
3345 			tree->ops->writepage_end_io_hook(page, start,  in __extent_writepage_io()
3357 			if (tree->ops && tree->ops->writepage_end_io_hook)  in __extent_writepage_io()
3358 				tree->ops->writepage_end_io_hook(page, cur,  in __extent_writepage_io()
3393 			if (!compressed && tree->ops &&  in __extent_writepage_io()
3394 			    tree->ops->writepage_end_io_hook)  in __extent_writepage_io()
3395 				tree->ops->writepage_end_io_hook(page, cur,  in __extent_writepage_io()
3411 		btrfs_set_range_writeback(tree, cur, cur + iosize - 1);  in __extent_writepage_io()
3418 		ret = submit_extent_page(REQ_OP_WRITE | write_flags, tree, wbc,  in __extent_writepage_io()
3698 	struct extent_io_tree *tree = &BTRFS_I(fs_info->btree_inode)->io_tree;  in write_one_eb()  local
3731 		ret = submit_extent_page(REQ_OP_WRITE | write_flags, tree, wbc,  in write_one_eb()
3764 	struct extent_io_tree *tree = &BTRFS_I(mapping->host)->io_tree;  in btree_write_cache_pages()  local
3769 		.tree = tree,  in btree_write_cache_pages()
4042 		.tree = &BTRFS_I(page->mapping->host)->io_tree,  in extent_write_full_page()
4058 	struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;  in extent_write_locked_range()  local
4065 		.tree = tree,  in extent_write_locked_range()
4081 			if (tree->ops && tree->ops->writepage_end_io_hook)  in extent_write_locked_range()
4082 				tree->ops->writepage_end_io_hook(page, start,  in extent_write_locked_range()
4101 		.tree = &BTRFS_I(mapping->host)->io_tree,  in extent_writepages()
4120 	struct extent_io_tree *tree = &BTRFS_I(mapping->host)->io_tree;  in extent_readpages()  local
4139 		__extent_readpages(tree, pagepool, nr, &em_cached, &bio,  in extent_readpages()
4144 		__extent_readpages(tree, pagepool, nr, &em_cached, &bio,  in extent_readpages()
4161 int extent_invalidatepage(struct extent_io_tree *tree,  in extent_invalidatepage()  argument
4173 	lock_extent_bits(tree, start, end, &cached_state);  in extent_invalidatepage()
4175 	clear_extent_bit(tree, start, end,  in extent_invalidatepage()
4187 static int try_release_extent_state(struct extent_io_tree *tree,  in try_release_extent_state()  argument
4194 	if (test_range_bit(tree, start, end,  in try_release_extent_state()
4202 		ret = __clear_extent_bit(tree, start, end,  in try_release_extent_state()
4228 	struct extent_io_tree *tree = &btrfs_inode->io_tree;  in try_release_extent_mapping()  local
4248 			if (!test_range_bit(tree, em->start,  in try_release_extent_mapping()
4265 	return try_release_extent_state(tree, page, mask);  in try_release_extent_mapping()
5216 int read_extent_buffer_pages(struct extent_io_tree *tree,  in read_extent_buffer_pages()  argument
5276 			err = __extent_read_full_page(tree, page,  in read_extent_buffer_pages()