Lines Matching refs:eb
69 struct extent_buffer *eb; in btrfs_extent_buffer_leak_debug_check() local
81 eb = list_first_entry(&fs_info->allocated_ebs, in btrfs_extent_buffer_leak_debug_check()
85 eb->start, eb->len, atomic_read(&eb->refs), eb->bflags, in btrfs_extent_buffer_leak_debug_check()
86 btrfs_header_owner(eb)); in btrfs_extent_buffer_leak_debug_check()
87 list_del(&eb->leak_list); in btrfs_extent_buffer_leak_debug_check()
88 kmem_cache_free(extent_buffer_cache, eb); in btrfs_extent_buffer_leak_debug_check()
2369 int btrfs_repair_eb_io_failure(const struct extent_buffer *eb, int mirror_num) in btrfs_repair_eb_io_failure() argument
2371 struct btrfs_fs_info *fs_info = eb->fs_info; in btrfs_repair_eb_io_failure()
2372 u64 start = eb->start; in btrfs_repair_eb_io_failure()
2373 int i, num_pages = num_extent_pages(eb); in btrfs_repair_eb_io_failure()
2380 struct page *p = eb->pages[i]; in btrfs_repair_eb_io_failure()
2945 struct extent_buffer *eb; in find_extent_buffer_readpage() local
2958 eb = radix_tree_lookup(&fs_info->buffer_radix, in find_extent_buffer_readpage()
2961 ASSERT(eb); in find_extent_buffer_readpage()
2962 return eb; in find_extent_buffer_readpage()
3067 struct extent_buffer *eb; in end_bio_extent_readpage() local
3069 eb = find_extent_buffer_readpage(fs_info, page, start); in end_bio_extent_readpage()
3070 set_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags); in end_bio_extent_readpage()
3071 eb->read_mirror = mirror; in end_bio_extent_readpage()
3072 atomic_dec(&eb->io_pages); in end_bio_extent_readpage()
3074 &eb->bflags)) in end_bio_extent_readpage()
3075 btree_readahead_hook(eb, -EIO); in end_bio_extent_readpage()
3442 static int attach_extent_buffer_page(struct extent_buffer *eb, in attach_extent_buffer_page() argument
3446 struct btrfs_fs_info *fs_info = eb->fs_info; in attach_extent_buffer_page()
3460 attach_page_private(page, eb); in attach_extent_buffer_page()
3462 WARN_ON(page->private != (unsigned long)eb); in attach_extent_buffer_page()
4150 void wait_on_extent_buffer_writeback(struct extent_buffer *eb) in wait_on_extent_buffer_writeback() argument
4152 wait_on_bit_io(&eb->bflags, EXTENT_BUFFER_WRITEBACK, in wait_on_extent_buffer_writeback()
4156 static void end_extent_buffer_writeback(struct extent_buffer *eb) in end_extent_buffer_writeback() argument
4158 clear_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags); in end_extent_buffer_writeback()
4160 wake_up_bit(&eb->bflags, EXTENT_BUFFER_WRITEBACK); in end_extent_buffer_writeback()
4173 static noinline_for_stack int lock_extent_buffer_for_io(struct extent_buffer *eb, in lock_extent_buffer_for_io() argument
4176 struct btrfs_fs_info *fs_info = eb->fs_info; in lock_extent_buffer_for_io()
4181 if (!btrfs_try_tree_write_lock(eb)) { in lock_extent_buffer_for_io()
4186 btrfs_tree_lock(eb); in lock_extent_buffer_for_io()
4189 if (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags)) { in lock_extent_buffer_for_io()
4190 btrfs_tree_unlock(eb); in lock_extent_buffer_for_io()
4200 wait_on_extent_buffer_writeback(eb); in lock_extent_buffer_for_io()
4201 btrfs_tree_lock(eb); in lock_extent_buffer_for_io()
4202 if (!test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags)) in lock_extent_buffer_for_io()
4204 btrfs_tree_unlock(eb); in lock_extent_buffer_for_io()
4213 spin_lock(&eb->refs_lock); in lock_extent_buffer_for_io()
4214 if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) { in lock_extent_buffer_for_io()
4215 set_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags); in lock_extent_buffer_for_io()
4216 spin_unlock(&eb->refs_lock); in lock_extent_buffer_for_io()
4217 btrfs_set_header_flag(eb, BTRFS_HEADER_FLAG_WRITTEN); in lock_extent_buffer_for_io()
4219 -eb->len, in lock_extent_buffer_for_io()
4223 spin_unlock(&eb->refs_lock); in lock_extent_buffer_for_io()
4226 btrfs_tree_unlock(eb); in lock_extent_buffer_for_io()
4237 num_pages = num_extent_pages(eb); in lock_extent_buffer_for_io()
4239 struct page *p = eb->pages[i]; in lock_extent_buffer_for_io()
4261 unlock_page(eb->pages[i]); in lock_extent_buffer_for_io()
4267 btrfs_tree_lock(eb); in lock_extent_buffer_for_io()
4268 spin_lock(&eb->refs_lock); in lock_extent_buffer_for_io()
4269 set_bit(EXTENT_BUFFER_DIRTY, &eb->bflags); in lock_extent_buffer_for_io()
4270 end_extent_buffer_writeback(eb); in lock_extent_buffer_for_io()
4271 spin_unlock(&eb->refs_lock); in lock_extent_buffer_for_io()
4272 percpu_counter_add_batch(&fs_info->dirty_metadata_bytes, eb->len, in lock_extent_buffer_for_io()
4274 btrfs_clear_header_flag(eb, BTRFS_HEADER_FLAG_WRITTEN); in lock_extent_buffer_for_io()
4275 btrfs_tree_unlock(eb); in lock_extent_buffer_for_io()
4279 static void set_btree_ioerr(struct page *page, struct extent_buffer *eb) in set_btree_ioerr() argument
4281 struct btrfs_fs_info *fs_info = eb->fs_info; in set_btree_ioerr()
4283 btrfs_page_set_error(fs_info, page, eb->start, eb->len); in set_btree_ioerr()
4284 if (test_and_set_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags)) in set_btree_ioerr()
4292 eb->len, fs_info->dirty_metadata_batch); in set_btree_ioerr()
4332 switch (eb->log_index) { in set_btree_ioerr()
4354 struct extent_buffer *eb; in find_extent_buffer_nolock() local
4357 eb = radix_tree_lookup(&fs_info->buffer_radix, in find_extent_buffer_nolock()
4359 if (eb && atomic_inc_not_zero(&eb->refs)) { in find_extent_buffer_nolock()
4361 return eb; in find_extent_buffer_nolock()
4393 struct extent_buffer *eb; in end_bio_subpage_eb_writepage() local
4401 eb = find_extent_buffer_nolock(fs_info, cur_bytenr); in end_bio_subpage_eb_writepage()
4402 ASSERT(eb); in end_bio_subpage_eb_writepage()
4404 cur_bytenr = eb->start + eb->len; in end_bio_subpage_eb_writepage()
4406 ASSERT(test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags)); in end_bio_subpage_eb_writepage()
4407 done = atomic_dec_and_test(&eb->io_pages); in end_bio_subpage_eb_writepage()
4411 test_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags)) { in end_bio_subpage_eb_writepage()
4413 set_btree_ioerr(page, eb); in end_bio_subpage_eb_writepage()
4416 btrfs_subpage_clear_writeback(fs_info, page, eb->start, in end_bio_subpage_eb_writepage()
4417 eb->len); in end_bio_subpage_eb_writepage()
4418 end_extent_buffer_writeback(eb); in end_bio_subpage_eb_writepage()
4424 atomic_dec(&eb->refs); in end_bio_subpage_eb_writepage()
4433 struct extent_buffer *eb; in end_bio_extent_buffer_writepage() local
4441 eb = (struct extent_buffer *)page->private; in end_bio_extent_buffer_writepage()
4442 BUG_ON(!eb); in end_bio_extent_buffer_writepage()
4443 done = atomic_dec_and_test(&eb->io_pages); in end_bio_extent_buffer_writepage()
4446 test_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags)) { in end_bio_extent_buffer_writepage()
4448 set_btree_ioerr(page, eb); in end_bio_extent_buffer_writepage()
4456 end_extent_buffer_writeback(eb); in end_bio_extent_buffer_writepage()
4462 static void prepare_eb_write(struct extent_buffer *eb) in prepare_eb_write() argument
4468 clear_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags); in prepare_eb_write()
4469 atomic_set(&eb->io_pages, num_extent_pages(eb)); in prepare_eb_write()
4472 nritems = btrfs_header_nritems(eb); in prepare_eb_write()
4473 if (btrfs_header_level(eb) > 0) { in prepare_eb_write()
4475 memzero_extent_buffer(eb, end, eb->len - end); in prepare_eb_write()
4482 end = BTRFS_LEAF_DATA_OFFSET + leaf_data_end(eb); in prepare_eb_write()
4483 memzero_extent_buffer(eb, start, end - start); in prepare_eb_write()
4491 static int write_one_subpage_eb(struct extent_buffer *eb, in write_one_subpage_eb() argument
4495 struct btrfs_fs_info *fs_info = eb->fs_info; in write_one_subpage_eb()
4496 struct page *page = eb->pages[0]; in write_one_subpage_eb()
4501 prepare_eb_write(eb); in write_one_subpage_eb()
4505 btrfs_subpage_set_writeback(fs_info, page, eb->start, eb->len); in write_one_subpage_eb()
4509 eb->start, eb->len); in write_one_subpage_eb()
4514 &epd->bio_ctrl, page, eb->start, eb->len, in write_one_subpage_eb()
4515 eb->start - page_offset(page), in write_one_subpage_eb()
4518 btrfs_subpage_clear_writeback(fs_info, page, eb->start, eb->len); in write_one_subpage_eb()
4519 set_btree_ioerr(page, eb); in write_one_subpage_eb()
4522 if (atomic_dec_and_test(&eb->io_pages)) in write_one_subpage_eb()
4523 end_extent_buffer_writeback(eb); in write_one_subpage_eb()
4536 static noinline_for_stack int write_one_eb(struct extent_buffer *eb, in write_one_eb() argument
4540 u64 disk_bytenr = eb->start; in write_one_eb()
4545 prepare_eb_write(eb); in write_one_eb()
4547 num_pages = num_extent_pages(eb); in write_one_eb()
4549 struct page *p = eb->pages[i]; in write_one_eb()
4559 set_btree_ioerr(p, eb); in write_one_eb()
4562 if (atomic_sub_and_test(num_pages - i, &eb->io_pages)) in write_one_eb()
4563 end_extent_buffer_writeback(eb); in write_one_eb()
4574 struct page *p = eb->pages[i]; in write_one_eb()
4612 struct extent_buffer *eb; in submit_eb_subpage() local
4640 eb = find_extent_buffer_nolock(fs_info, start); in submit_eb_subpage()
4649 if (!eb) in submit_eb_subpage()
4652 ret = lock_extent_buffer_for_io(eb, epd); in submit_eb_subpage()
4654 free_extent_buffer(eb); in submit_eb_subpage()
4658 free_extent_buffer(eb); in submit_eb_subpage()
4661 ret = write_one_subpage_eb(eb, wbc, epd); in submit_eb_subpage()
4662 free_extent_buffer(eb); in submit_eb_subpage()
4701 struct extent_buffer *eb; in submit_eb_page() local
4716 eb = (struct extent_buffer *)page->private; in submit_eb_page()
4722 if (WARN_ON(!eb)) { in submit_eb_page()
4727 if (eb == *eb_context) { in submit_eb_page()
4731 ret = atomic_inc_not_zero(&eb->refs); in submit_eb_page()
4736 if (!btrfs_check_meta_write_pointer(eb->fs_info, eb, &cache)) { in submit_eb_page()
4745 free_extent_buffer(eb); in submit_eb_page()
4749 *eb_context = eb; in submit_eb_page()
4751 ret = lock_extent_buffer_for_io(eb, epd); in submit_eb_page()
4753 btrfs_revert_meta_write_pointer(cache, eb); in submit_eb_page()
4756 free_extent_buffer(eb); in submit_eb_page()
4761 ret = write_one_eb(eb, wbc, epd); in submit_eb_page()
4762 free_extent_buffer(eb); in submit_eb_page()
5681 static void __free_extent_buffer(struct extent_buffer *eb) in __free_extent_buffer() argument
5683 kmem_cache_free(extent_buffer_cache, eb); in __free_extent_buffer()
5686 int extent_buffer_under_io(const struct extent_buffer *eb) in extent_buffer_under_io() argument
5688 return (atomic_read(&eb->io_pages) || in extent_buffer_under_io()
5689 test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags) || in extent_buffer_under_io()
5690 test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)); in extent_buffer_under_io()
5713 static void detach_extent_buffer_page(struct extent_buffer *eb, struct page *page) in detach_extent_buffer_page() argument
5715 struct btrfs_fs_info *fs_info = eb->fs_info; in detach_extent_buffer_page()
5716 const bool mapped = !test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags); in detach_extent_buffer_page()
5740 page->private == (unsigned long)eb) { in detach_extent_buffer_page()
5741 BUG_ON(test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)); in detach_extent_buffer_page()
5778 static void btrfs_release_extent_buffer_pages(struct extent_buffer *eb) in btrfs_release_extent_buffer_pages() argument
5783 ASSERT(!extent_buffer_under_io(eb)); in btrfs_release_extent_buffer_pages()
5785 num_pages = num_extent_pages(eb); in btrfs_release_extent_buffer_pages()
5787 struct page *page = eb->pages[i]; in btrfs_release_extent_buffer_pages()
5792 detach_extent_buffer_page(eb, page); in btrfs_release_extent_buffer_pages()
5802 static inline void btrfs_release_extent_buffer(struct extent_buffer *eb) in btrfs_release_extent_buffer() argument
5804 btrfs_release_extent_buffer_pages(eb); in btrfs_release_extent_buffer()
5805 btrfs_leak_debug_del(&eb->fs_info->eb_leak_lock, &eb->leak_list); in btrfs_release_extent_buffer()
5806 __free_extent_buffer(eb); in btrfs_release_extent_buffer()
5813 struct extent_buffer *eb = NULL; in __alloc_extent_buffer() local
5815 eb = kmem_cache_zalloc(extent_buffer_cache, GFP_NOFS|__GFP_NOFAIL); in __alloc_extent_buffer()
5816 eb->start = start; in __alloc_extent_buffer()
5817 eb->len = len; in __alloc_extent_buffer()
5818 eb->fs_info = fs_info; in __alloc_extent_buffer()
5819 eb->bflags = 0; in __alloc_extent_buffer()
5820 init_rwsem(&eb->lock); in __alloc_extent_buffer()
5822 btrfs_leak_debug_add(&fs_info->eb_leak_lock, &eb->leak_list, in __alloc_extent_buffer()
5824 INIT_LIST_HEAD(&eb->release_list); in __alloc_extent_buffer()
5826 spin_lock_init(&eb->refs_lock); in __alloc_extent_buffer()
5827 atomic_set(&eb->refs, 1); in __alloc_extent_buffer()
5828 atomic_set(&eb->io_pages, 0); in __alloc_extent_buffer()
5832 return eb; in __alloc_extent_buffer()
5879 struct extent_buffer *eb; in __alloc_dummy_extent_buffer() local
5883 eb = __alloc_extent_buffer(fs_info, start, len); in __alloc_dummy_extent_buffer()
5884 if (!eb) in __alloc_dummy_extent_buffer()
5887 num_pages = num_extent_pages(eb); in __alloc_dummy_extent_buffer()
5891 eb->pages[i] = alloc_page(GFP_NOFS); in __alloc_dummy_extent_buffer()
5892 if (!eb->pages[i]) in __alloc_dummy_extent_buffer()
5894 ret = attach_extent_buffer_page(eb, eb->pages[i], NULL); in __alloc_dummy_extent_buffer()
5898 set_extent_buffer_uptodate(eb); in __alloc_dummy_extent_buffer()
5899 btrfs_set_header_nritems(eb, 0); in __alloc_dummy_extent_buffer()
5900 set_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags); in __alloc_dummy_extent_buffer()
5902 return eb; in __alloc_dummy_extent_buffer()
5905 detach_extent_buffer_page(eb, eb->pages[i - 1]); in __alloc_dummy_extent_buffer()
5906 __free_page(eb->pages[i - 1]); in __alloc_dummy_extent_buffer()
5908 __free_extent_buffer(eb); in __alloc_dummy_extent_buffer()
5918 static void check_buffer_tree_ref(struct extent_buffer *eb) in check_buffer_tree_ref() argument
5944 refs = atomic_read(&eb->refs); in check_buffer_tree_ref()
5945 if (refs >= 2 && test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) in check_buffer_tree_ref()
5948 spin_lock(&eb->refs_lock); in check_buffer_tree_ref()
5949 if (!test_and_set_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) in check_buffer_tree_ref()
5950 atomic_inc(&eb->refs); in check_buffer_tree_ref()
5951 spin_unlock(&eb->refs_lock); in check_buffer_tree_ref()
5954 static void mark_extent_buffer_accessed(struct extent_buffer *eb, in mark_extent_buffer_accessed() argument
5959 check_buffer_tree_ref(eb); in mark_extent_buffer_accessed()
5961 num_pages = num_extent_pages(eb); in mark_extent_buffer_accessed()
5963 struct page *p = eb->pages[i]; in mark_extent_buffer_accessed()
5973 struct extent_buffer *eb; in find_extent_buffer() local
5975 eb = find_extent_buffer_nolock(fs_info, start); in find_extent_buffer()
5976 if (!eb) in find_extent_buffer()
5991 if (test_bit(EXTENT_BUFFER_STALE, &eb->bflags)) { in find_extent_buffer()
5992 spin_lock(&eb->refs_lock); in find_extent_buffer()
5993 spin_unlock(&eb->refs_lock); in find_extent_buffer()
5995 mark_extent_buffer_accessed(eb, NULL); in find_extent_buffer()
5996 return eb; in find_extent_buffer()
6003 struct extent_buffer *eb, *exists = NULL; in alloc_test_extent_buffer() local
6006 eb = find_extent_buffer(fs_info, start); in alloc_test_extent_buffer()
6007 if (eb) in alloc_test_extent_buffer()
6008 return eb; in alloc_test_extent_buffer()
6009 eb = alloc_dummy_extent_buffer(fs_info, start); in alloc_test_extent_buffer()
6010 if (!eb) in alloc_test_extent_buffer()
6012 eb->fs_info = fs_info; in alloc_test_extent_buffer()
6021 start >> fs_info->sectorsize_bits, eb); in alloc_test_extent_buffer()
6031 check_buffer_tree_ref(eb); in alloc_test_extent_buffer()
6032 set_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags); in alloc_test_extent_buffer()
6034 return eb; in alloc_test_extent_buffer()
6036 btrfs_release_extent_buffer(eb); in alloc_test_extent_buffer()
6080 struct extent_buffer *eb; in alloc_extent_buffer() local
6111 eb = find_extent_buffer(fs_info, start); in alloc_extent_buffer()
6112 if (eb) in alloc_extent_buffer()
6113 return eb; in alloc_extent_buffer()
6115 eb = __alloc_extent_buffer(fs_info, start, len); in alloc_extent_buffer()
6116 if (!eb) in alloc_extent_buffer()
6118 btrfs_set_buffer_lockdep_class(owner_root, eb, level); in alloc_extent_buffer()
6120 num_pages = num_extent_pages(eb); in alloc_extent_buffer()
6160 ret = attach_extent_buffer_page(eb, p, prealloc); in alloc_extent_buffer()
6174 WARN_ON(btrfs_page_test_dirty(fs_info, p, eb->start, eb->len)); in alloc_extent_buffer()
6175 eb->pages[i] = p; in alloc_extent_buffer()
6188 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags); in alloc_extent_buffer()
6198 start >> fs_info->sectorsize_bits, eb); in alloc_extent_buffer()
6209 check_buffer_tree_ref(eb); in alloc_extent_buffer()
6210 set_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags); in alloc_extent_buffer()
6218 unlock_page(eb->pages[i]); in alloc_extent_buffer()
6219 return eb; in alloc_extent_buffer()
6222 WARN_ON(!atomic_dec_and_test(&eb->refs)); in alloc_extent_buffer()
6224 if (eb->pages[i]) in alloc_extent_buffer()
6225 unlock_page(eb->pages[i]); in alloc_extent_buffer()
6228 btrfs_release_extent_buffer(eb); in alloc_extent_buffer()
6234 struct extent_buffer *eb = in btrfs_release_extent_buffer_rcu() local
6237 __free_extent_buffer(eb); in btrfs_release_extent_buffer_rcu()
6240 static int release_extent_buffer(struct extent_buffer *eb) in release_extent_buffer() argument
6241 __releases(&eb->refs_lock) in release_extent_buffer()
6243 lockdep_assert_held(&eb->refs_lock); in release_extent_buffer()
6245 WARN_ON(atomic_read(&eb->refs) == 0); in release_extent_buffer()
6246 if (atomic_dec_and_test(&eb->refs)) { in release_extent_buffer()
6247 if (test_and_clear_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags)) { in release_extent_buffer()
6248 struct btrfs_fs_info *fs_info = eb->fs_info; in release_extent_buffer()
6250 spin_unlock(&eb->refs_lock); in release_extent_buffer()
6254 eb->start >> fs_info->sectorsize_bits); in release_extent_buffer()
6257 spin_unlock(&eb->refs_lock); in release_extent_buffer()
6260 btrfs_leak_debug_del(&eb->fs_info->eb_leak_lock, &eb->leak_list); in release_extent_buffer()
6262 btrfs_release_extent_buffer_pages(eb); in release_extent_buffer()
6264 if (unlikely(test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags))) { in release_extent_buffer()
6265 __free_extent_buffer(eb); in release_extent_buffer()
6269 call_rcu(&eb->rcu_head, btrfs_release_extent_buffer_rcu); in release_extent_buffer()
6272 spin_unlock(&eb->refs_lock); in release_extent_buffer()
6277 void free_extent_buffer(struct extent_buffer *eb) in free_extent_buffer() argument
6281 if (!eb) in free_extent_buffer()
6285 refs = atomic_read(&eb->refs); in free_extent_buffer()
6286 if ((!test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags) && refs <= 3) in free_extent_buffer()
6287 || (test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags) && in free_extent_buffer()
6290 old = atomic_cmpxchg(&eb->refs, refs, refs - 1); in free_extent_buffer()
6295 spin_lock(&eb->refs_lock); in free_extent_buffer()
6296 if (atomic_read(&eb->refs) == 2 && in free_extent_buffer()
6297 test_bit(EXTENT_BUFFER_STALE, &eb->bflags) && in free_extent_buffer()
6298 !extent_buffer_under_io(eb) && in free_extent_buffer()
6299 test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) in free_extent_buffer()
6300 atomic_dec(&eb->refs); in free_extent_buffer()
6306 release_extent_buffer(eb); in free_extent_buffer()
6309 void free_extent_buffer_stale(struct extent_buffer *eb) in free_extent_buffer_stale() argument
6311 if (!eb) in free_extent_buffer_stale()
6314 spin_lock(&eb->refs_lock); in free_extent_buffer_stale()
6315 set_bit(EXTENT_BUFFER_STALE, &eb->bflags); in free_extent_buffer_stale()
6317 if (atomic_read(&eb->refs) == 2 && !extent_buffer_under_io(eb) && in free_extent_buffer_stale()
6318 test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) in free_extent_buffer_stale()
6319 atomic_dec(&eb->refs); in free_extent_buffer_stale()
6320 release_extent_buffer(eb); in free_extent_buffer_stale()
6335 static void clear_subpage_extent_buffer_dirty(const struct extent_buffer *eb) in clear_subpage_extent_buffer_dirty() argument
6337 struct btrfs_fs_info *fs_info = eb->fs_info; in clear_subpage_extent_buffer_dirty()
6338 struct page *page = eb->pages[0]; in clear_subpage_extent_buffer_dirty()
6343 last = btrfs_subpage_clear_and_test_dirty(fs_info, page, eb->start, in clear_subpage_extent_buffer_dirty()
6344 eb->len); in clear_subpage_extent_buffer_dirty()
6348 WARN_ON(atomic_read(&eb->refs) == 0); in clear_subpage_extent_buffer_dirty()
6351 void clear_extent_buffer_dirty(const struct extent_buffer *eb) in clear_extent_buffer_dirty() argument
6357 if (eb->fs_info->sectorsize < PAGE_SIZE) in clear_extent_buffer_dirty()
6358 return clear_subpage_extent_buffer_dirty(eb); in clear_extent_buffer_dirty()
6360 num_pages = num_extent_pages(eb); in clear_extent_buffer_dirty()
6363 page = eb->pages[i]; in clear_extent_buffer_dirty()
6371 WARN_ON(atomic_read(&eb->refs) == 0); in clear_extent_buffer_dirty()
6374 bool set_extent_buffer_dirty(struct extent_buffer *eb) in set_extent_buffer_dirty() argument
6380 check_buffer_tree_ref(eb); in set_extent_buffer_dirty()
6382 was_dirty = test_and_set_bit(EXTENT_BUFFER_DIRTY, &eb->bflags); in set_extent_buffer_dirty()
6384 num_pages = num_extent_pages(eb); in set_extent_buffer_dirty()
6385 WARN_ON(atomic_read(&eb->refs) == 0); in set_extent_buffer_dirty()
6386 WARN_ON(!test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)); in set_extent_buffer_dirty()
6389 bool subpage = eb->fs_info->sectorsize < PAGE_SIZE; in set_extent_buffer_dirty()
6403 lock_page(eb->pages[0]); in set_extent_buffer_dirty()
6405 btrfs_page_set_dirty(eb->fs_info, eb->pages[i], in set_extent_buffer_dirty()
6406 eb->start, eb->len); in set_extent_buffer_dirty()
6408 unlock_page(eb->pages[0]); in set_extent_buffer_dirty()
6412 ASSERT(PageDirty(eb->pages[i])); in set_extent_buffer_dirty()
6418 void clear_extent_buffer_uptodate(struct extent_buffer *eb) in clear_extent_buffer_uptodate() argument
6420 struct btrfs_fs_info *fs_info = eb->fs_info; in clear_extent_buffer_uptodate()
6425 clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags); in clear_extent_buffer_uptodate()
6426 num_pages = num_extent_pages(eb); in clear_extent_buffer_uptodate()
6428 page = eb->pages[i]; in clear_extent_buffer_uptodate()
6431 eb->start, eb->len); in clear_extent_buffer_uptodate()
6435 void set_extent_buffer_uptodate(struct extent_buffer *eb) in set_extent_buffer_uptodate() argument
6437 struct btrfs_fs_info *fs_info = eb->fs_info; in set_extent_buffer_uptodate()
6442 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags); in set_extent_buffer_uptodate()
6443 num_pages = num_extent_pages(eb); in set_extent_buffer_uptodate()
6445 page = eb->pages[i]; in set_extent_buffer_uptodate()
6446 btrfs_page_set_uptodate(fs_info, page, eb->start, eb->len); in set_extent_buffer_uptodate()
6450 static int read_extent_buffer_subpage(struct extent_buffer *eb, int wait, in read_extent_buffer_subpage() argument
6453 struct btrfs_fs_info *fs_info = eb->fs_info; in read_extent_buffer_subpage()
6455 struct page *page = eb->pages[0]; in read_extent_buffer_subpage()
6459 ASSERT(!test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags)); in read_extent_buffer_subpage()
6464 if (!try_lock_extent(io_tree, eb->start, eb->start + eb->len - 1)) in read_extent_buffer_subpage()
6467 ret = lock_extent(io_tree, eb->start, eb->start + eb->len - 1); in read_extent_buffer_subpage()
6473 if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags) || in read_extent_buffer_subpage()
6475 btrfs_subpage_test_uptodate(fs_info, page, eb->start, eb->len)) { in read_extent_buffer_subpage()
6476 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags); in read_extent_buffer_subpage()
6477 unlock_extent(io_tree, eb->start, eb->start + eb->len - 1); in read_extent_buffer_subpage()
6481 clear_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags); in read_extent_buffer_subpage()
6482 eb->read_mirror = 0; in read_extent_buffer_subpage()
6483 atomic_set(&eb->io_pages, 1); in read_extent_buffer_subpage()
6484 check_buffer_tree_ref(eb); in read_extent_buffer_subpage()
6485 btrfs_subpage_clear_error(fs_info, page, eb->start, eb->len); in read_extent_buffer_subpage()
6487 btrfs_subpage_start_reader(fs_info, page, eb->start, eb->len); in read_extent_buffer_subpage()
6489 page, eb->start, eb->len, in read_extent_buffer_subpage()
6490 eb->start - page_offset(page), in read_extent_buffer_subpage()
6499 atomic_dec(&eb->io_pages); in read_extent_buffer_subpage()
6512 wait_extent_bit(io_tree, eb->start, eb->start + eb->len - 1, EXTENT_LOCKED); in read_extent_buffer_subpage()
6513 if (!test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags)) in read_extent_buffer_subpage()
6518 int read_extent_buffer_pages(struct extent_buffer *eb, int wait, int mirror_num) in read_extent_buffer_pages() argument
6530 if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags)) in read_extent_buffer_pages()
6533 if (eb->fs_info->sectorsize < PAGE_SIZE) in read_extent_buffer_pages()
6534 return read_extent_buffer_subpage(eb, wait, mirror_num); in read_extent_buffer_pages()
6536 num_pages = num_extent_pages(eb); in read_extent_buffer_pages()
6538 page = eb->pages[i]; in read_extent_buffer_pages()
6560 page = eb->pages[i]; in read_extent_buffer_pages()
6568 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags); in read_extent_buffer_pages()
6572 clear_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags); in read_extent_buffer_pages()
6573 eb->read_mirror = 0; in read_extent_buffer_pages()
6574 atomic_set(&eb->io_pages, num_reads); in read_extent_buffer_pages()
6579 check_buffer_tree_ref(eb); in read_extent_buffer_pages()
6581 page = eb->pages[i]; in read_extent_buffer_pages()
6585 atomic_dec(&eb->io_pages); in read_extent_buffer_pages()
6604 atomic_dec(&eb->io_pages); in read_extent_buffer_pages()
6622 page = eb->pages[i]; in read_extent_buffer_pages()
6633 page = eb->pages[locked_pages]; in read_extent_buffer_pages()
6639 static bool report_eb_range(const struct extent_buffer *eb, unsigned long start, in report_eb_range() argument
6642 btrfs_warn(eb->fs_info, in report_eb_range()
6644 eb->start, eb->len, start, len); in report_eb_range()
6657 static inline int check_eb_range(const struct extent_buffer *eb, in check_eb_range() argument
6663 if (unlikely(check_add_overflow(start, len, &offset) || offset > eb->len)) in check_eb_range()
6664 return report_eb_range(eb, start, len); in check_eb_range()
6669 void read_extent_buffer(const struct extent_buffer *eb, void *dstv, in read_extent_buffer() argument
6679 if (check_eb_range(eb, start, len)) in read_extent_buffer()
6682 offset = get_eb_offset_in_page(eb, start); in read_extent_buffer()
6685 page = eb->pages[i]; in read_extent_buffer()
6698 int read_extent_buffer_to_user_nofault(const struct extent_buffer *eb, in read_extent_buffer_to_user_nofault() argument
6710 WARN_ON(start > eb->len); in read_extent_buffer_to_user_nofault()
6711 WARN_ON(start + len > eb->start + eb->len); in read_extent_buffer_to_user_nofault()
6713 offset = get_eb_offset_in_page(eb, start); in read_extent_buffer_to_user_nofault()
6716 page = eb->pages[i]; in read_extent_buffer_to_user_nofault()
6734 int memcmp_extent_buffer(const struct extent_buffer *eb, const void *ptrv, in memcmp_extent_buffer() argument
6745 if (check_eb_range(eb, start, len)) in memcmp_extent_buffer()
6748 offset = get_eb_offset_in_page(eb, start); in memcmp_extent_buffer()
6751 page = eb->pages[i]; in memcmp_extent_buffer()
6774 static void assert_eb_page_uptodate(const struct extent_buffer *eb, in assert_eb_page_uptodate() argument
6777 struct btrfs_fs_info *fs_info = eb->fs_info; in assert_eb_page_uptodate()
6783 eb->start, eb->len); in assert_eb_page_uptodate()
6790 void write_extent_buffer_chunk_tree_uuid(const struct extent_buffer *eb, in write_extent_buffer_chunk_tree_uuid() argument
6795 assert_eb_page_uptodate(eb, eb->pages[0]); in write_extent_buffer_chunk_tree_uuid()
6796 kaddr = page_address(eb->pages[0]) + in write_extent_buffer_chunk_tree_uuid()
6797 get_eb_offset_in_page(eb, offsetof(struct btrfs_header, in write_extent_buffer_chunk_tree_uuid()
6802 void write_extent_buffer_fsid(const struct extent_buffer *eb, const void *srcv) in write_extent_buffer_fsid() argument
6806 assert_eb_page_uptodate(eb, eb->pages[0]); in write_extent_buffer_fsid()
6807 kaddr = page_address(eb->pages[0]) + in write_extent_buffer_fsid()
6808 get_eb_offset_in_page(eb, offsetof(struct btrfs_header, fsid)); in write_extent_buffer_fsid()
6812 void write_extent_buffer(const struct extent_buffer *eb, const void *srcv, in write_extent_buffer() argument
6822 WARN_ON(test_bit(EXTENT_BUFFER_NO_CHECK, &eb->bflags)); in write_extent_buffer()
6824 if (check_eb_range(eb, start, len)) in write_extent_buffer()
6827 offset = get_eb_offset_in_page(eb, start); in write_extent_buffer()
6830 page = eb->pages[i]; in write_extent_buffer()
6831 assert_eb_page_uptodate(eb, page); in write_extent_buffer()
6844 void memzero_extent_buffer(const struct extent_buffer *eb, unsigned long start, in memzero_extent_buffer() argument
6853 if (check_eb_range(eb, start, len)) in memzero_extent_buffer()
6856 offset = get_eb_offset_in_page(eb, start); in memzero_extent_buffer()
6859 page = eb->pages[i]; in memzero_extent_buffer()
6860 assert_eb_page_uptodate(eb, page); in memzero_extent_buffer()
6945 static inline void eb_bitmap_offset(const struct extent_buffer *eb, in eb_bitmap_offset() argument
6958 offset = start + offset_in_page(eb->start) + byte_offset; in eb_bitmap_offset()
6970 int extent_buffer_test_bit(const struct extent_buffer *eb, unsigned long start, in extent_buffer_test_bit() argument
6978 eb_bitmap_offset(eb, start, nr, &i, &offset); in extent_buffer_test_bit()
6979 page = eb->pages[i]; in extent_buffer_test_bit()
6980 assert_eb_page_uptodate(eb, page); in extent_buffer_test_bit()
6992 void extent_buffer_bitmap_set(const struct extent_buffer *eb, unsigned long start, in extent_buffer_bitmap_set() argument
7003 eb_bitmap_offset(eb, start, pos, &i, &offset); in extent_buffer_bitmap_set()
7004 page = eb->pages[i]; in extent_buffer_bitmap_set()
7005 assert_eb_page_uptodate(eb, page); in extent_buffer_bitmap_set()
7015 page = eb->pages[++i]; in extent_buffer_bitmap_set()
7016 assert_eb_page_uptodate(eb, page); in extent_buffer_bitmap_set()
7034 void extent_buffer_bitmap_clear(const struct extent_buffer *eb, in extent_buffer_bitmap_clear() argument
7046 eb_bitmap_offset(eb, start, pos, &i, &offset); in extent_buffer_bitmap_clear()
7047 page = eb->pages[i]; in extent_buffer_bitmap_clear()
7048 assert_eb_page_uptodate(eb, page); in extent_buffer_bitmap_clear()
7058 page = eb->pages[++i]; in extent_buffer_bitmap_clear()
7059 assert_eb_page_uptodate(eb, page); in extent_buffer_bitmap_clear()
7207 struct extent_buffer *eb = NULL; in try_release_subpage_extent_buffer() local
7218 eb = get_next_extent_buffer(fs_info, page, cur); in try_release_subpage_extent_buffer()
7219 if (!eb) { in try_release_subpage_extent_buffer()
7224 cur = eb->start + eb->len; in try_release_subpage_extent_buffer()
7230 spin_lock(&eb->refs_lock); in try_release_subpage_extent_buffer()
7231 if (atomic_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) { in try_release_subpage_extent_buffer()
7232 spin_unlock(&eb->refs_lock); in try_release_subpage_extent_buffer()
7243 if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) { in try_release_subpage_extent_buffer()
7244 spin_unlock(&eb->refs_lock); in try_release_subpage_extent_buffer()
7253 release_extent_buffer(eb); in try_release_subpage_extent_buffer()
7271 struct extent_buffer *eb; in try_release_extent_buffer() local
7286 eb = (struct extent_buffer *)page->private; in try_release_extent_buffer()
7287 BUG_ON(!eb); in try_release_extent_buffer()
7294 spin_lock(&eb->refs_lock); in try_release_extent_buffer()
7295 if (atomic_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) { in try_release_extent_buffer()
7296 spin_unlock(&eb->refs_lock); in try_release_extent_buffer()
7306 if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) { in try_release_extent_buffer()
7307 spin_unlock(&eb->refs_lock); in try_release_extent_buffer()
7311 return release_extent_buffer(eb); in try_release_extent_buffer()
7329 struct extent_buffer *eb; in btrfs_readahead_tree_block() local
7332 eb = btrfs_find_create_tree_block(fs_info, bytenr, owner_root, level); in btrfs_readahead_tree_block()
7333 if (IS_ERR(eb)) in btrfs_readahead_tree_block()
7336 if (btrfs_buffer_uptodate(eb, gen, 1)) { in btrfs_readahead_tree_block()
7337 free_extent_buffer(eb); in btrfs_readahead_tree_block()
7341 ret = read_extent_buffer_pages(eb, WAIT_NONE, 0); in btrfs_readahead_tree_block()
7343 free_extent_buffer_stale(eb); in btrfs_readahead_tree_block()
7345 free_extent_buffer(eb); in btrfs_readahead_tree_block()