123 	struct btrfs_subpage *subpage;  in btrfs_attach_subpage()  local
136 	subpage = btrfs_alloc_subpage(fs_info, type);  in btrfs_attach_subpage()
137 	if (IS_ERR(subpage))  in btrfs_attach_subpage()
138 		return  PTR_ERR(subpage);  in btrfs_attach_subpage()
140 	attach_page_private(page, subpage);  in btrfs_attach_subpage()
147 	struct btrfs_subpage *subpage;  in btrfs_detach_subpage()  local
153 	subpage = detach_page_private(page);  in btrfs_detach_subpage()
154 	ASSERT(subpage);  in btrfs_detach_subpage()
155 	btrfs_free_subpage(subpage);  in btrfs_detach_subpage()
182 void btrfs_free_subpage(struct btrfs_subpage *subpage)  in btrfs_free_subpage()  argument
184 	kfree(subpage);  in btrfs_free_subpage()
199 	struct btrfs_subpage *subpage;  in btrfs_page_inc_eb_refs()  local
207 	subpage = (struct btrfs_subpage *)page->private;  in btrfs_page_inc_eb_refs()
208 	atomic_inc(&subpage->eb_refs);  in btrfs_page_inc_eb_refs()
214 	struct btrfs_subpage *subpage;  in btrfs_page_dec_eb_refs()  local
222 	subpage = (struct btrfs_subpage *)page->private;  in btrfs_page_dec_eb_refs()
223 	ASSERT(atomic_read(&subpage->eb_refs));  in btrfs_page_dec_eb_refs()
224 	atomic_dec(&subpage->eb_refs);  in btrfs_page_dec_eb_refs()
246 	struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;  in btrfs_subpage_start_reader()  local
251 	atomic_add(nbits, &subpage->readers);  in btrfs_subpage_start_reader()
257 	struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;  in btrfs_subpage_end_reader()  local
264 	ASSERT(atomic_read(&subpage->readers) >= nbits);  in btrfs_subpage_end_reader()
265 	last = atomic_sub_and_test(nbits, &subpage->readers);  in btrfs_subpage_end_reader()
299 	struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;  in btrfs_subpage_start_writer()  local
305 	ASSERT(atomic_read(&subpage->readers) == 0);  in btrfs_subpage_start_writer()
306 	ret = atomic_add_return(nbits, &subpage->writers);  in btrfs_subpage_start_writer()
313 	struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;  in btrfs_subpage_end_and_test_writer()  local
325 	if (atomic_read(&subpage->writers) == 0)  in btrfs_subpage_end_and_test_writer()
328 	ASSERT(atomic_read(&subpage->writers) >= nbits);  in btrfs_subpage_end_and_test_writer()
329 	return atomic_sub_and_test(nbits, &subpage->writers);  in btrfs_subpage_end_and_test_writer()
401 #define subpage_test_bitmap_all_set(fs_info, subpage, name)		\  argument
402 	bitmap_test_range_all_set(subpage->bitmaps,			\
406 #define subpage_test_bitmap_all_zero(fs_info, subpage, name)		\  argument
407 	bitmap_test_range_all_zero(subpage->bitmaps,			\
414 	struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;  in btrfs_subpage_set_uptodate()  local
419 	spin_lock_irqsave(&subpage->lock, flags);  in btrfs_subpage_set_uptodate()
420 	bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);  in btrfs_subpage_set_uptodate()
421 	if (subpage_test_bitmap_all_set(fs_info, subpage, uptodate))  in btrfs_subpage_set_uptodate()
423 	spin_unlock_irqrestore(&subpage->lock, flags);  in btrfs_subpage_set_uptodate()
429 	struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;  in btrfs_subpage_clear_uptodate()  local
434 	spin_lock_irqsave(&subpage->lock, flags);  in btrfs_subpage_clear_uptodate()
435 	bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);  in btrfs_subpage_clear_uptodate()
437 	spin_unlock_irqrestore(&subpage->lock, flags);  in btrfs_subpage_clear_uptodate()
443 	struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;  in btrfs_subpage_set_error()  local
448 	spin_lock_irqsave(&subpage->lock, flags);  in btrfs_subpage_set_error()
449 	bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);  in btrfs_subpage_set_error()
451 	spin_unlock_irqrestore(&subpage->lock, flags);  in btrfs_subpage_set_error()
457 	struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;  in btrfs_subpage_clear_error()  local
462 	spin_lock_irqsave(&subpage->lock, flags);  in btrfs_subpage_clear_error()
463 	bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);  in btrfs_subpage_clear_error()
464 	if (subpage_test_bitmap_all_zero(fs_info, subpage, error))  in btrfs_subpage_clear_error()
466 	spin_unlock_irqrestore(&subpage->lock, flags);  in btrfs_subpage_clear_error()
472 	struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;  in btrfs_subpage_set_dirty()  local
477 	spin_lock_irqsave(&subpage->lock, flags);  in btrfs_subpage_set_dirty()
478 	bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);  in btrfs_subpage_set_dirty()
479 	spin_unlock_irqrestore(&subpage->lock, flags);  in btrfs_subpage_set_dirty()
496 	struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;  in btrfs_subpage_clear_and_test_dirty()  local
502 	spin_lock_irqsave(&subpage->lock, flags);  in btrfs_subpage_clear_and_test_dirty()
503 	bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);  in btrfs_subpage_clear_and_test_dirty()
504 	if (subpage_test_bitmap_all_zero(fs_info, subpage, dirty))  in btrfs_subpage_clear_and_test_dirty()
506 	spin_unlock_irqrestore(&subpage->lock, flags);  in btrfs_subpage_clear_and_test_dirty()
523 	struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;  in btrfs_subpage_set_writeback()  local
528 	spin_lock_irqsave(&subpage->lock, flags);  in btrfs_subpage_set_writeback()
529 	bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);  in btrfs_subpage_set_writeback()
531 	spin_unlock_irqrestore(&subpage->lock, flags);  in btrfs_subpage_set_writeback()
537 	struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;  in btrfs_subpage_clear_writeback()  local
542 	spin_lock_irqsave(&subpage->lock, flags);  in btrfs_subpage_clear_writeback()
543 	bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);  in btrfs_subpage_clear_writeback()
544 	if (subpage_test_bitmap_all_zero(fs_info, subpage, writeback)) {  in btrfs_subpage_clear_writeback()
548 	spin_unlock_irqrestore(&subpage->lock, flags);  in btrfs_subpage_clear_writeback()
554 	struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;  in btrfs_subpage_set_ordered()  local
559 	spin_lock_irqsave(&subpage->lock, flags);  in btrfs_subpage_set_ordered()
560 	bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);  in btrfs_subpage_set_ordered()
562 	spin_unlock_irqrestore(&subpage->lock, flags);  in btrfs_subpage_set_ordered()
568 	struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;  in btrfs_subpage_clear_ordered()  local
573 	spin_lock_irqsave(&subpage->lock, flags);  in btrfs_subpage_clear_ordered()
574 	bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);  in btrfs_subpage_clear_ordered()
575 	if (subpage_test_bitmap_all_zero(fs_info, subpage, ordered))  in btrfs_subpage_clear_ordered()
577 	spin_unlock_irqrestore(&subpage->lock, flags);  in btrfs_subpage_clear_ordered()
583 	struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;  in btrfs_subpage_set_checked()  local
588 	spin_lock_irqsave(&subpage->lock, flags);  in btrfs_subpage_set_checked()
589 	bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);  in btrfs_subpage_set_checked()
590 	if (subpage_test_bitmap_all_set(fs_info, subpage, checked))  in btrfs_subpage_set_checked()
592 	spin_unlock_irqrestore(&subpage->lock, flags);  in btrfs_subpage_set_checked()
598 	struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;  in btrfs_subpage_clear_checked()  local
603 	spin_lock_irqsave(&subpage->lock, flags);  in btrfs_subpage_clear_checked()
604 	bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);  in btrfs_subpage_clear_checked()
606 	spin_unlock_irqrestore(&subpage->lock, flags);  in btrfs_subpage_clear_checked()
617 	struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; \
623 	spin_lock_irqsave(&subpage->lock, flags);			\
624 	ret = bitmap_test_range_all_set(subpage->bitmaps, start_bit,	\
626 	spin_unlock_irqrestore(&subpage->lock, flags);			\
714 	struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;  in btrfs_page_assert_not_dirty()  local
724 	ASSERT(subpage_test_bitmap_all_zero(fs_info, subpage, dirty));  in btrfs_page_assert_not_dirty()
745 	struct btrfs_subpage *subpage;  in btrfs_page_unlock_writer()  local
753 	subpage = (struct btrfs_subpage *)page->private;  in btrfs_page_unlock_writer()
762 	if (atomic_read(&subpage->writers) == 0)  in btrfs_page_unlock_writer()