43 #include "ordered-data.h"
179  * Cleanup all submitted ordered extents in specified range to handle errors
185  * to be released, which we want to happen only when finishing the ordered
207 		 * btrfs_mark_ordered_io_finished() to clear page Ordered and  in btrfs_cleanup_ordered_extents()
208 		 * run the ordered extent accounting.  in btrfs_cleanup_ordered_extents()
210 		 * Here we can't just clear the Ordered bit, or  in btrfs_cleanup_ordered_extents()
212 		 * for the page range, and the ordered extent will never finish.  in btrfs_cleanup_ordered_extents()
224 		 * Here we just clear all Ordered bits for every page in the  in btrfs_cleanup_ordered_extents()
226 		 * the ordered extent accounting for the range.  in btrfs_cleanup_ordered_extents()
600  * This is done inside an ordered work queue, and the compression
602  * two, and the ordered work queue takes care of making sure that
1073  * Phase two of compressed writeback.  This is the ordered portion of the code,
1139  * allocate extents on disk for the range, and create ordered data structs
1166  * while-loop, the ordered extents created in previous iterations are kept
1313 			 * at out_unlock label to free meta of this ordered  in cow_file_range()
1318 			 * skip current ordered extent.  in cow_file_range()
1333 		 * Do set the Ordered (Private2) bit so we know this page was  in cow_file_range()
1354 		 * free metadata of current ordered extent, we're OK to exit.  in cow_file_range()
1397 	 * For the range (1). We have already instantiated the ordered extents  in cow_file_range()
1417 	 * (or a subrange) and failed to create the respective ordered extent,  in cow_file_range()
2131 			 * from freeing metadata of created ordered extent.  in run_delalloc_nocow()
2146 		 * handler, as metadata for created ordered extent will only  in run_delalloc_nocow()
2511  * checksummed and sums are attached onto the ordered extent record.
2513  * At IO completion time the cums attached on the ordered extent record
2640 	struct btrfs_ordered_extent *ordered;  in extract_ordered_extent()  local
2649 	ordered = btrfs_lookup_ordered_extent(inode, file_offset);  in extract_ordered_extent()
2650 	if (WARN_ON_ONCE(!ordered))  in extract_ordered_extent()
2654 	if (ordered->disk_num_bytes == len)  in extract_ordered_extent()
2657 	/* We cannot split once end_bio'd ordered extent */  in extract_ordered_extent()
2658 	if (WARN_ON_ONCE(ordered->bytes_left != ordered->disk_num_bytes)) {  in extract_ordered_extent()
2663 	/* We cannot split a compressed ordered extent */  in extract_ordered_extent()
2664 	if (WARN_ON_ONCE(ordered->disk_num_bytes != ordered->num_bytes)) {  in extract_ordered_extent()
2669 	ordered_end = ordered->disk_bytenr + ordered->disk_num_bytes;  in extract_ordered_extent()
2670 	/* bio must be in one ordered extent */  in extract_ordered_extent()
2671 	if (WARN_ON_ONCE(start < ordered->disk_bytenr || end > ordered_end)) {  in extract_ordered_extent()
2677 	if (WARN_ON_ONCE(!list_empty(&ordered->list))) {  in extract_ordered_extent()
2682 	file_len = ordered->num_bytes;  in extract_ordered_extent()
2683 	pre = start - ordered->disk_bytenr;  in extract_ordered_extent()
2686 	ret = btrfs_split_ordered_extent(ordered, pre, post);  in extract_ordered_extent()
2692 	btrfs_put_ordered_extent(ordered);  in extract_ordered_extent()
2769  * given a list of ordered sums record them in the inode.  This happens
2869 	struct btrfs_ordered_extent *ordered;  in btrfs_writepage_fixup_worker()  local
2907 		 *    while waiting for the ordered extent.  in btrfs_writepage_fixup_worker()
2936 	/* already ordered? We're done */  in btrfs_writepage_fixup_worker()
2940 	ordered = btrfs_lookup_ordered_range(inode, page_start, PAGE_SIZE);  in btrfs_writepage_fixup_worker()
2941 	if (ordered) {  in btrfs_writepage_fixup_worker()
2945 		btrfs_start_ordered_extent(ordered, 1);  in btrfs_writepage_fixup_worker()
2946 		btrfs_put_ordered_extent(ordered);  in btrfs_writepage_fixup_worker()
2998  * properly happens and the data=ordered rules are followed.
3000  * In our case any range that doesn't have the ORDERED bit set
3002  * to fix it up.  The async helper will wait for ordered extents, set
3011 	/* This page has ordered extent covering it already */  in btrfs_writepage_cow_fixup()
3111 	 * EXTENT_DELALLOC_BIT bit through the ordered extent completion.  in insert_reserved_file_extent()
3181 	 * For delalloc, when completing an ordered extent we update the inode's  in insert_ordered_extent_file_extent()
3184 	 * except if the ordered extent was truncated.  in insert_ordered_extent_file_extent()
3196  * As ordered data IO finishes, this gets called so we can finish
3197  * an ordered extent if the range of bytes in the file it covers are
3349 		 * If we failed to finish this ordered extent for any reason we  in btrfs_finish_ordered_io()
3350 		 * need to make sure BTRFS_ORDERED_IOERR is set on the ordered  in btrfs_finish_ordered_io()
3354 		 * marking this ordered extent as failed.  in btrfs_finish_ordered_io()
3368 		 * If the ordered extent had an IOERR or something else went  in btrfs_finish_ordered_io()
3369 		 * wrong we need to return the space for this ordered extent  in btrfs_finish_ordered_io()
3398 	 * updating everything for this ordered extent.  in btrfs_finish_ordered_io()
4866 	struct btrfs_ordered_extent *ordered;  in btrfs_truncate_block()  local
4934 	ordered = btrfs_lookup_ordered_extent(inode, block_start);  in btrfs_truncate_block()
4935 	if (ordered) {  in btrfs_truncate_block()
4939 		btrfs_start_ordered_extent(ordered, 1);  in btrfs_truncate_block()
4940 		btrfs_put_ordered_extent(ordered);  in btrfs_truncate_block()
5275  * the ordered io finishing, while we release here the extent_map and
7144  *	 any ordered extents.
7253 	struct btrfs_ordered_extent *ordered;  in lock_extent_direct()  local
7265 		 * doing DIO to, so we need to make sure there's no ordered  in lock_extent_direct()
7268 		ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), lockstart,  in lock_extent_direct()
7278 		if (!ordered &&  in lock_extent_direct()
7285 		if (ordered) {  in lock_extent_direct()
7287 				btrfs_put_ordered_extent(ordered);  in lock_extent_direct()
7292 			 * If we are doing a DIO read and the ordered extent we  in lock_extent_direct()
7298 			 * created an ordered extent for a previous extent map  in lock_extent_direct()
7307 			    test_bit(BTRFS_ORDERED_DIRECT, &ordered->flags))  in lock_extent_direct()
7308 				btrfs_start_ordered_extent(ordered, 1);  in lock_extent_direct()
7311 			btrfs_put_ordered_extent(ordered);  in lock_extent_direct()
7320 			 * ordered dio extent we created before but did not have  in lock_extent_direct()
7323 			 * ordered extent to complete while holding a lock on  in lock_extent_direct()
7501 	 * We have created our ordered extent, so we can now release our reservation  in btrfs_get_blocks_direct_write()
7808 	 * this and loads of dio_bio->bi_status after this are fully ordered.  in btrfs_dio_private_put()
8114 	 * complete and writeback to start. We also need to wait for ordered  in btrfs_fiemap()
8225 	 * We have folio locked so no new ordered extent can be created on this  in btrfs_invalidate_folio()
8229 	 * Furthermore, endio function won't skip folio which has Ordered  in btrfs_invalidate_folio()
8231 	 * invalidate_folio to do the same ordered extent accounting twice  in btrfs_invalidate_folio()
8235 	 * do double ordered extent accounting on the same folio.  in btrfs_invalidate_folio()
8250 	 * still safe to wait for ordered extent to finish.  in btrfs_invalidate_folio()
8262 		struct btrfs_ordered_extent *ordered;  in btrfs_invalidate_folio()  local
8267 		ordered = btrfs_lookup_first_ordered_range(inode, cur,  in btrfs_invalidate_folio()
8269 		if (!ordered) {  in btrfs_invalidate_folio()
8272 			 * No ordered extent covering this range, we are safe  in btrfs_invalidate_folio()
8278 		if (ordered->file_offset > cur) {  in btrfs_invalidate_folio()
8281 			 * covered by any ordered extent.  in btrfs_invalidate_folio()
8283 			 * the ordered extent in the next iteration.  in btrfs_invalidate_folio()
8285 			range_end = ordered->file_offset - 1;  in btrfs_invalidate_folio()
8290 		range_end = min(ordered->file_offset + ordered->num_bytes - 1,  in btrfs_invalidate_folio()
8296 			 * If Ordered (Private2) is cleared, it means endio has  in btrfs_invalidate_folio()
8307 		 * for any ordered extents now. Don't clear EXTENT_DELALLOC_NEW  in btrfs_invalidate_folio()
8308 		 * here, must leave that up for the ordered extent completion.  in btrfs_invalidate_folio()
8320 		set_bit(BTRFS_ORDERED_TRUNCATED, &ordered->flags);  in btrfs_invalidate_folio()
8321 		ordered->truncated_len = min(ordered->truncated_len,  in btrfs_invalidate_folio()
8322 					     cur - ordered->file_offset);  in btrfs_invalidate_folio()
8326 		 * If the ordered extent has finished, we're safe to delete all  in btrfs_invalidate_folio()
8331 		if (btrfs_dec_test_ordered_pending(inode, &ordered,  in btrfs_invalidate_folio()
8333 			btrfs_finish_ordered_io(ordered);  in btrfs_invalidate_folio()
8335 			 * The ordered extent has finished, now we're again  in btrfs_invalidate_folio()
8341 		if (ordered)  in btrfs_invalidate_folio()
8342 			btrfs_put_ordered_extent(ordered);  in btrfs_invalidate_folio()
8368 	 * We have iterated through all ordered extents of the page, the page  in btrfs_invalidate_folio()
8369 	 * should not have Ordered (Private2) anymore, or the above iteration  in btrfs_invalidate_folio()
8400 	struct btrfs_ordered_extent *ordered;  in btrfs_page_mkwrite()  local
8463 	 * we can't set the delalloc bits if there are pending ordered  in btrfs_page_mkwrite()
8466 	ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), page_start,  in btrfs_page_mkwrite()
8468 	if (ordered) {  in btrfs_page_mkwrite()
8472 		btrfs_start_ordered_extent(ordered, 1);  in btrfs_page_mkwrite()
8473 		btrfs_put_ordered_extent(ordered);  in btrfs_page_mkwrite()
8817 	struct btrfs_ordered_extent *ordered;  in btrfs_destroy_inode()  local
8843 	 * If this is a free space inode do not take the ordered extents lockdep  in btrfs_destroy_inode()
8849 		ordered = btrfs_lookup_first_ordered_extent(inode, (u64)-1);  in btrfs_destroy_inode()
8850 		if (!ordered)  in btrfs_destroy_inode()
8854 				  "found ordered extent %llu %llu on inode cleanup",  in btrfs_destroy_inode()
8855 				  ordered->file_offset, ordered->num_bytes);  in btrfs_destroy_inode()
8860 			btrfs_remove_ordered_extent(inode, ordered);  in btrfs_destroy_inode()
8861 			btrfs_put_ordered_extent(ordered);  in btrfs_destroy_inode()
8862 			btrfs_put_ordered_extent(ordered);  in btrfs_destroy_inode()
10474 		struct btrfs_ordered_extent *ordered;  in btrfs_encoded_read()  local
10481 		ordered = btrfs_lookup_ordered_range(inode, start,  in btrfs_encoded_read()
10483 		if (!ordered)  in btrfs_encoded_read()
10485 		btrfs_put_ordered_extent(ordered);  in btrfs_encoded_read()
10709 		struct btrfs_ordered_extent *ordered;  in btrfs_do_encoded_write()  local
10720 		ordered = btrfs_lookup_ordered_range(inode, start, num_bytes);  in btrfs_do_encoded_write()
10721 		if (!ordered &&  in btrfs_do_encoded_write()
10724 		if (ordered)  in btrfs_do_encoded_write()
10725 			btrfs_put_ordered_extent(ordered);  in btrfs_do_encoded_write()
11251  * Verify that there are no ordered extents for a given file range.
11260  * we have flushed all delalloc in the range, we have waited for all ordered
11267 	struct btrfs_ordered_extent *ordered;  in btrfs_assert_inode_range_clean()  local
11272 	ordered = btrfs_lookup_first_ordered_range(inode, start, end + 1 - start);  in btrfs_assert_inode_range_clean()
11273 	if (ordered) {  in btrfs_assert_inode_range_clean()
11275 "found unexpected ordered extent in file range [%llu, %llu] for inode %llu root %llu (ordered range…  in btrfs_assert_inode_range_clean()
11277 			  ordered->file_offset,  in btrfs_assert_inode_range_clean()
11278 			  ordered->file_offset + ordered->num_bytes - 1);  in btrfs_assert_inode_range_clean()
11279 		btrfs_put_ordered_extent(ordered);  in btrfs_assert_inode_range_clean()
11282 	ASSERT(ordered == NULL);  in btrfs_assert_inode_range_clean()