389 static int io_ctl_init(struct btrfs_io_ctl *io_ctl, struct inode *inode,  in io_ctl_init()  argument
400 	memset(io_ctl, 0, sizeof(struct btrfs_io_ctl));  in io_ctl_init()
402 	io_ctl->pages = kcalloc(num_pages, sizeof(struct page *), GFP_NOFS);  in io_ctl_init()
403 	if (!io_ctl->pages)  in io_ctl_init()
406 	io_ctl->num_pages = num_pages;  in io_ctl_init()
407 	io_ctl->fs_info = btrfs_sb(inode->i_sb);  in io_ctl_init()
408 	io_ctl->inode = inode;  in io_ctl_init()
414 static void io_ctl_free(struct btrfs_io_ctl *io_ctl)  in io_ctl_free()  argument
416 	kfree(io_ctl->pages);  in io_ctl_free()
417 	io_ctl->pages = NULL;  in io_ctl_free()
420 static void io_ctl_unmap_page(struct btrfs_io_ctl *io_ctl)  in io_ctl_unmap_page()  argument
422 	if (io_ctl->cur) {  in io_ctl_unmap_page()
423 		io_ctl->cur = NULL;  in io_ctl_unmap_page()
424 		io_ctl->orig = NULL;  in io_ctl_unmap_page()
428 static void io_ctl_map_page(struct btrfs_io_ctl *io_ctl, int clear)  in io_ctl_map_page()  argument
430 	ASSERT(io_ctl->index < io_ctl->num_pages);  in io_ctl_map_page()
431 	io_ctl->page = io_ctl->pages[io_ctl->index++];  in io_ctl_map_page()
432 	io_ctl->cur = page_address(io_ctl->page);  in io_ctl_map_page()
433 	io_ctl->orig = io_ctl->cur;  in io_ctl_map_page()
434 	io_ctl->size = PAGE_SIZE;  in io_ctl_map_page()
436 		clear_page(io_ctl->cur);  in io_ctl_map_page()
439 static void io_ctl_drop_pages(struct btrfs_io_ctl *io_ctl)  in io_ctl_drop_pages()  argument
443 	io_ctl_unmap_page(io_ctl);  in io_ctl_drop_pages()
445 	for (i = 0; i < io_ctl->num_pages; i++) {  in io_ctl_drop_pages()
446 		if (io_ctl->pages[i]) {  in io_ctl_drop_pages()
447 			btrfs_page_clear_checked(io_ctl->fs_info,  in io_ctl_drop_pages()
448 					io_ctl->pages[i],  in io_ctl_drop_pages()
449 					page_offset(io_ctl->pages[i]),  in io_ctl_drop_pages()
451 			unlock_page(io_ctl->pages[i]);  in io_ctl_drop_pages()
452 			put_page(io_ctl->pages[i]);  in io_ctl_drop_pages()
457 static int io_ctl_prepare_pages(struct btrfs_io_ctl *io_ctl, bool uptodate)  in io_ctl_prepare_pages()  argument
460 	struct inode *inode = io_ctl->inode;  in io_ctl_prepare_pages()
464 	for (i = 0; i < io_ctl->num_pages; i++) {  in io_ctl_prepare_pages()
469 			io_ctl_drop_pages(io_ctl);  in io_ctl_prepare_pages()
477 			io_ctl_drop_pages(io_ctl);  in io_ctl_prepare_pages()
481 		io_ctl->pages[i] = page;  in io_ctl_prepare_pages()
488 				io_ctl_drop_pages(io_ctl);  in io_ctl_prepare_pages()
494 				io_ctl_drop_pages(io_ctl);  in io_ctl_prepare_pages()
500 	for (i = 0; i < io_ctl->num_pages; i++)  in io_ctl_prepare_pages()
501 		clear_page_dirty_for_io(io_ctl->pages[i]);  in io_ctl_prepare_pages()
506 static void io_ctl_set_generation(struct btrfs_io_ctl *io_ctl, u64 generation)  in io_ctl_set_generation()  argument
508 	io_ctl_map_page(io_ctl, 1);  in io_ctl_set_generation()
514 	io_ctl->cur += (sizeof(u32) * io_ctl->num_pages);  in io_ctl_set_generation()
515 	io_ctl->size -= sizeof(u64) + (sizeof(u32) * io_ctl->num_pages);  in io_ctl_set_generation()
517 	put_unaligned_le64(generation, io_ctl->cur);  in io_ctl_set_generation()
518 	io_ctl->cur += sizeof(u64);  in io_ctl_set_generation()
521 static int io_ctl_check_generation(struct btrfs_io_ctl *io_ctl, u64 generation)  in io_ctl_check_generation()  argument
529 	io_ctl->cur += sizeof(u32) * io_ctl->num_pages;  in io_ctl_check_generation()
530 	io_ctl->size -= sizeof(u64) + (sizeof(u32) * io_ctl->num_pages);  in io_ctl_check_generation()
532 	cache_gen = get_unaligned_le64(io_ctl->cur);  in io_ctl_check_generation()
534 		btrfs_err_rl(io_ctl->fs_info,  in io_ctl_check_generation()
537 		io_ctl_unmap_page(io_ctl);  in io_ctl_check_generation()
540 	io_ctl->cur += sizeof(u64);  in io_ctl_check_generation()
544 static void io_ctl_set_crc(struct btrfs_io_ctl *io_ctl, int index)  in io_ctl_set_crc()  argument
551 		offset = sizeof(u32) * io_ctl->num_pages;  in io_ctl_set_crc()
553 	crc = btrfs_crc32c(crc, io_ctl->orig + offset, PAGE_SIZE - offset);  in io_ctl_set_crc()
555 	io_ctl_unmap_page(io_ctl);  in io_ctl_set_crc()
556 	tmp = page_address(io_ctl->pages[0]);  in io_ctl_set_crc()
561 static int io_ctl_check_crc(struct btrfs_io_ctl *io_ctl, int index)  in io_ctl_check_crc()  argument
568 		offset = sizeof(u32) * io_ctl->num_pages;  in io_ctl_check_crc()
570 	tmp = page_address(io_ctl->pages[0]);  in io_ctl_check_crc()
574 	io_ctl_map_page(io_ctl, 0);  in io_ctl_check_crc()
575 	crc = btrfs_crc32c(crc, io_ctl->orig + offset, PAGE_SIZE - offset);  in io_ctl_check_crc()
578 		btrfs_err_rl(io_ctl->fs_info,  in io_ctl_check_crc()
580 		io_ctl_unmap_page(io_ctl);  in io_ctl_check_crc()
587 static int io_ctl_add_entry(struct btrfs_io_ctl *io_ctl, u64 offset, u64 bytes,  in io_ctl_add_entry()  argument
592 	if (!io_ctl->cur)  in io_ctl_add_entry()
595 	entry = io_ctl->cur;  in io_ctl_add_entry()
600 	io_ctl->cur += sizeof(struct btrfs_free_space_entry);  in io_ctl_add_entry()
601 	io_ctl->size -= sizeof(struct btrfs_free_space_entry);  in io_ctl_add_entry()
603 	if (io_ctl->size >= sizeof(struct btrfs_free_space_entry))  in io_ctl_add_entry()
606 	io_ctl_set_crc(io_ctl, io_ctl->index - 1);  in io_ctl_add_entry()
609 	if (io_ctl->index >= io_ctl->num_pages)  in io_ctl_add_entry()
613 	io_ctl_map_page(io_ctl, 1);  in io_ctl_add_entry()
617 static int io_ctl_add_bitmap(struct btrfs_io_ctl *io_ctl, void *bitmap)  in io_ctl_add_bitmap()  argument
619 	if (!io_ctl->cur)  in io_ctl_add_bitmap()
626 	if (io_ctl->cur != io_ctl->orig) {  in io_ctl_add_bitmap()
627 		io_ctl_set_crc(io_ctl, io_ctl->index - 1);  in io_ctl_add_bitmap()
628 		if (io_ctl->index >= io_ctl->num_pages)  in io_ctl_add_bitmap()
630 		io_ctl_map_page(io_ctl, 0);  in io_ctl_add_bitmap()
633 	copy_page(io_ctl->cur, bitmap);  in io_ctl_add_bitmap()
634 	io_ctl_set_crc(io_ctl, io_ctl->index - 1);  in io_ctl_add_bitmap()
635 	if (io_ctl->index < io_ctl->num_pages)  in io_ctl_add_bitmap()
636 		io_ctl_map_page(io_ctl, 0);  in io_ctl_add_bitmap()
640 static void io_ctl_zero_remaining_pages(struct btrfs_io_ctl *io_ctl)  in io_ctl_zero_remaining_pages()  argument
646 	if (io_ctl->cur != io_ctl->orig)  in io_ctl_zero_remaining_pages()
647 		io_ctl_set_crc(io_ctl, io_ctl->index - 1);  in io_ctl_zero_remaining_pages()
649 		io_ctl_unmap_page(io_ctl);  in io_ctl_zero_remaining_pages()
651 	while (io_ctl->index < io_ctl->num_pages) {  in io_ctl_zero_remaining_pages()
652 		io_ctl_map_page(io_ctl, 1);  in io_ctl_zero_remaining_pages()
653 		io_ctl_set_crc(io_ctl, io_ctl->index - 1);  in io_ctl_zero_remaining_pages()
657 static int io_ctl_read_entry(struct btrfs_io_ctl *io_ctl,  in io_ctl_read_entry()  argument
663 	if (!io_ctl->cur) {  in io_ctl_read_entry()
664 		ret = io_ctl_check_crc(io_ctl, io_ctl->index);  in io_ctl_read_entry()
669 	e = io_ctl->cur;  in io_ctl_read_entry()
673 	io_ctl->cur += sizeof(struct btrfs_free_space_entry);  in io_ctl_read_entry()
674 	io_ctl->size -= sizeof(struct btrfs_free_space_entry);  in io_ctl_read_entry()
676 	if (io_ctl->size >= sizeof(struct btrfs_free_space_entry))  in io_ctl_read_entry()
679 	io_ctl_unmap_page(io_ctl);  in io_ctl_read_entry()
684 static int io_ctl_read_bitmap(struct btrfs_io_ctl *io_ctl,  in io_ctl_read_bitmap()  argument
689 	ret = io_ctl_check_crc(io_ctl, io_ctl->index);  in io_ctl_read_bitmap()
693 	copy_page(entry->bitmap, io_ctl->cur);  in io_ctl_read_bitmap()
694 	io_ctl_unmap_page(io_ctl);  in io_ctl_read_bitmap()
750 	struct btrfs_io_ctl io_ctl;  in __load_free_space_cache()  local
803 	ret = io_ctl_init(&io_ctl, inode, 0);  in __load_free_space_cache()
809 	ret = io_ctl_prepare_pages(&io_ctl, true);  in __load_free_space_cache()
813 	ret = io_ctl_check_crc(&io_ctl, 0);  in __load_free_space_cache()
817 	ret = io_ctl_check_generation(&io_ctl, generation);  in __load_free_space_cache()
829 		ret = io_ctl_read_entry(&io_ctl, e, &type);  in __load_free_space_cache()
879 	io_ctl_unmap_page(&io_ctl);  in __load_free_space_cache()
887 		ret = io_ctl_read_bitmap(&io_ctl, e);  in __load_free_space_cache()
892 	io_ctl_drop_pages(&io_ctl);  in __load_free_space_cache()
895 	io_ctl_free(&io_ctl);  in __load_free_space_cache()
898 	io_ctl_drop_pages(&io_ctl);  in __load_free_space_cache()
1071 int write_cache_extent_entries(struct btrfs_io_ctl *io_ctl,  in write_cache_extent_entries()  argument
1104 		ret = io_ctl_add_entry(io_ctl, e->offset, e->bytes,  in write_cache_extent_entries()
1133 		ret = io_ctl_add_entry(io_ctl, trim_entry->start,  in write_cache_extent_entries()
1203 			    struct btrfs_io_ctl *io_ctl,  in write_pinned_extent_entries()  argument
1241 		ret = io_ctl_add_entry(io_ctl, extent_start, len, NULL);  in write_pinned_extent_entries()
1252 write_bitmap_entries(struct btrfs_io_ctl *io_ctl, struct list_head *bitmap_list)  in write_bitmap_entries()  argument
1259 		ret = io_ctl_add_bitmap(io_ctl, entry->bitmap);  in write_bitmap_entries()
1291 			   struct btrfs_io_ctl *io_ctl,  in cleanup_write_cache_enospc()  argument
1294 	io_ctl_drop_pages(io_ctl);  in cleanup_write_cache_enospc()
1302 				 struct btrfs_io_ctl *io_ctl,  in __btrfs_wait_cache_io()  argument
1306 	struct inode *inode = io_ctl->inode;  in __btrfs_wait_cache_io()
1318 				io_ctl->entries, io_ctl->bitmaps);  in __btrfs_wait_cache_io()
1349 		io_ctl->inode = NULL;  in __btrfs_wait_cache_io()
1362 				     block_group, &block_group->io_ctl,  in btrfs_wait_cache_io()
1383 				   struct btrfs_io_ctl *io_ctl,  in __btrfs_write_out_cache()  argument
1396 	WARN_ON(io_ctl->pages);  in __btrfs_write_out_cache()
1397 	ret = io_ctl_init(io_ctl, inode, 1);  in __btrfs_write_out_cache()
1417 	ret = io_ctl_prepare_pages(io_ctl, false);  in __btrfs_write_out_cache()
1424 	io_ctl_set_generation(io_ctl, trans->transid);  in __btrfs_write_out_cache()
1429 	ret = write_cache_extent_entries(io_ctl, ctl,  in __btrfs_write_out_cache()
1443 	ret = write_pinned_extent_entries(trans, block_group, io_ctl, &entries);  in __btrfs_write_out_cache()
1452 	ret = write_bitmap_entries(io_ctl, &bitmap_list);  in __btrfs_write_out_cache()
1459 	io_ctl_zero_remaining_pages(io_ctl);  in __btrfs_write_out_cache()
1462 	ret = btrfs_dirty_pages(BTRFS_I(inode), io_ctl->pages,  in __btrfs_write_out_cache()
1463 				io_ctl->num_pages, 0, i_size_read(inode),  in __btrfs_write_out_cache()
1474 	io_ctl_drop_pages(io_ctl);  in __btrfs_write_out_cache()
1475 	io_ctl_free(io_ctl);  in __btrfs_write_out_cache()
1485 	io_ctl->entries = entries;  in __btrfs_write_out_cache()
1486 	io_ctl->bitmaps = bitmaps;  in __btrfs_write_out_cache()
1500 	cleanup_write_cache_enospc(inode, io_ctl, &cached_state);  in __btrfs_write_out_cache()
1507 	io_ctl->inode = NULL;  in __btrfs_write_out_cache()
1508 	io_ctl_free(io_ctl);  in __btrfs_write_out_cache()
1540 				block_group, &block_group->io_ctl, trans);  in btrfs_write_out_cache()
1549 		block_group->io_ctl.inode = NULL;  in btrfs_write_out_cache()