379 static int io_ctl_init(struct btrfs_io_ctl *io_ctl, struct inode *inode,  in io_ctl_init()  argument
390 	memset(io_ctl, 0, sizeof(struct btrfs_io_ctl));  in io_ctl_init()
392 	io_ctl->pages = kcalloc(num_pages, sizeof(struct page *), GFP_NOFS);  in io_ctl_init()
393 	if (!io_ctl->pages)  in io_ctl_init()
396 	io_ctl->num_pages = num_pages;  in io_ctl_init()
397 	io_ctl->fs_info = btrfs_sb(inode->i_sb);  in io_ctl_init()
398 	io_ctl->inode = inode;  in io_ctl_init()
404 static void io_ctl_free(struct btrfs_io_ctl *io_ctl)  in io_ctl_free()  argument
406 	kfree(io_ctl->pages);  in io_ctl_free()
407 	io_ctl->pages = NULL;  in io_ctl_free()
410 static void io_ctl_unmap_page(struct btrfs_io_ctl *io_ctl)  in io_ctl_unmap_page()  argument
412 	if (io_ctl->cur) {  in io_ctl_unmap_page()
413 		io_ctl->cur = NULL;  in io_ctl_unmap_page()
414 		io_ctl->orig = NULL;  in io_ctl_unmap_page()
418 static void io_ctl_map_page(struct btrfs_io_ctl *io_ctl, int clear)  in io_ctl_map_page()  argument
420 	ASSERT(io_ctl->index < io_ctl->num_pages);  in io_ctl_map_page()
421 	io_ctl->page = io_ctl->pages[io_ctl->index++];  in io_ctl_map_page()
422 	io_ctl->cur = page_address(io_ctl->page);  in io_ctl_map_page()
423 	io_ctl->orig = io_ctl->cur;  in io_ctl_map_page()
424 	io_ctl->size = PAGE_SIZE;  in io_ctl_map_page()
426 		clear_page(io_ctl->cur);  in io_ctl_map_page()
429 static void io_ctl_drop_pages(struct btrfs_io_ctl *io_ctl)  in io_ctl_drop_pages()  argument
433 	io_ctl_unmap_page(io_ctl);  in io_ctl_drop_pages()
435 	for (i = 0; i < io_ctl->num_pages; i++) {  in io_ctl_drop_pages()
436 		if (io_ctl->pages[i]) {  in io_ctl_drop_pages()
437 			btrfs_page_clear_checked(io_ctl->fs_info,  in io_ctl_drop_pages()
438 					io_ctl->pages[i],  in io_ctl_drop_pages()
439 					page_offset(io_ctl->pages[i]),  in io_ctl_drop_pages()
441 			unlock_page(io_ctl->pages[i]);  in io_ctl_drop_pages()
442 			put_page(io_ctl->pages[i]);  in io_ctl_drop_pages()
447 static int io_ctl_prepare_pages(struct btrfs_io_ctl *io_ctl, bool uptodate)  in io_ctl_prepare_pages()  argument
450 	struct inode *inode = io_ctl->inode;  in io_ctl_prepare_pages()
454 	for (i = 0; i < io_ctl->num_pages; i++) {  in io_ctl_prepare_pages()
459 			io_ctl_drop_pages(io_ctl);  in io_ctl_prepare_pages()
467 			io_ctl_drop_pages(io_ctl);  in io_ctl_prepare_pages()
471 		io_ctl->pages[i] = page;  in io_ctl_prepare_pages()
478 				io_ctl_drop_pages(io_ctl);  in io_ctl_prepare_pages()
484 				io_ctl_drop_pages(io_ctl);  in io_ctl_prepare_pages()
490 	for (i = 0; i < io_ctl->num_pages; i++)  in io_ctl_prepare_pages()
491 		clear_page_dirty_for_io(io_ctl->pages[i]);  in io_ctl_prepare_pages()
496 static void io_ctl_set_generation(struct btrfs_io_ctl *io_ctl, u64 generation)  in io_ctl_set_generation()  argument
498 	io_ctl_map_page(io_ctl, 1);  in io_ctl_set_generation()
504 	io_ctl->cur += (sizeof(u32) * io_ctl->num_pages);  in io_ctl_set_generation()
505 	io_ctl->size -= sizeof(u64) + (sizeof(u32) * io_ctl->num_pages);  in io_ctl_set_generation()
507 	put_unaligned_le64(generation, io_ctl->cur);  in io_ctl_set_generation()
508 	io_ctl->cur += sizeof(u64);  in io_ctl_set_generation()
511 static int io_ctl_check_generation(struct btrfs_io_ctl *io_ctl, u64 generation)  in io_ctl_check_generation()  argument
519 	io_ctl->cur += sizeof(u32) * io_ctl->num_pages;  in io_ctl_check_generation()
520 	io_ctl->size -= sizeof(u64) + (sizeof(u32) * io_ctl->num_pages);  in io_ctl_check_generation()
522 	cache_gen = get_unaligned_le64(io_ctl->cur);  in io_ctl_check_generation()
524 		btrfs_err_rl(io_ctl->fs_info,  in io_ctl_check_generation()
527 		io_ctl_unmap_page(io_ctl);  in io_ctl_check_generation()
530 	io_ctl->cur += sizeof(u64);  in io_ctl_check_generation()
534 static void io_ctl_set_crc(struct btrfs_io_ctl *io_ctl, int index)  in io_ctl_set_crc()  argument
541 		offset = sizeof(u32) * io_ctl->num_pages;  in io_ctl_set_crc()
543 	crc = btrfs_crc32c(crc, io_ctl->orig + offset, PAGE_SIZE - offset);  in io_ctl_set_crc()
545 	io_ctl_unmap_page(io_ctl);  in io_ctl_set_crc()
546 	tmp = page_address(io_ctl->pages[0]);  in io_ctl_set_crc()
551 static int io_ctl_check_crc(struct btrfs_io_ctl *io_ctl, int index)  in io_ctl_check_crc()  argument
558 		offset = sizeof(u32) * io_ctl->num_pages;  in io_ctl_check_crc()
560 	tmp = page_address(io_ctl->pages[0]);  in io_ctl_check_crc()
564 	io_ctl_map_page(io_ctl, 0);  in io_ctl_check_crc()
565 	crc = btrfs_crc32c(crc, io_ctl->orig + offset, PAGE_SIZE - offset);  in io_ctl_check_crc()
568 		btrfs_err_rl(io_ctl->fs_info,  in io_ctl_check_crc()
570 		io_ctl_unmap_page(io_ctl);  in io_ctl_check_crc()
577 static int io_ctl_add_entry(struct btrfs_io_ctl *io_ctl, u64 offset, u64 bytes,  in io_ctl_add_entry()  argument
582 	if (!io_ctl->cur)  in io_ctl_add_entry()
585 	entry = io_ctl->cur;  in io_ctl_add_entry()
590 	io_ctl->cur += sizeof(struct btrfs_free_space_entry);  in io_ctl_add_entry()
591 	io_ctl->size -= sizeof(struct btrfs_free_space_entry);  in io_ctl_add_entry()
593 	if (io_ctl->size >= sizeof(struct btrfs_free_space_entry))  in io_ctl_add_entry()
596 	io_ctl_set_crc(io_ctl, io_ctl->index - 1);  in io_ctl_add_entry()
599 	if (io_ctl->index >= io_ctl->num_pages)  in io_ctl_add_entry()
603 	io_ctl_map_page(io_ctl, 1);  in io_ctl_add_entry()
607 static int io_ctl_add_bitmap(struct btrfs_io_ctl *io_ctl, void *bitmap)  in io_ctl_add_bitmap()  argument
609 	if (!io_ctl->cur)  in io_ctl_add_bitmap()
616 	if (io_ctl->cur != io_ctl->orig) {  in io_ctl_add_bitmap()
617 		io_ctl_set_crc(io_ctl, io_ctl->index - 1);  in io_ctl_add_bitmap()
618 		if (io_ctl->index >= io_ctl->num_pages)  in io_ctl_add_bitmap()
620 		io_ctl_map_page(io_ctl, 0);  in io_ctl_add_bitmap()
623 	copy_page(io_ctl->cur, bitmap);  in io_ctl_add_bitmap()
624 	io_ctl_set_crc(io_ctl, io_ctl->index - 1);  in io_ctl_add_bitmap()
625 	if (io_ctl->index < io_ctl->num_pages)  in io_ctl_add_bitmap()
626 		io_ctl_map_page(io_ctl, 0);  in io_ctl_add_bitmap()
630 static void io_ctl_zero_remaining_pages(struct btrfs_io_ctl *io_ctl)  in io_ctl_zero_remaining_pages()  argument
636 	if (io_ctl->cur != io_ctl->orig)  in io_ctl_zero_remaining_pages()
637 		io_ctl_set_crc(io_ctl, io_ctl->index - 1);  in io_ctl_zero_remaining_pages()
639 		io_ctl_unmap_page(io_ctl);  in io_ctl_zero_remaining_pages()
641 	while (io_ctl->index < io_ctl->num_pages) {  in io_ctl_zero_remaining_pages()
642 		io_ctl_map_page(io_ctl, 1);  in io_ctl_zero_remaining_pages()
643 		io_ctl_set_crc(io_ctl, io_ctl->index - 1);  in io_ctl_zero_remaining_pages()
647 static int io_ctl_read_entry(struct btrfs_io_ctl *io_ctl,  in io_ctl_read_entry()  argument
653 	if (!io_ctl->cur) {  in io_ctl_read_entry()
654 		ret = io_ctl_check_crc(io_ctl, io_ctl->index);  in io_ctl_read_entry()
659 	e = io_ctl->cur;  in io_ctl_read_entry()
663 	io_ctl->cur += sizeof(struct btrfs_free_space_entry);  in io_ctl_read_entry()
664 	io_ctl->size -= sizeof(struct btrfs_free_space_entry);  in io_ctl_read_entry()
666 	if (io_ctl->size >= sizeof(struct btrfs_free_space_entry))  in io_ctl_read_entry()
669 	io_ctl_unmap_page(io_ctl);  in io_ctl_read_entry()
674 static int io_ctl_read_bitmap(struct btrfs_io_ctl *io_ctl,  in io_ctl_read_bitmap()  argument
679 	ret = io_ctl_check_crc(io_ctl, io_ctl->index);  in io_ctl_read_bitmap()
683 	copy_page(entry->bitmap, io_ctl->cur);  in io_ctl_read_bitmap()
684 	io_ctl_unmap_page(io_ctl);  in io_ctl_read_bitmap()
740 	struct btrfs_io_ctl io_ctl;  in __load_free_space_cache()  local
793 	ret = io_ctl_init(&io_ctl, inode, 0);  in __load_free_space_cache()
799 	ret = io_ctl_prepare_pages(&io_ctl, true);  in __load_free_space_cache()
803 	ret = io_ctl_check_crc(&io_ctl, 0);  in __load_free_space_cache()
807 	ret = io_ctl_check_generation(&io_ctl, generation);  in __load_free_space_cache()
819 		ret = io_ctl_read_entry(&io_ctl, e, &type);  in __load_free_space_cache()
870 	io_ctl_unmap_page(&io_ctl);  in __load_free_space_cache()
878 		ret = io_ctl_read_bitmap(&io_ctl, e);  in __load_free_space_cache()
883 	io_ctl_drop_pages(&io_ctl);  in __load_free_space_cache()
886 	io_ctl_free(&io_ctl);  in __load_free_space_cache()
889 	io_ctl_drop_pages(&io_ctl);  in __load_free_space_cache()
1068 int write_cache_extent_entries(struct btrfs_io_ctl *io_ctl,  in write_cache_extent_entries()  argument
1101 		ret = io_ctl_add_entry(io_ctl, e->offset, e->bytes,  in write_cache_extent_entries()
1130 		ret = io_ctl_add_entry(io_ctl, trim_entry->start,  in write_cache_extent_entries()
1200 			    struct btrfs_io_ctl *io_ctl,  in write_pinned_extent_entries()  argument
1237 		ret = io_ctl_add_entry(io_ctl, extent_start, len, NULL);  in write_pinned_extent_entries()
1248 write_bitmap_entries(struct btrfs_io_ctl *io_ctl, struct list_head *bitmap_list)  in write_bitmap_entries()  argument
1255 		ret = io_ctl_add_bitmap(io_ctl, entry->bitmap);  in write_bitmap_entries()
1287 			   struct btrfs_io_ctl *io_ctl,  in cleanup_write_cache_enospc()  argument
1290 	io_ctl_drop_pages(io_ctl);  in cleanup_write_cache_enospc()
1298 				 struct btrfs_io_ctl *io_ctl,  in __btrfs_wait_cache_io()  argument
1302 	struct inode *inode = io_ctl->inode;  in __btrfs_wait_cache_io()
1314 				io_ctl->entries, io_ctl->bitmaps);  in __btrfs_wait_cache_io()
1345 		io_ctl->inode = NULL;  in __btrfs_wait_cache_io()
1358 				     block_group, &block_group->io_ctl,  in btrfs_wait_cache_io()
1379 				   struct btrfs_io_ctl *io_ctl,  in __btrfs_write_out_cache()  argument
1392 	WARN_ON(io_ctl->pages);  in __btrfs_write_out_cache()
1393 	ret = io_ctl_init(io_ctl, inode, 1);  in __btrfs_write_out_cache()
1413 	ret = io_ctl_prepare_pages(io_ctl, false);  in __btrfs_write_out_cache()
1420 	io_ctl_set_generation(io_ctl, trans->transid);  in __btrfs_write_out_cache()
1425 	ret = write_cache_extent_entries(io_ctl, ctl,  in __btrfs_write_out_cache()
1439 	ret = write_pinned_extent_entries(trans, block_group, io_ctl, &entries);  in __btrfs_write_out_cache()
1448 	ret = write_bitmap_entries(io_ctl, &bitmap_list);  in __btrfs_write_out_cache()
1455 	io_ctl_zero_remaining_pages(io_ctl);  in __btrfs_write_out_cache()
1458 	ret = btrfs_dirty_pages(BTRFS_I(inode), io_ctl->pages,  in __btrfs_write_out_cache()
1459 				io_ctl->num_pages, 0, i_size_read(inode),  in __btrfs_write_out_cache()
1470 	io_ctl_drop_pages(io_ctl);  in __btrfs_write_out_cache()
1471 	io_ctl_free(io_ctl);  in __btrfs_write_out_cache()
1481 	io_ctl->entries = entries;  in __btrfs_write_out_cache()
1482 	io_ctl->bitmaps = bitmaps;  in __btrfs_write_out_cache()
1496 	cleanup_write_cache_enospc(inode, io_ctl, &cached_state);  in __btrfs_write_out_cache()
1503 	io_ctl->inode = NULL;  in __btrfs_write_out_cache()
1504 	io_ctl_free(io_ctl);  in __btrfs_write_out_cache()
1536 				block_group, &block_group->io_ctl, trans);  in btrfs_write_out_cache()
1545 		block_group->io_ctl.inode = NULL;  in btrfs_write_out_cache()