356 static int io_ctl_init(struct btrfs_io_ctl *io_ctl, struct inode *inode,  in io_ctl_init()  argument
367 	memset(io_ctl, 0, sizeof(struct btrfs_io_ctl));  in io_ctl_init()
369 	io_ctl->pages = kcalloc(num_pages, sizeof(struct page *), GFP_NOFS);  in io_ctl_init()
370 	if (!io_ctl->pages)  in io_ctl_init()
373 	io_ctl->num_pages = num_pages;  in io_ctl_init()
374 	io_ctl->fs_info = btrfs_sb(inode->i_sb);  in io_ctl_init()
375 	io_ctl->inode = inode;  in io_ctl_init()
381 static void io_ctl_free(struct btrfs_io_ctl *io_ctl)  in io_ctl_free()  argument
383 	kfree(io_ctl->pages);  in io_ctl_free()
384 	io_ctl->pages = NULL;  in io_ctl_free()
387 static void io_ctl_unmap_page(struct btrfs_io_ctl *io_ctl)  in io_ctl_unmap_page()  argument
389 	if (io_ctl->cur) {  in io_ctl_unmap_page()
390 		io_ctl->cur = NULL;  in io_ctl_unmap_page()
391 		io_ctl->orig = NULL;  in io_ctl_unmap_page()
395 static void io_ctl_map_page(struct btrfs_io_ctl *io_ctl, int clear)  in io_ctl_map_page()  argument
397 	ASSERT(io_ctl->index < io_ctl->num_pages);  in io_ctl_map_page()
398 	io_ctl->page = io_ctl->pages[io_ctl->index++];  in io_ctl_map_page()
399 	io_ctl->cur = page_address(io_ctl->page);  in io_ctl_map_page()
400 	io_ctl->orig = io_ctl->cur;  in io_ctl_map_page()
401 	io_ctl->size = PAGE_SIZE;  in io_ctl_map_page()
403 		clear_page(io_ctl->cur);  in io_ctl_map_page()
406 static void io_ctl_drop_pages(struct btrfs_io_ctl *io_ctl)  in io_ctl_drop_pages()  argument
410 	io_ctl_unmap_page(io_ctl);  in io_ctl_drop_pages()
412 	for (i = 0; i < io_ctl->num_pages; i++) {  in io_ctl_drop_pages()
413 		if (io_ctl->pages[i]) {  in io_ctl_drop_pages()
414 			ClearPageChecked(io_ctl->pages[i]);  in io_ctl_drop_pages()
415 			unlock_page(io_ctl->pages[i]);  in io_ctl_drop_pages()
416 			put_page(io_ctl->pages[i]);  in io_ctl_drop_pages()
421 static int io_ctl_prepare_pages(struct btrfs_io_ctl *io_ctl, bool uptodate)  in io_ctl_prepare_pages()  argument
424 	struct inode *inode = io_ctl->inode;  in io_ctl_prepare_pages()
428 	for (i = 0; i < io_ctl->num_pages; i++) {  in io_ctl_prepare_pages()
433 			io_ctl_drop_pages(io_ctl);  in io_ctl_prepare_pages()
441 			io_ctl_drop_pages(io_ctl);  in io_ctl_prepare_pages()
445 		io_ctl->pages[i] = page;  in io_ctl_prepare_pages()
452 				io_ctl_drop_pages(io_ctl);  in io_ctl_prepare_pages()
458 				io_ctl_drop_pages(io_ctl);  in io_ctl_prepare_pages()
464 	for (i = 0; i < io_ctl->num_pages; i++)  in io_ctl_prepare_pages()
465 		clear_page_dirty_for_io(io_ctl->pages[i]);  in io_ctl_prepare_pages()
470 static void io_ctl_set_generation(struct btrfs_io_ctl *io_ctl, u64 generation)  in io_ctl_set_generation()  argument
472 	io_ctl_map_page(io_ctl, 1);  in io_ctl_set_generation()
478 	io_ctl->cur += (sizeof(u32) * io_ctl->num_pages);  in io_ctl_set_generation()
479 	io_ctl->size -= sizeof(u64) + (sizeof(u32) * io_ctl->num_pages);  in io_ctl_set_generation()
481 	put_unaligned_le64(generation, io_ctl->cur);  in io_ctl_set_generation()
482 	io_ctl->cur += sizeof(u64);  in io_ctl_set_generation()
485 static int io_ctl_check_generation(struct btrfs_io_ctl *io_ctl, u64 generation)  in io_ctl_check_generation()  argument
493 	io_ctl->cur += sizeof(u32) * io_ctl->num_pages;  in io_ctl_check_generation()
494 	io_ctl->size -= sizeof(u64) + (sizeof(u32) * io_ctl->num_pages);  in io_ctl_check_generation()
496 	cache_gen = get_unaligned_le64(io_ctl->cur);  in io_ctl_check_generation()
498 		btrfs_err_rl(io_ctl->fs_info,  in io_ctl_check_generation()
501 		io_ctl_unmap_page(io_ctl);  in io_ctl_check_generation()
504 	io_ctl->cur += sizeof(u64);  in io_ctl_check_generation()
508 static void io_ctl_set_crc(struct btrfs_io_ctl *io_ctl, int index)  in io_ctl_set_crc()  argument
515 		offset = sizeof(u32) * io_ctl->num_pages;  in io_ctl_set_crc()
517 	crc = btrfs_crc32c(crc, io_ctl->orig + offset, PAGE_SIZE - offset);  in io_ctl_set_crc()
519 	io_ctl_unmap_page(io_ctl);  in io_ctl_set_crc()
520 	tmp = page_address(io_ctl->pages[0]);  in io_ctl_set_crc()
525 static int io_ctl_check_crc(struct btrfs_io_ctl *io_ctl, int index)  in io_ctl_check_crc()  argument
532 		offset = sizeof(u32) * io_ctl->num_pages;  in io_ctl_check_crc()
534 	tmp = page_address(io_ctl->pages[0]);  in io_ctl_check_crc()
538 	io_ctl_map_page(io_ctl, 0);  in io_ctl_check_crc()
539 	crc = btrfs_crc32c(crc, io_ctl->orig + offset, PAGE_SIZE - offset);  in io_ctl_check_crc()
542 		btrfs_err_rl(io_ctl->fs_info,  in io_ctl_check_crc()
544 		io_ctl_unmap_page(io_ctl);  in io_ctl_check_crc()
551 static int io_ctl_add_entry(struct btrfs_io_ctl *io_ctl, u64 offset, u64 bytes,  in io_ctl_add_entry()  argument
556 	if (!io_ctl->cur)  in io_ctl_add_entry()
559 	entry = io_ctl->cur;  in io_ctl_add_entry()
564 	io_ctl->cur += sizeof(struct btrfs_free_space_entry);  in io_ctl_add_entry()
565 	io_ctl->size -= sizeof(struct btrfs_free_space_entry);  in io_ctl_add_entry()
567 	if (io_ctl->size >= sizeof(struct btrfs_free_space_entry))  in io_ctl_add_entry()
570 	io_ctl_set_crc(io_ctl, io_ctl->index - 1);  in io_ctl_add_entry()
573 	if (io_ctl->index >= io_ctl->num_pages)  in io_ctl_add_entry()
577 	io_ctl_map_page(io_ctl, 1);  in io_ctl_add_entry()
581 static int io_ctl_add_bitmap(struct btrfs_io_ctl *io_ctl, void *bitmap)  in io_ctl_add_bitmap()  argument
583 	if (!io_ctl->cur)  in io_ctl_add_bitmap()
590 	if (io_ctl->cur != io_ctl->orig) {  in io_ctl_add_bitmap()
591 		io_ctl_set_crc(io_ctl, io_ctl->index - 1);  in io_ctl_add_bitmap()
592 		if (io_ctl->index >= io_ctl->num_pages)  in io_ctl_add_bitmap()
594 		io_ctl_map_page(io_ctl, 0);  in io_ctl_add_bitmap()
597 	copy_page(io_ctl->cur, bitmap);  in io_ctl_add_bitmap()
598 	io_ctl_set_crc(io_ctl, io_ctl->index - 1);  in io_ctl_add_bitmap()
599 	if (io_ctl->index < io_ctl->num_pages)  in io_ctl_add_bitmap()
600 		io_ctl_map_page(io_ctl, 0);  in io_ctl_add_bitmap()
604 static void io_ctl_zero_remaining_pages(struct btrfs_io_ctl *io_ctl)  in io_ctl_zero_remaining_pages()  argument
610 	if (io_ctl->cur != io_ctl->orig)  in io_ctl_zero_remaining_pages()
611 		io_ctl_set_crc(io_ctl, io_ctl->index - 1);  in io_ctl_zero_remaining_pages()
613 		io_ctl_unmap_page(io_ctl);  in io_ctl_zero_remaining_pages()
615 	while (io_ctl->index < io_ctl->num_pages) {  in io_ctl_zero_remaining_pages()
616 		io_ctl_map_page(io_ctl, 1);  in io_ctl_zero_remaining_pages()
617 		io_ctl_set_crc(io_ctl, io_ctl->index - 1);  in io_ctl_zero_remaining_pages()
621 static int io_ctl_read_entry(struct btrfs_io_ctl *io_ctl,  in io_ctl_read_entry()  argument
627 	if (!io_ctl->cur) {  in io_ctl_read_entry()
628 		ret = io_ctl_check_crc(io_ctl, io_ctl->index);  in io_ctl_read_entry()
633 	e = io_ctl->cur;  in io_ctl_read_entry()
637 	io_ctl->cur += sizeof(struct btrfs_free_space_entry);  in io_ctl_read_entry()
638 	io_ctl->size -= sizeof(struct btrfs_free_space_entry);  in io_ctl_read_entry()
640 	if (io_ctl->size >= sizeof(struct btrfs_free_space_entry))  in io_ctl_read_entry()
643 	io_ctl_unmap_page(io_ctl);  in io_ctl_read_entry()
648 static int io_ctl_read_bitmap(struct btrfs_io_ctl *io_ctl,  in io_ctl_read_bitmap()  argument
653 	ret = io_ctl_check_crc(io_ctl, io_ctl->index);  in io_ctl_read_bitmap()
657 	copy_page(entry->bitmap, io_ctl->cur);  in io_ctl_read_bitmap()
658 	io_ctl_unmap_page(io_ctl);  in io_ctl_read_bitmap()
708 	struct btrfs_io_ctl io_ctl;  in __load_free_space_cache()  local
761 	ret = io_ctl_init(&io_ctl, inode, 0);  in __load_free_space_cache()
767 	ret = io_ctl_prepare_pages(&io_ctl, true);  in __load_free_space_cache()
771 	ret = io_ctl_check_crc(&io_ctl, 0);  in __load_free_space_cache()
775 	ret = io_ctl_check_generation(&io_ctl, generation);  in __load_free_space_cache()
787 		ret = io_ctl_read_entry(&io_ctl, e, &type);  in __load_free_space_cache()
837 	io_ctl_unmap_page(&io_ctl);  in __load_free_space_cache()
845 		ret = io_ctl_read_bitmap(&io_ctl, e);  in __load_free_space_cache()
850 	io_ctl_drop_pages(&io_ctl);  in __load_free_space_cache()
853 	io_ctl_free(&io_ctl);  in __load_free_space_cache()
856 	io_ctl_drop_pages(&io_ctl);  in __load_free_space_cache()
1010 int write_cache_extent_entries(struct btrfs_io_ctl *io_ctl,  in write_cache_extent_entries()  argument
1043 		ret = io_ctl_add_entry(io_ctl, e->offset, e->bytes,  in write_cache_extent_entries()
1072 		ret = io_ctl_add_entry(io_ctl, trim_entry->start,  in write_cache_extent_entries()
1142 			    struct btrfs_io_ctl *io_ctl,  in write_pinned_extent_entries()  argument
1180 		ret = io_ctl_add_entry(io_ctl, extent_start, len, NULL);  in write_pinned_extent_entries()
1191 write_bitmap_entries(struct btrfs_io_ctl *io_ctl, struct list_head *bitmap_list)  in write_bitmap_entries()  argument
1198 		ret = io_ctl_add_bitmap(io_ctl, entry->bitmap);  in write_bitmap_entries()
1230 			   struct btrfs_io_ctl *io_ctl,  in cleanup_write_cache_enospc()  argument
1233 	io_ctl_drop_pages(io_ctl);  in cleanup_write_cache_enospc()
1241 				 struct btrfs_io_ctl *io_ctl,  in __btrfs_wait_cache_io()  argument
1245 	struct inode *inode = io_ctl->inode;  in __btrfs_wait_cache_io()
1257 				io_ctl->entries, io_ctl->bitmaps);  in __btrfs_wait_cache_io()
1288 		io_ctl->inode = NULL;  in __btrfs_wait_cache_io()
1301 				     block_group, &block_group->io_ctl,  in btrfs_wait_cache_io()
1322 				   struct btrfs_io_ctl *io_ctl,  in __btrfs_write_out_cache()  argument
1335 	WARN_ON(io_ctl->pages);  in __btrfs_write_out_cache()
1336 	ret = io_ctl_init(io_ctl, inode, 1);  in __btrfs_write_out_cache()
1356 	ret = io_ctl_prepare_pages(io_ctl, false);  in __btrfs_write_out_cache()
1363 	io_ctl_set_generation(io_ctl, trans->transid);  in __btrfs_write_out_cache()
1368 	ret = write_cache_extent_entries(io_ctl, ctl,  in __btrfs_write_out_cache()
1382 	ret = write_pinned_extent_entries(trans, block_group, io_ctl, &entries);  in __btrfs_write_out_cache()
1391 	ret = write_bitmap_entries(io_ctl, &bitmap_list);  in __btrfs_write_out_cache()
1398 	io_ctl_zero_remaining_pages(io_ctl);  in __btrfs_write_out_cache()
1401 	ret = btrfs_dirty_pages(BTRFS_I(inode), io_ctl->pages,  in __btrfs_write_out_cache()
1402 				io_ctl->num_pages, 0, i_size_read(inode),  in __btrfs_write_out_cache()
1413 	io_ctl_drop_pages(io_ctl);  in __btrfs_write_out_cache()
1414 	io_ctl_free(io_ctl);  in __btrfs_write_out_cache()
1424 	io_ctl->entries = entries;  in __btrfs_write_out_cache()
1425 	io_ctl->bitmaps = bitmaps;  in __btrfs_write_out_cache()
1439 	cleanup_write_cache_enospc(inode, io_ctl, &cached_state);  in __btrfs_write_out_cache()
1446 	io_ctl->inode = NULL;  in __btrfs_write_out_cache()
1447 	io_ctl_free(io_ctl);  in __btrfs_write_out_cache()
1479 				block_group, &block_group->io_ctl, trans);  in btrfs_write_out_cache()
1488 		block_group->io_ctl.inode = NULL;  in btrfs_write_out_cache()