44 			     struct btrfs_io_ctl *io_ctl,
303 static int io_ctl_init(struct btrfs_io_ctl *io_ctl, struct inode *inode,  in io_ctl_init()  argument
319 	memset(io_ctl, 0, sizeof(struct btrfs_io_ctl));  in io_ctl_init()
321 	io_ctl->pages = kcalloc(num_pages, sizeof(struct page *), GFP_NOFS);  in io_ctl_init()
322 	if (!io_ctl->pages)  in io_ctl_init()
325 	io_ctl->num_pages = num_pages;  in io_ctl_init()
326 	io_ctl->fs_info = btrfs_sb(inode->i_sb);  in io_ctl_init()
327 	io_ctl->check_crcs = check_crcs;  in io_ctl_init()
328 	io_ctl->inode = inode;  in io_ctl_init()
334 static void io_ctl_free(struct btrfs_io_ctl *io_ctl)  in io_ctl_free()  argument
336 	kfree(io_ctl->pages);  in io_ctl_free()
337 	io_ctl->pages = NULL;  in io_ctl_free()
340 static void io_ctl_unmap_page(struct btrfs_io_ctl *io_ctl)  in io_ctl_unmap_page()  argument
342 	if (io_ctl->cur) {  in io_ctl_unmap_page()
343 		io_ctl->cur = NULL;  in io_ctl_unmap_page()
344 		io_ctl->orig = NULL;  in io_ctl_unmap_page()
348 static void io_ctl_map_page(struct btrfs_io_ctl *io_ctl, int clear)  in io_ctl_map_page()  argument
350 	ASSERT(io_ctl->index < io_ctl->num_pages);  in io_ctl_map_page()
351 	io_ctl->page = io_ctl->pages[io_ctl->index++];  in io_ctl_map_page()
352 	io_ctl->cur = page_address(io_ctl->page);  in io_ctl_map_page()
353 	io_ctl->orig = io_ctl->cur;  in io_ctl_map_page()
354 	io_ctl->size = PAGE_SIZE;  in io_ctl_map_page()
356 		clear_page(io_ctl->cur);  in io_ctl_map_page()
359 static void io_ctl_drop_pages(struct btrfs_io_ctl *io_ctl)  in io_ctl_drop_pages()  argument
363 	io_ctl_unmap_page(io_ctl);  in io_ctl_drop_pages()
365 	for (i = 0; i < io_ctl->num_pages; i++) {  in io_ctl_drop_pages()
366 		if (io_ctl->pages[i]) {  in io_ctl_drop_pages()
367 			ClearPageChecked(io_ctl->pages[i]);  in io_ctl_drop_pages()
368 			unlock_page(io_ctl->pages[i]);  in io_ctl_drop_pages()
369 			put_page(io_ctl->pages[i]);  in io_ctl_drop_pages()
374 static int io_ctl_prepare_pages(struct btrfs_io_ctl *io_ctl, bool uptodate)  in io_ctl_prepare_pages()  argument
377 	struct inode *inode = io_ctl->inode;  in io_ctl_prepare_pages()
381 	for (i = 0; i < io_ctl->num_pages; i++) {  in io_ctl_prepare_pages()
384 			io_ctl_drop_pages(io_ctl);  in io_ctl_prepare_pages()
387 		io_ctl->pages[i] = page;  in io_ctl_prepare_pages()
394 				io_ctl_drop_pages(io_ctl);  in io_ctl_prepare_pages()
400 				io_ctl_drop_pages(io_ctl);  in io_ctl_prepare_pages()
406 	for (i = 0; i < io_ctl->num_pages; i++) {  in io_ctl_prepare_pages()
407 		clear_page_dirty_for_io(io_ctl->pages[i]);  in io_ctl_prepare_pages()
408 		set_page_extent_mapped(io_ctl->pages[i]);  in io_ctl_prepare_pages()
414 static void io_ctl_set_generation(struct btrfs_io_ctl *io_ctl, u64 generation)  in io_ctl_set_generation()  argument
416 	io_ctl_map_page(io_ctl, 1);  in io_ctl_set_generation()
422 	if (io_ctl->check_crcs) {  in io_ctl_set_generation()
423 		io_ctl->cur += (sizeof(u32) * io_ctl->num_pages);  in io_ctl_set_generation()
424 		io_ctl->size -= sizeof(u64) + (sizeof(u32) * io_ctl->num_pages);  in io_ctl_set_generation()
426 		io_ctl->cur += sizeof(u64);  in io_ctl_set_generation()
427 		io_ctl->size -= sizeof(u64) * 2;  in io_ctl_set_generation()
430 	put_unaligned_le64(generation, io_ctl->cur);  in io_ctl_set_generation()
431 	io_ctl->cur += sizeof(u64);  in io_ctl_set_generation()
434 static int io_ctl_check_generation(struct btrfs_io_ctl *io_ctl, u64 generation)  in io_ctl_check_generation()  argument
442 	if (io_ctl->check_crcs) {  in io_ctl_check_generation()
443 		io_ctl->cur += sizeof(u32) * io_ctl->num_pages;  in io_ctl_check_generation()
444 		io_ctl->size -= sizeof(u64) +  in io_ctl_check_generation()
445 			(sizeof(u32) * io_ctl->num_pages);  in io_ctl_check_generation()
447 		io_ctl->cur += sizeof(u64);  in io_ctl_check_generation()
448 		io_ctl->size -= sizeof(u64) * 2;  in io_ctl_check_generation()
451 	cache_gen = get_unaligned_le64(io_ctl->cur);  in io_ctl_check_generation()
453 		btrfs_err_rl(io_ctl->fs_info,  in io_ctl_check_generation()
456 		io_ctl_unmap_page(io_ctl);  in io_ctl_check_generation()
459 	io_ctl->cur += sizeof(u64);  in io_ctl_check_generation()
463 static void io_ctl_set_crc(struct btrfs_io_ctl *io_ctl, int index)  in io_ctl_set_crc()  argument
469 	if (!io_ctl->check_crcs) {  in io_ctl_set_crc()
470 		io_ctl_unmap_page(io_ctl);  in io_ctl_set_crc()
475 		offset = sizeof(u32) * io_ctl->num_pages;  in io_ctl_set_crc()
477 	crc = btrfs_crc32c(crc, io_ctl->orig + offset, PAGE_SIZE - offset);  in io_ctl_set_crc()
479 	io_ctl_unmap_page(io_ctl);  in io_ctl_set_crc()
480 	tmp = page_address(io_ctl->pages[0]);  in io_ctl_set_crc()
485 static int io_ctl_check_crc(struct btrfs_io_ctl *io_ctl, int index)  in io_ctl_check_crc()  argument
491 	if (!io_ctl->check_crcs) {  in io_ctl_check_crc()
492 		io_ctl_map_page(io_ctl, 0);  in io_ctl_check_crc()
497 		offset = sizeof(u32) * io_ctl->num_pages;  in io_ctl_check_crc()
499 	tmp = page_address(io_ctl->pages[0]);  in io_ctl_check_crc()
503 	io_ctl_map_page(io_ctl, 0);  in io_ctl_check_crc()
504 	crc = btrfs_crc32c(crc, io_ctl->orig + offset, PAGE_SIZE - offset);  in io_ctl_check_crc()
507 		btrfs_err_rl(io_ctl->fs_info,  in io_ctl_check_crc()
509 		io_ctl_unmap_page(io_ctl);  in io_ctl_check_crc()
516 static int io_ctl_add_entry(struct btrfs_io_ctl *io_ctl, u64 offset, u64 bytes,  in io_ctl_add_entry()  argument
521 	if (!io_ctl->cur)  in io_ctl_add_entry()
524 	entry = io_ctl->cur;  in io_ctl_add_entry()
529 	io_ctl->cur += sizeof(struct btrfs_free_space_entry);  in io_ctl_add_entry()
530 	io_ctl->size -= sizeof(struct btrfs_free_space_entry);  in io_ctl_add_entry()
532 	if (io_ctl->size >= sizeof(struct btrfs_free_space_entry))  in io_ctl_add_entry()
535 	io_ctl_set_crc(io_ctl, io_ctl->index - 1);  in io_ctl_add_entry()
538 	if (io_ctl->index >= io_ctl->num_pages)  in io_ctl_add_entry()
542 	io_ctl_map_page(io_ctl, 1);  in io_ctl_add_entry()
546 static int io_ctl_add_bitmap(struct btrfs_io_ctl *io_ctl, void *bitmap)  in io_ctl_add_bitmap()  argument
548 	if (!io_ctl->cur)  in io_ctl_add_bitmap()
555 	if (io_ctl->cur != io_ctl->orig) {  in io_ctl_add_bitmap()
556 		io_ctl_set_crc(io_ctl, io_ctl->index - 1);  in io_ctl_add_bitmap()
557 		if (io_ctl->index >= io_ctl->num_pages)  in io_ctl_add_bitmap()
559 		io_ctl_map_page(io_ctl, 0);  in io_ctl_add_bitmap()
562 	copy_page(io_ctl->cur, bitmap);  in io_ctl_add_bitmap()
563 	io_ctl_set_crc(io_ctl, io_ctl->index - 1);  in io_ctl_add_bitmap()
564 	if (io_ctl->index < io_ctl->num_pages)  in io_ctl_add_bitmap()
565 		io_ctl_map_page(io_ctl, 0);  in io_ctl_add_bitmap()
569 static void io_ctl_zero_remaining_pages(struct btrfs_io_ctl *io_ctl)  in io_ctl_zero_remaining_pages()  argument
575 	if (io_ctl->cur != io_ctl->orig)  in io_ctl_zero_remaining_pages()
576 		io_ctl_set_crc(io_ctl, io_ctl->index - 1);  in io_ctl_zero_remaining_pages()
578 		io_ctl_unmap_page(io_ctl);  in io_ctl_zero_remaining_pages()
580 	while (io_ctl->index < io_ctl->num_pages) {  in io_ctl_zero_remaining_pages()
581 		io_ctl_map_page(io_ctl, 1);  in io_ctl_zero_remaining_pages()
582 		io_ctl_set_crc(io_ctl, io_ctl->index - 1);  in io_ctl_zero_remaining_pages()
586 static int io_ctl_read_entry(struct btrfs_io_ctl *io_ctl,  in io_ctl_read_entry()  argument
592 	if (!io_ctl->cur) {  in io_ctl_read_entry()
593 		ret = io_ctl_check_crc(io_ctl, io_ctl->index);  in io_ctl_read_entry()
598 	e = io_ctl->cur;  in io_ctl_read_entry()
602 	io_ctl->cur += sizeof(struct btrfs_free_space_entry);  in io_ctl_read_entry()
603 	io_ctl->size -= sizeof(struct btrfs_free_space_entry);  in io_ctl_read_entry()
605 	if (io_ctl->size >= sizeof(struct btrfs_free_space_entry))  in io_ctl_read_entry()
608 	io_ctl_unmap_page(io_ctl);  in io_ctl_read_entry()
613 static int io_ctl_read_bitmap(struct btrfs_io_ctl *io_ctl,  in io_ctl_read_bitmap()  argument
618 	ret = io_ctl_check_crc(io_ctl, io_ctl->index);  in io_ctl_read_bitmap()
622 	copy_page(entry->bitmap, io_ctl->cur);  in io_ctl_read_bitmap()
623 	io_ctl_unmap_page(io_ctl);  in io_ctl_read_bitmap()
673 	struct btrfs_io_ctl io_ctl;  in __load_free_space_cache()  local
726 	ret = io_ctl_init(&io_ctl, inode, 0);  in __load_free_space_cache()
732 	ret = io_ctl_prepare_pages(&io_ctl, true);  in __load_free_space_cache()
736 	ret = io_ctl_check_crc(&io_ctl, 0);  in __load_free_space_cache()
740 	ret = io_ctl_check_generation(&io_ctl, generation);  in __load_free_space_cache()
750 		ret = io_ctl_read_entry(&io_ctl, e, &type);  in __load_free_space_cache()
808 	io_ctl_unmap_page(&io_ctl);  in __load_free_space_cache()
816 		ret = io_ctl_read_bitmap(&io_ctl, e);  in __load_free_space_cache()
827 	io_ctl_drop_pages(&io_ctl);  in __load_free_space_cache()
832 	io_ctl_free(&io_ctl);  in __load_free_space_cache()
835 	io_ctl_drop_pages(&io_ctl);  in __load_free_space_cache()
937 int write_cache_extent_entries(struct btrfs_io_ctl *io_ctl,  in write_cache_extent_entries()  argument
970 		ret = io_ctl_add_entry(io_ctl, e->offset, e->bytes,  in write_cache_extent_entries()
999 		ret = io_ctl_add_entry(io_ctl, trim_entry->start,  in write_cache_extent_entries()
1069 			    struct btrfs_io_ctl *io_ctl,  in write_pinned_extent_entries()  argument
1107 		ret = io_ctl_add_entry(io_ctl, extent_start, len, NULL);  in write_pinned_extent_entries()
1118 write_bitmap_entries(struct btrfs_io_ctl *io_ctl, struct list_head *bitmap_list)  in write_bitmap_entries()  argument
1125 		ret = io_ctl_add_bitmap(io_ctl, entry->bitmap);  in write_bitmap_entries()
1157 			   struct btrfs_io_ctl *io_ctl,  in cleanup_write_cache_enospc()  argument
1160 	io_ctl_drop_pages(io_ctl);  in cleanup_write_cache_enospc()
1168 				 struct btrfs_io_ctl *io_ctl,  in __btrfs_wait_cache_io()  argument
1172 	struct inode *inode = io_ctl->inode;  in __btrfs_wait_cache_io()
1184 				io_ctl->entries, io_ctl->bitmaps);  in __btrfs_wait_cache_io()
1215 		io_ctl->inode = NULL;  in __btrfs_wait_cache_io()
1225 				    struct btrfs_io_ctl *io_ctl,  in btrfs_wait_cache_io_root()  argument
1228 	return __btrfs_wait_cache_io(root, trans, NULL, io_ctl, path, 0);  in btrfs_wait_cache_io_root()
1236 				     block_group, &block_group->io_ctl,  in btrfs_wait_cache_io()
1254 				   struct btrfs_io_ctl *io_ctl,  in __btrfs_write_out_cache()  argument
1267 	WARN_ON(io_ctl->pages);  in __btrfs_write_out_cache()
1268 	ret = io_ctl_init(io_ctl, inode, 1);  in __btrfs_write_out_cache()
1288 	ret = io_ctl_prepare_pages(io_ctl, false);  in __btrfs_write_out_cache()
1295 	io_ctl_set_generation(io_ctl, trans->transid);  in __btrfs_write_out_cache()
1300 	ret = write_cache_extent_entries(io_ctl, ctl,  in __btrfs_write_out_cache()
1314 	ret = write_pinned_extent_entries(trans, block_group, io_ctl, &entries);  in __btrfs_write_out_cache()
1323 	ret = write_bitmap_entries(io_ctl, &bitmap_list);  in __btrfs_write_out_cache()
1330 	io_ctl_zero_remaining_pages(io_ctl);  in __btrfs_write_out_cache()
1333 	ret = btrfs_dirty_pages(BTRFS_I(inode), io_ctl->pages,  in __btrfs_write_out_cache()
1334 				io_ctl->num_pages, 0, i_size_read(inode),  in __btrfs_write_out_cache()
1345 	io_ctl_drop_pages(io_ctl);  in __btrfs_write_out_cache()
1346 	io_ctl_free(io_ctl);  in __btrfs_write_out_cache()
1356 	io_ctl->entries = entries;  in __btrfs_write_out_cache()
1357 	io_ctl->bitmaps = bitmaps;  in __btrfs_write_out_cache()
1371 	cleanup_write_cache_enospc(inode, io_ctl, &cached_state);  in __btrfs_write_out_cache()
1378 	io_ctl->inode = NULL;  in __btrfs_write_out_cache()
1379 	io_ctl_free(io_ctl);  in __btrfs_write_out_cache()
1411 				block_group, &block_group->io_ctl, trans);  in btrfs_write_out_cache()
1420 		block_group->io_ctl.inode = NULL;  in btrfs_write_out_cache()
3964 	struct btrfs_io_ctl io_ctl;  in btrfs_write_out_ino_cache()  local
3970 	memset(&io_ctl, 0, sizeof(io_ctl));  in btrfs_write_out_ino_cache()
3971 	ret = __btrfs_write_out_cache(root, inode, ctl, NULL, &io_ctl, trans);  in btrfs_write_out_ino_cache()
3980 		ret = btrfs_wait_cache_io_root(root, trans, &io_ctl, path);  in btrfs_write_out_ino_cache()