36 			     struct btrfs_io_ctl *io_ctl,
288 static int io_ctl_init(struct btrfs_io_ctl *io_ctl, struct inode *inode,  in io_ctl_init()  argument
304 	memset(io_ctl, 0, sizeof(struct btrfs_io_ctl));  in io_ctl_init()
306 	io_ctl->pages = kcalloc(num_pages, sizeof(struct page *), GFP_NOFS);  in io_ctl_init()
307 	if (!io_ctl->pages)  in io_ctl_init()
310 	io_ctl->num_pages = num_pages;  in io_ctl_init()
311 	io_ctl->fs_info = btrfs_sb(inode->i_sb);  in io_ctl_init()
312 	io_ctl->check_crcs = check_crcs;  in io_ctl_init()
313 	io_ctl->inode = inode;  in io_ctl_init()
319 static void io_ctl_free(struct btrfs_io_ctl *io_ctl)  in io_ctl_free()  argument
321 	kfree(io_ctl->pages);  in io_ctl_free()
322 	io_ctl->pages = NULL;  in io_ctl_free()
325 static void io_ctl_unmap_page(struct btrfs_io_ctl *io_ctl)  in io_ctl_unmap_page()  argument
327 	if (io_ctl->cur) {  in io_ctl_unmap_page()
328 		io_ctl->cur = NULL;  in io_ctl_unmap_page()
329 		io_ctl->orig = NULL;  in io_ctl_unmap_page()
333 static void io_ctl_map_page(struct btrfs_io_ctl *io_ctl, int clear)  in io_ctl_map_page()  argument
335 	ASSERT(io_ctl->index < io_ctl->num_pages);  in io_ctl_map_page()
336 	io_ctl->page = io_ctl->pages[io_ctl->index++];  in io_ctl_map_page()
337 	io_ctl->cur = page_address(io_ctl->page);  in io_ctl_map_page()
338 	io_ctl->orig = io_ctl->cur;  in io_ctl_map_page()
339 	io_ctl->size = PAGE_SIZE;  in io_ctl_map_page()
341 		clear_page(io_ctl->cur);  in io_ctl_map_page()
344 static void io_ctl_drop_pages(struct btrfs_io_ctl *io_ctl)  in io_ctl_drop_pages()  argument
348 	io_ctl_unmap_page(io_ctl);  in io_ctl_drop_pages()
350 	for (i = 0; i < io_ctl->num_pages; i++) {  in io_ctl_drop_pages()
351 		if (io_ctl->pages[i]) {  in io_ctl_drop_pages()
352 			ClearPageChecked(io_ctl->pages[i]);  in io_ctl_drop_pages()
353 			unlock_page(io_ctl->pages[i]);  in io_ctl_drop_pages()
354 			put_page(io_ctl->pages[i]);  in io_ctl_drop_pages()
359 static int io_ctl_prepare_pages(struct btrfs_io_ctl *io_ctl, struct inode *inode,  in io_ctl_prepare_pages()  argument
366 	for (i = 0; i < io_ctl->num_pages; i++) {  in io_ctl_prepare_pages()
369 			io_ctl_drop_pages(io_ctl);  in io_ctl_prepare_pages()
372 		io_ctl->pages[i] = page;  in io_ctl_prepare_pages()
379 				io_ctl_drop_pages(io_ctl);  in io_ctl_prepare_pages()
385 	for (i = 0; i < io_ctl->num_pages; i++) {  in io_ctl_prepare_pages()
386 		clear_page_dirty_for_io(io_ctl->pages[i]);  in io_ctl_prepare_pages()
387 		set_page_extent_mapped(io_ctl->pages[i]);  in io_ctl_prepare_pages()
393 static void io_ctl_set_generation(struct btrfs_io_ctl *io_ctl, u64 generation)  in io_ctl_set_generation()  argument
397 	io_ctl_map_page(io_ctl, 1);  in io_ctl_set_generation()
403 	if (io_ctl->check_crcs) {  in io_ctl_set_generation()
404 		io_ctl->cur += (sizeof(u32) * io_ctl->num_pages);  in io_ctl_set_generation()
405 		io_ctl->size -= sizeof(u64) + (sizeof(u32) * io_ctl->num_pages);  in io_ctl_set_generation()
407 		io_ctl->cur += sizeof(u64);  in io_ctl_set_generation()
408 		io_ctl->size -= sizeof(u64) * 2;  in io_ctl_set_generation()
411 	val = io_ctl->cur;  in io_ctl_set_generation()
413 	io_ctl->cur += sizeof(u64);  in io_ctl_set_generation()
416 static int io_ctl_check_generation(struct btrfs_io_ctl *io_ctl, u64 generation)  in io_ctl_check_generation()  argument
424 	if (io_ctl->check_crcs) {  in io_ctl_check_generation()
425 		io_ctl->cur += sizeof(u32) * io_ctl->num_pages;  in io_ctl_check_generation()
426 		io_ctl->size -= sizeof(u64) +  in io_ctl_check_generation()
427 			(sizeof(u32) * io_ctl->num_pages);  in io_ctl_check_generation()
429 		io_ctl->cur += sizeof(u64);  in io_ctl_check_generation()
430 		io_ctl->size -= sizeof(u64) * 2;  in io_ctl_check_generation()
433 	gen = io_ctl->cur;  in io_ctl_check_generation()
435 		btrfs_err_rl(io_ctl->fs_info,  in io_ctl_check_generation()
438 		io_ctl_unmap_page(io_ctl);  in io_ctl_check_generation()
441 	io_ctl->cur += sizeof(u64);  in io_ctl_check_generation()
445 static void io_ctl_set_crc(struct btrfs_io_ctl *io_ctl, int index)  in io_ctl_set_crc()  argument
451 	if (!io_ctl->check_crcs) {  in io_ctl_set_crc()
452 		io_ctl_unmap_page(io_ctl);  in io_ctl_set_crc()
457 		offset = sizeof(u32) * io_ctl->num_pages;  in io_ctl_set_crc()
459 	crc = btrfs_csum_data(io_ctl->orig + offset, crc,  in io_ctl_set_crc()
462 	io_ctl_unmap_page(io_ctl);  in io_ctl_set_crc()
463 	tmp = page_address(io_ctl->pages[0]);  in io_ctl_set_crc()
468 static int io_ctl_check_crc(struct btrfs_io_ctl *io_ctl, int index)  in io_ctl_check_crc()  argument
474 	if (!io_ctl->check_crcs) {  in io_ctl_check_crc()
475 		io_ctl_map_page(io_ctl, 0);  in io_ctl_check_crc()
480 		offset = sizeof(u32) * io_ctl->num_pages;  in io_ctl_check_crc()
482 	tmp = page_address(io_ctl->pages[0]);  in io_ctl_check_crc()
486 	io_ctl_map_page(io_ctl, 0);  in io_ctl_check_crc()
487 	crc = btrfs_csum_data(io_ctl->orig + offset, crc,  in io_ctl_check_crc()
491 		btrfs_err_rl(io_ctl->fs_info,  in io_ctl_check_crc()
493 		io_ctl_unmap_page(io_ctl);  in io_ctl_check_crc()
500 static int io_ctl_add_entry(struct btrfs_io_ctl *io_ctl, u64 offset, u64 bytes,  in io_ctl_add_entry()  argument
505 	if (!io_ctl->cur)  in io_ctl_add_entry()
508 	entry = io_ctl->cur;  in io_ctl_add_entry()
513 	io_ctl->cur += sizeof(struct btrfs_free_space_entry);  in io_ctl_add_entry()
514 	io_ctl->size -= sizeof(struct btrfs_free_space_entry);  in io_ctl_add_entry()
516 	if (io_ctl->size >= sizeof(struct btrfs_free_space_entry))  in io_ctl_add_entry()
519 	io_ctl_set_crc(io_ctl, io_ctl->index - 1);  in io_ctl_add_entry()
522 	if (io_ctl->index >= io_ctl->num_pages)  in io_ctl_add_entry()
526 	io_ctl_map_page(io_ctl, 1);  in io_ctl_add_entry()
530 static int io_ctl_add_bitmap(struct btrfs_io_ctl *io_ctl, void *bitmap)  in io_ctl_add_bitmap()  argument
532 	if (!io_ctl->cur)  in io_ctl_add_bitmap()
539 	if (io_ctl->cur != io_ctl->orig) {  in io_ctl_add_bitmap()
540 		io_ctl_set_crc(io_ctl, io_ctl->index - 1);  in io_ctl_add_bitmap()
541 		if (io_ctl->index >= io_ctl->num_pages)  in io_ctl_add_bitmap()
543 		io_ctl_map_page(io_ctl, 0);  in io_ctl_add_bitmap()
546 	copy_page(io_ctl->cur, bitmap);  in io_ctl_add_bitmap()
547 	io_ctl_set_crc(io_ctl, io_ctl->index - 1);  in io_ctl_add_bitmap()
548 	if (io_ctl->index < io_ctl->num_pages)  in io_ctl_add_bitmap()
549 		io_ctl_map_page(io_ctl, 0);  in io_ctl_add_bitmap()
553 static void io_ctl_zero_remaining_pages(struct btrfs_io_ctl *io_ctl)  in io_ctl_zero_remaining_pages()  argument
559 	if (io_ctl->cur != io_ctl->orig)  in io_ctl_zero_remaining_pages()
560 		io_ctl_set_crc(io_ctl, io_ctl->index - 1);  in io_ctl_zero_remaining_pages()
562 		io_ctl_unmap_page(io_ctl);  in io_ctl_zero_remaining_pages()
564 	while (io_ctl->index < io_ctl->num_pages) {  in io_ctl_zero_remaining_pages()
565 		io_ctl_map_page(io_ctl, 1);  in io_ctl_zero_remaining_pages()
566 		io_ctl_set_crc(io_ctl, io_ctl->index - 1);  in io_ctl_zero_remaining_pages()
570 static int io_ctl_read_entry(struct btrfs_io_ctl *io_ctl,  in io_ctl_read_entry()  argument
576 	if (!io_ctl->cur) {  in io_ctl_read_entry()
577 		ret = io_ctl_check_crc(io_ctl, io_ctl->index);  in io_ctl_read_entry()
582 	e = io_ctl->cur;  in io_ctl_read_entry()
586 	io_ctl->cur += sizeof(struct btrfs_free_space_entry);  in io_ctl_read_entry()
587 	io_ctl->size -= sizeof(struct btrfs_free_space_entry);  in io_ctl_read_entry()
589 	if (io_ctl->size >= sizeof(struct btrfs_free_space_entry))  in io_ctl_read_entry()
592 	io_ctl_unmap_page(io_ctl);  in io_ctl_read_entry()
597 static int io_ctl_read_bitmap(struct btrfs_io_ctl *io_ctl,  in io_ctl_read_bitmap()  argument
602 	ret = io_ctl_check_crc(io_ctl, io_ctl->index);  in io_ctl_read_bitmap()
606 	copy_page(entry->bitmap, io_ctl->cur);  in io_ctl_read_bitmap()
607 	io_ctl_unmap_page(io_ctl);  in io_ctl_read_bitmap()
657 	struct btrfs_io_ctl io_ctl;  in __load_free_space_cache()  local
710 	ret = io_ctl_init(&io_ctl, inode, 0);  in __load_free_space_cache()
716 	ret = io_ctl_prepare_pages(&io_ctl, inode, 1);  in __load_free_space_cache()
720 	ret = io_ctl_check_crc(&io_ctl, 0);  in __load_free_space_cache()
724 	ret = io_ctl_check_generation(&io_ctl, generation);  in __load_free_space_cache()
734 		ret = io_ctl_read_entry(&io_ctl, e, &type);  in __load_free_space_cache()
781 	io_ctl_unmap_page(&io_ctl);  in __load_free_space_cache()
789 		ret = io_ctl_read_bitmap(&io_ctl, e);  in __load_free_space_cache()
794 	io_ctl_drop_pages(&io_ctl);  in __load_free_space_cache()
798 	io_ctl_free(&io_ctl);  in __load_free_space_cache()
801 	io_ctl_drop_pages(&io_ctl);  in __load_free_space_cache()
884 int write_cache_extent_entries(struct btrfs_io_ctl *io_ctl,  in write_cache_extent_entries()  argument
917 		ret = io_ctl_add_entry(io_ctl, e->offset, e->bytes,  in write_cache_extent_entries()
946 		ret = io_ctl_add_entry(io_ctl, trim_entry->start,  in write_cache_extent_entries()
1017 			    struct btrfs_io_ctl *io_ctl,  in write_pinned_extent_entries()  argument
1056 		ret = io_ctl_add_entry(io_ctl, extent_start, len, NULL);  in write_pinned_extent_entries()
1067 write_bitmap_entries(struct btrfs_io_ctl *io_ctl, struct list_head *bitmap_list)  in write_bitmap_entries()  argument
1074 		ret = io_ctl_add_bitmap(io_ctl, entry->bitmap);  in write_bitmap_entries()
1106 			   struct btrfs_io_ctl *io_ctl,  in cleanup_write_cache_enospc()  argument
1109 	io_ctl_drop_pages(io_ctl);  in cleanup_write_cache_enospc()
1117 				 struct btrfs_io_ctl *io_ctl,  in __btrfs_wait_cache_io()  argument
1121 	struct inode *inode = io_ctl->inode;  in __btrfs_wait_cache_io()
1133 				io_ctl->entries, io_ctl->bitmaps);  in __btrfs_wait_cache_io()
1135 	io_ctl_free(io_ctl);  in __btrfs_wait_cache_io()
1168 		io_ctl->inode = NULL;  in __btrfs_wait_cache_io()
1178 				    struct btrfs_io_ctl *io_ctl,  in btrfs_wait_cache_io_root()  argument
1181 	return __btrfs_wait_cache_io(root, trans, NULL, io_ctl, path, 0);  in btrfs_wait_cache_io_root()
1189 				     block_group, &block_group->io_ctl,  in btrfs_wait_cache_io()
1207 				   struct btrfs_io_ctl *io_ctl,  in __btrfs_write_out_cache()  argument
1221 	WARN_ON(io_ctl->pages);  in __btrfs_write_out_cache()
1222 	ret = io_ctl_init(io_ctl, inode, 1);  in __btrfs_write_out_cache()
1242 	ret = io_ctl_prepare_pages(io_ctl, inode, 0);  in __btrfs_write_out_cache()
1249 	io_ctl_set_generation(io_ctl, trans->transid);  in __btrfs_write_out_cache()
1254 	ret = write_cache_extent_entries(io_ctl, ctl,  in __btrfs_write_out_cache()
1269 					  io_ctl, &entries);  in __btrfs_write_out_cache()
1278 	ret = write_bitmap_entries(io_ctl, &bitmap_list);  in __btrfs_write_out_cache()
1285 	io_ctl_zero_remaining_pages(io_ctl);  in __btrfs_write_out_cache()
1288 	ret = btrfs_dirty_pages(inode, io_ctl->pages, io_ctl->num_pages, 0,  in __btrfs_write_out_cache()
1299 	io_ctl_drop_pages(io_ctl);  in __btrfs_write_out_cache()
1309 	io_ctl->entries = entries;  in __btrfs_write_out_cache()
1310 	io_ctl->bitmaps = bitmaps;  in __btrfs_write_out_cache()
1319 	io_ctl->inode = NULL;  in __btrfs_write_out_cache()
1320 	io_ctl_free(io_ctl);  in __btrfs_write_out_cache()
1336 	cleanup_write_cache_enospc(inode, io_ctl, &cached_state);  in __btrfs_write_out_cache()
1366 				block_group, &block_group->io_ctl, trans);  in btrfs_write_out_cache()
1377 		block_group->io_ctl.inode = NULL;  in btrfs_write_out_cache()
3506 	struct btrfs_io_ctl io_ctl;  in btrfs_write_out_ino_cache()  local
3512 	memset(&io_ctl, 0, sizeof(io_ctl));  in btrfs_write_out_ino_cache()
3513 	ret = __btrfs_write_out_cache(root, inode, ctl, NULL, &io_ctl, trans);  in btrfs_write_out_ino_cache()
3522 		ret = btrfs_wait_cache_io_root(root, trans, &io_ctl, path);  in btrfs_write_out_ino_cache()