440 static struct bio *__bio_alloc(struct f2fs_io_info *fio, int npages)  in __bio_alloc()  argument
442 	struct f2fs_sb_info *sbi = fio->sbi;  in __bio_alloc()
447 	f2fs_target_device(sbi, fio->new_blkaddr, bio);  in __bio_alloc()
448 	if (is_read_io(fio->op)) {  in __bio_alloc()
455 						fio->type, fio->temp);  in __bio_alloc()
457 	if (fio->io_wbc)  in __bio_alloc()
458 		wbc_init_bio(fio->io_wbc, bio);  in __bio_alloc()
465 				  const struct f2fs_io_info *fio,  in f2fs_set_bio_crypt_ctx()  argument
472 	if (!fio || !fio->encrypted_page)  in f2fs_set_bio_crypt_ctx()
478 				     const struct f2fs_io_info *fio)  in f2fs_crypt_mergeable_bio()  argument
484 	if (fio && fio->encrypted_page)  in f2fs_crypt_mergeable_bio()
546 static void __attach_io_flag(struct f2fs_io_info *fio)  in __attach_io_flag()  argument
548 	struct f2fs_sb_info *sbi = fio->sbi;  in __attach_io_flag()
552 	if (fio->type == DATA)  in __attach_io_flag()
554 	else if (fio->type == NODE)  in __attach_io_flag()
568 	if ((1 << fio->temp) & meta_flag)  in __attach_io_flag()
569 		fio->op_flags |= REQ_META;  in __attach_io_flag()
570 	if ((1 << fio->temp) & fua_flag)  in __attach_io_flag()
571 		fio->op_flags |= REQ_FUA;  in __attach_io_flag()
576 	struct f2fs_io_info *fio = &io->fio;  in __submit_merged_bio()  local
581 	__attach_io_flag(fio);  in __submit_merged_bio()
582 	bio_set_op_attrs(io->bio, fio->op, fio->op_flags);  in __submit_merged_bio()
584 	if (is_read_io(fio->op))  in __submit_merged_bio()
585 		trace_f2fs_prepare_read_bio(io->sbi->sb, fio->type, io->bio);  in __submit_merged_bio()
587 		trace_f2fs_prepare_write_bio(io->sbi->sb, fio->type, io->bio);  in __submit_merged_bio()
589 	__submit_bio(io->sbi, io->bio, fio->type);  in __submit_merged_bio()
640 		io->fio.type = META_FLUSH;  in __f2fs_submit_merged_write()
641 		io->fio.op = REQ_OP_WRITE;  in __f2fs_submit_merged_write()
642 		io->fio.op_flags = REQ_META | REQ_PRIO | REQ_SYNC;  in __f2fs_submit_merged_write()
644 			io->fio.op_flags |= REQ_PREFLUSH | REQ_FUA;  in __f2fs_submit_merged_write()
698 int f2fs_submit_page_bio(struct f2fs_io_info *fio)  in f2fs_submit_page_bio()  argument
701 	struct page *page = fio->encrypted_page ?  in f2fs_submit_page_bio()
702 			fio->encrypted_page : fio->page;  in f2fs_submit_page_bio()
704 	if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr,  in f2fs_submit_page_bio()
705 			fio->is_por ? META_POR : (__is_meta_io(fio) ?  in f2fs_submit_page_bio()
709 	trace_f2fs_submit_page_bio(page, fio);  in f2fs_submit_page_bio()
710 	f2fs_trace_ios(fio, 0);  in f2fs_submit_page_bio()
713 	bio = __bio_alloc(fio, 1);  in f2fs_submit_page_bio()
715 	f2fs_set_bio_crypt_ctx(bio, fio->page->mapping->host,  in f2fs_submit_page_bio()
716 			       fio->page->index, fio, GFP_NOIO);  in f2fs_submit_page_bio()
723 	if (fio->io_wbc && !is_read_io(fio->op))  in f2fs_submit_page_bio()
724 		wbc_account_cgroup_owner(fio->io_wbc, page, PAGE_SIZE);  in f2fs_submit_page_bio()
726 	__attach_io_flag(fio);  in f2fs_submit_page_bio()
727 	bio_set_op_attrs(bio, fio->op, fio->op_flags);  in f2fs_submit_page_bio()
729 	inc_page_count(fio->sbi, is_read_io(fio->op) ?  in f2fs_submit_page_bio()
730 			__read_io_type(page): WB_DATA_TYPE(fio->page));  in f2fs_submit_page_bio()
732 	__submit_bio(fio->sbi, bio, fio->type);  in f2fs_submit_page_bio()
745 						struct f2fs_io_info *fio)  in io_type_is_mergeable()  argument
747 	if (io->fio.op != fio->op)  in io_type_is_mergeable()
749 	return io->fio.op_flags == fio->op_flags;  in io_type_is_mergeable()
754 					struct f2fs_io_info *fio,  in io_is_mergeable()  argument
758 	if (F2FS_IO_ALIGNED(sbi) && (fio->type == DATA || fio->type == NODE)) {  in io_is_mergeable()
770 	return io_type_is_mergeable(io, fio);  in io_is_mergeable()
797 static int add_ipu_page(struct f2fs_io_info *fio, struct bio **bio,  in add_ipu_page()  argument
800 	struct f2fs_sb_info *sbi = fio->sbi;  in add_ipu_page()
818 							    *fio->last_block,  in add_ipu_page()
819 							    fio->new_blkaddr));  in add_ipu_page()
821 					fio->page->mapping->host,  in add_ipu_page()
822 					fio->page->index, fio) &&  in add_ipu_page()
901 int f2fs_merge_page_bio(struct f2fs_io_info *fio)  in f2fs_merge_page_bio()  argument
903 	struct bio *bio = *fio->bio;  in f2fs_merge_page_bio()
904 	struct page *page = fio->encrypted_page ?  in f2fs_merge_page_bio()
905 			fio->encrypted_page : fio->page;  in f2fs_merge_page_bio()
907 	if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr,  in f2fs_merge_page_bio()
908 			__is_meta_io(fio) ? META_GENERIC : DATA_GENERIC))  in f2fs_merge_page_bio()
911 	trace_f2fs_submit_page_bio(page, fio);  in f2fs_merge_page_bio()
912 	f2fs_trace_ios(fio, 0);  in f2fs_merge_page_bio()
914 	if (bio && !page_is_mergeable(fio->sbi, bio, *fio->last_block,  in f2fs_merge_page_bio()
915 						fio->new_blkaddr))  in f2fs_merge_page_bio()
916 		f2fs_submit_merged_ipu_write(fio->sbi, &bio, NULL);  in f2fs_merge_page_bio()
919 		bio = __bio_alloc(fio, BIO_MAX_PAGES);  in f2fs_merge_page_bio()
920 		__attach_io_flag(fio);  in f2fs_merge_page_bio()
921 		f2fs_set_bio_crypt_ctx(bio, fio->page->mapping->host,  in f2fs_merge_page_bio()
922 				       fio->page->index, fio, GFP_NOIO);  in f2fs_merge_page_bio()
923 		bio_set_op_attrs(bio, fio->op, fio->op_flags);  in f2fs_merge_page_bio()
925 		add_bio_entry(fio->sbi, bio, page, fio->temp);  in f2fs_merge_page_bio()
927 		if (add_ipu_page(fio, &bio, page))  in f2fs_merge_page_bio()
931 	if (fio->io_wbc)  in f2fs_merge_page_bio()
932 		wbc_account_cgroup_owner(fio->io_wbc, page, PAGE_SIZE);  in f2fs_merge_page_bio()
934 	inc_page_count(fio->sbi, WB_DATA_TYPE(page));  in f2fs_merge_page_bio()
936 	*fio->last_block = fio->new_blkaddr;  in f2fs_merge_page_bio()
937 	*fio->bio = bio;  in f2fs_merge_page_bio()
942 void f2fs_submit_page_write(struct f2fs_io_info *fio)  in f2fs_submit_page_write()  argument
944 	struct f2fs_sb_info *sbi = fio->sbi;  in f2fs_submit_page_write()
945 	enum page_type btype = PAGE_TYPE_OF_BIO(fio->type);  in f2fs_submit_page_write()
946 	struct f2fs_bio_info *io = sbi->write_io[btype] + fio->temp;  in f2fs_submit_page_write()
949 	f2fs_bug_on(sbi, is_read_io(fio->op));  in f2fs_submit_page_write()
953 	if (fio->in_list) {  in f2fs_submit_page_write()
959 		fio = list_first_entry(&io->io_list,  in f2fs_submit_page_write()
961 		list_del(&fio->list);  in f2fs_submit_page_write()
965 	verify_fio_blkaddr(fio);  in f2fs_submit_page_write()
967 	if (fio->encrypted_page)  in f2fs_submit_page_write()
968 		bio_page = fio->encrypted_page;  in f2fs_submit_page_write()
969 	else if (fio->compressed_page)  in f2fs_submit_page_write()
970 		bio_page = fio->compressed_page;  in f2fs_submit_page_write()
972 		bio_page = fio->page;  in f2fs_submit_page_write()
975 	fio->submitted = true;  in f2fs_submit_page_write()
980 	    (!io_is_mergeable(sbi, io->bio, io, fio, io->last_block_in_bio,  in f2fs_submit_page_write()
981 			      fio->new_blkaddr) ||  in f2fs_submit_page_write()
982 	     !f2fs_crypt_mergeable_bio(io->bio, fio->page->mapping->host,  in f2fs_submit_page_write()
983 				       bio_page->index, fio)))  in f2fs_submit_page_write()
988 				(fio->type == DATA || fio->type == NODE) &&  in f2fs_submit_page_write()
989 				fio->new_blkaddr & F2FS_IO_SIZE_MASK(sbi)) {  in f2fs_submit_page_write()
991 			fio->retry = true;  in f2fs_submit_page_write()
994 		io->bio = __bio_alloc(fio, BIO_MAX_PAGES);  in f2fs_submit_page_write()
995 		f2fs_set_bio_crypt_ctx(io->bio, fio->page->mapping->host,  in f2fs_submit_page_write()
996 				       bio_page->index, fio, GFP_NOIO);  in f2fs_submit_page_write()
997 		io->fio = *fio;  in f2fs_submit_page_write()
1005 	if (fio->io_wbc)  in f2fs_submit_page_write()
1006 		wbc_account_cgroup_owner(fio->io_wbc, bio_page, PAGE_SIZE);  in f2fs_submit_page_write()
1008 	io->last_block_in_bio = fio->new_blkaddr;  in f2fs_submit_page_write()
1009 	f2fs_trace_ios(fio, 0);  in f2fs_submit_page_write()
1011 	trace_f2fs_submit_page_write(fio->page, fio);  in f2fs_submit_page_write()
1013 	if (fio->in_list)  in f2fs_submit_page_write()
2480 int f2fs_encrypt_one_page(struct f2fs_io_info *fio)  in f2fs_encrypt_one_page()  argument
2482 	struct inode *inode = fio->page->mapping->host;  in f2fs_encrypt_one_page()
2489 	page = fio->compressed_page ? fio->compressed_page : fio->page;  in f2fs_encrypt_one_page()
2492 	f2fs_wait_on_block_writeback(inode, fio->old_blkaddr);  in f2fs_encrypt_one_page()
2498 	fio->encrypted_page = fscrypt_encrypt_pagecache_blocks(page,  in f2fs_encrypt_one_page()
2500 	if (IS_ERR(fio->encrypted_page)) {  in f2fs_encrypt_one_page()
2502 		if (PTR_ERR(fio->encrypted_page) == -ENOMEM) {  in f2fs_encrypt_one_page()
2503 			f2fs_flush_merged_writes(fio->sbi);  in f2fs_encrypt_one_page()
2508 		return PTR_ERR(fio->encrypted_page);  in f2fs_encrypt_one_page()
2511 	mpage = find_lock_page(META_MAPPING(fio->sbi), fio->old_blkaddr);  in f2fs_encrypt_one_page()
2515 				page_address(fio->encrypted_page), PAGE_SIZE);  in f2fs_encrypt_one_page()
2522 				struct f2fs_io_info *fio)  in check_inplace_update_policy()  argument
2542 			fio && fio->op == REQ_OP_WRITE &&  in check_inplace_update_policy()
2543 			!(fio->op_flags & REQ_SYNC) &&  in check_inplace_update_policy()
2552 	if (unlikely(fio && is_sbi_flag_set(sbi, SBI_CP_DISABLED) &&  in check_inplace_update_policy()
2553 			!f2fs_is_checkpointed_data(sbi, fio->old_blkaddr)))  in check_inplace_update_policy()
2559 bool f2fs_should_update_inplace(struct inode *inode, struct f2fs_io_info *fio)  in f2fs_should_update_inplace()  argument
2568 	return check_inplace_update_policy(inode, fio);  in f2fs_should_update_inplace()
2571 bool f2fs_should_update_outplace(struct inode *inode, struct f2fs_io_info *fio)  in f2fs_should_update_outplace()  argument
2583 	if (fio) {  in f2fs_should_update_outplace()
2584 		if (is_cold_data(fio->page))  in f2fs_should_update_outplace()
2586 		if (IS_ATOMIC_WRITTEN_PAGE(fio->page))  in f2fs_should_update_outplace()
2589 			f2fs_is_checkpointed_data(sbi, fio->old_blkaddr)))  in f2fs_should_update_outplace()
2595 static inline bool need_inplace_update(struct f2fs_io_info *fio)  in need_inplace_update()  argument
2597 	struct inode *inode = fio->page->mapping->host;  in need_inplace_update()
2599 	if (f2fs_should_update_outplace(inode, fio))  in need_inplace_update()
2602 	return f2fs_should_update_inplace(inode, fio);  in need_inplace_update()
2605 int f2fs_do_write_data_page(struct f2fs_io_info *fio)  in f2fs_do_write_data_page()  argument
2607 	struct page *page = fio->page;  in f2fs_do_write_data_page()
2616 	if (need_inplace_update(fio) &&  in f2fs_do_write_data_page()
2618 		fio->old_blkaddr = ei.blk + page->index - ei.fofs;  in f2fs_do_write_data_page()
2620 		if (!f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr,  in f2fs_do_write_data_page()
2625 		fio->need_lock = LOCK_DONE;  in f2fs_do_write_data_page()
2630 	if (fio->need_lock == LOCK_REQ && !f2fs_trylock_op(fio->sbi))  in f2fs_do_write_data_page()
2637 	fio->old_blkaddr = dn.data_blkaddr;  in f2fs_do_write_data_page()
2640 	if (fio->old_blkaddr == NULL_ADDR) {  in f2fs_do_write_data_page()
2646 	if (__is_valid_data_blkaddr(fio->old_blkaddr) &&  in f2fs_do_write_data_page()
2647 		!f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr,  in f2fs_do_write_data_page()
2657 		(__is_valid_data_blkaddr(fio->old_blkaddr) &&  in f2fs_do_write_data_page()
2658 					need_inplace_update(fio))) {  in f2fs_do_write_data_page()
2659 		err = f2fs_encrypt_one_page(fio);  in f2fs_do_write_data_page()
2666 		if (fio->need_lock == LOCK_REQ)  in f2fs_do_write_data_page()
2667 			f2fs_unlock_op(fio->sbi);  in f2fs_do_write_data_page()
2668 		err = f2fs_inplace_write_data(fio);  in f2fs_do_write_data_page()
2671 				fscrypt_finalize_bounce_page(&fio->encrypted_page);  in f2fs_do_write_data_page()
2677 		trace_f2fs_do_write_data_page(fio->page, IPU);  in f2fs_do_write_data_page()
2681 	if (fio->need_lock == LOCK_RETRY) {  in f2fs_do_write_data_page()
2682 		if (!f2fs_trylock_op(fio->sbi)) {  in f2fs_do_write_data_page()
2686 		fio->need_lock = LOCK_REQ;  in f2fs_do_write_data_page()
2689 	err = f2fs_get_node_info(fio->sbi, dn.nid, &ni);  in f2fs_do_write_data_page()
2693 	fio->version = ni.version;  in f2fs_do_write_data_page()
2695 	err = f2fs_encrypt_one_page(fio);  in f2fs_do_write_data_page()
2702 	if (fio->compr_blocks && fio->old_blkaddr == COMPRESS_ADDR)  in f2fs_do_write_data_page()
2703 		f2fs_i_compr_blocks_update(inode, fio->compr_blocks - 1, false);  in f2fs_do_write_data_page()
2706 	f2fs_outplace_write_data(&dn, fio);  in f2fs_do_write_data_page()
2714 	if (fio->need_lock == LOCK_REQ)  in f2fs_do_write_data_page()
2715 		f2fs_unlock_op(fio->sbi);  in f2fs_do_write_data_page()
2735 	struct f2fs_io_info fio = {  in f2fs_write_single_data_page()  local
2803 		fio.need_lock = LOCK_DONE;  in f2fs_write_single_data_page()
2804 		err = f2fs_do_write_data_page(&fio);  in f2fs_write_single_data_page()
2827 		err = f2fs_do_write_data_page(&fio);  in f2fs_write_single_data_page()
2829 			fio.need_lock = LOCK_REQ;  in f2fs_write_single_data_page()
2830 			err = f2fs_do_write_data_page(&fio);  in f2fs_write_single_data_page()
2872 		*submitted = fio.submitted ? 1 : 0;  in f2fs_write_single_data_page()