392 static struct bio *__bio_alloc(struct f2fs_io_info *fio, int npages)  in __bio_alloc()  argument
394 	struct f2fs_sb_info *sbi = fio->sbi;  in __bio_alloc()
399 	f2fs_target_device(sbi, fio->new_blkaddr, bio);  in __bio_alloc()
400 	if (is_read_io(fio->op)) {  in __bio_alloc()
407 						fio->type, fio->temp);  in __bio_alloc()
411 	if (fio->io_wbc)  in __bio_alloc()
412 		wbc_init_bio(fio->io_wbc, bio);  in __bio_alloc()
419 				  const struct f2fs_io_info *fio,  in f2fs_set_bio_crypt_ctx()  argument
426 	if (!fio || !fio->encrypted_page)  in f2fs_set_bio_crypt_ctx()
432 				     const struct f2fs_io_info *fio)  in f2fs_crypt_mergeable_bio()  argument
438 	if (fio && fio->encrypted_page)  in f2fs_crypt_mergeable_bio()
503 static void __attach_io_flag(struct f2fs_io_info *fio)  in __attach_io_flag()  argument
505 	struct f2fs_sb_info *sbi = fio->sbi;  in __attach_io_flag()
509 	if (fio->type == DATA)  in __attach_io_flag()
511 	else if (fio->type == NODE)  in __attach_io_flag()
525 	if ((1 << fio->temp) & meta_flag)  in __attach_io_flag()
526 		fio->op_flags |= REQ_META;  in __attach_io_flag()
527 	if ((1 << fio->temp) & fua_flag)  in __attach_io_flag()
528 		fio->op_flags |= REQ_FUA;  in __attach_io_flag()
533 	struct f2fs_io_info *fio = &io->fio;  in __submit_merged_bio()  local
538 	__attach_io_flag(fio);  in __submit_merged_bio()
539 	bio_set_op_attrs(io->bio, fio->op, fio->op_flags);  in __submit_merged_bio()
541 	if (is_read_io(fio->op))  in __submit_merged_bio()
542 		trace_f2fs_prepare_read_bio(io->sbi->sb, fio->type, io->bio);  in __submit_merged_bio()
544 		trace_f2fs_prepare_write_bio(io->sbi->sb, fio->type, io->bio);  in __submit_merged_bio()
546 	__submit_bio(io->sbi, io->bio, fio->type);  in __submit_merged_bio()
597 		io->fio.type = META_FLUSH;  in __f2fs_submit_merged_write()
598 		io->fio.op = REQ_OP_WRITE;  in __f2fs_submit_merged_write()
599 		io->fio.op_flags = REQ_META | REQ_PRIO | REQ_SYNC;  in __f2fs_submit_merged_write()
601 			io->fio.op_flags |= REQ_PREFLUSH | REQ_FUA;  in __f2fs_submit_merged_write()
655 int f2fs_submit_page_bio(struct f2fs_io_info *fio)  in f2fs_submit_page_bio()  argument
658 	struct page *page = fio->encrypted_page ?  in f2fs_submit_page_bio()
659 			fio->encrypted_page : fio->page;  in f2fs_submit_page_bio()
661 	if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr,  in f2fs_submit_page_bio()
662 			fio->is_por ? META_POR : (__is_meta_io(fio) ?  in f2fs_submit_page_bio()
666 	trace_f2fs_submit_page_bio(page, fio);  in f2fs_submit_page_bio()
669 	bio = __bio_alloc(fio, 1);  in f2fs_submit_page_bio()
671 	f2fs_set_bio_crypt_ctx(bio, fio->page->mapping->host,  in f2fs_submit_page_bio()
672 			       fio->page->index, fio, GFP_NOIO);  in f2fs_submit_page_bio()
679 	if (fio->io_wbc && !is_read_io(fio->op))  in f2fs_submit_page_bio()
680 		wbc_account_cgroup_owner(fio->io_wbc, page, PAGE_SIZE);  in f2fs_submit_page_bio()
682 	__attach_io_flag(fio);  in f2fs_submit_page_bio()
683 	bio_set_op_attrs(bio, fio->op, fio->op_flags);  in f2fs_submit_page_bio()
685 	inc_page_count(fio->sbi, is_read_io(fio->op) ?  in f2fs_submit_page_bio()
686 			__read_io_type(page): WB_DATA_TYPE(fio->page));  in f2fs_submit_page_bio()
688 	__submit_bio(fio->sbi, bio, fio->type);  in f2fs_submit_page_bio()
704 						struct f2fs_io_info *fio)  in io_type_is_mergeable()  argument
706 	if (io->fio.op != fio->op)  in io_type_is_mergeable()
708 	return io->fio.op_flags == fio->op_flags;  in io_type_is_mergeable()
713 					struct f2fs_io_info *fio,  in io_is_mergeable()  argument
717 	if (F2FS_IO_ALIGNED(sbi) && (fio->type == DATA || fio->type == NODE)) {  in io_is_mergeable()
729 	return io_type_is_mergeable(io, fio);  in io_is_mergeable()
756 static int add_ipu_page(struct f2fs_io_info *fio, struct bio **bio,  in add_ipu_page()  argument
759 	struct f2fs_sb_info *sbi = fio->sbi;  in add_ipu_page()
777 							    *fio->last_block,  in add_ipu_page()
778 							    fio->new_blkaddr));  in add_ipu_page()
780 					fio->page->mapping->host,  in add_ipu_page()
781 					fio->page->index, fio) &&  in add_ipu_page()
860 int f2fs_merge_page_bio(struct f2fs_io_info *fio)  in f2fs_merge_page_bio()  argument
862 	struct bio *bio = *fio->bio;  in f2fs_merge_page_bio()
863 	struct page *page = fio->encrypted_page ?  in f2fs_merge_page_bio()
864 			fio->encrypted_page : fio->page;  in f2fs_merge_page_bio()
866 	if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr,  in f2fs_merge_page_bio()
867 			__is_meta_io(fio) ? META_GENERIC : DATA_GENERIC))  in f2fs_merge_page_bio()
870 	trace_f2fs_submit_page_bio(page, fio);  in f2fs_merge_page_bio()
872 	if (bio && !page_is_mergeable(fio->sbi, bio, *fio->last_block,  in f2fs_merge_page_bio()
873 						fio->new_blkaddr))  in f2fs_merge_page_bio()
874 		f2fs_submit_merged_ipu_write(fio->sbi, &bio, NULL);  in f2fs_merge_page_bio()
877 		bio = __bio_alloc(fio, BIO_MAX_VECS);  in f2fs_merge_page_bio()
878 		__attach_io_flag(fio);  in f2fs_merge_page_bio()
879 		f2fs_set_bio_crypt_ctx(bio, fio->page->mapping->host,  in f2fs_merge_page_bio()
880 				       fio->page->index, fio, GFP_NOIO);  in f2fs_merge_page_bio()
881 		bio_set_op_attrs(bio, fio->op, fio->op_flags);  in f2fs_merge_page_bio()
883 		add_bio_entry(fio->sbi, bio, page, fio->temp);  in f2fs_merge_page_bio()
885 		if (add_ipu_page(fio, &bio, page))  in f2fs_merge_page_bio()
889 	if (fio->io_wbc)  in f2fs_merge_page_bio()
890 		wbc_account_cgroup_owner(fio->io_wbc, page, PAGE_SIZE);  in f2fs_merge_page_bio()
892 	inc_page_count(fio->sbi, WB_DATA_TYPE(page));  in f2fs_merge_page_bio()
894 	*fio->last_block = fio->new_blkaddr;  in f2fs_merge_page_bio()
895 	*fio->bio = bio;  in f2fs_merge_page_bio()
900 void f2fs_submit_page_write(struct f2fs_io_info *fio)  in f2fs_submit_page_write()  argument
902 	struct f2fs_sb_info *sbi = fio->sbi;  in f2fs_submit_page_write()
903 	enum page_type btype = PAGE_TYPE_OF_BIO(fio->type);  in f2fs_submit_page_write()
904 	struct f2fs_bio_info *io = sbi->write_io[btype] + fio->temp;  in f2fs_submit_page_write()
907 	f2fs_bug_on(sbi, is_read_io(fio->op));  in f2fs_submit_page_write()
911 	if (fio->in_list) {  in f2fs_submit_page_write()
917 		fio = list_first_entry(&io->io_list,  in f2fs_submit_page_write()
919 		list_del(&fio->list);  in f2fs_submit_page_write()
923 	verify_fio_blkaddr(fio);  in f2fs_submit_page_write()
925 	if (fio->encrypted_page)  in f2fs_submit_page_write()
926 		bio_page = fio->encrypted_page;  in f2fs_submit_page_write()
927 	else if (fio->compressed_page)  in f2fs_submit_page_write()
928 		bio_page = fio->compressed_page;  in f2fs_submit_page_write()
930 		bio_page = fio->page;  in f2fs_submit_page_write()
933 	fio->submitted = true;  in f2fs_submit_page_write()
938 	    (!io_is_mergeable(sbi, io->bio, io, fio, io->last_block_in_bio,  in f2fs_submit_page_write()
939 			      fio->new_blkaddr) ||  in f2fs_submit_page_write()
940 	     !f2fs_crypt_mergeable_bio(io->bio, fio->page->mapping->host,  in f2fs_submit_page_write()
941 				       bio_page->index, fio)))  in f2fs_submit_page_write()
946 				(fio->type == DATA || fio->type == NODE) &&  in f2fs_submit_page_write()
947 				fio->new_blkaddr & F2FS_IO_SIZE_MASK(sbi)) {  in f2fs_submit_page_write()
949 			fio->retry = true;  in f2fs_submit_page_write()
952 		io->bio = __bio_alloc(fio, BIO_MAX_VECS);  in f2fs_submit_page_write()
953 		f2fs_set_bio_crypt_ctx(io->bio, fio->page->mapping->host,  in f2fs_submit_page_write()
954 				       bio_page->index, fio, GFP_NOIO);  in f2fs_submit_page_write()
955 		io->fio = *fio;  in f2fs_submit_page_write()
963 	if (fio->io_wbc)  in f2fs_submit_page_write()
964 		wbc_account_cgroup_owner(fio->io_wbc, bio_page, PAGE_SIZE);  in f2fs_submit_page_write()
966 	io->last_block_in_bio = fio->new_blkaddr;  in f2fs_submit_page_write()
968 	trace_f2fs_submit_page_write(fio->page, fio);  in f2fs_submit_page_write()
970 	if (fio->in_list)  in f2fs_submit_page_write()
2468 int f2fs_encrypt_one_page(struct f2fs_io_info *fio)  in f2fs_encrypt_one_page()  argument
2470 	struct inode *inode = fio->page->mapping->host;  in f2fs_encrypt_one_page()
2477 	page = fio->compressed_page ? fio->compressed_page : fio->page;  in f2fs_encrypt_one_page()
2480 	f2fs_wait_on_block_writeback(inode, fio->old_blkaddr);  in f2fs_encrypt_one_page()
2486 	fio->encrypted_page = fscrypt_encrypt_pagecache_blocks(page,  in f2fs_encrypt_one_page()
2488 	if (IS_ERR(fio->encrypted_page)) {  in f2fs_encrypt_one_page()
2490 		if (PTR_ERR(fio->encrypted_page) == -ENOMEM) {  in f2fs_encrypt_one_page()
2491 			f2fs_flush_merged_writes(fio->sbi);  in f2fs_encrypt_one_page()
2496 		return PTR_ERR(fio->encrypted_page);  in f2fs_encrypt_one_page()
2499 	mpage = find_lock_page(META_MAPPING(fio->sbi), fio->old_blkaddr);  in f2fs_encrypt_one_page()
2503 				page_address(fio->encrypted_page), PAGE_SIZE);  in f2fs_encrypt_one_page()
2510 				struct f2fs_io_info *fio)  in check_inplace_update_policy()  argument
2530 			fio && fio->op == REQ_OP_WRITE &&  in check_inplace_update_policy()
2531 			!(fio->op_flags & REQ_SYNC) &&  in check_inplace_update_policy()
2540 	if (unlikely(fio && is_sbi_flag_set(sbi, SBI_CP_DISABLED) &&  in check_inplace_update_policy()
2541 			!f2fs_is_checkpointed_data(sbi, fio->old_blkaddr)))  in check_inplace_update_policy()
2547 bool f2fs_should_update_inplace(struct inode *inode, struct f2fs_io_info *fio)  in f2fs_should_update_inplace()  argument
2560 	return check_inplace_update_policy(inode, fio);  in f2fs_should_update_inplace()
2563 bool f2fs_should_update_outplace(struct inode *inode, struct f2fs_io_info *fio)  in f2fs_should_update_outplace()  argument
2582 	if (fio) {  in f2fs_should_update_outplace()
2583 		if (page_private_gcing(fio->page))  in f2fs_should_update_outplace()
2585 		if (page_private_dummy(fio->page))  in f2fs_should_update_outplace()
2588 			f2fs_is_checkpointed_data(sbi, fio->old_blkaddr)))  in f2fs_should_update_outplace()
2594 static inline bool need_inplace_update(struct f2fs_io_info *fio)  in need_inplace_update()  argument
2596 	struct inode *inode = fio->page->mapping->host;  in need_inplace_update()
2598 	if (f2fs_should_update_outplace(inode, fio))  in need_inplace_update()
2601 	return f2fs_should_update_inplace(inode, fio);  in need_inplace_update()
2604 int f2fs_do_write_data_page(struct f2fs_io_info *fio)  in f2fs_do_write_data_page()  argument
2606 	struct page *page = fio->page;  in f2fs_do_write_data_page()
2615 	if (need_inplace_update(fio) &&  in f2fs_do_write_data_page()
2617 		fio->old_blkaddr = ei.blk + page->index - ei.fofs;  in f2fs_do_write_data_page()
2619 		if (!f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr,  in f2fs_do_write_data_page()
2624 		fio->need_lock = LOCK_DONE;  in f2fs_do_write_data_page()
2629 	if (fio->need_lock == LOCK_REQ && !f2fs_trylock_op(fio->sbi))  in f2fs_do_write_data_page()
2636 	fio->old_blkaddr = dn.data_blkaddr;  in f2fs_do_write_data_page()
2639 	if (fio->old_blkaddr == NULL_ADDR) {  in f2fs_do_write_data_page()
2645 	if (__is_valid_data_blkaddr(fio->old_blkaddr) &&  in f2fs_do_write_data_page()
2646 		!f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr,  in f2fs_do_write_data_page()
2656 		(__is_valid_data_blkaddr(fio->old_blkaddr) &&  in f2fs_do_write_data_page()
2657 					need_inplace_update(fio))) {  in f2fs_do_write_data_page()
2658 		err = f2fs_encrypt_one_page(fio);  in f2fs_do_write_data_page()
2665 		if (fio->need_lock == LOCK_REQ)  in f2fs_do_write_data_page()
2666 			f2fs_unlock_op(fio->sbi);  in f2fs_do_write_data_page()
2667 		err = f2fs_inplace_write_data(fio);  in f2fs_do_write_data_page()
2670 				fscrypt_finalize_bounce_page(&fio->encrypted_page);  in f2fs_do_write_data_page()
2676 		trace_f2fs_do_write_data_page(fio->page, IPU);  in f2fs_do_write_data_page()
2680 	if (fio->need_lock == LOCK_RETRY) {  in f2fs_do_write_data_page()
2681 		if (!f2fs_trylock_op(fio->sbi)) {  in f2fs_do_write_data_page()
2685 		fio->need_lock = LOCK_REQ;  in f2fs_do_write_data_page()
2688 	err = f2fs_get_node_info(fio->sbi, dn.nid, &ni);  in f2fs_do_write_data_page()
2692 	fio->version = ni.version;  in f2fs_do_write_data_page()
2694 	err = f2fs_encrypt_one_page(fio);  in f2fs_do_write_data_page()
2701 	if (fio->compr_blocks && fio->old_blkaddr == COMPRESS_ADDR)  in f2fs_do_write_data_page()
2702 		f2fs_i_compr_blocks_update(inode, fio->compr_blocks - 1, false);  in f2fs_do_write_data_page()
2705 	f2fs_outplace_write_data(&dn, fio);  in f2fs_do_write_data_page()
2713 	if (fio->need_lock == LOCK_REQ)  in f2fs_do_write_data_page()
2714 		f2fs_unlock_op(fio->sbi);  in f2fs_do_write_data_page()
2735 	struct f2fs_io_info fio = {  in f2fs_write_single_data_page()  local
2803 		fio.need_lock = LOCK_DONE;  in f2fs_write_single_data_page()
2804 		err = f2fs_do_write_data_page(&fio);  in f2fs_write_single_data_page()
2827 		err = f2fs_do_write_data_page(&fio);  in f2fs_write_single_data_page()
2829 			fio.need_lock = LOCK_REQ;  in f2fs_write_single_data_page()
2830 			err = f2fs_do_write_data_page(&fio);  in f2fs_write_single_data_page()
2872 		*submitted = fio.submitted ? 1 : 0;  in f2fs_write_single_data_page()