397 	struct fuse_io_args ia;  member
418 		curr_index = wpa->ia.write.in.offset >> PAGE_SHIFT;  in fuse_find_writeback()
419 		if (idx_from >= curr_index + wpa->ia.ap.num_pages)  in fuse_find_writeback()
608 void fuse_read_args_fill(struct fuse_io_args *ia, struct file *file, loff_t pos,  in fuse_read_args_fill()  argument
612 	struct fuse_args *args = &ia->ap.args;  in fuse_read_args_fill()
614 	ia->read.in.fh = ff->fh;  in fuse_read_args_fill()
615 	ia->read.in.offset = pos;  in fuse_read_args_fill()
616 	ia->read.in.size = count;  in fuse_read_args_fill()
617 	ia->read.in.flags = file->f_flags;  in fuse_read_args_fill()
621 	args->in_args[0].size = sizeof(ia->read.in);  in fuse_read_args_fill()
622 	args->in_args[0].value = &ia->read.in;  in fuse_read_args_fill()
709 	struct fuse_io_args *ia;  in fuse_io_alloc()  local
711 	ia = kzalloc(sizeof(*ia), GFP_KERNEL);  in fuse_io_alloc()
712 	if (ia) {  in fuse_io_alloc()
713 		ia->io = io;  in fuse_io_alloc()
714 		ia->ap.pages = fuse_pages_alloc(npages, GFP_KERNEL,  in fuse_io_alloc()
715 						&ia->ap.descs);  in fuse_io_alloc()
716 		if (!ia->ap.pages) {  in fuse_io_alloc()
717 			kfree(ia);  in fuse_io_alloc()
718 			ia = NULL;  in fuse_io_alloc()
721 	return ia;  in fuse_io_alloc()
724 static void fuse_io_free(struct fuse_io_args *ia)  in fuse_io_free()  argument
726 	kfree(ia->ap.pages);  in fuse_io_free()
727 	kfree(ia);  in fuse_io_free()
733 	struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args);  in fuse_aio_complete_req()  local
734 	struct fuse_io_priv *io = ia->io;  in fuse_aio_complete_req()
737 	fuse_release_user_pages(&ia->ap, io->should_dirty);  in fuse_aio_complete_req()
742 		if (ia->write.out.size > ia->write.in.size) {  in fuse_aio_complete_req()
744 		} else if (ia->write.in.size != ia->write.out.size) {  in fuse_aio_complete_req()
745 			pos = ia->write.in.offset - io->offset +  in fuse_aio_complete_req()
746 				ia->write.out.size;  in fuse_aio_complete_req()
751 		if (ia->read.in.size != outsize)  in fuse_aio_complete_req()
752 			pos = ia->read.in.offset - io->offset + outsize;  in fuse_aio_complete_req()
756 	fuse_io_free(ia);  in fuse_aio_complete_req()
760 				   struct fuse_io_args *ia, size_t num_bytes)  in fuse_async_req_send()  argument
763 	struct fuse_io_priv *io = ia->io;  in fuse_async_req_send()
771 	ia->ap.args.end = fuse_aio_complete_req;  in fuse_async_req_send()
772 	ia->ap.args.may_block = io->should_dirty;  in fuse_async_req_send()
773 	err = fuse_simple_background(fm, &ia->ap.args, GFP_KERNEL);  in fuse_async_req_send()
775 		fuse_aio_complete_req(fm, &ia->ap.args, err);  in fuse_async_req_send()
780 static ssize_t fuse_send_read(struct fuse_io_args *ia, loff_t pos, size_t count,  in fuse_send_read()  argument
783 	struct file *file = ia->io->iocb->ki_filp;  in fuse_send_read()
787 	fuse_read_args_fill(ia, file, pos, count, FUSE_READ);  in fuse_send_read()
789 		ia->read.in.read_flags |= FUSE_READ_LOCKOWNER;  in fuse_send_read()
790 		ia->read.in.lock_owner = fuse_lock_owner_id(fm->fc, owner);  in fuse_send_read()
793 	if (ia->io->async)  in fuse_send_read()
794 		return fuse_async_req_send(fm, ia, count);  in fuse_send_read()
796 	return fuse_simple_request(fm, &ia->ap.args);  in fuse_send_read()
836 	struct fuse_io_args ia = {  in fuse_do_readpage()  local
859 	fuse_read_args_fill(&ia, file, pos, desc.length, FUSE_READ);  in fuse_do_readpage()
860 	res = fuse_simple_request(fm, &ia.ap.args);  in fuse_do_readpage()
867 		fuse_short_read(inode, attr_ver, res, &ia.ap);  in fuse_do_readpage()
895 	struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args);  in fuse_readpages_end()  local
896 	struct fuse_args_pages *ap = &ia->ap;  in fuse_readpages_end()
897 	size_t count = ia->read.in.size;  in fuse_readpages_end()
911 			fuse_short_read(inode, ia->read.attr_ver, num_read, ap);  in fuse_readpages_end()
926 	if (ia->ff)  in fuse_readpages_end()
927 		fuse_file_put(ia->ff, false, false);  in fuse_readpages_end()
929 	fuse_io_free(ia);  in fuse_readpages_end()
932 static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file)  in fuse_send_readpages()  argument
936 	struct fuse_args_pages *ap = &ia->ap;  in fuse_send_readpages()
953 	fuse_read_args_fill(ia, file, pos, count, FUSE_READ);  in fuse_send_readpages()
954 	ia->read.attr_ver = fuse_get_attr_version(fm->fc);  in fuse_send_readpages()
956 		ia->ff = fuse_file_get(ff);  in fuse_send_readpages()
981 		struct fuse_io_args *ia;  in fuse_readahead()  local
997 		ia = fuse_io_alloc(NULL, nr_pages);  in fuse_readahead()
998 		if (!ia)  in fuse_readahead()
1000 		ap = &ia->ap;  in fuse_readahead()
1008 		fuse_send_readpages(ia, rac->file);  in fuse_readahead()
1033 static void fuse_write_args_fill(struct fuse_io_args *ia, struct fuse_file *ff,  in fuse_write_args_fill()  argument
1036 	struct fuse_args *args = &ia->ap.args;  in fuse_write_args_fill()
1038 	ia->write.in.fh = ff->fh;  in fuse_write_args_fill()
1039 	ia->write.in.offset = pos;  in fuse_write_args_fill()
1040 	ia->write.in.size = count;  in fuse_write_args_fill()
1047 		args->in_args[0].size = sizeof(ia->write.in);  in fuse_write_args_fill()
1048 	args->in_args[0].value = &ia->write.in;  in fuse_write_args_fill()
1051 	args->out_args[0].size = sizeof(ia->write.out);  in fuse_write_args_fill()
1052 	args->out_args[0].value = &ia->write.out;  in fuse_write_args_fill()
1067 static ssize_t fuse_send_write(struct fuse_io_args *ia, loff_t pos,  in fuse_send_write()  argument
1070 	struct kiocb *iocb = ia->io->iocb;  in fuse_send_write()
1074 	struct fuse_write_in *inarg = &ia->write.in;  in fuse_send_write()
1077 	fuse_write_args_fill(ia, ff, pos, count);  in fuse_send_write()
1084 	if (ia->io->async)  in fuse_send_write()
1085 		return fuse_async_req_send(fm, ia, count);  in fuse_send_write()
1087 	err = fuse_simple_request(fm, &ia->ap.args);  in fuse_send_write()
1088 	if (!err && ia->write.out.size > count)  in fuse_send_write()
1091 	return err ?: ia->write.out.size;  in fuse_send_write()
1113 static ssize_t fuse_send_write_pages(struct fuse_io_args *ia,  in fuse_send_write_pages()  argument
1117 	struct fuse_args_pages *ap = &ia->ap;  in fuse_send_write_pages()
1128 	fuse_write_args_fill(ia, ff, pos, count);  in fuse_send_write_pages()
1129 	ia->write.in.flags = fuse_write_flags(iocb);  in fuse_send_write_pages()
1131 		ia->write.in.write_flags |= FUSE_WRITE_KILL_SUIDGID;  in fuse_send_write_pages()
1134 	if (!err && ia->write.out.size > count)  in fuse_send_write_pages()
1137 	short_write = ia->write.out.size < count;  in fuse_send_write_pages()
1139 	count = ia->write.out.size;  in fuse_send_write_pages()
1155 		if (ia->write.page_locked && (i == ap->num_pages - 1))  in fuse_send_write_pages()
1163 static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia,  in fuse_fill_write_pages()  argument
1168 	struct fuse_args_pages *ap = &ia->ap;  in fuse_fill_write_pages()
1226 			ia->write.page_locked = true;  in fuse_fill_write_pages()
1261 		struct fuse_io_args ia = {};  in fuse_perform_write()  local
1262 		struct fuse_args_pages *ap = &ia.ap;  in fuse_perform_write()
1272 		count = fuse_fill_write_pages(&ia, mapping, ii, pos, nr_pages);  in fuse_perform_write()
1276 			err = fuse_send_write_pages(&ia, iocb, inode,  in fuse_perform_write()
1279 				size_t num_written = ia.write.out.size;  in fuse_perform_write()
1442 	struct fuse_io_args *ia;  in fuse_direct_io()  local
1447 	ia = fuse_io_alloc(io, max_pages);  in fuse_direct_io()
1448 	if (!ia)  in fuse_direct_io()
1454 			fuse_io_free(ia);  in fuse_direct_io()
1469 			fuse_io_free(ia);  in fuse_direct_io()
1480 		err = fuse_get_user_pages(&ia->ap, iter, &nbytes, write,  in fuse_direct_io()
1487 				ia->write.in.write_flags |= FUSE_WRITE_KILL_SUIDGID;  in fuse_direct_io()
1489 			nres = fuse_send_write(ia, pos, nbytes, owner);  in fuse_direct_io()
1491 			nres = fuse_send_read(ia, pos, nbytes, owner);  in fuse_direct_io()
1495 			fuse_release_user_pages(&ia->ap, io->should_dirty);  in fuse_direct_io()
1496 			fuse_io_free(ia);  in fuse_direct_io()
1498 		ia = NULL;  in fuse_direct_io()
1515 			ia = fuse_io_alloc(io, max_pages);  in fuse_direct_io()
1516 			if (!ia)  in fuse_direct_io()
1520 	if (ia)  in fuse_direct_io()
1521 		fuse_io_free(ia);  in fuse_direct_io()
1657 	struct fuse_args_pages *ap = &wpa->ia.ap;  in fuse_writepage_free()
1666 	if (wpa->ia.ff)  in fuse_writepage_free()
1667 		fuse_file_put(wpa->ia.ff, false, false);  in fuse_writepage_free()
1676 	struct fuse_args_pages *ap = &wpa->ia.ap;  in fuse_writepage_finish()
1698 	struct fuse_write_in *inarg = &wpa->ia.write.in;  in fuse_send_writepage()
1699 	struct fuse_args *args = &wpa->ia.ap.args;  in fuse_send_writepage()
1700 	__u64 data_size = wpa->ia.ap.num_pages * PAGE_SIZE;  in fuse_send_writepage()
1773 	pgoff_t idx_from = wpa->ia.write.in.offset >> PAGE_SHIFT;  in fuse_insert_writeback()
1774 	pgoff_t idx_to = idx_from + wpa->ia.ap.num_pages - 1;  in fuse_insert_writeback()
1778 	WARN_ON(!wpa->ia.ap.num_pages);  in fuse_insert_writeback()
1787 		curr_index = curr->ia.write.in.offset >> PAGE_SHIFT;  in fuse_insert_writeback()
1789 		if (idx_from >= curr_index + curr->ia.ap.num_pages)  in fuse_insert_writeback()
1811 		container_of(args, typeof(*wpa), ia.ap.args);  in fuse_writepage_end()
1829 		struct fuse_write_in *inarg = &wpa->ia.write.in;  in fuse_writepage_end()
1834 		next->ia.ff = fuse_file_get(wpa->ia.ff);  in fuse_writepage_end()
1921 		ap = &wpa->ia.ap;  in fuse_writepage_args_alloc()
1963 	ap = &wpa->ia.ap;  in fuse_writepage_locked()
1970 	wpa->ia.ff = fuse_write_file_get(fi);  in fuse_writepage_locked()
1971 	if (!wpa->ia.ff)  in fuse_writepage_locked()
1975 	fuse_write_args_fill(&wpa->ia, wpa->ia.ff, page_offset(page), 0);  in fuse_writepage_locked()
1978 	wpa->ia.write.in.write_flags |= FUSE_WRITE_CACHE;  in fuse_writepage_locked()
2049 	struct fuse_args_pages *ap = &data->wpa->ia.ap;  in fuse_pages_realloc()
2078 	int num_pages = wpa->ia.ap.num_pages;  in fuse_writepages_send()
2081 	wpa->ia.ff = fuse_file_get(data->ff);  in fuse_writepages_send()
2104 	struct fuse_args_pages *new_ap = &new_wpa->ia.ap;  in fuse_writepage_add()
2120 		curr_index = tmp->ia.write.in.offset >> PAGE_SHIFT;  in fuse_writepage_add()
2122 			WARN_ON(tmp->ia.ap.num_pages != 1);  in fuse_writepage_add()
2123 			swap(tmp->ia.ap.pages[0], new_ap->pages[0]);  in fuse_writepage_add()
2186 	struct fuse_args_pages *ap = &wpa->ia.ap;  in fuse_writepages_fill()
2234 		ap = &wpa->ia.ap;  in fuse_writepages_fill()
2235 		fuse_write_args_fill(&wpa->ia, data->ff, folio_pos(folio), 0);  in fuse_writepages_fill()
2236 		wpa->ia.write.in.write_flags |= FUSE_WRITE_CACHE;  in fuse_writepages_fill()
2303 		WARN_ON(!data.wpa->ia.ap.num_pages);  in fuse_writepages()