396 	struct fuse_io_args ia;  member
417 		curr_index = wpa->ia.write.in.offset >> PAGE_SHIFT;  in fuse_find_writeback()
418 		if (idx_from >= curr_index + wpa->ia.ap.num_pages)  in fuse_find_writeback()
607 void fuse_read_args_fill(struct fuse_io_args *ia, struct file *file, loff_t pos,  in fuse_read_args_fill()  argument
611 	struct fuse_args *args = &ia->ap.args;  in fuse_read_args_fill()
613 	ia->read.in.fh = ff->fh;  in fuse_read_args_fill()
614 	ia->read.in.offset = pos;  in fuse_read_args_fill()
615 	ia->read.in.size = count;  in fuse_read_args_fill()
616 	ia->read.in.flags = file->f_flags;  in fuse_read_args_fill()
620 	args->in_args[0].size = sizeof(ia->read.in);  in fuse_read_args_fill()
621 	args->in_args[0].value = &ia->read.in;  in fuse_read_args_fill()
708 	struct fuse_io_args *ia;  in fuse_io_alloc()  local
710 	ia = kzalloc(sizeof(*ia), GFP_KERNEL);  in fuse_io_alloc()
711 	if (ia) {  in fuse_io_alloc()
712 		ia->io = io;  in fuse_io_alloc()
713 		ia->ap.pages = fuse_pages_alloc(npages, GFP_KERNEL,  in fuse_io_alloc()
714 						&ia->ap.descs);  in fuse_io_alloc()
715 		if (!ia->ap.pages) {  in fuse_io_alloc()
716 			kfree(ia);  in fuse_io_alloc()
717 			ia = NULL;  in fuse_io_alloc()
720 	return ia;  in fuse_io_alloc()
723 static void fuse_io_free(struct fuse_io_args *ia)  in fuse_io_free()  argument
725 	kfree(ia->ap.pages);  in fuse_io_free()
726 	kfree(ia);  in fuse_io_free()
732 	struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args);  in fuse_aio_complete_req()  local
733 	struct fuse_io_priv *io = ia->io;  in fuse_aio_complete_req()
736 	fuse_release_user_pages(&ia->ap, io->should_dirty);  in fuse_aio_complete_req()
741 		if (ia->write.out.size > ia->write.in.size) {  in fuse_aio_complete_req()
743 		} else if (ia->write.in.size != ia->write.out.size) {  in fuse_aio_complete_req()
744 			pos = ia->write.in.offset - io->offset +  in fuse_aio_complete_req()
745 				ia->write.out.size;  in fuse_aio_complete_req()
750 		if (ia->read.in.size != outsize)  in fuse_aio_complete_req()
751 			pos = ia->read.in.offset - io->offset + outsize;  in fuse_aio_complete_req()
755 	fuse_io_free(ia);  in fuse_aio_complete_req()
759 				   struct fuse_io_args *ia, size_t num_bytes)  in fuse_async_req_send()  argument
762 	struct fuse_io_priv *io = ia->io;  in fuse_async_req_send()
770 	ia->ap.args.end = fuse_aio_complete_req;  in fuse_async_req_send()
771 	ia->ap.args.may_block = io->should_dirty;  in fuse_async_req_send()
772 	err = fuse_simple_background(fm, &ia->ap.args, GFP_KERNEL);  in fuse_async_req_send()
774 		fuse_aio_complete_req(fm, &ia->ap.args, err);  in fuse_async_req_send()
779 static ssize_t fuse_send_read(struct fuse_io_args *ia, loff_t pos, size_t count,  in fuse_send_read()  argument
782 	struct file *file = ia->io->iocb->ki_filp;  in fuse_send_read()
786 	fuse_read_args_fill(ia, file, pos, count, FUSE_READ);  in fuse_send_read()
788 		ia->read.in.read_flags |= FUSE_READ_LOCKOWNER;  in fuse_send_read()
789 		ia->read.in.lock_owner = fuse_lock_owner_id(fm->fc, owner);  in fuse_send_read()
792 	if (ia->io->async)  in fuse_send_read()
793 		return fuse_async_req_send(fm, ia, count);  in fuse_send_read()
795 	return fuse_simple_request(fm, &ia->ap.args);  in fuse_send_read()
835 	struct fuse_io_args ia = {  in fuse_do_readpage()  local
858 	fuse_read_args_fill(&ia, file, pos, desc.length, FUSE_READ);  in fuse_do_readpage()
859 	res = fuse_simple_request(fm, &ia.ap.args);  in fuse_do_readpage()
866 		fuse_short_read(inode, attr_ver, res, &ia.ap);  in fuse_do_readpage()
894 	struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args);  in fuse_readpages_end()  local
895 	struct fuse_args_pages *ap = &ia->ap;  in fuse_readpages_end()
896 	size_t count = ia->read.in.size;  in fuse_readpages_end()
910 			fuse_short_read(inode, ia->read.attr_ver, num_read, ap);  in fuse_readpages_end()
925 	if (ia->ff)  in fuse_readpages_end()
926 		fuse_file_put(ia->ff, false, false);  in fuse_readpages_end()
928 	fuse_io_free(ia);  in fuse_readpages_end()
931 static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file)  in fuse_send_readpages()  argument
935 	struct fuse_args_pages *ap = &ia->ap;  in fuse_send_readpages()
952 	fuse_read_args_fill(ia, file, pos, count, FUSE_READ);  in fuse_send_readpages()
953 	ia->read.attr_ver = fuse_get_attr_version(fm->fc);  in fuse_send_readpages()
955 		ia->ff = fuse_file_get(ff);  in fuse_send_readpages()
980 		struct fuse_io_args *ia;  in fuse_readahead()  local
996 		ia = fuse_io_alloc(NULL, nr_pages);  in fuse_readahead()
997 		if (!ia)  in fuse_readahead()
999 		ap = &ia->ap;  in fuse_readahead()
1007 		fuse_send_readpages(ia, rac->file);  in fuse_readahead()
1032 static void fuse_write_args_fill(struct fuse_io_args *ia, struct fuse_file *ff,  in fuse_write_args_fill()  argument
1035 	struct fuse_args *args = &ia->ap.args;  in fuse_write_args_fill()
1037 	ia->write.in.fh = ff->fh;  in fuse_write_args_fill()
1038 	ia->write.in.offset = pos;  in fuse_write_args_fill()
1039 	ia->write.in.size = count;  in fuse_write_args_fill()
1046 		args->in_args[0].size = sizeof(ia->write.in);  in fuse_write_args_fill()
1047 	args->in_args[0].value = &ia->write.in;  in fuse_write_args_fill()
1050 	args->out_args[0].size = sizeof(ia->write.out);  in fuse_write_args_fill()
1051 	args->out_args[0].value = &ia->write.out;  in fuse_write_args_fill()
1066 static ssize_t fuse_send_write(struct fuse_io_args *ia, loff_t pos,  in fuse_send_write()  argument
1069 	struct kiocb *iocb = ia->io->iocb;  in fuse_send_write()
1073 	struct fuse_write_in *inarg = &ia->write.in;  in fuse_send_write()
1076 	fuse_write_args_fill(ia, ff, pos, count);  in fuse_send_write()
1083 	if (ia->io->async)  in fuse_send_write()
1084 		return fuse_async_req_send(fm, ia, count);  in fuse_send_write()
1086 	err = fuse_simple_request(fm, &ia->ap.args);  in fuse_send_write()
1087 	if (!err && ia->write.out.size > count)  in fuse_send_write()
1090 	return err ?: ia->write.out.size;  in fuse_send_write()
1112 static ssize_t fuse_send_write_pages(struct fuse_io_args *ia,  in fuse_send_write_pages()  argument
1116 	struct fuse_args_pages *ap = &ia->ap;  in fuse_send_write_pages()
1127 	fuse_write_args_fill(ia, ff, pos, count);  in fuse_send_write_pages()
1128 	ia->write.in.flags = fuse_write_flags(iocb);  in fuse_send_write_pages()
1130 		ia->write.in.write_flags |= FUSE_WRITE_KILL_SUIDGID;  in fuse_send_write_pages()
1133 	if (!err && ia->write.out.size > count)  in fuse_send_write_pages()
1136 	short_write = ia->write.out.size < count;  in fuse_send_write_pages()
1138 	count = ia->write.out.size;  in fuse_send_write_pages()
1154 		if (ia->write.page_locked && (i == ap->num_pages - 1))  in fuse_send_write_pages()
1162 static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia,  in fuse_fill_write_pages()  argument
1167 	struct fuse_args_pages *ap = &ia->ap;  in fuse_fill_write_pages()
1225 			ia->write.page_locked = true;  in fuse_fill_write_pages()
1260 		struct fuse_io_args ia = {};  in fuse_perform_write()  local
1261 		struct fuse_args_pages *ap = &ia.ap;  in fuse_perform_write()
1271 		count = fuse_fill_write_pages(&ia, mapping, ii, pos, nr_pages);  in fuse_perform_write()
1275 			err = fuse_send_write_pages(&ia, iocb, inode,  in fuse_perform_write()
1278 				size_t num_written = ia.write.out.size;  in fuse_perform_write()
1464 	struct fuse_io_args *ia;  in fuse_direct_io()  local
1468 	ia = fuse_io_alloc(io, max_pages);  in fuse_direct_io()
1469 	if (!ia)  in fuse_direct_io()
1486 		err = fuse_get_user_pages(&ia->ap, iter, &nbytes, write,  in fuse_direct_io()
1493 				ia->write.in.write_flags |= FUSE_WRITE_KILL_SUIDGID;  in fuse_direct_io()
1495 			nres = fuse_send_write(ia, pos, nbytes, owner);  in fuse_direct_io()
1497 			nres = fuse_send_read(ia, pos, nbytes, owner);  in fuse_direct_io()
1501 			fuse_release_user_pages(&ia->ap, io->should_dirty);  in fuse_direct_io()
1502 			fuse_io_free(ia);  in fuse_direct_io()
1504 		ia = NULL;  in fuse_direct_io()
1521 			ia = fuse_io_alloc(io, max_pages);  in fuse_direct_io()
1522 			if (!ia)  in fuse_direct_io()
1526 	if (ia)  in fuse_direct_io()
1527 		fuse_io_free(ia);  in fuse_direct_io()
1627 	struct fuse_args_pages *ap = &wpa->ia.ap;  in fuse_writepage_free()
1636 	if (wpa->ia.ff)  in fuse_writepage_free()
1637 		fuse_file_put(wpa->ia.ff, false, false);  in fuse_writepage_free()
1646 	struct fuse_args_pages *ap = &wpa->ia.ap;  in fuse_writepage_finish()
1668 	struct fuse_write_in *inarg = &wpa->ia.write.in;  in fuse_send_writepage()
1669 	struct fuse_args *args = &wpa->ia.ap.args;  in fuse_send_writepage()
1670 	__u64 data_size = wpa->ia.ap.num_pages * PAGE_SIZE;  in fuse_send_writepage()
1743 	pgoff_t idx_from = wpa->ia.write.in.offset >> PAGE_SHIFT;  in fuse_insert_writeback()
1744 	pgoff_t idx_to = idx_from + wpa->ia.ap.num_pages - 1;  in fuse_insert_writeback()
1748 	WARN_ON(!wpa->ia.ap.num_pages);  in fuse_insert_writeback()
1757 		curr_index = curr->ia.write.in.offset >> PAGE_SHIFT;  in fuse_insert_writeback()
1759 		if (idx_from >= curr_index + curr->ia.ap.num_pages)  in fuse_insert_writeback()
1781 		container_of(args, typeof(*wpa), ia.ap.args);  in fuse_writepage_end()
1799 		struct fuse_write_in *inarg = &wpa->ia.write.in;  in fuse_writepage_end()
1804 		next->ia.ff = fuse_file_get(wpa->ia.ff);  in fuse_writepage_end()
1891 		ap = &wpa->ia.ap;  in fuse_writepage_args_alloc()
1933 	ap = &wpa->ia.ap;  in fuse_writepage_locked()
1940 	wpa->ia.ff = fuse_write_file_get(fi);  in fuse_writepage_locked()
1941 	if (!wpa->ia.ff)  in fuse_writepage_locked()
1945 	fuse_write_args_fill(&wpa->ia, wpa->ia.ff, page_offset(page), 0);  in fuse_writepage_locked()
1948 	wpa->ia.write.in.write_flags |= FUSE_WRITE_CACHE;  in fuse_writepage_locked()
2019 	struct fuse_args_pages *ap = &data->wpa->ia.ap;  in fuse_pages_realloc()
2048 	int num_pages = wpa->ia.ap.num_pages;  in fuse_writepages_send()
2051 	wpa->ia.ff = fuse_file_get(data->ff);  in fuse_writepages_send()
2074 	struct fuse_args_pages *new_ap = &new_wpa->ia.ap;  in fuse_writepage_add()
2090 		curr_index = tmp->ia.write.in.offset >> PAGE_SHIFT;  in fuse_writepage_add()
2092 			WARN_ON(tmp->ia.ap.num_pages != 1);  in fuse_writepage_add()
2093 			swap(tmp->ia.ap.pages[0], new_ap->pages[0]);  in fuse_writepage_add()
2156 	struct fuse_args_pages *ap = &wpa->ia.ap;  in fuse_writepages_fill()
2204 		ap = &wpa->ia.ap;  in fuse_writepages_fill()
2205 		fuse_write_args_fill(&wpa->ia, data->ff, page_offset(page), 0);  in fuse_writepages_fill()
2206 		wpa->ia.write.in.write_flags |= FUSE_WRITE_CACHE;  in fuse_writepages_fill()
2273 		WARN_ON(!data.wpa->ia.ap.num_pages);  in fuse_writepages()