374 	struct fuse_io_args ia;  member
394 		curr_index = wpa->ia.write.in.offset >> PAGE_SHIFT;  in fuse_find_writeback()
395 		if (idx_from >= curr_index + wpa->ia.ap.num_pages)  in fuse_find_writeback()
581 void fuse_read_args_fill(struct fuse_io_args *ia, struct file *file, loff_t pos,  in fuse_read_args_fill()  argument
585 	struct fuse_args *args = &ia->ap.args;  in fuse_read_args_fill()
587 	ia->read.in.fh = ff->fh;  in fuse_read_args_fill()
588 	ia->read.in.offset = pos;  in fuse_read_args_fill()
589 	ia->read.in.size = count;  in fuse_read_args_fill()
590 	ia->read.in.flags = file->f_flags;  in fuse_read_args_fill()
594 	args->in_args[0].size = sizeof(ia->read.in);  in fuse_read_args_fill()
595 	args->in_args[0].value = &ia->read.in;  in fuse_read_args_fill()
682 	struct fuse_io_args *ia;  in fuse_io_alloc()  local
684 	ia = kzalloc(sizeof(*ia), GFP_KERNEL);  in fuse_io_alloc()
685 	if (ia) {  in fuse_io_alloc()
686 		ia->io = io;  in fuse_io_alloc()
687 		ia->ap.pages = fuse_pages_alloc(npages, GFP_KERNEL,  in fuse_io_alloc()
688 						&ia->ap.descs);  in fuse_io_alloc()
689 		if (!ia->ap.pages) {  in fuse_io_alloc()
690 			kfree(ia);  in fuse_io_alloc()
691 			ia = NULL;  in fuse_io_alloc()
694 	return ia;  in fuse_io_alloc()
697 static void fuse_io_free(struct fuse_io_args *ia)  in fuse_io_free()  argument
699 	kfree(ia->ap.pages);  in fuse_io_free()
700 	kfree(ia);  in fuse_io_free()
706 	struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args);  in fuse_aio_complete_req()  local
707 	struct fuse_io_priv *io = ia->io;  in fuse_aio_complete_req()
710 	fuse_release_user_pages(&ia->ap, io->should_dirty);  in fuse_aio_complete_req()
715 		if (ia->write.out.size > ia->write.in.size) {  in fuse_aio_complete_req()
717 		} else if (ia->write.in.size != ia->write.out.size) {  in fuse_aio_complete_req()
718 			pos = ia->write.in.offset - io->offset +  in fuse_aio_complete_req()
719 				ia->write.out.size;  in fuse_aio_complete_req()
724 		if (ia->read.in.size != outsize)  in fuse_aio_complete_req()
725 			pos = ia->read.in.offset - io->offset + outsize;  in fuse_aio_complete_req()
729 	fuse_io_free(ia);  in fuse_aio_complete_req()
733 				   struct fuse_io_args *ia, size_t num_bytes)  in fuse_async_req_send()  argument
736 	struct fuse_io_priv *io = ia->io;  in fuse_async_req_send()
744 	ia->ap.args.end = fuse_aio_complete_req;  in fuse_async_req_send()
745 	ia->ap.args.may_block = io->should_dirty;  in fuse_async_req_send()
746 	err = fuse_simple_background(fm, &ia->ap.args, GFP_KERNEL);  in fuse_async_req_send()
748 		fuse_aio_complete_req(fm, &ia->ap.args, err);  in fuse_async_req_send()
753 static ssize_t fuse_send_read(struct fuse_io_args *ia, loff_t pos, size_t count,  in fuse_send_read()  argument
756 	struct file *file = ia->io->iocb->ki_filp;  in fuse_send_read()
760 	fuse_read_args_fill(ia, file, pos, count, FUSE_READ);  in fuse_send_read()
762 		ia->read.in.read_flags |= FUSE_READ_LOCKOWNER;  in fuse_send_read()
763 		ia->read.in.lock_owner = fuse_lock_owner_id(fm->fc, owner);  in fuse_send_read()
766 	if (ia->io->async)  in fuse_send_read()
767 		return fuse_async_req_send(fm, ia, count);  in fuse_send_read()
769 	return fuse_simple_request(fm, &ia->ap.args);  in fuse_send_read()
818 	struct fuse_io_args ia = {  in fuse_do_readpage()  local
841 	fuse_read_args_fill(&ia, file, pos, desc.length, FUSE_READ);  in fuse_do_readpage()
842 	res = fuse_simple_request(fm, &ia.ap.args);  in fuse_do_readpage()
849 		fuse_short_read(inode, attr_ver, res, &ia.ap);  in fuse_do_readpage()
876 	struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args);  in fuse_readpages_end()  local
877 	struct fuse_args_pages *ap = &ia->ap;  in fuse_readpages_end()
878 	size_t count = ia->read.in.size;  in fuse_readpages_end()
892 			fuse_short_read(inode, ia->read.attr_ver, num_read, ap);  in fuse_readpages_end()
907 	if (ia->ff)  in fuse_readpages_end()
908 		fuse_file_put(ia->ff, false, false);  in fuse_readpages_end()
910 	fuse_io_free(ia);  in fuse_readpages_end()
913 static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file)  in fuse_send_readpages()  argument
917 	struct fuse_args_pages *ap = &ia->ap;  in fuse_send_readpages()
934 	fuse_read_args_fill(ia, file, pos, count, FUSE_READ);  in fuse_send_readpages()
935 	ia->read.attr_ver = fuse_get_attr_version(fm->fc);  in fuse_send_readpages()
937 		ia->ff = fuse_file_get(ff);  in fuse_send_readpages()
962 		struct fuse_io_args *ia;  in fuse_readahead()  local
970 		ia = fuse_io_alloc(NULL, nr_pages);  in fuse_readahead()
971 		if (!ia)  in fuse_readahead()
973 		ap = &ia->ap;  in fuse_readahead()
981 		fuse_send_readpages(ia, rac->file);  in fuse_readahead()
1006 static void fuse_write_args_fill(struct fuse_io_args *ia, struct fuse_file *ff,  in fuse_write_args_fill()  argument
1009 	struct fuse_args *args = &ia->ap.args;  in fuse_write_args_fill()
1011 	ia->write.in.fh = ff->fh;  in fuse_write_args_fill()
1012 	ia->write.in.offset = pos;  in fuse_write_args_fill()
1013 	ia->write.in.size = count;  in fuse_write_args_fill()
1020 		args->in_args[0].size = sizeof(ia->write.in);  in fuse_write_args_fill()
1021 	args->in_args[0].value = &ia->write.in;  in fuse_write_args_fill()
1024 	args->out_args[0].size = sizeof(ia->write.out);  in fuse_write_args_fill()
1025 	args->out_args[0].value = &ia->write.out;  in fuse_write_args_fill()
1040 static ssize_t fuse_send_write(struct fuse_io_args *ia, loff_t pos,  in fuse_send_write()  argument
1043 	struct kiocb *iocb = ia->io->iocb;  in fuse_send_write()
1047 	struct fuse_write_in *inarg = &ia->write.in;  in fuse_send_write()
1050 	fuse_write_args_fill(ia, ff, pos, count);  in fuse_send_write()
1057 	if (ia->io->async)  in fuse_send_write()
1058 		return fuse_async_req_send(fm, ia, count);  in fuse_send_write()
1060 	err = fuse_simple_request(fm, &ia->ap.args);  in fuse_send_write()
1061 	if (!err && ia->write.out.size > count)  in fuse_send_write()
1064 	return err ?: ia->write.out.size;  in fuse_send_write()
1084 static ssize_t fuse_send_write_pages(struct fuse_io_args *ia,  in fuse_send_write_pages()  argument
1088 	struct fuse_args_pages *ap = &ia->ap;  in fuse_send_write_pages()
1098 	fuse_write_args_fill(ia, ff, pos, count);  in fuse_send_write_pages()
1099 	ia->write.in.flags = fuse_write_flags(iocb);  in fuse_send_write_pages()
1102 	if (!err && ia->write.out.size > count)  in fuse_send_write_pages()
1106 	count = ia->write.out.size;  in fuse_send_write_pages()
1215 		struct fuse_io_args ia = {};  in fuse_perform_write()  local
1216 		struct fuse_args_pages *ap = &ia.ap;  in fuse_perform_write()
1230 			err = fuse_send_write_pages(&ia, iocb, inode,  in fuse_perform_write()
1233 				size_t num_written = ia.write.out.size;  in fuse_perform_write()
1424 	struct fuse_io_args *ia;  in fuse_direct_io()  local
1428 	ia = fuse_io_alloc(io, max_pages);  in fuse_direct_io()
1429 	if (!ia)  in fuse_direct_io()
1432 	ia->io = io;  in fuse_direct_io()
1447 		err = fuse_get_user_pages(&ia->ap, iter, &nbytes, write,  in fuse_direct_io()
1454 				ia->write.in.write_flags |= FUSE_WRITE_KILL_PRIV;  in fuse_direct_io()
1456 			nres = fuse_send_write(ia, pos, nbytes, owner);  in fuse_direct_io()
1458 			nres = fuse_send_read(ia, pos, nbytes, owner);  in fuse_direct_io()
1462 			fuse_release_user_pages(&ia->ap, io->should_dirty);  in fuse_direct_io()
1463 			fuse_io_free(ia);  in fuse_direct_io()
1465 		ia = NULL;  in fuse_direct_io()
1482 			ia = fuse_io_alloc(io, max_pages);  in fuse_direct_io()
1483 			if (!ia)  in fuse_direct_io()
1487 	if (ia)  in fuse_direct_io()
1488 		fuse_io_free(ia);  in fuse_direct_io()
1590 	struct fuse_args_pages *ap = &wpa->ia.ap;  in fuse_writepage_free()
1596 	if (wpa->ia.ff)  in fuse_writepage_free()
1597 		fuse_file_put(wpa->ia.ff, false, false);  in fuse_writepage_free()
1606 	struct fuse_args_pages *ap = &wpa->ia.ap;  in fuse_writepage_finish()
1628 	struct fuse_write_in *inarg = &wpa->ia.write.in;  in fuse_send_writepage()
1629 	struct fuse_args *args = &wpa->ia.ap.args;  in fuse_send_writepage()
1630 	__u64 data_size = wpa->ia.ap.num_pages * PAGE_SIZE;  in fuse_send_writepage()
1703 	pgoff_t idx_from = wpa->ia.write.in.offset >> PAGE_SHIFT;  in fuse_insert_writeback()
1704 	pgoff_t idx_to = idx_from + wpa->ia.ap.num_pages - 1;  in fuse_insert_writeback()
1708 	WARN_ON(!wpa->ia.ap.num_pages);  in fuse_insert_writeback()
1717 		curr_index = curr->ia.write.in.offset >> PAGE_SHIFT;  in fuse_insert_writeback()
1719 		if (idx_from >= curr_index + curr->ia.ap.num_pages)  in fuse_insert_writeback()
1741 		container_of(args, typeof(*wpa), ia.ap.args);  in fuse_writepage_end()
1750 		struct fuse_write_in *inarg = &wpa->ia.write.in;  in fuse_writepage_end()
1755 		next->ia.ff = fuse_file_get(wpa->ia.ff);  in fuse_writepage_end()
1835 		ap = &wpa->ia.ap;  in fuse_writepage_args_alloc()
1863 	ap = &wpa->ia.ap;  in fuse_writepage_locked()
1870 	wpa->ia.ff = fuse_write_file_get(fc, fi);  in fuse_writepage_locked()
1871 	if (!wpa->ia.ff)  in fuse_writepage_locked()
1874 	fuse_write_args_fill(&wpa->ia, wpa->ia.ff, page_offset(page), 0);  in fuse_writepage_locked()
1877 	wpa->ia.write.in.write_flags |= FUSE_WRITE_CACHE;  in fuse_writepage_locked()
1943 	struct fuse_args_pages *ap = &data->wpa->ia.ap;  in fuse_pages_realloc()
1972 	int num_pages = wpa->ia.ap.num_pages;  in fuse_writepages_send()
1975 	wpa->ia.ff = fuse_file_get(data->ff);  in fuse_writepages_send()
1998 	struct fuse_args_pages *new_ap = &new_wpa->ia.ap;  in fuse_writepage_add()
2014 		curr_index = tmp->ia.write.in.offset >> PAGE_SHIFT;  in fuse_writepage_add()
2016 			WARN_ON(tmp->ia.ap.num_pages != 1);  in fuse_writepage_add()
2017 			swap(tmp->ia.ap.pages[0], new_ap->pages[0]);  in fuse_writepage_add()
2080 	struct fuse_args_pages *ap = &wpa->ia.ap;  in fuse_writepages_fill()
2126 		ap = &wpa->ia.ap;  in fuse_writepages_fill()
2127 		fuse_write_args_fill(&wpa->ia, data->ff, page_offset(page), 0);  in fuse_writepages_fill()
2128 		wpa->ia.write.in.write_flags |= FUSE_WRITE_CACHE;  in fuse_writepages_fill()
2191 		WARN_ON(!data.wpa->ia.ap.num_pages);  in fuse_writepages()