37 static int io_iov_compat_buffer_select_prep(struct io_rw *rw)  in io_iov_compat_buffer_select_prep()  argument
42 	uiov = u64_to_user_ptr(rw->addr);  in io_iov_compat_buffer_select_prep()
50 	rw->len = clen;  in io_iov_compat_buffer_select_prep()
59 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_iov_buffer_select_prep()  local
61 	if (rw->len != 1)  in io_iov_buffer_select_prep()
66 		return io_iov_compat_buffer_select_prep(rw);  in io_iov_buffer_select_prep()
69 	uiov = u64_to_user_ptr(rw->addr);  in io_iov_buffer_select_prep()
72 	rw->len = iov.iov_len;  in io_iov_buffer_select_prep()
78 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_prep_rw()  local
82 	rw->kiocb.ki_pos = READ_ONCE(sqe->off);  in io_prep_rw()
104 		rw->kiocb.ki_ioprio = ioprio;  in io_prep_rw()
106 		rw->kiocb.ki_ioprio = get_current_ioprio();  in io_prep_rw()
109 	rw->addr = READ_ONCE(sqe->addr);  in io_prep_rw()
110 	rw->len = READ_ONCE(sqe->len);  in io_prep_rw()
111 	rw->flags = READ_ONCE(sqe->rw_flags);  in io_prep_rw()
155 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_kiocb_update_pos()  local
157 	if (rw->kiocb.ki_pos != -1)  in io_kiocb_update_pos()
158 		return &rw->kiocb.ki_pos;  in io_kiocb_update_pos()
162 		rw->kiocb.ki_pos = req->file->f_pos;  in io_kiocb_update_pos()
163 		return &rw->kiocb.ki_pos;  in io_kiocb_update_pos()
166 	rw->kiocb.ki_pos = 0;  in io_kiocb_update_pos()
243 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_req_io_end()  local
245 	if (rw->kiocb.ki_flags & IOCB_WRITE) {  in io_req_io_end()
294 	struct io_rw *rw = container_of(kiocb, struct io_rw, kiocb);  in io_complete_rw()  local
295 	struct io_kiocb *req = cmd_to_io_kiocb(rw);  in io_complete_rw()
306 	struct io_rw *rw = container_of(kiocb, struct io_rw, kiocb);  in io_complete_rw_iopoll()  local
307 	struct io_kiocb *req = cmd_to_io_kiocb(rw);  in io_complete_rw_iopoll()
326 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in kiocb_done()  local
330 		req->file->f_pos = rw->kiocb.ki_pos;  in kiocb_done()
331 	if (ret >= 0 && (rw->kiocb.ki_complete == io_complete_rw)) {  in kiocb_done()
343 		io_rw_done(&rw->kiocb, ret);  in kiocb_done()
360 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in __io_import_iovec()  local
369 		ret = io_import_fixed(ddir, iter, req->imu, rw->addr, rw->len);  in __io_import_iovec()
375 	buf = u64_to_user_ptr(rw->addr);  in __io_import_iovec()
376 	sqe_len = rw->len;  in __io_import_iovec()
384 			rw->addr = (unsigned long) buf;  in __io_import_iovec()
385 			rw->len = sqe_len;  in __io_import_iovec()
402 static inline int io_import_iovec(int rw, struct io_kiocb *req,  in io_import_iovec()  argument
406 	*iovec = __io_import_iovec(rw, req, s, issue_flags);  in io_import_iovec()
423 static ssize_t loop_rw_iter(int ddir, struct io_rw *rw, struct iov_iter *iter)  in loop_rw_iter()  argument
425 	struct kiocb *kiocb = &rw->kiocb;  in loop_rw_iter()
450 			iovec.iov_base = u64_to_user_ptr(rw->addr);  in loop_rw_iter()
451 			iovec.iov_len = rw->len;  in loop_rw_iter()
471 			rw->addr += nr;  in loop_rw_iter()
472 			rw->len -= nr;  in loop_rw_iter()
473 			if (!rw->len)  in loop_rw_iter()
531 static inline int io_rw_prep_async(struct io_kiocb *req, int rw)  in io_rw_prep_async()  argument
538 	ret = io_import_iovec(rw, req, &iov, &iorw->s, 0);  in io_rw_prep_async()
574 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_async_buf_func()  local
582 	rw->kiocb.ki_flags &= ~IOCB_WAITQ;  in io_async_buf_func()
604 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_rw_should_retry()  local
605 	struct kiocb *kiocb = &rw->kiocb;  in io_rw_should_retry()
632 static inline int io_iter_do_read(struct io_rw *rw, struct iov_iter *iter)  in io_iter_do_read()  argument
634 	struct file *file = rw->kiocb.ki_filp;  in io_iter_do_read()
637 		return call_read_iter(file, &rw->kiocb, iter);  in io_iter_do_read()
639 		return loop_rw_iter(READ, rw, iter);  in io_iter_do_read()
652 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_rw_init_file()  local
653 	struct kiocb *kiocb = &rw->kiocb;  in io_rw_init_file()
665 	ret = kiocb_set_rw_flags(kiocb, rw->flags);  in io_rw_init_file()
697 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_read()  local
700 	struct kiocb *kiocb = &rw->kiocb;  in io_read()
759 	ret = io_iter_do_read(rw, &s->iter);  in io_read()
829 		ret = io_iter_do_read(rw, &s->iter);  in io_read()
845 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_write()  local
848 	struct kiocb *kiocb = &rw->kiocb;  in io_write()
913 		ret2 = loop_rw_iter(WRITE, rw, &s->iter);  in io_write()
1026 			struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_do_iopoll()  local
1028 			ret = file->f_op->iopoll(&rw->kiocb, &iob, poll_flags);  in io_do_iopoll()