37 static int io_iov_compat_buffer_select_prep(struct io_rw *rw)  in io_iov_compat_buffer_select_prep()  argument
42 	uiov = u64_to_user_ptr(rw->addr);  in io_iov_compat_buffer_select_prep()
50 	rw->len = clen;  in io_iov_compat_buffer_select_prep()
59 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_iov_buffer_select_prep()  local
61 	if (rw->len != 1)  in io_iov_buffer_select_prep()
66 		return io_iov_compat_buffer_select_prep(rw);  in io_iov_buffer_select_prep()
69 	uiov = u64_to_user_ptr(rw->addr);  in io_iov_buffer_select_prep()
72 	rw->len = iov.iov_len;  in io_iov_buffer_select_prep()
78 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_prep_rw()  local
82 	rw->kiocb.ki_pos = READ_ONCE(sqe->off);  in io_prep_rw()
104 		rw->kiocb.ki_ioprio = ioprio;  in io_prep_rw()
106 		rw->kiocb.ki_ioprio = get_current_ioprio();  in io_prep_rw()
108 	rw->kiocb.dio_complete = NULL;  in io_prep_rw()
110 	rw->addr = READ_ONCE(sqe->addr);  in io_prep_rw()
111 	rw->len = READ_ONCE(sqe->len);  in io_prep_rw()
112 	rw->flags = READ_ONCE(sqe->rw_flags);  in io_prep_rw()
156 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_kiocb_update_pos()  local
158 	if (rw->kiocb.ki_pos != -1)  in io_kiocb_update_pos()
159 		return &rw->kiocb.ki_pos;  in io_kiocb_update_pos()
163 		rw->kiocb.ki_pos = req->file->f_pos;  in io_kiocb_update_pos()
164 		return &rw->kiocb.ki_pos;  in io_kiocb_update_pos()
167 	rw->kiocb.ki_pos = 0;  in io_kiocb_update_pos()
227 		struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_req_end_write()  local
229 		kiocb_end_write(&rw->kiocb);  in io_req_end_write()
239 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_req_io_end()  local
241 	if (rw->kiocb.ki_flags & IOCB_WRITE) {  in io_req_io_end()
284 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_req_rw_complete()  local
285 	struct kiocb *kiocb = &rw->kiocb;  in io_req_rw_complete()
288 		long res = kiocb->dio_complete(rw->kiocb.private);  in io_req_rw_complete()
305 	struct io_rw *rw = container_of(kiocb, struct io_rw, kiocb);  in io_complete_rw()  local
306 	struct io_kiocb *req = cmd_to_io_kiocb(rw);  in io_complete_rw()
319 	struct io_rw *rw = container_of(kiocb, struct io_rw, kiocb);  in io_complete_rw_iopoll()  local
320 	struct io_kiocb *req = cmd_to_io_kiocb(rw);  in io_complete_rw_iopoll()
339 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in kiocb_done()  local
343 		req->file->f_pos = rw->kiocb.ki_pos;  in kiocb_done()
344 	if (ret >= 0 && (rw->kiocb.ki_complete == io_complete_rw)) {  in kiocb_done()
356 		io_rw_done(&rw->kiocb, ret);  in kiocb_done()
373 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in __io_import_iovec()  local
382 		ret = io_import_fixed(ddir, iter, req->imu, rw->addr, rw->len);  in __io_import_iovec()
388 	buf = u64_to_user_ptr(rw->addr);  in __io_import_iovec()
389 	sqe_len = rw->len;  in __io_import_iovec()
397 			rw->addr = (unsigned long) buf;  in __io_import_iovec()
398 			rw->len = sqe_len;  in __io_import_iovec()
415 static inline int io_import_iovec(int rw, struct io_kiocb *req,  in io_import_iovec()  argument
419 	*iovec = __io_import_iovec(rw, req, s, issue_flags);  in io_import_iovec()
436 static ssize_t loop_rw_iter(int ddir, struct io_rw *rw, struct iov_iter *iter)  in loop_rw_iter()  argument
438 	struct kiocb *kiocb = &rw->kiocb;  in loop_rw_iter()
468 			addr = u64_to_user_ptr(rw->addr);  in loop_rw_iter()
469 			len = rw->len;  in loop_rw_iter()
486 			rw->addr += nr;  in loop_rw_iter()
487 			rw->len -= nr;  in loop_rw_iter()
488 			if (!rw->len)  in loop_rw_iter()
546 static inline int io_rw_prep_async(struct io_kiocb *req, int rw)  in io_rw_prep_async()  argument
553 	ret = io_import_iovec(rw, req, &iov, &iorw->s, 0);  in io_rw_prep_async()
589 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_async_buf_func()  local
597 	rw->kiocb.ki_flags &= ~IOCB_WAITQ;  in io_async_buf_func()
619 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_rw_should_retry()  local
620 	struct kiocb *kiocb = &rw->kiocb;  in io_rw_should_retry()
647 static inline int io_iter_do_read(struct io_rw *rw, struct iov_iter *iter)  in io_iter_do_read()  argument
649 	struct file *file = rw->kiocb.ki_filp;  in io_iter_do_read()
652 		return call_read_iter(file, &rw->kiocb, iter);  in io_iter_do_read()
654 		return loop_rw_iter(READ, rw, iter);  in io_iter_do_read()
667 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_rw_init_file()  local
668 	struct kiocb *kiocb = &rw->kiocb;  in io_rw_init_file()
680 	ret = kiocb_set_rw_flags(kiocb, rw->flags);  in io_rw_init_file()
713 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_read()  local
716 	struct kiocb *kiocb = &rw->kiocb;  in io_read()
775 	ret = io_iter_do_read(rw, &s->iter);  in io_read()
845 		ret = io_iter_do_read(rw, &s->iter);  in io_read()
861 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_write()  local
864 	struct kiocb *kiocb = &rw->kiocb;  in io_write()
919 		ret2 = loop_rw_iter(WRITE, rw, &s->iter);  in io_write()
1025 			struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_do_iopoll()  local
1027 			ret = file->f_op->iopoll(&rw->kiocb, &iob, poll_flags);  in io_do_iopoll()