31 static inline bool io_file_supports_nowait(struct io_kiocb *req)  in io_file_supports_nowait()  argument
33 	return req->flags & REQ_F_SUPPORT_NOWAIT;  in io_file_supports_nowait()
55 static int io_iov_buffer_select_prep(struct io_kiocb *req)  in io_iov_buffer_select_prep()  argument
59 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_iov_buffer_select_prep()
65 	if (req->ctx->compat)  in io_iov_buffer_select_prep()
76 int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_prep_rw()  argument
78 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_prep_rw()
84 	req->buf_index = READ_ONCE(sqe->buf_index);  in io_prep_rw()
86 	if (req->opcode == IORING_OP_READ_FIXED ||  in io_prep_rw()
87 	    req->opcode == IORING_OP_WRITE_FIXED) {  in io_prep_rw()
88 		struct io_ring_ctx *ctx = req->ctx;  in io_prep_rw()
91 		if (unlikely(req->buf_index >= ctx->nr_user_bufs))  in io_prep_rw()
93 		index = array_index_nospec(req->buf_index, ctx->nr_user_bufs);  in io_prep_rw()
94 		req->imu = ctx->user_bufs[index];  in io_prep_rw()
95 		io_req_set_rsrc_node(req, ctx, 0);  in io_prep_rw()
116 	if (req->opcode == IORING_OP_READV && req->flags & REQ_F_BUFFER_SELECT) {  in io_prep_rw()
117 		ret = io_iov_buffer_select_prep(req);  in io_prep_rw()
125 void io_readv_writev_cleanup(struct io_kiocb *req)  in io_readv_writev_cleanup()  argument
127 	struct io_async_rw *io = req->async_data;  in io_readv_writev_cleanup()
153 static inline loff_t *io_kiocb_update_pos(struct io_kiocb *req)  in io_kiocb_update_pos()  argument
155 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_kiocb_update_pos()
160 	if (!(req->file->f_mode & FMODE_STREAM)) {  in io_kiocb_update_pos()
161 		req->flags |= REQ_F_CUR_POS;  in io_kiocb_update_pos()
162 		rw->kiocb.ki_pos = req->file->f_pos;  in io_kiocb_update_pos()
170 static void io_req_task_queue_reissue(struct io_kiocb *req)  in io_req_task_queue_reissue()  argument
172 	req->io_task_work.func = io_queue_iowq;  in io_req_task_queue_reissue()
173 	io_req_task_work_add(req);  in io_req_task_queue_reissue()
177 static bool io_resubmit_prep(struct io_kiocb *req)  in io_resubmit_prep()  argument
179 	struct io_async_rw *io = req->async_data;  in io_resubmit_prep()
181 	if (!req_has_async_data(req))  in io_resubmit_prep()
182 		return !io_req_prep_async(req);  in io_resubmit_prep()
187 static bool io_rw_should_reissue(struct io_kiocb *req)  in io_rw_should_reissue()  argument
189 	umode_t mode = file_inode(req->file)->i_mode;  in io_rw_should_reissue()
190 	struct io_ring_ctx *ctx = req->ctx;  in io_rw_should_reissue()
194 	if ((req->flags & REQ_F_NOWAIT) || (io_wq_current_is_worker() &&  in io_rw_should_reissue()
208 	if (!same_thread_group(req->task, current) || !in_task())  in io_rw_should_reissue()
213 static bool io_resubmit_prep(struct io_kiocb *req)  in io_resubmit_prep()  argument
217 static bool io_rw_should_reissue(struct io_kiocb *req)  in io_rw_should_reissue()  argument
223 static void kiocb_end_write(struct io_kiocb *req)  in kiocb_end_write()  argument
229 	if (req->flags & REQ_F_ISREG) {  in kiocb_end_write()
230 		struct super_block *sb = file_inode(req->file)->i_sb;  in kiocb_end_write()
241 static void io_req_io_end(struct io_kiocb *req)  in io_req_io_end()  argument
243 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_req_io_end()
246 		kiocb_end_write(req);  in io_req_io_end()
247 		fsnotify_modify(req->file);  in io_req_io_end()
249 		fsnotify_access(req->file);  in io_req_io_end()
253 static bool __io_complete_rw_common(struct io_kiocb *req, long res)  in __io_complete_rw_common()  argument
255 	if (unlikely(res != req->cqe.res)) {  in __io_complete_rw_common()
257 		    io_rw_should_reissue(req)) {  in __io_complete_rw_common()
262 			io_req_io_end(req);  in __io_complete_rw_common()
263 			req->flags |= REQ_F_REISSUE | REQ_F_PARTIAL_IO;  in __io_complete_rw_common()
266 		req_set_fail(req);  in __io_complete_rw_common()
267 		req->cqe.res = res;  in __io_complete_rw_common()
272 static inline int io_fixup_rw_res(struct io_kiocb *req, long res)  in io_fixup_rw_res()  argument
274 	struct io_async_rw *io = req->async_data;  in io_fixup_rw_res()
277 	if (req_has_async_data(req) && io->bytes_done > 0) {  in io_fixup_rw_res()
286 static void io_req_rw_complete(struct io_kiocb *req, bool *locked)  in io_req_rw_complete()  argument
288 	io_req_io_end(req);  in io_req_rw_complete()
289 	io_req_task_complete(req, locked);  in io_req_rw_complete()
295 	struct io_kiocb *req = cmd_to_io_kiocb(rw);  in io_complete_rw()  local
297 	if (__io_complete_rw_common(req, res))  in io_complete_rw()
299 	io_req_set_res(req, io_fixup_rw_res(req, res), 0);  in io_complete_rw()
300 	req->io_task_work.func = io_req_rw_complete;  in io_complete_rw()
301 	io_req_task_work_add(req);  in io_complete_rw()
307 	struct io_kiocb *req = cmd_to_io_kiocb(rw);  in io_complete_rw_iopoll()  local
310 		kiocb_end_write(req);  in io_complete_rw_iopoll()
311 	if (unlikely(res != req->cqe.res)) {  in io_complete_rw_iopoll()
312 		if (res == -EAGAIN && io_rw_should_reissue(req)) {  in io_complete_rw_iopoll()
313 			req->flags |= REQ_F_REISSUE | REQ_F_PARTIAL_IO;  in io_complete_rw_iopoll()
316 		req->cqe.res = res;  in io_complete_rw_iopoll()
320 	smp_store_release(&req->iopoll_completed, 1);  in io_complete_rw_iopoll()
323 static int kiocb_done(struct io_kiocb *req, ssize_t ret,  in kiocb_done()  argument
326 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in kiocb_done()
327 	unsigned final_ret = io_fixup_rw_res(req, ret);  in kiocb_done()
329 	if (req->flags & REQ_F_CUR_POS)  in kiocb_done()
330 		req->file->f_pos = rw->kiocb.ki_pos;  in kiocb_done()
332 		if (!__io_complete_rw_common(req, ret)) {  in kiocb_done()
337 			io_req_io_end(req);  in kiocb_done()
338 			io_req_set_res(req, final_ret,  in kiocb_done()
339 				       io_put_kbuf(req, issue_flags));  in kiocb_done()
346 	if (req->flags & REQ_F_REISSUE) {  in kiocb_done()
347 		req->flags &= ~REQ_F_REISSUE;  in kiocb_done()
348 		if (io_resubmit_prep(req))  in kiocb_done()
349 			io_req_task_queue_reissue(req);  in kiocb_done()
351 			io_req_task_queue_fail(req, final_ret);  in kiocb_done()
356 static struct iovec *__io_import_iovec(int ddir, struct io_kiocb *req,  in __io_import_iovec()  argument
360 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in __io_import_iovec()
362 	u8 opcode = req->opcode;  in __io_import_iovec()
369 		ret = io_import_fixed(ddir, iter, req->imu, rw->addr, rw->len);  in __io_import_iovec()
379 	    (req->flags & REQ_F_BUFFER_SELECT)) {  in __io_import_iovec()
380 		if (io_do_buffer_select(req)) {  in __io_import_iovec()
381 			buf = io_buffer_select(req, &sqe_len, issue_flags);  in __io_import_iovec()
396 			      req->ctx->compat);  in __io_import_iovec()
402 static inline int io_import_iovec(int rw, struct io_kiocb *req,  in io_import_iovec()  argument
406 	*iovec = __io_import_iovec(rw, req, s, issue_flags);  in io_import_iovec()
483 static void io_req_map_rw(struct io_kiocb *req, const struct iovec *iovec,  in io_req_map_rw()  argument
486 	struct io_async_rw *io = req->async_data;  in io_req_map_rw()
506 		req->flags |= REQ_F_NEED_CLEANUP;  in io_req_map_rw()
510 static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec,  in io_setup_async_rw()  argument
513 	if (!force && !io_op_defs[req->opcode].prep_async)  in io_setup_async_rw()
515 	if (!req_has_async_data(req)) {  in io_setup_async_rw()
518 		if (io_alloc_async_data(req)) {  in io_setup_async_rw()
523 		io_req_map_rw(req, iovec, s->fast_iov, &s->iter);  in io_setup_async_rw()
524 		iorw = req->async_data;  in io_setup_async_rw()
531 static inline int io_rw_prep_async(struct io_kiocb *req, int rw)  in io_rw_prep_async()  argument
533 	struct io_async_rw *iorw = req->async_data;  in io_rw_prep_async()
538 	ret = io_import_iovec(rw, req, &iov, &iorw->s, 0);  in io_rw_prep_async()
545 		req->flags |= REQ_F_NEED_CLEANUP;  in io_rw_prep_async()
549 int io_readv_prep_async(struct io_kiocb *req)  in io_readv_prep_async()  argument
551 	return io_rw_prep_async(req, READ);  in io_readv_prep_async()
554 int io_writev_prep_async(struct io_kiocb *req)  in io_writev_prep_async()  argument
556 	return io_rw_prep_async(req, WRITE);  in io_writev_prep_async()
573 	struct io_kiocb *req = wait->private;  in io_async_buf_func()  local
574 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_async_buf_func()
584 	io_req_task_queue(req);  in io_async_buf_func()
600 static bool io_rw_should_retry(struct io_kiocb *req)  in io_rw_should_retry()  argument
602 	struct io_async_rw *io = req->async_data;  in io_rw_should_retry()
604 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_rw_should_retry()
608 	if (req->flags & REQ_F_NOWAIT)  in io_rw_should_retry()
619 	if (file_can_poll(req->file) || !(req->file->f_mode & FMODE_BUF_RASYNC))  in io_rw_should_retry()
623 	wait->wait.private = req;  in io_rw_should_retry()
644 static bool need_complete_io(struct io_kiocb *req)  in need_complete_io()  argument
646 	return req->flags & REQ_F_ISREG ||  in need_complete_io()
647 		S_ISBLK(file_inode(req->file)->i_mode);  in need_complete_io()
650 static int io_rw_init_file(struct io_kiocb *req, fmode_t mode)  in io_rw_init_file()  argument
652 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_rw_init_file()
654 	struct io_ring_ctx *ctx = req->ctx;  in io_rw_init_file()
655 	struct file *file = req->file;  in io_rw_init_file()
661 	if (!io_req_ffs_set(req))  in io_rw_init_file()
662 		req->flags |= io_file_get_flags(file) << REQ_F_SUPPORT_NOWAIT_BIT;  in io_rw_init_file()
675 	    ((file->f_flags & O_NONBLOCK) && !io_file_supports_nowait(req)))  in io_rw_init_file()
676 		req->flags |= REQ_F_NOWAIT;  in io_rw_init_file()
685 		req->iopoll_completed = 0;  in io_rw_init_file()
695 int io_read(struct io_kiocb *req, unsigned int issue_flags)  in io_read()  argument
697 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_read()
706 	if (!req_has_async_data(req)) {  in io_read()
707 		ret = io_import_iovec(READ, req, &iovec, s, issue_flags);  in io_read()
711 		io = req->async_data;  in io_read()
718 		if (io_do_buffer_select(req)) {  in io_read()
719 			ret = io_import_iovec(READ, req, &iovec, s, issue_flags);  in io_read()
732 	ret = io_rw_init_file(req, FMODE_READ);  in io_read()
737 	req->cqe.res = iov_iter_count(&s->iter);  in io_read()
741 		if (unlikely(!io_file_supports_nowait(req))) {  in io_read()
742 			ret = io_setup_async_rw(req, iovec, s, true);  in io_read()
751 	ppos = io_kiocb_update_pos(req);  in io_read()
753 	ret = rw_verify_area(READ, req->file, ppos, req->cqe.res);  in io_read()
761 	if (ret == -EAGAIN || (req->flags & REQ_F_REISSUE)) {  in io_read()
762 		req->flags &= ~REQ_F_REISSUE;  in io_read()
764 		if (req->opcode == IORING_OP_READ && file_can_poll(req->file))  in io_read()
767 		if (!force_nonblock && !(req->ctx->flags & IORING_SETUP_IOPOLL))  in io_read()
770 		if (req->flags & REQ_F_NOWAIT)  in io_read()
777 	} else if (ret == req->cqe.res || ret <= 0 || !force_nonblock ||  in io_read()
778 		   (req->flags & REQ_F_NOWAIT) || !need_complete_io(req)) {  in io_read()
790 	ret2 = io_setup_async_rw(req, iovec, s, true);  in io_read()
797 	io = req->async_data;  in io_read()
817 		if (!io_rw_should_retry(req)) {  in io_read()
822 		req->cqe.res = iov_iter_count(&s->iter);  in io_read()
840 	return kiocb_done(req, ret, issue_flags);  in io_read()
843 int io_write(struct io_kiocb *req, unsigned int issue_flags)  in io_write()  argument
845 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_write()
853 	if (!req_has_async_data(req)) {  in io_write()
854 		ret = io_import_iovec(WRITE, req, &iovec, s, issue_flags);  in io_write()
858 		struct io_async_rw *io = req->async_data;  in io_write()
864 	ret = io_rw_init_file(req, FMODE_WRITE);  in io_write()
869 	req->cqe.res = iov_iter_count(&s->iter);  in io_write()
873 		if (unlikely(!io_file_supports_nowait(req)))  in io_write()
879 			(req->flags & REQ_F_ISREG))  in io_write()
888 	ppos = io_kiocb_update_pos(req);  in io_write()
890 	ret = rw_verify_area(WRITE, req->file, ppos, req->cqe.res);  in io_write()
903 	if (req->flags & REQ_F_ISREG) {  in io_write()
904 		sb_start_write(file_inode(req->file)->i_sb);  in io_write()
905 		__sb_writers_release(file_inode(req->file)->i_sb,  in io_write()
910 	if (likely(req->file->f_op->write_iter))  in io_write()
911 		ret2 = call_write_iter(req->file, kiocb, &s->iter);  in io_write()
912 	else if (req->file->f_op->write)  in io_write()
917 	if (req->flags & REQ_F_REISSUE) {  in io_write()
918 		req->flags &= ~REQ_F_REISSUE;  in io_write()
929 	if (ret2 == -EAGAIN && (req->flags & REQ_F_NOWAIT))  in io_write()
933 		if (ret2 == -EAGAIN && (req->ctx->flags & IORING_SETUP_IOPOLL))  in io_write()
936 		if (ret2 != req->cqe.res && ret2 >= 0 && need_complete_io(req)) {  in io_write()
939 			trace_io_uring_short_write(req->ctx, kiocb->ki_pos - ret2,  in io_write()
940 						req->cqe.res, ret2);  in io_write()
948 			ret = io_setup_async_rw(req, iovec, s, true);  in io_write()
950 			io = req->async_data;  in io_write()
955 				kiocb_end_write(req);  in io_write()
959 		ret = kiocb_done(req, ret2, issue_flags);  in io_write()
963 		ret = io_setup_async_rw(req, iovec, s, false);  in io_write()
966 				kiocb_end_write(req);  in io_write()
984 void io_rw_fail(struct io_kiocb *req)  in io_rw_fail()  argument
988 	res = io_fixup_rw_res(req, req->cqe.res);  in io_rw_fail()
989 	io_req_set_res(req, res, req->cqe.flags);  in io_rw_fail()
1007 		struct io_kiocb *req = container_of(pos, struct io_kiocb, comp_list);  in io_do_iopoll()  local
1008 		struct file *file = req->file;  in io_do_iopoll()
1016 		if (READ_ONCE(req->iopoll_completed))  in io_do_iopoll()
1019 		if (req->opcode == IORING_OP_URING_CMD) {  in io_do_iopoll()
1022 			ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);  in io_do_iopoll()
1026 			struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_do_iopoll()
1037 		    READ_ONCE(req->iopoll_completed))  in io_do_iopoll()
1048 		struct io_kiocb *req = container_of(pos, struct io_kiocb, comp_list);  in io_do_iopoll()  local
1051 		if (!smp_load_acquire(&req->iopoll_completed))  in io_do_iopoll()
1054 		if (unlikely(req->flags & REQ_F_CQE_SKIP))  in io_do_iopoll()
1057 		req->cqe.flags = io_put_kbuf(req, 0);  in io_do_iopoll()
1058 		__io_fill_cqe_req(req->ctx, req);  in io_do_iopoll()