Lines Matching +full:- +full:- +full:retry
1 // SPDX-License-Identifier: GPL-2.0
6 #include <linux/blk-mq.h>
33 return req->flags & REQ_F_SUPPORT_NOWAIT; in io_file_supports_nowait()
42 uiov = u64_to_user_ptr(rw->addr); in io_iov_compat_buffer_select_prep()
44 return -EFAULT; in io_iov_compat_buffer_select_prep()
45 if (__get_user(clen, &uiov->iov_len)) in io_iov_compat_buffer_select_prep()
46 return -EFAULT; in io_iov_compat_buffer_select_prep()
48 return -EINVAL; in io_iov_compat_buffer_select_prep()
50 rw->len = clen; in io_iov_compat_buffer_select_prep()
61 if (rw->len != 1) in io_iov_buffer_select_prep()
62 return -EINVAL; in io_iov_buffer_select_prep()
65 if (req->ctx->compat) in io_iov_buffer_select_prep()
69 uiov = u64_to_user_ptr(rw->addr); in io_iov_buffer_select_prep()
71 return -EFAULT; in io_iov_buffer_select_prep()
72 rw->len = iov.iov_len; in io_iov_buffer_select_prep()
82 rw->kiocb.ki_pos = READ_ONCE(sqe->off); in io_prep_rw()
83 /* used for fixed read/write too - just read unconditionally */ in io_prep_rw()
84 req->buf_index = READ_ONCE(sqe->buf_index); in io_prep_rw()
86 if (req->opcode == IORING_OP_READ_FIXED || in io_prep_rw()
87 req->opcode == IORING_OP_WRITE_FIXED) { in io_prep_rw()
88 struct io_ring_ctx *ctx = req->ctx; in io_prep_rw()
91 if (unlikely(req->buf_index >= ctx->nr_user_bufs)) in io_prep_rw()
92 return -EFAULT; in io_prep_rw()
93 index = array_index_nospec(req->buf_index, ctx->nr_user_bufs); in io_prep_rw()
94 req->imu = ctx->user_bufs[index]; in io_prep_rw()
98 ioprio = READ_ONCE(sqe->ioprio); in io_prep_rw()
104 rw->kiocb.ki_ioprio = ioprio; in io_prep_rw()
106 rw->kiocb.ki_ioprio = get_current_ioprio(); in io_prep_rw()
108 rw->kiocb.dio_complete = NULL; in io_prep_rw()
110 rw->addr = READ_ONCE(sqe->addr); in io_prep_rw()
111 rw->len = READ_ONCE(sqe->len); in io_prep_rw()
112 rw->flags = READ_ONCE(sqe->rw_flags); in io_prep_rw()
114 /* Have to do this validation here, as this is in io_read() rw->len might in io_prep_rw()
117 if (req->opcode == IORING_OP_READV && req->flags & REQ_F_BUFFER_SELECT) { in io_prep_rw()
128 struct io_async_rw *io = req->async_data; in io_readv_writev_cleanup()
130 kfree(io->free_iovec); in io_readv_writev_cleanup()
136 case -EIOCBQUEUED: in io_rw_done()
138 case -ERESTARTSYS: in io_rw_done()
139 case -ERESTARTNOINTR: in io_rw_done()
140 case -ERESTARTNOHAND: in io_rw_done()
141 case -ERESTART_RESTARTBLOCK: in io_rw_done()
147 ret = -EINTR; in io_rw_done()
150 kiocb->ki_complete(kiocb, ret); in io_rw_done()
158 if (rw->kiocb.ki_pos != -1) in io_kiocb_update_pos()
159 return &rw->kiocb.ki_pos; in io_kiocb_update_pos()
161 if (!(req->file->f_mode & FMODE_STREAM)) { in io_kiocb_update_pos()
162 req->flags |= REQ_F_CUR_POS; in io_kiocb_update_pos()
163 rw->kiocb.ki_pos = req->file->f_pos; in io_kiocb_update_pos()
164 return &rw->kiocb.ki_pos; in io_kiocb_update_pos()
167 rw->kiocb.ki_pos = 0; in io_kiocb_update_pos()
173 req->io_task_work.func = io_queue_iowq; in io_req_task_queue_reissue()
180 struct io_async_rw *io = req->async_data; in io_resubmit_prep()
184 iov_iter_restore(&io->s.iter, &io->s.iter_state); in io_resubmit_prep()
190 umode_t mode = file_inode(req->file)->i_mode; in io_rw_should_reissue()
191 struct io_ring_ctx *ctx = req->ctx; in io_rw_should_reissue()
195 if ((req->flags & REQ_F_NOWAIT) || (io_wq_current_is_worker() && in io_rw_should_reissue()
196 !(ctx->flags & IORING_SETUP_IOPOLL))) in io_rw_should_reissue()
201 * -EAGAIN. in io_rw_should_reissue()
203 if (percpu_ref_is_dying(&ctx->refs)) in io_rw_should_reissue()
206 * Play it safe and assume not safe to re-import and reissue if we're in io_rw_should_reissue()
209 if (!same_thread_group(req->task, current) || !in_task()) in io_rw_should_reissue()
226 if (req->flags & REQ_F_ISREG) { in io_req_end_write()
229 kiocb_end_write(&rw->kiocb); in io_req_end_write()
241 if (rw->kiocb.ki_flags & IOCB_WRITE) { in io_req_io_end()
243 fsnotify_modify(req->file); in io_req_io_end()
245 fsnotify_access(req->file); in io_req_io_end()
251 if (unlikely(res != req->cqe.res)) { in __io_complete_rw_common()
252 if ((res == -EAGAIN || res == -EOPNOTSUPP) && in __io_complete_rw_common()
259 req->flags |= REQ_F_REISSUE | REQ_F_PARTIAL_IO; in __io_complete_rw_common()
263 req->cqe.res = res; in __io_complete_rw_common()
270 struct io_async_rw *io = req->async_data; in io_fixup_rw_res()
273 if (req_has_async_data(req) && io->bytes_done > 0) { in io_fixup_rw_res()
275 res = io->bytes_done; in io_fixup_rw_res()
277 res += io->bytes_done; in io_fixup_rw_res()
285 struct kiocb *kiocb = &rw->kiocb; in io_req_rw_complete()
287 if ((kiocb->ki_flags & IOCB_DIO_CALLER_COMP) && kiocb->dio_complete) { in io_req_rw_complete()
288 long res = kiocb->dio_complete(rw->kiocb.private); in io_req_rw_complete()
295 if (req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)) { in io_req_rw_complete()
296 unsigned issue_flags = ts->locked ? 0 : IO_URING_F_UNLOCKED; in io_req_rw_complete()
298 req->cqe.flags |= io_put_kbuf(req, issue_flags); in io_req_rw_complete()
308 if (!kiocb->dio_complete || !(kiocb->ki_flags & IOCB_DIO_CALLER_COMP)) { in io_complete_rw()
313 req->io_task_work.func = io_req_rw_complete; in io_complete_rw()
322 if (kiocb->ki_flags & IOCB_WRITE) in io_complete_rw_iopoll()
324 if (unlikely(res != req->cqe.res)) { in io_complete_rw_iopoll()
325 if (res == -EAGAIN && io_rw_should_reissue(req)) { in io_complete_rw_iopoll()
326 req->flags |= REQ_F_REISSUE | REQ_F_PARTIAL_IO; in io_complete_rw_iopoll()
329 req->cqe.res = res; in io_complete_rw_iopoll()
332 /* order with io_iopoll_complete() checking ->iopoll_completed */ in io_complete_rw_iopoll()
333 smp_store_release(&req->iopoll_completed, 1); in io_complete_rw_iopoll()
342 if (ret >= 0 && req->flags & REQ_F_CUR_POS) in kiocb_done()
343 req->file->f_pos = rw->kiocb.ki_pos; in kiocb_done()
344 if (ret >= 0 && (rw->kiocb.ki_complete == io_complete_rw)) { in kiocb_done()
356 io_rw_done(&rw->kiocb, ret); in kiocb_done()
359 if (req->flags & REQ_F_REISSUE) { in kiocb_done()
360 req->flags &= ~REQ_F_REISSUE; in kiocb_done()
374 struct iov_iter *iter = &s->iter; in __io_import_iovec()
375 u8 opcode = req->opcode; in __io_import_iovec()
382 ret = io_import_fixed(ddir, iter, req->imu, rw->addr, rw->len); in __io_import_iovec()
388 buf = u64_to_user_ptr(rw->addr); in __io_import_iovec()
389 sqe_len = rw->len; in __io_import_iovec()
392 (req->flags & REQ_F_BUFFER_SELECT)) { in __io_import_iovec()
396 return ERR_PTR(-ENOBUFS); in __io_import_iovec()
397 rw->addr = (unsigned long) buf; in __io_import_iovec()
398 rw->len = sqe_len; in __io_import_iovec()
407 iovec = s->fast_iov; in __io_import_iovec()
409 req->ctx->compat); in __io_import_iovec()
423 iov_iter_save_state(&s->iter, &s->iter_state); in io_import_iovec()
429 return (kiocb->ki_filp->f_mode & FMODE_STREAM) ? NULL : &kiocb->ki_pos; in io_kiocb_ppos()
433 * For files that don't have ->read_iter() and ->write_iter(), handle them
434 * by looping over ->read() or ->write() manually.
438 struct kiocb *kiocb = &rw->kiocb; in loop_rw_iter()
439 struct file *file = kiocb->ki_filp; in loop_rw_iter()
445 * support non-blocking either. For the latter, this just causes in loop_rw_iter()
448 if (kiocb->ki_flags & IOCB_HIPRI) in loop_rw_iter()
449 return -EOPNOTSUPP; in loop_rw_iter()
450 if ((kiocb->ki_flags & IOCB_NOWAIT) && in loop_rw_iter()
451 !(kiocb->ki_filp->f_flags & O_NONBLOCK)) in loop_rw_iter()
452 return -EAGAIN; in loop_rw_iter()
462 addr = iter->ubuf + iter->iov_offset; in loop_rw_iter()
468 addr = u64_to_user_ptr(rw->addr); in loop_rw_iter()
469 len = rw->len; in loop_rw_iter()
473 nr = file->f_op->read(file, addr, len, ppos); in loop_rw_iter()
475 nr = file->f_op->write(file, addr, len, ppos); in loop_rw_iter()
486 rw->addr += nr; in loop_rw_iter()
487 rw->len -= nr; in loop_rw_iter()
488 if (!rw->len) in loop_rw_iter()
501 struct io_async_rw *io = req->async_data; in io_req_map_rw()
503 memcpy(&io->s.iter, iter, sizeof(*iter)); in io_req_map_rw()
504 io->free_iovec = iovec; in io_req_map_rw()
505 io->bytes_done = 0; in io_req_map_rw()
512 io->s.iter.__iov = io->s.fast_iov; in io_req_map_rw()
513 if (iter->__iov != fast_iov) { in io_req_map_rw()
514 iov_off = iter_iov(iter) - fast_iov; in io_req_map_rw()
515 io->s.iter.__iov += iov_off; in io_req_map_rw()
517 if (io->s.fast_iov != fast_iov) in io_req_map_rw()
518 memcpy(io->s.fast_iov + iov_off, fast_iov + iov_off, in io_req_map_rw()
519 sizeof(struct iovec) * iter->nr_segs); in io_req_map_rw()
521 req->flags |= REQ_F_NEED_CLEANUP; in io_req_map_rw()
528 if (!force && !io_cold_defs[req->opcode].prep_async) in io_setup_async_rw()
535 return -ENOMEM; in io_setup_async_rw()
538 io_req_map_rw(req, iovec, s->fast_iov, &s->iter); in io_setup_async_rw()
539 iorw = req->async_data; in io_setup_async_rw()
541 iov_iter_save_state(&iorw->s.iter, &iorw->s.iter_state); in io_setup_async_rw()
548 struct io_async_rw *iorw = req->async_data; in io_rw_prep_async()
552 /* submission path, ->uring_lock should already be taken */ in io_rw_prep_async()
553 ret = io_import_iovec(rw, req, &iov, &iorw->s, 0); in io_rw_prep_async()
557 iorw->bytes_done = 0; in io_rw_prep_async()
558 iorw->free_iovec = iov; in io_rw_prep_async()
560 req->flags |= REQ_F_NEED_CLEANUP; in io_rw_prep_async()
579 * queue a task_work based retry of the operation, attempting to copy the data
581 * do a thread based blocking retry of the operation. That's the unexpected
588 struct io_kiocb *req = wait->private; in io_async_buf_func()
597 rw->kiocb.ki_flags &= ~IOCB_WAITQ; in io_async_buf_func()
598 list_del_init(&wait->entry); in io_async_buf_func()
605 * based retry. If we return false here, the request is handed to the async
606 * worker threads for retry. If we're doing buffered reads on a regular file,
607 * we prepare a private wait_page_queue entry and retry the operation. This
610 * that callback, io_uring uses task_work to setup a retry of the operation.
611 * That retry will attempt the buffered read again. The retry will generally
613 * async worker threads for a blocking retry.
617 struct io_async_rw *io = req->async_data; in io_rw_should_retry()
618 struct wait_page_queue *wait = &io->wpq; in io_rw_should_retry()
620 struct kiocb *kiocb = &rw->kiocb; in io_rw_should_retry()
622 /* never retry for NOWAIT, we just complete with -EAGAIN */ in io_rw_should_retry()
623 if (req->flags & REQ_F_NOWAIT) in io_rw_should_retry()
627 if (kiocb->ki_flags & (IOCB_DIRECT | IOCB_HIPRI)) in io_rw_should_retry()
634 if (file_can_poll(req->file) || !(req->file->f_mode & FMODE_BUF_RASYNC)) in io_rw_should_retry()
637 wait->wait.func = io_async_buf_func; in io_rw_should_retry()
638 wait->wait.private = req; in io_rw_should_retry()
639 wait->wait.flags = 0; in io_rw_should_retry()
640 INIT_LIST_HEAD(&wait->wait.entry); in io_rw_should_retry()
641 kiocb->ki_flags |= IOCB_WAITQ; in io_rw_should_retry()
642 kiocb->ki_flags &= ~IOCB_NOWAIT; in io_rw_should_retry()
643 kiocb->ki_waitq = wait; in io_rw_should_retry()
649 struct file *file = rw->kiocb.ki_filp; in io_iter_do_read()
651 if (likely(file->f_op->read_iter)) in io_iter_do_read()
652 return call_read_iter(file, &rw->kiocb, iter); in io_iter_do_read()
653 else if (file->f_op->read) in io_iter_do_read()
656 return -EINVAL; in io_iter_do_read()
661 return req->flags & REQ_F_ISREG || in need_complete_io()
662 S_ISBLK(file_inode(req->file)->i_mode); in need_complete_io()
668 struct kiocb *kiocb = &rw->kiocb; in io_rw_init_file()
669 struct io_ring_ctx *ctx = req->ctx; in io_rw_init_file()
670 struct file *file = req->file; in io_rw_init_file()
673 if (unlikely(!file || !(file->f_mode & mode))) in io_rw_init_file()
674 return -EBADF; in io_rw_init_file()
676 if (!(req->flags & REQ_F_FIXED_FILE)) in io_rw_init_file()
677 req->flags |= io_file_get_flags(file); in io_rw_init_file()
679 kiocb->ki_flags = file->f_iocb_flags; in io_rw_init_file()
680 ret = kiocb_set_rw_flags(kiocb, rw->flags); in io_rw_init_file()
683 kiocb->ki_flags |= IOCB_ALLOC_CACHE; in io_rw_init_file()
686 * If the file is marked O_NONBLOCK, still allow retry for it if it in io_rw_init_file()
688 * reliably. If not, or it IOCB_NOWAIT is set, don't retry. in io_rw_init_file()
690 if ((kiocb->ki_flags & IOCB_NOWAIT) || in io_rw_init_file()
691 ((file->f_flags & O_NONBLOCK) && !io_file_supports_nowait(req))) in io_rw_init_file()
692 req->flags |= REQ_F_NOWAIT; in io_rw_init_file()
694 if (ctx->flags & IORING_SETUP_IOPOLL) { in io_rw_init_file()
695 if (!(kiocb->ki_flags & IOCB_DIRECT) || !file->f_op->iopoll) in io_rw_init_file()
696 return -EOPNOTSUPP; in io_rw_init_file()
698 kiocb->private = NULL; in io_rw_init_file()
699 kiocb->ki_flags |= IOCB_HIPRI; in io_rw_init_file()
700 kiocb->ki_complete = io_complete_rw_iopoll; in io_rw_init_file()
701 req->iopoll_completed = 0; in io_rw_init_file()
703 if (kiocb->ki_flags & IOCB_HIPRI) in io_rw_init_file()
704 return -EINVAL; in io_rw_init_file()
705 kiocb->ki_complete = io_complete_rw; in io_rw_init_file()
716 struct kiocb *kiocb = &rw->kiocb; in io_read()
727 io = req->async_data; in io_read()
728 s = &io->s; in io_read()
731 * Safe and required to re-import if we're using provided in io_read()
732 * buffers, as we dropped the selected one before retry. in io_read()
745 iov_iter_restore(&s->iter, &s->iter_state); in io_read()
753 req->cqe.res = iov_iter_count(&s->iter); in io_read()
759 return ret ?: -EAGAIN; in io_read()
761 kiocb->ki_flags |= IOCB_NOWAIT; in io_read()
763 /* Ensure we clear previously set non-block flag */ in io_read()
764 kiocb->ki_flags &= ~IOCB_NOWAIT; in io_read()
769 ret = rw_verify_area(READ, req->file, ppos, req->cqe.res); in io_read()
775 ret = io_iter_do_read(rw, &s->iter); in io_read()
777 if (ret == -EAGAIN || (req->flags & REQ_F_REISSUE)) { in io_read()
778 req->flags &= ~REQ_F_REISSUE; in io_read()
780 if (req->opcode == IORING_OP_READ && file_can_poll(req->file)) in io_read()
781 return -EAGAIN; in io_read()
782 /* IOPOLL retry should happen for io-wq threads */ in io_read()
783 if (!force_nonblock && !(req->ctx->flags & IORING_SETUP_IOPOLL)) in io_read()
785 /* no retry on NONBLOCK nor RWF_NOWAIT */ in io_read()
786 if (req->flags & REQ_F_NOWAIT) in io_read()
789 } else if (ret == -EIOCBQUEUED) { in io_read()
793 } else if (ret == req->cqe.res || ret <= 0 || !force_nonblock || in io_read()
794 (req->flags & REQ_F_NOWAIT) || !need_complete_io(req)) { in io_read()
795 /* read all, failed, already did sync or don't want to retry */ in io_read()
804 iov_iter_restore(&s->iter, &s->iter_state); in io_read()
813 io = req->async_data; in io_read()
814 s = &io->s; in io_read()
826 iov_iter_advance(&s->iter, ret); in io_read()
827 if (!iov_iter_count(&s->iter)) in io_read()
829 io->bytes_done += ret; in io_read()
830 iov_iter_save_state(&s->iter, &s->iter_state); in io_read()
832 /* if we can retry, do so with the callbacks armed */ in io_read()
834 kiocb->ki_flags &= ~IOCB_WAITQ; in io_read()
835 return -EAGAIN; in io_read()
838 req->cqe.res = iov_iter_count(&s->iter); in io_read()
840 * Now retry read with the IOCB_WAITQ parts set in the iocb. If in io_read()
841 * we get -EIOCBQUEUED, then we'll get a notification when the in io_read()
843 * here, and if we do, then just retry at the new offset. in io_read()
845 ret = io_iter_do_read(rw, &s->iter); in io_read()
846 if (ret == -EIOCBQUEUED) in io_read()
848 /* we got some bytes, but not all. retry. */ in io_read()
849 kiocb->ki_flags &= ~IOCB_WAITQ; in io_read()
850 iov_iter_restore(&s->iter, &s->iter_state); in io_read()
864 struct kiocb *kiocb = &rw->kiocb; in io_write()
874 struct io_async_rw *io = req->async_data; in io_write()
876 s = &io->s; in io_write()
877 iov_iter_restore(&s->iter, &s->iter_state); in io_write()
885 req->cqe.res = iov_iter_count(&s->iter); in io_write()
892 /* File path supports NOWAIT for non-direct_IO only for block devices. */ in io_write()
893 if (!(kiocb->ki_flags & IOCB_DIRECT) && in io_write()
894 !(kiocb->ki_filp->f_mode & FMODE_BUF_WASYNC) && in io_write()
895 (req->flags & REQ_F_ISREG)) in io_write()
898 kiocb->ki_flags |= IOCB_NOWAIT; in io_write()
900 /* Ensure we clear previously set non-block flag */ in io_write()
901 kiocb->ki_flags &= ~IOCB_NOWAIT; in io_write()
906 ret = rw_verify_area(WRITE, req->file, ppos, req->cqe.res); in io_write()
912 if (req->flags & REQ_F_ISREG) in io_write()
914 kiocb->ki_flags |= IOCB_WRITE; in io_write()
916 if (likely(req->file->f_op->write_iter)) in io_write()
917 ret2 = call_write_iter(req->file, kiocb, &s->iter); in io_write()
918 else if (req->file->f_op->write) in io_write()
919 ret2 = loop_rw_iter(WRITE, rw, &s->iter); in io_write()
921 ret2 = -EINVAL; in io_write()
923 if (req->flags & REQ_F_REISSUE) { in io_write()
924 req->flags &= ~REQ_F_REISSUE; in io_write()
925 ret2 = -EAGAIN; in io_write()
929 * Raw bdev writes will return -EOPNOTSUPP for IOCB_NOWAIT. Just in io_write()
930 * retry them without IOCB_NOWAIT. in io_write()
932 if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT)) in io_write()
933 ret2 = -EAGAIN; in io_write()
934 /* no retry on NONBLOCK nor RWF_NOWAIT */ in io_write()
935 if (ret2 == -EAGAIN && (req->flags & REQ_F_NOWAIT)) in io_write()
937 if (!force_nonblock || ret2 != -EAGAIN) { in io_write()
938 /* IOPOLL retry should happen for io-wq threads */ in io_write()
939 if (ret2 == -EAGAIN && (req->ctx->flags & IORING_SETUP_IOPOLL)) in io_write()
942 if (ret2 != req->cqe.res && ret2 >= 0 && need_complete_io(req)) { in io_write()
945 trace_io_uring_short_write(req->ctx, kiocb->ki_pos - ret2, in io_write()
946 req->cqe.res, ret2); in io_write()
953 iov_iter_save_state(&s->iter, &s->iter_state); in io_write()
956 io = req->async_data; in io_write()
958 io->bytes_done += ret2; in io_write()
960 if (kiocb->ki_flags & IOCB_WRITE) in io_write()
962 return ret ? ret : -EAGAIN; in io_write()
968 iov_iter_restore(&s->iter, &s->iter_state); in io_write()
971 if (kiocb->ki_flags & IOCB_WRITE) in io_write()
973 return -EAGAIN; in io_write()
987 res = io_fixup_rw_res(req, req->cqe.res); in io_rw_fail()
988 io_req_set_res(req, res, req->cqe.flags); in io_rw_fail()
1002 if (ctx->poll_multi_queue || force_nonspin) in io_do_iopoll()
1005 wq_list_for_each(pos, start, &ctx->iopoll_list) { in io_do_iopoll()
1007 struct file *file = req->file; in io_do_iopoll()
1015 if (READ_ONCE(req->iopoll_completed)) in io_do_iopoll()
1018 if (req->opcode == IORING_OP_URING_CMD) { in io_do_iopoll()
1022 ret = file->f_op->uring_cmd_iopoll(ioucmd, &iob, in io_do_iopoll()
1027 ret = file->f_op->iopoll(&rw->kiocb, &iob, poll_flags); in io_do_iopoll()
1036 READ_ONCE(req->iopoll_completed)) in io_do_iopoll()
1049 /* order with io_complete_rw_iopoll(), e.g. ->result updates */ in io_do_iopoll()
1050 if (!smp_load_acquire(&req->iopoll_completed)) in io_do_iopoll()
1053 req->cqe.flags = io_put_kbuf(req, 0); in io_do_iopoll()
1058 pos = start ? start->next : ctx->iopoll_list.first; in io_do_iopoll()
1059 wq_list_cut(&ctx->iopoll_list, prev, start); in io_do_iopoll()
1061 if (WARN_ON_ONCE(!wq_list_empty(&ctx->submit_state.compl_reqs))) in io_do_iopoll()
1063 ctx->submit_state.compl_reqs.first = pos; in io_do_iopoll()