456 	struct kiocb			kiocb;  member
2376 		struct kiocb *kiocb = &req->rw.kiocb;  in io_do_iopoll()  local
2390 		ret = kiocb->ki_filp->f_op->iopoll(kiocb, spin);  in io_do_iopoll()
2522 static void io_complete_rw_common(struct kiocb *kiocb, long res,  in io_complete_rw_common()  argument
2525 	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);  in io_complete_rw_common()
2528 	if (kiocb->ki_flags & IOCB_WRITE)  in io_complete_rw_common()
2612 		io_complete_rw_common(&req->rw.kiocb, res, cs);  in __io_complete_rw()
2615 static void io_complete_rw(struct kiocb *kiocb, long res, long res2)  in io_complete_rw()  argument
2617 	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);  in io_complete_rw()
2622 static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2)  in io_complete_rw_iopoll()  argument
2624 	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);  in io_complete_rw_iopoll()
2626 	if (kiocb->ki_flags & IOCB_WRITE)  in io_complete_rw_iopoll()
2765 	struct kiocb *kiocb = &req->rw.kiocb;  in io_prep_rw()  local
2772 	kiocb->ki_pos = READ_ONCE(sqe->off);  in io_prep_rw()
2773 	if (kiocb->ki_pos == -1 && !(req->file->f_mode & FMODE_STREAM)) {  in io_prep_rw()
2775 		kiocb->ki_pos = req->file->f_pos;  in io_prep_rw()
2777 	kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp));  in io_prep_rw()
2778 	kiocb->ki_flags = iocb_flags(kiocb->ki_filp);  in io_prep_rw()
2779 	ret = kiocb_set_rw_flags(kiocb, READ_ONCE(sqe->rw_flags));  in io_prep_rw()
2789 		kiocb->ki_ioprio = ioprio;  in io_prep_rw()
2791 		kiocb->ki_ioprio = get_current_ioprio();  in io_prep_rw()
2794 	if (kiocb->ki_flags & IOCB_NOWAIT)  in io_prep_rw()
2798 		if (!(kiocb->ki_flags & IOCB_DIRECT) ||  in io_prep_rw()
2799 		    !kiocb->ki_filp->f_op->iopoll)  in io_prep_rw()
2802 		kiocb->ki_flags |= IOCB_HIPRI;  in io_prep_rw()
2803 		kiocb->ki_complete = io_complete_rw_iopoll;  in io_prep_rw()
2806 		if (kiocb->ki_flags & IOCB_HIPRI)  in io_prep_rw()
2808 		kiocb->ki_complete = io_complete_rw;  in io_prep_rw()
2817 static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret)  in io_rw_done()  argument
2834 		kiocb->ki_complete(kiocb, ret, 0);  in io_rw_done()
2838 static void kiocb_done(struct kiocb *kiocb, ssize_t ret,  in kiocb_done()  argument
2841 	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);  in kiocb_done()
2853 		req->file->f_pos = kiocb->ki_pos;  in kiocb_done()
2854 	if (ret >= 0 && kiocb->ki_complete == io_complete_rw)  in kiocb_done()
2857 		io_rw_done(kiocb, ret);  in kiocb_done()
3126 static inline loff_t *io_kiocb_ppos(struct kiocb *kiocb)  in io_kiocb_ppos()  argument
3128 	return (kiocb->ki_filp->f_mode & FMODE_STREAM) ? NULL : &kiocb->ki_pos;  in io_kiocb_ppos()
3137 	struct kiocb *kiocb = &req->rw.kiocb;  in loop_rw_iter()  local
3146 	if (kiocb->ki_flags & IOCB_HIPRI)  in loop_rw_iter()
3148 	if (kiocb->ki_flags & IOCB_NOWAIT)  in loop_rw_iter()
3164 					      iovec.iov_len, io_kiocb_ppos(kiocb));  in loop_rw_iter()
3167 					       iovec.iov_len, io_kiocb_ppos(kiocb));  in loop_rw_iter()
3300 	req->rw.kiocb.ki_flags &= ~IOCB_WAITQ;  in io_async_buf_func()
3337 	struct kiocb *kiocb = &req->rw.kiocb;  in io_rw_should_retry()  local
3344 	if (kiocb->ki_flags & (IOCB_DIRECT | IOCB_HIPRI))  in io_rw_should_retry()
3358 	kiocb->ki_flags |= IOCB_WAITQ;  in io_rw_should_retry()
3359 	kiocb->ki_flags &= ~IOCB_NOWAIT;  in io_rw_should_retry()
3360 	kiocb->ki_waitq = wait;  in io_rw_should_retry()
3367 		return call_read_iter(req->file, &req->rw.kiocb, iter);  in io_iter_do_read()
3378 	struct kiocb *kiocb = &req->rw.kiocb;  in io_read()  local
3398 		kiocb->ki_flags &= ~IOCB_NOWAIT;  in io_read()
3400 		kiocb->ki_flags |= IOCB_NOWAIT;  in io_read()
3408 	ret = rw_verify_area(READ, req->file, io_kiocb_ppos(kiocb), iov_count);  in io_read()
3458 		kiocb->ki_flags &= ~IOCB_WAITQ;  in io_read()
3477 	kiocb_done(kiocb, ret, cs);  in io_read()
3507 	struct kiocb *kiocb = &req->rw.kiocb;  in io_write()  local
3525 		kiocb->ki_flags &= ~IOCB_NOWAIT;  in io_write()
3527 		kiocb->ki_flags |= IOCB_NOWAIT;  in io_write()
3534 	if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT) &&  in io_write()
3538 	ret = rw_verify_area(WRITE, req->file, io_kiocb_ppos(kiocb), iov_count);  in io_write()
3554 	kiocb->ki_flags |= IOCB_WRITE;  in io_write()
3557 		ret2 = call_write_iter(req->file, kiocb, iter);  in io_write()
3567 	if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT))  in io_write()
3577 		kiocb_done(kiocb, ret2, cs);  in io_write()