562 	struct kiocb			kiocb;  member
2467 		struct kiocb *kiocb = &req->rw.kiocb;  in io_do_iopoll()  local
2482 		ret = kiocb->ki_filp->f_op->iopoll(kiocb, spin);  in io_do_iopoll()
2644 	if (req->rw.kiocb.ki_flags & IOCB_WRITE)  in __io_complete_rw_common()
2684 static void io_complete_rw(struct kiocb *kiocb, long res, long res2)  in io_complete_rw()  argument
2686 	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);  in io_complete_rw()
2695 static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2)  in io_complete_rw_iopoll()  argument
2697 	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);  in io_complete_rw_iopoll()
2699 	if (kiocb->ki_flags & IOCB_WRITE)  in io_complete_rw_iopoll()
2746 			queue_num0 = blk_qc_t_to_queue_num(list_req->rw.kiocb.ki_cookie);  in io_iopoll_req_issued()
2747 			queue_num1 = blk_qc_t_to_queue_num(req->rw.kiocb.ki_cookie);  in io_iopoll_req_issued()
2834 	struct kiocb *kiocb = &req->rw.kiocb;  in io_prep_rw()  local
2842 	kiocb->ki_pos = READ_ONCE(sqe->off);  in io_prep_rw()
2843 	if (kiocb->ki_pos == -1 && !(file->f_mode & FMODE_STREAM)) {  in io_prep_rw()
2845 		kiocb->ki_pos = file->f_pos;  in io_prep_rw()
2847 	kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp));  in io_prep_rw()
2848 	kiocb->ki_flags = iocb_flags(kiocb->ki_filp);  in io_prep_rw()
2849 	ret = kiocb_set_rw_flags(kiocb, READ_ONCE(sqe->rw_flags));  in io_prep_rw()
2858 	if ((kiocb->ki_flags & IOCB_NOWAIT) ||  in io_prep_rw()
2868 		kiocb->ki_ioprio = ioprio;  in io_prep_rw()
2870 		kiocb->ki_ioprio = get_current_ioprio();  in io_prep_rw()
2873 		if (!(kiocb->ki_flags & IOCB_DIRECT) ||  in io_prep_rw()
2874 		    !kiocb->ki_filp->f_op->iopoll)  in io_prep_rw()
2877 		kiocb->ki_flags |= IOCB_HIPRI | IOCB_ALLOC_CACHE;  in io_prep_rw()
2878 		kiocb->ki_complete = io_complete_rw_iopoll;  in io_prep_rw()
2881 		if (kiocb->ki_flags & IOCB_HIPRI)  in io_prep_rw()
2883 		kiocb->ki_complete = io_complete_rw;  in io_prep_rw()
2898 static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret)  in io_rw_done()  argument
2915 		kiocb->ki_complete(kiocb, ret, 0);  in io_rw_done()
2919 static void kiocb_done(struct kiocb *kiocb, ssize_t ret,  in kiocb_done()  argument
2922 	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);  in kiocb_done()
2934 		req->file->f_pos = kiocb->ki_pos;  in kiocb_done()
2935 	if (ret >= 0 && (kiocb->ki_complete == io_complete_rw))  in kiocb_done()
2938 		io_rw_done(kiocb, ret);  in kiocb_done()
3214 static inline loff_t *io_kiocb_ppos(struct kiocb *kiocb)  in io_kiocb_ppos()  argument
3216 	return (kiocb->ki_filp->f_mode & FMODE_STREAM) ? NULL : &kiocb->ki_pos;  in io_kiocb_ppos()
3225 	struct kiocb *kiocb = &req->rw.kiocb;  in loop_rw_iter()  local
3234 	if (kiocb->ki_flags & IOCB_HIPRI)  in loop_rw_iter()
3236 	if (kiocb->ki_flags & IOCB_NOWAIT)  in loop_rw_iter()
3252 					      iovec.iov_len, io_kiocb_ppos(kiocb));  in loop_rw_iter()
3255 					       iovec.iov_len, io_kiocb_ppos(kiocb));  in loop_rw_iter()
3380 	req->rw.kiocb.ki_flags &= ~IOCB_WAITQ;  in io_async_buf_func()
3402 	struct kiocb *kiocb = &req->rw.kiocb;  in io_rw_should_retry()  local
3409 	if (kiocb->ki_flags & (IOCB_DIRECT | IOCB_HIPRI))  in io_rw_should_retry()
3423 	kiocb->ki_flags |= IOCB_WAITQ;  in io_rw_should_retry()
3424 	kiocb->ki_flags &= ~IOCB_NOWAIT;  in io_rw_should_retry()
3425 	kiocb->ki_waitq = wait;  in io_rw_should_retry()
3432 		return call_read_iter(req->file, &req->rw.kiocb, iter);  in io_iter_do_read()
3448 	struct kiocb *kiocb = &req->rw.kiocb;  in io_read()  local
3476 		kiocb->ki_flags &= ~IOCB_NOWAIT;  in io_read()
3478 		kiocb->ki_flags |= IOCB_NOWAIT;  in io_read()
3486 	ret = rw_verify_area(READ, req->file, io_kiocb_ppos(kiocb), req->result);  in io_read()
3547 			kiocb->ki_flags &= ~IOCB_WAITQ;  in io_read()
3561 		kiocb->ki_flags &= ~IOCB_WAITQ;  in io_read()
3565 	kiocb_done(kiocb, ret, issue_flags);  in io_read()
3583 	struct kiocb *kiocb = &req->rw.kiocb;  in io_write()  local
3606 		kiocb->ki_flags &= ~IOCB_NOWAIT;  in io_write()
3608 		kiocb->ki_flags |= IOCB_NOWAIT;  in io_write()
3615 	if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT) &&  in io_write()
3619 	ret = rw_verify_area(WRITE, req->file, io_kiocb_ppos(kiocb), req->result);  in io_write()
3635 	kiocb->ki_flags |= IOCB_WRITE;  in io_write()
3638 		ret2 = call_write_iter(req->file, kiocb, iter);  in io_write()
3653 	if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT))  in io_write()
3663 		kiocb_done(kiocb, ret2, issue_flags);  in io_write()