2830 static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,  in io_prep_rw()  argument
2842 	kiocb->ki_pos = READ_ONCE(sqe->off);  in io_prep_rw()
2849 	ret = kiocb_set_rw_flags(kiocb, READ_ONCE(sqe->rw_flags));  in io_prep_rw()
2862 	ioprio = READ_ONCE(sqe->ioprio);  in io_prep_rw()
2892 	req->rw.addr = READ_ONCE(sqe->addr);  in io_prep_rw()
2893 	req->rw.len = READ_ONCE(sqe->len);  in io_prep_rw()
2894 	req->buf_index = READ_ONCE(sqe->buf_index);  in io_prep_rw()
3351 static int io_read_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_read_prep()  argument
3355 	return io_prep_rw(req, sqe, READ);  in io_read_prep()
3573 static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_write_prep()  argument
3577 	return io_prep_rw(req, sqe, WRITE);  in io_write_prep()
3678 			    const struct io_uring_sqe *sqe)  in io_renameat_prep()  argument
3685 	if (sqe->ioprio || sqe->buf_index || sqe->splice_fd_in)  in io_renameat_prep()
3690 	ren->old_dfd = READ_ONCE(sqe->fd);  in io_renameat_prep()
3691 	oldf = u64_to_user_ptr(READ_ONCE(sqe->addr));  in io_renameat_prep()
3692 	newf = u64_to_user_ptr(READ_ONCE(sqe->addr2));  in io_renameat_prep()
3693 	ren->new_dfd = READ_ONCE(sqe->len);  in io_renameat_prep()
3694 	ren->flags = READ_ONCE(sqe->rename_flags);  in io_renameat_prep()
3729 			    const struct io_uring_sqe *sqe)  in io_unlinkat_prep()  argument
3736 	if (sqe->ioprio || sqe->off || sqe->len || sqe->buf_index ||  in io_unlinkat_prep()
3737 	    sqe->splice_fd_in)  in io_unlinkat_prep()
3742 	un->dfd = READ_ONCE(sqe->fd);  in io_unlinkat_prep()
3744 	un->flags = READ_ONCE(sqe->unlink_flags);  in io_unlinkat_prep()
3748 	fname = u64_to_user_ptr(READ_ONCE(sqe->addr));  in io_unlinkat_prep()
3778 			    const struct io_uring_sqe *sqe)  in io_mkdirat_prep()  argument
3785 	if (sqe->ioprio || sqe->off || sqe->rw_flags || sqe->buf_index ||  in io_mkdirat_prep()
3786 	    sqe->splice_fd_in)  in io_mkdirat_prep()
3791 	mkd->dfd = READ_ONCE(sqe->fd);  in io_mkdirat_prep()
3792 	mkd->mode = READ_ONCE(sqe->len);  in io_mkdirat_prep()
3794 	fname = u64_to_user_ptr(READ_ONCE(sqe->addr));  in io_mkdirat_prep()
3821 			    const struct io_uring_sqe *sqe)  in io_symlinkat_prep()  argument
3828 	if (sqe->ioprio || sqe->len || sqe->rw_flags || sqe->buf_index ||  in io_symlinkat_prep()
3829 	    sqe->splice_fd_in)  in io_symlinkat_prep()
3834 	sl->new_dfd = READ_ONCE(sqe->fd);  in io_symlinkat_prep()
3835 	oldpath = u64_to_user_ptr(READ_ONCE(sqe->addr));  in io_symlinkat_prep()
3836 	newpath = u64_to_user_ptr(READ_ONCE(sqe->addr2));  in io_symlinkat_prep()
3870 			    const struct io_uring_sqe *sqe)  in io_linkat_prep()  argument
3877 	if (sqe->ioprio || sqe->rw_flags || sqe->buf_index || sqe->splice_fd_in)  in io_linkat_prep()
3882 	lnk->old_dfd = READ_ONCE(sqe->fd);  in io_linkat_prep()
3883 	lnk->new_dfd = READ_ONCE(sqe->len);  in io_linkat_prep()
3884 	oldf = u64_to_user_ptr(READ_ONCE(sqe->addr));  in io_linkat_prep()
3885 	newf = u64_to_user_ptr(READ_ONCE(sqe->addr2));  in io_linkat_prep()
3886 	lnk->flags = READ_ONCE(sqe->hardlink_flags);  in io_linkat_prep()
3921 			    const struct io_uring_sqe *sqe)  in io_shutdown_prep()  argument
3926 	if (unlikely(sqe->ioprio || sqe->off || sqe->addr || sqe->rw_flags ||  in io_shutdown_prep()
3927 		     sqe->buf_index || sqe->splice_fd_in))  in io_shutdown_prep()
3930 	req->shutdown.how = READ_ONCE(sqe->len);  in io_shutdown_prep()
3961 			    const struct io_uring_sqe *sqe)  in __io_splice_prep()  argument
3970 	sp->len = READ_ONCE(sqe->len);  in __io_splice_prep()
3971 	sp->flags = READ_ONCE(sqe->splice_flags);  in __io_splice_prep()
3976 	sp->file_in = io_file_get(req->ctx, req, READ_ONCE(sqe->splice_fd_in),  in __io_splice_prep()
3985 		       const struct io_uring_sqe *sqe)  in io_tee_prep()  argument
3987 	if (READ_ONCE(sqe->splice_off_in) || READ_ONCE(sqe->off))  in io_tee_prep()
3989 	return __io_splice_prep(req, sqe);  in io_tee_prep()
4015 static int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_splice_prep()  argument
4019 	sp->off_in = READ_ONCE(sqe->splice_off_in);  in io_splice_prep()
4020 	sp->off_out = READ_ONCE(sqe->off);  in io_splice_prep()
4021 	return __io_splice_prep(req, sqe);  in io_splice_prep()
4066 static int io_fsync_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_fsync_prep()  argument
4075 	if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index ||  in io_fsync_prep()
4076 		     sqe->splice_fd_in))  in io_fsync_prep()
4079 	req->sync.flags = READ_ONCE(sqe->fsync_flags);  in io_fsync_prep()
4083 	req->sync.off = READ_ONCE(sqe->off);  in io_fsync_prep()
4084 	req->sync.len = READ_ONCE(sqe->len);  in io_fsync_prep()
4107 			     const struct io_uring_sqe *sqe)  in io_fallocate_prep()  argument
4109 	if (sqe->ioprio || sqe->buf_index || sqe->rw_flags ||  in io_fallocate_prep()
4110 	    sqe->splice_fd_in)  in io_fallocate_prep()
4115 	req->sync.off = READ_ONCE(sqe->off);  in io_fallocate_prep()
4116 	req->sync.len = READ_ONCE(sqe->addr);  in io_fallocate_prep()
4117 	req->sync.mode = READ_ONCE(sqe->len);  in io_fallocate_prep()
4136 static int __io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in __io_openat_prep()  argument
4143 	if (unlikely(sqe->ioprio || sqe->buf_index))  in __io_openat_prep()
4152 	req->open.dfd = READ_ONCE(sqe->fd);  in __io_openat_prep()
4153 	fname = u64_to_user_ptr(READ_ONCE(sqe->addr));  in __io_openat_prep()
4161 	req->open.file_slot = READ_ONCE(sqe->file_index);  in __io_openat_prep()
4170 static int io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_openat_prep()  argument
4172 	u64 mode = READ_ONCE(sqe->len);  in io_openat_prep()
4173 	u64 flags = READ_ONCE(sqe->open_flags);  in io_openat_prep()
4176 	return __io_openat_prep(req, sqe);  in io_openat_prep()
4179 static int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_openat2_prep()  argument
4185 	how = u64_to_user_ptr(READ_ONCE(sqe->addr2));  in io_openat2_prep()
4186 	len = READ_ONCE(sqe->len);  in io_openat2_prep()
4195 	return __io_openat_prep(req, sqe);  in io_openat2_prep()
4270 				  const struct io_uring_sqe *sqe)  in io_remove_buffers_prep()  argument
4275 	if (sqe->ioprio || sqe->rw_flags || sqe->addr || sqe->len || sqe->off ||  in io_remove_buffers_prep()
4276 	    sqe->splice_fd_in)  in io_remove_buffers_prep()
4279 	tmp = READ_ONCE(sqe->fd);  in io_remove_buffers_prep()
4285 	p->bgid = READ_ONCE(sqe->buf_group);  in io_remove_buffers_prep()
4341 				   const struct io_uring_sqe *sqe)  in io_provide_buffers_prep()  argument
4347 	if (sqe->ioprio || sqe->rw_flags || sqe->splice_fd_in)  in io_provide_buffers_prep()
4350 	tmp = READ_ONCE(sqe->fd);  in io_provide_buffers_prep()
4354 	p->addr = READ_ONCE(sqe->addr);  in io_provide_buffers_prep()
4355 	p->len = READ_ONCE(sqe->len);  in io_provide_buffers_prep()
4367 	p->bgid = READ_ONCE(sqe->buf_group);  in io_provide_buffers_prep()
4368 	tmp = READ_ONCE(sqe->off);  in io_provide_buffers_prep()
4431 			     const struct io_uring_sqe *sqe)  in io_epoll_ctl_prep()  argument
4434 	if (sqe->ioprio || sqe->buf_index || sqe->splice_fd_in)  in io_epoll_ctl_prep()
4439 	req->epoll.epfd = READ_ONCE(sqe->fd);  in io_epoll_ctl_prep()
4440 	req->epoll.op = READ_ONCE(sqe->len);  in io_epoll_ctl_prep()
4441 	req->epoll.fd = READ_ONCE(sqe->off);  in io_epoll_ctl_prep()
4446 		ev = u64_to_user_ptr(READ_ONCE(sqe->addr));  in io_epoll_ctl_prep()
4477 static int io_madvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_madvise_prep()  argument
4480 	if (sqe->ioprio || sqe->buf_index || sqe->off || sqe->splice_fd_in)  in io_madvise_prep()
4485 	req->madvise.addr = READ_ONCE(sqe->addr);  in io_madvise_prep()
4486 	req->madvise.len = READ_ONCE(sqe->len);  in io_madvise_prep()
4487 	req->madvise.advice = READ_ONCE(sqe->fadvise_advice);  in io_madvise_prep()
4513 static int io_fadvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_fadvise_prep()  argument
4515 	if (sqe->ioprio || sqe->buf_index || sqe->addr || sqe->splice_fd_in)  in io_fadvise_prep()
4520 	req->fadvise.offset = READ_ONCE(sqe->off);  in io_fadvise_prep()
4521 	req->fadvise.len = READ_ONCE(sqe->len);  in io_fadvise_prep()
4522 	req->fadvise.advice = READ_ONCE(sqe->fadvise_advice);  in io_fadvise_prep()
4549 static int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_statx_prep()  argument
4553 	if (sqe->ioprio || sqe->buf_index || sqe->splice_fd_in)  in io_statx_prep()
4558 	req->statx.dfd = READ_ONCE(sqe->fd);  in io_statx_prep()
4559 	req->statx.mask = READ_ONCE(sqe->len);  in io_statx_prep()
4560 	req->statx.filename = u64_to_user_ptr(READ_ONCE(sqe->addr));  in io_statx_prep()
4561 	req->statx.buffer = u64_to_user_ptr(READ_ONCE(sqe->addr2));  in io_statx_prep()
4562 	req->statx.flags = READ_ONCE(sqe->statx_flags);  in io_statx_prep()
4584 static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_close_prep()  argument
4588 	if (sqe->ioprio || sqe->off || sqe->addr || sqe->len ||  in io_close_prep()
4589 	    sqe->rw_flags || sqe->buf_index)  in io_close_prep()
4594 	req->close.fd = READ_ONCE(sqe->fd);  in io_close_prep()
4595 	req->close.file_slot = READ_ONCE(sqe->file_index);  in io_close_prep()
4653 static int io_sfr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_sfr_prep()  argument
4659 	if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index ||  in io_sfr_prep()
4660 		     sqe->splice_fd_in))  in io_sfr_prep()
4663 	req->sync.off = READ_ONCE(sqe->off);  in io_sfr_prep()
4664 	req->sync.len = READ_ONCE(sqe->len);  in io_sfr_prep()
4665 	req->sync.flags = READ_ONCE(sqe->sync_range_flags);  in io_sfr_prep()
4727 static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_sendmsg_prep()  argument
4734 	sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));  in io_sendmsg_prep()
4735 	sr->len = READ_ONCE(sqe->len);  in io_sendmsg_prep()
4736 	sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL;  in io_sendmsg_prep()
4948 static int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_recvmsg_prep()  argument
4955 	sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));  in io_recvmsg_prep()
4956 	sr->len = READ_ONCE(sqe->len);  in io_recvmsg_prep()
4957 	sr->bgid = READ_ONCE(sqe->buf_group);  in io_recvmsg_prep()
4958 	sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL;  in io_recvmsg_prep()
5081 static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_accept_prep()  argument
5087 	if (sqe->ioprio || sqe->len || sqe->buf_index)  in io_accept_prep()
5090 	accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));  in io_accept_prep()
5091 	accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2));  in io_accept_prep()
5092 	accept->flags = READ_ONCE(sqe->accept_flags);  in io_accept_prep()
5095 	accept->file_slot = READ_ONCE(sqe->file_index);  in io_accept_prep()
5153 static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_connect_prep()  argument
5159 	if (sqe->ioprio || sqe->len || sqe->buf_index || sqe->rw_flags ||  in io_connect_prep()
5160 	    sqe->splice_fd_in)  in io_connect_prep()
5163 	conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));  in io_connect_prep()
5164 	conn->addr_len =  READ_ONCE(sqe->addr2);  in io_connect_prep()
5217 static int io_##op##_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) \
5750 static __poll_t io_poll_parse_events(const struct io_uring_sqe *sqe,  in io_poll_parse_events()  argument
5755 	events = READ_ONCE(sqe->poll32_events);  in io_poll_parse_events()
5765 			       const struct io_uring_sqe *sqe)  in io_poll_update_prep()  argument
5772 	if (sqe->ioprio || sqe->buf_index || sqe->splice_fd_in)  in io_poll_update_prep()
5774 	flags = READ_ONCE(sqe->len);  in io_poll_update_prep()
5782 	upd->old_user_data = READ_ONCE(sqe->addr);  in io_poll_update_prep()
5786 	upd->new_user_data = READ_ONCE(sqe->off);  in io_poll_update_prep()
5790 		upd->events = io_poll_parse_events(sqe, flags);  in io_poll_update_prep()
5791 	else if (sqe->poll32_events)  in io_poll_update_prep()
5814 static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_poll_add_prep()  argument
5821 	if (sqe->ioprio || sqe->buf_index || sqe->off || sqe->addr)  in io_poll_add_prep()
5823 	flags = READ_ONCE(sqe->len);  in io_poll_add_prep()
5828 	poll->events = io_poll_parse_events(sqe, flags);  in io_poll_add_prep()
6045 				  const struct io_uring_sqe *sqe)  in io_timeout_remove_prep()  argument
6053 	if (sqe->ioprio || sqe->buf_index || sqe->len || sqe->splice_fd_in)  in io_timeout_remove_prep()
6057 	tr->addr = READ_ONCE(sqe->addr);  in io_timeout_remove_prep()
6058 	tr->flags = READ_ONCE(sqe->timeout_flags);  in io_timeout_remove_prep()
6066 		if (get_timespec64(&tr->ts, u64_to_user_ptr(sqe->addr2)))  in io_timeout_remove_prep()
6114 static int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,  in io_timeout_prep()  argument
6119 	u32 off = READ_ONCE(sqe->off);  in io_timeout_prep()
6123 	if (sqe->ioprio || sqe->buf_index || sqe->len != 1 ||  in io_timeout_prep()
6124 	    sqe->splice_fd_in)  in io_timeout_prep()
6128 	flags = READ_ONCE(sqe->timeout_flags);  in io_timeout_prep()
6147 	if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr)))  in io_timeout_prep()
6279 				const struct io_uring_sqe *sqe)  in io_async_cancel_prep()  argument
6285 	if (sqe->ioprio || sqe->off || sqe->len || sqe->cancel_flags ||  in io_async_cancel_prep()
6286 	    sqe->splice_fd_in)  in io_async_cancel_prep()
6289 	req->cancel.addr = READ_ONCE(sqe->addr);  in io_async_cancel_prep()
6323 				const struct io_uring_sqe *sqe)  in io_rsrc_update_prep()  argument
6327 	if (sqe->ioprio || sqe->rw_flags || sqe->splice_fd_in)  in io_rsrc_update_prep()
6330 	req->rsrc_update.offset = READ_ONCE(sqe->off);  in io_rsrc_update_prep()
6331 	req->rsrc_update.nr_args = READ_ONCE(sqe->len);  in io_rsrc_update_prep()
6334 	req->rsrc_update.arg = READ_ONCE(sqe->addr);  in io_rsrc_update_prep()
6361 static int io_req_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_req_prep()  argument
6369 		return io_read_prep(req, sqe);  in io_req_prep()
6373 		return io_write_prep(req, sqe);  in io_req_prep()
6375 		return io_poll_add_prep(req, sqe);  in io_req_prep()
6377 		return io_poll_update_prep(req, sqe);  in io_req_prep()
6379 		return io_fsync_prep(req, sqe);  in io_req_prep()
6381 		return io_sfr_prep(req, sqe);  in io_req_prep()
6384 		return io_sendmsg_prep(req, sqe);  in io_req_prep()
6387 		return io_recvmsg_prep(req, sqe);  in io_req_prep()
6389 		return io_connect_prep(req, sqe);  in io_req_prep()
6391 		return io_timeout_prep(req, sqe, false);  in io_req_prep()
6393 		return io_timeout_remove_prep(req, sqe);  in io_req_prep()
6395 		return io_async_cancel_prep(req, sqe);  in io_req_prep()
6397 		return io_timeout_prep(req, sqe, true);  in io_req_prep()
6399 		return io_accept_prep(req, sqe);  in io_req_prep()
6401 		return io_fallocate_prep(req, sqe);  in io_req_prep()
6403 		return io_openat_prep(req, sqe);  in io_req_prep()
6405 		return io_close_prep(req, sqe);  in io_req_prep()
6407 		return io_rsrc_update_prep(req, sqe);  in io_req_prep()
6409 		return io_statx_prep(req, sqe);  in io_req_prep()
6411 		return io_fadvise_prep(req, sqe);  in io_req_prep()
6413 		return io_madvise_prep(req, sqe);  in io_req_prep()
6415 		return io_openat2_prep(req, sqe);  in io_req_prep()
6417 		return io_epoll_ctl_prep(req, sqe);  in io_req_prep()
6419 		return io_splice_prep(req, sqe);  in io_req_prep()
6421 		return io_provide_buffers_prep(req, sqe);  in io_req_prep()
6423 		return io_remove_buffers_prep(req, sqe);  in io_req_prep()
6425 		return io_tee_prep(req, sqe);  in io_req_prep()
6427 		return io_shutdown_prep(req, sqe);  in io_req_prep()
6429 		return io_renameat_prep(req, sqe);  in io_req_prep()
6431 		return io_unlinkat_prep(req, sqe);  in io_req_prep()
6433 		return io_mkdirat_prep(req, sqe);  in io_req_prep()
6435 		return io_symlinkat_prep(req, sqe);  in io_req_prep()
6437 		return io_linkat_prep(req, sqe);  in io_req_prep()
7048 		       const struct io_uring_sqe *sqe)  in io_init_req()  argument
7056 	req->opcode = READ_ONCE(sqe->opcode);  in io_init_req()
7058 	req->flags = sqe_flags = READ_ONCE(sqe->flags);  in io_init_req()
7059 	req->user_data = READ_ONCE(sqe->user_data);  in io_init_req()
7078 	personality = READ_ONCE(sqe->personality);  in io_init_req()
7099 		req->file = io_file_get(ctx, req, READ_ONCE(sqe->fd),  in io_init_req()
7110 			 const struct io_uring_sqe *sqe)  in io_submit_sqe()  argument
7116 	ret = io_init_req(ctx, req, sqe);  in io_submit_sqe()
7142 		ret = io_req_prep(req, sqe);  in io_submit_sqe()
7274 		const struct io_uring_sqe *sqe;  in io_submit_sqes()  local
7283 		sqe = io_get_sqe(ctx);  in io_submit_sqes()
7284 		if (unlikely(!sqe)) {  in io_submit_sqes()
7290 		if (io_submit_sqe(ctx, req, sqe))  in io_submit_sqes()