180 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);  in io_sendmsg_copy_hdr()  local
184 	return sendmsg_copy_msghdr(&iomsg->msg, sr->umsg, sr->msg_flags,  in io_sendmsg_copy_hdr()
207 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);  in io_setup_async_addr()  local
210 	if (!sr->addr || req_has_async_data(req))  in io_setup_async_addr()
240 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);  in io_sendmsg_prep()  local
245 		sr->addr = u64_to_user_ptr(READ_ONCE(sqe->addr2));  in io_sendmsg_prep()
246 		sr->addr_len = READ_ONCE(sqe->addr_len);  in io_sendmsg_prep()
251 	sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));  in io_sendmsg_prep()
252 	sr->len = READ_ONCE(sqe->len);  in io_sendmsg_prep()
253 	sr->flags = READ_ONCE(sqe->ioprio);  in io_sendmsg_prep()
254 	if (sr->flags & ~IORING_RECVSEND_POLL_FIRST)  in io_sendmsg_prep()
256 	sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL;  in io_sendmsg_prep()
257 	if (sr->msg_flags & MSG_DONTWAIT)  in io_sendmsg_prep()
262 		sr->msg_flags |= MSG_CMSG_COMPAT;  in io_sendmsg_prep()
264 	sr->done_io = 0;  in io_sendmsg_prep()
270 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);  in io_sendmsg()  local
291 	    (sr->flags & IORING_RECVSEND_POLL_FIRST))  in io_sendmsg()
294 	flags = sr->msg_flags;  in io_sendmsg()
306 			sr->done_io += ret;  in io_sendmsg()
320 		ret += sr->done_io;  in io_sendmsg()
321 	else if (sr->done_io)  in io_sendmsg()
322 		ret = sr->done_io;  in io_sendmsg()
330 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);  in io_send()  local
344 	if (sr->addr) {  in io_send()
350 			ret = move_addr_to_kernel(sr->addr, sr->addr_len, &__address);  in io_send()
355 		msg.msg_namelen = sr->addr_len;  in io_send()
359 	    (sr->flags & IORING_RECVSEND_POLL_FIRST))  in io_send()
366 	ret = import_single_range(WRITE, sr->buf, sr->len, &iov, &msg.msg_iter);  in io_send()
370 	flags = sr->msg_flags;  in io_send()
383 			sr->len -= ret;  in io_send()
384 			sr->buf += ret;  in io_send()
385 			sr->done_io += ret;  in io_send()
394 		ret += sr->done_io;  in io_send()
395 	else if (sr->done_io)  in io_send()
396 		ret = sr->done_io;  in io_send()
419 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);  in __io_recvmsg_copy_hdr()  local
423 	if (copy_from_user(&msg, sr->umsg, sizeof(*sr->umsg)))  in __io_recvmsg_copy_hdr()
432 			sr->len = iomsg->fast_iov[0].iov_len = 0;  in __io_recvmsg_copy_hdr()
440 			sr->len = iomsg->fast_iov[0].iov_len;  in __io_recvmsg_copy_hdr()
466 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);  in __io_compat_recvmsg_copy_hdr()  local
471 	if (copy_from_user(&msg, sr->umsg_compat, sizeof(msg)))  in __io_compat_recvmsg_copy_hdr()
483 			sr->len = 0;  in __io_compat_recvmsg_copy_hdr()
493 			sr->len = clen;  in __io_compat_recvmsg_copy_hdr()
544 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);  in io_recvmsg_prep()  local
549 	sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));  in io_recvmsg_prep()
550 	sr->len = READ_ONCE(sqe->len);  in io_recvmsg_prep()
551 	sr->flags = READ_ONCE(sqe->ioprio);  in io_recvmsg_prep()
552 	if (sr->flags & ~(RECVMSG_FLAGS))  in io_recvmsg_prep()
554 	sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL;  in io_recvmsg_prep()
555 	if (sr->msg_flags & MSG_DONTWAIT)  in io_recvmsg_prep()
557 	if (sr->msg_flags & MSG_ERRQUEUE)  in io_recvmsg_prep()
559 	if (sr->flags & IORING_RECV_MULTISHOT) {  in io_recvmsg_prep()
562 		if (sr->msg_flags & MSG_WAITALL)  in io_recvmsg_prep()
564 		if (req->opcode == IORING_OP_RECV && sr->len)  in io_recvmsg_prep()
571 		sr->msg_flags |= MSG_CMSG_COMPAT;  in io_recvmsg_prep()
573 	sr->done_io = 0;  in io_recvmsg_prep()
579 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);  in io_recv_prep_retry()  local
581 	sr->done_io = 0;  in io_recv_prep_retry()
582 	sr->len = 0; /* get from the provided buffer */  in io_recv_prep_retry()
624 				     struct io_sr_msg *sr, void __user **buf,  in io_recvmsg_prep_multishot()  argument
642 	sr->buf = *buf; /* stash for later copy */  in io_recvmsg_prep_multishot()
709 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);  in io_recvmsg()  local
732 	    (sr->flags & IORING_RECVSEND_POLL_FIRST))  in io_recvmsg()
738 		size_t len = sr->len;  in io_recvmsg()
745 			ret = io_recvmsg_prep_multishot(kmsg, sr, &buf, &len);  in io_recvmsg()
758 	flags = sr->msg_flags;  in io_recvmsg()
766 		ret = io_recvmsg_multishot(sock, sr, kmsg, flags,  in io_recvmsg()
769 		ret = __sys_recvmsg_sock(sock, &kmsg->msg, sr->umsg,  in io_recvmsg()
782 			sr->done_io += ret;  in io_recvmsg()
794 		ret += sr->done_io;  in io_recvmsg()
795 	else if (sr->done_io)  in io_recvmsg()
796 		ret = sr->done_io;  in io_recvmsg()
820 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);  in io_recv()  local
828 	size_t len = sr->len;  in io_recv()
831 	    (sr->flags & IORING_RECVSEND_POLL_FIRST))  in io_recv()
845 		sr->buf = buf;  in io_recv()
848 	ret = import_single_range(READ, sr->buf, len, &iov, &msg.msg_iter);  in io_recv()
861 	flags = sr->msg_flags;  in io_recv()
878 			sr->len -= ret;  in io_recv()
879 			sr->buf += ret;  in io_recv()
880 			sr->done_io += ret;  in io_recv()
893 		ret += sr->done_io;  in io_recv()
894 	else if (sr->done_io)  in io_recv()
895 		ret = sr->done_io;  in io_recv()
1145 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);  in io_sendmsg_zc()  local
1167 	    (sr->flags & IORING_RECVSEND_POLL_FIRST))  in io_sendmsg_zc()
1170 	flags = sr->msg_flags | MSG_ZEROCOPY;  in io_sendmsg_zc()
1176 	kmsg->msg.msg_ubuf = &io_notif_to_data(sr->notif)->uarg;  in io_sendmsg_zc()
1185 			sr->done_io += ret;  in io_sendmsg_zc()
1201 		ret += sr->done_io;  in io_sendmsg_zc()
1202 	else if (sr->done_io)  in io_sendmsg_zc()
1203 		ret = sr->done_io;  in io_sendmsg_zc()
1210 		io_notif_flush(sr->notif);  in io_sendmsg_zc()
1219 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);  in io_sendrecv_fail()  local
1222 		req->cqe.res = sr->done_io;  in io_sendrecv_fail()