Lines Matching +full:rx +full:- +full:sched +full:- +full:sp
1 // SPDX-License-Identifier: GPL-2.0-or-later
14 #include <linux/sched/signal.h>
18 #include "ar-internal.h"
26 min_t(unsigned int, call->tx_winsize, in rxrpc_check_tx_space()
27 call->cong_cwnd + call->cong_extra); in rxrpc_check_tx_space()
28 rxrpc_seq_t tx_win = READ_ONCE(call->tx_hard_ack); in rxrpc_check_tx_space()
32 return call->tx_top - tx_win < win_size; in rxrpc_check_tx_space()
38 static int rxrpc_wait_for_tx_window_intr(struct rxrpc_sock *rx, in rxrpc_wait_for_tx_window_intr() argument
47 if (call->state >= RXRPC_CALL_COMPLETE) in rxrpc_wait_for_tx_window_intr()
48 return call->error; in rxrpc_wait_for_tx_window_intr()
62 static int rxrpc_wait_for_tx_window_waitall(struct rxrpc_sock *rx, in rxrpc_wait_for_tx_window_waitall() argument
68 rtt = READ_ONCE(call->peer->srtt_us) >> 3; in rxrpc_wait_for_tx_window_waitall()
74 tx_start = READ_ONCE(call->tx_hard_ack); in rxrpc_wait_for_tx_window_waitall()
79 tx_win = READ_ONCE(call->tx_hard_ack); in rxrpc_wait_for_tx_window_waitall()
83 if (call->state >= RXRPC_CALL_COMPLETE) in rxrpc_wait_for_tx_window_waitall()
84 return call->error; in rxrpc_wait_for_tx_window_waitall()
88 return -EINTR; in rxrpc_wait_for_tx_window_waitall()
103 static int rxrpc_wait_for_tx_window_nonintr(struct rxrpc_sock *rx, in rxrpc_wait_for_tx_window_nonintr() argument
112 if (call->state >= RXRPC_CALL_COMPLETE) in rxrpc_wait_for_tx_window_nonintr()
113 return call->error; in rxrpc_wait_for_tx_window_nonintr()
122 * - caller holds the socket locked
124 static int rxrpc_wait_for_tx_window(struct rxrpc_sock *rx, in rxrpc_wait_for_tx_window() argument
133 call->tx_hard_ack, call->tx_top, call->tx_winsize); in rxrpc_wait_for_tx_window()
135 add_wait_queue(&call->waitq, &myself); in rxrpc_wait_for_tx_window()
137 switch (call->interruptibility) { in rxrpc_wait_for_tx_window()
140 ret = rxrpc_wait_for_tx_window_waitall(rx, call); in rxrpc_wait_for_tx_window()
142 ret = rxrpc_wait_for_tx_window_intr(rx, call, timeo); in rxrpc_wait_for_tx_window()
147 ret = rxrpc_wait_for_tx_window_nonintr(rx, call, timeo); in rxrpc_wait_for_tx_window()
151 remove_wait_queue(&call->waitq, &myself); in rxrpc_wait_for_tx_window()
162 spin_lock_bh(&call->lock); in rxrpc_instant_resend()
164 if (call->state < RXRPC_CALL_COMPLETE) { in rxrpc_instant_resend()
165 call->rxtx_annotations[ix] = in rxrpc_instant_resend()
166 (call->rxtx_annotations[ix] & RXRPC_TX_ANNO_LAST) | in rxrpc_instant_resend()
168 if (!test_and_set_bit(RXRPC_CALL_EV_RESEND, &call->events)) in rxrpc_instant_resend()
172 spin_unlock_bh(&call->lock); in rxrpc_instant_resend()
179 static void rxrpc_notify_end_tx(struct rxrpc_sock *rx, struct rxrpc_call *call, in rxrpc_notify_end_tx() argument
183 notify_end_tx(&rx->sk, call, call->user_call_ID); in rxrpc_notify_end_tx()
191 static int rxrpc_queue_packet(struct rxrpc_sock *rx, struct rxrpc_call *call, in rxrpc_queue_packet() argument
195 struct rxrpc_skb_priv *sp = rxrpc_skb(skb); in rxrpc_queue_packet() local
197 rxrpc_seq_t seq = sp->hdr.seq; in rxrpc_queue_packet()
203 ASSERTCMP(seq, ==, call->tx_top + 1); in rxrpc_queue_packet()
211 skb->tstamp = ktime_get_real(); in rxrpc_queue_packet()
215 call->rxtx_annotations[ix] = annotation; in rxrpc_queue_packet()
217 call->rxtx_buffer[ix] = skb; in rxrpc_queue_packet()
218 call->tx_top = seq; in rxrpc_queue_packet()
224 if (last || call->state == RXRPC_CALL_SERVER_ACK_REQUEST) { in rxrpc_queue_packet()
226 write_lock_bh(&call->state_lock); in rxrpc_queue_packet()
227 switch (call->state) { in rxrpc_queue_packet()
229 call->state = RXRPC_CALL_CLIENT_AWAIT_REPLY; in rxrpc_queue_packet()
230 rxrpc_notify_end_tx(rx, call, notify_end_tx); in rxrpc_queue_packet()
233 call->state = RXRPC_CALL_SERVER_SEND_REPLY; in rxrpc_queue_packet()
235 WRITE_ONCE(call->ack_at, now + MAX_JIFFY_OFFSET); in rxrpc_queue_packet()
236 if (call->ackr_reason == RXRPC_ACK_DELAY) in rxrpc_queue_packet()
237 call->ackr_reason = 0; in rxrpc_queue_packet()
243 call->state = RXRPC_CALL_SERVER_AWAIT_ACK; in rxrpc_queue_packet()
244 rxrpc_notify_end_tx(rx, call, notify_end_tx); in rxrpc_queue_packet()
249 write_unlock_bh(&call->state_lock); in rxrpc_queue_packet()
258 case -ENETUNREACH: in rxrpc_queue_packet()
259 case -EHOSTUNREACH: in rxrpc_queue_packet()
260 case -ECONNREFUSED: in rxrpc_queue_packet()
269 unsigned long resend_at = now + call->peer->rto_j; in rxrpc_queue_packet()
271 WRITE_ONCE(call->resend_at, resend_at); in rxrpc_queue_packet()
284 * - must be called in process context
285 * - The caller holds the call user access mutex, but not the socket lock.
287 static int rxrpc_send_data(struct rxrpc_sock *rx, in rxrpc_send_data() argument
293 struct rxrpc_skb_priv *sp; in rxrpc_send_data() local
295 struct sock *sk = &rx->sk; in rxrpc_send_data()
298 bool more = msg->msg_flags & MSG_MORE; in rxrpc_send_data()
301 timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT); in rxrpc_send_data()
307 ret = -EPIPE; in rxrpc_send_data()
308 if (sk->sk_shutdown & SEND_SHUTDOWN) in rxrpc_send_data()
310 state = READ_ONCE(call->state); in rxrpc_send_data()
311 ret = -ESHUTDOWN; in rxrpc_send_data()
314 ret = -EPROTO; in rxrpc_send_data()
320 ret = -EMSGSIZE; in rxrpc_send_data()
321 if (call->tx_total_len != -1) { in rxrpc_send_data()
322 if (len - copied > call->tx_total_len) in rxrpc_send_data()
324 if (!more && len - copied != call->tx_total_len) in rxrpc_send_data()
328 skb = call->tx_pending; in rxrpc_send_data()
329 call->tx_pending = NULL; in rxrpc_send_data()
334 if (call->ackr_reason == RXRPC_ACK_PING_RESPONSE) in rxrpc_send_data()
350 ret = call->conn->security->how_much_data(call, remain, in rxrpc_send_data()
359 sk, bufsize, msg->msg_flags & MSG_DONTWAIT, &ret); in rxrpc_send_data()
363 sp = rxrpc_skb(skb); in rxrpc_send_data()
364 sp->rx_flags |= RXRPC_SKB_TX_BUFFER; in rxrpc_send_data()
369 ASSERTCMP(skb->mark, ==, 0); in rxrpc_send_data()
373 sp->remain = chunk; in rxrpc_send_data()
374 if (sp->remain > skb_tailroom(skb)) in rxrpc_send_data()
375 sp->remain = skb_tailroom(skb); in rxrpc_send_data()
381 sp->remain); in rxrpc_send_data()
383 skb->ip_summed = CHECKSUM_UNNECESSARY; in rxrpc_send_data()
387 sp = rxrpc_skb(skb); in rxrpc_send_data()
395 if (copy > sp->remain) in rxrpc_send_data()
396 copy = sp->remain; in rxrpc_send_data()
399 ret = skb_add_data(skb, &msg->msg_iter, copy); in rxrpc_send_data()
403 sp->remain -= copy; in rxrpc_send_data()
404 skb->mark += copy; in rxrpc_send_data()
406 if (call->tx_total_len != -1) in rxrpc_send_data()
407 call->tx_total_len -= copy; in rxrpc_send_data()
412 if (call->state == RXRPC_CALL_COMPLETE) in rxrpc_send_data()
416 if (sp->remain <= 0 || in rxrpc_send_data()
418 struct rxrpc_connection *conn = call->conn; in rxrpc_send_data()
421 seq = call->tx_top + 1; in rxrpc_send_data()
423 sp->hdr.seq = seq; in rxrpc_send_data()
424 sp->hdr._rsvd = 0; in rxrpc_send_data()
425 sp->hdr.flags = conn->out_clientflag; in rxrpc_send_data()
428 sp->hdr.flags |= RXRPC_LAST_PACKET; in rxrpc_send_data()
429 else if (call->tx_top - call->tx_hard_ack < in rxrpc_send_data()
430 call->tx_winsize) in rxrpc_send_data()
431 sp->hdr.flags |= RXRPC_MORE_PACKETS; in rxrpc_send_data()
433 ret = call->security->secure_packet(call, skb, skb->mark); in rxrpc_send_data()
437 ret = rxrpc_queue_packet(rx, call, skb, in rxrpc_send_data()
447 if (READ_ONCE(call->state) == RXRPC_CALL_COMPLETE) { in rxrpc_send_data()
448 read_lock_bh(&call->state_lock); in rxrpc_send_data()
449 if (call->error < 0) in rxrpc_send_data()
450 ret = call->error; in rxrpc_send_data()
451 read_unlock_bh(&call->state_lock); in rxrpc_send_data()
454 call->tx_pending = skb; in rxrpc_send_data()
460 _leave(" = %d", call->error); in rxrpc_send_data()
461 return call->error; in rxrpc_send_data()
469 ret = -EFAULT; in rxrpc_send_data()
473 ret = -EAGAIN; in rxrpc_send_data()
474 if (msg->msg_flags & MSG_DONTWAIT) in rxrpc_send_data()
476 mutex_unlock(&call->user_mutex); in rxrpc_send_data()
478 ret = rxrpc_wait_for_tx_window(rx, call, &timeo, in rxrpc_send_data()
479 msg->msg_flags & MSG_WAITALL); in rxrpc_send_data()
482 if (call->interruptibility == RXRPC_INTERRUPTIBLE) { in rxrpc_send_data()
483 if (mutex_lock_interruptible(&call->user_mutex) < 0) { in rxrpc_send_data()
488 mutex_lock(&call->user_mutex); in rxrpc_send_data()
503 if (msg->msg_controllen == 0) in rxrpc_sendmsg_cmsg()
504 return -EINVAL; in rxrpc_sendmsg_cmsg()
508 return -EINVAL; in rxrpc_sendmsg_cmsg()
510 len = cmsg->cmsg_len - sizeof(struct cmsghdr); in rxrpc_sendmsg_cmsg()
512 cmsg->cmsg_level, cmsg->cmsg_type, len); in rxrpc_sendmsg_cmsg()
514 if (cmsg->cmsg_level != SOL_RXRPC) in rxrpc_sendmsg_cmsg()
517 switch (cmsg->cmsg_type) { in rxrpc_sendmsg_cmsg()
519 if (msg->msg_flags & MSG_CMSG_COMPAT) { in rxrpc_sendmsg_cmsg()
521 return -EINVAL; in rxrpc_sendmsg_cmsg()
522 p->call.user_call_ID = *(u32 *)CMSG_DATA(cmsg); in rxrpc_sendmsg_cmsg()
525 return -EINVAL; in rxrpc_sendmsg_cmsg()
526 p->call.user_call_ID = *(unsigned long *) in rxrpc_sendmsg_cmsg()
533 if (p->command != RXRPC_CMD_SEND_DATA) in rxrpc_sendmsg_cmsg()
534 return -EINVAL; in rxrpc_sendmsg_cmsg()
535 p->command = RXRPC_CMD_SEND_ABORT; in rxrpc_sendmsg_cmsg()
536 if (len != sizeof(p->abort_code)) in rxrpc_sendmsg_cmsg()
537 return -EINVAL; in rxrpc_sendmsg_cmsg()
538 p->abort_code = *(unsigned int *)CMSG_DATA(cmsg); in rxrpc_sendmsg_cmsg()
539 if (p->abort_code == 0) in rxrpc_sendmsg_cmsg()
540 return -EINVAL; in rxrpc_sendmsg_cmsg()
544 if (p->command != RXRPC_CMD_SEND_DATA) in rxrpc_sendmsg_cmsg()
545 return -EINVAL; in rxrpc_sendmsg_cmsg()
546 p->command = RXRPC_CMD_CHARGE_ACCEPT; in rxrpc_sendmsg_cmsg()
548 return -EINVAL; in rxrpc_sendmsg_cmsg()
552 p->exclusive = true; in rxrpc_sendmsg_cmsg()
554 return -EINVAL; in rxrpc_sendmsg_cmsg()
558 p->upgrade = true; in rxrpc_sendmsg_cmsg()
560 return -EINVAL; in rxrpc_sendmsg_cmsg()
564 if (p->call.tx_total_len != -1 || len != sizeof(__s64)) in rxrpc_sendmsg_cmsg()
565 return -EINVAL; in rxrpc_sendmsg_cmsg()
566 p->call.tx_total_len = *(__s64 *)CMSG_DATA(cmsg); in rxrpc_sendmsg_cmsg()
567 if (p->call.tx_total_len < 0) in rxrpc_sendmsg_cmsg()
568 return -EINVAL; in rxrpc_sendmsg_cmsg()
573 return -EINVAL; in rxrpc_sendmsg_cmsg()
574 memcpy(&p->call.timeouts, CMSG_DATA(cmsg), len); in rxrpc_sendmsg_cmsg()
575 p->call.nr_timeouts = len / 4; in rxrpc_sendmsg_cmsg()
576 if (p->call.timeouts.hard > INT_MAX / HZ) in rxrpc_sendmsg_cmsg()
577 return -ERANGE; in rxrpc_sendmsg_cmsg()
578 if (p->call.nr_timeouts >= 2 && p->call.timeouts.idle > 60 * 60 * 1000) in rxrpc_sendmsg_cmsg()
579 return -ERANGE; in rxrpc_sendmsg_cmsg()
580 if (p->call.nr_timeouts >= 3 && p->call.timeouts.normal > 60 * 60 * 1000) in rxrpc_sendmsg_cmsg()
581 return -ERANGE; in rxrpc_sendmsg_cmsg()
585 return -EINVAL; in rxrpc_sendmsg_cmsg()
590 return -EINVAL; in rxrpc_sendmsg_cmsg()
591 if (p->call.tx_total_len != -1 && p->command != RXRPC_CMD_SEND_DATA) in rxrpc_sendmsg_cmsg()
592 return -EINVAL; in rxrpc_sendmsg_cmsg()
599 * - Called with the socket lock held, which it must release.
600 * - If it returns a call, the call's lock will need releasing by the caller.
603 rxrpc_new_client_call_for_sendmsg(struct rxrpc_sock *rx, struct msghdr *msg, in rxrpc_new_client_call_for_sendmsg() argument
605 __releases(&rx->sk.sk_lock.slock) in rxrpc_new_client_call_for_sendmsg()
606 __acquires(&call->user_mutex) in rxrpc_new_client_call_for_sendmsg()
612 DECLARE_SOCKADDR(struct sockaddr_rxrpc *, srx, msg->msg_name); in rxrpc_new_client_call_for_sendmsg()
616 if (!msg->msg_name) { in rxrpc_new_client_call_for_sendmsg()
617 release_sock(&rx->sk); in rxrpc_new_client_call_for_sendmsg()
618 return ERR_PTR(-EDESTADDRREQ); in rxrpc_new_client_call_for_sendmsg()
621 key = rx->key; in rxrpc_new_client_call_for_sendmsg()
622 if (key && !rx->key->payload.data[0]) in rxrpc_new_client_call_for_sendmsg()
626 cp.local = rx->local; in rxrpc_new_client_call_for_sendmsg()
627 cp.key = rx->key; in rxrpc_new_client_call_for_sendmsg()
628 cp.security_level = rx->min_sec_level; in rxrpc_new_client_call_for_sendmsg()
629 cp.exclusive = rx->exclusive | p->exclusive; in rxrpc_new_client_call_for_sendmsg()
630 cp.upgrade = p->upgrade; in rxrpc_new_client_call_for_sendmsg()
631 cp.service_id = srx->srx_service; in rxrpc_new_client_call_for_sendmsg()
632 call = rxrpc_new_client_call(rx, &cp, srx, &p->call, GFP_KERNEL, in rxrpc_new_client_call_for_sendmsg()
643 * - caller holds the socket locked
644 * - the socket may be either a client socket or a server socket
646 int rxrpc_do_sendmsg(struct rxrpc_sock *rx, struct msghdr *msg, size_t len) in rxrpc_do_sendmsg() argument
647 __releases(&rx->sk.sk_lock.slock) in rxrpc_do_sendmsg()
648 __releases(&call->user_mutex) in rxrpc_do_sendmsg()
657 .call.tx_total_len = -1, in rxrpc_do_sendmsg()
674 ret = -EINVAL; in rxrpc_do_sendmsg()
675 if (rx->sk.sk_state != RXRPC_SERVER_LISTENING) in rxrpc_do_sendmsg()
677 ret = rxrpc_user_charge_accept(rx, p.call.user_call_ID); in rxrpc_do_sendmsg()
681 call = rxrpc_find_call_by_user_ID(rx, p.call.user_call_ID); in rxrpc_do_sendmsg()
683 ret = -EBADSLT; in rxrpc_do_sendmsg()
686 call = rxrpc_new_client_call_for_sendmsg(rx, msg, &p); in rxrpc_do_sendmsg()
692 if (READ_ONCE(call->state) == RXRPC_CALL_COMPLETE) in rxrpc_do_sendmsg()
695 switch (READ_ONCE(call->state)) { in rxrpc_do_sendmsg()
701 ret = -EBUSY; in rxrpc_do_sendmsg()
707 ret = mutex_lock_interruptible(&call->user_mutex); in rxrpc_do_sendmsg()
708 release_sock(&rx->sk); in rxrpc_do_sendmsg()
710 ret = -ERESTARTSYS; in rxrpc_do_sendmsg()
714 if (p.call.tx_total_len != -1) { in rxrpc_do_sendmsg()
715 ret = -EINVAL; in rxrpc_do_sendmsg()
716 if (call->tx_total_len != -1 || in rxrpc_do_sendmsg()
717 call->tx_pending || in rxrpc_do_sendmsg()
718 call->tx_top != 0) in rxrpc_do_sendmsg()
720 call->tx_total_len = p.call.tx_total_len; in rxrpc_do_sendmsg()
729 WRITE_ONCE(call->next_rx_timo, j); in rxrpc_do_sendmsg()
735 WRITE_ONCE(call->next_req_timo, j); in rxrpc_do_sendmsg()
742 WRITE_ONCE(call->expect_term_by, j); in rxrpc_do_sendmsg()
749 state = READ_ONCE(call->state); in rxrpc_do_sendmsg()
751 call->debug_id, call->user_call_ID, state, call->conn); in rxrpc_do_sendmsg()
755 ret = -ESHUTDOWN; in rxrpc_do_sendmsg()
758 if (rxrpc_abort_call("CMD", call, 0, p.abort_code, -ECONNABORTED)) in rxrpc_do_sendmsg()
761 ret = -EINVAL; in rxrpc_do_sendmsg()
763 ret = rxrpc_send_data(rx, call, msg, len, NULL, &dropped_lock); in rxrpc_do_sendmsg()
768 mutex_unlock(&call->user_mutex); in rxrpc_do_sendmsg()
775 release_sock(&rx->sk); in rxrpc_do_sendmsg()
780 * rxrpc_kernel_send_data - Allow a kernel service to send data on a call
799 _enter("{%d,%s},", call->debug_id, rxrpc_call_states[call->state]); in rxrpc_kernel_send_data()
801 ASSERTCMP(msg->msg_name, ==, NULL); in rxrpc_kernel_send_data()
802 ASSERTCMP(msg->msg_control, ==, NULL); in rxrpc_kernel_send_data()
804 mutex_lock(&call->user_mutex); in rxrpc_kernel_send_data()
807 call->debug_id, call->user_call_ID, call->state, call->conn); in rxrpc_kernel_send_data()
809 switch (READ_ONCE(call->state)) { in rxrpc_kernel_send_data()
813 ret = rxrpc_send_data(rxrpc_sk(sock->sk), call, msg, len, in rxrpc_kernel_send_data()
817 read_lock_bh(&call->state_lock); in rxrpc_kernel_send_data()
818 ret = call->error; in rxrpc_kernel_send_data()
819 read_unlock_bh(&call->state_lock); in rxrpc_kernel_send_data()
824 ret = -EPROTO; in rxrpc_kernel_send_data()
829 mutex_unlock(&call->user_mutex); in rxrpc_kernel_send_data()
836 * rxrpc_kernel_abort_call - Allow a kernel service to abort a call
841 * @why: 3-char string indicating why.
851 _enter("{%d},%d,%d,%s", call->debug_id, abort_code, error, why); in rxrpc_kernel_abort_call()
853 mutex_lock(&call->user_mutex); in rxrpc_kernel_abort_call()
859 mutex_unlock(&call->user_mutex); in rxrpc_kernel_abort_call()
865 * rxrpc_kernel_set_tx_length - Set the total Tx length on a call
871 * allows buffer-to-packet encrypt-and-copy to be performed.
879 WARN_ON(call->tx_total_len != -1); in rxrpc_kernel_set_tx_length()
880 call->tx_total_len = tx_total_len; in rxrpc_kernel_set_tx_length()