23 static bool rxrpc_check_tx_space(struct rxrpc_call *call, rxrpc_seq_t *_tx_win)  in rxrpc_check_tx_space()  argument
26 		min_t(unsigned int, call->tx_winsize,  in rxrpc_check_tx_space()
27 		      call->cong_cwnd + call->cong_extra);  in rxrpc_check_tx_space()
28 	rxrpc_seq_t tx_win = READ_ONCE(call->tx_hard_ack);  in rxrpc_check_tx_space()
32 	return call->tx_top - tx_win < win_size;  in rxrpc_check_tx_space()
39 					 struct rxrpc_call *call,  in rxrpc_wait_for_tx_window_intr()  argument
44 		if (rxrpc_check_tx_space(call, NULL))  in rxrpc_wait_for_tx_window_intr()
47 		if (call->state >= RXRPC_CALL_COMPLETE)  in rxrpc_wait_for_tx_window_intr()
48 			return call->error;  in rxrpc_wait_for_tx_window_intr()
53 		trace_rxrpc_transmit(call, rxrpc_transmit_wait);  in rxrpc_wait_for_tx_window_intr()
63 					    struct rxrpc_call *call)  in rxrpc_wait_for_tx_window_waitall()  argument
68 	rtt = READ_ONCE(call->peer->srtt_us) >> 3;  in rxrpc_wait_for_tx_window_waitall()
74 	tx_start = READ_ONCE(call->tx_hard_ack);  in rxrpc_wait_for_tx_window_waitall()
79 		tx_win = READ_ONCE(call->tx_hard_ack);  in rxrpc_wait_for_tx_window_waitall()
80 		if (rxrpc_check_tx_space(call, &tx_win))  in rxrpc_wait_for_tx_window_waitall()
83 		if (call->state >= RXRPC_CALL_COMPLETE)  in rxrpc_wait_for_tx_window_waitall()
84 			return call->error;  in rxrpc_wait_for_tx_window_waitall()
95 		trace_rxrpc_transmit(call, rxrpc_transmit_wait);  in rxrpc_wait_for_tx_window_waitall()
104 					    struct rxrpc_call *call,  in rxrpc_wait_for_tx_window_nonintr()  argument
109 		if (rxrpc_check_tx_space(call, NULL))  in rxrpc_wait_for_tx_window_nonintr()
112 		if (call->state >= RXRPC_CALL_COMPLETE)  in rxrpc_wait_for_tx_window_nonintr()
113 			return call->error;  in rxrpc_wait_for_tx_window_nonintr()
115 		trace_rxrpc_transmit(call, rxrpc_transmit_wait);  in rxrpc_wait_for_tx_window_nonintr()
125 				    struct rxrpc_call *call,  in rxrpc_wait_for_tx_window()  argument
133 	       call->tx_hard_ack, call->tx_top, call->tx_winsize);  in rxrpc_wait_for_tx_window()
135 	add_wait_queue(&call->waitq, &myself);  in rxrpc_wait_for_tx_window()
137 	switch (call->interruptibility) {  in rxrpc_wait_for_tx_window()
140 			ret = rxrpc_wait_for_tx_window_waitall(rx, call);  in rxrpc_wait_for_tx_window()
142 			ret = rxrpc_wait_for_tx_window_intr(rx, call, timeo);  in rxrpc_wait_for_tx_window()
147 		ret = rxrpc_wait_for_tx_window_nonintr(rx, call, timeo);  in rxrpc_wait_for_tx_window()
151 	remove_wait_queue(&call->waitq, &myself);  in rxrpc_wait_for_tx_window()
160 static inline void rxrpc_instant_resend(struct rxrpc_call *call, int ix)  in rxrpc_instant_resend()  argument
162 	spin_lock_bh(&call->lock);  in rxrpc_instant_resend()
164 	if (call->state < RXRPC_CALL_COMPLETE) {  in rxrpc_instant_resend()
165 		call->rxtx_annotations[ix] =  in rxrpc_instant_resend()
166 			(call->rxtx_annotations[ix] & RXRPC_TX_ANNO_LAST) |  in rxrpc_instant_resend()
168 		if (!test_and_set_bit(RXRPC_CALL_EV_RESEND, &call->events))  in rxrpc_instant_resend()
169 			rxrpc_queue_call(call);  in rxrpc_instant_resend()
172 	spin_unlock_bh(&call->lock);  in rxrpc_instant_resend()
176  * Notify the owner of the call that the transmit phase is ended and the last
179 static void rxrpc_notify_end_tx(struct rxrpc_sock *rx, struct rxrpc_call *call,  in rxrpc_notify_end_tx()  argument
183 		notify_end_tx(&rx->sk, call, call->user_call_ID);  in rxrpc_notify_end_tx()
191 static int rxrpc_queue_packet(struct rxrpc_sock *rx, struct rxrpc_call *call,  in rxrpc_queue_packet()  argument
203 	ASSERTCMP(seq, ==, call->tx_top + 1);  in rxrpc_queue_packet()
215 	call->rxtx_annotations[ix] = annotation;  in rxrpc_queue_packet()
217 	call->rxtx_buffer[ix] = skb;  in rxrpc_queue_packet()
218 	call->tx_top = seq;  in rxrpc_queue_packet()
220 		trace_rxrpc_transmit(call, rxrpc_transmit_queue_last);  in rxrpc_queue_packet()
222 		trace_rxrpc_transmit(call, rxrpc_transmit_queue);  in rxrpc_queue_packet()
224 	if (last || call->state == RXRPC_CALL_SERVER_ACK_REQUEST) {  in rxrpc_queue_packet()
226 		write_lock_bh(&call->state_lock);  in rxrpc_queue_packet()
227 		switch (call->state) {  in rxrpc_queue_packet()
229 			call->state = RXRPC_CALL_CLIENT_AWAIT_REPLY;  in rxrpc_queue_packet()
230 			rxrpc_notify_end_tx(rx, call, notify_end_tx);  in rxrpc_queue_packet()
233 			call->state = RXRPC_CALL_SERVER_SEND_REPLY;  in rxrpc_queue_packet()
235 			WRITE_ONCE(call->ack_at, now + MAX_JIFFY_OFFSET);  in rxrpc_queue_packet()
236 			if (call->ackr_reason == RXRPC_ACK_DELAY)  in rxrpc_queue_packet()
237 				call->ackr_reason = 0;  in rxrpc_queue_packet()
238 			trace_rxrpc_timer(call, rxrpc_timer_init_for_send_reply, now);  in rxrpc_queue_packet()
243 			call->state = RXRPC_CALL_SERVER_AWAIT_ACK;  in rxrpc_queue_packet()
244 			rxrpc_notify_end_tx(rx, call, notify_end_tx);  in rxrpc_queue_packet()
249 		write_unlock_bh(&call->state_lock);  in rxrpc_queue_packet()
252 	if (seq == 1 && rxrpc_is_client_call(call))  in rxrpc_queue_packet()
253 		rxrpc_expose_client_call(call);  in rxrpc_queue_packet()
255 	ret = rxrpc_send_data_packet(call, skb, false);  in rxrpc_queue_packet()
261 			rxrpc_set_call_completion(call, RXRPC_CALL_LOCAL_ERROR,  in rxrpc_queue_packet()
266 		rxrpc_instant_resend(call, ix);  in rxrpc_queue_packet()
269 		unsigned long resend_at = now + call->peer->rto_j;  in rxrpc_queue_packet()
271 		WRITE_ONCE(call->resend_at, resend_at);  in rxrpc_queue_packet()
272 		rxrpc_reduce_call_timer(call, resend_at, now,  in rxrpc_queue_packet()
285  * - The caller holds the call user access mutex, but not the socket lock.
288 			   struct rxrpc_call *call,  in rxrpc_send_data()  argument
310 	state = READ_ONCE(call->state);  in rxrpc_send_data()
321 	if (call->tx_total_len != -1) {  in rxrpc_send_data()
322 		if (len - copied > call->tx_total_len)  in rxrpc_send_data()
324 		if (!more && len - copied != call->tx_total_len)  in rxrpc_send_data()
328 	skb = call->tx_pending;  in rxrpc_send_data()
329 	call->tx_pending = NULL;  in rxrpc_send_data()
334 		if (call->ackr_reason == RXRPC_ACK_PING_RESPONSE)  in rxrpc_send_data()
335 			rxrpc_send_ack_packet(call, false, NULL);  in rxrpc_send_data()
342 			if (!rxrpc_check_tx_space(call, NULL))  in rxrpc_send_data()
350 			ret = call->conn->security->how_much_data(call, remain,  in rxrpc_send_data()
406 			if (call->tx_total_len != -1)  in rxrpc_send_data()
407 				call->tx_total_len -= copy;  in rxrpc_send_data()
410 		/* check for the far side aborting the call or a network error  in rxrpc_send_data()
412 		if (call->state == RXRPC_CALL_COMPLETE)  in rxrpc_send_data()
418 			struct rxrpc_connection *conn = call->conn;  in rxrpc_send_data()
421 			seq = call->tx_top + 1;  in rxrpc_send_data()
429 			else if (call->tx_top - call->tx_hard_ack <  in rxrpc_send_data()
430 				 call->tx_winsize)  in rxrpc_send_data()
433 			ret = call->security->secure_packet(call, skb, skb->mark);  in rxrpc_send_data()
437 			ret = rxrpc_queue_packet(rx, call, skb,  in rxrpc_send_data()
447 	if (READ_ONCE(call->state) == RXRPC_CALL_COMPLETE) {  in rxrpc_send_data()
448 		read_lock_bh(&call->state_lock);  in rxrpc_send_data()
449 		if (call->error < 0)  in rxrpc_send_data()
450 			ret = call->error;  in rxrpc_send_data()
451 		read_unlock_bh(&call->state_lock);  in rxrpc_send_data()
454 	call->tx_pending = skb;  in rxrpc_send_data()
460 	_leave(" = %d", call->error);  in rxrpc_send_data()
461 	return call->error;  in rxrpc_send_data()
476 	mutex_unlock(&call->user_mutex);  in rxrpc_send_data()
478 	ret = rxrpc_wait_for_tx_window(rx, call, &timeo,  in rxrpc_send_data()
482 	if (call->interruptibility == RXRPC_INTERRUPTIBLE) {  in rxrpc_send_data()
483 		if (mutex_lock_interruptible(&call->user_mutex) < 0) {  in rxrpc_send_data()
488 		mutex_lock(&call->user_mutex);  in rxrpc_send_data()
522 				p->call.user_call_ID = *(u32 *)CMSG_DATA(cmsg);  in rxrpc_sendmsg_cmsg()
526 				p->call.user_call_ID = *(unsigned long *)  in rxrpc_sendmsg_cmsg()
564 			if (p->call.tx_total_len != -1 || len != sizeof(__s64))  in rxrpc_sendmsg_cmsg()
566 			p->call.tx_total_len = *(__s64 *)CMSG_DATA(cmsg);  in rxrpc_sendmsg_cmsg()
567 			if (p->call.tx_total_len < 0)  in rxrpc_sendmsg_cmsg()
574 			memcpy(&p->call.timeouts, CMSG_DATA(cmsg), len);  in rxrpc_sendmsg_cmsg()
575 			p->call.nr_timeouts = len / 4;  in rxrpc_sendmsg_cmsg()
576 			if (p->call.timeouts.hard > INT_MAX / HZ)  in rxrpc_sendmsg_cmsg()
578 			if (p->call.nr_timeouts >= 2 && p->call.timeouts.idle > 60 * 60 * 1000)  in rxrpc_sendmsg_cmsg()
580 			if (p->call.nr_timeouts >= 3 && p->call.timeouts.normal > 60 * 60 * 1000)  in rxrpc_sendmsg_cmsg()
591 	if (p->call.tx_total_len != -1 && p->command != RXRPC_CMD_SEND_DATA)  in rxrpc_sendmsg_cmsg()
598  * Create a new client call for sendmsg().
600  * - If it returns a call, the call's lock will need releasing by the caller.
606 	__acquires(&call->user_mutex)  in rxrpc_new_client_call_for_sendmsg()
609 	struct rxrpc_call *call;  in rxrpc_new_client_call_for_sendmsg()  local
632 	call = rxrpc_new_client_call(rx, &cp, srx, &p->call, GFP_KERNEL,  in rxrpc_new_client_call_for_sendmsg()
637 	_leave(" = %p\n", call);  in rxrpc_new_client_call_for_sendmsg()
638 	return call;  in rxrpc_new_client_call_for_sendmsg()
642  * send a message forming part of a client call through an RxRPC socket
648 	__releases(&call->user_mutex)  in rxrpc_do_sendmsg()
651 	struct rxrpc_call *call;  in rxrpc_do_sendmsg()  local
657 		.call.tx_total_len	= -1,  in rxrpc_do_sendmsg()
658 		.call.user_call_ID	= 0,  in rxrpc_do_sendmsg()
659 		.call.nr_timeouts	= 0,  in rxrpc_do_sendmsg()
660 		.call.interruptibility	= RXRPC_INTERRUPTIBLE,  in rxrpc_do_sendmsg()
677 		ret = rxrpc_user_charge_accept(rx, p.call.user_call_ID);  in rxrpc_do_sendmsg()
681 	call = rxrpc_find_call_by_user_ID(rx, p.call.user_call_ID);  in rxrpc_do_sendmsg()
682 	if (!call) {  in rxrpc_do_sendmsg()
686 		call = rxrpc_new_client_call_for_sendmsg(rx, msg, &p);  in rxrpc_do_sendmsg()
688 		if (IS_ERR(call))  in rxrpc_do_sendmsg()
689 			return PTR_ERR(call);  in rxrpc_do_sendmsg()
690 		/* ... and we have the call lock. */  in rxrpc_do_sendmsg()
692 		if (READ_ONCE(call->state) == RXRPC_CALL_COMPLETE)  in rxrpc_do_sendmsg()
695 		switch (READ_ONCE(call->state)) {  in rxrpc_do_sendmsg()
700 			rxrpc_put_call(call, rxrpc_call_put);  in rxrpc_do_sendmsg()
707 		ret = mutex_lock_interruptible(&call->user_mutex);  in rxrpc_do_sendmsg()
714 		if (p.call.tx_total_len != -1) {  in rxrpc_do_sendmsg()
716 			if (call->tx_total_len != -1 ||  in rxrpc_do_sendmsg()
717 			    call->tx_pending ||  in rxrpc_do_sendmsg()
718 			    call->tx_top != 0)  in rxrpc_do_sendmsg()
720 			call->tx_total_len = p.call.tx_total_len;  in rxrpc_do_sendmsg()
724 	switch (p.call.nr_timeouts) {  in rxrpc_do_sendmsg()
726 		j = msecs_to_jiffies(p.call.timeouts.normal);  in rxrpc_do_sendmsg()
727 		if (p.call.timeouts.normal > 0 && j == 0)  in rxrpc_do_sendmsg()
729 		WRITE_ONCE(call->next_rx_timo, j);  in rxrpc_do_sendmsg()
732 		j = msecs_to_jiffies(p.call.timeouts.idle);  in rxrpc_do_sendmsg()
733 		if (p.call.timeouts.idle > 0 && j == 0)  in rxrpc_do_sendmsg()
735 		WRITE_ONCE(call->next_req_timo, j);  in rxrpc_do_sendmsg()
738 		if (p.call.timeouts.hard > 0) {  in rxrpc_do_sendmsg()
739 			j = msecs_to_jiffies(p.call.timeouts.hard);  in rxrpc_do_sendmsg()
742 			WRITE_ONCE(call->expect_term_by, j);  in rxrpc_do_sendmsg()
743 			rxrpc_reduce_call_timer(call, j, now,  in rxrpc_do_sendmsg()
749 	state = READ_ONCE(call->state);  in rxrpc_do_sendmsg()
750 	_debug("CALL %d USR %lx ST %d on CONN %p",  in rxrpc_do_sendmsg()
751 	       call->debug_id, call->user_call_ID, state, call->conn);  in rxrpc_do_sendmsg()
754 		/* it's too late for this call */  in rxrpc_do_sendmsg()
758 		if (rxrpc_abort_call("CMD", call, 0, p.abort_code, -ECONNABORTED))  in rxrpc_do_sendmsg()
759 			ret = rxrpc_send_abort_packet(call);  in rxrpc_do_sendmsg()
763 		ret = rxrpc_send_data(rx, call, msg, len, NULL, &dropped_lock);  in rxrpc_do_sendmsg()
768 		mutex_unlock(&call->user_mutex);  in rxrpc_do_sendmsg()
770 	rxrpc_put_call(call, rxrpc_call_put);  in rxrpc_do_sendmsg()
780  * rxrpc_kernel_send_data - Allow a kernel service to send data on a call
781  * @sock: The socket the call is on
782  * @call: The call to send data through
787  * Allow a kernel service to send data on a call.  The call must be in an state
792 int rxrpc_kernel_send_data(struct socket *sock, struct rxrpc_call *call,  in rxrpc_kernel_send_data()  argument
799 	_enter("{%d,%s},", call->debug_id, rxrpc_call_states[call->state]);  in rxrpc_kernel_send_data()
804 	mutex_lock(&call->user_mutex);  in rxrpc_kernel_send_data()
806 	_debug("CALL %d USR %lx ST %d on CONN %p",  in rxrpc_kernel_send_data()
807 	       call->debug_id, call->user_call_ID, call->state, call->conn);  in rxrpc_kernel_send_data()
809 	switch (READ_ONCE(call->state)) {  in rxrpc_kernel_send_data()
813 		ret = rxrpc_send_data(rxrpc_sk(sock->sk), call, msg, len,  in rxrpc_kernel_send_data()
817 		read_lock_bh(&call->state_lock);  in rxrpc_kernel_send_data()
818 		ret = call->error;  in rxrpc_kernel_send_data()
819 		read_unlock_bh(&call->state_lock);  in rxrpc_kernel_send_data()
822 		/* Request phase complete for this client call */  in rxrpc_kernel_send_data()
823 		trace_rxrpc_rx_eproto(call, 0, tracepoint_string("late_send"));  in rxrpc_kernel_send_data()
829 		mutex_unlock(&call->user_mutex);  in rxrpc_kernel_send_data()
836  * rxrpc_kernel_abort_call - Allow a kernel service to abort a call
837  * @sock: The socket the call is on
838  * @call: The call to be aborted
843  * Allow a kernel service to abort a call, if it's still in an abortable state
844  * and return true if the call was aborted, false if it was already complete.
846 bool rxrpc_kernel_abort_call(struct socket *sock, struct rxrpc_call *call,  in rxrpc_kernel_abort_call()  argument
851 	_enter("{%d},%d,%d,%s", call->debug_id, abort_code, error, why);  in rxrpc_kernel_abort_call()
853 	mutex_lock(&call->user_mutex);  in rxrpc_kernel_abort_call()
855 	aborted = rxrpc_abort_call(why, call, 0, abort_code, error);  in rxrpc_kernel_abort_call()
857 		rxrpc_send_abort_packet(call);  in rxrpc_kernel_abort_call()
859 	mutex_unlock(&call->user_mutex);  in rxrpc_kernel_abort_call()
865  * rxrpc_kernel_set_tx_length - Set the total Tx length on a call
866  * @sock: The socket the call is on
867  * @call: The call to be informed
868  * @tx_total_len: The amount of data to be transmitted for this call
870  * Allow a kernel service to set the total transmit length on a call.  This
874  * request length can be set when beginning the call.
876 void rxrpc_kernel_set_tx_length(struct socket *sock, struct rxrpc_call *call,  in rxrpc_kernel_set_tx_length()  argument
879 	WARN_ON(call->tx_total_len != -1);  in rxrpc_kernel_set_tx_length()
880 	call->tx_total_len = tx_total_len;  in rxrpc_kernel_set_tx_length()