20  * Post a call for attention by the socket or kernel service.  Further
23 void rxrpc_notify_socket(struct rxrpc_call *call)  in rxrpc_notify_socket()  argument
28 	_enter("%d", call->debug_id);  in rxrpc_notify_socket()
30 	if (!list_empty(&call->recvmsg_link))  in rxrpc_notify_socket()
35 	rx = rcu_dereference(call->socket);  in rxrpc_notify_socket()
38 		if (call->notify_rx) {  in rxrpc_notify_socket()
39 			spin_lock_bh(&call->notify_lock);  in rxrpc_notify_socket()
40 			call->notify_rx(sk, call, call->user_call_ID);  in rxrpc_notify_socket()
41 			spin_unlock_bh(&call->notify_lock);  in rxrpc_notify_socket()
44 			if (list_empty(&call->recvmsg_link)) {  in rxrpc_notify_socket()
45 				rxrpc_get_call(call, rxrpc_call_got);  in rxrpc_notify_socket()
46 				list_add_tail(&call->recvmsg_link, &rx->recvmsg_q);  in rxrpc_notify_socket()
51 				_debug("call %ps", sk->sk_data_ready);  in rxrpc_notify_socket()
62  * Pass a call terminating message to userspace.
64 static int rxrpc_recvmsg_term(struct rxrpc_call *call, struct msghdr *msg)  in rxrpc_recvmsg_term()  argument
69 	switch (call->completion) {  in rxrpc_recvmsg_term()
72 		if (rxrpc_is_service_call(call))  in rxrpc_recvmsg_term()
76 		tmp = call->abort_code;  in rxrpc_recvmsg_term()
80 		tmp = call->abort_code;  in rxrpc_recvmsg_term()
84 		tmp = -call->error;  in rxrpc_recvmsg_term()
88 		tmp = -call->error;  in rxrpc_recvmsg_term()
92 		pr_err("Invalid terminal call state %u\n", call->state);  in rxrpc_recvmsg_term()
97 	trace_rxrpc_recvmsg(call, rxrpc_recvmsg_terminal, call->rx_hard_ack,  in rxrpc_recvmsg_term()
98 			    call->rx_pkt_offset, call->rx_pkt_len, ret);  in rxrpc_recvmsg_term()
103  * Pass back notification of a new call.  The call is added to the
104  * to-be-accepted list.  This means that the next call to be accepted might not
105  * be the last call seen awaiting acceptance, but unless we leave this on the
110 				  struct rxrpc_call *call,  in rxrpc_recvmsg_new_call()  argument
120 		list_del_init(&call->recvmsg_link);  in rxrpc_recvmsg_new_call()
123 		rxrpc_get_call(call, rxrpc_call_got);  in rxrpc_recvmsg_new_call()
125 		list_add_tail(&call->accept_link, &rx->to_be_accepted);  in rxrpc_recvmsg_new_call()
129 	trace_rxrpc_recvmsg(call, rxrpc_recvmsg_to_be_accepted, 1, 0, 0, ret);  in rxrpc_recvmsg_new_call()
136 static void rxrpc_end_rx_phase(struct rxrpc_call *call, rxrpc_serial_t serial)  in rxrpc_end_rx_phase()  argument
138 	_enter("%d,%s", call->debug_id, rxrpc_call_states[call->state]);  in rxrpc_end_rx_phase()
140 	trace_rxrpc_receive(call, rxrpc_receive_end, 0, call->rx_top);  in rxrpc_end_rx_phase()
141 	ASSERTCMP(call->rx_hard_ack, ==, call->rx_top);  in rxrpc_end_rx_phase()
143 	if (call->state == RXRPC_CALL_CLIENT_RECV_REPLY) {  in rxrpc_end_rx_phase()
144 		rxrpc_propose_ACK(call, RXRPC_ACK_IDLE, serial, false, true,  in rxrpc_end_rx_phase()
146 		//rxrpc_send_ack_packet(call, false, NULL);  in rxrpc_end_rx_phase()
149 	write_lock_bh(&call->state_lock);  in rxrpc_end_rx_phase()
151 	switch (call->state) {  in rxrpc_end_rx_phase()
153 		__rxrpc_call_completed(call);  in rxrpc_end_rx_phase()
154 		write_unlock_bh(&call->state_lock);  in rxrpc_end_rx_phase()
158 		call->tx_phase = true;  in rxrpc_end_rx_phase()
159 		call->state = RXRPC_CALL_SERVER_ACK_REQUEST;  in rxrpc_end_rx_phase()
160 		call->expect_req_by = jiffies + MAX_JIFFY_OFFSET;  in rxrpc_end_rx_phase()
161 		write_unlock_bh(&call->state_lock);  in rxrpc_end_rx_phase()
162 		rxrpc_propose_ACK(call, RXRPC_ACK_DELAY, serial, false, true,  in rxrpc_end_rx_phase()
166 		write_unlock_bh(&call->state_lock);  in rxrpc_end_rx_phase()
174 static void rxrpc_rotate_rx_window(struct rxrpc_call *call)  in rxrpc_rotate_rx_window()  argument
184 	_enter("%d", call->debug_id);  in rxrpc_rotate_rx_window()
186 	hard_ack = call->rx_hard_ack;  in rxrpc_rotate_rx_window()
187 	top = smp_load_acquire(&call->rx_top);  in rxrpc_rotate_rx_window()
192 	skb = call->rxtx_buffer[ix];  in rxrpc_rotate_rx_window()
196 	subpacket = call->rxtx_annotations[ix] & RXRPC_RX_ANNO_SUBPACKET;  in rxrpc_rotate_rx_window()
203 	call->rxtx_buffer[ix] = NULL;  in rxrpc_rotate_rx_window()
204 	call->rxtx_annotations[ix] = 0;  in rxrpc_rotate_rx_window()
206 	smp_store_release(&call->rx_hard_ack, hard_ack);  in rxrpc_rotate_rx_window()
210 	trace_rxrpc_receive(call, rxrpc_receive_rotate, serial, hard_ack);  in rxrpc_rotate_rx_window()
212 		rxrpc_end_rx_phase(call, serial);  in rxrpc_rotate_rx_window()
215 		if (after_eq(hard_ack, call->ackr_consumed + 2) ||  in rxrpc_rotate_rx_window()
216 		    after_eq(top, call->ackr_seen + 2) ||  in rxrpc_rotate_rx_window()
217 		    (hard_ack == top && after(hard_ack, call->ackr_consumed)))  in rxrpc_rotate_rx_window()
218 			rxrpc_propose_ACK(call, RXRPC_ACK_DELAY, serial,  in rxrpc_rotate_rx_window()
221 		if (call->ackr_reason && call->ackr_reason != RXRPC_ACK_DELAY)  in rxrpc_rotate_rx_window()
222 			rxrpc_send_ack_packet(call, false, NULL);  in rxrpc_rotate_rx_window()
232 static int rxrpc_verify_packet(struct rxrpc_call *call, struct sk_buff *skb,  in rxrpc_verify_packet()  argument
254 	return call->security->verify_packet(call, skb, offset, len,  in rxrpc_verify_packet()
268 static int rxrpc_locate_data(struct rxrpc_call *call, struct sk_buff *skb,  in rxrpc_locate_data()  argument
290 		ret = rxrpc_verify_packet(call, skb, annotation, offset, len);  in rxrpc_locate_data()
299 	call->security->locate_data(call, skb, _offset, _len);  in rxrpc_locate_data()
304  * Deliver messages to a call.  This keeps processing packets until the buffer
308 static int rxrpc_recvmsg_data(struct socket *sock, struct rxrpc_call *call,  in rxrpc_recvmsg_data()  argument
321 	if (test_and_clear_bit(RXRPC_CALL_RX_UNDERRUN, &call->flags) &&  in rxrpc_recvmsg_data()
322 	    call->ackr_reason)  in rxrpc_recvmsg_data()
323 		rxrpc_send_ack_packet(call, false, NULL);  in rxrpc_recvmsg_data()
325 	rx_pkt_offset = call->rx_pkt_offset;  in rxrpc_recvmsg_data()
326 	rx_pkt_len = call->rx_pkt_len;  in rxrpc_recvmsg_data()
327 	rx_pkt_last = call->rx_pkt_last;  in rxrpc_recvmsg_data()
329 	if (call->state >= RXRPC_CALL_SERVER_ACK_REQUEST) {  in rxrpc_recvmsg_data()
330 		seq = call->rx_hard_ack;  in rxrpc_recvmsg_data()
336 	hard_ack = call->rx_hard_ack;  in rxrpc_recvmsg_data()
339 	while (top = smp_load_acquire(&call->rx_top),  in rxrpc_recvmsg_data()
343 		skb = call->rxtx_buffer[ix];  in rxrpc_recvmsg_data()
345 			trace_rxrpc_recvmsg(call, rxrpc_recvmsg_hole, seq,  in rxrpc_recvmsg_data()
355 			serial += call->rxtx_annotations[ix] & RXRPC_RX_ANNO_SUBPACKET;  in rxrpc_recvmsg_data()
356 			trace_rxrpc_receive(call, rxrpc_receive_front,  in rxrpc_recvmsg_data()
364 			ret2 = rxrpc_locate_data(call, skb,  in rxrpc_recvmsg_data()
365 						 &call->rxtx_annotations[ix],  in rxrpc_recvmsg_data()
368 			trace_rxrpc_recvmsg(call, rxrpc_recvmsg_next, seq,  in rxrpc_recvmsg_data()
375 			trace_rxrpc_recvmsg(call, rxrpc_recvmsg_cont, seq,  in rxrpc_recvmsg_data()
399 			trace_rxrpc_recvmsg(call, rxrpc_recvmsg_full, seq,  in rxrpc_recvmsg_data()
408 			rxrpc_rotate_rx_window(call);  in rxrpc_recvmsg_data()
413 			ASSERTCMP(seq, ==, READ_ONCE(call->rx_top));  in rxrpc_recvmsg_data()
423 		call->rx_pkt_offset = rx_pkt_offset;  in rxrpc_recvmsg_data()
424 		call->rx_pkt_len = rx_pkt_len;  in rxrpc_recvmsg_data()
425 		call->rx_pkt_last = rx_pkt_last;  in rxrpc_recvmsg_data()
428 	trace_rxrpc_recvmsg(call, rxrpc_recvmsg_data_return, seq,  in rxrpc_recvmsg_data()
431 		set_bit(RXRPC_CALL_RX_UNDERRUN, &call->flags);  in rxrpc_recvmsg_data()
443 	struct rxrpc_call *call;  in rxrpc_recvmsg()  local
473 			call = NULL;  in rxrpc_recvmsg()
497 	/* Find the next call and dequeue it if we're not just peeking.  If we  in rxrpc_recvmsg()
502 	call = list_entry(l, struct rxrpc_call, recvmsg_link);  in rxrpc_recvmsg()
504 		list_del_init(&call->recvmsg_link);  in rxrpc_recvmsg()
506 		rxrpc_get_call(call, rxrpc_call_got);  in rxrpc_recvmsg()
509 	trace_rxrpc_recvmsg(call, rxrpc_recvmsg_dequeue, 0, 0, 0, 0);  in rxrpc_recvmsg()
511 	/* We're going to drop the socket lock, so we need to lock the call  in rxrpc_recvmsg()
514 	if (!mutex_trylock(&call->user_mutex)) {  in rxrpc_recvmsg()
519 		if (mutex_lock_interruptible(&call->user_mutex) < 0)  in rxrpc_recvmsg()
525 	if (test_bit(RXRPC_CALL_RELEASED, &call->flags))  in rxrpc_recvmsg()
528 	if (test_bit(RXRPC_CALL_HAS_USERID, &call->flags)) {  in rxrpc_recvmsg()
530 			unsigned int id32 = call->user_call_ID;  in rxrpc_recvmsg()
535 			unsigned long idl = call->user_call_ID;  in rxrpc_recvmsg()
546 		size_t len = sizeof(call->peer->srx);  in rxrpc_recvmsg()
548 		memcpy(msg->msg_name, &call->peer->srx, len);  in rxrpc_recvmsg()
549 		srx->srx_service = call->service_id;  in rxrpc_recvmsg()
553 	switch (READ_ONCE(call->state)) {  in rxrpc_recvmsg()
555 		ret = rxrpc_recvmsg_new_call(rx, call, msg, flags);  in rxrpc_recvmsg()
560 		ret = rxrpc_recvmsg_data(sock, call, msg, &msg->msg_iter, len,  in rxrpc_recvmsg()
565 		if (after(call->rx_top, call->rx_hard_ack) &&  in rxrpc_recvmsg()
566 		    call->rxtx_buffer[(call->rx_hard_ack + 1) & RXRPC_RXTX_BUFF_MASK])  in rxrpc_recvmsg()
567 			rxrpc_notify_socket(call);  in rxrpc_recvmsg()
577 	if (call->state == RXRPC_CALL_COMPLETE) {  in rxrpc_recvmsg()
578 		ret = rxrpc_recvmsg_term(call, msg);  in rxrpc_recvmsg()
582 			rxrpc_release_call(rx, call);  in rxrpc_recvmsg()
594 	mutex_unlock(&call->user_mutex);  in rxrpc_recvmsg()
595 	rxrpc_put_call(call, rxrpc_call_put);  in rxrpc_recvmsg()
596 	trace_rxrpc_recvmsg(call, rxrpc_recvmsg_return, 0, 0, 0, ret);  in rxrpc_recvmsg()
602 		list_add(&call->recvmsg_link, &rx->recvmsg_q);  in rxrpc_recvmsg()
604 		trace_rxrpc_recvmsg(call, rxrpc_recvmsg_requeue, 0, 0, 0, 0);  in rxrpc_recvmsg()
606 		rxrpc_put_call(call, rxrpc_call_put);  in rxrpc_recvmsg()
611 	trace_rxrpc_recvmsg(call, rxrpc_recvmsg_return, 0, 0, 0, ret);  in rxrpc_recvmsg()
618 	call = NULL;  in rxrpc_recvmsg()
624  * @sock: The socket that the call exists on
625  * @call: The call to send data through
632  * state of a call.  Returns 0 if got what was asked for and there's more
641 int rxrpc_kernel_recv_data(struct socket *sock, struct rxrpc_call *call,  in rxrpc_kernel_recv_data()  argument
649 	       call->debug_id, rxrpc_call_states[call->state],  in rxrpc_kernel_recv_data()
652 	ASSERTCMP(call->state, !=, RXRPC_CALL_SERVER_ACCEPTING);  in rxrpc_kernel_recv_data()
654 	mutex_lock(&call->user_mutex);  in rxrpc_kernel_recv_data()
656 	switch (READ_ONCE(call->state)) {  in rxrpc_kernel_recv_data()
660 		ret = rxrpc_recvmsg_data(sock, call, NULL, iter,  in rxrpc_kernel_recv_data()
694 	switch (call->ackr_reason) {  in rxrpc_kernel_recv_data()
702 		rxrpc_send_ack_packet(call, false, NULL);  in rxrpc_kernel_recv_data()
706 		*_service = call->service_id;  in rxrpc_kernel_recv_data()
707 	mutex_unlock(&call->user_mutex);  in rxrpc_kernel_recv_data()
712 	trace_rxrpc_rx_eproto(call, 0, tracepoint_string("short_data"));  in rxrpc_kernel_recv_data()
716 	trace_rxrpc_rx_eproto(call, 0, tracepoint_string("excess_data"));  in rxrpc_kernel_recv_data()
720 	*_abort = call->abort_code;  in rxrpc_kernel_recv_data()
721 	ret = call->error;  in rxrpc_kernel_recv_data()
722 	if (call->completion == RXRPC_CALL_SUCCEEDED) {  in rxrpc_kernel_recv_data()
733  * @sock: The socket that the call exists on
734  * @call: The call to query
740 bool rxrpc_kernel_get_reply_time(struct socket *sock, struct rxrpc_call *call,  in rxrpc_kernel_get_reply_time()  argument
747 	mutex_lock(&call->user_mutex);  in rxrpc_kernel_get_reply_time()
749 	if (READ_ONCE(call->state) != RXRPC_CALL_CLIENT_RECV_REPLY)  in rxrpc_kernel_get_reply_time()
752 	hard_ack = call->rx_hard_ack;  in rxrpc_kernel_get_reply_time()
757 	top = smp_load_acquire(&call->rx_top);  in rxrpc_kernel_get_reply_time()
761 	skb = call->rxtx_buffer[seq & RXRPC_RXTX_BUFF_MASK];  in rxrpc_kernel_get_reply_time()
769 	mutex_unlock(&call->user_mutex);  in rxrpc_kernel_get_reply_time()