87 static void rds_recv_rcvbuf_delta(struct rds_sock *rs, struct sock *sk,  in rds_recv_rcvbuf_delta()  argument
96 	rs->rs_rcv_bytes += delta;  in rds_recv_rcvbuf_delta()
103 	if (rs->rs_transport->t_type == RDS_TRANS_LOOP)  in rds_recv_rcvbuf_delta()
106 	now_congested = rs->rs_rcv_bytes > rds_sk_rcvbuf(rs);  in rds_recv_rcvbuf_delta()
108 	rdsdebug("rs %p (%pI6c:%u) recv bytes %d buf %d "  in rds_recv_rcvbuf_delta()
110 	  rs, &rs->rs_bound_addr,  in rds_recv_rcvbuf_delta()
111 	  ntohs(rs->rs_bound_port), rs->rs_rcv_bytes,  in rds_recv_rcvbuf_delta()
112 	  rds_sk_rcvbuf(rs), now_congested, delta);  in rds_recv_rcvbuf_delta()
115 	if (!rs->rs_congested && now_congested) {  in rds_recv_rcvbuf_delta()
116 		rs->rs_congested = 1;  in rds_recv_rcvbuf_delta()
123 	else if (rs->rs_congested && (rs->rs_rcv_bytes < (rds_sk_rcvbuf(rs)/2))) {  in rds_recv_rcvbuf_delta()
124 		rs->rs_congested = 0;  in rds_recv_rcvbuf_delta()
165 static void rds_recv_incoming_exthdrs(struct rds_incoming *inc, struct rds_sock *rs)  in rds_recv_incoming_exthdrs()  argument
183 			rds_rdma_unuse(rs, be32_to_cpu(buffer.rdma.h_rdma_rkey), 0);  in rds_recv_incoming_exthdrs()
286 	struct rds_sock *rs = NULL;  in rds_recv_incoming()  local
362 	rs = rds_find_bound(daddr, inc->i_hdr.h_dport, conn->c_bound_if);  in rds_recv_incoming()
363 	if (!rs) {  in rds_recv_incoming()
369 	rds_recv_incoming_exthdrs(inc, rs);  in rds_recv_incoming()
372 	sk = rds_rs_to_sk(rs);  in rds_recv_incoming()
375 	write_lock_irqsave(&rs->rs_recv_lock, flags);  in rds_recv_incoming()
377 		rdsdebug("adding inc %p to rs %p's recv queue\n", inc, rs);  in rds_recv_incoming()
379 		rds_recv_rcvbuf_delta(rs, sk, inc->i_conn->c_lcong,  in rds_recv_incoming()
386 		list_add_tail(&inc->i_item, &rs->rs_recv_queue);  in rds_recv_incoming()
391 	write_unlock_irqrestore(&rs->rs_recv_lock, flags);  in rds_recv_incoming()
394 	if (rs)  in rds_recv_incoming()
395 		rds_sock_put(rs);  in rds_recv_incoming()
403 static int rds_next_incoming(struct rds_sock *rs, struct rds_incoming **inc)  in rds_next_incoming()  argument
408 		read_lock_irqsave(&rs->rs_recv_lock, flags);  in rds_next_incoming()
409 		if (!list_empty(&rs->rs_recv_queue)) {  in rds_next_incoming()
410 			*inc = list_entry(rs->rs_recv_queue.next,  in rds_next_incoming()
415 		read_unlock_irqrestore(&rs->rs_recv_lock, flags);  in rds_next_incoming()
421 static int rds_still_queued(struct rds_sock *rs, struct rds_incoming *inc,  in rds_still_queued()  argument
424 	struct sock *sk = rds_rs_to_sk(rs);  in rds_still_queued()
428 	write_lock_irqsave(&rs->rs_recv_lock, flags);  in rds_still_queued()
433 			rds_recv_rcvbuf_delta(rs, sk, inc->i_conn->c_lcong,  in rds_still_queued()
440 	write_unlock_irqrestore(&rs->rs_recv_lock, flags);  in rds_still_queued()
442 	rdsdebug("inc %p rs %p still %d dropped %d\n", inc, rs, ret, drop);  in rds_still_queued()
450 int rds_notify_queue_get(struct rds_sock *rs, struct msghdr *msghdr)  in rds_notify_queue_get()  argument
474 	spin_lock_irqsave(&rs->rs_lock, flags);  in rds_notify_queue_get()
475 	while (!list_empty(&rs->rs_notify_queue) && count < max_messages) {  in rds_notify_queue_get()
476 		notifier = list_entry(rs->rs_notify_queue.next,  in rds_notify_queue_get()
481 	spin_unlock_irqrestore(&rs->rs_lock, flags);  in rds_notify_queue_get()
507 		spin_lock_irqsave(&rs->rs_lock, flags);  in rds_notify_queue_get()
508 		list_splice(&copy, &rs->rs_notify_queue);  in rds_notify_queue_get()
509 		spin_unlock_irqrestore(&rs->rs_lock, flags);  in rds_notify_queue_get()
518 static int rds_notify_cong(struct rds_sock *rs, struct msghdr *msghdr)  in rds_notify_cong()  argument
520 	uint64_t notify = rs->rs_cong_notify;  in rds_notify_cong()
529 	spin_lock_irqsave(&rs->rs_lock, flags);  in rds_notify_cong()
530 	rs->rs_cong_notify &= ~notify;  in rds_notify_cong()
531 	spin_unlock_irqrestore(&rs->rs_lock, flags);  in rds_notify_cong()
540 			 struct rds_sock *rs)  in rds_cmsg_recv()  argument
553 	    sock_flag(rds_rs_to_sk(rs), SOCK_RCVTSTAMP)) {  in rds_cmsg_recv()
557 		if (!sock_flag(rds_rs_to_sk(rs), SOCK_TSTAMP_NEW)) {  in rds_cmsg_recv()
574 	if (rs->rs_rx_traces) {  in rds_cmsg_recv()
580 		t.rx_traces =  rs->rs_rx_traces;  in rds_cmsg_recv()
581 		for (i = 0; i < rs->rs_rx_traces; i++) {  in rds_cmsg_recv()
582 			j = rs->rs_rx_trace[i];  in rds_cmsg_recv()
598 static bool rds_recvmsg_zcookie(struct rds_sock *rs, struct msghdr *msg)  in rds_recvmsg_zcookie()  argument
600 	struct rds_msg_zcopy_queue *q = &rs->rs_zcookie_queue;  in rds_recvmsg_zcookie()
608 	if (!sock_flag(rds_rs_to_sk(rs), SOCK_ZEROCOPY) ||  in rds_recvmsg_zcookie()
637 	struct rds_sock *rs = rds_sk_to_rs(sk);  in rds_recvmsg()  local
656 		if (!list_empty(&rs->rs_notify_queue)) {  in rds_recvmsg()
657 			ret = rds_notify_queue_get(rs, msg);  in rds_recvmsg()
661 		if (rs->rs_cong_notify) {  in rds_recvmsg()
662 			ret = rds_notify_cong(rs, msg);  in rds_recvmsg()
666 		if (!rds_next_incoming(rs, &inc)) {  in rds_recvmsg()
668 				bool reaped = rds_recvmsg_zcookie(rs, msg);  in rds_recvmsg()
675 					(!list_empty(&rs->rs_notify_queue) ||  in rds_recvmsg()
676 					 rs->rs_cong_notify ||  in rds_recvmsg()
677 					 rds_next_incoming(rs, &inc)), timeo);  in rds_recvmsg()
701 		if (!rds_still_queued(rs, inc, !(msg_flags & MSG_PEEK))) {  in rds_recvmsg()
715 		if (rds_cmsg_recv(inc, msg, rs)) {  in rds_recvmsg()
719 		rds_recvmsg_zcookie(rs, msg);  in rds_recvmsg()
736 				sin6->sin6_scope_id = rs->rs_bound_scope_id;  in rds_recvmsg()
755 void rds_clear_recv_queue(struct rds_sock *rs)  in rds_clear_recv_queue()  argument
757 	struct sock *sk = rds_rs_to_sk(rs);  in rds_clear_recv_queue()
761 	write_lock_irqsave(&rs->rs_recv_lock, flags);  in rds_clear_recv_queue()
762 	list_for_each_entry_safe(inc, tmp, &rs->rs_recv_queue, i_item) {  in rds_clear_recv_queue()
763 		rds_recv_rcvbuf_delta(rs, sk, inc->i_conn->c_lcong,  in rds_clear_recv_queue()
769 	write_unlock_irqrestore(&rs->rs_recv_lock, flags);  in rds_clear_recv_queue()