72 static void rds_ib_send_unmap_data(struct rds_ib_connection *ic,  in rds_ib_send_unmap_data()  argument
77 		ib_dma_unmap_sg(ic->i_cm_id->device,  in rds_ib_send_unmap_data()
82 static void rds_ib_send_unmap_rdma(struct rds_ib_connection *ic,  in rds_ib_send_unmap_rdma()  argument
87 		ib_dma_unmap_sg(ic->i_cm_id->device,  in rds_ib_send_unmap_rdma()
122 static void rds_ib_send_unmap_atomic(struct rds_ib_connection *ic,  in rds_ib_send_unmap_atomic()  argument
128 		ib_dma_unmap_sg(ic->i_cm_id->device, op->op_sg, 1,  in rds_ib_send_unmap_atomic()
149 static struct rds_message *rds_ib_send_unmap_op(struct rds_ib_connection *ic,  in rds_ib_send_unmap_op()  argument
160 			rds_ib_send_unmap_data(ic, send->s_op, wc_status);  in rds_ib_send_unmap_op()
167 			rds_ib_send_unmap_rdma(ic, send->s_op, wc_status);  in rds_ib_send_unmap_op()
174 			rds_ib_send_unmap_atomic(ic, send->s_op, wc_status);  in rds_ib_send_unmap_op()
189 void rds_ib_send_init_ring(struct rds_ib_connection *ic)  in rds_ib_send_init_ring()  argument
194 	for (i = 0, send = ic->i_sends; i < ic->i_send_ring.w_nr; i++, send++) {  in rds_ib_send_init_ring()
204 		sge->addr = ic->i_send_hdrs_dma + (i * sizeof(struct rds_header));  in rds_ib_send_init_ring()
206 		sge->lkey = ic->i_pd->local_dma_lkey;  in rds_ib_send_init_ring()
208 		send->s_sge[1].lkey = ic->i_pd->local_dma_lkey;  in rds_ib_send_init_ring()
212 void rds_ib_send_clear_ring(struct rds_ib_connection *ic)  in rds_ib_send_clear_ring()  argument
217 	for (i = 0, send = ic->i_sends; i < ic->i_send_ring.w_nr; i++, send++) {  in rds_ib_send_clear_ring()
219 			rds_ib_send_unmap_op(ic, send, IB_WC_WR_FLUSH_ERR);  in rds_ib_send_clear_ring()
227 static void rds_ib_sub_signaled(struct rds_ib_connection *ic, int nr)  in rds_ib_sub_signaled()  argument
229 	if ((atomic_sub_return(nr, &ic->i_signaled_sends) == 0) &&  in rds_ib_sub_signaled()
232 	BUG_ON(atomic_read(&ic->i_signaled_sends) < 0);  in rds_ib_sub_signaled()
241 void rds_ib_send_cqe_handler(struct rds_ib_connection *ic, struct ib_wc *wc)  in rds_ib_send_cqe_handler()  argument
244 	struct rds_connection *conn = ic->conn;  in rds_ib_send_cqe_handler()
259 		if (time_after(jiffies, ic->i_ack_queued + HZ / 2))  in rds_ib_send_cqe_handler()
261 		rds_ib_ack_send_complete(ic);  in rds_ib_send_cqe_handler()
265 	oldest = rds_ib_ring_oldest(&ic->i_send_ring);  in rds_ib_send_cqe_handler()
267 	completed = rds_ib_ring_completed(&ic->i_send_ring, wc->wr_id, oldest);  in rds_ib_send_cqe_handler()
270 		send = &ic->i_sends[oldest];  in rds_ib_send_cqe_handler()
274 		rm = rds_ib_send_unmap_op(ic, send, wc->status);  in rds_ib_send_cqe_handler()
290 		oldest = (oldest + 1) % ic->i_send_ring.w_nr;  in rds_ib_send_cqe_handler()
293 	rds_ib_ring_free(&ic->i_send_ring, completed);  in rds_ib_send_cqe_handler()
294 	rds_ib_sub_signaled(ic, nr_sig);  in rds_ib_send_cqe_handler()
354 int rds_ib_send_grab_credits(struct rds_ib_connection *ic,  in rds_ib_send_grab_credits()  argument
361 	if (!ic->i_flowctl)  in rds_ib_send_grab_credits()
366 	oldval = newval = atomic_read(&ic->i_credits);  in rds_ib_send_grab_credits()
378 		struct rds_connection *conn = ic->i_cm_id->context;  in rds_ib_send_grab_credits()
400 	if (atomic_cmpxchg(&ic->i_credits, oldval, newval) != oldval)  in rds_ib_send_grab_credits()
409 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_send_add_credits()  local
416 			IB_GET_SEND_CREDITS(atomic_read(&ic->i_credits)),  in rds_ib_send_add_credits()
419 	atomic_add(IB_SET_SEND_CREDITS(credits), &ic->i_credits);  in rds_ib_send_add_credits()
430 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_advertise_credits()  local
435 	atomic_add(IB_SET_POST_CREDITS(posted), &ic->i_credits);  in rds_ib_advertise_credits()
449 	if (IB_GET_POST_CREDITS(atomic_read(&ic->i_credits)) >= 16)  in rds_ib_advertise_credits()
450 		set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);  in rds_ib_advertise_credits()
453 static inline int rds_ib_set_wr_signal_state(struct rds_ib_connection *ic,  in rds_ib_set_wr_signal_state()  argument
462 	if (ic->i_unsignaled_wrs-- == 0 || notify) {  in rds_ib_set_wr_signal_state()
463 		ic->i_unsignaled_wrs = rds_ib_sysctl_max_unsig_wrs;  in rds_ib_set_wr_signal_state()
486 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_xmit()  local
487 	struct ib_device *dev = ic->i_cm_id->device;  in rds_ib_xmit()
523 	work_alloc = rds_ib_ring_alloc(&ic->i_send_ring, i, &pos);  in rds_ib_xmit()
531 	if (ic->i_flowctl) {  in rds_ib_xmit()
532 		credit_alloc = rds_ib_send_grab_credits(ic, work_alloc, &posted, 0, RDS_MAX_ADV_CREDIT);  in rds_ib_xmit()
535 			rds_ib_ring_unalloc(&ic->i_send_ring, work_alloc - credit_alloc);  in rds_ib_xmit()
548 	if (!ic->i_data_op) {  in rds_ib_xmit()
554 			rdsdebug("ic %p mapping rm %p: %d\n", ic, rm, rm->data.op_count);  in rds_ib_xmit()
557 				rds_ib_ring_unalloc(&ic->i_send_ring, work_alloc);  in rds_ib_xmit()
568 		ic->i_data_op = &rm->data;  in rds_ib_xmit()
595 		rm->m_inc.i_hdr.h_ack = cpu_to_be64(rds_ib_piggyb_ack(ic));  in rds_ib_xmit()
601 		if (ic->i_flowctl) {  in rds_ib_xmit()
602 			rds_ib_send_grab_credits(ic, 0, &posted, 1, RDS_MAX_ADV_CREDIT - adv_credits);  in rds_ib_xmit()
618 	send = &ic->i_sends[pos];  in rds_ib_xmit()
621 	scat = &ic->i_data_op->op_sg[rm->data.op_dmasg];  in rds_ib_xmit()
634 		send->s_sge[0].addr = ic->i_send_hdrs_dma  in rds_ib_xmit()
638 		memcpy(&ic->i_send_hdrs[pos], &rm->m_inc.i_hdr, sizeof(struct rds_header));  in rds_ib_xmit()
660 		rds_ib_set_wr_signal_state(ic, send, false);  in rds_ib_xmit()
665 		if (ic->i_flowctl && flow_controlled && i == (work_alloc - 1)) {  in rds_ib_xmit()
666 			rds_ib_set_wr_signal_state(ic, send, true);  in rds_ib_xmit()
676 		if (ic->i_flowctl && adv_credits) {  in rds_ib_xmit()
677 			struct rds_header *hdr = &ic->i_send_hdrs[pos];  in rds_ib_xmit()
690 		pos = (pos + 1) % ic->i_send_ring.w_nr;  in rds_ib_xmit()
691 		send = &ic->i_sends[pos];  in rds_ib_xmit()
704 		prev->s_op = ic->i_data_op;  in rds_ib_xmit()
707 			nr_sig += rds_ib_set_wr_signal_state(ic, prev, true);  in rds_ib_xmit()
708 		ic->i_data_op = NULL;  in rds_ib_xmit()
713 		rds_ib_ring_unalloc(&ic->i_send_ring, work_alloc - i);  in rds_ib_xmit()
716 	if (ic->i_flowctl && i < credit_alloc)  in rds_ib_xmit()
720 		atomic_add(nr_sig, &ic->i_signaled_sends);  in rds_ib_xmit()
724 	ret = ib_post_send(ic->i_cm_id->qp, &first->s_wr, &failed_wr);  in rds_ib_xmit()
725 	rdsdebug("ic %p first %p (wr %p) ret %d wr %p\n", ic,  in rds_ib_xmit()
731 		rds_ib_ring_unalloc(&ic->i_send_ring, work_alloc);  in rds_ib_xmit()
732 		rds_ib_sub_signaled(ic, nr_sig);  in rds_ib_xmit()
734 			ic->i_data_op = prev->s_op;  in rds_ib_xmit()
738 		rds_ib_conn_error(ic->conn, "ib_post_send failed\n");  in rds_ib_xmit()
755 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_xmit_atomic()  local
763 	work_alloc = rds_ib_ring_alloc(&ic->i_send_ring, 1, &pos);  in rds_ib_xmit_atomic()
771 	send = &ic->i_sends[pos];  in rds_ib_xmit_atomic()
788 	nr_sig = rds_ib_set_wr_signal_state(ic, send, op->op_notify);  in rds_ib_xmit_atomic()
797 	ret = ib_dma_map_sg(ic->i_cm_id->device, op->op_sg, 1, DMA_FROM_DEVICE);  in rds_ib_xmit_atomic()
798 	rdsdebug("ic %p mapping atomic op %p. mapped %d pg\n", ic, op, ret);  in rds_ib_xmit_atomic()
800 		rds_ib_ring_unalloc(&ic->i_send_ring, work_alloc);  in rds_ib_xmit_atomic()
809 	send->s_sge[0].lkey = ic->i_pd->local_dma_lkey;  in rds_ib_xmit_atomic()
815 		atomic_add(nr_sig, &ic->i_signaled_sends);  in rds_ib_xmit_atomic()
818 	ret = ib_post_send(ic->i_cm_id->qp, &send->s_atomic_wr.wr, &failed_wr);  in rds_ib_xmit_atomic()
819 	rdsdebug("ic %p send %p (wr %p) ret %d wr %p\n", ic,  in rds_ib_xmit_atomic()
825 		rds_ib_ring_unalloc(&ic->i_send_ring, work_alloc);  in rds_ib_xmit_atomic()
826 		rds_ib_sub_signaled(ic, nr_sig);  in rds_ib_xmit_atomic()
841 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_xmit_rdma()  local
849 	u32 max_sge = ic->rds_ibdev->max_sge;  in rds_ib_xmit_rdma()
861 		op->op_count = ib_dma_map_sg(ic->i_cm_id->device,  in rds_ib_xmit_rdma()
864 		rdsdebug("ic %p mapping op %p: %d\n", ic, op, op->op_count);  in rds_ib_xmit_rdma()
880 	work_alloc = rds_ib_ring_alloc(&ic->i_send_ring, i, &pos);  in rds_ib_xmit_rdma()
882 		rds_ib_ring_unalloc(&ic->i_send_ring, work_alloc);  in rds_ib_xmit_rdma()
888 	send = &ic->i_sends[pos];  in rds_ib_xmit_rdma()
901 			nr_sig += rds_ib_set_wr_signal_state(ic, send,  in rds_ib_xmit_rdma()
925 			send->s_sge[j].lkey = ic->i_pd->local_dma_lkey;  in rds_ib_xmit_rdma()
928 			rdsdebug("ic %p sent %d remote_addr %llu\n", ic, sent, remote_addr);  in rds_ib_xmit_rdma()
940 		if (++send == &ic->i_sends[ic->i_send_ring.w_nr])  in rds_ib_xmit_rdma()
941 			send = ic->i_sends;  in rds_ib_xmit_rdma()
951 		rds_ib_ring_unalloc(&ic->i_send_ring, work_alloc - i);  in rds_ib_xmit_rdma()
956 		atomic_add(nr_sig, &ic->i_signaled_sends);  in rds_ib_xmit_rdma()
959 	ret = ib_post_send(ic->i_cm_id->qp, &first->s_rdma_wr.wr, &failed_wr);  in rds_ib_xmit_rdma()
960 	rdsdebug("ic %p first %p (wr %p) ret %d wr %p\n", ic,  in rds_ib_xmit_rdma()
966 		rds_ib_ring_unalloc(&ic->i_send_ring, work_alloc);  in rds_ib_xmit_rdma()
967 		rds_ib_sub_signaled(ic, nr_sig);  in rds_ib_xmit_rdma()
984 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_xmit_path_complete()  local
988 	rds_ib_attempt_ack(ic);  in rds_ib_xmit_path_complete()