155 						struct rds_ib_send_work *send,  in rds_ib_send_unmap_op()  argument
161 	switch (send->s_wr.opcode) {  in rds_ib_send_unmap_op()
163 		if (send->s_op) {  in rds_ib_send_unmap_op()
164 			rm = container_of(send->s_op, struct rds_message, data);  in rds_ib_send_unmap_op()
165 			rds_ib_send_unmap_data(ic, send->s_op, wc_status);  in rds_ib_send_unmap_op()
170 		if (send->s_op) {  in rds_ib_send_unmap_op()
171 			rm = container_of(send->s_op, struct rds_message, rdma);  in rds_ib_send_unmap_op()
172 			rds_ib_send_unmap_rdma(ic, send->s_op, wc_status);  in rds_ib_send_unmap_op()
177 		if (send->s_op) {  in rds_ib_send_unmap_op()
178 			rm = container_of(send->s_op, struct rds_message, atomic);  in rds_ib_send_unmap_op()
179 			rds_ib_send_unmap_atomic(ic, send->s_op, wc_status);  in rds_ib_send_unmap_op()
185 			       __func__, send->s_wr.opcode);  in rds_ib_send_unmap_op()
189 	send->s_wr.opcode = 0xdead;  in rds_ib_send_unmap_op()
196 	struct rds_ib_send_work *send;  in rds_ib_send_init_ring()  local
199 	for (i = 0, send = ic->i_sends; i < ic->i_send_ring.w_nr; i++, send++) {  in rds_ib_send_init_ring()
202 		send->s_op = NULL;  in rds_ib_send_init_ring()
204 		send->s_wr.wr_id = i;  in rds_ib_send_init_ring()
205 		send->s_wr.sg_list = send->s_sge;  in rds_ib_send_init_ring()
206 		send->s_wr.ex.imm_data = 0;  in rds_ib_send_init_ring()
208 		sge = &send->s_sge[0];  in rds_ib_send_init_ring()
213 		send->s_sge[1].lkey = ic->i_pd->local_dma_lkey;  in rds_ib_send_init_ring()
219 	struct rds_ib_send_work *send;  in rds_ib_send_clear_ring()  local
222 	for (i = 0, send = ic->i_sends; i < ic->i_send_ring.w_nr; i++, send++) {  in rds_ib_send_clear_ring()
223 		if (send->s_op && send->s_wr.opcode != 0xdead)  in rds_ib_send_clear_ring()
224 			rds_ib_send_unmap_op(ic, send, IB_WC_WR_FLUSH_ERR);  in rds_ib_send_clear_ring()
250 	struct rds_ib_send_work *send;  in rds_ib_send_cqe_handler()  local
275 		send = &ic->i_sends[oldest];  in rds_ib_send_cqe_handler()
276 		if (send->s_wr.send_flags & IB_SEND_SIGNALED)  in rds_ib_send_cqe_handler()
279 		rm = rds_ib_send_unmap_op(ic, send, wc->status);  in rds_ib_send_cqe_handler()
281 		if (time_after(jiffies, send->s_queued + HZ / 2))  in rds_ib_send_cqe_handler()
284 		if (send->s_op) {  in rds_ib_send_cqe_handler()
285 			if (send->s_op == rm->m_final_op) {  in rds_ib_send_cqe_handler()
292 			send->s_op = NULL;  in rds_ib_send_cqe_handler()
458 					     struct rds_ib_send_work *send,  in rds_ib_set_wr_signal_state()  argument
468 		send->s_wr.send_flags |= IB_SEND_SIGNALED;  in rds_ib_set_wr_signal_state()
492 	struct rds_ib_send_work *send = NULL;  in rds_ib_xmit()  local
622 	send = &ic->i_sends[pos];  in rds_ib_xmit()
623 	first = send;  in rds_ib_xmit()
631 		send->s_wr.send_flags = send_flags;  in rds_ib_xmit()
632 		send->s_wr.opcode = IB_WR_SEND;  in rds_ib_xmit()
633 		send->s_wr.num_sge = 1;  in rds_ib_xmit()
634 		send->s_wr.next = NULL;  in rds_ib_xmit()
635 		send->s_queued = jiffies;  in rds_ib_xmit()
636 		send->s_op = NULL;  in rds_ib_xmit()
638 		send->s_sge[0].addr = ic->i_send_hdrs_dma  in rds_ib_xmit()
640 		send->s_sge[0].length = sizeof(struct rds_header);  in rds_ib_xmit()
649 			send->s_wr.num_sge = 2;  in rds_ib_xmit()
651 			send->s_sge[1].addr = ib_sg_dma_address(dev, scat);  in rds_ib_xmit()
652 			send->s_sge[1].addr += rm->data.op_dmaoff;  in rds_ib_xmit()
653 			send->s_sge[1].length = len;  in rds_ib_xmit()
664 		rds_ib_set_wr_signal_state(ic, send, false);  in rds_ib_xmit()
670 			rds_ib_set_wr_signal_state(ic, send, true);  in rds_ib_xmit()
671 			send->s_wr.send_flags |= IB_SEND_SOLICITED;  in rds_ib_xmit()
674 		if (send->s_wr.send_flags & IB_SEND_SIGNALED)  in rds_ib_xmit()
677 		rdsdebug("send %p wr %p num_sge %u next %p\n", send,  in rds_ib_xmit()
678 			 &send->s_wr, send->s_wr.num_sge, send->s_wr.next);  in rds_ib_xmit()
691 			prev->s_wr.next = &send->s_wr;  in rds_ib_xmit()
692 		prev = send;  in rds_ib_xmit()
695 		send = &ic->i_sends[pos];  in rds_ib_xmit()
760 	struct rds_ib_send_work *send = NULL;  in rds_ib_xmit_atomic()  local
775 	send = &ic->i_sends[pos];  in rds_ib_xmit_atomic()
776 	send->s_queued = jiffies;  in rds_ib_xmit_atomic()
779 		send->s_atomic_wr.wr.opcode = IB_WR_MASKED_ATOMIC_CMP_AND_SWP;  in rds_ib_xmit_atomic()
780 		send->s_atomic_wr.compare_add = op->op_m_cswp.compare;  in rds_ib_xmit_atomic()
781 		send->s_atomic_wr.swap = op->op_m_cswp.swap;  in rds_ib_xmit_atomic()
782 		send->s_atomic_wr.compare_add_mask = op->op_m_cswp.compare_mask;  in rds_ib_xmit_atomic()
783 		send->s_atomic_wr.swap_mask = op->op_m_cswp.swap_mask;  in rds_ib_xmit_atomic()
785 		send->s_atomic_wr.wr.opcode = IB_WR_MASKED_ATOMIC_FETCH_AND_ADD;  in rds_ib_xmit_atomic()
786 		send->s_atomic_wr.compare_add = op->op_m_fadd.add;  in rds_ib_xmit_atomic()
787 		send->s_atomic_wr.swap = 0;  in rds_ib_xmit_atomic()
788 		send->s_atomic_wr.compare_add_mask = op->op_m_fadd.nocarry_mask;  in rds_ib_xmit_atomic()
789 		send->s_atomic_wr.swap_mask = 0;  in rds_ib_xmit_atomic()
791 	send->s_wr.send_flags = 0;  in rds_ib_xmit_atomic()
792 	nr_sig = rds_ib_set_wr_signal_state(ic, send, op->op_notify);  in rds_ib_xmit_atomic()
793 	send->s_atomic_wr.wr.num_sge = 1;  in rds_ib_xmit_atomic()
794 	send->s_atomic_wr.wr.next = NULL;  in rds_ib_xmit_atomic()
795 	send->s_atomic_wr.remote_addr = op->op_remote_addr;  in rds_ib_xmit_atomic()
796 	send->s_atomic_wr.rkey = op->op_rkey;  in rds_ib_xmit_atomic()
797 	send->s_op = op;  in rds_ib_xmit_atomic()
798 	rds_message_addref(container_of(send->s_op, struct rds_message, atomic));  in rds_ib_xmit_atomic()
811 	send->s_sge[0].addr = ib_sg_dma_address(ic->i_cm_id->device, op->op_sg);  in rds_ib_xmit_atomic()
812 	send->s_sge[0].length = ib_sg_dma_len(ic->i_cm_id->device, op->op_sg);  in rds_ib_xmit_atomic()
813 	send->s_sge[0].lkey = ic->i_pd->local_dma_lkey;  in rds_ib_xmit_atomic()
816 		 send->s_sge[0].addr, send->s_sge[0].length);  in rds_ib_xmit_atomic()
821 	failed_wr = &send->s_atomic_wr.wr;  in rds_ib_xmit_atomic()
822 	ret = ib_post_send(ic->i_cm_id->qp, &send->s_atomic_wr.wr, &failed_wr);  in rds_ib_xmit_atomic()
824 		 send, &send->s_atomic_wr, ret, failed_wr);  in rds_ib_xmit_atomic()
825 	BUG_ON(failed_wr != &send->s_atomic_wr.wr);  in rds_ib_xmit_atomic()
834 	if (unlikely(failed_wr != &send->s_atomic_wr.wr)) {  in rds_ib_xmit_atomic()
836 		BUG_ON(failed_wr != &send->s_atomic_wr.wr);  in rds_ib_xmit_atomic()
846 	struct rds_ib_send_work *send = NULL;  in rds_ib_xmit_rdma()  local
892 	send = &ic->i_sends[pos];  in rds_ib_xmit_rdma()
893 	first = send;  in rds_ib_xmit_rdma()
900 		send->s_wr.send_flags = 0;  in rds_ib_xmit_rdma()
901 		send->s_queued = jiffies;  in rds_ib_xmit_rdma()
902 		send->s_op = NULL;  in rds_ib_xmit_rdma()
904 		nr_sig += rds_ib_set_wr_signal_state(ic, send, op->op_notify);  in rds_ib_xmit_rdma()
906 		send->s_wr.opcode = op->op_write ? IB_WR_RDMA_WRITE : IB_WR_RDMA_READ;  in rds_ib_xmit_rdma()
907 		send->s_rdma_wr.remote_addr = remote_addr;  in rds_ib_xmit_rdma()
908 		send->s_rdma_wr.rkey = op->op_rkey;  in rds_ib_xmit_rdma()
911 			send->s_rdma_wr.wr.num_sge = max_sge;  in rds_ib_xmit_rdma()
914 			send->s_rdma_wr.wr.num_sge = num_sge;  in rds_ib_xmit_rdma()
917 		send->s_rdma_wr.wr.next = NULL;  in rds_ib_xmit_rdma()
920 			prev->s_rdma_wr.wr.next = &send->s_rdma_wr.wr;  in rds_ib_xmit_rdma()
922 		for (j = 0; j < send->s_rdma_wr.wr.num_sge &&  in rds_ib_xmit_rdma()
925 			send->s_sge[j].addr =  in rds_ib_xmit_rdma()
927 			send->s_sge[j].length = len;  in rds_ib_xmit_rdma()
928 			send->s_sge[j].lkey = ic->i_pd->local_dma_lkey;  in rds_ib_xmit_rdma()
937 		rdsdebug("send %p wr %p num_sge %u next %p\n", send,  in rds_ib_xmit_rdma()
938 			&send->s_rdma_wr.wr,  in rds_ib_xmit_rdma()
939 			send->s_rdma_wr.wr.num_sge,  in rds_ib_xmit_rdma()
940 			send->s_rdma_wr.wr.next);  in rds_ib_xmit_rdma()
942 		prev = send;  in rds_ib_xmit_rdma()
943 		if (++send == &ic->i_sends[ic->i_send_ring.w_nr])  in rds_ib_xmit_rdma()
944 			send = ic->i_sends;  in rds_ib_xmit_rdma()