Lines Matching +full:retain +full:- +full:state +full:- +full:shutdown
14 * - Redistributions of source code must retain the above
18 * - Redistributions in binary form must reproduce the above
52 conn->c_version = version; in rds_ib_set_protocol()
60 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_set_flow_control()
64 ic->i_flowctl = 1; in rds_ib_set_flow_control()
67 ic->i_flowctl = 0; in rds_ib_set_flow_control()
73 * low timeout, but not the absolute minimum - this should
77 * smallest infinite number :-) above.
82 * Caller passes in a qp_attr pointer - don't waste stack spacv
90 attr->min_rnr_timer = IB_RNR_TIMER_000_32; in rds_ib_tune_rnr()
91 ret = ib_modify_qp(ic->i_cm_id->qp, attr, IB_QP_MIN_RNR_TIMER); in rds_ib_tune_rnr()
93 printk(KERN_NOTICE "ib_modify_qp(IB_QP_MIN_RNR_TIMER): err=%d\n", -ret); in rds_ib_tune_rnr()
102 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_cm_connect_complete()
111 dp = event->param.conn.private_data; in rds_ib_cm_connect_complete()
112 if (conn->c_isv6) { in rds_ib_cm_connect_complete()
113 if (event->param.conn.private_data_len >= in rds_ib_cm_connect_complete()
115 major = dp->ricp_v6.dp_protocol_major; in rds_ib_cm_connect_complete()
116 minor = dp->ricp_v6.dp_protocol_minor; in rds_ib_cm_connect_complete()
117 credit = dp->ricp_v6.dp_credit; in rds_ib_cm_connect_complete()
119 * aligned. Since dp_ack_seq is 64-bit extended load in rds_ib_cm_connect_complete()
123 ack_seq = get_unaligned(&dp->ricp_v6.dp_ack_seq); in rds_ib_cm_connect_complete()
125 } else if (event->param.conn.private_data_len >= in rds_ib_cm_connect_complete()
127 major = dp->ricp_v4.dp_protocol_major; in rds_ib_cm_connect_complete()
128 minor = dp->ricp_v4.dp_protocol_minor; in rds_ib_cm_connect_complete()
129 credit = dp->ricp_v4.dp_credit; in rds_ib_cm_connect_complete()
130 ack_seq = get_unaligned(&dp->ricp_v4.dp_ack_seq); in rds_ib_cm_connect_complete()
139 if (conn->c_version < RDS_PROTOCOL_VERSION) { in rds_ib_cm_connect_complete()
140 if (conn->c_version != RDS_PROTOCOL_COMPAT_VERSION) { in rds_ib_cm_connect_complete()
142 &conn->c_laddr, &conn->c_faddr, in rds_ib_cm_connect_complete()
143 RDS_PROTOCOL_MAJOR(conn->c_version), in rds_ib_cm_connect_complete()
144 RDS_PROTOCOL_MINOR(conn->c_version)); in rds_ib_cm_connect_complete()
151 ic->i_active_side ? "Active" : "Passive", in rds_ib_cm_connect_complete()
152 &conn->c_laddr, &conn->c_faddr, conn->c_tos, in rds_ib_cm_connect_complete()
153 RDS_PROTOCOL_MAJOR(conn->c_version), in rds_ib_cm_connect_complete()
154 RDS_PROTOCOL_MINOR(conn->c_version), in rds_ib_cm_connect_complete()
155 ic->i_flowctl ? ", flow control" : ""); in rds_ib_cm_connect_complete()
158 ic->i_sl = ic->i_cm_id->route.path_rec->sl; in rds_ib_cm_connect_complete()
160 atomic_set(&ic->i_cq_quiesce, 0); in rds_ib_cm_connect_complete()
168 /* Post receive buffers - as a side effect, this will update in rds_ib_cm_connect_complete()
176 err = ib_modify_qp(ic->i_cm_id->qp, &qp_attr, IB_QP_STATE); in rds_ib_cm_connect_complete()
181 err = rds_ib_update_ipaddr(ic->rds_ibdev, &conn->c_laddr); in rds_ib_cm_connect_complete()
194 conn->c_proposed_version = conn->c_version; in rds_ib_cm_connect_complete()
206 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_cm_fill_conn_param()
207 struct rds_ib_device *rds_ibdev = ic->rds_ibdev; in rds_ib_cm_fill_conn_param()
211 conn_param->responder_resources = in rds_ib_cm_fill_conn_param()
212 min_t(u32, rds_ibdev->max_responder_resources, max_responder_resources); in rds_ib_cm_fill_conn_param()
213 conn_param->initiator_depth = in rds_ib_cm_fill_conn_param()
214 min_t(u32, rds_ibdev->max_initiator_depth, max_initiator_depth); in rds_ib_cm_fill_conn_param()
215 conn_param->retry_count = min_t(unsigned int, rds_ib_retry_count, 7); in rds_ib_cm_fill_conn_param()
216 conn_param->rnr_retry_count = 7; in rds_ib_cm_fill_conn_param()
221 dp->ricp_v6.dp_saddr = conn->c_laddr; in rds_ib_cm_fill_conn_param()
222 dp->ricp_v6.dp_daddr = conn->c_faddr; in rds_ib_cm_fill_conn_param()
223 dp->ricp_v6.dp_protocol_major = in rds_ib_cm_fill_conn_param()
225 dp->ricp_v6.dp_protocol_minor = in rds_ib_cm_fill_conn_param()
227 dp->ricp_v6.dp_protocol_minor_mask = in rds_ib_cm_fill_conn_param()
229 dp->ricp_v6.dp_ack_seq = in rds_ib_cm_fill_conn_param()
231 dp->ricp_v6.dp_cmn.ricpc_dp_toss = conn->c_tos; in rds_ib_cm_fill_conn_param()
233 conn_param->private_data = &dp->ricp_v6; in rds_ib_cm_fill_conn_param()
234 conn_param->private_data_len = sizeof(dp->ricp_v6); in rds_ib_cm_fill_conn_param()
236 dp->ricp_v4.dp_saddr = conn->c_laddr.s6_addr32[3]; in rds_ib_cm_fill_conn_param()
237 dp->ricp_v4.dp_daddr = conn->c_faddr.s6_addr32[3]; in rds_ib_cm_fill_conn_param()
238 dp->ricp_v4.dp_protocol_major = in rds_ib_cm_fill_conn_param()
240 dp->ricp_v4.dp_protocol_minor = in rds_ib_cm_fill_conn_param()
242 dp->ricp_v4.dp_protocol_minor_mask = in rds_ib_cm_fill_conn_param()
244 dp->ricp_v4.dp_ack_seq = in rds_ib_cm_fill_conn_param()
246 dp->ricp_v4.dp_cmn.ricpc_dp_toss = conn->c_tos; in rds_ib_cm_fill_conn_param()
248 conn_param->private_data = &dp->ricp_v4; in rds_ib_cm_fill_conn_param()
249 conn_param->private_data_len = sizeof(dp->ricp_v4); in rds_ib_cm_fill_conn_param()
253 if (ic->i_flowctl) { in rds_ib_cm_fill_conn_param()
257 (atomic_read(&ic->i_credits)); in rds_ib_cm_fill_conn_param()
259 dp->ricp_v6.dp_credit = cpu_to_be32(credits); in rds_ib_cm_fill_conn_param()
261 dp->ricp_v4.dp_credit = cpu_to_be32(credits); in rds_ib_cm_fill_conn_param()
263 &ic->i_credits); in rds_ib_cm_fill_conn_param()
271 event->event, ib_event_msg(event->event), data); in rds_ib_cq_event_handler()
276 * spinlocks and the transient state of refilling doesn't change the
285 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_cq_comp_handler_recv()
291 tasklet_schedule(&ic->i_recv_tasklet); in rds_ib_cq_comp_handler_recv()
304 (unsigned long long)wc->wr_id, wc->status, in poll_scq()
305 wc->byte_len, be32_to_cpu(wc->ex.imm_data)); in poll_scq()
307 if (wc->wr_id <= ic->i_send_ring.w_nr || in poll_scq()
308 wc->wr_id == RDS_IB_ACK_WR_ID) in poll_scq()
320 struct rds_connection *conn = ic->conn; in rds_ib_tasklet_fn_send()
325 if (atomic_read(&ic->i_cq_quiesce)) in rds_ib_tasklet_fn_send()
328 poll_scq(ic, ic->i_send_cq, ic->i_send_wc); in rds_ib_tasklet_fn_send()
329 ib_req_notify_cq(ic->i_send_cq, IB_CQ_NEXT_COMP); in rds_ib_tasklet_fn_send()
330 poll_scq(ic, ic->i_send_cq, ic->i_send_wc); in rds_ib_tasklet_fn_send()
333 (!test_bit(RDS_LL_SEND_FULL, &conn->c_flags) || in rds_ib_tasklet_fn_send()
334 test_bit(0, &conn->c_map_queued))) in rds_ib_tasklet_fn_send()
335 rds_send_xmit(&ic->conn->c_path[0]); in rds_ib_tasklet_fn_send()
349 (unsigned long long)wc->wr_id, wc->status, in poll_rcq()
350 wc->byte_len, be32_to_cpu(wc->ex.imm_data)); in poll_rcq()
360 struct rds_connection *conn = ic->conn; in rds_ib_tasklet_fn_recv()
361 struct rds_ib_device *rds_ibdev = ic->rds_ibdev; in rds_ib_tasklet_fn_recv()
362 struct rds_ib_ack_state state; in rds_ib_tasklet_fn_recv() local
370 if (atomic_read(&ic->i_cq_quiesce)) in rds_ib_tasklet_fn_recv()
373 memset(&state, 0, sizeof(state)); in rds_ib_tasklet_fn_recv()
374 poll_rcq(ic, ic->i_recv_cq, ic->i_recv_wc, &state); in rds_ib_tasklet_fn_recv()
375 ib_req_notify_cq(ic->i_recv_cq, IB_CQ_SOLICITED); in rds_ib_tasklet_fn_recv()
376 poll_rcq(ic, ic->i_recv_cq, ic->i_recv_wc, &state); in rds_ib_tasklet_fn_recv()
378 if (state.ack_next_valid) in rds_ib_tasklet_fn_recv()
379 rds_ib_set_ack(ic, state.ack_next, state.ack_required); in rds_ib_tasklet_fn_recv()
380 if (state.ack_recv_valid && state.ack_recv > ic->i_ack_recv) { in rds_ib_tasklet_fn_recv()
381 rds_send_drop_acked(conn, state.ack_recv, NULL); in rds_ib_tasklet_fn_recv()
382 ic->i_ack_recv = state.ack_recv; in rds_ib_tasklet_fn_recv()
392 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_qp_event_handler()
394 rdsdebug("conn %p ic %p event %u (%s)\n", conn, ic, event->event, in rds_ib_qp_event_handler()
395 ib_event_msg(event->event)); in rds_ib_qp_event_handler()
397 switch (event->event) { in rds_ib_qp_event_handler()
399 rdma_notify(ic->i_cm_id, IB_EVENT_COMM_EST); in rds_ib_qp_event_handler()
402 rdsdebug("Fatal QP Event %u (%s) - connection %pI6c->%pI6c, reconnecting\n", in rds_ib_qp_event_handler()
403 event->event, ib_event_msg(event->event), in rds_ib_qp_event_handler()
404 &conn->c_laddr, &conn->c_faddr); in rds_ib_qp_event_handler()
413 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_cq_comp_handler_send()
419 tasklet_schedule(&ic->i_send_tasklet); in rds_ib_cq_comp_handler_send()
424 int min = rds_ibdev->vector_load[rds_ibdev->dev->num_comp_vectors - 1]; in ibdev_get_unused_vector()
425 int index = rds_ibdev->dev->num_comp_vectors - 1; in ibdev_get_unused_vector()
428 for (i = rds_ibdev->dev->num_comp_vectors - 1; i >= 0; i--) { in ibdev_get_unused_vector()
429 if (rds_ibdev->vector_load[i] < min) { in ibdev_get_unused_vector()
431 min = rds_ibdev->vector_load[i]; in ibdev_get_unused_vector()
435 rds_ibdev->vector_load[index]++; in ibdev_get_unused_vector()
441 rds_ibdev->vector_load[index]--; in ibdev_put_vector()
512 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_setup_qp()
513 struct ib_device *dev = ic->i_cm_id->device; in rds_ib_setup_qp()
527 return -EOPNOTSUPP; in rds_ib_setup_qp()
538 max_wrs = rds_ibdev->max_wrs < rds_ib_sysctl_max_send_wr + 1 ? in rds_ib_setup_qp()
539 rds_ibdev->max_wrs - 1 : rds_ib_sysctl_max_send_wr; in rds_ib_setup_qp()
540 if (ic->i_send_ring.w_nr != max_wrs) in rds_ib_setup_qp()
541 rds_ib_ring_resize(&ic->i_send_ring, max_wrs); in rds_ib_setup_qp()
543 max_wrs = rds_ibdev->max_wrs < rds_ib_sysctl_max_recv_wr + 1 ? in rds_ib_setup_qp()
544 rds_ibdev->max_wrs - 1 : rds_ib_sysctl_max_recv_wr; in rds_ib_setup_qp()
545 if (ic->i_recv_ring.w_nr != max_wrs) in rds_ib_setup_qp()
546 rds_ib_ring_resize(&ic->i_recv_ring, max_wrs); in rds_ib_setup_qp()
549 ic->i_pd = rds_ibdev->pd; in rds_ib_setup_qp()
551 ic->i_scq_vector = ibdev_get_unused_vector(rds_ibdev); in rds_ib_setup_qp()
552 cq_attr.cqe = ic->i_send_ring.w_nr + fr_queue_space + 1; in rds_ib_setup_qp()
553 cq_attr.comp_vector = ic->i_scq_vector; in rds_ib_setup_qp()
554 ic->i_send_cq = ib_create_cq(dev, rds_ib_cq_comp_handler_send, in rds_ib_setup_qp()
557 if (IS_ERR(ic->i_send_cq)) { in rds_ib_setup_qp()
558 ret = PTR_ERR(ic->i_send_cq); in rds_ib_setup_qp()
559 ic->i_send_cq = NULL; in rds_ib_setup_qp()
560 ibdev_put_vector(rds_ibdev, ic->i_scq_vector); in rds_ib_setup_qp()
565 ic->i_rcq_vector = ibdev_get_unused_vector(rds_ibdev); in rds_ib_setup_qp()
566 cq_attr.cqe = ic->i_recv_ring.w_nr; in rds_ib_setup_qp()
567 cq_attr.comp_vector = ic->i_rcq_vector; in rds_ib_setup_qp()
568 ic->i_recv_cq = ib_create_cq(dev, rds_ib_cq_comp_handler_recv, in rds_ib_setup_qp()
571 if (IS_ERR(ic->i_recv_cq)) { in rds_ib_setup_qp()
572 ret = PTR_ERR(ic->i_recv_cq); in rds_ib_setup_qp()
573 ic->i_recv_cq = NULL; in rds_ib_setup_qp()
574 ibdev_put_vector(rds_ibdev, ic->i_rcq_vector); in rds_ib_setup_qp()
579 ret = ib_req_notify_cq(ic->i_send_cq, IB_CQ_NEXT_COMP); in rds_ib_setup_qp()
585 ret = ib_req_notify_cq(ic->i_recv_cq, IB_CQ_SOLICITED); in rds_ib_setup_qp()
596 attr.cap.max_send_wr = ic->i_send_ring.w_nr + fr_queue_space + 1; in rds_ib_setup_qp()
597 attr.cap.max_recv_wr = ic->i_recv_ring.w_nr + 1; in rds_ib_setup_qp()
598 attr.cap.max_send_sge = rds_ibdev->max_sge; in rds_ib_setup_qp()
602 attr.send_cq = ic->i_send_cq; in rds_ib_setup_qp()
603 attr.recv_cq = ic->i_recv_cq; in rds_ib_setup_qp()
609 ret = rdma_create_qp(ic->i_cm_id, ic->i_pd, &attr); in rds_ib_setup_qp()
615 pool = rds_ibdev->rid_hdrs_pool; in rds_ib_setup_qp()
616 ic->i_send_hdrs = rds_dma_hdrs_alloc(dev, pool, &ic->i_send_hdrs_dma, in rds_ib_setup_qp()
617 ic->i_send_ring.w_nr); in rds_ib_setup_qp()
618 if (!ic->i_send_hdrs) { in rds_ib_setup_qp()
619 ret = -ENOMEM; in rds_ib_setup_qp()
624 ic->i_recv_hdrs = rds_dma_hdrs_alloc(dev, pool, &ic->i_recv_hdrs_dma, in rds_ib_setup_qp()
625 ic->i_recv_ring.w_nr); in rds_ib_setup_qp()
626 if (!ic->i_recv_hdrs) { in rds_ib_setup_qp()
627 ret = -ENOMEM; in rds_ib_setup_qp()
632 ic->i_ack = dma_pool_zalloc(pool, GFP_KERNEL, in rds_ib_setup_qp()
633 &ic->i_ack_dma); in rds_ib_setup_qp()
634 if (!ic->i_ack) { in rds_ib_setup_qp()
635 ret = -ENOMEM; in rds_ib_setup_qp()
640 ic->i_sends = vzalloc_node(array_size(sizeof(struct rds_ib_send_work), in rds_ib_setup_qp()
641 ic->i_send_ring.w_nr), in rds_ib_setup_qp()
643 if (!ic->i_sends) { in rds_ib_setup_qp()
644 ret = -ENOMEM; in rds_ib_setup_qp()
649 ic->i_recvs = vzalloc_node(array_size(sizeof(struct rds_ib_recv_work), in rds_ib_setup_qp()
650 ic->i_recv_ring.w_nr), in rds_ib_setup_qp()
652 if (!ic->i_recvs) { in rds_ib_setup_qp()
653 ret = -ENOMEM; in rds_ib_setup_qp()
660 rdsdebug("conn %p pd %p cq %p %p\n", conn, ic->i_pd, in rds_ib_setup_qp()
661 ic->i_send_cq, ic->i_recv_cq); in rds_ib_setup_qp()
666 vfree(ic->i_sends); in rds_ib_setup_qp()
669 dma_pool_free(pool, ic->i_ack, ic->i_ack_dma); in rds_ib_setup_qp()
670 ic->i_ack = NULL; in rds_ib_setup_qp()
673 rds_dma_hdrs_free(pool, ic->i_recv_hdrs, ic->i_recv_hdrs_dma, in rds_ib_setup_qp()
674 ic->i_recv_ring.w_nr); in rds_ib_setup_qp()
675 ic->i_recv_hdrs = NULL; in rds_ib_setup_qp()
676 ic->i_recv_hdrs_dma = NULL; in rds_ib_setup_qp()
679 rds_dma_hdrs_free(pool, ic->i_send_hdrs, ic->i_send_hdrs_dma, in rds_ib_setup_qp()
680 ic->i_send_ring.w_nr); in rds_ib_setup_qp()
681 ic->i_send_hdrs = NULL; in rds_ib_setup_qp()
682 ic->i_send_hdrs_dma = NULL; in rds_ib_setup_qp()
685 rdma_destroy_qp(ic->i_cm_id); in rds_ib_setup_qp()
687 ib_destroy_cq(ic->i_recv_cq); in rds_ib_setup_qp()
688 ic->i_recv_cq = NULL; in rds_ib_setup_qp()
690 ib_destroy_cq(ic->i_send_cq); in rds_ib_setup_qp()
691 ic->i_send_cq = NULL; in rds_ib_setup_qp()
702 const union rds_ib_conn_priv *dp = event->param.conn.private_data; in rds_ib_protocol_compatible()
709 * rdma_cm private data is odd - when there is any private data in the in rds_ib_protocol_compatible()
714 * from an older version. This could be 3.0 or 2.0 - we can't tell. in rds_ib_protocol_compatible()
715 * We really should have changed this for OFED 1.3 :-( in rds_ib_protocol_compatible()
719 if (!event->param.conn.private_data_len) { in rds_ib_protocol_compatible()
727 major = dp->ricp_v6.dp_protocol_major; in rds_ib_protocol_compatible()
728 minor = dp->ricp_v6.dp_protocol_minor; in rds_ib_protocol_compatible()
729 mask = dp->ricp_v6.dp_protocol_minor_mask; in rds_ib_protocol_compatible()
732 major = dp->ricp_v4.dp_protocol_major; in rds_ib_protocol_compatible()
733 minor = dp->ricp_v4.dp_protocol_minor; in rds_ib_protocol_compatible()
734 mask = dp->ricp_v4.dp_protocol_minor_mask; in rds_ib_protocol_compatible()
737 /* Even if len is crap *now* I still want to check it. -ASG */ in rds_ib_protocol_compatible()
738 if (event->param.conn.private_data_len < data_len || major == 0) in rds_ib_protocol_compatible()
752 &dp->ricp_v6.dp_saddr, major, minor); in rds_ib_protocol_compatible()
755 &dp->ricp_v4.dp_saddr, major, minor); in rds_ib_protocol_compatible()
779 idx = dev->ifindex; in __rds_find_ifindex()
792 __be64 lguid = cm_id->route.path_rec->sgid.global.interface_id; in rds_ib_cm_handle_connect()
793 __be64 fguid = cm_id->route.path_rec->dgid.global.interface_id; in rds_ib_cm_handle_connect()
816 dp = event->param.conn.private_data; in rds_ib_cm_handle_connect()
819 dp_cmn = &dp->ricp_v6.dp_cmn; in rds_ib_cm_handle_connect()
820 saddr6 = &dp->ricp_v6.dp_saddr; in rds_ib_cm_handle_connect()
821 daddr6 = &dp->ricp_v6.dp_daddr; in rds_ib_cm_handle_connect()
831 err = -EOPNOTSUPP; in rds_ib_cm_handle_connect()
839 err = -EOPNOTSUPP; in rds_ib_cm_handle_connect()
844 err = -EOPNOTSUPP; in rds_ib_cm_handle_connect()
848 dp_cmn = &dp->ricp_v4.dp_cmn; in rds_ib_cm_handle_connect()
849 ipv6_addr_set_v4mapped(dp->ricp_v4.dp_saddr, &s_mapped_addr); in rds_ib_cm_handle_connect()
850 ipv6_addr_set_v4mapped(dp->ricp_v4.dp_daddr, &d_mapped_addr); in rds_ib_cm_handle_connect()
859 (unsigned long long)be64_to_cpu(fguid), dp_cmn->ricpc_dp_toss); in rds_ib_cm_handle_connect()
863 &rds_ib_transport, dp_cmn->ricpc_dp_toss, in rds_ib_cm_handle_connect()
875 * by both hosts, we have a random backoff mechanism - in rds_ib_cm_handle_connect()
878 mutex_lock(&conn->c_cm_lock); in rds_ib_cm_handle_connect()
886 /* Wait and see - our connect may still be succeeding */ in rds_ib_cm_handle_connect()
892 ic = conn->c_transport_data; in rds_ib_cm_handle_connect()
895 rds_ib_set_flow_control(conn, be32_to_cpu(dp_cmn->ricpc_credit)); in rds_ib_cm_handle_connect()
899 if (dp_cmn->ricpc_ack_seq) in rds_ib_cm_handle_connect()
900 rds_send_drop_acked(conn, be64_to_cpu(dp_cmn->ricpc_ack_seq), in rds_ib_cm_handle_connect()
903 BUG_ON(cm_id->context); in rds_ib_cm_handle_connect()
904 BUG_ON(ic->i_cm_id); in rds_ib_cm_handle_connect()
906 ic->i_cm_id = cm_id; in rds_ib_cm_handle_connect()
907 cm_id->context = conn; in rds_ib_cm_handle_connect()
920 event->param.conn.responder_resources, in rds_ib_cm_handle_connect()
921 event->param.conn.initiator_depth, isv6); in rds_ib_cm_handle_connect()
929 mutex_unlock(&conn->c_cm_lock); in rds_ib_cm_handle_connect()
939 struct rds_connection *conn = cm_id->context; in rds_ib_cm_initiate_connect()
940 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_cm_initiate_connect()
948 ic->i_flowctl = rds_ib_sysctl_flow_control; /* advertise flow control */ in rds_ib_cm_initiate_connect()
957 conn->c_proposed_version, in rds_ib_cm_initiate_connect()
965 /* Beware - returning non-zero tells the rdma_cm to destroy in rds_ib_cm_initiate_connect()
969 if (ic->i_cm_id == cm_id) in rds_ib_cm_initiate_connect()
972 ic->i_active_side = true; in rds_ib_cm_initiate_connect()
978 struct rds_connection *conn = cp->cp_conn; in rds_ib_conn_path_connect()
984 ic = conn->c_transport_data; in rds_ib_conn_path_connect()
989 if (conn->c_isv6) in rds_ib_conn_path_connect()
994 ic->i_cm_id = rdma_create_id(&init_net, handler, conn, in rds_ib_conn_path_connect()
996 if (IS_ERR(ic->i_cm_id)) { in rds_ib_conn_path_connect()
997 ret = PTR_ERR(ic->i_cm_id); in rds_ib_conn_path_connect()
998 ic->i_cm_id = NULL; in rds_ib_conn_path_connect()
1003 rdsdebug("created cm id %p for conn %p\n", ic->i_cm_id, conn); in rds_ib_conn_path_connect()
1005 if (ipv6_addr_v4mapped(&conn->c_faddr)) { in rds_ib_conn_path_connect()
1009 sin->sin_family = AF_INET; in rds_ib_conn_path_connect()
1010 sin->sin_addr.s_addr = conn->c_laddr.s6_addr32[3]; in rds_ib_conn_path_connect()
1011 sin->sin_port = 0; in rds_ib_conn_path_connect()
1014 sin->sin_family = AF_INET; in rds_ib_conn_path_connect()
1015 sin->sin_addr.s_addr = conn->c_faddr.s6_addr32[3]; in rds_ib_conn_path_connect()
1016 sin->sin_port = htons(RDS_PORT); in rds_ib_conn_path_connect()
1021 sin6->sin6_family = AF_INET6; in rds_ib_conn_path_connect()
1022 sin6->sin6_addr = conn->c_laddr; in rds_ib_conn_path_connect()
1023 sin6->sin6_port = 0; in rds_ib_conn_path_connect()
1024 sin6->sin6_scope_id = conn->c_dev_if; in rds_ib_conn_path_connect()
1027 sin6->sin6_family = AF_INET6; in rds_ib_conn_path_connect()
1028 sin6->sin6_addr = conn->c_faddr; in rds_ib_conn_path_connect()
1029 sin6->sin6_port = htons(RDS_CM_PORT); in rds_ib_conn_path_connect()
1030 sin6->sin6_scope_id = conn->c_dev_if; in rds_ib_conn_path_connect()
1033 ret = rdma_resolve_addr(ic->i_cm_id, (struct sockaddr *)&src, in rds_ib_conn_path_connect()
1037 rdsdebug("addr resolve failed for cm id %p: %d\n", ic->i_cm_id, in rds_ib_conn_path_connect()
1039 rdma_destroy_id(ic->i_cm_id); in rds_ib_conn_path_connect()
1040 ic->i_cm_id = NULL; in rds_ib_conn_path_connect()
1054 struct rds_connection *conn = cp->cp_conn; in rds_ib_conn_path_shutdown()
1055 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_conn_path_shutdown()
1058 rdsdebug("cm %p pd %p cq %p %p qp %p\n", ic->i_cm_id, in rds_ib_conn_path_shutdown()
1059 ic->i_pd, ic->i_send_cq, ic->i_recv_cq, in rds_ib_conn_path_shutdown()
1060 ic->i_cm_id ? ic->i_cm_id->qp : NULL); in rds_ib_conn_path_shutdown()
1062 if (ic->i_cm_id) { in rds_ib_conn_path_shutdown()
1063 rdsdebug("disconnecting cm %p\n", ic->i_cm_id); in rds_ib_conn_path_shutdown()
1064 err = rdma_disconnect(ic->i_cm_id); in rds_ib_conn_path_shutdown()
1070 ic->i_cm_id, err); in rds_ib_conn_path_shutdown()
1082 * only has unsignaled sends in it. We've shutdown new in rds_ib_conn_path_shutdown()
1088 rds_ib_ring_empty(&ic->i_recv_ring) && in rds_ib_conn_path_shutdown()
1089 (atomic_read(&ic->i_signaled_sends) == 0) && in rds_ib_conn_path_shutdown()
1090 (atomic_read(&ic->i_fastreg_inuse_count) == 0) && in rds_ib_conn_path_shutdown()
1091 (atomic_read(&ic->i_fastreg_wrs) == RDS_IB_DEFAULT_FR_WR)); in rds_ib_conn_path_shutdown()
1092 tasklet_kill(&ic->i_send_tasklet); in rds_ib_conn_path_shutdown()
1093 tasklet_kill(&ic->i_recv_tasklet); in rds_ib_conn_path_shutdown()
1095 atomic_set(&ic->i_cq_quiesce, 1); in rds_ib_conn_path_shutdown()
1097 /* first destroy the ib state that generates callbacks */ in rds_ib_conn_path_shutdown()
1098 if (ic->i_cm_id->qp) in rds_ib_conn_path_shutdown()
1099 rdma_destroy_qp(ic->i_cm_id); in rds_ib_conn_path_shutdown()
1100 if (ic->i_send_cq) { in rds_ib_conn_path_shutdown()
1101 if (ic->rds_ibdev) in rds_ib_conn_path_shutdown()
1102 ibdev_put_vector(ic->rds_ibdev, ic->i_scq_vector); in rds_ib_conn_path_shutdown()
1103 ib_destroy_cq(ic->i_send_cq); in rds_ib_conn_path_shutdown()
1106 if (ic->i_recv_cq) { in rds_ib_conn_path_shutdown()
1107 if (ic->rds_ibdev) in rds_ib_conn_path_shutdown()
1108 ibdev_put_vector(ic->rds_ibdev, ic->i_rcq_vector); in rds_ib_conn_path_shutdown()
1109 ib_destroy_cq(ic->i_recv_cq); in rds_ib_conn_path_shutdown()
1112 if (ic->rds_ibdev) { in rds_ib_conn_path_shutdown()
1115 pool = ic->rds_ibdev->rid_hdrs_pool; in rds_ib_conn_path_shutdown()
1118 if (ic->i_send_hdrs) { in rds_ib_conn_path_shutdown()
1119 rds_dma_hdrs_free(pool, ic->i_send_hdrs, in rds_ib_conn_path_shutdown()
1120 ic->i_send_hdrs_dma, in rds_ib_conn_path_shutdown()
1121 ic->i_send_ring.w_nr); in rds_ib_conn_path_shutdown()
1122 ic->i_send_hdrs = NULL; in rds_ib_conn_path_shutdown()
1123 ic->i_send_hdrs_dma = NULL; in rds_ib_conn_path_shutdown()
1126 if (ic->i_recv_hdrs) { in rds_ib_conn_path_shutdown()
1127 rds_dma_hdrs_free(pool, ic->i_recv_hdrs, in rds_ib_conn_path_shutdown()
1128 ic->i_recv_hdrs_dma, in rds_ib_conn_path_shutdown()
1129 ic->i_recv_ring.w_nr); in rds_ib_conn_path_shutdown()
1130 ic->i_recv_hdrs = NULL; in rds_ib_conn_path_shutdown()
1131 ic->i_recv_hdrs_dma = NULL; in rds_ib_conn_path_shutdown()
1134 if (ic->i_ack) { in rds_ib_conn_path_shutdown()
1135 dma_pool_free(pool, ic->i_ack, ic->i_ack_dma); in rds_ib_conn_path_shutdown()
1136 ic->i_ack = NULL; in rds_ib_conn_path_shutdown()
1139 WARN_ON(ic->i_send_hdrs); in rds_ib_conn_path_shutdown()
1140 WARN_ON(ic->i_send_hdrs_dma); in rds_ib_conn_path_shutdown()
1141 WARN_ON(ic->i_recv_hdrs); in rds_ib_conn_path_shutdown()
1142 WARN_ON(ic->i_recv_hdrs_dma); in rds_ib_conn_path_shutdown()
1143 WARN_ON(ic->i_ack); in rds_ib_conn_path_shutdown()
1146 if (ic->i_sends) in rds_ib_conn_path_shutdown()
1148 if (ic->i_recvs) in rds_ib_conn_path_shutdown()
1151 rdma_destroy_id(ic->i_cm_id); in rds_ib_conn_path_shutdown()
1156 if (ic->rds_ibdev) in rds_ib_conn_path_shutdown()
1157 rds_ib_remove_conn(ic->rds_ibdev, conn); in rds_ib_conn_path_shutdown()
1159 ic->i_cm_id = NULL; in rds_ib_conn_path_shutdown()
1160 ic->i_pd = NULL; in rds_ib_conn_path_shutdown()
1161 ic->i_send_cq = NULL; in rds_ib_conn_path_shutdown()
1162 ic->i_recv_cq = NULL; in rds_ib_conn_path_shutdown()
1164 BUG_ON(ic->rds_ibdev); in rds_ib_conn_path_shutdown()
1167 if (ic->i_data_op) { in rds_ib_conn_path_shutdown()
1170 rm = container_of(ic->i_data_op, struct rds_message, data); in rds_ib_conn_path_shutdown()
1172 ic->i_data_op = NULL; in rds_ib_conn_path_shutdown()
1175 /* Clear the ACK state */ in rds_ib_conn_path_shutdown()
1176 clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags); in rds_ib_conn_path_shutdown()
1178 atomic64_set(&ic->i_ack_next, 0); in rds_ib_conn_path_shutdown()
1180 ic->i_ack_next = 0; in rds_ib_conn_path_shutdown()
1182 ic->i_ack_recv = 0; in rds_ib_conn_path_shutdown()
1184 /* Clear flow control state */ in rds_ib_conn_path_shutdown()
1185 ic->i_flowctl = 0; in rds_ib_conn_path_shutdown()
1186 atomic_set(&ic->i_credits, 0); in rds_ib_conn_path_shutdown()
1188 /* Re-init rings, but retain sizes. */ in rds_ib_conn_path_shutdown()
1189 rds_ib_ring_init(&ic->i_send_ring, ic->i_send_ring.w_nr); in rds_ib_conn_path_shutdown()
1190 rds_ib_ring_init(&ic->i_recv_ring, ic->i_recv_ring.w_nr); in rds_ib_conn_path_shutdown()
1192 if (ic->i_ibinc) { in rds_ib_conn_path_shutdown()
1193 rds_inc_put(&ic->i_ibinc->ii_inc); in rds_ib_conn_path_shutdown()
1194 ic->i_ibinc = NULL; in rds_ib_conn_path_shutdown()
1197 vfree(ic->i_sends); in rds_ib_conn_path_shutdown()
1198 ic->i_sends = NULL; in rds_ib_conn_path_shutdown()
1199 vfree(ic->i_recvs); in rds_ib_conn_path_shutdown()
1200 ic->i_recvs = NULL; in rds_ib_conn_path_shutdown()
1201 ic->i_active_side = false; in rds_ib_conn_path_shutdown()
1213 return -ENOMEM; in rds_ib_conn_alloc()
1221 INIT_LIST_HEAD(&ic->ib_node); in rds_ib_conn_alloc()
1222 tasklet_init(&ic->i_send_tasklet, rds_ib_tasklet_fn_send, in rds_ib_conn_alloc()
1224 tasklet_init(&ic->i_recv_tasklet, rds_ib_tasklet_fn_recv, in rds_ib_conn_alloc()
1226 mutex_init(&ic->i_recv_mutex); in rds_ib_conn_alloc()
1228 spin_lock_init(&ic->i_ack_lock); in rds_ib_conn_alloc()
1230 atomic_set(&ic->i_signaled_sends, 0); in rds_ib_conn_alloc()
1231 atomic_set(&ic->i_fastreg_wrs, RDS_IB_DEFAULT_FR_WR); in rds_ib_conn_alloc()
1237 rds_ib_ring_init(&ic->i_send_ring, 0); in rds_ib_conn_alloc()
1238 rds_ib_ring_init(&ic->i_recv_ring, 0); in rds_ib_conn_alloc()
1240 ic->conn = conn; in rds_ib_conn_alloc()
1241 conn->c_transport_data = ic; in rds_ib_conn_alloc()
1244 list_add_tail(&ic->ib_node, &ib_nodev_conns); in rds_ib_conn_alloc()
1248 rdsdebug("conn %p conn ic %p\n", conn, conn->c_transport_data); in rds_ib_conn_alloc()
1264 * A race with shutdown() or connect() would cause problems in rds_ib_conn_free()
1267 lock_ptr = ic->rds_ibdev ? &ic->rds_ibdev->spinlock : &ib_nodev_conns_lock; in rds_ib_conn_free()
1270 list_del(&ic->ib_node); in rds_ib_conn_free()