58 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_set_flow_control()  local
62 		ic->i_flowctl = 1;  in rds_ib_set_flow_control()
65 		ic->i_flowctl = 0;  in rds_ib_set_flow_control()
84 rds_ib_tune_rnr(struct rds_ib_connection *ic, struct ib_qp_attr *attr)  in rds_ib_tune_rnr()  argument
89 	ret = ib_modify_qp(ic->i_cm_id->qp, attr, IB_QP_MIN_RNR_TIMER);  in rds_ib_tune_rnr()
100 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_cm_connect_complete()  local
149 		  ic->i_active_side ? "Active" : "Passive",  in rds_ib_cm_connect_complete()
153 		  ic->i_flowctl ? ", flow control" : "");  in rds_ib_cm_connect_complete()
156 	ic->i_sl = ic->i_cm_id->route.path_rec->sl;  in rds_ib_cm_connect_complete()
158 	atomic_set(&ic->i_cq_quiesce, 0);  in rds_ib_cm_connect_complete()
164 	rds_ib_send_init_ring(ic);  in rds_ib_cm_connect_complete()
165 	rds_ib_recv_init_ring(ic);  in rds_ib_cm_connect_complete()
171 	rds_ib_tune_rnr(ic, &qp_attr);  in rds_ib_cm_connect_complete()
174 	err = ib_modify_qp(ic->i_cm_id->qp, &qp_attr, IB_QP_STATE);  in rds_ib_cm_connect_complete()
179 	err = rds_ib_update_ipaddr(ic->rds_ibdev, &conn->c_laddr);  in rds_ib_cm_connect_complete()
204 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_cm_fill_conn_param()  local
205 	struct rds_ib_device *rds_ibdev = ic->rds_ibdev;  in rds_ib_cm_fill_conn_param()
228 			    cpu_to_be64(rds_ib_piggyb_ack(ic));  in rds_ib_cm_fill_conn_param()
243 			    cpu_to_be64(rds_ib_piggyb_ack(ic));  in rds_ib_cm_fill_conn_param()
251 		if (ic->i_flowctl) {  in rds_ib_cm_fill_conn_param()
255 				(atomic_read(&ic->i_credits));  in rds_ib_cm_fill_conn_param()
261 				   &ic->i_credits);  in rds_ib_cm_fill_conn_param()
283 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_cq_comp_handler_recv()  local
289 	tasklet_schedule(&ic->i_recv_tasklet);  in rds_ib_cq_comp_handler_recv()
292 static void poll_scq(struct rds_ib_connection *ic, struct ib_cq *cq,  in poll_scq()  argument
305 			if (wc->wr_id <= ic->i_send_ring.w_nr ||  in poll_scq()
307 				rds_ib_send_cqe_handler(ic, wc);  in poll_scq()
309 				rds_ib_mr_cqe_handler(ic, wc);  in poll_scq()
317 	struct rds_ib_connection *ic = (struct rds_ib_connection *)data;  in rds_ib_tasklet_fn_send()  local
318 	struct rds_connection *conn = ic->conn;  in rds_ib_tasklet_fn_send()
323 	if (atomic_read(&ic->i_cq_quiesce))  in rds_ib_tasklet_fn_send()
326 	poll_scq(ic, ic->i_send_cq, ic->i_send_wc);  in rds_ib_tasklet_fn_send()
327 	ib_req_notify_cq(ic->i_send_cq, IB_CQ_NEXT_COMP);  in rds_ib_tasklet_fn_send()
328 	poll_scq(ic, ic->i_send_cq, ic->i_send_wc);  in rds_ib_tasklet_fn_send()
333 		rds_send_xmit(&ic->conn->c_path[0]);  in rds_ib_tasklet_fn_send()
336 static void poll_rcq(struct rds_ib_connection *ic, struct ib_cq *cq,  in poll_rcq()  argument
350 			rds_ib_recv_cqe_handler(ic, wc, ack_state);  in poll_rcq()
357 	struct rds_ib_connection *ic = (struct rds_ib_connection *)data;  in rds_ib_tasklet_fn_recv()  local
358 	struct rds_connection *conn = ic->conn;  in rds_ib_tasklet_fn_recv()
359 	struct rds_ib_device *rds_ibdev = ic->rds_ibdev;  in rds_ib_tasklet_fn_recv()
368 	if (atomic_read(&ic->i_cq_quiesce))  in rds_ib_tasklet_fn_recv()
372 	poll_rcq(ic, ic->i_recv_cq, ic->i_recv_wc, &state);  in rds_ib_tasklet_fn_recv()
373 	ib_req_notify_cq(ic->i_recv_cq, IB_CQ_SOLICITED);  in rds_ib_tasklet_fn_recv()
374 	poll_rcq(ic, ic->i_recv_cq, ic->i_recv_wc, &state);  in rds_ib_tasklet_fn_recv()
377 		rds_ib_set_ack(ic, state.ack_next, state.ack_required);  in rds_ib_tasklet_fn_recv()
378 	if (state.ack_recv_valid && state.ack_recv > ic->i_ack_recv) {  in rds_ib_tasklet_fn_recv()
380 		ic->i_ack_recv = state.ack_recv;  in rds_ib_tasklet_fn_recv()
384 		rds_ib_attempt_ack(ic);  in rds_ib_tasklet_fn_recv()
390 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_qp_event_handler()  local
392 	rdsdebug("conn %p ic %p event %u (%s)\n", conn, ic, event->event,  in rds_ib_qp_event_handler()
397 		rdma_notify(ic->i_cm_id, IB_EVENT_COMM_EST);  in rds_ib_qp_event_handler()
411 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_cq_comp_handler_send()  local
417 	tasklet_schedule(&ic->i_send_tasklet);  in rds_ib_cq_comp_handler_send()
448 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_setup_qp()  local
449 	struct ib_device *dev = ic->i_cm_id->device;  in rds_ib_setup_qp()
475 	if (ic->i_send_ring.w_nr != max_wrs)  in rds_ib_setup_qp()
476 		rds_ib_ring_resize(&ic->i_send_ring, max_wrs);  in rds_ib_setup_qp()
480 	if (ic->i_recv_ring.w_nr != max_wrs)  in rds_ib_setup_qp()
481 		rds_ib_ring_resize(&ic->i_recv_ring, max_wrs);  in rds_ib_setup_qp()
484 	ic->i_pd = rds_ibdev->pd;  in rds_ib_setup_qp()
486 	ic->i_scq_vector = ibdev_get_unused_vector(rds_ibdev);  in rds_ib_setup_qp()
487 	cq_attr.cqe = ic->i_send_ring.w_nr + fr_queue_space + 1;  in rds_ib_setup_qp()
488 	cq_attr.comp_vector = ic->i_scq_vector;  in rds_ib_setup_qp()
489 	ic->i_send_cq = ib_create_cq(dev, rds_ib_cq_comp_handler_send,  in rds_ib_setup_qp()
492 	if (IS_ERR(ic->i_send_cq)) {  in rds_ib_setup_qp()
493 		ret = PTR_ERR(ic->i_send_cq);  in rds_ib_setup_qp()
494 		ic->i_send_cq = NULL;  in rds_ib_setup_qp()
495 		ibdev_put_vector(rds_ibdev, ic->i_scq_vector);  in rds_ib_setup_qp()
500 	ic->i_rcq_vector = ibdev_get_unused_vector(rds_ibdev);  in rds_ib_setup_qp()
501 	cq_attr.cqe = ic->i_recv_ring.w_nr;  in rds_ib_setup_qp()
502 	cq_attr.comp_vector = ic->i_rcq_vector;  in rds_ib_setup_qp()
503 	ic->i_recv_cq = ib_create_cq(dev, rds_ib_cq_comp_handler_recv,  in rds_ib_setup_qp()
506 	if (IS_ERR(ic->i_recv_cq)) {  in rds_ib_setup_qp()
507 		ret = PTR_ERR(ic->i_recv_cq);  in rds_ib_setup_qp()
508 		ic->i_recv_cq = NULL;  in rds_ib_setup_qp()
509 		ibdev_put_vector(rds_ibdev, ic->i_rcq_vector);  in rds_ib_setup_qp()
514 	ret = ib_req_notify_cq(ic->i_send_cq, IB_CQ_NEXT_COMP);  in rds_ib_setup_qp()
520 	ret = ib_req_notify_cq(ic->i_recv_cq, IB_CQ_SOLICITED);  in rds_ib_setup_qp()
531 	attr.cap.max_send_wr = ic->i_send_ring.w_nr + fr_queue_space + 1;  in rds_ib_setup_qp()
532 	attr.cap.max_recv_wr = ic->i_recv_ring.w_nr + 1;  in rds_ib_setup_qp()
537 	attr.send_cq = ic->i_send_cq;  in rds_ib_setup_qp()
538 	attr.recv_cq = ic->i_recv_cq;  in rds_ib_setup_qp()
544 	ret = rdma_create_qp(ic->i_cm_id, ic->i_pd, &attr);  in rds_ib_setup_qp()
550 	ic->i_send_hdrs = ib_dma_alloc_coherent(dev,  in rds_ib_setup_qp()
551 					   ic->i_send_ring.w_nr *  in rds_ib_setup_qp()
553 					   &ic->i_send_hdrs_dma, GFP_KERNEL);  in rds_ib_setup_qp()
554 	if (!ic->i_send_hdrs) {  in rds_ib_setup_qp()
560 	ic->i_recv_hdrs = ib_dma_alloc_coherent(dev,  in rds_ib_setup_qp()
561 					   ic->i_recv_ring.w_nr *  in rds_ib_setup_qp()
563 					   &ic->i_recv_hdrs_dma, GFP_KERNEL);  in rds_ib_setup_qp()
564 	if (!ic->i_recv_hdrs) {  in rds_ib_setup_qp()
570 	ic->i_ack = ib_dma_alloc_coherent(dev, sizeof(struct rds_header),  in rds_ib_setup_qp()
571 				       &ic->i_ack_dma, GFP_KERNEL);  in rds_ib_setup_qp()
572 	if (!ic->i_ack) {  in rds_ib_setup_qp()
578 	ic->i_sends = vzalloc_node(array_size(sizeof(struct rds_ib_send_work),  in rds_ib_setup_qp()
579 					      ic->i_send_ring.w_nr),  in rds_ib_setup_qp()
581 	if (!ic->i_sends) {  in rds_ib_setup_qp()
587 	ic->i_recvs = vzalloc_node(array_size(sizeof(struct rds_ib_recv_work),  in rds_ib_setup_qp()
588 					      ic->i_recv_ring.w_nr),  in rds_ib_setup_qp()
590 	if (!ic->i_recvs) {  in rds_ib_setup_qp()
596 	rds_ib_recv_init_ack(ic);  in rds_ib_setup_qp()
598 	rdsdebug("conn %p pd %p cq %p %p\n", conn, ic->i_pd,  in rds_ib_setup_qp()
599 		 ic->i_send_cq, ic->i_recv_cq);  in rds_ib_setup_qp()
604 	vfree(ic->i_sends);  in rds_ib_setup_qp()
607 			     ic->i_ack, ic->i_ack_dma);  in rds_ib_setup_qp()
609 	ib_dma_free_coherent(dev, ic->i_recv_ring.w_nr *  in rds_ib_setup_qp()
611 					ic->i_recv_hdrs, ic->i_recv_hdrs_dma);  in rds_ib_setup_qp()
613 	ib_dma_free_coherent(dev, ic->i_send_ring.w_nr *  in rds_ib_setup_qp()
615 					ic->i_send_hdrs, ic->i_send_hdrs_dma);  in rds_ib_setup_qp()
617 	rdma_destroy_qp(ic->i_cm_id);  in rds_ib_setup_qp()
619 	ib_destroy_cq(ic->i_recv_cq);  in rds_ib_setup_qp()
620 	ic->i_recv_cq = NULL;  in rds_ib_setup_qp()
622 	ib_destroy_cq(ic->i_send_cq);  in rds_ib_setup_qp()
623 	ic->i_send_cq = NULL;  in rds_ib_setup_qp()
728 	struct rds_ib_connection *ic = NULL;  in rds_ib_cm_handle_connect()  local
824 	ic = conn->c_transport_data;  in rds_ib_cm_handle_connect()
836 	BUG_ON(ic->i_cm_id);  in rds_ib_cm_handle_connect()
838 	ic->i_cm_id = cm_id;  in rds_ib_cm_handle_connect()
871 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_cm_initiate_connect()  local
879 	ic->i_flowctl = rds_ib_sysctl_flow_control;	/* advertise flow control */  in rds_ib_cm_initiate_connect()
899 		if (ic->i_cm_id == cm_id)  in rds_ib_cm_initiate_connect()
902 	ic->i_active_side = true;  in rds_ib_cm_initiate_connect()
911 	struct rds_ib_connection *ic;  in rds_ib_conn_path_connect()  local
914 	ic = conn->c_transport_data;  in rds_ib_conn_path_connect()
924 	ic->i_cm_id = rdma_create_id(&init_net, handler, conn,  in rds_ib_conn_path_connect()
926 	if (IS_ERR(ic->i_cm_id)) {  in rds_ib_conn_path_connect()
927 		ret = PTR_ERR(ic->i_cm_id);  in rds_ib_conn_path_connect()
928 		ic->i_cm_id = NULL;  in rds_ib_conn_path_connect()
933 	rdsdebug("created cm id %p for conn %p\n", ic->i_cm_id, conn);  in rds_ib_conn_path_connect()
963 	ret = rdma_resolve_addr(ic->i_cm_id, (struct sockaddr *)&src,  in rds_ib_conn_path_connect()
967 		rdsdebug("addr resolve failed for cm id %p: %d\n", ic->i_cm_id,  in rds_ib_conn_path_connect()
969 		rdma_destroy_id(ic->i_cm_id);  in rds_ib_conn_path_connect()
970 		ic->i_cm_id = NULL;  in rds_ib_conn_path_connect()
985 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_conn_path_shutdown()  local
988 	rdsdebug("cm %p pd %p cq %p %p qp %p\n", ic->i_cm_id,  in rds_ib_conn_path_shutdown()
989 		 ic->i_pd, ic->i_send_cq, ic->i_recv_cq,  in rds_ib_conn_path_shutdown()
990 		 ic->i_cm_id ? ic->i_cm_id->qp : NULL);  in rds_ib_conn_path_shutdown()
992 	if (ic->i_cm_id) {  in rds_ib_conn_path_shutdown()
993 		struct ib_device *dev = ic->i_cm_id->device;  in rds_ib_conn_path_shutdown()
995 		rdsdebug("disconnecting cm %p\n", ic->i_cm_id);  in rds_ib_conn_path_shutdown()
996 		err = rdma_disconnect(ic->i_cm_id);  in rds_ib_conn_path_shutdown()
1002 				ic->i_cm_id, err);  in rds_ib_conn_path_shutdown()
1020 			   rds_ib_ring_empty(&ic->i_recv_ring) &&  in rds_ib_conn_path_shutdown()
1021 			   (atomic_read(&ic->i_signaled_sends) == 0) &&  in rds_ib_conn_path_shutdown()
1022 			   (atomic_read(&ic->i_fastreg_inuse_count) == 0) &&  in rds_ib_conn_path_shutdown()
1023 			   (atomic_read(&ic->i_fastreg_wrs) == RDS_IB_DEFAULT_FR_WR));  in rds_ib_conn_path_shutdown()
1024 		tasklet_kill(&ic->i_send_tasklet);  in rds_ib_conn_path_shutdown()
1025 		tasklet_kill(&ic->i_recv_tasklet);  in rds_ib_conn_path_shutdown()
1027 		atomic_set(&ic->i_cq_quiesce, 1);  in rds_ib_conn_path_shutdown()
1030 		if (ic->i_cm_id->qp)  in rds_ib_conn_path_shutdown()
1031 			rdma_destroy_qp(ic->i_cm_id);  in rds_ib_conn_path_shutdown()
1032 		if (ic->i_send_cq) {  in rds_ib_conn_path_shutdown()
1033 			if (ic->rds_ibdev)  in rds_ib_conn_path_shutdown()
1034 				ibdev_put_vector(ic->rds_ibdev, ic->i_scq_vector);  in rds_ib_conn_path_shutdown()
1035 			ib_destroy_cq(ic->i_send_cq);  in rds_ib_conn_path_shutdown()
1038 		if (ic->i_recv_cq) {  in rds_ib_conn_path_shutdown()
1039 			if (ic->rds_ibdev)  in rds_ib_conn_path_shutdown()
1040 				ibdev_put_vector(ic->rds_ibdev, ic->i_rcq_vector);  in rds_ib_conn_path_shutdown()
1041 			ib_destroy_cq(ic->i_recv_cq);  in rds_ib_conn_path_shutdown()
1045 		if (ic->i_send_hdrs)  in rds_ib_conn_path_shutdown()
1047 					   ic->i_send_ring.w_nr *  in rds_ib_conn_path_shutdown()
1049 					   ic->i_send_hdrs,  in rds_ib_conn_path_shutdown()
1050 					   ic->i_send_hdrs_dma);  in rds_ib_conn_path_shutdown()
1052 		if (ic->i_recv_hdrs)  in rds_ib_conn_path_shutdown()
1054 					   ic->i_recv_ring.w_nr *  in rds_ib_conn_path_shutdown()
1056 					   ic->i_recv_hdrs,  in rds_ib_conn_path_shutdown()
1057 					   ic->i_recv_hdrs_dma);  in rds_ib_conn_path_shutdown()
1059 		if (ic->i_ack)  in rds_ib_conn_path_shutdown()
1061 					     ic->i_ack, ic->i_ack_dma);  in rds_ib_conn_path_shutdown()
1063 		if (ic->i_sends)  in rds_ib_conn_path_shutdown()
1064 			rds_ib_send_clear_ring(ic);  in rds_ib_conn_path_shutdown()
1065 		if (ic->i_recvs)  in rds_ib_conn_path_shutdown()
1066 			rds_ib_recv_clear_ring(ic);  in rds_ib_conn_path_shutdown()
1068 		rdma_destroy_id(ic->i_cm_id);  in rds_ib_conn_path_shutdown()
1073 		if (ic->rds_ibdev)  in rds_ib_conn_path_shutdown()
1074 			rds_ib_remove_conn(ic->rds_ibdev, conn);  in rds_ib_conn_path_shutdown()
1076 		ic->i_cm_id = NULL;  in rds_ib_conn_path_shutdown()
1077 		ic->i_pd = NULL;  in rds_ib_conn_path_shutdown()
1078 		ic->i_send_cq = NULL;  in rds_ib_conn_path_shutdown()
1079 		ic->i_recv_cq = NULL;  in rds_ib_conn_path_shutdown()
1080 		ic->i_send_hdrs = NULL;  in rds_ib_conn_path_shutdown()
1081 		ic->i_recv_hdrs = NULL;  in rds_ib_conn_path_shutdown()
1082 		ic->i_ack = NULL;  in rds_ib_conn_path_shutdown()
1084 	BUG_ON(ic->rds_ibdev);  in rds_ib_conn_path_shutdown()
1087 	if (ic->i_data_op) {  in rds_ib_conn_path_shutdown()
1090 		rm = container_of(ic->i_data_op, struct rds_message, data);  in rds_ib_conn_path_shutdown()
1092 		ic->i_data_op = NULL;  in rds_ib_conn_path_shutdown()
1096 	clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags);  in rds_ib_conn_path_shutdown()
1098 	atomic64_set(&ic->i_ack_next, 0);  in rds_ib_conn_path_shutdown()
1100 	ic->i_ack_next = 0;  in rds_ib_conn_path_shutdown()
1102 	ic->i_ack_recv = 0;  in rds_ib_conn_path_shutdown()
1105 	ic->i_flowctl = 0;  in rds_ib_conn_path_shutdown()
1106 	atomic_set(&ic->i_credits, 0);  in rds_ib_conn_path_shutdown()
1109 	rds_ib_ring_init(&ic->i_send_ring, ic->i_send_ring.w_nr);  in rds_ib_conn_path_shutdown()
1110 	rds_ib_ring_init(&ic->i_recv_ring, ic->i_recv_ring.w_nr);  in rds_ib_conn_path_shutdown()
1112 	if (ic->i_ibinc) {  in rds_ib_conn_path_shutdown()
1113 		rds_inc_put(&ic->i_ibinc->ii_inc);  in rds_ib_conn_path_shutdown()
1114 		ic->i_ibinc = NULL;  in rds_ib_conn_path_shutdown()
1117 	vfree(ic->i_sends);  in rds_ib_conn_path_shutdown()
1118 	ic->i_sends = NULL;  in rds_ib_conn_path_shutdown()
1119 	vfree(ic->i_recvs);  in rds_ib_conn_path_shutdown()
1120 	ic->i_recvs = NULL;  in rds_ib_conn_path_shutdown()
1121 	ic->i_active_side = false;  in rds_ib_conn_path_shutdown()
1126 	struct rds_ib_connection *ic;  in rds_ib_conn_alloc()  local
1131 	ic = kzalloc(sizeof(struct rds_ib_connection), gfp);  in rds_ib_conn_alloc()
1132 	if (!ic)  in rds_ib_conn_alloc()
1135 	ret = rds_ib_recv_alloc_caches(ic, gfp);  in rds_ib_conn_alloc()
1137 		kfree(ic);  in rds_ib_conn_alloc()
1141 	INIT_LIST_HEAD(&ic->ib_node);  in rds_ib_conn_alloc()
1142 	tasklet_init(&ic->i_send_tasklet, rds_ib_tasklet_fn_send,  in rds_ib_conn_alloc()
1143 		     (unsigned long)ic);  in rds_ib_conn_alloc()
1144 	tasklet_init(&ic->i_recv_tasklet, rds_ib_tasklet_fn_recv,  in rds_ib_conn_alloc()
1145 		     (unsigned long)ic);  in rds_ib_conn_alloc()
1146 	mutex_init(&ic->i_recv_mutex);  in rds_ib_conn_alloc()
1148 	spin_lock_init(&ic->i_ack_lock);  in rds_ib_conn_alloc()
1150 	atomic_set(&ic->i_signaled_sends, 0);  in rds_ib_conn_alloc()
1151 	atomic_set(&ic->i_fastreg_wrs, RDS_IB_DEFAULT_FR_WR);  in rds_ib_conn_alloc()
1157 	rds_ib_ring_init(&ic->i_send_ring, 0);  in rds_ib_conn_alloc()
1158 	rds_ib_ring_init(&ic->i_recv_ring, 0);  in rds_ib_conn_alloc()
1160 	ic->conn = conn;  in rds_ib_conn_alloc()
1161 	conn->c_transport_data = ic;  in rds_ib_conn_alloc()
1164 	list_add_tail(&ic->ib_node, &ib_nodev_conns);  in rds_ib_conn_alloc()
1177 	struct rds_ib_connection *ic = arg;  in rds_ib_conn_free()  local
1180 	rdsdebug("ic %p\n", ic);  in rds_ib_conn_free()
1187 	lock_ptr = ic->rds_ibdev ? &ic->rds_ibdev->spinlock : &ib_nodev_conns_lock;  in rds_ib_conn_free()
1190 	list_del(&ic->ib_node);  in rds_ib_conn_free()
1193 	rds_ib_recv_free_caches(ic);  in rds_ib_conn_free()
1195 	kfree(ic);  in rds_ib_conn_free()