77 static int rpcrdma_sendctxs_create(struct rpcrdma_xprt *r_xprt);
78 static void rpcrdma_sendctxs_destroy(struct rpcrdma_xprt *r_xprt);
79 static void rpcrdma_sendctx_put_locked(struct rpcrdma_xprt *r_xprt,
81 static int rpcrdma_reqs_setup(struct rpcrdma_xprt *r_xprt);
82 static void rpcrdma_reqs_reset(struct rpcrdma_xprt *r_xprt);
84 static void rpcrdma_reps_unmap(struct rpcrdma_xprt *r_xprt);
85 static void rpcrdma_mrs_create(struct rpcrdma_xprt *r_xprt);
86 static void rpcrdma_mrs_destroy(struct rpcrdma_xprt *r_xprt);
99 static void rpcrdma_xprt_drain(struct rpcrdma_xprt *r_xprt)  in rpcrdma_xprt_drain()  argument
101 	struct rpcrdma_ep *ep = r_xprt->rx_ep;  in rpcrdma_xprt_drain()
150 void rpcrdma_flush_disconnect(struct rpcrdma_xprt *r_xprt, struct ib_wc *wc)  in rpcrdma_flush_disconnect()  argument
153 		rpcrdma_force_disconnect(r_xprt->rx_ep);  in rpcrdma_flush_disconnect()
167 	struct rpcrdma_xprt *r_xprt = cq->cq_context;  in rpcrdma_wc_send()  local
171 	rpcrdma_sendctx_put_locked(r_xprt, sc);  in rpcrdma_wc_send()
172 	rpcrdma_flush_disconnect(r_xprt, wc);  in rpcrdma_wc_send()
186 	struct rpcrdma_xprt *r_xprt = cq->cq_context;  in rpcrdma_wc_receive()  local
190 	--r_xprt->rx_ep->re_receive_count;  in rpcrdma_wc_receive()
207 	rpcrdma_flush_disconnect(r_xprt, wc);  in rpcrdma_wc_receive()
311 static struct rdma_cm_id *rpcrdma_create_id(struct rpcrdma_xprt *r_xprt,  in rpcrdma_create_id()  argument
315 	struct rpc_xprt *xprt = &r_xprt->rx_xprt;  in rpcrdma_create_id()
395 static int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt)  in rpcrdma_ep_create()  argument
406 	ep->re_xprt = &r_xprt->rx_xprt;  in rpcrdma_ep_create()
409 	id = rpcrdma_create_id(r_xprt, ep);  in rpcrdma_ep_create()
418 	ep->re_max_requests = r_xprt->rx_xprt.max_reqs;  in rpcrdma_ep_create()
425 	r_xprt->rx_buf.rb_max_requests = cpu_to_be32(ep->re_max_requests);  in rpcrdma_ep_create()
447 	ep->re_attr.send_cq = ib_alloc_cq_any(device, r_xprt,  in rpcrdma_ep_create()
455 	ep->re_attr.recv_cq = ib_alloc_cq_any(device, r_xprt,  in rpcrdma_ep_create()
505 	r_xprt->rx_ep = ep;  in rpcrdma_ep_create()
520 int rpcrdma_xprt_connect(struct rpcrdma_xprt *r_xprt)  in rpcrdma_xprt_connect()  argument
522 	struct rpc_xprt *xprt = &r_xprt->rx_xprt;  in rpcrdma_xprt_connect()
526 	rc = rpcrdma_ep_create(r_xprt);  in rpcrdma_xprt_connect()
529 	ep = r_xprt->rx_ep;  in rpcrdma_xprt_connect()
532 	rpcrdma_reset_cwnd(r_xprt);  in rpcrdma_xprt_connect()
538 	rpcrdma_post_recvs(r_xprt, true);  in rpcrdma_xprt_connect()
553 	rc = rpcrdma_sendctxs_create(r_xprt);  in rpcrdma_xprt_connect()
559 	rc = rpcrdma_reqs_setup(r_xprt);  in rpcrdma_xprt_connect()
564 	rpcrdma_mrs_create(r_xprt);  in rpcrdma_xprt_connect()
567 	trace_xprtrdma_connect(r_xprt, rc);  in rpcrdma_xprt_connect()
581 void rpcrdma_xprt_disconnect(struct rpcrdma_xprt *r_xprt)  in rpcrdma_xprt_disconnect()  argument
583 	struct rpcrdma_ep *ep = r_xprt->rx_ep;  in rpcrdma_xprt_disconnect()
592 	trace_xprtrdma_disconnect(r_xprt, rc);  in rpcrdma_xprt_disconnect()
594 	rpcrdma_xprt_drain(r_xprt);  in rpcrdma_xprt_disconnect()
595 	rpcrdma_reps_unmap(r_xprt);  in rpcrdma_xprt_disconnect()
596 	rpcrdma_reqs_reset(r_xprt);  in rpcrdma_xprt_disconnect()
597 	rpcrdma_mrs_destroy(r_xprt);  in rpcrdma_xprt_disconnect()
598 	rpcrdma_sendctxs_destroy(r_xprt);  in rpcrdma_xprt_disconnect()
603 	r_xprt->rx_ep = NULL;  in rpcrdma_xprt_disconnect()
623 static void rpcrdma_sendctxs_destroy(struct rpcrdma_xprt *r_xprt)  in rpcrdma_sendctxs_destroy()  argument
625 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_sendctxs_destroy()
649 static int rpcrdma_sendctxs_create(struct rpcrdma_xprt *r_xprt)  in rpcrdma_sendctxs_create()  argument
651 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_sendctxs_create()
660 	i = r_xprt->rx_ep->re_max_requests + RPCRDMA_MAX_BC_REQUESTS;  in rpcrdma_sendctxs_create()
667 		sc = rpcrdma_sendctx_create(r_xprt->rx_ep);  in rpcrdma_sendctxs_create()
702 struct rpcrdma_sendctx *rpcrdma_sendctx_get_locked(struct rpcrdma_xprt *r_xprt)  in rpcrdma_sendctx_get_locked()  argument
704 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_sendctx_get_locked()
728 	xprt_wait_for_buffer_space(&r_xprt->rx_xprt);  in rpcrdma_sendctx_get_locked()
729 	r_xprt->rx_stats.empty_sendctx_q++;  in rpcrdma_sendctx_get_locked()
743 static void rpcrdma_sendctx_put_locked(struct rpcrdma_xprt *r_xprt,  in rpcrdma_sendctx_put_locked()  argument
746 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_sendctx_put_locked()
764 	xprt_write_space(&r_xprt->rx_xprt);  in rpcrdma_sendctx_put_locked()
768 rpcrdma_mrs_create(struct rpcrdma_xprt *r_xprt)  in rpcrdma_mrs_create()  argument
770 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_mrs_create()
771 	struct rpcrdma_ep *ep = r_xprt->rx_ep;  in rpcrdma_mrs_create()
782 		rc = frwr_mr_init(r_xprt, mr);  in rpcrdma_mrs_create()
794 	r_xprt->rx_stats.mrs_allocated += count;  in rpcrdma_mrs_create()
795 	trace_xprtrdma_createmrs(r_xprt, count);  in rpcrdma_mrs_create()
803 	struct rpcrdma_xprt *r_xprt = container_of(buf, struct rpcrdma_xprt,  in rpcrdma_mr_refresh_worker()  local
806 	rpcrdma_mrs_create(r_xprt);  in rpcrdma_mr_refresh_worker()
807 	xprt_write_space(&r_xprt->rx_xprt);  in rpcrdma_mr_refresh_worker()
815 void rpcrdma_mrs_refresh(struct rpcrdma_xprt *r_xprt)  in rpcrdma_mrs_refresh()  argument
817 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_mrs_refresh()
818 	struct rpcrdma_ep *ep = r_xprt->rx_ep;  in rpcrdma_mrs_refresh()
840 struct rpcrdma_req *rpcrdma_req_create(struct rpcrdma_xprt *r_xprt, size_t size,  in rpcrdma_req_create()  argument
843 	struct rpcrdma_buffer *buffer = &r_xprt->rx_buf;  in rpcrdma_req_create()
880 int rpcrdma_req_setup(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req)  in rpcrdma_req_setup()  argument
887 		     r_xprt->rx_ep->re_max_rdma_segs * rpcrdma_readchunk_maxsz;  in rpcrdma_req_setup()
894 	if (!__rpcrdma_regbuf_dma_map(r_xprt, rb))  in rpcrdma_req_setup()
912 static int rpcrdma_reqs_setup(struct rpcrdma_xprt *r_xprt)  in rpcrdma_reqs_setup()  argument
914 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_reqs_setup()
919 		rc = rpcrdma_req_setup(r_xprt, req);  in rpcrdma_reqs_setup()
945 static void rpcrdma_reqs_reset(struct rpcrdma_xprt *r_xprt)  in rpcrdma_reqs_reset()  argument
947 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_reqs_reset()
958 struct rpcrdma_rep *rpcrdma_rep_create(struct rpcrdma_xprt *r_xprt,  in rpcrdma_rep_create()  argument
967 	rep->rr_rdmabuf = rpcrdma_regbuf_alloc(r_xprt->rx_ep->re_inline_recv,  in rpcrdma_rep_create()
972 	if (!rpcrdma_regbuf_dma_map(r_xprt, rep->rr_rdmabuf))  in rpcrdma_rep_create()
978 	rep->rr_rxprt = r_xprt;  in rpcrdma_rep_create()
984 	list_add(&rep->rr_all, &r_xprt->rx_buf.rb_all_reps);  in rpcrdma_rep_create()
1022 static void rpcrdma_reps_unmap(struct rpcrdma_xprt *r_xprt)  in rpcrdma_reps_unmap()  argument
1024 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_reps_unmap()
1047 int rpcrdma_buffer_create(struct rpcrdma_xprt *r_xprt)  in rpcrdma_buffer_create()  argument
1049 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_buffer_create()
1063 	for (i = 0; i < r_xprt->rx_xprt.max_reqs; i++) {  in rpcrdma_buffer_create()
1066 		req = rpcrdma_req_create(r_xprt, RPCRDMA_V1_DEF_INLINE_SIZE * 2,  in rpcrdma_buffer_create()
1117 static void rpcrdma_mrs_destroy(struct rpcrdma_xprt *r_xprt)  in rpcrdma_mrs_destroy()  argument
1119 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_mrs_destroy()
1170 rpcrdma_mr_get(struct rpcrdma_xprt *r_xprt)  in rpcrdma_mr_get()  argument
1172 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_mr_get()
1188 	struct rpcrdma_xprt *r_xprt = mr->mr_xprt;  in rpcrdma_mr_put()  local
1192 		ib_dma_unmap_sg(r_xprt->rx_ep->re_id->device,  in rpcrdma_mr_put()
1307 bool __rpcrdma_regbuf_dma_map(struct rpcrdma_xprt *r_xprt,  in __rpcrdma_regbuf_dma_map()  argument
1310 	struct ib_device *device = r_xprt->rx_ep->re_id->device;  in __rpcrdma_regbuf_dma_map()
1323 	rb->rg_iov.lkey = r_xprt->rx_ep->re_pd->local_dma_lkey;  in __rpcrdma_regbuf_dma_map()
1356 int rpcrdma_post_sends(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req)  in rpcrdma_post_sends()  argument
1359 	struct rpcrdma_ep *ep = r_xprt->rx_ep;  in rpcrdma_post_sends()
1371 	rc = frwr_send(r_xprt, req);  in rpcrdma_post_sends()
1383 void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, bool temp)  in rpcrdma_post_recvs()  argument
1385 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_post_recvs()
1386 	struct rpcrdma_ep *ep = r_xprt->rx_ep;  in rpcrdma_post_recvs()
1410 			rep = rpcrdma_rep_create(r_xprt, temp);  in rpcrdma_post_recvs()
1426 	trace_xprtrdma_post_recvs(r_xprt, count, rc);  in rpcrdma_post_recvs()