77 static int rpcrdma_sendctxs_create(struct rpcrdma_xprt *r_xprt);
78 static void rpcrdma_sendctxs_destroy(struct rpcrdma_xprt *r_xprt);
79 static void rpcrdma_sendctx_put_locked(struct rpcrdma_xprt *r_xprt,
81 static int rpcrdma_reqs_setup(struct rpcrdma_xprt *r_xprt);
82 static void rpcrdma_reqs_reset(struct rpcrdma_xprt *r_xprt);
84 static void rpcrdma_reps_unmap(struct rpcrdma_xprt *r_xprt);
85 static void rpcrdma_mrs_create(struct rpcrdma_xprt *r_xprt);
86 static void rpcrdma_mrs_destroy(struct rpcrdma_xprt *r_xprt);
99 static void rpcrdma_xprt_drain(struct rpcrdma_xprt *r_xprt)  in rpcrdma_xprt_drain()  argument
101 	struct rpcrdma_ep *ep = r_xprt->rx_ep;  in rpcrdma_xprt_drain()
140 void rpcrdma_flush_disconnect(struct rpcrdma_xprt *r_xprt, struct ib_wc *wc)  in rpcrdma_flush_disconnect()  argument
143 		rpcrdma_force_disconnect(r_xprt->rx_ep);  in rpcrdma_flush_disconnect()
157 	struct rpcrdma_xprt *r_xprt = cq->cq_context;  in rpcrdma_wc_send()  local
161 	rpcrdma_sendctx_put_locked(r_xprt, sc);  in rpcrdma_wc_send()
162 	rpcrdma_flush_disconnect(r_xprt, wc);  in rpcrdma_wc_send()
176 	struct rpcrdma_xprt *r_xprt = cq->cq_context;  in rpcrdma_wc_receive()  local
180 	--r_xprt->rx_ep->re_receive_count;  in rpcrdma_wc_receive()
197 	rpcrdma_flush_disconnect(r_xprt, wc);  in rpcrdma_wc_receive()
198 	rpcrdma_rep_put(&r_xprt->rx_buf, rep);  in rpcrdma_wc_receive()
301 static struct rdma_cm_id *rpcrdma_create_id(struct rpcrdma_xprt *r_xprt,  in rpcrdma_create_id()  argument
305 	struct rpc_xprt *xprt = &r_xprt->rx_xprt;  in rpcrdma_create_id()
385 static int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt)  in rpcrdma_ep_create()  argument
396 	ep->re_xprt = &r_xprt->rx_xprt;  in rpcrdma_ep_create()
399 	id = rpcrdma_create_id(r_xprt, ep);  in rpcrdma_ep_create()
409 	ep->re_max_requests = r_xprt->rx_xprt.max_reqs;  in rpcrdma_ep_create()
416 	r_xprt->rx_buf.rb_max_requests = cpu_to_be32(ep->re_max_requests);  in rpcrdma_ep_create()
436 	ep->re_attr.send_cq = ib_alloc_cq_any(device, r_xprt,  in rpcrdma_ep_create()
444 	ep->re_attr.recv_cq = ib_alloc_cq_any(device, r_xprt,  in rpcrdma_ep_create()
494 	r_xprt->rx_ep = ep;  in rpcrdma_ep_create()
509 int rpcrdma_xprt_connect(struct rpcrdma_xprt *r_xprt)  in rpcrdma_xprt_connect()  argument
511 	struct rpc_xprt *xprt = &r_xprt->rx_xprt;  in rpcrdma_xprt_connect()
515 	rc = rpcrdma_ep_create(r_xprt);  in rpcrdma_xprt_connect()
518 	ep = r_xprt->rx_ep;  in rpcrdma_xprt_connect()
521 	rpcrdma_reset_cwnd(r_xprt);  in rpcrdma_xprt_connect()
527 	rpcrdma_post_recvs(r_xprt, 1, true);  in rpcrdma_xprt_connect()
542 	rc = rpcrdma_sendctxs_create(r_xprt);  in rpcrdma_xprt_connect()
548 	rc = rpcrdma_reqs_setup(r_xprt);  in rpcrdma_xprt_connect()
553 	rpcrdma_mrs_create(r_xprt);  in rpcrdma_xprt_connect()
556 	trace_xprtrdma_connect(r_xprt, rc);  in rpcrdma_xprt_connect()
570 void rpcrdma_xprt_disconnect(struct rpcrdma_xprt *r_xprt)  in rpcrdma_xprt_disconnect()  argument
572 	struct rpcrdma_ep *ep = r_xprt->rx_ep;  in rpcrdma_xprt_disconnect()
581 	trace_xprtrdma_disconnect(r_xprt, rc);  in rpcrdma_xprt_disconnect()
583 	rpcrdma_xprt_drain(r_xprt);  in rpcrdma_xprt_disconnect()
584 	rpcrdma_reps_unmap(r_xprt);  in rpcrdma_xprt_disconnect()
585 	rpcrdma_reqs_reset(r_xprt);  in rpcrdma_xprt_disconnect()
586 	rpcrdma_mrs_destroy(r_xprt);  in rpcrdma_xprt_disconnect()
587 	rpcrdma_sendctxs_destroy(r_xprt);  in rpcrdma_xprt_disconnect()
592 	r_xprt->rx_ep = NULL;  in rpcrdma_xprt_disconnect()
612 static void rpcrdma_sendctxs_destroy(struct rpcrdma_xprt *r_xprt)  in rpcrdma_sendctxs_destroy()  argument
614 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_sendctxs_destroy()
641 static int rpcrdma_sendctxs_create(struct rpcrdma_xprt *r_xprt)  in rpcrdma_sendctxs_create()  argument
643 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_sendctxs_create()
652 	i = r_xprt->rx_ep->re_max_requests + RPCRDMA_MAX_BC_REQUESTS;  in rpcrdma_sendctxs_create()
659 		sc = rpcrdma_sendctx_create(r_xprt->rx_ep);  in rpcrdma_sendctxs_create()
694 struct rpcrdma_sendctx *rpcrdma_sendctx_get_locked(struct rpcrdma_xprt *r_xprt)  in rpcrdma_sendctx_get_locked()  argument
696 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_sendctx_get_locked()
720 	xprt_wait_for_buffer_space(&r_xprt->rx_xprt);  in rpcrdma_sendctx_get_locked()
721 	r_xprt->rx_stats.empty_sendctx_q++;  in rpcrdma_sendctx_get_locked()
735 static void rpcrdma_sendctx_put_locked(struct rpcrdma_xprt *r_xprt,  in rpcrdma_sendctx_put_locked()  argument
738 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_sendctx_put_locked()
756 	xprt_write_space(&r_xprt->rx_xprt);  in rpcrdma_sendctx_put_locked()
760 rpcrdma_mrs_create(struct rpcrdma_xprt *r_xprt)  in rpcrdma_mrs_create()  argument
762 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_mrs_create()
763 	struct rpcrdma_ep *ep = r_xprt->rx_ep;  in rpcrdma_mrs_create()
774 		rc = frwr_mr_init(r_xprt, mr);  in rpcrdma_mrs_create()
786 	r_xprt->rx_stats.mrs_allocated += count;  in rpcrdma_mrs_create()
787 	trace_xprtrdma_createmrs(r_xprt, count);  in rpcrdma_mrs_create()
795 	struct rpcrdma_xprt *r_xprt = container_of(buf, struct rpcrdma_xprt,  in rpcrdma_mr_refresh_worker()  local
798 	rpcrdma_mrs_create(r_xprt);  in rpcrdma_mr_refresh_worker()
799 	xprt_write_space(&r_xprt->rx_xprt);  in rpcrdma_mr_refresh_worker()
807 void rpcrdma_mrs_refresh(struct rpcrdma_xprt *r_xprt)  in rpcrdma_mrs_refresh()  argument
809 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_mrs_refresh()
810 	struct rpcrdma_ep *ep = r_xprt->rx_ep;  in rpcrdma_mrs_refresh()
832 struct rpcrdma_req *rpcrdma_req_create(struct rpcrdma_xprt *r_xprt, size_t size,  in rpcrdma_req_create()  argument
835 	struct rpcrdma_buffer *buffer = &r_xprt->rx_buf;  in rpcrdma_req_create()
872 int rpcrdma_req_setup(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req)  in rpcrdma_req_setup()  argument
879 		     r_xprt->rx_ep->re_max_rdma_segs * rpcrdma_readchunk_maxsz;  in rpcrdma_req_setup()
886 	if (!__rpcrdma_regbuf_dma_map(r_xprt, rb))  in rpcrdma_req_setup()
904 static int rpcrdma_reqs_setup(struct rpcrdma_xprt *r_xprt)  in rpcrdma_reqs_setup()  argument
906 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_reqs_setup()
911 		rc = rpcrdma_req_setup(r_xprt, req);  in rpcrdma_reqs_setup()
937 static void rpcrdma_reqs_reset(struct rpcrdma_xprt *r_xprt)  in rpcrdma_reqs_reset()  argument
939 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_reqs_reset()
947 struct rpcrdma_rep *rpcrdma_rep_create(struct rpcrdma_xprt *r_xprt,  in rpcrdma_rep_create()  argument
950 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_rep_create()
957 	rep->rr_rdmabuf = rpcrdma_regbuf_alloc(r_xprt->rx_ep->re_inline_recv,  in rpcrdma_rep_create()
962 	if (!rpcrdma_regbuf_dma_map(r_xprt, rep->rr_rdmabuf))  in rpcrdma_rep_create()
966 		atomic_inc_return(&r_xprt->rx_ep->re_completion_ids);  in rpcrdma_rep_create()
971 	rep->rr_rxprt = r_xprt;  in rpcrdma_rep_create()
1034 static void rpcrdma_reps_unmap(struct rpcrdma_xprt *r_xprt)  in rpcrdma_reps_unmap()  argument
1036 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_reps_unmap()
1069 int rpcrdma_buffer_create(struct rpcrdma_xprt *r_xprt)  in rpcrdma_buffer_create()  argument
1071 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_buffer_create()
1085 	for (i = 0; i < r_xprt->rx_xprt.max_reqs; i++) {  in rpcrdma_buffer_create()
1088 		req = rpcrdma_req_create(r_xprt, RPCRDMA_V1_DEF_INLINE_SIZE * 2,  in rpcrdma_buffer_create()
1139 static void rpcrdma_mrs_destroy(struct rpcrdma_xprt *r_xprt)  in rpcrdma_mrs_destroy()  argument
1141 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_mrs_destroy()
1192 rpcrdma_mr_get(struct rpcrdma_xprt *r_xprt)  in rpcrdma_mr_get()  argument
1194 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_mr_get()
1311 bool __rpcrdma_regbuf_dma_map(struct rpcrdma_xprt *r_xprt,  in __rpcrdma_regbuf_dma_map()  argument
1314 	struct ib_device *device = r_xprt->rx_ep->re_id->device;  in __rpcrdma_regbuf_dma_map()
1327 	rb->rg_iov.lkey = r_xprt->rx_ep->re_pd->local_dma_lkey;  in __rpcrdma_regbuf_dma_map()
1359 void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, int needed, bool temp)  in rpcrdma_post_recvs()  argument
1361 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_post_recvs()
1362 	struct rpcrdma_ep *ep = r_xprt->rx_ep;  in rpcrdma_post_recvs()
1388 			rep = rpcrdma_rep_create(r_xprt, temp);  in rpcrdma_post_recvs()
1405 		trace_xprtrdma_post_recvs_err(r_xprt, rc);  in rpcrdma_post_recvs()
1419 	trace_xprtrdma_post_recvs(r_xprt, count);  in rpcrdma_post_recvs()