Lines Matching refs:cep
38 static int siw_cm_upcall(struct siw_cep *cep, enum iw_cm_event_type reason,
53 struct siw_cep *cep = sk_to_cep(sk); in siw_sk_save_upcalls() local
56 cep->sk_state_change = sk->sk_state_change; in siw_sk_save_upcalls()
57 cep->sk_data_ready = sk->sk_data_ready; in siw_sk_save_upcalls()
58 cep->sk_write_space = sk->sk_write_space; in siw_sk_save_upcalls()
59 cep->sk_error_report = sk->sk_error_report; in siw_sk_save_upcalls()
63 static void siw_sk_restore_upcalls(struct sock *sk, struct siw_cep *cep) in siw_sk_restore_upcalls() argument
65 sk->sk_state_change = cep->sk_state_change; in siw_sk_restore_upcalls()
66 sk->sk_data_ready = cep->sk_data_ready; in siw_sk_restore_upcalls()
67 sk->sk_write_space = cep->sk_write_space; in siw_sk_restore_upcalls()
68 sk->sk_error_report = cep->sk_error_report; in siw_sk_restore_upcalls()
72 static void siw_qp_socket_assoc(struct siw_cep *cep, struct siw_qp *qp) in siw_qp_socket_assoc() argument
74 struct socket *s = cep->sock; in siw_qp_socket_assoc()
89 struct siw_cep *cep; in siw_socket_disassoc() local
93 cep = sk_to_cep(sk); in siw_socket_disassoc()
94 if (cep) { in siw_socket_disassoc()
95 siw_sk_restore_upcalls(sk, cep); in siw_socket_disassoc()
96 siw_cep_put(cep); in siw_socket_disassoc()
108 struct siw_cep *cep; in siw_rtr_data_ready() local
114 cep = sk_to_cep(sk); in siw_rtr_data_ready()
115 if (!cep) { in siw_rtr_data_ready()
133 siw_cm_upcall(cep, IW_CM_EVENT_ESTABLISHED, 0); in siw_rtr_data_ready()
137 siw_qp_socket_assoc(cep, qp); in siw_rtr_data_ready()
140 static void siw_sk_assign_rtr_upcalls(struct siw_cep *cep) in siw_sk_assign_rtr_upcalls() argument
142 struct sock *sk = cep->sock->sk; in siw_sk_assign_rtr_upcalls()
150 static void siw_cep_socket_assoc(struct siw_cep *cep, struct socket *s) in siw_cep_socket_assoc() argument
152 cep->sock = s; in siw_cep_socket_assoc()
153 siw_cep_get(cep); in siw_cep_socket_assoc()
154 s->sk->sk_user_data = cep; in siw_cep_socket_assoc()
162 struct siw_cep *cep = kzalloc(sizeof(*cep), GFP_KERNEL); in siw_cep_alloc() local
165 if (!cep) in siw_cep_alloc()
168 INIT_LIST_HEAD(&cep->listenq); in siw_cep_alloc()
169 INIT_LIST_HEAD(&cep->devq); in siw_cep_alloc()
170 INIT_LIST_HEAD(&cep->work_freelist); in siw_cep_alloc()
172 kref_init(&cep->ref); in siw_cep_alloc()
173 cep->state = SIW_EPSTATE_IDLE; in siw_cep_alloc()
174 init_waitqueue_head(&cep->waitq); in siw_cep_alloc()
175 spin_lock_init(&cep->lock); in siw_cep_alloc()
176 cep->sdev = sdev; in siw_cep_alloc()
177 cep->enhanced_rdma_conn_est = false; in siw_cep_alloc()
180 list_add_tail(&cep->devq, &sdev->cep_list); in siw_cep_alloc()
183 siw_dbg_cep(cep, "new endpoint\n"); in siw_cep_alloc()
184 return cep; in siw_cep_alloc()
187 static void siw_cm_free_work(struct siw_cep *cep) in siw_cm_free_work() argument
192 list_for_each_safe(w, tmp, &cep->work_freelist) { in siw_cm_free_work()
199 static void siw_cancel_mpatimer(struct siw_cep *cep) in siw_cancel_mpatimer() argument
201 spin_lock_bh(&cep->lock); in siw_cancel_mpatimer()
202 if (cep->mpa_timer) { in siw_cancel_mpatimer()
203 if (cancel_delayed_work(&cep->mpa_timer->work)) { in siw_cancel_mpatimer()
204 siw_cep_put(cep); in siw_cancel_mpatimer()
205 kfree(cep->mpa_timer); /* not needed again */ in siw_cancel_mpatimer()
207 cep->mpa_timer = NULL; in siw_cancel_mpatimer()
209 spin_unlock_bh(&cep->lock); in siw_cancel_mpatimer()
215 spin_lock_bh(&work->cep->lock); in siw_put_work()
216 list_add(&work->list, &work->cep->work_freelist); in siw_put_work()
217 spin_unlock_bh(&work->cep->lock); in siw_put_work()
220 static void siw_cep_set_inuse(struct siw_cep *cep) in siw_cep_set_inuse() argument
224 spin_lock_irqsave(&cep->lock, flags); in siw_cep_set_inuse()
226 if (cep->in_use) { in siw_cep_set_inuse()
227 spin_unlock_irqrestore(&cep->lock, flags); in siw_cep_set_inuse()
228 wait_event_interruptible(cep->waitq, !cep->in_use); in siw_cep_set_inuse()
233 cep->in_use = 1; in siw_cep_set_inuse()
234 spin_unlock_irqrestore(&cep->lock, flags); in siw_cep_set_inuse()
238 static void siw_cep_set_free(struct siw_cep *cep) in siw_cep_set_free() argument
242 spin_lock_irqsave(&cep->lock, flags); in siw_cep_set_free()
243 cep->in_use = 0; in siw_cep_set_free()
244 spin_unlock_irqrestore(&cep->lock, flags); in siw_cep_set_free()
246 wake_up(&cep->waitq); in siw_cep_set_free()
251 struct siw_cep *cep = container_of(ref, struct siw_cep, ref); in __siw_cep_dealloc() local
252 struct siw_device *sdev = cep->sdev; in __siw_cep_dealloc()
255 WARN_ON(cep->listen_cep); in __siw_cep_dealloc()
258 kfree(cep->mpa.pdata); in __siw_cep_dealloc()
259 spin_lock_bh(&cep->lock); in __siw_cep_dealloc()
260 if (!list_empty(&cep->work_freelist)) in __siw_cep_dealloc()
261 siw_cm_free_work(cep); in __siw_cep_dealloc()
262 spin_unlock_bh(&cep->lock); in __siw_cep_dealloc()
265 list_del(&cep->devq); in __siw_cep_dealloc()
268 siw_dbg_cep(cep, "free endpoint\n"); in __siw_cep_dealloc()
269 kfree(cep); in __siw_cep_dealloc()
272 static struct siw_cm_work *siw_get_work(struct siw_cep *cep) in siw_get_work() argument
276 spin_lock_bh(&cep->lock); in siw_get_work()
277 if (!list_empty(&cep->work_freelist)) { in siw_get_work()
278 work = list_entry(cep->work_freelist.next, struct siw_cm_work, in siw_get_work()
282 spin_unlock_bh(&cep->lock); in siw_get_work()
286 static int siw_cm_alloc_work(struct siw_cep *cep, int num) in siw_cm_alloc_work() argument
293 if (!(list_empty(&cep->work_freelist))) in siw_cm_alloc_work()
294 siw_cm_free_work(cep); in siw_cm_alloc_work()
297 work->cep = cep; in siw_cm_alloc_work()
299 list_add(&work->list, &cep->work_freelist); in siw_cm_alloc_work()
309 static int siw_cm_upcall(struct siw_cep *cep, enum iw_cm_event_type reason, in siw_cm_upcall() argument
320 event.provider_data = cep; in siw_cm_upcall()
321 id = cep->listen_cep->cm_id; in siw_cm_upcall()
323 id = cep->cm_id; in siw_cm_upcall()
329 event.ird = cep->ird; in siw_cm_upcall()
330 event.ord = cep->ord; in siw_cm_upcall()
332 event.ird = cep->ord; in siw_cm_upcall()
333 event.ord = cep->ird; in siw_cm_upcall()
338 u16 pd_len = be16_to_cpu(cep->mpa.hdr.params.pd_len); in siw_cm_upcall()
345 event.private_data = cep->mpa.pdata; in siw_cm_upcall()
348 if (cep->enhanced_rdma_conn_est) { in siw_cm_upcall()
355 getname_local(cep->sock, &event.local_addr); in siw_cm_upcall()
356 getname_peer(cep->sock, &event.remote_addr); in siw_cm_upcall()
358 siw_dbg_cep(cep, "[QP %u]: reason=%d, status=%d\n", in siw_cm_upcall()
359 cep->qp ? qp_id(cep->qp) : UINT_MAX, reason, status); in siw_cm_upcall()
374 struct siw_cep *cep = qp->cep; in siw_qp_cm_drop() local
379 if (!qp->cep) in siw_qp_cm_drop()
383 siw_cm_queue_work(cep, SIW_CM_WORK_CLOSE_LLP); in siw_qp_cm_drop()
385 siw_cep_set_inuse(cep); in siw_qp_cm_drop()
387 if (cep->state == SIW_EPSTATE_CLOSED) { in siw_qp_cm_drop()
388 siw_dbg_cep(cep, "already closed\n"); in siw_qp_cm_drop()
391 siw_dbg_cep(cep, "immediate close, state %d\n", cep->state); in siw_qp_cm_drop()
396 if (cep->cm_id) { in siw_qp_cm_drop()
397 switch (cep->state) { in siw_qp_cm_drop()
399 siw_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, in siw_qp_cm_drop()
404 siw_cm_upcall(cep, IW_CM_EVENT_CLOSE, 0); in siw_qp_cm_drop()
416 cep->cm_id->rem_ref(cep->cm_id); in siw_qp_cm_drop()
417 cep->cm_id = NULL; in siw_qp_cm_drop()
418 siw_cep_put(cep); in siw_qp_cm_drop()
420 cep->state = SIW_EPSTATE_CLOSED; in siw_qp_cm_drop()
422 if (cep->sock) { in siw_qp_cm_drop()
423 siw_socket_disassoc(cep->sock); in siw_qp_cm_drop()
427 sock_release(cep->sock); in siw_qp_cm_drop()
428 cep->sock = NULL; in siw_qp_cm_drop()
430 if (cep->qp) { in siw_qp_cm_drop()
431 cep->qp = NULL; in siw_qp_cm_drop()
435 siw_cep_set_free(cep); in siw_qp_cm_drop()
439 void siw_cep_put(struct siw_cep *cep) in siw_cep_put() argument
441 WARN_ON(kref_read(&cep->ref) < 1); in siw_cep_put()
442 kref_put(&cep->ref, __siw_cep_dealloc); in siw_cep_put()
445 void siw_cep_get(struct siw_cep *cep) in siw_cep_get() argument
447 kref_get(&cep->ref); in siw_cep_get()
453 static int siw_send_mpareqrep(struct siw_cep *cep, const void *pdata, u8 pd_len) in siw_send_mpareqrep() argument
455 struct socket *s = cep->sock; in siw_send_mpareqrep()
456 struct mpa_rr *rr = &cep->mpa.hdr; in siw_send_mpareqrep()
469 if (cep->enhanced_rdma_conn_est) { in siw_send_mpareqrep()
471 iov[iovec_num].iov_base = &cep->mpa.v2_ctrl; in siw_send_mpareqrep()
472 iov[iovec_num].iov_len = sizeof(cep->mpa.v2_ctrl); in siw_send_mpareqrep()
473 mpa_len += sizeof(cep->mpa.v2_ctrl); in siw_send_mpareqrep()
481 if (cep->enhanced_rdma_conn_est) in siw_send_mpareqrep()
482 pd_len += sizeof(cep->mpa.v2_ctrl); in siw_send_mpareqrep()
500 static int siw_recv_mpa_rr(struct siw_cep *cep) in siw_recv_mpa_rr() argument
502 struct mpa_rr *hdr = &cep->mpa.hdr; in siw_recv_mpa_rr()
503 struct socket *s = cep->sock; in siw_recv_mpa_rr()
507 if (cep->mpa.bytes_rcvd < sizeof(struct mpa_rr)) { in siw_recv_mpa_rr()
508 rcvd = ksock_recv(s, (char *)hdr + cep->mpa.bytes_rcvd, in siw_recv_mpa_rr()
509 sizeof(struct mpa_rr) - cep->mpa.bytes_rcvd, in siw_recv_mpa_rr()
514 cep->mpa.bytes_rcvd += rcvd; in siw_recv_mpa_rr()
516 if (cep->mpa.bytes_rcvd < sizeof(struct mpa_rr)) in siw_recv_mpa_rr()
529 to_rcv = pd_len - (cep->mpa.bytes_rcvd - sizeof(struct mpa_rr)); in siw_recv_mpa_rr()
544 siw_dbg_cep(cep, "peer EOF\n"); in siw_recv_mpa_rr()
548 siw_dbg_cep(cep, "error: %d\n", rcvd); in siw_recv_mpa_rr()
551 siw_dbg_cep(cep, "peer sent extra data: %d\n", rcvd); in siw_recv_mpa_rr()
560 if (!cep->mpa.pdata) { in siw_recv_mpa_rr()
561 cep->mpa.pdata = kmalloc(pd_len + 4, GFP_KERNEL); in siw_recv_mpa_rr()
562 if (!cep->mpa.pdata) in siw_recv_mpa_rr()
566 s, cep->mpa.pdata + cep->mpa.bytes_rcvd - sizeof(struct mpa_rr), in siw_recv_mpa_rr()
575 cep->mpa.bytes_rcvd += rcvd; in siw_recv_mpa_rr()
578 siw_dbg_cep(cep, "%d bytes private data received\n", pd_len); in siw_recv_mpa_rr()
590 static int siw_proc_mpareq(struct siw_cep *cep) in siw_proc_mpareq() argument
596 rv = siw_recv_mpa_rr(cep); in siw_proc_mpareq()
600 req = &cep->mpa.hdr; in siw_proc_mpareq()
625 cep->enhanced_rdma_conn_est = true; in siw_proc_mpareq()
646 if (cep->enhanced_rdma_conn_est) { in siw_proc_mpareq()
647 struct mpa_v2_data *v2 = (struct mpa_v2_data *)cep->mpa.pdata; in siw_proc_mpareq()
654 cep->ord = ntohs(v2->ird) & MPA_IRD_ORD_MASK; in siw_proc_mpareq()
655 cep->ord = min(cep->ord, SIW_MAX_ORD_QP); in siw_proc_mpareq()
656 cep->ird = ntohs(v2->ord) & MPA_IRD_ORD_MASK; in siw_proc_mpareq()
657 cep->ird = min(cep->ird, SIW_MAX_IRD_QP); in siw_proc_mpareq()
660 cep->mpa.v2_ctrl.ird = htons(cep->ird); in siw_proc_mpareq()
661 cep->mpa.v2_ctrl.ord = htons(cep->ord); in siw_proc_mpareq()
672 cep->mpa.v2_ctrl.ird |= MPA_V2_PEER_TO_PEER; in siw_proc_mpareq()
675 cep->mpa.v2_ctrl.ord |= MPA_V2_RDMA_WRITE_RTR; in siw_proc_mpareq()
677 cep->mpa.v2_ctrl.ord |= MPA_V2_RDMA_READ_RTR; in siw_proc_mpareq()
679 cep->mpa.v2_ctrl.ord |= MPA_V2_RDMA_WRITE_RTR; in siw_proc_mpareq()
683 cep->state = SIW_EPSTATE_RECVD_MPAREQ; in siw_proc_mpareq()
686 siw_cep_get(cep); in siw_proc_mpareq()
687 rv = siw_cm_upcall(cep, IW_CM_EVENT_CONNECT_REQUEST, 0); in siw_proc_mpareq()
689 siw_cep_put(cep); in siw_proc_mpareq()
694 siw_dbg_cep(cep, "reject: crc %d:%d:%d, m %d:%d\n", in siw_proc_mpareq()
706 kfree(cep->mpa.pdata); in siw_proc_mpareq()
708 cep->mpa.pdata = NULL; in siw_proc_mpareq()
710 siw_send_mpareqrep(cep, NULL, 0); in siw_proc_mpareq()
715 static int siw_proc_mpareply(struct siw_cep *cep) in siw_proc_mpareply() argument
719 struct siw_qp *qp = cep->qp; in siw_proc_mpareply()
727 rv = siw_recv_mpa_rr(cep); in siw_proc_mpareply()
731 siw_cancel_mpatimer(cep); in siw_proc_mpareply()
733 rep = &cep->mpa.hdr; in siw_proc_mpareply()
748 siw_dbg_cep(cep, "got mpa reject\n"); in siw_proc_mpareply()
749 siw_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, -ECONNRESET); in siw_proc_mpareply()
754 siw_dbg_cep(cep, "peer allows GSO on TX\n"); in siw_proc_mpareply()
761 siw_dbg_cep(cep, "reply unsupp: crc %d:%d:%d, m %d:%d\n", in siw_proc_mpareply()
766 siw_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, -ECONNREFUSED); in siw_proc_mpareply()
770 if (cep->enhanced_rdma_conn_est) { in siw_proc_mpareply()
779 siw_dbg_cep(cep, "mpa reply error: vers %d, enhcd %d\n", in siw_proc_mpareply()
785 siw_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, in siw_proc_mpareply()
789 v2 = (struct mpa_v2_data *)cep->mpa.pdata; in siw_proc_mpareply()
793 if (cep->ird < rep_ord && in siw_proc_mpareply()
795 rep_ord > cep->sdev->attrs.max_ird)) { in siw_proc_mpareply()
796 siw_dbg_cep(cep, "ird %d, rep_ord %d, max_ord %d\n", in siw_proc_mpareply()
797 cep->ird, rep_ord, in siw_proc_mpareply()
798 cep->sdev->attrs.max_ord); in siw_proc_mpareply()
801 if (cep->ord > rep_ird && relaxed_ird_negotiation == false) { in siw_proc_mpareply()
802 siw_dbg_cep(cep, "ord %d, rep_ird %d\n", cep->ord, in siw_proc_mpareply()
810 cep->ird = rep_ord; in siw_proc_mpareply()
811 cep->ord = rep_ird; in siw_proc_mpareply()
825 if (cep->mpa.v2_ctrl_req.ird & MPA_V2_PEER_TO_PEER) in siw_proc_mpareply()
827 cep->mpa.v2_ctrl_req.ord & in siw_proc_mpareply()
839 siw_dbg_cep(cep, in siw_proc_mpareply()
862 qp_attrs.irq_size = cep->ird; in siw_proc_mpareply()
863 qp_attrs.orq_size = cep->ord; in siw_proc_mpareply()
864 qp_attrs.sk = cep->sock; in siw_proc_mpareply()
879 siw_qp_socket_assoc(cep, qp); in siw_proc_mpareply()
890 rv = siw_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, 0); in siw_proc_mpareply()
892 cep->state = SIW_EPSTATE_RDMA_MODE; in siw_proc_mpareply()
899 siw_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, -EINVAL); in siw_proc_mpareply()
908 static void siw_accept_newconn(struct siw_cep *cep) in siw_accept_newconn() argument
910 struct socket *s = cep->sock; in siw_accept_newconn()
915 if (cep->state != SIW_EPSTATE_LISTENING) in siw_accept_newconn()
918 new_cep = siw_cep_alloc(cep->sdev); in siw_accept_newconn()
934 new_cep->sk_state_change = cep->sk_state_change; in siw_accept_newconn()
935 new_cep->sk_data_ready = cep->sk_data_ready; in siw_accept_newconn()
936 new_cep->sk_write_space = cep->sk_write_space; in siw_accept_newconn()
937 new_cep->sk_error_report = cep->sk_error_report; in siw_accept_newconn()
944 siw_dbg_cep(cep, "kernel_accept() error: %d\n", rv); in siw_accept_newconn()
961 new_cep->listen_cep = cep; in siw_accept_newconn()
962 siw_cep_get(cep); in siw_accept_newconn()
968 siw_dbg_cep(cep, "immediate mpa request\n"); in siw_accept_newconn()
973 siw_cep_put(cep); in siw_accept_newconn()
993 siw_dbg_cep(cep, "error %d\n", rv); in siw_accept_newconn()
999 struct siw_cep *cep; in siw_cm_work_handler() local
1003 cep = work->cep; in siw_cm_work_handler()
1005 siw_dbg_cep(cep, "[QP %u]: work type: %d, state %d\n", in siw_cm_work_handler()
1006 cep->qp ? qp_id(cep->qp) : UINT_MAX, in siw_cm_work_handler()
1007 work->type, cep->state); in siw_cm_work_handler()
1009 siw_cep_set_inuse(cep); in siw_cm_work_handler()
1013 siw_accept_newconn(cep); in siw_cm_work_handler()
1017 if (cep->state == SIW_EPSTATE_AWAIT_MPAREQ) { in siw_cm_work_handler()
1018 if (cep->listen_cep) { in siw_cm_work_handler()
1019 siw_cep_set_inuse(cep->listen_cep); in siw_cm_work_handler()
1021 if (cep->listen_cep->state == in siw_cm_work_handler()
1023 rv = siw_proc_mpareq(cep); in siw_cm_work_handler()
1027 siw_cep_set_free(cep->listen_cep); in siw_cm_work_handler()
1030 siw_cep_put(cep->listen_cep); in siw_cm_work_handler()
1031 cep->listen_cep = NULL; in siw_cm_work_handler()
1033 siw_cep_put(cep); in siw_cm_work_handler()
1036 } else if (cep->state == SIW_EPSTATE_AWAIT_MPAREP) { in siw_cm_work_handler()
1037 rv = siw_proc_mpareply(cep); in siw_cm_work_handler()
1044 if (cep->state == SIW_EPSTATE_RDMA_MODE) { in siw_cm_work_handler()
1045 cep->sock->sk->sk_data_ready(cep->sock->sk); in siw_cm_work_handler()
1046 siw_dbg_cep(cep, "already in RDMA mode"); in siw_cm_work_handler()
1048 siw_dbg_cep(cep, "out of state: %d\n", in siw_cm_work_handler()
1049 cep->state); in siw_cm_work_handler()
1060 if (cep->qp && cep->qp->term_info.valid) in siw_cm_work_handler()
1061 siw_send_terminate(cep->qp); in siw_cm_work_handler()
1063 if (cep->cm_id) in siw_cm_work_handler()
1064 siw_cm_upcall(cep, IW_CM_EVENT_CLOSE, 0); in siw_cm_work_handler()
1070 if (cep->cm_id) { in siw_cm_work_handler()
1071 if (cep->state == SIW_EPSTATE_AWAIT_MPAREP) { in siw_cm_work_handler()
1075 siw_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, in siw_cm_work_handler()
1077 } else if (cep->state == SIW_EPSTATE_RDMA_MODE) { in siw_cm_work_handler()
1082 siw_cm_upcall(cep, IW_CM_EVENT_DISCONNECT, 0); in siw_cm_work_handler()
1083 siw_cm_upcall(cep, IW_CM_EVENT_CLOSE, 0); in siw_cm_work_handler()
1090 if (cep->state == SIW_EPSTATE_RECVD_MPAREQ) { in siw_cm_work_handler()
1094 siw_dbg_cep(cep, in siw_cm_work_handler()
1096 } else if (cep->state == SIW_EPSTATE_AWAIT_MPAREQ) { in siw_cm_work_handler()
1100 siw_dbg_cep(cep, "no mpareq: drop listener\n"); in siw_cm_work_handler()
1101 siw_cep_put(cep->listen_cep); in siw_cm_work_handler()
1102 cep->listen_cep = NULL; in siw_cm_work_handler()
1109 cep->mpa_timer = NULL; in siw_cm_work_handler()
1111 if (cep->state == SIW_EPSTATE_AWAIT_MPAREP) { in siw_cm_work_handler()
1117 cep->mpa.hdr.params.pd_len = 0; in siw_cm_work_handler()
1119 if (cep->cm_id) in siw_cm_work_handler()
1120 siw_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, in siw_cm_work_handler()
1124 } else if (cep->state == SIW_EPSTATE_AWAIT_MPAREQ) { in siw_cm_work_handler()
1128 if (cep->listen_cep) { in siw_cm_work_handler()
1129 siw_cep_put(cep->listen_cep); in siw_cm_work_handler()
1130 cep->listen_cep = NULL; in siw_cm_work_handler()
1140 siw_dbg_cep(cep, in siw_cm_work_handler()
1142 cep->mpa_timer ? "y" : "n", in siw_cm_work_handler()
1143 cep->qp ? qp_id(cep->qp) : UINT_MAX); in siw_cm_work_handler()
1145 siw_cancel_mpatimer(cep); in siw_cm_work_handler()
1147 cep->state = SIW_EPSTATE_CLOSED; in siw_cm_work_handler()
1149 if (cep->qp) { in siw_cm_work_handler()
1150 struct siw_qp *qp = cep->qp; in siw_cm_work_handler()
1156 siw_cep_set_free(cep); in siw_cm_work_handler()
1161 siw_cep_set_inuse(cep); in siw_cm_work_handler()
1162 cep->qp = NULL; in siw_cm_work_handler()
1165 if (cep->sock) { in siw_cm_work_handler()
1166 siw_socket_disassoc(cep->sock); in siw_cm_work_handler()
1167 sock_release(cep->sock); in siw_cm_work_handler()
1168 cep->sock = NULL; in siw_cm_work_handler()
1170 if (cep->cm_id) { in siw_cm_work_handler()
1171 cep->cm_id->rem_ref(cep->cm_id); in siw_cm_work_handler()
1172 cep->cm_id = NULL; in siw_cm_work_handler()
1173 siw_cep_put(cep); in siw_cm_work_handler()
1176 siw_cep_set_free(cep); in siw_cm_work_handler()
1178 siw_cep_put(cep); in siw_cm_work_handler()
1183 int siw_cm_queue_work(struct siw_cep *cep, enum siw_work_type type) in siw_cm_queue_work() argument
1185 struct siw_cm_work *work = siw_get_work(cep); in siw_cm_queue_work()
1189 siw_dbg_cep(cep, "failed with no work available\n"); in siw_cm_queue_work()
1193 work->cep = cep; in siw_cm_queue_work()
1195 siw_cep_get(cep); in siw_cm_queue_work()
1200 cep->mpa_timer = work; in siw_cm_queue_work()
1202 if (cep->state == SIW_EPSTATE_AWAIT_MPAREP) in siw_cm_queue_work()
1207 siw_dbg_cep(cep, "[QP %u]: work type: %d, timeout %lu\n", in siw_cm_queue_work()
1208 cep->qp ? qp_id(cep->qp) : -1, type, delay); in siw_cm_queue_work()
1217 struct siw_cep *cep; in siw_cm_llp_data_ready() local
1221 cep = sk_to_cep(sk); in siw_cm_llp_data_ready()
1222 if (!cep) in siw_cm_llp_data_ready()
1225 siw_dbg_cep(cep, "state: %d\n", cep->state); in siw_cm_llp_data_ready()
1227 switch (cep->state) { in siw_cm_llp_data_ready()
1234 siw_cm_queue_work(cep, SIW_CM_WORK_READ_MPAHDR); in siw_cm_llp_data_ready()
1238 siw_dbg_cep(cep, "unexpected data, state %d\n", cep->state); in siw_cm_llp_data_ready()
1247 struct siw_cep *cep = sk_to_cep(sk); in siw_cm_llp_write_space() local
1249 if (cep) in siw_cm_llp_write_space()
1250 siw_dbg_cep(cep, "state: %d\n", cep->state); in siw_cm_llp_write_space()
1255 struct siw_cep *cep = sk_to_cep(sk); in siw_cm_llp_error_report() local
1257 if (cep) { in siw_cm_llp_error_report()
1258 siw_dbg_cep(cep, "error %d, socket state: %d, cep state: %d\n", in siw_cm_llp_error_report()
1259 sk->sk_err, sk->sk_state, cep->state); in siw_cm_llp_error_report()
1260 cep->sk_error_report(sk); in siw_cm_llp_error_report()
1266 struct siw_cep *cep; in siw_cm_llp_state_change() local
1271 cep = sk_to_cep(sk); in siw_cm_llp_state_change()
1272 if (!cep) { in siw_cm_llp_state_change()
1277 orig_state_change = cep->sk_state_change; in siw_cm_llp_state_change()
1279 siw_dbg_cep(cep, "state: %d\n", cep->state); in siw_cm_llp_state_change()
1287 siw_cm_queue_work(cep, SIW_CM_WORK_ACCEPT); in siw_cm_llp_state_change()
1292 if (cep->qp) in siw_cm_llp_state_change()
1293 cep->qp->tx_ctx.tx_suspend = 1; in siw_cm_llp_state_change()
1294 siw_cm_queue_work(cep, SIW_CM_WORK_PEER_CLOSE); in siw_cm_llp_state_change()
1298 siw_dbg_cep(cep, "unexpected socket state %d\n", sk->sk_state); in siw_cm_llp_state_change()
1335 struct siw_cep *cep = NULL; in siw_connect() local
1389 cep = siw_cep_alloc(sdev); in siw_connect()
1390 if (!cep) { in siw_connect()
1394 siw_cep_set_inuse(cep); in siw_connect()
1397 siw_cep_get(cep); in siw_connect()
1398 qp->cep = cep; in siw_connect()
1401 cep->qp = qp; in siw_connect()
1404 cep->cm_id = id; in siw_connect()
1411 rv = siw_cm_alloc_work(cep, 4); in siw_connect()
1416 cep->ird = params->ird; in siw_connect()
1417 cep->ord = params->ord; in siw_connect()
1419 if (p2p_mode && cep->ord == 0) in siw_connect()
1420 cep->ord = 1; in siw_connect()
1422 cep->state = SIW_EPSTATE_CONNECTING; in siw_connect()
1427 siw_cep_socket_assoc(cep, s); in siw_connect()
1429 cep->state = SIW_EPSTATE_AWAIT_MPAREP; in siw_connect()
1435 cep->mpa.hdr.params.bits = 0; in siw_connect()
1442 __mpa_rr_set_revision(&cep->mpa.hdr.params.bits, version); in siw_connect()
1445 cep->mpa.hdr.params.bits |= MPA_RR_FLAG_GSO_EXP; in siw_connect()
1448 cep->mpa.hdr.params.bits |= MPA_RR_FLAG_CRC; in siw_connect()
1457 cep->enhanced_rdma_conn_est = true; in siw_connect()
1458 cep->mpa.hdr.params.bits |= MPA_RR_FLAG_ENHANCED; in siw_connect()
1460 cep->mpa.v2_ctrl.ird = htons(cep->ird); in siw_connect()
1461 cep->mpa.v2_ctrl.ord = htons(cep->ord); in siw_connect()
1464 cep->mpa.v2_ctrl.ird |= MPA_V2_PEER_TO_PEER; in siw_connect()
1465 cep->mpa.v2_ctrl.ord |= rtr_type; in siw_connect()
1468 cep->mpa.v2_ctrl_req.ird = cep->mpa.v2_ctrl.ird; in siw_connect()
1469 cep->mpa.v2_ctrl_req.ord = cep->mpa.v2_ctrl.ord; in siw_connect()
1471 memcpy(cep->mpa.hdr.key, MPA_KEY_REQ, 16); in siw_connect()
1473 rv = siw_send_mpareqrep(cep, params->private_data, pd_len); in siw_connect()
1477 cep->mpa.hdr.params.pd_len = 0; in siw_connect()
1480 rv = siw_cm_queue_work(cep, SIW_CM_WORK_MPATIMEOUT); in siw_connect()
1482 siw_dbg_cep(cep, "[QP %u]: exit\n", qp_id(qp)); in siw_connect()
1483 siw_cep_set_free(cep); in siw_connect()
1490 if (cep) { in siw_connect()
1493 cep->sock = NULL; in siw_connect()
1495 cep->qp = NULL; in siw_connect()
1497 cep->cm_id = NULL; in siw_connect()
1499 siw_cep_put(cep); in siw_connect()
1501 qp->cep = NULL; in siw_connect()
1502 siw_cep_put(cep); in siw_connect()
1504 cep->state = SIW_EPSTATE_CLOSED; in siw_connect()
1506 siw_cep_set_free(cep); in siw_connect()
1508 siw_cep_put(cep); in siw_connect()
1536 struct siw_cep *cep = (struct siw_cep *)id->provider_data; in siw_accept() local
1542 siw_cep_set_inuse(cep); in siw_accept()
1543 siw_cep_put(cep); in siw_accept()
1546 if (cep->mpa.hdr.params.pd_len) { in siw_accept()
1547 cep->mpa.hdr.params.pd_len = 0; in siw_accept()
1548 kfree(cep->mpa.pdata); in siw_accept()
1549 cep->mpa.pdata = NULL; in siw_accept()
1551 siw_cancel_mpatimer(cep); in siw_accept()
1553 if (cep->state != SIW_EPSTATE_RECVD_MPAREQ) { in siw_accept()
1554 siw_dbg_cep(cep, "out of state\n"); in siw_accept()
1556 siw_cep_set_free(cep); in siw_accept()
1557 siw_cep_put(cep); in siw_accept()
1564 siw_cep_set_free(cep); in siw_accept()
1565 siw_cep_put(cep); in siw_accept()
1575 siw_dbg_cep(cep, "[QP %d]\n", params->qpn); in siw_accept()
1577 if (try_gso && cep->mpa.hdr.params.bits & MPA_RR_FLAG_GSO_EXP) { in siw_accept()
1578 siw_dbg_cep(cep, "peer allows GSO on TX\n"); in siw_accept()
1584 cep, in siw_accept()
1592 if (cep->enhanced_rdma_conn_est) in siw_accept()
1597 cep, in siw_accept()
1604 if (cep->enhanced_rdma_conn_est) { in siw_accept()
1605 if (params->ord > cep->ord) { in siw_accept()
1607 params->ord = cep->ord; in siw_accept()
1609 cep->ird = params->ird; in siw_accept()
1610 cep->ord = params->ord; in siw_accept()
1616 if (params->ird < cep->ird) { in siw_accept()
1618 cep->ird <= sdev->attrs.max_ird) in siw_accept()
1619 params->ird = cep->ird; in siw_accept()
1626 if (cep->mpa.v2_ctrl.ord & in siw_accept()
1632 cep->mpa.v2_ctrl.ord = in siw_accept()
1634 (cep->mpa.v2_ctrl.ord & ~MPA_V2_MASK_IRD_ORD); in siw_accept()
1635 cep->mpa.v2_ctrl.ird = in siw_accept()
1637 (cep->mpa.v2_ctrl.ird & ~MPA_V2_MASK_IRD_ORD); in siw_accept()
1639 cep->ird = params->ird; in siw_accept()
1640 cep->ord = params->ord; in siw_accept()
1642 cep->cm_id = id; in siw_accept()
1646 qp_attrs.orq_size = cep->ord; in siw_accept()
1647 qp_attrs.irq_size = cep->ird; in siw_accept()
1648 qp_attrs.sk = cep->sock; in siw_accept()
1649 if (cep->mpa.hdr.params.bits & MPA_RR_FLAG_CRC) in siw_accept()
1653 siw_dbg_cep(cep, "[QP%u]: moving to rts\n", qp_id(qp)); in siw_accept()
1656 siw_cep_get(cep); in siw_accept()
1657 qp->cep = cep; in siw_accept()
1660 cep->qp = qp; in siw_accept()
1662 cep->state = SIW_EPSTATE_RDMA_MODE; in siw_accept()
1674 siw_dbg_cep(cep, "[QP %u]: send mpa reply, %d byte pdata\n", in siw_accept()
1677 rv = siw_send_mpareqrep(cep, params->private_data, in siw_accept()
1683 siw_sk_assign_rtr_upcalls(cep); in siw_accept()
1685 siw_qp_socket_assoc(cep, qp); in siw_accept()
1686 rv = siw_cm_upcall(cep, IW_CM_EVENT_ESTABLISHED, 0); in siw_accept()
1690 siw_cep_set_free(cep); in siw_accept()
1694 siw_socket_disassoc(cep->sock); in siw_accept()
1695 sock_release(cep->sock); in siw_accept()
1696 cep->sock = NULL; in siw_accept()
1698 cep->state = SIW_EPSTATE_CLOSED; in siw_accept()
1700 if (cep->cm_id) { in siw_accept()
1701 cep->cm_id->rem_ref(id); in siw_accept()
1702 cep->cm_id = NULL; in siw_accept()
1704 if (qp->cep) { in siw_accept()
1705 siw_cep_put(cep); in siw_accept()
1706 qp->cep = NULL; in siw_accept()
1708 cep->qp = NULL; in siw_accept()
1711 siw_cep_set_free(cep); in siw_accept()
1712 siw_cep_put(cep); in siw_accept()
1725 struct siw_cep *cep = (struct siw_cep *)id->provider_data; in siw_reject() local
1727 siw_cep_set_inuse(cep); in siw_reject()
1728 siw_cep_put(cep); in siw_reject()
1730 siw_cancel_mpatimer(cep); in siw_reject()
1732 if (cep->state != SIW_EPSTATE_RECVD_MPAREQ) { in siw_reject()
1733 siw_dbg_cep(cep, "out of state\n"); in siw_reject()
1735 siw_cep_set_free(cep); in siw_reject()
1736 siw_cep_put(cep); /* put last reference */ in siw_reject()
1740 siw_dbg_cep(cep, "cep->state %d, pd_len %d\n", cep->state, in siw_reject()
1743 if (__mpa_rr_revision(cep->mpa.hdr.params.bits) >= MPA_REVISION_1) { in siw_reject()
1744 cep->mpa.hdr.params.bits |= MPA_RR_FLAG_REJECT; /* reject */ in siw_reject()
1745 siw_send_mpareqrep(cep, pdata, pd_len); in siw_reject()
1747 siw_socket_disassoc(cep->sock); in siw_reject()
1748 sock_release(cep->sock); in siw_reject()
1749 cep->sock = NULL; in siw_reject()
1751 cep->state = SIW_EPSTATE_CLOSED; in siw_reject()
1753 siw_cep_set_free(cep); in siw_reject()
1754 siw_cep_put(cep); in siw_reject()
1768 struct siw_cep *cep = NULL; in siw_create_listen() local
1817 cep = siw_cep_alloc(sdev); in siw_create_listen()
1818 if (!cep) { in siw_create_listen()
1822 siw_cep_socket_assoc(cep, s); in siw_create_listen()
1824 rv = siw_cm_alloc_work(cep, backlog); in siw_create_listen()
1836 cep->cm_id = id; in siw_create_listen()
1868 list_add_tail(&cep->listenq, (struct list_head *)id->provider_data); in siw_create_listen()
1869 cep->state = SIW_EPSTATE_LISTENING; in siw_create_listen()
1878 if (cep) { in siw_create_listen()
1879 siw_cep_set_inuse(cep); in siw_create_listen()
1881 if (cep->cm_id) { in siw_create_listen()
1882 cep->cm_id->rem_ref(cep->cm_id); in siw_create_listen()
1883 cep->cm_id = NULL; in siw_create_listen()
1885 cep->sock = NULL; in siw_create_listen()
1887 cep->state = SIW_EPSTATE_CLOSED; in siw_create_listen()
1889 siw_cep_set_free(cep); in siw_create_listen()
1890 siw_cep_put(cep); in siw_create_listen()
1906 struct siw_cep *cep = list_entry(p, struct siw_cep, listenq); in siw_drop_listeners() local
1910 siw_dbg_cep(cep, "drop cep, state %d\n", cep->state); in siw_drop_listeners()
1912 siw_cep_set_inuse(cep); in siw_drop_listeners()
1914 if (cep->cm_id) { in siw_drop_listeners()
1915 cep->cm_id->rem_ref(cep->cm_id); in siw_drop_listeners()
1916 cep->cm_id = NULL; in siw_drop_listeners()
1918 if (cep->sock) { in siw_drop_listeners()
1919 siw_socket_disassoc(cep->sock); in siw_drop_listeners()
1920 sock_release(cep->sock); in siw_drop_listeners()
1921 cep->sock = NULL; in siw_drop_listeners()
1923 cep->state = SIW_EPSTATE_CLOSED; in siw_drop_listeners()
1924 siw_cep_set_free(cep); in siw_drop_listeners()
1925 siw_cep_put(cep); in siw_drop_listeners()