Lines Matching +full:post +full:-
2 * Copyright (c) 2015-2016 QLogic Corporation
14 * - Redistributions of source code must retain the above
18 * - Redistributions in binary form must reproduce the above
32 #include <linux/dma-mapping.h>
50 #include <rdma/qedr-abi.h>
55 info->gsi_cons = (info->gsi_cons + 1) % info->max_wr; in qedr_inc_sw_gsi_cons()
61 dev->gsi_qp_created = 1; in qedr_store_gsi_qp_cq()
62 dev->gsi_sqcq = get_qedr_cq(attrs->send_cq); in qedr_store_gsi_qp_cq()
63 dev->gsi_rqcq = get_qedr_cq(attrs->recv_cq); in qedr_store_gsi_qp_cq()
64 dev->gsi_qp = qp; in qedr_store_gsi_qp_cq()
75 struct qedr_cq *cq = dev->gsi_sqcq; in qedr_ll2_complete_tx_packet()
76 struct qedr_qp *qp = dev->gsi_qp; in qedr_ll2_complete_tx_packet()
81 dev->gsi_sqcq, dev->gsi_rqcq, qp->sq.gsi_cons, in qedr_ll2_complete_tx_packet()
82 cq->ibcq.comp_handler ? "Yes" : "No"); in qedr_ll2_complete_tx_packet()
84 dma_free_coherent(&dev->pdev->dev, pkt->header.len, pkt->header.vaddr, in qedr_ll2_complete_tx_packet()
85 pkt->header.baddr); in qedr_ll2_complete_tx_packet()
88 spin_lock_irqsave(&qp->q_lock, flags); in qedr_ll2_complete_tx_packet()
89 qedr_inc_sw_gsi_cons(&qp->sq); in qedr_ll2_complete_tx_packet()
90 spin_unlock_irqrestore(&qp->q_lock, flags); in qedr_ll2_complete_tx_packet()
92 if (cq->ibcq.comp_handler) in qedr_ll2_complete_tx_packet()
93 (*cq->ibcq.comp_handler) (&cq->ibcq, cq->ibcq.cq_context); in qedr_ll2_complete_tx_packet()
100 struct qedr_cq *cq = dev->gsi_rqcq; in qedr_ll2_complete_rx_packet()
101 struct qedr_qp *qp = dev->gsi_qp; in qedr_ll2_complete_rx_packet()
104 spin_lock_irqsave(&qp->q_lock, flags); in qedr_ll2_complete_rx_packet()
106 qp->rqe_wr_id[qp->rq.gsi_cons].rc = data->u.data_length_error ? in qedr_ll2_complete_rx_packet()
107 -EINVAL : 0; in qedr_ll2_complete_rx_packet()
108 qp->rqe_wr_id[qp->rq.gsi_cons].vlan = data->vlan; in qedr_ll2_complete_rx_packet()
110 qp->rqe_wr_id[qp->rq.gsi_cons].sg_list[0].length = in qedr_ll2_complete_rx_packet()
111 data->length.data_length; in qedr_ll2_complete_rx_packet()
112 *((u32 *)&qp->rqe_wr_id[qp->rq.gsi_cons].smac[0]) = in qedr_ll2_complete_rx_packet()
113 ntohl(data->opaque_data_0); in qedr_ll2_complete_rx_packet()
114 *((u16 *)&qp->rqe_wr_id[qp->rq.gsi_cons].smac[4]) = in qedr_ll2_complete_rx_packet()
115 ntohs((u16)data->opaque_data_1); in qedr_ll2_complete_rx_packet()
117 qedr_inc_sw_gsi_cons(&qp->rq); in qedr_ll2_complete_rx_packet()
119 spin_unlock_irqrestore(&qp->q_lock, flags); in qedr_ll2_complete_rx_packet()
121 if (cq->ibcq.comp_handler) in qedr_ll2_complete_rx_packet()
122 (*cq->ibcq.comp_handler) (&cq->ibcq, cq->ibcq.cq_context); in qedr_ll2_complete_rx_packet()
139 cq = get_qedr_cq(attrs->send_cq); in qedr_destroy_gsi_cq()
140 iparams.icid = cq->icid; in qedr_destroy_gsi_cq()
141 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams); in qedr_destroy_gsi_cq()
142 dev->ops->common->chain_free(dev->cdev, &cq->pbl); in qedr_destroy_gsi_cq()
144 cq = get_qedr_cq(attrs->recv_cq); in qedr_destroy_gsi_cq()
146 if (iparams.icid != cq->icid) { in qedr_destroy_gsi_cq()
147 iparams.icid = cq->icid; in qedr_destroy_gsi_cq()
148 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams); in qedr_destroy_gsi_cq()
149 dev->ops->common->chain_free(dev->cdev, &cq->pbl); in qedr_destroy_gsi_cq()
156 if (attrs->cap.max_recv_sge > QEDR_GSI_MAX_RECV_SGE) { in qedr_check_gsi_qp_attrs()
159 attrs->cap.max_recv_sge, QEDR_GSI_MAX_RECV_SGE); in qedr_check_gsi_qp_attrs()
160 return -EINVAL; in qedr_check_gsi_qp_attrs()
163 if (attrs->cap.max_recv_wr > QEDR_GSI_MAX_RECV_WR) { in qedr_check_gsi_qp_attrs()
166 attrs->cap.max_recv_wr, QEDR_GSI_MAX_RECV_WR); in qedr_check_gsi_qp_attrs()
167 return -EINVAL; in qedr_check_gsi_qp_attrs()
170 if (attrs->cap.max_send_wr > QEDR_GSI_MAX_SEND_WR) { in qedr_check_gsi_qp_attrs()
173 attrs->cap.max_send_wr, QEDR_GSI_MAX_SEND_WR); in qedr_check_gsi_qp_attrs()
174 return -EINVAL; in qedr_check_gsi_qp_attrs()
190 roce_flavor = (pkt->roce_mode == ROCE_V1) ? in qedr_ll2_post_tx()
193 if (pkt->roce_mode == ROCE_V2_IPV4) in qedr_ll2_post_tx()
196 ll2_tx_pkt.num_of_bds = 1 /* hdr */ + pkt->n_seg; in qedr_ll2_post_tx()
198 ll2_tx_pkt.tx_dest = pkt->tx_dest; in qedr_ll2_post_tx()
200 ll2_tx_pkt.first_frag = pkt->header.baddr; in qedr_ll2_post_tx()
201 ll2_tx_pkt.first_frag_len = pkt->header.len; in qedr_ll2_post_tx()
205 rc = dev->ops->ll2_prepare_tx_packet(dev->rdma_ctx, in qedr_ll2_post_tx()
206 dev->gsi_ll2_handle, in qedr_ll2_post_tx()
209 /* TX failed while posting header - release resources */ in qedr_ll2_post_tx()
210 dma_free_coherent(&dev->pdev->dev, pkt->header.len, in qedr_ll2_post_tx()
211 pkt->header.vaddr, pkt->header.baddr); in qedr_ll2_post_tx()
219 for (i = 0; i < pkt->n_seg; i++) { in qedr_ll2_post_tx()
220 rc = dev->ops->ll2_set_fragment_of_tx_packet( in qedr_ll2_post_tx()
221 dev->rdma_ctx, in qedr_ll2_post_tx()
222 dev->gsi_ll2_handle, in qedr_ll2_post_tx()
223 pkt->payload[i].baddr, in qedr_ll2_post_tx()
224 pkt->payload[i].len); in qedr_ll2_post_tx()
243 if (dev->gsi_ll2_handle == QED_LL2_UNUSED_HANDLE) in qedr_ll2_stop()
247 rc = dev->ops->ll2_set_mac_filter(dev->cdev, in qedr_ll2_stop()
248 dev->gsi_ll2_mac_address, NULL); in qedr_ll2_stop()
250 rc = dev->ops->ll2_terminate_connection(dev->rdma_ctx, in qedr_ll2_stop()
251 dev->gsi_ll2_handle); in qedr_ll2_stop()
255 dev->ops->ll2_release_connection(dev->rdma_ctx, dev->gsi_ll2_handle); in qedr_ll2_stop()
257 dev->gsi_ll2_handle = QED_LL2_UNUSED_HANDLE; in qedr_ll2_stop()
278 data.input.mtu = dev->ndev->mtu; in qedr_ll2_start()
279 data.input.rx_num_desc = attrs->cap.max_recv_wr; in qedr_ll2_start()
282 data.input.tx_num_desc = attrs->cap.max_send_wr; in qedr_ll2_start()
288 data.p_connection_handle = &dev->gsi_ll2_handle; in qedr_ll2_start()
291 rc = dev->ops->ll2_acquire_connection(dev->rdma_ctx, &data); in qedr_ll2_start()
299 rc = dev->ops->ll2_establish_connection(dev->rdma_ctx, in qedr_ll2_start()
300 dev->gsi_ll2_handle); in qedr_ll2_start()
308 rc = dev->ops->ll2_set_mac_filter(dev->cdev, NULL, dev->ndev->dev_addr); in qedr_ll2_start()
315 dev->ops->ll2_terminate_connection(dev->rdma_ctx, dev->gsi_ll2_handle); in qedr_ll2_start()
317 dev->ops->ll2_release_connection(dev->rdma_ctx, dev->gsi_ll2_handle); in qedr_ll2_start()
338 qp->ibqp.qp_num = 1; in qedr_create_gsi_qp()
339 qp->rq.max_wr = attrs->cap.max_recv_wr; in qedr_create_gsi_qp()
340 qp->sq.max_wr = attrs->cap.max_send_wr; in qedr_create_gsi_qp()
342 qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id), in qedr_create_gsi_qp()
344 if (!qp->rqe_wr_id) in qedr_create_gsi_qp()
346 qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id), in qedr_create_gsi_qp()
348 if (!qp->wqe_wr_id) in qedr_create_gsi_qp()
352 ether_addr_copy(dev->gsi_ll2_mac_address, dev->ndev->dev_addr); in qedr_create_gsi_qp()
356 dev->gsi_rqcq->cq_type = QEDR_CQ_TYPE_GSI; in qedr_create_gsi_qp()
357 dev->gsi_rqcq->cq_type = QEDR_CQ_TYPE_GSI; in qedr_create_gsi_qp()
364 kfree(qp->rqe_wr_id); in qedr_create_gsi_qp()
370 return -ENOMEM; in qedr_create_gsi_qp()
387 struct rdma_ah_attr *ah_attr = &get_qedr_ah(ud_wr(swr)->ah)->attr; in qedr_gsi_build_header()
389 const struct ib_gid_attr *sgid_attr = grh->sgid_attr; in qedr_gsi_build_header()
407 for (i = 0; i < swr->num_sge; ++i) in qedr_gsi_build_header()
408 send_size += swr->sg_list[i].length; in qedr_gsi_build_header()
410 has_udp = (sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP); in qedr_gsi_build_header()
415 } else if (ipv6_addr_v4mapped((struct in6_addr *)&sgid_attr->gid)) { in qedr_gsi_build_header()
431 DP_ERR(dev, "gsi post send: failed to init header\n"); in qedr_gsi_build_header()
436 ether_addr_copy(udh->eth.dmac_h, ah_attr->roce.dmac); in qedr_gsi_build_header()
437 ether_addr_copy(udh->eth.smac_h, dev->ndev->dev_addr); in qedr_gsi_build_header()
439 udh->eth.type = htons(ETH_P_8021Q); in qedr_gsi_build_header()
440 udh->vlan.tag = htons(vlan_id); in qedr_gsi_build_header()
441 udh->vlan.type = htons(ether_type); in qedr_gsi_build_header()
443 udh->eth.type = htons(ether_type); in qedr_gsi_build_header()
447 udh->bth.solicited_event = !!(swr->send_flags & IB_SEND_SOLICITED); in qedr_gsi_build_header()
448 udh->bth.pkey = QEDR_ROCE_PKEY_DEFAULT; in qedr_gsi_build_header()
449 udh->bth.destination_qpn = htonl(ud_wr(swr)->remote_qpn); in qedr_gsi_build_header()
450 udh->bth.psn = htonl((qp->sq_psn++) & ((1 << 24) - 1)); in qedr_gsi_build_header()
451 udh->bth.opcode = IB_OPCODE_UD_SEND_ONLY; in qedr_gsi_build_header()
454 udh->deth.qkey = htonl(0x80010000); in qedr_gsi_build_header()
455 udh->deth.source_qpn = htonl(QEDR_GSI_QPN); in qedr_gsi_build_header()
459 udh->grh.traffic_class = grh->traffic_class; in qedr_gsi_build_header()
460 udh->grh.flow_label = grh->flow_label; in qedr_gsi_build_header()
461 udh->grh.hop_limit = grh->hop_limit; in qedr_gsi_build_header()
462 udh->grh.destination_gid = grh->dgid; in qedr_gsi_build_header()
463 memcpy(&udh->grh.source_gid.raw, sgid_attr->gid.raw, in qedr_gsi_build_header()
464 sizeof(udh->grh.source_gid.raw)); in qedr_gsi_build_header()
469 udh->ip4.protocol = IPPROTO_UDP; in qedr_gsi_build_header()
470 udh->ip4.tos = htonl(grh->flow_label); in qedr_gsi_build_header()
471 udh->ip4.frag_off = htons(IP_DF); in qedr_gsi_build_header()
472 udh->ip4.ttl = grh->hop_limit; in qedr_gsi_build_header()
474 ipv4_addr = qedr_get_ipv4_from_gid(sgid_attr->gid.raw); in qedr_gsi_build_header()
475 udh->ip4.saddr = ipv4_addr; in qedr_gsi_build_header()
476 ipv4_addr = qedr_get_ipv4_from_gid(grh->dgid.raw); in qedr_gsi_build_header()
477 udh->ip4.daddr = ipv4_addr; in qedr_gsi_build_header()
483 udh->udp.sport = htons(QEDR_ROCE_V2_UDP_SPORT); in qedr_gsi_build_header()
484 udh->udp.dport = htons(ROCE_V2_UDP_DPORT); in qedr_gsi_build_header()
485 udh->udp.csum = 0; in qedr_gsi_build_header()
498 struct pci_dev *pdev = dev->pdev; in qedr_gsi_build_packet()
513 return -ENOMEM; in qedr_gsi_build_packet()
515 packet->header.vaddr = dma_alloc_coherent(&pdev->dev, header_size, in qedr_gsi_build_packet()
516 &packet->header.baddr, in qedr_gsi_build_packet()
518 if (!packet->header.vaddr) { in qedr_gsi_build_packet()
520 return -ENOMEM; in qedr_gsi_build_packet()
524 packet->tx_dest = QED_LL2_TX_DEST_LB; in qedr_gsi_build_packet()
526 packet->tx_dest = QED_LL2_TX_DEST_NW; in qedr_gsi_build_packet()
528 packet->roce_mode = roce_mode; in qedr_gsi_build_packet()
529 memcpy(packet->header.vaddr, ud_header_buffer, header_size); in qedr_gsi_build_packet()
530 packet->header.len = header_size; in qedr_gsi_build_packet()
531 packet->n_seg = swr->num_sge; in qedr_gsi_build_packet()
532 for (i = 0; i < packet->n_seg; i++) { in qedr_gsi_build_packet()
533 packet->payload[i].baddr = swr->sg_list[i].addr; in qedr_gsi_build_packet()
534 packet->payload[i].len = swr->sg_list[i].length; in qedr_gsi_build_packet()
547 struct qedr_dev *dev = qp->dev; in qedr_gsi_post_send()
551 if (qp->state != QED_ROCE_QP_STATE_RTS) { in qedr_gsi_post_send()
554 "gsi post recv: failed to post rx buffer. state is %d and not QED_ROCE_QP_STATE_RTS\n", in qedr_gsi_post_send()
555 qp->state); in qedr_gsi_post_send()
556 return -EINVAL; in qedr_gsi_post_send()
559 if (wr->num_sge > RDMA_MAX_SGE_PER_SQ_WQE) { in qedr_gsi_post_send()
560 DP_ERR(dev, "gsi post send: num_sge is too large (%d>%d)\n", in qedr_gsi_post_send()
561 wr->num_sge, RDMA_MAX_SGE_PER_SQ_WQE); in qedr_gsi_post_send()
562 rc = -EINVAL; in qedr_gsi_post_send()
566 if (wr->opcode != IB_WR_SEND) { in qedr_gsi_post_send()
568 "gsi post send: failed due to unsupported opcode %d\n", in qedr_gsi_post_send()
569 wr->opcode); in qedr_gsi_post_send()
570 rc = -EINVAL; in qedr_gsi_post_send()
574 spin_lock_irqsave(&qp->q_lock, flags); in qedr_gsi_post_send()
578 spin_unlock_irqrestore(&qp->q_lock, flags); in qedr_gsi_post_send()
585 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id; in qedr_gsi_post_send()
586 qedr_inc_sw_prod(&qp->sq); in qedr_gsi_post_send()
587 DP_DEBUG(qp->dev, QEDR_MSG_GSI, in qedr_gsi_post_send()
588 "gsi post send: opcode=%d, wr_id=%llx\n", wr->opcode, in qedr_gsi_post_send()
589 wr->wr_id); in qedr_gsi_post_send()
591 DP_ERR(dev, "gsi post send: failed to transmit (rc=%d)\n", rc); in qedr_gsi_post_send()
592 rc = -EAGAIN; in qedr_gsi_post_send()
596 spin_unlock_irqrestore(&qp->q_lock, flags); in qedr_gsi_post_send()
598 if (wr->next) { in qedr_gsi_post_send()
600 "gsi post send: failed second WR. Only one WR may be passed at a time\n"); in qedr_gsi_post_send()
601 *bad_wr = wr->next; in qedr_gsi_post_send()
602 rc = -EINVAL; in qedr_gsi_post_send()
615 struct qedr_dev *dev = get_qedr_dev(ibqp->device); in qedr_gsi_post_recv()
620 if ((qp->state != QED_ROCE_QP_STATE_RTR) && in qedr_gsi_post_recv()
621 (qp->state != QED_ROCE_QP_STATE_RTS)) { in qedr_gsi_post_recv()
624 "gsi post recv: failed to post rx buffer. state is %d and not QED_ROCE_QP_STATE_RTR/S\n", in qedr_gsi_post_recv()
625 qp->state); in qedr_gsi_post_recv()
626 return -EINVAL; in qedr_gsi_post_recv()
629 spin_lock_irqsave(&qp->q_lock, flags); in qedr_gsi_post_recv()
632 if (wr->num_sge > QEDR_GSI_MAX_RECV_SGE) { in qedr_gsi_post_recv()
634 "gsi post recv: failed to post rx buffer. too many sges %d>%d\n", in qedr_gsi_post_recv()
635 wr->num_sge, QEDR_GSI_MAX_RECV_SGE); in qedr_gsi_post_recv()
639 rc = dev->ops->ll2_post_rx_buffer(dev->rdma_ctx, in qedr_gsi_post_recv()
640 dev->gsi_ll2_handle, in qedr_gsi_post_recv()
641 wr->sg_list[0].addr, in qedr_gsi_post_recv()
642 wr->sg_list[0].length, in qedr_gsi_post_recv()
647 "gsi post recv: failed to post rx buffer (rc=%d)\n", in qedr_gsi_post_recv()
652 memset(&qp->rqe_wr_id[qp->rq.prod], 0, in qedr_gsi_post_recv()
653 sizeof(qp->rqe_wr_id[qp->rq.prod])); in qedr_gsi_post_recv()
654 qp->rqe_wr_id[qp->rq.prod].sg_list[0] = wr->sg_list[0]; in qedr_gsi_post_recv()
655 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id; in qedr_gsi_post_recv()
657 qedr_inc_sw_prod(&qp->rq); in qedr_gsi_post_recv()
659 wr = wr->next; in qedr_gsi_post_recv()
662 spin_unlock_irqrestore(&qp->q_lock, flags); in qedr_gsi_post_recv()
666 spin_unlock_irqrestore(&qp->q_lock, flags); in qedr_gsi_post_recv()
668 return -ENOMEM; in qedr_gsi_post_recv()
673 struct qedr_dev *dev = get_qedr_dev(ibcq->device); in qedr_gsi_poll_cq()
675 struct qedr_qp *qp = dev->gsi_qp; in qedr_gsi_poll_cq()
680 spin_lock_irqsave(&cq->cq_lock, flags); in qedr_gsi_poll_cq()
682 while (i < num_entries && qp->rq.cons != qp->rq.gsi_cons) { in qedr_gsi_poll_cq()
685 wc[i].qp = &qp->ibqp; in qedr_gsi_poll_cq()
686 wc[i].wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; in qedr_gsi_poll_cq()
689 wc[i].status = (qp->rqe_wr_id[qp->rq.cons].rc) ? in qedr_gsi_poll_cq()
691 /* 0 - currently only one recv sg is supported */ in qedr_gsi_poll_cq()
692 wc[i].byte_len = qp->rqe_wr_id[qp->rq.cons].sg_list[0].length; in qedr_gsi_poll_cq()
694 ether_addr_copy(wc[i].smac, qp->rqe_wr_id[qp->rq.cons].smac); in qedr_gsi_poll_cq()
697 vlan_id = qp->rqe_wr_id[qp->rq.cons].vlan & in qedr_gsi_poll_cq()
702 wc[i].sl = (qp->rqe_wr_id[qp->rq.cons].vlan & in qedr_gsi_poll_cq()
706 qedr_inc_sw_cons(&qp->rq); in qedr_gsi_poll_cq()
710 while (i < num_entries && qp->sq.cons != qp->sq.gsi_cons) { in qedr_gsi_poll_cq()
713 wc[i].qp = &qp->ibqp; in qedr_gsi_poll_cq()
714 wc[i].wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id; in qedr_gsi_poll_cq()
718 qedr_inc_sw_cons(&qp->sq); in qedr_gsi_poll_cq()
722 spin_unlock_irqrestore(&cq->cq_lock, flags); in qedr_gsi_poll_cq()
725 …"gsi poll_cq: requested entries=%d, actual=%d, qp->rq.cons=%d, qp->rq.gsi_cons=%x, qp->sq.cons=%d,… in qedr_gsi_poll_cq()
726 num_entries, i, qp->rq.cons, qp->rq.gsi_cons, qp->sq.cons, in qedr_gsi_poll_cq()
727 qp->sq.gsi_cons, qp->ibqp.qp_num); in qedr_gsi_poll_cq()