Lines Matching full:lnk
524 void smc_wr_remember_qp_attr(struct smc_link *lnk) in smc_wr_remember_qp_attr() argument
526 struct ib_qp_attr *attr = &lnk->qp_attr; in smc_wr_remember_qp_attr()
531 ib_query_qp(lnk->roce_qp, attr, in smc_wr_remember_qp_attr()
551 lnk->wr_tx_cnt = min_t(size_t, SMC_WR_BUF_CNT, in smc_wr_remember_qp_attr()
552 lnk->qp_attr.cap.max_send_wr); in smc_wr_remember_qp_attr()
553 lnk->wr_rx_cnt = min_t(size_t, SMC_WR_BUF_CNT * 3, in smc_wr_remember_qp_attr()
554 lnk->qp_attr.cap.max_recv_wr); in smc_wr_remember_qp_attr()
557 static void smc_wr_init_sge(struct smc_link *lnk) in smc_wr_init_sge() argument
559 int sges_per_buf = (lnk->lgr->smc_version == SMC_V2) ? 2 : 1; in smc_wr_init_sge()
560 bool send_inline = (lnk->qp_attr.cap.max_inline_data > SMC_WR_TX_SIZE); in smc_wr_init_sge()
563 for (i = 0; i < lnk->wr_tx_cnt; i++) { in smc_wr_init_sge()
564 lnk->wr_tx_sges[i].addr = send_inline ? (uintptr_t)(&lnk->wr_tx_bufs[i]) : in smc_wr_init_sge()
565 lnk->wr_tx_dma_addr + i * SMC_WR_BUF_SIZE; in smc_wr_init_sge()
566 lnk->wr_tx_sges[i].length = SMC_WR_TX_SIZE; in smc_wr_init_sge()
567 lnk->wr_tx_sges[i].lkey = lnk->roce_pd->local_dma_lkey; in smc_wr_init_sge()
568 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[0].wr_tx_rdma_sge[0].lkey = in smc_wr_init_sge()
569 lnk->roce_pd->local_dma_lkey; in smc_wr_init_sge()
570 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[0].wr_tx_rdma_sge[1].lkey = in smc_wr_init_sge()
571 lnk->roce_pd->local_dma_lkey; in smc_wr_init_sge()
572 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[1].wr_tx_rdma_sge[0].lkey = in smc_wr_init_sge()
573 lnk->roce_pd->local_dma_lkey; in smc_wr_init_sge()
574 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[1].wr_tx_rdma_sge[1].lkey = in smc_wr_init_sge()
575 lnk->roce_pd->local_dma_lkey; in smc_wr_init_sge()
576 lnk->wr_tx_ibs[i].next = NULL; in smc_wr_init_sge()
577 lnk->wr_tx_ibs[i].sg_list = &lnk->wr_tx_sges[i]; in smc_wr_init_sge()
578 lnk->wr_tx_ibs[i].num_sge = 1; in smc_wr_init_sge()
579 lnk->wr_tx_ibs[i].opcode = IB_WR_SEND; in smc_wr_init_sge()
580 lnk->wr_tx_ibs[i].send_flags = in smc_wr_init_sge()
583 lnk->wr_tx_ibs[i].send_flags |= IB_SEND_INLINE; in smc_wr_init_sge()
584 lnk->wr_tx_rdmas[i].wr_tx_rdma[0].wr.opcode = IB_WR_RDMA_WRITE; in smc_wr_init_sge()
585 lnk->wr_tx_rdmas[i].wr_tx_rdma[1].wr.opcode = IB_WR_RDMA_WRITE; in smc_wr_init_sge()
586 lnk->wr_tx_rdmas[i].wr_tx_rdma[0].wr.sg_list = in smc_wr_init_sge()
587 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[0].wr_tx_rdma_sge; in smc_wr_init_sge()
588 lnk->wr_tx_rdmas[i].wr_tx_rdma[1].wr.sg_list = in smc_wr_init_sge()
589 lnk->wr_tx_rdma_sges[i].tx_rdma_sge[1].wr_tx_rdma_sge; in smc_wr_init_sge()
592 if (lnk->lgr->smc_version == SMC_V2) { in smc_wr_init_sge()
593 lnk->wr_tx_v2_sge->addr = lnk->wr_tx_v2_dma_addr; in smc_wr_init_sge()
594 lnk->wr_tx_v2_sge->length = SMC_WR_BUF_V2_SIZE; in smc_wr_init_sge()
595 lnk->wr_tx_v2_sge->lkey = lnk->roce_pd->local_dma_lkey; in smc_wr_init_sge()
597 lnk->wr_tx_v2_ib->next = NULL; in smc_wr_init_sge()
598 lnk->wr_tx_v2_ib->sg_list = lnk->wr_tx_v2_sge; in smc_wr_init_sge()
599 lnk->wr_tx_v2_ib->num_sge = 1; in smc_wr_init_sge()
600 lnk->wr_tx_v2_ib->opcode = IB_WR_SEND; in smc_wr_init_sge()
601 lnk->wr_tx_v2_ib->send_flags = in smc_wr_init_sge()
611 for (i = 0; i < lnk->wr_rx_cnt; i++) { in smc_wr_init_sge()
614 lnk->wr_rx_sges[x].addr = in smc_wr_init_sge()
615 lnk->wr_rx_dma_addr + i * SMC_WR_BUF_SIZE; in smc_wr_init_sge()
616 lnk->wr_rx_sges[x].length = SMC_WR_TX_SIZE; in smc_wr_init_sge()
617 lnk->wr_rx_sges[x].lkey = lnk->roce_pd->local_dma_lkey; in smc_wr_init_sge()
618 if (lnk->lgr->smc_version == SMC_V2) { in smc_wr_init_sge()
619 lnk->wr_rx_sges[x + 1].addr = in smc_wr_init_sge()
620 lnk->wr_rx_v2_dma_addr + SMC_WR_TX_SIZE; in smc_wr_init_sge()
621 lnk->wr_rx_sges[x + 1].length = in smc_wr_init_sge()
623 lnk->wr_rx_sges[x + 1].lkey = in smc_wr_init_sge()
624 lnk->roce_pd->local_dma_lkey; in smc_wr_init_sge()
626 lnk->wr_rx_ibs[i].next = NULL; in smc_wr_init_sge()
627 lnk->wr_rx_ibs[i].sg_list = &lnk->wr_rx_sges[x]; in smc_wr_init_sge()
628 lnk->wr_rx_ibs[i].num_sge = sges_per_buf; in smc_wr_init_sge()
630 lnk->wr_reg.wr.next = NULL; in smc_wr_init_sge()
631 lnk->wr_reg.wr.num_sge = 0; in smc_wr_init_sge()
632 lnk->wr_reg.wr.send_flags = IB_SEND_SIGNALED; in smc_wr_init_sge()
633 lnk->wr_reg.wr.opcode = IB_WR_REG_MR; in smc_wr_init_sge()
634 lnk->wr_reg.access = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE; in smc_wr_init_sge()
637 void smc_wr_free_link(struct smc_link *lnk) in smc_wr_free_link() argument
641 if (!lnk->smcibdev) in smc_wr_free_link()
643 ibdev = lnk->smcibdev->ibdev; in smc_wr_free_link()
645 smc_wr_drain_cq(lnk); in smc_wr_free_link()
646 smc_wr_wakeup_reg_wait(lnk); in smc_wr_free_link()
647 smc_wr_wakeup_tx_wait(lnk); in smc_wr_free_link()
649 smc_wr_tx_wait_no_pending_sends(lnk); in smc_wr_free_link()
650 wait_event(lnk->wr_reg_wait, (!atomic_read(&lnk->wr_reg_refcnt))); in smc_wr_free_link()
651 wait_event(lnk->wr_tx_wait, (!atomic_read(&lnk->wr_tx_refcnt))); in smc_wr_free_link()
653 if (lnk->wr_rx_dma_addr) { in smc_wr_free_link()
654 ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr, in smc_wr_free_link()
655 SMC_WR_BUF_SIZE * lnk->wr_rx_cnt, in smc_wr_free_link()
657 lnk->wr_rx_dma_addr = 0; in smc_wr_free_link()
659 if (lnk->wr_rx_v2_dma_addr) { in smc_wr_free_link()
660 ib_dma_unmap_single(ibdev, lnk->wr_rx_v2_dma_addr, in smc_wr_free_link()
663 lnk->wr_rx_v2_dma_addr = 0; in smc_wr_free_link()
665 if (lnk->wr_tx_dma_addr) { in smc_wr_free_link()
666 ib_dma_unmap_single(ibdev, lnk->wr_tx_dma_addr, in smc_wr_free_link()
667 SMC_WR_BUF_SIZE * lnk->wr_tx_cnt, in smc_wr_free_link()
669 lnk->wr_tx_dma_addr = 0; in smc_wr_free_link()
671 if (lnk->wr_tx_v2_dma_addr) { in smc_wr_free_link()
672 ib_dma_unmap_single(ibdev, lnk->wr_tx_v2_dma_addr, in smc_wr_free_link()
675 lnk->wr_tx_v2_dma_addr = 0; in smc_wr_free_link()
690 void smc_wr_free_link_mem(struct smc_link *lnk) in smc_wr_free_link_mem() argument
692 kfree(lnk->wr_tx_v2_ib); in smc_wr_free_link_mem()
693 lnk->wr_tx_v2_ib = NULL; in smc_wr_free_link_mem()
694 kfree(lnk->wr_tx_v2_sge); in smc_wr_free_link_mem()
695 lnk->wr_tx_v2_sge = NULL; in smc_wr_free_link_mem()
696 kfree(lnk->wr_tx_v2_pend); in smc_wr_free_link_mem()
697 lnk->wr_tx_v2_pend = NULL; in smc_wr_free_link_mem()
698 kfree(lnk->wr_tx_compl); in smc_wr_free_link_mem()
699 lnk->wr_tx_compl = NULL; in smc_wr_free_link_mem()
700 kfree(lnk->wr_tx_pends); in smc_wr_free_link_mem()
701 lnk->wr_tx_pends = NULL; in smc_wr_free_link_mem()
702 bitmap_free(lnk->wr_tx_mask); in smc_wr_free_link_mem()
703 lnk->wr_tx_mask = NULL; in smc_wr_free_link_mem()
704 kfree(lnk->wr_tx_sges); in smc_wr_free_link_mem()
705 lnk->wr_tx_sges = NULL; in smc_wr_free_link_mem()
706 kfree(lnk->wr_tx_rdma_sges); in smc_wr_free_link_mem()
707 lnk->wr_tx_rdma_sges = NULL; in smc_wr_free_link_mem()
708 kfree(lnk->wr_rx_sges); in smc_wr_free_link_mem()
709 lnk->wr_rx_sges = NULL; in smc_wr_free_link_mem()
710 kfree(lnk->wr_tx_rdmas); in smc_wr_free_link_mem()
711 lnk->wr_tx_rdmas = NULL; in smc_wr_free_link_mem()
712 kfree(lnk->wr_rx_ibs); in smc_wr_free_link_mem()
713 lnk->wr_rx_ibs = NULL; in smc_wr_free_link_mem()
714 kfree(lnk->wr_tx_ibs); in smc_wr_free_link_mem()
715 lnk->wr_tx_ibs = NULL; in smc_wr_free_link_mem()
716 kfree(lnk->wr_tx_bufs); in smc_wr_free_link_mem()
717 lnk->wr_tx_bufs = NULL; in smc_wr_free_link_mem()
718 kfree(lnk->wr_rx_bufs); in smc_wr_free_link_mem()
719 lnk->wr_rx_bufs = NULL; in smc_wr_free_link_mem()
850 int smc_wr_create_link(struct smc_link *lnk) in smc_wr_create_link() argument
852 struct ib_device *ibdev = lnk->smcibdev->ibdev; in smc_wr_create_link()
855 smc_wr_tx_set_wr_id(&lnk->wr_tx_id, 0); in smc_wr_create_link()
856 lnk->wr_rx_id = 0; in smc_wr_create_link()
857 lnk->wr_rx_dma_addr = ib_dma_map_single( in smc_wr_create_link()
858 ibdev, lnk->wr_rx_bufs, SMC_WR_BUF_SIZE * lnk->wr_rx_cnt, in smc_wr_create_link()
860 if (ib_dma_mapping_error(ibdev, lnk->wr_rx_dma_addr)) { in smc_wr_create_link()
861 lnk->wr_rx_dma_addr = 0; in smc_wr_create_link()
865 if (lnk->lgr->smc_version == SMC_V2) { in smc_wr_create_link()
866 lnk->wr_rx_v2_dma_addr = ib_dma_map_single(ibdev, in smc_wr_create_link()
867 lnk->lgr->wr_rx_buf_v2, SMC_WR_BUF_V2_SIZE, in smc_wr_create_link()
869 if (ib_dma_mapping_error(ibdev, lnk->wr_rx_v2_dma_addr)) { in smc_wr_create_link()
870 lnk->wr_rx_v2_dma_addr = 0; in smc_wr_create_link()
874 lnk->wr_tx_v2_dma_addr = ib_dma_map_single(ibdev, in smc_wr_create_link()
875 lnk->lgr->wr_tx_buf_v2, SMC_WR_BUF_V2_SIZE, in smc_wr_create_link()
877 if (ib_dma_mapping_error(ibdev, lnk->wr_tx_v2_dma_addr)) { in smc_wr_create_link()
878 lnk->wr_tx_v2_dma_addr = 0; in smc_wr_create_link()
883 lnk->wr_tx_dma_addr = ib_dma_map_single( in smc_wr_create_link()
884 ibdev, lnk->wr_tx_bufs, SMC_WR_BUF_SIZE * lnk->wr_tx_cnt, in smc_wr_create_link()
886 if (ib_dma_mapping_error(ibdev, lnk->wr_tx_dma_addr)) { in smc_wr_create_link()
890 smc_wr_init_sge(lnk); in smc_wr_create_link()
891 bitmap_zero(lnk->wr_tx_mask, SMC_WR_BUF_CNT); in smc_wr_create_link()
892 init_waitqueue_head(&lnk->wr_tx_wait); in smc_wr_create_link()
893 atomic_set(&lnk->wr_tx_refcnt, 0); in smc_wr_create_link()
894 init_waitqueue_head(&lnk->wr_reg_wait); in smc_wr_create_link()
895 atomic_set(&lnk->wr_reg_refcnt, 0); in smc_wr_create_link()
896 init_waitqueue_head(&lnk->wr_rx_empty_wait); in smc_wr_create_link()
900 if (lnk->wr_rx_v2_dma_addr) { in smc_wr_create_link()
901 ib_dma_unmap_single(ibdev, lnk->wr_rx_v2_dma_addr, in smc_wr_create_link()
904 lnk->wr_rx_v2_dma_addr = 0; in smc_wr_create_link()
906 if (lnk->wr_tx_v2_dma_addr) { in smc_wr_create_link()
907 ib_dma_unmap_single(ibdev, lnk->wr_tx_v2_dma_addr, in smc_wr_create_link()
910 lnk->wr_tx_v2_dma_addr = 0; in smc_wr_create_link()
912 ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr, in smc_wr_create_link()
913 SMC_WR_BUF_SIZE * lnk->wr_rx_cnt, in smc_wr_create_link()
915 lnk->wr_rx_dma_addr = 0; in smc_wr_create_link()