Lines Matching full:rf
69 * @rf: RDMA PCI function
72 static void irdma_puda_ce_handler(struct irdma_pci_f *rf, in irdma_puda_ce_handler() argument
75 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_puda_ce_handler()
99 * @rf: RDMA PCI function
102 static void irdma_process_ceq(struct irdma_pci_f *rf, struct irdma_ceq *ceq) in irdma_process_ceq() argument
104 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_process_ceq()
124 queue_work(rf->cqp_cmpl_wq, &rf->cqp_cmpl_work); in irdma_process_ceq()
127 irdma_puda_ce_handler(rf, cq); in irdma_process_ceq()
195 * @rf: RDMA PCI function
197 static void irdma_process_aeq(struct irdma_pci_f *rf) in irdma_process_aeq() argument
199 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_process_aeq()
200 struct irdma_aeq *aeq = &rf->aeq; in irdma_process_aeq()
210 struct irdma_device *iwdev = rf->iwdev; in irdma_process_aeq()
231 spin_lock_irqsave(&rf->qptable_lock, flags); in irdma_process_aeq()
232 iwqp = rf->qp_table[info->qp_cq_id]; in irdma_process_aeq()
234 spin_unlock_irqrestore(&rf->qptable_lock, in irdma_process_aeq()
246 spin_unlock_irqrestore(&rf->qptable_lock, flags); in irdma_process_aeq()
426 struct irdma_pci_f *rf = from_tasklet(rf, t, dpc_tasklet); in irdma_dpc() local
428 if (rf->msix_shared) in irdma_dpc()
429 irdma_process_ceq(rf, rf->ceqlist); in irdma_dpc()
430 irdma_process_aeq(rf); in irdma_dpc()
431 irdma_ena_intr(&rf->sc_dev, rf->iw_msixtbl[0].idx); in irdma_dpc()
441 struct irdma_pci_f *rf = iwceq->rf; in irdma_ceq_dpc() local
443 irdma_process_ceq(rf, iwceq); in irdma_ceq_dpc()
444 irdma_ena_intr(&rf->sc_dev, iwceq->msix_idx); in irdma_ceq_dpc()
449 * @rf: RDMA PCI function
454 static enum irdma_status_code irdma_save_msix_info(struct irdma_pci_f *rf) in irdma_save_msix_info() argument
463 if (!rf->msix_count) in irdma_save_msix_info()
466 size = sizeof(struct irdma_msix_vector) * rf->msix_count; in irdma_save_msix_info()
467 size += struct_size(iw_qvlist, qv_info, rf->msix_count); in irdma_save_msix_info()
468 rf->iw_msixtbl = kzalloc(size, GFP_KERNEL); in irdma_save_msix_info()
469 if (!rf->iw_msixtbl) in irdma_save_msix_info()
472 rf->iw_qvlist = (struct irdma_qvlist_info *) in irdma_save_msix_info()
473 (&rf->iw_msixtbl[rf->msix_count]); in irdma_save_msix_info()
474 iw_qvlist = rf->iw_qvlist; in irdma_save_msix_info()
476 iw_qvlist->num_vectors = rf->msix_count; in irdma_save_msix_info()
477 if (rf->msix_count <= num_online_cpus()) in irdma_save_msix_info()
478 rf->msix_shared = true; in irdma_save_msix_info()
480 pmsix = rf->msix_entries; in irdma_save_msix_info()
481 for (i = 0, ceq_idx = 0; i < rf->msix_count; i++, iw_qvinfo++) { in irdma_save_msix_info()
482 rf->iw_msixtbl[i].idx = pmsix->entry; in irdma_save_msix_info()
483 rf->iw_msixtbl[i].irq = pmsix->vector; in irdma_save_msix_info()
484 rf->iw_msixtbl[i].cpu_affinity = ceq_idx; in irdma_save_msix_info()
487 if (rf->msix_shared) in irdma_save_msix_info()
496 iw_qvinfo->v_idx = rf->iw_msixtbl[i].idx; in irdma_save_msix_info()
510 struct irdma_pci_f *rf = data; in irdma_irq_handler() local
512 tasklet_schedule(&rf->dpc_tasklet); in irdma_irq_handler()
527 ibdev_err(to_ibdev(&iwceq->rf->sc_dev), "expected irq = %d received irq = %d\n", in irdma_ceq_handler()
536 * @rf: RDMA PCI function
542 static void irdma_destroy_irq(struct irdma_pci_f *rf, in irdma_destroy_irq() argument
545 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_destroy_irq()
554 * @rf: RDMA PCI function
560 static void irdma_destroy_cqp(struct irdma_pci_f *rf, bool free_hwcqp) in irdma_destroy_cqp() argument
563 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_destroy_cqp()
564 struct irdma_cqp *cqp = &rf->cqp; in irdma_destroy_cqp()
566 if (rf->cqp_cmpl_wq) in irdma_destroy_cqp()
567 destroy_workqueue(rf->cqp_cmpl_wq); in irdma_destroy_cqp()
573 irdma_cleanup_pending_cqp_op(rf); in irdma_destroy_cqp()
583 static void irdma_destroy_virt_aeq(struct irdma_pci_f *rf) in irdma_destroy_virt_aeq() argument
585 struct irdma_aeq *aeq = &rf->aeq; in irdma_destroy_virt_aeq()
589 irdma_unmap_vm_page_list(&rf->hw, pg_arr, pg_cnt); in irdma_destroy_virt_aeq()
590 irdma_free_pble(rf->pble_rsrc, &aeq->palloc); in irdma_destroy_virt_aeq()
596 * @rf: RDMA PCI function
602 static void irdma_destroy_aeq(struct irdma_pci_f *rf) in irdma_destroy_aeq() argument
605 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_destroy_aeq()
606 struct irdma_aeq *aeq = &rf->aeq; in irdma_destroy_aeq()
608 if (!rf->msix_shared) { in irdma_destroy_aeq()
609 rf->sc_dev.irq_ops->irdma_cfg_aeq(&rf->sc_dev, rf->iw_msixtbl->idx, false); in irdma_destroy_aeq()
610 irdma_destroy_irq(rf, rf->iw_msixtbl, rf); in irdma_destroy_aeq()
612 if (rf->reset) in irdma_destroy_aeq()
622 irdma_destroy_virt_aeq(rf); in irdma_destroy_aeq()
632 * @rf: RDMA PCI function
638 static void irdma_destroy_ceq(struct irdma_pci_f *rf, struct irdma_ceq *iwceq) in irdma_destroy_ceq() argument
641 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_destroy_ceq()
643 if (rf->reset) in irdma_destroy_ceq()
664 * @rf: RDMA PCI function
668 static void irdma_del_ceq_0(struct irdma_pci_f *rf) in irdma_del_ceq_0() argument
670 struct irdma_ceq *iwceq = rf->ceqlist; in irdma_del_ceq_0()
673 if (rf->msix_shared) { in irdma_del_ceq_0()
674 msix_vec = &rf->iw_msixtbl[0]; in irdma_del_ceq_0()
675 rf->sc_dev.irq_ops->irdma_cfg_ceq(&rf->sc_dev, in irdma_del_ceq_0()
678 irdma_destroy_irq(rf, msix_vec, rf); in irdma_del_ceq_0()
680 msix_vec = &rf->iw_msixtbl[1]; in irdma_del_ceq_0()
681 irdma_destroy_irq(rf, msix_vec, iwceq); in irdma_del_ceq_0()
684 irdma_destroy_ceq(rf, iwceq); in irdma_del_ceq_0()
685 rf->sc_dev.ceq_valid = false; in irdma_del_ceq_0()
686 rf->ceqs_count = 0; in irdma_del_ceq_0()
691 * @rf: RDMA PCI function
696 static void irdma_del_ceqs(struct irdma_pci_f *rf) in irdma_del_ceqs() argument
698 struct irdma_ceq *iwceq = &rf->ceqlist[1]; in irdma_del_ceqs()
702 if (rf->msix_shared) in irdma_del_ceqs()
703 msix_vec = &rf->iw_msixtbl[1]; in irdma_del_ceqs()
705 msix_vec = &rf->iw_msixtbl[2]; in irdma_del_ceqs()
707 for (i = 1; i < rf->ceqs_count; i++, msix_vec++, iwceq++) { in irdma_del_ceqs()
708 rf->sc_dev.irq_ops->irdma_cfg_ceq(&rf->sc_dev, msix_vec->ceq_id, in irdma_del_ceqs()
710 irdma_destroy_irq(rf, msix_vec, iwceq); in irdma_del_ceqs()
711 irdma_cqp_ceq_cmd(&rf->sc_dev, &iwceq->sc_ceq, in irdma_del_ceqs()
713 dma_free_coherent(rf->sc_dev.hw->device, iwceq->mem.size, in irdma_del_ceqs()
717 rf->ceqs_count = 1; in irdma_del_ceqs()
722 * @rf: RDMA PCI function
727 static void irdma_destroy_ccq(struct irdma_pci_f *rf) in irdma_destroy_ccq() argument
729 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_destroy_ccq()
730 struct irdma_ccq *ccq = &rf->ccq; in irdma_destroy_ccq()
733 if (!rf->reset) in irdma_destroy_ccq()
803 * @rf: RDMA PCI function
811 irdma_create_hmc_objs(struct irdma_pci_f *rf, bool privileged, enum irdma_vers vers) in irdma_create_hmc_objs() argument
813 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_create_hmc_objs()
820 info.entry_type = rf->sd_type; in irdma_create_hmc_objs()
857 * @rf: RDMA PCI function
867 irdma_obj_aligned_mem(struct irdma_pci_f *rf, struct irdma_dma_mem *memptr, in irdma_obj_aligned_mem() argument
873 va = (unsigned long)rf->obj_next.va; in irdma_obj_aligned_mem()
879 memptr->pa = rf->obj_next.pa + extra; in irdma_obj_aligned_mem()
881 if (((u8 *)memptr->va + size) > ((u8 *)rf->obj_mem.va + rf->obj_mem.size)) in irdma_obj_aligned_mem()
884 rf->obj_next.va = (u8 *)memptr->va + size; in irdma_obj_aligned_mem()
885 rf->obj_next.pa = memptr->pa + size; in irdma_obj_aligned_mem()
892 * @rf: RDMA PCI function
897 static enum irdma_status_code irdma_create_cqp(struct irdma_pci_f *rf) in irdma_create_cqp() argument
902 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_create_cqp()
904 struct irdma_cqp *cqp = &rf->cqp; in irdma_create_cqp()
930 status = irdma_obj_aligned_mem(rf, &mem, sizeof(struct irdma_cqp_ctx), in irdma_create_cqp()
944 cqp_init_info.hmc_profile = rf->rsrc_profile; in irdma_create_cqp()
946 cqp_init_info.protocol_used = rf->protocol_used; in irdma_create_cqp()
948 switch (rf->rdma_ver) { in irdma_create_cqp()
985 irdma_destroy_cqp(rf, false); in irdma_create_cqp()
992 * @rf: RDMA PCI function
997 static enum irdma_status_code irdma_create_ccq(struct irdma_pci_f *rf) in irdma_create_ccq() argument
999 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_create_ccq()
1002 struct irdma_ccq *ccq = &rf->ccq; in irdma_create_ccq()
1015 status = irdma_obj_aligned_mem(rf, &ccq->shadow_area, in irdma_create_ccq()
1031 info.vsi = &rf->default_vsi; in irdma_create_ccq()
1056 status = irdma_alloc_local_mac_entry(iwdev->rf, in irdma_alloc_set_mac()
1059 status = irdma_add_local_mac_entry(iwdev->rf, in irdma_alloc_set_mac()
1063 irdma_del_local_mac_entry(iwdev->rf, in irdma_alloc_set_mac()
1072 * @rf: RDMA PCI function
1081 irdma_cfg_ceq_vector(struct irdma_pci_f *rf, struct irdma_ceq *iwceq, in irdma_cfg_ceq_vector() argument
1086 if (rf->msix_shared && !ceq_id) { in irdma_cfg_ceq_vector()
1087 tasklet_setup(&rf->dpc_tasklet, irdma_dpc); in irdma_cfg_ceq_vector()
1089 "AEQCEQ", rf); in irdma_cfg_ceq_vector()
1100 ibdev_dbg(&rf->iwdev->ibdev, "ERR: ceq irq config fail\n"); in irdma_cfg_ceq_vector()
1105 rf->sc_dev.irq_ops->irdma_cfg_ceq(&rf->sc_dev, ceq_id, msix_vec->idx, true); in irdma_cfg_ceq_vector()
1112 * @rf: RDMA PCI function
1117 static enum irdma_status_code irdma_cfg_aeq_vector(struct irdma_pci_f *rf) in irdma_cfg_aeq_vector() argument
1119 struct irdma_msix_vector *msix_vec = rf->iw_msixtbl; in irdma_cfg_aeq_vector()
1122 if (!rf->msix_shared) { in irdma_cfg_aeq_vector()
1123 tasklet_setup(&rf->dpc_tasklet, irdma_dpc); in irdma_cfg_aeq_vector()
1125 "irdma", rf); in irdma_cfg_aeq_vector()
1128 ibdev_dbg(&rf->iwdev->ibdev, "ERR: aeq irq config fail\n"); in irdma_cfg_aeq_vector()
1132 rf->sc_dev.irq_ops->irdma_cfg_aeq(&rf->sc_dev, msix_vec->idx, true); in irdma_cfg_aeq_vector()
1139 * @rf: RDMA PCI function
1147 static enum irdma_status_code irdma_create_ceq(struct irdma_pci_f *rf, in irdma_create_ceq() argument
1154 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_create_ceq()
1159 iwceq->rf = rf; in irdma_create_ceq()
1160 ceq_size = min(rf->sc_dev.hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt, in irdma_create_ceq()
1176 scratch = (uintptr_t)&rf->cqp.sc_cqp; in irdma_create_ceq()
1180 status = irdma_cqp_ceq_cmd(&rf->sc_dev, &iwceq->sc_ceq, in irdma_create_ceq()
1197 * @rf: RDMA PCI function
1203 static enum irdma_status_code irdma_setup_ceq_0(struct irdma_pci_f *rf) in irdma_setup_ceq_0() argument
1211 num_ceqs = min(rf->msix_count, rf->sc_dev.hmc_fpm_misc.max_ceqs); in irdma_setup_ceq_0()
1212 rf->ceqlist = kcalloc(num_ceqs, sizeof(*rf->ceqlist), GFP_KERNEL); in irdma_setup_ceq_0()
1213 if (!rf->ceqlist) { in irdma_setup_ceq_0()
1218 iwceq = &rf->ceqlist[0]; in irdma_setup_ceq_0()
1219 status = irdma_create_ceq(rf, iwceq, 0, &rf->default_vsi); in irdma_setup_ceq_0()
1221 ibdev_dbg(&rf->iwdev->ibdev, "ERR: create ceq status = %d\n", in irdma_setup_ceq_0()
1227 i = rf->msix_shared ? 0 : 1; in irdma_setup_ceq_0()
1228 msix_vec = &rf->iw_msixtbl[i]; in irdma_setup_ceq_0()
1231 status = irdma_cfg_ceq_vector(rf, iwceq, 0, msix_vec); in irdma_setup_ceq_0()
1233 irdma_destroy_ceq(rf, iwceq); in irdma_setup_ceq_0()
1237 irdma_ena_intr(&rf->sc_dev, msix_vec->idx); in irdma_setup_ceq_0()
1238 rf->ceqs_count++; in irdma_setup_ceq_0()
1241 if (status && !rf->ceqs_count) { in irdma_setup_ceq_0()
1242 kfree(rf->ceqlist); in irdma_setup_ceq_0()
1243 rf->ceqlist = NULL; in irdma_setup_ceq_0()
1246 rf->sc_dev.ceq_valid = true; in irdma_setup_ceq_0()
1253 * @rf: RDMA PCI function
1260 static enum irdma_status_code irdma_setup_ceqs(struct irdma_pci_f *rf, in irdma_setup_ceqs() argument
1270 num_ceqs = min(rf->msix_count, rf->sc_dev.hmc_fpm_misc.max_ceqs); in irdma_setup_ceqs()
1271 i = (rf->msix_shared) ? 1 : 2; in irdma_setup_ceqs()
1273 iwceq = &rf->ceqlist[ceq_id]; in irdma_setup_ceqs()
1274 status = irdma_create_ceq(rf, iwceq, ceq_id, vsi); in irdma_setup_ceqs()
1276 ibdev_dbg(&rf->iwdev->ibdev, in irdma_setup_ceqs()
1281 msix_vec = &rf->iw_msixtbl[i]; in irdma_setup_ceqs()
1284 status = irdma_cfg_ceq_vector(rf, iwceq, ceq_id, msix_vec); in irdma_setup_ceqs()
1286 irdma_destroy_ceq(rf, iwceq); in irdma_setup_ceqs()
1289 irdma_ena_intr(&rf->sc_dev, msix_vec->idx); in irdma_setup_ceqs()
1290 rf->ceqs_count++; in irdma_setup_ceqs()
1296 irdma_del_ceqs(rf); in irdma_setup_ceqs()
1301 static enum irdma_status_code irdma_create_virt_aeq(struct irdma_pci_f *rf, in irdma_create_virt_aeq() argument
1305 struct irdma_aeq *aeq = &rf->aeq; in irdma_create_virt_aeq()
1309 if (rf->rdma_ver < IRDMA_GEN_2) in irdma_create_virt_aeq()
1319 status = irdma_get_pble(rf->pble_rsrc, &aeq->palloc, pg_cnt, true); in irdma_create_virt_aeq()
1326 status = irdma_map_vm_page_list(&rf->hw, aeq->mem.va, pg_arr, pg_cnt); in irdma_create_virt_aeq()
1328 irdma_free_pble(rf->pble_rsrc, &aeq->palloc); in irdma_create_virt_aeq()
1338 * @rf: RDMA PCI function
1343 static enum irdma_status_code irdma_create_aeq(struct irdma_pci_f *rf) in irdma_create_aeq() argument
1347 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_create_aeq()
1348 struct irdma_aeq *aeq = &rf->aeq; in irdma_create_aeq()
1349 struct irdma_hmc_info *hmc_info = rf->sc_dev.hmc_info; in irdma_create_aeq()
1351 u8 multiplier = (rf->protocol_used == IRDMA_IWARP_PROTOCOL_ONLY) ? 2 : 1; in irdma_create_aeq()
1366 status = irdma_create_virt_aeq(rf, aeq_size); in irdma_create_aeq()
1380 info.msix_idx = rf->iw_msixtbl->idx; in irdma_create_aeq()
1393 irdma_destroy_virt_aeq(rf); in irdma_create_aeq()
1405 * @rf: RDMA PCI function
1410 static enum irdma_status_code irdma_setup_aeq(struct irdma_pci_f *rf) in irdma_setup_aeq() argument
1412 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_setup_aeq()
1415 status = irdma_create_aeq(rf); in irdma_setup_aeq()
1419 status = irdma_cfg_aeq_vector(rf); in irdma_setup_aeq()
1421 irdma_destroy_aeq(rf); in irdma_setup_aeq()
1425 if (!rf->msix_shared) in irdma_setup_aeq()
1426 irdma_ena_intr(dev, rf->iw_msixtbl[0].idx); in irdma_setup_aeq()
1448 info.sq_size = min(iwdev->rf->max_qp / 2, (u32)32768); in irdma_initialize_ilq()
1478 info.sq_size = min(iwdev->rf->max_qp / 2, (u32)32768); in irdma_initialize_ieq()
1496 struct irdma_pci_f *rf = iwdev->rf; in irdma_reinitialize_ieq() local
1500 iwdev->rf->reset = true; in irdma_reinitialize_ieq()
1501 rf->gen_ops.request_reset(rf); in irdma_reinitialize_ieq()
1507 * @rf: RDMA PCI function
1513 static enum irdma_status_code irdma_hmc_setup(struct irdma_pci_f *rf) in irdma_hmc_setup() argument
1518 if (rf->rdma_ver == IRDMA_GEN_1) in irdma_hmc_setup()
1519 qpcnt = rsrc_limits_table[rf->limits_sel].qplimit * 2; in irdma_hmc_setup()
1521 qpcnt = rsrc_limits_table[rf->limits_sel].qplimit; in irdma_hmc_setup()
1523 rf->sd_type = IRDMA_SD_TYPE_DIRECT; in irdma_hmc_setup()
1524 status = irdma_cfg_fpm_val(&rf->sc_dev, qpcnt); in irdma_hmc_setup()
1528 status = irdma_create_hmc_objs(rf, true, rf->rdma_ver); in irdma_hmc_setup()
1535 * @rf: RDMA PCI function
1537 static void irdma_del_init_mem(struct irdma_pci_f *rf) in irdma_del_init_mem() argument
1539 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_del_init_mem()
1543 kfree(rf->mem_rsrc); in irdma_del_init_mem()
1544 rf->mem_rsrc = NULL; in irdma_del_init_mem()
1545 dma_free_coherent(rf->hw.device, rf->obj_mem.size, rf->obj_mem.va, in irdma_del_init_mem()
1546 rf->obj_mem.pa); in irdma_del_init_mem()
1547 rf->obj_mem.va = NULL; in irdma_del_init_mem()
1548 if (rf->rdma_ver != IRDMA_GEN_1) { in irdma_del_init_mem()
1549 kfree(rf->allocated_ws_nodes); in irdma_del_init_mem()
1550 rf->allocated_ws_nodes = NULL; in irdma_del_init_mem()
1552 kfree(rf->ceqlist); in irdma_del_init_mem()
1553 rf->ceqlist = NULL; in irdma_del_init_mem()
1554 kfree(rf->iw_msixtbl); in irdma_del_init_mem()
1555 rf->iw_msixtbl = NULL; in irdma_del_init_mem()
1556 kfree(rf->hmc_info_mem); in irdma_del_init_mem()
1557 rf->hmc_info_mem = NULL; in irdma_del_init_mem()
1562 * @rf: RDMA PCI function
1568 static enum irdma_status_code irdma_initialize_dev(struct irdma_pci_f *rf) in irdma_initialize_dev() argument
1571 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_initialize_dev()
1580 rf->hmc_info_mem = kzalloc(size, GFP_KERNEL); in irdma_initialize_dev()
1581 if (!rf->hmc_info_mem) in irdma_initialize_dev()
1584 rf->pble_rsrc = (struct irdma_hmc_pble_rsrc *)rf->hmc_info_mem; in irdma_initialize_dev()
1585 dev->hmc_info = &rf->hw.hmc; in irdma_initialize_dev()
1587 (rf->pble_rsrc + 1); in irdma_initialize_dev()
1589 status = irdma_obj_aligned_mem(rf, &mem, IRDMA_QUERY_FPM_BUF_SIZE, in irdma_initialize_dev()
1597 status = irdma_obj_aligned_mem(rf, &mem, IRDMA_COMMIT_FPM_BUF_SIZE, in irdma_initialize_dev()
1605 info.bar0 = rf->hw.hw_addr; in irdma_initialize_dev()
1606 info.hmc_fn_id = PCI_FUNC(rf->pcidev->devfn); in irdma_initialize_dev()
1607 info.hw = &rf->hw; in irdma_initialize_dev()
1608 status = irdma_sc_dev_init(rf->rdma_ver, &rf->sc_dev, &info); in irdma_initialize_dev()
1614 kfree(rf->hmc_info_mem); in irdma_initialize_dev()
1615 rf->hmc_info_mem = NULL; in irdma_initialize_dev()
1633 if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) in irdma_rt_deinit_hw()
1634 irdma_del_local_mac_entry(iwdev->rf, in irdma_rt_deinit_hw()
1643 iwdev->rf->reset); in irdma_rt_deinit_hw()
1649 iwdev->rf->reset); in irdma_rt_deinit_hw()
1665 static enum irdma_status_code irdma_setup_init_state(struct irdma_pci_f *rf) in irdma_setup_init_state() argument
1669 status = irdma_save_msix_info(rf); in irdma_setup_init_state()
1673 rf->hw.device = &rf->pcidev->dev; in irdma_setup_init_state()
1674 rf->obj_mem.size = ALIGN(8192, IRDMA_HW_PAGE_SIZE); in irdma_setup_init_state()
1675 rf->obj_mem.va = dma_alloc_coherent(rf->hw.device, rf->obj_mem.size, in irdma_setup_init_state()
1676 &rf->obj_mem.pa, GFP_KERNEL); in irdma_setup_init_state()
1677 if (!rf->obj_mem.va) { in irdma_setup_init_state()
1682 rf->obj_next = rf->obj_mem; in irdma_setup_init_state()
1683 status = irdma_initialize_dev(rf); in irdma_setup_init_state()
1690 dma_free_coherent(rf->hw.device, rf->obj_mem.size, rf->obj_mem.va, in irdma_setup_init_state()
1691 rf->obj_mem.pa); in irdma_setup_init_state()
1692 rf->obj_mem.va = NULL; in irdma_setup_init_state()
1694 kfree(rf->iw_msixtbl); in irdma_setup_init_state()
1695 rf->iw_msixtbl = NULL; in irdma_setup_init_state()
1707 iwdev->rf->used_pds = find_next_zero_bit(iwdev->rf->allocated_pds, in irdma_get_used_rsrc()
1708 iwdev->rf->max_pd, 0); in irdma_get_used_rsrc()
1709 iwdev->rf->used_qps = find_next_zero_bit(iwdev->rf->allocated_qps, in irdma_get_used_rsrc()
1710 iwdev->rf->max_qp, 0); in irdma_get_used_rsrc()
1711 iwdev->rf->used_cqs = find_next_zero_bit(iwdev->rf->allocated_cqs, in irdma_get_used_rsrc()
1712 iwdev->rf->max_cq, 0); in irdma_get_used_rsrc()
1713 iwdev->rf->used_mrs = find_next_zero_bit(iwdev->rf->allocated_mrs, in irdma_get_used_rsrc()
1714 iwdev->rf->max_mr, 0); in irdma_get_used_rsrc()
1717 void irdma_ctrl_deinit_hw(struct irdma_pci_f *rf) in irdma_ctrl_deinit_hw() argument
1719 enum init_completion_state state = rf->init_state; in irdma_ctrl_deinit_hw()
1721 rf->init_state = INVALID_STATE; in irdma_ctrl_deinit_hw()
1722 if (rf->rsrc_created) { in irdma_ctrl_deinit_hw()
1723 irdma_destroy_aeq(rf); in irdma_ctrl_deinit_hw()
1724 irdma_destroy_pble_prm(rf->pble_rsrc); in irdma_ctrl_deinit_hw()
1725 irdma_del_ceqs(rf); in irdma_ctrl_deinit_hw()
1726 rf->rsrc_created = false; in irdma_ctrl_deinit_hw()
1730 irdma_del_ceq_0(rf); in irdma_ctrl_deinit_hw()
1733 irdma_destroy_ccq(rf); in irdma_ctrl_deinit_hw()
1737 irdma_del_hmc_objects(&rf->sc_dev, rf->sc_dev.hmc_info, true, in irdma_ctrl_deinit_hw()
1738 rf->reset, rf->rdma_ver); in irdma_ctrl_deinit_hw()
1741 irdma_destroy_cqp(rf, true); in irdma_ctrl_deinit_hw()
1744 irdma_del_init_mem(rf); in irdma_ctrl_deinit_hw()
1748 ibdev_warn(&rf->iwdev->ibdev, "bad init_state = %d\n", rf->init_state); in irdma_ctrl_deinit_hw()
1764 struct irdma_pci_f *rf = iwdev->rf; in irdma_rt_init_hw() local
1765 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_rt_init_hw()
1774 vsi_info.register_qset = rf->gen_ops.register_qset; in irdma_rt_init_hw()
1775 vsi_info.unregister_qset = rf->gen_ops.unregister_qset; in irdma_rt_init_hw()
1779 status = irdma_setup_cm_core(iwdev, rf->rdma_ver); in irdma_rt_init_hw()
1807 if (!rf->rsrc_created) { in irdma_rt_init_hw()
1808 status = irdma_setup_ceqs(rf, &iwdev->vsi); in irdma_rt_init_hw()
1814 status = irdma_hmc_init_pble(&rf->sc_dev, in irdma_rt_init_hw()
1815 rf->pble_rsrc); in irdma_rt_init_hw()
1817 irdma_del_ceqs(rf); in irdma_rt_init_hw()
1823 status = irdma_setup_aeq(rf); in irdma_rt_init_hw()
1825 irdma_destroy_pble_prm(rf->pble_rsrc); in irdma_rt_init_hw()
1826 irdma_del_ceqs(rf); in irdma_rt_init_hw()
1830 rf->rsrc_created = true; in irdma_rt_init_hw()
1837 if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) in irdma_rt_init_hw()
1855 dev_err(&rf->pcidev->dev, "HW runtime init FAIL status = %d last cmpl = %d\n", in irdma_rt_init_hw()
1864 * @rf: RDMA PCI function
1866 * Create admin queues, HMC obejcts and RF resource objects
1868 enum irdma_status_code irdma_ctrl_init_hw(struct irdma_pci_f *rf) in irdma_ctrl_init_hw() argument
1870 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_ctrl_init_hw()
1873 status = irdma_setup_init_state(rf); in irdma_ctrl_init_hw()
1876 rf->init_state = INITIAL_STATE; in irdma_ctrl_init_hw()
1878 status = irdma_create_cqp(rf); in irdma_ctrl_init_hw()
1881 rf->init_state = CQP_CREATED; in irdma_ctrl_init_hw()
1883 status = irdma_hmc_setup(rf); in irdma_ctrl_init_hw()
1886 rf->init_state = HMC_OBJS_CREATED; in irdma_ctrl_init_hw()
1888 status = irdma_initialize_hw_rsrc(rf); in irdma_ctrl_init_hw()
1891 rf->init_state = HW_RSRC_INITIALIZED; in irdma_ctrl_init_hw()
1893 status = irdma_create_ccq(rf); in irdma_ctrl_init_hw()
1896 rf->init_state = CCQ_CREATED; in irdma_ctrl_init_hw()
1899 if (rf->rdma_ver != IRDMA_GEN_1) { in irdma_ctrl_init_hw()
1905 status = irdma_setup_ceq_0(rf); in irdma_ctrl_init_hw()
1908 rf->init_state = CEQ0_CREATED; in irdma_ctrl_init_hw()
1910 rf->cqp_cmpl_wq = alloc_ordered_workqueue("cqp_cmpl_wq", in irdma_ctrl_init_hw()
1912 if (!rf->cqp_cmpl_wq) { in irdma_ctrl_init_hw()
1916 INIT_WORK(&rf->cqp_cmpl_work, cqp_compl_worker); in irdma_ctrl_init_hw()
1921 dev_err(&rf->pcidev->dev, "IRDMA hardware initialization FAILED init_state=%d status=%d\n", in irdma_ctrl_init_hw()
1922 rf->init_state, status); in irdma_ctrl_init_hw()
1923 irdma_ctrl_deinit_hw(rf); in irdma_ctrl_init_hw()
1929 * @rf: RDMA PCI function
1931 static void irdma_set_hw_rsrc(struct irdma_pci_f *rf) in irdma_set_hw_rsrc() argument
1933 rf->allocated_qps = (void *)(rf->mem_rsrc + in irdma_set_hw_rsrc()
1934 (sizeof(struct irdma_arp_entry) * rf->arp_table_size)); in irdma_set_hw_rsrc()
1935 rf->allocated_cqs = &rf->allocated_qps[BITS_TO_LONGS(rf->max_qp)]; in irdma_set_hw_rsrc()
1936 rf->allocated_mrs = &rf->allocated_cqs[BITS_TO_LONGS(rf->max_cq)]; in irdma_set_hw_rsrc()
1937 rf->allocated_pds = &rf->allocated_mrs[BITS_TO_LONGS(rf->max_mr)]; in irdma_set_hw_rsrc()
1938 rf->allocated_ahs = &rf->allocated_pds[BITS_TO_LONGS(rf->max_pd)]; in irdma_set_hw_rsrc()
1939 rf->allocated_mcgs = &rf->allocated_ahs[BITS_TO_LONGS(rf->max_ah)]; in irdma_set_hw_rsrc()
1940 rf->allocated_arps = &rf->allocated_mcgs[BITS_TO_LONGS(rf->max_mcg)]; in irdma_set_hw_rsrc()
1941 rf->qp_table = (struct irdma_qp **) in irdma_set_hw_rsrc()
1942 (&rf->allocated_arps[BITS_TO_LONGS(rf->arp_table_size)]); in irdma_set_hw_rsrc()
1944 spin_lock_init(&rf->rsrc_lock); in irdma_set_hw_rsrc()
1945 spin_lock_init(&rf->arp_lock); in irdma_set_hw_rsrc()
1946 spin_lock_init(&rf->qptable_lock); in irdma_set_hw_rsrc()
1947 spin_lock_init(&rf->qh_list_lock); in irdma_set_hw_rsrc()
1952 * @rf: RDMA PCI function
1954 static u32 irdma_calc_mem_rsrc_size(struct irdma_pci_f *rf) in irdma_calc_mem_rsrc_size() argument
1958 rsrc_size = sizeof(struct irdma_arp_entry) * rf->arp_table_size; in irdma_calc_mem_rsrc_size()
1959 rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(rf->max_qp); in irdma_calc_mem_rsrc_size()
1960 rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(rf->max_mr); in irdma_calc_mem_rsrc_size()
1961 rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(rf->max_cq); in irdma_calc_mem_rsrc_size()
1962 rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(rf->max_pd); in irdma_calc_mem_rsrc_size()
1963 rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(rf->arp_table_size); in irdma_calc_mem_rsrc_size()
1964 rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(rf->max_ah); in irdma_calc_mem_rsrc_size()
1965 rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(rf->max_mcg); in irdma_calc_mem_rsrc_size()
1966 rsrc_size += sizeof(struct irdma_qp **) * rf->max_qp; in irdma_calc_mem_rsrc_size()
1973 * @rf: RDMA PCI function
1975 u32 irdma_initialize_hw_rsrc(struct irdma_pci_f *rf) in irdma_initialize_hw_rsrc() argument
1981 if (rf->rdma_ver != IRDMA_GEN_1) { in irdma_initialize_hw_rsrc()
1982 rf->allocated_ws_nodes = in irdma_initialize_hw_rsrc()
1985 if (!rf->allocated_ws_nodes) in irdma_initialize_hw_rsrc()
1988 set_bit(0, rf->allocated_ws_nodes); in irdma_initialize_hw_rsrc()
1989 rf->max_ws_node_id = IRDMA_MAX_WS_NODES; in irdma_initialize_hw_rsrc()
1991 rf->max_cqe = rf->sc_dev.hw_attrs.uk_attrs.max_hw_cq_size; in irdma_initialize_hw_rsrc()
1992 rf->max_qp = rf->sc_dev.hmc_info->hmc_obj[IRDMA_HMC_IW_QP].cnt; in irdma_initialize_hw_rsrc()
1993 rf->max_mr = rf->sc_dev.hmc_info->hmc_obj[IRDMA_HMC_IW_MR].cnt; in irdma_initialize_hw_rsrc()
1994 rf->max_cq = rf->sc_dev.hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt; in irdma_initialize_hw_rsrc()
1995 rf->max_pd = rf->sc_dev.hw_attrs.max_hw_pds; in irdma_initialize_hw_rsrc()
1996 rf->arp_table_size = rf->sc_dev.hmc_info->hmc_obj[IRDMA_HMC_IW_ARP].cnt; in irdma_initialize_hw_rsrc()
1997 rf->max_ah = rf->sc_dev.hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt; in irdma_initialize_hw_rsrc()
1998 rf->max_mcg = rf->max_qp; in irdma_initialize_hw_rsrc()
2000 rsrc_size = irdma_calc_mem_rsrc_size(rf); in irdma_initialize_hw_rsrc()
2001 rf->mem_rsrc = kzalloc(rsrc_size, GFP_KERNEL); in irdma_initialize_hw_rsrc()
2002 if (!rf->mem_rsrc) { in irdma_initialize_hw_rsrc()
2007 rf->arp_table = (struct irdma_arp_entry *)rf->mem_rsrc; in irdma_initialize_hw_rsrc()
2009 irdma_set_hw_rsrc(rf); in irdma_initialize_hw_rsrc()
2011 set_bit(0, rf->allocated_mrs); in irdma_initialize_hw_rsrc()
2012 set_bit(0, rf->allocated_qps); in irdma_initialize_hw_rsrc()
2013 set_bit(0, rf->allocated_cqs); in irdma_initialize_hw_rsrc()
2014 set_bit(0, rf->allocated_pds); in irdma_initialize_hw_rsrc()
2015 set_bit(0, rf->allocated_arps); in irdma_initialize_hw_rsrc()
2016 set_bit(0, rf->allocated_ahs); in irdma_initialize_hw_rsrc()
2017 set_bit(0, rf->allocated_mcgs); in irdma_initialize_hw_rsrc()
2018 set_bit(2, rf->allocated_qps); /* qp 2 IEQ */ in irdma_initialize_hw_rsrc()
2019 set_bit(1, rf->allocated_qps); /* qp 1 ILQ */ in irdma_initialize_hw_rsrc()
2020 set_bit(1, rf->allocated_cqs); in irdma_initialize_hw_rsrc()
2021 set_bit(1, rf->allocated_pds); in irdma_initialize_hw_rsrc()
2022 set_bit(2, rf->allocated_cqs); in irdma_initialize_hw_rsrc()
2023 set_bit(2, rf->allocated_pds); in irdma_initialize_hw_rsrc()
2025 INIT_LIST_HEAD(&rf->mc_qht_list.list); in irdma_initialize_hw_rsrc()
2027 mrdrvbits = 24 - max(get_count_order(rf->max_mr), 14); in irdma_initialize_hw_rsrc()
2028 rf->mr_stagmask = ~(((1 << mrdrvbits) - 1) << (32 - mrdrvbits)); in irdma_initialize_hw_rsrc()
2033 kfree(rf->allocated_ws_nodes); in irdma_initialize_hw_rsrc()
2034 rf->allocated_ws_nodes = NULL; in irdma_initialize_hw_rsrc()
2041 * @rf: RDMA PCI function
2044 void irdma_cqp_ce_handler(struct irdma_pci_f *rf, struct irdma_sc_cq *cq) in irdma_cqp_ce_handler() argument
2047 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_cqp_ce_handler()
2055 spin_lock_irqsave(&rf->cqp.compl_lock, flags); in irdma_cqp_ce_handler()
2057 spin_unlock_irqrestore(&rf->cqp.compl_lock, flags); in irdma_cqp_ce_handler()
2066 ibdev_err(&rf->iwdev->ibdev, "cqp opcode = 0x%x maj_err_code = 0x%x min_err_code = 0x%x\n", in irdma_cqp_ce_handler()
2077 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_cqp_ce_handler()
2081 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_cqp_ce_handler()
2100 struct irdma_pci_f *rf = container_of(work, struct irdma_pci_f, in cqp_compl_worker() local
2102 struct irdma_sc_cq *cq = &rf->ccq.sc_cq; in cqp_compl_worker()
2104 irdma_cqp_ce_handler(rf, cq); in cqp_compl_worker()
2162 * @rf: RDMA PCI function
2165 void irdma_del_local_mac_entry(struct irdma_pci_f *rf, u16 idx) in irdma_del_local_mac_entry() argument
2167 struct irdma_cqp *iwcqp = &rf->cqp; in irdma_del_local_mac_entry()
2183 irdma_handle_cqp_op(rf, cqp_request); in irdma_del_local_mac_entry()
2190 * @rf: RDMA PCI function
2194 int irdma_add_local_mac_entry(struct irdma_pci_f *rf, u8 *mac_addr, u16 idx) in irdma_add_local_mac_entry() argument
2197 struct irdma_cqp *iwcqp = &rf->cqp; in irdma_add_local_mac_entry()
2216 status = irdma_handle_cqp_op(rf, cqp_request); in irdma_add_local_mac_entry()
2224 * @rf: RDMA PCI function
2231 int irdma_alloc_local_mac_entry(struct irdma_pci_f *rf, u16 *mac_tbl_idx) in irdma_alloc_local_mac_entry() argument
2233 struct irdma_cqp *iwcqp = &rf->cqp; in irdma_alloc_local_mac_entry()
2247 status = irdma_handle_cqp_op(rf, cqp_request); in irdma_alloc_local_mac_entry()
2271 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, add_port); in irdma_cqp_manage_apbvt_cmd()
2282 cqp_info->in.u.manage_apbvt_entry.cqp = &iwdev->rf->cqp.sc_cqp; in irdma_cqp_manage_apbvt_cmd()
2287 status = irdma_handle_cqp_op(iwdev->rf, cqp_request); in irdma_cqp_manage_apbvt_cmd()
2288 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); in irdma_cqp_manage_apbvt_cmd()
2359 * @rf: RDMA PCI function
2365 void irdma_manage_arp_cache(struct irdma_pci_f *rf, unsigned char *mac_addr, in irdma_manage_arp_cache() argument
2373 arp_index = irdma_arp_table(rf, ip_addr, ipv4, mac_addr, action); in irdma_manage_arp_cache()
2377 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false); in irdma_manage_arp_cache()
2391 cqp_info->in.u.add_arp_cache_entry.cqp = &rf->cqp.sc_cqp; in irdma_manage_arp_cache()
2396 cqp_info->in.u.del_arp_cache_entry.cqp = &rf->cqp.sc_cqp; in irdma_manage_arp_cache()
2401 irdma_handle_cqp_op(rf, cqp_request); in irdma_manage_arp_cache()
2402 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_manage_arp_cache()
2434 struct irdma_cqp *iwcqp = &iwdev->rf->cqp; in irdma_manage_qhash()
2498 cqp_info->in.u.manage_qhash_table_entry.cqp = &iwdev->rf->cqp.sc_cqp; in irdma_manage_qhash()
2502 status = irdma_handle_cqp_op(iwdev->rf, cqp_request); in irdma_manage_qhash()
2550 * @rf: RDMA PCI function
2555 enum irdma_status_code irdma_hw_flush_wqes(struct irdma_pci_f *rf, in irdma_hw_flush_wqes() argument
2566 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait); in irdma_hw_flush_wqes()
2579 status = irdma_handle_cqp_op(rf, cqp_request); in irdma_hw_flush_wqes()
2583 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_hw_flush_wqes()
2614 new_req = irdma_alloc_and_get_cqp_request(&rf->cqp, true); in irdma_hw_flush_wqes()
2627 status = irdma_handle_cqp_op(rf, new_req); in irdma_hw_flush_wqes()
2636 irdma_put_cqp_request(&rf->cqp, new_req); in irdma_hw_flush_wqes()
2647 ibdev_dbg(&rf->iwdev->ibdev, in irdma_hw_flush_wqes()
2649 iwqp->ibqp.qp_num, rf->protocol_used, iwqp->iwarp_state, in irdma_hw_flush_wqes()
2654 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_hw_flush_wqes()
2661 * @rf: RDMA PCI function
2666 void irdma_gen_ae(struct irdma_pci_f *rf, struct irdma_sc_qp *qp, in irdma_gen_ae() argument
2673 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait); in irdma_gen_ae()
2685 irdma_handle_cqp_op(rf, cqp_request); in irdma_gen_ae()
2686 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_gen_ae()
2692 struct irdma_pci_f *rf = iwqp->iwdev->rf; in irdma_flush_wqes() local
2723 (void)irdma_hw_flush_wqes(rf, &iwqp->sc_qp, &info, in irdma_flush_wqes()