23 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_init_pf_regs()  local
27 	gc->db_page_size = mana_gd_r32(gc, GDMA_PF_REG_DB_PAGE_SIZE) & 0xFFFF;  in mana_gd_init_pf_regs()
28 	gc->db_page_base = gc->bar0_va +  in mana_gd_init_pf_regs()
29 				mana_gd_r64(gc, GDMA_PF_REG_DB_PAGE_OFF);  in mana_gd_init_pf_regs()
31 	sriov_base_off = mana_gd_r64(gc, GDMA_SRIOV_REG_CFG_BASE_OFF);  in mana_gd_init_pf_regs()
33 	sriov_base_va = gc->bar0_va + sriov_base_off;  in mana_gd_init_pf_regs()
34 	gc->shm_base = sriov_base_va +  in mana_gd_init_pf_regs()
35 			mana_gd_r64(gc, sriov_base_off + GDMA_PF_REG_SHM_OFF);  in mana_gd_init_pf_regs()
40 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_init_vf_regs()  local
42 	gc->db_page_size = mana_gd_r32(gc, GDMA_REG_DB_PAGE_SIZE) & 0xFFFF;  in mana_gd_init_vf_regs()
44 	gc->db_page_base = gc->bar0_va +  in mana_gd_init_vf_regs()
45 				mana_gd_r64(gc, GDMA_REG_DB_PAGE_OFFSET);  in mana_gd_init_vf_regs()
47 	gc->shm_base = gc->bar0_va + mana_gd_r64(gc, GDMA_REG_SHM_OFFSET);  in mana_gd_init_vf_regs()
52 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_init_registers()  local
54 	if (gc->is_pf)  in mana_gd_init_registers()
62 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_query_max_resources()  local
70 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);  in mana_gd_query_max_resources()
72 		dev_err(gc->dev, "Failed to query resource info: %d, 0x%x\n",  in mana_gd_query_max_resources()
77 	if (gc->num_msix_usable > resp.max_msix)  in mana_gd_query_max_resources()
78 		gc->num_msix_usable = resp.max_msix;  in mana_gd_query_max_resources()
80 	if (gc->num_msix_usable <= 1)  in mana_gd_query_max_resources()
83 	gc->max_num_queues = num_online_cpus();  in mana_gd_query_max_resources()
84 	if (gc->max_num_queues > MANA_MAX_NUM_QUEUES)  in mana_gd_query_max_resources()
85 		gc->max_num_queues = MANA_MAX_NUM_QUEUES;  in mana_gd_query_max_resources()
87 	if (gc->max_num_queues > resp.max_eq)  in mana_gd_query_max_resources()
88 		gc->max_num_queues = resp.max_eq;  in mana_gd_query_max_resources()
90 	if (gc->max_num_queues > resp.max_cq)  in mana_gd_query_max_resources()
91 		gc->max_num_queues = resp.max_cq;  in mana_gd_query_max_resources()
93 	if (gc->max_num_queues > resp.max_sq)  in mana_gd_query_max_resources()
94 		gc->max_num_queues = resp.max_sq;  in mana_gd_query_max_resources()
96 	if (gc->max_num_queues > resp.max_rq)  in mana_gd_query_max_resources()
97 		gc->max_num_queues = resp.max_rq;  in mana_gd_query_max_resources()
100 	if (gc->max_num_queues > gc->num_msix_usable - 1)  in mana_gd_query_max_resources()
101 		gc->max_num_queues = gc->num_msix_usable - 1;  in mana_gd_query_max_resources()
108 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_detect_devices()  local
119 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);  in mana_gd_detect_devices()
121 		dev_err(gc->dev, "Failed to detect devices: %d, 0x%x\n", err,  in mana_gd_detect_devices()
137 			gc->mana.gdma_context = gc;  in mana_gd_detect_devices()
138 			gc->mana.dev_id = dev;  in mana_gd_detect_devices()
142 	return gc->mana.dev_id.type == 0 ? -ENODEV : 0;  in mana_gd_detect_devices()
145 int mana_gd_send_request(struct gdma_context *gc, u32 req_len, const void *req,  in mana_gd_send_request()  argument
148 	struct hw_channel_context *hwc = gc->hwc.driver_data;  in mana_gd_send_request()
153 int mana_gd_alloc_memory(struct gdma_context *gc, unsigned int length,  in mana_gd_alloc_memory()  argument
162 	gmi->dev = gc->dev;  in mana_gd_alloc_memory()
180 static int mana_gd_create_hw_eq(struct gdma_context *gc,  in mana_gd_create_hw_eq()  argument
202 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);  in mana_gd_create_hw_eq()
204 		dev_err(gc->dev, "Failed to create queue: %d, 0x%x\n", err,  in mana_gd_create_hw_eq()
217 	struct gdma_context *gc = queue->gdma_dev->gdma_context;  in mana_gd_disable_queue()  local
232 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);  in mana_gd_disable_queue()
234 		dev_err(gc->dev, "Failed to disable queue: %d, 0x%x\n", err,  in mana_gd_disable_queue()
247 static void mana_gd_ring_doorbell(struct gdma_context *gc, u32 db_index,  in mana_gd_ring_doorbell()  argument
251 	void __iomem *addr = gc->db_page_base + gc->db_page_size * db_index;  in mana_gd_ring_doorbell()
297 void mana_gd_wq_ring_doorbell(struct gdma_context *gc, struct gdma_queue *queue)  in mana_gd_wq_ring_doorbell()  argument
299 	mana_gd_ring_doorbell(gc, queue->gdma_dev->doorbell, queue->type,  in mana_gd_wq_ring_doorbell()
305 	struct gdma_context *gc = cq->gdma_dev->gdma_context;  in mana_gd_ring_cq()  local
311 	mana_gd_ring_doorbell(gc, cq->gdma_dev->doorbell, cq->type, cq->id,  in mana_gd_ring_cq()
318 	struct gdma_context *gc = eq->gdma_dev->gdma_context;  in mana_gd_process_eqe()  local
334 		if (WARN_ON_ONCE(cq_id >= gc->max_num_cqs))  in mana_gd_process_eqe()
337 		cq = gc->cq_table[cq_id];  in mana_gd_process_eqe()
347 		gc->test_event_eq_id = eq->id;  in mana_gd_process_eqe()
348 		complete(&gc->eq_test_event);  in mana_gd_process_eqe()
373 	struct gdma_context *gc;  in mana_gd_process_eq_events()  local
378 	gc = eq->gdma_dev->gdma_context;  in mana_gd_process_eq_events()
396 			dev_err(gc->dev, "EQ %d: overflow detected\n", eq->id);  in mana_gd_process_eq_events()
412 	mana_gd_ring_doorbell(gc, eq->gdma_dev->doorbell, eq->type, eq->id,  in mana_gd_process_eq_events()
421 	struct gdma_context *gc;  in mana_gd_register_irq()  local
428 	gc = gd->gdma_context;  in mana_gd_register_irq()
429 	r = &gc->msix_resource;  in mana_gd_register_irq()
430 	dev = gc->dev;  in mana_gd_register_irq()
435 	if (msi_index >= r->size || msi_index >= gc->num_msix_usable) {  in mana_gd_register_irq()
446 			err, msi_index, r->size, gc->num_msix_usable);  in mana_gd_register_irq()
451 	gic = &gc->irq_contexts[msi_index];  in mana_gd_register_irq()
466 	struct gdma_context *gc;  in mana_gd_deregiser_irq()  local
471 	gc = gd->gdma_context;  in mana_gd_deregiser_irq()
472 	r = &gc->msix_resource;  in mana_gd_deregiser_irq()
476 	if (WARN_ON(msix_index >= gc->num_msix_usable))  in mana_gd_deregiser_irq()
479 	gic = &gc->irq_contexts[msix_index];  in mana_gd_deregiser_irq()
490 int mana_gd_test_eq(struct gdma_context *gc, struct gdma_queue *eq)  in mana_gd_test_eq()  argument
494 	struct device *dev = gc->dev;  in mana_gd_test_eq()
497 	mutex_lock(&gc->eq_test_event_mutex);  in mana_gd_test_eq()
499 	init_completion(&gc->eq_test_event);  in mana_gd_test_eq()
500 	gc->test_event_eq_id = INVALID_QUEUE_ID;  in mana_gd_test_eq()
508 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);  in mana_gd_test_eq()
521 	if (!wait_for_completion_timeout(&gc->eq_test_event, 30 * HZ)) {  in mana_gd_test_eq()
526 	if (eq->id != gc->test_event_eq_id) {  in mana_gd_test_eq()
528 			gc->test_event_eq_id, eq->id);  in mana_gd_test_eq()
534 	mutex_unlock(&gc->eq_test_event_mutex);  in mana_gd_test_eq()
538 static void mana_gd_destroy_eq(struct gdma_context *gc, bool flush_evenets,  in mana_gd_destroy_eq()  argument
544 		err = mana_gd_test_eq(gc, queue);  in mana_gd_destroy_eq()
546 			dev_warn(gc->dev, "Failed to flush EQ: %d\n", err);  in mana_gd_destroy_eq()
559 	struct gdma_context *gc = gd->gdma_context;  in mana_gd_create_eq()  local
560 	struct device *dev = gc->dev;  in mana_gd_create_eq()
586 		err = mana_gd_create_hw_eq(gc, queue);  in mana_gd_create_eq()
590 		err = mana_gd_test_eq(gc, queue);  in mana_gd_create_eq()
598 	mana_gd_destroy_eq(gc, false, queue);  in mana_gd_create_eq()
613 static void mana_gd_destroy_cq(struct gdma_context *gc,  in mana_gd_destroy_cq()  argument
618 	if (id >= gc->max_num_cqs)  in mana_gd_destroy_cq()
621 	if (!gc->cq_table[id])  in mana_gd_destroy_cq()
624 	gc->cq_table[id] = NULL;  in mana_gd_destroy_cq()
631 	struct gdma_context *gc = gd->gdma_context;  in mana_gd_create_hwc_queue()  local
641 	err = mana_gd_alloc_memory(gc, spec->queue_size, gmi);  in mana_gd_create_hwc_queue()
670 static void mana_gd_destroy_dma_region(struct gdma_context *gc, u64 gdma_region)  in mana_gd_destroy_dma_region()  argument
683 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);  in mana_gd_destroy_dma_region()
685 		dev_err(gc->dev, "Failed to destroy DMA region: %d, 0x%x\n",  in mana_gd_destroy_dma_region()
695 	struct gdma_context *gc = gd->gdma_context;  in mana_gd_create_dma_region()  local
708 	hwc = gc->hwc.driver_data;  in mana_gd_create_dma_region()
728 	err = mana_gd_send_request(gc, req_msg_size, req, sizeof(resp), &resp);  in mana_gd_create_dma_region()
733 		dev_err(gc->dev, "Failed to create DMA region: 0x%x\n",  in mana_gd_create_dma_region()
749 	struct gdma_context *gc = gd->gdma_context;  in mana_gd_create_mana_eq()  local
762 	err = mana_gd_alloc_memory(gc, spec->queue_size, gmi);  in mana_gd_create_mana_eq()
795 	struct gdma_context *gc = gd->gdma_context;  in mana_gd_create_mana_wq_cq()  local
809 	err = mana_gd_alloc_memory(gc, spec->queue_size, gmi);  in mana_gd_create_mana_wq_cq()
837 void mana_gd_destroy_queue(struct gdma_context *gc, struct gdma_queue *queue)  in mana_gd_destroy_queue()  argument
843 		mana_gd_destroy_eq(gc, queue->eq.disable_needed, queue);  in mana_gd_destroy_queue()
847 		mana_gd_destroy_cq(gc, queue);  in mana_gd_destroy_queue()
857 		dev_err(gc->dev, "Can't destroy unknown queue: type=%d\n",  in mana_gd_destroy_queue()
862 	mana_gd_destroy_dma_region(gc, gmi->gdma_region);  in mana_gd_destroy_queue()
869 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_verify_vf_version()  local
894 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);  in mana_gd_verify_vf_version()
896 		dev_err(gc->dev, "VfVerifyVersionOutput: %d, status=0x%x\n",  in mana_gd_verify_vf_version()
906 	struct gdma_context *gc = gd->gdma_context;  in mana_gd_register_device()  local
920 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);  in mana_gd_register_device()
922 		dev_err(gc->dev, "gdma_register_device_resp failed: %d, 0x%x\n",  in mana_gd_register_device()
936 	struct gdma_context *gc = gd->gdma_context;  in mana_gd_deregister_device()  local
949 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);  in mana_gd_deregister_device()
951 		dev_err(gc->dev, "Failed to deregister device: %d, 0x%x\n",  in mana_gd_deregister_device()
1054 	struct gdma_context *gc;  in mana_gd_post_work_request()  local
1085 		gc = wq->gdma_dev->gdma_context;  in mana_gd_post_work_request()
1086 		dev_err(gc->dev, "unsuccessful flow control!\n");  in mana_gd_post_work_request()
1110 	struct gdma_context *gc = queue->gdma_dev->gdma_context;  in mana_gd_post_and_ring()  local
1117 	mana_gd_wq_ring_doorbell(gc, queue);  in mana_gd_post_and_ring()
1208 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_setup_irqs()  local
1224 	gc->irq_contexts = kcalloc(nvec, sizeof(struct gdma_irq_context),  in mana_gd_setup_irqs()
1226 	if (!gc->irq_contexts) {  in mana_gd_setup_irqs()
1232 		gic = &gc->irq_contexts[i];  in mana_gd_setup_irqs()
1247 	err = mana_gd_alloc_res_map(nvec, &gc->msix_resource);  in mana_gd_setup_irqs()
1251 	gc->max_num_msix = nvec;  in mana_gd_setup_irqs()
1252 	gc->num_msix_usable = nvec;  in mana_gd_setup_irqs()
1259 		gic = &gc->irq_contexts[j];  in mana_gd_setup_irqs()
1263 	kfree(gc->irq_contexts);  in mana_gd_setup_irqs()
1264 	gc->irq_contexts = NULL;  in mana_gd_setup_irqs()
1272 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_remove_irqs()  local
1276 	if (gc->max_num_msix < 1)  in mana_gd_remove_irqs()
1279 	mana_gd_free_res_map(&gc->msix_resource);  in mana_gd_remove_irqs()
1281 	for (i = 0; i < gc->max_num_msix; i++) {  in mana_gd_remove_irqs()
1286 		gic = &gc->irq_contexts[i];  in mana_gd_remove_irqs()
1292 	gc->max_num_msix = 0;  in mana_gd_remove_irqs()
1293 	gc->num_msix_usable = 0;  in mana_gd_remove_irqs()
1294 	kfree(gc->irq_contexts);  in mana_gd_remove_irqs()
1295 	gc->irq_contexts = NULL;  in mana_gd_remove_irqs()
1300 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_setup()  local
1304 	mana_smc_init(&gc->shm_channel, gc->dev, gc->shm_base);  in mana_gd_setup()
1310 	err = mana_hwc_create_channel(gc);  in mana_gd_setup()
1329 	mana_hwc_destroy_channel(gc);  in mana_gd_setup()
1337 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_cleanup()  local
1339 	mana_hwc_destroy_channel(gc);  in mana_gd_cleanup()
1351 	struct gdma_context *gc;  in mana_gd_probe()  local
1374 	gc = vzalloc(sizeof(*gc));  in mana_gd_probe()
1375 	if (!gc)  in mana_gd_probe()
1378 	mutex_init(&gc->eq_test_event_mutex);  in mana_gd_probe()
1379 	pci_set_drvdata(pdev, gc);  in mana_gd_probe()
1385 	gc->is_pf = mana_is_pf(pdev->device);  in mana_gd_probe()
1386 	gc->bar0_va = bar0_va;  in mana_gd_probe()
1387 	gc->dev = &pdev->dev;  in mana_gd_probe()
1393 	err = mana_probe(&gc->mana, false);  in mana_gd_probe()
1405 	vfree(gc);  in mana_gd_probe()
1417 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_remove()  local
1419 	mana_remove(&gc->mana, false);  in mana_gd_remove()
1423 	pci_iounmap(pdev, gc->bar0_va);  in mana_gd_remove()
1425 	vfree(gc);  in mana_gd_remove()
1435 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_suspend()  local
1437 	mana_remove(&gc->mana, true);  in mana_gd_suspend()
1450 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_resume()  local
1457 	err = mana_probe(&gc->mana, true);  in mana_gd_resume()
1467 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_shutdown()  local
1471 	mana_remove(&gc->mana, true);  in mana_gd_shutdown()