Lines Matching +full:rx +full:- +full:eq
1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
9 struct gdma_resource *r = &hwc->inflight_msg_res; in mana_hwc_get_msg_index()
13 down(&hwc->sema); in mana_hwc_get_msg_index()
15 spin_lock_irqsave(&r->lock, flags); in mana_hwc_get_msg_index()
17 index = find_first_zero_bit(hwc->inflight_msg_res.map, in mana_hwc_get_msg_index()
18 hwc->inflight_msg_res.size); in mana_hwc_get_msg_index()
20 bitmap_set(hwc->inflight_msg_res.map, index, 1); in mana_hwc_get_msg_index()
22 spin_unlock_irqrestore(&r->lock, flags); in mana_hwc_get_msg_index()
31 struct gdma_resource *r = &hwc->inflight_msg_res; in mana_hwc_put_msg_index()
34 spin_lock_irqsave(&r->lock, flags); in mana_hwc_put_msg_index()
35 bitmap_clear(hwc->inflight_msg_res.map, msg_id, 1); in mana_hwc_put_msg_index()
36 spin_unlock_irqrestore(&r->lock, flags); in mana_hwc_put_msg_index()
38 up(&hwc->sema); in mana_hwc_put_msg_index()
46 return -EPROTO; in mana_hwc_verify_resp_msg()
48 if (resp_len > caller_ctx->output_buflen) in mana_hwc_verify_resp_msg()
49 return -EPROTO; in mana_hwc_verify_resp_msg()
60 if (!test_bit(resp_msg->response.hwc_msg_id, in mana_hwc_handle_resp()
61 hwc->inflight_msg_res.map)) { in mana_hwc_handle_resp()
62 dev_err(hwc->dev, "hwc_rx: invalid msg_id = %u\n", in mana_hwc_handle_resp()
63 resp_msg->response.hwc_msg_id); in mana_hwc_handle_resp()
67 ctx = hwc->caller_ctx + resp_msg->response.hwc_msg_id; in mana_hwc_handle_resp()
72 ctx->status_code = resp_msg->status; in mana_hwc_handle_resp()
74 memcpy(ctx->output_buf, resp_msg, resp_len); in mana_hwc_handle_resp()
76 ctx->error = err; in mana_hwc_handle_resp()
77 complete(&ctx->comp_event); in mana_hwc_handle_resp()
83 struct device *dev = hwc_rxq->hwc->dev; in mana_hwc_post_rx_wqe()
87 sge = &req->sge; in mana_hwc_post_rx_wqe()
88 sge->address = (u64)req->buf_sge_addr; in mana_hwc_post_rx_wqe()
89 sge->mem_key = hwc_rxq->msg_buf->gpa_mkey; in mana_hwc_post_rx_wqe()
90 sge->size = req->buf_len; in mana_hwc_post_rx_wqe()
92 memset(&req->wqe_req, 0, sizeof(struct gdma_wqe_request)); in mana_hwc_post_rx_wqe()
93 req->wqe_req.sgl = sge; in mana_hwc_post_rx_wqe()
94 req->wqe_req.num_sge = 1; in mana_hwc_post_rx_wqe()
95 req->wqe_req.client_data_unit = 0; in mana_hwc_post_rx_wqe()
97 err = mana_gd_post_and_ring(hwc_rxq->gdma_wq, &req->wqe_req, NULL); in mana_hwc_post_rx_wqe()
107 struct gdma_dev *gd = hwc->gdma_dev; in mana_hwc_init_event_handler()
112 switch (event->type) { in mana_hwc_init_event_handler()
114 eq_db.as_uint32 = event->details[0]; in mana_hwc_init_event_handler()
115 hwc->cq->gdma_eq->id = eq_db.eq_id; in mana_hwc_init_event_handler()
116 gd->doorbell = eq_db.doorbell; in mana_hwc_init_event_handler()
120 type_data.as_uint32 = event->details[0]; in mana_hwc_init_event_handler()
126 hwc->cq->gdma_cq->id = val; in mana_hwc_init_event_handler()
130 hwc->rxq->gdma_wq->id = val; in mana_hwc_init_event_handler()
134 hwc->txq->gdma_wq->id = val; in mana_hwc_init_event_handler()
138 hwc->hwc_init_q_depth_max = (u16)val; in mana_hwc_init_event_handler()
142 hwc->hwc_init_max_req_msg_size = val; in mana_hwc_init_event_handler()
146 hwc->hwc_init_max_resp_msg_size = val; in mana_hwc_init_event_handler()
150 gd->gdma_context->max_num_cqs = val; in mana_hwc_init_event_handler()
154 hwc->gdma_dev->pdid = val; in mana_hwc_init_event_handler()
158 hwc->rxq->msg_buf->gpa_mkey = val; in mana_hwc_init_event_handler()
159 hwc->txq->msg_buf->gpa_mkey = val; in mana_hwc_init_event_handler()
166 complete(&hwc->hwc_init_eqe_comp); in mana_hwc_init_event_handler()
179 struct hwc_wq *hwc_rxq = hwc->rxq; in mana_hwc_rx_event_handler()
189 if (WARN_ON_ONCE(hwc_rxq->gdma_wq->id != gdma_rxq_id)) in mana_hwc_rx_event_handler()
192 rq = hwc_rxq->gdma_wq; in mana_hwc_rx_event_handler()
193 wqe = mana_gd_get_wqe_ptr(rq, rx_oob->wqe_offset / GDMA_WQE_BU_SIZE); in mana_hwc_rx_event_handler()
196 sge = (struct gdma_sge *)(wqe + 8 + dma_oob->inline_oob_size_div4 * 4); in mana_hwc_rx_event_handler()
198 /* Select the RX work request for virtual address and for reposting. */ in mana_hwc_rx_event_handler()
199 rq_base_addr = hwc_rxq->msg_buf->mem_info.dma_handle; in mana_hwc_rx_event_handler()
200 rx_req_idx = (sge->address - rq_base_addr) / hwc->max_req_msg_size; in mana_hwc_rx_event_handler()
202 rx_req = &hwc_rxq->msg_buf->reqs[rx_req_idx]; in mana_hwc_rx_event_handler()
203 resp = (struct gdma_resp_hdr *)rx_req->buf_va; in mana_hwc_rx_event_handler()
205 if (resp->response.hwc_msg_id >= hwc->num_inflight_msg) { in mana_hwc_rx_event_handler()
206 dev_err(hwc->dev, "HWC RX: wrong msg_id=%u\n", in mana_hwc_rx_event_handler()
207 resp->response.hwc_msg_id); in mana_hwc_rx_event_handler()
211 mana_hwc_handle_resp(hwc, rx_oob->tx_oob_data_size, resp); in mana_hwc_rx_event_handler()
225 struct hwc_wq *hwc_txq = hwc->txq; in mana_hwc_tx_event_handler()
227 WARN_ON_ONCE(!hwc_txq || hwc_txq->gdma_wq->id != gdma_txq_id); in mana_hwc_tx_event_handler()
237 return -EINVAL; in mana_hwc_create_gdma_wq()
243 return mana_gd_create_hwc_queue(hwc->gdma_dev, &spec, queue); in mana_hwc_create_gdma_wq()
261 return mana_gd_create_hwc_queue(hwc->gdma_dev, &spec, queue); in mana_hwc_create_gdma_cq()
274 spec.eq.context = ctx; in mana_hwc_create_gdma_eq()
275 spec.eq.callback = cb; in mana_hwc_create_gdma_eq()
276 spec.eq.log2_throttle_limit = DEFAULT_LOG2_THROTTLING_FOR_ERROR_EQ; in mana_hwc_create_gdma_eq()
278 return mana_gd_create_hwc_queue(hwc->gdma_dev, &spec, queue); in mana_hwc_create_gdma_eq()
288 WARN_ON_ONCE(hwc_cq->gdma_cq != q_self); in mana_hwc_comp_event()
290 completions = hwc_cq->comp_buf; in mana_hwc_comp_event()
291 comp_read = mana_gd_poll_cq(q_self, completions, hwc_cq->queue_depth); in mana_hwc_comp_event()
292 WARN_ON_ONCE(comp_read <= 0 || comp_read > hwc_cq->queue_depth); in mana_hwc_comp_event()
298 hwc_cq->tx_event_handler(hwc_cq->tx_event_ctx, in mana_hwc_comp_event()
302 hwc_cq->rx_event_handler(hwc_cq->rx_event_ctx, in mana_hwc_comp_event()
315 kfree(hwc_cq->comp_buf); in mana_hwc_destroy_cq()
317 if (hwc_cq->gdma_cq) in mana_hwc_destroy_cq()
318 mana_gd_destroy_queue(gc, hwc_cq->gdma_cq); in mana_hwc_destroy_cq()
320 if (hwc_cq->gdma_eq) in mana_hwc_destroy_cq()
321 mana_gd_destroy_queue(gc, hwc_cq->gdma_eq); in mana_hwc_destroy_cq()
333 struct gdma_queue *eq, *cq; in mana_hwc_create_cq() local
349 return -ENOMEM; in mana_hwc_create_cq()
351 err = mana_hwc_create_gdma_eq(hwc, eq_size, ctx, callback, &eq); in mana_hwc_create_cq()
353 dev_err(hwc->dev, "Failed to create HWC EQ for RQ: %d\n", err); in mana_hwc_create_cq()
356 hwc_cq->gdma_eq = eq; in mana_hwc_create_cq()
359 eq, &cq); in mana_hwc_create_cq()
361 dev_err(hwc->dev, "Failed to create HWC CQ for RQ: %d\n", err); in mana_hwc_create_cq()
364 hwc_cq->gdma_cq = cq; in mana_hwc_create_cq()
368 err = -ENOMEM; in mana_hwc_create_cq()
372 hwc_cq->hwc = hwc; in mana_hwc_create_cq()
373 hwc_cq->comp_buf = comp_buf; in mana_hwc_create_cq()
374 hwc_cq->queue_depth = q_depth; in mana_hwc_create_cq()
375 hwc_cq->rx_event_handler = rx_ev_hdlr; in mana_hwc_create_cq()
376 hwc_cq->rx_event_ctx = rx_ev_ctx; in mana_hwc_create_cq()
377 hwc_cq->tx_event_handler = tx_ev_hdlr; in mana_hwc_create_cq()
378 hwc_cq->tx_event_ctx = tx_ev_ctx; in mana_hwc_create_cq()
383 mana_hwc_destroy_cq(hwc->gdma_dev->gdma_context, hwc_cq); in mana_hwc_create_cq()
391 struct gdma_context *gc = hwc->gdma_dev->gdma_context; in mana_hwc_alloc_dma_buf()
403 return -ENOMEM; in mana_hwc_alloc_dma_buf()
405 dma_buf->num_reqs = q_depth; in mana_hwc_alloc_dma_buf()
409 gmi = &dma_buf->mem_info; in mana_hwc_alloc_dma_buf()
412 dev_err(hwc->dev, "Failed to allocate DMA buffer: %d\n", err); in mana_hwc_alloc_dma_buf()
416 virt_addr = dma_buf->mem_info.virt_addr; in mana_hwc_alloc_dma_buf()
417 base_pa = (u8 *)dma_buf->mem_info.dma_handle; in mana_hwc_alloc_dma_buf()
420 hwc_wr = &dma_buf->reqs[i]; in mana_hwc_alloc_dma_buf()
422 hwc_wr->buf_va = virt_addr + i * max_msg_size; in mana_hwc_alloc_dma_buf()
423 hwc_wr->buf_sge_addr = base_pa + i * max_msg_size; in mana_hwc_alloc_dma_buf()
425 hwc_wr->buf_len = max_msg_size; in mana_hwc_alloc_dma_buf()
441 mana_gd_free_memory(&dma_buf->mem_info); in mana_hwc_dealloc_dma_buf()
452 mana_hwc_dealloc_dma_buf(hwc, hwc_wq->msg_buf); in mana_hwc_destroy_wq()
454 if (hwc_wq->gdma_wq) in mana_hwc_destroy_wq()
455 mana_gd_destroy_queue(hwc->gdma_dev->gdma_context, in mana_hwc_destroy_wq()
456 hwc_wq->gdma_wq); in mana_hwc_destroy_wq()
483 return -ENOMEM; in mana_hwc_create_wq()
490 &hwc_wq->msg_buf); in mana_hwc_create_wq()
494 hwc_wq->hwc = hwc; in mana_hwc_create_wq()
495 hwc_wq->gdma_wq = queue; in mana_hwc_create_wq()
496 hwc_wq->queue_depth = q_depth; in mana_hwc_create_wq()
497 hwc_wq->hwc_cq = hwc_cq; in mana_hwc_create_wq()
512 struct device *dev = hwc_txq->hwc->dev; in mana_hwc_post_tx_wqe()
517 if (req->msg_size == 0 || req->msg_size > req->buf_len) { in mana_hwc_post_tx_wqe()
519 req->msg_size, req->buf_len); in mana_hwc_post_tx_wqe()
520 return -EINVAL; in mana_hwc_post_tx_wqe()
523 tx_oob = &req->tx_oob; in mana_hwc_post_tx_wqe()
525 tx_oob->vrq_id = dest_virt_rq_id; in mana_hwc_post_tx_wqe()
526 tx_oob->dest_vfid = 0; in mana_hwc_post_tx_wqe()
527 tx_oob->vrcq_id = dest_virt_rcq_id; in mana_hwc_post_tx_wqe()
528 tx_oob->vscq_id = hwc_txq->hwc_cq->gdma_cq->id; in mana_hwc_post_tx_wqe()
529 tx_oob->loopback = false; in mana_hwc_post_tx_wqe()
530 tx_oob->lso_override = false; in mana_hwc_post_tx_wqe()
531 tx_oob->dest_pf = dest_pf; in mana_hwc_post_tx_wqe()
532 tx_oob->vsq_id = hwc_txq->gdma_wq->id; in mana_hwc_post_tx_wqe()
534 sge = &req->sge; in mana_hwc_post_tx_wqe()
535 sge->address = (u64)req->buf_sge_addr; in mana_hwc_post_tx_wqe()
536 sge->mem_key = hwc_txq->msg_buf->gpa_mkey; in mana_hwc_post_tx_wqe()
537 sge->size = req->msg_size; in mana_hwc_post_tx_wqe()
539 memset(&req->wqe_req, 0, sizeof(struct gdma_wqe_request)); in mana_hwc_post_tx_wqe()
540 req->wqe_req.sgl = sge; in mana_hwc_post_tx_wqe()
541 req->wqe_req.num_sge = 1; in mana_hwc_post_tx_wqe()
542 req->wqe_req.inline_oob_size = sizeof(struct hwc_tx_oob); in mana_hwc_post_tx_wqe()
543 req->wqe_req.inline_oob_data = tx_oob; in mana_hwc_post_tx_wqe()
544 req->wqe_req.client_data_unit = 0; in mana_hwc_post_tx_wqe()
546 err = mana_gd_post_and_ring(hwc_txq->gdma_wq, &req->wqe_req, NULL); in mana_hwc_post_tx_wqe()
557 sema_init(&hwc->sema, num_msg); in mana_hwc_init_inflight_msg()
559 err = mana_gd_alloc_res_map(num_msg, &hwc->inflight_msg_res); in mana_hwc_init_inflight_msg()
561 dev_err(hwc->dev, "Failed to init inflight_msg_res: %d\n", err); in mana_hwc_init_inflight_msg()
568 struct gdma_context *gc = hwc->gdma_dev->gdma_context; in mana_hwc_test_channel()
569 struct hwc_wq *hwc_rxq = hwc->rxq; in mana_hwc_test_channel()
577 req = &hwc_rxq->msg_buf->reqs[i]; in mana_hwc_test_channel()
585 return -ENOMEM; in mana_hwc_test_channel()
590 hwc->caller_ctx = ctx; in mana_hwc_test_channel()
592 return mana_gd_test_eq(gc, hwc->cq->gdma_eq); in mana_hwc_test_channel()
599 struct hw_channel_context *hwc = gc->hwc.driver_data; in mana_hwc_establish_channel()
600 struct gdma_queue *rq = hwc->rxq->gdma_wq; in mana_hwc_establish_channel()
601 struct gdma_queue *sq = hwc->txq->gdma_wq; in mana_hwc_establish_channel()
602 struct gdma_queue *eq = hwc->cq->gdma_eq; in mana_hwc_establish_channel() local
603 struct gdma_queue *cq = hwc->cq->gdma_cq; in mana_hwc_establish_channel()
606 init_completion(&hwc->hwc_init_eqe_comp); in mana_hwc_establish_channel()
608 err = mana_smc_setup_hwc(&gc->shm_channel, false, in mana_hwc_establish_channel()
609 eq->mem_info.dma_handle, in mana_hwc_establish_channel()
610 cq->mem_info.dma_handle, in mana_hwc_establish_channel()
611 rq->mem_info.dma_handle, in mana_hwc_establish_channel()
612 sq->mem_info.dma_handle, in mana_hwc_establish_channel()
613 eq->eq.msix_index); in mana_hwc_establish_channel()
617 if (!wait_for_completion_timeout(&hwc->hwc_init_eqe_comp, 60 * HZ)) in mana_hwc_establish_channel()
618 return -ETIMEDOUT; in mana_hwc_establish_channel()
620 *q_depth = hwc->hwc_init_q_depth_max; in mana_hwc_establish_channel()
621 *max_req_msg_size = hwc->hwc_init_max_req_msg_size; in mana_hwc_establish_channel()
622 *max_resp_msg_size = hwc->hwc_init_max_resp_msg_size; in mana_hwc_establish_channel()
624 if (WARN_ON(cq->id >= gc->max_num_cqs)) in mana_hwc_establish_channel()
625 return -EPROTO; in mana_hwc_establish_channel()
627 gc->cq_table = vzalloc(gc->max_num_cqs * sizeof(struct gdma_queue *)); in mana_hwc_establish_channel()
628 if (!gc->cq_table) in mana_hwc_establish_channel()
629 return -ENOMEM; in mana_hwc_establish_channel()
631 gc->cq_table[cq->id] = cq; in mana_hwc_establish_channel()
656 dev_err(hwc->dev, "Failed to create HWC CQ: %d\n", err); in mana_hwc_init_queues()
659 hwc->cq = hwc_cq; in mana_hwc_init_queues()
664 dev_err(hwc->dev, "Failed to create HWC RQ: %d\n", err); in mana_hwc_init_queues()
667 hwc->rxq = hwc_rxq; in mana_hwc_init_queues()
672 dev_err(hwc->dev, "Failed to create HWC SQ: %d\n", err); in mana_hwc_init_queues()
675 hwc->txq = hwc_txq; in mana_hwc_init_queues()
677 hwc->num_inflight_msg = q_depth; in mana_hwc_init_queues()
678 hwc->max_req_msg_size = max_req_msg_size; in mana_hwc_init_queues()
689 mana_hwc_destroy_cq(hwc->gdma_dev->gdma_context, hwc_cq); in mana_hwc_init_queues()
691 mana_gd_free_res_map(&hwc->inflight_msg_res); in mana_hwc_init_queues()
698 struct gdma_dev *gd = &gc->hwc; in mana_hwc_create_channel()
705 return -ENOMEM; in mana_hwc_create_channel()
707 gd->gdma_context = gc; in mana_hwc_create_channel()
708 gd->driver_data = hwc; in mana_hwc_create_channel()
709 hwc->gdma_dev = gd; in mana_hwc_create_channel()
710 hwc->dev = gc->dev; in mana_hwc_create_channel()
713 gd->dev_id.as_uint32 = 0; in mana_hwc_create_channel()
714 gd->dev_id.type = GDMA_DEVICE_HWC; in mana_hwc_create_channel()
716 gd->pdid = INVALID_PDID; in mana_hwc_create_channel()
717 gd->doorbell = INVALID_DOORBELL; in mana_hwc_create_channel()
723 dev_err(hwc->dev, "Failed to initialize HWC: %d\n", err); in mana_hwc_create_channel()
730 dev_err(hwc->dev, "Failed to establish HWC: %d\n", err); in mana_hwc_create_channel()
734 err = mana_hwc_test_channel(gc->hwc.driver_data, in mana_hwc_create_channel()
738 dev_err(hwc->dev, "Failed to test HWC: %d\n", err); in mana_hwc_create_channel()
750 struct hw_channel_context *hwc = gc->hwc.driver_data; in mana_hwc_destroy_channel()
753 mana_smc_teardown_hwc(&gc->shm_channel, false); in mana_hwc_destroy_channel()
755 ctx = hwc->caller_ctx; in mana_hwc_destroy_channel()
757 hwc->caller_ctx = NULL; in mana_hwc_destroy_channel()
759 mana_hwc_destroy_wq(hwc, hwc->txq); in mana_hwc_destroy_channel()
760 hwc->txq = NULL; in mana_hwc_destroy_channel()
762 mana_hwc_destroy_wq(hwc, hwc->rxq); in mana_hwc_destroy_channel()
763 hwc->rxq = NULL; in mana_hwc_destroy_channel()
765 mana_hwc_destroy_cq(hwc->gdma_dev->gdma_context, hwc->cq); in mana_hwc_destroy_channel()
766 hwc->cq = NULL; in mana_hwc_destroy_channel()
768 mana_gd_free_res_map(&hwc->inflight_msg_res); in mana_hwc_destroy_channel()
770 hwc->num_inflight_msg = 0; in mana_hwc_destroy_channel()
772 if (hwc->gdma_dev->pdid != INVALID_PDID) { in mana_hwc_destroy_channel()
773 hwc->gdma_dev->doorbell = INVALID_DOORBELL; in mana_hwc_destroy_channel()
774 hwc->gdma_dev->pdid = INVALID_PDID; in mana_hwc_destroy_channel()
778 gc->hwc.driver_data = NULL; in mana_hwc_destroy_channel()
779 gc->hwc.gdma_context = NULL; in mana_hwc_destroy_channel()
786 struct hwc_wq *txq = hwc->txq; in mana_hwc_send_request()
794 tx_wr = &txq->msg_buf->reqs[msg_id]; in mana_hwc_send_request()
796 if (req_len > tx_wr->buf_len) { in mana_hwc_send_request()
797 dev_err(hwc->dev, "HWC: req msg size: %d > %d\n", req_len, in mana_hwc_send_request()
798 tx_wr->buf_len); in mana_hwc_send_request()
799 err = -EINVAL; in mana_hwc_send_request()
803 ctx = hwc->caller_ctx + msg_id; in mana_hwc_send_request()
804 ctx->output_buf = resp; in mana_hwc_send_request()
805 ctx->output_buflen = resp_len; in mana_hwc_send_request()
807 req_msg = (struct gdma_req_hdr *)tx_wr->buf_va; in mana_hwc_send_request()
811 req_msg->req.hwc_msg_id = msg_id; in mana_hwc_send_request()
813 tx_wr->msg_size = req_len; in mana_hwc_send_request()
817 dev_err(hwc->dev, "HWC: Failed to post send WQE: %d\n", err); in mana_hwc_send_request()
821 if (!wait_for_completion_timeout(&ctx->comp_event, 30 * HZ)) { in mana_hwc_send_request()
822 dev_err(hwc->dev, "HWC: Request timed out!\n"); in mana_hwc_send_request()
823 err = -ETIMEDOUT; in mana_hwc_send_request()
827 if (ctx->error) { in mana_hwc_send_request()
828 err = ctx->error; in mana_hwc_send_request()
832 if (ctx->status_code) { in mana_hwc_send_request()
833 dev_err(hwc->dev, "HWC: Failed hw_channel req: 0x%x\n", in mana_hwc_send_request()
834 ctx->status_code); in mana_hwc_send_request()
835 err = -EPROTO; in mana_hwc_send_request()