Lines Matching refs:rx_ring
70 struct ena_ring *rx_ring);
72 struct ena_ring *rx_ring);
115 adapter->rx_ring[i].mtu = mtu; in update_rx_ring_mtu()
349 if (!READ_ONCE(adapter->rx_ring->xdp_bpf_prog)) in ena_xdp_xmit()
375 static int ena_xdp_execute(struct ena_ring *rx_ring, struct xdp_buff *xdp) in ena_xdp_execute() argument
383 xdp_prog = READ_ONCE(rx_ring->xdp_bpf_prog); in ena_xdp_execute()
394 trace_xdp_exception(rx_ring->netdev, xdp_prog, verdict); in ena_xdp_execute()
395 xdp_stat = &rx_ring->rx_stats.xdp_aborted; in ena_xdp_execute()
401 xdp_ring = rx_ring->xdp_ring; in ena_xdp_execute()
406 if (ena_xdp_xmit_frame(xdp_ring, rx_ring->netdev, xdpf, in ena_xdp_execute()
411 xdp_stat = &rx_ring->rx_stats.xdp_tx; in ena_xdp_execute()
414 if (likely(!xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog))) { in ena_xdp_execute()
415 xdp_stat = &rx_ring->rx_stats.xdp_redirect; in ena_xdp_execute()
418 trace_xdp_exception(rx_ring->netdev, xdp_prog, verdict); in ena_xdp_execute()
419 xdp_stat = &rx_ring->rx_stats.xdp_aborted; in ena_xdp_execute()
423 trace_xdp_exception(rx_ring->netdev, xdp_prog, verdict); in ena_xdp_execute()
424 xdp_stat = &rx_ring->rx_stats.xdp_aborted; in ena_xdp_execute()
427 xdp_stat = &rx_ring->rx_stats.xdp_drop; in ena_xdp_execute()
430 xdp_stat = &rx_ring->rx_stats.xdp_pass; in ena_xdp_execute()
433 bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, verdict); in ena_xdp_execute()
434 xdp_stat = &rx_ring->rx_stats.xdp_invalid; in ena_xdp_execute()
437 ena_increase_stat(xdp_stat, 1, &rx_ring->syncp); in ena_xdp_execute()
478 static int ena_xdp_register_rxq_info(struct ena_ring *rx_ring) in ena_xdp_register_rxq_info() argument
482 rc = xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, rx_ring->qid, 0); in ena_xdp_register_rxq_info()
485 netif_err(rx_ring->adapter, ifup, rx_ring->netdev, in ena_xdp_register_rxq_info()
487 rx_ring->qid, rc); in ena_xdp_register_rxq_info()
491 rc = xdp_rxq_info_reg_mem_model(&rx_ring->xdp_rxq, MEM_TYPE_PAGE_SHARED, in ena_xdp_register_rxq_info()
495 netif_err(rx_ring->adapter, ifup, rx_ring->netdev, in ena_xdp_register_rxq_info()
497 rx_ring->qid, rc); in ena_xdp_register_rxq_info()
498 xdp_rxq_info_unreg(&rx_ring->xdp_rxq); in ena_xdp_register_rxq_info()
505 static void ena_xdp_unregister_rxq_info(struct ena_ring *rx_ring) in ena_xdp_unregister_rxq_info() argument
507 xdp_rxq_info_unreg_mem_model(&rx_ring->xdp_rxq); in ena_xdp_unregister_rxq_info()
508 xdp_rxq_info_unreg(&rx_ring->xdp_rxq); in ena_xdp_unregister_rxq_info()
515 struct ena_ring *rx_ring; in ena_xdp_exchange_program_rx_in_range() local
519 rx_ring = &adapter->rx_ring[i]; in ena_xdp_exchange_program_rx_in_range()
520 xchg(&rx_ring->xdp_bpf_prog, prog); in ena_xdp_exchange_program_rx_in_range()
522 ena_xdp_register_rxq_info(rx_ring); in ena_xdp_exchange_program_rx_in_range()
523 rx_ring->rx_headroom = XDP_PACKET_HEADROOM; in ena_xdp_exchange_program_rx_in_range()
525 ena_xdp_unregister_rxq_info(rx_ring); in ena_xdp_exchange_program_rx_in_range()
526 rx_ring->rx_headroom = NET_SKB_PAD; in ena_xdp_exchange_program_rx_in_range()
690 rxr = &adapter->rx_ring[i]; in ena_init_io_rings()
866 struct ena_ring *rx_ring = &adapter->rx_ring[qid]; in ena_setup_rx_resources() local
870 if (rx_ring->rx_buffer_info) { in ena_setup_rx_resources()
879 size = sizeof(struct ena_rx_buffer) * (rx_ring->ring_size + 1); in ena_setup_rx_resources()
882 rx_ring->rx_buffer_info = vzalloc_node(size, node); in ena_setup_rx_resources()
883 if (!rx_ring->rx_buffer_info) { in ena_setup_rx_resources()
884 rx_ring->rx_buffer_info = vzalloc(size); in ena_setup_rx_resources()
885 if (!rx_ring->rx_buffer_info) in ena_setup_rx_resources()
889 size = sizeof(u16) * rx_ring->ring_size; in ena_setup_rx_resources()
890 rx_ring->free_ids = vzalloc_node(size, node); in ena_setup_rx_resources()
891 if (!rx_ring->free_ids) { in ena_setup_rx_resources()
892 rx_ring->free_ids = vzalloc(size); in ena_setup_rx_resources()
893 if (!rx_ring->free_ids) { in ena_setup_rx_resources()
894 vfree(rx_ring->rx_buffer_info); in ena_setup_rx_resources()
895 rx_ring->rx_buffer_info = NULL; in ena_setup_rx_resources()
901 for (i = 0; i < rx_ring->ring_size; i++) in ena_setup_rx_resources()
902 rx_ring->free_ids[i] = i; in ena_setup_rx_resources()
905 memset(&rx_ring->rx_stats, 0x0, sizeof(rx_ring->rx_stats)); in ena_setup_rx_resources()
907 rx_ring->next_to_clean = 0; in ena_setup_rx_resources()
908 rx_ring->next_to_use = 0; in ena_setup_rx_resources()
909 rx_ring->cpu = ena_irq->cpu; in ena_setup_rx_resources()
923 struct ena_ring *rx_ring = &adapter->rx_ring[qid]; in ena_free_rx_resources() local
925 vfree(rx_ring->rx_buffer_info); in ena_free_rx_resources()
926 rx_ring->rx_buffer_info = NULL; in ena_free_rx_resources()
928 vfree(rx_ring->free_ids); in ena_free_rx_resources()
929 rx_ring->free_ids = NULL; in ena_free_rx_resources()
973 static struct page *ena_alloc_map_page(struct ena_ring *rx_ring, in ena_alloc_map_page() argument
983 ena_increase_stat(&rx_ring->rx_stats.page_alloc_fail, 1, in ena_alloc_map_page()
984 &rx_ring->syncp); in ena_alloc_map_page()
991 *dma = dma_map_page(rx_ring->dev, page, 0, ENA_PAGE_SIZE, in ena_alloc_map_page()
993 if (unlikely(dma_mapping_error(rx_ring->dev, *dma))) { in ena_alloc_map_page()
994 ena_increase_stat(&rx_ring->rx_stats.dma_mapping_err, 1, in ena_alloc_map_page()
995 &rx_ring->syncp); in ena_alloc_map_page()
1003 static int ena_alloc_rx_buffer(struct ena_ring *rx_ring, in ena_alloc_rx_buffer() argument
1006 int headroom = rx_ring->rx_headroom; in ena_alloc_rx_buffer()
1020 page = ena_alloc_map_page(rx_ring, &dma); in ena_alloc_rx_buffer()
1024 netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev, in ena_alloc_rx_buffer()
1037 static void ena_unmap_rx_buff(struct ena_ring *rx_ring, in ena_unmap_rx_buff() argument
1042 dma_unmap_page(rx_ring->dev, ena_buf->paddr - rx_ring->rx_headroom, in ena_unmap_rx_buff()
1047 static void ena_free_rx_page(struct ena_ring *rx_ring, in ena_free_rx_page() argument
1053 netif_warn(rx_ring->adapter, rx_err, rx_ring->netdev, in ena_free_rx_page()
1058 ena_unmap_rx_buff(rx_ring, rx_info); in ena_free_rx_page()
1064 static int ena_refill_rx_bufs(struct ena_ring *rx_ring, u32 num) in ena_refill_rx_bufs() argument
1070 next_to_use = rx_ring->next_to_use; in ena_refill_rx_bufs()
1075 req_id = rx_ring->free_ids[next_to_use]; in ena_refill_rx_bufs()
1077 rx_info = &rx_ring->rx_buffer_info[req_id]; in ena_refill_rx_bufs()
1079 rc = ena_alloc_rx_buffer(rx_ring, rx_info); in ena_refill_rx_bufs()
1081 netif_warn(rx_ring->adapter, rx_err, rx_ring->netdev, in ena_refill_rx_bufs()
1083 rx_ring->qid); in ena_refill_rx_bufs()
1086 rc = ena_com_add_single_rx_desc(rx_ring->ena_com_io_sq, in ena_refill_rx_bufs()
1090 netif_warn(rx_ring->adapter, rx_status, rx_ring->netdev, in ena_refill_rx_bufs()
1092 rx_ring->qid); in ena_refill_rx_bufs()
1096 rx_ring->ring_size); in ena_refill_rx_bufs()
1100 ena_increase_stat(&rx_ring->rx_stats.refil_partial, 1, in ena_refill_rx_bufs()
1101 &rx_ring->syncp); in ena_refill_rx_bufs()
1102 netif_warn(rx_ring->adapter, rx_err, rx_ring->netdev, in ena_refill_rx_bufs()
1104 rx_ring->qid, i, num); in ena_refill_rx_bufs()
1109 ena_com_write_sq_doorbell(rx_ring->ena_com_io_sq); in ena_refill_rx_bufs()
1111 rx_ring->next_to_use = next_to_use; in ena_refill_rx_bufs()
1119 struct ena_ring *rx_ring = &adapter->rx_ring[qid]; in ena_free_rx_bufs() local
1122 for (i = 0; i < rx_ring->ring_size; i++) { in ena_free_rx_bufs()
1123 struct ena_rx_buffer *rx_info = &rx_ring->rx_buffer_info[i]; in ena_free_rx_bufs()
1126 ena_free_rx_page(rx_ring, rx_info); in ena_free_rx_bufs()
1135 struct ena_ring *rx_ring; in ena_refill_all_rx_bufs() local
1139 rx_ring = &adapter->rx_ring[i]; in ena_refill_all_rx_bufs()
1140 bufs_num = rx_ring->ring_size - 1; in ena_refill_all_rx_bufs()
1141 rc = ena_refill_rx_bufs(rx_ring, bufs_num); in ena_refill_all_rx_bufs()
1144 netif_warn(rx_ring->adapter, rx_status, rx_ring->netdev, in ena_refill_all_rx_bufs()
1396 static struct sk_buff *ena_alloc_skb(struct ena_ring *rx_ring, void *first_frag) in ena_alloc_skb() argument
1401 skb = napi_alloc_skb(rx_ring->napi, rx_ring->rx_copybreak); in ena_alloc_skb()
1406 ena_increase_stat(&rx_ring->rx_stats.skb_alloc_fail, 1, in ena_alloc_skb()
1407 &rx_ring->syncp); in ena_alloc_skb()
1409 netif_dbg(rx_ring->adapter, rx_err, rx_ring->netdev, in ena_alloc_skb()
1418 static struct sk_buff *ena_rx_skb(struct ena_ring *rx_ring, in ena_rx_skb() argument
1434 rx_info = &rx_ring->rx_buffer_info[req_id]; in ena_rx_skb()
1437 adapter = rx_ring->adapter; in ena_rx_skb()
1438 netif_err(adapter, rx_err, rx_ring->netdev, in ena_rx_skb()
1439 "Page is NULL. qid %u req_id %u\n", rx_ring->qid, req_id); in ena_rx_skb()
1440 ena_increase_stat(&rx_ring->rx_stats.bad_req_id, 1, &rx_ring->syncp); in ena_rx_skb()
1445 netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev, in ena_rx_skb()
1456 if (len <= rx_ring->rx_copybreak) { in ena_rx_skb()
1457 skb = ena_alloc_skb(rx_ring, NULL); in ena_rx_skb()
1461 netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev, in ena_rx_skb()
1466 dma_sync_single_for_cpu(rx_ring->dev, in ena_rx_skb()
1471 dma_sync_single_for_device(rx_ring->dev, in ena_rx_skb()
1477 skb->protocol = eth_type_trans(skb, rx_ring->netdev); in ena_rx_skb()
1478 rx_ring->free_ids[*next_to_clean] = req_id; in ena_rx_skb()
1480 rx_ring->ring_size); in ena_rx_skb()
1484 ena_unmap_rx_buff(rx_ring, rx_info); in ena_rx_skb()
1486 skb = ena_alloc_skb(rx_ring, page_addr); in ena_rx_skb()
1493 skb->protocol = eth_type_trans(skb, rx_ring->netdev); in ena_rx_skb()
1496 netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev, in ena_rx_skb()
1502 rx_ring->free_ids[*next_to_clean] = req_id; in ena_rx_skb()
1505 rx_ring->ring_size); in ena_rx_skb()
1513 rx_info = &rx_ring->rx_buffer_info[req_id]; in ena_rx_skb()
1515 ena_unmap_rx_buff(rx_ring, rx_info); in ena_rx_skb()
1530 static void ena_rx_checksum(struct ena_ring *rx_ring, in ena_rx_checksum() argument
1535 if (unlikely(!(rx_ring->netdev->features & NETIF_F_RXCSUM))) { in ena_rx_checksum()
1551 ena_increase_stat(&rx_ring->rx_stats.csum_bad, 1, in ena_rx_checksum()
1552 &rx_ring->syncp); in ena_rx_checksum()
1553 netif_dbg(rx_ring->adapter, rx_err, rx_ring->netdev, in ena_rx_checksum()
1563 ena_increase_stat(&rx_ring->rx_stats.csum_bad, 1, in ena_rx_checksum()
1564 &rx_ring->syncp); in ena_rx_checksum()
1565 netif_dbg(rx_ring->adapter, rx_err, rx_ring->netdev, in ena_rx_checksum()
1573 ena_increase_stat(&rx_ring->rx_stats.csum_good, 1, in ena_rx_checksum()
1574 &rx_ring->syncp); in ena_rx_checksum()
1576 ena_increase_stat(&rx_ring->rx_stats.csum_unchecked, 1, in ena_rx_checksum()
1577 &rx_ring->syncp); in ena_rx_checksum()
1587 static void ena_set_rx_hash(struct ena_ring *rx_ring, in ena_set_rx_hash() argument
1593 if (likely(rx_ring->netdev->features & NETIF_F_RXHASH)) { in ena_set_rx_hash()
1609 static int ena_xdp_handle_buff(struct ena_ring *rx_ring, struct xdp_buff *xdp) in ena_xdp_handle_buff() argument
1614 rx_info = &rx_ring->rx_buffer_info[rx_ring->ena_bufs[0].req_id]; in ena_xdp_handle_buff()
1617 rx_ring->ena_bufs[0].len, false); in ena_xdp_handle_buff()
1621 if (unlikely(rx_ring->ena_bufs[0].len > ENA_XDP_MAX_MTU)) in ena_xdp_handle_buff()
1624 ret = ena_xdp_execute(rx_ring, xdp); in ena_xdp_handle_buff()
1629 rx_ring->ena_bufs[0].len = xdp->data_end - xdp->data; in ena_xdp_handle_buff()
1641 static int ena_clean_rx_irq(struct ena_ring *rx_ring, struct napi_struct *napi, in ena_clean_rx_irq() argument
1644 u16 next_to_clean = rx_ring->next_to_clean; in ena_clean_rx_irq()
1660 netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev, in ena_clean_rx_irq()
1661 "%s qid %d\n", __func__, rx_ring->qid); in ena_clean_rx_irq()
1663 xdp_init_buff(&xdp, ENA_PAGE_SIZE, &rx_ring->xdp_rxq); in ena_clean_rx_irq()
1668 ena_rx_ctx.ena_bufs = rx_ring->ena_bufs; in ena_clean_rx_irq()
1669 ena_rx_ctx.max_bufs = rx_ring->sgl_size; in ena_clean_rx_irq()
1672 rc = ena_com_rx_pkt(rx_ring->ena_com_io_cq, in ena_clean_rx_irq()
1673 rx_ring->ena_com_io_sq, in ena_clean_rx_irq()
1682 rx_info = &rx_ring->rx_buffer_info[rx_ring->ena_bufs[0].req_id]; in ena_clean_rx_irq()
1685 netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev, in ena_clean_rx_irq()
1687 rx_ring->qid, ena_rx_ctx.descs, ena_rx_ctx.l3_proto, in ena_clean_rx_irq()
1690 if (ena_xdp_present_ring(rx_ring)) in ena_clean_rx_irq()
1691 xdp_verdict = ena_xdp_handle_buff(rx_ring, &xdp); in ena_clean_rx_irq()
1695 skb = ena_rx_skb(rx_ring, in ena_clean_rx_irq()
1696 rx_ring->ena_bufs, in ena_clean_rx_irq()
1702 int req_id = rx_ring->ena_bufs[i].req_id; in ena_clean_rx_irq()
1704 rx_ring->free_ids[next_to_clean] = req_id; in ena_clean_rx_irq()
1707 rx_ring->ring_size); in ena_clean_rx_irq()
1713 ena_unmap_rx_buff(rx_ring, in ena_clean_rx_irq()
1714 &rx_ring->rx_buffer_info[req_id]); in ena_clean_rx_irq()
1715 rx_ring->rx_buffer_info[req_id].page = NULL; in ena_clean_rx_irq()
1726 ena_rx_checksum(rx_ring, &ena_rx_ctx, skb); in ena_clean_rx_irq()
1728 ena_set_rx_hash(rx_ring, &ena_rx_ctx, skb); in ena_clean_rx_irq()
1730 skb_record_rx_queue(skb, rx_ring->qid); in ena_clean_rx_irq()
1732 if (rx_ring->ena_bufs[0].len <= rx_ring->rx_copybreak) in ena_clean_rx_irq()
1743 rx_ring->per_napi_packets += work_done; in ena_clean_rx_irq()
1744 u64_stats_update_begin(&rx_ring->syncp); in ena_clean_rx_irq()
1745 rx_ring->rx_stats.bytes += total_len; in ena_clean_rx_irq()
1746 rx_ring->rx_stats.cnt += work_done; in ena_clean_rx_irq()
1747 rx_ring->rx_stats.rx_copybreak_pkt += rx_copybreak_pkt; in ena_clean_rx_irq()
1748 u64_stats_update_end(&rx_ring->syncp); in ena_clean_rx_irq()
1750 rx_ring->next_to_clean = next_to_clean; in ena_clean_rx_irq()
1752 refill_required = ena_com_free_q_entries(rx_ring->ena_com_io_sq); in ena_clean_rx_irq()
1754 min_t(int, rx_ring->ring_size / ENA_RX_REFILL_THRESH_DIVIDER, in ena_clean_rx_irq()
1759 ena_com_update_dev_comp_head(rx_ring->ena_com_io_cq); in ena_clean_rx_irq()
1760 ena_refill_rx_bufs(rx_ring, refill_required); in ena_clean_rx_irq()
1769 adapter = netdev_priv(rx_ring->netdev); in ena_clean_rx_irq()
1772 ena_increase_stat(&rx_ring->rx_stats.bad_desc_num, 1, in ena_clean_rx_irq()
1773 &rx_ring->syncp); in ena_clean_rx_irq()
1776 ena_increase_stat(&rx_ring->rx_stats.bad_req_id, 1, in ena_clean_rx_irq()
1777 &rx_ring->syncp); in ena_clean_rx_irq()
1790 ena_napi->rx_ring->smoothed_interval = cur_moder.usec; in ena_dim_work()
1797 struct ena_ring *rx_ring = ena_napi->rx_ring; in ena_adjust_adaptive_rx_intr_moderation() local
1799 if (!rx_ring->per_napi_packets) in ena_adjust_adaptive_rx_intr_moderation()
1802 rx_ring->non_empty_napi_events++; in ena_adjust_adaptive_rx_intr_moderation()
1804 dim_update_sample(rx_ring->non_empty_napi_events, in ena_adjust_adaptive_rx_intr_moderation()
1805 rx_ring->rx_stats.cnt, in ena_adjust_adaptive_rx_intr_moderation()
1806 rx_ring->rx_stats.bytes, in ena_adjust_adaptive_rx_intr_moderation()
1811 rx_ring->per_napi_packets = 0; in ena_adjust_adaptive_rx_intr_moderation()
1815 struct ena_ring *rx_ring) in ena_unmask_interrupt() argument
1822 if (rx_ring) in ena_unmask_interrupt()
1823 rx_interval = ena_com_get_adaptive_moderation_enabled(rx_ring->ena_dev) ? in ena_unmask_interrupt()
1824 rx_ring->smoothed_interval : in ena_unmask_interrupt()
1825 ena_com_get_nonadaptive_moderation_interval_rx(rx_ring->ena_dev); in ena_unmask_interrupt()
1847 struct ena_ring *rx_ring) in ena_update_ring_numa_node() argument
1861 if (rx_ring) in ena_update_ring_numa_node()
1862 ena_com_update_numa_node(rx_ring->ena_com_io_cq, in ena_update_ring_numa_node()
1867 if (rx_ring) in ena_update_ring_numa_node()
1868 rx_ring->cpu = cpu; in ena_update_ring_numa_node()
1941 struct ena_ring *tx_ring, *rx_ring; in ena_io_poll() local
1949 rx_ring = ena_napi->rx_ring; in ena_io_poll()
1964 rx_work_done = ena_clean_rx_irq(rx_ring, napi, budget); in ena_io_poll()
1987 if (ena_com_get_adaptive_moderation_enabled(rx_ring->ena_dev)) in ena_io_poll()
1990 ena_unmask_interrupt(tx_ring, rx_ring); in ena_io_poll()
1993 ena_update_ring_numa_node(tx_ring, rx_ring); in ena_io_poll()
2272 napi->rx_ring = &adapter->rx_ring[i]; in ena_init_napi_in_range()
2429 struct ena_ring *rx_ring; in ena_create_io_rx_queue() local
2436 rx_ring = &adapter->rx_ring[qid]; in ena_create_io_rx_queue()
2446 ctx.queue_size = rx_ring->ring_size; in ena_create_io_rx_queue()
2447 ctx.numa_node = cpu_to_node(rx_ring->cpu); in ena_create_io_rx_queue()
2458 &rx_ring->ena_com_io_sq, in ena_create_io_rx_queue()
2459 &rx_ring->ena_com_io_cq); in ena_create_io_rx_queue()
2467 ena_com_update_numa_node(rx_ring->ena_com_io_cq, ctx.numa_node); in ena_create_io_rx_queue()
2506 adapter->rx_ring[i].ring_size = new_rx_size; in set_io_rings_size()
2580 cur_rx_ring_size = adapter->rx_ring[0].ring_size; in create_queues_with_size_backoff()
2654 &adapter->rx_ring[i]); in ena_up()
3256 struct ena_ring *rx_ring, *tx_ring; in ena_get_stats64() local
3279 rx_ring = &adapter->rx_ring[i]; in ena_get_stats64()
3282 start = u64_stats_fetch_begin_irq(&rx_ring->syncp); in ena_get_stats64()
3283 packets = rx_ring->rx_stats.cnt; in ena_get_stats64()
3284 bytes = rx_ring->rx_stats.bytes; in ena_get_stats64()
3285 } while (u64_stats_fetch_retry_irq(&rx_ring->syncp, start)); in ena_get_stats64()
3679 struct ena_ring *rx_ring) in check_for_rx_interrupt_queue() argument
3681 struct ena_napi *ena_napi = container_of(rx_ring->napi, struct ena_napi, napi); in check_for_rx_interrupt_queue()
3686 if (ena_com_cq_empty(rx_ring->ena_com_io_cq)) in check_for_rx_interrupt_queue()
3689 rx_ring->no_interrupt_event_cnt++; in check_for_rx_interrupt_queue()
3691 if (rx_ring->no_interrupt_event_cnt == ENA_MAX_NO_INTERRUPT_ITERATIONS) { in check_for_rx_interrupt_queue()
3694 rx_ring->qid); in check_for_rx_interrupt_queue()
3772 struct ena_ring *rx_ring; in check_for_missing_completions() local
3793 rx_ring = &adapter->rx_ring[i]; in check_for_missing_completions()
3800 check_for_rx_interrupt_queue(adapter, rx_ring) : 0; in check_for_missing_completions()
3830 struct ena_ring *rx_ring; in check_for_empty_rx_ring() local
3840 rx_ring = &adapter->rx_ring[i]; in check_for_empty_rx_ring()
3842 refill_required = ena_com_free_q_entries(rx_ring->ena_com_io_sq); in check_for_empty_rx_ring()
3843 if (unlikely(refill_required == (rx_ring->ring_size - 1))) { in check_for_empty_rx_ring()
3844 rx_ring->empty_rx_queue++; in check_for_empty_rx_ring()
3846 if (rx_ring->empty_rx_queue >= EMPTY_RX_REFILL) { in check_for_empty_rx_ring()
3847 ena_increase_stat(&rx_ring->rx_stats.empty_rx_ring, 1, in check_for_empty_rx_ring()
3848 &rx_ring->syncp); in check_for_empty_rx_ring()
3853 napi_schedule(rx_ring->napi); in check_for_empty_rx_ring()
3854 rx_ring->empty_rx_queue = 0; in check_for_empty_rx_ring()
3857 rx_ring->empty_rx_queue = 0; in check_for_empty_rx_ring()