555 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts)  in bnxt_tx_int()  argument
557 	struct bnxt_tx_ring_info *txr = bnapi->tx_ring;  in bnxt_tx_int()
779 static void bnxt_reuse_rx_agg_bufs(struct bnxt_napi *bnapi, u16 cp_cons,  in bnxt_reuse_rx_agg_bufs()  argument
782 	struct bnxt *bp = bnapi->bp;  in bnxt_reuse_rx_agg_bufs()
783 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;  in bnxt_reuse_rx_agg_bufs()
784 	struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;  in bnxt_reuse_rx_agg_bufs()
857 	skb = napi_alloc_skb(&rxr->bnapi->napi, payload);  in bnxt_rx_page_skb()
906 static struct sk_buff *bnxt_rx_pages(struct bnxt *bp, struct bnxt_napi *bnapi,  in bnxt_rx_pages()  argument
911 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;  in bnxt_rx_pages()
912 	struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;  in bnxt_rx_pages()
958 			bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs - i);  in bnxt_rx_pages()
990 static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data,  in bnxt_copy_skb()  argument
994 	struct bnxt *bp = bnapi->bp;  in bnxt_copy_skb()
998 	skb = napi_alloc_skb(&bnapi->napi, len);  in bnxt_copy_skb()
1015 static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_napi *bnapi,  in bnxt_discard_rx()  argument
1018 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;  in bnxt_discard_rx()
1063 	if (!rxr->bnapi->in_reset) {  in bnxt_sched_reset()
1064 		rxr->bnapi->in_reset = true;  in bnxt_sched_reset()
1144 static void bnxt_abort_tpa(struct bnxt *bp, struct bnxt_napi *bnapi,  in bnxt_abort_tpa()  argument
1148 		bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs);  in bnxt_abort_tpa()
1342 					   struct bnxt_napi *bnapi,  in bnxt_tpa_end()  argument
1348 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;  in bnxt_tpa_end()
1349 	struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;  in bnxt_tpa_end()
1359 	if (unlikely(bnapi->in_reset)) {  in bnxt_tpa_end()
1360 		int rc = bnxt_discard_rx(bp, bnapi, raw_cons, tpa_end);  in bnxt_tpa_end()
1386 		bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs);  in bnxt_tpa_end()
1394 		skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping);  in bnxt_tpa_end()
1396 			bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs);  in bnxt_tpa_end()
1405 			bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs);  in bnxt_tpa_end()
1420 			bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs);  in bnxt_tpa_end()
1428 		skb = bnxt_rx_pages(bp, bnapi, skb, cp_cons, agg_bufs);  in bnxt_tpa_end()
1463 static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi,  in bnxt_deliver_skb()  argument
1471 	skb_record_rx_queue(skb, bnapi->index);  in bnxt_deliver_skb()
1472 	napi_gro_receive(&bnapi->napi, skb);  in bnxt_deliver_skb()
1482 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_napi *bnapi, u32 *raw_cons,  in bnxt_rx_pkt()  argument
1485 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;  in bnxt_rx_pkt()
1486 	struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;  in bnxt_rx_pkt()
1524 		skb = bnxt_tpa_end(bp, bnapi, &tmp_raw_cons,  in bnxt_rx_pkt()
1533 			bnxt_deliver_skb(bp, bnapi, skb);  in bnxt_rx_pkt()
1545 		int rc1 = bnxt_discard_rx(bp, bnapi, raw_cons, rxcmp);  in bnxt_rx_pkt()
1568 			bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs);  in bnxt_rx_pkt()
1583 		skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr);  in bnxt_rx_pkt()
1605 		skb = bnxt_rx_pages(bp, bnapi, skb, cp_cons, agg_bufs);  in bnxt_rx_pkt()
1648 	bnxt_deliver_skb(bp, bnapi, skb);  in bnxt_rx_pkt()
1667 static int bnxt_force_rx_discard(struct bnxt *bp, struct bnxt_napi *bnapi,  in bnxt_force_rx_discard()  argument
1670 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;  in bnxt_force_rx_discard()
1700 	return bnxt_rx_pkt(bp, bnapi, raw_cons, event);  in bnxt_force_rx_discard()
1811 	struct bnxt_napi *bnapi = dev_instance;  in bnxt_msix()  local
1812 	struct bnxt *bp = bnapi->bp;  in bnxt_msix()
1813 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;  in bnxt_msix()
1818 	napi_schedule(&bnapi->napi);  in bnxt_msix()
1835 	struct bnxt_napi *bnapi = dev_instance;  in bnxt_inta()  local
1836 	struct bnxt *bp = bnapi->bp;  in bnxt_inta()
1837 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;  in bnxt_inta()
1857 	napi_schedule(&bnapi->napi);  in bnxt_inta()
1861 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)  in bnxt_poll_work()  argument
1863 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;  in bnxt_poll_work()
1894 				rc = bnxt_rx_pkt(bp, bnapi, &raw_cons, &event);  in bnxt_poll_work()
1896 				rc = bnxt_force_rx_discard(bp, bnapi, &raw_cons,  in bnxt_poll_work()
1924 		struct bnxt_tx_ring_info *txr = bnapi->tx_ring;  in bnxt_poll_work()
1942 		bnapi->tx_int(bp, bnapi, tx_pkts);  in bnxt_poll_work()
1945 		struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;  in bnxt_poll_work()
1957 	struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);  in bnxt_poll_nitroa0()  local
1958 	struct bnxt *bp = bnapi->bp;  in bnxt_poll_nitroa0()
1959 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;  in bnxt_poll_nitroa0()
1960 	struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;  in bnxt_poll_nitroa0()
1990 			rc = bnxt_rx_pkt(bp, bnapi, &raw_cons, &event);  in bnxt_poll_nitroa0()
2025 	struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);  in bnxt_poll()  local
2026 	struct bnxt *bp = bnapi->bp;  in bnxt_poll()
2027 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;  in bnxt_poll()
2031 		work_done += bnxt_poll_work(bp, bnapi, budget - work_done);  in bnxt_poll()
2404 		ring->grp_idx = txr->bnapi->index;  in bnxt_alloc_tx_rings()
2439 	if (!bp->bnapi)  in bnxt_free_cp_rings()
2443 		struct bnxt_napi *bnapi = bp->bnapi[i];  in bnxt_free_cp_rings()  local
2447 		if (!bnapi)  in bnxt_free_cp_rings()
2450 		cpr = &bnapi->cp_ring;  in bnxt_free_cp_rings()
2464 		struct bnxt_napi *bnapi = bp->bnapi[i];  in bnxt_alloc_cp_rings()  local
2468 		if (!bnapi)  in bnxt_alloc_cp_rings()
2471 		cpr = &bnapi->cp_ring;  in bnxt_alloc_cp_rings()
2491 		struct bnxt_napi *bnapi = bp->bnapi[i];  in bnxt_init_ring_struct()  local
2497 		if (!bnapi)  in bnxt_init_ring_struct()
2500 		cpr = &bnapi->cp_ring;  in bnxt_init_ring_struct()
2508 		rxr = bnapi->rx_ring;  in bnxt_init_ring_struct()
2529 		txr = bnapi->tx_ring;  in bnxt_init_ring_struct()
2655 		struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;  in bnxt_init_cp_rings()
3088 	if (!bp->bnapi)  in bnxt_free_stats()
3094 		struct bnxt_napi *bnapi = bp->bnapi[i];  in bnxt_free_stats()  local
3095 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;  in bnxt_free_stats()
3113 		struct bnxt_napi *bnapi = bp->bnapi[i];  in bnxt_alloc_stats()  local
3114 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;  in bnxt_alloc_stats()
3164 	if (!bp->bnapi)  in bnxt_clear_ring_indices()
3168 		struct bnxt_napi *bnapi = bp->bnapi[i];  in bnxt_clear_ring_indices()  local
3173 		if (!bnapi)  in bnxt_clear_ring_indices()
3176 		cpr = &bnapi->cp_ring;  in bnxt_clear_ring_indices()
3179 		txr = bnapi->tx_ring;  in bnxt_clear_ring_indices()
3185 		rxr = bnapi->rx_ring;  in bnxt_clear_ring_indices()
3264 		kfree(bp->bnapi);  in bnxt_free_mem()
3265 		bp->bnapi = NULL;  in bnxt_free_mem()
3274 	void *bnapi;  in bnxt_alloc_mem()  local
3283 		bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL);  in bnxt_alloc_mem()
3284 		if (!bnapi)  in bnxt_alloc_mem()
3287 		bp->bnapi = bnapi;  in bnxt_alloc_mem()
3288 		bnapi += arr_size;  in bnxt_alloc_mem()
3289 		for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) {  in bnxt_alloc_mem()
3290 			bp->bnapi[i] = bnapi;  in bnxt_alloc_mem()
3291 			bp->bnapi[i]->index = i;  in bnxt_alloc_mem()
3292 			bp->bnapi[i]->bp = bp;  in bnxt_alloc_mem()
3302 			bp->rx_ring[i].bnapi = bp->bnapi[i];  in bnxt_alloc_mem()
3303 			bp->bnapi[i]->rx_ring = &bp->rx_ring[i];  in bnxt_alloc_mem()
3324 			bp->tx_ring[i].bnapi = bp->bnapi[j];  in bnxt_alloc_mem()
3325 			bp->bnapi[j]->tx_ring = &bp->tx_ring[i];  in bnxt_alloc_mem()
3330 				bp->bnapi[j]->tx_int = bnxt_tx_int;  in bnxt_alloc_mem()
3332 				bp->bnapi[j]->flags |= BNXT_NAPI_FLAG_XDP;  in bnxt_alloc_mem()
3333 				bp->bnapi[j]->tx_int = bnxt_tx_int_xdp;  in bnxt_alloc_mem()
3380 	if (!bp->bnapi)  in bnxt_disable_int()
3384 		struct bnxt_napi *bnapi = bp->bnapi[i];  in bnxt_disable_int()  local
3385 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;  in bnxt_disable_int()
3395 	struct bnxt_napi *bnapi = bp->bnapi[n];  in bnxt_cp_num_to_irq_num()  local
3398 	cpr = &bnapi->cp_ring;  in bnxt_cp_num_to_irq_num()
3422 		struct bnxt_napi *bnapi = bp->bnapi[i];  in bnxt_enable_int()  local
3423 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;  in bnxt_enable_int()
4143 	grp_idx = bp->rx_ring[ring].bnapi->index;  in bnxt_hwrm_vnic_cfg()
4201 		grp_idx = bp->rx_ring[i].bnapi->index;  in bnxt_hwrm_vnic_alloc()
4261 		unsigned int grp_idx = bp->rx_ring[i].bnapi->index;  in bnxt_hwrm_ring_grp_alloc()
4409 		struct bnxt_napi *bnapi = bp->bnapi[i];  in bnxt_hwrm_ring_alloc()  local
4410 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;  in bnxt_hwrm_ring_alloc()
4444 		u32 map_idx = rxr->bnapi->index;  in bnxt_hwrm_ring_alloc()
4509 	if (!bp->bnapi)  in bnxt_hwrm_ring_free()
4515 		u32 grp_idx = txr->bnapi->index;  in bnxt_hwrm_ring_free()
4530 		u32 grp_idx = rxr->bnapi->index;  in bnxt_hwrm_ring_free()
4547 		u32 grp_idx = rxr->bnapi->index;  in bnxt_hwrm_ring_free()
4568 		struct bnxt_napi *bnapi = bp->bnapi[i];  in bnxt_hwrm_ring_free()  local
4569 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;  in bnxt_hwrm_ring_free()
4960 int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi)  in bnxt_hwrm_set_ring_coal()  argument
4963 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;  in bnxt_hwrm_set_ring_coal()
4975 	if (!bnapi->rx_ring)  in bnxt_hwrm_set_ring_coal()
4983 	grp_idx = bnapi->index;  in bnxt_hwrm_set_ring_coal()
5006 		struct bnxt_napi *bnapi = bp->bnapi[i];  in bnxt_hwrm_set_coal()  local
5009 		if (!bnapi->rx_ring)  in bnxt_hwrm_set_coal()
5027 	if (!bp->bnapi)  in bnxt_hwrm_stat_ctx_free()
5037 		struct bnxt_napi *bnapi = bp->bnapi[i];  in bnxt_hwrm_stat_ctx_free()  local
5038 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;  in bnxt_hwrm_stat_ctx_free()
5070 		struct bnxt_napi *bnapi = bp->bnapi[i];  in bnxt_hwrm_stat_ctx_alloc()  local
5071 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;  in bnxt_hwrm_stat_ctx_alloc()
6113 	if (!bp->irq_tbl || !bp->bnapi)  in bnxt_free_irq()
6126 			free_irq(irq->vector, bp->bnapi[i]);  in bnxt_free_irq()
6158 		if (rmap && bp->bnapi[i]->rx_ring) {  in bnxt_request_irq()
6167 				 bp->bnapi[i]);  in bnxt_request_irq()
6195 	if (!bp->bnapi)  in bnxt_del_napi()
6199 		struct bnxt_napi *bnapi = bp->bnapi[i];  in bnxt_del_napi()  local
6201 		napi_hash_del(&bnapi->napi);  in bnxt_del_napi()
6202 		netif_napi_del(&bnapi->napi);  in bnxt_del_napi()
6214 	struct bnxt_napi *bnapi;  in bnxt_init_napi()  local
6220 			bnapi = bp->bnapi[i];  in bnxt_init_napi()
6221 			netif_napi_add(bp->dev, &bnapi->napi,  in bnxt_init_napi()
6225 			bnapi = bp->bnapi[cp_nr_rings];  in bnxt_init_napi()
6226 			netif_napi_add(bp->dev, &bnapi->napi,  in bnxt_init_napi()
6230 		bnapi = bp->bnapi[0];  in bnxt_init_napi()
6231 		netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll, 64);  in bnxt_init_napi()
6239 	if (!bp->bnapi)  in bnxt_disable_napi()
6243 		struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;  in bnxt_disable_napi()
6245 		if (bp->bnapi[i]->rx_ring)  in bnxt_disable_napi()
6248 		napi_disable(&bp->bnapi[i]->napi);  in bnxt_disable_napi()
6257 		struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;  in bnxt_enable_napi()
6258 		bp->bnapi[i]->in_reset = false;  in bnxt_enable_napi()
6260 		if (bp->bnapi[i]->rx_ring) {  in bnxt_enable_napi()
6264 		napi_enable(&bp->bnapi[i]->napi);  in bnxt_enable_napi()
7233 		struct bnxt_napi *bnapi = bp->bnapi[i];  in bnxt_get_stats64()  local
7234 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;  in bnxt_get_stats64()
7607 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi)  in bnxt_dump_tx_sw_state()  argument
7609 	struct bnxt_tx_ring_info *txr = bnapi->tx_ring;  in bnxt_dump_tx_sw_state()
7610 	int i = bnapi->index;  in bnxt_dump_tx_sw_state()
7615 	netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n",  in bnxt_dump_tx_sw_state()
7620 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi)  in bnxt_dump_rx_sw_state()  argument
7622 	struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;  in bnxt_dump_rx_sw_state()
7623 	int i = bnapi->index;  in bnxt_dump_rx_sw_state()
7628 …netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg…  in bnxt_dump_rx_sw_state()
7634 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi)  in bnxt_dump_cp_sw_state()  argument
7636 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;  in bnxt_dump_cp_sw_state()
7637 	int i = bnapi->index;  in bnxt_dump_cp_sw_state()
7639 	netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n",  in bnxt_dump_cp_sw_state()
7646 	struct bnxt_napi *bnapi;  in bnxt_dbg_dump_states()  local
7649 		bnapi = bp->bnapi[i];  in bnxt_dbg_dump_states()
7651 			bnxt_dump_tx_sw_state(bnapi);  in bnxt_dbg_dump_states()
7652 			bnxt_dump_rx_sw_state(bnapi);  in bnxt_dbg_dump_states()
7653 			bnxt_dump_cp_sw_state(bnapi);  in bnxt_dbg_dump_states()