63 static int ena_clean_xdp_irq(struct ena_ring *xdp_ring, u32 budget);
187 	struct ena_ring *xdp_ring;  in ena_xdp_io_poll()  local
191 	xdp_ring = ena_napi->xdp_ring;  in ena_xdp_io_poll()
192 	xdp_ring->first_interrupt = ena_napi->first_interrupt;  in ena_xdp_io_poll()
196 	if (!test_bit(ENA_FLAG_DEV_UP, &xdp_ring->adapter->flags) ||  in ena_xdp_io_poll()
197 	    test_bit(ENA_FLAG_TRIGGER_RESET, &xdp_ring->adapter->flags)) {  in ena_xdp_io_poll()
202 	xdp_work_done = ena_clean_xdp_irq(xdp_ring, xdp_budget);  in ena_xdp_io_poll()
207 	if (unlikely(!test_bit(ENA_FLAG_DEV_UP, &xdp_ring->adapter->flags))) {  in ena_xdp_io_poll()
213 			ena_unmask_interrupt(xdp_ring, NULL);  in ena_xdp_io_poll()
214 		ena_update_ring_numa_node(xdp_ring, NULL);  in ena_xdp_io_poll()
220 	u64_stats_update_begin(&xdp_ring->syncp);  in ena_xdp_io_poll()
221 	xdp_ring->tx_stats.napi_comp += napi_comp_call;  in ena_xdp_io_poll()
222 	xdp_ring->tx_stats.tx_poll++;  in ena_xdp_io_poll()
223 	u64_stats_update_end(&xdp_ring->syncp);  in ena_xdp_io_poll()
228 static int ena_xdp_tx_map_buff(struct ena_ring *xdp_ring,  in ena_xdp_tx_map_buff()  argument
234 	struct ena_adapter *adapter = xdp_ring->adapter;  in ena_xdp_tx_map_buff()
244 	*push_len = min_t(u32, size, xdp_ring->tx_max_header_size);  in ena_xdp_tx_map_buff()
248 		dma = dma_map_single(xdp_ring->dev,  in ena_xdp_tx_map_buff()
252 		if (unlikely(dma_mapping_error(xdp_ring->dev, dma)))  in ena_xdp_tx_map_buff()
265 	u64_stats_update_begin(&xdp_ring->syncp);  in ena_xdp_tx_map_buff()
266 	xdp_ring->tx_stats.dma_mapping_err++;  in ena_xdp_tx_map_buff()
267 	u64_stats_update_end(&xdp_ring->syncp);  in ena_xdp_tx_map_buff()
285 	struct ena_ring *xdp_ring;  in ena_xdp_xmit_buff()  local
291 	xdp_ring = &adapter->tx_ring[qid];  in ena_xdp_xmit_buff()
292 	next_to_use = xdp_ring->next_to_use;  in ena_xdp_xmit_buff()
293 	req_id = xdp_ring->free_ids[next_to_use];  in ena_xdp_xmit_buff()
294 	tx_info = &xdp_ring->tx_buffer_info[req_id];  in ena_xdp_xmit_buff()
299 	rc = ena_xdp_tx_map_buff(xdp_ring, tx_info, xdp, &push_hdr, &push_len);  in ena_xdp_xmit_buff()
310 			     xdp_ring,  in ena_xdp_xmit_buff()
320 	ena_com_write_sq_doorbell(xdp_ring->ena_com_io_sq);  in ena_xdp_xmit_buff()
321 	u64_stats_update_begin(&xdp_ring->syncp);  in ena_xdp_xmit_buff()
322 	xdp_ring->tx_stats.doorbells++;  in ena_xdp_xmit_buff()
323 	u64_stats_update_end(&xdp_ring->syncp);  in ena_xdp_xmit_buff()
328 	ena_unmap_tx_buff(xdp_ring, tx_info);  in ena_xdp_xmit_buff()
1213 static int validate_xdp_req_id(struct ena_ring *xdp_ring, u16 req_id)  in validate_xdp_req_id()  argument
1217 	if (likely(req_id < xdp_ring->ring_size)) {  in validate_xdp_req_id()
1218 		tx_info = &xdp_ring->tx_buffer_info[req_id];  in validate_xdp_req_id()
1223 	return handle_invalid_req_id(xdp_ring, req_id, tx_info, true);  in validate_xdp_req_id()
1785 static int ena_clean_xdp_irq(struct ena_ring *xdp_ring, u32 budget)  in ena_clean_xdp_irq()  argument
1794 	if (unlikely(!xdp_ring))  in ena_clean_xdp_irq()
1796 	next_to_clean = xdp_ring->next_to_clean;  in ena_clean_xdp_irq()
1802 		rc = ena_com_tx_comp_req_id_get(xdp_ring->ena_com_io_cq,  in ena_clean_xdp_irq()
1807 		rc = validate_xdp_req_id(xdp_ring, req_id);  in ena_clean_xdp_irq()
1811 		tx_info = &xdp_ring->tx_buffer_info[req_id];  in ena_clean_xdp_irq()
1816 		ena_unmap_tx_buff(xdp_ring, tx_info);  in ena_clean_xdp_irq()
1818 		netif_dbg(xdp_ring->adapter, tx_done, xdp_ring->netdev,  in ena_clean_xdp_irq()
1819 			  "tx_poll: q %d skb %p completed\n", xdp_ring->qid,  in ena_clean_xdp_irq()
1827 		xdp_ring->free_ids[next_to_clean] = req_id;  in ena_clean_xdp_irq()
1829 						     xdp_ring->ring_size);  in ena_clean_xdp_irq()
1832 	xdp_ring->next_to_clean = next_to_clean;  in ena_clean_xdp_irq()
1833 	ena_com_comp_ack(xdp_ring->ena_com_io_sq, total_done);  in ena_clean_xdp_irq()
1834 	ena_com_update_dev_comp_head(xdp_ring->ena_com_io_cq);  in ena_clean_xdp_irq()
1836 	netif_dbg(xdp_ring->adapter, tx_done, xdp_ring->netdev,  in ena_clean_xdp_irq()
1838 		  xdp_ring->qid, tx_pkts);  in ena_clean_xdp_irq()
2161 			adapter->ena_napi[i].xdp_ring);  in ena_del_napi_in_range()
2182 			napi->xdp_ring = &adapter->tx_ring[i];  in ena_init_napi_in_range()