178 		txr = &adapter->tx_ring[i];  in ena_init_io_rings()
211 	struct ena_ring *tx_ring = &adapter->tx_ring[qid];  in ena_setup_tx_resources()  local
215 	if (tx_ring->tx_buffer_info) {  in ena_setup_tx_resources()
221 	size = sizeof(struct ena_tx_buffer) * tx_ring->ring_size;  in ena_setup_tx_resources()
224 	tx_ring->tx_buffer_info = vzalloc_node(size, node);  in ena_setup_tx_resources()
225 	if (!tx_ring->tx_buffer_info) {  in ena_setup_tx_resources()
226 		tx_ring->tx_buffer_info = vzalloc(size);  in ena_setup_tx_resources()
227 		if (!tx_ring->tx_buffer_info)  in ena_setup_tx_resources()
231 	size = sizeof(u16) * tx_ring->ring_size;  in ena_setup_tx_resources()
232 	tx_ring->free_tx_ids = vzalloc_node(size, node);  in ena_setup_tx_resources()
233 	if (!tx_ring->free_tx_ids) {  in ena_setup_tx_resources()
234 		tx_ring->free_tx_ids = vzalloc(size);  in ena_setup_tx_resources()
235 		if (!tx_ring->free_tx_ids) {  in ena_setup_tx_resources()
236 			vfree(tx_ring->tx_buffer_info);  in ena_setup_tx_resources()
242 	for (i = 0; i < tx_ring->ring_size; i++)  in ena_setup_tx_resources()
243 		tx_ring->free_tx_ids[i] = i;  in ena_setup_tx_resources()
246 	memset(&tx_ring->tx_stats, 0x0, sizeof(tx_ring->tx_stats));  in ena_setup_tx_resources()
248 	tx_ring->next_to_use = 0;  in ena_setup_tx_resources()
249 	tx_ring->next_to_clean = 0;  in ena_setup_tx_resources()
250 	tx_ring->cpu = ena_irq->cpu;  in ena_setup_tx_resources()
262 	struct ena_ring *tx_ring = &adapter->tx_ring[qid];  in ena_free_tx_resources()  local
264 	vfree(tx_ring->tx_buffer_info);  in ena_free_tx_resources()
265 	tx_ring->tx_buffer_info = NULL;  in ena_free_tx_resources()
267 	vfree(tx_ring->free_tx_ids);  in ena_free_tx_resources()
268 	tx_ring->free_tx_ids = NULL;  in ena_free_tx_resources()
609 static void ena_free_tx_bufs(struct ena_ring *tx_ring)  in ena_free_tx_bufs()  argument
614 	for (i = 0; i < tx_ring->ring_size; i++) {  in ena_free_tx_bufs()
615 		struct ena_tx_buffer *tx_info = &tx_ring->tx_buffer_info[i];  in ena_free_tx_bufs()
624 			netdev_notice(tx_ring->netdev,  in ena_free_tx_bufs()
626 				      tx_ring->qid, i);  in ena_free_tx_bufs()
629 			netdev_dbg(tx_ring->netdev,  in ena_free_tx_bufs()
631 				   tx_ring->qid, i);  in ena_free_tx_bufs()
635 		dma_unmap_single(tx_ring->dev,  in ena_free_tx_bufs()
644 			dma_unmap_page(tx_ring->dev,  in ena_free_tx_bufs()
652 	netdev_tx_reset_queue(netdev_get_tx_queue(tx_ring->netdev,  in ena_free_tx_bufs()
653 						  tx_ring->qid));  in ena_free_tx_bufs()
658 	struct ena_ring *tx_ring;  in ena_free_all_tx_bufs()  local
662 		tx_ring = &adapter->tx_ring[i];  in ena_free_all_tx_bufs()
663 		ena_free_tx_bufs(tx_ring);  in ena_free_all_tx_bufs()
695 static int validate_tx_req_id(struct ena_ring *tx_ring, u16 req_id)  in validate_tx_req_id()  argument
699 	if (likely(req_id < tx_ring->ring_size)) {  in validate_tx_req_id()
700 		tx_info = &tx_ring->tx_buffer_info[req_id];  in validate_tx_req_id()
706 		netif_err(tx_ring->adapter, tx_done, tx_ring->netdev,  in validate_tx_req_id()
709 		netif_err(tx_ring->adapter, tx_done, tx_ring->netdev,  in validate_tx_req_id()
712 	u64_stats_update_begin(&tx_ring->syncp);  in validate_tx_req_id()
713 	tx_ring->tx_stats.bad_req_id++;  in validate_tx_req_id()
714 	u64_stats_update_end(&tx_ring->syncp);  in validate_tx_req_id()
717 	tx_ring->adapter->reset_reason = ENA_REGS_RESET_INV_TX_REQ_ID;  in validate_tx_req_id()
718 	set_bit(ENA_FLAG_TRIGGER_RESET, &tx_ring->adapter->flags);  in validate_tx_req_id()
722 static int ena_clean_tx_irq(struct ena_ring *tx_ring, u32 budget)  in ena_clean_tx_irq()  argument
733 	next_to_clean = tx_ring->next_to_clean;  in ena_clean_tx_irq()
734 	txq = netdev_get_tx_queue(tx_ring->netdev, tx_ring->qid);  in ena_clean_tx_irq()
742 		rc = ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq,  in ena_clean_tx_irq()
747 		rc = validate_tx_req_id(tx_ring, req_id);  in ena_clean_tx_irq()
751 		tx_info = &tx_ring->tx_buffer_info[req_id];  in ena_clean_tx_irq()
763 			dma_unmap_single(tx_ring->dev,  in ena_clean_tx_irq()
772 				dma_unmap_page(tx_ring->dev,  in ena_clean_tx_irq()
779 		netif_dbg(tx_ring->adapter, tx_done, tx_ring->netdev,  in ena_clean_tx_irq()
780 			  "tx_poll: q %d skb %p completed\n", tx_ring->qid,  in ena_clean_tx_irq()
788 		tx_ring->free_tx_ids[next_to_clean] = req_id;  in ena_clean_tx_irq()
790 						     tx_ring->ring_size);  in ena_clean_tx_irq()
793 	tx_ring->next_to_clean = next_to_clean;  in ena_clean_tx_irq()
794 	ena_com_comp_ack(tx_ring->ena_com_io_sq, total_done);  in ena_clean_tx_irq()
795 	ena_com_update_dev_comp_head(tx_ring->ena_com_io_cq);  in ena_clean_tx_irq()
799 	netif_dbg(tx_ring->adapter, tx_done, tx_ring->netdev,  in ena_clean_tx_irq()
801 		  tx_ring->qid, tx_pkts);  in ena_clean_tx_irq()
808 	above_thresh = ena_com_sq_empty_space(tx_ring->ena_com_io_sq) >  in ena_clean_tx_irq()
812 		above_thresh = ena_com_sq_empty_space(tx_ring->ena_com_io_sq) >  in ena_clean_tx_irq()
816 			u64_stats_update_begin(&tx_ring->syncp);  in ena_clean_tx_irq()
817 			tx_ring->tx_stats.queue_wakeup++;  in ena_clean_tx_irq()
818 			u64_stats_update_end(&tx_ring->syncp);  in ena_clean_tx_irq()
823 	tx_ring->per_napi_bytes += tx_bytes;  in ena_clean_tx_irq()
824 	tx_ring->per_napi_packets += tx_pkts;  in ena_clean_tx_irq()
1131 				       struct ena_ring *tx_ring)  in ena_adjust_intr_moderation()  argument
1143 	tx_ring->per_napi_packets = 0;  in ena_adjust_intr_moderation()
1144 	tx_ring->per_napi_bytes = 0;  in ena_adjust_intr_moderation()
1149 static inline void ena_unmask_interrupt(struct ena_ring *tx_ring,  in ena_unmask_interrupt()  argument
1159 				tx_ring->smoothed_interval,  in ena_unmask_interrupt()
1169 static inline void ena_update_ring_numa_node(struct ena_ring *tx_ring,  in ena_update_ring_numa_node()  argument
1176 	if (likely(tx_ring->cpu == cpu))  in ena_update_ring_numa_node()
1183 		ena_com_update_numa_node(tx_ring->ena_com_io_cq, numa_node);  in ena_update_ring_numa_node()
1187 	tx_ring->cpu = cpu;  in ena_update_ring_numa_node()
1198 	struct ena_ring *tx_ring, *rx_ring;  in ena_io_poll()  local
1206 	tx_ring = ena_napi->tx_ring;  in ena_io_poll()
1209 	tx_budget = tx_ring->ring_size / ENA_TX_POLL_BUDGET_DIVIDER;  in ena_io_poll()
1211 	if (!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags) ||  in ena_io_poll()
1212 	    test_bit(ENA_FLAG_TRIGGER_RESET, &tx_ring->adapter->flags)) {  in ena_io_poll()
1217 	tx_work_done = ena_clean_tx_irq(tx_ring, tx_budget);  in ena_io_poll()
1223 	if (unlikely(!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags) ||  in ena_io_poll()
1224 		     test_bit(ENA_FLAG_TRIGGER_RESET, &tx_ring->adapter->flags))) {  in ena_io_poll()
1237 				ena_adjust_intr_moderation(rx_ring, tx_ring);  in ena_io_poll()
1239 			ena_unmask_interrupt(tx_ring, rx_ring);  in ena_io_poll()
1242 		ena_update_ring_numa_node(tx_ring, rx_ring);  in ena_io_poll()
1249 	u64_stats_update_begin(&tx_ring->syncp);  in ena_io_poll()
1250 	tx_ring->tx_stats.napi_comp += napi_comp_call;  in ena_io_poll()
1251 	tx_ring->tx_stats.tx_poll++;  in ena_io_poll()
1252 	u64_stats_update_end(&tx_ring->syncp);  in ena_io_poll()
1278 	ena_napi->tx_ring->first_interrupt = true;  in ena_intr_msix_io()
1507 		napi->tx_ring = &adapter->tx_ring[i];  in ena_init_napi()
1596 	struct ena_ring *tx_ring;  in ena_create_io_tx_queue()  local
1603 	tx_ring = &adapter->tx_ring[qid];  in ena_create_io_tx_queue()
1612 	ctx.numa_node = cpu_to_node(tx_ring->cpu);  in ena_create_io_tx_queue()
1623 				     &tx_ring->ena_com_io_sq,  in ena_create_io_tx_queue()
1624 				     &tx_ring->ena_com_io_cq);  in ena_create_io_tx_queue()
1633 	ena_com_update_numa_node(tx_ring->ena_com_io_cq, ctx.numa_node);  in ena_create_io_tx_queue()
1777 		ena_unmask_interrupt(&adapter->tx_ring[i],  in ena_up()
1960 static int ena_check_and_linearize_skb(struct ena_ring *tx_ring,  in ena_check_and_linearize_skb()  argument
1968 	if (num_frags < tx_ring->sgl_size)  in ena_check_and_linearize_skb()
1971 	if ((num_frags == tx_ring->sgl_size) &&  in ena_check_and_linearize_skb()
1972 	    (header_len < tx_ring->tx_max_header_size))  in ena_check_and_linearize_skb()
1975 	u64_stats_update_begin(&tx_ring->syncp);  in ena_check_and_linearize_skb()
1976 	tx_ring->tx_stats.linearize++;  in ena_check_and_linearize_skb()
1977 	u64_stats_update_end(&tx_ring->syncp);  in ena_check_and_linearize_skb()
1981 		u64_stats_update_begin(&tx_ring->syncp);  in ena_check_and_linearize_skb()
1982 		tx_ring->tx_stats.linearize_failed++;  in ena_check_and_linearize_skb()
1983 		u64_stats_update_end(&tx_ring->syncp);  in ena_check_and_linearize_skb()
1995 	struct ena_ring *tx_ring;  in ena_start_xmit()  local
2011 	tx_ring = &adapter->tx_ring[qid];  in ena_start_xmit()
2014 	rc = ena_check_and_linearize_skb(tx_ring, skb);  in ena_start_xmit()
2021 	next_to_use = tx_ring->next_to_use;  in ena_start_xmit()
2022 	req_id = tx_ring->free_tx_ids[next_to_use];  in ena_start_xmit()
2023 	tx_info = &tx_ring->tx_buffer_info[req_id];  in ena_start_xmit()
2030 	if (tx_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {  in ena_start_xmit()
2032 		push_len = min_t(u32, len, tx_ring->tx_max_header_size);  in ena_start_xmit()
2037 		header_len = min_t(u32, len, tx_ring->tx_max_header_size);  in ena_start_xmit()
2046 		dma = dma_map_single(tx_ring->dev, skb->data + push_len,  in ena_start_xmit()
2048 		if (dma_mapping_error(tx_ring->dev, dma))  in ena_start_xmit()
2064 		dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len,  in ena_start_xmit()
2066 		if (dma_mapping_error(tx_ring->dev, dma))  in ena_start_xmit()
2087 	rc = ena_com_prepare_tx(tx_ring->ena_com_io_sq, &ena_tx_ctx,  in ena_start_xmit()
2093 		u64_stats_update_begin(&tx_ring->syncp);  in ena_start_xmit()
2094 		tx_ring->tx_stats.queue_stop++;  in ena_start_xmit()
2095 		tx_ring->tx_stats.prepare_ctx_err++;  in ena_start_xmit()
2096 		u64_stats_update_end(&tx_ring->syncp);  in ena_start_xmit()
2103 	u64_stats_update_begin(&tx_ring->syncp);  in ena_start_xmit()
2104 	tx_ring->tx_stats.cnt++;  in ena_start_xmit()
2105 	tx_ring->tx_stats.bytes += skb->len;  in ena_start_xmit()
2106 	u64_stats_update_end(&tx_ring->syncp);  in ena_start_xmit()
2112 	tx_ring->next_to_use = ENA_TX_RING_IDX_NEXT(next_to_use,  in ena_start_xmit()
2113 		tx_ring->ring_size);  in ena_start_xmit()
2119 	if (unlikely(ena_com_sq_empty_space(tx_ring->ena_com_io_sq) <  in ena_start_xmit()
2120 		     (tx_ring->sgl_size + 2))) {  in ena_start_xmit()
2125 		u64_stats_update_begin(&tx_ring->syncp);  in ena_start_xmit()
2126 		tx_ring->tx_stats.queue_stop++;  in ena_start_xmit()
2127 		u64_stats_update_end(&tx_ring->syncp);  in ena_start_xmit()
2139 		if (ena_com_sq_empty_space(tx_ring->ena_com_io_sq)  in ena_start_xmit()
2142 			u64_stats_update_begin(&tx_ring->syncp);  in ena_start_xmit()
2143 			tx_ring->tx_stats.queue_wakeup++;  in ena_start_xmit()
2144 			u64_stats_update_end(&tx_ring->syncp);  in ena_start_xmit()
2152 		ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq);  in ena_start_xmit()
2153 		u64_stats_update_begin(&tx_ring->syncp);  in ena_start_xmit()
2154 		tx_ring->tx_stats.doorbells++;  in ena_start_xmit()
2155 		u64_stats_update_end(&tx_ring->syncp);  in ena_start_xmit()
2161 	u64_stats_update_begin(&tx_ring->syncp);  in ena_start_xmit()
2162 	tx_ring->tx_stats.dma_mapping_err++;  in ena_start_xmit()
2163 	u64_stats_update_end(&tx_ring->syncp);  in ena_start_xmit()
2176 		dma_unmap_single(tx_ring->dev, dma_unmap_addr(ena_buf, paddr),  in ena_start_xmit()
2182 			dma_unmap_page(tx_ring->dev, dma_unmap_addr(ena_buf, paddr),  in ena_start_xmit()
2293 	struct ena_ring *rx_ring, *tx_ring;  in ena_get_stats64()  local
2304 		tx_ring = &adapter->tx_ring[i];  in ena_get_stats64()
2307 			start = u64_stats_fetch_begin_irq(&tx_ring->syncp);  in ena_get_stats64()
2308 			packets = tx_ring->tx_stats.cnt;  in ena_get_stats64()
2309 			bytes = tx_ring->tx_stats.bytes;  in ena_get_stats64()
2310 		} while (u64_stats_fetch_retry_irq(&tx_ring->syncp, start));  in ena_get_stats64()
2683 					  struct ena_ring *tx_ring)  in check_missing_comp_in_tx_queue()  argument
2690 	for (i = 0; i < tx_ring->ring_size; i++) {  in check_missing_comp_in_tx_queue()
2691 		tx_buf = &tx_ring->tx_buffer_info[i];  in check_missing_comp_in_tx_queue()
2698 		if (unlikely(!tx_ring->first_interrupt && time_is_before_jiffies(last_jiffies +  in check_missing_comp_in_tx_queue()
2705 				  tx_ring->qid);  in check_missing_comp_in_tx_queue()
2717 					     tx_ring->qid, i);  in check_missing_comp_in_tx_queue()
2735 	u64_stats_update_begin(&tx_ring->syncp);  in check_missing_comp_in_tx_queue()
2736 	tx_ring->tx_stats.missed_tx = missed_tx;  in check_missing_comp_in_tx_queue()
2737 	u64_stats_update_end(&tx_ring->syncp);  in check_missing_comp_in_tx_queue()
2744 	struct ena_ring *tx_ring;  in check_for_missing_completions()  local
2763 		tx_ring = &adapter->tx_ring[i];  in check_for_missing_completions()
2766 		rc = check_missing_comp_in_tx_queue(adapter, tx_ring);  in check_for_missing_completions()