69 static void ena_unmask_interrupt(struct ena_ring *tx_ring,
71 static void ena_update_ring_numa_node(struct ena_ring *tx_ring,
73 static void ena_unmap_tx_buff(struct ena_ring *tx_ring,
87 static void ena_ring_tx_doorbell(struct ena_ring *tx_ring)  in ena_ring_tx_doorbell()  argument
89 	ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq);  in ena_ring_tx_doorbell()
90 	ena_increase_stat(&tx_ring->tx_stats.doorbells, 1, &tx_ring->syncp);  in ena_ring_tx_doorbell()
354 	xdp_ring = &adapter->tx_ring[qid];  in ena_xdp_xmit()
689 		txr = &adapter->tx_ring[i];  in ena_init_io_rings()
719 			rxr->xdp_ring = &adapter->tx_ring[i + adapter->num_io_queues];  in ena_init_io_rings()
732 	struct ena_ring *tx_ring = &adapter->tx_ring[qid];  in ena_setup_tx_resources()  local
736 	if (tx_ring->tx_buffer_info) {  in ena_setup_tx_resources()
742 	size = sizeof(struct ena_tx_buffer) * tx_ring->ring_size;  in ena_setup_tx_resources()
745 	tx_ring->tx_buffer_info = vzalloc_node(size, node);  in ena_setup_tx_resources()
746 	if (!tx_ring->tx_buffer_info) {  in ena_setup_tx_resources()
747 		tx_ring->tx_buffer_info = vzalloc(size);  in ena_setup_tx_resources()
748 		if (!tx_ring->tx_buffer_info)  in ena_setup_tx_resources()
752 	size = sizeof(u16) * tx_ring->ring_size;  in ena_setup_tx_resources()
753 	tx_ring->free_ids = vzalloc_node(size, node);  in ena_setup_tx_resources()
754 	if (!tx_ring->free_ids) {  in ena_setup_tx_resources()
755 		tx_ring->free_ids = vzalloc(size);  in ena_setup_tx_resources()
756 		if (!tx_ring->free_ids)  in ena_setup_tx_resources()
760 	size = tx_ring->tx_max_header_size;  in ena_setup_tx_resources()
761 	tx_ring->push_buf_intermediate_buf = vzalloc_node(size, node);  in ena_setup_tx_resources()
762 	if (!tx_ring->push_buf_intermediate_buf) {  in ena_setup_tx_resources()
763 		tx_ring->push_buf_intermediate_buf = vzalloc(size);  in ena_setup_tx_resources()
764 		if (!tx_ring->push_buf_intermediate_buf)  in ena_setup_tx_resources()
769 	for (i = 0; i < tx_ring->ring_size; i++)  in ena_setup_tx_resources()
770 		tx_ring->free_ids[i] = i;  in ena_setup_tx_resources()
773 	memset(&tx_ring->tx_stats, 0x0, sizeof(tx_ring->tx_stats));  in ena_setup_tx_resources()
775 	tx_ring->next_to_use = 0;  in ena_setup_tx_resources()
776 	tx_ring->next_to_clean = 0;  in ena_setup_tx_resources()
777 	tx_ring->cpu = ena_irq->cpu;  in ena_setup_tx_resources()
781 	vfree(tx_ring->free_ids);  in ena_setup_tx_resources()
782 	tx_ring->free_ids = NULL;  in ena_setup_tx_resources()
784 	vfree(tx_ring->tx_buffer_info);  in ena_setup_tx_resources()
785 	tx_ring->tx_buffer_info = NULL;  in ena_setup_tx_resources()
798 	struct ena_ring *tx_ring = &adapter->tx_ring[qid];  in ena_free_tx_resources()  local
800 	vfree(tx_ring->tx_buffer_info);  in ena_free_tx_resources()
801 	tx_ring->tx_buffer_info = NULL;  in ena_free_tx_resources()
803 	vfree(tx_ring->free_ids);  in ena_free_tx_resources()
804 	tx_ring->free_ids = NULL;  in ena_free_tx_resources()
806 	vfree(tx_ring->push_buf_intermediate_buf);  in ena_free_tx_resources()
807 	tx_ring->push_buf_intermediate_buf = NULL;  in ena_free_tx_resources()
1158 static void ena_unmap_tx_buff(struct ena_ring *tx_ring,  in ena_unmap_tx_buff()  argument
1172 		dma_unmap_single(tx_ring->dev,  in ena_unmap_tx_buff()
1182 		dma_unmap_page(tx_ring->dev, dma_unmap_addr(ena_buf, paddr),  in ena_unmap_tx_buff()
1191 static void ena_free_tx_bufs(struct ena_ring *tx_ring)  in ena_free_tx_bufs()  argument
1196 	for (i = 0; i < tx_ring->ring_size; i++) {  in ena_free_tx_bufs()
1197 		struct ena_tx_buffer *tx_info = &tx_ring->tx_buffer_info[i];  in ena_free_tx_bufs()
1203 			netif_notice(tx_ring->adapter, ifdown, tx_ring->netdev,  in ena_free_tx_bufs()
1205 				     tx_ring->qid, i);  in ena_free_tx_bufs()
1208 			netif_dbg(tx_ring->adapter, ifdown, tx_ring->netdev,  in ena_free_tx_bufs()
1210 				  tx_ring->qid, i);  in ena_free_tx_bufs()
1213 		ena_unmap_tx_buff(tx_ring, tx_info);  in ena_free_tx_bufs()
1217 	netdev_tx_reset_queue(netdev_get_tx_queue(tx_ring->netdev,  in ena_free_tx_bufs()
1218 						  tx_ring->qid));  in ena_free_tx_bufs()
1223 	struct ena_ring *tx_ring;  in ena_free_all_tx_bufs()  local
1227 		tx_ring = &adapter->tx_ring[i];  in ena_free_all_tx_bufs()
1228 		ena_free_tx_bufs(tx_ring);  in ena_free_all_tx_bufs()
1283 static int validate_tx_req_id(struct ena_ring *tx_ring, u16 req_id)  in validate_tx_req_id()  argument
1287 	tx_info = &tx_ring->tx_buffer_info[req_id];  in validate_tx_req_id()
1291 	return handle_invalid_req_id(tx_ring, req_id, tx_info, false);  in validate_tx_req_id()
1305 static int ena_clean_tx_irq(struct ena_ring *tx_ring, u32 budget)  in ena_clean_tx_irq()  argument
1316 	next_to_clean = tx_ring->next_to_clean;  in ena_clean_tx_irq()
1317 	txq = netdev_get_tx_queue(tx_ring->netdev, tx_ring->qid);  in ena_clean_tx_irq()
1323 		rc = ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq,  in ena_clean_tx_irq()
1327 				handle_invalid_req_id(tx_ring, req_id, NULL,  in ena_clean_tx_irq()
1333 		rc = validate_tx_req_id(tx_ring, req_id);  in ena_clean_tx_irq()
1337 		tx_info = &tx_ring->tx_buffer_info[req_id];  in ena_clean_tx_irq()
1346 		ena_unmap_tx_buff(tx_ring, tx_info);  in ena_clean_tx_irq()
1348 		netif_dbg(tx_ring->adapter, tx_done, tx_ring->netdev,  in ena_clean_tx_irq()
1349 			  "tx_poll: q %d skb %p completed\n", tx_ring->qid,  in ena_clean_tx_irq()
1357 		tx_ring->free_ids[next_to_clean] = req_id;  in ena_clean_tx_irq()
1359 						     tx_ring->ring_size);  in ena_clean_tx_irq()
1362 	tx_ring->next_to_clean = next_to_clean;  in ena_clean_tx_irq()
1363 	ena_com_comp_ack(tx_ring->ena_com_io_sq, total_done);  in ena_clean_tx_irq()
1364 	ena_com_update_dev_comp_head(tx_ring->ena_com_io_cq);  in ena_clean_tx_irq()
1368 	netif_dbg(tx_ring->adapter, tx_done, tx_ring->netdev,  in ena_clean_tx_irq()
1370 		  tx_ring->qid, tx_pkts);  in ena_clean_tx_irq()
1377 	above_thresh = ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,  in ena_clean_tx_irq()
1382 			ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,  in ena_clean_tx_irq()
1385 		    test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags)) {  in ena_clean_tx_irq()
1387 			ena_increase_stat(&tx_ring->tx_stats.queue_wakeup, 1,  in ena_clean_tx_irq()
1388 					  &tx_ring->syncp);  in ena_clean_tx_irq()
1814 static void ena_unmask_interrupt(struct ena_ring *tx_ring,  in ena_unmask_interrupt()  argument
1832 				tx_ring->smoothed_interval,  in ena_unmask_interrupt()
1835 	ena_increase_stat(&tx_ring->tx_stats.unmask_interrupt, 1,  in ena_unmask_interrupt()
1836 			  &tx_ring->syncp);  in ena_unmask_interrupt()
1843 	ena_com_unmask_intr(tx_ring->ena_com_io_cq, &intr_reg);  in ena_unmask_interrupt()
1846 static void ena_update_ring_numa_node(struct ena_ring *tx_ring,  in ena_update_ring_numa_node()  argument
1853 	if (likely(tx_ring->cpu == cpu))  in ena_update_ring_numa_node()
1860 		ena_com_update_numa_node(tx_ring->ena_com_io_cq, numa_node);  in ena_update_ring_numa_node()
1866 	tx_ring->cpu = cpu;  in ena_update_ring_numa_node()
1941 	struct ena_ring *tx_ring, *rx_ring;  in ena_io_poll()  local
1948 	tx_ring = ena_napi->tx_ring;  in ena_io_poll()
1951 	tx_budget = tx_ring->ring_size / ENA_TX_POLL_BUDGET_DIVIDER;  in ena_io_poll()
1953 	if (!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags) ||  in ena_io_poll()
1954 	    test_bit(ENA_FLAG_TRIGGER_RESET, &tx_ring->adapter->flags)) {  in ena_io_poll()
1959 	tx_work_done = ena_clean_tx_irq(tx_ring, tx_budget);  in ena_io_poll()
1969 	if (unlikely(!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags) ||  in ena_io_poll()
1970 		     test_bit(ENA_FLAG_TRIGGER_RESET, &tx_ring->adapter->flags))) {  in ena_io_poll()
1990 			ena_unmask_interrupt(tx_ring, rx_ring);  in ena_io_poll()
1993 		ena_update_ring_numa_node(tx_ring, rx_ring);  in ena_io_poll()
2000 	u64_stats_update_begin(&tx_ring->syncp);  in ena_io_poll()
2001 	tx_ring->tx_stats.napi_comp += napi_comp_call;  in ena_io_poll()
2002 	tx_ring->tx_stats.tx_poll++;  in ena_io_poll()
2003 	u64_stats_update_end(&tx_ring->syncp);  in ena_io_poll()
2005 	tx_ring->tx_stats.last_napi_jiffies = jiffies;  in ena_io_poll()
2273 			napi->tx_ring = &adapter->tx_ring[i];  in ena_init_napi_in_range()
2275 			napi->xdp_ring = &adapter->tx_ring[i];  in ena_init_napi_in_range()
2361 	struct ena_ring *tx_ring;  in ena_create_io_tx_queue()  local
2368 	tx_ring = &adapter->tx_ring[qid];  in ena_create_io_tx_queue()
2378 	ctx.queue_size = tx_ring->ring_size;  in ena_create_io_tx_queue()
2379 	ctx.numa_node = cpu_to_node(tx_ring->cpu);  in ena_create_io_tx_queue()
2390 				     &tx_ring->ena_com_io_sq,  in ena_create_io_tx_queue()
2391 				     &tx_ring->ena_com_io_cq);  in ena_create_io_tx_queue()
2400 	ena_com_update_numa_node(tx_ring->ena_com_io_cq, ctx.numa_node);  in ena_create_io_tx_queue()
2505 		adapter->tx_ring[i].ring_size = new_tx_size;  in set_io_rings_size()
2579 		cur_tx_ring_size = adapter->tx_ring[0].ring_size;  in create_queues_with_size_backoff()
2653 		ena_unmask_interrupt(&adapter->tx_ring[i],  in ena_up()
2899 static int ena_check_and_linearize_skb(struct ena_ring *tx_ring,  in ena_check_and_linearize_skb()  argument
2907 	if (num_frags < tx_ring->sgl_size)  in ena_check_and_linearize_skb()
2910 	if ((num_frags == tx_ring->sgl_size) &&  in ena_check_and_linearize_skb()
2911 	    (header_len < tx_ring->tx_max_header_size))  in ena_check_and_linearize_skb()
2914 	ena_increase_stat(&tx_ring->tx_stats.linearize, 1, &tx_ring->syncp);  in ena_check_and_linearize_skb()
2918 		ena_increase_stat(&tx_ring->tx_stats.linearize_failed, 1,  in ena_check_and_linearize_skb()
2919 				  &tx_ring->syncp);  in ena_check_and_linearize_skb()
2925 static int ena_tx_map_skb(struct ena_ring *tx_ring,  in ena_tx_map_skb()  argument
2931 	struct ena_adapter *adapter = tx_ring->adapter;  in ena_tx_map_skb()
2943 	if (tx_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {  in ena_tx_map_skb()
2954 		push_len = min_t(u32, skb->len, tx_ring->tx_max_header_size);  in ena_tx_map_skb()
2956 					       tx_ring->push_buf_intermediate_buf);  in ena_tx_map_skb()
2959 			ena_increase_stat(&tx_ring->tx_stats.llq_buffer_copy, 1,  in ena_tx_map_skb()
2960 					  &tx_ring->syncp);  in ena_tx_map_skb()
2967 				    tx_ring->tx_max_header_size);  in ena_tx_map_skb()
2975 		dma = dma_map_single(tx_ring->dev, skb->data + push_len,  in ena_tx_map_skb()
2977 		if (unlikely(dma_mapping_error(tx_ring->dev, dma)))  in ena_tx_map_skb()
3002 		dma = skb_frag_dma_map(tx_ring->dev, frag, delta,  in ena_tx_map_skb()
3004 		if (unlikely(dma_mapping_error(tx_ring->dev, dma)))  in ena_tx_map_skb()
3017 	ena_increase_stat(&tx_ring->tx_stats.dma_mapping_err, 1,  in ena_tx_map_skb()
3018 			  &tx_ring->syncp);  in ena_tx_map_skb()
3024 	ena_unmap_tx_buff(tx_ring, tx_info);  in ena_tx_map_skb()
3035 	struct ena_ring *tx_ring;  in ena_start_xmit()  local
3044 	tx_ring = &adapter->tx_ring[qid];  in ena_start_xmit()
3047 	rc = ena_check_and_linearize_skb(tx_ring, skb);  in ena_start_xmit()
3053 	next_to_use = tx_ring->next_to_use;  in ena_start_xmit()
3054 	req_id = tx_ring->free_ids[next_to_use];  in ena_start_xmit()
3055 	tx_info = &tx_ring->tx_buffer_info[req_id];  in ena_start_xmit()
3060 	rc = ena_tx_map_skb(tx_ring, tx_info, skb, &push_hdr, &header_len);  in ena_start_xmit()
3072 	ena_tx_csum(&ena_tx_ctx, skb, tx_ring->disable_meta_caching);  in ena_start_xmit()
3075 			     tx_ring,  in ena_start_xmit()
3089 	if (unlikely(!ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,  in ena_start_xmit()
3090 						   tx_ring->sgl_size + 2))) {  in ena_start_xmit()
3095 		ena_increase_stat(&tx_ring->tx_stats.queue_stop, 1,  in ena_start_xmit()
3096 				  &tx_ring->syncp);  in ena_start_xmit()
3108 		if (ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,  in ena_start_xmit()
3111 			ena_increase_stat(&tx_ring->tx_stats.queue_wakeup, 1,  in ena_start_xmit()
3112 					  &tx_ring->syncp);  in ena_start_xmit()
3120 		ena_ring_tx_doorbell(tx_ring);  in ena_start_xmit()
3125 	ena_unmap_tx_buff(tx_ring, tx_info);  in ena_start_xmit()
3256 	struct ena_ring *rx_ring, *tx_ring;  in ena_get_stats64()  local
3268 		tx_ring = &adapter->tx_ring[i];  in ena_get_stats64()
3271 			start = u64_stats_fetch_begin_irq(&tx_ring->syncp);  in ena_get_stats64()
3272 			packets = tx_ring->tx_stats.cnt;  in ena_get_stats64()
3273 			bytes = tx_ring->tx_stats.bytes;  in ena_get_stats64()
3274 		} while (u64_stats_fetch_retry_irq(&tx_ring->syncp, start));  in ena_get_stats64()
3704 					  struct ena_ring *tx_ring)  in check_missing_comp_in_tx_queue()  argument
3706 	struct ena_napi *ena_napi = container_of(tx_ring->napi, struct ena_napi, napi);  in check_missing_comp_in_tx_queue()
3715 	for (i = 0; i < tx_ring->ring_size; i++) {  in check_missing_comp_in_tx_queue()
3716 		tx_buf = &tx_ring->tx_buffer_info[i];  in check_missing_comp_in_tx_queue()
3732 				  tx_ring->qid);  in check_missing_comp_in_tx_queue()
3742 				time_since_last_napi = jiffies_to_usecs(jiffies - tx_ring->tx_stats.last_napi_jiffies);  in check_missing_comp_in_tx_queue()
3746 					     tx_ring->qid, i, time_since_last_napi, missing_tx_comp_to);  in check_missing_comp_in_tx_queue()
3763 	ena_increase_stat(&tx_ring->tx_stats.missed_tx, missed_tx,  in check_missing_comp_in_tx_queue()
3764 			  &tx_ring->syncp);  in check_missing_comp_in_tx_queue()
3771 	struct ena_ring *tx_ring;  in check_for_missing_completions()  local
3792 		tx_ring = &adapter->tx_ring[i];  in check_for_missing_completions()
3795 		rc = check_missing_comp_in_tx_queue(adapter, tx_ring);  in check_for_missing_completions()