69 static void ena_unmask_interrupt(struct ena_ring *tx_ring,
71 static void ena_update_ring_numa_node(struct ena_ring *tx_ring,
73 static void ena_unmap_tx_buff(struct ena_ring *tx_ring,
87 static void ena_ring_tx_doorbell(struct ena_ring *tx_ring)  in ena_ring_tx_doorbell()  argument
89 	ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq);  in ena_ring_tx_doorbell()
90 	ena_increase_stat(&tx_ring->tx_stats.doorbells, 1, &tx_ring->syncp);  in ena_ring_tx_doorbell()
354 	xdp_ring = &adapter->tx_ring[qid];  in ena_xdp_xmit()
700 		txr = &adapter->tx_ring[i];  in ena_init_io_rings()
730 			rxr->xdp_ring = &adapter->tx_ring[i + adapter->num_io_queues];  in ena_init_io_rings()
743 	struct ena_ring *tx_ring = &adapter->tx_ring[qid];  in ena_setup_tx_resources()  local
747 	if (tx_ring->tx_buffer_info) {  in ena_setup_tx_resources()
753 	size = sizeof(struct ena_tx_buffer) * tx_ring->ring_size;  in ena_setup_tx_resources()
756 	tx_ring->tx_buffer_info = vzalloc_node(size, node);  in ena_setup_tx_resources()
757 	if (!tx_ring->tx_buffer_info) {  in ena_setup_tx_resources()
758 		tx_ring->tx_buffer_info = vzalloc(size);  in ena_setup_tx_resources()
759 		if (!tx_ring->tx_buffer_info)  in ena_setup_tx_resources()
763 	size = sizeof(u16) * tx_ring->ring_size;  in ena_setup_tx_resources()
764 	tx_ring->free_ids = vzalloc_node(size, node);  in ena_setup_tx_resources()
765 	if (!tx_ring->free_ids) {  in ena_setup_tx_resources()
766 		tx_ring->free_ids = vzalloc(size);  in ena_setup_tx_resources()
767 		if (!tx_ring->free_ids)  in ena_setup_tx_resources()
771 	size = tx_ring->tx_max_header_size;  in ena_setup_tx_resources()
772 	tx_ring->push_buf_intermediate_buf = vzalloc_node(size, node);  in ena_setup_tx_resources()
773 	if (!tx_ring->push_buf_intermediate_buf) {  in ena_setup_tx_resources()
774 		tx_ring->push_buf_intermediate_buf = vzalloc(size);  in ena_setup_tx_resources()
775 		if (!tx_ring->push_buf_intermediate_buf)  in ena_setup_tx_resources()
780 	for (i = 0; i < tx_ring->ring_size; i++)  in ena_setup_tx_resources()
781 		tx_ring->free_ids[i] = i;  in ena_setup_tx_resources()
784 	memset(&tx_ring->tx_stats, 0x0, sizeof(tx_ring->tx_stats));  in ena_setup_tx_resources()
786 	tx_ring->next_to_use = 0;  in ena_setup_tx_resources()
787 	tx_ring->next_to_clean = 0;  in ena_setup_tx_resources()
788 	tx_ring->cpu = ena_irq->cpu;  in ena_setup_tx_resources()
789 	tx_ring->numa_node = node;  in ena_setup_tx_resources()
793 	vfree(tx_ring->free_ids);  in ena_setup_tx_resources()
794 	tx_ring->free_ids = NULL;  in ena_setup_tx_resources()
796 	vfree(tx_ring->tx_buffer_info);  in ena_setup_tx_resources()
797 	tx_ring->tx_buffer_info = NULL;  in ena_setup_tx_resources()
810 	struct ena_ring *tx_ring = &adapter->tx_ring[qid];  in ena_free_tx_resources()  local
812 	vfree(tx_ring->tx_buffer_info);  in ena_free_tx_resources()
813 	tx_ring->tx_buffer_info = NULL;  in ena_free_tx_resources()
815 	vfree(tx_ring->free_ids);  in ena_free_tx_resources()
816 	tx_ring->free_ids = NULL;  in ena_free_tx_resources()
818 	vfree(tx_ring->push_buf_intermediate_buf);  in ena_free_tx_resources()
819 	tx_ring->push_buf_intermediate_buf = NULL;  in ena_free_tx_resources()
1171 static void ena_unmap_tx_buff(struct ena_ring *tx_ring,  in ena_unmap_tx_buff()  argument
1185 		dma_unmap_single(tx_ring->dev,  in ena_unmap_tx_buff()
1195 		dma_unmap_page(tx_ring->dev, dma_unmap_addr(ena_buf, paddr),  in ena_unmap_tx_buff()
1204 static void ena_free_tx_bufs(struct ena_ring *tx_ring)  in ena_free_tx_bufs()  argument
1209 	for (i = 0; i < tx_ring->ring_size; i++) {  in ena_free_tx_bufs()
1210 		struct ena_tx_buffer *tx_info = &tx_ring->tx_buffer_info[i];  in ena_free_tx_bufs()
1216 			netif_notice(tx_ring->adapter, ifdown, tx_ring->netdev,  in ena_free_tx_bufs()
1218 				     tx_ring->qid, i);  in ena_free_tx_bufs()
1221 			netif_dbg(tx_ring->adapter, ifdown, tx_ring->netdev,  in ena_free_tx_bufs()
1223 				  tx_ring->qid, i);  in ena_free_tx_bufs()
1226 		ena_unmap_tx_buff(tx_ring, tx_info);  in ena_free_tx_bufs()
1230 	netdev_tx_reset_queue(netdev_get_tx_queue(tx_ring->netdev,  in ena_free_tx_bufs()
1231 						  tx_ring->qid));  in ena_free_tx_bufs()
1236 	struct ena_ring *tx_ring;  in ena_free_all_tx_bufs()  local
1240 		tx_ring = &adapter->tx_ring[i];  in ena_free_all_tx_bufs()
1241 		ena_free_tx_bufs(tx_ring);  in ena_free_all_tx_bufs()
1296 static int validate_tx_req_id(struct ena_ring *tx_ring, u16 req_id)  in validate_tx_req_id()  argument
1300 	tx_info = &tx_ring->tx_buffer_info[req_id];  in validate_tx_req_id()
1304 	return handle_invalid_req_id(tx_ring, req_id, tx_info, false);  in validate_tx_req_id()
1318 static int ena_clean_tx_irq(struct ena_ring *tx_ring, u32 budget)  in ena_clean_tx_irq()  argument
1329 	next_to_clean = tx_ring->next_to_clean;  in ena_clean_tx_irq()
1330 	txq = netdev_get_tx_queue(tx_ring->netdev, tx_ring->qid);  in ena_clean_tx_irq()
1336 		rc = ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq,  in ena_clean_tx_irq()
1340 				handle_invalid_req_id(tx_ring, req_id, NULL,  in ena_clean_tx_irq()
1346 		rc = validate_tx_req_id(tx_ring, req_id);  in ena_clean_tx_irq()
1350 		tx_info = &tx_ring->tx_buffer_info[req_id];  in ena_clean_tx_irq()
1359 		ena_unmap_tx_buff(tx_ring, tx_info);  in ena_clean_tx_irq()
1361 		netif_dbg(tx_ring->adapter, tx_done, tx_ring->netdev,  in ena_clean_tx_irq()
1362 			  "tx_poll: q %d skb %p completed\n", tx_ring->qid,  in ena_clean_tx_irq()
1370 		tx_ring->free_ids[next_to_clean] = req_id;  in ena_clean_tx_irq()
1372 						     tx_ring->ring_size);  in ena_clean_tx_irq()
1375 	tx_ring->next_to_clean = next_to_clean;  in ena_clean_tx_irq()
1376 	ena_com_comp_ack(tx_ring->ena_com_io_sq, total_done);  in ena_clean_tx_irq()
1377 	ena_com_update_dev_comp_head(tx_ring->ena_com_io_cq);  in ena_clean_tx_irq()
1381 	netif_dbg(tx_ring->adapter, tx_done, tx_ring->netdev,  in ena_clean_tx_irq()
1383 		  tx_ring->qid, tx_pkts);  in ena_clean_tx_irq()
1390 	above_thresh = ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,  in ena_clean_tx_irq()
1395 			ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,  in ena_clean_tx_irq()
1398 		    test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags)) {  in ena_clean_tx_irq()
1400 			ena_increase_stat(&tx_ring->tx_stats.queue_wakeup, 1,  in ena_clean_tx_irq()
1401 					  &tx_ring->syncp);  in ena_clean_tx_irq()
1884 static void ena_unmask_interrupt(struct ena_ring *tx_ring,  in ena_unmask_interrupt()  argument
1887 	u32 rx_interval = tx_ring->smoothed_interval;  in ena_unmask_interrupt()
1903 				tx_ring->smoothed_interval,  in ena_unmask_interrupt()
1906 	ena_increase_stat(&tx_ring->tx_stats.unmask_interrupt, 1,  in ena_unmask_interrupt()
1907 			  &tx_ring->syncp);  in ena_unmask_interrupt()
1914 	ena_com_unmask_intr(tx_ring->ena_com_io_cq, &intr_reg);  in ena_unmask_interrupt()
1917 static void ena_update_ring_numa_node(struct ena_ring *tx_ring,  in ena_update_ring_numa_node()  argument
1924 	if (likely(tx_ring->cpu == cpu))  in ena_update_ring_numa_node()
1927 	tx_ring->cpu = cpu;  in ena_update_ring_numa_node()
1933 	if (likely(tx_ring->numa_node == numa_node))  in ena_update_ring_numa_node()
1939 		ena_com_update_numa_node(tx_ring->ena_com_io_cq, numa_node);  in ena_update_ring_numa_node()
1940 		tx_ring->numa_node = numa_node;  in ena_update_ring_numa_node()
2017 	struct ena_ring *tx_ring, *rx_ring;  in ena_io_poll()  local
2024 	tx_ring = ena_napi->tx_ring;  in ena_io_poll()
2027 	tx_budget = tx_ring->ring_size / ENA_TX_POLL_BUDGET_DIVIDER;  in ena_io_poll()
2029 	if (!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags) ||  in ena_io_poll()
2030 	    test_bit(ENA_FLAG_TRIGGER_RESET, &tx_ring->adapter->flags)) {  in ena_io_poll()
2035 	tx_work_done = ena_clean_tx_irq(tx_ring, tx_budget);  in ena_io_poll()
2045 	if (unlikely(!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags) ||  in ena_io_poll()
2046 		     test_bit(ENA_FLAG_TRIGGER_RESET, &tx_ring->adapter->flags))) {  in ena_io_poll()
2066 			ena_update_ring_numa_node(tx_ring, rx_ring);  in ena_io_poll()
2067 			ena_unmask_interrupt(tx_ring, rx_ring);  in ena_io_poll()
2075 	u64_stats_update_begin(&tx_ring->syncp);  in ena_io_poll()
2076 	tx_ring->tx_stats.napi_comp += napi_comp_call;  in ena_io_poll()
2077 	tx_ring->tx_stats.tx_poll++;  in ena_io_poll()
2078 	u64_stats_update_end(&tx_ring->syncp);  in ena_io_poll()
2080 	tx_ring->tx_stats.last_napi_jiffies = jiffies;  in ena_io_poll()
2348 			napi->tx_ring = &adapter->tx_ring[i];  in ena_init_napi_in_range()
2350 			napi->xdp_ring = &adapter->tx_ring[i];  in ena_init_napi_in_range()
2436 	struct ena_ring *tx_ring;  in ena_create_io_tx_queue()  local
2443 	tx_ring = &adapter->tx_ring[qid];  in ena_create_io_tx_queue()
2453 	ctx.queue_size = tx_ring->ring_size;  in ena_create_io_tx_queue()
2454 	ctx.numa_node = tx_ring->numa_node;  in ena_create_io_tx_queue()
2465 				     &tx_ring->ena_com_io_sq,  in ena_create_io_tx_queue()
2466 				     &tx_ring->ena_com_io_cq);  in ena_create_io_tx_queue()
2475 	ena_com_update_numa_node(tx_ring->ena_com_io_cq, ctx.numa_node);  in ena_create_io_tx_queue()
2580 		adapter->tx_ring[i].ring_size = new_tx_size;  in set_io_rings_size()
2654 		cur_tx_ring_size = adapter->tx_ring[0].ring_size;  in create_queues_with_size_backoff()
2728 		ena_unmask_interrupt(&adapter->tx_ring[i],  in ena_up()
3008 static int ena_check_and_linearize_skb(struct ena_ring *tx_ring,  in ena_check_and_linearize_skb()  argument
3016 	if (num_frags < tx_ring->sgl_size)  in ena_check_and_linearize_skb()
3019 	if ((num_frags == tx_ring->sgl_size) &&  in ena_check_and_linearize_skb()
3020 	    (header_len < tx_ring->tx_max_header_size))  in ena_check_and_linearize_skb()
3023 	ena_increase_stat(&tx_ring->tx_stats.linearize, 1, &tx_ring->syncp);  in ena_check_and_linearize_skb()
3027 		ena_increase_stat(&tx_ring->tx_stats.linearize_failed, 1,  in ena_check_and_linearize_skb()
3028 				  &tx_ring->syncp);  in ena_check_and_linearize_skb()
3034 static int ena_tx_map_skb(struct ena_ring *tx_ring,  in ena_tx_map_skb()  argument
3040 	struct ena_adapter *adapter = tx_ring->adapter;  in ena_tx_map_skb()
3052 	if (tx_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {  in ena_tx_map_skb()
3063 		push_len = min_t(u32, skb->len, tx_ring->tx_max_header_size);  in ena_tx_map_skb()
3065 					       tx_ring->push_buf_intermediate_buf);  in ena_tx_map_skb()
3068 			ena_increase_stat(&tx_ring->tx_stats.llq_buffer_copy, 1,  in ena_tx_map_skb()
3069 					  &tx_ring->syncp);  in ena_tx_map_skb()
3076 				    tx_ring->tx_max_header_size);  in ena_tx_map_skb()
3084 		dma = dma_map_single(tx_ring->dev, skb->data + push_len,  in ena_tx_map_skb()
3086 		if (unlikely(dma_mapping_error(tx_ring->dev, dma)))  in ena_tx_map_skb()
3111 		dma = skb_frag_dma_map(tx_ring->dev, frag, delta,  in ena_tx_map_skb()
3113 		if (unlikely(dma_mapping_error(tx_ring->dev, dma)))  in ena_tx_map_skb()
3126 	ena_increase_stat(&tx_ring->tx_stats.dma_mapping_err, 1,  in ena_tx_map_skb()
3127 			  &tx_ring->syncp);  in ena_tx_map_skb()
3133 	ena_unmap_tx_buff(tx_ring, tx_info);  in ena_tx_map_skb()
3144 	struct ena_ring *tx_ring;  in ena_start_xmit()  local
3153 	tx_ring = &adapter->tx_ring[qid];  in ena_start_xmit()
3156 	rc = ena_check_and_linearize_skb(tx_ring, skb);  in ena_start_xmit()
3162 	next_to_use = tx_ring->next_to_use;  in ena_start_xmit()
3163 	req_id = tx_ring->free_ids[next_to_use];  in ena_start_xmit()
3164 	tx_info = &tx_ring->tx_buffer_info[req_id];  in ena_start_xmit()
3169 	rc = ena_tx_map_skb(tx_ring, tx_info, skb, &push_hdr, &header_len);  in ena_start_xmit()
3181 	ena_tx_csum(&ena_tx_ctx, skb, tx_ring->disable_meta_caching);  in ena_start_xmit()
3184 			     tx_ring,  in ena_start_xmit()
3198 	if (unlikely(!ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,  in ena_start_xmit()
3199 						   tx_ring->sgl_size + 2))) {  in ena_start_xmit()
3204 		ena_increase_stat(&tx_ring->tx_stats.queue_stop, 1,  in ena_start_xmit()
3205 				  &tx_ring->syncp);  in ena_start_xmit()
3217 		if (ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,  in ena_start_xmit()
3220 			ena_increase_stat(&tx_ring->tx_stats.queue_wakeup, 1,  in ena_start_xmit()
3221 					  &tx_ring->syncp);  in ena_start_xmit()
3229 		ena_ring_tx_doorbell(tx_ring);  in ena_start_xmit()
3234 	ena_unmap_tx_buff(tx_ring, tx_info);  in ena_start_xmit()
3366 	struct ena_ring *rx_ring, *tx_ring;  in ena_get_stats64()  local
3378 		tx_ring = &adapter->tx_ring[i];  in ena_get_stats64()
3381 			start = u64_stats_fetch_begin(&tx_ring->syncp);  in ena_get_stats64()
3382 			packets = tx_ring->tx_stats.cnt;  in ena_get_stats64()
3383 			bytes = tx_ring->tx_stats.bytes;  in ena_get_stats64()
3384 		} while (u64_stats_fetch_retry(&tx_ring->syncp, start));  in ena_get_stats64()
3451 	if (adapter->tx_ring->ring_size)  in ena_calc_io_queue_size()
3452 		tx_queue_size = adapter->tx_ring->ring_size;  in ena_calc_io_queue_size()
3838 		txr = &adapter->tx_ring[i];  in ena_restore_device()
3934 					  struct ena_ring *tx_ring)  in check_missing_comp_in_tx_queue()  argument
3936 	struct ena_napi *ena_napi = container_of(tx_ring->napi, struct ena_napi, napi);  in check_missing_comp_in_tx_queue()
3945 	for (i = 0; i < tx_ring->ring_size; i++) {  in check_missing_comp_in_tx_queue()
3946 		tx_buf = &tx_ring->tx_buffer_info[i];  in check_missing_comp_in_tx_queue()
3962 				  tx_ring->qid);  in check_missing_comp_in_tx_queue()
3972 				time_since_last_napi = jiffies_to_usecs(jiffies - tx_ring->tx_stats.last_napi_jiffies);  in check_missing_comp_in_tx_queue()
3976 					     tx_ring->qid, i, time_since_last_napi, missing_tx_comp_to);  in check_missing_comp_in_tx_queue()
3993 	ena_increase_stat(&tx_ring->tx_stats.missed_tx, missed_tx,  in check_missing_comp_in_tx_queue()
3994 			  &tx_ring->syncp);  in check_missing_comp_in_tx_queue()
4001 	struct ena_ring *tx_ring;  in check_for_missing_completions()  local
4022 		tx_ring = &adapter->tx_ring[i];  in check_for_missing_completions()
4025 		rc = check_missing_comp_in_tx_queue(adapter, tx_ring);  in check_for_missing_completions()