210 static inline bool ixgbevf_check_tx_hang(struct ixgbevf_ring *tx_ring)  in ixgbevf_check_tx_hang()  argument
212 	u32 tx_done = ixgbevf_get_tx_completed(tx_ring);  in ixgbevf_check_tx_hang()
213 	u32 tx_done_old = tx_ring->tx_stats.tx_done_old;  in ixgbevf_check_tx_hang()
214 	u32 tx_pending = ixgbevf_get_tx_pending(tx_ring);  in ixgbevf_check_tx_hang()
216 	clear_check_for_tx_hang(tx_ring);  in ixgbevf_check_tx_hang()
226 					&tx_ring->state);  in ixgbevf_check_tx_hang()
229 	clear_bit(__IXGBEVF_HANG_CHECK_ARMED, &tx_ring->state);  in ixgbevf_check_tx_hang()
232 	tx_ring->tx_stats.tx_done_old = tx_done;  in ixgbevf_check_tx_hang()
265 				 struct ixgbevf_ring *tx_ring, int napi_budget)  in ixgbevf_clean_tx_irq()  argument
271 	unsigned int budget = tx_ring->count / 2;  in ixgbevf_clean_tx_irq()
272 	unsigned int i = tx_ring->next_to_clean;  in ixgbevf_clean_tx_irq()
277 	tx_buffer = &tx_ring->tx_buffer_info[i];  in ixgbevf_clean_tx_irq()
278 	tx_desc = IXGBEVF_TX_DESC(tx_ring, i);  in ixgbevf_clean_tx_irq()
279 	i -= tx_ring->count;  in ixgbevf_clean_tx_irq()
305 		if (ring_is_xdp(tx_ring))  in ixgbevf_clean_tx_irq()
311 		dma_unmap_single(tx_ring->dev,  in ixgbevf_clean_tx_irq()
325 				i -= tx_ring->count;  in ixgbevf_clean_tx_irq()
326 				tx_buffer = tx_ring->tx_buffer_info;  in ixgbevf_clean_tx_irq()
327 				tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);  in ixgbevf_clean_tx_irq()
332 				dma_unmap_page(tx_ring->dev,  in ixgbevf_clean_tx_irq()
345 			i -= tx_ring->count;  in ixgbevf_clean_tx_irq()
346 			tx_buffer = tx_ring->tx_buffer_info;  in ixgbevf_clean_tx_irq()
347 			tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);  in ixgbevf_clean_tx_irq()
357 	i += tx_ring->count;  in ixgbevf_clean_tx_irq()
358 	tx_ring->next_to_clean = i;  in ixgbevf_clean_tx_irq()
359 	u64_stats_update_begin(&tx_ring->syncp);  in ixgbevf_clean_tx_irq()
360 	tx_ring->stats.bytes += total_bytes;  in ixgbevf_clean_tx_irq()
361 	tx_ring->stats.packets += total_packets;  in ixgbevf_clean_tx_irq()
362 	u64_stats_update_end(&tx_ring->syncp);  in ixgbevf_clean_tx_irq()
367 	if (check_for_tx_hang(tx_ring) && ixgbevf_check_tx_hang(tx_ring)) {  in ixgbevf_clean_tx_irq()
371 		eop_desc = tx_ring->tx_buffer_info[i].next_to_watch;  in ixgbevf_clean_tx_irq()
383 		       ring_is_xdp(tx_ring) ? " XDP" : "",  in ixgbevf_clean_tx_irq()
384 		       tx_ring->queue_index,  in ixgbevf_clean_tx_irq()
385 		       IXGBE_READ_REG(hw, IXGBE_VFTDH(tx_ring->reg_idx)),  in ixgbevf_clean_tx_irq()
386 		       IXGBE_READ_REG(hw, IXGBE_VFTDT(tx_ring->reg_idx)),  in ixgbevf_clean_tx_irq()
387 		       tx_ring->next_to_use, i,  in ixgbevf_clean_tx_irq()
389 		       tx_ring->tx_buffer_info[i].time_stamp, jiffies);  in ixgbevf_clean_tx_irq()
391 		if (!ring_is_xdp(tx_ring))  in ixgbevf_clean_tx_irq()
392 			netif_stop_subqueue(tx_ring->netdev,  in ixgbevf_clean_tx_irq()
393 					    tx_ring->queue_index);  in ixgbevf_clean_tx_irq()
401 	if (ring_is_xdp(tx_ring))  in ixgbevf_clean_tx_irq()
405 	if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&  in ixgbevf_clean_tx_irq()
406 		     (ixgbevf_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) {  in ixgbevf_clean_tx_irq()
412 		if (__netif_subqueue_stopped(tx_ring->netdev,  in ixgbevf_clean_tx_irq()
413 					     tx_ring->queue_index) &&  in ixgbevf_clean_tx_irq()
415 			netif_wake_subqueue(tx_ring->netdev,  in ixgbevf_clean_tx_irq()
416 					    tx_ring->queue_index);  in ixgbevf_clean_tx_irq()
417 			++tx_ring->tx_stats.restart_queue;  in ixgbevf_clean_tx_irq()
1760 		ixgbevf_configure_tx_ring(adapter, adapter->tx_ring[i]);  in ixgbevf_configure_tx()
2195 		adapter->tx_ring[0]->reg_idx = def_q;  in ixgbevf_configure_dcb()
2384 static void ixgbevf_clean_tx_ring(struct ixgbevf_ring *tx_ring)  in ixgbevf_clean_tx_ring()  argument
2386 	u16 i = tx_ring->next_to_clean;  in ixgbevf_clean_tx_ring()
2387 	struct ixgbevf_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i];  in ixgbevf_clean_tx_ring()
2389 	while (i != tx_ring->next_to_use) {  in ixgbevf_clean_tx_ring()
2393 		if (ring_is_xdp(tx_ring))  in ixgbevf_clean_tx_ring()
2399 		dma_unmap_single(tx_ring->dev,  in ixgbevf_clean_tx_ring()
2406 		tx_desc = IXGBEVF_TX_DESC(tx_ring, i);  in ixgbevf_clean_tx_ring()
2413 			if (unlikely(i == tx_ring->count)) {  in ixgbevf_clean_tx_ring()
2415 				tx_buffer = tx_ring->tx_buffer_info;  in ixgbevf_clean_tx_ring()
2416 				tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);  in ixgbevf_clean_tx_ring()
2421 				dma_unmap_page(tx_ring->dev,  in ixgbevf_clean_tx_ring()
2430 		if (unlikely(i == tx_ring->count)) {  in ixgbevf_clean_tx_ring()
2432 			tx_buffer = tx_ring->tx_buffer_info;  in ixgbevf_clean_tx_ring()
2437 	tx_ring->next_to_use = 0;  in ixgbevf_clean_tx_ring()
2438 	tx_ring->next_to_clean = 0;  in ixgbevf_clean_tx_ring()
2463 		ixgbevf_clean_tx_ring(adapter->tx_ring[i]);  in ixgbevf_clean_all_tx_rings()
2498 		u8 reg_idx = adapter->tx_ring[i]->reg_idx;  in ixgbevf_down()
2747 		 adapter->tx_ring[txr_idx] = ring;  in ixgbevf_alloc_q_vector()
2835 			adapter->tx_ring[ring->queue_index] = NULL;  in ixgbevf_free_q_vector()
3205 			set_check_for_tx_hang(adapter->tx_ring[i]);  in ixgbevf_check_hang_subtask()
3349 void ixgbevf_free_tx_resources(struct ixgbevf_ring *tx_ring)  in ixgbevf_free_tx_resources()  argument
3351 	ixgbevf_clean_tx_ring(tx_ring);  in ixgbevf_free_tx_resources()
3353 	vfree(tx_ring->tx_buffer_info);  in ixgbevf_free_tx_resources()
3354 	tx_ring->tx_buffer_info = NULL;  in ixgbevf_free_tx_resources()
3357 	if (!tx_ring->desc)  in ixgbevf_free_tx_resources()
3360 	dma_free_coherent(tx_ring->dev, tx_ring->size, tx_ring->desc,  in ixgbevf_free_tx_resources()
3361 			  tx_ring->dma);  in ixgbevf_free_tx_resources()
3363 	tx_ring->desc = NULL;  in ixgbevf_free_tx_resources()
3377 		if (adapter->tx_ring[i]->desc)  in ixgbevf_free_all_tx_resources()
3378 			ixgbevf_free_tx_resources(adapter->tx_ring[i]);  in ixgbevf_free_all_tx_resources()
3390 int ixgbevf_setup_tx_resources(struct ixgbevf_ring *tx_ring)  in ixgbevf_setup_tx_resources()  argument
3392 	struct ixgbevf_adapter *adapter = netdev_priv(tx_ring->netdev);  in ixgbevf_setup_tx_resources()
3395 	size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count;  in ixgbevf_setup_tx_resources()
3396 	tx_ring->tx_buffer_info = vmalloc(size);  in ixgbevf_setup_tx_resources()
3397 	if (!tx_ring->tx_buffer_info)  in ixgbevf_setup_tx_resources()
3400 	u64_stats_init(&tx_ring->syncp);  in ixgbevf_setup_tx_resources()
3403 	tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc);  in ixgbevf_setup_tx_resources()
3404 	tx_ring->size = ALIGN(tx_ring->size, 4096);  in ixgbevf_setup_tx_resources()
3406 	tx_ring->desc = dma_alloc_coherent(tx_ring->dev, tx_ring->size,  in ixgbevf_setup_tx_resources()
3407 					   &tx_ring->dma, GFP_KERNEL);  in ixgbevf_setup_tx_resources()
3408 	if (!tx_ring->desc)  in ixgbevf_setup_tx_resources()
3414 	vfree(tx_ring->tx_buffer_info);  in ixgbevf_setup_tx_resources()
3415 	tx_ring->tx_buffer_info = NULL;  in ixgbevf_setup_tx_resources()
3435 		err = ixgbevf_setup_tx_resources(adapter->tx_ring[i]);  in ixgbevf_setup_all_tx_resources()
3456 		ixgbevf_free_tx_resources(adapter->tx_ring[i]);  in ixgbevf_setup_all_tx_resources()
3725 static void ixgbevf_tx_ctxtdesc(struct ixgbevf_ring *tx_ring,  in ixgbevf_tx_ctxtdesc()  argument
3730 	u16 i = tx_ring->next_to_use;  in ixgbevf_tx_ctxtdesc()
3732 	context_desc = IXGBEVF_TX_CTXTDESC(tx_ring, i);  in ixgbevf_tx_ctxtdesc()
3735 	tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;  in ixgbevf_tx_ctxtdesc()
3746 static int ixgbevf_tso(struct ixgbevf_ring *tx_ring,  in ixgbevf_tso()  argument
3837 	ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens, fceof_saidx, type_tucmd,  in ixgbevf_tso()
3843 static void ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring,  in ixgbevf_tx_csum()  argument
3888 	ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens,  in ixgbevf_tx_csum()
3939 static void ixgbevf_tx_map(struct ixgbevf_ring *tx_ring,  in ixgbevf_tx_map()  argument
3951 	u16 i = tx_ring->next_to_use;  in ixgbevf_tx_map()
3953 	tx_desc = IXGBEVF_TX_DESC(tx_ring, i);  in ixgbevf_tx_map()
3960 	dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);  in ixgbevf_tx_map()
3965 		if (dma_mapping_error(tx_ring->dev, dma))  in ixgbevf_tx_map()
3980 			if (i == tx_ring->count) {  in ixgbevf_tx_map()
3981 				tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);  in ixgbevf_tx_map()
3999 		if (i == tx_ring->count) {  in ixgbevf_tx_map()
4000 			tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);  in ixgbevf_tx_map()
4008 		dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,  in ixgbevf_tx_map()
4011 		tx_buffer = &tx_ring->tx_buffer_info[i];  in ixgbevf_tx_map()
4036 	if (i == tx_ring->count)  in ixgbevf_tx_map()
4039 	tx_ring->next_to_use = i;  in ixgbevf_tx_map()
4042 	ixgbevf_write_tail(tx_ring, i);  in ixgbevf_tx_map()
4046 	dev_err(tx_ring->dev, "TX DMA map failed\n");  in ixgbevf_tx_map()
4047 	tx_buffer = &tx_ring->tx_buffer_info[i];  in ixgbevf_tx_map()
4052 			dma_unmap_page(tx_ring->dev,  in ixgbevf_tx_map()
4059 			i += tx_ring->count;  in ixgbevf_tx_map()
4060 		tx_buffer = &tx_ring->tx_buffer_info[i];  in ixgbevf_tx_map()
4064 		dma_unmap_single(tx_ring->dev,  in ixgbevf_tx_map()
4073 	tx_ring->next_to_use = i;  in ixgbevf_tx_map()
4076 static int __ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size)  in __ixgbevf_maybe_stop_tx()  argument
4078 	netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);  in __ixgbevf_maybe_stop_tx()
4088 	if (likely(ixgbevf_desc_unused(tx_ring) < size))  in __ixgbevf_maybe_stop_tx()
4092 	netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);  in __ixgbevf_maybe_stop_tx()
4093 	++tx_ring->tx_stats.restart_queue;  in __ixgbevf_maybe_stop_tx()
4098 static int ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size)  in ixgbevf_maybe_stop_tx()  argument
4100 	if (likely(ixgbevf_desc_unused(tx_ring) >= size))  in ixgbevf_maybe_stop_tx()
4102 	return __ixgbevf_maybe_stop_tx(tx_ring, size);  in ixgbevf_maybe_stop_tx()
4106 				   struct ixgbevf_ring *tx_ring)  in ixgbevf_xmit_frame_ring()  argument
4139 	if (ixgbevf_maybe_stop_tx(tx_ring, count + 3)) {  in ixgbevf_xmit_frame_ring()
4140 		tx_ring->tx_stats.tx_busy++;  in ixgbevf_xmit_frame_ring()
4145 	first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];  in ixgbevf_xmit_frame_ring()
4161 	if (xfrm_offload(skb) && !ixgbevf_ipsec_tx(tx_ring, first, &ipsec_tx))  in ixgbevf_xmit_frame_ring()
4164 	tso = ixgbevf_tso(tx_ring, first, &hdr_len, &ipsec_tx);  in ixgbevf_xmit_frame_ring()
4168 		ixgbevf_tx_csum(tx_ring, first, &ipsec_tx);  in ixgbevf_xmit_frame_ring()
4170 	ixgbevf_tx_map(tx_ring, first, hdr_len);  in ixgbevf_xmit_frame_ring()
4172 	ixgbevf_maybe_stop_tx(tx_ring, DESC_NEEDED);  in ixgbevf_xmit_frame_ring()
4186 	struct ixgbevf_ring *tx_ring;  in ixgbevf_xmit_frame()  local
4202 	tx_ring = adapter->tx_ring[skb->queue_mapping];  in ixgbevf_xmit_frame()
4203 	return ixgbevf_xmit_frame_ring(skb, tx_ring);  in ixgbevf_xmit_frame()
4370 		ring = adapter->tx_ring[i];  in ixgbevf_get_stats()