512 			       rx_ring->q_vector->napi.napi_id);  in igc_setup_rx_resources()
1765 	skb = napi_alloc_skb(&rx_ring->q_vector->napi,  in igc_construct_skb()
1953 	struct igc_adapter *adapter = rx_ring->q_vector->adapter;  in igc_rx_offset()
2343 static void igc_update_rx_stats(struct igc_q_vector *q_vector,  in igc_update_rx_stats()  argument
2346 	struct igc_ring *ring = q_vector->rx.ring;  in igc_update_rx_stats()
2353 	q_vector->rx.total_packets += packets;  in igc_update_rx_stats()
2354 	q_vector->rx.total_bytes += bytes;  in igc_update_rx_stats()
2357 static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget)  in igc_clean_rx_irq()  argument
2360 	struct igc_adapter *adapter = q_vector->adapter;  in igc_clean_rx_irq()
2361 	struct igc_ring *rx_ring = q_vector->rx.ring;  in igc_clean_rx_irq()
2398 			timestamp = igc_ptp_rx_pktstamp(q_vector->adapter,  in igc_clean_rx_irq()
2464 		napi_gro_receive(&q_vector->napi, skb);  in igc_clean_rx_irq()
2479 	igc_update_rx_stats(q_vector, total_packets, total_bytes);  in igc_clean_rx_irq()
2496 	skb = __napi_alloc_skb(&ring->q_vector->napi, totalsize,  in igc_construct_skb_zc()
2512 static void igc_dispatch_skb_zc(struct igc_q_vector *q_vector,  in igc_dispatch_skb_zc()  argument
2517 	struct igc_ring *ring = q_vector->rx.ring;  in igc_dispatch_skb_zc()
2533 	napi_gro_receive(&q_vector->napi, skb);  in igc_dispatch_skb_zc()
2536 static int igc_clean_rx_irq_zc(struct igc_q_vector *q_vector, const int budget)  in igc_clean_rx_irq_zc()  argument
2538 	struct igc_adapter *adapter = q_vector->adapter;  in igc_clean_rx_irq_zc()
2539 	struct igc_ring *ring = q_vector->rx.ring;  in igc_clean_rx_irq_zc()
2572 			timestamp = igc_ptp_rx_pktstamp(q_vector->adapter,  in igc_clean_rx_irq_zc()
2590 			igc_dispatch_skb_zc(q_vector, desc, bi->xdp, timestamp);  in igc_clean_rx_irq_zc()
2619 	igc_update_rx_stats(q_vector, total_packets, total_bytes);  in igc_clean_rx_irq_zc()
2632 static void igc_update_tx_stats(struct igc_q_vector *q_vector,  in igc_update_tx_stats()  argument
2635 	struct igc_ring *ring = q_vector->tx.ring;  in igc_update_tx_stats()
2642 	q_vector->tx.total_bytes += bytes;  in igc_update_tx_stats()
2643 	q_vector->tx.total_packets += packets;  in igc_update_tx_stats()
2712 static bool igc_clean_tx_irq(struct igc_q_vector *q_vector, int napi_budget)  in igc_clean_tx_irq()  argument
2714 	struct igc_adapter *adapter = q_vector->adapter;  in igc_clean_tx_irq()
2716 	unsigned int budget = q_vector->tx.work_limit;  in igc_clean_tx_irq()
2717 	struct igc_ring *tx_ring = q_vector->tx.ring;  in igc_clean_tx_irq()
2807 	igc_update_tx_stats(q_vector, total_packets, total_bytes);  in igc_clean_tx_irq()
3743 static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector)  in igc_assign_vector()  argument
3745 	struct igc_adapter *adapter = q_vector->adapter;  in igc_assign_vector()
3750 	if (q_vector->rx.ring)  in igc_assign_vector()
3751 		rx_queue = q_vector->rx.ring->reg_idx;  in igc_assign_vector()
3752 	if (q_vector->tx.ring)  in igc_assign_vector()
3753 		tx_queue = q_vector->tx.ring->reg_idx;  in igc_assign_vector()
3765 		q_vector->eims_value = BIT(msix_vector);  in igc_assign_vector()
3773 	adapter->eims_enable_mask |= q_vector->eims_value;  in igc_assign_vector()
3776 	q_vector->set_itr = 1;  in igc_assign_vector()
3818 		igc_assign_vector(adapter->q_vector[i], vector++);  in igc_configure_msix()
3917 	struct igc_q_vector *q_vector = adapter->q_vector[v_idx];  in igc_reset_q_vector()  local
3922 	if (!q_vector)  in igc_reset_q_vector()
3925 	if (q_vector->tx.ring)  in igc_reset_q_vector()
3926 		adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL;  in igc_reset_q_vector()
3928 	if (q_vector->rx.ring)  in igc_reset_q_vector()
3929 		adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL;  in igc_reset_q_vector()
3931 	netif_napi_del(&q_vector->napi);  in igc_reset_q_vector()
3943 	struct igc_q_vector *q_vector = adapter->q_vector[v_idx];  in igc_free_q_vector()  local
3945 	adapter->q_vector[v_idx] = NULL;  in igc_free_q_vector()
3950 	if (q_vector)  in igc_free_q_vector()
3951 		kfree_rcu(q_vector, rcu);  in igc_free_q_vector()
3991 static void igc_update_itr(struct igc_q_vector *q_vector,  in igc_update_itr()  argument
4043 static void igc_set_itr(struct igc_q_vector *q_vector)  in igc_set_itr()  argument
4045 	struct igc_adapter *adapter = q_vector->adapter;  in igc_set_itr()
4046 	u32 new_itr = q_vector->itr_val;  in igc_set_itr()
4060 	igc_update_itr(q_vector, &q_vector->tx);  in igc_set_itr()
4061 	igc_update_itr(q_vector, &q_vector->rx);  in igc_set_itr()
4063 	current_itr = max(q_vector->rx.itr, q_vector->tx.itr);  in igc_set_itr()
4067 	    ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||  in igc_set_itr()
4068 	    (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))  in igc_set_itr()
4087 	if (new_itr != q_vector->itr_val) {  in igc_set_itr()
4092 		new_itr = new_itr > q_vector->itr_val ?  in igc_set_itr()
4093 			  max((new_itr * q_vector->itr_val) /  in igc_set_itr()
4094 			  (new_itr + (q_vector->itr_val >> 2)),  in igc_set_itr()
4102 		q_vector->itr_val = new_itr;  in igc_set_itr()
4103 		q_vector->set_itr = 1;  in igc_set_itr()
4207 static void igc_update_ring_itr(struct igc_q_vector *q_vector)  in igc_update_ring_itr()  argument
4209 	struct igc_adapter *adapter = q_vector->adapter;  in igc_update_ring_itr()
4210 	int new_val = q_vector->itr_val;  in igc_update_ring_itr()
4226 	packets = q_vector->rx.total_packets;  in igc_update_ring_itr()
4228 		avg_wire_size = q_vector->rx.total_bytes / packets;  in igc_update_ring_itr()
4230 	packets = q_vector->tx.total_packets;  in igc_update_ring_itr()
4233 				      q_vector->tx.total_bytes / packets);  in igc_update_ring_itr()
4253 	    ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||  in igc_update_ring_itr()
4254 	    (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))  in igc_update_ring_itr()
4258 	if (new_val != q_vector->itr_val) {  in igc_update_ring_itr()
4259 		q_vector->itr_val = new_val;  in igc_update_ring_itr()
4260 		q_vector->set_itr = 1;  in igc_update_ring_itr()
4263 	q_vector->rx.total_bytes = 0;  in igc_update_ring_itr()
4264 	q_vector->rx.total_packets = 0;  in igc_update_ring_itr()
4265 	q_vector->tx.total_bytes = 0;  in igc_update_ring_itr()
4266 	q_vector->tx.total_packets = 0;  in igc_update_ring_itr()
4269 static void igc_ring_irq_enable(struct igc_q_vector *q_vector)  in igc_ring_irq_enable()  argument
4271 	struct igc_adapter *adapter = q_vector->adapter;  in igc_ring_irq_enable()
4274 	if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) ||  in igc_ring_irq_enable()
4275 	    (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) {  in igc_ring_irq_enable()
4277 			igc_set_itr(q_vector);  in igc_ring_irq_enable()
4279 			igc_update_ring_itr(q_vector);  in igc_ring_irq_enable()
4284 			wr32(IGC_EIMS, q_vector->eims_value);  in igc_ring_irq_enable()
4326 	struct igc_q_vector *q_vector = container_of(napi,  in igc_poll()  local
4329 	struct igc_ring *rx_ring = q_vector->rx.ring;  in igc_poll()
4333 	if (q_vector->tx.ring)  in igc_poll()
4334 		clean_complete = igc_clean_tx_irq(q_vector, budget);  in igc_poll()
4338 			      igc_clean_rx_irq_zc(q_vector, budget) :  in igc_poll()
4339 			      igc_clean_rx_irq(q_vector, budget);  in igc_poll()
4354 		igc_ring_irq_enable(q_vector);  in igc_poll()
4376 	struct igc_q_vector *q_vector;  in igc_alloc_q_vector()  local
4387 	q_vector = adapter->q_vector[v_idx];  in igc_alloc_q_vector()
4388 	if (!q_vector)  in igc_alloc_q_vector()
4389 		q_vector = kzalloc(struct_size(q_vector, ring, ring_count),  in igc_alloc_q_vector()
4392 		memset(q_vector, 0, struct_size(q_vector, ring, ring_count));  in igc_alloc_q_vector()
4393 	if (!q_vector)  in igc_alloc_q_vector()
4397 	netif_napi_add(adapter->netdev, &q_vector->napi, igc_poll);  in igc_alloc_q_vector()
4400 	adapter->q_vector[v_idx] = q_vector;  in igc_alloc_q_vector()
4401 	q_vector->adapter = adapter;  in igc_alloc_q_vector()
4404 	q_vector->tx.work_limit = adapter->tx_work_limit;  in igc_alloc_q_vector()
4407 	q_vector->itr_register = adapter->io_addr + IGC_EITR(0);  in igc_alloc_q_vector()
4408 	q_vector->itr_val = IGC_START_ITR;  in igc_alloc_q_vector()
4411 	ring = q_vector->ring;  in igc_alloc_q_vector()
4417 			q_vector->itr_val = adapter->rx_itr_setting;  in igc_alloc_q_vector()
4421 			q_vector->itr_val = adapter->tx_itr_setting;  in igc_alloc_q_vector()
4430 		ring->q_vector = q_vector;  in igc_alloc_q_vector()
4433 		igc_add_ring(ring, &q_vector->tx);  in igc_alloc_q_vector()
4452 		ring->q_vector = q_vector;  in igc_alloc_q_vector()
4455 		igc_add_ring(ring, &q_vector->rx);  in igc_alloc_q_vector()
4627 		napi_enable(&adapter->q_vector[i]->napi);  in igc_up()
4632 		igc_assign_vector(adapter->q_vector[0], 0);  in igc_up()
4853 		if (adapter->q_vector[i]) {  in igc_down()
4854 			napi_synchronize(&adapter->q_vector[i]->napi);  in igc_down()
4855 			napi_disable(&adapter->q_vector[i]->napi);  in igc_down()
5154 static void igc_write_itr(struct igc_q_vector *q_vector)  in igc_write_itr()  argument
5156 	u32 itr_val = q_vector->itr_val & IGC_QVECTOR_MASK;  in igc_write_itr()
5158 	if (!q_vector->set_itr)  in igc_write_itr()
5166 	writel(itr_val, q_vector->itr_register);  in igc_write_itr()
5167 	q_vector->set_itr = 0;  in igc_write_itr()
5172 	struct igc_q_vector *q_vector = data;  in igc_msix_ring()  local
5175 	igc_write_itr(q_vector);  in igc_msix_ring()
5177 	napi_schedule(&q_vector->napi);  in igc_msix_ring()
5207 		struct igc_q_vector *q_vector = adapter->q_vector[i];  in igc_request_msix()  local
5211 		q_vector->itr_register = adapter->io_addr + IGC_EITR(vector);  in igc_request_msix()
5213 		if (q_vector->rx.ring && q_vector->tx.ring)  in igc_request_msix()
5214 			sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,  in igc_request_msix()
5215 				q_vector->rx.ring->queue_index);  in igc_request_msix()
5216 		else if (q_vector->tx.ring)  in igc_request_msix()
5217 			sprintf(q_vector->name, "%s-tx-%u", netdev->name,  in igc_request_msix()
5218 				q_vector->tx.ring->queue_index);  in igc_request_msix()
5219 		else if (q_vector->rx.ring)  in igc_request_msix()
5220 			sprintf(q_vector->name, "%s-rx-%u", netdev->name,  in igc_request_msix()
5221 				q_vector->rx.ring->queue_index);  in igc_request_msix()
5223 			sprintf(q_vector->name, "%s-unused", netdev->name);  in igc_request_msix()
5226 				  igc_msix_ring, 0, q_vector->name,  in igc_request_msix()
5227 				  q_vector);  in igc_request_msix()
5242 			 adapter->q_vector[i]);  in igc_request_msix()
5476 			eics |= adapter->q_vector[i]->eims_value;  in igc_watchdog_task()
5503 	struct igc_q_vector *q_vector = adapter->q_vector[0];  in igc_intr_msi()  local
5508 	igc_write_itr(q_vector);  in igc_intr_msi()
5527 	napi_schedule(&q_vector->napi);  in igc_intr_msi()
5540 	struct igc_q_vector *q_vector = adapter->q_vector[0];  in igc_intr()  local
5553 	igc_write_itr(q_vector);  in igc_intr()
5573 	napi_schedule(&q_vector->napi);  in igc_intr()
5587 				 adapter->q_vector[i]);  in igc_free_irq()
5623 	igc_assign_vector(adapter->q_vector[0], 0);  in igc_request_irq()
5709 		napi_enable(&adapter->q_vector[i]->napi);  in __igc_open()
6123 					struct igc_q_vector *q_vector)  in igc_trigger_rxtxq_interrupt()  argument
6128 	eics |= q_vector->eims_value;  in igc_trigger_rxtxq_interrupt()
6135 	struct igc_q_vector *q_vector;  in igc_xsk_wakeup()  local
6152 	q_vector = adapter->q_vector[queue_id];  in igc_xsk_wakeup()
6153 	if (!napi_if_scheduled_mark_missed(&q_vector->napi))  in igc_xsk_wakeup()
6154 		igc_trigger_rxtxq_interrupt(adapter, q_vector);  in igc_xsk_wakeup()
6888 	struct igc_hw *hw = &ring->q_vector->adapter->hw;  in igc_disable_rx_ring_hw()
6906 	struct igc_adapter *adapter = ring->q_vector->adapter;  in igc_enable_rx_ring()
6918 	struct igc_hw *hw = &ring->q_vector->adapter->hw;  in igc_disable_tx_ring_hw()
6936 	struct igc_adapter *adapter = ring->q_vector->adapter;  in igc_enable_tx_ring()