508 			       rx_ring->q_vector->napi.napi_id);  in igc_setup_rx_resources()
1758 	skb = napi_alloc_skb(&rx_ring->q_vector->napi, IGC_RX_HDR_LEN);  in igc_construct_skb()
1939 	struct igc_adapter *adapter = rx_ring->q_vector->adapter;  in igc_rx_offset()
2292 static void igc_update_rx_stats(struct igc_q_vector *q_vector,  in igc_update_rx_stats()  argument
2295 	struct igc_ring *ring = q_vector->rx.ring;  in igc_update_rx_stats()
2302 	q_vector->rx.total_packets += packets;  in igc_update_rx_stats()
2303 	q_vector->rx.total_bytes += bytes;  in igc_update_rx_stats()
2306 static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget)  in igc_clean_rx_irq()  argument
2309 	struct igc_adapter *adapter = q_vector->adapter;  in igc_clean_rx_irq()
2310 	struct igc_ring *rx_ring = q_vector->rx.ring;  in igc_clean_rx_irq()
2347 			timestamp = igc_ptp_rx_pktstamp(q_vector->adapter,  in igc_clean_rx_irq()
2411 		napi_gro_receive(&q_vector->napi, skb);  in igc_clean_rx_irq()
2426 	igc_update_rx_stats(q_vector, total_packets, total_bytes);  in igc_clean_rx_irq()
2442 	skb = __napi_alloc_skb(&ring->q_vector->napi,  in igc_construct_skb_zc()
2456 static void igc_dispatch_skb_zc(struct igc_q_vector *q_vector,  in igc_dispatch_skb_zc()  argument
2461 	struct igc_ring *ring = q_vector->rx.ring;  in igc_dispatch_skb_zc()
2477 	napi_gro_receive(&q_vector->napi, skb);  in igc_dispatch_skb_zc()
2480 static int igc_clean_rx_irq_zc(struct igc_q_vector *q_vector, const int budget)  in igc_clean_rx_irq_zc()  argument
2482 	struct igc_adapter *adapter = q_vector->adapter;  in igc_clean_rx_irq_zc()
2483 	struct igc_ring *ring = q_vector->rx.ring;  in igc_clean_rx_irq_zc()
2516 			timestamp = igc_ptp_rx_pktstamp(q_vector->adapter,  in igc_clean_rx_irq_zc()
2534 			igc_dispatch_skb_zc(q_vector, desc, bi->xdp, timestamp);  in igc_clean_rx_irq_zc()
2563 	igc_update_rx_stats(q_vector, total_packets, total_bytes);  in igc_clean_rx_irq_zc()
2576 static void igc_update_tx_stats(struct igc_q_vector *q_vector,  in igc_update_tx_stats()  argument
2579 	struct igc_ring *ring = q_vector->tx.ring;  in igc_update_tx_stats()
2586 	q_vector->tx.total_bytes += bytes;  in igc_update_tx_stats()
2587 	q_vector->tx.total_packets += packets;  in igc_update_tx_stats()
2656 static bool igc_clean_tx_irq(struct igc_q_vector *q_vector, int napi_budget)  in igc_clean_tx_irq()  argument
2658 	struct igc_adapter *adapter = q_vector->adapter;  in igc_clean_tx_irq()
2660 	unsigned int budget = q_vector->tx.work_limit;  in igc_clean_tx_irq()
2661 	struct igc_ring *tx_ring = q_vector->tx.ring;  in igc_clean_tx_irq()
2751 	igc_update_tx_stats(q_vector, total_packets, total_bytes);  in igc_clean_tx_irq()
3687 static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector)  in igc_assign_vector()  argument
3689 	struct igc_adapter *adapter = q_vector->adapter;  in igc_assign_vector()
3694 	if (q_vector->rx.ring)  in igc_assign_vector()
3695 		rx_queue = q_vector->rx.ring->reg_idx;  in igc_assign_vector()
3696 	if (q_vector->tx.ring)  in igc_assign_vector()
3697 		tx_queue = q_vector->tx.ring->reg_idx;  in igc_assign_vector()
3709 		q_vector->eims_value = BIT(msix_vector);  in igc_assign_vector()
3717 	adapter->eims_enable_mask |= q_vector->eims_value;  in igc_assign_vector()
3720 	q_vector->set_itr = 1;  in igc_assign_vector()
3762 		igc_assign_vector(adapter->q_vector[i], vector++);  in igc_configure_msix()
3861 	struct igc_q_vector *q_vector = adapter->q_vector[v_idx];  in igc_reset_q_vector()  local
3866 	if (!q_vector)  in igc_reset_q_vector()
3869 	if (q_vector->tx.ring)  in igc_reset_q_vector()
3870 		adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL;  in igc_reset_q_vector()
3872 	if (q_vector->rx.ring)  in igc_reset_q_vector()
3873 		adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL;  in igc_reset_q_vector()
3875 	netif_napi_del(&q_vector->napi);  in igc_reset_q_vector()
3887 	struct igc_q_vector *q_vector = adapter->q_vector[v_idx];  in igc_free_q_vector()  local
3889 	adapter->q_vector[v_idx] = NULL;  in igc_free_q_vector()
3894 	if (q_vector)  in igc_free_q_vector()
3895 		kfree_rcu(q_vector, rcu);  in igc_free_q_vector()
3935 static void igc_update_itr(struct igc_q_vector *q_vector,  in igc_update_itr()  argument
3987 static void igc_set_itr(struct igc_q_vector *q_vector)  in igc_set_itr()  argument
3989 	struct igc_adapter *adapter = q_vector->adapter;  in igc_set_itr()
3990 	u32 new_itr = q_vector->itr_val;  in igc_set_itr()
4004 	igc_update_itr(q_vector, &q_vector->tx);  in igc_set_itr()
4005 	igc_update_itr(q_vector, &q_vector->rx);  in igc_set_itr()
4007 	current_itr = max(q_vector->rx.itr, q_vector->tx.itr);  in igc_set_itr()
4011 	    ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||  in igc_set_itr()
4012 	    (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))  in igc_set_itr()
4031 	if (new_itr != q_vector->itr_val) {  in igc_set_itr()
4036 		new_itr = new_itr > q_vector->itr_val ?  in igc_set_itr()
4037 			  max((new_itr * q_vector->itr_val) /  in igc_set_itr()
4038 			  (new_itr + (q_vector->itr_val >> 2)),  in igc_set_itr()
4046 		q_vector->itr_val = new_itr;  in igc_set_itr()
4047 		q_vector->set_itr = 1;  in igc_set_itr()
4151 static void igc_update_ring_itr(struct igc_q_vector *q_vector)  in igc_update_ring_itr()  argument
4153 	struct igc_adapter *adapter = q_vector->adapter;  in igc_update_ring_itr()
4154 	int new_val = q_vector->itr_val;  in igc_update_ring_itr()
4170 	packets = q_vector->rx.total_packets;  in igc_update_ring_itr()
4172 		avg_wire_size = q_vector->rx.total_bytes / packets;  in igc_update_ring_itr()
4174 	packets = q_vector->tx.total_packets;  in igc_update_ring_itr()
4177 				      q_vector->tx.total_bytes / packets);  in igc_update_ring_itr()
4197 	    ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||  in igc_update_ring_itr()
4198 	    (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))  in igc_update_ring_itr()
4202 	if (new_val != q_vector->itr_val) {  in igc_update_ring_itr()
4203 		q_vector->itr_val = new_val;  in igc_update_ring_itr()
4204 		q_vector->set_itr = 1;  in igc_update_ring_itr()
4207 	q_vector->rx.total_bytes = 0;  in igc_update_ring_itr()
4208 	q_vector->rx.total_packets = 0;  in igc_update_ring_itr()
4209 	q_vector->tx.total_bytes = 0;  in igc_update_ring_itr()
4210 	q_vector->tx.total_packets = 0;  in igc_update_ring_itr()
4213 static void igc_ring_irq_enable(struct igc_q_vector *q_vector)  in igc_ring_irq_enable()  argument
4215 	struct igc_adapter *adapter = q_vector->adapter;  in igc_ring_irq_enable()
4218 	if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) ||  in igc_ring_irq_enable()
4219 	    (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) {  in igc_ring_irq_enable()
4221 			igc_set_itr(q_vector);  in igc_ring_irq_enable()
4223 			igc_update_ring_itr(q_vector);  in igc_ring_irq_enable()
4228 			wr32(IGC_EIMS, q_vector->eims_value);  in igc_ring_irq_enable()
4270 	struct igc_q_vector *q_vector = container_of(napi,  in igc_poll()  local
4273 	struct igc_ring *rx_ring = q_vector->rx.ring;  in igc_poll()
4277 	if (q_vector->tx.ring)  in igc_poll()
4278 		clean_complete = igc_clean_tx_irq(q_vector, budget);  in igc_poll()
4282 			      igc_clean_rx_irq_zc(q_vector, budget) :  in igc_poll()
4283 			      igc_clean_rx_irq(q_vector, budget);  in igc_poll()
4298 		igc_ring_irq_enable(q_vector);  in igc_poll()
4320 	struct igc_q_vector *q_vector;  in igc_alloc_q_vector()  local
4331 	q_vector = adapter->q_vector[v_idx];  in igc_alloc_q_vector()
4332 	if (!q_vector)  in igc_alloc_q_vector()
4333 		q_vector = kzalloc(struct_size(q_vector, ring, ring_count),  in igc_alloc_q_vector()
4336 		memset(q_vector, 0, struct_size(q_vector, ring, ring_count));  in igc_alloc_q_vector()
4337 	if (!q_vector)  in igc_alloc_q_vector()
4341 	netif_napi_add(adapter->netdev, &q_vector->napi,  in igc_alloc_q_vector()
4345 	adapter->q_vector[v_idx] = q_vector;  in igc_alloc_q_vector()
4346 	q_vector->adapter = adapter;  in igc_alloc_q_vector()
4349 	q_vector->tx.work_limit = adapter->tx_work_limit;  in igc_alloc_q_vector()
4352 	q_vector->itr_register = adapter->io_addr + IGC_EITR(0);  in igc_alloc_q_vector()
4353 	q_vector->itr_val = IGC_START_ITR;  in igc_alloc_q_vector()
4356 	ring = q_vector->ring;  in igc_alloc_q_vector()
4362 			q_vector->itr_val = adapter->rx_itr_setting;  in igc_alloc_q_vector()
4366 			q_vector->itr_val = adapter->tx_itr_setting;  in igc_alloc_q_vector()
4375 		ring->q_vector = q_vector;  in igc_alloc_q_vector()
4378 		igc_add_ring(ring, &q_vector->tx);  in igc_alloc_q_vector()
4397 		ring->q_vector = q_vector;  in igc_alloc_q_vector()
4400 		igc_add_ring(ring, &q_vector->rx);  in igc_alloc_q_vector()
4572 		napi_enable(&adapter->q_vector[i]->napi);  in igc_up()
4577 		igc_assign_vector(adapter->q_vector[0], 0);  in igc_up()
4798 		if (adapter->q_vector[i]) {  in igc_down()
4799 			napi_synchronize(&adapter->q_vector[i]->napi);  in igc_down()
4800 			napi_disable(&adapter->q_vector[i]->napi);  in igc_down()
5099 static void igc_write_itr(struct igc_q_vector *q_vector)  in igc_write_itr()  argument
5101 	u32 itr_val = q_vector->itr_val & IGC_QVECTOR_MASK;  in igc_write_itr()
5103 	if (!q_vector->set_itr)  in igc_write_itr()
5111 	writel(itr_val, q_vector->itr_register);  in igc_write_itr()
5112 	q_vector->set_itr = 0;  in igc_write_itr()
5117 	struct igc_q_vector *q_vector = data;  in igc_msix_ring()  local
5120 	igc_write_itr(q_vector);  in igc_msix_ring()
5122 	napi_schedule(&q_vector->napi);  in igc_msix_ring()
5152 		struct igc_q_vector *q_vector = adapter->q_vector[i];  in igc_request_msix()  local
5156 		q_vector->itr_register = adapter->io_addr + IGC_EITR(vector);  in igc_request_msix()
5158 		if (q_vector->rx.ring && q_vector->tx.ring)  in igc_request_msix()
5159 			sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,  in igc_request_msix()
5160 				q_vector->rx.ring->queue_index);  in igc_request_msix()
5161 		else if (q_vector->tx.ring)  in igc_request_msix()
5162 			sprintf(q_vector->name, "%s-tx-%u", netdev->name,  in igc_request_msix()
5163 				q_vector->tx.ring->queue_index);  in igc_request_msix()
5164 		else if (q_vector->rx.ring)  in igc_request_msix()
5165 			sprintf(q_vector->name, "%s-rx-%u", netdev->name,  in igc_request_msix()
5166 				q_vector->rx.ring->queue_index);  in igc_request_msix()
5168 			sprintf(q_vector->name, "%s-unused", netdev->name);  in igc_request_msix()
5171 				  igc_msix_ring, 0, q_vector->name,  in igc_request_msix()
5172 				  q_vector);  in igc_request_msix()
5187 			 adapter->q_vector[i]);  in igc_request_msix()
5421 			eics |= adapter->q_vector[i]->eims_value;  in igc_watchdog_task()
5448 	struct igc_q_vector *q_vector = adapter->q_vector[0];  in igc_intr_msi()  local
5453 	igc_write_itr(q_vector);  in igc_intr_msi()
5469 	napi_schedule(&q_vector->napi);  in igc_intr_msi()
5482 	struct igc_q_vector *q_vector = adapter->q_vector[0];  in igc_intr()  local
5495 	igc_write_itr(q_vector);  in igc_intr()
5512 	napi_schedule(&q_vector->napi);  in igc_intr()
5526 				 adapter->q_vector[i]);  in igc_free_irq()
5562 	igc_assign_vector(adapter->q_vector[0], 0);  in igc_request_irq()
5648 		napi_enable(&adapter->q_vector[i]->napi);  in __igc_open()
6051 					struct igc_q_vector *q_vector)  in igc_trigger_rxtxq_interrupt()  argument
6056 	eics |= q_vector->eims_value;  in igc_trigger_rxtxq_interrupt()
6063 	struct igc_q_vector *q_vector;  in igc_xsk_wakeup()  local
6080 	q_vector = adapter->q_vector[queue_id];  in igc_xsk_wakeup()
6081 	if (!napi_if_scheduled_mark_missed(&q_vector->napi))  in igc_xsk_wakeup()
6082 		igc_trigger_rxtxq_interrupt(adapter, q_vector);  in igc_xsk_wakeup()
6870 	struct igc_hw *hw = &ring->q_vector->adapter->hw;  in igc_disable_rx_ring_hw()
6888 	struct igc_adapter *adapter = ring->q_vector->adapter;  in igc_enable_rx_ring()
6900 	struct igc_hw *hw = &ring->q_vector->adapter->hw;  in igc_disable_tx_ring_hw()
6918 	struct igc_adapter *adapter = ring->q_vector->adapter;  in igc_enable_tx_ring()