1597 	skb = napi_alloc_skb(&rx_ring->q_vector->napi, IGC_RX_HDR_LEN);  in igc_construct_skb()
1602 		igc_ptp_rx_pktstamp(rx_ring->q_vector, va, skb);  in igc_construct_skb()
1898 static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget)  in igc_clean_rx_irq()  argument
1901 	struct igc_ring *rx_ring = q_vector->rx.ring;  in igc_clean_rx_irq()
1964 		napi_gro_receive(&q_vector->napi, skb);  in igc_clean_rx_irq()
1980 	q_vector->rx.total_packets += total_packets;  in igc_clean_rx_irq()
1981 	q_vector->rx.total_bytes += total_bytes;  in igc_clean_rx_irq()
1996 static bool igc_clean_tx_irq(struct igc_q_vector *q_vector, int napi_budget)  in igc_clean_tx_irq()  argument
1998 	struct igc_adapter *adapter = q_vector->adapter;  in igc_clean_tx_irq()
2000 	unsigned int budget = q_vector->tx.work_limit;  in igc_clean_tx_irq()
2001 	struct igc_ring *tx_ring = q_vector->tx.ring;  in igc_clean_tx_irq()
2093 	q_vector->tx.total_bytes += total_bytes;  in igc_clean_tx_irq()
2094 	q_vector->tx.total_packets += total_packets;  in igc_clean_tx_irq()
2703 static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector)  in igc_assign_vector()  argument
2705 	struct igc_adapter *adapter = q_vector->adapter;  in igc_assign_vector()
2710 	if (q_vector->rx.ring)  in igc_assign_vector()
2711 		rx_queue = q_vector->rx.ring->reg_idx;  in igc_assign_vector()
2712 	if (q_vector->tx.ring)  in igc_assign_vector()
2713 		tx_queue = q_vector->tx.ring->reg_idx;  in igc_assign_vector()
2725 		q_vector->eims_value = BIT(msix_vector);  in igc_assign_vector()
2733 	adapter->eims_enable_mask |= q_vector->eims_value;  in igc_assign_vector()
2736 	q_vector->set_itr = 1;  in igc_assign_vector()
2778 		igc_assign_vector(adapter->q_vector[i], vector++);  in igc_configure_msix()
2877 	struct igc_q_vector *q_vector = adapter->q_vector[v_idx];  in igc_reset_q_vector()  local
2882 	if (!q_vector)  in igc_reset_q_vector()
2885 	if (q_vector->tx.ring)  in igc_reset_q_vector()
2886 		adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL;  in igc_reset_q_vector()
2888 	if (q_vector->rx.ring)  in igc_reset_q_vector()
2889 		adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL;  in igc_reset_q_vector()
2891 	netif_napi_del(&q_vector->napi);  in igc_reset_q_vector()
2903 	struct igc_q_vector *q_vector = adapter->q_vector[v_idx];  in igc_free_q_vector()  local
2905 	adapter->q_vector[v_idx] = NULL;  in igc_free_q_vector()
2910 	if (q_vector)  in igc_free_q_vector()
2911 		kfree_rcu(q_vector, rcu);  in igc_free_q_vector()
2951 static void igc_update_itr(struct igc_q_vector *q_vector,  in igc_update_itr()  argument
3003 static void igc_set_itr(struct igc_q_vector *q_vector)  in igc_set_itr()  argument
3005 	struct igc_adapter *adapter = q_vector->adapter;  in igc_set_itr()
3006 	u32 new_itr = q_vector->itr_val;  in igc_set_itr()
3020 	igc_update_itr(q_vector, &q_vector->tx);  in igc_set_itr()
3021 	igc_update_itr(q_vector, &q_vector->rx);  in igc_set_itr()
3023 	current_itr = max(q_vector->rx.itr, q_vector->tx.itr);  in igc_set_itr()
3027 	    ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||  in igc_set_itr()
3028 	    (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))  in igc_set_itr()
3047 	if (new_itr != q_vector->itr_val) {  in igc_set_itr()
3052 		new_itr = new_itr > q_vector->itr_val ?  in igc_set_itr()
3053 			  max((new_itr * q_vector->itr_val) /  in igc_set_itr()
3054 			  (new_itr + (q_vector->itr_val >> 2)),  in igc_set_itr()
3062 		q_vector->itr_val = new_itr;  in igc_set_itr()
3063 		q_vector->set_itr = 1;  in igc_set_itr()
3167 static void igc_update_ring_itr(struct igc_q_vector *q_vector)  in igc_update_ring_itr()  argument
3169 	struct igc_adapter *adapter = q_vector->adapter;  in igc_update_ring_itr()
3170 	int new_val = q_vector->itr_val;  in igc_update_ring_itr()
3186 	packets = q_vector->rx.total_packets;  in igc_update_ring_itr()
3188 		avg_wire_size = q_vector->rx.total_bytes / packets;  in igc_update_ring_itr()
3190 	packets = q_vector->tx.total_packets;  in igc_update_ring_itr()
3193 				      q_vector->tx.total_bytes / packets);  in igc_update_ring_itr()
3213 	    ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||  in igc_update_ring_itr()
3214 	    (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))  in igc_update_ring_itr()
3218 	if (new_val != q_vector->itr_val) {  in igc_update_ring_itr()
3219 		q_vector->itr_val = new_val;  in igc_update_ring_itr()
3220 		q_vector->set_itr = 1;  in igc_update_ring_itr()
3223 	q_vector->rx.total_bytes = 0;  in igc_update_ring_itr()
3224 	q_vector->rx.total_packets = 0;  in igc_update_ring_itr()
3225 	q_vector->tx.total_bytes = 0;  in igc_update_ring_itr()
3226 	q_vector->tx.total_packets = 0;  in igc_update_ring_itr()
3229 static void igc_ring_irq_enable(struct igc_q_vector *q_vector)  in igc_ring_irq_enable()  argument
3231 	struct igc_adapter *adapter = q_vector->adapter;  in igc_ring_irq_enable()
3234 	if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) ||  in igc_ring_irq_enable()
3235 	    (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) {  in igc_ring_irq_enable()
3237 			igc_set_itr(q_vector);  in igc_ring_irq_enable()
3239 			igc_update_ring_itr(q_vector);  in igc_ring_irq_enable()
3244 			wr32(IGC_EIMS, q_vector->eims_value);  in igc_ring_irq_enable()
3286 	struct igc_q_vector *q_vector = container_of(napi,  in igc_poll()  local
3292 	if (q_vector->tx.ring)  in igc_poll()
3293 		clean_complete = igc_clean_tx_irq(q_vector, budget);  in igc_poll()
3295 	if (q_vector->rx.ring) {  in igc_poll()
3296 		int cleaned = igc_clean_rx_irq(q_vector, budget);  in igc_poll()
3311 		igc_ring_irq_enable(q_vector);  in igc_poll()
3333 	struct igc_q_vector *q_vector;  in igc_alloc_q_vector()  local
3344 	q_vector = adapter->q_vector[v_idx];  in igc_alloc_q_vector()
3345 	if (!q_vector)  in igc_alloc_q_vector()
3346 		q_vector = kzalloc(struct_size(q_vector, ring, ring_count),  in igc_alloc_q_vector()
3349 		memset(q_vector, 0, struct_size(q_vector, ring, ring_count));  in igc_alloc_q_vector()
3350 	if (!q_vector)  in igc_alloc_q_vector()
3354 	netif_napi_add(adapter->netdev, &q_vector->napi,  in igc_alloc_q_vector()
3358 	adapter->q_vector[v_idx] = q_vector;  in igc_alloc_q_vector()
3359 	q_vector->adapter = adapter;  in igc_alloc_q_vector()
3362 	q_vector->tx.work_limit = adapter->tx_work_limit;  in igc_alloc_q_vector()
3365 	q_vector->itr_register = adapter->io_addr + IGC_EITR(0);  in igc_alloc_q_vector()
3366 	q_vector->itr_val = IGC_START_ITR;  in igc_alloc_q_vector()
3369 	ring = q_vector->ring;  in igc_alloc_q_vector()
3375 			q_vector->itr_val = adapter->rx_itr_setting;  in igc_alloc_q_vector()
3379 			q_vector->itr_val = adapter->tx_itr_setting;  in igc_alloc_q_vector()
3388 		ring->q_vector = q_vector;  in igc_alloc_q_vector()
3391 		igc_add_ring(ring, &q_vector->tx);  in igc_alloc_q_vector()
3410 		ring->q_vector = q_vector;  in igc_alloc_q_vector()
3413 		igc_add_ring(ring, &q_vector->rx);  in igc_alloc_q_vector()
3585 		napi_enable(&adapter->q_vector[i]->napi);  in igc_up()
3590 		igc_assign_vector(adapter->q_vector[0], 0);  in igc_up()
3807 		if (adapter->q_vector[i]) {  in igc_down()
3808 			napi_synchronize(&adapter->q_vector[i]->napi);  in igc_down()
3809 			napi_disable(&adapter->q_vector[i]->napi);  in igc_down()
4032 static void igc_write_itr(struct igc_q_vector *q_vector)  in igc_write_itr()  argument
4034 	u32 itr_val = q_vector->itr_val & IGC_QVECTOR_MASK;  in igc_write_itr()
4036 	if (!q_vector->set_itr)  in igc_write_itr()
4044 	writel(itr_val, q_vector->itr_register);  in igc_write_itr()
4045 	q_vector->set_itr = 0;  in igc_write_itr()
4050 	struct igc_q_vector *q_vector = data;  in igc_msix_ring()  local
4053 	igc_write_itr(q_vector);  in igc_msix_ring()
4055 	napi_schedule(&q_vector->napi);  in igc_msix_ring()
4078 		struct igc_q_vector *q_vector = adapter->q_vector[i];  in igc_request_msix()  local
4082 		q_vector->itr_register = adapter->io_addr + IGC_EITR(vector);  in igc_request_msix()
4084 		if (q_vector->rx.ring && q_vector->tx.ring)  in igc_request_msix()
4085 			sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,  in igc_request_msix()
4086 				q_vector->rx.ring->queue_index);  in igc_request_msix()
4087 		else if (q_vector->tx.ring)  in igc_request_msix()
4088 			sprintf(q_vector->name, "%s-tx-%u", netdev->name,  in igc_request_msix()
4089 				q_vector->tx.ring->queue_index);  in igc_request_msix()
4090 		else if (q_vector->rx.ring)  in igc_request_msix()
4091 			sprintf(q_vector->name, "%s-rx-%u", netdev->name,  in igc_request_msix()
4092 				q_vector->rx.ring->queue_index);  in igc_request_msix()
4094 			sprintf(q_vector->name, "%s-unused", netdev->name);  in igc_request_msix()
4097 				  igc_msix_ring, 0, q_vector->name,  in igc_request_msix()
4098 				  q_vector);  in igc_request_msix()
4113 			 adapter->q_vector[i]);  in igc_request_msix()
4353 			eics |= adapter->q_vector[i]->eims_value;  in igc_watchdog_task()
4380 	struct igc_q_vector *q_vector = adapter->q_vector[0];  in igc_intr_msi()  local
4385 	igc_write_itr(q_vector);  in igc_intr_msi()
4401 	napi_schedule(&q_vector->napi);  in igc_intr_msi()
4414 	struct igc_q_vector *q_vector = adapter->q_vector[0];  in igc_intr()  local
4427 	igc_write_itr(q_vector);  in igc_intr()
4444 	napi_schedule(&q_vector->napi);  in igc_intr()
4458 				 adapter->q_vector[i]);  in igc_free_irq()
4494 	igc_assign_vector(adapter->q_vector[0], 0);  in igc_request_irq()
4580 		napi_enable(&adapter->q_vector[i]->napi);  in __igc_open()