60 static void igc_write_itr(struct igc_q_vector *q_vector);
61 static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector);
1307 	skb = napi_alloc_skb(&rx_ring->q_vector->napi, IGC_RX_HDR_LEN);  in igc_construct_skb()
1557 static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget)  in igc_clean_rx_irq()  argument
1560 	struct igc_ring *rx_ring = q_vector->rx.ring;  in igc_clean_rx_irq()
1623 		napi_gro_receive(&q_vector->napi, skb);  in igc_clean_rx_irq()
1639 	q_vector->rx.total_packets += total_packets;  in igc_clean_rx_irq()
1640 	q_vector->rx.total_bytes += total_bytes;  in igc_clean_rx_irq()
1701 static bool igc_clean_tx_irq(struct igc_q_vector *q_vector, int napi_budget)  in igc_clean_tx_irq()  argument
1703 	struct igc_adapter *adapter = q_vector->adapter;  in igc_clean_tx_irq()
1705 	unsigned int budget = q_vector->tx.work_limit;  in igc_clean_tx_irq()
1706 	struct igc_ring *tx_ring = q_vector->tx.ring;  in igc_clean_tx_irq()
1798 	q_vector->tx.total_bytes += total_bytes;  in igc_clean_tx_irq()
1799 	q_vector->tx.total_packets += total_packets;  in igc_clean_tx_irq()
1880 		napi_enable(&adapter->q_vector[i]->napi);  in igc_up()
1885 		igc_assign_vector(adapter->q_vector[0], 0);  in igc_up()
2137 		if (adapter->q_vector[i]) {  in igc_down()
2138 			napi_synchronize(&adapter->q_vector[i]->napi);  in igc_down()
2139 			napi_disable(&adapter->q_vector[i]->napi);  in igc_down()
2591 static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector)  in igc_assign_vector()  argument
2593 	struct igc_adapter *adapter = q_vector->adapter;  in igc_assign_vector()
2598 	if (q_vector->rx.ring)  in igc_assign_vector()
2599 		rx_queue = q_vector->rx.ring->reg_idx;  in igc_assign_vector()
2600 	if (q_vector->tx.ring)  in igc_assign_vector()
2601 		tx_queue = q_vector->tx.ring->reg_idx;  in igc_assign_vector()
2613 		q_vector->eims_value = BIT(msix_vector);  in igc_assign_vector()
2621 	adapter->eims_enable_mask |= q_vector->eims_value;  in igc_assign_vector()
2624 	q_vector->set_itr = 1;  in igc_assign_vector()
2666 		igc_assign_vector(adapter->q_vector[i], vector++);  in igc_configure_msix()
2673 	struct igc_q_vector *q_vector = data;  in igc_msix_ring()  local
2676 	igc_write_itr(q_vector);  in igc_msix_ring()
2678 	napi_schedule(&q_vector->napi);  in igc_msix_ring()
2701 		struct igc_q_vector *q_vector = adapter->q_vector[i];  in igc_request_msix()  local
2705 		q_vector->itr_register = adapter->io_addr + IGC_EITR(vector);  in igc_request_msix()
2707 		if (q_vector->rx.ring && q_vector->tx.ring)  in igc_request_msix()
2708 			sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,  in igc_request_msix()
2709 				q_vector->rx.ring->queue_index);  in igc_request_msix()
2710 		else if (q_vector->tx.ring)  in igc_request_msix()
2711 			sprintf(q_vector->name, "%s-tx-%u", netdev->name,  in igc_request_msix()
2712 				q_vector->tx.ring->queue_index);  in igc_request_msix()
2713 		else if (q_vector->rx.ring)  in igc_request_msix()
2714 			sprintf(q_vector->name, "%s-rx-%u", netdev->name,  in igc_request_msix()
2715 				q_vector->rx.ring->queue_index);  in igc_request_msix()
2717 			sprintf(q_vector->name, "%s-unused", netdev->name);  in igc_request_msix()
2720 				  igc_msix_ring, 0, q_vector->name,  in igc_request_msix()
2721 				  q_vector);  in igc_request_msix()
2736 			 adapter->q_vector[i]);  in igc_request_msix()
2752 	struct igc_q_vector *q_vector = adapter->q_vector[v_idx];  in igc_reset_q_vector()  local
2757 	if (!q_vector)  in igc_reset_q_vector()
2760 	if (q_vector->tx.ring)  in igc_reset_q_vector()
2761 		adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL;  in igc_reset_q_vector()
2763 	if (q_vector->rx.ring)  in igc_reset_q_vector()
2764 		adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL;  in igc_reset_q_vector()
2766 	netif_napi_del(&q_vector->napi);  in igc_reset_q_vector()
2829 	struct igc_q_vector *q_vector = adapter->q_vector[v_idx];  in igc_free_q_vector()  local
2831 	adapter->q_vector[v_idx] = NULL;  in igc_free_q_vector()
2836 	if (q_vector)  in igc_free_q_vector()
2837 		kfree_rcu(q_vector, rcu);  in igc_free_q_vector()
3058 			eics |= adapter->q_vector[i]->eims_value;  in igc_watchdog_task()
3089 static void igc_update_ring_itr(struct igc_q_vector *q_vector)  in igc_update_ring_itr()  argument
3091 	struct igc_adapter *adapter = q_vector->adapter;  in igc_update_ring_itr()
3092 	int new_val = q_vector->itr_val;  in igc_update_ring_itr()
3108 	packets = q_vector->rx.total_packets;  in igc_update_ring_itr()
3110 		avg_wire_size = q_vector->rx.total_bytes / packets;  in igc_update_ring_itr()
3112 	packets = q_vector->tx.total_packets;  in igc_update_ring_itr()
3115 				      q_vector->tx.total_bytes / packets);  in igc_update_ring_itr()
3135 	    ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||  in igc_update_ring_itr()
3136 	    (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))  in igc_update_ring_itr()
3140 	if (new_val != q_vector->itr_val) {  in igc_update_ring_itr()
3141 		q_vector->itr_val = new_val;  in igc_update_ring_itr()
3142 		q_vector->set_itr = 1;  in igc_update_ring_itr()
3145 	q_vector->rx.total_bytes = 0;  in igc_update_ring_itr()
3146 	q_vector->rx.total_packets = 0;  in igc_update_ring_itr()
3147 	q_vector->tx.total_bytes = 0;  in igc_update_ring_itr()
3148 	q_vector->tx.total_packets = 0;  in igc_update_ring_itr()
3166 static void igc_update_itr(struct igc_q_vector *q_vector,  in igc_update_itr()  argument
3226 	struct igc_q_vector *q_vector = adapter->q_vector[0];  in igc_intr_msi()  local
3231 	igc_write_itr(q_vector);  in igc_intr_msi()
3247 	napi_schedule(&q_vector->napi);  in igc_intr_msi()
3260 	struct igc_q_vector *q_vector = adapter->q_vector[0];  in igc_intr()  local
3273 	igc_write_itr(q_vector);  in igc_intr()
3290 	napi_schedule(&q_vector->napi);  in igc_intr()
3295 static void igc_set_itr(struct igc_q_vector *q_vector)  in igc_set_itr()  argument
3297 	struct igc_adapter *adapter = q_vector->adapter;  in igc_set_itr()
3298 	u32 new_itr = q_vector->itr_val;  in igc_set_itr()
3312 	igc_update_itr(q_vector, &q_vector->tx);  in igc_set_itr()
3313 	igc_update_itr(q_vector, &q_vector->rx);  in igc_set_itr()
3315 	current_itr = max(q_vector->rx.itr, q_vector->tx.itr);  in igc_set_itr()
3319 	    ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||  in igc_set_itr()
3320 	    (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))  in igc_set_itr()
3339 	if (new_itr != q_vector->itr_val) {  in igc_set_itr()
3344 		new_itr = new_itr > q_vector->itr_val ?  in igc_set_itr()
3345 			  max((new_itr * q_vector->itr_val) /  in igc_set_itr()
3346 			  (new_itr + (q_vector->itr_val >> 2)),  in igc_set_itr()
3354 		q_vector->itr_val = new_itr;  in igc_set_itr()
3355 		q_vector->set_itr = 1;  in igc_set_itr()
3359 static void igc_ring_irq_enable(struct igc_q_vector *q_vector)  in igc_ring_irq_enable()  argument
3361 	struct igc_adapter *adapter = q_vector->adapter;  in igc_ring_irq_enable()
3364 	if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) ||  in igc_ring_irq_enable()
3365 	    (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) {  in igc_ring_irq_enable()
3367 			igc_set_itr(q_vector);  in igc_ring_irq_enable()
3369 			igc_update_ring_itr(q_vector);  in igc_ring_irq_enable()
3374 			wr32(IGC_EIMS, q_vector->eims_value);  in igc_ring_irq_enable()
3387 	struct igc_q_vector *q_vector = container_of(napi,  in igc_poll()  local
3393 	if (q_vector->tx.ring)  in igc_poll()
3394 		clean_complete = igc_clean_tx_irq(q_vector, budget);  in igc_poll()
3396 	if (q_vector->rx.ring) {  in igc_poll()
3397 		int cleaned = igc_clean_rx_irq(q_vector, budget);  in igc_poll()
3412 		igc_ring_irq_enable(q_vector);  in igc_poll()
3510 	struct igc_q_vector *q_vector;  in igc_alloc_q_vector()  local
3521 	q_vector = adapter->q_vector[v_idx];  in igc_alloc_q_vector()
3522 	if (!q_vector)  in igc_alloc_q_vector()
3523 		q_vector = kzalloc(struct_size(q_vector, ring, ring_count),  in igc_alloc_q_vector()
3526 		memset(q_vector, 0, struct_size(q_vector, ring, ring_count));  in igc_alloc_q_vector()
3527 	if (!q_vector)  in igc_alloc_q_vector()
3531 	netif_napi_add(adapter->netdev, &q_vector->napi,  in igc_alloc_q_vector()
3535 	adapter->q_vector[v_idx] = q_vector;  in igc_alloc_q_vector()
3536 	q_vector->adapter = adapter;  in igc_alloc_q_vector()
3539 	q_vector->tx.work_limit = adapter->tx_work_limit;  in igc_alloc_q_vector()
3542 	q_vector->itr_register = adapter->io_addr + IGC_EITR(0);  in igc_alloc_q_vector()
3543 	q_vector->itr_val = IGC_START_ITR;  in igc_alloc_q_vector()
3546 	ring = q_vector->ring;  in igc_alloc_q_vector()
3552 			q_vector->itr_val = adapter->rx_itr_setting;  in igc_alloc_q_vector()
3556 			q_vector->itr_val = adapter->tx_itr_setting;  in igc_alloc_q_vector()
3565 		ring->q_vector = q_vector;  in igc_alloc_q_vector()
3568 		igc_add_ring(ring, &q_vector->tx);  in igc_alloc_q_vector()
3587 		ring->q_vector = q_vector;  in igc_alloc_q_vector()
3590 		igc_add_ring(ring, &q_vector->rx);  in igc_alloc_q_vector()
3722 				 adapter->q_vector[i]);  in igc_free_irq()
3814 	igc_assign_vector(adapter->q_vector[0], 0);  in igc_request_irq()
3838 static void igc_write_itr(struct igc_q_vector *q_vector)  in igc_write_itr()  argument
3840 	u32 itr_val = q_vector->itr_val & IGC_QVECTOR_MASK;  in igc_write_itr()
3842 	if (!q_vector->set_itr)  in igc_write_itr()
3850 	writel(itr_val, q_vector->itr_register);  in igc_write_itr()
3851 	q_vector->set_itr = 0;  in igc_write_itr()
3912 		napi_enable(&adapter->q_vector[i]->napi);  in __igc_open()