135 static void iavf_force_wb(struct iavf_vsi *vsi, struct iavf_q_vector *q_vector)  in iavf_force_wb()  argument
144 	     IAVF_VFINT_DYN_CTLN1(q_vector->reg_idx),  in iavf_force_wb()
187 				iavf_force_wb(vsi, tx_ring->q_vector);  in iavf_detect_recover_hung()
306 	tx_ring->q_vector->tx.total_bytes += total_bytes;  in iavf_clean_tx_irq()
307 	tx_ring->q_vector->tx.total_packets += total_packets;  in iavf_clean_tx_irq()
354 				  struct iavf_q_vector *q_vector)  in iavf_enable_wb_on_itr()  argument
356 	u16 flags = q_vector->tx.ring[0].flags;  in iavf_enable_wb_on_itr()
362 	if (q_vector->arm_wb_state)  in iavf_enable_wb_on_itr()
369 	     IAVF_VFINT_DYN_CTLN1(q_vector->reg_idx), val);  in iavf_enable_wb_on_itr()
370 	q_vector->arm_wb_state = true;  in iavf_enable_wb_on_itr()
373 static inline bool iavf_container_is_rx(struct iavf_q_vector *q_vector,  in iavf_container_is_rx()  argument
376 	return &q_vector->rx == rc;  in iavf_container_is_rx()
448 static void iavf_update_itr(struct iavf_q_vector *q_vector,  in iavf_update_itr()  argument
463 	itr = iavf_container_is_rx(q_vector, rc) ?  in iavf_update_itr()
481 	if (q_vector->itr_countdown) {  in iavf_update_itr()
489 	if (iavf_container_is_rx(q_vector, rc)) {  in iavf_update_itr()
496 		    (q_vector->tx.target_itr & IAVF_ITR_ADAPTIVE_LATENCY)) {  in iavf_update_itr()
507 		    (q_vector->rx.target_itr & IAVF_ITR_MASK) ==  in iavf_update_itr()
535 		itr = min(q_vector->tx.current_itr, q_vector->rx.current_itr);  in iavf_update_itr()
623 			    iavf_itr_divisor(q_vector->adapter)) *  in iavf_update_itr()
897 	struct iavf_q_vector *q_vector = rx_ring->q_vector;  in iavf_receive_skb()  local
906 	napi_gro_receive(&q_vector->napi, skb);  in iavf_receive_skb()
1338 	skb = __napi_alloc_skb(&rx_ring->q_vector->napi,  in iavf_construct_skb()
1612 	rx_ring->q_vector->rx.total_packets += total_rx_packets;  in iavf_clean_rx_irq()
1613 	rx_ring->q_vector->rx.total_bytes += total_rx_bytes;  in iavf_clean_rx_irq()
1666 					  struct iavf_q_vector *q_vector)  in iavf_update_enable_itr()  argument
1672 	iavf_update_itr(q_vector, &q_vector->tx);  in iavf_update_enable_itr()
1673 	iavf_update_itr(q_vector, &q_vector->rx);  in iavf_update_enable_itr()
1683 	if (q_vector->rx.target_itr < q_vector->rx.current_itr) {  in iavf_update_enable_itr()
1686 					   q_vector->rx.target_itr);  in iavf_update_enable_itr()
1687 		q_vector->rx.current_itr = q_vector->rx.target_itr;  in iavf_update_enable_itr()
1688 		q_vector->itr_countdown = ITR_COUNTDOWN_START;  in iavf_update_enable_itr()
1689 	} else if ((q_vector->tx.target_itr < q_vector->tx.current_itr) ||  in iavf_update_enable_itr()
1690 		   ((q_vector->rx.target_itr - q_vector->rx.current_itr) <  in iavf_update_enable_itr()
1691 		    (q_vector->tx.target_itr - q_vector->tx.current_itr))) {  in iavf_update_enable_itr()
1696 					   q_vector->tx.target_itr);  in iavf_update_enable_itr()
1697 		q_vector->tx.current_itr = q_vector->tx.target_itr;  in iavf_update_enable_itr()
1698 		q_vector->itr_countdown = ITR_COUNTDOWN_START;  in iavf_update_enable_itr()
1699 	} else if (q_vector->rx.current_itr != q_vector->rx.target_itr) {  in iavf_update_enable_itr()
1702 					   q_vector->rx.target_itr);  in iavf_update_enable_itr()
1703 		q_vector->rx.current_itr = q_vector->rx.target_itr;  in iavf_update_enable_itr()
1704 		q_vector->itr_countdown = ITR_COUNTDOWN_START;  in iavf_update_enable_itr()
1708 		if (q_vector->itr_countdown)  in iavf_update_enable_itr()
1709 			q_vector->itr_countdown--;  in iavf_update_enable_itr()
1713 		wr32(hw, INTREG(q_vector->reg_idx), intval);  in iavf_update_enable_itr()
1727 	struct iavf_q_vector *q_vector =  in iavf_napi_poll()  local
1729 	struct iavf_vsi *vsi = q_vector->vsi;  in iavf_napi_poll()
1744 	iavf_for_each_ring(ring, q_vector->tx) {  in iavf_napi_poll()
1760 	budget_per_ring = max(budget/q_vector->num_ringpairs, 1);  in iavf_napi_poll()
1762 	iavf_for_each_ring(ring, q_vector->rx) {  in iavf_napi_poll()
1782 		if (!cpumask_test_cpu(cpu_id, &q_vector->affinity_mask)) {  in iavf_napi_poll()
1787 			iavf_force_wb(vsi, q_vector);  in iavf_napi_poll()
1794 			q_vector->tx.ring[0].tx_stats.tx_force_wb++;  in iavf_napi_poll()
1795 			iavf_enable_wb_on_itr(vsi, q_vector);  in iavf_napi_poll()
1801 		q_vector->arm_wb_state = false;  in iavf_napi_poll()
1807 		iavf_update_enable_itr(vsi, q_vector);  in iavf_napi_poll()