87 	struct hns3_enet_tqp_vector *tqp_vector = vector;  in hns3_irq_handle()  local
89 	napi_schedule_irqoff(&tqp_vector->napi);  in hns3_irq_handle()
100 		tqp_vectors = &priv->tqp_vector[i];  in hns3_nic_uninit_irq()
124 		tqp_vectors = &priv->tqp_vector[i];  in hns3_nic_init_irq()
167 static void hns3_mask_vector_irq(struct hns3_enet_tqp_vector *tqp_vector,  in hns3_mask_vector_irq()  argument
170 	writel(mask_en, tqp_vector->mask_addr);  in hns3_mask_vector_irq()
173 static void hns3_vector_enable(struct hns3_enet_tqp_vector *tqp_vector)  in hns3_vector_enable()  argument
175 	napi_enable(&tqp_vector->napi);  in hns3_vector_enable()
178 	hns3_mask_vector_irq(tqp_vector, 1);  in hns3_vector_enable()
181 static void hns3_vector_disable(struct hns3_enet_tqp_vector *tqp_vector)  in hns3_vector_disable()  argument
184 	hns3_mask_vector_irq(tqp_vector, 0);  in hns3_vector_disable()
186 	disable_irq(tqp_vector->vector_irq);  in hns3_vector_disable()
187 	napi_disable(&tqp_vector->napi);  in hns3_vector_disable()
190 void hns3_set_vector_coalesce_rl(struct hns3_enet_tqp_vector *tqp_vector,  in hns3_set_vector_coalesce_rl()  argument
200 	if (rl_reg > 0 && !tqp_vector->tx_group.coal.gl_adapt_enable &&  in hns3_set_vector_coalesce_rl()
201 	    !tqp_vector->rx_group.coal.gl_adapt_enable)  in hns3_set_vector_coalesce_rl()
207 	writel(rl_reg, tqp_vector->mask_addr + HNS3_VECTOR_RL_OFFSET);  in hns3_set_vector_coalesce_rl()
210 void hns3_set_vector_coalesce_rx_gl(struct hns3_enet_tqp_vector *tqp_vector,  in hns3_set_vector_coalesce_rx_gl()  argument
215 	writel(rx_gl_reg, tqp_vector->mask_addr + HNS3_VECTOR_GL0_OFFSET);  in hns3_set_vector_coalesce_rx_gl()
218 void hns3_set_vector_coalesce_tx_gl(struct hns3_enet_tqp_vector *tqp_vector,  in hns3_set_vector_coalesce_tx_gl()  argument
223 	writel(tx_gl_reg, tqp_vector->mask_addr + HNS3_VECTOR_GL1_OFFSET);  in hns3_set_vector_coalesce_tx_gl()
226 static void hns3_vector_gl_rl_init(struct hns3_enet_tqp_vector *tqp_vector,  in hns3_vector_gl_rl_init()  argument
235 	tqp_vector->tx_group.coal.gl_adapt_enable = 1;  in hns3_vector_gl_rl_init()
236 	tqp_vector->rx_group.coal.gl_adapt_enable = 1;  in hns3_vector_gl_rl_init()
238 	tqp_vector->tx_group.coal.int_gl = HNS3_INT_GL_50K;  in hns3_vector_gl_rl_init()
239 	tqp_vector->rx_group.coal.int_gl = HNS3_INT_GL_50K;  in hns3_vector_gl_rl_init()
241 	tqp_vector->rx_group.coal.flow_level = HNS3_FLOW_LOW;  in hns3_vector_gl_rl_init()
242 	tqp_vector->tx_group.coal.flow_level = HNS3_FLOW_LOW;  in hns3_vector_gl_rl_init()
245 static void hns3_vector_gl_rl_init_hw(struct hns3_enet_tqp_vector *tqp_vector,  in hns3_vector_gl_rl_init_hw()  argument
250 	hns3_set_vector_coalesce_tx_gl(tqp_vector,  in hns3_vector_gl_rl_init_hw()
251 				       tqp_vector->tx_group.coal.int_gl);  in hns3_vector_gl_rl_init_hw()
252 	hns3_set_vector_coalesce_rx_gl(tqp_vector,  in hns3_vector_gl_rl_init_hw()
253 				       tqp_vector->rx_group.coal.int_gl);  in hns3_vector_gl_rl_init_hw()
254 	hns3_set_vector_coalesce_rl(tqp_vector, h->kinfo.int_rl_setting);  in hns3_vector_gl_rl_init_hw()
342 	struct hns3_enet_tqp_vector *tqp_vector;  in hns3_set_rx_cpu_rmap()  local
352 		tqp_vector = &priv->tqp_vector[i];  in hns3_set_rx_cpu_rmap()
354 				       tqp_vector->vector_irq);  in hns3_set_rx_cpu_rmap()
391 		hns3_vector_enable(&priv->tqp_vector[i]);  in hns3_nic_net_up()
410 		hns3_vector_disable(&priv->tqp_vector[j]);  in hns3_nic_net_up()
423 		struct hns3_enet_tqp_vector *tqp_vector = &priv->tqp_vector[i];  in hns3_config_xps()  local
424 		struct hns3_enet_ring *ring = tqp_vector->tx_group.ring;  in hns3_config_xps()
430 						  &tqp_vector->affinity_mask,  in hns3_config_xps()
500 		hns3_vector_disable(&priv->tqp_vector[i]);  in hns3_nic_net_down()
1709 	napi = &tx_ring->tqp_vector->napi;  in hns3_get_tx_timeo_queue_info()
1759 		    readl(tx_ring->tqp_vector->mask_addr));  in hns3_get_tx_timeo_queue_info()
2349 	ring->tqp_vector->tx_group.total_bytes += bytes;  in hns3_clean_tx_ring()
2350 	ring->tqp_vector->tx_group.total_packets += pkts;  in hns3_clean_tx_ring()
2404 				hns3_rl_err(ring->tqp_vector->napi.dev,  in hns3_nic_alloc_rx_buffers()
2567 		napi_gro_flush(&ring->tqp_vector->napi, false);  in hns3_rx_skb()
2569 	napi_gro_receive(&ring->tqp_vector->napi, skb);  in hns3_rx_skb()
2632 	ring->skb = napi_alloc_skb(&ring->tqp_vector->napi, HNS3_RX_HEAD_SIZE);  in hns3_alloc_skb()
2707 			new_skb = napi_alloc_skb(&ring->tqp_vector->napi,  in hns3_add_frag()
2710 				hns3_rl_err(ring->tqp_vector->napi.dev,  in hns3_add_frag()
2855 	ring->tqp_vector->rx_group.total_bytes += len;  in hns3_handle_bdinfo()
3002 	struct hns3_enet_tqp_vector *tqp_vector;  in hns3_get_new_flow_lvl()  local
3006 	tqp_vector = ring_group->ring->tqp_vector;  in hns3_get_new_flow_lvl()
3008 		jiffies_to_msecs(jiffies - tqp_vector->last_jiffies);  in hns3_get_new_flow_lvl()
3046 	    &tqp_vector->rx_group == ring_group)  in hns3_get_new_flow_lvl()
3058 	struct hns3_enet_tqp_vector *tqp_vector;  in hns3_get_new_int_gl()  local
3064 	tqp_vector = ring_group->ring->tqp_vector;  in hns3_get_new_int_gl()
3065 	if (!tqp_vector->last_jiffies)  in hns3_get_new_int_gl()
3102 static void hns3_update_new_int_gl(struct hns3_enet_tqp_vector *tqp_vector)  in hns3_update_new_int_gl()  argument
3104 	struct hns3_enet_ring_group *rx_group = &tqp_vector->rx_group;  in hns3_update_new_int_gl()
3105 	struct hns3_enet_ring_group *tx_group = &tqp_vector->tx_group;  in hns3_update_new_int_gl()
3110 			tqp_vector->last_jiffies + msecs_to_jiffies(1000)))  in hns3_update_new_int_gl()
3116 			hns3_set_vector_coalesce_rx_gl(tqp_vector,  in hns3_update_new_int_gl()
3123 			hns3_set_vector_coalesce_tx_gl(tqp_vector,  in hns3_update_new_int_gl()
3127 	tqp_vector->last_jiffies = jiffies;  in hns3_update_new_int_gl()
3136 	struct hns3_enet_tqp_vector *tqp_vector =  in hns3_nic_common_poll()  local
3149 	hns3_for_each_ring(ring, tqp_vector->tx_group)  in hns3_nic_common_poll()
3153 	if (tqp_vector->num_tqps > 1)  in hns3_nic_common_poll()
3154 		rx_budget = max(budget / tqp_vector->num_tqps, 1);  in hns3_nic_common_poll()
3156 	hns3_for_each_ring(ring, tqp_vector->rx_group) {  in hns3_nic_common_poll()
3166 	tqp_vector->rx_group.total_packets += rx_pkt_total;  in hns3_nic_common_poll()
3173 		hns3_update_new_int_gl(tqp_vector);  in hns3_nic_common_poll()
3174 		hns3_mask_vector_irq(tqp_vector, 1);  in hns3_nic_common_poll()
3180 static int hns3_get_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,  in hns3_get_vector_ring_chain()  argument
3183 	struct pci_dev *pdev = tqp_vector->handle->pdev;  in hns3_get_vector_ring_chain()
3189 	tx_ring = tqp_vector->tx_group.ring;  in hns3_get_vector_ring_chain()
3220 	rx_ring = tqp_vector->rx_group.ring;  in hns3_get_vector_ring_chain()
3263 static void hns3_free_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,  in hns3_free_vector_ring_chain()  argument
3266 	struct pci_dev *pdev = tqp_vector->handle->pdev;  in hns3_free_vector_ring_chain()
3290 	struct hns3_enet_tqp_vector *tqp_vector;  in hns3_nic_set_cpumask()  local
3298 		tqp_vector = &priv->tqp_vector[vector_i];  in hns3_nic_set_cpumask()
3300 				&tqp_vector->affinity_mask);  in hns3_nic_set_cpumask()
3308 	struct hns3_enet_tqp_vector *tqp_vector;  in hns3_nic_init_vector_data()  local
3315 		tqp_vector = &priv->tqp_vector[i];  in hns3_nic_init_vector_data()
3316 		hns3_vector_gl_rl_init_hw(tqp_vector, priv);  in hns3_nic_init_vector_data()
3317 		tqp_vector->num_tqps = 0;  in hns3_nic_init_vector_data()
3324 		tqp_vector = &priv->tqp_vector[vector_i];  in hns3_nic_init_vector_data()
3326 		hns3_add_ring_to_group(&tqp_vector->tx_group,  in hns3_nic_init_vector_data()
3329 		hns3_add_ring_to_group(&tqp_vector->rx_group,  in hns3_nic_init_vector_data()
3332 		priv->ring_data[i].ring->tqp_vector = tqp_vector;  in hns3_nic_init_vector_data()
3333 		priv->ring_data[i + tqp_num].ring->tqp_vector = tqp_vector;  in hns3_nic_init_vector_data()
3334 		tqp_vector->num_tqps++;  in hns3_nic_init_vector_data()
3338 		tqp_vector = &priv->tqp_vector[i];  in hns3_nic_init_vector_data()
3340 		tqp_vector->rx_group.total_bytes = 0;  in hns3_nic_init_vector_data()
3341 		tqp_vector->rx_group.total_packets = 0;  in hns3_nic_init_vector_data()
3342 		tqp_vector->tx_group.total_bytes = 0;  in hns3_nic_init_vector_data()
3343 		tqp_vector->tx_group.total_packets = 0;  in hns3_nic_init_vector_data()
3344 		tqp_vector->handle = h;  in hns3_nic_init_vector_data()
3346 		ret = hns3_get_vector_ring_chain(tqp_vector,  in hns3_nic_init_vector_data()
3352 			tqp_vector->vector_irq, &vector_ring_chain);  in hns3_nic_init_vector_data()
3354 		hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);  in hns3_nic_init_vector_data()
3359 		netif_napi_add(priv->netdev, &tqp_vector->napi,  in hns3_nic_init_vector_data()
3367 		netif_napi_del(&priv->tqp_vector[i].napi);  in hns3_nic_init_vector_data()
3377 	struct hns3_enet_tqp_vector *tqp_vector;  in hns3_nic_alloc_vector_data()  local
3399 	priv->tqp_vector = (struct hns3_enet_tqp_vector *)  in hns3_nic_alloc_vector_data()
3400 		devm_kcalloc(&pdev->dev, vector_num, sizeof(*priv->tqp_vector),  in hns3_nic_alloc_vector_data()
3402 	if (!priv->tqp_vector) {  in hns3_nic_alloc_vector_data()
3408 		tqp_vector = &priv->tqp_vector[i];  in hns3_nic_alloc_vector_data()
3409 		tqp_vector->idx = i;  in hns3_nic_alloc_vector_data()
3410 		tqp_vector->mask_addr = vector[i].io_addr;  in hns3_nic_alloc_vector_data()
3411 		tqp_vector->vector_irq = vector[i].vector;  in hns3_nic_alloc_vector_data()
3412 		hns3_vector_gl_rl_init(tqp_vector, priv);  in hns3_nic_alloc_vector_data()
3430 	struct hns3_enet_tqp_vector *tqp_vector;  in hns3_nic_uninit_vector_data()  local
3434 		tqp_vector = &priv->tqp_vector[i];  in hns3_nic_uninit_vector_data()
3436 		if (!tqp_vector->rx_group.ring && !tqp_vector->tx_group.ring)  in hns3_nic_uninit_vector_data()
3439 		hns3_get_vector_ring_chain(tqp_vector, &vector_ring_chain);  in hns3_nic_uninit_vector_data()
3442 			tqp_vector->vector_irq, &vector_ring_chain);  in hns3_nic_uninit_vector_data()
3444 		hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);  in hns3_nic_uninit_vector_data()
3446 		if (tqp_vector->irq_init_flag == HNS3_VECTOR_INITED) {  in hns3_nic_uninit_vector_data()
3447 			irq_set_affinity_hint(tqp_vector->vector_irq, NULL);  in hns3_nic_uninit_vector_data()
3448 			free_irq(tqp_vector->vector_irq, tqp_vector);  in hns3_nic_uninit_vector_data()
3449 			tqp_vector->irq_init_flag = HNS3_VECTOR_NOT_INITED;  in hns3_nic_uninit_vector_data()
3452 		hns3_clear_ring_group(&tqp_vector->rx_group);  in hns3_nic_uninit_vector_data()
3453 		hns3_clear_ring_group(&tqp_vector->tx_group);  in hns3_nic_uninit_vector_data()
3454 		netif_napi_del(&priv->tqp_vector[i].napi);  in hns3_nic_uninit_vector_data()
3465 		struct hns3_enet_tqp_vector *tqp_vector;  in hns3_nic_dealloc_vector_data()  local
3467 		tqp_vector = &priv->tqp_vector[i];  in hns3_nic_dealloc_vector_data()
3468 		ret = h->ae_algo->ops->put_vector(h, tqp_vector->vector_irq);  in hns3_nic_dealloc_vector_data()
3473 	devm_kfree(&pdev->dev, priv->tqp_vector);  in hns3_nic_dealloc_vector_data()
4214 	memcpy(&priv->tx_coal, &priv->tqp_vector[0].tx_group.coal,  in hns3_store_coal()
4216 	memcpy(&priv->rx_coal, &priv->tqp_vector[0].rx_group.coal,  in hns3_store_coal()
4226 		memcpy(&priv->tqp_vector[i].tx_group.coal, &priv->tx_coal,  in hns3_restore_coal()
4228 		memcpy(&priv->tqp_vector[i].rx_group.coal, &priv->rx_coal,  in hns3_restore_coal()