385 	struct hns3_enet_tqp_vector *tqp_vector = vector;  in hns3_irq_handle()  local
387 	napi_schedule_irqoff(&tqp_vector->napi);  in hns3_irq_handle()
388 	tqp_vector->event_cnt++;  in hns3_irq_handle()
399 		tqp_vectors = &priv->tqp_vector[i];  in hns3_nic_uninit_irq()
423 		tqp_vectors = &priv->tqp_vector[i];  in hns3_nic_init_irq()
470 static void hns3_mask_vector_irq(struct hns3_enet_tqp_vector *tqp_vector,  in hns3_mask_vector_irq()  argument
473 	writel(mask_en, tqp_vector->mask_addr);  in hns3_mask_vector_irq()
476 static void hns3_vector_enable(struct hns3_enet_tqp_vector *tqp_vector)  in hns3_vector_enable()  argument
478 	napi_enable(&tqp_vector->napi);  in hns3_vector_enable()
479 	enable_irq(tqp_vector->vector_irq);  in hns3_vector_enable()
482 	hns3_mask_vector_irq(tqp_vector, 1);  in hns3_vector_enable()
485 static void hns3_vector_disable(struct hns3_enet_tqp_vector *tqp_vector)  in hns3_vector_disable()  argument
488 	hns3_mask_vector_irq(tqp_vector, 0);  in hns3_vector_disable()
490 	disable_irq(tqp_vector->vector_irq);  in hns3_vector_disable()
491 	napi_disable(&tqp_vector->napi);  in hns3_vector_disable()
492 	cancel_work_sync(&tqp_vector->rx_group.dim.work);  in hns3_vector_disable()
493 	cancel_work_sync(&tqp_vector->tx_group.dim.work);  in hns3_vector_disable()
496 void hns3_set_vector_coalesce_rl(struct hns3_enet_tqp_vector *tqp_vector,  in hns3_set_vector_coalesce_rl()  argument
505 	if (rl_reg > 0 && !tqp_vector->tx_group.coal.adapt_enable &&  in hns3_set_vector_coalesce_rl()
506 	    !tqp_vector->rx_group.coal.adapt_enable)  in hns3_set_vector_coalesce_rl()
512 	writel(rl_reg, tqp_vector->mask_addr + HNS3_VECTOR_RL_OFFSET);  in hns3_set_vector_coalesce_rl()
515 void hns3_set_vector_coalesce_rx_gl(struct hns3_enet_tqp_vector *tqp_vector,  in hns3_set_vector_coalesce_rx_gl()  argument
520 	if (tqp_vector->rx_group.coal.unit_1us)  in hns3_set_vector_coalesce_rx_gl()
525 	writel(new_val, tqp_vector->mask_addr + HNS3_VECTOR_GL0_OFFSET);  in hns3_set_vector_coalesce_rx_gl()
528 void hns3_set_vector_coalesce_tx_gl(struct hns3_enet_tqp_vector *tqp_vector,  in hns3_set_vector_coalesce_tx_gl()  argument
533 	if (tqp_vector->tx_group.coal.unit_1us)  in hns3_set_vector_coalesce_tx_gl()
538 	writel(new_val, tqp_vector->mask_addr + HNS3_VECTOR_GL1_OFFSET);  in hns3_set_vector_coalesce_tx_gl()
541 void hns3_set_vector_coalesce_tx_ql(struct hns3_enet_tqp_vector *tqp_vector,  in hns3_set_vector_coalesce_tx_ql()  argument
544 	writel(ql_value, tqp_vector->mask_addr + HNS3_VECTOR_TX_QL_OFFSET);  in hns3_set_vector_coalesce_tx_ql()
547 void hns3_set_vector_coalesce_rx_ql(struct hns3_enet_tqp_vector *tqp_vector,  in hns3_set_vector_coalesce_rx_ql()  argument
550 	writel(ql_value, tqp_vector->mask_addr + HNS3_VECTOR_RX_QL_OFFSET);  in hns3_set_vector_coalesce_rx_ql()
553 static void hns3_vector_coalesce_init(struct hns3_enet_tqp_vector *tqp_vector,  in hns3_vector_coalesce_init()  argument
557 	struct hns3_enet_coalesce *tx_coal = &tqp_vector->tx_group.coal;  in hns3_vector_coalesce_init()
558 	struct hns3_enet_coalesce *rx_coal = &tqp_vector->rx_group.coal;  in hns3_vector_coalesce_init()
590 hns3_vector_coalesce_init_hw(struct hns3_enet_tqp_vector *tqp_vector,  in hns3_vector_coalesce_init_hw()  argument
593 	struct hns3_enet_coalesce *tx_coal = &tqp_vector->tx_group.coal;  in hns3_vector_coalesce_init_hw()
594 	struct hns3_enet_coalesce *rx_coal = &tqp_vector->rx_group.coal;  in hns3_vector_coalesce_init_hw()
597 	hns3_set_vector_coalesce_tx_gl(tqp_vector, tx_coal->int_gl);  in hns3_vector_coalesce_init_hw()
598 	hns3_set_vector_coalesce_rx_gl(tqp_vector, rx_coal->int_gl);  in hns3_vector_coalesce_init_hw()
599 	hns3_set_vector_coalesce_rl(tqp_vector, h->kinfo.int_rl_setting);  in hns3_vector_coalesce_init_hw()
602 		hns3_set_vector_coalesce_tx_ql(tqp_vector, tx_coal->int_ql);  in hns3_vector_coalesce_init_hw()
605 		hns3_set_vector_coalesce_rx_ql(tqp_vector, rx_coal->int_ql);  in hns3_vector_coalesce_init_hw()
688 	struct hns3_enet_tqp_vector *tqp_vector;  in hns3_set_rx_cpu_rmap()  local
698 		tqp_vector = &priv->tqp_vector[i];  in hns3_set_rx_cpu_rmap()
700 				       tqp_vector->vector_irq);  in hns3_set_rx_cpu_rmap()
725 		hns3_vector_enable(&priv->tqp_vector[i]);  in hns3_nic_net_up()
739 			hns3_vector_disable(&priv->tqp_vector[j]);  in hns3_nic_net_up()
750 		struct hns3_enet_tqp_vector *tqp_vector = &priv->tqp_vector[i];  in hns3_config_xps()  local
751 		struct hns3_enet_ring *ring = tqp_vector->tx_group.ring;  in hns3_config_xps()
757 						  &tqp_vector->affinity_mask,  in hns3_config_xps()
832 		hns3_vector_disable(&priv->tqp_vector[i]);  in hns3_nic_net_down()
2703 	napi = &tx_ring->tqp_vector->napi;  in hns3_get_tx_timeo_queue_info()
2753 		    readl(tx_ring->tqp_vector->mask_addr));  in hns3_get_tx_timeo_queue_info()
3453 	ring->tqp_vector->tx_group.total_bytes += bytes;  in hns3_clean_tx_ring()
3454 	ring->tqp_vector->tx_group.total_packets += pkts;  in hns3_clean_tx_ring()
3774 		napi_gro_flush(&ring->tqp_vector->napi, false);  in hns3_rx_skb()
3776 	napi_gro_receive(&ring->tqp_vector->napi, skb);  in hns3_rx_skb()
3850 	ring->skb = napi_alloc_skb(&ring->tqp_vector->napi, HNS3_RX_HEAD_SIZE);  in hns3_alloc_skb()
3920 			new_skb = napi_alloc_skb(&ring->tqp_vector->napi, 0);  in hns3_add_frag()
4100 	ring->tqp_vector->rx_group.total_bytes += len;  in hns3_handle_bdinfo()
4216 static void hns3_update_rx_int_coalesce(struct hns3_enet_tqp_vector *tqp_vector)  in hns3_update_rx_int_coalesce()  argument
4218 	struct hns3_enet_ring_group *rx_group = &tqp_vector->rx_group;  in hns3_update_rx_int_coalesce()
4224 	dim_update_sample(tqp_vector->event_cnt, rx_group->total_packets,  in hns3_update_rx_int_coalesce()
4229 static void hns3_update_tx_int_coalesce(struct hns3_enet_tqp_vector *tqp_vector)  in hns3_update_tx_int_coalesce()  argument
4231 	struct hns3_enet_ring_group *tx_group = &tqp_vector->tx_group;  in hns3_update_tx_int_coalesce()
4237 	dim_update_sample(tqp_vector->event_cnt, tx_group->total_packets,  in hns3_update_tx_int_coalesce()
4248 	struct hns3_enet_tqp_vector *tqp_vector =  in hns3_nic_common_poll()  local
4261 	hns3_for_each_ring(ring, tqp_vector->tx_group)  in hns3_nic_common_poll()
4265 	if (tqp_vector->num_tqps > 1)  in hns3_nic_common_poll()
4266 		rx_budget = max(budget / tqp_vector->num_tqps, 1);  in hns3_nic_common_poll()
4268 	hns3_for_each_ring(ring, tqp_vector->rx_group) {  in hns3_nic_common_poll()
4277 	tqp_vector->rx_group.total_packets += rx_pkt_total;  in hns3_nic_common_poll()
4284 		hns3_update_rx_int_coalesce(tqp_vector);  in hns3_nic_common_poll()
4285 		hns3_update_tx_int_coalesce(tqp_vector);  in hns3_nic_common_poll()
4287 		hns3_mask_vector_irq(tqp_vector, 1);  in hns3_nic_common_poll()
4293 static int hns3_get_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,  in hns3_get_vector_ring_chain()  argument
4296 	struct pci_dev *pdev = tqp_vector->handle->pdev;  in hns3_get_vector_ring_chain()
4302 	tx_ring = tqp_vector->tx_group.ring;  in hns3_get_vector_ring_chain()
4333 	rx_ring = tqp_vector->rx_group.ring;  in hns3_get_vector_ring_chain()
4376 static void hns3_free_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,  in hns3_free_vector_ring_chain()  argument
4379 	struct pci_dev *pdev = tqp_vector->handle->pdev;  in hns3_free_vector_ring_chain()
4403 	struct hns3_enet_tqp_vector *tqp_vector;  in hns3_nic_set_cpumask()  local
4411 		tqp_vector = &priv->tqp_vector[vector_i];  in hns3_nic_set_cpumask()
4413 				&tqp_vector->affinity_mask);  in hns3_nic_set_cpumask()
4422 	struct hns3_enet_tqp_vector *tqp_vector = group->ring->tqp_vector;  in hns3_rx_dim_work()  local
4426 	hns3_set_vector_coalesce_rx_gl(group->ring->tqp_vector, cur_moder.usec);  in hns3_rx_dim_work()
4427 	tqp_vector->rx_group.coal.int_gl = cur_moder.usec;  in hns3_rx_dim_work()
4429 	if (cur_moder.pkts < tqp_vector->rx_group.coal.int_ql_max) {  in hns3_rx_dim_work()
4430 		hns3_set_vector_coalesce_rx_ql(tqp_vector, cur_moder.pkts);  in hns3_rx_dim_work()
4431 		tqp_vector->rx_group.coal.int_ql = cur_moder.pkts;  in hns3_rx_dim_work()
4442 	struct hns3_enet_tqp_vector *tqp_vector = group->ring->tqp_vector;  in hns3_tx_dim_work()  local
4446 	hns3_set_vector_coalesce_tx_gl(tqp_vector, cur_moder.usec);  in hns3_tx_dim_work()
4447 	tqp_vector->tx_group.coal.int_gl = cur_moder.usec;  in hns3_tx_dim_work()
4449 	if (cur_moder.pkts < tqp_vector->tx_group.coal.int_ql_max) {  in hns3_tx_dim_work()
4450 		hns3_set_vector_coalesce_tx_ql(tqp_vector, cur_moder.pkts);  in hns3_tx_dim_work()
4451 		tqp_vector->tx_group.coal.int_ql = cur_moder.pkts;  in hns3_tx_dim_work()
4457 static void hns3_nic_init_dim(struct hns3_enet_tqp_vector *tqp_vector)  in hns3_nic_init_dim()  argument
4459 	INIT_WORK(&tqp_vector->rx_group.dim.work, hns3_rx_dim_work);  in hns3_nic_init_dim()
4460 	INIT_WORK(&tqp_vector->tx_group.dim.work, hns3_tx_dim_work);  in hns3_nic_init_dim()
4466 	struct hns3_enet_tqp_vector *tqp_vector;  in hns3_nic_init_vector_data()  local
4473 		tqp_vector = &priv->tqp_vector[i];  in hns3_nic_init_vector_data()
4474 		hns3_vector_coalesce_init_hw(tqp_vector, priv);  in hns3_nic_init_vector_data()
4475 		tqp_vector->num_tqps = 0;  in hns3_nic_init_vector_data()
4476 		hns3_nic_init_dim(tqp_vector);  in hns3_nic_init_vector_data()
4483 		tqp_vector = &priv->tqp_vector[vector_i];  in hns3_nic_init_vector_data()
4485 		hns3_add_ring_to_group(&tqp_vector->tx_group,  in hns3_nic_init_vector_data()
4488 		hns3_add_ring_to_group(&tqp_vector->rx_group,  in hns3_nic_init_vector_data()
4491 		priv->ring[i].tqp_vector = tqp_vector;  in hns3_nic_init_vector_data()
4492 		priv->ring[i + tqp_num].tqp_vector = tqp_vector;  in hns3_nic_init_vector_data()
4493 		tqp_vector->num_tqps++;  in hns3_nic_init_vector_data()
4499 		tqp_vector = &priv->tqp_vector[i];  in hns3_nic_init_vector_data()
4501 		tqp_vector->rx_group.total_bytes = 0;  in hns3_nic_init_vector_data()
4502 		tqp_vector->rx_group.total_packets = 0;  in hns3_nic_init_vector_data()
4503 		tqp_vector->tx_group.total_bytes = 0;  in hns3_nic_init_vector_data()
4504 		tqp_vector->tx_group.total_packets = 0;  in hns3_nic_init_vector_data()
4505 		tqp_vector->handle = h;  in hns3_nic_init_vector_data()
4507 		ret = hns3_get_vector_ring_chain(tqp_vector,  in hns3_nic_init_vector_data()
4513 			tqp_vector->vector_irq, &vector_ring_chain);  in hns3_nic_init_vector_data()
4515 		hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);  in hns3_nic_init_vector_data()
4520 		netif_napi_add(priv->netdev, &tqp_vector->napi,  in hns3_nic_init_vector_data()
4528 		netif_napi_del(&priv->tqp_vector[i].napi);  in hns3_nic_init_vector_data()
4564 	struct hns3_enet_tqp_vector *tqp_vector;  in hns3_nic_alloc_vector_data()  local
4585 	priv->tqp_vector = (struct hns3_enet_tqp_vector *)  in hns3_nic_alloc_vector_data()
4586 		devm_kcalloc(&pdev->dev, vector_num, sizeof(*priv->tqp_vector),  in hns3_nic_alloc_vector_data()
4588 	if (!priv->tqp_vector) {  in hns3_nic_alloc_vector_data()
4594 		tqp_vector = &priv->tqp_vector[i];  in hns3_nic_alloc_vector_data()
4595 		tqp_vector->idx = i;  in hns3_nic_alloc_vector_data()
4596 		tqp_vector->mask_addr = vector[i].io_addr;  in hns3_nic_alloc_vector_data()
4597 		tqp_vector->vector_irq = vector[i].vector;  in hns3_nic_alloc_vector_data()
4598 		hns3_vector_coalesce_init(tqp_vector, priv);  in hns3_nic_alloc_vector_data()
4616 	struct hns3_enet_tqp_vector *tqp_vector;  in hns3_nic_uninit_vector_data()  local
4620 		tqp_vector = &priv->tqp_vector[i];  in hns3_nic_uninit_vector_data()
4622 		if (!tqp_vector->rx_group.ring && !tqp_vector->tx_group.ring)  in hns3_nic_uninit_vector_data()
4629 		if (hns3_get_vector_ring_chain(tqp_vector, &vector_ring_chain))  in hns3_nic_uninit_vector_data()
4633 			tqp_vector->vector_irq, &vector_ring_chain);  in hns3_nic_uninit_vector_data()
4635 		hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);  in hns3_nic_uninit_vector_data()
4637 		hns3_clear_ring_group(&tqp_vector->rx_group);  in hns3_nic_uninit_vector_data()
4638 		hns3_clear_ring_group(&tqp_vector->tx_group);  in hns3_nic_uninit_vector_data()
4639 		netif_napi_del(&priv->tqp_vector[i].napi);  in hns3_nic_uninit_vector_data()
4650 		struct hns3_enet_tqp_vector *tqp_vector;  in hns3_nic_dealloc_vector_data()  local
4652 		tqp_vector = &priv->tqp_vector[i];  in hns3_nic_dealloc_vector_data()
4653 		ret = h->ae_algo->ops->put_vector(h, tqp_vector->vector_irq);  in hns3_nic_dealloc_vector_data()
4658 	devm_kfree(&pdev->dev, priv->tqp_vector);  in hns3_nic_dealloc_vector_data()
5017 			priv->tqp_vector[i].tx_group.dim.mode = mode;  in hns3_set_cq_period_mode()
5022 			priv->tqp_vector[i].rx_group.dim.mode = mode;  in hns3_set_cq_period_mode()