384 	struct hns3_enet_tqp_vector *tqp_vector = vector;  in hns3_irq_handle()  local
386 	napi_schedule_irqoff(&tqp_vector->napi);  in hns3_irq_handle()
387 	tqp_vector->event_cnt++;  in hns3_irq_handle()
398 		tqp_vectors = &priv->tqp_vector[i];  in hns3_nic_uninit_irq()
422 		tqp_vectors = &priv->tqp_vector[i];  in hns3_nic_init_irq()
469 static void hns3_mask_vector_irq(struct hns3_enet_tqp_vector *tqp_vector,  in hns3_mask_vector_irq()  argument
472 	writel(mask_en, tqp_vector->mask_addr);  in hns3_mask_vector_irq()
475 static void hns3_vector_enable(struct hns3_enet_tqp_vector *tqp_vector)  in hns3_vector_enable()  argument
477 	napi_enable(&tqp_vector->napi);  in hns3_vector_enable()
478 	enable_irq(tqp_vector->vector_irq);  in hns3_vector_enable()
481 	hns3_mask_vector_irq(tqp_vector, 1);  in hns3_vector_enable()
484 static void hns3_vector_disable(struct hns3_enet_tqp_vector *tqp_vector)  in hns3_vector_disable()  argument
487 	hns3_mask_vector_irq(tqp_vector, 0);  in hns3_vector_disable()
489 	disable_irq(tqp_vector->vector_irq);  in hns3_vector_disable()
490 	napi_disable(&tqp_vector->napi);  in hns3_vector_disable()
491 	cancel_work_sync(&tqp_vector->rx_group.dim.work);  in hns3_vector_disable()
492 	cancel_work_sync(&tqp_vector->tx_group.dim.work);  in hns3_vector_disable()
495 void hns3_set_vector_coalesce_rl(struct hns3_enet_tqp_vector *tqp_vector,  in hns3_set_vector_coalesce_rl()  argument
504 	if (rl_reg > 0 && !tqp_vector->tx_group.coal.adapt_enable &&  in hns3_set_vector_coalesce_rl()
505 	    !tqp_vector->rx_group.coal.adapt_enable)  in hns3_set_vector_coalesce_rl()
511 	writel(rl_reg, tqp_vector->mask_addr + HNS3_VECTOR_RL_OFFSET);  in hns3_set_vector_coalesce_rl()
514 void hns3_set_vector_coalesce_rx_gl(struct hns3_enet_tqp_vector *tqp_vector,  in hns3_set_vector_coalesce_rx_gl()  argument
519 	if (tqp_vector->rx_group.coal.unit_1us)  in hns3_set_vector_coalesce_rx_gl()
524 	writel(new_val, tqp_vector->mask_addr + HNS3_VECTOR_GL0_OFFSET);  in hns3_set_vector_coalesce_rx_gl()
527 void hns3_set_vector_coalesce_tx_gl(struct hns3_enet_tqp_vector *tqp_vector,  in hns3_set_vector_coalesce_tx_gl()  argument
532 	if (tqp_vector->tx_group.coal.unit_1us)  in hns3_set_vector_coalesce_tx_gl()
537 	writel(new_val, tqp_vector->mask_addr + HNS3_VECTOR_GL1_OFFSET);  in hns3_set_vector_coalesce_tx_gl()
540 void hns3_set_vector_coalesce_tx_ql(struct hns3_enet_tqp_vector *tqp_vector,  in hns3_set_vector_coalesce_tx_ql()  argument
543 	writel(ql_value, tqp_vector->mask_addr + HNS3_VECTOR_TX_QL_OFFSET);  in hns3_set_vector_coalesce_tx_ql()
546 void hns3_set_vector_coalesce_rx_ql(struct hns3_enet_tqp_vector *tqp_vector,  in hns3_set_vector_coalesce_rx_ql()  argument
549 	writel(ql_value, tqp_vector->mask_addr + HNS3_VECTOR_RX_QL_OFFSET);  in hns3_set_vector_coalesce_rx_ql()
552 static void hns3_vector_coalesce_init(struct hns3_enet_tqp_vector *tqp_vector,  in hns3_vector_coalesce_init()  argument
556 	struct hns3_enet_coalesce *tx_coal = &tqp_vector->tx_group.coal;  in hns3_vector_coalesce_init()
557 	struct hns3_enet_coalesce *rx_coal = &tqp_vector->rx_group.coal;  in hns3_vector_coalesce_init()
589 hns3_vector_coalesce_init_hw(struct hns3_enet_tqp_vector *tqp_vector,  in hns3_vector_coalesce_init_hw()  argument
592 	struct hns3_enet_coalesce *tx_coal = &tqp_vector->tx_group.coal;  in hns3_vector_coalesce_init_hw()
593 	struct hns3_enet_coalesce *rx_coal = &tqp_vector->rx_group.coal;  in hns3_vector_coalesce_init_hw()
596 	hns3_set_vector_coalesce_tx_gl(tqp_vector, tx_coal->int_gl);  in hns3_vector_coalesce_init_hw()
597 	hns3_set_vector_coalesce_rx_gl(tqp_vector, rx_coal->int_gl);  in hns3_vector_coalesce_init_hw()
598 	hns3_set_vector_coalesce_rl(tqp_vector, h->kinfo.int_rl_setting);  in hns3_vector_coalesce_init_hw()
601 		hns3_set_vector_coalesce_tx_ql(tqp_vector, tx_coal->int_ql);  in hns3_vector_coalesce_init_hw()
604 		hns3_set_vector_coalesce_rx_ql(tqp_vector, rx_coal->int_ql);  in hns3_vector_coalesce_init_hw()
687 	struct hns3_enet_tqp_vector *tqp_vector;  in hns3_set_rx_cpu_rmap()  local
697 		tqp_vector = &priv->tqp_vector[i];  in hns3_set_rx_cpu_rmap()
699 				       tqp_vector->vector_irq);  in hns3_set_rx_cpu_rmap()
724 		hns3_vector_enable(&priv->tqp_vector[i]);  in hns3_nic_net_up()
738 			hns3_vector_disable(&priv->tqp_vector[j]);  in hns3_nic_net_up()
749 		struct hns3_enet_tqp_vector *tqp_vector = &priv->tqp_vector[i];  in hns3_config_xps()  local
750 		struct hns3_enet_ring *ring = tqp_vector->tx_group.ring;  in hns3_config_xps()
756 						  &tqp_vector->affinity_mask,  in hns3_config_xps()
831 		hns3_vector_disable(&priv->tqp_vector[i]);  in hns3_nic_net_down()
2795 	struct napi_struct *napi = &tx_ring->tqp_vector->napi;  in hns3_dump_queue_stats()
2826 		    readl(tx_ring->tqp_vector->mask_addr));  in hns3_dump_queue_reg()
3628 	ring->tqp_vector->tx_group.total_bytes += bytes;  in hns3_clean_tx_ring()
3629 	ring->tqp_vector->tx_group.total_packets += pkts;  in hns3_clean_tx_ring()
3950 		napi_gro_flush(&ring->tqp_vector->napi, false);  in hns3_rx_skb()
3952 	napi_gro_receive(&ring->tqp_vector->napi, skb);  in hns3_rx_skb()
4026 	ring->skb = napi_alloc_skb(&ring->tqp_vector->napi, HNS3_RX_HEAD_SIZE);  in hns3_alloc_skb()
4091 			new_skb = napi_alloc_skb(&ring->tqp_vector->napi, 0);  in hns3_add_frag()
4305 	ring->tqp_vector->rx_group.total_bytes += len;  in hns3_handle_bdinfo()
4429 static void hns3_update_rx_int_coalesce(struct hns3_enet_tqp_vector *tqp_vector)  in hns3_update_rx_int_coalesce()  argument
4431 	struct hns3_enet_ring_group *rx_group = &tqp_vector->rx_group;  in hns3_update_rx_int_coalesce()
4437 	dim_update_sample(tqp_vector->event_cnt, rx_group->total_packets,  in hns3_update_rx_int_coalesce()
4442 static void hns3_update_tx_int_coalesce(struct hns3_enet_tqp_vector *tqp_vector)  in hns3_update_tx_int_coalesce()  argument
4444 	struct hns3_enet_ring_group *tx_group = &tqp_vector->tx_group;  in hns3_update_tx_int_coalesce()
4450 	dim_update_sample(tqp_vector->event_cnt, tx_group->total_packets,  in hns3_update_tx_int_coalesce()
4461 	struct hns3_enet_tqp_vector *tqp_vector =  in hns3_nic_common_poll()  local
4474 	hns3_for_each_ring(ring, tqp_vector->tx_group)  in hns3_nic_common_poll()
4478 	if (tqp_vector->num_tqps > 1)  in hns3_nic_common_poll()
4479 		rx_budget = max(budget / tqp_vector->num_tqps, 1);  in hns3_nic_common_poll()
4481 	hns3_for_each_ring(ring, tqp_vector->rx_group) {  in hns3_nic_common_poll()
4490 	tqp_vector->rx_group.total_packets += rx_pkt_total;  in hns3_nic_common_poll()
4497 		hns3_update_rx_int_coalesce(tqp_vector);  in hns3_nic_common_poll()
4498 		hns3_update_tx_int_coalesce(tqp_vector);  in hns3_nic_common_poll()
4500 		hns3_mask_vector_irq(tqp_vector, 1);  in hns3_nic_common_poll()
4506 static int hns3_create_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,  in hns3_create_ring_chain()  argument
4513 	struct pci_dev *pdev = tqp_vector->handle->pdev;  in hns3_create_ring_chain()
4517 	ring = is_tx ? tqp_vector->tx_group.ring : tqp_vector->rx_group.ring;  in hns3_create_ring_chain()
4548 hns3_get_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector)  in hns3_get_vector_ring_chain()  argument
4550 	struct pci_dev *pdev = tqp_vector->handle->pdev;  in hns3_get_vector_ring_chain()
4554 	if (hns3_create_ring_chain(tqp_vector, &cur_chain, true))  in hns3_get_vector_ring_chain()
4557 	if (hns3_create_ring_chain(tqp_vector, &cur_chain, false))  in hns3_get_vector_ring_chain()
4572 static void hns3_free_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,  in hns3_free_vector_ring_chain()  argument
4575 	struct pci_dev *pdev = tqp_vector->handle->pdev;  in hns3_free_vector_ring_chain()
4599 	struct hns3_enet_tqp_vector *tqp_vector;  in hns3_nic_set_cpumask()  local
4607 		tqp_vector = &priv->tqp_vector[vector_i];  in hns3_nic_set_cpumask()
4609 				&tqp_vector->affinity_mask);  in hns3_nic_set_cpumask()
4618 	struct hns3_enet_tqp_vector *tqp_vector = group->ring->tqp_vector;  in hns3_rx_dim_work()  local
4622 	hns3_set_vector_coalesce_rx_gl(group->ring->tqp_vector, cur_moder.usec);  in hns3_rx_dim_work()
4623 	tqp_vector->rx_group.coal.int_gl = cur_moder.usec;  in hns3_rx_dim_work()
4625 	if (cur_moder.pkts < tqp_vector->rx_group.coal.int_ql_max) {  in hns3_rx_dim_work()
4626 		hns3_set_vector_coalesce_rx_ql(tqp_vector, cur_moder.pkts);  in hns3_rx_dim_work()
4627 		tqp_vector->rx_group.coal.int_ql = cur_moder.pkts;  in hns3_rx_dim_work()
4638 	struct hns3_enet_tqp_vector *tqp_vector = group->ring->tqp_vector;  in hns3_tx_dim_work()  local
4642 	hns3_set_vector_coalesce_tx_gl(tqp_vector, cur_moder.usec);  in hns3_tx_dim_work()
4643 	tqp_vector->tx_group.coal.int_gl = cur_moder.usec;  in hns3_tx_dim_work()
4645 	if (cur_moder.pkts < tqp_vector->tx_group.coal.int_ql_max) {  in hns3_tx_dim_work()
4646 		hns3_set_vector_coalesce_tx_ql(tqp_vector, cur_moder.pkts);  in hns3_tx_dim_work()
4647 		tqp_vector->tx_group.coal.int_ql = cur_moder.pkts;  in hns3_tx_dim_work()
4653 static void hns3_nic_init_dim(struct hns3_enet_tqp_vector *tqp_vector)  in hns3_nic_init_dim()  argument
4655 	INIT_WORK(&tqp_vector->rx_group.dim.work, hns3_rx_dim_work);  in hns3_nic_init_dim()
4656 	INIT_WORK(&tqp_vector->tx_group.dim.work, hns3_tx_dim_work);  in hns3_nic_init_dim()
4662 	struct hns3_enet_tqp_vector *tqp_vector;  in hns3_nic_init_vector_data()  local
4669 		tqp_vector = &priv->tqp_vector[i];  in hns3_nic_init_vector_data()
4670 		hns3_vector_coalesce_init_hw(tqp_vector, priv);  in hns3_nic_init_vector_data()
4671 		tqp_vector->num_tqps = 0;  in hns3_nic_init_vector_data()
4672 		hns3_nic_init_dim(tqp_vector);  in hns3_nic_init_vector_data()
4679 		tqp_vector = &priv->tqp_vector[vector_i];  in hns3_nic_init_vector_data()
4681 		hns3_add_ring_to_group(&tqp_vector->tx_group,  in hns3_nic_init_vector_data()
4684 		hns3_add_ring_to_group(&tqp_vector->rx_group,  in hns3_nic_init_vector_data()
4687 		priv->ring[i].tqp_vector = tqp_vector;  in hns3_nic_init_vector_data()
4688 		priv->ring[i + tqp_num].tqp_vector = tqp_vector;  in hns3_nic_init_vector_data()
4689 		tqp_vector->num_tqps++;  in hns3_nic_init_vector_data()
4695 		tqp_vector = &priv->tqp_vector[i];  in hns3_nic_init_vector_data()
4697 		tqp_vector->rx_group.total_bytes = 0;  in hns3_nic_init_vector_data()
4698 		tqp_vector->rx_group.total_packets = 0;  in hns3_nic_init_vector_data()
4699 		tqp_vector->tx_group.total_bytes = 0;  in hns3_nic_init_vector_data()
4700 		tqp_vector->tx_group.total_packets = 0;  in hns3_nic_init_vector_data()
4701 		tqp_vector->handle = h;  in hns3_nic_init_vector_data()
4703 		vector_ring_chain = hns3_get_vector_ring_chain(tqp_vector);  in hns3_nic_init_vector_data()
4710 			tqp_vector->vector_irq, vector_ring_chain);  in hns3_nic_init_vector_data()
4712 		hns3_free_vector_ring_chain(tqp_vector, vector_ring_chain);  in hns3_nic_init_vector_data()
4717 		netif_napi_add(priv->netdev, &tqp_vector->napi,  in hns3_nic_init_vector_data()
4725 		netif_napi_del(&priv->tqp_vector[i].napi);  in hns3_nic_init_vector_data()
4761 	struct hns3_enet_tqp_vector *tqp_vector;  in hns3_nic_alloc_vector_data()  local
4782 	priv->tqp_vector = (struct hns3_enet_tqp_vector *)  in hns3_nic_alloc_vector_data()
4783 		devm_kcalloc(&pdev->dev, vector_num, sizeof(*priv->tqp_vector),  in hns3_nic_alloc_vector_data()
4785 	if (!priv->tqp_vector) {  in hns3_nic_alloc_vector_data()
4791 		tqp_vector = &priv->tqp_vector[i];  in hns3_nic_alloc_vector_data()
4792 		tqp_vector->idx = i;  in hns3_nic_alloc_vector_data()
4793 		tqp_vector->mask_addr = vector[i].io_addr;  in hns3_nic_alloc_vector_data()
4794 		tqp_vector->vector_irq = vector[i].vector;  in hns3_nic_alloc_vector_data()
4795 		hns3_vector_coalesce_init(tqp_vector, priv);  in hns3_nic_alloc_vector_data()
4813 	struct hns3_enet_tqp_vector *tqp_vector;  in hns3_nic_uninit_vector_data()  local
4817 		tqp_vector = &priv->tqp_vector[i];  in hns3_nic_uninit_vector_data()
4819 		if (!tqp_vector->rx_group.ring && !tqp_vector->tx_group.ring)  in hns3_nic_uninit_vector_data()
4826 		vector_ring_chain = hns3_get_vector_ring_chain(tqp_vector);  in hns3_nic_uninit_vector_data()
4831 			tqp_vector->vector_irq, vector_ring_chain);  in hns3_nic_uninit_vector_data()
4833 		hns3_free_vector_ring_chain(tqp_vector, vector_ring_chain);  in hns3_nic_uninit_vector_data()
4835 		hns3_clear_ring_group(&tqp_vector->rx_group);  in hns3_nic_uninit_vector_data()
4836 		hns3_clear_ring_group(&tqp_vector->tx_group);  in hns3_nic_uninit_vector_data()
4837 		netif_napi_del(&priv->tqp_vector[i].napi);  in hns3_nic_uninit_vector_data()
4848 		struct hns3_enet_tqp_vector *tqp_vector;  in hns3_nic_dealloc_vector_data()  local
4850 		tqp_vector = &priv->tqp_vector[i];  in hns3_nic_dealloc_vector_data()
4851 		ret = h->ae_algo->ops->put_vector(h, tqp_vector->vector_irq);  in hns3_nic_dealloc_vector_data()
4856 	devm_kfree(&pdev->dev, priv->tqp_vector);  in hns3_nic_dealloc_vector_data()
5219 			priv->tqp_vector[i].tx_group.dim.mode = mode;  in hns3_set_cq_period_mode()
5224 			priv->tqp_vector[i].rx_group.dim.mode = mode;  in hns3_set_cq_period_mode()
5863 		hns3_vector_disable(&priv->tqp_vector[i]);  in hns3_external_lb_prepare()
5890 		hns3_vector_enable(&priv->tqp_vector[i]);  in hns3_external_lb_restore()