385 	struct hns3_enet_tqp_vector *tqp_vector = vector;  in hns3_irq_handle()  local
387 	napi_schedule_irqoff(&tqp_vector->napi);  in hns3_irq_handle()
388 	tqp_vector->event_cnt++;  in hns3_irq_handle()
399 		tqp_vectors = &priv->tqp_vector[i];  in hns3_nic_uninit_irq()
423 		tqp_vectors = &priv->tqp_vector[i];  in hns3_nic_init_irq()
470 static void hns3_mask_vector_irq(struct hns3_enet_tqp_vector *tqp_vector,  in hns3_mask_vector_irq()  argument
473 	writel(mask_en, tqp_vector->mask_addr);  in hns3_mask_vector_irq()
476 static void hns3_vector_enable(struct hns3_enet_tqp_vector *tqp_vector)  in hns3_vector_enable()  argument
478 	napi_enable(&tqp_vector->napi);  in hns3_vector_enable()
479 	enable_irq(tqp_vector->vector_irq);  in hns3_vector_enable()
482 	hns3_mask_vector_irq(tqp_vector, 1);  in hns3_vector_enable()
485 static void hns3_vector_disable(struct hns3_enet_tqp_vector *tqp_vector)  in hns3_vector_disable()  argument
488 	hns3_mask_vector_irq(tqp_vector, 0);  in hns3_vector_disable()
490 	disable_irq(tqp_vector->vector_irq);  in hns3_vector_disable()
491 	napi_disable(&tqp_vector->napi);  in hns3_vector_disable()
492 	cancel_work_sync(&tqp_vector->rx_group.dim.work);  in hns3_vector_disable()
493 	cancel_work_sync(&tqp_vector->tx_group.dim.work);  in hns3_vector_disable()
496 void hns3_set_vector_coalesce_rl(struct hns3_enet_tqp_vector *tqp_vector,  in hns3_set_vector_coalesce_rl()  argument
505 	if (rl_reg > 0 && !tqp_vector->tx_group.coal.adapt_enable &&  in hns3_set_vector_coalesce_rl()
506 	    !tqp_vector->rx_group.coal.adapt_enable)  in hns3_set_vector_coalesce_rl()
512 	writel(rl_reg, tqp_vector->mask_addr + HNS3_VECTOR_RL_OFFSET);  in hns3_set_vector_coalesce_rl()
515 void hns3_set_vector_coalesce_rx_gl(struct hns3_enet_tqp_vector *tqp_vector,  in hns3_set_vector_coalesce_rx_gl()  argument
520 	if (tqp_vector->rx_group.coal.unit_1us)  in hns3_set_vector_coalesce_rx_gl()
525 	writel(new_val, tqp_vector->mask_addr + HNS3_VECTOR_GL0_OFFSET);  in hns3_set_vector_coalesce_rx_gl()
528 void hns3_set_vector_coalesce_tx_gl(struct hns3_enet_tqp_vector *tqp_vector,  in hns3_set_vector_coalesce_tx_gl()  argument
533 	if (tqp_vector->tx_group.coal.unit_1us)  in hns3_set_vector_coalesce_tx_gl()
538 	writel(new_val, tqp_vector->mask_addr + HNS3_VECTOR_GL1_OFFSET);  in hns3_set_vector_coalesce_tx_gl()
541 void hns3_set_vector_coalesce_tx_ql(struct hns3_enet_tqp_vector *tqp_vector,  in hns3_set_vector_coalesce_tx_ql()  argument
544 	writel(ql_value, tqp_vector->mask_addr + HNS3_VECTOR_TX_QL_OFFSET);  in hns3_set_vector_coalesce_tx_ql()
547 void hns3_set_vector_coalesce_rx_ql(struct hns3_enet_tqp_vector *tqp_vector,  in hns3_set_vector_coalesce_rx_ql()  argument
550 	writel(ql_value, tqp_vector->mask_addr + HNS3_VECTOR_RX_QL_OFFSET);  in hns3_set_vector_coalesce_rx_ql()
553 static void hns3_vector_coalesce_init(struct hns3_enet_tqp_vector *tqp_vector,  in hns3_vector_coalesce_init()  argument
557 	struct hns3_enet_coalesce *tx_coal = &tqp_vector->tx_group.coal;  in hns3_vector_coalesce_init()
558 	struct hns3_enet_coalesce *rx_coal = &tqp_vector->rx_group.coal;  in hns3_vector_coalesce_init()
590 hns3_vector_coalesce_init_hw(struct hns3_enet_tqp_vector *tqp_vector,  in hns3_vector_coalesce_init_hw()  argument
593 	struct hns3_enet_coalesce *tx_coal = &tqp_vector->tx_group.coal;  in hns3_vector_coalesce_init_hw()
594 	struct hns3_enet_coalesce *rx_coal = &tqp_vector->rx_group.coal;  in hns3_vector_coalesce_init_hw()
597 	hns3_set_vector_coalesce_tx_gl(tqp_vector, tx_coal->int_gl);  in hns3_vector_coalesce_init_hw()
598 	hns3_set_vector_coalesce_rx_gl(tqp_vector, rx_coal->int_gl);  in hns3_vector_coalesce_init_hw()
599 	hns3_set_vector_coalesce_rl(tqp_vector, h->kinfo.int_rl_setting);  in hns3_vector_coalesce_init_hw()
602 		hns3_set_vector_coalesce_tx_ql(tqp_vector, tx_coal->int_ql);  in hns3_vector_coalesce_init_hw()
605 		hns3_set_vector_coalesce_rx_ql(tqp_vector, rx_coal->int_ql);  in hns3_vector_coalesce_init_hw()
688 	struct hns3_enet_tqp_vector *tqp_vector;  in hns3_set_rx_cpu_rmap()  local
698 		tqp_vector = &priv->tqp_vector[i];  in hns3_set_rx_cpu_rmap()
700 				       tqp_vector->vector_irq);  in hns3_set_rx_cpu_rmap()
725 		hns3_vector_enable(&priv->tqp_vector[i]);  in hns3_nic_net_up()
739 			hns3_vector_disable(&priv->tqp_vector[j]);  in hns3_nic_net_up()
750 		struct hns3_enet_tqp_vector *tqp_vector = &priv->tqp_vector[i];  in hns3_config_xps()  local
751 		struct hns3_enet_ring *ring = tqp_vector->tx_group.ring;  in hns3_config_xps()
757 						  &tqp_vector->affinity_mask,  in hns3_config_xps()
832 		hns3_vector_disable(&priv->tqp_vector[i]);  in hns3_nic_net_down()
2804 	struct napi_struct *napi = &tx_ring->tqp_vector->napi;  in hns3_dump_queue_stats()
2835 		    readl(tx_ring->tqp_vector->mask_addr));  in hns3_dump_queue_reg()
3643 	ring->tqp_vector->tx_group.total_bytes += bytes;  in hns3_clean_tx_ring()
3644 	ring->tqp_vector->tx_group.total_packets += pkts;  in hns3_clean_tx_ring()
3963 		napi_gro_flush(&ring->tqp_vector->napi, false);  in hns3_rx_skb()
3965 	napi_gro_receive(&ring->tqp_vector->napi, skb);  in hns3_rx_skb()
4039 	ring->skb = napi_alloc_skb(&ring->tqp_vector->napi, HNS3_RX_HEAD_SIZE);  in hns3_alloc_skb()
4104 			new_skb = napi_alloc_skb(&ring->tqp_vector->napi, 0);  in hns3_add_frag()
4318 	ring->tqp_vector->rx_group.total_bytes += len;  in hns3_handle_bdinfo()
4442 static void hns3_update_rx_int_coalesce(struct hns3_enet_tqp_vector *tqp_vector)  in hns3_update_rx_int_coalesce()  argument
4444 	struct hns3_enet_ring_group *rx_group = &tqp_vector->rx_group;  in hns3_update_rx_int_coalesce()
4450 	dim_update_sample(tqp_vector->event_cnt, rx_group->total_packets,  in hns3_update_rx_int_coalesce()
4455 static void hns3_update_tx_int_coalesce(struct hns3_enet_tqp_vector *tqp_vector)  in hns3_update_tx_int_coalesce()  argument
4457 	struct hns3_enet_ring_group *tx_group = &tqp_vector->tx_group;  in hns3_update_tx_int_coalesce()
4463 	dim_update_sample(tqp_vector->event_cnt, tx_group->total_packets,  in hns3_update_tx_int_coalesce()
4474 	struct hns3_enet_tqp_vector *tqp_vector =  in hns3_nic_common_poll()  local
4487 	hns3_for_each_ring(ring, tqp_vector->tx_group)  in hns3_nic_common_poll()
4491 	if (tqp_vector->num_tqps > 1)  in hns3_nic_common_poll()
4492 		rx_budget = max(budget / tqp_vector->num_tqps, 1);  in hns3_nic_common_poll()
4494 	hns3_for_each_ring(ring, tqp_vector->rx_group) {  in hns3_nic_common_poll()
4503 	tqp_vector->rx_group.total_packets += rx_pkt_total;  in hns3_nic_common_poll()
4510 		hns3_update_rx_int_coalesce(tqp_vector);  in hns3_nic_common_poll()
4511 		hns3_update_tx_int_coalesce(tqp_vector);  in hns3_nic_common_poll()
4513 		hns3_mask_vector_irq(tqp_vector, 1);  in hns3_nic_common_poll()
4519 static int hns3_create_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,  in hns3_create_ring_chain()  argument
4526 	struct pci_dev *pdev = tqp_vector->handle->pdev;  in hns3_create_ring_chain()
4530 	ring = is_tx ? tqp_vector->tx_group.ring : tqp_vector->rx_group.ring;  in hns3_create_ring_chain()
4561 hns3_get_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector)  in hns3_get_vector_ring_chain()  argument
4563 	struct pci_dev *pdev = tqp_vector->handle->pdev;  in hns3_get_vector_ring_chain()
4567 	if (hns3_create_ring_chain(tqp_vector, &cur_chain, true))  in hns3_get_vector_ring_chain()
4570 	if (hns3_create_ring_chain(tqp_vector, &cur_chain, false))  in hns3_get_vector_ring_chain()
4585 static void hns3_free_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,  in hns3_free_vector_ring_chain()  argument
4588 	struct pci_dev *pdev = tqp_vector->handle->pdev;  in hns3_free_vector_ring_chain()
4612 	struct hns3_enet_tqp_vector *tqp_vector;  in hns3_nic_set_cpumask()  local
4620 		tqp_vector = &priv->tqp_vector[vector_i];  in hns3_nic_set_cpumask()
4622 				&tqp_vector->affinity_mask);  in hns3_nic_set_cpumask()
4631 	struct hns3_enet_tqp_vector *tqp_vector = group->ring->tqp_vector;  in hns3_rx_dim_work()  local
4635 	hns3_set_vector_coalesce_rx_gl(group->ring->tqp_vector, cur_moder.usec);  in hns3_rx_dim_work()
4636 	tqp_vector->rx_group.coal.int_gl = cur_moder.usec;  in hns3_rx_dim_work()
4638 	if (cur_moder.pkts < tqp_vector->rx_group.coal.int_ql_max) {  in hns3_rx_dim_work()
4639 		hns3_set_vector_coalesce_rx_ql(tqp_vector, cur_moder.pkts);  in hns3_rx_dim_work()
4640 		tqp_vector->rx_group.coal.int_ql = cur_moder.pkts;  in hns3_rx_dim_work()
4651 	struct hns3_enet_tqp_vector *tqp_vector = group->ring->tqp_vector;  in hns3_tx_dim_work()  local
4655 	hns3_set_vector_coalesce_tx_gl(tqp_vector, cur_moder.usec);  in hns3_tx_dim_work()
4656 	tqp_vector->tx_group.coal.int_gl = cur_moder.usec;  in hns3_tx_dim_work()
4658 	if (cur_moder.pkts < tqp_vector->tx_group.coal.int_ql_max) {  in hns3_tx_dim_work()
4659 		hns3_set_vector_coalesce_tx_ql(tqp_vector, cur_moder.pkts);  in hns3_tx_dim_work()
4660 		tqp_vector->tx_group.coal.int_ql = cur_moder.pkts;  in hns3_tx_dim_work()
4666 static void hns3_nic_init_dim(struct hns3_enet_tqp_vector *tqp_vector)  in hns3_nic_init_dim()  argument
4668 	INIT_WORK(&tqp_vector->rx_group.dim.work, hns3_rx_dim_work);  in hns3_nic_init_dim()
4669 	INIT_WORK(&tqp_vector->tx_group.dim.work, hns3_tx_dim_work);  in hns3_nic_init_dim()
4675 	struct hns3_enet_tqp_vector *tqp_vector;  in hns3_nic_init_vector_data()  local
4682 		tqp_vector = &priv->tqp_vector[i];  in hns3_nic_init_vector_data()
4683 		hns3_vector_coalesce_init_hw(tqp_vector, priv);  in hns3_nic_init_vector_data()
4684 		tqp_vector->num_tqps = 0;  in hns3_nic_init_vector_data()
4685 		hns3_nic_init_dim(tqp_vector);  in hns3_nic_init_vector_data()
4692 		tqp_vector = &priv->tqp_vector[vector_i];  in hns3_nic_init_vector_data()
4694 		hns3_add_ring_to_group(&tqp_vector->tx_group,  in hns3_nic_init_vector_data()
4697 		hns3_add_ring_to_group(&tqp_vector->rx_group,  in hns3_nic_init_vector_data()
4700 		priv->ring[i].tqp_vector = tqp_vector;  in hns3_nic_init_vector_data()
4701 		priv->ring[i + tqp_num].tqp_vector = tqp_vector;  in hns3_nic_init_vector_data()
4702 		tqp_vector->num_tqps++;  in hns3_nic_init_vector_data()
4708 		tqp_vector = &priv->tqp_vector[i];  in hns3_nic_init_vector_data()
4710 		tqp_vector->rx_group.total_bytes = 0;  in hns3_nic_init_vector_data()
4711 		tqp_vector->rx_group.total_packets = 0;  in hns3_nic_init_vector_data()
4712 		tqp_vector->tx_group.total_bytes = 0;  in hns3_nic_init_vector_data()
4713 		tqp_vector->tx_group.total_packets = 0;  in hns3_nic_init_vector_data()
4714 		tqp_vector->handle = h;  in hns3_nic_init_vector_data()
4716 		vector_ring_chain = hns3_get_vector_ring_chain(tqp_vector);  in hns3_nic_init_vector_data()
4723 			tqp_vector->vector_irq, vector_ring_chain);  in hns3_nic_init_vector_data()
4725 		hns3_free_vector_ring_chain(tqp_vector, vector_ring_chain);  in hns3_nic_init_vector_data()
4730 		netif_napi_add(priv->netdev, &tqp_vector->napi,  in hns3_nic_init_vector_data()
4738 		netif_napi_del(&priv->tqp_vector[i].napi);  in hns3_nic_init_vector_data()
4774 	struct hns3_enet_tqp_vector *tqp_vector;  in hns3_nic_alloc_vector_data()  local
4795 	priv->tqp_vector = (struct hns3_enet_tqp_vector *)  in hns3_nic_alloc_vector_data()
4796 		devm_kcalloc(&pdev->dev, vector_num, sizeof(*priv->tqp_vector),  in hns3_nic_alloc_vector_data()
4798 	if (!priv->tqp_vector) {  in hns3_nic_alloc_vector_data()
4804 		tqp_vector = &priv->tqp_vector[i];  in hns3_nic_alloc_vector_data()
4805 		tqp_vector->idx = i;  in hns3_nic_alloc_vector_data()
4806 		tqp_vector->mask_addr = vector[i].io_addr;  in hns3_nic_alloc_vector_data()
4807 		tqp_vector->vector_irq = vector[i].vector;  in hns3_nic_alloc_vector_data()
4808 		hns3_vector_coalesce_init(tqp_vector, priv);  in hns3_nic_alloc_vector_data()
4826 	struct hns3_enet_tqp_vector *tqp_vector;  in hns3_nic_uninit_vector_data()  local
4830 		tqp_vector = &priv->tqp_vector[i];  in hns3_nic_uninit_vector_data()
4832 		if (!tqp_vector->rx_group.ring && !tqp_vector->tx_group.ring)  in hns3_nic_uninit_vector_data()
4839 		vector_ring_chain = hns3_get_vector_ring_chain(tqp_vector);  in hns3_nic_uninit_vector_data()
4844 			tqp_vector->vector_irq, vector_ring_chain);  in hns3_nic_uninit_vector_data()
4846 		hns3_free_vector_ring_chain(tqp_vector, vector_ring_chain);  in hns3_nic_uninit_vector_data()
4848 		hns3_clear_ring_group(&tqp_vector->rx_group);  in hns3_nic_uninit_vector_data()
4849 		hns3_clear_ring_group(&tqp_vector->tx_group);  in hns3_nic_uninit_vector_data()
4850 		netif_napi_del(&priv->tqp_vector[i].napi);  in hns3_nic_uninit_vector_data()
4861 		struct hns3_enet_tqp_vector *tqp_vector;  in hns3_nic_dealloc_vector_data()  local
4863 		tqp_vector = &priv->tqp_vector[i];  in hns3_nic_dealloc_vector_data()
4864 		ret = h->ae_algo->ops->put_vector(h, tqp_vector->vector_irq);  in hns3_nic_dealloc_vector_data()
4869 	devm_kfree(&pdev->dev, priv->tqp_vector);  in hns3_nic_dealloc_vector_data()
5232 			priv->tqp_vector[i].tx_group.dim.mode = mode;  in hns3_set_cq_period_mode()
5237 			priv->tqp_vector[i].rx_group.dim.mode = mode;  in hns3_set_cq_period_mode()
5879 		hns3_vector_disable(&priv->tqp_vector[i]);  in hns3_external_lb_prepare()
5915 		hns3_vector_enable(&priv->tqp_vector[i]);  in hns3_external_lb_restore()