279 	struct am65_cpsw_tx_chn *tx_chn;  in am65_cpsw_nuss_ndo_host_tx_timeout()  local
284 	tx_chn = &common->tx_chns[txqueue];  in am65_cpsw_nuss_ndo_host_tx_timeout()
292 		   k3_cppi_desc_pool_avail(tx_chn->desc_pool));  in am65_cpsw_nuss_ndo_host_tx_timeout()
451 		ret = k3_udma_glue_enable_tx_chn(common->tx_chns[i].tx_chn);  in am65_cpsw_nuss_common_open()
487 		k3_udma_glue_tdown_tx_chn(common->tx_chns[i].tx_chn, false);  in am65_cpsw_nuss_common_stop()
497 		k3_udma_glue_reset_tx_chn(common->tx_chns[i].tx_chn,  in am65_cpsw_nuss_common_stop()
500 		k3_udma_glue_disable_tx_chn(common->tx_chns[i].tx_chn);  in am65_cpsw_nuss_common_stop()
803 static void am65_cpsw_nuss_xmit_free(struct am65_cpsw_tx_chn *tx_chn,  in am65_cpsw_nuss_xmit_free()  argument
814 	k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &buf_dma);  in am65_cpsw_nuss_xmit_free()
816 	dma_unmap_single(tx_chn->dma_dev, buf_dma, buf_dma_len, DMA_TO_DEVICE);  in am65_cpsw_nuss_xmit_free()
819 	k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &next_desc_dma);  in am65_cpsw_nuss_xmit_free()
821 		next_desc = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool,  in am65_cpsw_nuss_xmit_free()
824 		k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &buf_dma);  in am65_cpsw_nuss_xmit_free()
826 		dma_unmap_page(tx_chn->dma_dev, buf_dma, buf_dma_len,  in am65_cpsw_nuss_xmit_free()
830 		k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &next_desc_dma);  in am65_cpsw_nuss_xmit_free()
832 		k3_cppi_desc_pool_free(tx_chn->desc_pool, next_desc);  in am65_cpsw_nuss_xmit_free()
835 	k3_cppi_desc_pool_free(tx_chn->desc_pool, first_desc);  in am65_cpsw_nuss_xmit_free()
840 	struct am65_cpsw_tx_chn *tx_chn = data;  in am65_cpsw_nuss_tx_cleanup()  local
845 	desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool, desc_dma);  in am65_cpsw_nuss_tx_cleanup()
848 	am65_cpsw_nuss_xmit_free(tx_chn, desc_tx);  in am65_cpsw_nuss_tx_cleanup()
854 am65_cpsw_nuss_tx_compl_packet(struct am65_cpsw_tx_chn *tx_chn,  in am65_cpsw_nuss_tx_compl_packet()  argument
864 	desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool,  in am65_cpsw_nuss_tx_compl_packet()
868 	am65_cpsw_nuss_xmit_free(tx_chn, desc_tx);  in am65_cpsw_nuss_tx_compl_packet()
872 	am65_cpts_tx_timestamp(tx_chn->common->cpts, skb);  in am65_cpsw_nuss_tx_compl_packet()
884 static void am65_cpsw_nuss_tx_wake(struct am65_cpsw_tx_chn *tx_chn, struct net_device *ndev,  in am65_cpsw_nuss_tx_wake()  argument
894 		    (k3_cppi_desc_pool_avail(tx_chn->desc_pool) >= MAX_SKB_FRAGS))  in am65_cpsw_nuss_tx_wake()
905 	struct am65_cpsw_tx_chn *tx_chn;  in am65_cpsw_nuss_tx_compl_packets()  local
913 	tx_chn = &common->tx_chns[chn];  in am65_cpsw_nuss_tx_compl_packets()
916 		spin_lock(&tx_chn->lock);  in am65_cpsw_nuss_tx_compl_packets()
917 		res = k3_udma_glue_pop_tx_chn(tx_chn->tx_chn, &desc_dma);  in am65_cpsw_nuss_tx_compl_packets()
918 		spin_unlock(&tx_chn->lock);  in am65_cpsw_nuss_tx_compl_packets()
928 		skb = am65_cpsw_nuss_tx_compl_packet(tx_chn, desc_dma);  in am65_cpsw_nuss_tx_compl_packets()
938 		am65_cpsw_nuss_tx_wake(tx_chn, ndev, netif_txq);  in am65_cpsw_nuss_tx_compl_packets()
950 	struct am65_cpsw_tx_chn *tx_chn;  in am65_cpsw_nuss_tx_compl_packets_2g()  local
958 	tx_chn = &common->tx_chns[chn];  in am65_cpsw_nuss_tx_compl_packets_2g()
961 		res = k3_udma_glue_pop_tx_chn(tx_chn->tx_chn, &desc_dma);  in am65_cpsw_nuss_tx_compl_packets_2g()
971 		skb = am65_cpsw_nuss_tx_compl_packet(tx_chn, desc_dma);  in am65_cpsw_nuss_tx_compl_packets_2g()
986 	am65_cpsw_nuss_tx_wake(tx_chn, ndev, netif_txq);  in am65_cpsw_nuss_tx_compl_packets_2g()
995 	struct am65_cpsw_tx_chn *tx_chn = am65_cpsw_napi_to_tx_chn(napi_tx);  in am65_cpsw_nuss_tx_poll()  local
998 	if (AM65_CPSW_IS_CPSW2G(tx_chn->common))  in am65_cpsw_nuss_tx_poll()
999 		num_tx = am65_cpsw_nuss_tx_compl_packets_2g(tx_chn->common, tx_chn->id, budget);  in am65_cpsw_nuss_tx_poll()
1001 		num_tx = am65_cpsw_nuss_tx_compl_packets(tx_chn->common, tx_chn->id, budget);  in am65_cpsw_nuss_tx_poll()
1007 		enable_irq(tx_chn->irq);  in am65_cpsw_nuss_tx_poll()
1025 	struct am65_cpsw_tx_chn *tx_chn = dev_id;  in am65_cpsw_nuss_tx_irq()  local
1028 	napi_schedule(&tx_chn->napi_tx);  in am65_cpsw_nuss_tx_irq()
1040 	struct am65_cpsw_tx_chn *tx_chn;  in am65_cpsw_nuss_ndo_slave_xmit()  local
1058 	tx_chn = &common->tx_chns[q_idx];  in am65_cpsw_nuss_ndo_slave_xmit()
1062 	buf_dma = dma_map_single(tx_chn->dma_dev, skb->data, pkt_len,  in am65_cpsw_nuss_ndo_slave_xmit()
1064 	if (unlikely(dma_mapping_error(tx_chn->dma_dev, buf_dma))) {  in am65_cpsw_nuss_ndo_slave_xmit()
1070 	first_desc = k3_cppi_desc_pool_alloc(tx_chn->desc_pool);  in am65_cpsw_nuss_ndo_slave_xmit()
1073 		dma_unmap_single(tx_chn->dma_dev, buf_dma, pkt_len,  in am65_cpsw_nuss_ndo_slave_xmit()
1084 	k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &buf_dma);  in am65_cpsw_nuss_ndo_slave_xmit()
1114 		next_desc = k3_cppi_desc_pool_alloc(tx_chn->desc_pool);  in am65_cpsw_nuss_ndo_slave_xmit()
1120 		buf_dma = skb_frag_dma_map(tx_chn->dma_dev, frag, 0, frag_size,  in am65_cpsw_nuss_ndo_slave_xmit()
1122 		if (unlikely(dma_mapping_error(tx_chn->dma_dev, buf_dma))) {  in am65_cpsw_nuss_ndo_slave_xmit()
1124 			k3_cppi_desc_pool_free(tx_chn->desc_pool, next_desc);  in am65_cpsw_nuss_ndo_slave_xmit()
1130 		k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &buf_dma);  in am65_cpsw_nuss_ndo_slave_xmit()
1134 		desc_dma = k3_cppi_desc_pool_virt2dma(tx_chn->desc_pool,  in am65_cpsw_nuss_ndo_slave_xmit()
1136 		k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &desc_dma);  in am65_cpsw_nuss_ndo_slave_xmit()
1151 	desc_dma = k3_cppi_desc_pool_virt2dma(tx_chn->desc_pool, first_desc);  in am65_cpsw_nuss_ndo_slave_xmit()
1153 		ret = k3_udma_glue_push_tx_chn(tx_chn->tx_chn, first_desc, desc_dma);  in am65_cpsw_nuss_ndo_slave_xmit()
1155 		spin_lock_bh(&tx_chn->lock);  in am65_cpsw_nuss_ndo_slave_xmit()
1156 		ret = k3_udma_glue_push_tx_chn(tx_chn->tx_chn, first_desc, desc_dma);  in am65_cpsw_nuss_ndo_slave_xmit()
1157 		spin_unlock_bh(&tx_chn->lock);  in am65_cpsw_nuss_ndo_slave_xmit()
1167 	if (k3_cppi_desc_pool_avail(tx_chn->desc_pool) < MAX_SKB_FRAGS) {  in am65_cpsw_nuss_ndo_slave_xmit()
1174 		if (k3_cppi_desc_pool_avail(tx_chn->desc_pool) >=  in am65_cpsw_nuss_ndo_slave_xmit()
1184 	am65_cpsw_nuss_xmit_free(tx_chn, first_desc);  in am65_cpsw_nuss_ndo_slave_xmit()
1191 	am65_cpsw_nuss_xmit_free(tx_chn, first_desc);  in am65_cpsw_nuss_ndo_slave_xmit()
1508 		struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i];  in am65_cpsw_nuss_free_tx_chns()  local
1510 		if (!IS_ERR_OR_NULL(tx_chn->desc_pool))  in am65_cpsw_nuss_free_tx_chns()
1511 			k3_cppi_desc_pool_destroy(tx_chn->desc_pool);  in am65_cpsw_nuss_free_tx_chns()
1513 		if (!IS_ERR_OR_NULL(tx_chn->tx_chn))  in am65_cpsw_nuss_free_tx_chns()
1514 			k3_udma_glue_release_tx_chn(tx_chn->tx_chn);  in am65_cpsw_nuss_free_tx_chns()
1516 		memset(tx_chn, 0, sizeof(*tx_chn));  in am65_cpsw_nuss_free_tx_chns()
1528 		struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i];  in am65_cpsw_nuss_remove_tx_chns()  local
1530 		if (tx_chn->irq)  in am65_cpsw_nuss_remove_tx_chns()
1531 			devm_free_irq(dev, tx_chn->irq, tx_chn);  in am65_cpsw_nuss_remove_tx_chns()
1533 		netif_napi_del(&tx_chn->napi_tx);  in am65_cpsw_nuss_remove_tx_chns()
1535 		if (!IS_ERR_OR_NULL(tx_chn->desc_pool))  in am65_cpsw_nuss_remove_tx_chns()
1536 			k3_cppi_desc_pool_destroy(tx_chn->desc_pool);  in am65_cpsw_nuss_remove_tx_chns()
1538 		if (!IS_ERR_OR_NULL(tx_chn->tx_chn))  in am65_cpsw_nuss_remove_tx_chns()
1539 			k3_udma_glue_release_tx_chn(tx_chn->tx_chn);  in am65_cpsw_nuss_remove_tx_chns()
1541 		memset(tx_chn, 0, sizeof(*tx_chn));  in am65_cpsw_nuss_remove_tx_chns()
1568 		struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i];  in am65_cpsw_nuss_init_tx_chns()  local
1570 		snprintf(tx_chn->tx_chn_name,  in am65_cpsw_nuss_init_tx_chns()
1571 			 sizeof(tx_chn->tx_chn_name), "tx%d", i);  in am65_cpsw_nuss_init_tx_chns()
1573 		spin_lock_init(&tx_chn->lock);  in am65_cpsw_nuss_init_tx_chns()
1574 		tx_chn->common = common;  in am65_cpsw_nuss_init_tx_chns()
1575 		tx_chn->id = i;  in am65_cpsw_nuss_init_tx_chns()
1576 		tx_chn->descs_num = max_desc_num;  in am65_cpsw_nuss_init_tx_chns()
1578 		tx_chn->tx_chn =  in am65_cpsw_nuss_init_tx_chns()
1580 						    tx_chn->tx_chn_name,  in am65_cpsw_nuss_init_tx_chns()
1582 		if (IS_ERR(tx_chn->tx_chn)) {  in am65_cpsw_nuss_init_tx_chns()
1583 			ret = dev_err_probe(dev, PTR_ERR(tx_chn->tx_chn),  in am65_cpsw_nuss_init_tx_chns()
1587 		tx_chn->dma_dev = k3_udma_glue_tx_get_dma_device(tx_chn->tx_chn);  in am65_cpsw_nuss_init_tx_chns()
1589 		tx_chn->desc_pool = k3_cppi_desc_pool_create_name(tx_chn->dma_dev,  in am65_cpsw_nuss_init_tx_chns()
1590 								  tx_chn->descs_num,  in am65_cpsw_nuss_init_tx_chns()
1592 								  tx_chn->tx_chn_name);  in am65_cpsw_nuss_init_tx_chns()
1593 		if (IS_ERR(tx_chn->desc_pool)) {  in am65_cpsw_nuss_init_tx_chns()
1594 			ret = PTR_ERR(tx_chn->desc_pool);  in am65_cpsw_nuss_init_tx_chns()
1599 		tx_chn->irq = k3_udma_glue_tx_get_irq(tx_chn->tx_chn);  in am65_cpsw_nuss_init_tx_chns()
1600 		if (tx_chn->irq <= 0) {  in am65_cpsw_nuss_init_tx_chns()
1602 				tx_chn->irq);  in am65_cpsw_nuss_init_tx_chns()
1606 		snprintf(tx_chn->tx_chn_name,  in am65_cpsw_nuss_init_tx_chns()
1607 			 sizeof(tx_chn->tx_chn_name), "%s-tx%d",  in am65_cpsw_nuss_init_tx_chns()
1608 			 dev_name(dev), tx_chn->id);  in am65_cpsw_nuss_init_tx_chns()
2058 		struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i];  in am65_cpsw_nuss_ndev_add_tx_napi()  local
2060 		netif_napi_add_tx(common->dma_ndev, &tx_chn->napi_tx,  in am65_cpsw_nuss_ndev_add_tx_napi()
2063 		ret = devm_request_irq(dev, tx_chn->irq,  in am65_cpsw_nuss_ndev_add_tx_napi()
2066 				       tx_chn->tx_chn_name, tx_chn);  in am65_cpsw_nuss_ndev_add_tx_napi()
2069 				tx_chn->id, tx_chn->irq, ret);  in am65_cpsw_nuss_ndev_add_tx_napi()