342 	struct am65_cpsw_tx_chn *tx_chn;  in am65_cpsw_nuss_ndo_host_tx_timeout()  local
347 	tx_chn = &common->tx_chns[txqueue];  in am65_cpsw_nuss_ndo_host_tx_timeout()
355 		   k3_cppi_desc_pool_avail(tx_chn->desc_pool));  in am65_cpsw_nuss_ndo_host_tx_timeout()
515 		ret = k3_udma_glue_enable_tx_chn(common->tx_chns[i].tx_chn);  in am65_cpsw_nuss_common_open()
551 		k3_udma_glue_tdown_tx_chn(common->tx_chns[i].tx_chn, false);  in am65_cpsw_nuss_common_stop()
561 		k3_udma_glue_reset_tx_chn(common->tx_chns[i].tx_chn,  in am65_cpsw_nuss_common_stop()
564 		k3_udma_glue_disable_tx_chn(common->tx_chns[i].tx_chn);  in am65_cpsw_nuss_common_stop()
889 static void am65_cpsw_nuss_xmit_free(struct am65_cpsw_tx_chn *tx_chn,  in am65_cpsw_nuss_xmit_free()  argument
900 	k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &buf_dma);  in am65_cpsw_nuss_xmit_free()
902 	dma_unmap_single(tx_chn->dma_dev, buf_dma, buf_dma_len, DMA_TO_DEVICE);  in am65_cpsw_nuss_xmit_free()
905 	k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &next_desc_dma);  in am65_cpsw_nuss_xmit_free()
907 		next_desc = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool,  in am65_cpsw_nuss_xmit_free()
910 		k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &buf_dma);  in am65_cpsw_nuss_xmit_free()
912 		dma_unmap_page(tx_chn->dma_dev, buf_dma, buf_dma_len,  in am65_cpsw_nuss_xmit_free()
916 		k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &next_desc_dma);  in am65_cpsw_nuss_xmit_free()
918 		k3_cppi_desc_pool_free(tx_chn->desc_pool, next_desc);  in am65_cpsw_nuss_xmit_free()
921 	k3_cppi_desc_pool_free(tx_chn->desc_pool, first_desc);  in am65_cpsw_nuss_xmit_free()
926 	struct am65_cpsw_tx_chn *tx_chn = data;  in am65_cpsw_nuss_tx_cleanup()  local
931 	desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool, desc_dma);  in am65_cpsw_nuss_tx_cleanup()
934 	am65_cpsw_nuss_xmit_free(tx_chn, desc_tx);  in am65_cpsw_nuss_tx_cleanup()
940 am65_cpsw_nuss_tx_compl_packet(struct am65_cpsw_tx_chn *tx_chn,  in am65_cpsw_nuss_tx_compl_packet()  argument
950 	desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool,  in am65_cpsw_nuss_tx_compl_packet()
954 	am65_cpsw_nuss_xmit_free(tx_chn, desc_tx);  in am65_cpsw_nuss_tx_compl_packet()
958 	am65_cpts_tx_timestamp(tx_chn->common->cpts, skb);  in am65_cpsw_nuss_tx_compl_packet()
970 static void am65_cpsw_nuss_tx_wake(struct am65_cpsw_tx_chn *tx_chn, struct net_device *ndev,  in am65_cpsw_nuss_tx_wake()  argument
980 		    (k3_cppi_desc_pool_avail(tx_chn->desc_pool) >= MAX_SKB_FRAGS))  in am65_cpsw_nuss_tx_wake()
991 	struct am65_cpsw_tx_chn *tx_chn;  in am65_cpsw_nuss_tx_compl_packets()  local
999 	tx_chn = &common->tx_chns[chn];  in am65_cpsw_nuss_tx_compl_packets()
1002 		spin_lock(&tx_chn->lock);  in am65_cpsw_nuss_tx_compl_packets()
1003 		res = k3_udma_glue_pop_tx_chn(tx_chn->tx_chn, &desc_dma);  in am65_cpsw_nuss_tx_compl_packets()
1004 		spin_unlock(&tx_chn->lock);  in am65_cpsw_nuss_tx_compl_packets()
1014 		skb = am65_cpsw_nuss_tx_compl_packet(tx_chn, desc_dma);  in am65_cpsw_nuss_tx_compl_packets()
1024 		am65_cpsw_nuss_tx_wake(tx_chn, ndev, netif_txq);  in am65_cpsw_nuss_tx_compl_packets()
1036 	struct am65_cpsw_tx_chn *tx_chn;  in am65_cpsw_nuss_tx_compl_packets_2g()  local
1044 	tx_chn = &common->tx_chns[chn];  in am65_cpsw_nuss_tx_compl_packets_2g()
1047 		res = k3_udma_glue_pop_tx_chn(tx_chn->tx_chn, &desc_dma);  in am65_cpsw_nuss_tx_compl_packets_2g()
1057 		skb = am65_cpsw_nuss_tx_compl_packet(tx_chn, desc_dma);  in am65_cpsw_nuss_tx_compl_packets_2g()
1072 	am65_cpsw_nuss_tx_wake(tx_chn, ndev, netif_txq);  in am65_cpsw_nuss_tx_compl_packets_2g()
1081 	struct am65_cpsw_tx_chn *tx_chn = am65_cpsw_napi_to_tx_chn(napi_tx);  in am65_cpsw_nuss_tx_poll()  local
1084 	if (AM65_CPSW_IS_CPSW2G(tx_chn->common))  in am65_cpsw_nuss_tx_poll()
1085 		num_tx = am65_cpsw_nuss_tx_compl_packets_2g(tx_chn->common, tx_chn->id, budget);  in am65_cpsw_nuss_tx_poll()
1087 		num_tx = am65_cpsw_nuss_tx_compl_packets(tx_chn->common, tx_chn->id, budget);  in am65_cpsw_nuss_tx_poll()
1093 		enable_irq(tx_chn->irq);  in am65_cpsw_nuss_tx_poll()
1111 	struct am65_cpsw_tx_chn *tx_chn = dev_id;  in am65_cpsw_nuss_tx_irq()  local
1114 	napi_schedule(&tx_chn->napi_tx);  in am65_cpsw_nuss_tx_irq()
1126 	struct am65_cpsw_tx_chn *tx_chn;  in am65_cpsw_nuss_ndo_slave_xmit()  local
1144 	tx_chn = &common->tx_chns[q_idx];  in am65_cpsw_nuss_ndo_slave_xmit()
1148 	buf_dma = dma_map_single(tx_chn->dma_dev, skb->data, pkt_len,  in am65_cpsw_nuss_ndo_slave_xmit()
1150 	if (unlikely(dma_mapping_error(tx_chn->dma_dev, buf_dma))) {  in am65_cpsw_nuss_ndo_slave_xmit()
1156 	first_desc = k3_cppi_desc_pool_alloc(tx_chn->desc_pool);  in am65_cpsw_nuss_ndo_slave_xmit()
1159 		dma_unmap_single(tx_chn->dma_dev, buf_dma, pkt_len,  in am65_cpsw_nuss_ndo_slave_xmit()
1170 	k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &buf_dma);  in am65_cpsw_nuss_ndo_slave_xmit()
1200 		next_desc = k3_cppi_desc_pool_alloc(tx_chn->desc_pool);  in am65_cpsw_nuss_ndo_slave_xmit()
1206 		buf_dma = skb_frag_dma_map(tx_chn->dma_dev, frag, 0, frag_size,  in am65_cpsw_nuss_ndo_slave_xmit()
1208 		if (unlikely(dma_mapping_error(tx_chn->dma_dev, buf_dma))) {  in am65_cpsw_nuss_ndo_slave_xmit()
1210 			k3_cppi_desc_pool_free(tx_chn->desc_pool, next_desc);  in am65_cpsw_nuss_ndo_slave_xmit()
1216 		k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &buf_dma);  in am65_cpsw_nuss_ndo_slave_xmit()
1220 		desc_dma = k3_cppi_desc_pool_virt2dma(tx_chn->desc_pool,  in am65_cpsw_nuss_ndo_slave_xmit()
1222 		k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &desc_dma);  in am65_cpsw_nuss_ndo_slave_xmit()
1237 	desc_dma = k3_cppi_desc_pool_virt2dma(tx_chn->desc_pool, first_desc);  in am65_cpsw_nuss_ndo_slave_xmit()
1239 		ret = k3_udma_glue_push_tx_chn(tx_chn->tx_chn, first_desc, desc_dma);  in am65_cpsw_nuss_ndo_slave_xmit()
1241 		spin_lock_bh(&tx_chn->lock);  in am65_cpsw_nuss_ndo_slave_xmit()
1242 		ret = k3_udma_glue_push_tx_chn(tx_chn->tx_chn, first_desc, desc_dma);  in am65_cpsw_nuss_ndo_slave_xmit()
1243 		spin_unlock_bh(&tx_chn->lock);  in am65_cpsw_nuss_ndo_slave_xmit()
1253 	if (k3_cppi_desc_pool_avail(tx_chn->desc_pool) < MAX_SKB_FRAGS) {  in am65_cpsw_nuss_ndo_slave_xmit()
1260 		if (k3_cppi_desc_pool_avail(tx_chn->desc_pool) >=  in am65_cpsw_nuss_ndo_slave_xmit()
1270 	am65_cpsw_nuss_xmit_free(tx_chn, first_desc);  in am65_cpsw_nuss_ndo_slave_xmit()
1277 	am65_cpsw_nuss_xmit_free(tx_chn, first_desc);  in am65_cpsw_nuss_ndo_slave_xmit()
1517 		struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i];  in am65_cpsw_nuss_free_tx_chns()  local
1519 		if (!IS_ERR_OR_NULL(tx_chn->desc_pool))  in am65_cpsw_nuss_free_tx_chns()
1520 			k3_cppi_desc_pool_destroy(tx_chn->desc_pool);  in am65_cpsw_nuss_free_tx_chns()
1522 		if (!IS_ERR_OR_NULL(tx_chn->tx_chn))  in am65_cpsw_nuss_free_tx_chns()
1523 			k3_udma_glue_release_tx_chn(tx_chn->tx_chn);  in am65_cpsw_nuss_free_tx_chns()
1525 		memset(tx_chn, 0, sizeof(*tx_chn));  in am65_cpsw_nuss_free_tx_chns()
1537 		struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i];  in am65_cpsw_nuss_remove_tx_chns()  local
1539 		if (tx_chn->irq)  in am65_cpsw_nuss_remove_tx_chns()
1540 			devm_free_irq(dev, tx_chn->irq, tx_chn);  in am65_cpsw_nuss_remove_tx_chns()
1542 		netif_napi_del(&tx_chn->napi_tx);  in am65_cpsw_nuss_remove_tx_chns()
1544 		if (!IS_ERR_OR_NULL(tx_chn->desc_pool))  in am65_cpsw_nuss_remove_tx_chns()
1545 			k3_cppi_desc_pool_destroy(tx_chn->desc_pool);  in am65_cpsw_nuss_remove_tx_chns()
1547 		if (!IS_ERR_OR_NULL(tx_chn->tx_chn))  in am65_cpsw_nuss_remove_tx_chns()
1548 			k3_udma_glue_release_tx_chn(tx_chn->tx_chn);  in am65_cpsw_nuss_remove_tx_chns()
1550 		memset(tx_chn, 0, sizeof(*tx_chn));  in am65_cpsw_nuss_remove_tx_chns()
1577 		struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i];  in am65_cpsw_nuss_init_tx_chns()  local
1579 		snprintf(tx_chn->tx_chn_name,  in am65_cpsw_nuss_init_tx_chns()
1580 			 sizeof(tx_chn->tx_chn_name), "tx%d", i);  in am65_cpsw_nuss_init_tx_chns()
1582 		spin_lock_init(&tx_chn->lock);  in am65_cpsw_nuss_init_tx_chns()
1583 		tx_chn->common = common;  in am65_cpsw_nuss_init_tx_chns()
1584 		tx_chn->id = i;  in am65_cpsw_nuss_init_tx_chns()
1585 		tx_chn->descs_num = max_desc_num;  in am65_cpsw_nuss_init_tx_chns()
1587 		tx_chn->tx_chn =  in am65_cpsw_nuss_init_tx_chns()
1589 						    tx_chn->tx_chn_name,  in am65_cpsw_nuss_init_tx_chns()
1591 		if (IS_ERR(tx_chn->tx_chn)) {  in am65_cpsw_nuss_init_tx_chns()
1592 			ret = dev_err_probe(dev, PTR_ERR(tx_chn->tx_chn),  in am65_cpsw_nuss_init_tx_chns()
1596 		tx_chn->dma_dev = k3_udma_glue_tx_get_dma_device(tx_chn->tx_chn);  in am65_cpsw_nuss_init_tx_chns()
1598 		tx_chn->desc_pool = k3_cppi_desc_pool_create_name(tx_chn->dma_dev,  in am65_cpsw_nuss_init_tx_chns()
1599 								  tx_chn->descs_num,  in am65_cpsw_nuss_init_tx_chns()
1601 								  tx_chn->tx_chn_name);  in am65_cpsw_nuss_init_tx_chns()
1602 		if (IS_ERR(tx_chn->desc_pool)) {  in am65_cpsw_nuss_init_tx_chns()
1603 			ret = PTR_ERR(tx_chn->desc_pool);  in am65_cpsw_nuss_init_tx_chns()
1608 		tx_chn->irq = k3_udma_glue_tx_get_irq(tx_chn->tx_chn);  in am65_cpsw_nuss_init_tx_chns()
1609 		if (tx_chn->irq <= 0) {  in am65_cpsw_nuss_init_tx_chns()
1611 				tx_chn->irq);  in am65_cpsw_nuss_init_tx_chns()
1615 		snprintf(tx_chn->tx_chn_name,  in am65_cpsw_nuss_init_tx_chns()
1616 			 sizeof(tx_chn->tx_chn_name), "%s-tx%d",  in am65_cpsw_nuss_init_tx_chns()
1617 			 dev_name(dev), tx_chn->id);  in am65_cpsw_nuss_init_tx_chns()
2029 		struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i];  in am65_cpsw_nuss_ndev_add_tx_napi()  local
2031 		netif_tx_napi_add(common->dma_ndev, &tx_chn->napi_tx,  in am65_cpsw_nuss_ndev_add_tx_napi()
2034 		ret = devm_request_irq(dev, tx_chn->irq,  in am65_cpsw_nuss_ndev_add_tx_napi()
2037 				       tx_chn->tx_chn_name, tx_chn);  in am65_cpsw_nuss_ndev_add_tx_napi()
2040 				tx_chn->id, tx_chn->irq, ret);  in am65_cpsw_nuss_ndev_add_tx_napi()