1 // SPDX-License-Identifier: GPL-2.0
4  * Copyright (C) 2020 Texas Instruments Incorporated - http://www.ti.com/
28 #include <linux/dma/ti-cppi5.h>
29 #include <linux/dma/k3-udma-glue.h>
33 #include "am65-cpsw-nuss.h"
34 #include "k3-cppi-desc-pool.h"
35 #include "am65-cpts.h"
107 /* The PTP event messages - Sync, Delay_Req, Pdelay_Req, and Pdelay_Resp. */
136 	writel(mac_hi, slave->port_base + AM65_CPSW_PORTN_REG_SA_H);  in am65_cpsw_port_set_sl_mac()
137 	writel(mac_lo, slave->port_base + AM65_CPSW_PORTN_REG_SA_L);  in am65_cpsw_port_set_sl_mac()
142 	cpsw_sl_reset(port->slave.mac_sl, 100);  in am65_cpsw_sl_ctl_reset()
145 	       port->port_base + AM65_CPSW_PORT_REG_RX_MAXLEN);  in am65_cpsw_sl_ctl_reset()
150 	common->nuss_ver = readl(common->ss_base);  in am65_cpsw_nuss_get_ver()
151 	common->cpsw_ver = readl(common->cpsw_base);  in am65_cpsw_nuss_get_ver()
152 	dev_info(common->dev,  in am65_cpsw_nuss_get_ver()
153 		 "initializing am65 cpsw nuss version 0x%08X, cpsw version 0x%08X Ports: %u quirks:%08x\n",  in am65_cpsw_nuss_get_ver()
154 		common->nuss_ver,  in am65_cpsw_nuss_get_ver()
155 		common->cpsw_ver,  in am65_cpsw_nuss_get_ver()
156 		common->port_num + 1,  in am65_cpsw_nuss_get_ver()
157 		common->pdata.quirks);  in am65_cpsw_nuss_get_ver()
164 	struct phy_device *phy = port->slave.phy;  in am65_cpsw_nuss_adjust_link()
170 	if (phy->link) {  in am65_cpsw_nuss_adjust_link()
173 		if (phy->speed == 1000)  in am65_cpsw_nuss_adjust_link()
175 		if (phy->speed == 10 && phy_interface_is_rgmii(phy))  in am65_cpsw_nuss_adjust_link()
178 		if (phy->speed == 100 && phy->interface == PHY_INTERFACE_MODE_RMII)  in am65_cpsw_nuss_adjust_link()
180 		if (phy->duplex)  in am65_cpsw_nuss_adjust_link()
186 		if (port->slave.rx_pause)  in am65_cpsw_nuss_adjust_link()
189 		if (port->slave.tx_pause)  in am65_cpsw_nuss_adjust_link()
192 		cpsw_sl_ctl_set(port->slave.mac_sl, mac_control);  in am65_cpsw_nuss_adjust_link()
195 		cpsw_ale_control_set(common->ale, port->port_id,  in am65_cpsw_nuss_adjust_link()
198 		am65_cpsw_qos_link_up(ndev, phy->speed);  in am65_cpsw_nuss_adjust_link()
204 		cpsw_ale_control_set(common->ale, port->port_id,  in am65_cpsw_nuss_adjust_link()
207 		cpsw_sl_ctl_set(port->slave.mac_sl, CPSW_SL_CTL_CMD_IDLE);  in am65_cpsw_nuss_adjust_link()
209 		tmo = cpsw_sl_wait_for_idle(port->slave.mac_sl, 100);  in am65_cpsw_nuss_adjust_link()
210 		dev_dbg(common->dev, "donw msc_sl %08x tmo %d\n",  in am65_cpsw_nuss_adjust_link()
211 			cpsw_sl_reg_read(port->slave.mac_sl, CPSW_SL_MACSTATUS),  in am65_cpsw_nuss_adjust_link()
214 		cpsw_sl_ctl_reset(port->slave.mac_sl);  in am65_cpsw_nuss_adjust_link()
234 	ret = pm_runtime_get_sync(common->dev);  in am65_cpsw_nuss_ndo_slave_add_vid()
236 		pm_runtime_put_noidle(common->dev);  in am65_cpsw_nuss_ndo_slave_add_vid()
240 	port_mask = BIT(port->port_id) | ALE_PORT_HOST;  in am65_cpsw_nuss_ndo_slave_add_vid()
243 	dev_info(common->dev, "Adding vlan %d to vlan filter\n", vid);  in am65_cpsw_nuss_ndo_slave_add_vid()
244 	ret = cpsw_ale_add_vlan(common->ale, vid, port_mask,  in am65_cpsw_nuss_ndo_slave_add_vid()
247 	pm_runtime_put(common->dev);  in am65_cpsw_nuss_ndo_slave_add_vid()
260 	ret = pm_runtime_get_sync(common->dev);  in am65_cpsw_nuss_ndo_slave_kill_vid()
262 		pm_runtime_put_noidle(common->dev);  in am65_cpsw_nuss_ndo_slave_kill_vid()
266 	dev_info(common->dev, "Removing vlan %d from vlan filter\n", vid);  in am65_cpsw_nuss_ndo_slave_kill_vid()
267 	ret = cpsw_ale_del_vlan(common->ale, vid, 0);  in am65_cpsw_nuss_ndo_slave_kill_vid()
269 	pm_runtime_put(common->dev);  in am65_cpsw_nuss_ndo_slave_kill_vid()
276 	struct am65_cpsw_common *common = port->common;  in am65_cpsw_slave_set_promisc_2g()
280 		cpsw_ale_control_set(common->ale, port->port_id,  in am65_cpsw_slave_set_promisc_2g()
282 		dev_dbg(common->dev, "promisc enabled\n");  in am65_cpsw_slave_set_promisc_2g()
285 		cpsw_ale_control_set(common->ale, port->port_id,  in am65_cpsw_slave_set_promisc_2g()
287 		dev_dbg(common->dev, "promisc disabled\n");  in am65_cpsw_slave_set_promisc_2g()
298 	promisc = !!(ndev->flags & IFF_PROMISC);  in am65_cpsw_nuss_ndo_slave_set_rx_mode()
305 	cpsw_ale_set_allmulti(common->ale,  in am65_cpsw_nuss_ndo_slave_set_rx_mode()
306 			      ndev->flags & IFF_ALLMULTI, port->port_id);  in am65_cpsw_nuss_ndo_slave_set_rx_mode()
310 	cpsw_ale_flush_multicast(common->ale, port_mask, -1);  in am65_cpsw_nuss_ndo_slave_set_rx_mode()
317 			cpsw_ale_add_mcast(common->ale, ha->addr,  in am65_cpsw_nuss_ndo_slave_set_rx_mode()
332 	tx_chn = &common->tx_chns[txqueue];  in am65_cpsw_nuss_ndo_host_tx_timeout()
333 	trans_start = netif_txq->trans_start;  in am65_cpsw_nuss_ndo_host_tx_timeout()
338 		   jiffies_to_msecs(jiffies - trans_start),  in am65_cpsw_nuss_ndo_host_tx_timeout()
339 		   dql_avail(&netif_txq->dql),  in am65_cpsw_nuss_ndo_host_tx_timeout()
340 		   k3_cppi_desc_pool_avail(tx_chn->desc_pool));  in am65_cpsw_nuss_ndo_host_tx_timeout()
352 	struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;  in am65_cpsw_nuss_rx_push()
354 	struct device *dev = common->dev;  in am65_cpsw_nuss_rx_push()
360 	desc_rx = k3_cppi_desc_pool_alloc(rx_chn->desc_pool);  in am65_cpsw_nuss_rx_push()
363 		return -ENOMEM;  in am65_cpsw_nuss_rx_push()
365 	desc_dma = k3_cppi_desc_pool_virt2dma(rx_chn->desc_pool, desc_rx);  in am65_cpsw_nuss_rx_push()
367 	buf_dma = dma_map_single(dev, skb->data, pkt_len, DMA_FROM_DEVICE);  in am65_cpsw_nuss_rx_push()
369 		k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx);  in am65_cpsw_nuss_rx_push()
371 		return -EINVAL;  in am65_cpsw_nuss_rx_push()
380 	return k3_udma_glue_push_rx_chn(rx_chn->rx_chn, 0, desc_rx, desc_dma);  in am65_cpsw_nuss_rx_push()
389 	val = readl(host_p->port_base + AM65_CPSW_PORT_REG_PRI_CTL);  in am65_cpsw_nuss_set_p0_ptype()
391 	if (common->pf_p0_rx_ptype_rrobin) {  in am65_cpsw_nuss_set_p0_ptype()
403 	writel(pri_map, host_p->port_base + AM65_CPSW_PORT_REG_RX_PRI_MAP);  in am65_cpsw_nuss_set_p0_ptype()
404 	writel(val, host_p->port_base + AM65_CPSW_PORT_REG_PRI_CTL);  in am65_cpsw_nuss_set_p0_ptype()
415 	if (common->usage_count)  in am65_cpsw_nuss_common_open()
421 	       common->cpsw_base + AM65_CPSW_REG_CTL);  in am65_cpsw_nuss_common_open()
424 	       host_p->port_base + AM65_CPSW_PORT_REG_RX_MAXLEN);  in am65_cpsw_nuss_common_open()
426 	writel(common->rx_flow_id_base,  in am65_cpsw_nuss_common_open()
427 	       host_p->port_base + AM65_CPSW_PORT0_REG_FLOW_ID_OFFSET);  in am65_cpsw_nuss_common_open()
431 		       host_p->port_base + AM65_CPSW_P0_REG_CTL);  in am65_cpsw_nuss_common_open()
437 	for (port_idx = 0; port_idx < common->port_num; port_idx++) {  in am65_cpsw_nuss_common_open()
438 		struct am65_cpsw_port *port = &common->ports[port_idx];  in am65_cpsw_nuss_common_open()
440 		if (!port->disabled)  in am65_cpsw_nuss_common_open()
441 			val |=  BIT(port->port_id);  in am65_cpsw_nuss_common_open()
443 	writel(val, common->cpsw_base + AM65_CPSW_REG_STAT_PORT_EN);  in am65_cpsw_nuss_common_open()
446 	writel(0, common->cpsw_base + AM65_CPSW_REG_PTYPE);  in am65_cpsw_nuss_common_open()
448 	cpsw_ale_start(common->ale);  in am65_cpsw_nuss_common_open()
451 	cpsw_ale_control_set(common->ale, HOST_PORT_NUM,  in am65_cpsw_nuss_common_open()
453 	cpsw_ale_control_set(common->ale, HOST_PORT_NUM,  in am65_cpsw_nuss_common_open()
456 		cpsw_ale_control_set(common->ale, HOST_PORT_NUM,  in am65_cpsw_nuss_common_open()
459 	cpsw_ale_control_set(common->ale, HOST_PORT_NUM, ALE_VLAN_AWARE, 1);  in am65_cpsw_nuss_common_open()
460 	cpsw_ale_control_set(common->ale, HOST_PORT_NUM,  in am65_cpsw_nuss_common_open()
464 	port_mask = GENMASK(common->port_num, 0) &  in am65_cpsw_nuss_common_open()
465 		    ~common->disabled_ports_mask;  in am65_cpsw_nuss_common_open()
467 	cpsw_ale_add_vlan(common->ale, 0, port_mask,  in am65_cpsw_nuss_common_open()
471 	for (i = 0; i < common->rx_chns.descs_num; i++) {  in am65_cpsw_nuss_common_open()
476 			dev_err(common->dev, "cannot allocate skb\n");  in am65_cpsw_nuss_common_open()
477 			return -ENOMEM;  in am65_cpsw_nuss_common_open()
482 			dev_err(common->dev,  in am65_cpsw_nuss_common_open()
490 	k3_udma_glue_enable_rx_chn(common->rx_chns.rx_chn);  in am65_cpsw_nuss_common_open()
492 	for (i = 0; i < common->tx_ch_num; i++) {  in am65_cpsw_nuss_common_open()
493 		ret = k3_udma_glue_enable_tx_chn(common->tx_chns[i].tx_chn);  in am65_cpsw_nuss_common_open()
496 		napi_enable(&common->tx_chns[i].napi_tx);  in am65_cpsw_nuss_common_open()
499 	napi_enable(&common->napi_rx);  in am65_cpsw_nuss_common_open()
501 	dev_dbg(common->dev, "cpsw_nuss started\n");  in am65_cpsw_nuss_common_open()
512 	if (common->usage_count != 1)  in am65_cpsw_nuss_common_stop()
515 	cpsw_ale_control_set(common->ale, HOST_PORT_NUM,  in am65_cpsw_nuss_common_stop()
519 	atomic_set(&common->tdown_cnt, common->tx_ch_num);  in am65_cpsw_nuss_common_stop()
522 	reinit_completion(&common->tdown_complete);  in am65_cpsw_nuss_common_stop()
524 	for (i = 0; i < common->tx_ch_num; i++)  in am65_cpsw_nuss_common_stop()
525 		k3_udma_glue_tdown_tx_chn(common->tx_chns[i].tx_chn, false);  in am65_cpsw_nuss_common_stop()
527 	i = wait_for_completion_timeout(&common->tdown_complete,  in am65_cpsw_nuss_common_stop()
530 		dev_err(common->dev, "tx timeout\n");  in am65_cpsw_nuss_common_stop()
531 	for (i = 0; i < common->tx_ch_num; i++)  in am65_cpsw_nuss_common_stop()
532 		napi_disable(&common->tx_chns[i].napi_tx);  in am65_cpsw_nuss_common_stop()
534 	for (i = 0; i < common->tx_ch_num; i++) {  in am65_cpsw_nuss_common_stop()
535 		k3_udma_glue_reset_tx_chn(common->tx_chns[i].tx_chn,  in am65_cpsw_nuss_common_stop()
536 					  &common->tx_chns[i],  in am65_cpsw_nuss_common_stop()
538 		k3_udma_glue_disable_tx_chn(common->tx_chns[i].tx_chn);  in am65_cpsw_nuss_common_stop()
541 	k3_udma_glue_tdown_rx_chn(common->rx_chns.rx_chn, true);  in am65_cpsw_nuss_common_stop()
542 	napi_disable(&common->napi_rx);  in am65_cpsw_nuss_common_stop()
545 		k3_udma_glue_reset_rx_chn(common->rx_chns.rx_chn, i,  in am65_cpsw_nuss_common_stop()
546 					  &common->rx_chns,  in am65_cpsw_nuss_common_stop()
549 	k3_udma_glue_disable_rx_chn(common->rx_chns.rx_chn);  in am65_cpsw_nuss_common_stop()
551 	cpsw_ale_stop(common->ale);  in am65_cpsw_nuss_common_stop()
553 	writel(0, common->cpsw_base + AM65_CPSW_REG_CTL);  in am65_cpsw_nuss_common_stop()
554 	writel(0, common->cpsw_base + AM65_CPSW_REG_STAT_PORT_EN);  in am65_cpsw_nuss_common_stop()
556 	dev_dbg(common->dev, "cpsw_nuss stopped\n");  in am65_cpsw_nuss_common_stop()
566 	if (port->slave.phy)  in am65_cpsw_nuss_ndo_slave_stop()
567 		phy_stop(port->slave.phy);  in am65_cpsw_nuss_ndo_slave_stop()
571 	if (port->slave.phy) {  in am65_cpsw_nuss_ndo_slave_stop()
572 		phy_disconnect(port->slave.phy);  in am65_cpsw_nuss_ndo_slave_stop()
573 		port->slave.phy = NULL;  in am65_cpsw_nuss_ndo_slave_stop()
580 	common->usage_count--;  in am65_cpsw_nuss_ndo_slave_stop()
581 	pm_runtime_put(common->dev);  in am65_cpsw_nuss_ndo_slave_stop()
592 	return am65_cpsw_nuss_ndo_slave_add_vid(port->ndev, 0, vid);  in cpsw_restore_vlans()
602 	ret = pm_runtime_get_sync(common->dev);  in am65_cpsw_nuss_ndo_slave_open()
604 		pm_runtime_put_noidle(common->dev);  in am65_cpsw_nuss_ndo_slave_open()
609 	ret = netif_set_real_num_tx_queues(ndev, common->tx_ch_num);  in am65_cpsw_nuss_ndo_slave_open()
611 		dev_err(common->dev, "cannot set real number of tx queues\n");  in am65_cpsw_nuss_ndo_slave_open()
617 		dev_err(common->dev, "cannot set real number of rx queues\n");  in am65_cpsw_nuss_ndo_slave_open()
621 	for (i = 0; i < common->tx_ch_num; i++)  in am65_cpsw_nuss_ndo_slave_open()
624 	ret = am65_cpsw_nuss_common_open(common, ndev->features);  in am65_cpsw_nuss_ndo_slave_open()
628 	common->usage_count++;  in am65_cpsw_nuss_ndo_slave_open()
630 	am65_cpsw_port_set_sl_mac(port, ndev->dev_addr);  in am65_cpsw_nuss_ndo_slave_open()
632 	if (port->slave.mac_only)  in am65_cpsw_nuss_ndo_slave_open()
633 		/* enable mac-only mode on port */  in am65_cpsw_nuss_ndo_slave_open()
634 		cpsw_ale_control_set(common->ale, port->port_id,  in am65_cpsw_nuss_ndo_slave_open()
637 		cpsw_ale_control_set(common->ale, port->port_id,  in am65_cpsw_nuss_ndo_slave_open()
640 	port_mask = BIT(port->port_id) | ALE_PORT_HOST;  in am65_cpsw_nuss_ndo_slave_open()
641 	cpsw_ale_add_ucast(common->ale, ndev->dev_addr,  in am65_cpsw_nuss_ndo_slave_open()
643 	cpsw_ale_add_mcast(common->ale, ndev->broadcast,  in am65_cpsw_nuss_ndo_slave_open()
646 	/* mac_sl should be configured via phy-link interface */  in am65_cpsw_nuss_ndo_slave_open()
649 	ret = phy_set_mode_ext(port->slave.ifphy, PHY_MODE_ETHERNET,  in am65_cpsw_nuss_ndo_slave_open()
650 			       port->slave.phy_if);  in am65_cpsw_nuss_ndo_slave_open()
654 	if (port->slave.phy_node) {  in am65_cpsw_nuss_ndo_slave_open()
655 		port->slave.phy = of_phy_connect(ndev,  in am65_cpsw_nuss_ndo_slave_open()
656 						 port->slave.phy_node,  in am65_cpsw_nuss_ndo_slave_open()
658 						 0, port->slave.phy_if);  in am65_cpsw_nuss_ndo_slave_open()
659 		if (!port->slave.phy) {  in am65_cpsw_nuss_ndo_slave_open()
660 			dev_err(common->dev, "phy %pOF not found on slave %d\n",  in am65_cpsw_nuss_ndo_slave_open()
661 				port->slave.phy_node,  in am65_cpsw_nuss_ndo_slave_open()
662 				port->port_id);  in am65_cpsw_nuss_ndo_slave_open()
663 			ret = -ENODEV;  in am65_cpsw_nuss_ndo_slave_open()
671 	phy_attached_info(port->slave.phy);  in am65_cpsw_nuss_ndo_slave_open()
672 	phy_start(port->slave.phy);  in am65_cpsw_nuss_ndo_slave_open()
690 	desc_rx = k3_cppi_desc_pool_dma2virt(rx_chn->desc_pool, desc_dma);  in am65_cpsw_nuss_rx_cleanup()
695 	dma_unmap_single(rx_chn->dev, buf_dma, buf_dma_len, DMA_FROM_DEVICE);  in am65_cpsw_nuss_rx_cleanup()
696 	k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx);  in am65_cpsw_nuss_rx_cleanup()
710 	ssh->hwtstamp = ns_to_ktime(ns);  in am65_cpsw_nuss_rx_ts()
713 /* RX psdata[2] word format - checksum information */
725 	 * AM65_CPSW_RX_PSD_CSUM_ERR bit - indicates csum error  in am65_cpsw_nuss_rx_csum()
727 	 * bits - indicates IPv4/IPv6 packet  in am65_cpsw_nuss_rx_csum()
728 	 * AM65_CPSW_RX_PSD_IS_FRAGMENT bit - indicates fragmented packet  in am65_cpsw_nuss_rx_csum()
734 	if (unlikely(!(skb->dev->features & NETIF_F_RXCSUM)))  in am65_cpsw_nuss_rx_csum()
742 			skb->ip_summed = CHECKSUM_UNNECESSARY;  in am65_cpsw_nuss_rx_csum()
749 	struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;  in am65_cpsw_nuss_rx_packets()
754 	struct device *dev = common->dev;  in am65_cpsw_nuss_rx_packets()
763 	ret = k3_udma_glue_pop_rx_chn(rx_chn->rx_chn, flow_idx, &desc_dma);  in am65_cpsw_nuss_rx_packets()
765 		if (ret != -ENODATA)  in am65_cpsw_nuss_rx_packets()
775 	desc_rx = k3_cppi_desc_pool_dma2virt(rx_chn->desc_pool, desc_dma);  in am65_cpsw_nuss_rx_packets()
783 	cppi5_desc_get_tags_ids(&desc_rx->hdr, &port_id, NULL);  in am65_cpsw_nuss_rx_packets()
786 	ndev = port->ndev;  in am65_cpsw_nuss_rx_packets()
787 	skb->dev = ndev;  in am65_cpsw_nuss_rx_packets()
791 	if (port->rx_ts_enabled)  in am65_cpsw_nuss_rx_packets()
798 	k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx);  in am65_cpsw_nuss_rx_packets()
803 		skb->protocol = eth_type_trans(skb, ndev);  in am65_cpsw_nuss_rx_packets()
805 		napi_gro_receive(&common->napi_rx, skb);  in am65_cpsw_nuss_rx_packets()
808 		stats = this_cpu_ptr(ndev_priv->stats);  in am65_cpsw_nuss_rx_packets()
810 		u64_stats_update_begin(&stats->syncp);  in am65_cpsw_nuss_rx_packets()
811 		stats->rx_packets++;  in am65_cpsw_nuss_rx_packets()
812 		stats->rx_bytes += pkt_len;  in am65_cpsw_nuss_rx_packets()
813 		u64_stats_update_end(&stats->syncp);  in am65_cpsw_nuss_rx_packets()
816 		ndev->stats.rx_dropped++;  in am65_cpsw_nuss_rx_packets()
822 		ndev->stats.rx_dropped++;  in am65_cpsw_nuss_rx_packets()
829 		ndev->stats.rx_errors++;  in am65_cpsw_nuss_rx_packets()
830 		ndev->stats.rx_dropped++;  in am65_cpsw_nuss_rx_packets()
844 	while (flow--) {  in am65_cpsw_nuss_rx_poll()
845 		cur_budget = budget - num_rx;  in am65_cpsw_nuss_rx_poll()
847 		while (cur_budget--) {  in am65_cpsw_nuss_rx_poll()
858 	dev_dbg(common->dev, "%s num_rx:%d %d\n", __func__, num_rx, budget);  in am65_cpsw_nuss_rx_poll()
861 		enable_irq(common->rx_chns.irq);  in am65_cpsw_nuss_rx_poll()
884 		next_desc = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool,  in am65_cpsw_nuss_xmit_free()
893 		k3_cppi_desc_pool_free(tx_chn->desc_pool, next_desc);  in am65_cpsw_nuss_xmit_free()
896 	k3_cppi_desc_pool_free(tx_chn->desc_pool, first_desc);  in am65_cpsw_nuss_xmit_free()
906 	desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool, desc_dma);  in am65_cpsw_nuss_tx_cleanup()
909 	am65_cpsw_nuss_xmit_free(tx_chn, tx_chn->common->dev, desc_tx);  in am65_cpsw_nuss_tx_cleanup()
918 	struct device *dev = common->dev;  in am65_cpsw_nuss_tx_compl_packets()
928 	tx_chn = &common->tx_chns[chn];  in am65_cpsw_nuss_tx_compl_packets()
934 		res = k3_udma_glue_pop_tx_chn(tx_chn->tx_chn, &desc_dma);  in am65_cpsw_nuss_tx_compl_packets()
935 		if (res == -ENODATA)  in am65_cpsw_nuss_tx_compl_packets()
939 			if (atomic_dec_and_test(&common->tdown_cnt))  in am65_cpsw_nuss_tx_compl_packets()
940 				complete(&common->tdown_complete);  in am65_cpsw_nuss_tx_compl_packets()
944 		desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool,  in am65_cpsw_nuss_tx_compl_packets()
950 		ndev = skb->dev;  in am65_cpsw_nuss_tx_compl_packets()
952 		am65_cpts_tx_timestamp(common->cpts, skb);  in am65_cpsw_nuss_tx_compl_packets()
955 		stats = this_cpu_ptr(ndev_priv->stats);  in am65_cpsw_nuss_tx_compl_packets()
956 		u64_stats_update_begin(&stats->syncp);  in am65_cpsw_nuss_tx_compl_packets()
957 		stats->tx_packets++;  in am65_cpsw_nuss_tx_compl_packets()
958 		stats->tx_bytes += skb->len;  in am65_cpsw_nuss_tx_compl_packets()
959 		u64_stats_update_end(&stats->syncp);  in am65_cpsw_nuss_tx_compl_packets()
961 		total_bytes += skb->len;  in am65_cpsw_nuss_tx_compl_packets()
980 		    (k3_cppi_desc_pool_avail(tx_chn->desc_pool) >=  in am65_cpsw_nuss_tx_compl_packets()
996 	num_tx = am65_cpsw_nuss_tx_compl_packets(tx_chn->common, tx_chn->id,  in am65_cpsw_nuss_tx_poll()
1001 		enable_irq(tx_chn->irq);  in am65_cpsw_nuss_tx_poll()
1012 	napi_schedule(&common->napi_rx);  in am65_cpsw_nuss_rx_irq()
1022 	napi_schedule(&tx_chn->napi_tx);  in am65_cpsw_nuss_tx_irq()
1033 	struct device *dev = common->dev;  in am65_cpsw_nuss_ndo_slave_xmit()
1046 	if (port->tx_ts_enabled)  in am65_cpsw_nuss_ndo_slave_xmit()
1047 		am65_cpts_prep_tx_timestamp(common->cpts, skb);  in am65_cpsw_nuss_ndo_slave_xmit()
1052 	tx_chn = &common->tx_chns[q_idx];  in am65_cpsw_nuss_ndo_slave_xmit()
1056 	buf_dma = dma_map_single(dev, skb->data, pkt_len,  in am65_cpsw_nuss_ndo_slave_xmit()
1060 		ndev->stats.tx_errors++;  in am65_cpsw_nuss_ndo_slave_xmit()
1064 	first_desc = k3_cppi_desc_pool_alloc(tx_chn->desc_pool);  in am65_cpsw_nuss_ndo_slave_xmit()
1073 	cppi5_desc_set_pktids(&first_desc->hdr, 0, 0x3FFF);  in am65_cpsw_nuss_ndo_slave_xmit()
1075 	cppi5_desc_set_tags_ids(&first_desc->hdr, 0, port->port_id);  in am65_cpsw_nuss_ndo_slave_xmit()
1084 	if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) {  in am65_cpsw_nuss_ndo_slave_xmit()
1088 		cs_offset = cs_start + skb->csum_offset;  in am65_cpsw_nuss_ndo_slave_xmit()
1091 			    ((cs_start + 1) << 16) | (skb->len - cs_start);  in am65_cpsw_nuss_ndo_slave_xmit()
1102 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {  in am65_cpsw_nuss_ndo_slave_xmit()
1103 		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];  in am65_cpsw_nuss_ndo_slave_xmit()
1106 		next_desc = k3_cppi_desc_pool_alloc(tx_chn->desc_pool);  in am65_cpsw_nuss_ndo_slave_xmit()
1116 			k3_cppi_desc_pool_free(tx_chn->desc_pool, next_desc);  in am65_cpsw_nuss_ndo_slave_xmit()
1117 			ndev->stats.tx_errors++;  in am65_cpsw_nuss_ndo_slave_xmit()
1125 		desc_dma = k3_cppi_desc_pool_virt2dma(tx_chn->desc_pool,  in am65_cpsw_nuss_ndo_slave_xmit()
1132 	WARN_ON(pkt_len != skb->len);  in am65_cpsw_nuss_ndo_slave_xmit()
1141 	desc_dma = k3_cppi_desc_pool_virt2dma(tx_chn->desc_pool, first_desc);  in am65_cpsw_nuss_ndo_slave_xmit()
1142 	ret = k3_udma_glue_push_tx_chn(tx_chn->tx_chn, first_desc, desc_dma);  in am65_cpsw_nuss_ndo_slave_xmit()
1147 		ndev->stats.tx_errors++;  in am65_cpsw_nuss_ndo_slave_xmit()
1151 	if (k3_cppi_desc_pool_avail(tx_chn->desc_pool) < MAX_SKB_FRAGS) {  in am65_cpsw_nuss_ndo_slave_xmit()
1157 		/* re-check for smp */  in am65_cpsw_nuss_ndo_slave_xmit()
1158 		if (k3_cppi_desc_pool_avail(tx_chn->desc_pool) >=  in am65_cpsw_nuss_ndo_slave_xmit()
1170 	ndev->stats.tx_dropped++;  in am65_cpsw_nuss_ndo_slave_xmit()
1193 	ret = pm_runtime_get_sync(common->dev);  in am65_cpsw_nuss_ndo_slave_set_mac_address()
1195 		pm_runtime_put_noidle(common->dev);  in am65_cpsw_nuss_ndo_slave_set_mac_address()
1199 	cpsw_ale_del_ucast(common->ale, ndev->dev_addr,  in am65_cpsw_nuss_ndo_slave_set_mac_address()
1201 	cpsw_ale_add_ucast(common->ale, sockaddr->sa_data,  in am65_cpsw_nuss_ndo_slave_set_mac_address()
1207 	pm_runtime_put(common->dev);  in am65_cpsw_nuss_ndo_slave_set_mac_address()
1221 		return -EOPNOTSUPP;  in am65_cpsw_nuss_hwtstamp_set()
1223 	if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg)))  in am65_cpsw_nuss_hwtstamp_set()
1224 		return -EFAULT;  in am65_cpsw_nuss_hwtstamp_set()
1232 		return -ERANGE;  in am65_cpsw_nuss_hwtstamp_set()
1237 		port->rx_ts_enabled = false;  in am65_cpsw_nuss_hwtstamp_set()
1254 		port->rx_ts_enabled = true;  in am65_cpsw_nuss_hwtstamp_set()
1258 		return -ERANGE;  in am65_cpsw_nuss_hwtstamp_set()
1261 	port->tx_ts_enabled = (cfg.tx_type == HWTSTAMP_TX_ON);  in am65_cpsw_nuss_hwtstamp_set()
1282 	if (port->tx_ts_enabled)  in am65_cpsw_nuss_hwtstamp_set()
1286 	writel(seq_id, port->port_base + AM65_CPSW_PORTN_REG_TS_SEQ_LTYPE_REG);  in am65_cpsw_nuss_hwtstamp_set()
1287 	writel(ts_vlan_ltype, port->port_base +  in am65_cpsw_nuss_hwtstamp_set()
1289 	writel(ts_ctrl_ltype2, port->port_base +  in am65_cpsw_nuss_hwtstamp_set()
1291 	writel(ts_ctrl, port->port_base + AM65_CPSW_PORTN_REG_TS_CTL);  in am65_cpsw_nuss_hwtstamp_set()
1294 	am65_cpts_rx_enable(common->cpts, port->rx_ts_enabled);  in am65_cpsw_nuss_hwtstamp_set()
1296 	return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0;  in am65_cpsw_nuss_hwtstamp_set()
1306 		return -EOPNOTSUPP;  in am65_cpsw_nuss_hwtstamp_get()
1309 	cfg.tx_type = port->tx_ts_enabled ?  in am65_cpsw_nuss_hwtstamp_get()
1311 	cfg.rx_filter = port->rx_ts_enabled ?  in am65_cpsw_nuss_hwtstamp_get()
1314 	return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0;  in am65_cpsw_nuss_hwtstamp_get()
1323 		return -EINVAL;  in am65_cpsw_nuss_ndo_slave_ioctl()
1332 	if (!port->slave.phy)  in am65_cpsw_nuss_ndo_slave_ioctl()
1333 		return -EOPNOTSUPP;  in am65_cpsw_nuss_ndo_slave_ioctl()
1335 	return phy_mii_ioctl(port->slave.phy, req, cmd);  in am65_cpsw_nuss_ndo_slave_ioctl()
1352 		cpu_stats = per_cpu_ptr(ndev_priv->stats, cpu);  in am65_cpsw_nuss_ndo_get_stats()
1354 			start = u64_stats_fetch_begin_irq(&cpu_stats->syncp);  in am65_cpsw_nuss_ndo_get_stats()
1355 			rx_packets = cpu_stats->rx_packets;  in am65_cpsw_nuss_ndo_get_stats()
1356 			rx_bytes   = cpu_stats->rx_bytes;  in am65_cpsw_nuss_ndo_get_stats()
1357 			tx_packets = cpu_stats->tx_packets;  in am65_cpsw_nuss_ndo_get_stats()
1358 			tx_bytes   = cpu_stats->tx_bytes;  in am65_cpsw_nuss_ndo_get_stats()
1359 		} while (u64_stats_fetch_retry_irq(&cpu_stats->syncp, start));  in am65_cpsw_nuss_ndo_get_stats()
1361 		stats->rx_packets += rx_packets;  in am65_cpsw_nuss_ndo_get_stats()
1362 		stats->rx_bytes   += rx_bytes;  in am65_cpsw_nuss_ndo_get_stats()
1363 		stats->tx_packets += tx_packets;  in am65_cpsw_nuss_ndo_get_stats()
1364 		stats->tx_bytes   += tx_bytes;  in am65_cpsw_nuss_ndo_get_stats()
1367 	stats->rx_errors	= dev->stats.rx_errors;  in am65_cpsw_nuss_ndo_get_stats()
1368 	stats->rx_dropped	= dev->stats.rx_dropped;  in am65_cpsw_nuss_ndo_get_stats()
1369 	stats->tx_dropped	= dev->stats.tx_dropped;  in am65_cpsw_nuss_ndo_get_stats()
1376 	netdev_features_t changes = features ^ ndev->features;  in am65_cpsw_nuss_ndo_slave_set_features()
1384 		dev_info(common->dev, "Turn %s tx-checksum-ip-generic\n",  in am65_cpsw_nuss_ndo_slave_set_features()
1388 			       host_p->port_base + AM65_CPSW_P0_REG_CTL);  in am65_cpsw_nuss_ndo_slave_set_features()
1391 			       host_p->port_base + AM65_CPSW_P0_REG_CTL);  in am65_cpsw_nuss_ndo_slave_set_features()
1415 	struct am65_cpsw_common *common = port->common;  in am65_cpsw_nuss_slave_disable_unused()
1417 	if (!port->disabled)  in am65_cpsw_nuss_slave_disable_unused()
1420 	common->disabled_ports_mask |= BIT(port->port_id);  in am65_cpsw_nuss_slave_disable_unused()
1421 	cpsw_ale_control_set(common->ale, port->port_id,  in am65_cpsw_nuss_slave_disable_unused()
1424 	cpsw_sl_reset(port->slave.mac_sl, 100);  in am65_cpsw_nuss_slave_disable_unused()
1425 	cpsw_sl_ctl_reset(port->slave.mac_sl);  in am65_cpsw_nuss_slave_disable_unused()
1433 	for (i = 0; i < common->tx_ch_num; i++) {  in am65_cpsw_nuss_free_tx_chns()
1434 		struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i];  in am65_cpsw_nuss_free_tx_chns()
1436 		if (!IS_ERR_OR_NULL(tx_chn->tx_chn))  in am65_cpsw_nuss_free_tx_chns()
1437 			k3_udma_glue_release_tx_chn(tx_chn->tx_chn);  in am65_cpsw_nuss_free_tx_chns()
1439 		if (!IS_ERR_OR_NULL(tx_chn->desc_pool))  in am65_cpsw_nuss_free_tx_chns()
1440 			k3_cppi_desc_pool_destroy(tx_chn->desc_pool);  in am65_cpsw_nuss_free_tx_chns()
1448 	struct device *dev = common->dev;  in am65_cpsw_nuss_remove_tx_chns()
1453 	for (i = 0; i < common->tx_ch_num; i++) {  in am65_cpsw_nuss_remove_tx_chns()
1454 		struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i];  in am65_cpsw_nuss_remove_tx_chns()
1456 		if (tx_chn->irq)  in am65_cpsw_nuss_remove_tx_chns()
1457 			devm_free_irq(dev, tx_chn->irq, tx_chn);  in am65_cpsw_nuss_remove_tx_chns()
1459 		netif_napi_del(&tx_chn->napi_tx);  in am65_cpsw_nuss_remove_tx_chns()
1461 		if (!IS_ERR_OR_NULL(tx_chn->tx_chn))  in am65_cpsw_nuss_remove_tx_chns()
1462 			k3_udma_glue_release_tx_chn(tx_chn->tx_chn);  in am65_cpsw_nuss_remove_tx_chns()
1464 		if (!IS_ERR_OR_NULL(tx_chn->desc_pool))  in am65_cpsw_nuss_remove_tx_chns()
1465 			k3_cppi_desc_pool_destroy(tx_chn->desc_pool);  in am65_cpsw_nuss_remove_tx_chns()
1475 	struct device *dev = common->dev;  in am65_cpsw_nuss_init_tx_chns()
1493 	for (i = 0; i < common->tx_ch_num; i++) {  in am65_cpsw_nuss_init_tx_chns()
1494 		struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i];  in am65_cpsw_nuss_init_tx_chns()
1496 		snprintf(tx_chn->tx_chn_name,  in am65_cpsw_nuss_init_tx_chns()
1497 			 sizeof(tx_chn->tx_chn_name), "tx%d", i);  in am65_cpsw_nuss_init_tx_chns()
1499 		tx_chn->common = common;  in am65_cpsw_nuss_init_tx_chns()
1500 		tx_chn->id = i;  in am65_cpsw_nuss_init_tx_chns()
1501 		tx_chn->descs_num = max_desc_num;  in am65_cpsw_nuss_init_tx_chns()
1502 		tx_chn->desc_pool =  in am65_cpsw_nuss_init_tx_chns()
1504 						      tx_chn->descs_num,  in am65_cpsw_nuss_init_tx_chns()
1506 						      tx_chn->tx_chn_name);  in am65_cpsw_nuss_init_tx_chns()
1507 		if (IS_ERR(tx_chn->desc_pool)) {  in am65_cpsw_nuss_init_tx_chns()
1508 			ret = PTR_ERR(tx_chn->desc_pool);  in am65_cpsw_nuss_init_tx_chns()
1513 		tx_chn->tx_chn =  in am65_cpsw_nuss_init_tx_chns()
1515 						    tx_chn->tx_chn_name,  in am65_cpsw_nuss_init_tx_chns()
1517 		if (IS_ERR(tx_chn->tx_chn)) {  in am65_cpsw_nuss_init_tx_chns()
1518 			ret = PTR_ERR(tx_chn->tx_chn);  in am65_cpsw_nuss_init_tx_chns()
1524 		tx_chn->irq = k3_udma_glue_tx_get_irq(tx_chn->tx_chn);  in am65_cpsw_nuss_init_tx_chns()
1525 		if (tx_chn->irq <= 0) {  in am65_cpsw_nuss_init_tx_chns()
1527 				tx_chn->irq);  in am65_cpsw_nuss_init_tx_chns()
1531 		snprintf(tx_chn->tx_chn_name,  in am65_cpsw_nuss_init_tx_chns()
1532 			 sizeof(tx_chn->tx_chn_name), "%s-tx%d",  in am65_cpsw_nuss_init_tx_chns()
1533 			 dev_name(dev), tx_chn->id);  in am65_cpsw_nuss_init_tx_chns()
1551 	rx_chn = &common->rx_chns;  in am65_cpsw_nuss_free_rx_chns()
1553 	if (!IS_ERR_OR_NULL(rx_chn->rx_chn))  in am65_cpsw_nuss_free_rx_chns()
1554 		k3_udma_glue_release_rx_chn(rx_chn->rx_chn);  in am65_cpsw_nuss_free_rx_chns()
1556 	if (!IS_ERR_OR_NULL(rx_chn->desc_pool))  in am65_cpsw_nuss_free_rx_chns()
1557 		k3_cppi_desc_pool_destroy(rx_chn->desc_pool);  in am65_cpsw_nuss_free_rx_chns()
1562 	struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;  in am65_cpsw_nuss_init_rx_chns()
1565 	struct device *dev = common->dev;  in am65_cpsw_nuss_init_rx_chns()
1575 	rx_cfg.flow_id_base = common->rx_flow_id_base;  in am65_cpsw_nuss_init_rx_chns()
1578 	rx_chn->dev = dev;  in am65_cpsw_nuss_init_rx_chns()
1579 	rx_chn->descs_num = max_desc_num;  in am65_cpsw_nuss_init_rx_chns()
1580 	rx_chn->desc_pool = k3_cppi_desc_pool_create_name(dev,  in am65_cpsw_nuss_init_rx_chns()
1581 							  rx_chn->descs_num,  in am65_cpsw_nuss_init_rx_chns()
1583 	if (IS_ERR(rx_chn->desc_pool)) {  in am65_cpsw_nuss_init_rx_chns()
1584 		ret = PTR_ERR(rx_chn->desc_pool);  in am65_cpsw_nuss_init_rx_chns()
1589 	rx_chn->rx_chn = k3_udma_glue_request_rx_chn(dev, "rx", &rx_cfg);  in am65_cpsw_nuss_init_rx_chns()
1590 	if (IS_ERR(rx_chn->rx_chn)) {  in am65_cpsw_nuss_init_rx_chns()
1591 		ret = PTR_ERR(rx_chn->rx_chn);  in am65_cpsw_nuss_init_rx_chns()
1596 	common->rx_flow_id_base =  in am65_cpsw_nuss_init_rx_chns()
1597 			k3_udma_glue_rx_get_flow_id_base(rx_chn->rx_chn);  in am65_cpsw_nuss_init_rx_chns()
1598 	dev_info(dev, "set new flow-id-base %u\n", common->rx_flow_id_base);  in am65_cpsw_nuss_init_rx_chns()
1624 		ret = k3_udma_glue_rx_flow_init(rx_chn->rx_chn,  in am65_cpsw_nuss_init_rx_chns()
1632 				k3_udma_glue_rx_flow_get_fdq_id(rx_chn->rx_chn,  in am65_cpsw_nuss_init_rx_chns()
1635 		rx_chn->irq = k3_udma_glue_rx_get_irq(rx_chn->rx_chn, i);  in am65_cpsw_nuss_init_rx_chns()
1637 		if (rx_chn->irq <= 0) {  in am65_cpsw_nuss_init_rx_chns()
1639 				rx_chn->irq);  in am65_cpsw_nuss_init_rx_chns()
1640 			ret = -ENXIO;  in am65_cpsw_nuss_init_rx_chns()
1659 	host_p->common = common;  in am65_cpsw_nuss_init_host_p()
1660 	host_p->port_base = common->cpsw_base + AM65_CPSW_NU_PORTS_BASE;  in am65_cpsw_nuss_init_host_p()
1661 	host_p->stat_base = common->cpsw_base + AM65_CPSW_NU_STATS_BASE;  in am65_cpsw_nuss_init_host_p()
1673 	syscon = syscon_regmap_lookup_by_phandle(of_node, "ti,syscon-efuse");  in am65_cpsw_am654_get_efuse_macid()
1675 		if (PTR_ERR(syscon) == -ENODEV)  in am65_cpsw_am654_get_efuse_macid()
1680 	ret = of_property_read_u32_index(of_node, "ti,syscon-efuse", 1,  in am65_cpsw_am654_get_efuse_macid()
1700 	struct device *dev = common->dev;  in am65_cpsw_init_cpts()
1708 	node = of_get_child_by_name(dev->of_node, "cpts");  in am65_cpsw_init_cpts()
1711 		return -ENOENT;  in am65_cpsw_init_cpts()
1714 	reg_base = common->cpsw_base + AM65_CPSW_NU_CPTS_BASE;  in am65_cpsw_init_cpts()
1719 		if (ret == -EOPNOTSUPP) {  in am65_cpsw_init_cpts()
1727 	common->cpts = cpts;  in am65_cpsw_init_cpts()
1735 	struct device *dev = common->dev;  in am65_cpsw_nuss_init_slave_ports()
1738 	node = of_get_child_by_name(dev->of_node, "ethernet-ports");  in am65_cpsw_nuss_init_slave_ports()
1740 		return -ENOENT;  in am65_cpsw_nuss_init_slave_ports()
1748 		if (strcmp(port_np->name, "port"))  in am65_cpsw_nuss_init_slave_ports()
1758 		if (!port_id || port_id > common->port_num) {  in am65_cpsw_nuss_init_slave_ports()
1760 				port_np, port_id, port_np->name);  in am65_cpsw_nuss_init_slave_ports()
1761 			return -EINVAL;  in am65_cpsw_nuss_init_slave_ports()
1765 		port->port_id = port_id;  in am65_cpsw_nuss_init_slave_ports()
1766 		port->common = common;  in am65_cpsw_nuss_init_slave_ports()
1767 		port->port_base = common->cpsw_base + AM65_CPSW_NU_PORTS_BASE +  in am65_cpsw_nuss_init_slave_ports()
1769 		port->stat_base = common->cpsw_base + AM65_CPSW_NU_STATS_BASE +  in am65_cpsw_nuss_init_slave_ports()
1771 		port->name = of_get_property(port_np, "label", NULL);  in am65_cpsw_nuss_init_slave_ports()
1772 		port->fetch_ram_base =  in am65_cpsw_nuss_init_slave_ports()
1773 				common->cpsw_base + AM65_CPSW_NU_FRAM_BASE +  in am65_cpsw_nuss_init_slave_ports()
1774 				(AM65_CPSW_NU_FRAM_PORT_OFFSET * (port_id - 1));  in am65_cpsw_nuss_init_slave_ports()
1776 		port->slave.mac_sl = cpsw_sl_get("am65", dev, port->port_base);  in am65_cpsw_nuss_init_slave_ports()
1777 		if (IS_ERR(port->slave.mac_sl))  in am65_cpsw_nuss_init_slave_ports()
1778 			return PTR_ERR(port->slave.mac_sl);  in am65_cpsw_nuss_init_slave_ports()
1780 		port->disabled = !of_device_is_available(port_np);  in am65_cpsw_nuss_init_slave_ports()
1781 		if (port->disabled)  in am65_cpsw_nuss_init_slave_ports()
1784 		port->slave.ifphy = devm_of_phy_get(dev, port_np, NULL);  in am65_cpsw_nuss_init_slave_ports()
1785 		if (IS_ERR(port->slave.ifphy)) {  in am65_cpsw_nuss_init_slave_ports()
1786 			ret = PTR_ERR(port->slave.ifphy);  in am65_cpsw_nuss_init_slave_ports()
1792 		port->slave.mac_only =  in am65_cpsw_nuss_init_slave_ports()
1793 				of_property_read_bool(port_np, "ti,mac-only");  in am65_cpsw_nuss_init_slave_ports()
1799 				if (ret != -EPROBE_DEFER)  in am65_cpsw_nuss_init_slave_ports()
1800 					dev_err(dev, "%pOF failed to register fixed-link phy: %d\n",  in am65_cpsw_nuss_init_slave_ports()
1804 			port->slave.phy_node = of_node_get(port_np);  in am65_cpsw_nuss_init_slave_ports()
1806 			port->slave.phy_node =  in am65_cpsw_nuss_init_slave_ports()
1807 				of_parse_phandle(port_np, "phy-handle", 0);  in am65_cpsw_nuss_init_slave_ports()
1810 		if (!port->slave.phy_node) {  in am65_cpsw_nuss_init_slave_ports()
1813 			return -ENODEV;  in am65_cpsw_nuss_init_slave_ports()
1816 		ret = of_get_phy_mode(port_np, &port->slave.phy_if);  in am65_cpsw_nuss_init_slave_ports()
1818 			dev_err(dev, "%pOF read phy-mode err %d\n",  in am65_cpsw_nuss_init_slave_ports()
1825 			ether_addr_copy(port->slave.mac_addr, mac_addr);  in am65_cpsw_nuss_init_slave_ports()
1827 							   port->port_id,  in am65_cpsw_nuss_init_slave_ports()
1828 							   port->slave.mac_addr) ||  in am65_cpsw_nuss_init_slave_ports()
1829 			   !is_valid_ether_addr(port->slave.mac_addr)) {  in am65_cpsw_nuss_init_slave_ports()
1830 			random_ether_addr(port->slave.mac_addr);  in am65_cpsw_nuss_init_slave_ports()
1849 	struct device *dev = common->dev;  in am65_cpsw_nuss_init_ndev_2g()
1856 	port->ndev = devm_alloc_etherdev_mqs(common->dev,  in am65_cpsw_nuss_init_ndev_2g()
1860 	if (!port->ndev) {  in am65_cpsw_nuss_init_ndev_2g()
1862 			port->port_id);  in am65_cpsw_nuss_init_ndev_2g()
1863 		return -ENOMEM;  in am65_cpsw_nuss_init_ndev_2g()
1866 	ndev_priv = netdev_priv(port->ndev);  in am65_cpsw_nuss_init_ndev_2g()
1867 	ndev_priv->port = port;  in am65_cpsw_nuss_init_ndev_2g()
1868 	ndev_priv->msg_enable = AM65_CPSW_DEBUG;  in am65_cpsw_nuss_init_ndev_2g()
1869 	SET_NETDEV_DEV(port->ndev, dev);  in am65_cpsw_nuss_init_ndev_2g()
1871 	ether_addr_copy(port->ndev->dev_addr, port->slave.mac_addr);  in am65_cpsw_nuss_init_ndev_2g()
1873 	port->ndev->min_mtu = AM65_CPSW_MIN_PACKET_SIZE;  in am65_cpsw_nuss_init_ndev_2g()
1874 	port->ndev->max_mtu = AM65_CPSW_MAX_PACKET_SIZE;  in am65_cpsw_nuss_init_ndev_2g()
1875 	port->ndev->hw_features = NETIF_F_SG |  in am65_cpsw_nuss_init_ndev_2g()
1879 	port->ndev->features = port->ndev->hw_features |  in am65_cpsw_nuss_init_ndev_2g()
1881 	port->ndev->vlan_features |=  NETIF_F_SG;  in am65_cpsw_nuss_init_ndev_2g()
1882 	port->ndev->netdev_ops = &am65_cpsw_nuss_netdev_ops_2g;  in am65_cpsw_nuss_init_ndev_2g()
1883 	port->ndev->ethtool_ops = &am65_cpsw_ethtool_ops_slave;  in am65_cpsw_nuss_init_ndev_2g()
1886 	if (common->pdata.quirks & AM65_CPSW_QUIRK_I2027_NO_TX_CSUM)  in am65_cpsw_nuss_init_ndev_2g()
1887 		port->ndev->features &= ~NETIF_F_HW_CSUM;  in am65_cpsw_nuss_init_ndev_2g()
1889 	ndev_priv->stats = netdev_alloc_pcpu_stats(struct am65_cpsw_ndev_stats);  in am65_cpsw_nuss_init_ndev_2g()
1890 	if (!ndev_priv->stats)  in am65_cpsw_nuss_init_ndev_2g()
1891 		return -ENOMEM;  in am65_cpsw_nuss_init_ndev_2g()
1894 				       ndev_priv->stats);  in am65_cpsw_nuss_init_ndev_2g()
1900 	netif_napi_add(port->ndev, &common->napi_rx,  in am65_cpsw_nuss_init_ndev_2g()
1908 	struct device *dev = common->dev;  in am65_cpsw_nuss_ndev_add_napi_2g()
1914 	for (i = 0; i < common->tx_ch_num; i++) {  in am65_cpsw_nuss_ndev_add_napi_2g()
1915 		struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i];  in am65_cpsw_nuss_ndev_add_napi_2g()
1917 		netif_tx_napi_add(port->ndev, &tx_chn->napi_tx,  in am65_cpsw_nuss_ndev_add_napi_2g()
1920 		ret = devm_request_irq(dev, tx_chn->irq,  in am65_cpsw_nuss_ndev_add_napi_2g()
1923 				       tx_chn->tx_chn_name, tx_chn);  in am65_cpsw_nuss_ndev_add_napi_2g()
1926 				tx_chn->id, tx_chn->irq, ret);  in am65_cpsw_nuss_ndev_add_napi_2g()
1937 	struct device *dev = common->dev;  in am65_cpsw_nuss_ndev_reg_2g()
1946 	ret = devm_request_irq(dev, common->rx_chns.irq,  in am65_cpsw_nuss_ndev_reg_2g()
1951 			common->rx_chns.irq, ret);  in am65_cpsw_nuss_ndev_reg_2g()
1955 	ret = register_netdev(port->ndev);  in am65_cpsw_nuss_ndev_reg_2g()
1970 	common->tx_ch_num = num_tx;  in am65_cpsw_nuss_update_tx_chns()
1983 	for (i = 0; i < common->port_num; i++) {  in am65_cpsw_nuss_cleanup_ndev()
1984 		port = &common->ports[i];  in am65_cpsw_nuss_cleanup_ndev()
1985 		if (port->ndev)  in am65_cpsw_nuss_cleanup_ndev()
1986 			unregister_netdev(port->ndev);  in am65_cpsw_nuss_cleanup_ndev()
2015 	{ .compatible = "ti,am654-cpsw-nuss", .data = &am65x_sr1_0},
2016 	{ .compatible = "ti,j721e-cpsw-nuss", .data = &j721e_pdata},
2026 	if (soc && soc->data) {  in am65_cpsw_nuss_apply_socinfo()
2027 		const struct am65_cpsw_soc_pdata *socdata = soc->data;  in am65_cpsw_nuss_apply_socinfo()
2030 		common->pdata.quirks &= ~socdata->quirks_dis;  in am65_cpsw_nuss_apply_socinfo()
2038 	struct device *dev = &pdev->dev;  in am65_cpsw_nuss_probe()
2047 		return -ENOMEM;  in am65_cpsw_nuss_probe()
2048 	common->dev = dev;  in am65_cpsw_nuss_probe()
2052 		return -EINVAL;  in am65_cpsw_nuss_probe()
2053 	common->pdata = *(const struct am65_cpsw_pdata *)of_id->data;  in am65_cpsw_nuss_probe()
2058 	common->ss_base = devm_ioremap_resource(&pdev->dev, res);  in am65_cpsw_nuss_probe()
2059 	if (IS_ERR(common->ss_base))  in am65_cpsw_nuss_probe()
2060 		return PTR_ERR(common->ss_base);  in am65_cpsw_nuss_probe()
2061 	common->cpsw_base = common->ss_base + AM65_CPSW_CPSW_NU_BASE;  in am65_cpsw_nuss_probe()
2063 	node = of_get_child_by_name(dev->of_node, "ethernet-ports");  in am65_cpsw_nuss_probe()
2065 		return -ENOENT;  in am65_cpsw_nuss_probe()
2066 	common->port_num = of_get_child_count(node);  in am65_cpsw_nuss_probe()
2067 	if (common->port_num < 1 || common->port_num > AM65_CPSW_MAX_PORTS)  in am65_cpsw_nuss_probe()
2068 		return -ENOENT;  in am65_cpsw_nuss_probe()
2071 	if (common->port_num != 1)  in am65_cpsw_nuss_probe()
2072 		return -EOPNOTSUPP;  in am65_cpsw_nuss_probe()
2074 	common->rx_flow_id_base = -1;  in am65_cpsw_nuss_probe()
2075 	init_completion(&common->tdown_complete);  in am65_cpsw_nuss_probe()
2076 	common->tx_ch_num = 1;  in am65_cpsw_nuss_probe()
2077 	common->pf_p0_rx_ptype_rrobin = false;  in am65_cpsw_nuss_probe()
2085 	common->ports = devm_kcalloc(dev, common->port_num,  in am65_cpsw_nuss_probe()
2086 				     sizeof(*common->ports),  in am65_cpsw_nuss_probe()
2088 	if (!common->ports)  in am65_cpsw_nuss_probe()
2089 		return -ENOMEM;  in am65_cpsw_nuss_probe()
2095 		if (ret != -EPROBE_DEFER)  in am65_cpsw_nuss_probe()
2099 	common->bus_freq = clk_get_rate(clk);  in am65_cpsw_nuss_probe()
2109 	node = of_get_child_by_name(dev->of_node, "mdio");  in am65_cpsw_nuss_probe()
2117 			ret = -ENODEV;  in am65_cpsw_nuss_probe()
2121 		common->mdio_dev =  &mdio_pdev->dev;  in am65_cpsw_nuss_probe()
2146 	ale_params.ale_ports = common->port_num + 1;  in am65_cpsw_nuss_probe()
2147 	ale_params.ale_regs = common->cpsw_base + AM65_CPSW_NU_ALE_BASE;  in am65_cpsw_nuss_probe()
2148 	ale_params.dev_id = "am65x-cpsw2g";  in am65_cpsw_nuss_probe()
2149 	ale_params.bus_freq = common->bus_freq;  in am65_cpsw_nuss_probe()
2151 	common->ale = cpsw_ale_create(&ale_params);  in am65_cpsw_nuss_probe()
2152 	if (IS_ERR(common->ale)) {  in am65_cpsw_nuss_probe()
2154 		ret = PTR_ERR(common->ale);  in am65_cpsw_nuss_probe()
2163 	for (i = 0; i < common->port_num; i++)  in am65_cpsw_nuss_probe()
2164 		am65_cpsw_nuss_slave_disable_unused(&common->ports[i]);  in am65_cpsw_nuss_probe()
2180 	of_platform_device_destroy(common->mdio_dev, NULL);  in am65_cpsw_nuss_probe()
2189 	struct device *dev = &pdev->dev;  in am65_cpsw_nuss_remove()
2195 	ret = pm_runtime_get_sync(&pdev->dev);  in am65_cpsw_nuss_remove()
2197 		pm_runtime_put_noidle(&pdev->dev);  in am65_cpsw_nuss_remove()
2206 	of_platform_device_destroy(common->mdio_dev, NULL);  in am65_cpsw_nuss_remove()
2208 	pm_runtime_put_sync(&pdev->dev);  in am65_cpsw_nuss_remove()
2209 	pm_runtime_disable(&pdev->dev);  in am65_cpsw_nuss_remove()
2226 MODULE_DESCRIPTION("TI AM65 CPSW Ethernet driver");