1 // SPDX-License-Identifier: GPL-2.0-only
5  * Copyright (C) 2004-2006 Atmel Corporation
10 #include <linux/clk-provider.h>
20 #include <linux/gpio.h>
21 #include <linux/gpio/consumer.h>
25 #include <linux/dma-mapping.h>
41 #include <linux/firmware/xlnx-zynqmp.h>
44 /* This structure is only used for MACB on SiFive FU540 devices */
58 				 * (bp)->rx_ring_size)
64 				 * (bp)->tx_ring_size)
67 #define MACB_TX_WAKEUP_THRESH(bp)	(3 * (bp)->tx_ring_size / 4)
78 …MAX_TX_LEN		((unsigned int)((1 << MACB_TX_FRMLEN_SIZE) - 1) & ~((unsigned int)(MACB_TX_LEN_ALIGN -…
95  * 1 frame time (10 Mbits/s, full-duplex, ignoring collisions)
135 	switch (bp->hw_dma_cap) {  in macb_dma_desc_get_size()
160 	switch (bp->hw_dma_cap) {  in macb_adj_dma_desc_idx()
186 	return index & (bp->tx_ring_size - 1);  in macb_tx_ring_wrap()
192 	index = macb_tx_ring_wrap(queue->bp, index);  in macb_tx_desc()
193 	index = macb_adj_dma_desc_idx(queue->bp, index);  in macb_tx_desc()
194 	return &queue->tx_ring[index];  in macb_tx_desc()
200 	return &queue->tx_skb[macb_tx_ring_wrap(queue->bp, index)];  in macb_tx_skb()
207 	offset = macb_tx_ring_wrap(queue->bp, index) *  in macb_tx_dma()
208 			macb_dma_desc_get_size(queue->bp);  in macb_tx_dma()
210 	return queue->tx_ring_dma + offset;  in macb_tx_dma()
215 	return index & (bp->rx_ring_size - 1);  in macb_rx_ring_wrap()
220 	index = macb_rx_ring_wrap(queue->bp, index);  in macb_rx_desc()
221 	index = macb_adj_dma_desc_idx(queue->bp, index);  in macb_rx_desc()
222 	return &queue->rx_ring[index];  in macb_rx_desc()
227 	return queue->rx_buffers + queue->bp->rx_buffer_size *  in macb_rx_buffer()
228 	       macb_rx_ring_wrap(queue->bp, index);  in macb_rx_buffer()
234 	return __raw_readl(bp->regs + offset);  in hw_readl_native()
239 	__raw_writel(value, bp->regs + offset);  in hw_writel_native()
244 	return readl_relaxed(bp->regs + offset);  in hw_readl()
249 	writel_relaxed(value, bp->regs + offset);  in hw_writel()
286 	bottom = cpu_to_le32(*((u32 *)bp->dev->dev_addr));  in macb_set_hwaddr()
288 	top = cpu_to_le16(*((u16 *)(bp->dev->dev_addr + 4)));  in macb_set_hwaddr()
320 			eth_hw_addr_set(bp->dev, addr);  in macb_get_hwaddr()
325 	dev_info(&bp->pdev->dev, "invalid hw address, using random\n");  in macb_get_hwaddr()
326 	eth_hw_addr_random(bp->dev);  in macb_get_hwaddr()
339 	struct macb *bp = bus->priv;  in macb_mdio_read()
342 	status = pm_runtime_resume_and_get(&bp->pdev->dev);  in macb_mdio_read()
382 	pm_runtime_mark_last_busy(&bp->pdev->dev);  in macb_mdio_read()
383 	pm_runtime_put_autosuspend(&bp->pdev->dev);  in macb_mdio_read()
391 	struct macb *bp = bus->priv;  in macb_mdio_write()
394 	status = pm_runtime_resume_and_get(&bp->pdev->dev);  in macb_mdio_write()
434 	pm_runtime_mark_last_busy(&bp->pdev->dev);  in macb_mdio_write()
435 	pm_runtime_put_autosuspend(&bp->pdev->dev);  in macb_mdio_write()
445 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {  in macb_init_buffers()
446 		queue_writel(queue, RBQP, lower_32_bits(queue->rx_ring_dma));  in macb_init_buffers()
448 		if (bp->hw_dma_cap & HW_DMA_CAP_64B)  in macb_init_buffers()
450 				     upper_32_bits(queue->rx_ring_dma));  in macb_init_buffers()
452 		queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma));  in macb_init_buffers()
454 		if (bp->hw_dma_cap & HW_DMA_CAP_64B)  in macb_init_buffers()
456 				     upper_32_bits(queue->tx_ring_dma));  in macb_init_buffers()
462  * macb_set_tx_clk() - Set a clock to a new frequency
470 	if (!bp->tx_clk || (bp->caps & MACB_CAPS_CLK_HW_CHG))  in macb_set_tx_clk()
474 	if (bp->phy_interface == PHY_INTERFACE_MODE_MII)  in macb_set_tx_clk()
491 	rate_rounded = clk_round_rate(bp->tx_clk, rate);  in macb_set_tx_clk()
498 	ferr = abs(rate_rounded - rate);  in macb_set_tx_clk()
501 		netdev_warn(bp->dev,  in macb_set_tx_clk()
505 	if (clk_set_rate(bp->tx_clk, rate_rounded))  in macb_set_tx_clk()
506 		netdev_err(bp->dev, "adjusting tx_clk failed.\n");  in macb_set_tx_clk()
530 	state->speed = SPEED_10000;  in macb_usx_pcs_get_state()
531 	state->duplex = 1;  in macb_usx_pcs_get_state()
532 	state->an_complete = 1;  in macb_usx_pcs_get_state()
535 	state->link = !!(val & GEM_BIT(USX_BLOCK_LOCK));  in macb_usx_pcs_get_state()
538 		state->pause = MLO_PAUSE_RX;  in macb_usx_pcs_get_state()
558 	state->link = 0;  in macb_pcs_get_state()
590 	struct net_device *ndev = to_net_dev(config->dev);  in macb_mac_config()
596 	spin_lock_irqsave(&bp->lock, flags);  in macb_mac_config()
601 	if (bp->caps & MACB_CAPS_MACB_IS_EMAC) {  in macb_mac_config()
602 		if (state->interface == PHY_INTERFACE_MODE_RMII)  in macb_mac_config()
608 		if (state->interface == PHY_INTERFACE_MODE_SGMII) {  in macb_mac_config()
610 		} else if (state->interface == PHY_INTERFACE_MODE_10GBASER) {  in macb_mac_config()
613 		} else if (bp->caps & MACB_CAPS_MIIONRGMII &&  in macb_mac_config()
614 			   bp->phy_interface == PHY_INTERFACE_MODE_MII) {  in macb_mac_config()
630 	if (macb_is_gem(bp) && state->interface == PHY_INTERFACE_MODE_SGMII) {  in macb_mac_config()
642 	spin_unlock_irqrestore(&bp->lock, flags);  in macb_mac_config()
648 	struct net_device *ndev = to_net_dev(config->dev);  in macb_mac_link_down()
654 	if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC))  in macb_mac_link_down()
655 		for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)  in macb_mac_link_down()
657 				     bp->rx_intr_mask | MACB_TX_INT_FLAGS | MACB_BIT(HRESP));  in macb_mac_link_down()
672 	struct net_device *ndev = to_net_dev(config->dev);  in macb_mac_link_up()
679 	spin_lock_irqsave(&bp->lock, flags);  in macb_mac_link_up()
691 	if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) {  in macb_mac_link_up()
708 		bp->macbgem_ops.mog_init_rings(bp);  in macb_mac_link_up()
711 		for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)  in macb_mac_link_up()
713 				     bp->rx_intr_mask | MACB_TX_INT_FLAGS | MACB_BIT(HRESP));  in macb_mac_link_up()
718 	if (bp->phy_interface == PHY_INTERFACE_MODE_10GBASER)  in macb_mac_link_up()
722 	spin_unlock_irqrestore(&bp->lock, flags);  in macb_mac_link_up()
733 	struct net_device *ndev = to_net_dev(config->dev);  in macb_mac_select_pcs()
737 		return &bp->phylink_usx_pcs;  in macb_mac_select_pcs()
739 		return &bp->phylink_sgmii_pcs;  in macb_mac_select_pcs()
754 	dn = of_parse_phandle(dn, "phy-handle", 0);  in macb_phy_handle_exists()
761 	struct device_node *dn = bp->pdev->dev.of_node;  in macb_phylink_connect()
762 	struct net_device *dev = bp->dev;  in macb_phylink_connect()
767 		ret = phylink_of_phy_connect(bp->phylink, dn, 0);  in macb_phylink_connect()
770 		phydev = phy_find_first(bp->mii_bus);  in macb_phylink_connect()
773 			return -ENXIO;  in macb_phylink_connect()
777 		ret = phylink_connect_phy(bp->phylink, phydev);  in macb_phylink_connect()
785 	phylink_start(bp->phylink);  in macb_phylink_connect()
793 	struct net_device *ndev = to_net_dev(config->dev);  in macb_get_pcs_fixed_state()
796 	state->link = (macb_readl(bp, NSR) & MACB_BIT(NSR_LINK)) != 0;  in macb_get_pcs_fixed_state()
804 	bp->phylink_sgmii_pcs.ops = &macb_phylink_pcs_ops;  in macb_mii_probe()
805 	bp->phylink_usx_pcs.ops = &macb_phylink_usx_pcs_ops;  in macb_mii_probe()
807 	bp->phylink_config.dev = &dev->dev;  in macb_mii_probe()
808 	bp->phylink_config.type = PHYLINK_NETDEV;  in macb_mii_probe()
809 	bp->phylink_config.mac_managed_pm = true;  in macb_mii_probe()
811 	if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) {  in macb_mii_probe()
812 		bp->phylink_config.poll_fixed_state = true;  in macb_mii_probe()
813 		bp->phylink_config.get_fixed_state = macb_get_pcs_fixed_state;  in macb_mii_probe()
816 	bp->phylink_config.mac_capabilities = MAC_ASYM_PAUSE |  in macb_mii_probe()
820 		  bp->phylink_config.supported_interfaces);  in macb_mii_probe()
822 		  bp->phylink_config.supported_interfaces);  in macb_mii_probe()
825 	if (macb_is_gem(bp) && (bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE)) {  in macb_mii_probe()
826 		bp->phylink_config.mac_capabilities |= MAC_1000FD;  in macb_mii_probe()
827 		if (!(bp->caps & MACB_CAPS_NO_GIGABIT_HALF))  in macb_mii_probe()
828 			bp->phylink_config.mac_capabilities |= MAC_1000HD;  in macb_mii_probe()
831 			  bp->phylink_config.supported_interfaces);  in macb_mii_probe()
832 		phy_interface_set_rgmii(bp->phylink_config.supported_interfaces);  in macb_mii_probe()
834 		if (bp->caps & MACB_CAPS_PCS)  in macb_mii_probe()
836 				  bp->phylink_config.supported_interfaces);  in macb_mii_probe()
838 		if (bp->caps & MACB_CAPS_HIGH_SPEED) {  in macb_mii_probe()
840 				  bp->phylink_config.supported_interfaces);  in macb_mii_probe()
841 			bp->phylink_config.mac_capabilities |= MAC_10000FD;  in macb_mii_probe()
845 	bp->phylink = phylink_create(&bp->phylink_config, bp->pdev->dev.fwnode,  in macb_mii_probe()
846 				     bp->phy_interface, &macb_phylink_ops);  in macb_mii_probe()
847 	if (IS_ERR(bp->phylink)) {  in macb_mii_probe()
849 			   PTR_ERR(bp->phylink));  in macb_mii_probe()
850 		return PTR_ERR(bp->phylink);  in macb_mii_probe()
858 	struct device_node *child, *np = bp->pdev->dev.of_node;  in macb_mdiobus_register()
865 		int ret = of_mdiobus_register(bp->mii_bus, child);  in macb_mdiobus_register()
872 		return mdiobus_register(bp->mii_bus);  in macb_mdiobus_register()
886 			return of_mdiobus_register(bp->mii_bus, np);  in macb_mdiobus_register()
889 	return mdiobus_register(bp->mii_bus);  in macb_mdiobus_register()
894 	int err = -ENXIO;  in macb_mii_init()
899 	bp->mii_bus = mdiobus_alloc();  in macb_mii_init()
900 	if (!bp->mii_bus) {  in macb_mii_init()
901 		err = -ENOMEM;  in macb_mii_init()
905 	bp->mii_bus->name = "MACB_mii_bus";  in macb_mii_init()
906 	bp->mii_bus->read = &macb_mdio_read;  in macb_mii_init()
907 	bp->mii_bus->write = &macb_mdio_write;  in macb_mii_init()
908 	snprintf(bp->mii_bus->id, MII_BUS_ID_SIZE, "%s-%x",  in macb_mii_init()
909 		 bp->pdev->name, bp->pdev->id);  in macb_mii_init()
910 	bp->mii_bus->priv = bp;  in macb_mii_init()
911 	bp->mii_bus->parent = &bp->pdev->dev;  in macb_mii_init()
913 	dev_set_drvdata(&bp->dev->dev, bp->mii_bus);  in macb_mii_init()
919 	err = macb_mii_probe(bp->dev);  in macb_mii_init()
926 	mdiobus_unregister(bp->mii_bus);  in macb_mii_init()
928 	mdiobus_free(bp->mii_bus);  in macb_mii_init()
935 	u32 *p = &bp->hw_stats.macb.rx_pause_frames;  in macb_update_stats()
936 	u32 *end = &bp->hw_stats.macb.tx_pause_frames + 1;  in macb_update_stats()
939 	WARN_ON((unsigned long)(end - p - 1) != (MACB_TPF - MACB_PFR) / 4);  in macb_update_stats()
942 		*p += bp->macb_reg_readl(bp, offset);  in macb_update_stats()
962 	return -ETIMEDOUT;  in macb_halt_tx()
967 	if (tx_skb->mapping) {  in macb_tx_unmap()
968 		if (tx_skb->mapped_as_page)  in macb_tx_unmap()
969 			dma_unmap_page(&bp->pdev->dev, tx_skb->mapping,  in macb_tx_unmap()
970 				       tx_skb->size, DMA_TO_DEVICE);  in macb_tx_unmap()
972 			dma_unmap_single(&bp->pdev->dev, tx_skb->mapping,  in macb_tx_unmap()
973 					 tx_skb->size, DMA_TO_DEVICE);  in macb_tx_unmap()
974 		tx_skb->mapping = 0;  in macb_tx_unmap()
977 	if (tx_skb->skb) {  in macb_tx_unmap()
978 		napi_consume_skb(tx_skb->skb, budget);  in macb_tx_unmap()
979 		tx_skb->skb = NULL;  in macb_tx_unmap()
988 	if (bp->hw_dma_cap & HW_DMA_CAP_64B) {  in macb_set_addr()
990 		desc_64->addrh = upper_32_bits(addr);  in macb_set_addr()
998 	desc->addr = lower_32_bits(addr);  in macb_set_addr()
1007 	if (bp->hw_dma_cap & HW_DMA_CAP_64B) {  in macb_get_addr()
1009 		addr = ((u64)(desc_64->addrh) << 32);  in macb_get_addr()
1012 	addr |= MACB_BF(RX_WADDR, MACB_BFEXT(RX_WADDR, desc->addr));  in macb_get_addr()
1020 	struct macb		*bp = queue->bp;  in macb_tx_error_task()
1027 	netdev_vdbg(bp->dev, "macb_tx_error_task: q = %u, t = %u, h = %u\n",  in macb_tx_error_task()
1028 		    (unsigned int)(queue - bp->queues),  in macb_tx_error_task()
1029 		    queue->tx_tail, queue->tx_head);  in macb_tx_error_task()
1037 	napi_disable(&queue->napi_tx);  in macb_tx_error_task()
1038 	spin_lock_irqsave(&bp->lock, flags);  in macb_tx_error_task()
1041 	netif_tx_stop_all_queues(bp->dev);  in macb_tx_error_task()
1049 		netdev_err(bp->dev, "BUG: halt tx timed out\n");  in macb_tx_error_task()
1054 	for (tail = queue->tx_tail; tail != queue->tx_head; tail++) {  in macb_tx_error_task()
1058 		ctrl = desc->ctrl;  in macb_tx_error_task()
1060 		skb = tx_skb->skb;  in macb_tx_error_task()
1068 				skb = tx_skb->skb;  in macb_tx_error_task()
1075 				netdev_vdbg(bp->dev, "txerr skb %u (data %p) TX complete\n",  in macb_tx_error_task()
1077 					    skb->data);  in macb_tx_error_task()
1078 				bp->dev->stats.tx_packets++;  in macb_tx_error_task()
1079 				queue->stats.tx_packets++;  in macb_tx_error_task()
1080 				bp->dev->stats.tx_bytes += skb->len;  in macb_tx_error_task()
1081 				queue->stats.tx_bytes += skb->len;  in macb_tx_error_task()
1084 			/* "Buffers exhausted mid-frame" errors may only happen  in macb_tx_error_task()
1089 				netdev_err(bp->dev,  in macb_tx_error_task()
1090 					   "BUG: TX buffers exhausted mid-frame\n");  in macb_tx_error_task()
1092 			desc->ctrl = ctrl | MACB_BIT(TX_USED);  in macb_tx_error_task()
1101 	desc->ctrl = MACB_BIT(TX_USED);  in macb_tx_error_task()
1107 	queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma));  in macb_tx_error_task()
1109 	if (bp->hw_dma_cap & HW_DMA_CAP_64B)  in macb_tx_error_task()
1110 		queue_writel(queue, TBQPH, upper_32_bits(queue->tx_ring_dma));  in macb_tx_error_task()
1113 	queue->tx_head = 0;  in macb_tx_error_task()
1114 	queue->tx_tail = 0;  in macb_tx_error_task()
1121 	netif_tx_start_all_queues(bp->dev);  in macb_tx_error_task()
1124 	spin_unlock_irqrestore(&bp->lock, flags);  in macb_tx_error_task()
1125 	napi_enable(&queue->napi_tx);  in macb_tx_error_task()
1135 	if (likely(!(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)))  in ptp_one_step_sync()
1147 	if (hdr->flag_field[0] & PTP_FLAG_TWOSTEP)  in ptp_one_step_sync()
1160 	struct macb *bp = queue->bp;  in macb_tx_complete()
1161 	u16 queue_index = queue - bp->queues;  in macb_tx_complete()
1166 	spin_lock(&queue->tx_ptr_lock);  in macb_tx_complete()
1167 	head = queue->tx_head;  in macb_tx_complete()
1168 	for (tail = queue->tx_tail; tail != head && packets < budget; tail++) {  in macb_tx_complete()
1179 		ctrl = desc->ctrl;  in macb_tx_complete()
1190 			skb = tx_skb->skb;  in macb_tx_complete()
1194 				if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&  in macb_tx_complete()
1200 					tx_skb->skb = NULL;  in macb_tx_complete()
1202 				netdev_vdbg(bp->dev, "skb %u (data %p) TX complete\n",  in macb_tx_complete()
1204 					    skb->data);  in macb_tx_complete()
1205 				bp->dev->stats.tx_packets++;  in macb_tx_complete()
1206 				queue->stats.tx_packets++;  in macb_tx_complete()
1207 				bp->dev->stats.tx_bytes += skb->len;  in macb_tx_complete()
1208 				queue->stats.tx_bytes += skb->len;  in macb_tx_complete()
1224 	queue->tx_tail = tail;  in macb_tx_complete()
1225 	if (__netif_subqueue_stopped(bp->dev, queue_index) &&  in macb_tx_complete()
1226 	    CIRC_CNT(queue->tx_head, queue->tx_tail,  in macb_tx_complete()
1227 		     bp->tx_ring_size) <= MACB_TX_WAKEUP_THRESH(bp))  in macb_tx_complete()
1228 		netif_wake_subqueue(bp->dev, queue_index);  in macb_tx_complete()
1229 	spin_unlock(&queue->tx_ptr_lock);  in macb_tx_complete()
1239 	struct macb *bp = queue->bp;  in gem_rx_refill()
1242 	while (CIRC_SPACE(queue->rx_prepared_head, queue->rx_tail,  in gem_rx_refill()
1243 			bp->rx_ring_size) > 0) {  in gem_rx_refill()
1244 		entry = macb_rx_ring_wrap(bp, queue->rx_prepared_head);  in gem_rx_refill()
1251 		if (!queue->rx_skbuff[entry]) {  in gem_rx_refill()
1253 			skb = netdev_alloc_skb(bp->dev, bp->rx_buffer_size);  in gem_rx_refill()
1255 				netdev_err(bp->dev,  in gem_rx_refill()
1261 			paddr = dma_map_single(&bp->pdev->dev, skb->data,  in gem_rx_refill()
1262 					       bp->rx_buffer_size,  in gem_rx_refill()
1264 			if (dma_mapping_error(&bp->pdev->dev, paddr)) {  in gem_rx_refill()
1269 			queue->rx_skbuff[entry] = skb;  in gem_rx_refill()
1271 			if (entry == bp->rx_ring_size - 1)  in gem_rx_refill()
1273 			desc->ctrl = 0;  in gem_rx_refill()
1283 			desc->ctrl = 0;  in gem_rx_refill()
1285 			desc->addr &= ~MACB_BIT(RX_USED);  in gem_rx_refill()
1287 		queue->rx_prepared_head++;  in gem_rx_refill()
1293 	netdev_vdbg(bp->dev, "rx ring: queue: %p, prepared head %d, tail %d\n",  in gem_rx_refill()
1294 			queue, queue->rx_prepared_head, queue->rx_tail);  in gem_rx_refill()
1306 		desc->addr &= ~MACB_BIT(RX_USED);  in discard_partial_frame()
1321 	struct macb *bp = queue->bp;  in gem_rx()
1333 		entry = macb_rx_ring_wrap(bp, queue->rx_tail);  in gem_rx()
1339 		rxused = (desc->addr & MACB_BIT(RX_USED)) ? true : false;  in gem_rx()
1345 		/* Ensure ctrl is at least as up-to-date as rxused */  in gem_rx()
1348 		ctrl = desc->ctrl;  in gem_rx()
1350 		queue->rx_tail++;  in gem_rx()
1354 			netdev_err(bp->dev,  in gem_rx()
1356 			bp->dev->stats.rx_dropped++;  in gem_rx()
1357 			queue->stats.rx_dropped++;  in gem_rx()
1360 		skb = queue->rx_skbuff[entry];  in gem_rx()
1362 			netdev_err(bp->dev,  in gem_rx()
1364 			bp->dev->stats.rx_dropped++;  in gem_rx()
1365 			queue->stats.rx_dropped++;  in gem_rx()
1369 		queue->rx_skbuff[entry] = NULL;  in gem_rx()
1370 		len = ctrl & bp->rx_frm_len_mask;  in gem_rx()
1372 		netdev_vdbg(bp->dev, "gem_rx %u (len %u)\n", entry, len);  in gem_rx()
1375 		dma_unmap_single(&bp->pdev->dev, addr,  in gem_rx()
1376 				 bp->rx_buffer_size, DMA_FROM_DEVICE);  in gem_rx()
1378 		skb->protocol = eth_type_trans(skb, bp->dev);  in gem_rx()
1380 		if (bp->dev->features & NETIF_F_RXCSUM &&  in gem_rx()
1381 		    !(bp->dev->flags & IFF_PROMISC) &&  in gem_rx()
1383 			skb->ip_summed = CHECKSUM_UNNECESSARY;  in gem_rx()
1385 		bp->dev->stats.rx_packets++;  in gem_rx()
1386 		queue->stats.rx_packets++;  in gem_rx()
1387 		bp->dev->stats.rx_bytes += skb->len;  in gem_rx()
1388 		queue->stats.rx_bytes += skb->len;  in gem_rx()
1393 		netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n",  in gem_rx()
1394 			    skb->len, skb->csum);  in gem_rx()
1398 			       skb->data, 32, true);  in gem_rx()
1417 	struct macb *bp = queue->bp;  in macb_rx_frame()
1420 	len = desc->ctrl & bp->rx_frm_len_mask;  in macb_rx_frame()
1422 	netdev_vdbg(bp->dev, "macb_rx_frame frags %u - %u (len %u)\n",  in macb_rx_frame()
1428 	 * payload word-aligned.  in macb_rx_frame()
1434 	skb = netdev_alloc_skb(bp->dev, len + NET_IP_ALIGN);  in macb_rx_frame()
1436 		bp->dev->stats.rx_dropped++;  in macb_rx_frame()
1439 			desc->addr &= ~MACB_BIT(RX_USED);  in macb_rx_frame()
1456 		unsigned int frag_len = bp->rx_buffer_size;  in macb_rx_frame()
1461 				return -1;  in macb_rx_frame()
1463 			frag_len = len - offset;  in macb_rx_frame()
1468 		offset += bp->rx_buffer_size;  in macb_rx_frame()
1470 		desc->addr &= ~MACB_BIT(RX_USED);  in macb_rx_frame()
1480 	skb->protocol = eth_type_trans(skb, bp->dev);  in macb_rx_frame()
1482 	bp->dev->stats.rx_packets++;  in macb_rx_frame()
1483 	bp->dev->stats.rx_bytes += skb->len;  in macb_rx_frame()
1484 	netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n",  in macb_rx_frame()
1485 		    skb->len, skb->csum);  in macb_rx_frame()
1493 	struct macb *bp = queue->bp;  in macb_init_rx_ring()
1498 	addr = queue->rx_buffers_dma;  in macb_init_rx_ring()
1499 	for (i = 0; i < bp->rx_ring_size; i++) {  in macb_init_rx_ring()
1502 		desc->ctrl = 0;  in macb_init_rx_ring()
1503 		addr += bp->rx_buffer_size;  in macb_init_rx_ring()
1505 	desc->addr |= MACB_BIT(RX_WRAP);  in macb_init_rx_ring()
1506 	queue->rx_tail = 0;  in macb_init_rx_ring()
1512 	struct macb *bp = queue->bp;  in macb_rx()
1516 	int first_frag = -1;  in macb_rx()
1518 	for (tail = queue->rx_tail; budget > 0; tail++) {  in macb_rx()
1525 		if (!(desc->addr & MACB_BIT(RX_USED)))  in macb_rx()
1528 		/* Ensure ctrl is at least as up-to-date as addr */  in macb_rx()
1531 		ctrl = desc->ctrl;  in macb_rx()
1534 			if (first_frag != -1)  in macb_rx()
1542 			if (unlikely(first_frag == -1)) {  in macb_rx()
1548 			first_frag = -1;  in macb_rx()
1555 				budget--;  in macb_rx()
1564 		netdev_err(bp->dev, "RX queue corruption: reset it\n");  in macb_rx()
1566 		spin_lock_irqsave(&bp->lock, flags);  in macb_rx()
1572 		queue_writel(queue, RBQP, queue->rx_ring_dma);  in macb_rx()
1576 		spin_unlock_irqrestore(&bp->lock, flags);  in macb_rx()
1580 	if (first_frag != -1)  in macb_rx()
1581 		queue->rx_tail = first_frag;  in macb_rx()
1583 		queue->rx_tail = tail;  in macb_rx()
1590 	struct macb *bp = queue->bp;  in macb_rx_pending()
1594 	entry = macb_rx_ring_wrap(bp, queue->rx_tail);  in macb_rx_pending()
1600 	return (desc->addr & MACB_BIT(RX_USED)) != 0;  in macb_rx_pending()
1606 	struct macb *bp = queue->bp;  in macb_rx_poll()
1609 	work_done = bp->macbgem_ops.mog_rx(queue, napi, budget);  in macb_rx_poll()
1611 	netdev_vdbg(bp->dev, "RX poll: queue = %u, work_done = %d, budget = %d\n",  in macb_rx_poll()
1612 		    (unsigned int)(queue - bp->queues), work_done, budget);  in macb_rx_poll()
1615 		queue_writel(queue, IER, bp->rx_intr_mask);  in macb_rx_poll()
1621 		 * interrupts are re-enabled.  in macb_rx_poll()
1628 			queue_writel(queue, IDR, bp->rx_intr_mask);  in macb_rx_poll()
1629 			if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)  in macb_rx_poll()
1631 			netdev_vdbg(bp->dev, "poll: packets pending, reschedule\n");  in macb_rx_poll()
1643 	struct macb *bp = queue->bp;  in macb_tx_restart()
1646 	spin_lock(&queue->tx_ptr_lock);  in macb_tx_restart()
1648 	if (queue->tx_head == queue->tx_tail)  in macb_tx_restart()
1653 	head_idx = macb_adj_dma_desc_idx(bp, macb_tx_ring_wrap(bp, queue->tx_head));  in macb_tx_restart()
1658 	spin_lock_irq(&bp->lock);  in macb_tx_restart()
1660 	spin_unlock_irq(&bp->lock);  in macb_tx_restart()
1663 	spin_unlock(&queue->tx_ptr_lock);  in macb_tx_restart()
1670 	spin_lock(&queue->tx_ptr_lock);  in macb_tx_complete_pending()
1671 	if (queue->tx_head != queue->tx_tail) {  in macb_tx_complete_pending()
1675 		if (macb_tx_desc(queue, queue->tx_tail)->ctrl & MACB_BIT(TX_USED))  in macb_tx_complete_pending()
1678 	spin_unlock(&queue->tx_ptr_lock);  in macb_tx_complete_pending()
1685 	struct macb *bp = queue->bp;  in macb_tx_poll()
1691 	if (queue->txubr_pending) {  in macb_tx_poll()
1692 		queue->txubr_pending = false;  in macb_tx_poll()
1693 		netdev_vdbg(bp->dev, "poll: tx restart\n");  in macb_tx_poll()
1697 	netdev_vdbg(bp->dev, "TX poll: queue = %u, work_done = %d, budget = %d\n",  in macb_tx_poll()
1698 		    (unsigned int)(queue - bp->queues), work_done, budget);  in macb_tx_poll()
1707 		 * interrupts are re-enabled.  in macb_tx_poll()
1715 			if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)  in macb_tx_poll()
1717 			netdev_vdbg(bp->dev, "TX poll: packets pending, reschedule\n");  in macb_tx_poll()
1728 	struct net_device *dev = bp->dev;  in macb_hresp_error_task()
1733 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {  in macb_hresp_error_task()
1734 		queue_writel(queue, IDR, bp->rx_intr_mask |  in macb_hresp_error_task()
1745 	bp->macbgem_ops.mog_init_rings(bp);  in macb_hresp_error_task()
1751 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)  in macb_hresp_error_task()
1753 			     bp->rx_intr_mask |  in macb_hresp_error_task()
1767 	struct macb *bp = queue->bp;  in macb_wol_interrupt()
1775 	spin_lock(&bp->lock);  in macb_wol_interrupt()
1780 		netdev_vdbg(bp->dev, "MACB WoL: queue = %u, isr = 0x%08lx\n",  in macb_wol_interrupt()
1781 			    (unsigned int)(queue - bp->queues),  in macb_wol_interrupt()
1783 		if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)  in macb_wol_interrupt()
1785 		pm_wakeup_event(&bp->pdev->dev, 0);  in macb_wol_interrupt()
1788 	spin_unlock(&bp->lock);  in macb_wol_interrupt()
1796 	struct macb *bp = queue->bp;  in gem_wol_interrupt()
1804 	spin_lock(&bp->lock);  in gem_wol_interrupt()
1809 		netdev_vdbg(bp->dev, "GEM WoL: queue = %u, isr = 0x%08lx\n",  in gem_wol_interrupt()
1810 			    (unsigned int)(queue - bp->queues),  in gem_wol_interrupt()
1812 		if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)  in gem_wol_interrupt()
1814 		pm_wakeup_event(&bp->pdev->dev, 0);  in gem_wol_interrupt()
1817 	spin_unlock(&bp->lock);  in gem_wol_interrupt()
1825 	struct macb *bp = queue->bp;  in macb_interrupt()
1826 	struct net_device *dev = bp->dev;  in macb_interrupt()
1834 	spin_lock(&bp->lock);  in macb_interrupt()
1839 			queue_writel(queue, IDR, -1);  in macb_interrupt()
1840 			if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)  in macb_interrupt()
1841 				queue_writel(queue, ISR, -1);  in macb_interrupt()
1845 		netdev_vdbg(bp->dev, "queue = %u, isr = 0x%08lx\n",  in macb_interrupt()
1846 			    (unsigned int)(queue - bp->queues),  in macb_interrupt()
1849 		if (status & bp->rx_intr_mask) {  in macb_interrupt()
1856 			queue_writel(queue, IDR, bp->rx_intr_mask);  in macb_interrupt()
1857 			if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)  in macb_interrupt()
1860 			if (napi_schedule_prep(&queue->napi_rx)) {  in macb_interrupt()
1861 				netdev_vdbg(bp->dev, "scheduling RX softirq\n");  in macb_interrupt()
1862 				__napi_schedule(&queue->napi_rx);  in macb_interrupt()
1869 			if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)  in macb_interrupt()
1874 				queue->txubr_pending = true;  in macb_interrupt()
1878 			if (napi_schedule_prep(&queue->napi_tx)) {  in macb_interrupt()
1879 				netdev_vdbg(bp->dev, "scheduling TX softirq\n");  in macb_interrupt()
1880 				__napi_schedule(&queue->napi_tx);  in macb_interrupt()
1886 			schedule_work(&queue->tx_error_task);  in macb_interrupt()
1888 			if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)  in macb_interrupt()
1895 		 * add that if/when we get our hands on a full-blown MII PHY.  in macb_interrupt()
1900 		 * interrupts but it can be cleared by re-enabling RX. See  in macb_interrupt()
1911 			if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)  in macb_interrupt()
1918 				bp->hw_stats.gem.rx_overruns++;  in macb_interrupt()
1920 				bp->hw_stats.macb.rx_overruns++;  in macb_interrupt()
1922 			if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)  in macb_interrupt()
1927 			tasklet_schedule(&bp->hresp_err_tasklet);  in macb_interrupt()
1930 			if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)  in macb_interrupt()
1936 	spin_unlock(&bp->lock);  in macb_interrupt()
1942 /* Polling receive - used by netconsole and other diagnostic tools
1953 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)  in macb_poll_controller()
1954 		macb_interrupt(dev->irq, queue);  in macb_poll_controller()
1965 	unsigned int len, entry, i, tx_head = queue->tx_head;  in macb_tx_map()
1969 	unsigned int f, nr_frags = skb_shinfo(skb)->nr_frags;  in macb_tx_map()
1974 	if (skb_shinfo(skb)->gso_size != 0) {  in macb_tx_map()
1975 		if (ip_hdr(skb)->protocol == IPPROTO_UDP)  in macb_tx_map()
1976 			/* UDP - UFO */  in macb_tx_map()
1979 			/* TCP - TSO */  in macb_tx_map()
1983 	/* First, map non-paged data */  in macb_tx_map()
1992 		tx_skb = &queue->tx_skb[entry];  in macb_tx_map()
1994 		mapping = dma_map_single(&bp->pdev->dev,  in macb_tx_map()
1995 					 skb->data + offset,  in macb_tx_map()
1997 		if (dma_mapping_error(&bp->pdev->dev, mapping))  in macb_tx_map()
2001 		tx_skb->skb = NULL;  in macb_tx_map()
2002 		tx_skb->mapping = mapping;  in macb_tx_map()
2003 		tx_skb->size = size;  in macb_tx_map()
2004 		tx_skb->mapped_as_page = false;  in macb_tx_map()
2006 		len -= size;  in macb_tx_map()
2011 		size = min(len, bp->max_tx_length);  in macb_tx_map()
2016 		const skb_frag_t *frag = &skb_shinfo(skb)->frags[f];  in macb_tx_map()
2021 			size = min(len, bp->max_tx_length);  in macb_tx_map()
2023 			tx_skb = &queue->tx_skb[entry];  in macb_tx_map()
2025 			mapping = skb_frag_dma_map(&bp->pdev->dev, frag,  in macb_tx_map()
2027 			if (dma_mapping_error(&bp->pdev->dev, mapping))  in macb_tx_map()
2031 			tx_skb->skb = NULL;  in macb_tx_map()
2032 			tx_skb->mapping = mapping;  in macb_tx_map()
2033 			tx_skb->size = size;  in macb_tx_map()
2034 			tx_skb->mapped_as_page = true;  in macb_tx_map()
2036 			len -= size;  in macb_tx_map()
2045 		netdev_err(bp->dev, "BUG! empty skb!\n");  in macb_tx_map()
2050 	tx_skb->skb = skb;  in macb_tx_map()
2063 	desc->ctrl = ctrl;  in macb_tx_map()
2068 			mss_mfs = skb_shinfo(skb)->gso_size +  in macb_tx_map()
2072 			mss_mfs = skb_shinfo(skb)->gso_size;  in macb_tx_map()
2081 		i--;  in macb_tx_map()
2083 		tx_skb = &queue->tx_skb[entry];  in macb_tx_map()
2086 		ctrl = (u32)tx_skb->size;  in macb_tx_map()
2091 		if (unlikely(entry == (bp->tx_ring_size - 1)))  in macb_tx_map()
2095 		if (i == queue->tx_head) {  in macb_tx_map()
2098 			if ((bp->dev->features & NETIF_F_HW_CSUM) &&  in macb_tx_map()
2099 			    skb->ip_summed != CHECKSUM_PARTIAL && !lso_ctrl &&  in macb_tx_map()
2109 		macb_set_addr(bp, desc, tx_skb->mapping);  in macb_tx_map()
2110 		/* desc->addr must be visible to hardware before clearing  in macb_tx_map()
2111 		 * 'TX_USED' bit in desc->ctrl.  in macb_tx_map()
2114 		desc->ctrl = ctrl;  in macb_tx_map()
2115 	} while (i != queue->tx_head);  in macb_tx_map()
2117 	queue->tx_head = tx_head;  in macb_tx_map()
2122 	netdev_err(bp->dev, "TX DMA map failed\n");  in macb_tx_map()
2124 	for (i = queue->tx_head; i != tx_head; i++) {  in macb_tx_map()
2143 	if (!skb_is_nonlinear(skb) || (ip_hdr(skb)->protocol != IPPROTO_UDP))  in macb_features_check()
2153 	if (!IS_ALIGNED(skb_headlen(skb) - hdrlen, MACB_TX_LEN_ALIGN))  in macb_features_check()
2156 	nr_frags = skb_shinfo(skb)->nr_frags;  in macb_features_check()
2158 	nr_frags--;  in macb_features_check()
2160 		const skb_frag_t *frag = &skb_shinfo(skb)->frags[f];  in macb_features_check()
2171 	if (skb->ip_summed != CHECKSUM_PARTIAL)  in macb_clear_csum()
2176 		return -1;  in macb_clear_csum()
2179 	 * This is required - at least for Zynq, which otherwise calculates  in macb_clear_csum()
2182 	*(__sum16 *)(skb_checksum_start(skb) + skb->csum_offset) = 0;  in macb_clear_csum()
2190 	int padlen = ETH_ZLEN - (*skb)->len;  in macb_pad_and_fcs()
2196 	if (!(ndev->features & NETIF_F_HW_CSUM) ||  in macb_pad_and_fcs()
2197 	    !((*skb)->ip_summed != CHECKSUM_PARTIAL) ||  in macb_pad_and_fcs()
2198 	    skb_shinfo(*skb)->gso_size || ptp_one_step_sync(*skb))  in macb_pad_and_fcs()
2217 		(*skb)->data = memmove((*skb)->head, (*skb)->data, (*skb)->len);  in macb_pad_and_fcs()
2218 		skb_set_tail_pointer(*skb, (*skb)->len);  in macb_pad_and_fcs()
2222 			return -ENOMEM;  in macb_pad_and_fcs()
2229 		skb_put_zero(*skb, padlen - ETH_FCS_LEN);  in macb_pad_and_fcs()
2233 	fcs = crc32_le(~0, (*skb)->data, (*skb)->len);  in macb_pad_and_fcs()
2248 	struct macb_queue *queue = &bp->queues[queue_index];  in macb_start_xmit()
2264 	is_lso = (skb_shinfo(skb)->gso_size != 0);  in macb_start_xmit()
2268 		if (ip_hdr(skb)->protocol == IPPROTO_UDP)  in macb_start_xmit()
2274 			netdev_err(bp->dev, "Error - LSO headers fragmented!!!\n");  in macb_start_xmit()
2279 		hdrlen = min(skb_headlen(skb), bp->max_tx_length);  in macb_start_xmit()
2282 	netdev_vdbg(bp->dev,  in macb_start_xmit()
2284 		    queue_index, skb->len, skb->head, skb->data,  in macb_start_xmit()
2287 		       skb->data, 16, true);  in macb_start_xmit()
2296 		desc_cnt = DIV_ROUND_UP((skb_headlen(skb) - hdrlen), bp->max_tx_length) + 1;  in macb_start_xmit()
2298 		desc_cnt = DIV_ROUND_UP(skb_headlen(skb), bp->max_tx_length);  in macb_start_xmit()
2299 	nr_frags = skb_shinfo(skb)->nr_frags;  in macb_start_xmit()
2301 		frag_size = skb_frag_size(&skb_shinfo(skb)->frags[f]);  in macb_start_xmit()
2302 		desc_cnt += DIV_ROUND_UP(frag_size, bp->max_tx_length);  in macb_start_xmit()
2305 	spin_lock_bh(&queue->tx_ptr_lock);  in macb_start_xmit()
2308 	if (CIRC_SPACE(queue->tx_head, queue->tx_tail,  in macb_start_xmit()
2309 		       bp->tx_ring_size) < desc_cnt) {  in macb_start_xmit()
2311 		netdev_dbg(bp->dev, "tx_head = %u, tx_tail = %u\n",  in macb_start_xmit()
2312 			   queue->tx_head, queue->tx_tail);  in macb_start_xmit()
2327 	spin_lock_irq(&bp->lock);  in macb_start_xmit()
2329 	spin_unlock_irq(&bp->lock);  in macb_start_xmit()
2331 	if (CIRC_SPACE(queue->tx_head, queue->tx_tail, bp->tx_ring_size) < 1)  in macb_start_xmit()
2335 	spin_unlock_bh(&queue->tx_ptr_lock);  in macb_start_xmit()
2343 		bp->rx_buffer_size = MACB_RX_BUFFER_SIZE;  in macb_init_rx_buffer_size()
2345 		bp->rx_buffer_size = size;  in macb_init_rx_buffer_size()
2347 		if (bp->rx_buffer_size % RX_BUFFER_MULTIPLE) {  in macb_init_rx_buffer_size()
2348 			netdev_dbg(bp->dev,  in macb_init_rx_buffer_size()
2351 			bp->rx_buffer_size =  in macb_init_rx_buffer_size()
2352 				roundup(bp->rx_buffer_size, RX_BUFFER_MULTIPLE);  in macb_init_rx_buffer_size()
2356 	netdev_dbg(bp->dev, "mtu [%u] rx_buffer_size [%zu]\n",  in macb_init_rx_buffer_size()
2357 		   bp->dev->mtu, bp->rx_buffer_size);  in macb_init_rx_buffer_size()
2369 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {  in gem_free_rx_buffers()
2370 		if (!queue->rx_skbuff)  in gem_free_rx_buffers()
2373 		for (i = 0; i < bp->rx_ring_size; i++) {  in gem_free_rx_buffers()
2374 			skb = queue->rx_skbuff[i];  in gem_free_rx_buffers()
2382 			dma_unmap_single(&bp->pdev->dev, addr, bp->rx_buffer_size,  in gem_free_rx_buffers()
2388 		kfree(queue->rx_skbuff);  in gem_free_rx_buffers()
2389 		queue->rx_skbuff = NULL;  in gem_free_rx_buffers()
2395 	struct macb_queue *queue = &bp->queues[0];  in macb_free_rx_buffers()
2397 	if (queue->rx_buffers) {  in macb_free_rx_buffers()
2398 		dma_free_coherent(&bp->pdev->dev,  in macb_free_rx_buffers()
2399 				  bp->rx_ring_size * bp->rx_buffer_size,  in macb_free_rx_buffers()
2400 				  queue->rx_buffers, queue->rx_buffers_dma);  in macb_free_rx_buffers()
2401 		queue->rx_buffers = NULL;  in macb_free_rx_buffers()
2411 	bp->macbgem_ops.mog_free_rx_buffers(bp);  in macb_free_consistent()
2413 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {  in macb_free_consistent()
2414 		kfree(queue->tx_skb);  in macb_free_consistent()
2415 		queue->tx_skb = NULL;  in macb_free_consistent()
2416 		if (queue->tx_ring) {  in macb_free_consistent()
2417 			size = TX_RING_BYTES(bp) + bp->tx_bd_rd_prefetch;  in macb_free_consistent()
2418 			dma_free_coherent(&bp->pdev->dev, size,  in macb_free_consistent()
2419 					  queue->tx_ring, queue->tx_ring_dma);  in macb_free_consistent()
2420 			queue->tx_ring = NULL;  in macb_free_consistent()
2422 		if (queue->rx_ring) {  in macb_free_consistent()
2423 			size = RX_RING_BYTES(bp) + bp->rx_bd_rd_prefetch;  in macb_free_consistent()
2424 			dma_free_coherent(&bp->pdev->dev, size,  in macb_free_consistent()
2425 					  queue->rx_ring, queue->rx_ring_dma);  in macb_free_consistent()
2426 			queue->rx_ring = NULL;  in macb_free_consistent()
2437 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {  in gem_alloc_rx_buffers()
2438 		size = bp->rx_ring_size * sizeof(struct sk_buff *);  in gem_alloc_rx_buffers()
2439 		queue->rx_skbuff = kzalloc(size, GFP_KERNEL);  in gem_alloc_rx_buffers()
2440 		if (!queue->rx_skbuff)  in gem_alloc_rx_buffers()
2441 			return -ENOMEM;  in gem_alloc_rx_buffers()
2443 			netdev_dbg(bp->dev,  in gem_alloc_rx_buffers()
2445 				   bp->rx_ring_size, queue->rx_skbuff);  in gem_alloc_rx_buffers()
2452 	struct macb_queue *queue = &bp->queues[0];  in macb_alloc_rx_buffers()
2455 	size = bp->rx_ring_size * bp->rx_buffer_size;  in macb_alloc_rx_buffers()
2456 	queue->rx_buffers = dma_alloc_coherent(&bp->pdev->dev, size,  in macb_alloc_rx_buffers()
2457 					    &queue->rx_buffers_dma, GFP_KERNEL);  in macb_alloc_rx_buffers()
2458 	if (!queue->rx_buffers)  in macb_alloc_rx_buffers()
2459 		return -ENOMEM;  in macb_alloc_rx_buffers()
2461 	netdev_dbg(bp->dev,  in macb_alloc_rx_buffers()
2463 		   size, (unsigned long)queue->rx_buffers_dma, queue->rx_buffers);  in macb_alloc_rx_buffers()
2473 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {  in macb_alloc_consistent()
2474 		size = TX_RING_BYTES(bp) + bp->tx_bd_rd_prefetch;  in macb_alloc_consistent()
2475 		queue->tx_ring = dma_alloc_coherent(&bp->pdev->dev, size,  in macb_alloc_consistent()
2476 						    &queue->tx_ring_dma,  in macb_alloc_consistent()
2478 		if (!queue->tx_ring)  in macb_alloc_consistent()
2480 		netdev_dbg(bp->dev,  in macb_alloc_consistent()
2482 			   q, size, (unsigned long)queue->tx_ring_dma,  in macb_alloc_consistent()
2483 			   queue->tx_ring);  in macb_alloc_consistent()
2485 		size = bp->tx_ring_size * sizeof(struct macb_tx_skb);  in macb_alloc_consistent()
2486 		queue->tx_skb = kmalloc(size, GFP_KERNEL);  in macb_alloc_consistent()
2487 		if (!queue->tx_skb)  in macb_alloc_consistent()
2490 		size = RX_RING_BYTES(bp) + bp->rx_bd_rd_prefetch;  in macb_alloc_consistent()
2491 		queue->rx_ring = dma_alloc_coherent(&bp->pdev->dev, size,  in macb_alloc_consistent()
2492 						 &queue->rx_ring_dma, GFP_KERNEL);  in macb_alloc_consistent()
2493 		if (!queue->rx_ring)  in macb_alloc_consistent()
2495 		netdev_dbg(bp->dev,  in macb_alloc_consistent()
2497 			   size, (unsigned long)queue->rx_ring_dma, queue->rx_ring);  in macb_alloc_consistent()
2499 	if (bp->macbgem_ops.mog_alloc_rx_buffers(bp))  in macb_alloc_consistent()
2506 	return -ENOMEM;  in macb_alloc_consistent()
2516 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {  in gem_init_rings()
2517 		for (i = 0; i < bp->tx_ring_size; i++) {  in gem_init_rings()
2520 			desc->ctrl = MACB_BIT(TX_USED);  in gem_init_rings()
2522 		desc->ctrl |= MACB_BIT(TX_WRAP);  in gem_init_rings()
2523 		queue->tx_head = 0;  in gem_init_rings()
2524 		queue->tx_tail = 0;  in gem_init_rings()
2526 		queue->rx_tail = 0;  in gem_init_rings()
2527 		queue->rx_prepared_head = 0;  in gem_init_rings()
2539 	macb_init_rx_ring(&bp->queues[0]);  in macb_init_rings()
2541 	for (i = 0; i < bp->tx_ring_size; i++) {  in macb_init_rings()
2542 		desc = macb_tx_desc(&bp->queues[0], i);  in macb_init_rings()
2544 		desc->ctrl = MACB_BIT(TX_USED);  in macb_init_rings()
2546 	bp->queues[0].tx_head = 0;  in macb_init_rings()
2547 	bp->queues[0].tx_tail = 0;  in macb_init_rings()
2548 	desc->ctrl |= MACB_BIT(TX_WRAP);  in macb_init_rings()
2568 	macb_writel(bp, TSR, -1);  in macb_reset_hw()
2569 	macb_writel(bp, RSR, -1);  in macb_reset_hw()
2572 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {  in macb_reset_hw()
2573 		queue_writel(queue, IDR, -1);  in macb_reset_hw()
2575 		if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)  in macb_reset_hw()
2576 			queue_writel(queue, ISR, -1);  in macb_reset_hw()
2583 	unsigned long pclk_hz = clk_get_rate(bp->pclk);  in gem_mdc_clk_div()
2609 	pclk_hz = clk_get_rate(bp->pclk);  in macb_mdc_clk_div()
2643  * - use the correct receive buffer size
2644  * - set best burst length for DMA operations
2646  * - set both rx/tx packet buffers to full memory size
2656 	buffer_size = bp->rx_buffer_size / RX_BUFFER_MULTIPLE;  in macb_configure_dma()
2658 		dmacfg = gem_readl(bp, DMACFG) & ~GEM_BF(RXBS, -1L);  in macb_configure_dma()
2659 		for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {  in macb_configure_dma()
2665 		if (bp->dma_burst_length)  in macb_configure_dma()
2666 			dmacfg = GEM_BFINS(FBLDO, bp->dma_burst_length, dmacfg);  in macb_configure_dma()
2667 		dmacfg |= GEM_BIT(TXPBMS) | GEM_BF(RXBMS, -1L);  in macb_configure_dma()
2670 		if (bp->native_io)  in macb_configure_dma()
2675 		if (bp->dev->features & NETIF_F_HW_CSUM)  in macb_configure_dma()
2682 		if (bp->hw_dma_cap & HW_DMA_CAP_64B)  in macb_configure_dma()
2686 		if (bp->hw_dma_cap & HW_DMA_CAP_PTP)  in macb_configure_dma()
2689 		netdev_dbg(bp->dev, "Cadence configure DMA with 0x%08x\n",  in macb_configure_dma()
2705 	if (bp->caps & MACB_CAPS_JUMBO)  in macb_init_hw()
2709 	if (bp->dev->flags & IFF_PROMISC)  in macb_init_hw()
2711 	else if (macb_is_gem(bp) && bp->dev->features & NETIF_F_RXCSUM)  in macb_init_hw()
2713 	if (!(bp->dev->flags & IFF_BROADCAST))  in macb_init_hw()
2717 	if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len)  in macb_init_hw()
2718 		gem_writel(bp, JML, bp->jumbo_max_len);  in macb_init_hw()
2719 	bp->rx_frm_len_mask = MACB_RX_FRMLEN_MASK;  in macb_init_hw()
2720 	if (bp->caps & MACB_CAPS_JUMBO)  in macb_init_hw()
2721 		bp->rx_frm_len_mask = MACB_RX_JFRMLEN_MASK;  in macb_init_hw()
2782 /* Add multicast addresses to the internal multicast-hash table. */
2794 		bitnr = hash_get_index(ha->addr);  in macb_sethashtable()
2810 	if (dev->flags & IFF_PROMISC) {  in macb_set_rx_mode()
2822 		if (macb_is_gem(bp) && dev->features & NETIF_F_RXCSUM)  in macb_set_rx_mode()
2826 	if (dev->flags & IFF_ALLMULTI) {  in macb_set_rx_mode()
2828 		macb_or_gem_writel(bp, HRB, -1);  in macb_set_rx_mode()
2829 		macb_or_gem_writel(bp, HRT, -1);  in macb_set_rx_mode()
2835 	} else if (dev->flags & (~IFF_ALLMULTI)) {  in macb_set_rx_mode()
2847 	size_t bufsz = dev->mtu + ETH_HLEN + ETH_FCS_LEN + NET_IP_ALIGN;  in macb_open()
2853 	netdev_dbg(bp->dev, "open\n");  in macb_open()
2855 	err = pm_runtime_resume_and_get(&bp->pdev->dev);  in macb_open()
2869 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {  in macb_open()
2870 		napi_enable(&queue->napi_rx);  in macb_open()
2871 		napi_enable(&queue->napi_tx);  in macb_open()
2876 	err = phy_power_on(bp->sgmii_phy);  in macb_open()
2886 	if (bp->ptp_info)  in macb_open()
2887 		bp->ptp_info->ptp_init(dev);  in macb_open()
2892 	phy_power_off(bp->sgmii_phy);  in macb_open()
2896 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {  in macb_open()
2897 		napi_disable(&queue->napi_rx);  in macb_open()
2898 		napi_disable(&queue->napi_tx);  in macb_open()
2902 	pm_runtime_put_sync(&bp->pdev->dev);  in macb_open()
2915 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {  in macb_close()
2916 		napi_disable(&queue->napi_rx);  in macb_close()
2917 		napi_disable(&queue->napi_tx);  in macb_close()
2920 	phylink_stop(bp->phylink);  in macb_close()
2921 	phylink_disconnect_phy(bp->phylink);  in macb_close()
2923 	phy_power_off(bp->sgmii_phy);  in macb_close()
2925 	spin_lock_irqsave(&bp->lock, flags);  in macb_close()
2928 	spin_unlock_irqrestore(&bp->lock, flags);  in macb_close()
2932 	if (bp->ptp_info)  in macb_close()
2933 		bp->ptp_info->ptp_remove(dev);  in macb_close()
2935 	pm_runtime_put(&bp->pdev->dev);  in macb_close()
2943 		return -EBUSY;  in macb_change_mtu()
2945 	dev->mtu = new_mtu;  in macb_change_mtu()
2956 	u32 *p = &bp->hw_stats.gem.tx_octets_31_0;  in gem_update_stats()
2960 		u64 val = bp->macb_reg_readl(bp, offset);  in gem_update_stats()
2962 		bp->ethtool_stats[i] += val;  in gem_update_stats()
2967 			val = bp->macb_reg_readl(bp, offset + 4);  in gem_update_stats()
2968 			bp->ethtool_stats[i] += ((u64)val) << 32;  in gem_update_stats()
2974 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)  in gem_update_stats()
2975 		for (i = 0, stat = &queue->stats.first; i < QUEUE_STATS_LEN; ++i, ++stat)  in gem_update_stats()
2976 			bp->ethtool_stats[idx++] = *stat;  in gem_update_stats()
2981 	struct gem_stats *hwstat = &bp->hw_stats.gem;  in gem_get_stats()
2982 	struct net_device_stats *nstat = &bp->dev->stats;  in gem_get_stats()
2984 	if (!netif_running(bp->dev))  in gem_get_stats()
2989 	nstat->rx_errors = (hwstat->rx_frame_check_sequence_errors +  in gem_get_stats()
2990 			    hwstat->rx_alignment_errors +  in gem_get_stats()
2991 			    hwstat->rx_resource_errors +  in gem_get_stats()
2992 			    hwstat->rx_overruns +  in gem_get_stats()
2993 			    hwstat->rx_oversize_frames +  in gem_get_stats()
2994 			    hwstat->rx_jabbers +  in gem_get_stats()
2995 			    hwstat->rx_undersized_frames +  in gem_get_stats()
2996 			    hwstat->rx_length_field_frame_errors);  in gem_get_stats()
2997 	nstat->tx_errors = (hwstat->tx_late_collisions +  in gem_get_stats()
2998 			    hwstat->tx_excessive_collisions +  in gem_get_stats()
2999 			    hwstat->tx_underrun +  in gem_get_stats()
3000 			    hwstat->tx_carrier_sense_errors);  in gem_get_stats()
3001 	nstat->multicast = hwstat->rx_multicast_frames;  in gem_get_stats()
3002 	nstat->collisions = (hwstat->tx_single_collision_frames +  in gem_get_stats()
3003 			     hwstat->tx_multiple_collision_frames +  in gem_get_stats()
3004 			     hwstat->tx_excessive_collisions);  in gem_get_stats()
3005 	nstat->rx_length_errors = (hwstat->rx_oversize_frames +  in gem_get_stats()
3006 				   hwstat->rx_jabbers +  in gem_get_stats()
3007 				   hwstat->rx_undersized_frames +  in gem_get_stats()
3008 				   hwstat->rx_length_field_frame_errors);  in gem_get_stats()
3009 	nstat->rx_over_errors = hwstat->rx_resource_errors;  in gem_get_stats()
3010 	nstat->rx_crc_errors = hwstat->rx_frame_check_sequence_errors;  in gem_get_stats()
3011 	nstat->rx_frame_errors = hwstat->rx_alignment_errors;  in gem_get_stats()
3012 	nstat->rx_fifo_errors = hwstat->rx_overruns;  in gem_get_stats()
3013 	nstat->tx_aborted_errors = hwstat->tx_excessive_collisions;  in gem_get_stats()
3014 	nstat->tx_carrier_errors = hwstat->tx_carrier_sense_errors;  in gem_get_stats()
3015 	nstat->tx_fifo_errors = hwstat->tx_underrun;  in gem_get_stats()
3027 	memcpy(data, &bp->ethtool_stats, sizeof(u64)  in gem_get_ethtool_stats()
3037 		return GEM_STATS_LEN + bp->num_queues * QUEUE_STATS_LEN;  in gem_get_sset_count()
3039 		return -EOPNOTSUPP;  in gem_get_sset_count()
3057 		for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {  in gem_get_ethtool_strings()
3071 	struct net_device_stats *nstat = &bp->dev->stats;  in macb_get_stats()
3072 	struct macb_stats *hwstat = &bp->hw_stats.macb;  in macb_get_stats()
3081 	nstat->rx_errors = (hwstat->rx_fcs_errors +  in macb_get_stats()
3082 			    hwstat->rx_align_errors +  in macb_get_stats()
3083 			    hwstat->rx_resource_errors +  in macb_get_stats()
3084 			    hwstat->rx_overruns +  in macb_get_stats()
3085 			    hwstat->rx_oversize_pkts +  in macb_get_stats()
3086 			    hwstat->rx_jabbers +  in macb_get_stats()
3087 			    hwstat->rx_undersize_pkts +  in macb_get_stats()
3088 			    hwstat->rx_length_mismatch);  in macb_get_stats()
3089 	nstat->tx_errors = (hwstat->tx_late_cols +  in macb_get_stats()
3090 			    hwstat->tx_excessive_cols +  in macb_get_stats()
3091 			    hwstat->tx_underruns +  in macb_get_stats()
3092 			    hwstat->tx_carrier_errors +  in macb_get_stats()
3093 			    hwstat->sqe_test_errors);  in macb_get_stats()
3094 	nstat->collisions = (hwstat->tx_single_cols +  in macb_get_stats()
3095 			     hwstat->tx_multiple_cols +  in macb_get_stats()
3096 			     hwstat->tx_excessive_cols);  in macb_get_stats()
3097 	nstat->rx_length_errors = (hwstat->rx_oversize_pkts +  in macb_get_stats()
3098 				   hwstat->rx_jabbers +  in macb_get_stats()
3099 				   hwstat->rx_undersize_pkts +  in macb_get_stats()
3100 				   hwstat->rx_length_mismatch);  in macb_get_stats()
3101 	nstat->rx_over_errors = hwstat->rx_resource_errors +  in macb_get_stats()
3102 				   hwstat->rx_overruns;  in macb_get_stats()
3103 	nstat->rx_crc_errors = hwstat->rx_fcs_errors;  in macb_get_stats()
3104 	nstat->rx_frame_errors = hwstat->rx_align_errors;  in macb_get_stats()
3105 	nstat->rx_fifo_errors = hwstat->rx_overruns;  in macb_get_stats()
3107 	nstat->tx_aborted_errors = hwstat->tx_excessive_cols;  in macb_get_stats()
3108 	nstat->tx_carrier_errors = hwstat->tx_carrier_errors;  in macb_get_stats()
3109 	nstat->tx_fifo_errors = hwstat->tx_underruns;  in macb_get_stats()
3127 	regs->version = (macb_readl(bp, MID) & ((1 << MACB_REV_SIZE) - 1))  in macb_get_regs()
3130 	tail = macb_tx_ring_wrap(bp, bp->queues[0].tx_tail);  in macb_get_regs()
3131 	head = macb_tx_ring_wrap(bp, bp->queues[0].tx_head);  in macb_get_regs()
3144 	regs_buff[10] = macb_tx_dma(&bp->queues[0], tail);  in macb_get_regs()
3145 	regs_buff[11] = macb_tx_dma(&bp->queues[0], head);  in macb_get_regs()
3147 	if (!(bp->caps & MACB_CAPS_USRIO_DISABLED))  in macb_get_regs()
3157 	if (bp->wol & MACB_WOL_HAS_MAGIC_PACKET) {  in macb_get_wol()
3158 		phylink_ethtool_get_wol(bp->phylink, wol);  in macb_get_wol()
3159 		wol->supported |= WAKE_MAGIC;  in macb_get_wol()
3161 		if (bp->wol & MACB_WOL_ENABLED)  in macb_get_wol()
3162 			wol->wolopts |= WAKE_MAGIC;  in macb_get_wol()
3172 	ret = phylink_ethtool_set_wol(bp->phylink, wol);  in macb_set_wol()
3176 	if (!ret || ret != -EOPNOTSUPP)  in macb_set_wol()
3179 	if (!(bp->wol & MACB_WOL_HAS_MAGIC_PACKET) ||  in macb_set_wol()
3180 	    (wol->wolopts & ~WAKE_MAGIC))  in macb_set_wol()
3181 		return -EOPNOTSUPP;  in macb_set_wol()
3183 	if (wol->wolopts & WAKE_MAGIC)  in macb_set_wol()
3184 		bp->wol |= MACB_WOL_ENABLED;  in macb_set_wol()
3186 		bp->wol &= ~MACB_WOL_ENABLED;  in macb_set_wol()
3188 	device_set_wakeup_enable(&bp->pdev->dev, bp->wol & MACB_WOL_ENABLED);  in macb_set_wol()
3198 	return phylink_ethtool_ksettings_get(bp->phylink, kset);  in macb_get_link_ksettings()
3206 	return phylink_ethtool_ksettings_set(bp->phylink, kset);  in macb_set_link_ksettings()
3216 	ring->rx_max_pending = MAX_RX_RING_SIZE;  in macb_get_ringparam()
3217 	ring->tx_max_pending = MAX_TX_RING_SIZE;  in macb_get_ringparam()
3219 	ring->rx_pending = bp->rx_ring_size;  in macb_get_ringparam()
3220 	ring->tx_pending = bp->tx_ring_size;  in macb_get_ringparam()
3232 	if ((ring->rx_mini_pending) || (ring->rx_jumbo_pending))  in macb_set_ringparam()
3233 		return -EINVAL;  in macb_set_ringparam()
3235 	new_rx_size = clamp_t(u32, ring->rx_pending,  in macb_set_ringparam()
3239 	new_tx_size = clamp_t(u32, ring->tx_pending,  in macb_set_ringparam()
3243 	if ((new_tx_size == bp->tx_ring_size) &&  in macb_set_ringparam()
3244 	    (new_rx_size == bp->rx_ring_size)) {  in macb_set_ringparam()
3249 	if (netif_running(bp->dev)) {  in macb_set_ringparam()
3251 		macb_close(bp->dev);  in macb_set_ringparam()
3254 	bp->rx_ring_size = new_rx_size;  in macb_set_ringparam()
3255 	bp->tx_ring_size = new_tx_size;  in macb_set_ringparam()
3258 		macb_open(bp->dev);  in macb_set_ringparam()
3269 	tsu_clk = devm_clk_get(&bp->pdev->dev, "tsu_clk");  in gem_get_tsu_rate()
3273 	else if (!IS_ERR(bp->pclk)) {  in gem_get_tsu_rate()
3274 		tsu_clk = bp->pclk;  in gem_get_tsu_rate()
3277 		return -ENOTSUPP;  in gem_get_tsu_rate()
3291 	if ((bp->hw_dma_cap & HW_DMA_CAP_PTP) == 0) {  in gem_get_ts_info()
3296 	info->so_timestamping =  in gem_get_ts_info()
3303 	info->tx_types =  in gem_get_ts_info()
3307 	info->rx_filters =  in gem_get_ts_info()
3311 	info->phc_index = bp->ptp_clock ? ptp_clock_index(bp->ptp_clock) : -1;  in gem_get_ts_info()
3332 	if (bp->ptp_info)  in macb_get_ts_info()
3333 		return bp->ptp_info->get_ts_info(netdev, info);  in macb_get_ts_info()
3340 	struct net_device *netdev = bp->dev;  in gem_enable_flow_filters()
3345 	if (!(netdev->features & NETIF_F_NTUPLE))  in gem_enable_flow_filters()
3350 	list_for_each_entry(item, &bp->rx_fs_list.list, list) {  in gem_enable_flow_filters()
3351 		struct ethtool_rx_flow_spec *fs = &item->fs;  in gem_enable_flow_filters()
3354 		if (fs->location >= num_t2_scr)  in gem_enable_flow_filters()
3357 		t2_scr = gem_readl_n(bp, SCRT2, fs->location);  in gem_enable_flow_filters()
3363 		tp4sp_m = &(fs->m_u.tcp_ip4_spec);  in gem_enable_flow_filters()
3365 		if (enable && (tp4sp_m->ip4src == 0xFFFFFFFF))  in gem_enable_flow_filters()
3370 		if (enable && (tp4sp_m->ip4dst == 0xFFFFFFFF))  in gem_enable_flow_filters()
3375 		if (enable && ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF)))  in gem_enable_flow_filters()
3380 		gem_writel_n(bp, SCRT2, fs->location, t2_scr);  in gem_enable_flow_filters()
3387 	uint16_t index = fs->location;  in gem_prog_cmp_regs()
3396 	tp4sp_v = &(fs->h_u.tcp_ip4_spec);  in gem_prog_cmp_regs()
3397 	tp4sp_m = &(fs->m_u.tcp_ip4_spec);  in gem_prog_cmp_regs()
3400 	if (tp4sp_m->ip4src == 0xFFFFFFFF) {  in gem_prog_cmp_regs()
3401 		/* 1st compare reg - IP source address */  in gem_prog_cmp_regs()
3404 		w0 = tp4sp_v->ip4src;  in gem_prog_cmp_regs()
3405 		w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */  in gem_prog_cmp_regs()
3414 	if (tp4sp_m->ip4dst == 0xFFFFFFFF) {  in gem_prog_cmp_regs()
3415 		/* 2nd compare reg - IP destination address */  in gem_prog_cmp_regs()
3418 		w0 = tp4sp_v->ip4dst;  in gem_prog_cmp_regs()
3419 		w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */  in gem_prog_cmp_regs()
3428 	if ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF)) {  in gem_prog_cmp_regs()
3429 		/* 3rd compare reg - source port, destination port */  in gem_prog_cmp_regs()
3433 		if (tp4sp_m->psrc == tp4sp_m->pdst) {  in gem_prog_cmp_regs()
3434 			w0 = GEM_BFINS(T2MASK, tp4sp_v->psrc, w0);  in gem_prog_cmp_regs()
3435 			w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0);  in gem_prog_cmp_regs()
3436 			w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */  in gem_prog_cmp_regs()
3440 			w1 = GEM_BFINS(T2DISMSK, 0, w1); /* 16-bit compare */  in gem_prog_cmp_regs()
3442 			if (tp4sp_m->psrc == 0xFFFF) { /* src port */  in gem_prog_cmp_regs()
3443 				w0 = GEM_BFINS(T2CMP, tp4sp_v->psrc, w0);  in gem_prog_cmp_regs()
3446 				w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0);  in gem_prog_cmp_regs()
3456 	t2_scr = GEM_BFINS(QUEUE, (fs->ring_cookie) & 0xFF, t2_scr);  in gem_prog_cmp_regs()
3471 	struct ethtool_rx_flow_spec *fs = &cmd->fs;  in gem_add_flow_filter()
3474 	int ret = -EINVAL;  in gem_add_flow_filter()
3479 		return -ENOMEM;  in gem_add_flow_filter()
3480 	memcpy(&newfs->fs, fs, sizeof(newfs->fs));  in gem_add_flow_filter()
3484 			fs->flow_type, (int)fs->ring_cookie, fs->location,  in gem_add_flow_filter()
3485 			htonl(fs->h_u.tcp_ip4_spec.ip4src),  in gem_add_flow_filter()
3486 			htonl(fs->h_u.tcp_ip4_spec.ip4dst),  in gem_add_flow_filter()
3487 			be16_to_cpu(fs->h_u.tcp_ip4_spec.psrc),  in gem_add_flow_filter()
3488 			be16_to_cpu(fs->h_u.tcp_ip4_spec.pdst));  in gem_add_flow_filter()
3490 	spin_lock_irqsave(&bp->rx_fs_lock, flags);  in gem_add_flow_filter()
3493 	list_for_each_entry(item, &bp->rx_fs_list.list, list) {  in gem_add_flow_filter()
3494 		if (item->fs.location > newfs->fs.location) {  in gem_add_flow_filter()
3495 			list_add_tail(&newfs->list, &item->list);  in gem_add_flow_filter()
3498 		} else if (item->fs.location == fs->location) {  in gem_add_flow_filter()
3500 					fs->location);  in gem_add_flow_filter()
3501 			ret = -EBUSY;  in gem_add_flow_filter()
3506 		list_add_tail(&newfs->list, &bp->rx_fs_list.list);  in gem_add_flow_filter()
3509 	bp->rx_fs_list.count++;  in gem_add_flow_filter()
3513 	spin_unlock_irqrestore(&bp->rx_fs_lock, flags);  in gem_add_flow_filter()
3517 	spin_unlock_irqrestore(&bp->rx_fs_lock, flags);  in gem_add_flow_filter()
3530 	spin_lock_irqsave(&bp->rx_fs_lock, flags);  in gem_del_flow_filter()
3532 	list_for_each_entry(item, &bp->rx_fs_list.list, list) {  in gem_del_flow_filter()
3533 		if (item->fs.location == cmd->fs.location) {  in gem_del_flow_filter()
3535 			fs = &(item->fs);  in gem_del_flow_filter()
3538 					fs->flow_type, (int)fs->ring_cookie, fs->location,  in gem_del_flow_filter()
3539 					htonl(fs->h_u.tcp_ip4_spec.ip4src),  in gem_del_flow_filter()
3540 					htonl(fs->h_u.tcp_ip4_spec.ip4dst),  in gem_del_flow_filter()
3541 					be16_to_cpu(fs->h_u.tcp_ip4_spec.psrc),  in gem_del_flow_filter()
3542 					be16_to_cpu(fs->h_u.tcp_ip4_spec.pdst));  in gem_del_flow_filter()
3544 			gem_writel_n(bp, SCRT2, fs->location, 0);  in gem_del_flow_filter()
3546 			list_del(&item->list);  in gem_del_flow_filter()
3547 			bp->rx_fs_list.count--;  in gem_del_flow_filter()
3548 			spin_unlock_irqrestore(&bp->rx_fs_lock, flags);  in gem_del_flow_filter()
3554 	spin_unlock_irqrestore(&bp->rx_fs_lock, flags);  in gem_del_flow_filter()
3555 	return -EINVAL;  in gem_del_flow_filter()
3564 	list_for_each_entry(item, &bp->rx_fs_list.list, list) {  in gem_get_flow_entry()
3565 		if (item->fs.location == cmd->fs.location) {  in gem_get_flow_entry()
3566 			memcpy(&cmd->fs, &item->fs, sizeof(cmd->fs));  in gem_get_flow_entry()
3570 	return -EINVAL;  in gem_get_flow_entry()
3580 	list_for_each_entry(item, &bp->rx_fs_list.list, list) {  in gem_get_all_flow_entries()
3581 		if (cnt == cmd->rule_cnt)  in gem_get_all_flow_entries()
3582 			return -EMSGSIZE;  in gem_get_all_flow_entries()
3583 		rule_locs[cnt] = item->fs.location;  in gem_get_all_flow_entries()
3586 	cmd->data = bp->max_tuples;  in gem_get_all_flow_entries()
3587 	cmd->rule_cnt = cnt;  in gem_get_all_flow_entries()
3598 	switch (cmd->cmd) {  in gem_get_rxnfc()
3600 		cmd->data = bp->num_queues;  in gem_get_rxnfc()
3603 		cmd->rule_cnt = bp->rx_fs_list.count;  in gem_get_rxnfc()
3613 			  "Command parameter %d is not supported\n", cmd->cmd);  in gem_get_rxnfc()
3614 		ret = -EOPNOTSUPP;  in gem_get_rxnfc()
3625 	switch (cmd->cmd) {  in gem_set_rxnfc()
3627 		if ((cmd->fs.location >= bp->max_tuples)  in gem_set_rxnfc()
3628 				|| (cmd->fs.ring_cookie >= bp->num_queues)) {  in gem_set_rxnfc()
3629 			ret = -EINVAL;  in gem_set_rxnfc()
3639 			  "Command parameter %d is not supported\n", cmd->cmd);  in gem_set_rxnfc()
3640 		ret = -EOPNOTSUPP;  in gem_set_rxnfc()
3682 		return -EINVAL;  in macb_ioctl()
3684 	if (bp->ptp_info) {  in macb_ioctl()
3687 			return bp->ptp_info->set_hwtst(dev, rq, cmd);  in macb_ioctl()
3689 			return bp->ptp_info->get_hwtst(dev, rq);  in macb_ioctl()
3693 	return phylink_mii_ioctl(bp->phylink, rq, cmd);  in macb_ioctl()
3716 	struct net_device *netdev = bp->dev;  in macb_set_rxcsum_feature()
3723 	if ((features & NETIF_F_RXCSUM) && !(netdev->flags & IFF_PROMISC))  in macb_set_rxcsum_feature()
3744 	netdev_features_t changed = features ^ netdev->features;  in macb_set_features()
3763 	struct net_device *netdev = bp->dev;  in macb_restore_features()
3764 	netdev_features_t features = netdev->features;  in macb_restore_features()
3774 	list_for_each_entry(item, &bp->rx_fs_list.list, list)  in macb_restore_features()
3775 		gem_prog_cmp_regs(bp, &item->fs);  in macb_restore_features()
3806 		bp->caps = dt_conf->caps;  in macb_configure_caps()
3808 	if (hw_is_gem(bp->regs, bp->native_io)) {  in macb_configure_caps()
3809 		bp->caps |= MACB_CAPS_MACB_IS_GEM;  in macb_configure_caps()
3813 			bp->caps |= MACB_CAPS_ISR_CLEAR_ON_WRITE;  in macb_configure_caps()
3815 			bp->caps |= MACB_CAPS_PCS;  in macb_configure_caps()
3818 			bp->caps |= MACB_CAPS_HIGH_SPEED;  in macb_configure_caps()
3821 			bp->caps |= MACB_CAPS_FIFO_MODE;  in macb_configure_caps()
3825 				dev_err(&bp->pdev->dev,  in macb_configure_caps()
3828 				bp->hw_dma_cap |= HW_DMA_CAP_PTP;  in macb_configure_caps()
3829 				bp->ptp_info = &gem_ptp_info;  in macb_configure_caps()
3835 	dev_dbg(&bp->pdev->dev, "Cadence caps 0x%08x\n", bp->caps);  in macb_configure_caps()
3881 	pdata = dev_get_platdata(&pdev->dev);  in macb_clk_init()
3883 		*pclk = pdata->pclk;  in macb_clk_init()
3884 		*hclk = pdata->hclk;  in macb_clk_init()
3886 		*pclk = devm_clk_get(&pdev->dev, "pclk");  in macb_clk_init()
3887 		*hclk = devm_clk_get(&pdev->dev, "hclk");  in macb_clk_init()
3891 		return dev_err_probe(&pdev->dev,  in macb_clk_init()
3892 				     IS_ERR(*pclk) ? PTR_ERR(*pclk) : -ENODEV,  in macb_clk_init()
3896 		return dev_err_probe(&pdev->dev,  in macb_clk_init()
3897 				     IS_ERR(*hclk) ? PTR_ERR(*hclk) : -ENODEV,  in macb_clk_init()
3900 	*tx_clk = devm_clk_get_optional(&pdev->dev, "tx_clk");  in macb_clk_init()
3904 	*rx_clk = devm_clk_get_optional(&pdev->dev, "rx_clk");  in macb_clk_init()
3908 	*tsu_clk = devm_clk_get_optional(&pdev->dev, "tsu_clk");  in macb_clk_init()
3914 		dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err);  in macb_clk_init()
3920 		dev_err(&pdev->dev, "failed to enable hclk (%d)\n", err);  in macb_clk_init()
3926 		dev_err(&pdev->dev, "failed to enable tx_clk (%d)\n", err);  in macb_clk_init()
3932 		dev_err(&pdev->dev, "failed to enable rx_clk (%d)\n", err);  in macb_clk_init()
3938 		dev_err(&pdev->dev, "failed to enable tsu_clk (%d)\n", err);  in macb_clk_init()
3968 	bp->tx_ring_size = DEFAULT_TX_RING_SIZE;  in macb_init()
3969 	bp->rx_ring_size = DEFAULT_RX_RING_SIZE;  in macb_init()
3976 		if (!(bp->queue_mask & (1 << hw_q)))  in macb_init()
3979 		queue = &bp->queues[q];  in macb_init()
3980 		queue->bp = bp;  in macb_init()
3981 		spin_lock_init(&queue->tx_ptr_lock);  in macb_init()
3982 		netif_napi_add(dev, &queue->napi_rx, macb_rx_poll);  in macb_init()
3983 		netif_napi_add(dev, &queue->napi_tx, macb_tx_poll);  in macb_init()
3985 			queue->ISR  = GEM_ISR(hw_q - 1);  in macb_init()
3986 			queue->IER  = GEM_IER(hw_q - 1);  in macb_init()
3987 			queue->IDR  = GEM_IDR(hw_q - 1);  in macb_init()
3988 			queue->IMR  = GEM_IMR(hw_q - 1);  in macb_init()
3989 			queue->TBQP = GEM_TBQP(hw_q - 1);  in macb_init()
3990 			queue->RBQP = GEM_RBQP(hw_q - 1);  in macb_init()
3991 			queue->RBQS = GEM_RBQS(hw_q - 1);  in macb_init()
3993 			if (bp->hw_dma_cap & HW_DMA_CAP_64B) {  in macb_init()
3994 				queue->TBQPH = GEM_TBQPH(hw_q - 1);  in macb_init()
3995 				queue->RBQPH = GEM_RBQPH(hw_q - 1);  in macb_init()
4000 			queue->ISR  = MACB_ISR;  in macb_init()
4001 			queue->IER  = MACB_IER;  in macb_init()
4002 			queue->IDR  = MACB_IDR;  in macb_init()
4003 			queue->IMR  = MACB_IMR;  in macb_init()
4004 			queue->TBQP = MACB_TBQP;  in macb_init()
4005 			queue->RBQP = MACB_RBQP;  in macb_init()
4007 			if (bp->hw_dma_cap & HW_DMA_CAP_64B) {  in macb_init()
4008 				queue->TBQPH = MACB_TBQPH;  in macb_init()
4009 				queue->RBQPH = MACB_RBQPH;  in macb_init()
4019 		queue->irq = platform_get_irq(pdev, q);  in macb_init()
4020 		err = devm_request_irq(&pdev->dev, queue->irq, macb_interrupt,  in macb_init()
4021 				       IRQF_SHARED, dev->name, queue);  in macb_init()
4023 			dev_err(&pdev->dev,  in macb_init()
4025 				queue->irq, err);  in macb_init()
4029 		INIT_WORK(&queue->tx_error_task, macb_tx_error_task);  in macb_init()
4033 	dev->netdev_ops = &macb_netdev_ops;  in macb_init()
4037 		bp->max_tx_length = GEM_MAX_TX_LEN;  in macb_init()
4038 		bp->macbgem_ops.mog_alloc_rx_buffers = gem_alloc_rx_buffers;  in macb_init()
4039 		bp->macbgem_ops.mog_free_rx_buffers = gem_free_rx_buffers;  in macb_init()
4040 		bp->macbgem_ops.mog_init_rings = gem_init_rings;  in macb_init()
4041 		bp->macbgem_ops.mog_rx = gem_rx;  in macb_init()
4042 		dev->ethtool_ops = &gem_ethtool_ops;  in macb_init()
4044 		bp->max_tx_length = MACB_MAX_TX_LEN;  in macb_init()
4045 		bp->macbgem_ops.mog_alloc_rx_buffers = macb_alloc_rx_buffers;  in macb_init()
4046 		bp->macbgem_ops.mog_free_rx_buffers = macb_free_rx_buffers;  in macb_init()
4047 		bp->macbgem_ops.mog_init_rings = macb_init_rings;  in macb_init()
4048 		bp->macbgem_ops.mog_rx = macb_rx;  in macb_init()
4049 		dev->ethtool_ops = &macb_ethtool_ops;  in macb_init()
4053 	dev->hw_features = NETIF_F_SG;  in macb_init()
4057 		dev->hw_features |= MACB_NETIF_LSO;  in macb_init()
4060 	if (macb_is_gem(bp) && !(bp->caps & MACB_CAPS_FIFO_MODE))  in macb_init()
4061 		dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_RXCSUM;  in macb_init()
4062 	if (bp->caps & MACB_CAPS_SG_DISABLED)  in macb_init()
4063 		dev->hw_features &= ~NETIF_F_SG;  in macb_init()
4064 	dev->features = dev->hw_features;  in macb_init()
4068 	 * each 4-tuple define requires 1 T2 screener reg + 3 compare regs  in macb_init()
4071 	bp->max_tuples = min((GEM_BFEXT(SCR2CMP, reg) / 3),  in macb_init()
4073 	INIT_LIST_HEAD(&bp->rx_fs_list.list);  in macb_init()
4074 	if (bp->max_tuples > 0) {  in macb_init()
4082 			dev->hw_features |= NETIF_F_NTUPLE;  in macb_init()
4084 			bp->rx_fs_list.count = 0;  in macb_init()
4085 			spin_lock_init(&bp->rx_fs_lock);  in macb_init()
4087 			bp->max_tuples = 0;  in macb_init()
4090 	if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) {  in macb_init()
4092 		if (phy_interface_mode_is_rgmii(bp->phy_interface))  in macb_init()
4093 			val = bp->usrio->rgmii;  in macb_init()
4094 		else if (bp->phy_interface == PHY_INTERFACE_MODE_RMII &&  in macb_init()
4095 			 (bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII))  in macb_init()
4096 			val = bp->usrio->rmii;  in macb_init()
4097 		else if (!(bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII))  in macb_init()
4098 			val = bp->usrio->mii;  in macb_init()
4100 		if (bp->caps & MACB_CAPS_USRIO_HAS_CLKEN)  in macb_init()
4101 			val |= bp->usrio->refclk;  in macb_init()
4109 	if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII)  in macb_init()
4133 	struct macb_queue *q = &lp->queues[0];  in at91ether_alloc_coherent()
4135 	q->rx_ring = dma_alloc_coherent(&lp->pdev->dev,  in at91ether_alloc_coherent()
4138 					 &q->rx_ring_dma, GFP_KERNEL);  in at91ether_alloc_coherent()
4139 	if (!q->rx_ring)  in at91ether_alloc_coherent()
4140 		return -ENOMEM;  in at91ether_alloc_coherent()
4142 	q->rx_buffers = dma_alloc_coherent(&lp->pdev->dev,  in at91ether_alloc_coherent()
4145 					    &q->rx_buffers_dma, GFP_KERNEL);  in at91ether_alloc_coherent()
4146 	if (!q->rx_buffers) {  in at91ether_alloc_coherent()
4147 		dma_free_coherent(&lp->pdev->dev,  in at91ether_alloc_coherent()
4150 				  q->rx_ring, q->rx_ring_dma);  in at91ether_alloc_coherent()
4151 		q->rx_ring = NULL;  in at91ether_alloc_coherent()
4152 		return -ENOMEM;  in at91ether_alloc_coherent()
4160 	struct macb_queue *q = &lp->queues[0];  in at91ether_free_coherent()
4162 	if (q->rx_ring) {  in at91ether_free_coherent()
4163 		dma_free_coherent(&lp->pdev->dev,  in at91ether_free_coherent()
4166 				  q->rx_ring, q->rx_ring_dma);  in at91ether_free_coherent()
4167 		q->rx_ring = NULL;  in at91ether_free_coherent()
4170 	if (q->rx_buffers) {  in at91ether_free_coherent()
4171 		dma_free_coherent(&lp->pdev->dev,  in at91ether_free_coherent()
4174 				  q->rx_buffers, q->rx_buffers_dma);  in at91ether_free_coherent()
4175 		q->rx_buffers = NULL;  in at91ether_free_coherent()
4182 	struct macb_queue *q = &lp->queues[0];  in at91ether_start()
4192 	addr = q->rx_buffers_dma;  in at91ether_start()
4196 		desc->ctrl = 0;  in at91ether_start()
4201 	desc->addr |= MACB_BIT(RX_WRAP);  in at91ether_start()
4204 	q->rx_tail = 0;  in at91ether_start()
4207 	macb_writel(lp, RBQP, q->rx_ring_dma);  in at91ether_start()
4253 	ret = pm_runtime_resume_and_get(&lp->pdev->dev);  in at91ether_open()
4278 	pm_runtime_put_sync(&lp->pdev->dev);  in at91ether_open()
4289 	phylink_stop(lp->phylink);  in at91ether_close()
4290 	phylink_disconnect_phy(lp->phylink);  in at91ether_close()
4294 	return pm_runtime_put(&lp->pdev->dev);  in at91ether_close()
4309 		lp->rm9200_txq[desc].skb = skb;  in at91ether_start_xmit()
4310 		lp->rm9200_txq[desc].size = skb->len;  in at91ether_start_xmit()
4311 		lp->rm9200_txq[desc].mapping = dma_map_single(&lp->pdev->dev, skb->data,  in at91ether_start_xmit()
4312 							      skb->len, DMA_TO_DEVICE);  in at91ether_start_xmit()
4313 		if (dma_mapping_error(&lp->pdev->dev, lp->rm9200_txq[desc].mapping)) {  in at91ether_start_xmit()
4315 			dev->stats.tx_dropped++;  in at91ether_start_xmit()
4321 		macb_writel(lp, TAR, lp->rm9200_txq[desc].mapping);  in at91ether_start_xmit()
4323 		macb_writel(lp, TCR, skb->len);  in at91ether_start_xmit()
4339 	struct macb_queue *q = &lp->queues[0];  in at91ether_rx()
4345 	desc = macb_rx_desc(q, q->rx_tail);  in at91ether_rx()
4346 	while (desc->addr & MACB_BIT(RX_USED)) {  in at91ether_rx()
4347 		p_recv = q->rx_buffers + q->rx_tail * AT91ETHER_MAX_RBUFF_SZ;  in at91ether_rx()
4348 		pktlen = MACB_BF(RX_FRMLEN, desc->ctrl);  in at91ether_rx()
4354 			skb->protocol = eth_type_trans(skb, dev);  in at91ether_rx()
4355 			dev->stats.rx_packets++;  in at91ether_rx()
4356 			dev->stats.rx_bytes += pktlen;  in at91ether_rx()
4359 			dev->stats.rx_dropped++;  in at91ether_rx()
4362 		if (desc->ctrl & MACB_BIT(RX_MHASH_MATCH))  in at91ether_rx()
4363 			dev->stats.multicast++;  in at91ether_rx()
4366 		desc->addr &= ~MACB_BIT(RX_USED);  in at91ether_rx()
4369 		if (q->rx_tail == AT91ETHER_MAX_RX_DESCR - 1)  in at91ether_rx()
4370 			q->rx_tail = 0;  in at91ether_rx()
4372 			q->rx_tail++;  in at91ether_rx()
4374 		desc = macb_rx_desc(q, q->rx_tail);  in at91ether_rx()
4399 			dev->stats.tx_errors++;  in at91ether_interrupt()
4402 		if (lp->rm9200_txq[desc].skb) {  in at91ether_interrupt()
4403 			dev_consume_skb_irq(lp->rm9200_txq[desc].skb);  in at91ether_interrupt()
4404 			lp->rm9200_txq[desc].skb = NULL;  in at91ether_interrupt()
4405 			dma_unmap_single(&lp->pdev->dev, lp->rm9200_txq[desc].mapping,  in at91ether_interrupt()
4406 					 lp->rm9200_txq[desc].size, DMA_TO_DEVICE);  in at91ether_interrupt()
4407 			dev->stats.tx_packets++;  in at91ether_interrupt()
4408 			dev->stats.tx_bytes += lp->rm9200_txq[desc].size;  in at91ether_interrupt()
4413 	/* Work-around for EMAC Errata section 41.3.1 */  in at91ether_interrupt()
4433 	at91ether_interrupt(dev->irq, dev);  in at91ether_poll_controller()
4463 	*pclk = devm_clk_get(&pdev->dev, "ether_clk");  in at91ether_clk_init()
4469 		dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err);  in at91ether_clk_init()
4482 	bp->queues[0].bp = bp;  in at91ether_init()
4484 	dev->netdev_ops = &at91ether_netdev_ops;  in at91ether_init()
4485 	dev->ethtool_ops = &macb_ethtool_ops;  in at91ether_init()
4487 	err = devm_request_irq(&pdev->dev, dev->irq, at91ether_interrupt,  in at91ether_init()
4488 			       0, dev->name, dev);  in at91ether_init()
4502 	return mgmt->rate;  in fu540_macb_tx_recalc_rate()
4535 		iowrite32(1, mgmt->reg);  in fu540_macb_tx_set_rate()
4537 		iowrite32(0, mgmt->reg);  in fu540_macb_tx_set_rate()
4538 	mgmt->rate = rate;  in fu540_macb_tx_set_rate()
4560 	mgmt = devm_kzalloc(&pdev->dev, sizeof(*mgmt), GFP_KERNEL);  in fu540_c000_clk_init()
4562 		err = -ENOMEM;  in fu540_c000_clk_init()
4566 	init.name = "sifive-gemgxl-mgmt";  in fu540_c000_clk_init()
4571 	mgmt->rate = 0;  in fu540_c000_clk_init()
4572 	mgmt->hw.init = &init;  in fu540_c000_clk_init()
4574 	*tx_clk = devm_clk_register(&pdev->dev, &mgmt->hw);  in fu540_c000_clk_init()
4582 		dev_err(&pdev->dev, "failed to enable tx_clk (%u)\n", err);  in fu540_c000_clk_init()
4586 		dev_info(&pdev->dev, "Registered clk switch '%s'\n", init.name);  in fu540_c000_clk_init()
4599 	mgmt->reg = devm_platform_ioremap_resource(pdev, 1);  in fu540_c000_init()
4600 	if (IS_ERR(mgmt->reg))  in fu540_c000_init()
4601 		return PTR_ERR(mgmt->reg);  in fu540_c000_init()
4612 	if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) {  in init_reset_optional()
4614 		bp->sgmii_phy = devm_phy_optional_get(&pdev->dev, NULL);  in init_reset_optional()
4616 		if (IS_ERR(bp->sgmii_phy))  in init_reset_optional()
4617 			return dev_err_probe(&pdev->dev, PTR_ERR(bp->sgmii_phy),  in init_reset_optional()
4620 		ret = phy_init(bp->sgmii_phy);  in init_reset_optional()
4622 			return dev_err_probe(&pdev->dev, ret,  in init_reset_optional()
4630 		ret = of_property_read_u32_array(pdev->dev.of_node, "power-domains",  in init_reset_optional()
4633 			dev_err(&pdev->dev, "Failed to read power management information\n");  in init_reset_optional()
4646 	ret = device_reset_optional(&pdev->dev);  in init_reset_optional()
4648 		phy_exit(bp->sgmii_phy);  in init_reset_optional()
4649 		return dev_err_probe(&pdev->dev, ret, "failed to reset controller");  in init_reset_optional()
4656 		phy_exit(bp->sgmii_phy);  in init_reset_optional()
4810 	{ .compatible = "cdns,at91sam9260-macb", .data = &at91sam9260_config },
4812 	{ .compatible = "cdns,np4-macb", .data = &np4_config },
4813 	{ .compatible = "cdns,pc302-gem", .data = &pc302gem_config },
4815 	{ .compatible = "cdns,sam9x60-macb", .data = &at91sam9260_config },
4816 	{ .compatible = "atmel,sama5d2-gem", .data = &sama5d2_config },
4817 	{ .compatible = "atmel,sama5d29-gem", .data = &sama5d29_config },
4818 	{ .compatible = "atmel,sama5d3-gem", .data = &sama5d3_config },
4819 	{ .compatible = "atmel,sama5d3-macb", .data = &sama5d3macb_config },
4820 	{ .compatible = "atmel,sama5d4-gem", .data = &sama5d4_config },
4821 	{ .compatible = "cdns,at91rm9200-emac", .data = &emac_config },
4823 	{ .compatible = "cdns,zynqmp-gem", .data = &zynqmp_config}, /* deprecated */
4824 	{ .compatible = "cdns,zynq-gem", .data = &zynq_config }, /* deprecated */
4825 	{ .compatible = "sifive,fu540-c000-gem", .data = &fu540_c000_config },
4826 	{ .compatible = "microchip,mpfs-macb", .data = &mpfs_config },
4827 	{ .compatible = "microchip,sama7g5-gem", .data = &sama7g5_gem_config },
4828 	{ .compatible = "microchip,sama7g5-emac", .data = &sama7g5_emac_config },
4829 	{ .compatible = "xlnx,zynqmp-gem", .data = &zynqmp_config},
4830 	{ .compatible = "xlnx,zynq-gem", .data = &zynq_config },
4831 	{ .compatible = "xlnx,versal-gem", .data = &versal_config},
4853 			struct clk **) = macb_config->clk_init;  in macb_probe()
4854 	int (*init)(struct platform_device *) = macb_config->init;  in macb_probe()
4855 	struct device_node *np = pdev->dev.of_node;  in macb_probe()
4875 		if (match && match->data) {  in macb_probe()
4876 			macb_config = match->data;  in macb_probe()
4877 			clk_init = macb_config->clk_init;  in macb_probe()
4878 			init = macb_config->init;  in macb_probe()
4886 	pm_runtime_set_autosuspend_delay(&pdev->dev, MACB_PM_TIMEOUT);  in macb_probe()
4887 	pm_runtime_use_autosuspend(&pdev->dev);  in macb_probe()
4888 	pm_runtime_get_noresume(&pdev->dev);  in macb_probe()
4889 	pm_runtime_set_active(&pdev->dev);  in macb_probe()
4890 	pm_runtime_enable(&pdev->dev);  in macb_probe()
4896 		err = -ENOMEM;  in macb_probe()
4900 	dev->base_addr = regs->start;  in macb_probe()
4902 	SET_NETDEV_DEV(dev, &pdev->dev);  in macb_probe()
4905 	bp->pdev = pdev;  in macb_probe()
4906 	bp->dev = dev;  in macb_probe()
4907 	bp->regs = mem;  in macb_probe()
4908 	bp->native_io = native_io;  in macb_probe()
4910 		bp->macb_reg_readl = hw_readl_native;  in macb_probe()
4911 		bp->macb_reg_writel = hw_writel_native;  in macb_probe()
4913 		bp->macb_reg_readl = hw_readl;  in macb_probe()
4914 		bp->macb_reg_writel = hw_writel;  in macb_probe()
4916 	bp->num_queues = num_queues;  in macb_probe()
4917 	bp->queue_mask = queue_mask;  in macb_probe()
4919 		bp->dma_burst_length = macb_config->dma_burst_length;  in macb_probe()
4920 	bp->pclk = pclk;  in macb_probe()
4921 	bp->hclk = hclk;  in macb_probe()
4922 	bp->tx_clk = tx_clk;  in macb_probe()
4923 	bp->rx_clk = rx_clk;  in macb_probe()
4924 	bp->tsu_clk = tsu_clk;  in macb_probe()
4926 		bp->jumbo_max_len = macb_config->jumbo_max_len;  in macb_probe()
4928 	bp->wol = 0;  in macb_probe()
4929 	if (of_get_property(np, "magic-packet", NULL))  in macb_probe()
4930 		bp->wol |= MACB_WOL_HAS_MAGIC_PACKET;  in macb_probe()
4931 	device_set_wakeup_capable(&pdev->dev, bp->wol & MACB_WOL_HAS_MAGIC_PACKET);  in macb_probe()
4933 	bp->usrio = macb_config->usrio;  in macb_probe()
4935 	spin_lock_init(&bp->lock);  in macb_probe()
4942 		dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(44));  in macb_probe()
4943 		bp->hw_dma_cap |= HW_DMA_CAP_64B;  in macb_probe()
4948 	dev->irq = platform_get_irq(pdev, 0);  in macb_probe()
4949 	if (dev->irq < 0) {  in macb_probe()
4950 		err = dev->irq;  in macb_probe()
4954 	/* MTU range: 68 - 1500 or 10240 */  in macb_probe()
4955 	dev->min_mtu = GEM_MTU_MIN_SIZE;  in macb_probe()
4956 	if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len)  in macb_probe()
4957 		dev->max_mtu = bp->jumbo_max_len - ETH_HLEN - ETH_FCS_LEN;  in macb_probe()
4959 		dev->max_mtu = ETH_DATA_LEN;  in macb_probe()
4961 	if (bp->caps & MACB_CAPS_BD_RD_PREFETCH) {  in macb_probe()
4964 			bp->rx_bd_rd_prefetch = (2 << (val - 1)) *  in macb_probe()
4969 			bp->tx_bd_rd_prefetch = (2 << (val - 1)) *  in macb_probe()
4973 	bp->rx_intr_mask = MACB_RX_INT_FLAGS;  in macb_probe()
4974 	if (bp->caps & MACB_CAPS_NEEDS_RSTONUBR)  in macb_probe()
4975 		bp->rx_intr_mask |= MACB_BIT(RXUBR);  in macb_probe()
4977 	err = of_get_ethdev_address(np, bp->dev);  in macb_probe()
4978 	if (err == -EPROBE_DEFER)  in macb_probe()
4986 		bp->phy_interface = PHY_INTERFACE_MODE_MII;  in macb_probe()
4988 		bp->phy_interface = interface;  in macb_probe()
5003 		dev_err(&pdev->dev, "Cannot register net device, aborting.\n");  in macb_probe()
5007 	tasklet_setup(&bp->hresp_err_tasklet, macb_hresp_error_task);  in macb_probe()
5011 		    dev->base_addr, dev->irq, dev->dev_addr);  in macb_probe()
5013 	pm_runtime_mark_last_busy(&bp->pdev->dev);  in macb_probe()
5014 	pm_runtime_put_autosuspend(&bp->pdev->dev);  in macb_probe()
5019 	mdiobus_unregister(bp->mii_bus);  in macb_probe()
5020 	mdiobus_free(bp->mii_bus);  in macb_probe()
5023 	phy_exit(bp->sgmii_phy);  in macb_probe()
5030 	pm_runtime_disable(&pdev->dev);  in macb_probe()
5031 	pm_runtime_set_suspended(&pdev->dev);  in macb_probe()
5032 	pm_runtime_dont_use_autosuspend(&pdev->dev);  in macb_probe()
5046 		phy_exit(bp->sgmii_phy);  in macb_remove()
5047 		mdiobus_unregister(bp->mii_bus);  in macb_remove()
5048 		mdiobus_free(bp->mii_bus);  in macb_remove()
5051 		tasklet_kill(&bp->hresp_err_tasklet);  in macb_remove()
5052 		pm_runtime_disable(&pdev->dev);  in macb_remove()
5053 		pm_runtime_dont_use_autosuspend(&pdev->dev);  in macb_remove()
5054 		if (!pm_runtime_suspended(&pdev->dev)) {  in macb_remove()
5055 			macb_clks_disable(bp->pclk, bp->hclk, bp->tx_clk,  in macb_remove()
5056 					  bp->rx_clk, bp->tsu_clk);  in macb_remove()
5057 			pm_runtime_set_suspended(&pdev->dev);  in macb_remove()
5059 		phylink_destroy(bp->phylink);  in macb_remove()
5078 	if (bp->wol & MACB_WOL_ENABLED) {  in macb_suspend()
5079 		spin_lock_irqsave(&bp->lock, flags);  in macb_suspend()
5081 		macb_writel(bp, TSR, -1);  in macb_suspend()
5082 		macb_writel(bp, RSR, -1);  in macb_suspend()
5083 		for (q = 0, queue = bp->queues; q < bp->num_queues;  in macb_suspend()
5086 			queue_writel(queue, IDR, -1);  in macb_suspend()
5088 			if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)  in macb_suspend()
5089 				queue_writel(queue, ISR, -1);  in macb_suspend()
5094 		devm_free_irq(dev, bp->queues[0].irq, bp->queues);  in macb_suspend()
5096 			err = devm_request_irq(dev, bp->queues[0].irq, gem_wol_interrupt,  in macb_suspend()
5097 					       IRQF_SHARED, netdev->name, bp->queues);  in macb_suspend()
5101 					bp->queues[0].irq, err);  in macb_suspend()
5102 				spin_unlock_irqrestore(&bp->lock, flags);  in macb_suspend()
5105 			queue_writel(bp->queues, IER, GEM_BIT(WOL));  in macb_suspend()
5108 			err = devm_request_irq(dev, bp->queues[0].irq, macb_wol_interrupt,  in macb_suspend()
5109 					       IRQF_SHARED, netdev->name, bp->queues);  in macb_suspend()
5113 					bp->queues[0].irq, err);  in macb_suspend()
5114 				spin_unlock_irqrestore(&bp->lock, flags);  in macb_suspend()
5117 			queue_writel(bp->queues, IER, MACB_BIT(WOL));  in macb_suspend()
5120 		spin_unlock_irqrestore(&bp->lock, flags);  in macb_suspend()
5122 		enable_irq_wake(bp->queues[0].irq);  in macb_suspend()
5126 	for (q = 0, queue = bp->queues; q < bp->num_queues;  in macb_suspend()
5128 		napi_disable(&queue->napi_rx);  in macb_suspend()
5129 		napi_disable(&queue->napi_tx);  in macb_suspend()
5132 	if (!(bp->wol & MACB_WOL_ENABLED)) {  in macb_suspend()
5134 		phylink_stop(bp->phylink);  in macb_suspend()
5135 		phy_exit(bp->sgmii_phy);  in macb_suspend()
5137 		spin_lock_irqsave(&bp->lock, flags);  in macb_suspend()
5139 		spin_unlock_irqrestore(&bp->lock, flags);  in macb_suspend()
5142 	if (!(bp->caps & MACB_CAPS_USRIO_DISABLED))  in macb_suspend()
5143 		bp->pm_data.usrio = macb_or_gem_readl(bp, USRIO);  in macb_suspend()
5145 	if (netdev->hw_features & NETIF_F_NTUPLE)  in macb_suspend()
5146 		bp->pm_data.scrt2 = gem_readl_n(bp, ETHT, SCRT2_ETHT);  in macb_suspend()
5148 	if (bp->ptp_info)  in macb_suspend()
5149 		bp->ptp_info->ptp_remove(netdev);  in macb_suspend()
5171 	if (bp->wol & MACB_WOL_ENABLED) {  in macb_resume()
5172 		spin_lock_irqsave(&bp->lock, flags);  in macb_resume()
5175 			queue_writel(bp->queues, IDR, GEM_BIT(WOL));  in macb_resume()
5178 			queue_writel(bp->queues, IDR, MACB_BIT(WOL));  in macb_resume()
5182 		queue_readl(bp->queues, ISR);  in macb_resume()
5183 		if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)  in macb_resume()
5184 			queue_writel(bp->queues, ISR, -1);  in macb_resume()
5186 		devm_free_irq(dev, bp->queues[0].irq, bp->queues);  in macb_resume()
5187 		err = devm_request_irq(dev, bp->queues[0].irq, macb_interrupt,  in macb_resume()
5188 				       IRQF_SHARED, netdev->name, bp->queues);  in macb_resume()
5192 				bp->queues[0].irq, err);  in macb_resume()
5193 			spin_unlock_irqrestore(&bp->lock, flags);  in macb_resume()
5196 		spin_unlock_irqrestore(&bp->lock, flags);  in macb_resume()
5198 		disable_irq_wake(bp->queues[0].irq);  in macb_resume()
5204 		phylink_stop(bp->phylink);  in macb_resume()
5208 	for (q = 0, queue = bp->queues; q < bp->num_queues;  in macb_resume()
5210 		napi_enable(&queue->napi_rx);  in macb_resume()
5211 		napi_enable(&queue->napi_tx);  in macb_resume()
5214 	if (netdev->hw_features & NETIF_F_NTUPLE)  in macb_resume()
5215 		gem_writel_n(bp, ETHT, SCRT2_ETHT, bp->pm_data.scrt2);  in macb_resume()
5217 	if (!(bp->caps & MACB_CAPS_USRIO_DISABLED))  in macb_resume()
5218 		macb_or_gem_writel(bp, USRIO, bp->pm_data.usrio);  in macb_resume()
5225 	if (!device_may_wakeup(&bp->dev->dev))  in macb_resume()
5226 		phy_init(bp->sgmii_phy);  in macb_resume()
5228 	phylink_start(bp->phylink);  in macb_resume()
5232 	if (bp->ptp_info)  in macb_resume()
5233 		bp->ptp_info->ptp_init(netdev);  in macb_resume()
5244 		macb_clks_disable(bp->pclk, bp->hclk, bp->tx_clk, bp->rx_clk, bp->tsu_clk);  in macb_runtime_suspend()
5245 	else if (!(bp->caps & MACB_CAPS_NEED_TSUCLK))  in macb_runtime_suspend()
5246 		macb_clks_disable(NULL, NULL, NULL, NULL, bp->tsu_clk);  in macb_runtime_suspend()
5257 		clk_prepare_enable(bp->pclk);  in macb_runtime_resume()
5258 		clk_prepare_enable(bp->hclk);  in macb_runtime_resume()
5259 		clk_prepare_enable(bp->tx_clk);  in macb_runtime_resume()
5260 		clk_prepare_enable(bp->rx_clk);  in macb_runtime_resume()
5261 		clk_prepare_enable(bp->tsu_clk);  in macb_runtime_resume()
5262 	} else if (!(bp->caps & MACB_CAPS_NEED_TSUCLK)) {  in macb_runtime_resume()
5263 		clk_prepare_enable(bp->tsu_clk);  in macb_runtime_resume()