Lines Matching refs:fep
80 static int fec_enet_xdp_tx_xmit(struct fec_enet_private *fep,
384 struct fec_enet_private *fep = netdev_priv(ndev); in fec_dump() local
392 txq = fep->tx_queue[0]; in fec_dump()
432 fec_enet_create_page_pool(struct fec_enet_private *fep, in fec_enet_create_page_pool() argument
435 struct bpf_prog *xdp_prog = READ_ONCE(fep->xdp_prog); in fec_enet_create_page_pool()
440 .nid = dev_to_node(&fep->pdev->dev), in fec_enet_create_page_pool()
441 .dev = &fep->pdev->dev, in fec_enet_create_page_pool()
455 err = xdp_rxq_info_reg(&rxq->xdp_rxq, fep->netdev, rxq->id, 0); in fec_enet_create_page_pool()
479 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_txq_submit_frag_skb() local
505 if (fep->bufdesc_ex) { in fec_enet_txq_submit_frag_skb()
508 SKBTX_HW_TSTAMP && fep->hwts_tx_en)) in fec_enet_txq_submit_frag_skb()
513 if (fep->bufdesc_ex) { in fec_enet_txq_submit_frag_skb()
514 if (fep->quirks & FEC_QUIRK_HAS_AVB) in fec_enet_txq_submit_frag_skb()
526 if (((unsigned long) bufaddr) & fep->tx_align || in fec_enet_txq_submit_frag_skb()
527 fep->quirks & FEC_QUIRK_SWAP_FRAME) { in fec_enet_txq_submit_frag_skb()
531 if (fep->quirks & FEC_QUIRK_SWAP_FRAME) in fec_enet_txq_submit_frag_skb()
535 addr = dma_map_single(&fep->pdev->dev, bufaddr, frag_len, in fec_enet_txq_submit_frag_skb()
537 if (dma_mapping_error(&fep->pdev->dev, addr)) { in fec_enet_txq_submit_frag_skb()
557 dma_unmap_single(&fep->pdev->dev, fec32_to_cpu(bdp->cbd_bufaddr), in fec_enet_txq_submit_frag_skb()
566 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_txq_submit_skb() local
602 if (((unsigned long) bufaddr) & fep->tx_align || in fec_enet_txq_submit_skb()
603 fep->quirks & FEC_QUIRK_SWAP_FRAME) { in fec_enet_txq_submit_skb()
607 if (fep->quirks & FEC_QUIRK_SWAP_FRAME) in fec_enet_txq_submit_skb()
612 addr = dma_map_single(&fep->pdev->dev, bufaddr, buflen, DMA_TO_DEVICE); in fec_enet_txq_submit_skb()
613 if (dma_mapping_error(&fep->pdev->dev, addr)) { in fec_enet_txq_submit_skb()
623 dma_unmap_single(&fep->pdev->dev, addr, in fec_enet_txq_submit_skb()
630 if (fep->bufdesc_ex) { in fec_enet_txq_submit_skb()
633 SKBTX_HW_TSTAMP && fep->hwts_tx_en)) in fec_enet_txq_submit_skb()
640 if (fep->bufdesc_ex) { in fec_enet_txq_submit_skb()
645 fep->hwts_tx_en)) in fec_enet_txq_submit_skb()
648 if (fep->quirks & FEC_QUIRK_HAS_AVB) in fec_enet_txq_submit_skb()
694 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_txq_put_data_tso() local
705 if (((unsigned long) data) & fep->tx_align || in fec_enet_txq_put_data_tso()
706 fep->quirks & FEC_QUIRK_SWAP_FRAME) { in fec_enet_txq_put_data_tso()
710 if (fep->quirks & FEC_QUIRK_SWAP_FRAME) in fec_enet_txq_put_data_tso()
714 addr = dma_map_single(&fep->pdev->dev, data, size, DMA_TO_DEVICE); in fec_enet_txq_put_data_tso()
715 if (dma_mapping_error(&fep->pdev->dev, addr)) { in fec_enet_txq_put_data_tso()
725 if (fep->bufdesc_ex) { in fec_enet_txq_put_data_tso()
726 if (fep->quirks & FEC_QUIRK_HAS_AVB) in fec_enet_txq_put_data_tso()
739 if (fep->bufdesc_ex) in fec_enet_txq_put_data_tso()
753 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_txq_put_hdr_tso() local
767 if (((unsigned long)bufaddr) & fep->tx_align || in fec_enet_txq_put_hdr_tso()
768 fep->quirks & FEC_QUIRK_SWAP_FRAME) { in fec_enet_txq_put_hdr_tso()
772 if (fep->quirks & FEC_QUIRK_SWAP_FRAME) in fec_enet_txq_put_hdr_tso()
775 dmabuf = dma_map_single(&fep->pdev->dev, bufaddr, in fec_enet_txq_put_hdr_tso()
777 if (dma_mapping_error(&fep->pdev->dev, dmabuf)) { in fec_enet_txq_put_hdr_tso()
788 if (fep->bufdesc_ex) { in fec_enet_txq_put_hdr_tso()
789 if (fep->quirks & FEC_QUIRK_HAS_AVB) in fec_enet_txq_put_hdr_tso()
806 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_txq_submit_tso() local
872 if (!(fep->quirks & FEC_QUIRK_ERR007885) || in fec_enet_txq_submit_tso()
889 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_start_xmit() local
897 txq = fep->tx_queue[queue]; in fec_enet_start_xmit()
918 struct fec_enet_private *fep = netdev_priv(dev); in fec_enet_bd_init() local
925 for (q = 0; q < fep->num_rx_queues; q++) { in fec_enet_bd_init()
927 rxq = fep->rx_queue[q]; in fec_enet_bd_init()
947 for (q = 0; q < fep->num_tx_queues; q++) { in fec_enet_bd_init()
949 txq = fep->tx_queue[q]; in fec_enet_bd_init()
959 dma_unmap_single(&fep->pdev->dev, in fec_enet_bd_init()
967 dma_unmap_single(&fep->pdev->dev, in fec_enet_bd_init()
997 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_active_rxring() local
1000 for (i = 0; i < fep->num_rx_queues; i++) in fec_enet_active_rxring()
1001 writel(0, fep->rx_queue[i]->bd.reg_desc_active); in fec_enet_active_rxring()
1006 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_enable_ring() local
1011 for (i = 0; i < fep->num_rx_queues; i++) { in fec_enet_enable_ring()
1012 rxq = fep->rx_queue[i]; in fec_enet_enable_ring()
1013 writel(rxq->bd.dma, fep->hwp + FEC_R_DES_START(i)); in fec_enet_enable_ring()
1014 writel(PKT_MAXBUF_SIZE, fep->hwp + FEC_R_BUFF_SIZE(i)); in fec_enet_enable_ring()
1019 fep->hwp + FEC_RCMR(i)); in fec_enet_enable_ring()
1022 for (i = 0; i < fep->num_tx_queues; i++) { in fec_enet_enable_ring()
1023 txq = fep->tx_queue[i]; in fec_enet_enable_ring()
1024 writel(txq->bd.dma, fep->hwp + FEC_X_DES_START(i)); in fec_enet_enable_ring()
1029 fep->hwp + FEC_DMA_CFG(i)); in fec_enet_enable_ring()
1041 struct fec_enet_private *fep = netdev_priv(ndev); in fec_restart() local
1050 if (fep->quirks & FEC_QUIRK_HAS_MULTI_QUEUES || in fec_restart()
1051 ((fep->quirks & FEC_QUIRK_NO_HARD_RESET) && fep->link)) { in fec_restart()
1052 writel(0, fep->hwp + FEC_ECNTRL); in fec_restart()
1054 writel(1, fep->hwp + FEC_ECNTRL); in fec_restart()
1064 fep->hwp + FEC_ADDR_LOW); in fec_restart()
1066 fep->hwp + FEC_ADDR_HIGH); in fec_restart()
1069 writel((0xffffffff & ~FEC_ENET_MII), fep->hwp + FEC_IEVENT); in fec_restart()
1076 if (fep->full_duplex == DUPLEX_FULL) { in fec_restart()
1078 writel(0x04, fep->hwp + FEC_X_CNTRL); in fec_restart()
1082 writel(0x0, fep->hwp + FEC_X_CNTRL); in fec_restart()
1086 writel(fep->phy_speed, fep->hwp + FEC_MII_SPEED); in fec_restart()
1089 if (fep->quirks & FEC_QUIRK_HAS_RACC) { in fec_restart()
1090 u32 val = readl(fep->hwp + FEC_RACC); in fec_restart()
1094 if (fep->csum_flags & FLAG_RX_CSUM_ENABLED) in fec_restart()
1099 writel(val, fep->hwp + FEC_RACC); in fec_restart()
1100 writel(PKT_MAXBUF_SIZE, fep->hwp + FEC_FTRL); in fec_restart()
1108 if (fep->quirks & FEC_QUIRK_ENET_MAC) { in fec_restart()
1113 if (fep->phy_interface == PHY_INTERFACE_MODE_RGMII || in fec_restart()
1114 fep->phy_interface == PHY_INTERFACE_MODE_RGMII_ID || in fec_restart()
1115 fep->phy_interface == PHY_INTERFACE_MODE_RGMII_RXID || in fec_restart()
1116 fep->phy_interface == PHY_INTERFACE_MODE_RGMII_TXID) in fec_restart()
1118 else if (fep->phy_interface == PHY_INTERFACE_MODE_RMII) in fec_restart()
1134 if (fep->quirks & FEC_QUIRK_USE_GASKET) { in fec_restart()
1137 writel(0, fep->hwp + FEC_MIIGSK_ENR); in fec_restart()
1138 while (readl(fep->hwp + FEC_MIIGSK_ENR) & 4) in fec_restart()
1146 cfgr = (fep->phy_interface == PHY_INTERFACE_MODE_RMII) in fec_restart()
1150 writel(cfgr, fep->hwp + FEC_MIIGSK_CFGR); in fec_restart()
1153 writel(2, fep->hwp + FEC_MIIGSK_ENR); in fec_restart()
1160 if ((fep->pause_flag & FEC_PAUSE_FLAG_ENABLE) || in fec_restart()
1161 ((fep->pause_flag & FEC_PAUSE_FLAG_AUTONEG) && in fec_restart()
1166 writel(FEC_ENET_RSEM_V, fep->hwp + FEC_R_FIFO_RSEM); in fec_restart()
1167 writel(FEC_ENET_RSFL_V, fep->hwp + FEC_R_FIFO_RSFL); in fec_restart()
1168 writel(FEC_ENET_RAEM_V, fep->hwp + FEC_R_FIFO_RAEM); in fec_restart()
1169 writel(FEC_ENET_RAFL_V, fep->hwp + FEC_R_FIFO_RAFL); in fec_restart()
1172 writel(FEC_ENET_OPD_V, fep->hwp + FEC_OPD); in fec_restart()
1178 writel(rcntl, fep->hwp + FEC_R_CNTRL); in fec_restart()
1183 writel(0, fep->hwp + FEC_HASH_TABLE_HIGH); in fec_restart()
1184 writel(0, fep->hwp + FEC_HASH_TABLE_LOW); in fec_restart()
1187 if (fep->quirks & FEC_QUIRK_ENET_MAC) { in fec_restart()
1191 writel(1 << 8, fep->hwp + FEC_X_WMRK); in fec_restart()
1194 if (fep->bufdesc_ex) in fec_restart()
1197 if (fep->quirks & FEC_QUIRK_DELAYED_CLKS_SUPPORT && in fec_restart()
1198 fep->rgmii_txc_dly) in fec_restart()
1200 if (fep->quirks & FEC_QUIRK_DELAYED_CLKS_SUPPORT && in fec_restart()
1201 fep->rgmii_rxc_dly) in fec_restart()
1206 writel(0 << 31, fep->hwp + FEC_MIB_CTRLSTAT); in fec_restart()
1210 writel(ecntl, fep->hwp + FEC_ECNTRL); in fec_restart()
1213 if (fep->bufdesc_ex) in fec_restart()
1217 if (fep->link) in fec_restart()
1218 writel(FEC_DEFAULT_IMASK, fep->hwp + FEC_IMASK); in fec_restart()
1220 writel(0, fep->hwp + FEC_IMASK); in fec_restart()
1223 if (fep->quirks & FEC_QUIRK_HAS_COALESCE) in fec_restart()
1227 static int fec_enet_ipc_handle_init(struct fec_enet_private *fep) in fec_enet_ipc_handle_init() argument
1234 return imx_scu_get_handle(&fep->ipc_handle); in fec_enet_ipc_handle_init()
1237 static void fec_enet_ipg_stop_set(struct fec_enet_private *fep, bool enabled) in fec_enet_ipg_stop_set() argument
1239 struct device_node *np = fep->pdev->dev.of_node; in fec_enet_ipg_stop_set()
1243 if (!np || !fep->ipc_handle) in fec_enet_ipg_stop_set()
1252 imx_sc_misc_set_control(fep->ipc_handle, rsrc_id, IMX_SC_C_IPG_STOP, val); in fec_enet_ipg_stop_set()
1255 static void fec_enet_stop_mode(struct fec_enet_private *fep, bool enabled) in fec_enet_stop_mode() argument
1257 struct fec_platform_data *pdata = fep->pdev->dev.platform_data; in fec_enet_stop_mode()
1258 struct fec_stop_mode_gpr *stop_gpr = &fep->stop_gpr; in fec_enet_stop_mode()
1271 fec_enet_ipg_stop_set(fep, enabled); in fec_enet_stop_mode()
1277 struct fec_enet_private *fep = netdev_priv(ndev); in fec_irqs_disable() local
1279 writel(0, fep->hwp + FEC_IMASK); in fec_irqs_disable()
1284 struct fec_enet_private *fep = netdev_priv(ndev); in fec_irqs_disable_except_wakeup() local
1286 writel(0, fep->hwp + FEC_IMASK); in fec_irqs_disable_except_wakeup()
1287 writel(FEC_ENET_WAKEUP, fep->hwp + FEC_IMASK); in fec_irqs_disable_except_wakeup()
1293 struct fec_enet_private *fep = netdev_priv(ndev); in fec_stop() local
1294 u32 rmii_mode = readl(fep->hwp + FEC_R_CNTRL) & (1 << 8); in fec_stop()
1298 if (fep->link) { in fec_stop()
1299 writel(1, fep->hwp + FEC_X_CNTRL); /* Graceful transmit stop */ in fec_stop()
1301 if (!(readl(fep->hwp + FEC_IEVENT) & FEC_ENET_GRA)) in fec_stop()
1309 if (!(fep->wol_flag & FEC_WOL_FLAG_SLEEP_ON)) { in fec_stop()
1310 if (fep->quirks & FEC_QUIRK_HAS_MULTI_QUEUES) { in fec_stop()
1311 writel(0, fep->hwp + FEC_ECNTRL); in fec_stop()
1313 writel(1, fep->hwp + FEC_ECNTRL); in fec_stop()
1317 val = readl(fep->hwp + FEC_ECNTRL); in fec_stop()
1319 writel(val, fep->hwp + FEC_ECNTRL); in fec_stop()
1321 writel(fep->phy_speed, fep->hwp + FEC_MII_SPEED); in fec_stop()
1322 writel(FEC_DEFAULT_IMASK, fep->hwp + FEC_IMASK); in fec_stop()
1325 if (fep->quirks & FEC_QUIRK_ENET_MAC && in fec_stop()
1326 !(fep->wol_flag & FEC_WOL_FLAG_SLEEP_ON)) { in fec_stop()
1327 writel(2, fep->hwp + FEC_ECNTRL); in fec_stop()
1328 writel(rmii_mode, fep->hwp + FEC_R_CNTRL); in fec_stop()
1336 struct fec_enet_private *fep = netdev_priv(ndev); in fec_timeout() local
1342 schedule_work(&fep->tx_timeout_work); in fec_timeout()
1347 struct fec_enet_private *fep = in fec_enet_timeout_work() local
1349 struct net_device *ndev = fep->netdev; in fec_enet_timeout_work()
1353 napi_disable(&fep->napi); in fec_enet_timeout_work()
1358 napi_enable(&fep->napi); in fec_enet_timeout_work()
1364 fec_enet_hwtstamp(struct fec_enet_private *fep, unsigned ts, in fec_enet_hwtstamp() argument
1370 spin_lock_irqsave(&fep->tmreg_lock, flags); in fec_enet_hwtstamp()
1371 ns = timecounter_cyc2time(&fep->tc, ts); in fec_enet_hwtstamp()
1372 spin_unlock_irqrestore(&fep->tmreg_lock, flags); in fec_enet_hwtstamp()
1381 struct fec_enet_private *fep; in fec_enet_tx_queue() local
1393 fep = netdev_priv(ndev); in fec_enet_tx_queue()
1395 txq = fep->tx_queue[queue_id]; in fec_enet_tx_queue()
1416 dma_unmap_single(&fep->pdev->dev, in fec_enet_tx_queue()
1435 dma_unmap_single(&fep->pdev->dev, in fec_enet_tx_queue()
1488 fep->hwts_tx_en) && fep->bufdesc_ex) { in fec_enet_tx_queue()
1492 fec_enet_hwtstamp(fep, fec32_to_cpu(ebdp->ts), &shhwtstamps); in fec_enet_tx_queue()
1536 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_tx() local
1540 for (i = fep->num_tx_queues - 1; i >= 0; i--) in fec_enet_tx()
1560 fec_enet_run_xdp(struct fec_enet_private *fep, struct bpf_prog *prog, in fec_enet_run_xdp() argument
1585 err = xdp_do_redirect(fep->netdev, xdp, prog); in fec_enet_run_xdp()
1594 err = fec_enet_xdp_tx_xmit(fep, cpu, xdp, sync); in fec_enet_run_xdp()
1604 bpf_warn_invalid_xdp_action(fep->netdev, prog, act); in fec_enet_run_xdp()
1617 trace_xdp_exception(fep->netdev, prog, act); in fec_enet_run_xdp()
1632 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_rx_queue() local
1644 bool need_swap = fep->quirks & FEC_QUIRK_SWAP_FRAME; in fec_enet_rx_queue()
1645 struct bpf_prog *xdp_prog = READ_ONCE(fep->xdp_prog); in fec_enet_rx_queue()
1657 if (fep->quirks & FEC_QUIRK_HAS_RACC) { in fec_enet_rx_queue()
1666 rxq = fep->rx_queue[queue_id]; in fec_enet_rx_queue()
1680 writel(FEC_ENET_RXF_GET(queue_id), fep->hwp + FEC_IEVENT); in fec_enet_rx_queue()
1715 dma_sync_single_for_cpu(&fep->pdev->dev, in fec_enet_rx_queue()
1727 ret = fec_enet_run_xdp(fep, xdp_prog, &xdp, rxq, cpu); in fec_enet_rx_queue()
1758 if (fep->bufdesc_ex) in fec_enet_rx_queue()
1764 fep->bufdesc_ex && in fec_enet_rx_queue()
1780 if (fep->hwts_rx_en && fep->bufdesc_ex) in fec_enet_rx_queue()
1781 fec_enet_hwtstamp(fep, fec32_to_cpu(ebdp->ts), in fec_enet_rx_queue()
1784 if (fep->bufdesc_ex && in fec_enet_rx_queue()
1785 (fep->csum_flags & FLAG_RX_CSUM_ENABLED)) { in fec_enet_rx_queue()
1801 napi_gro_receive(&fep->napi, skb); in fec_enet_rx_queue()
1810 if (fep->bufdesc_ex) { in fec_enet_rx_queue()
1842 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_rx() local
1846 for (i = fep->num_rx_queues - 1; i >= 0; i--) in fec_enet_rx()
1852 static bool fec_enet_collect_events(struct fec_enet_private *fep) in fec_enet_collect_events() argument
1856 int_events = readl(fep->hwp + FEC_IEVENT); in fec_enet_collect_events()
1861 writel(int_events, fep->hwp + FEC_IEVENT); in fec_enet_collect_events()
1870 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_interrupt() local
1873 if (fec_enet_collect_events(fep) && fep->link) { in fec_enet_interrupt()
1876 if (napi_schedule_prep(&fep->napi)) { in fec_enet_interrupt()
1878 writel(0, fep->hwp + FEC_IMASK); in fec_enet_interrupt()
1879 __napi_schedule(&fep->napi); in fec_enet_interrupt()
1889 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_rx_napi() local
1895 } while ((done < budget) && fec_enet_collect_events(fep)); in fec_enet_rx_napi()
1899 writel(FEC_DEFAULT_IMASK, fep->hwp + FEC_IMASK); in fec_enet_rx_napi()
1908 struct fec_enet_private *fep = netdev_priv(ndev); in fec_get_mac() local
1924 struct device_node *np = fep->pdev->dev.of_node; in fec_get_mac()
1942 struct fec_platform_data *pdata = dev_get_platdata(&fep->pdev->dev); in fec_get_mac()
1954 cpu_to_be32(readl(fep->hwp + FEC_ADDR_LOW)); in fec_get_mac()
1956 cpu_to_be16(readl(fep->hwp + FEC_ADDR_HIGH) >> 16); in fec_get_mac()
1965 dev_err(&fep->pdev->dev, "Invalid MAC address: %pM\n", iap); in fec_get_mac()
1967 dev_info(&fep->pdev->dev, "Using random MAC address: %pM\n", in fec_get_mac()
1973 eth_hw_addr_gen(ndev, iap, iap == macaddr ? fep->dev_id : 0); in fec_get_mac()
1985 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_adjust_link() local
1995 fep->link = 0; in fec_enet_adjust_link()
1997 if (!fep->link) { in fec_enet_adjust_link()
1998 fep->link = phy_dev->link; in fec_enet_adjust_link()
2002 if (fep->full_duplex != phy_dev->duplex) { in fec_enet_adjust_link()
2003 fep->full_duplex = phy_dev->duplex; in fec_enet_adjust_link()
2007 if (phy_dev->speed != fep->speed) { in fec_enet_adjust_link()
2008 fep->speed = phy_dev->speed; in fec_enet_adjust_link()
2014 napi_disable(&fep->napi); in fec_enet_adjust_link()
2019 napi_enable(&fep->napi); in fec_enet_adjust_link()
2022 if (fep->link) { in fec_enet_adjust_link()
2023 napi_disable(&fep->napi); in fec_enet_adjust_link()
2027 napi_enable(&fep->napi); in fec_enet_adjust_link()
2028 fep->link = phy_dev->link; in fec_enet_adjust_link()
2037 static int fec_enet_mdio_wait(struct fec_enet_private *fep) in fec_enet_mdio_wait() argument
2042 ret = readl_poll_timeout_atomic(fep->hwp + FEC_IEVENT, ievent, in fec_enet_mdio_wait()
2046 writel(FEC_ENET_MII, fep->hwp + FEC_IEVENT); in fec_enet_mdio_wait()
2053 struct fec_enet_private *fep = bus->priv; in fec_enet_mdio_read_c22() local
2054 struct device *dev = &fep->pdev->dev; in fec_enet_mdio_read_c22()
2069 FEC_MMFR_TA, fep->hwp + FEC_MII_DATA); in fec_enet_mdio_read_c22()
2072 ret = fec_enet_mdio_wait(fep); in fec_enet_mdio_read_c22()
2074 netdev_err(fep->netdev, "MDIO read timeout\n"); in fec_enet_mdio_read_c22()
2078 ret = FEC_MMFR_DATA(readl(fep->hwp + FEC_MII_DATA)); in fec_enet_mdio_read_c22()
2090 struct fec_enet_private *fep = bus->priv; in fec_enet_mdio_read_c45() local
2091 struct device *dev = &fep->pdev->dev; in fec_enet_mdio_read_c45()
2104 fep->hwp + FEC_MII_DATA); in fec_enet_mdio_read_c45()
2107 ret = fec_enet_mdio_wait(fep); in fec_enet_mdio_read_c45()
2109 netdev_err(fep->netdev, "MDIO address write timeout\n"); in fec_enet_mdio_read_c45()
2118 FEC_MMFR_TA, fep->hwp + FEC_MII_DATA); in fec_enet_mdio_read_c45()
2121 ret = fec_enet_mdio_wait(fep); in fec_enet_mdio_read_c45()
2123 netdev_err(fep->netdev, "MDIO read timeout\n"); in fec_enet_mdio_read_c45()
2127 ret = FEC_MMFR_DATA(readl(fep->hwp + FEC_MII_DATA)); in fec_enet_mdio_read_c45()
2139 struct fec_enet_private *fep = bus->priv; in fec_enet_mdio_write_c22() local
2140 struct device *dev = &fep->pdev->dev; in fec_enet_mdio_write_c22()
2155 fep->hwp + FEC_MII_DATA); in fec_enet_mdio_write_c22()
2158 ret = fec_enet_mdio_wait(fep); in fec_enet_mdio_write_c22()
2160 netdev_err(fep->netdev, "MDIO write timeout\n"); in fec_enet_mdio_write_c22()
2171 struct fec_enet_private *fep = bus->priv; in fec_enet_mdio_write_c45() local
2172 struct device *dev = &fep->pdev->dev; in fec_enet_mdio_write_c45()
2185 fep->hwp + FEC_MII_DATA); in fec_enet_mdio_write_c45()
2188 ret = fec_enet_mdio_wait(fep); in fec_enet_mdio_write_c45()
2190 netdev_err(fep->netdev, "MDIO address write timeout\n"); in fec_enet_mdio_write_c45()
2198 fep->hwp + FEC_MII_DATA); in fec_enet_mdio_write_c45()
2201 ret = fec_enet_mdio_wait(fep); in fec_enet_mdio_write_c45()
2203 netdev_err(fep->netdev, "MDIO write timeout\n"); in fec_enet_mdio_write_c45()
2214 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_phy_reset_after_clk_enable() local
2219 } else if (fep->phy_node) { in fec_enet_phy_reset_after_clk_enable()
2227 phy_dev = of_phy_find_device(fep->phy_node); in fec_enet_phy_reset_after_clk_enable()
2235 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_clk_enable() local
2239 ret = clk_prepare_enable(fep->clk_enet_out); in fec_enet_clk_enable()
2243 if (fep->clk_ptp) { in fec_enet_clk_enable()
2244 mutex_lock(&fep->ptp_clk_mutex); in fec_enet_clk_enable()
2245 ret = clk_prepare_enable(fep->clk_ptp); in fec_enet_clk_enable()
2247 mutex_unlock(&fep->ptp_clk_mutex); in fec_enet_clk_enable()
2250 fep->ptp_clk_on = true; in fec_enet_clk_enable()
2252 mutex_unlock(&fep->ptp_clk_mutex); in fec_enet_clk_enable()
2255 ret = clk_prepare_enable(fep->clk_ref); in fec_enet_clk_enable()
2259 ret = clk_prepare_enable(fep->clk_2x_txclk); in fec_enet_clk_enable()
2265 clk_disable_unprepare(fep->clk_enet_out); in fec_enet_clk_enable()
2266 if (fep->clk_ptp) { in fec_enet_clk_enable()
2267 mutex_lock(&fep->ptp_clk_mutex); in fec_enet_clk_enable()
2268 clk_disable_unprepare(fep->clk_ptp); in fec_enet_clk_enable()
2269 fep->ptp_clk_on = false; in fec_enet_clk_enable()
2270 mutex_unlock(&fep->ptp_clk_mutex); in fec_enet_clk_enable()
2272 clk_disable_unprepare(fep->clk_ref); in fec_enet_clk_enable()
2273 clk_disable_unprepare(fep->clk_2x_txclk); in fec_enet_clk_enable()
2279 if (fep->clk_ref) in fec_enet_clk_enable()
2280 clk_disable_unprepare(fep->clk_ref); in fec_enet_clk_enable()
2282 if (fep->clk_ptp) { in fec_enet_clk_enable()
2283 mutex_lock(&fep->ptp_clk_mutex); in fec_enet_clk_enable()
2284 clk_disable_unprepare(fep->clk_ptp); in fec_enet_clk_enable()
2285 fep->ptp_clk_on = false; in fec_enet_clk_enable()
2286 mutex_unlock(&fep->ptp_clk_mutex); in fec_enet_clk_enable()
2289 clk_disable_unprepare(fep->clk_enet_out); in fec_enet_clk_enable()
2294 static int fec_enet_parse_rgmii_delay(struct fec_enet_private *fep, in fec_enet_parse_rgmii_delay() argument
2302 dev_err(&fep->pdev->dev, "The only allowed RGMII TX delay values are: 0ps, 2000ps"); in fec_enet_parse_rgmii_delay()
2305 fep->rgmii_txc_dly = true; in fec_enet_parse_rgmii_delay()
2312 dev_err(&fep->pdev->dev, "The only allowed RGMII RX delay values are: 0ps, 2000ps"); in fec_enet_parse_rgmii_delay()
2315 fep->rgmii_rxc_dly = true; in fec_enet_parse_rgmii_delay()
2324 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_mii_probe() local
2329 int dev_id = fep->dev_id; in fec_enet_mii_probe()
2331 if (fep->phy_node) { in fec_enet_mii_probe()
2332 phy_dev = of_phy_connect(ndev, fep->phy_node, in fec_enet_mii_probe()
2334 fep->phy_interface); in fec_enet_mii_probe()
2342 if (!mdiobus_is_registered_device(fep->mii_bus, phy_id)) in fec_enet_mii_probe()
2346 strscpy(mdio_bus_id, fep->mii_bus->id, MII_BUS_ID_SIZE); in fec_enet_mii_probe()
2359 fep->phy_interface); in fec_enet_mii_probe()
2368 if (fep->quirks & FEC_QUIRK_HAS_GBIT) { in fec_enet_mii_probe()
2379 fep->link = 0; in fec_enet_mii_probe()
2380 fep->full_duplex = 0; in fec_enet_mii_probe()
2393 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_mii_init() local
2416 if ((fep->quirks & FEC_QUIRK_SINGLE_MDIO) && fep->dev_id > 0) { in fec_enet_mii_init()
2419 fep->mii_bus = fec0_mii_bus; in fec_enet_mii_init()
2442 mii_speed = DIV_ROUND_UP(clk_get_rate(fep->clk_ipg), bus_freq * 2); in fec_enet_mii_init()
2443 if (fep->quirks & FEC_QUIRK_ENET_MAC) in fec_enet_mii_init()
2448 clk_get_rate(fep->clk_ipg)); in fec_enet_mii_init()
2465 holdtime = DIV_ROUND_UP(clk_get_rate(fep->clk_ipg), 100000000) - 1; in fec_enet_mii_init()
2467 fep->phy_speed = mii_speed << 1 | holdtime << 8; in fec_enet_mii_init()
2470 fep->phy_speed |= BIT(7); in fec_enet_mii_init()
2472 if (fep->quirks & FEC_QUIRK_CLEAR_SETUP_MII) { in fec_enet_mii_init()
2481 writel(0, fep->hwp + FEC_MII_DATA); in fec_enet_mii_init()
2484 writel(fep->phy_speed, fep->hwp + FEC_MII_SPEED); in fec_enet_mii_init()
2487 writel(FEC_ENET_MII, fep->hwp + FEC_IEVENT); in fec_enet_mii_init()
2489 fep->mii_bus = mdiobus_alloc(); in fec_enet_mii_init()
2490 if (fep->mii_bus == NULL) { in fec_enet_mii_init()
2495 fep->mii_bus->name = "fec_enet_mii_bus"; in fec_enet_mii_init()
2496 fep->mii_bus->read = fec_enet_mdio_read_c22; in fec_enet_mii_init()
2497 fep->mii_bus->write = fec_enet_mdio_write_c22; in fec_enet_mii_init()
2498 if (fep->quirks & FEC_QUIRK_HAS_MDIO_C45) { in fec_enet_mii_init()
2499 fep->mii_bus->read_c45 = fec_enet_mdio_read_c45; in fec_enet_mii_init()
2500 fep->mii_bus->write_c45 = fec_enet_mdio_write_c45; in fec_enet_mii_init()
2502 snprintf(fep->mii_bus->id, MII_BUS_ID_SIZE, "%s-%x", in fec_enet_mii_init()
2503 pdev->name, fep->dev_id + 1); in fec_enet_mii_init()
2504 fep->mii_bus->priv = fep; in fec_enet_mii_init()
2505 fep->mii_bus->parent = &pdev->dev; in fec_enet_mii_init()
2507 err = of_mdiobus_register(fep->mii_bus, node); in fec_enet_mii_init()
2515 if (fep->quirks & FEC_QUIRK_SINGLE_MDIO) in fec_enet_mii_init()
2516 fec0_mii_bus = fep->mii_bus; in fec_enet_mii_init()
2521 mdiobus_free(fep->mii_bus); in fec_enet_mii_init()
2527 static void fec_enet_mii_remove(struct fec_enet_private *fep) in fec_enet_mii_remove() argument
2530 mdiobus_unregister(fep->mii_bus); in fec_enet_mii_remove()
2531 mdiobus_free(fep->mii_bus); in fec_enet_mii_remove()
2538 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_get_drvinfo() local
2540 strscpy(info->driver, fep->pdev->dev.driver->name, in fec_enet_get_drvinfo()
2547 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_get_regs_len() local
2551 r = platform_get_resource(fep->pdev, IORESOURCE_MEM, 0); in fec_enet_get_regs_len()
2635 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_get_regs() local
2636 u32 __iomem *theregs = (u32 __iomem *)fep->hwp; in fec_enet_get_regs()
2637 struct device *dev = &fep->pdev->dev; in fec_enet_get_regs()
2671 !(fep->quirks & FEC_QUIRK_HAS_FRREG)) in fec_enet_get_regs()
2685 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_get_ts_info() local
2687 if (fep->bufdesc_ex) { in fec_enet_get_ts_info()
2695 if (fep->ptp_clock) in fec_enet_get_ts_info()
2696 info->phc_index = ptp_clock_index(fep->ptp_clock); in fec_enet_get_ts_info()
2716 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_get_pauseparam() local
2718 pause->autoneg = (fep->pause_flag & FEC_PAUSE_FLAG_AUTONEG) != 0; in fec_enet_get_pauseparam()
2719 pause->tx_pause = (fep->pause_flag & FEC_PAUSE_FLAG_ENABLE) != 0; in fec_enet_get_pauseparam()
2726 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_set_pauseparam() local
2737 fep->pause_flag = 0; in fec_enet_set_pauseparam()
2740 fep->pause_flag |= pause->rx_pause ? FEC_PAUSE_FLAG_ENABLE : 0; in fec_enet_set_pauseparam()
2741 fep->pause_flag |= pause->autoneg ? FEC_PAUSE_FLAG_AUTONEG : 0; in fec_enet_set_pauseparam()
2752 napi_disable(&fep->napi); in fec_enet_set_pauseparam()
2757 napi_enable(&fep->napi); in fec_enet_set_pauseparam()
2843 struct fec_enet_private *fep = netdev_priv(dev); in fec_enet_update_ethtool_stats() local
2847 fep->ethtool_stats[i] = readl(fep->hwp + fec_stats[i].offset); in fec_enet_update_ethtool_stats()
2850 static void fec_enet_get_xdp_stats(struct fec_enet_private *fep, u64 *data) in fec_enet_get_xdp_stats() argument
2856 for (i = fep->num_rx_queues - 1; i >= 0; i--) { in fec_enet_get_xdp_stats()
2857 rxq = fep->rx_queue[i]; in fec_enet_get_xdp_stats()
2866 static void fec_enet_page_pool_stats(struct fec_enet_private *fep, u64 *data) in fec_enet_page_pool_stats() argument
2873 for (i = fep->num_rx_queues - 1; i >= 0; i--) { in fec_enet_page_pool_stats()
2874 rxq = fep->rx_queue[i]; in fec_enet_page_pool_stats()
2889 struct fec_enet_private *fep = netdev_priv(dev); in fec_enet_get_ethtool_stats() local
2894 memcpy(data, fep->ethtool_stats, FEC_STATS_SIZE); in fec_enet_get_ethtool_stats()
2897 fec_enet_get_xdp_stats(fep, data); in fec_enet_get_ethtool_stats()
2900 fec_enet_page_pool_stats(fep, data); in fec_enet_get_ethtool_stats()
2945 struct fec_enet_private *fep = netdev_priv(dev); in fec_enet_clear_ethtool_stats() local
2950 writel(FEC_MIB_CTRLSTAT_DISABLE, fep->hwp + FEC_MIB_CTRLSTAT); in fec_enet_clear_ethtool_stats()
2953 writel(0, fep->hwp + fec_stats[i].offset); in fec_enet_clear_ethtool_stats()
2955 for (i = fep->num_rx_queues - 1; i >= 0; i--) { in fec_enet_clear_ethtool_stats()
2956 rxq = fep->rx_queue[i]; in fec_enet_clear_ethtool_stats()
2962 writel(0, fep->hwp + FEC_MIB_CTRLSTAT); in fec_enet_clear_ethtool_stats()
2982 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_us_to_itr_clock() local
2984 return us * (fep->itr_clk_rate / 64000) / 1000; in fec_enet_us_to_itr_clock()
2990 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_itr_coal_set() local
2994 if (!fep->rx_time_itr || !fep->rx_pkts_itr || in fec_enet_itr_coal_set()
2995 !fep->tx_time_itr || !fep->tx_pkts_itr) in fec_enet_itr_coal_set()
3005 rx_itr |= FEC_ITR_ICFT(fep->rx_pkts_itr); in fec_enet_itr_coal_set()
3006 rx_itr |= FEC_ITR_ICTT(fec_enet_us_to_itr_clock(ndev, fep->rx_time_itr)); in fec_enet_itr_coal_set()
3007 tx_itr |= FEC_ITR_ICFT(fep->tx_pkts_itr); in fec_enet_itr_coal_set()
3008 tx_itr |= FEC_ITR_ICTT(fec_enet_us_to_itr_clock(ndev, fep->tx_time_itr)); in fec_enet_itr_coal_set()
3013 writel(tx_itr, fep->hwp + FEC_TXIC0); in fec_enet_itr_coal_set()
3014 writel(rx_itr, fep->hwp + FEC_RXIC0); in fec_enet_itr_coal_set()
3015 if (fep->quirks & FEC_QUIRK_HAS_MULTI_QUEUES) { in fec_enet_itr_coal_set()
3016 writel(tx_itr, fep->hwp + FEC_TXIC1); in fec_enet_itr_coal_set()
3017 writel(rx_itr, fep->hwp + FEC_RXIC1); in fec_enet_itr_coal_set()
3018 writel(tx_itr, fep->hwp + FEC_TXIC2); in fec_enet_itr_coal_set()
3019 writel(rx_itr, fep->hwp + FEC_RXIC2); in fec_enet_itr_coal_set()
3028 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_get_coalesce() local
3030 if (!(fep->quirks & FEC_QUIRK_HAS_COALESCE)) in fec_enet_get_coalesce()
3033 ec->rx_coalesce_usecs = fep->rx_time_itr; in fec_enet_get_coalesce()
3034 ec->rx_max_coalesced_frames = fep->rx_pkts_itr; in fec_enet_get_coalesce()
3036 ec->tx_coalesce_usecs = fep->tx_time_itr; in fec_enet_get_coalesce()
3037 ec->tx_max_coalesced_frames = fep->tx_pkts_itr; in fec_enet_get_coalesce()
3047 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_set_coalesce() local
3048 struct device *dev = &fep->pdev->dev; in fec_enet_set_coalesce()
3051 if (!(fep->quirks & FEC_QUIRK_HAS_COALESCE)) in fec_enet_set_coalesce()
3076 fep->rx_time_itr = ec->rx_coalesce_usecs; in fec_enet_set_coalesce()
3077 fep->rx_pkts_itr = ec->rx_max_coalesced_frames; in fec_enet_set_coalesce()
3079 fep->tx_time_itr = ec->tx_coalesce_usecs; in fec_enet_set_coalesce()
3080 fep->tx_pkts_itr = ec->tx_max_coalesced_frames; in fec_enet_set_coalesce()
3092 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_us_to_tx_cycle() local
3094 return us * (fep->clk_ref_rate / 1000) / 1000; in fec_enet_us_to_tx_cycle()
3099 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_eee_mode_set() local
3100 struct ethtool_eee *p = &fep->eee; in fec_enet_eee_mode_set()
3120 writel(sleep_cycle, fep->hwp + FEC_LPI_SLEEP); in fec_enet_eee_mode_set()
3121 writel(wake_cycle, fep->hwp + FEC_LPI_WAKE); in fec_enet_eee_mode_set()
3129 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_get_eee() local
3130 struct ethtool_eee *p = &fep->eee; in fec_enet_get_eee()
3132 if (!(fep->quirks & FEC_QUIRK_HAS_EEE)) in fec_enet_get_eee()
3149 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_set_eee() local
3150 struct ethtool_eee *p = &fep->eee; in fec_enet_set_eee()
3153 if (!(fep->quirks & FEC_QUIRK_HAS_EEE)) in fec_enet_set_eee()
3176 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_get_wol() local
3178 if (fep->wol_flag & FEC_WOL_HAS_MAGIC_PACKET) { in fec_enet_get_wol()
3180 wol->wolopts = fep->wol_flag & FEC_WOL_FLAG_ENABLE ? WAKE_MAGIC : 0; in fec_enet_get_wol()
3189 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_set_wol() local
3191 if (!(fep->wol_flag & FEC_WOL_HAS_MAGIC_PACKET)) in fec_enet_set_wol()
3199 fep->wol_flag |= FEC_WOL_FLAG_ENABLE; in fec_enet_set_wol()
3201 fep->wol_flag &= (~FEC_WOL_FLAG_ENABLE); in fec_enet_set_wol()
3235 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_free_buffers() local
3241 for (q = 0; q < fep->num_rx_queues; q++) { in fec_enet_free_buffers()
3242 rxq = fep->rx_queue[q]; in fec_enet_free_buffers()
3255 for (q = 0; q < fep->num_tx_queues; q++) { in fec_enet_free_buffers()
3256 txq = fep->tx_queue[q]; in fec_enet_free_buffers()
3284 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_free_queue() local
3288 for (i = 0; i < fep->num_tx_queues; i++) in fec_enet_free_queue()
3289 if (fep->tx_queue[i] && fep->tx_queue[i]->tso_hdrs) { in fec_enet_free_queue()
3290 txq = fep->tx_queue[i]; in fec_enet_free_queue()
3291 dma_free_coherent(&fep->pdev->dev, in fec_enet_free_queue()
3297 for (i = 0; i < fep->num_rx_queues; i++) in fec_enet_free_queue()
3298 kfree(fep->rx_queue[i]); in fec_enet_free_queue()
3299 for (i = 0; i < fep->num_tx_queues; i++) in fec_enet_free_queue()
3300 kfree(fep->tx_queue[i]); in fec_enet_free_queue()
3305 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_alloc_queue() local
3310 for (i = 0; i < fep->num_tx_queues; i++) { in fec_enet_alloc_queue()
3317 fep->tx_queue[i] = txq; in fec_enet_alloc_queue()
3319 fep->total_tx_ring_size += fep->tx_queue[i]->bd.ring_size; in fec_enet_alloc_queue()
3324 txq->tso_hdrs = dma_alloc_coherent(&fep->pdev->dev, in fec_enet_alloc_queue()
3334 for (i = 0; i < fep->num_rx_queues; i++) { in fec_enet_alloc_queue()
3335 fep->rx_queue[i] = kzalloc(sizeof(*fep->rx_queue[i]), in fec_enet_alloc_queue()
3337 if (!fep->rx_queue[i]) { in fec_enet_alloc_queue()
3342 fep->rx_queue[i]->bd.ring_size = RX_RING_SIZE; in fec_enet_alloc_queue()
3343 fep->total_rx_ring_size += fep->rx_queue[i]->bd.ring_size; in fec_enet_alloc_queue()
3355 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_alloc_rxq_buffers() local
3362 rxq = fep->rx_queue[queue]; in fec_enet_alloc_rxq_buffers()
3365 err = fec_enet_create_page_pool(fep, rxq, rxq->bd.ring_size); in fec_enet_alloc_rxq_buffers()
3383 if (fep->bufdesc_ex) { in fec_enet_alloc_rxq_buffers()
3404 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_alloc_txq_buffers() local
3409 txq = fep->tx_queue[queue]; in fec_enet_alloc_txq_buffers()
3419 if (fep->bufdesc_ex) { in fec_enet_alloc_txq_buffers()
3440 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_alloc_buffers() local
3443 for (i = 0; i < fep->num_rx_queues; i++) in fec_enet_alloc_buffers()
3447 for (i = 0; i < fep->num_tx_queues; i++) in fec_enet_alloc_buffers()
3456 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_open() local
3460 ret = pm_runtime_resume_and_get(&fep->pdev->dev); in fec_enet_open()
3464 pinctrl_pm_select_default_state(&fep->pdev->dev); in fec_enet_open()
3502 if (fep->quirks & FEC_QUIRK_ERR006687) in fec_enet_open()
3505 if (fep->quirks & FEC_QUIRK_HAS_PMQOS) in fec_enet_open()
3506 cpu_latency_qos_add_request(&fep->pm_qos_req, 0); in fec_enet_open()
3508 napi_enable(&fep->napi); in fec_enet_open()
3512 device_set_wakeup_enable(&ndev->dev, fep->wol_flag & in fec_enet_open()
3522 pm_runtime_mark_last_busy(&fep->pdev->dev); in fec_enet_open()
3523 pm_runtime_put_autosuspend(&fep->pdev->dev); in fec_enet_open()
3524 pinctrl_pm_select_sleep_state(&fep->pdev->dev); in fec_enet_open()
3531 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_close() local
3536 napi_disable(&fep->napi); in fec_enet_close()
3543 if (fep->quirks & FEC_QUIRK_ERR006687) in fec_enet_close()
3549 if (fep->quirks & FEC_QUIRK_HAS_PMQOS) in fec_enet_close()
3550 cpu_latency_qos_remove_request(&fep->pm_qos_req); in fec_enet_close()
3552 pinctrl_pm_select_sleep_state(&fep->pdev->dev); in fec_enet_close()
3553 pm_runtime_mark_last_busy(&fep->pdev->dev); in fec_enet_close()
3554 pm_runtime_put_autosuspend(&fep->pdev->dev); in fec_enet_close()
3575 struct fec_enet_private *fep = netdev_priv(ndev); in set_multicast_list() local
3582 tmp = readl(fep->hwp + FEC_R_CNTRL); in set_multicast_list()
3584 writel(tmp, fep->hwp + FEC_R_CNTRL); in set_multicast_list()
3588 tmp = readl(fep->hwp + FEC_R_CNTRL); in set_multicast_list()
3590 writel(tmp, fep->hwp + FEC_R_CNTRL); in set_multicast_list()
3596 writel(0xffffffff, fep->hwp + FEC_GRP_HASH_TABLE_HIGH); in set_multicast_list()
3597 writel(0xffffffff, fep->hwp + FEC_GRP_HASH_TABLE_LOW); in set_multicast_list()
3618 writel(hash_high, fep->hwp + FEC_GRP_HASH_TABLE_HIGH); in set_multicast_list()
3619 writel(hash_low, fep->hwp + FEC_GRP_HASH_TABLE_LOW); in set_multicast_list()
3626 struct fec_enet_private *fep = netdev_priv(ndev); in fec_set_mac_address() local
3645 fep->hwp + FEC_ADDR_LOW); in fec_set_mac_address()
3647 fep->hwp + FEC_ADDR_HIGH); in fec_set_mac_address()
3662 struct fec_enet_private *fep = netdev_priv(dev); in fec_poll_controller() local
3665 if (fep->irq[i] > 0) { in fec_poll_controller()
3666 disable_irq(fep->irq[i]); in fec_poll_controller()
3667 fec_enet_interrupt(fep->irq[i], dev); in fec_poll_controller()
3668 enable_irq(fep->irq[i]); in fec_poll_controller()
3677 struct fec_enet_private *fep = netdev_priv(netdev); in fec_enet_set_netdev_features() local
3685 fep->csum_flags |= FLAG_RX_CSUM_ENABLED; in fec_enet_set_netdev_features()
3687 fep->csum_flags &= ~FLAG_RX_CSUM_ENABLED; in fec_enet_set_netdev_features()
3694 struct fec_enet_private *fep = netdev_priv(netdev); in fec_set_features() local
3698 napi_disable(&fep->napi); in fec_set_features()
3705 napi_enable(&fep->napi); in fec_set_features()
3729 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_select_queue() local
3732 if (!(fep->quirks & FEC_QUIRK_HAS_AVB)) in fec_enet_select_queue()
3744 struct fec_enet_private *fep = netdev_priv(dev); in fec_enet_bpf() local
3754 if (fep->quirks & FEC_QUIRK_SWAP_FRAME) in fec_enet_bpf()
3761 napi_disable(&fep->napi); in fec_enet_bpf()
3765 old_prog = xchg(&fep->xdp_prog, bpf->prog); in fec_enet_bpf()
3772 napi_enable(&fep->napi); in fec_enet_bpf()
3790 fec_enet_xdp_get_tx_queue(struct fec_enet_private *fep, int index) in fec_enet_xdp_get_tx_queue() argument
3795 return (index % fep->num_tx_queues); in fec_enet_xdp_get_tx_queue()
3798 static int fec_enet_txq_xmit_frame(struct fec_enet_private *fep, in fec_enet_txq_xmit_frame() argument
3811 netdev_err_once(fep->netdev, "NOT enough BD for SG!\n"); in fec_enet_txq_xmit_frame()
3825 dma_addr = dma_map_single(&fep->pdev->dev, xdpf->data, in fec_enet_txq_xmit_frame()
3827 if (dma_mapping_error(&fep->pdev->dev, dma_addr)) in fec_enet_txq_xmit_frame()
3840 dma_sync_single_for_device(&fep->pdev->dev, dma_addr, in fec_enet_txq_xmit_frame()
3848 if (fep->bufdesc_ex) in fec_enet_txq_xmit_frame()
3854 if (fep->bufdesc_ex) { in fec_enet_txq_xmit_frame()
3857 if (fep->quirks & FEC_QUIRK_HAS_AVB) in fec_enet_txq_xmit_frame()
3889 static int fec_enet_xdp_tx_xmit(struct fec_enet_private *fep, in fec_enet_xdp_tx_xmit() argument
3897 queue = fec_enet_xdp_get_tx_queue(fep, cpu); in fec_enet_xdp_tx_xmit()
3898 txq = fep->tx_queue[queue]; in fec_enet_xdp_tx_xmit()
3899 nq = netdev_get_tx_queue(fep->netdev, queue); in fec_enet_xdp_tx_xmit()
3905 ret = fec_enet_txq_xmit_frame(fep, txq, xdp, dma_sync_len, false); in fec_enet_xdp_tx_xmit()
3917 struct fec_enet_private *fep = netdev_priv(dev); in fec_enet_xdp_xmit() local
3925 queue = fec_enet_xdp_get_tx_queue(fep, cpu); in fec_enet_xdp_xmit()
3926 txq = fep->tx_queue[queue]; in fec_enet_xdp_xmit()
3927 nq = netdev_get_tx_queue(fep->netdev, queue); in fec_enet_xdp_xmit()
3934 if (fec_enet_txq_xmit_frame(fep, txq, frames[i], 0, true) < 0) in fec_enet_xdp_xmit()
3947 struct fec_enet_private *fep = netdev_priv(ndev); in fec_hwtstamp_get() local
3952 if (!fep->bufdesc_ex) in fec_hwtstamp_get()
3964 struct fec_enet_private *fep = netdev_priv(ndev); in fec_hwtstamp_set() local
3969 if (!fep->bufdesc_ex) in fec_hwtstamp_set()
4009 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_init() local
4014 unsigned dsize = fep->bufdesc_ex ? sizeof(struct bufdesc_ex) : in fec_enet_init()
4021 fep->rx_align = 0xf; in fec_enet_init()
4022 fep->tx_align = 0xf; in fec_enet_init()
4024 fep->rx_align = 0x3; in fec_enet_init()
4025 fep->tx_align = 0x3; in fec_enet_init()
4027 fep->rx_pkts_itr = FEC_ITR_ICFT_DEFAULT; in fec_enet_init()
4028 fep->tx_pkts_itr = FEC_ITR_ICFT_DEFAULT; in fec_enet_init()
4029 fep->rx_time_itr = FEC_ITR_ICTT_DEFAULT; in fec_enet_init()
4030 fep->tx_time_itr = FEC_ITR_ICTT_DEFAULT; in fec_enet_init()
4033 ret = dma_set_mask_and_coherent(&fep->pdev->dev, DMA_BIT_MASK(32)); in fec_enet_init()
4035 dev_warn(&fep->pdev->dev, "No suitable DMA available\n"); in fec_enet_init()
4043 bd_size = (fep->total_tx_ring_size + fep->total_rx_ring_size) * dsize; in fec_enet_init()
4046 cbd_base = dmam_alloc_coherent(&fep->pdev->dev, bd_size, &bd_dma, in fec_enet_init()
4059 for (i = 0; i < fep->num_rx_queues; i++) { in fec_enet_init()
4060 struct fec_enet_priv_rx_q *rxq = fep->rx_queue[i]; in fec_enet_init()
4069 rxq->bd.reg_desc_active = fep->hwp + offset_des_active_rxq[i]; in fec_enet_init()
4075 for (i = 0; i < fep->num_tx_queues; i++) { in fec_enet_init()
4076 struct fec_enet_priv_tx_q *txq = fep->tx_queue[i]; in fec_enet_init()
4085 txq->bd.reg_desc_active = fep->hwp + offset_des_active_txq[i]; in fec_enet_init()
4097 writel(FEC_RX_DISABLED_IMASK, fep->hwp + FEC_IMASK); in fec_enet_init()
4098 netif_napi_add(ndev, &fep->napi, fec_enet_rx_napi); in fec_enet_init()
4100 if (fep->quirks & FEC_QUIRK_HAS_VLAN) in fec_enet_init()
4104 if (fep->quirks & FEC_QUIRK_HAS_CSUM) { in fec_enet_init()
4110 fep->csum_flags |= FLAG_RX_CSUM_ENABLED; in fec_enet_init()
4113 if (fep->quirks & FEC_QUIRK_HAS_MULTI_QUEUES) { in fec_enet_init()
4114 fep->tx_align = 0; in fec_enet_init()
4115 fep->rx_align = 0x3f; in fec_enet_init()
4120 if (!(fep->quirks & FEC_QUIRK_SWAP_FRAME)) in fec_enet_init()
4126 if (fep->quirks & FEC_QUIRK_MIB_CLEAR) in fec_enet_init()
4244 struct fec_enet_private *fep = netdev_priv(ndev); in fec_enet_get_wakeup_irq() local
4246 if (fep->quirks & FEC_QUIRK_WAKEUP_FROM_INT2) in fec_enet_get_wakeup_irq()
4247 fep->wake_irq = fep->irq[2]; in fec_enet_get_wakeup_irq()
4249 fep->wake_irq = fep->irq[0]; in fec_enet_get_wakeup_irq()
4252 static int fec_enet_init_stop_mode(struct fec_enet_private *fep, in fec_enet_init_stop_mode() argument
4266 dev_dbg(&fep->pdev->dev, "no stop mode property\n"); in fec_enet_init_stop_mode()
4270 fep->stop_gpr.gpr = syscon_node_to_regmap(gpr_np); in fec_enet_init_stop_mode()
4271 if (IS_ERR(fep->stop_gpr.gpr)) { in fec_enet_init_stop_mode()
4272 dev_err(&fep->pdev->dev, "could not find gpr regmap\n"); in fec_enet_init_stop_mode()
4273 ret = PTR_ERR(fep->stop_gpr.gpr); in fec_enet_init_stop_mode()
4274 fep->stop_gpr.gpr = NULL; in fec_enet_init_stop_mode()
4278 fep->stop_gpr.reg = out_val[1]; in fec_enet_init_stop_mode()
4279 fep->stop_gpr.bit = out_val[2]; in fec_enet_init_stop_mode()
4290 struct fec_enet_private *fep; in fec_probe() local
4315 fep = netdev_priv(ndev); in fec_probe()
4322 fep->quirks = dev_info->quirks; in fec_probe()
4324 fep->netdev = ndev; in fec_probe()
4325 fep->num_rx_queues = num_rx_qs; in fec_probe()
4326 fep->num_tx_queues = num_tx_qs; in fec_probe()
4330 if (fep->quirks & FEC_QUIRK_HAS_GBIT) in fec_probe()
4331 fep->pause_flag |= FEC_PAUSE_FLAG_AUTONEG; in fec_probe()
4337 fep->hwp = devm_platform_ioremap_resource(pdev, 0); in fec_probe()
4338 if (IS_ERR(fep->hwp)) { in fec_probe()
4339 ret = PTR_ERR(fep->hwp); in fec_probe()
4343 fep->pdev = pdev; in fec_probe()
4344 fep->dev_id = dev_id++; in fec_probe()
4351 fep->quirks |= FEC_QUIRK_ERR006687; in fec_probe()
4353 ret = fec_enet_ipc_handle_init(fep); in fec_probe()
4358 fep->wol_flag |= FEC_WOL_HAS_MAGIC_PACKET; in fec_probe()
4360 ret = fec_enet_init_stop_mode(fep, np); in fec_probe()
4374 fep->phy_node = phy_node; in fec_probe()
4380 fep->phy_interface = pdata->phy; in fec_probe()
4382 fep->phy_interface = PHY_INTERFACE_MODE_MII; in fec_probe()
4384 fep->phy_interface = interface; in fec_probe()
4387 ret = fec_enet_parse_rgmii_delay(fep, np); in fec_probe()
4391 fep->clk_ipg = devm_clk_get(&pdev->dev, "ipg"); in fec_probe()
4392 if (IS_ERR(fep->clk_ipg)) { in fec_probe()
4393 ret = PTR_ERR(fep->clk_ipg); in fec_probe()
4397 fep->clk_ahb = devm_clk_get(&pdev->dev, "ahb"); in fec_probe()
4398 if (IS_ERR(fep->clk_ahb)) { in fec_probe()
4399 ret = PTR_ERR(fep->clk_ahb); in fec_probe()
4403 fep->itr_clk_rate = clk_get_rate(fep->clk_ahb); in fec_probe()
4406 fep->clk_enet_out = devm_clk_get_optional(&pdev->dev, "enet_out"); in fec_probe()
4407 if (IS_ERR(fep->clk_enet_out)) { in fec_probe()
4408 ret = PTR_ERR(fep->clk_enet_out); in fec_probe()
4412 fep->ptp_clk_on = false; in fec_probe()
4413 mutex_init(&fep->ptp_clk_mutex); in fec_probe()
4416 fep->clk_ref = devm_clk_get_optional(&pdev->dev, "enet_clk_ref"); in fec_probe()
4417 if (IS_ERR(fep->clk_ref)) { in fec_probe()
4418 ret = PTR_ERR(fep->clk_ref); in fec_probe()
4421 fep->clk_ref_rate = clk_get_rate(fep->clk_ref); in fec_probe()
4424 if (fep->rgmii_txc_dly || fep->rgmii_rxc_dly) { in fec_probe()
4425 fep->clk_2x_txclk = devm_clk_get(&pdev->dev, "enet_2x_txclk"); in fec_probe()
4426 if (IS_ERR(fep->clk_2x_txclk)) in fec_probe()
4427 fep->clk_2x_txclk = NULL; in fec_probe()
4430 fep->bufdesc_ex = fep->quirks & FEC_QUIRK_HAS_BUFDESC_EX; in fec_probe()
4431 fep->clk_ptp = devm_clk_get(&pdev->dev, "ptp"); in fec_probe()
4432 if (IS_ERR(fep->clk_ptp)) { in fec_probe()
4433 fep->clk_ptp = NULL; in fec_probe()
4434 fep->bufdesc_ex = false; in fec_probe()
4441 ret = clk_prepare_enable(fep->clk_ipg); in fec_probe()
4444 ret = clk_prepare_enable(fep->clk_ahb); in fec_probe()
4448 fep->reg_phy = devm_regulator_get_optional(&pdev->dev, "phy"); in fec_probe()
4449 if (!IS_ERR(fep->reg_phy)) { in fec_probe()
4450 ret = regulator_enable(fep->reg_phy); in fec_probe()
4457 if (PTR_ERR(fep->reg_phy) == -EPROBE_DEFER) { in fec_probe()
4461 fep->reg_phy = NULL; in fec_probe()
4475 if (fep->bufdesc_ex) in fec_probe()
4496 fep->irq[i] = irq; in fec_probe()
4517 device_init_wakeup(&ndev->dev, fep->wol_flag & in fec_probe()
4520 if (fep->bufdesc_ex && fep->ptp_clock) in fec_probe()
4521 netdev_info(ndev, "registered PHC device %d\n", fep->dev_id); in fec_probe()
4523 INIT_WORK(&fep->tx_timeout_work, fec_enet_timeout_work); in fec_probe()
4531 fec_enet_mii_remove(fep); in fec_probe()
4539 if (fep->reg_phy) in fec_probe()
4540 regulator_disable(fep->reg_phy); in fec_probe()
4542 clk_disable_unprepare(fep->clk_ahb); in fec_probe()
4544 clk_disable_unprepare(fep->clk_ipg); in fec_probe()
4566 struct fec_enet_private *fep = netdev_priv(ndev); in fec_drv_remove() local
4576 cancel_work_sync(&fep->tx_timeout_work); in fec_drv_remove()
4579 fec_enet_mii_remove(fep); in fec_drv_remove()
4580 if (fep->reg_phy) in fec_drv_remove()
4581 regulator_disable(fep->reg_phy); in fec_drv_remove()
4585 of_node_put(fep->phy_node); in fec_drv_remove()
4591 clk_disable_unprepare(fep->clk_ahb); in fec_drv_remove()
4592 clk_disable_unprepare(fep->clk_ipg); in fec_drv_remove()
4603 struct fec_enet_private *fep = netdev_priv(ndev); in fec_suspend() local
4608 if (fep->wol_flag & FEC_WOL_FLAG_ENABLE) in fec_suspend()
4609 fep->wol_flag |= FEC_WOL_FLAG_SLEEP_ON; in fec_suspend()
4611 napi_disable(&fep->napi); in fec_suspend()
4616 if (!(fep->wol_flag & FEC_WOL_FLAG_ENABLE)) { in fec_suspend()
4618 pinctrl_pm_select_sleep_state(&fep->pdev->dev); in fec_suspend()
4621 if (fep->wake_irq > 0) { in fec_suspend()
4622 disable_irq(fep->wake_irq); in fec_suspend()
4623 enable_irq_wake(fep->wake_irq); in fec_suspend()
4625 fec_enet_stop_mode(fep, true); in fec_suspend()
4630 fep->rpm_active = !pm_runtime_status_suspended(dev); in fec_suspend()
4631 if (fep->rpm_active) { in fec_suspend()
4641 if (fep->reg_phy && !(fep->wol_flag & FEC_WOL_FLAG_ENABLE)) in fec_suspend()
4642 regulator_disable(fep->reg_phy); in fec_suspend()
4647 if (fep->clk_enet_out || fep->reg_phy) in fec_suspend()
4648 fep->link = 0; in fec_suspend()
4656 struct fec_enet_private *fep = netdev_priv(ndev); in fec_resume() local
4660 if (fep->reg_phy && !(fep->wol_flag & FEC_WOL_FLAG_ENABLE)) { in fec_resume()
4661 ret = regulator_enable(fep->reg_phy); in fec_resume()
4668 if (fep->rpm_active) in fec_resume()
4676 if (fep->wol_flag & FEC_WOL_FLAG_ENABLE) { in fec_resume()
4677 fec_enet_stop_mode(fep, false); in fec_resume()
4678 if (fep->wake_irq) { in fec_resume()
4679 disable_irq_wake(fep->wake_irq); in fec_resume()
4680 enable_irq(fep->wake_irq); in fec_resume()
4683 val = readl(fep->hwp + FEC_ECNTRL); in fec_resume()
4685 writel(val, fep->hwp + FEC_ECNTRL); in fec_resume()
4686 fep->wol_flag &= ~FEC_WOL_FLAG_SLEEP_ON; in fec_resume()
4688 pinctrl_pm_select_default_state(&fep->pdev->dev); in fec_resume()
4694 napi_enable(&fep->napi); in fec_resume()
4703 if (fep->reg_phy) in fec_resume()
4704 regulator_disable(fep->reg_phy); in fec_resume()
4711 struct fec_enet_private *fep = netdev_priv(ndev); in fec_runtime_suspend() local
4713 clk_disable_unprepare(fep->clk_ahb); in fec_runtime_suspend()
4714 clk_disable_unprepare(fep->clk_ipg); in fec_runtime_suspend()
4722 struct fec_enet_private *fep = netdev_priv(ndev); in fec_runtime_resume() local
4725 ret = clk_prepare_enable(fep->clk_ahb); in fec_runtime_resume()
4728 ret = clk_prepare_enable(fep->clk_ipg); in fec_runtime_resume()
4735 clk_disable_unprepare(fep->clk_ahb); in fec_runtime_resume()