Lines Matching +full:queue +full:- +full:rx
1 // SPDX-License-Identifier: GPL-2.0-only
3 This is the driver for the ST MAC 10/100/1000 on-chip Ethernet controllers.
6 Copyright(C) 2007-2011 STMicroelectronics Ltd
29 #include <linux/dma-mapping.h>
54 #define TSO_MAX_BUFF_SIZE (SZ_16K - 1)
62 static int debug = -1;
64 MODULE_PARM_DESC(debug, "Message Level (-1: default, 0: no output, 16: all)");
66 static int phyaddr = -1;
70 #define STMMAC_TX_THRESH(x) ((x)->dma_tx_size / 4)
71 #define STMMAC_RX_THRESH(x) ((x)->dma_rx_size / 4)
113 /* By default the driver will use the ring mode to manage tx and rx descriptors,
126 static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue);
127 static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue);
142 ret = clk_prepare_enable(priv->plat->stmmac_clk); in stmmac_bus_clks_config()
145 ret = clk_prepare_enable(priv->plat->pclk); in stmmac_bus_clks_config()
147 clk_disable_unprepare(priv->plat->stmmac_clk); in stmmac_bus_clks_config()
150 if (priv->plat->clks_config) { in stmmac_bus_clks_config()
151 ret = priv->plat->clks_config(priv->plat->bsp_priv, enabled); in stmmac_bus_clks_config()
153 clk_disable_unprepare(priv->plat->stmmac_clk); in stmmac_bus_clks_config()
154 clk_disable_unprepare(priv->plat->pclk); in stmmac_bus_clks_config()
159 clk_disable_unprepare(priv->plat->stmmac_clk); in stmmac_bus_clks_config()
160 clk_disable_unprepare(priv->plat->pclk); in stmmac_bus_clks_config()
161 if (priv->plat->clks_config) in stmmac_bus_clks_config()
162 priv->plat->clks_config(priv->plat->bsp_priv, enabled); in stmmac_bus_clks_config()
170 * stmmac_verify_args - verify the driver parameters.
192 u32 rx_queues_cnt = priv->plat->rx_queues_to_use; in __stmmac_disable_all_queues()
193 u32 tx_queues_cnt = priv->plat->tx_queues_to_use; in __stmmac_disable_all_queues()
195 u32 queue; in __stmmac_disable_all_queues() local
197 for (queue = 0; queue < maxq; queue++) { in __stmmac_disable_all_queues()
198 struct stmmac_channel *ch = &priv->channel[queue]; in __stmmac_disable_all_queues()
201 test_bit(queue, priv->af_xdp_zc_qps)) { in __stmmac_disable_all_queues()
202 napi_disable(&ch->rxtx_napi); in __stmmac_disable_all_queues()
206 if (queue < rx_queues_cnt) in __stmmac_disable_all_queues()
207 napi_disable(&ch->rx_napi); in __stmmac_disable_all_queues()
208 if (queue < tx_queues_cnt) in __stmmac_disable_all_queues()
209 napi_disable(&ch->tx_napi); in __stmmac_disable_all_queues()
214 * stmmac_disable_all_queues - Disable all queues
219 u32 rx_queues_cnt = priv->plat->rx_queues_to_use; in stmmac_disable_all_queues()
221 u32 queue; in stmmac_disable_all_queues() local
224 for (queue = 0; queue < rx_queues_cnt; queue++) { in stmmac_disable_all_queues()
225 rx_q = &priv->rx_queue[queue]; in stmmac_disable_all_queues()
226 if (rx_q->xsk_pool) { in stmmac_disable_all_queues()
236 * stmmac_enable_all_queues - Enable all queues
241 u32 rx_queues_cnt = priv->plat->rx_queues_to_use; in stmmac_enable_all_queues()
242 u32 tx_queues_cnt = priv->plat->tx_queues_to_use; in stmmac_enable_all_queues()
244 u32 queue; in stmmac_enable_all_queues() local
246 for (queue = 0; queue < maxq; queue++) { in stmmac_enable_all_queues()
247 struct stmmac_channel *ch = &priv->channel[queue]; in stmmac_enable_all_queues()
250 test_bit(queue, priv->af_xdp_zc_qps)) { in stmmac_enable_all_queues()
251 napi_enable(&ch->rxtx_napi); in stmmac_enable_all_queues()
255 if (queue < rx_queues_cnt) in stmmac_enable_all_queues()
256 napi_enable(&ch->rx_napi); in stmmac_enable_all_queues()
257 if (queue < tx_queues_cnt) in stmmac_enable_all_queues()
258 napi_enable(&ch->tx_napi); in stmmac_enable_all_queues()
264 if (!test_bit(STMMAC_DOWN, &priv->state) && in stmmac_service_event_schedule()
265 !test_and_set_bit(STMMAC_SERVICE_SCHED, &priv->state)) in stmmac_service_event_schedule()
266 queue_work(priv->wq, &priv->service_task); in stmmac_service_event_schedule()
271 netif_carrier_off(priv->dev); in stmmac_global_err()
272 set_bit(STMMAC_RESET_REQUESTED, &priv->state); in stmmac_global_err()
277 * stmmac_clk_csr_set - dynamically set the MDC clock
284 * changed at run-time and it is fixed (as reported in the driver
292 clk_rate = clk_get_rate(priv->plat->stmmac_clk); in stmmac_clk_csr_set()
301 if (!(priv->clk_csr & MAC_CSR_H_FRQ_MASK)) { in stmmac_clk_csr_set()
303 priv->clk_csr = STMMAC_CSR_20_35M; in stmmac_clk_csr_set()
305 priv->clk_csr = STMMAC_CSR_35_60M; in stmmac_clk_csr_set()
307 priv->clk_csr = STMMAC_CSR_60_100M; in stmmac_clk_csr_set()
309 priv->clk_csr = STMMAC_CSR_100_150M; in stmmac_clk_csr_set()
311 priv->clk_csr = STMMAC_CSR_150_250M; in stmmac_clk_csr_set()
313 priv->clk_csr = STMMAC_CSR_250_300M; in stmmac_clk_csr_set()
316 if (priv->plat->has_sun8i) { in stmmac_clk_csr_set()
318 priv->clk_csr = 0x03; in stmmac_clk_csr_set()
320 priv->clk_csr = 0x02; in stmmac_clk_csr_set()
322 priv->clk_csr = 0x01; in stmmac_clk_csr_set()
324 priv->clk_csr = 0; in stmmac_clk_csr_set()
327 if (priv->plat->has_xgmac) { in stmmac_clk_csr_set()
329 priv->clk_csr = 0x5; in stmmac_clk_csr_set()
331 priv->clk_csr = 0x4; in stmmac_clk_csr_set()
333 priv->clk_csr = 0x3; in stmmac_clk_csr_set()
335 priv->clk_csr = 0x2; in stmmac_clk_csr_set()
337 priv->clk_csr = 0x1; in stmmac_clk_csr_set()
339 priv->clk_csr = 0x0; in stmmac_clk_csr_set()
349 static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue) in stmmac_tx_avail() argument
351 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; in stmmac_tx_avail()
354 if (tx_q->dirty_tx > tx_q->cur_tx) in stmmac_tx_avail()
355 avail = tx_q->dirty_tx - tx_q->cur_tx - 1; in stmmac_tx_avail()
357 avail = priv->dma_tx_size - tx_q->cur_tx + tx_q->dirty_tx - 1; in stmmac_tx_avail()
363 * stmmac_rx_dirty - Get RX queue dirty
365 * @queue: RX queue index
367 static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue) in stmmac_rx_dirty() argument
369 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; in stmmac_rx_dirty()
372 if (rx_q->dirty_rx <= rx_q->cur_rx) in stmmac_rx_dirty()
373 dirty = rx_q->cur_rx - rx_q->dirty_rx; in stmmac_rx_dirty()
375 dirty = priv->dma_rx_size - rx_q->dirty_rx + rx_q->cur_rx; in stmmac_rx_dirty()
385 priv->eee_sw_timer_en = en ? 0 : 1; in stmmac_lpi_entry_timer_config()
386 tx_lpi_timer = en ? priv->tx_lpi_timer : 0; in stmmac_lpi_entry_timer_config()
387 stmmac_set_eee_lpi_timer(priv, priv->hw, tx_lpi_timer); in stmmac_lpi_entry_timer_config()
391 * stmmac_enable_eee_mode - check and enter in LPI mode
398 u32 tx_cnt = priv->plat->tx_queues_to_use; in stmmac_enable_eee_mode()
399 u32 queue; in stmmac_enable_eee_mode() local
402 for (queue = 0; queue < tx_cnt; queue++) { in stmmac_enable_eee_mode()
403 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; in stmmac_enable_eee_mode()
405 if (tx_q->dirty_tx != tx_q->cur_tx) in stmmac_enable_eee_mode()
410 if (!priv->tx_path_in_lpi_mode) in stmmac_enable_eee_mode()
411 stmmac_set_eee_mode(priv, priv->hw, in stmmac_enable_eee_mode()
412 priv->plat->en_tx_lpi_clockgating); in stmmac_enable_eee_mode()
416 * stmmac_disable_eee_mode - disable and exit from LPI mode
423 if (!priv->eee_sw_timer_en) { in stmmac_disable_eee_mode()
428 stmmac_reset_eee_mode(priv, priv->hw); in stmmac_disable_eee_mode()
429 del_timer_sync(&priv->eee_ctrl_timer); in stmmac_disable_eee_mode()
430 priv->tx_path_in_lpi_mode = false; in stmmac_disable_eee_mode()
434 * stmmac_eee_ctrl_timer - EEE TX SW timer.
445 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer)); in stmmac_eee_ctrl_timer()
449 * stmmac_eee_init - init EEE
458 int eee_tw_timer = priv->eee_tw_timer; in stmmac_eee_init()
463 if (priv->hw->pcs == STMMAC_PCS_TBI || in stmmac_eee_init()
464 priv->hw->pcs == STMMAC_PCS_RTBI) in stmmac_eee_init()
468 if (!priv->dma_cap.eee) in stmmac_eee_init()
471 mutex_lock(&priv->lock); in stmmac_eee_init()
474 if (!priv->eee_active) { in stmmac_eee_init()
475 if (priv->eee_enabled) { in stmmac_eee_init()
476 netdev_dbg(priv->dev, "disable EEE\n"); in stmmac_eee_init()
478 del_timer_sync(&priv->eee_ctrl_timer); in stmmac_eee_init()
479 stmmac_set_eee_timer(priv, priv->hw, 0, eee_tw_timer); in stmmac_eee_init()
480 if (priv->hw->xpcs) in stmmac_eee_init()
481 xpcs_config_eee(priv->hw->xpcs, in stmmac_eee_init()
482 priv->plat->mult_fact_100ns, in stmmac_eee_init()
485 mutex_unlock(&priv->lock); in stmmac_eee_init()
489 if (priv->eee_active && !priv->eee_enabled) { in stmmac_eee_init()
490 timer_setup(&priv->eee_ctrl_timer, stmmac_eee_ctrl_timer, 0); in stmmac_eee_init()
491 stmmac_set_eee_timer(priv, priv->hw, STMMAC_DEFAULT_LIT_LS, in stmmac_eee_init()
493 if (priv->hw->xpcs) in stmmac_eee_init()
494 xpcs_config_eee(priv->hw->xpcs, in stmmac_eee_init()
495 priv->plat->mult_fact_100ns, in stmmac_eee_init()
499 if (priv->plat->has_gmac4 && priv->tx_lpi_timer <= STMMAC_ET_MAX) { in stmmac_eee_init()
500 del_timer_sync(&priv->eee_ctrl_timer); in stmmac_eee_init()
501 priv->tx_path_in_lpi_mode = false; in stmmac_eee_init()
505 mod_timer(&priv->eee_ctrl_timer, in stmmac_eee_init()
506 STMMAC_LPI_T(priv->tx_lpi_timer)); in stmmac_eee_init()
509 mutex_unlock(&priv->lock); in stmmac_eee_init()
510 netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n"); in stmmac_eee_init()
514 /* stmmac_get_tx_hwtstamp - get HW TX timestamps
530 if (!priv->hwts_tx_en) in stmmac_get_tx_hwtstamp()
534 if (likely(!skb || !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS))) in stmmac_get_tx_hwtstamp()
539 stmmac_get_timestamp(priv, p, priv->adv_ts, &ns); in stmmac_get_tx_hwtstamp()
541 } else if (!stmmac_get_mac_tx_timestamp(priv, priv->hw, &ns)) { in stmmac_get_tx_hwtstamp()
547 if (priv->plat->has_gmac4 && priv->plat->clk_ptp_rate) { in stmmac_get_tx_hwtstamp()
548 adjust += -(2 * (NSEC_PER_SEC / in stmmac_get_tx_hwtstamp()
549 priv->plat->clk_ptp_rate)); in stmmac_get_tx_hwtstamp()
556 netdev_dbg(priv->dev, "get valid TX hw timestamp %llu\n", ns); in stmmac_get_tx_hwtstamp()
562 /* stmmac_get_rx_hwtstamp - get HW RX timestamps
579 if (!priv->hwts_rx_en) in stmmac_get_rx_hwtstamp()
582 if (priv->plat->has_gmac4 || priv->plat->has_xgmac) in stmmac_get_rx_hwtstamp()
586 if (stmmac_get_rx_timestamp_status(priv, p, np, priv->adv_ts)) { in stmmac_get_rx_hwtstamp()
587 stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns); in stmmac_get_rx_hwtstamp()
590 if (priv->plat->has_gmac4 && priv->plat->clk_ptp_rate) { in stmmac_get_rx_hwtstamp()
591 adjust += 2 * (NSEC_PER_SEC / priv->plat->clk_ptp_rate); in stmmac_get_rx_hwtstamp()
592 ns -= adjust; in stmmac_get_rx_hwtstamp()
595 netdev_dbg(priv->dev, "get valid RX hw timestamp %llu\n", ns); in stmmac_get_rx_hwtstamp()
598 shhwtstamp->hwtstamp = ns_to_ktime(ns); in stmmac_get_rx_hwtstamp()
600 netdev_dbg(priv->dev, "cannot get RX hw timestamp\n"); in stmmac_get_rx_hwtstamp()
605 * stmmac_hwtstamp_set - control hardware timestamping.
611 * and incoming(RX) packets time stamping based on user input.
613 * 0 on success and an appropriate -ve integer on failure.
633 xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; in stmmac_hwtstamp_set()
635 if (!(priv->dma_cap.time_stamp || priv->adv_ts)) { in stmmac_hwtstamp_set()
636 netdev_alert(priv->dev, "No support for HW time stamping\n"); in stmmac_hwtstamp_set()
637 priv->hwts_tx_en = 0; in stmmac_hwtstamp_set()
638 priv->hwts_rx_en = 0; in stmmac_hwtstamp_set()
640 return -EOPNOTSUPP; in stmmac_hwtstamp_set()
643 if (copy_from_user(&config, ifr->ifr_data, in stmmac_hwtstamp_set()
645 return -EFAULT; in stmmac_hwtstamp_set()
647 netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n", in stmmac_hwtstamp_set()
652 return -EINVAL; in stmmac_hwtstamp_set()
656 return -ERANGE; in stmmac_hwtstamp_set()
658 if (priv->adv_ts) { in stmmac_hwtstamp_set()
739 if (priv->synopsys_id < DWMAC_CORE_4_10) in stmmac_hwtstamp_set()
779 return -ERANGE; in stmmac_hwtstamp_set()
792 priv->hwts_rx_en = ((config.rx_filter == HWTSTAMP_FILTER_NONE) ? 0 : 1); in stmmac_hwtstamp_set()
793 priv->hwts_tx_en = config.tx_type == HWTSTAMP_TX_ON; in stmmac_hwtstamp_set()
795 if (!priv->hwts_tx_en && !priv->hwts_rx_en) in stmmac_hwtstamp_set()
796 stmmac_config_hw_tstamping(priv, priv->ptpaddr, 0); in stmmac_hwtstamp_set()
802 stmmac_config_hw_tstamping(priv, priv->ptpaddr, value); in stmmac_hwtstamp_set()
806 priv->ptpaddr, priv->plat->clk_ptp_rate, in stmmac_hwtstamp_set()
811 priv->sub_second_inc = sec_inc; in stmmac_hwtstamp_set()
812 priv->systime_flags = value; in stmmac_hwtstamp_set()
820 priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate); in stmmac_hwtstamp_set()
821 stmmac_config_addend(priv, priv->ptpaddr, priv->default_addend); in stmmac_hwtstamp_set()
827 stmmac_init_systime(priv, priv->ptpaddr, in stmmac_hwtstamp_set()
831 memcpy(&priv->tstamp_config, &config, sizeof(config)); in stmmac_hwtstamp_set()
833 return copy_to_user(ifr->ifr_data, &config, in stmmac_hwtstamp_set()
834 sizeof(config)) ? -EFAULT : 0; in stmmac_hwtstamp_set()
838 * stmmac_hwtstamp_get - read hardware timestamping.
849 struct hwtstamp_config *config = &priv->tstamp_config; in stmmac_hwtstamp_get()
851 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) in stmmac_hwtstamp_get()
852 return -EOPNOTSUPP; in stmmac_hwtstamp_get()
854 return copy_to_user(ifr->ifr_data, config, in stmmac_hwtstamp_get()
855 sizeof(*config)) ? -EFAULT : 0; in stmmac_hwtstamp_get()
859 * stmmac_init_ptp - init PTP
867 bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; in stmmac_init_ptp()
869 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) in stmmac_init_ptp()
870 return -EOPNOTSUPP; in stmmac_init_ptp()
872 priv->adv_ts = 0; in stmmac_init_ptp()
874 if (xmac && priv->dma_cap.atime_stamp) in stmmac_init_ptp()
875 priv->adv_ts = 1; in stmmac_init_ptp()
877 else if (priv->extend_desc && priv->dma_cap.atime_stamp) in stmmac_init_ptp()
878 priv->adv_ts = 1; in stmmac_init_ptp()
880 if (priv->dma_cap.time_stamp) in stmmac_init_ptp()
881 netdev_info(priv->dev, "IEEE 1588-2002 Timestamp supported\n"); in stmmac_init_ptp()
883 if (priv->adv_ts) in stmmac_init_ptp()
884 netdev_info(priv->dev, in stmmac_init_ptp()
885 "IEEE 1588-2008 Advanced Timestamp supported\n"); in stmmac_init_ptp()
887 priv->hwts_tx_en = 0; in stmmac_init_ptp()
888 priv->hwts_rx_en = 0; in stmmac_init_ptp()
897 clk_disable_unprepare(priv->plat->clk_ptp_ref); in stmmac_release_ptp()
902 * stmmac_mac_flow_ctrl - Configure flow control in all queues
909 u32 tx_cnt = priv->plat->tx_queues_to_use; in stmmac_mac_flow_ctrl()
911 stmmac_flow_ctrl(priv, priv->hw, duplex, priv->flow_ctrl, in stmmac_mac_flow_ctrl()
912 priv->pause, tx_cnt); in stmmac_mac_flow_ctrl()
919 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); in stmmac_validate()
922 int tx_cnt = priv->plat->tx_queues_to_use; in stmmac_validate()
923 int max_speed = priv->plat->max_speed; in stmmac_validate()
942 } else if (priv->plat->has_gmac4) { in stmmac_validate()
947 } else if (priv->plat->has_xgmac) { in stmmac_validate()
998 /* Half-Duplex can only work with single queue */ in stmmac_validate()
1008 linkmode_and(state->advertising, state->advertising, mac_supported); in stmmac_validate()
1009 linkmode_andnot(state->advertising, state->advertising, mask); in stmmac_validate()
1012 if (priv->hw->xpcs) in stmmac_validate()
1013 xpcs_validate(priv->hw->xpcs, supported, state); in stmmac_validate()
1024 struct stmmac_fpe_cfg *fpe_cfg = priv->plat->fpe_cfg; in stmmac_fpe_link_state_handle()
1025 enum stmmac_fpe_state *lo_state = &fpe_cfg->lo_fpe_state; in stmmac_fpe_link_state_handle()
1026 enum stmmac_fpe_state *lp_state = &fpe_cfg->lp_fpe_state; in stmmac_fpe_link_state_handle()
1027 bool *hs_enable = &fpe_cfg->hs_enable; in stmmac_fpe_link_state_handle()
1030 stmmac_fpe_send_mpacket(priv, priv->ioaddr, MPACKET_VERIFY); in stmmac_fpe_link_state_handle()
1040 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); in stmmac_mac_link_down()
1042 stmmac_mac_set(priv, priv->ioaddr, false); in stmmac_mac_link_down()
1043 priv->eee_active = false; in stmmac_mac_link_down()
1044 priv->tx_lpi_enabled = false; in stmmac_mac_link_down()
1045 priv->eee_enabled = stmmac_eee_init(priv); in stmmac_mac_link_down()
1046 stmmac_set_eee_pls(priv, priv->hw, false); in stmmac_mac_link_down()
1048 if (priv->dma_cap.fpesel) in stmmac_mac_link_down()
1058 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); in stmmac_mac_link_up()
1061 ctrl = readl(priv->ioaddr + MAC_CTRL_REG); in stmmac_mac_link_up()
1062 ctrl &= ~priv->hw->link.speed_mask; in stmmac_mac_link_up()
1067 ctrl |= priv->hw->link.xgmii.speed10000; in stmmac_mac_link_up()
1070 ctrl |= priv->hw->link.xgmii.speed5000; in stmmac_mac_link_up()
1073 ctrl |= priv->hw->link.xgmii.speed2500; in stmmac_mac_link_up()
1081 ctrl |= priv->hw->link.xlgmii.speed100000; in stmmac_mac_link_up()
1084 ctrl |= priv->hw->link.xlgmii.speed50000; in stmmac_mac_link_up()
1087 ctrl |= priv->hw->link.xlgmii.speed40000; in stmmac_mac_link_up()
1090 ctrl |= priv->hw->link.xlgmii.speed25000; in stmmac_mac_link_up()
1093 ctrl |= priv->hw->link.xgmii.speed10000; in stmmac_mac_link_up()
1096 ctrl |= priv->hw->link.speed2500; in stmmac_mac_link_up()
1099 ctrl |= priv->hw->link.speed1000; in stmmac_mac_link_up()
1107 ctrl |= priv->hw->link.speed2500; in stmmac_mac_link_up()
1110 ctrl |= priv->hw->link.speed1000; in stmmac_mac_link_up()
1113 ctrl |= priv->hw->link.speed100; in stmmac_mac_link_up()
1116 ctrl |= priv->hw->link.speed10; in stmmac_mac_link_up()
1123 priv->speed = speed; in stmmac_mac_link_up()
1125 if (priv->plat->fix_mac_speed) in stmmac_mac_link_up()
1126 priv->plat->fix_mac_speed(priv->plat->bsp_priv, speed); in stmmac_mac_link_up()
1129 ctrl &= ~priv->hw->link.duplex; in stmmac_mac_link_up()
1131 ctrl |= priv->hw->link.duplex; in stmmac_mac_link_up()
1137 writel(ctrl, priv->ioaddr + MAC_CTRL_REG); in stmmac_mac_link_up()
1139 stmmac_mac_set(priv, priv->ioaddr, true); in stmmac_mac_link_up()
1140 if (phy && priv->dma_cap.eee) { in stmmac_mac_link_up()
1141 priv->eee_active = phy_init_eee(phy, 1) >= 0; in stmmac_mac_link_up()
1142 priv->eee_enabled = stmmac_eee_init(priv); in stmmac_mac_link_up()
1143 priv->tx_lpi_enabled = priv->eee_enabled; in stmmac_mac_link_up()
1144 stmmac_set_eee_pls(priv, priv->hw, true); in stmmac_mac_link_up()
1147 if (priv->dma_cap.fpesel) in stmmac_mac_link_up()
1159 * stmmac_check_pcs_mode - verify if RGMII/SGMII is supported
1167 int interface = priv->plat->interface; in stmmac_check_pcs_mode()
1169 if (priv->dma_cap.pcs) { in stmmac_check_pcs_mode()
1174 netdev_dbg(priv->dev, "PCS RGMII support enabled\n"); in stmmac_check_pcs_mode()
1175 priv->hw->pcs = STMMAC_PCS_RGMII; in stmmac_check_pcs_mode()
1177 netdev_dbg(priv->dev, "PCS SGMII support enabled\n"); in stmmac_check_pcs_mode()
1178 priv->hw->pcs = STMMAC_PCS_SGMII; in stmmac_check_pcs_mode()
1184 * stmmac_init_phy - PHY initialization
1197 node = priv->plat->phylink_node; in stmmac_init_phy()
1200 ret = phylink_of_phy_connect(priv->phylink, node, 0); in stmmac_init_phy()
1202 /* Some DT bindings do not set-up the PHY handle. Let's try to in stmmac_init_phy()
1206 int addr = priv->plat->phy_addr; in stmmac_init_phy()
1209 phydev = mdiobus_get_phy(priv->mii, addr); in stmmac_init_phy()
1211 netdev_err(priv->dev, "no phy at addr %d\n", addr); in stmmac_init_phy()
1212 return -ENODEV; in stmmac_init_phy()
1215 ret = phylink_connect_phy(priv->phylink, phydev); in stmmac_init_phy()
1218 if (!priv->plat->pmt) { in stmmac_init_phy()
1221 phylink_ethtool_get_wol(priv->phylink, &wol); in stmmac_init_phy()
1222 device_set_wakeup_capable(priv->device, !!wol.supported); in stmmac_init_phy()
1230 struct stmmac_mdio_bus_data *mdio_bus_data = priv->plat->mdio_bus_data; in stmmac_phy_setup()
1231 struct fwnode_handle *fwnode = of_fwnode_handle(priv->plat->phylink_node); in stmmac_phy_setup()
1232 int mode = priv->plat->phy_interface; in stmmac_phy_setup()
1235 priv->phylink_config.dev = &priv->dev->dev; in stmmac_phy_setup()
1236 priv->phylink_config.type = PHYLINK_NETDEV; in stmmac_phy_setup()
1237 priv->phylink_config.pcs_poll = true; in stmmac_phy_setup()
1238 if (priv->plat->mdio_bus_data) in stmmac_phy_setup()
1239 priv->phylink_config.ovr_an_inband = in stmmac_phy_setup()
1240 mdio_bus_data->xpcs_an_inband; in stmmac_phy_setup()
1243 fwnode = dev_fwnode(priv->device); in stmmac_phy_setup()
1245 phylink = phylink_create(&priv->phylink_config, fwnode, in stmmac_phy_setup()
1250 if (priv->hw->xpcs) in stmmac_phy_setup()
1251 phylink_set_pcs(phylink, &priv->hw->xpcs->pcs); in stmmac_phy_setup()
1253 priv->phylink = phylink; in stmmac_phy_setup()
1259 u32 rx_cnt = priv->plat->rx_queues_to_use; in stmmac_display_rx_rings()
1262 u32 queue; in stmmac_display_rx_rings() local
1264 /* Display RX rings */ in stmmac_display_rx_rings()
1265 for (queue = 0; queue < rx_cnt; queue++) { in stmmac_display_rx_rings()
1266 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; in stmmac_display_rx_rings()
1268 pr_info("\tRX Queue %u rings\n", queue); in stmmac_display_rx_rings()
1270 if (priv->extend_desc) { in stmmac_display_rx_rings()
1271 head_rx = (void *)rx_q->dma_erx; in stmmac_display_rx_rings()
1274 head_rx = (void *)rx_q->dma_rx; in stmmac_display_rx_rings()
1278 /* Display RX ring */ in stmmac_display_rx_rings()
1279 stmmac_display_ring(priv, head_rx, priv->dma_rx_size, true, in stmmac_display_rx_rings()
1280 rx_q->dma_rx_phy, desc_size); in stmmac_display_rx_rings()
1286 u32 tx_cnt = priv->plat->tx_queues_to_use; in stmmac_display_tx_rings()
1289 u32 queue; in stmmac_display_tx_rings() local
1292 for (queue = 0; queue < tx_cnt; queue++) { in stmmac_display_tx_rings()
1293 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; in stmmac_display_tx_rings()
1295 pr_info("\tTX Queue %d rings\n", queue); in stmmac_display_tx_rings()
1297 if (priv->extend_desc) { in stmmac_display_tx_rings()
1298 head_tx = (void *)tx_q->dma_etx; in stmmac_display_tx_rings()
1300 } else if (tx_q->tbs & STMMAC_TBS_AVAIL) { in stmmac_display_tx_rings()
1301 head_tx = (void *)tx_q->dma_entx; in stmmac_display_tx_rings()
1304 head_tx = (void *)tx_q->dma_tx; in stmmac_display_tx_rings()
1308 stmmac_display_ring(priv, head_tx, priv->dma_tx_size, false, in stmmac_display_tx_rings()
1309 tx_q->dma_tx_phy, desc_size); in stmmac_display_tx_rings()
1315 /* Display RX ring */ in stmmac_display_rings()
1341 * stmmac_clear_rx_descriptors - clear RX descriptors
1343 * @queue: RX queue index
1344 * Description: this function is called to clear the RX descriptors
1347 static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv, u32 queue) in stmmac_clear_rx_descriptors() argument
1349 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; in stmmac_clear_rx_descriptors()
1352 /* Clear the RX descriptors */ in stmmac_clear_rx_descriptors()
1353 for (i = 0; i < priv->dma_rx_size; i++) in stmmac_clear_rx_descriptors()
1354 if (priv->extend_desc) in stmmac_clear_rx_descriptors()
1355 stmmac_init_rx_desc(priv, &rx_q->dma_erx[i].basic, in stmmac_clear_rx_descriptors()
1356 priv->use_riwt, priv->mode, in stmmac_clear_rx_descriptors()
1357 (i == priv->dma_rx_size - 1), in stmmac_clear_rx_descriptors()
1358 priv->dma_buf_sz); in stmmac_clear_rx_descriptors()
1360 stmmac_init_rx_desc(priv, &rx_q->dma_rx[i], in stmmac_clear_rx_descriptors()
1361 priv->use_riwt, priv->mode, in stmmac_clear_rx_descriptors()
1362 (i == priv->dma_rx_size - 1), in stmmac_clear_rx_descriptors()
1363 priv->dma_buf_sz); in stmmac_clear_rx_descriptors()
1367 * stmmac_clear_tx_descriptors - clear tx descriptors
1369 * @queue: TX queue index.
1373 static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, u32 queue) in stmmac_clear_tx_descriptors() argument
1375 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; in stmmac_clear_tx_descriptors()
1379 for (i = 0; i < priv->dma_tx_size; i++) { in stmmac_clear_tx_descriptors()
1380 int last = (i == (priv->dma_tx_size - 1)); in stmmac_clear_tx_descriptors()
1383 if (priv->extend_desc) in stmmac_clear_tx_descriptors()
1384 p = &tx_q->dma_etx[i].basic; in stmmac_clear_tx_descriptors()
1385 else if (tx_q->tbs & STMMAC_TBS_AVAIL) in stmmac_clear_tx_descriptors()
1386 p = &tx_q->dma_entx[i].basic; in stmmac_clear_tx_descriptors()
1388 p = &tx_q->dma_tx[i]; in stmmac_clear_tx_descriptors()
1390 stmmac_init_tx_desc(priv, p, priv->mode, last); in stmmac_clear_tx_descriptors()
1395 * stmmac_clear_descriptors - clear descriptors
1397 * Description: this function is called to clear the TX and RX descriptors
1402 u32 rx_queue_cnt = priv->plat->rx_queues_to_use; in stmmac_clear_descriptors()
1403 u32 tx_queue_cnt = priv->plat->tx_queues_to_use; in stmmac_clear_descriptors()
1404 u32 queue; in stmmac_clear_descriptors() local
1406 /* Clear the RX descriptors */ in stmmac_clear_descriptors()
1407 for (queue = 0; queue < rx_queue_cnt; queue++) in stmmac_clear_descriptors()
1408 stmmac_clear_rx_descriptors(priv, queue); in stmmac_clear_descriptors()
1411 for (queue = 0; queue < tx_queue_cnt; queue++) in stmmac_clear_descriptors()
1412 stmmac_clear_tx_descriptors(priv, queue); in stmmac_clear_descriptors()
1416 * stmmac_init_rx_buffers - init the RX descriptor buffer.
1421 * @queue: RX queue index
1426 int i, gfp_t flags, u32 queue) in stmmac_init_rx_buffers() argument
1428 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; in stmmac_init_rx_buffers()
1429 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; in stmmac_init_rx_buffers()
1431 if (!buf->page) { in stmmac_init_rx_buffers()
1432 buf->page = page_pool_dev_alloc_pages(rx_q->page_pool); in stmmac_init_rx_buffers()
1433 if (!buf->page) in stmmac_init_rx_buffers()
1434 return -ENOMEM; in stmmac_init_rx_buffers()
1435 buf->page_offset = stmmac_rx_offset(priv); in stmmac_init_rx_buffers()
1438 if (priv->sph && !buf->sec_page) { in stmmac_init_rx_buffers()
1439 buf->sec_page = page_pool_dev_alloc_pages(rx_q->page_pool); in stmmac_init_rx_buffers()
1440 if (!buf->sec_page) in stmmac_init_rx_buffers()
1441 return -ENOMEM; in stmmac_init_rx_buffers()
1443 buf->sec_addr = page_pool_get_dma_addr(buf->sec_page); in stmmac_init_rx_buffers()
1444 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true); in stmmac_init_rx_buffers()
1446 buf->sec_page = NULL; in stmmac_init_rx_buffers()
1447 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false); in stmmac_init_rx_buffers()
1450 buf->addr = page_pool_get_dma_addr(buf->page) + buf->page_offset; in stmmac_init_rx_buffers()
1452 stmmac_set_desc_addr(priv, p, buf->addr); in stmmac_init_rx_buffers()
1453 if (priv->dma_buf_sz == BUF_SIZE_16KiB) in stmmac_init_rx_buffers()
1460 * stmmac_free_rx_buffer - free RX dma buffers
1462 * @queue: RX queue index
1465 static void stmmac_free_rx_buffer(struct stmmac_priv *priv, u32 queue, int i) in stmmac_free_rx_buffer() argument
1467 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; in stmmac_free_rx_buffer()
1468 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; in stmmac_free_rx_buffer()
1470 if (buf->page) in stmmac_free_rx_buffer()
1471 page_pool_put_full_page(rx_q->page_pool, buf->page, false); in stmmac_free_rx_buffer()
1472 buf->page = NULL; in stmmac_free_rx_buffer()
1474 if (buf->sec_page) in stmmac_free_rx_buffer()
1475 page_pool_put_full_page(rx_q->page_pool, buf->sec_page, false); in stmmac_free_rx_buffer()
1476 buf->sec_page = NULL; in stmmac_free_rx_buffer()
1480 * stmmac_free_tx_buffer - free RX dma buffers
1482 * @queue: RX queue index
1485 static void stmmac_free_tx_buffer(struct stmmac_priv *priv, u32 queue, int i) in stmmac_free_tx_buffer() argument
1487 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; in stmmac_free_tx_buffer()
1489 if (tx_q->tx_skbuff_dma[i].buf && in stmmac_free_tx_buffer()
1490 tx_q->tx_skbuff_dma[i].buf_type != STMMAC_TXBUF_T_XDP_TX) { in stmmac_free_tx_buffer()
1491 if (tx_q->tx_skbuff_dma[i].map_as_page) in stmmac_free_tx_buffer()
1492 dma_unmap_page(priv->device, in stmmac_free_tx_buffer()
1493 tx_q->tx_skbuff_dma[i].buf, in stmmac_free_tx_buffer()
1494 tx_q->tx_skbuff_dma[i].len, in stmmac_free_tx_buffer()
1497 dma_unmap_single(priv->device, in stmmac_free_tx_buffer()
1498 tx_q->tx_skbuff_dma[i].buf, in stmmac_free_tx_buffer()
1499 tx_q->tx_skbuff_dma[i].len, in stmmac_free_tx_buffer()
1503 if (tx_q->xdpf[i] && in stmmac_free_tx_buffer()
1504 (tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XDP_TX || in stmmac_free_tx_buffer()
1505 tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XDP_NDO)) { in stmmac_free_tx_buffer()
1506 xdp_return_frame(tx_q->xdpf[i]); in stmmac_free_tx_buffer()
1507 tx_q->xdpf[i] = NULL; in stmmac_free_tx_buffer()
1510 if (tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XSK_TX) in stmmac_free_tx_buffer()
1511 tx_q->xsk_frames_done++; in stmmac_free_tx_buffer()
1513 if (tx_q->tx_skbuff[i] && in stmmac_free_tx_buffer()
1514 tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_SKB) { in stmmac_free_tx_buffer()
1515 dev_kfree_skb_any(tx_q->tx_skbuff[i]); in stmmac_free_tx_buffer()
1516 tx_q->tx_skbuff[i] = NULL; in stmmac_free_tx_buffer()
1519 tx_q->tx_skbuff_dma[i].buf = 0; in stmmac_free_tx_buffer()
1520 tx_q->tx_skbuff_dma[i].map_as_page = false; in stmmac_free_tx_buffer()
1524 * dma_free_rx_skbufs - free RX dma buffers
1526 * @queue: RX queue index
1528 static void dma_free_rx_skbufs(struct stmmac_priv *priv, u32 queue) in dma_free_rx_skbufs() argument
1532 for (i = 0; i < priv->dma_rx_size; i++) in dma_free_rx_skbufs()
1533 stmmac_free_rx_buffer(priv, queue, i); in dma_free_rx_skbufs()
1536 static int stmmac_alloc_rx_buffers(struct stmmac_priv *priv, u32 queue, in stmmac_alloc_rx_buffers() argument
1539 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; in stmmac_alloc_rx_buffers()
1542 for (i = 0; i < priv->dma_rx_size; i++) { in stmmac_alloc_rx_buffers()
1546 if (priv->extend_desc) in stmmac_alloc_rx_buffers()
1547 p = &((rx_q->dma_erx + i)->basic); in stmmac_alloc_rx_buffers()
1549 p = rx_q->dma_rx + i; in stmmac_alloc_rx_buffers()
1552 queue); in stmmac_alloc_rx_buffers()
1556 rx_q->buf_alloc_num++; in stmmac_alloc_rx_buffers()
1563 * dma_free_rx_xskbufs - free RX dma buffers from XSK pool
1565 * @queue: RX queue index
1567 static void dma_free_rx_xskbufs(struct stmmac_priv *priv, u32 queue) in dma_free_rx_xskbufs() argument
1569 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; in dma_free_rx_xskbufs()
1572 for (i = 0; i < priv->dma_rx_size; i++) { in dma_free_rx_xskbufs()
1573 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; in dma_free_rx_xskbufs()
1575 if (!buf->xdp) in dma_free_rx_xskbufs()
1578 xsk_buff_free(buf->xdp); in dma_free_rx_xskbufs()
1579 buf->xdp = NULL; in dma_free_rx_xskbufs()
1583 static int stmmac_alloc_rx_buffers_zc(struct stmmac_priv *priv, u32 queue) in stmmac_alloc_rx_buffers_zc() argument
1585 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; in stmmac_alloc_rx_buffers_zc()
1588 for (i = 0; i < priv->dma_rx_size; i++) { in stmmac_alloc_rx_buffers_zc()
1593 if (priv->extend_desc) in stmmac_alloc_rx_buffers_zc()
1594 p = (struct dma_desc *)(rx_q->dma_erx + i); in stmmac_alloc_rx_buffers_zc()
1596 p = rx_q->dma_rx + i; in stmmac_alloc_rx_buffers_zc()
1598 buf = &rx_q->buf_pool[i]; in stmmac_alloc_rx_buffers_zc()
1600 buf->xdp = xsk_buff_alloc(rx_q->xsk_pool); in stmmac_alloc_rx_buffers_zc()
1601 if (!buf->xdp) in stmmac_alloc_rx_buffers_zc()
1602 return -ENOMEM; in stmmac_alloc_rx_buffers_zc()
1604 dma_addr = xsk_buff_xdp_get_dma(buf->xdp); in stmmac_alloc_rx_buffers_zc()
1606 rx_q->buf_alloc_num++; in stmmac_alloc_rx_buffers_zc()
1612 static struct xsk_buff_pool *stmmac_get_xsk_pool(struct stmmac_priv *priv, u32 queue) in stmmac_get_xsk_pool() argument
1614 if (!stmmac_xdp_is_enabled(priv) || !test_bit(queue, priv->af_xdp_zc_qps)) in stmmac_get_xsk_pool()
1617 return xsk_get_pool_from_qid(priv->dev, queue); in stmmac_get_xsk_pool()
1621 * __init_dma_rx_desc_rings - init the RX descriptor ring (per queue)
1623 * @queue: RX queue index
1625 * Description: this function initializes the DMA RX descriptors
1629 static int __init_dma_rx_desc_rings(struct stmmac_priv *priv, u32 queue, gfp_t flags) in __init_dma_rx_desc_rings() argument
1631 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; in __init_dma_rx_desc_rings()
1634 netif_dbg(priv, probe, priv->dev, in __init_dma_rx_desc_rings()
1636 (u32)rx_q->dma_rx_phy); in __init_dma_rx_desc_rings()
1638 stmmac_clear_rx_descriptors(priv, queue); in __init_dma_rx_desc_rings()
1640 xdp_rxq_info_unreg_mem_model(&rx_q->xdp_rxq); in __init_dma_rx_desc_rings()
1642 rx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue); in __init_dma_rx_desc_rings()
1644 if (rx_q->xsk_pool) { in __init_dma_rx_desc_rings()
1645 WARN_ON(xdp_rxq_info_reg_mem_model(&rx_q->xdp_rxq, in __init_dma_rx_desc_rings()
1648 netdev_info(priv->dev, in __init_dma_rx_desc_rings()
1649 "Register MEM_TYPE_XSK_BUFF_POOL RxQ-%d\n", in __init_dma_rx_desc_rings()
1650 rx_q->queue_index); in __init_dma_rx_desc_rings()
1651 xsk_pool_set_rxq_info(rx_q->xsk_pool, &rx_q->xdp_rxq); in __init_dma_rx_desc_rings()
1653 WARN_ON(xdp_rxq_info_reg_mem_model(&rx_q->xdp_rxq, in __init_dma_rx_desc_rings()
1655 rx_q->page_pool)); in __init_dma_rx_desc_rings()
1656 netdev_info(priv->dev, in __init_dma_rx_desc_rings()
1657 "Register MEM_TYPE_PAGE_POOL RxQ-%d\n", in __init_dma_rx_desc_rings()
1658 rx_q->queue_index); in __init_dma_rx_desc_rings()
1661 if (rx_q->xsk_pool) { in __init_dma_rx_desc_rings()
1662 /* RX XDP ZC buffer pool may not be populated, e.g. in __init_dma_rx_desc_rings()
1663 * xdpsock TX-only. in __init_dma_rx_desc_rings()
1665 stmmac_alloc_rx_buffers_zc(priv, queue); in __init_dma_rx_desc_rings()
1667 ret = stmmac_alloc_rx_buffers(priv, queue, flags); in __init_dma_rx_desc_rings()
1669 return -ENOMEM; in __init_dma_rx_desc_rings()
1672 rx_q->cur_rx = 0; in __init_dma_rx_desc_rings()
1673 rx_q->dirty_rx = 0; in __init_dma_rx_desc_rings()
1676 if (priv->mode == STMMAC_CHAIN_MODE) { in __init_dma_rx_desc_rings()
1677 if (priv->extend_desc) in __init_dma_rx_desc_rings()
1678 stmmac_mode_init(priv, rx_q->dma_erx, in __init_dma_rx_desc_rings()
1679 rx_q->dma_rx_phy, in __init_dma_rx_desc_rings()
1680 priv->dma_rx_size, 1); in __init_dma_rx_desc_rings()
1682 stmmac_mode_init(priv, rx_q->dma_rx, in __init_dma_rx_desc_rings()
1683 rx_q->dma_rx_phy, in __init_dma_rx_desc_rings()
1684 priv->dma_rx_size, 0); in __init_dma_rx_desc_rings()
1693 u32 rx_count = priv->plat->rx_queues_to_use; in init_dma_rx_desc_rings()
1694 u32 queue; in init_dma_rx_desc_rings() local
1697 /* RX INITIALIZATION */ in init_dma_rx_desc_rings()
1698 netif_dbg(priv, probe, priv->dev, in init_dma_rx_desc_rings()
1701 for (queue = 0; queue < rx_count; queue++) { in init_dma_rx_desc_rings()
1702 ret = __init_dma_rx_desc_rings(priv, queue, flags); in init_dma_rx_desc_rings()
1710 while (queue >= 0) { in init_dma_rx_desc_rings()
1711 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; in init_dma_rx_desc_rings()
1713 if (rx_q->xsk_pool) in init_dma_rx_desc_rings()
1714 dma_free_rx_xskbufs(priv, queue); in init_dma_rx_desc_rings()
1716 dma_free_rx_skbufs(priv, queue); in init_dma_rx_desc_rings()
1718 rx_q->buf_alloc_num = 0; in init_dma_rx_desc_rings()
1719 rx_q->xsk_pool = NULL; in init_dma_rx_desc_rings()
1721 if (queue == 0) in init_dma_rx_desc_rings()
1724 queue--; in init_dma_rx_desc_rings()
1731 * __init_dma_tx_desc_rings - init the TX descriptor ring (per queue)
1733 * @queue : TX queue index
1738 static int __init_dma_tx_desc_rings(struct stmmac_priv *priv, u32 queue) in __init_dma_tx_desc_rings() argument
1740 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; in __init_dma_tx_desc_rings()
1743 netif_dbg(priv, probe, priv->dev, in __init_dma_tx_desc_rings()
1745 (u32)tx_q->dma_tx_phy); in __init_dma_tx_desc_rings()
1748 if (priv->mode == STMMAC_CHAIN_MODE) { in __init_dma_tx_desc_rings()
1749 if (priv->extend_desc) in __init_dma_tx_desc_rings()
1750 stmmac_mode_init(priv, tx_q->dma_etx, in __init_dma_tx_desc_rings()
1751 tx_q->dma_tx_phy, in __init_dma_tx_desc_rings()
1752 priv->dma_tx_size, 1); in __init_dma_tx_desc_rings()
1753 else if (!(tx_q->tbs & STMMAC_TBS_AVAIL)) in __init_dma_tx_desc_rings()
1754 stmmac_mode_init(priv, tx_q->dma_tx, in __init_dma_tx_desc_rings()
1755 tx_q->dma_tx_phy, in __init_dma_tx_desc_rings()
1756 priv->dma_tx_size, 0); in __init_dma_tx_desc_rings()
1759 tx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue); in __init_dma_tx_desc_rings()
1761 for (i = 0; i < priv->dma_tx_size; i++) { in __init_dma_tx_desc_rings()
1764 if (priv->extend_desc) in __init_dma_tx_desc_rings()
1765 p = &((tx_q->dma_etx + i)->basic); in __init_dma_tx_desc_rings()
1766 else if (tx_q->tbs & STMMAC_TBS_AVAIL) in __init_dma_tx_desc_rings()
1767 p = &((tx_q->dma_entx + i)->basic); in __init_dma_tx_desc_rings()
1769 p = tx_q->dma_tx + i; in __init_dma_tx_desc_rings()
1773 tx_q->tx_skbuff_dma[i].buf = 0; in __init_dma_tx_desc_rings()
1774 tx_q->tx_skbuff_dma[i].map_as_page = false; in __init_dma_tx_desc_rings()
1775 tx_q->tx_skbuff_dma[i].len = 0; in __init_dma_tx_desc_rings()
1776 tx_q->tx_skbuff_dma[i].last_segment = false; in __init_dma_tx_desc_rings()
1777 tx_q->tx_skbuff[i] = NULL; in __init_dma_tx_desc_rings()
1780 tx_q->dirty_tx = 0; in __init_dma_tx_desc_rings()
1781 tx_q->cur_tx = 0; in __init_dma_tx_desc_rings()
1782 tx_q->mss = 0; in __init_dma_tx_desc_rings()
1784 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue)); in __init_dma_tx_desc_rings()
1793 u32 queue; in init_dma_tx_desc_rings() local
1795 tx_queue_cnt = priv->plat->tx_queues_to_use; in init_dma_tx_desc_rings()
1797 for (queue = 0; queue < tx_queue_cnt; queue++) in init_dma_tx_desc_rings()
1798 __init_dma_tx_desc_rings(priv, queue); in init_dma_tx_desc_rings()
1804 * init_dma_desc_rings - init the RX/TX descriptor rings
1807 * Description: this function initializes the DMA RX/TX descriptors
1831 * dma_free_tx_skbufs - free TX dma buffers
1833 * @queue: TX queue index
1835 static void dma_free_tx_skbufs(struct stmmac_priv *priv, u32 queue) in dma_free_tx_skbufs() argument
1837 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; in dma_free_tx_skbufs()
1840 tx_q->xsk_frames_done = 0; in dma_free_tx_skbufs()
1842 for (i = 0; i < priv->dma_tx_size; i++) in dma_free_tx_skbufs()
1843 stmmac_free_tx_buffer(priv, queue, i); in dma_free_tx_skbufs()
1845 if (tx_q->xsk_pool && tx_q->xsk_frames_done) { in dma_free_tx_skbufs()
1846 xsk_tx_completed(tx_q->xsk_pool, tx_q->xsk_frames_done); in dma_free_tx_skbufs()
1847 tx_q->xsk_frames_done = 0; in dma_free_tx_skbufs()
1848 tx_q->xsk_pool = NULL; in dma_free_tx_skbufs()
1853 * stmmac_free_tx_skbufs - free TX skb buffers
1858 u32 tx_queue_cnt = priv->plat->tx_queues_to_use; in stmmac_free_tx_skbufs()
1859 u32 queue; in stmmac_free_tx_skbufs() local
1861 for (queue = 0; queue < tx_queue_cnt; queue++) in stmmac_free_tx_skbufs()
1862 dma_free_tx_skbufs(priv, queue); in stmmac_free_tx_skbufs()
1866 * __free_dma_rx_desc_resources - free RX dma desc resources (per queue)
1868 * @queue: RX queue index
1870 static void __free_dma_rx_desc_resources(struct stmmac_priv *priv, u32 queue) in __free_dma_rx_desc_resources() argument
1872 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; in __free_dma_rx_desc_resources()
1874 /* Release the DMA RX socket buffers */ in __free_dma_rx_desc_resources()
1875 if (rx_q->xsk_pool) in __free_dma_rx_desc_resources()
1876 dma_free_rx_xskbufs(priv, queue); in __free_dma_rx_desc_resources()
1878 dma_free_rx_skbufs(priv, queue); in __free_dma_rx_desc_resources()
1880 rx_q->buf_alloc_num = 0; in __free_dma_rx_desc_resources()
1881 rx_q->xsk_pool = NULL; in __free_dma_rx_desc_resources()
1884 if (!priv->extend_desc) in __free_dma_rx_desc_resources()
1885 dma_free_coherent(priv->device, priv->dma_rx_size * in __free_dma_rx_desc_resources()
1887 rx_q->dma_rx, rx_q->dma_rx_phy); in __free_dma_rx_desc_resources()
1889 dma_free_coherent(priv->device, priv->dma_rx_size * in __free_dma_rx_desc_resources()
1891 rx_q->dma_erx, rx_q->dma_rx_phy); in __free_dma_rx_desc_resources()
1893 if (xdp_rxq_info_is_reg(&rx_q->xdp_rxq)) in __free_dma_rx_desc_resources()
1894 xdp_rxq_info_unreg(&rx_q->xdp_rxq); in __free_dma_rx_desc_resources()
1896 kfree(rx_q->buf_pool); in __free_dma_rx_desc_resources()
1897 if (rx_q->page_pool) in __free_dma_rx_desc_resources()
1898 page_pool_destroy(rx_q->page_pool); in __free_dma_rx_desc_resources()
1903 u32 rx_count = priv->plat->rx_queues_to_use; in free_dma_rx_desc_resources()
1904 u32 queue; in free_dma_rx_desc_resources() local
1906 /* Free RX queue resources */ in free_dma_rx_desc_resources()
1907 for (queue = 0; queue < rx_count; queue++) in free_dma_rx_desc_resources()
1908 __free_dma_rx_desc_resources(priv, queue); in free_dma_rx_desc_resources()
1912 * __free_dma_tx_desc_resources - free TX dma desc resources (per queue)
1914 * @queue: TX queue index
1916 static void __free_dma_tx_desc_resources(struct stmmac_priv *priv, u32 queue) in __free_dma_tx_desc_resources() argument
1918 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; in __free_dma_tx_desc_resources()
1923 dma_free_tx_skbufs(priv, queue); in __free_dma_tx_desc_resources()
1925 if (priv->extend_desc) { in __free_dma_tx_desc_resources()
1927 addr = tx_q->dma_etx; in __free_dma_tx_desc_resources()
1928 } else if (tx_q->tbs & STMMAC_TBS_AVAIL) { in __free_dma_tx_desc_resources()
1930 addr = tx_q->dma_entx; in __free_dma_tx_desc_resources()
1933 addr = tx_q->dma_tx; in __free_dma_tx_desc_resources()
1936 size *= priv->dma_tx_size; in __free_dma_tx_desc_resources()
1938 dma_free_coherent(priv->device, size, addr, tx_q->dma_tx_phy); in __free_dma_tx_desc_resources()
1940 kfree(tx_q->tx_skbuff_dma); in __free_dma_tx_desc_resources()
1941 kfree(tx_q->tx_skbuff); in __free_dma_tx_desc_resources()
1946 u32 tx_count = priv->plat->tx_queues_to_use; in free_dma_tx_desc_resources()
1947 u32 queue; in free_dma_tx_desc_resources() local
1949 /* Free TX queue resources */ in free_dma_tx_desc_resources()
1950 for (queue = 0; queue < tx_count; queue++) in free_dma_tx_desc_resources()
1951 __free_dma_tx_desc_resources(priv, queue); in free_dma_tx_desc_resources()
1955 * __alloc_dma_rx_desc_resources - alloc RX resources (per queue).
1957 * @queue: RX queue index
1959 * this function allocates the resources for TX and RX paths. In case of
1960 * reception, for example, it pre-allocated the RX socket buffer in order to
1961 * allow zero-copy mechanism.
1963 static int __alloc_dma_rx_desc_resources(struct stmmac_priv *priv, u32 queue) in __alloc_dma_rx_desc_resources() argument
1965 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; in __alloc_dma_rx_desc_resources()
1966 struct stmmac_channel *ch = &priv->channel[queue]; in __alloc_dma_rx_desc_resources()
1973 rx_q->queue_index = queue; in __alloc_dma_rx_desc_resources()
1974 rx_q->priv_data = priv; in __alloc_dma_rx_desc_resources()
1977 pp_params.pool_size = priv->dma_rx_size; in __alloc_dma_rx_desc_resources()
1978 num_pages = DIV_ROUND_UP(priv->dma_buf_sz, PAGE_SIZE); in __alloc_dma_rx_desc_resources()
1980 pp_params.nid = dev_to_node(priv->device); in __alloc_dma_rx_desc_resources()
1981 pp_params.dev = priv->device; in __alloc_dma_rx_desc_resources()
1986 rx_q->page_pool = page_pool_create(&pp_params); in __alloc_dma_rx_desc_resources()
1987 if (IS_ERR(rx_q->page_pool)) { in __alloc_dma_rx_desc_resources()
1988 ret = PTR_ERR(rx_q->page_pool); in __alloc_dma_rx_desc_resources()
1989 rx_q->page_pool = NULL; in __alloc_dma_rx_desc_resources()
1993 rx_q->buf_pool = kcalloc(priv->dma_rx_size, in __alloc_dma_rx_desc_resources()
1994 sizeof(*rx_q->buf_pool), in __alloc_dma_rx_desc_resources()
1996 if (!rx_q->buf_pool) in __alloc_dma_rx_desc_resources()
1997 return -ENOMEM; in __alloc_dma_rx_desc_resources()
1999 if (priv->extend_desc) { in __alloc_dma_rx_desc_resources()
2000 rx_q->dma_erx = dma_alloc_coherent(priv->device, in __alloc_dma_rx_desc_resources()
2001 priv->dma_rx_size * in __alloc_dma_rx_desc_resources()
2003 &rx_q->dma_rx_phy, in __alloc_dma_rx_desc_resources()
2005 if (!rx_q->dma_erx) in __alloc_dma_rx_desc_resources()
2006 return -ENOMEM; in __alloc_dma_rx_desc_resources()
2009 rx_q->dma_rx = dma_alloc_coherent(priv->device, in __alloc_dma_rx_desc_resources()
2010 priv->dma_rx_size * in __alloc_dma_rx_desc_resources()
2012 &rx_q->dma_rx_phy, in __alloc_dma_rx_desc_resources()
2014 if (!rx_q->dma_rx) in __alloc_dma_rx_desc_resources()
2015 return -ENOMEM; in __alloc_dma_rx_desc_resources()
2019 test_bit(queue, priv->af_xdp_zc_qps)) in __alloc_dma_rx_desc_resources()
2020 napi_id = ch->rxtx_napi.napi_id; in __alloc_dma_rx_desc_resources()
2022 napi_id = ch->rx_napi.napi_id; in __alloc_dma_rx_desc_resources()
2024 ret = xdp_rxq_info_reg(&rx_q->xdp_rxq, priv->dev, in __alloc_dma_rx_desc_resources()
2025 rx_q->queue_index, in __alloc_dma_rx_desc_resources()
2028 netdev_err(priv->dev, "Failed to register xdp rxq info\n"); in __alloc_dma_rx_desc_resources()
2029 return -EINVAL; in __alloc_dma_rx_desc_resources()
2037 u32 rx_count = priv->plat->rx_queues_to_use; in alloc_dma_rx_desc_resources()
2038 u32 queue; in alloc_dma_rx_desc_resources() local
2041 /* RX queues buffers and DMA */ in alloc_dma_rx_desc_resources()
2042 for (queue = 0; queue < rx_count; queue++) { in alloc_dma_rx_desc_resources()
2043 ret = __alloc_dma_rx_desc_resources(priv, queue); in alloc_dma_rx_desc_resources()
2057 * __alloc_dma_tx_desc_resources - alloc TX resources (per queue).
2059 * @queue: TX queue index
2061 * this function allocates the resources for TX and RX paths. In case of
2062 * reception, for example, it pre-allocated the RX socket buffer in order to
2063 * allow zero-copy mechanism.
2065 static int __alloc_dma_tx_desc_resources(struct stmmac_priv *priv, u32 queue) in __alloc_dma_tx_desc_resources() argument
2067 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; in __alloc_dma_tx_desc_resources()
2071 tx_q->queue_index = queue; in __alloc_dma_tx_desc_resources()
2072 tx_q->priv_data = priv; in __alloc_dma_tx_desc_resources()
2074 tx_q->tx_skbuff_dma = kcalloc(priv->dma_tx_size, in __alloc_dma_tx_desc_resources()
2075 sizeof(*tx_q->tx_skbuff_dma), in __alloc_dma_tx_desc_resources()
2077 if (!tx_q->tx_skbuff_dma) in __alloc_dma_tx_desc_resources()
2078 return -ENOMEM; in __alloc_dma_tx_desc_resources()
2080 tx_q->tx_skbuff = kcalloc(priv->dma_tx_size, in __alloc_dma_tx_desc_resources()
2083 if (!tx_q->tx_skbuff) in __alloc_dma_tx_desc_resources()
2084 return -ENOMEM; in __alloc_dma_tx_desc_resources()
2086 if (priv->extend_desc) in __alloc_dma_tx_desc_resources()
2088 else if (tx_q->tbs & STMMAC_TBS_AVAIL) in __alloc_dma_tx_desc_resources()
2093 size *= priv->dma_tx_size; in __alloc_dma_tx_desc_resources()
2095 addr = dma_alloc_coherent(priv->device, size, in __alloc_dma_tx_desc_resources()
2096 &tx_q->dma_tx_phy, GFP_KERNEL); in __alloc_dma_tx_desc_resources()
2098 return -ENOMEM; in __alloc_dma_tx_desc_resources()
2100 if (priv->extend_desc) in __alloc_dma_tx_desc_resources()
2101 tx_q->dma_etx = addr; in __alloc_dma_tx_desc_resources()
2102 else if (tx_q->tbs & STMMAC_TBS_AVAIL) in __alloc_dma_tx_desc_resources()
2103 tx_q->dma_entx = addr; in __alloc_dma_tx_desc_resources()
2105 tx_q->dma_tx = addr; in __alloc_dma_tx_desc_resources()
2112 u32 tx_count = priv->plat->tx_queues_to_use; in alloc_dma_tx_desc_resources()
2113 u32 queue; in alloc_dma_tx_desc_resources() local
2117 for (queue = 0; queue < tx_count; queue++) { in alloc_dma_tx_desc_resources()
2118 ret = __alloc_dma_tx_desc_resources(priv, queue); in alloc_dma_tx_desc_resources()
2131 * alloc_dma_desc_resources - alloc TX/RX resources.
2134 * this function allocates the resources for TX and RX paths. In case of
2135 * reception, for example, it pre-allocated the RX socket buffer in order to
2136 * allow zero-copy mechanism.
2140 /* RX Allocation */ in alloc_dma_desc_resources()
2152 * free_dma_desc_resources - free dma desc resources
2160 /* Release the DMA RX socket buffers later in free_dma_desc_resources()
2167 * stmmac_mac_enable_rx_queues - Enable MAC rx queues
2169 * Description: It is used for enabling the rx queues in the MAC
2173 u32 rx_queues_count = priv->plat->rx_queues_to_use; in stmmac_mac_enable_rx_queues()
2174 int queue; in stmmac_mac_enable_rx_queues() local
2177 for (queue = 0; queue < rx_queues_count; queue++) { in stmmac_mac_enable_rx_queues()
2178 mode = priv->plat->rx_queues_cfg[queue].mode_to_use; in stmmac_mac_enable_rx_queues()
2179 stmmac_rx_queue_enable(priv, priv->hw, mode, queue); in stmmac_mac_enable_rx_queues()
2184 * stmmac_start_rx_dma - start RX DMA channel
2186 * @chan: RX channel index
2188 * This starts a RX DMA channel
2192 netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan); in stmmac_start_rx_dma()
2193 stmmac_start_rx(priv, priv->ioaddr, chan); in stmmac_start_rx_dma()
2197 * stmmac_start_tx_dma - start TX DMA channel
2205 netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan); in stmmac_start_tx_dma()
2206 stmmac_start_tx(priv, priv->ioaddr, chan); in stmmac_start_tx_dma()
2210 * stmmac_stop_rx_dma - stop RX DMA channel
2212 * @chan: RX channel index
2214 * This stops a RX DMA channel
2218 netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan); in stmmac_stop_rx_dma()
2219 stmmac_stop_rx(priv, priv->ioaddr, chan); in stmmac_stop_rx_dma()
2223 * stmmac_stop_tx_dma - stop TX DMA channel
2231 netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan); in stmmac_stop_tx_dma()
2232 stmmac_stop_tx(priv, priv->ioaddr, chan); in stmmac_stop_tx_dma()
2236 * stmmac_start_all_dma - start all RX and TX DMA channels
2239 * This starts all the RX and TX DMA channels
2243 u32 rx_channels_count = priv->plat->rx_queues_to_use; in stmmac_start_all_dma()
2244 u32 tx_channels_count = priv->plat->tx_queues_to_use; in stmmac_start_all_dma()
2255 * stmmac_stop_all_dma - stop all RX and TX DMA channels
2258 * This stops the RX and TX DMA channels
2262 u32 rx_channels_count = priv->plat->rx_queues_to_use; in stmmac_stop_all_dma()
2263 u32 tx_channels_count = priv->plat->tx_queues_to_use; in stmmac_stop_all_dma()
2274 * stmmac_dma_operation_mode - HW DMA operation mode
2277 * order to program the tx/rx DMA thresholds or Store-And-Forward mode.
2281 u32 rx_channels_count = priv->plat->rx_queues_to_use; in stmmac_dma_operation_mode()
2282 u32 tx_channels_count = priv->plat->tx_queues_to_use; in stmmac_dma_operation_mode()
2283 int rxfifosz = priv->plat->rx_fifo_size; in stmmac_dma_operation_mode()
2284 int txfifosz = priv->plat->tx_fifo_size; in stmmac_dma_operation_mode()
2291 rxfifosz = priv->dma_cap.rx_fifo_size; in stmmac_dma_operation_mode()
2293 txfifosz = priv->dma_cap.tx_fifo_size; in stmmac_dma_operation_mode()
2295 /* Adjust for real per queue fifo size */ in stmmac_dma_operation_mode()
2299 if (priv->plat->force_thresh_dma_mode) { in stmmac_dma_operation_mode()
2302 } else if (priv->plat->force_sf_dma_mode || priv->plat->tx_coe) { in stmmac_dma_operation_mode()
2312 priv->xstats.threshold = SF_DMA_MODE; in stmmac_dma_operation_mode()
2320 struct stmmac_rx_queue *rx_q = &priv->rx_queue[chan]; in stmmac_dma_operation_mode()
2323 qmode = priv->plat->rx_queues_cfg[chan].mode_to_use; in stmmac_dma_operation_mode()
2325 stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, in stmmac_dma_operation_mode()
2328 if (rx_q->xsk_pool) { in stmmac_dma_operation_mode()
2329 buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool); in stmmac_dma_operation_mode()
2330 stmmac_set_dma_bfsize(priv, priv->ioaddr, in stmmac_dma_operation_mode()
2334 stmmac_set_dma_bfsize(priv, priv->ioaddr, in stmmac_dma_operation_mode()
2335 priv->dma_buf_sz, in stmmac_dma_operation_mode()
2341 qmode = priv->plat->tx_queues_cfg[chan].mode_to_use; in stmmac_dma_operation_mode()
2343 stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, in stmmac_dma_operation_mode()
2348 static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget) in stmmac_xdp_xmit_zc() argument
2350 struct netdev_queue *nq = netdev_get_tx_queue(priv->dev, queue); in stmmac_xdp_xmit_zc()
2351 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; in stmmac_xdp_xmit_zc()
2352 struct xsk_buff_pool *pool = tx_q->xsk_pool; in stmmac_xdp_xmit_zc()
2353 unsigned int entry = tx_q->cur_tx; in stmmac_xdp_xmit_zc()
2358 /* Avoids TX time-out as we are sharing with slow path */ in stmmac_xdp_xmit_zc()
2359 nq->trans_start = jiffies; in stmmac_xdp_xmit_zc()
2361 budget = min(budget, stmmac_tx_avail(priv, queue)); in stmmac_xdp_xmit_zc()
2363 while (budget-- > 0) { in stmmac_xdp_xmit_zc()
2370 if (unlikely(stmmac_tx_avail(priv, queue) < STMMAC_TX_XSK_AVAIL) || in stmmac_xdp_xmit_zc()
2371 !netif_carrier_ok(priv->dev)) { in stmmac_xdp_xmit_zc()
2379 if (likely(priv->extend_desc)) in stmmac_xdp_xmit_zc()
2380 tx_desc = (struct dma_desc *)(tx_q->dma_etx + entry); in stmmac_xdp_xmit_zc()
2381 else if (tx_q->tbs & STMMAC_TBS_AVAIL) in stmmac_xdp_xmit_zc()
2382 tx_desc = &tx_q->dma_entx[entry].basic; in stmmac_xdp_xmit_zc()
2384 tx_desc = tx_q->dma_tx + entry; in stmmac_xdp_xmit_zc()
2389 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XSK_TX; in stmmac_xdp_xmit_zc()
2395 tx_q->tx_skbuff_dma[entry].buf = 0; in stmmac_xdp_xmit_zc()
2396 tx_q->xdpf[entry] = NULL; in stmmac_xdp_xmit_zc()
2398 tx_q->tx_skbuff_dma[entry].map_as_page = false; in stmmac_xdp_xmit_zc()
2399 tx_q->tx_skbuff_dma[entry].len = xdp_desc.len; in stmmac_xdp_xmit_zc()
2400 tx_q->tx_skbuff_dma[entry].last_segment = true; in stmmac_xdp_xmit_zc()
2401 tx_q->tx_skbuff_dma[entry].is_jumbo = false; in stmmac_xdp_xmit_zc()
2405 tx_q->tx_count_frames++; in stmmac_xdp_xmit_zc()
2407 if (!priv->tx_coal_frames[queue]) in stmmac_xdp_xmit_zc()
2409 else if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0) in stmmac_xdp_xmit_zc()
2415 tx_q->tx_count_frames = 0; in stmmac_xdp_xmit_zc()
2417 priv->xstats.tx_set_ic_bit++; in stmmac_xdp_xmit_zc()
2421 true, priv->mode, true, true, in stmmac_xdp_xmit_zc()
2424 stmmac_enable_dma_transmission(priv, priv->ioaddr); in stmmac_xdp_xmit_zc()
2426 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_tx_size); in stmmac_xdp_xmit_zc()
2427 entry = tx_q->cur_tx; in stmmac_xdp_xmit_zc()
2431 stmmac_flush_tx_descriptors(priv, queue); in stmmac_xdp_xmit_zc()
2444 * stmmac_tx_clean - to manage the transmission completion
2447 * @queue: TX queue index
2450 static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue) in stmmac_tx_clean() argument
2452 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; in stmmac_tx_clean()
2456 __netif_tx_lock_bh(netdev_get_tx_queue(priv->dev, queue)); in stmmac_tx_clean()
2458 priv->xstats.tx_clean++; in stmmac_tx_clean()
2460 tx_q->xsk_frames_done = 0; in stmmac_tx_clean()
2462 entry = tx_q->dirty_tx; in stmmac_tx_clean()
2465 while ((entry != tx_q->cur_tx) && count < priv->dma_tx_size) { in stmmac_tx_clean()
2471 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_TX || in stmmac_tx_clean()
2472 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_NDO) { in stmmac_tx_clean()
2473 xdpf = tx_q->xdpf[entry]; in stmmac_tx_clean()
2475 } else if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) { in stmmac_tx_clean()
2477 skb = tx_q->tx_skbuff[entry]; in stmmac_tx_clean()
2483 if (priv->extend_desc) in stmmac_tx_clean()
2484 p = (struct dma_desc *)(tx_q->dma_etx + entry); in stmmac_tx_clean()
2485 else if (tx_q->tbs & STMMAC_TBS_AVAIL) in stmmac_tx_clean()
2486 p = &tx_q->dma_entx[entry].basic; in stmmac_tx_clean()
2488 p = tx_q->dma_tx + entry; in stmmac_tx_clean()
2490 status = stmmac_tx_status(priv, &priv->dev->stats, in stmmac_tx_clean()
2491 &priv->xstats, p, priv->ioaddr); in stmmac_tx_clean()
2507 priv->dev->stats.tx_errors++; in stmmac_tx_clean()
2509 priv->dev->stats.tx_packets++; in stmmac_tx_clean()
2510 priv->xstats.tx_pkt_n++; in stmmac_tx_clean()
2511 priv->xstats.txq_stats[queue].tx_pkt_n++; in stmmac_tx_clean()
2517 if (likely(tx_q->tx_skbuff_dma[entry].buf && in stmmac_tx_clean()
2518 tx_q->tx_skbuff_dma[entry].buf_type != STMMAC_TXBUF_T_XDP_TX)) { in stmmac_tx_clean()
2519 if (tx_q->tx_skbuff_dma[entry].map_as_page) in stmmac_tx_clean()
2520 dma_unmap_page(priv->device, in stmmac_tx_clean()
2521 tx_q->tx_skbuff_dma[entry].buf, in stmmac_tx_clean()
2522 tx_q->tx_skbuff_dma[entry].len, in stmmac_tx_clean()
2525 dma_unmap_single(priv->device, in stmmac_tx_clean()
2526 tx_q->tx_skbuff_dma[entry].buf, in stmmac_tx_clean()
2527 tx_q->tx_skbuff_dma[entry].len, in stmmac_tx_clean()
2529 tx_q->tx_skbuff_dma[entry].buf = 0; in stmmac_tx_clean()
2530 tx_q->tx_skbuff_dma[entry].len = 0; in stmmac_tx_clean()
2531 tx_q->tx_skbuff_dma[entry].map_as_page = false; in stmmac_tx_clean()
2536 tx_q->tx_skbuff_dma[entry].last_segment = false; in stmmac_tx_clean()
2537 tx_q->tx_skbuff_dma[entry].is_jumbo = false; in stmmac_tx_clean()
2540 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_TX) { in stmmac_tx_clean()
2542 tx_q->xdpf[entry] = NULL; in stmmac_tx_clean()
2546 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_NDO) { in stmmac_tx_clean()
2548 tx_q->xdpf[entry] = NULL; in stmmac_tx_clean()
2551 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XSK_TX) in stmmac_tx_clean()
2552 tx_q->xsk_frames_done++; in stmmac_tx_clean()
2554 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) { in stmmac_tx_clean()
2557 bytes_compl += skb->len; in stmmac_tx_clean()
2559 tx_q->tx_skbuff[entry] = NULL; in stmmac_tx_clean()
2563 stmmac_release_tx_desc(priv, p, priv->mode); in stmmac_tx_clean()
2565 entry = STMMAC_GET_ENTRY(entry, priv->dma_tx_size); in stmmac_tx_clean()
2567 tx_q->dirty_tx = entry; in stmmac_tx_clean()
2569 netdev_tx_completed_queue(netdev_get_tx_queue(priv->dev, queue), in stmmac_tx_clean()
2572 if (unlikely(netif_tx_queue_stopped(netdev_get_tx_queue(priv->dev, in stmmac_tx_clean()
2573 queue))) && in stmmac_tx_clean()
2574 stmmac_tx_avail(priv, queue) > STMMAC_TX_THRESH(priv)) { in stmmac_tx_clean()
2576 netif_dbg(priv, tx_done, priv->dev, in stmmac_tx_clean()
2578 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, queue)); in stmmac_tx_clean()
2581 if (tx_q->xsk_pool) { in stmmac_tx_clean()
2584 if (tx_q->xsk_frames_done) in stmmac_tx_clean()
2585 xsk_tx_completed(tx_q->xsk_pool, tx_q->xsk_frames_done); in stmmac_tx_clean()
2587 if (xsk_uses_need_wakeup(tx_q->xsk_pool)) in stmmac_tx_clean()
2588 xsk_set_tx_need_wakeup(tx_q->xsk_pool); in stmmac_tx_clean()
2592 * available), return "budget - 1" to reenable TX IRQ. in stmmac_tx_clean()
2595 work_done = stmmac_xdp_xmit_zc(priv, queue, in stmmac_tx_clean()
2598 xmits = budget - 1; in stmmac_tx_clean()
2603 if (priv->eee_enabled && !priv->tx_path_in_lpi_mode && in stmmac_tx_clean()
2604 priv->eee_sw_timer_en) { in stmmac_tx_clean()
2606 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer)); in stmmac_tx_clean()
2610 if (tx_q->dirty_tx != tx_q->cur_tx) in stmmac_tx_clean()
2611 hrtimer_start(&tx_q->txtimer, in stmmac_tx_clean()
2612 STMMAC_COAL_TIMER(priv->tx_coal_timer[queue]), in stmmac_tx_clean()
2615 __netif_tx_unlock_bh(netdev_get_tx_queue(priv->dev, queue)); in stmmac_tx_clean()
2622 * stmmac_tx_err - to manage the tx error
2630 struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan]; in stmmac_tx_err()
2632 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, chan)); in stmmac_tx_err()
2637 tx_q->dirty_tx = 0; in stmmac_tx_err()
2638 tx_q->cur_tx = 0; in stmmac_tx_err()
2639 tx_q->mss = 0; in stmmac_tx_err()
2640 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, chan)); in stmmac_tx_err()
2641 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, in stmmac_tx_err()
2642 tx_q->dma_tx_phy, chan); in stmmac_tx_err()
2645 priv->dev->stats.tx_errors++; in stmmac_tx_err()
2646 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, chan)); in stmmac_tx_err()
2650 * stmmac_set_dma_operation_mode - Set DMA operation mode by channel
2653 * @rxmode: RX operating mode
2656 * runtime in order to program the tx/rx DMA thresholds or Store-And-Forward
2662 u8 rxqmode = priv->plat->rx_queues_cfg[chan].mode_to_use; in stmmac_set_dma_operation_mode()
2663 u8 txqmode = priv->plat->tx_queues_cfg[chan].mode_to_use; in stmmac_set_dma_operation_mode()
2664 u32 rx_channels_count = priv->plat->rx_queues_to_use; in stmmac_set_dma_operation_mode()
2665 u32 tx_channels_count = priv->plat->tx_queues_to_use; in stmmac_set_dma_operation_mode()
2666 int rxfifosz = priv->plat->rx_fifo_size; in stmmac_set_dma_operation_mode()
2667 int txfifosz = priv->plat->tx_fifo_size; in stmmac_set_dma_operation_mode()
2670 rxfifosz = priv->dma_cap.rx_fifo_size; in stmmac_set_dma_operation_mode()
2672 txfifosz = priv->dma_cap.tx_fifo_size; in stmmac_set_dma_operation_mode()
2674 /* Adjust for real per queue fifo size */ in stmmac_set_dma_operation_mode()
2678 stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, rxfifosz, rxqmode); in stmmac_set_dma_operation_mode()
2679 stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, txfifosz, txqmode); in stmmac_set_dma_operation_mode()
2686 ret = stmmac_safety_feat_irq_status(priv, priv->dev, in stmmac_safety_feat_interrupt()
2687 priv->ioaddr, priv->dma_cap.asp, &priv->sstats); in stmmac_safety_feat_interrupt()
2688 if (ret && (ret != -EINVAL)) { in stmmac_safety_feat_interrupt()
2698 int status = stmmac_dma_interrupt_status(priv, priv->ioaddr, in stmmac_napi_check()
2699 &priv->xstats, chan, dir); in stmmac_napi_check()
2700 struct stmmac_rx_queue *rx_q = &priv->rx_queue[chan]; in stmmac_napi_check()
2701 struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan]; in stmmac_napi_check()
2702 struct stmmac_channel *ch = &priv->channel[chan]; in stmmac_napi_check()
2707 rx_napi = rx_q->xsk_pool ? &ch->rxtx_napi : &ch->rx_napi; in stmmac_napi_check()
2708 tx_napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi; in stmmac_napi_check()
2710 if ((status & handle_rx) && (chan < priv->plat->rx_queues_to_use)) { in stmmac_napi_check()
2712 spin_lock_irqsave(&ch->lock, flags); in stmmac_napi_check()
2713 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 0); in stmmac_napi_check()
2714 spin_unlock_irqrestore(&ch->lock, flags); in stmmac_napi_check()
2719 if ((status & handle_tx) && (chan < priv->plat->tx_queues_to_use)) { in stmmac_napi_check()
2721 spin_lock_irqsave(&ch->lock, flags); in stmmac_napi_check()
2722 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 0, 1); in stmmac_napi_check()
2723 spin_unlock_irqrestore(&ch->lock, flags); in stmmac_napi_check()
2732 * stmmac_dma_interrupt - DMA ISR
2740 u32 tx_channel_count = priv->plat->tx_queues_to_use; in stmmac_dma_interrupt()
2741 u32 rx_channel_count = priv->plat->rx_queues_to_use; in stmmac_dma_interrupt()
2758 if (unlikely(priv->xstats.threshold != SF_DMA_MODE) && in stmmac_dma_interrupt()
2761 if (priv->plat->force_thresh_dma_mode) in stmmac_dma_interrupt()
2771 priv->xstats.threshold = tc; in stmmac_dma_interrupt()
2789 stmmac_mmc_intr_all_mask(priv, priv->mmcaddr); in stmmac_mmc_setup()
2791 if (priv->dma_cap.rmon) { in stmmac_mmc_setup()
2792 stmmac_mmc_ctrl(priv, priv->mmcaddr, mode); in stmmac_mmc_setup()
2793 memset(&priv->mmc, 0, sizeof(struct stmmac_counters)); in stmmac_mmc_setup()
2795 netdev_info(priv->dev, "No MAC Management Counters available\n"); in stmmac_mmc_setup()
2799 * stmmac_get_hw_features - get MAC capabilities from the HW cap. register.
2809 return stmmac_get_hw_feature(priv, priv->ioaddr, &priv->dma_cap) == 0; in stmmac_get_hw_features()
2813 * stmmac_check_ether_addr - check if the MAC addr is valid
2821 if (!is_valid_ether_addr(priv->dev->dev_addr)) { in stmmac_check_ether_addr()
2822 stmmac_get_umac_addr(priv, priv->hw, priv->dev->dev_addr, 0); in stmmac_check_ether_addr()
2823 if (!is_valid_ether_addr(priv->dev->dev_addr)) in stmmac_check_ether_addr()
2824 eth_hw_addr_random(priv->dev); in stmmac_check_ether_addr()
2825 dev_info(priv->device, "device MAC address %pM\n", in stmmac_check_ether_addr()
2826 priv->dev->dev_addr); in stmmac_check_ether_addr()
2831 * stmmac_init_dma_engine - DMA init.
2840 u32 rx_channels_count = priv->plat->rx_queues_to_use; in stmmac_init_dma_engine()
2841 u32 tx_channels_count = priv->plat->tx_queues_to_use; in stmmac_init_dma_engine()
2849 if (!priv->plat->dma_cfg || !priv->plat->dma_cfg->pbl) { in stmmac_init_dma_engine()
2850 dev_err(priv->device, "Invalid DMA configuration\n"); in stmmac_init_dma_engine()
2851 return -EINVAL; in stmmac_init_dma_engine()
2854 if (priv->extend_desc && (priv->mode == STMMAC_RING_MODE)) in stmmac_init_dma_engine()
2857 ret = stmmac_reset(priv, priv->ioaddr); in stmmac_init_dma_engine()
2859 dev_err(priv->device, "Failed to reset the dma\n"); in stmmac_init_dma_engine()
2864 stmmac_dma_init(priv, priv->ioaddr, priv->plat->dma_cfg, atds); in stmmac_init_dma_engine()
2866 if (priv->plat->axi) in stmmac_init_dma_engine()
2867 stmmac_axi(priv, priv->ioaddr, priv->plat->axi); in stmmac_init_dma_engine()
2871 stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan); in stmmac_init_dma_engine()
2873 /* DMA RX Channel Configuration */ in stmmac_init_dma_engine()
2875 rx_q = &priv->rx_queue[chan]; in stmmac_init_dma_engine()
2877 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, in stmmac_init_dma_engine()
2878 rx_q->dma_rx_phy, chan); in stmmac_init_dma_engine()
2880 rx_q->rx_tail_addr = rx_q->dma_rx_phy + in stmmac_init_dma_engine()
2881 (rx_q->buf_alloc_num * in stmmac_init_dma_engine()
2883 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, in stmmac_init_dma_engine()
2884 rx_q->rx_tail_addr, chan); in stmmac_init_dma_engine()
2889 tx_q = &priv->tx_queue[chan]; in stmmac_init_dma_engine()
2891 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, in stmmac_init_dma_engine()
2892 tx_q->dma_tx_phy, chan); in stmmac_init_dma_engine()
2894 tx_q->tx_tail_addr = tx_q->dma_tx_phy; in stmmac_init_dma_engine()
2895 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, in stmmac_init_dma_engine()
2896 tx_q->tx_tail_addr, chan); in stmmac_init_dma_engine()
2902 static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue) in stmmac_tx_timer_arm() argument
2904 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; in stmmac_tx_timer_arm()
2906 hrtimer_start(&tx_q->txtimer, in stmmac_tx_timer_arm()
2907 STMMAC_COAL_TIMER(priv->tx_coal_timer[queue]), in stmmac_tx_timer_arm()
2912 * stmmac_tx_timer - mitigation sw timer for tx.
2920 struct stmmac_priv *priv = tx_q->priv_data; in stmmac_tx_timer()
2924 ch = &priv->channel[tx_q->queue_index]; in stmmac_tx_timer()
2925 napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi; in stmmac_tx_timer()
2930 spin_lock_irqsave(&ch->lock, flags); in stmmac_tx_timer()
2931 stmmac_disable_dma_irq(priv, priv->ioaddr, ch->index, 0, 1); in stmmac_tx_timer()
2932 spin_unlock_irqrestore(&ch->lock, flags); in stmmac_tx_timer()
2940 * stmmac_init_coalesce - init mitigation options.
2949 u32 tx_channel_count = priv->plat->tx_queues_to_use; in stmmac_init_coalesce()
2950 u32 rx_channel_count = priv->plat->rx_queues_to_use; in stmmac_init_coalesce()
2954 struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan]; in stmmac_init_coalesce()
2956 priv->tx_coal_frames[chan] = STMMAC_TX_FRAMES; in stmmac_init_coalesce()
2957 priv->tx_coal_timer[chan] = STMMAC_COAL_TX_TIMER; in stmmac_init_coalesce()
2959 hrtimer_init(&tx_q->txtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); in stmmac_init_coalesce()
2960 tx_q->txtimer.function = stmmac_tx_timer; in stmmac_init_coalesce()
2964 priv->rx_coal_frames[chan] = STMMAC_RX_FRAMES; in stmmac_init_coalesce()
2969 u32 rx_channels_count = priv->plat->rx_queues_to_use; in stmmac_set_rings_length()
2970 u32 tx_channels_count = priv->plat->tx_queues_to_use; in stmmac_set_rings_length()
2975 stmmac_set_tx_ring_len(priv, priv->ioaddr, in stmmac_set_rings_length()
2976 (priv->dma_tx_size - 1), chan); in stmmac_set_rings_length()
2978 /* set RX ring length */ in stmmac_set_rings_length()
2980 stmmac_set_rx_ring_len(priv, priv->ioaddr, in stmmac_set_rings_length()
2981 (priv->dma_rx_size - 1), chan); in stmmac_set_rings_length()
2985 * stmmac_set_tx_queue_weight - Set TX queue weight
2991 u32 tx_queues_count = priv->plat->tx_queues_to_use; in stmmac_set_tx_queue_weight()
2993 u32 queue; in stmmac_set_tx_queue_weight() local
2995 for (queue = 0; queue < tx_queues_count; queue++) { in stmmac_set_tx_queue_weight()
2996 weight = priv->plat->tx_queues_cfg[queue].weight; in stmmac_set_tx_queue_weight()
2997 stmmac_set_mtl_tx_queue_weight(priv, priv->hw, weight, queue); in stmmac_set_tx_queue_weight()
3002 * stmmac_configure_cbs - Configure CBS in TX queue
3008 u32 tx_queues_count = priv->plat->tx_queues_to_use; in stmmac_configure_cbs()
3010 u32 queue; in stmmac_configure_cbs() local
3012 /* queue 0 is reserved for legacy traffic */ in stmmac_configure_cbs()
3013 for (queue = 1; queue < tx_queues_count; queue++) { in stmmac_configure_cbs()
3014 mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use; in stmmac_configure_cbs()
3018 stmmac_config_cbs(priv, priv->hw, in stmmac_configure_cbs()
3019 priv->plat->tx_queues_cfg[queue].send_slope, in stmmac_configure_cbs()
3020 priv->plat->tx_queues_cfg[queue].idle_slope, in stmmac_configure_cbs()
3021 priv->plat->tx_queues_cfg[queue].high_credit, in stmmac_configure_cbs()
3022 priv->plat->tx_queues_cfg[queue].low_credit, in stmmac_configure_cbs()
3023 queue); in stmmac_configure_cbs()
3028 * stmmac_rx_queue_dma_chan_map - Map RX queue to RX dma channel
3030 * Description: It is used for mapping RX queues to RX dma channels
3034 u32 rx_queues_count = priv->plat->rx_queues_to_use; in stmmac_rx_queue_dma_chan_map()
3035 u32 queue; in stmmac_rx_queue_dma_chan_map() local
3038 for (queue = 0; queue < rx_queues_count; queue++) { in stmmac_rx_queue_dma_chan_map()
3039 chan = priv->plat->rx_queues_cfg[queue].chan; in stmmac_rx_queue_dma_chan_map()
3040 stmmac_map_mtl_to_dma(priv, priv->hw, queue, chan); in stmmac_rx_queue_dma_chan_map()
3045 * stmmac_mac_config_rx_queues_prio - Configure RX Queue priority
3047 * Description: It is used for configuring the RX Queue Priority
3051 u32 rx_queues_count = priv->plat->rx_queues_to_use; in stmmac_mac_config_rx_queues_prio()
3052 u32 queue; in stmmac_mac_config_rx_queues_prio() local
3055 for (queue = 0; queue < rx_queues_count; queue++) { in stmmac_mac_config_rx_queues_prio()
3056 if (!priv->plat->rx_queues_cfg[queue].use_prio) in stmmac_mac_config_rx_queues_prio()
3059 prio = priv->plat->rx_queues_cfg[queue].prio; in stmmac_mac_config_rx_queues_prio()
3060 stmmac_rx_queue_prio(priv, priv->hw, prio, queue); in stmmac_mac_config_rx_queues_prio()
3065 * stmmac_mac_config_tx_queues_prio - Configure TX Queue priority
3067 * Description: It is used for configuring the TX Queue Priority
3071 u32 tx_queues_count = priv->plat->tx_queues_to_use; in stmmac_mac_config_tx_queues_prio()
3072 u32 queue; in stmmac_mac_config_tx_queues_prio() local
3075 for (queue = 0; queue < tx_queues_count; queue++) { in stmmac_mac_config_tx_queues_prio()
3076 if (!priv->plat->tx_queues_cfg[queue].use_prio) in stmmac_mac_config_tx_queues_prio()
3079 prio = priv->plat->tx_queues_cfg[queue].prio; in stmmac_mac_config_tx_queues_prio()
3080 stmmac_tx_queue_prio(priv, priv->hw, prio, queue); in stmmac_mac_config_tx_queues_prio()
3085 * stmmac_mac_config_rx_queues_routing - Configure RX Queue Routing
3087 * Description: It is used for configuring the RX queue routing
3091 u32 rx_queues_count = priv->plat->rx_queues_to_use; in stmmac_mac_config_rx_queues_routing()
3092 u32 queue; in stmmac_mac_config_rx_queues_routing() local
3095 for (queue = 0; queue < rx_queues_count; queue++) { in stmmac_mac_config_rx_queues_routing()
3096 /* no specific packet type routing specified for the queue */ in stmmac_mac_config_rx_queues_routing()
3097 if (priv->plat->rx_queues_cfg[queue].pkt_route == 0x0) in stmmac_mac_config_rx_queues_routing()
3100 packet = priv->plat->rx_queues_cfg[queue].pkt_route; in stmmac_mac_config_rx_queues_routing()
3101 stmmac_rx_queue_routing(priv, priv->hw, packet, queue); in stmmac_mac_config_rx_queues_routing()
3107 if (!priv->dma_cap.rssen || !priv->plat->rss_en) { in stmmac_mac_config_rss()
3108 priv->rss.enable = false; in stmmac_mac_config_rss()
3112 if (priv->dev->features & NETIF_F_RXHASH) in stmmac_mac_config_rss()
3113 priv->rss.enable = true; in stmmac_mac_config_rss()
3115 priv->rss.enable = false; in stmmac_mac_config_rss()
3117 stmmac_rss_configure(priv, priv->hw, &priv->rss, in stmmac_mac_config_rss()
3118 priv->plat->rx_queues_to_use); in stmmac_mac_config_rss()
3122 * stmmac_mtl_configuration - Configure MTL
3128 u32 rx_queues_count = priv->plat->rx_queues_to_use; in stmmac_mtl_configuration()
3129 u32 tx_queues_count = priv->plat->tx_queues_to_use; in stmmac_mtl_configuration()
3134 /* Configure MTL RX algorithms */ in stmmac_mtl_configuration()
3136 stmmac_prog_mtl_rx_algorithms(priv, priv->hw, in stmmac_mtl_configuration()
3137 priv->plat->rx_sched_algorithm); in stmmac_mtl_configuration()
3141 stmmac_prog_mtl_tx_algorithms(priv, priv->hw, in stmmac_mtl_configuration()
3142 priv->plat->tx_sched_algorithm); in stmmac_mtl_configuration()
3148 /* Map RX MTL to DMA channels */ in stmmac_mtl_configuration()
3151 /* Enable MAC RX Queues */ in stmmac_mtl_configuration()
3154 /* Set RX priorities */ in stmmac_mtl_configuration()
3162 /* Set RX routing */ in stmmac_mtl_configuration()
3173 if (priv->dma_cap.asp) { in stmmac_safety_feat_configuration()
3174 netdev_info(priv->dev, "Enabling Safety Features\n"); in stmmac_safety_feat_configuration()
3175 stmmac_safety_feat_config(priv, priv->ioaddr, priv->dma_cap.asp, in stmmac_safety_feat_configuration()
3176 priv->plat->safety_feat_cfg); in stmmac_safety_feat_configuration()
3178 netdev_info(priv->dev, "No Safety Features support found\n"); in stmmac_safety_feat_configuration()
3186 clear_bit(__FPE_TASK_SCHED, &priv->fpe_task_state); in stmmac_fpe_start_wq()
3187 clear_bit(__FPE_REMOVING, &priv->fpe_task_state); in stmmac_fpe_start_wq()
3189 name = priv->wq_name; in stmmac_fpe_start_wq()
3190 sprintf(name, "%s-fpe", priv->dev->name); in stmmac_fpe_start_wq()
3192 priv->fpe_wq = create_singlethread_workqueue(name); in stmmac_fpe_start_wq()
3193 if (!priv->fpe_wq) { in stmmac_fpe_start_wq()
3194 netdev_err(priv->dev, "%s: Failed to create workqueue\n", name); in stmmac_fpe_start_wq()
3196 return -ENOMEM; in stmmac_fpe_start_wq()
3198 netdev_info(priv->dev, "FPE workqueue start"); in stmmac_fpe_start_wq()
3204 * stmmac_hw_setup - setup mac in a usable state.
3213 * 0 on success and an appropriate (-)ve integer as defined in errno.h
3219 u32 rx_cnt = priv->plat->rx_queues_to_use; in stmmac_hw_setup()
3220 u32 tx_cnt = priv->plat->tx_queues_to_use; in stmmac_hw_setup()
3228 netdev_err(priv->dev, "%s: DMA engine initialization failed\n", in stmmac_hw_setup()
3234 stmmac_set_umac_addr(priv, priv->hw, dev->dev_addr, 0); in stmmac_hw_setup()
3237 if (priv->hw->pcs) { in stmmac_hw_setup()
3238 int speed = priv->plat->mac_port_sel_speed; in stmmac_hw_setup()
3242 priv->hw->ps = speed; in stmmac_hw_setup()
3244 dev_warn(priv->device, "invalid port speed\n"); in stmmac_hw_setup()
3245 priv->hw->ps = 0; in stmmac_hw_setup()
3250 stmmac_core_init(priv, priv->hw, dev); in stmmac_hw_setup()
3258 ret = stmmac_rx_ipc(priv, priv->hw); in stmmac_hw_setup()
3260 netdev_warn(priv->dev, "RX IPC Checksum Offload disabled\n"); in stmmac_hw_setup()
3261 priv->plat->rx_coe = STMMAC_RX_COE_NONE; in stmmac_hw_setup()
3262 priv->hw->rx_csum = 0; in stmmac_hw_setup()
3265 /* Enable the MAC Rx/Tx */ in stmmac_hw_setup()
3266 stmmac_mac_set(priv, priv->ioaddr, true); in stmmac_hw_setup()
3274 ret = clk_prepare_enable(priv->plat->clk_ptp_ref); in stmmac_hw_setup()
3276 netdev_warn(priv->dev, "failed to enable PTP reference clock: %d\n", ret); in stmmac_hw_setup()
3279 if (ret == -EOPNOTSUPP) in stmmac_hw_setup()
3280 netdev_warn(priv->dev, "PTP not supported by HW\n"); in stmmac_hw_setup()
3282 netdev_warn(priv->dev, "PTP init failed\n"); in stmmac_hw_setup()
3285 priv->eee_tw_timer = STMMAC_DEFAULT_TWT_LS; in stmmac_hw_setup()
3288 if (!priv->tx_lpi_timer) in stmmac_hw_setup()
3289 priv->tx_lpi_timer = eee_timer * 1000; in stmmac_hw_setup()
3291 if (priv->use_riwt) { in stmmac_hw_setup()
3292 u32 queue; in stmmac_hw_setup() local
3294 for (queue = 0; queue < rx_cnt; queue++) { in stmmac_hw_setup()
3295 if (!priv->rx_riwt[queue]) in stmmac_hw_setup()
3296 priv->rx_riwt[queue] = DEF_DMA_RIWT; in stmmac_hw_setup()
3298 stmmac_rx_watchdog(priv, priv->ioaddr, in stmmac_hw_setup()
3299 priv->rx_riwt[queue], queue); in stmmac_hw_setup()
3303 if (priv->hw->pcs) in stmmac_hw_setup()
3304 stmmac_pcs_ctrl_ane(priv, priv->ioaddr, 1, priv->hw->ps, 0); in stmmac_hw_setup()
3306 /* set TX and RX rings length */ in stmmac_hw_setup()
3310 if (priv->tso) { in stmmac_hw_setup()
3312 struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan]; in stmmac_hw_setup()
3314 /* TSO and TBS cannot co-exist */ in stmmac_hw_setup()
3315 if (tx_q->tbs & STMMAC_TBS_AVAIL) in stmmac_hw_setup()
3318 stmmac_enable_tso(priv, priv->ioaddr, 1, chan); in stmmac_hw_setup()
3323 sph_en = (priv->hw->rx_csum > 0) && priv->sph; in stmmac_hw_setup()
3325 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); in stmmac_hw_setup()
3329 if (priv->dma_cap.vlins) in stmmac_hw_setup()
3330 stmmac_enable_vlan(priv, priv->hw, STMMAC_VLAN_INSERT); in stmmac_hw_setup()
3334 struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan]; in stmmac_hw_setup()
3335 int enable = tx_q->tbs & STMMAC_TBS_AVAIL; in stmmac_hw_setup()
3337 stmmac_enable_tbs(priv, priv->ioaddr, enable, chan); in stmmac_hw_setup()
3340 /* Configure real RX and TX queues */ in stmmac_hw_setup()
3341 netif_set_real_num_rx_queues(dev, priv->plat->rx_queues_to_use); in stmmac_hw_setup()
3342 netif_set_real_num_tx_queues(dev, priv->plat->tx_queues_to_use); in stmmac_hw_setup()
3347 if (priv->dma_cap.fpesel) { in stmmac_hw_setup()
3350 if (priv->plat->fpe_cfg->enable) in stmmac_hw_setup()
3361 clk_disable_unprepare(priv->plat->clk_ptp_ref); in stmmac_hw_teardown()
3372 irq_idx = priv->plat->tx_queues_to_use; in stmmac_free_irq()
3375 for (j = irq_idx - 1; j >= 0; j--) { in stmmac_free_irq()
3376 if (priv->tx_irq[j] > 0) { in stmmac_free_irq()
3377 irq_set_affinity_hint(priv->tx_irq[j], NULL); in stmmac_free_irq()
3378 free_irq(priv->tx_irq[j], &priv->tx_queue[j]); in stmmac_free_irq()
3381 irq_idx = priv->plat->rx_queues_to_use; in stmmac_free_irq()
3384 for (j = irq_idx - 1; j >= 0; j--) { in stmmac_free_irq()
3385 if (priv->rx_irq[j] > 0) { in stmmac_free_irq()
3386 irq_set_affinity_hint(priv->rx_irq[j], NULL); in stmmac_free_irq()
3387 free_irq(priv->rx_irq[j], &priv->rx_queue[j]); in stmmac_free_irq()
3391 if (priv->sfty_ue_irq > 0 && priv->sfty_ue_irq != dev->irq) in stmmac_free_irq()
3392 free_irq(priv->sfty_ue_irq, dev); in stmmac_free_irq()
3395 if (priv->sfty_ce_irq > 0 && priv->sfty_ce_irq != dev->irq) in stmmac_free_irq()
3396 free_irq(priv->sfty_ce_irq, dev); in stmmac_free_irq()
3399 if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) in stmmac_free_irq()
3400 free_irq(priv->lpi_irq, dev); in stmmac_free_irq()
3403 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) in stmmac_free_irq()
3404 free_irq(priv->wol_irq, dev); in stmmac_free_irq()
3407 free_irq(dev->irq, dev); in stmmac_free_irq()
3427 int_name = priv->int_name_mac; in stmmac_request_irq_multi_msi()
3428 sprintf(int_name, "%s:%s", dev->name, "mac"); in stmmac_request_irq_multi_msi()
3429 ret = request_irq(dev->irq, stmmac_mac_interrupt, in stmmac_request_irq_multi_msi()
3432 netdev_err(priv->dev, in stmmac_request_irq_multi_msi()
3434 __func__, dev->irq, ret); in stmmac_request_irq_multi_msi()
3442 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) { in stmmac_request_irq_multi_msi()
3443 int_name = priv->int_name_wol; in stmmac_request_irq_multi_msi()
3444 sprintf(int_name, "%s:%s", dev->name, "wol"); in stmmac_request_irq_multi_msi()
3445 ret = request_irq(priv->wol_irq, in stmmac_request_irq_multi_msi()
3449 netdev_err(priv->dev, in stmmac_request_irq_multi_msi()
3451 __func__, priv->wol_irq, ret); in stmmac_request_irq_multi_msi()
3460 if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) { in stmmac_request_irq_multi_msi()
3461 int_name = priv->int_name_lpi; in stmmac_request_irq_multi_msi()
3462 sprintf(int_name, "%s:%s", dev->name, "lpi"); in stmmac_request_irq_multi_msi()
3463 ret = request_irq(priv->lpi_irq, in stmmac_request_irq_multi_msi()
3467 netdev_err(priv->dev, in stmmac_request_irq_multi_msi()
3469 __func__, priv->lpi_irq, ret); in stmmac_request_irq_multi_msi()
3478 if (priv->sfty_ce_irq > 0 && priv->sfty_ce_irq != dev->irq) { in stmmac_request_irq_multi_msi()
3479 int_name = priv->int_name_sfty_ce; in stmmac_request_irq_multi_msi()
3480 sprintf(int_name, "%s:%s", dev->name, "safety-ce"); in stmmac_request_irq_multi_msi()
3481 ret = request_irq(priv->sfty_ce_irq, in stmmac_request_irq_multi_msi()
3485 netdev_err(priv->dev, in stmmac_request_irq_multi_msi()
3487 __func__, priv->sfty_ce_irq, ret); in stmmac_request_irq_multi_msi()
3496 if (priv->sfty_ue_irq > 0 && priv->sfty_ue_irq != dev->irq) { in stmmac_request_irq_multi_msi()
3497 int_name = priv->int_name_sfty_ue; in stmmac_request_irq_multi_msi()
3498 sprintf(int_name, "%s:%s", dev->name, "safety-ue"); in stmmac_request_irq_multi_msi()
3499 ret = request_irq(priv->sfty_ue_irq, in stmmac_request_irq_multi_msi()
3503 netdev_err(priv->dev, in stmmac_request_irq_multi_msi()
3505 __func__, priv->sfty_ue_irq, ret); in stmmac_request_irq_multi_msi()
3511 /* Request Rx MSI irq */ in stmmac_request_irq_multi_msi()
3512 for (i = 0; i < priv->plat->rx_queues_to_use; i++) { in stmmac_request_irq_multi_msi()
3513 if (priv->rx_irq[i] == 0) in stmmac_request_irq_multi_msi()
3516 int_name = priv->int_name_rx_irq[i]; in stmmac_request_irq_multi_msi()
3517 sprintf(int_name, "%s:%s-%d", dev->name, "rx", i); in stmmac_request_irq_multi_msi()
3518 ret = request_irq(priv->rx_irq[i], in stmmac_request_irq_multi_msi()
3520 0, int_name, &priv->rx_queue[i]); in stmmac_request_irq_multi_msi()
3522 netdev_err(priv->dev, in stmmac_request_irq_multi_msi()
3523 "%s: alloc rx-%d MSI %d (error: %d)\n", in stmmac_request_irq_multi_msi()
3524 __func__, i, priv->rx_irq[i], ret); in stmmac_request_irq_multi_msi()
3531 irq_set_affinity_hint(priv->rx_irq[i], &cpu_mask); in stmmac_request_irq_multi_msi()
3535 for (i = 0; i < priv->plat->tx_queues_to_use; i++) { in stmmac_request_irq_multi_msi()
3536 if (priv->tx_irq[i] == 0) in stmmac_request_irq_multi_msi()
3539 int_name = priv->int_name_tx_irq[i]; in stmmac_request_irq_multi_msi()
3540 sprintf(int_name, "%s:%s-%d", dev->name, "tx", i); in stmmac_request_irq_multi_msi()
3541 ret = request_irq(priv->tx_irq[i], in stmmac_request_irq_multi_msi()
3543 0, int_name, &priv->tx_queue[i]); in stmmac_request_irq_multi_msi()
3545 netdev_err(priv->dev, in stmmac_request_irq_multi_msi()
3546 "%s: alloc tx-%d MSI %d (error: %d)\n", in stmmac_request_irq_multi_msi()
3547 __func__, i, priv->tx_irq[i], ret); in stmmac_request_irq_multi_msi()
3554 irq_set_affinity_hint(priv->tx_irq[i], &cpu_mask); in stmmac_request_irq_multi_msi()
3570 ret = request_irq(dev->irq, stmmac_interrupt, in stmmac_request_irq_single()
3571 IRQF_SHARED, dev->name, dev); in stmmac_request_irq_single()
3573 netdev_err(priv->dev, in stmmac_request_irq_single()
3575 __func__, dev->irq, ret); in stmmac_request_irq_single()
3583 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) { in stmmac_request_irq_single()
3584 ret = request_irq(priv->wol_irq, stmmac_interrupt, in stmmac_request_irq_single()
3585 IRQF_SHARED, dev->name, dev); in stmmac_request_irq_single()
3587 netdev_err(priv->dev, in stmmac_request_irq_single()
3589 __func__, priv->wol_irq, ret); in stmmac_request_irq_single()
3596 if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) { in stmmac_request_irq_single()
3597 ret = request_irq(priv->lpi_irq, stmmac_interrupt, in stmmac_request_irq_single()
3598 IRQF_SHARED, dev->name, dev); in stmmac_request_irq_single()
3600 netdev_err(priv->dev, in stmmac_request_irq_single()
3602 __func__, priv->lpi_irq, ret); in stmmac_request_irq_single()
3621 if (priv->plat->multi_msi_en) in stmmac_request_irq()
3630 * stmmac_open - open entry point of the driver
3635 * 0 on success and an appropriate (-)ve integer as defined in errno.h
3641 int mode = priv->plat->phy_interface; in stmmac_open()
3646 ret = pm_runtime_get_sync(priv->device); in stmmac_open()
3648 pm_runtime_put_noidle(priv->device); in stmmac_open()
3652 if (priv->hw->pcs != STMMAC_PCS_TBI && in stmmac_open()
3653 priv->hw->pcs != STMMAC_PCS_RTBI && in stmmac_open()
3654 (!priv->hw->xpcs || in stmmac_open()
3655 xpcs_get_an_mode(priv->hw->xpcs, mode) != DW_AN_C73)) { in stmmac_open()
3658 netdev_err(priv->dev, in stmmac_open()
3666 memset(&priv->xstats, 0, sizeof(struct stmmac_extra_stats)); in stmmac_open()
3667 priv->xstats.threshold = tc; in stmmac_open()
3669 bfsize = stmmac_set_16kib_bfsize(priv, dev->mtu); in stmmac_open()
3674 bfsize = stmmac_set_bfsize(dev->mtu, priv->dma_buf_sz); in stmmac_open()
3676 priv->dma_buf_sz = bfsize; in stmmac_open()
3679 priv->rx_copybreak = STMMAC_RX_COPYBREAK; in stmmac_open()
3681 if (!priv->dma_tx_size) in stmmac_open()
3682 priv->dma_tx_size = DMA_DEFAULT_TX_SIZE; in stmmac_open()
3683 if (!priv->dma_rx_size) in stmmac_open()
3684 priv->dma_rx_size = DMA_DEFAULT_RX_SIZE; in stmmac_open()
3687 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) { in stmmac_open()
3688 struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan]; in stmmac_open()
3689 int tbs_en = priv->plat->tx_queues_cfg[chan].tbs_en; in stmmac_open()
3691 /* Setup per-TXQ tbs flag before TX descriptor alloc */ in stmmac_open()
3692 tx_q->tbs |= tbs_en ? STMMAC_TBS_AVAIL : 0; in stmmac_open()
3697 netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n", in stmmac_open()
3704 netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n", in stmmac_open()
3711 netdev_err(priv->dev, "%s: Hw setup failed\n", __func__); in stmmac_open()
3717 phylink_start(priv->phylink); in stmmac_open()
3719 phylink_speed_up(priv->phylink); in stmmac_open()
3726 netif_tx_start_all_queues(priv->dev); in stmmac_open()
3731 phylink_stop(priv->phylink); in stmmac_open()
3733 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) in stmmac_open()
3734 hrtimer_cancel(&priv->tx_queue[chan].txtimer); in stmmac_open()
3740 phylink_disconnect_phy(priv->phylink); in stmmac_open()
3742 pm_runtime_put(priv->device); in stmmac_open()
3748 set_bit(__FPE_REMOVING, &priv->fpe_task_state); in stmmac_fpe_stop_wq()
3750 if (priv->fpe_wq) in stmmac_fpe_stop_wq()
3751 destroy_workqueue(priv->fpe_wq); in stmmac_fpe_stop_wq()
3753 netdev_info(priv->dev, "FPE workqueue stop"); in stmmac_fpe_stop_wq()
3757 * stmmac_release - close entry point of the driver
3767 if (device_may_wakeup(priv->device)) in stmmac_release()
3768 phylink_speed_down(priv->phylink, false); in stmmac_release()
3770 phylink_stop(priv->phylink); in stmmac_release()
3771 phylink_disconnect_phy(priv->phylink); in stmmac_release()
3775 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) in stmmac_release()
3776 hrtimer_cancel(&priv->tx_queue[chan].txtimer); in stmmac_release()
3781 if (priv->eee_enabled) { in stmmac_release()
3782 priv->tx_path_in_lpi_mode = false; in stmmac_release()
3783 del_timer_sync(&priv->eee_ctrl_timer); in stmmac_release()
3786 /* Stop TX/RX DMA and clear the descriptors */ in stmmac_release()
3789 /* Release and free the Rx/Tx resources */ in stmmac_release()
3792 /* Disable the MAC Rx/Tx */ in stmmac_release()
3793 stmmac_mac_set(priv, priv->ioaddr, false); in stmmac_release()
3799 pm_runtime_put(priv->device); in stmmac_release()
3801 if (priv->dma_cap.fpesel) in stmmac_release()
3814 if (!priv->dma_cap.vlins) in stmmac_vlan_insert()
3818 if (skb->vlan_proto == htons(ETH_P_8021AD)) { in stmmac_vlan_insert()
3825 if (tx_q->tbs & STMMAC_TBS_AVAIL) in stmmac_vlan_insert()
3826 p = &tx_q->dma_entx[tx_q->cur_tx].basic; in stmmac_vlan_insert()
3828 p = &tx_q->dma_tx[tx_q->cur_tx]; in stmmac_vlan_insert()
3834 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_tx_size); in stmmac_vlan_insert()
3839 * stmmac_tso_allocator - close entry point of the driver
3844 * @queue: TX queue index
3850 int total_len, bool last_segment, u32 queue) in stmmac_tso_allocator() argument
3852 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; in stmmac_tso_allocator()
3862 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, in stmmac_tso_allocator()
3863 priv->dma_tx_size); in stmmac_tso_allocator()
3864 WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]); in stmmac_tso_allocator()
3866 if (tx_q->tbs & STMMAC_TBS_AVAIL) in stmmac_tso_allocator()
3867 desc = &tx_q->dma_entx[tx_q->cur_tx].basic; in stmmac_tso_allocator()
3869 desc = &tx_q->dma_tx[tx_q->cur_tx]; in stmmac_tso_allocator()
3871 curr_addr = des + (total_len - tmp_len); in stmmac_tso_allocator()
3872 if (priv->dma_cap.addr64 <= 32) in stmmac_tso_allocator()
3873 desc->des0 = cpu_to_le32(curr_addr); in stmmac_tso_allocator()
3885 tmp_len -= TSO_MAX_BUFF_SIZE; in stmmac_tso_allocator()
3889 static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue) in stmmac_flush_tx_descriptors() argument
3891 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; in stmmac_flush_tx_descriptors()
3894 if (likely(priv->extend_desc)) in stmmac_flush_tx_descriptors()
3896 else if (tx_q->tbs & STMMAC_TBS_AVAIL) in stmmac_flush_tx_descriptors()
3907 tx_q->tx_tail_addr = tx_q->dma_tx_phy + (tx_q->cur_tx * desc_size); in stmmac_flush_tx_descriptors()
3908 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue); in stmmac_flush_tx_descriptors()
3912 * stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO)
3920 * --------
3921 * | DES0 |---> buffer1 = L2/L3/L4 header
3922 * | DES1 |---> TCP Payload (can continue on next descr...)
3923 * | DES2 |---> buffer 1 and 2 len
3924 * | DES3 |---> must set TSE, TCP hdr len-> [22:19]. TCP payload len [17:0]
3925 * --------
3929 * --------
3930 * | DES0 | --| Split TCP Payload on Buffers 1 and 2
3931 * | DES1 | --|
3932 * | DES2 | --> buffer 1 and 2 len
3934 * --------
3942 int nfrags = skb_shinfo(skb)->nr_frags; in stmmac_tso_xmit()
3943 u32 queue = skb_get_queue_mapping(skb); in stmmac_tso_xmit() local
3953 tx_q = &priv->tx_queue[queue]; in stmmac_tso_xmit()
3954 first_tx = tx_q->cur_tx; in stmmac_tso_xmit()
3957 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) { in stmmac_tso_xmit()
3966 if (unlikely(stmmac_tx_avail(priv, queue) < in stmmac_tso_xmit()
3967 (((skb->len - proto_hdr_len) / TSO_MAX_BUFF_SIZE + 1)))) { in stmmac_tso_xmit()
3968 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) { in stmmac_tso_xmit()
3969 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, in stmmac_tso_xmit()
3970 queue)); in stmmac_tso_xmit()
3972 netdev_err(priv->dev, in stmmac_tso_xmit()
3973 "%s: Tx Ring full when queue awake\n", in stmmac_tso_xmit()
3979 pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */ in stmmac_tso_xmit()
3981 mss = skb_shinfo(skb)->gso_size; in stmmac_tso_xmit()
3984 if (mss != tx_q->mss) { in stmmac_tso_xmit()
3985 if (tx_q->tbs & STMMAC_TBS_AVAIL) in stmmac_tso_xmit()
3986 mss_desc = &tx_q->dma_entx[tx_q->cur_tx].basic; in stmmac_tso_xmit()
3988 mss_desc = &tx_q->dma_tx[tx_q->cur_tx]; in stmmac_tso_xmit()
3991 tx_q->mss = mss; in stmmac_tso_xmit()
3992 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, in stmmac_tso_xmit()
3993 priv->dma_tx_size); in stmmac_tso_xmit()
3994 WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]); in stmmac_tso_xmit()
4000 pr_info("\tskb->len %d, skb->data_len %d\n", skb->len, in stmmac_tso_xmit()
4001 skb->data_len); in stmmac_tso_xmit()
4007 first_entry = tx_q->cur_tx; in stmmac_tso_xmit()
4008 WARN_ON(tx_q->tx_skbuff[first_entry]); in stmmac_tso_xmit()
4010 if (tx_q->tbs & STMMAC_TBS_AVAIL) in stmmac_tso_xmit()
4011 desc = &tx_q->dma_entx[first_entry].basic; in stmmac_tso_xmit()
4013 desc = &tx_q->dma_tx[first_entry]; in stmmac_tso_xmit()
4020 des = dma_map_single(priv->device, skb->data, skb_headlen(skb), in stmmac_tso_xmit()
4022 if (dma_mapping_error(priv->device, des)) in stmmac_tso_xmit()
4025 tx_q->tx_skbuff_dma[first_entry].buf = des; in stmmac_tso_xmit()
4026 tx_q->tx_skbuff_dma[first_entry].len = skb_headlen(skb); in stmmac_tso_xmit()
4027 tx_q->tx_skbuff_dma[first_entry].map_as_page = false; in stmmac_tso_xmit()
4028 tx_q->tx_skbuff_dma[first_entry].buf_type = STMMAC_TXBUF_T_SKB; in stmmac_tso_xmit()
4030 if (priv->dma_cap.addr64 <= 32) { in stmmac_tso_xmit()
4031 first->des0 = cpu_to_le32(des); in stmmac_tso_xmit()
4035 first->des1 = cpu_to_le32(des + proto_hdr_len); in stmmac_tso_xmit()
4038 tmp_pay_len = pay_len - TSO_MAX_BUFF_SIZE; in stmmac_tso_xmit()
4046 stmmac_tso_allocator(priv, des, tmp_pay_len, (nfrags == 0), queue); in stmmac_tso_xmit()
4050 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; in stmmac_tso_xmit()
4052 des = skb_frag_dma_map(priv->device, frag, 0, in stmmac_tso_xmit()
4055 if (dma_mapping_error(priv->device, des)) in stmmac_tso_xmit()
4059 (i == nfrags - 1), queue); in stmmac_tso_xmit()
4061 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des; in stmmac_tso_xmit()
4062 tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_frag_size(frag); in stmmac_tso_xmit()
4063 tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = true; in stmmac_tso_xmit()
4064 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB; in stmmac_tso_xmit()
4067 tx_q->tx_skbuff_dma[tx_q->cur_tx].last_segment = true; in stmmac_tso_xmit()
4070 tx_q->tx_skbuff[tx_q->cur_tx] = skb; in stmmac_tso_xmit()
4071 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB; in stmmac_tso_xmit()
4074 tx_packets = (tx_q->cur_tx + 1) - first_tx; in stmmac_tso_xmit()
4075 tx_q->tx_count_frames += tx_packets; in stmmac_tso_xmit()
4077 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en) in stmmac_tso_xmit()
4079 else if (!priv->tx_coal_frames[queue]) in stmmac_tso_xmit()
4081 else if (tx_packets > priv->tx_coal_frames[queue]) in stmmac_tso_xmit()
4083 else if ((tx_q->tx_count_frames % in stmmac_tso_xmit()
4084 priv->tx_coal_frames[queue]) < tx_packets) in stmmac_tso_xmit()
4090 if (tx_q->tbs & STMMAC_TBS_AVAIL) in stmmac_tso_xmit()
4091 desc = &tx_q->dma_entx[tx_q->cur_tx].basic; in stmmac_tso_xmit()
4093 desc = &tx_q->dma_tx[tx_q->cur_tx]; in stmmac_tso_xmit()
4095 tx_q->tx_count_frames = 0; in stmmac_tso_xmit()
4097 priv->xstats.tx_set_ic_bit++; in stmmac_tso_xmit()
4105 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_tx_size); in stmmac_tso_xmit()
4107 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) { in stmmac_tso_xmit()
4108 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", in stmmac_tso_xmit()
4110 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); in stmmac_tso_xmit()
4113 dev->stats.tx_bytes += skb->len; in stmmac_tso_xmit()
4114 priv->xstats.tx_tso_frames++; in stmmac_tso_xmit()
4115 priv->xstats.tx_tso_nfrags += nfrags; in stmmac_tso_xmit()
4117 if (priv->sarc_type) in stmmac_tso_xmit()
4118 stmmac_set_desc_sarc(priv, first, priv->sarc_type); in stmmac_tso_xmit()
4122 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && in stmmac_tso_xmit()
4123 priv->hwts_tx_en)) { in stmmac_tso_xmit()
4125 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; in stmmac_tso_xmit()
4133 1, tx_q->tx_skbuff_dma[first_entry].last_segment, in stmmac_tso_xmit()
4134 hdr / 4, (skb->len - proto_hdr_len)); in stmmac_tso_xmit()
4149 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, in stmmac_tso_xmit()
4150 tx_q->cur_tx, first, nfrags); in stmmac_tso_xmit()
4152 print_pkt(skb->data, skb_headlen(skb)); in stmmac_tso_xmit()
4155 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len); in stmmac_tso_xmit()
4157 stmmac_flush_tx_descriptors(priv, queue); in stmmac_tso_xmit()
4158 stmmac_tx_timer_arm(priv, queue); in stmmac_tso_xmit()
4163 dev_err(priv->device, "Tx dma map failed\n"); in stmmac_tso_xmit()
4165 priv->dev->stats.tx_dropped++; in stmmac_tso_xmit()
4170 * stmmac_xmit - Tx entry point of the driver
4183 u32 queue = skb_get_queue_mapping(skb); in stmmac_xmit() local
4184 int nfrags = skb_shinfo(skb)->nr_frags; in stmmac_xmit()
4185 int gso = skb_shinfo(skb)->gso_type; in stmmac_xmit()
4193 tx_q = &priv->tx_queue[queue]; in stmmac_xmit()
4194 first_tx = tx_q->cur_tx; in stmmac_xmit()
4196 if (priv->tx_path_in_lpi_mode && priv->eee_sw_timer_en) in stmmac_xmit()
4200 if (skb_is_gso(skb) && priv->tso) { in stmmac_xmit()
4203 if (priv->plat->has_gmac4 && (gso & SKB_GSO_UDP_L4)) in stmmac_xmit()
4207 if (unlikely(stmmac_tx_avail(priv, queue) < nfrags + 1)) { in stmmac_xmit()
4208 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) { in stmmac_xmit()
4209 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, in stmmac_xmit()
4210 queue)); in stmmac_xmit()
4212 netdev_err(priv->dev, in stmmac_xmit()
4213 "%s: Tx Ring full when queue awake\n", in stmmac_xmit()
4222 entry = tx_q->cur_tx; in stmmac_xmit()
4224 WARN_ON(tx_q->tx_skbuff[first_entry]); in stmmac_xmit()
4226 csum_insertion = (skb->ip_summed == CHECKSUM_PARTIAL); in stmmac_xmit()
4228 if (likely(priv->extend_desc)) in stmmac_xmit()
4229 desc = (struct dma_desc *)(tx_q->dma_etx + entry); in stmmac_xmit()
4230 else if (tx_q->tbs & STMMAC_TBS_AVAIL) in stmmac_xmit()
4231 desc = &tx_q->dma_entx[entry].basic; in stmmac_xmit()
4233 desc = tx_q->dma_tx + entry; in stmmac_xmit()
4240 enh_desc = priv->plat->enh_desc; in stmmac_xmit()
4243 is_jumbo = stmmac_is_jumbo_frm(priv, skb->len, enh_desc); in stmmac_xmit()
4247 if (unlikely(entry < 0) && (entry != -EINVAL)) in stmmac_xmit()
4252 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; in stmmac_xmit()
4254 bool last_segment = (i == (nfrags - 1)); in stmmac_xmit()
4256 entry = STMMAC_GET_ENTRY(entry, priv->dma_tx_size); in stmmac_xmit()
4257 WARN_ON(tx_q->tx_skbuff[entry]); in stmmac_xmit()
4259 if (likely(priv->extend_desc)) in stmmac_xmit()
4260 desc = (struct dma_desc *)(tx_q->dma_etx + entry); in stmmac_xmit()
4261 else if (tx_q->tbs & STMMAC_TBS_AVAIL) in stmmac_xmit()
4262 desc = &tx_q->dma_entx[entry].basic; in stmmac_xmit()
4264 desc = tx_q->dma_tx + entry; in stmmac_xmit()
4266 des = skb_frag_dma_map(priv->device, frag, 0, len, in stmmac_xmit()
4268 if (dma_mapping_error(priv->device, des)) in stmmac_xmit()
4271 tx_q->tx_skbuff_dma[entry].buf = des; in stmmac_xmit()
4275 tx_q->tx_skbuff_dma[entry].map_as_page = true; in stmmac_xmit()
4276 tx_q->tx_skbuff_dma[entry].len = len; in stmmac_xmit()
4277 tx_q->tx_skbuff_dma[entry].last_segment = last_segment; in stmmac_xmit()
4278 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_SKB; in stmmac_xmit()
4282 priv->mode, 1, last_segment, skb->len); in stmmac_xmit()
4286 tx_q->tx_skbuff[entry] = skb; in stmmac_xmit()
4287 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_SKB; in stmmac_xmit()
4290 * segment is reset and the timer re-started to clean the tx status. in stmmac_xmit()
4294 tx_packets = (entry + 1) - first_tx; in stmmac_xmit()
4295 tx_q->tx_count_frames += tx_packets; in stmmac_xmit()
4297 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en) in stmmac_xmit()
4299 else if (!priv->tx_coal_frames[queue]) in stmmac_xmit()
4301 else if (tx_packets > priv->tx_coal_frames[queue]) in stmmac_xmit()
4303 else if ((tx_q->tx_count_frames % in stmmac_xmit()
4304 priv->tx_coal_frames[queue]) < tx_packets) in stmmac_xmit()
4310 if (likely(priv->extend_desc)) in stmmac_xmit()
4311 desc = &tx_q->dma_etx[entry].basic; in stmmac_xmit()
4312 else if (tx_q->tbs & STMMAC_TBS_AVAIL) in stmmac_xmit()
4313 desc = &tx_q->dma_entx[entry].basic; in stmmac_xmit()
4315 desc = &tx_q->dma_tx[entry]; in stmmac_xmit()
4317 tx_q->tx_count_frames = 0; in stmmac_xmit()
4319 priv->xstats.tx_set_ic_bit++; in stmmac_xmit()
4327 entry = STMMAC_GET_ENTRY(entry, priv->dma_tx_size); in stmmac_xmit()
4328 tx_q->cur_tx = entry; in stmmac_xmit()
4331 netdev_dbg(priv->dev, in stmmac_xmit()
4333 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, in stmmac_xmit()
4336 netdev_dbg(priv->dev, ">>> frame to be transmitted: "); in stmmac_xmit()
4337 print_pkt(skb->data, skb->len); in stmmac_xmit()
4340 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) { in stmmac_xmit()
4341 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", in stmmac_xmit()
4343 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); in stmmac_xmit()
4346 dev->stats.tx_bytes += skb->len; in stmmac_xmit()
4348 if (priv->sarc_type) in stmmac_xmit()
4349 stmmac_set_desc_sarc(priv, first, priv->sarc_type); in stmmac_xmit()
4360 des = dma_map_single(priv->device, skb->data, in stmmac_xmit()
4362 if (dma_mapping_error(priv->device, des)) in stmmac_xmit()
4365 tx_q->tx_skbuff_dma[first_entry].buf = des; in stmmac_xmit()
4366 tx_q->tx_skbuff_dma[first_entry].buf_type = STMMAC_TXBUF_T_SKB; in stmmac_xmit()
4367 tx_q->tx_skbuff_dma[first_entry].map_as_page = false; in stmmac_xmit()
4371 tx_q->tx_skbuff_dma[first_entry].len = nopaged_len; in stmmac_xmit()
4372 tx_q->tx_skbuff_dma[first_entry].last_segment = last_segment; in stmmac_xmit()
4374 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && in stmmac_xmit()
4375 priv->hwts_tx_en)) { in stmmac_xmit()
4377 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; in stmmac_xmit()
4383 csum_insertion, priv->mode, 0, last_segment, in stmmac_xmit()
4384 skb->len); in stmmac_xmit()
4387 if (tx_q->tbs & STMMAC_TBS_EN) { in stmmac_xmit()
4388 struct timespec64 ts = ns_to_timespec64(skb->tstamp); in stmmac_xmit()
4390 tbs_desc = &tx_q->dma_entx[first_entry]; in stmmac_xmit()
4396 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len); in stmmac_xmit()
4398 stmmac_enable_dma_transmission(priv, priv->ioaddr); in stmmac_xmit()
4400 stmmac_flush_tx_descriptors(priv, queue); in stmmac_xmit()
4401 stmmac_tx_timer_arm(priv, queue); in stmmac_xmit()
4406 netdev_err(priv->dev, "Tx DMA map failed\n"); in stmmac_xmit()
4408 priv->dev->stats.tx_dropped++; in stmmac_xmit()
4418 veth = (struct vlan_ethhdr *)skb->data; in stmmac_rx_vlan()
4419 vlan_proto = veth->h_vlan_proto; in stmmac_rx_vlan()
4422 dev->features & NETIF_F_HW_VLAN_CTAG_RX) || in stmmac_rx_vlan()
4424 dev->features & NETIF_F_HW_VLAN_STAG_RX)) { in stmmac_rx_vlan()
4426 vlanid = ntohs(veth->h_vlan_TCI); in stmmac_rx_vlan()
4427 memmove(skb->data + VLAN_HLEN, veth, ETH_ALEN * 2); in stmmac_rx_vlan()
4434 * stmmac_rx_refill - refill used skb preallocated buffers
4436 * @queue: RX queue index
4438 * that is based on zero-copy.
4440 static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue) in stmmac_rx_refill() argument
4442 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; in stmmac_rx_refill()
4443 int dirty = stmmac_rx_dirty(priv, queue); in stmmac_rx_refill()
4444 unsigned int entry = rx_q->dirty_rx; in stmmac_rx_refill()
4446 while (dirty-- > 0) { in stmmac_rx_refill()
4447 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry]; in stmmac_rx_refill()
4451 if (priv->extend_desc) in stmmac_rx_refill()
4452 p = (struct dma_desc *)(rx_q->dma_erx + entry); in stmmac_rx_refill()
4454 p = rx_q->dma_rx + entry; in stmmac_rx_refill()
4456 if (!buf->page) { in stmmac_rx_refill()
4457 buf->page = page_pool_dev_alloc_pages(rx_q->page_pool); in stmmac_rx_refill()
4458 if (!buf->page) in stmmac_rx_refill()
4462 if (priv->sph && !buf->sec_page) { in stmmac_rx_refill()
4463 buf->sec_page = page_pool_dev_alloc_pages(rx_q->page_pool); in stmmac_rx_refill()
4464 if (!buf->sec_page) in stmmac_rx_refill()
4467 buf->sec_addr = page_pool_get_dma_addr(buf->sec_page); in stmmac_rx_refill()
4470 buf->addr = page_pool_get_dma_addr(buf->page) + buf->page_offset; in stmmac_rx_refill()
4472 stmmac_set_desc_addr(priv, p, buf->addr); in stmmac_rx_refill()
4473 if (priv->sph) in stmmac_rx_refill()
4474 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true); in stmmac_rx_refill()
4476 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false); in stmmac_rx_refill()
4479 rx_q->rx_count_frames++; in stmmac_rx_refill()
4480 rx_q->rx_count_frames += priv->rx_coal_frames[queue]; in stmmac_rx_refill()
4481 if (rx_q->rx_count_frames > priv->rx_coal_frames[queue]) in stmmac_rx_refill()
4482 rx_q->rx_count_frames = 0; in stmmac_rx_refill()
4484 use_rx_wd = !priv->rx_coal_frames[queue]; in stmmac_rx_refill()
4485 use_rx_wd |= rx_q->rx_count_frames > 0; in stmmac_rx_refill()
4486 if (!priv->use_riwt) in stmmac_rx_refill()
4492 entry = STMMAC_GET_ENTRY(entry, priv->dma_rx_size); in stmmac_rx_refill()
4494 rx_q->dirty_rx = entry; in stmmac_rx_refill()
4495 rx_q->rx_tail_addr = rx_q->dma_rx_phy + in stmmac_rx_refill()
4496 (rx_q->dirty_rx * sizeof(struct dma_desc)); in stmmac_rx_refill()
4497 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue); in stmmac_rx_refill()
4505 int coe = priv->hw->rx_csum; in stmmac_rx_buf1_len()
4508 if (priv->sph && len) in stmmac_rx_buf1_len()
4513 if (priv->sph && hlen) { in stmmac_rx_buf1_len()
4514 priv->xstats.rx_split_hdr_pkt_n++; in stmmac_rx_buf1_len()
4520 return priv->dma_buf_sz; in stmmac_rx_buf1_len()
4525 return min_t(unsigned int, priv->dma_buf_sz, plen); in stmmac_rx_buf1_len()
4532 int coe = priv->hw->rx_csum; in stmmac_rx_buf2_len()
4536 if (!priv->sph) in stmmac_rx_buf2_len()
4541 return priv->dma_buf_sz; in stmmac_rx_buf2_len()
4546 return plen - len; in stmmac_rx_buf2_len()
4549 static int stmmac_xdp_xmit_xdpf(struct stmmac_priv *priv, int queue, in stmmac_xdp_xmit_xdpf() argument
4552 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; in stmmac_xdp_xmit_xdpf()
4553 unsigned int entry = tx_q->cur_tx; in stmmac_xdp_xmit_xdpf()
4558 if (stmmac_tx_avail(priv, queue) < STMMAC_TX_THRESH(priv)) in stmmac_xdp_xmit_xdpf()
4561 if (likely(priv->extend_desc)) in stmmac_xdp_xmit_xdpf()
4562 tx_desc = (struct dma_desc *)(tx_q->dma_etx + entry); in stmmac_xdp_xmit_xdpf()
4563 else if (tx_q->tbs & STMMAC_TBS_AVAIL) in stmmac_xdp_xmit_xdpf()
4564 tx_desc = &tx_q->dma_entx[entry].basic; in stmmac_xdp_xmit_xdpf()
4566 tx_desc = tx_q->dma_tx + entry; in stmmac_xdp_xmit_xdpf()
4569 dma_addr = dma_map_single(priv->device, xdpf->data, in stmmac_xdp_xmit_xdpf()
4570 xdpf->len, DMA_TO_DEVICE); in stmmac_xdp_xmit_xdpf()
4571 if (dma_mapping_error(priv->device, dma_addr)) in stmmac_xdp_xmit_xdpf()
4574 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XDP_NDO; in stmmac_xdp_xmit_xdpf()
4576 struct page *page = virt_to_page(xdpf->data); in stmmac_xdp_xmit_xdpf()
4579 xdpf->headroom; in stmmac_xdp_xmit_xdpf()
4580 dma_sync_single_for_device(priv->device, dma_addr, in stmmac_xdp_xmit_xdpf()
4581 xdpf->len, DMA_BIDIRECTIONAL); in stmmac_xdp_xmit_xdpf()
4583 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XDP_TX; in stmmac_xdp_xmit_xdpf()
4586 tx_q->tx_skbuff_dma[entry].buf = dma_addr; in stmmac_xdp_xmit_xdpf()
4587 tx_q->tx_skbuff_dma[entry].map_as_page = false; in stmmac_xdp_xmit_xdpf()
4588 tx_q->tx_skbuff_dma[entry].len = xdpf->len; in stmmac_xdp_xmit_xdpf()
4589 tx_q->tx_skbuff_dma[entry].last_segment = true; in stmmac_xdp_xmit_xdpf()
4590 tx_q->tx_skbuff_dma[entry].is_jumbo = false; in stmmac_xdp_xmit_xdpf()
4592 tx_q->xdpf[entry] = xdpf; in stmmac_xdp_xmit_xdpf()
4596 stmmac_prepare_tx_desc(priv, tx_desc, 1, xdpf->len, in stmmac_xdp_xmit_xdpf()
4597 true, priv->mode, true, true, in stmmac_xdp_xmit_xdpf()
4598 xdpf->len); in stmmac_xdp_xmit_xdpf()
4600 tx_q->tx_count_frames++; in stmmac_xdp_xmit_xdpf()
4602 if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0) in stmmac_xdp_xmit_xdpf()
4608 tx_q->tx_count_frames = 0; in stmmac_xdp_xmit_xdpf()
4610 priv->xstats.tx_set_ic_bit++; in stmmac_xdp_xmit_xdpf()
4613 stmmac_enable_dma_transmission(priv, priv->ioaddr); in stmmac_xdp_xmit_xdpf()
4615 entry = STMMAC_GET_ENTRY(entry, priv->dma_tx_size); in stmmac_xdp_xmit_xdpf()
4616 tx_q->cur_tx = entry; in stmmac_xdp_xmit_xdpf()
4629 while (index >= priv->plat->tx_queues_to_use) in stmmac_xdp_get_tx_queue()
4630 index -= priv->plat->tx_queues_to_use; in stmmac_xdp_get_tx_queue()
4641 int queue; in stmmac_xdp_xmit_back() local
4647 queue = stmmac_xdp_get_tx_queue(priv, cpu); in stmmac_xdp_xmit_back()
4648 nq = netdev_get_tx_queue(priv->dev, queue); in stmmac_xdp_xmit_back()
4651 /* Avoids TX time-out as we are sharing with slow path */ in stmmac_xdp_xmit_back()
4652 nq->trans_start = jiffies; in stmmac_xdp_xmit_back()
4654 res = stmmac_xdp_xmit_xdpf(priv, queue, xdpf, false); in stmmac_xdp_xmit_back()
4656 stmmac_flush_tx_descriptors(priv, queue); in stmmac_xdp_xmit_back()
4679 if (xdp_do_redirect(priv->dev, xdp, prog) < 0) in __stmmac_xdp_run_prog()
4688 trace_xdp_exception(priv->dev, prog, act); in __stmmac_xdp_run_prog()
4704 prog = READ_ONCE(priv->xdp_prog); in stmmac_xdp_run_prog()
4712 return ERR_PTR(-res); in stmmac_xdp_run_prog()
4719 int queue; in stmmac_finalize_xdp_rx() local
4721 queue = stmmac_xdp_get_tx_queue(priv, cpu); in stmmac_finalize_xdp_rx()
4724 stmmac_tx_timer_arm(priv, queue); in stmmac_finalize_xdp_rx()
4733 unsigned int metasize = xdp->data - xdp->data_meta; in stmmac_construct_skb_zc()
4734 unsigned int datasize = xdp->data_end - xdp->data; in stmmac_construct_skb_zc()
4737 skb = __napi_alloc_skb(&ch->rxtx_napi, in stmmac_construct_skb_zc()
4738 xdp->data_end - xdp->data_hard_start, in stmmac_construct_skb_zc()
4743 skb_reserve(skb, xdp->data - xdp->data_hard_start); in stmmac_construct_skb_zc()
4744 memcpy(__skb_put(skb, datasize), xdp->data, datasize); in stmmac_construct_skb_zc()
4751 static void stmmac_dispatch_skb_zc(struct stmmac_priv *priv, u32 queue, in stmmac_dispatch_skb_zc() argument
4755 struct stmmac_channel *ch = &priv->channel[queue]; in stmmac_dispatch_skb_zc()
4756 unsigned int len = xdp->data_end - xdp->data; in stmmac_dispatch_skb_zc()
4758 int coe = priv->hw->rx_csum; in stmmac_dispatch_skb_zc()
4764 priv->dev->stats.rx_dropped++; in stmmac_dispatch_skb_zc()
4769 stmmac_rx_vlan(priv->dev, skb); in stmmac_dispatch_skb_zc()
4770 skb->protocol = eth_type_trans(skb, priv->dev); in stmmac_dispatch_skb_zc()
4775 skb->ip_summed = CHECKSUM_UNNECESSARY; in stmmac_dispatch_skb_zc()
4780 skb_record_rx_queue(skb, queue); in stmmac_dispatch_skb_zc()
4781 napi_gro_receive(&ch->rxtx_napi, skb); in stmmac_dispatch_skb_zc()
4783 priv->dev->stats.rx_packets++; in stmmac_dispatch_skb_zc()
4784 priv->dev->stats.rx_bytes += len; in stmmac_dispatch_skb_zc()
4787 static bool stmmac_rx_refill_zc(struct stmmac_priv *priv, u32 queue, u32 budget) in stmmac_rx_refill_zc() argument
4789 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; in stmmac_rx_refill_zc()
4790 unsigned int entry = rx_q->dirty_rx; in stmmac_rx_refill_zc()
4794 budget = min(budget, stmmac_rx_dirty(priv, queue)); in stmmac_rx_refill_zc()
4796 while (budget-- > 0 && entry != rx_q->cur_rx) { in stmmac_rx_refill_zc()
4797 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry]; in stmmac_rx_refill_zc()
4801 if (!buf->xdp) { in stmmac_rx_refill_zc()
4802 buf->xdp = xsk_buff_alloc(rx_q->xsk_pool); in stmmac_rx_refill_zc()
4803 if (!buf->xdp) { in stmmac_rx_refill_zc()
4809 if (priv->extend_desc) in stmmac_rx_refill_zc()
4810 rx_desc = (struct dma_desc *)(rx_q->dma_erx + entry); in stmmac_rx_refill_zc()
4812 rx_desc = rx_q->dma_rx + entry; in stmmac_rx_refill_zc()
4814 dma_addr = xsk_buff_xdp_get_dma(buf->xdp); in stmmac_rx_refill_zc()
4819 rx_q->rx_count_frames++; in stmmac_rx_refill_zc()
4820 rx_q->rx_count_frames += priv->rx_coal_frames[queue]; in stmmac_rx_refill_zc()
4821 if (rx_q->rx_count_frames > priv->rx_coal_frames[queue]) in stmmac_rx_refill_zc()
4822 rx_q->rx_count_frames = 0; in stmmac_rx_refill_zc()
4824 use_rx_wd = !priv->rx_coal_frames[queue]; in stmmac_rx_refill_zc()
4825 use_rx_wd |= rx_q->rx_count_frames > 0; in stmmac_rx_refill_zc()
4826 if (!priv->use_riwt) in stmmac_rx_refill_zc()
4832 entry = STMMAC_GET_ENTRY(entry, priv->dma_rx_size); in stmmac_rx_refill_zc()
4836 rx_q->dirty_rx = entry; in stmmac_rx_refill_zc()
4837 rx_q->rx_tail_addr = rx_q->dma_rx_phy + in stmmac_rx_refill_zc()
4838 (rx_q->dirty_rx * sizeof(struct dma_desc)); in stmmac_rx_refill_zc()
4839 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue); in stmmac_rx_refill_zc()
4845 static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue) in stmmac_rx_zc() argument
4847 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; in stmmac_rx_zc()
4849 int dirty = stmmac_rx_dirty(priv, queue); in stmmac_rx_zc()
4850 unsigned int next_entry = rx_q->cur_rx; in stmmac_rx_zc()
4860 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__); in stmmac_rx_zc()
4861 if (priv->extend_desc) { in stmmac_rx_zc()
4862 rx_head = (void *)rx_q->dma_erx; in stmmac_rx_zc()
4865 rx_head = (void *)rx_q->dma_rx; in stmmac_rx_zc()
4869 stmmac_display_ring(priv, rx_head, priv->dma_rx_size, true, in stmmac_rx_zc()
4870 rx_q->dma_rx_phy, desc_size); in stmmac_rx_zc()
4879 if (!count && rx_q->state_saved) { in stmmac_rx_zc()
4880 error = rx_q->state.error; in stmmac_rx_zc()
4881 len = rx_q->state.len; in stmmac_rx_zc()
4883 rx_q->state_saved = false; in stmmac_rx_zc()
4894 buf = &rx_q->buf_pool[entry]; in stmmac_rx_zc()
4898 !stmmac_rx_refill_zc(priv, queue, dirty); in stmmac_rx_zc()
4902 if (priv->extend_desc) in stmmac_rx_zc()
4903 p = (struct dma_desc *)(rx_q->dma_erx + entry); in stmmac_rx_zc()
4905 p = rx_q->dma_rx + entry; in stmmac_rx_zc()
4908 status = stmmac_rx_status(priv, &priv->dev->stats, in stmmac_rx_zc()
4909 &priv->xstats, p); in stmmac_rx_zc()
4914 /* Prefetch the next RX descriptor */ in stmmac_rx_zc()
4915 rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, in stmmac_rx_zc()
4916 priv->dma_rx_size); in stmmac_rx_zc()
4917 next_entry = rx_q->cur_rx; in stmmac_rx_zc()
4919 if (priv->extend_desc) in stmmac_rx_zc()
4920 np = (struct dma_desc *)(rx_q->dma_erx + next_entry); in stmmac_rx_zc()
4922 np = rx_q->dma_rx + next_entry; in stmmac_rx_zc()
4927 if (!buf->xdp) in stmmac_rx_zc()
4930 if (priv->extend_desc) in stmmac_rx_zc()
4931 stmmac_rx_extended_status(priv, &priv->dev->stats, in stmmac_rx_zc()
4932 &priv->xstats, in stmmac_rx_zc()
4933 rx_q->dma_erx + entry); in stmmac_rx_zc()
4935 xsk_buff_free(buf->xdp); in stmmac_rx_zc()
4936 buf->xdp = NULL; in stmmac_rx_zc()
4939 if (!priv->hwts_rx_en) in stmmac_rx_zc()
4940 priv->dev->stats.rx_errors++; in stmmac_rx_zc()
4950 /* XSK pool expects RX frame 1:1 mapped to XSK buffer */ in stmmac_rx_zc()
4952 xsk_buff_free(buf->xdp); in stmmac_rx_zc()
4953 buf->xdp = NULL; in stmmac_rx_zc()
4964 * Type frames (LLC/LLC-SNAP) in stmmac_rx_zc()
4971 (likely(priv->synopsys_id >= DWMAC_CORE_4_00) || in stmmac_rx_zc()
4973 buf1_len -= ETH_FCS_LEN; in stmmac_rx_zc()
4974 len -= ETH_FCS_LEN; in stmmac_rx_zc()
4977 /* RX buffer is good and fit into a XSK pool buffer */ in stmmac_rx_zc()
4978 buf->xdp->data_end = buf->xdp->data + buf1_len; in stmmac_rx_zc()
4979 xsk_buff_dma_sync_for_cpu(buf->xdp, rx_q->xsk_pool); in stmmac_rx_zc()
4981 prog = READ_ONCE(priv->xdp_prog); in stmmac_rx_zc()
4982 res = __stmmac_xdp_run_prog(priv, prog, buf->xdp); in stmmac_rx_zc()
4986 stmmac_dispatch_skb_zc(priv, queue, p, np, buf->xdp); in stmmac_rx_zc()
4987 xsk_buff_free(buf->xdp); in stmmac_rx_zc()
4990 xsk_buff_free(buf->xdp); in stmmac_rx_zc()
4991 priv->dev->stats.rx_dropped++; in stmmac_rx_zc()
4999 buf->xdp = NULL; in stmmac_rx_zc()
5005 rx_q->state_saved = true; in stmmac_rx_zc()
5006 rx_q->state.error = error; in stmmac_rx_zc()
5007 rx_q->state.len = len; in stmmac_rx_zc()
5012 priv->xstats.rx_pkt_n += count; in stmmac_rx_zc()
5013 priv->xstats.rxq_stats[queue].rx_pkt_n += count; in stmmac_rx_zc()
5015 if (xsk_uses_need_wakeup(rx_q->xsk_pool)) { in stmmac_rx_zc()
5016 if (failure || stmmac_rx_dirty(priv, queue) > 0) in stmmac_rx_zc()
5017 xsk_set_rx_need_wakeup(rx_q->xsk_pool); in stmmac_rx_zc()
5019 xsk_clear_rx_need_wakeup(rx_q->xsk_pool); in stmmac_rx_zc()
5028 * stmmac_rx - manage the receive process
5031 * @queue: RX queue index.
5035 static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) in stmmac_rx() argument
5037 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; in stmmac_rx()
5038 struct stmmac_channel *ch = &priv->channel[queue]; in stmmac_rx()
5040 int status = 0, coe = priv->hw->rx_csum; in stmmac_rx()
5041 unsigned int next_entry = rx_q->cur_rx; in stmmac_rx()
5049 dma_dir = page_pool_get_dma_dir(rx_q->page_pool); in stmmac_rx()
5050 buf_sz = DIV_ROUND_UP(priv->dma_buf_sz, PAGE_SIZE) * PAGE_SIZE; in stmmac_rx()
5055 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__); in stmmac_rx()
5056 if (priv->extend_desc) { in stmmac_rx()
5057 rx_head = (void *)rx_q->dma_erx; in stmmac_rx()
5060 rx_head = (void *)rx_q->dma_rx; in stmmac_rx()
5064 stmmac_display_ring(priv, rx_head, priv->dma_rx_size, true, in stmmac_rx()
5065 rx_q->dma_rx_phy, desc_size); in stmmac_rx()
5075 if (!count && rx_q->state_saved) { in stmmac_rx()
5076 skb = rx_q->state.skb; in stmmac_rx()
5077 error = rx_q->state.error; in stmmac_rx()
5078 len = rx_q->state.len; in stmmac_rx()
5080 rx_q->state_saved = false; in stmmac_rx()
5093 buf = &rx_q->buf_pool[entry]; in stmmac_rx()
5095 if (priv->extend_desc) in stmmac_rx()
5096 p = (struct dma_desc *)(rx_q->dma_erx + entry); in stmmac_rx()
5098 p = rx_q->dma_rx + entry; in stmmac_rx()
5101 status = stmmac_rx_status(priv, &priv->dev->stats, in stmmac_rx()
5102 &priv->xstats, p); in stmmac_rx()
5107 rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, in stmmac_rx()
5108 priv->dma_rx_size); in stmmac_rx()
5109 next_entry = rx_q->cur_rx; in stmmac_rx()
5111 if (priv->extend_desc) in stmmac_rx()
5112 np = (struct dma_desc *)(rx_q->dma_erx + next_entry); in stmmac_rx()
5114 np = rx_q->dma_rx + next_entry; in stmmac_rx()
5118 if (priv->extend_desc) in stmmac_rx()
5119 stmmac_rx_extended_status(priv, &priv->dev->stats, in stmmac_rx()
5120 &priv->xstats, rx_q->dma_erx + entry); in stmmac_rx()
5122 page_pool_recycle_direct(rx_q->page_pool, buf->page); in stmmac_rx()
5123 buf->page = NULL; in stmmac_rx()
5125 if (!priv->hwts_rx_en) in stmmac_rx()
5126 priv->dev->stats.rx_errors++; in stmmac_rx()
5140 prefetch(page_address(buf->page) + buf->page_offset); in stmmac_rx()
5141 if (buf->sec_page) in stmmac_rx()
5142 prefetch(page_address(buf->sec_page)); in stmmac_rx()
5150 * Type frames (LLC/LLC-SNAP) in stmmac_rx()
5157 (likely(priv->synopsys_id >= DWMAC_CORE_4_00) || in stmmac_rx()
5160 buf2_len -= ETH_FCS_LEN; in stmmac_rx()
5162 buf1_len -= ETH_FCS_LEN; in stmmac_rx()
5164 len -= ETH_FCS_LEN; in stmmac_rx()
5170 dma_sync_single_for_cpu(priv->device, buf->addr, in stmmac_rx()
5173 xdp_init_buff(&xdp, buf_sz, &rx_q->xdp_rxq); in stmmac_rx()
5174 xdp_prepare_buff(&xdp, page_address(buf->page), in stmmac_rx()
5175 buf->page_offset, buf1_len, false); in stmmac_rx()
5177 pre_len = xdp.data_end - xdp.data_hard_start - in stmmac_rx()
5178 buf->page_offset; in stmmac_rx()
5183 sync_len = xdp.data_end - xdp.data_hard_start - in stmmac_rx()
5184 buf->page_offset; in stmmac_rx()
5189 unsigned int xdp_res = -PTR_ERR(skb); in stmmac_rx()
5192 page_pool_put_page(rx_q->page_pool, in stmmac_rx()
5195 buf->page = NULL; in stmmac_rx()
5196 priv->dev->stats.rx_dropped++; in stmmac_rx()
5211 buf->page = NULL; in stmmac_rx()
5221 buf1_len = xdp.data_end - xdp.data; in stmmac_rx()
5223 skb = napi_alloc_skb(&ch->rx_napi, buf1_len); in stmmac_rx()
5225 priv->dev->stats.rx_dropped++; in stmmac_rx()
5235 page_pool_recycle_direct(rx_q->page_pool, buf->page); in stmmac_rx()
5236 buf->page = NULL; in stmmac_rx()
5238 dma_sync_single_for_cpu(priv->device, buf->addr, in stmmac_rx()
5240 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, in stmmac_rx()
5241 buf->page, buf->page_offset, buf1_len, in stmmac_rx()
5242 priv->dma_buf_sz); in stmmac_rx()
5245 page_pool_release_page(rx_q->page_pool, buf->page); in stmmac_rx()
5246 buf->page = NULL; in stmmac_rx()
5250 dma_sync_single_for_cpu(priv->device, buf->sec_addr, in stmmac_rx()
5252 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, in stmmac_rx()
5253 buf->sec_page, 0, buf2_len, in stmmac_rx()
5254 priv->dma_buf_sz); in stmmac_rx()
5257 page_pool_release_page(rx_q->page_pool, buf->sec_page); in stmmac_rx()
5258 buf->sec_page = NULL; in stmmac_rx()
5270 stmmac_rx_vlan(priv->dev, skb); in stmmac_rx()
5271 skb->protocol = eth_type_trans(skb, priv->dev); in stmmac_rx()
5276 skb->ip_summed = CHECKSUM_UNNECESSARY; in stmmac_rx()
5281 skb_record_rx_queue(skb, queue); in stmmac_rx()
5282 napi_gro_receive(&ch->rx_napi, skb); in stmmac_rx()
5285 priv->dev->stats.rx_packets++; in stmmac_rx()
5286 priv->dev->stats.rx_bytes += len; in stmmac_rx()
5291 rx_q->state_saved = true; in stmmac_rx()
5292 rx_q->state.skb = skb; in stmmac_rx()
5293 rx_q->state.error = error; in stmmac_rx()
5294 rx_q->state.len = len; in stmmac_rx()
5299 stmmac_rx_refill(priv, queue); in stmmac_rx()
5301 priv->xstats.rx_pkt_n += count; in stmmac_rx()
5302 priv->xstats.rxq_stats[queue].rx_pkt_n += count; in stmmac_rx()
5311 struct stmmac_priv *priv = ch->priv_data; in stmmac_napi_poll_rx()
5312 u32 chan = ch->index; in stmmac_napi_poll_rx()
5315 priv->xstats.napi_poll++; in stmmac_napi_poll_rx()
5321 spin_lock_irqsave(&ch->lock, flags); in stmmac_napi_poll_rx()
5322 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 0); in stmmac_napi_poll_rx()
5323 spin_unlock_irqrestore(&ch->lock, flags); in stmmac_napi_poll_rx()
5333 struct stmmac_priv *priv = ch->priv_data; in stmmac_napi_poll_tx()
5334 u32 chan = ch->index; in stmmac_napi_poll_tx()
5337 priv->xstats.napi_poll++; in stmmac_napi_poll_tx()
5345 spin_lock_irqsave(&ch->lock, flags); in stmmac_napi_poll_tx()
5346 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 0, 1); in stmmac_napi_poll_tx()
5347 spin_unlock_irqrestore(&ch->lock, flags); in stmmac_napi_poll_tx()
5357 struct stmmac_priv *priv = ch->priv_data; in stmmac_napi_poll_rxtx()
5359 u32 chan = ch->index; in stmmac_napi_poll_rxtx()
5361 priv->xstats.napi_poll++; in stmmac_napi_poll_rxtx()
5370 /* If either TX or RX work is not complete, return budget in stmmac_napi_poll_rxtx()
5380 spin_lock_irqsave(&ch->lock, flags); in stmmac_napi_poll_rxtx()
5381 /* Both RX and TX work done are compelte, in stmmac_napi_poll_rxtx()
5382 * so enable both RX & TX IRQs. in stmmac_napi_poll_rxtx()
5384 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1); in stmmac_napi_poll_rxtx()
5385 spin_unlock_irqrestore(&ch->lock, flags); in stmmac_napi_poll_rxtx()
5388 return min(rxtx_done, budget - 1); in stmmac_napi_poll_rxtx()
5394 * @txqueue: the index of the hanging transmit queue
5408 * stmmac_set_rx_mode - entry point for multicast addressing
5420 stmmac_set_filter(priv, priv->hw, dev); in stmmac_set_rx_mode()
5424 * stmmac_change_mtu - entry point to change MTU size for the device.
5431 * 0 on success and an appropriate (-)ve integer as defined in errno.h
5437 int txfifosz = priv->plat->tx_fifo_size; in stmmac_change_mtu()
5441 txfifosz = priv->dma_cap.tx_fifo_size; in stmmac_change_mtu()
5443 txfifosz /= priv->plat->tx_queues_to_use; in stmmac_change_mtu()
5446 netdev_err(priv->dev, "must be stopped to change its MTU\n"); in stmmac_change_mtu()
5447 return -EBUSY; in stmmac_change_mtu()
5451 netdev_dbg(priv->dev, "Jumbo frames not supported for XDP\n"); in stmmac_change_mtu()
5452 return -EINVAL; in stmmac_change_mtu()
5459 return -EINVAL; in stmmac_change_mtu()
5461 dev->mtu = mtu; in stmmac_change_mtu()
5473 if (priv->plat->rx_coe == STMMAC_RX_COE_NONE) in stmmac_fix_features()
5476 if (!priv->plat->tx_coe) in stmmac_fix_features()
5484 if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN)) in stmmac_fix_features()
5488 if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) { in stmmac_fix_features()
5490 priv->tso = true; in stmmac_fix_features()
5492 priv->tso = false; in stmmac_fix_features()
5507 priv->hw->rx_csum = priv->plat->rx_coe; in stmmac_set_features()
5509 priv->hw->rx_csum = 0; in stmmac_set_features()
5513 stmmac_rx_ipc(priv, priv->hw); in stmmac_set_features()
5515 sph_en = (priv->hw->rx_csum > 0) && priv->sph; in stmmac_set_features()
5517 for (chan = 0; chan < priv->plat->rx_queues_to_use; chan++) in stmmac_set_features()
5518 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); in stmmac_set_features()
5525 struct stmmac_fpe_cfg *fpe_cfg = priv->plat->fpe_cfg; in stmmac_fpe_event_status()
5526 enum stmmac_fpe_state *lo_state = &fpe_cfg->lo_fpe_state; in stmmac_fpe_event_status()
5527 enum stmmac_fpe_state *lp_state = &fpe_cfg->lp_fpe_state; in stmmac_fpe_event_status()
5528 bool *hs_enable = &fpe_cfg->hs_enable; in stmmac_fpe_event_status()
5540 stmmac_fpe_send_mpacket(priv, priv->ioaddr, in stmmac_fpe_event_status()
5558 if (!test_bit(__FPE_REMOVING, &priv->fpe_task_state) && in stmmac_fpe_event_status()
5559 !test_and_set_bit(__FPE_TASK_SCHED, &priv->fpe_task_state) && in stmmac_fpe_event_status()
5560 priv->fpe_wq) { in stmmac_fpe_event_status()
5561 queue_work(priv->fpe_wq, &priv->fpe_task); in stmmac_fpe_event_status()
5567 u32 rx_cnt = priv->plat->rx_queues_to_use; in stmmac_common_interrupt()
5568 u32 tx_cnt = priv->plat->tx_queues_to_use; in stmmac_common_interrupt()
5570 u32 queue; in stmmac_common_interrupt() local
5573 xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; in stmmac_common_interrupt()
5576 if (priv->irq_wake) in stmmac_common_interrupt()
5577 pm_wakeup_event(priv->device, 0); in stmmac_common_interrupt()
5579 if (priv->dma_cap.estsel) in stmmac_common_interrupt()
5580 stmmac_est_irq_status(priv, priv->ioaddr, priv->dev, in stmmac_common_interrupt()
5581 &priv->xstats, tx_cnt); in stmmac_common_interrupt()
5583 if (priv->dma_cap.fpesel) { in stmmac_common_interrupt()
5584 int status = stmmac_fpe_irq_status(priv, priv->ioaddr, in stmmac_common_interrupt()
5585 priv->dev); in stmmac_common_interrupt()
5591 if ((priv->plat->has_gmac) || xmac) { in stmmac_common_interrupt()
5592 int status = stmmac_host_irq_status(priv, priv->hw, &priv->xstats); in stmmac_common_interrupt()
5597 priv->tx_path_in_lpi_mode = true; in stmmac_common_interrupt()
5599 priv->tx_path_in_lpi_mode = false; in stmmac_common_interrupt()
5602 for (queue = 0; queue < queues_count; queue++) { in stmmac_common_interrupt()
5603 status = stmmac_host_mtl_irq_status(priv, priv->hw, in stmmac_common_interrupt()
5604 queue); in stmmac_common_interrupt()
5608 if (priv->hw->pcs) { in stmmac_common_interrupt()
5609 if (priv->xstats.pcs_link) in stmmac_common_interrupt()
5610 netif_carrier_on(priv->dev); in stmmac_common_interrupt()
5612 netif_carrier_off(priv->dev); in stmmac_common_interrupt()
5620 * stmmac_interrupt - main ISR
5627 * o Core interrupts to manage: remote wake-up, management counter, LPI
5636 if (test_bit(STMMAC_DOWN, &priv->state)) in stmmac_interrupt()
5658 netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__); in stmmac_mac_interrupt()
5663 if (test_bit(STMMAC_DOWN, &priv->state)) in stmmac_mac_interrupt()
5678 netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__); in stmmac_safety_interrupt()
5683 if (test_bit(STMMAC_DOWN, &priv->state)) in stmmac_safety_interrupt()
5695 int chan = tx_q->queue_index; in stmmac_msi_intr_tx()
5702 netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__); in stmmac_msi_intr_tx()
5707 if (test_bit(STMMAC_DOWN, &priv->state)) in stmmac_msi_intr_tx()
5714 if (unlikely(priv->xstats.threshold != SF_DMA_MODE) && in stmmac_msi_intr_tx()
5717 if (priv->plat->force_thresh_dma_mode) in stmmac_msi_intr_tx()
5727 priv->xstats.threshold = tc; in stmmac_msi_intr_tx()
5739 int chan = rx_q->queue_index; in stmmac_msi_intr_rx()
5745 netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__); in stmmac_msi_intr_rx()
5750 if (test_bit(STMMAC_DOWN, &priv->state)) in stmmac_msi_intr_rx()
5759 /* Polling receive - used by NETCONSOLE and other diagnostic tools
5768 if (test_bit(STMMAC_DOWN, &priv->state)) in stmmac_poll_controller()
5771 if (priv->plat->multi_msi_en) { in stmmac_poll_controller()
5772 for (i = 0; i < priv->plat->rx_queues_to_use; i++) in stmmac_poll_controller()
5773 stmmac_msi_intr_rx(0, &priv->rx_queue[i]); in stmmac_poll_controller()
5775 for (i = 0; i < priv->plat->tx_queues_to_use; i++) in stmmac_poll_controller()
5776 stmmac_msi_intr_tx(0, &priv->tx_queue[i]); in stmmac_poll_controller()
5778 disable_irq(dev->irq); in stmmac_poll_controller()
5779 stmmac_interrupt(dev->irq, dev); in stmmac_poll_controller()
5780 enable_irq(dev->irq); in stmmac_poll_controller()
5786 * stmmac_ioctl - Entry point for the Ioctl
5797 int ret = -EOPNOTSUPP; in stmmac_ioctl()
5800 return -EINVAL; in stmmac_ioctl()
5806 ret = phylink_mii_ioctl(priv->phylink, rq, cmd); in stmmac_ioctl()
5825 int ret = -EOPNOTSUPP; in stmmac_setup_tc_block_cb()
5827 if (!tc_cls_can_offload_and_chain0(priv->dev, type_data)) in stmmac_setup_tc_block_cb()
5867 return -EOPNOTSUPP; in stmmac_setup_tc()
5874 int gso = skb_shinfo(skb)->gso_type; in stmmac_select_queue()
5879 * capable Queues. Let's use always the Queue 0 in stmmac_select_queue()
5886 return netdev_pick_tx(dev, skb, NULL) % dev->real_num_tx_queues; in stmmac_select_queue()
5894 ret = pm_runtime_get_sync(priv->device); in stmmac_set_mac_address()
5896 pm_runtime_put_noidle(priv->device); in stmmac_set_mac_address()
5904 stmmac_set_umac_addr(priv, priv->hw, ndev->dev_addr, 0); in stmmac_set_mac_address()
5907 pm_runtime_put(priv->device); in stmmac_set_mac_address()
5928 le32_to_cpu(ep->basic.des0), in sysfs_display_ring()
5929 le32_to_cpu(ep->basic.des1), in sysfs_display_ring()
5930 le32_to_cpu(ep->basic.des2), in sysfs_display_ring()
5931 le32_to_cpu(ep->basic.des3)); in sysfs_display_ring()
5937 le32_to_cpu(p->des0), le32_to_cpu(p->des1), in sysfs_display_ring()
5938 le32_to_cpu(p->des2), le32_to_cpu(p->des3)); in sysfs_display_ring()
5947 struct net_device *dev = seq->private; in stmmac_rings_status_show()
5949 u32 rx_count = priv->plat->rx_queues_to_use; in stmmac_rings_status_show()
5950 u32 tx_count = priv->plat->tx_queues_to_use; in stmmac_rings_status_show()
5951 u32 queue; in stmmac_rings_status_show() local
5953 if ((dev->flags & IFF_UP) == 0) in stmmac_rings_status_show()
5956 for (queue = 0; queue < rx_count; queue++) { in stmmac_rings_status_show()
5957 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; in stmmac_rings_status_show()
5959 seq_printf(seq, "RX Queue %d:\n", queue); in stmmac_rings_status_show()
5961 if (priv->extend_desc) { in stmmac_rings_status_show()
5963 sysfs_display_ring((void *)rx_q->dma_erx, in stmmac_rings_status_show()
5964 priv->dma_rx_size, 1, seq, rx_q->dma_rx_phy); in stmmac_rings_status_show()
5967 sysfs_display_ring((void *)rx_q->dma_rx, in stmmac_rings_status_show()
5968 priv->dma_rx_size, 0, seq, rx_q->dma_rx_phy); in stmmac_rings_status_show()
5972 for (queue = 0; queue < tx_count; queue++) { in stmmac_rings_status_show()
5973 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; in stmmac_rings_status_show()
5975 seq_printf(seq, "TX Queue %d:\n", queue); in stmmac_rings_status_show()
5977 if (priv->extend_desc) { in stmmac_rings_status_show()
5979 sysfs_display_ring((void *)tx_q->dma_etx, in stmmac_rings_status_show()
5980 priv->dma_tx_size, 1, seq, tx_q->dma_tx_phy); in stmmac_rings_status_show()
5981 } else if (!(tx_q->tbs & STMMAC_TBS_AVAIL)) { in stmmac_rings_status_show()
5983 sysfs_display_ring((void *)tx_q->dma_tx, in stmmac_rings_status_show()
5984 priv->dma_tx_size, 0, seq, tx_q->dma_tx_phy); in stmmac_rings_status_show()
5994 struct net_device *dev = seq->private; in stmmac_dma_cap_show()
5997 if (!priv->hw_cap_support) { in stmmac_dma_cap_show()
6007 (priv->dma_cap.mbps_10_100) ? "Y" : "N"); in stmmac_dma_cap_show()
6009 (priv->dma_cap.mbps_1000) ? "Y" : "N"); in stmmac_dma_cap_show()
6011 (priv->dma_cap.half_duplex) ? "Y" : "N"); in stmmac_dma_cap_show()
6013 (priv->dma_cap.hash_filter) ? "Y" : "N"); in stmmac_dma_cap_show()
6015 (priv->dma_cap.multi_addr) ? "Y" : "N"); in stmmac_dma_cap_show()
6017 (priv->dma_cap.pcs) ? "Y" : "N"); in stmmac_dma_cap_show()
6019 (priv->dma_cap.sma_mdio) ? "Y" : "N"); in stmmac_dma_cap_show()
6021 (priv->dma_cap.pmt_remote_wake_up) ? "Y" : "N"); in stmmac_dma_cap_show()
6023 (priv->dma_cap.pmt_magic_frame) ? "Y" : "N"); in stmmac_dma_cap_show()
6025 (priv->dma_cap.rmon) ? "Y" : "N"); in stmmac_dma_cap_show()
6026 seq_printf(seq, "\tIEEE 1588-2002 Time Stamp: %s\n", in stmmac_dma_cap_show()
6027 (priv->dma_cap.time_stamp) ? "Y" : "N"); in stmmac_dma_cap_show()
6028 seq_printf(seq, "\tIEEE 1588-2008 Advanced Time Stamp: %s\n", in stmmac_dma_cap_show()
6029 (priv->dma_cap.atime_stamp) ? "Y" : "N"); in stmmac_dma_cap_show()
6030 seq_printf(seq, "\t802.3az - Energy-Efficient Ethernet (EEE): %s\n", in stmmac_dma_cap_show()
6031 (priv->dma_cap.eee) ? "Y" : "N"); in stmmac_dma_cap_show()
6032 seq_printf(seq, "\tAV features: %s\n", (priv->dma_cap.av) ? "Y" : "N"); in stmmac_dma_cap_show()
6034 (priv->dma_cap.tx_coe) ? "Y" : "N"); in stmmac_dma_cap_show()
6035 if (priv->synopsys_id >= DWMAC_CORE_4_00) { in stmmac_dma_cap_show()
6036 seq_printf(seq, "\tIP Checksum Offload in RX: %s\n", in stmmac_dma_cap_show()
6037 (priv->dma_cap.rx_coe) ? "Y" : "N"); in stmmac_dma_cap_show()
6039 seq_printf(seq, "\tIP Checksum Offload (type1) in RX: %s\n", in stmmac_dma_cap_show()
6040 (priv->dma_cap.rx_coe_type1) ? "Y" : "N"); in stmmac_dma_cap_show()
6041 seq_printf(seq, "\tIP Checksum Offload (type2) in RX: %s\n", in stmmac_dma_cap_show()
6042 (priv->dma_cap.rx_coe_type2) ? "Y" : "N"); in stmmac_dma_cap_show()
6045 (priv->dma_cap.rxfifo_over_2048) ? "Y" : "N"); in stmmac_dma_cap_show()
6046 seq_printf(seq, "\tNumber of Additional RX channel: %d\n", in stmmac_dma_cap_show()
6047 priv->dma_cap.number_rx_channel); in stmmac_dma_cap_show()
6049 priv->dma_cap.number_tx_channel); in stmmac_dma_cap_show()
6050 seq_printf(seq, "\tNumber of Additional RX queues: %d\n", in stmmac_dma_cap_show()
6051 priv->dma_cap.number_rx_queues); in stmmac_dma_cap_show()
6053 priv->dma_cap.number_tx_queues); in stmmac_dma_cap_show()
6055 (priv->dma_cap.enh_desc) ? "Y" : "N"); in stmmac_dma_cap_show()
6056 seq_printf(seq, "\tTX Fifo Size: %d\n", priv->dma_cap.tx_fifo_size); in stmmac_dma_cap_show()
6057 seq_printf(seq, "\tRX Fifo Size: %d\n", priv->dma_cap.rx_fifo_size); in stmmac_dma_cap_show()
6058 seq_printf(seq, "\tHash Table Size: %d\n", priv->dma_cap.hash_tb_sz); in stmmac_dma_cap_show()
6059 seq_printf(seq, "\tTSO: %s\n", priv->dma_cap.tsoen ? "Y" : "N"); in stmmac_dma_cap_show()
6061 priv->dma_cap.pps_out_num); in stmmac_dma_cap_show()
6063 priv->dma_cap.asp ? "Y" : "N"); in stmmac_dma_cap_show()
6064 seq_printf(seq, "\tFlexible RX Parser: %s\n", in stmmac_dma_cap_show()
6065 priv->dma_cap.frpsel ? "Y" : "N"); in stmmac_dma_cap_show()
6067 priv->dma_cap.addr64); in stmmac_dma_cap_show()
6069 priv->dma_cap.rssen ? "Y" : "N"); in stmmac_dma_cap_show()
6071 priv->dma_cap.vlhash ? "Y" : "N"); in stmmac_dma_cap_show()
6073 priv->dma_cap.sphen ? "Y" : "N"); in stmmac_dma_cap_show()
6075 priv->dma_cap.vlins ? "Y" : "N"); in stmmac_dma_cap_show()
6077 priv->dma_cap.dvlan ? "Y" : "N"); in stmmac_dma_cap_show()
6079 priv->dma_cap.l3l4fnum); in stmmac_dma_cap_show()
6081 priv->dma_cap.arpoffsel ? "Y" : "N"); in stmmac_dma_cap_show()
6083 priv->dma_cap.estsel ? "Y" : "N"); in stmmac_dma_cap_show()
6085 priv->dma_cap.fpesel ? "Y" : "N"); in stmmac_dma_cap_show()
6086 seq_printf(seq, "\tTime-Based Scheduling (TBS): %s\n", in stmmac_dma_cap_show()
6087 priv->dma_cap.tbssel ? "Y" : "N"); in stmmac_dma_cap_show()
6100 if (dev->netdev_ops != &stmmac_netdev_ops) in stmmac_device_event()
6105 if (priv->dbgfs_dir) in stmmac_device_event()
6106 priv->dbgfs_dir = debugfs_rename(stmmac_fs_dir, in stmmac_device_event()
6107 priv->dbgfs_dir, in stmmac_device_event()
6109 dev->name); in stmmac_device_event()
6127 priv->dbgfs_dir = debugfs_create_dir(dev->name, stmmac_fs_dir); in stmmac_init_fs()
6129 /* Entry to report DMA RX/TX rings */ in stmmac_init_fs()
6130 debugfs_create_file("descriptors_status", 0444, priv->dbgfs_dir, dev, in stmmac_init_fs()
6134 debugfs_create_file("dma_cap", 0444, priv->dbgfs_dir, dev, in stmmac_init_fs()
6144 debugfs_remove_recursive(priv->dbgfs_dir); in stmmac_exit_fs()
6179 for_each_set_bit(vid, priv->active_vlans, VLAN_N_VID) { in stmmac_vlan_update()
6186 if (!priv->dma_cap.vlhash) { in stmmac_vlan_update()
6188 return -EOPNOTSUPP; in stmmac_vlan_update()
6194 return stmmac_update_vlan_hash(priv, priv->hw, hash, pmatch, is_double); in stmmac_vlan_update()
6206 set_bit(vid, priv->active_vlans); in stmmac_vlan_rx_add_vid()
6209 clear_bit(vid, priv->active_vlans); in stmmac_vlan_rx_add_vid()
6213 if (priv->hw->num_vlan) { in stmmac_vlan_rx_add_vid()
6214 ret = stmmac_add_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid); in stmmac_vlan_rx_add_vid()
6228 ret = pm_runtime_get_sync(priv->device); in stmmac_vlan_rx_kill_vid()
6230 pm_runtime_put_noidle(priv->device); in stmmac_vlan_rx_kill_vid()
6237 clear_bit(vid, priv->active_vlans); in stmmac_vlan_rx_kill_vid()
6239 if (priv->hw->num_vlan) { in stmmac_vlan_rx_kill_vid()
6240 ret = stmmac_del_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid); in stmmac_vlan_rx_kill_vid()
6248 pm_runtime_put(priv->device); in stmmac_vlan_rx_kill_vid()
6257 switch (bpf->command) { in stmmac_bpf()
6259 return stmmac_xdp_set_prog(priv, bpf->prog, bpf->extack); in stmmac_bpf()
6261 return stmmac_xdp_setup_pool(priv, bpf->xsk.pool, in stmmac_bpf()
6262 bpf->xsk.queue_id); in stmmac_bpf()
6264 return -EOPNOTSUPP; in stmmac_bpf()
6275 int queue; in stmmac_xdp_xmit() local
6277 if (unlikely(test_bit(STMMAC_DOWN, &priv->state))) in stmmac_xdp_xmit()
6278 return -ENETDOWN; in stmmac_xdp_xmit()
6281 return -EINVAL; in stmmac_xdp_xmit()
6283 queue = stmmac_xdp_get_tx_queue(priv, cpu); in stmmac_xdp_xmit()
6284 nq = netdev_get_tx_queue(priv->dev, queue); in stmmac_xdp_xmit()
6287 /* Avoids TX time-out as we are sharing with slow path */ in stmmac_xdp_xmit()
6288 nq->trans_start = jiffies; in stmmac_xdp_xmit()
6293 res = stmmac_xdp_xmit_xdpf(priv, queue, frames[i], true); in stmmac_xdp_xmit()
6301 stmmac_flush_tx_descriptors(priv, queue); in stmmac_xdp_xmit()
6302 stmmac_tx_timer_arm(priv, queue); in stmmac_xdp_xmit()
6310 void stmmac_disable_rx_queue(struct stmmac_priv *priv, u32 queue) in stmmac_disable_rx_queue() argument
6312 struct stmmac_channel *ch = &priv->channel[queue]; in stmmac_disable_rx_queue()
6315 spin_lock_irqsave(&ch->lock, flags); in stmmac_disable_rx_queue()
6316 stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 1, 0); in stmmac_disable_rx_queue()
6317 spin_unlock_irqrestore(&ch->lock, flags); in stmmac_disable_rx_queue()
6319 stmmac_stop_rx_dma(priv, queue); in stmmac_disable_rx_queue()
6320 __free_dma_rx_desc_resources(priv, queue); in stmmac_disable_rx_queue()
6323 void stmmac_enable_rx_queue(struct stmmac_priv *priv, u32 queue) in stmmac_enable_rx_queue() argument
6325 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; in stmmac_enable_rx_queue()
6326 struct stmmac_channel *ch = &priv->channel[queue]; in stmmac_enable_rx_queue()
6331 ret = __alloc_dma_rx_desc_resources(priv, queue); in stmmac_enable_rx_queue()
6333 netdev_err(priv->dev, "Failed to alloc RX desc.\n"); in stmmac_enable_rx_queue()
6337 ret = __init_dma_rx_desc_rings(priv, queue, GFP_KERNEL); in stmmac_enable_rx_queue()
6339 __free_dma_rx_desc_resources(priv, queue); in stmmac_enable_rx_queue()
6340 netdev_err(priv->dev, "Failed to init RX desc.\n"); in stmmac_enable_rx_queue()
6344 stmmac_clear_rx_descriptors(priv, queue); in stmmac_enable_rx_queue()
6346 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, in stmmac_enable_rx_queue()
6347 rx_q->dma_rx_phy, rx_q->queue_index); in stmmac_enable_rx_queue()
6349 rx_q->rx_tail_addr = rx_q->dma_rx_phy + (rx_q->buf_alloc_num * in stmmac_enable_rx_queue()
6351 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, in stmmac_enable_rx_queue()
6352 rx_q->rx_tail_addr, rx_q->queue_index); in stmmac_enable_rx_queue()
6354 if (rx_q->xsk_pool && rx_q->buf_alloc_num) { in stmmac_enable_rx_queue()
6355 buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool); in stmmac_enable_rx_queue()
6356 stmmac_set_dma_bfsize(priv, priv->ioaddr, in stmmac_enable_rx_queue()
6358 rx_q->queue_index); in stmmac_enable_rx_queue()
6360 stmmac_set_dma_bfsize(priv, priv->ioaddr, in stmmac_enable_rx_queue()
6361 priv->dma_buf_sz, in stmmac_enable_rx_queue()
6362 rx_q->queue_index); in stmmac_enable_rx_queue()
6365 stmmac_start_rx_dma(priv, queue); in stmmac_enable_rx_queue()
6367 spin_lock_irqsave(&ch->lock, flags); in stmmac_enable_rx_queue()
6368 stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 1, 0); in stmmac_enable_rx_queue()
6369 spin_unlock_irqrestore(&ch->lock, flags); in stmmac_enable_rx_queue()
6372 void stmmac_disable_tx_queue(struct stmmac_priv *priv, u32 queue) in stmmac_disable_tx_queue() argument
6374 struct stmmac_channel *ch = &priv->channel[queue]; in stmmac_disable_tx_queue()
6377 spin_lock_irqsave(&ch->lock, flags); in stmmac_disable_tx_queue()
6378 stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 0, 1); in stmmac_disable_tx_queue()
6379 spin_unlock_irqrestore(&ch->lock, flags); in stmmac_disable_tx_queue()
6381 stmmac_stop_tx_dma(priv, queue); in stmmac_disable_tx_queue()
6382 __free_dma_tx_desc_resources(priv, queue); in stmmac_disable_tx_queue()
6385 void stmmac_enable_tx_queue(struct stmmac_priv *priv, u32 queue) in stmmac_enable_tx_queue() argument
6387 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; in stmmac_enable_tx_queue()
6388 struct stmmac_channel *ch = &priv->channel[queue]; in stmmac_enable_tx_queue()
6392 ret = __alloc_dma_tx_desc_resources(priv, queue); in stmmac_enable_tx_queue()
6394 netdev_err(priv->dev, "Failed to alloc TX desc.\n"); in stmmac_enable_tx_queue()
6398 ret = __init_dma_tx_desc_rings(priv, queue); in stmmac_enable_tx_queue()
6400 __free_dma_tx_desc_resources(priv, queue); in stmmac_enable_tx_queue()
6401 netdev_err(priv->dev, "Failed to init TX desc.\n"); in stmmac_enable_tx_queue()
6405 stmmac_clear_tx_descriptors(priv, queue); in stmmac_enable_tx_queue()
6407 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, in stmmac_enable_tx_queue()
6408 tx_q->dma_tx_phy, tx_q->queue_index); in stmmac_enable_tx_queue()
6410 if (tx_q->tbs & STMMAC_TBS_AVAIL) in stmmac_enable_tx_queue()
6411 stmmac_enable_tbs(priv, priv->ioaddr, 1, tx_q->queue_index); in stmmac_enable_tx_queue()
6413 tx_q->tx_tail_addr = tx_q->dma_tx_phy; in stmmac_enable_tx_queue()
6414 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, in stmmac_enable_tx_queue()
6415 tx_q->tx_tail_addr, tx_q->queue_index); in stmmac_enable_tx_queue()
6417 stmmac_start_tx_dma(priv, queue); in stmmac_enable_tx_queue()
6419 spin_lock_irqsave(&ch->lock, flags); in stmmac_enable_tx_queue()
6420 stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 0, 1); in stmmac_enable_tx_queue()
6421 spin_unlock_irqrestore(&ch->lock, flags); in stmmac_enable_tx_queue()
6424 int stmmac_xsk_wakeup(struct net_device *dev, u32 queue, u32 flags) in stmmac_xsk_wakeup() argument
6431 if (test_bit(STMMAC_DOWN, &priv->state) || in stmmac_xsk_wakeup()
6432 !netif_carrier_ok(priv->dev)) in stmmac_xsk_wakeup()
6433 return -ENETDOWN; in stmmac_xsk_wakeup()
6436 return -ENXIO; in stmmac_xsk_wakeup()
6438 if (queue >= priv->plat->rx_queues_to_use || in stmmac_xsk_wakeup()
6439 queue >= priv->plat->tx_queues_to_use) in stmmac_xsk_wakeup()
6440 return -EINVAL; in stmmac_xsk_wakeup()
6442 rx_q = &priv->rx_queue[queue]; in stmmac_xsk_wakeup()
6443 tx_q = &priv->tx_queue[queue]; in stmmac_xsk_wakeup()
6444 ch = &priv->channel[queue]; in stmmac_xsk_wakeup()
6446 if (!rx_q->xsk_pool && !tx_q->xsk_pool) in stmmac_xsk_wakeup()
6447 return -ENXIO; in stmmac_xsk_wakeup()
6449 if (!napi_if_scheduled_mark_missed(&ch->rxtx_napi)) { in stmmac_xsk_wakeup()
6450 /* EQoS does not have per-DMA channel SW interrupt, in stmmac_xsk_wakeup()
6451 * so we schedule RX Napi straight-away. in stmmac_xsk_wakeup()
6453 if (likely(napi_schedule_prep(&ch->rxtx_napi))) in stmmac_xsk_wakeup()
6454 __napi_schedule(&ch->rxtx_napi); in stmmac_xsk_wakeup()
6485 if (!test_and_clear_bit(STMMAC_RESET_REQUESTED, &priv->state)) in stmmac_reset_subtask()
6487 if (test_bit(STMMAC_DOWN, &priv->state)) in stmmac_reset_subtask()
6490 netdev_err(priv->dev, "Reset adapter.\n"); in stmmac_reset_subtask()
6493 netif_trans_update(priv->dev); in stmmac_reset_subtask()
6494 while (test_and_set_bit(STMMAC_RESETING, &priv->state)) in stmmac_reset_subtask()
6497 set_bit(STMMAC_DOWN, &priv->state); in stmmac_reset_subtask()
6498 dev_close(priv->dev); in stmmac_reset_subtask()
6499 dev_open(priv->dev, NULL); in stmmac_reset_subtask()
6500 clear_bit(STMMAC_DOWN, &priv->state); in stmmac_reset_subtask()
6501 clear_bit(STMMAC_RESETING, &priv->state); in stmmac_reset_subtask()
6511 clear_bit(STMMAC_SERVICE_SCHED, &priv->state); in stmmac_service_task()
6515 * stmmac_hw_init - Init the MAC device
6526 /* dwmac-sun8i only work in chain mode */ in stmmac_hw_init()
6527 if (priv->plat->has_sun8i) in stmmac_hw_init()
6529 priv->chain_mode = chain_mode; in stmmac_hw_init()
6537 priv->hw_cap_support = stmmac_get_hw_features(priv); in stmmac_hw_init()
6538 if (priv->hw_cap_support) { in stmmac_hw_init()
6539 dev_info(priv->device, "DMA HW capability register supported\n"); in stmmac_hw_init()
6546 priv->plat->enh_desc = priv->dma_cap.enh_desc; in stmmac_hw_init()
6547 priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up && in stmmac_hw_init()
6548 !priv->plat->use_phy_wol; in stmmac_hw_init()
6549 priv->hw->pmt = priv->plat->pmt; in stmmac_hw_init()
6550 if (priv->dma_cap.hash_tb_sz) { in stmmac_hw_init()
6551 priv->hw->multicast_filter_bins = in stmmac_hw_init()
6552 (BIT(priv->dma_cap.hash_tb_sz) << 5); in stmmac_hw_init()
6553 priv->hw->mcast_bits_log2 = in stmmac_hw_init()
6554 ilog2(priv->hw->multicast_filter_bins); in stmmac_hw_init()
6558 if (priv->plat->force_thresh_dma_mode) in stmmac_hw_init()
6559 priv->plat->tx_coe = 0; in stmmac_hw_init()
6561 priv->plat->tx_coe = priv->dma_cap.tx_coe; in stmmac_hw_init()
6564 priv->plat->rx_coe = priv->dma_cap.rx_coe; in stmmac_hw_init()
6566 if (priv->dma_cap.rx_coe_type2) in stmmac_hw_init()
6567 priv->plat->rx_coe = STMMAC_RX_COE_TYPE2; in stmmac_hw_init()
6568 else if (priv->dma_cap.rx_coe_type1) in stmmac_hw_init()
6569 priv->plat->rx_coe = STMMAC_RX_COE_TYPE1; in stmmac_hw_init()
6572 dev_info(priv->device, "No HW DMA feature register supported\n"); in stmmac_hw_init()
6575 if (priv->plat->rx_coe) { in stmmac_hw_init()
6576 priv->hw->rx_csum = priv->plat->rx_coe; in stmmac_hw_init()
6577 dev_info(priv->device, "RX Checksum Offload Engine supported\n"); in stmmac_hw_init()
6578 if (priv->synopsys_id < DWMAC_CORE_4_00) in stmmac_hw_init()
6579 dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum); in stmmac_hw_init()
6581 if (priv->plat->tx_coe) in stmmac_hw_init()
6582 dev_info(priv->device, "TX Checksum insertion supported\n"); in stmmac_hw_init()
6584 if (priv->plat->pmt) { in stmmac_hw_init()
6585 dev_info(priv->device, "Wake-Up On Lan supported\n"); in stmmac_hw_init()
6586 device_set_wakeup_capable(priv->device, 1); in stmmac_hw_init()
6589 if (priv->dma_cap.tsoen) in stmmac_hw_init()
6590 dev_info(priv->device, "TSO supported\n"); in stmmac_hw_init()
6592 priv->hw->vlan_fail_q_en = priv->plat->vlan_fail_q_en; in stmmac_hw_init()
6593 priv->hw->vlan_fail_q = priv->plat->vlan_fail_q; in stmmac_hw_init()
6596 if (priv->hwif_quirks) { in stmmac_hw_init()
6597 ret = priv->hwif_quirks(priv); in stmmac_hw_init()
6602 /* Rx Watchdog is available in the COREs newer than the 3.40. in stmmac_hw_init()
6607 if (((priv->synopsys_id >= DWMAC_CORE_3_50) || in stmmac_hw_init()
6608 (priv->plat->has_xgmac)) && (!priv->plat->riwt_off)) { in stmmac_hw_init()
6609 priv->use_riwt = 1; in stmmac_hw_init()
6610 dev_info(priv->device, in stmmac_hw_init()
6611 "Enable RX Mitigation via HW Watchdog Timer\n"); in stmmac_hw_init()
6620 u32 queue, maxq; in stmmac_napi_add() local
6622 maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use); in stmmac_napi_add()
6624 for (queue = 0; queue < maxq; queue++) { in stmmac_napi_add()
6625 struct stmmac_channel *ch = &priv->channel[queue]; in stmmac_napi_add()
6627 ch->priv_data = priv; in stmmac_napi_add()
6628 ch->index = queue; in stmmac_napi_add()
6629 spin_lock_init(&ch->lock); in stmmac_napi_add()
6631 if (queue < priv->plat->rx_queues_to_use) { in stmmac_napi_add()
6632 netif_napi_add(dev, &ch->rx_napi, stmmac_napi_poll_rx, in stmmac_napi_add()
6635 if (queue < priv->plat->tx_queues_to_use) { in stmmac_napi_add()
6636 netif_tx_napi_add(dev, &ch->tx_napi, in stmmac_napi_add()
6640 if (queue < priv->plat->rx_queues_to_use && in stmmac_napi_add()
6641 queue < priv->plat->tx_queues_to_use) { in stmmac_napi_add()
6642 netif_napi_add(dev, &ch->rxtx_napi, in stmmac_napi_add()
6652 u32 queue, maxq; in stmmac_napi_del() local
6654 maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use); in stmmac_napi_del()
6656 for (queue = 0; queue < maxq; queue++) { in stmmac_napi_del()
6657 struct stmmac_channel *ch = &priv->channel[queue]; in stmmac_napi_del()
6659 if (queue < priv->plat->rx_queues_to_use) in stmmac_napi_del()
6660 netif_napi_del(&ch->rx_napi); in stmmac_napi_del()
6661 if (queue < priv->plat->tx_queues_to_use) in stmmac_napi_del()
6662 netif_napi_del(&ch->tx_napi); in stmmac_napi_del()
6663 if (queue < priv->plat->rx_queues_to_use && in stmmac_napi_del()
6664 queue < priv->plat->tx_queues_to_use) { in stmmac_napi_del()
6665 netif_napi_del(&ch->rxtx_napi); in stmmac_napi_del()
6680 priv->plat->rx_queues_to_use = rx_cnt; in stmmac_reinit_queues()
6681 priv->plat->tx_queues_to_use = tx_cnt; in stmmac_reinit_queues()
6699 priv->dma_rx_size = rx_size; in stmmac_reinit_ringparam()
6700 priv->dma_tx_size = tx_size; in stmmac_reinit_ringparam()
6713 struct stmmac_fpe_cfg *fpe_cfg = priv->plat->fpe_cfg; in stmmac_fpe_lp_task()
6714 enum stmmac_fpe_state *lo_state = &fpe_cfg->lo_fpe_state; in stmmac_fpe_lp_task()
6715 enum stmmac_fpe_state *lp_state = &fpe_cfg->lp_fpe_state; in stmmac_fpe_lp_task()
6716 bool *hs_enable = &fpe_cfg->hs_enable; in stmmac_fpe_lp_task()
6717 bool *enable = &fpe_cfg->enable; in stmmac_fpe_lp_task()
6720 while (retries-- > 0) { in stmmac_fpe_lp_task()
6727 stmmac_fpe_configure(priv, priv->ioaddr, in stmmac_fpe_lp_task()
6728 priv->plat->tx_queues_to_use, in stmmac_fpe_lp_task()
6729 priv->plat->rx_queues_to_use, in stmmac_fpe_lp_task()
6732 netdev_info(priv->dev, "configured FPE\n"); in stmmac_fpe_lp_task()
6736 netdev_info(priv->dev, "!!! BOTH FPE stations ON\n"); in stmmac_fpe_lp_task()
6743 netdev_info(priv->dev, SEND_VERIFY_MPAKCET_FMT, in stmmac_fpe_lp_task()
6745 stmmac_fpe_send_mpacket(priv, priv->ioaddr, in stmmac_fpe_lp_task()
6752 clear_bit(__FPE_TASK_SCHED, &priv->fpe_task_state); in stmmac_fpe_lp_task()
6757 if (priv->plat->fpe_cfg->hs_enable != enable) { in stmmac_fpe_handshake()
6759 stmmac_fpe_send_mpacket(priv, priv->ioaddr, in stmmac_fpe_handshake()
6762 priv->plat->fpe_cfg->lo_fpe_state = FPE_STATE_OFF; in stmmac_fpe_handshake()
6763 priv->plat->fpe_cfg->lp_fpe_state = FPE_STATE_OFF; in stmmac_fpe_handshake()
6766 priv->plat->fpe_cfg->hs_enable = enable; in stmmac_fpe_handshake()
6792 return -ENOMEM; in stmmac_dvr_probe()
6797 priv->device = device; in stmmac_dvr_probe()
6798 priv->dev = ndev; in stmmac_dvr_probe()
6801 priv->pause = pause; in stmmac_dvr_probe()
6802 priv->plat = plat_dat; in stmmac_dvr_probe()
6803 priv->ioaddr = res->addr; in stmmac_dvr_probe()
6804 priv->dev->base_addr = (unsigned long)res->addr; in stmmac_dvr_probe()
6805 priv->plat->dma_cfg->multi_msi_en = priv->plat->multi_msi_en; in stmmac_dvr_probe()
6807 priv->dev->irq = res->irq; in stmmac_dvr_probe()
6808 priv->wol_irq = res->wol_irq; in stmmac_dvr_probe()
6809 priv->lpi_irq = res->lpi_irq; in stmmac_dvr_probe()
6810 priv->sfty_ce_irq = res->sfty_ce_irq; in stmmac_dvr_probe()
6811 priv->sfty_ue_irq = res->sfty_ue_irq; in stmmac_dvr_probe()
6813 priv->rx_irq[i] = res->rx_irq[i]; in stmmac_dvr_probe()
6815 priv->tx_irq[i] = res->tx_irq[i]; in stmmac_dvr_probe()
6817 if (!is_zero_ether_addr(res->mac)) in stmmac_dvr_probe()
6818 memcpy(priv->dev->dev_addr, res->mac, ETH_ALEN); in stmmac_dvr_probe()
6820 dev_set_drvdata(device, priv->dev); in stmmac_dvr_probe()
6825 priv->af_xdp_zc_qps = bitmap_zalloc(MTL_MAX_TX_QUEUES, GFP_KERNEL); in stmmac_dvr_probe()
6826 if (!priv->af_xdp_zc_qps) in stmmac_dvr_probe()
6827 return -ENOMEM; in stmmac_dvr_probe()
6830 priv->wq = create_singlethread_workqueue("stmmac_wq"); in stmmac_dvr_probe()
6831 if (!priv->wq) { in stmmac_dvr_probe()
6832 dev_err(priv->device, "failed to create workqueue\n"); in stmmac_dvr_probe()
6833 return -ENOMEM; in stmmac_dvr_probe()
6836 INIT_WORK(&priv->service_task, stmmac_service_task); in stmmac_dvr_probe()
6839 INIT_WORK(&priv->fpe_task, stmmac_fpe_lp_task); in stmmac_dvr_probe()
6845 priv->plat->phy_addr = phyaddr; in stmmac_dvr_probe()
6847 if (priv->plat->stmmac_rst) { in stmmac_dvr_probe()
6848 ret = reset_control_assert(priv->plat->stmmac_rst); in stmmac_dvr_probe()
6849 reset_control_deassert(priv->plat->stmmac_rst); in stmmac_dvr_probe()
6853 if (ret == -ENOTSUPP) in stmmac_dvr_probe()
6854 reset_control_reset(priv->plat->stmmac_rst); in stmmac_dvr_probe()
6857 ret = reset_control_deassert(priv->plat->stmmac_ahb_rst); in stmmac_dvr_probe()
6858 if (ret == -ENOTSUPP) in stmmac_dvr_probe()
6859 dev_err(priv->device, "unable to bring out of ahb reset: %pe\n", in stmmac_dvr_probe()
6869 if (priv->synopsys_id < DWMAC_CORE_5_20) in stmmac_dvr_probe()
6870 priv->plat->dma_cfg->dche = false; in stmmac_dvr_probe()
6874 ndev->netdev_ops = &stmmac_netdev_ops; in stmmac_dvr_probe()
6876 ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | in stmmac_dvr_probe()
6881 ndev->hw_features |= NETIF_F_HW_TC; in stmmac_dvr_probe()
6884 if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) { in stmmac_dvr_probe()
6885 ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6; in stmmac_dvr_probe()
6886 if (priv->plat->has_gmac4) in stmmac_dvr_probe()
6887 ndev->hw_features |= NETIF_F_GSO_UDP_L4; in stmmac_dvr_probe()
6888 priv->tso = true; in stmmac_dvr_probe()
6889 dev_info(priv->device, "TSO feature enabled\n"); in stmmac_dvr_probe()
6892 if (priv->dma_cap.sphen) { in stmmac_dvr_probe()
6893 ndev->hw_features |= NETIF_F_GRO; in stmmac_dvr_probe()
6894 priv->sph_cap = true; in stmmac_dvr_probe()
6895 priv->sph = priv->sph_cap; in stmmac_dvr_probe()
6896 dev_info(priv->device, "SPH feature enabled\n"); in stmmac_dvr_probe()
6904 if (priv->plat->addr64) in stmmac_dvr_probe()
6905 priv->dma_cap.addr64 = priv->plat->addr64; in stmmac_dvr_probe()
6907 if (priv->dma_cap.addr64) { in stmmac_dvr_probe()
6909 DMA_BIT_MASK(priv->dma_cap.addr64)); in stmmac_dvr_probe()
6911 dev_info(priv->device, "Using %d bits DMA width\n", in stmmac_dvr_probe()
6912 priv->dma_cap.addr64); in stmmac_dvr_probe()
6919 priv->plat->dma_cfg->eame = true; in stmmac_dvr_probe()
6923 dev_err(priv->device, "Failed to set DMA Mask\n"); in stmmac_dvr_probe()
6927 priv->dma_cap.addr64 = 32; in stmmac_dvr_probe()
6931 ndev->features |= ndev->hw_features | NETIF_F_HIGHDMA; in stmmac_dvr_probe()
6932 ndev->watchdog_timeo = msecs_to_jiffies(watchdog); in stmmac_dvr_probe()
6935 ndev->features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX; in stmmac_dvr_probe()
6936 if (priv->dma_cap.vlhash) { in stmmac_dvr_probe()
6937 ndev->features |= NETIF_F_HW_VLAN_CTAG_FILTER; in stmmac_dvr_probe()
6938 ndev->features |= NETIF_F_HW_VLAN_STAG_FILTER; in stmmac_dvr_probe()
6940 if (priv->dma_cap.vlins) { in stmmac_dvr_probe()
6941 ndev->features |= NETIF_F_HW_VLAN_CTAG_TX; in stmmac_dvr_probe()
6942 if (priv->dma_cap.dvlan) in stmmac_dvr_probe()
6943 ndev->features |= NETIF_F_HW_VLAN_STAG_TX; in stmmac_dvr_probe()
6946 priv->msg_enable = netif_msg_init(debug, default_msg_level); in stmmac_dvr_probe()
6949 rxq = priv->plat->rx_queues_to_use; in stmmac_dvr_probe()
6950 netdev_rss_key_fill(priv->rss.key, sizeof(priv->rss.key)); in stmmac_dvr_probe()
6951 for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++) in stmmac_dvr_probe()
6952 priv->rss.table[i] = ethtool_rxfh_indir_default(i, rxq); in stmmac_dvr_probe()
6954 if (priv->dma_cap.rssen && priv->plat->rss_en) in stmmac_dvr_probe()
6955 ndev->features |= NETIF_F_RXHASH; in stmmac_dvr_probe()
6957 /* MTU range: 46 - hw-specific max */ in stmmac_dvr_probe()
6958 ndev->min_mtu = ETH_ZLEN - ETH_HLEN; in stmmac_dvr_probe()
6959 if (priv->plat->has_xgmac) in stmmac_dvr_probe()
6960 ndev->max_mtu = XGMAC_JUMBO_LEN; in stmmac_dvr_probe()
6961 else if ((priv->plat->enh_desc) || (priv->synopsys_id >= DWMAC_CORE_4_00)) in stmmac_dvr_probe()
6962 ndev->max_mtu = JUMBO_LEN; in stmmac_dvr_probe()
6964 ndev->max_mtu = SKB_MAX_HEAD(NET_SKB_PAD + NET_IP_ALIGN); in stmmac_dvr_probe()
6965 /* Will not overwrite ndev->max_mtu if plat->maxmtu > ndev->max_mtu in stmmac_dvr_probe()
6966 * as well as plat->maxmtu < ndev->min_mtu which is a invalid range. in stmmac_dvr_probe()
6968 if ((priv->plat->maxmtu < ndev->max_mtu) && in stmmac_dvr_probe()
6969 (priv->plat->maxmtu >= ndev->min_mtu)) in stmmac_dvr_probe()
6970 ndev->max_mtu = priv->plat->maxmtu; in stmmac_dvr_probe()
6971 else if (priv->plat->maxmtu < ndev->min_mtu) in stmmac_dvr_probe()
6972 dev_warn(priv->device, in stmmac_dvr_probe()
6974 __func__, priv->plat->maxmtu); in stmmac_dvr_probe()
6977 priv->flow_ctrl = FLOW_AUTO; /* RX/TX pause on */ in stmmac_dvr_probe()
6982 mutex_init(&priv->lock); in stmmac_dvr_probe()
6986 * changed at run-time and it is fixed. Viceversa the driver'll try to in stmmac_dvr_probe()
6990 if (priv->plat->clk_csr >= 0) in stmmac_dvr_probe()
6991 priv->clk_csr = priv->plat->clk_csr; in stmmac_dvr_probe()
7001 if (priv->hw->pcs != STMMAC_PCS_TBI && in stmmac_dvr_probe()
7002 priv->hw->pcs != STMMAC_PCS_RTBI) { in stmmac_dvr_probe()
7006 dev_err(priv->device, in stmmac_dvr_probe()
7008 __func__, priv->plat->bus_id); in stmmac_dvr_probe()
7013 if (priv->plat->speed_mode_2500) in stmmac_dvr_probe()
7014 priv->plat->speed_mode_2500(ndev, priv->plat->bsp_priv); in stmmac_dvr_probe()
7016 if (priv->plat->mdio_bus_data && priv->plat->mdio_bus_data->has_xpcs) { in stmmac_dvr_probe()
7017 ret = stmmac_xpcs_setup(priv->mii); in stmmac_dvr_probe()
7030 dev_err(priv->device, "%s: ERROR %i registering the device\n", in stmmac_dvr_probe()
7035 if (priv->plat->serdes_powerup) { in stmmac_dvr_probe()
7036 ret = priv->plat->serdes_powerup(ndev, in stmmac_dvr_probe()
7037 priv->plat->bsp_priv); in stmmac_dvr_probe()
7057 phylink_destroy(priv->phylink); in stmmac_dvr_probe()
7060 if (priv->hw->pcs != STMMAC_PCS_TBI && in stmmac_dvr_probe()
7061 priv->hw->pcs != STMMAC_PCS_RTBI) in stmmac_dvr_probe()
7066 destroy_workqueue(priv->wq); in stmmac_dvr_probe()
7067 bitmap_free(priv->af_xdp_zc_qps); in stmmac_dvr_probe()
7076 * Description: this function resets the TX/RX processes, disables the MAC RX/TX
7084 netdev_info(priv->dev, "%s: removing driver", __func__); in stmmac_dvr_remove()
7087 stmmac_mac_set(priv, priv->ioaddr, false); in stmmac_dvr_remove()
7094 if (priv->plat->serdes_powerdown) in stmmac_dvr_remove()
7095 priv->plat->serdes_powerdown(ndev, priv->plat->bsp_priv); in stmmac_dvr_remove()
7100 phylink_destroy(priv->phylink); in stmmac_dvr_remove()
7101 if (priv->plat->stmmac_rst) in stmmac_dvr_remove()
7102 reset_control_assert(priv->plat->stmmac_rst); in stmmac_dvr_remove()
7103 reset_control_assert(priv->plat->stmmac_ahb_rst); in stmmac_dvr_remove()
7106 if (priv->hw->pcs != STMMAC_PCS_TBI && in stmmac_dvr_remove()
7107 priv->hw->pcs != STMMAC_PCS_RTBI) in stmmac_dvr_remove()
7109 destroy_workqueue(priv->wq); in stmmac_dvr_remove()
7110 mutex_destroy(&priv->lock); in stmmac_dvr_remove()
7111 bitmap_free(priv->af_xdp_zc_qps); in stmmac_dvr_remove()
7118 * stmmac_suspend - suspend callback
7121 * by the platform driver to stop the network queue, release the resources,
7133 mutex_lock(&priv->lock); in stmmac_suspend()
7139 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) in stmmac_suspend()
7140 hrtimer_cancel(&priv->tx_queue[chan].txtimer); in stmmac_suspend()
7142 if (priv->eee_enabled) { in stmmac_suspend()
7143 priv->tx_path_in_lpi_mode = false; in stmmac_suspend()
7144 del_timer_sync(&priv->eee_ctrl_timer); in stmmac_suspend()
7147 /* Stop TX/RX DMA */ in stmmac_suspend()
7150 if (priv->plat->serdes_powerdown) in stmmac_suspend()
7151 priv->plat->serdes_powerdown(ndev, priv->plat->bsp_priv); in stmmac_suspend()
7154 if (device_may_wakeup(priv->device) && priv->plat->pmt) { in stmmac_suspend()
7155 stmmac_pmt(priv, priv->hw, priv->wolopts); in stmmac_suspend()
7156 priv->irq_wake = 1; in stmmac_suspend()
7158 stmmac_mac_set(priv, priv->ioaddr, false); in stmmac_suspend()
7159 pinctrl_pm_select_sleep_state(priv->device); in stmmac_suspend()
7162 mutex_unlock(&priv->lock); in stmmac_suspend()
7165 if (device_may_wakeup(priv->device) && priv->plat->pmt) { in stmmac_suspend()
7166 phylink_suspend(priv->phylink, true); in stmmac_suspend()
7168 if (device_may_wakeup(priv->device)) in stmmac_suspend()
7169 phylink_speed_down(priv->phylink, false); in stmmac_suspend()
7170 phylink_suspend(priv->phylink, false); in stmmac_suspend()
7174 if (priv->dma_cap.fpesel) { in stmmac_suspend()
7176 stmmac_fpe_configure(priv, priv->ioaddr, in stmmac_suspend()
7177 priv->plat->tx_queues_to_use, in stmmac_suspend()
7178 priv->plat->rx_queues_to_use, false); in stmmac_suspend()
7184 priv->speed = SPEED_UNKNOWN; in stmmac_suspend()
7190 * stmmac_reset_queues_param - reset queue parameters
7195 u32 rx_cnt = priv->plat->rx_queues_to_use; in stmmac_reset_queues_param()
7196 u32 tx_cnt = priv->plat->tx_queues_to_use; in stmmac_reset_queues_param()
7197 u32 queue; in stmmac_reset_queues_param() local
7199 for (queue = 0; queue < rx_cnt; queue++) { in stmmac_reset_queues_param()
7200 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue]; in stmmac_reset_queues_param()
7202 rx_q->cur_rx = 0; in stmmac_reset_queues_param()
7203 rx_q->dirty_rx = 0; in stmmac_reset_queues_param()
7206 for (queue = 0; queue < tx_cnt; queue++) { in stmmac_reset_queues_param()
7207 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue]; in stmmac_reset_queues_param()
7209 tx_q->cur_tx = 0; in stmmac_reset_queues_param()
7210 tx_q->dirty_tx = 0; in stmmac_reset_queues_param()
7211 tx_q->mss = 0; in stmmac_reset_queues_param()
7213 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue)); in stmmac_reset_queues_param()
7218 * stmmac_resume - resume callback
7233 * automatically as soon as a magic packet or a Wake-up frame in stmmac_resume()
7238 if (device_may_wakeup(priv->device) && priv->plat->pmt) { in stmmac_resume()
7239 mutex_lock(&priv->lock); in stmmac_resume()
7240 stmmac_pmt(priv, priv->hw, 0); in stmmac_resume()
7241 mutex_unlock(&priv->lock); in stmmac_resume()
7242 priv->irq_wake = 0; in stmmac_resume()
7244 pinctrl_pm_select_default_state(priv->device); in stmmac_resume()
7246 if (priv->mii) in stmmac_resume()
7247 stmmac_mdio_reset(priv->mii); in stmmac_resume()
7250 if (priv->plat->serdes_powerup) { in stmmac_resume()
7251 ret = priv->plat->serdes_powerup(ndev, in stmmac_resume()
7252 priv->plat->bsp_priv); in stmmac_resume()
7259 if (device_may_wakeup(priv->device) && priv->plat->pmt) { in stmmac_resume()
7260 phylink_resume(priv->phylink); in stmmac_resume()
7262 phylink_resume(priv->phylink); in stmmac_resume()
7263 if (device_may_wakeup(priv->device)) in stmmac_resume()
7264 phylink_speed_up(priv->phylink); in stmmac_resume()
7269 mutex_lock(&priv->lock); in stmmac_resume()
7280 stmmac_restore_hw_vlan_rx_fltr(priv, ndev, priv->hw); in stmmac_resume()
7284 mutex_unlock(&priv->lock); in stmmac_resume()
7299 return -EINVAL; in stmmac_cmdline_opt()
7334 return -EINVAL; in stmmac_cmdline_opt()