133 static void stmmac_reset_rx_queue(struct stmmac_priv *priv, u32 queue);
134 static void stmmac_reset_tx_queue(struct stmmac_priv *priv, u32 queue);
135 static void stmmac_reset_queues_param(struct stmmac_priv *priv);
136 static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue);
137 static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue);
138 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode,
149 int stmmac_bus_clks_config(struct stmmac_priv *priv, bool enabled)  in stmmac_bus_clks_config()
202 static void __stmmac_disable_all_queues(struct stmmac_priv *priv)  in __stmmac_disable_all_queues()
229 static void stmmac_disable_all_queues(struct stmmac_priv *priv)  in stmmac_disable_all_queues()
251 static void stmmac_enable_all_queues(struct stmmac_priv *priv)  in stmmac_enable_all_queues()
274 static void stmmac_service_event_schedule(struct stmmac_priv *priv)  in stmmac_service_event_schedule()
281 static void stmmac_global_err(struct stmmac_priv *priv)  in stmmac_global_err()
300 static void stmmac_clk_csr_set(struct stmmac_priv *priv)  in stmmac_clk_csr_set()
361 static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue)  in stmmac_tx_avail()
379 static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue)  in stmmac_rx_dirty()
392 static void stmmac_lpi_entry_timer_config(struct stmmac_priv *priv, bool en)  in stmmac_lpi_entry_timer_config()
408 static int stmmac_enable_eee_mode(struct stmmac_priv *priv)  in stmmac_enable_eee_mode()
434 void stmmac_disable_eee_mode(struct stmmac_priv *priv)  in stmmac_disable_eee_mode()
455 	struct stmmac_priv *priv = from_timer(priv, t, eee_ctrl_timer);  in stmmac_eee_ctrl_timer()
469 bool stmmac_eee_init(struct stmmac_priv *priv)  in stmmac_eee_init()
535 static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv,  in stmmac_get_tx_hwtstamp()
578 static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p,  in stmmac_get_rx_hwtstamp()
619 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_hwtstamp_set()
814 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_hwtstamp_get()
834 int stmmac_init_tstamp_counter(struct stmmac_priv *priv, u32 systime_flags)  in stmmac_init_tstamp_counter()
882 static int stmmac_init_ptp(struct stmmac_priv *priv)  in stmmac_init_ptp()
915 static void stmmac_release_ptp(struct stmmac_priv *priv)  in stmmac_release_ptp()
927 static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex)  in stmmac_mac_flow_ctrl()
938 	struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));  in stmmac_mac_select_pcs()
952 static void stmmac_fpe_link_state_handle(struct stmmac_priv *priv, bool is_up)  in stmmac_fpe_link_state_handle()
970 	struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));  in stmmac_mac_link_down()
988 	struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));  in stmmac_mac_link_up()
1105 static void stmmac_check_pcs_mode(struct stmmac_priv *priv)  in stmmac_check_pcs_mode()
1133 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_init_phy()
1170 static int stmmac_phy_setup(struct stmmac_priv *priv)  in stmmac_phy_setup()
1236 static void stmmac_display_rx_rings(struct stmmac_priv *priv,  in stmmac_display_rx_rings()
1264 static void stmmac_display_tx_rings(struct stmmac_priv *priv,  in stmmac_display_tx_rings()
1294 static void stmmac_display_rings(struct stmmac_priv *priv,  in stmmac_display_rings()
1330 static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv,  in stmmac_clear_rx_descriptors()
1359 static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv,  in stmmac_clear_tx_descriptors()
1389 static void stmmac_clear_descriptors(struct stmmac_priv *priv,  in stmmac_clear_descriptors()
1416 static int stmmac_init_rx_buffers(struct stmmac_priv *priv,  in stmmac_init_rx_buffers()
1462 static void stmmac_free_rx_buffer(struct stmmac_priv *priv,  in stmmac_free_rx_buffer()
1484 static void stmmac_free_tx_buffer(struct stmmac_priv *priv,  in stmmac_free_tx_buffer()
1530 static void dma_free_rx_skbufs(struct stmmac_priv *priv,  in dma_free_rx_skbufs()
1541 static int stmmac_alloc_rx_buffers(struct stmmac_priv *priv,  in stmmac_alloc_rx_buffers()
1574 static void dma_free_rx_xskbufs(struct stmmac_priv *priv,  in dma_free_rx_xskbufs()
1592 static int stmmac_alloc_rx_buffers_zc(struct stmmac_priv *priv,  in stmmac_alloc_rx_buffers_zc()
1623 static struct xsk_buff_pool *stmmac_get_xsk_pool(struct stmmac_priv *priv, u32 queue)  in stmmac_get_xsk_pool()
1641 static int __init_dma_rx_desc_rings(struct stmmac_priv *priv,  in __init_dma_rx_desc_rings()
1705 	struct stmmac_priv *priv = netdev_priv(dev);  in init_dma_rx_desc_rings()
1749 static int __init_dma_tx_desc_rings(struct stmmac_priv *priv,  in __init_dma_tx_desc_rings()
1799 	struct stmmac_priv *priv = netdev_priv(dev);  in init_dma_tx_desc_rings()
1824 	struct stmmac_priv *priv = netdev_priv(dev);  in init_dma_desc_rings()
1847 static void dma_free_tx_skbufs(struct stmmac_priv *priv,  in dma_free_tx_skbufs()
1870 static void stmmac_free_tx_skbufs(struct stmmac_priv *priv)  in stmmac_free_tx_skbufs()
1885 static void __free_dma_rx_desc_resources(struct stmmac_priv *priv,  in __free_dma_rx_desc_resources()
1918 static void free_dma_rx_desc_resources(struct stmmac_priv *priv,  in free_dma_rx_desc_resources()
1935 static void __free_dma_tx_desc_resources(struct stmmac_priv *priv,  in __free_dma_tx_desc_resources()
1965 static void free_dma_tx_desc_resources(struct stmmac_priv *priv,  in free_dma_tx_desc_resources()
1986 static int __alloc_dma_rx_desc_resources(struct stmmac_priv *priv,  in __alloc_dma_rx_desc_resources()
2060 static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv,  in alloc_dma_rx_desc_resources()
2092 static int __alloc_dma_tx_desc_resources(struct stmmac_priv *priv,  in __alloc_dma_tx_desc_resources()
2139 static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv,  in alloc_dma_tx_desc_resources()
2169 static int alloc_dma_desc_resources(struct stmmac_priv *priv,  in alloc_dma_desc_resources()
2188 static void free_dma_desc_resources(struct stmmac_priv *priv,  in free_dma_desc_resources()
2205 static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv)  in stmmac_mac_enable_rx_queues()
2224 static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan)  in stmmac_start_rx_dma()
2237 static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan)  in stmmac_start_tx_dma()
2250 static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan)  in stmmac_stop_rx_dma()
2263 static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan)  in stmmac_stop_tx_dma()
2269 static void stmmac_enable_all_dma_irq(struct stmmac_priv *priv)  in stmmac_enable_all_dma_irq()
2292 static void stmmac_start_all_dma(struct stmmac_priv *priv)  in stmmac_start_all_dma()
2311 static void stmmac_stop_all_dma(struct stmmac_priv *priv)  in stmmac_stop_all_dma()
2330 static void stmmac_dma_operation_mode(struct stmmac_priv *priv)  in stmmac_dma_operation_mode()
2399 static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget)  in stmmac_xdp_xmit_zc()
2494 static void stmmac_bump_dma_threshold(struct stmmac_priv *priv, u32 chan)  in stmmac_bump_dma_threshold()
2516 static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue)  in stmmac_tx_clean()
2696 static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan)  in stmmac_tx_err()
2724 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode,  in stmmac_set_dma_operation_mode()
2747 static bool stmmac_safety_feat_interrupt(struct stmmac_priv *priv)  in stmmac_safety_feat_interrupt()
2761 static int stmmac_napi_check(struct stmmac_priv *priv, u32 chan, u32 dir)  in stmmac_napi_check()
2803 static void stmmac_dma_interrupt(struct stmmac_priv *priv)  in stmmac_dma_interrupt()
2835 static void stmmac_mmc_setup(struct stmmac_priv *priv)  in stmmac_mmc_setup()
2858 static int stmmac_get_hw_features(struct stmmac_priv *priv)  in stmmac_get_hw_features()
2870 static void stmmac_check_ether_addr(struct stmmac_priv *priv)  in stmmac_check_ether_addr()
2893 static int stmmac_init_dma_engine(struct stmmac_priv *priv)  in stmmac_init_dma_engine()
2959 static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue)  in stmmac_tx_timer_arm()
2977 	struct stmmac_priv *priv = tx_q->priv_data;  in stmmac_tx_timer()
3004 static void stmmac_init_coalesce(struct stmmac_priv *priv)  in stmmac_init_coalesce()
3024 static void stmmac_set_rings_length(struct stmmac_priv *priv)  in stmmac_set_rings_length()
3046 static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv)  in stmmac_set_tx_queue_weight()
3063 static void stmmac_configure_cbs(struct stmmac_priv *priv)  in stmmac_configure_cbs()
3089 static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv)  in stmmac_rx_queue_dma_chan_map()
3106 static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv)  in stmmac_mac_config_rx_queues_prio()
3126 static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv)  in stmmac_mac_config_tx_queues_prio()
3146 static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv)  in stmmac_mac_config_rx_queues_routing()
3162 static void stmmac_mac_config_rss(struct stmmac_priv *priv)  in stmmac_mac_config_rss()
3183 static void stmmac_mtl_configuration(struct stmmac_priv *priv)  in stmmac_mtl_configuration()
3228 static void stmmac_safety_feat_configuration(struct stmmac_priv *priv)  in stmmac_safety_feat_configuration()
3239 static int stmmac_fpe_start_wq(struct stmmac_priv *priv)  in stmmac_fpe_start_wq()
3275 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_hw_setup()
3420 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_hw_teardown()
3428 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_free_irq()
3479 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_request_irq_multi_msi()
3631 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_request_irq_single()
3682 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_request_irq()
3704 stmmac_setup_dma_desc(struct stmmac_priv *priv, unsigned int mtu)  in stmmac_setup_dma_desc()
3780 	struct stmmac_priv *priv = netdev_priv(dev);  in __stmmac_open()
3861 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_open()
3874 static void stmmac_fpe_stop_wq(struct stmmac_priv *priv)  in stmmac_fpe_stop_wq()
3892 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_release()
3941 static bool stmmac_vlan_insert(struct stmmac_priv *priv, struct sk_buff *skb,  in stmmac_vlan_insert()
3983 static void stmmac_tso_allocator(struct stmmac_priv *priv, dma_addr_t des,  in stmmac_tso_allocator()
4023 static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue)  in stmmac_flush_tx_descriptors()
4075 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_tso_xmit()
4314 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_xmit()
4574 static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue)  in stmmac_rx_refill()
4638 static unsigned int stmmac_rx_buf1_len(struct stmmac_priv *priv,  in stmmac_rx_buf1_len()
4666 static unsigned int stmmac_rx_buf2_len(struct stmmac_priv *priv,  in stmmac_rx_buf2_len()
4687 static int stmmac_xdp_xmit_xdpf(struct stmmac_priv *priv, int queue,  in stmmac_xdp_xmit_xdpf()
4759 static int stmmac_xdp_get_tx_queue(struct stmmac_priv *priv,  in stmmac_xdp_get_tx_queue()
4773 static int stmmac_xdp_xmit_back(struct stmmac_priv *priv,  in stmmac_xdp_xmit_back()
4801 static int __stmmac_xdp_run_prog(struct stmmac_priv *priv,  in __stmmac_xdp_run_prog()
4836 static struct sk_buff *stmmac_xdp_run_prog(struct stmmac_priv *priv,  in stmmac_xdp_run_prog()
4853 static void stmmac_finalize_xdp_rx(struct stmmac_priv *priv,  in stmmac_finalize_xdp_rx()
4889 static void stmmac_dispatch_skb_zc(struct stmmac_priv *priv, u32 queue,  in stmmac_dispatch_skb_zc()
4925 static bool stmmac_rx_refill_zc(struct stmmac_priv *priv, u32 queue, u32 budget)  in stmmac_rx_refill_zc()
4983 static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue)  in stmmac_rx_zc()
5165 static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)  in stmmac_rx()
5434 	struct stmmac_priv *priv = ch->priv_data;  in stmmac_napi_poll_rx()
5456 	struct stmmac_priv *priv = ch->priv_data;  in stmmac_napi_poll_tx()
5480 	struct stmmac_priv *priv = ch->priv_data;  in stmmac_napi_poll_rxtx()
5525 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_tx_timeout()
5541 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_set_rx_mode()
5559 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_change_mtu()
5612 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_fix_features()
5642 	struct stmmac_priv *priv = netdev_priv(netdev);  in stmmac_set_features()
5665 static void stmmac_fpe_event_status(struct stmmac_priv *priv, int status)  in stmmac_fpe_event_status()
5707 static void stmmac_common_interrupt(struct stmmac_priv *priv)  in stmmac_common_interrupt()
5775 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_interrupt()
5797 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_mac_interrupt()
5817 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_safety_interrupt()
5839 	struct stmmac_priv *priv;  in stmmac_msi_intr_tx()
5843 	priv = container_of(dma_conf, struct stmmac_priv, dma_conf);  in stmmac_msi_intr_tx()
5871 	struct stmmac_priv *priv;  in stmmac_msi_intr_rx()
5874 	priv = container_of(dma_conf, struct stmmac_priv, dma_conf);  in stmmac_msi_intr_rx()
5896 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_poll_controller()
5928 	struct stmmac_priv *priv = netdev_priv (dev);  in stmmac_ioctl()
5956 	struct stmmac_priv *priv = cb_priv;  in stmmac_setup_tc_block_cb()
5984 	struct stmmac_priv *priv = netdev_priv(ndev);  in stmmac_setup_tc()
6023 	struct stmmac_priv *priv = netdev_priv(ndev);  in stmmac_set_mac_address()
6078 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_rings_status_show()
6125 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_dma_cap_show()
6228 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_device_event()
6252 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_init_fs()
6272 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_exit_fs()
6302 static int stmmac_vlan_update(struct stmmac_priv *priv, bool is_double)  in stmmac_vlan_update()
6329 	struct stmmac_priv *priv = netdev_priv(ndev);  in stmmac_vlan_rx_add_vid()
6354 	struct stmmac_priv *priv = netdev_priv(ndev);  in stmmac_vlan_rx_kill_vid()
6383 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_bpf()
6399 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_xdp_xmit()
6438 void stmmac_disable_rx_queue(struct stmmac_priv *priv, u32 queue)  in stmmac_disable_rx_queue()
6451 void stmmac_enable_rx_queue(struct stmmac_priv *priv, u32 queue)  in stmmac_enable_rx_queue()
6501 void stmmac_disable_tx_queue(struct stmmac_priv *priv, u32 queue)  in stmmac_disable_tx_queue()
6514 void stmmac_enable_tx_queue(struct stmmac_priv *priv, u32 queue)  in stmmac_enable_tx_queue()
6556 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_xdp_release()
6589 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_xdp_open()
6696 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_xsk_wakeup()
6753 static void stmmac_reset_subtask(struct stmmac_priv *priv)  in stmmac_reset_subtask()
6777 	struct stmmac_priv *priv = container_of(work, struct stmmac_priv,  in stmmac_service_task()
6792 static int stmmac_hw_init(struct stmmac_priv *priv)  in stmmac_hw_init()
6889 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_napi_add()
6918 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_napi_del()
6939 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_reinit_queues()
6960 	struct stmmac_priv *priv = netdev_priv(dev);  in stmmac_reinit_ringparam()
6978 	struct stmmac_priv *priv = container_of(work, struct stmmac_priv,  in stmmac_fpe_lp_task()
7022 void stmmac_fpe_handshake(struct stmmac_priv *priv, bool enable)  in stmmac_fpe_handshake()
7052 	struct stmmac_priv *priv;  in stmmac_dvr_probe()
7056 	ndev = devm_alloc_etherdev_mqs(device, sizeof(struct stmmac_priv),  in stmmac_dvr_probe()
7343 	struct stmmac_priv *priv = netdev_priv(ndev);  in stmmac_dvr_remove()
7391 	struct stmmac_priv *priv = netdev_priv(ndev);  in stmmac_suspend()
7453 static void stmmac_reset_rx_queue(struct stmmac_priv *priv, u32 queue)  in stmmac_reset_rx_queue()
7461 static void stmmac_reset_tx_queue(struct stmmac_priv *priv, u32 queue)  in stmmac_reset_tx_queue()
7476 static void stmmac_reset_queues_param(struct stmmac_priv *priv)  in stmmac_reset_queues_param()
7498 	struct stmmac_priv *priv = netdev_priv(ndev);  in stmmac_resume()