203 static int dpaa_netdev_init(struct net_device *net_dev,  in dpaa_netdev_init()  argument
207 	struct dpaa_priv *priv = netdev_priv(net_dev);  in dpaa_netdev_init()
208 	struct device *dev = net_dev->dev.parent;  in dpaa_netdev_init()
219 		percpu_priv->net_dev = net_dev;  in dpaa_netdev_init()
222 	net_dev->netdev_ops = dpaa_ops;  in dpaa_netdev_init()
225 	net_dev->mem_start = (unsigned long)priv->mac_dev->res->start;  in dpaa_netdev_init()
226 	net_dev->mem_end = (unsigned long)priv->mac_dev->res->end;  in dpaa_netdev_init()
228 	net_dev->min_mtu = ETH_MIN_MTU;  in dpaa_netdev_init()
229 	net_dev->max_mtu = dpaa_get_max_mtu();  in dpaa_netdev_init()
231 	net_dev->hw_features |= (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |  in dpaa_netdev_init()
234 	net_dev->hw_features |= NETIF_F_SG | NETIF_F_HIGHDMA;  in dpaa_netdev_init()
238 	net_dev->features |= NETIF_F_GSO;  in dpaa_netdev_init()
239 	net_dev->features |= NETIF_F_RXCSUM;  in dpaa_netdev_init()
241 	net_dev->priv_flags |= IFF_LIVE_ADDR_CHANGE;  in dpaa_netdev_init()
243 	net_dev->priv_flags &= ~IFF_TX_SKB_SHARING;  in dpaa_netdev_init()
245 	net_dev->features |= net_dev->hw_features;  in dpaa_netdev_init()
246 	net_dev->vlan_features = net_dev->features;  in dpaa_netdev_init()
248 	net_dev->xdp_features = NETDEV_XDP_ACT_BASIC |  in dpaa_netdev_init()
253 		memcpy(net_dev->perm_addr, mac_addr, net_dev->addr_len);  in dpaa_netdev_init()
254 		eth_hw_addr_set(net_dev, mac_addr);  in dpaa_netdev_init()
256 		eth_hw_addr_random(net_dev);  in dpaa_netdev_init()
258 			(const enet_addr_t *)net_dev->dev_addr);  in dpaa_netdev_init()
264 			 net_dev->dev_addr);  in dpaa_netdev_init()
267 	net_dev->ethtool_ops = &dpaa_ethtool_ops;  in dpaa_netdev_init()
269 	net_dev->needed_headroom = priv->tx_headroom;  in dpaa_netdev_init()
270 	net_dev->watchdog_timeo = msecs_to_jiffies(tx_timeout);  in dpaa_netdev_init()
273 	mac_dev->phylink_config.dev = &net_dev->dev;  in dpaa_netdev_init()
287 	netif_carrier_off(net_dev);  in dpaa_netdev_init()
289 	err = register_netdev(net_dev);  in dpaa_netdev_init()
299 static int dpaa_stop(struct net_device *net_dev)  in dpaa_stop()  argument
306 	priv = netdev_priv(net_dev);  in dpaa_stop()
309 	netif_tx_stop_all_queues(net_dev);  in dpaa_stop()
325 	net_dev->phydev = NULL;  in dpaa_stop()
332 static void dpaa_tx_timeout(struct net_device *net_dev, unsigned int txqueue)  in dpaa_tx_timeout()  argument
337 	priv = netdev_priv(net_dev);  in dpaa_tx_timeout()
340 	netif_crit(priv, timer, net_dev, "Transmit timeout latency: %u ms\n",  in dpaa_tx_timeout()
341 		   jiffies_to_msecs(jiffies - dev_trans_start(net_dev)));  in dpaa_tx_timeout()
349 static void dpaa_get_stats64(struct net_device *net_dev,  in dpaa_get_stats64()  argument
353 	struct dpaa_priv *priv = netdev_priv(net_dev);  in dpaa_get_stats64()
370 static int dpaa_setup_tc(struct net_device *net_dev, enum tc_setup_type type,  in dpaa_setup_tc()  argument
373 	struct dpaa_priv *priv = netdev_priv(net_dev);  in dpaa_setup_tc()
388 		netdev_reset_tc(net_dev);  in dpaa_setup_tc()
393 		netdev_err(net_dev, "Too many traffic classes: max %d supported.\n",  in dpaa_setup_tc()
398 	netdev_set_num_tc(net_dev, num_tc);  in dpaa_setup_tc()
401 		netdev_set_tc_queue(net_dev, i, DPAA_TC_TXQ_NUM,  in dpaa_setup_tc()
406 	netif_set_real_num_tx_queues(net_dev, priv->num_tc * DPAA_TC_TXQ_NUM);  in dpaa_setup_tc()
431 static int dpaa_set_mac_address(struct net_device *net_dev, void *addr)  in dpaa_set_mac_address()  argument
438 	priv = netdev_priv(net_dev);  in dpaa_set_mac_address()
440 	memcpy(old_addr.sa_data, net_dev->dev_addr,  ETH_ALEN);  in dpaa_set_mac_address()
442 	err = eth_mac_addr(net_dev, addr);  in dpaa_set_mac_address()
444 		netif_err(priv, drv, net_dev, "eth_mac_addr() = %d\n", err);  in dpaa_set_mac_address()
451 				   (const enet_addr_t *)net_dev->dev_addr);  in dpaa_set_mac_address()
453 		netif_err(priv, drv, net_dev, "mac_dev->change_addr() = %d\n",  in dpaa_set_mac_address()
456 		eth_mac_addr(net_dev, &old_addr);  in dpaa_set_mac_address()
464 static void dpaa_set_rx_mode(struct net_device *net_dev)  in dpaa_set_rx_mode()  argument
469 	priv = netdev_priv(net_dev);  in dpaa_set_rx_mode()
471 	if (!!(net_dev->flags & IFF_PROMISC) != priv->mac_dev->promisc) {  in dpaa_set_rx_mode()
476 			netif_err(priv, drv, net_dev,  in dpaa_set_rx_mode()
481 	if (!!(net_dev->flags & IFF_ALLMULTI) != priv->mac_dev->allmulti) {  in dpaa_set_rx_mode()
486 			netif_err(priv, drv, net_dev,  in dpaa_set_rx_mode()
491 	err = priv->mac_dev->set_multi(net_dev, priv->mac_dev);  in dpaa_set_rx_mode()
493 		netif_err(priv, drv, net_dev, "mac_dev->set_multi() = %d\n",  in dpaa_set_rx_mode()
821 		netif_tx_stop_all_queues(priv->net_dev);  in dpaa_eth_cgscn()
826 		netif_tx_wake_all_queues(priv->net_dev);  in dpaa_eth_cgscn()
884 	struct net_device *net_dev = to_net_dev(mac_dev->phylink_config.dev);  in dpaa_eth_cgr_set_speed()  local
885 	struct dpaa_priv *priv = netdev_priv(net_dev);  in dpaa_eth_cgr_set_speed()
904 		netdev_err(net_dev, "could not update speed: %d\n", err);  in dpaa_eth_cgr_set_speed()
912 	fq->net_dev = priv->net_dev;  in dpaa_setup_ingress()
924 	fq->net_dev = priv->net_dev;  in dpaa_setup_egress()
947 		dev_err(priv->net_dev->dev.parent,  in dpaa_fq_setup()
984 			dev_warn(priv->net_dev->dev.parent,  in dpaa_fq_setup()
1024 	priv = netdev_priv(dpaa_fq->net_dev);  in dpaa_fq_init()
1025 	dev = dpaa_fq->net_dev->dev.parent;  in dpaa_fq_init()
1156 		err = xdp_rxq_info_reg(&dpaa_fq->xdp_rxq, dpaa_fq->net_dev,  in dpaa_fq_init()
1185 	priv = netdev_priv(dpaa_fq->net_dev);  in dpaa_fq_free_entry()
1383 static void dpaa_fd_release(const struct net_device *net_dev,  in dpaa_fd_release()  argument
1412 			netdev_err(net_dev, "DMA mapping failed\n");  in dpaa_fd_release()
1509 			netif_alert(priv, tx_err, priv->net_dev,  in dpaa_enable_tx_csum()
1526 			netif_alert(priv, tx_err, priv->net_dev,  in dpaa_enable_tx_csum()
1552 	struct net_device *net_dev = dpaa_bp->priv->net_dev;  in dpaa_bp_add_8_bufs()  local
1561 			netdev_err(net_dev, "dev_alloc_pages() failed\n");  in dpaa_bp_add_8_bufs()
1569 			netdev_err(net_dev, "DMA map failed\n");  in dpaa_bp_add_8_bufs()
1672 	struct device *dev = priv->net_dev->dev.parent;  in dpaa_cleanup_tx_fd()
1748 	if ((priv->net_dev->features & NETIF_F_RXCSUM) &&  in rx_csum_offload()
1930 	struct net_device *net_dev = priv->net_dev;  in skb_to_contig_fd()  local
1956 			netif_err(priv, tx_err, net_dev, "HW csum error: %d\n",  in skb_to_contig_fd()
1970 			netif_err(priv, tx_err, net_dev, "dma_map_single() failed\n");  in skb_to_contig_fd()
1983 	struct net_device *net_dev = priv->net_dev;  in skb_to_sg_fd()  local
1996 		netdev_err(net_dev, "dev_alloc_pages() failed\n");  in skb_to_sg_fd()
2010 			netif_err(priv, tx_err, net_dev, "HW csum error: %d\n",  in skb_to_sg_fd()
2024 		netdev_err(priv->net_dev, "DMA mapping failed\n");  in skb_to_sg_fd()
2038 			netdev_err(priv->net_dev, "DMA mapping failed\n");  in skb_to_sg_fd()
2064 		netdev_err(priv->net_dev, "DMA mapping failed\n");  in skb_to_sg_fd()
2100 	trace_dpaa_tx_fd(priv->net_dev, egress_fq, fd);  in dpaa_xmit()
2120 static int dpaa_a050385_wa_skb(struct net_device *net_dev, struct sk_buff **s)  in dpaa_a050385_wa_skb()  argument
2122 	struct dpaa_priv *priv = netdev_priv(net_dev);  in dpaa_a050385_wa_skb()
2155 	new_skb = netdev_alloc_skb(net_dev, skb->len + DPAA_A050385_ALIGN - 1 +  in dpaa_a050385_wa_skb()
2277 dpaa_start_xmit(struct sk_buff *skb, struct net_device *net_dev)  in dpaa_start_xmit()  argument
2289 	priv = netdev_priv(net_dev);  in dpaa_start_xmit()
2324 		if (dpaa_a050385_wa_skb(net_dev, &skb))  in dpaa_start_xmit()
2341 	txq = netdev_get_tx_queue(net_dev, queue_mapping);  in dpaa_start_xmit()
2362 static void dpaa_rx_error(struct net_device *net_dev,  in dpaa_rx_error()  argument
2369 		netif_err(priv, hw, net_dev, "Err FD status = 0x%08x\n",  in dpaa_rx_error()
2383 	dpaa_fd_release(net_dev, fd);  in dpaa_rx_error()
2386 static void dpaa_tx_error(struct net_device *net_dev,  in dpaa_tx_error()  argument
2395 		netif_warn(priv, hw, net_dev, "FD status = 0x%08x\n",  in dpaa_tx_error()
2427 static void dpaa_tx_conf(struct net_device *net_dev,  in dpaa_tx_conf()  argument
2437 			netif_warn(priv, hw, net_dev, "FD status = 0x%08x\n",  in dpaa_tx_conf()
2473 	struct net_device *net_dev;  in rx_error_dqrr()  local
2477 	net_dev = dpaa_fq->net_dev;  in rx_error_dqrr()
2478 	priv = netdev_priv(net_dev);  in rx_error_dqrr()
2489 	dpaa_rx_error(net_dev, priv, percpu_priv, &dq->fd, fq->fqid);  in rx_error_dqrr()
2494 static int dpaa_xdp_xmit_frame(struct net_device *net_dev,  in dpaa_xdp_xmit_frame()  argument
2497 	struct dpaa_priv *priv = netdev_priv(net_dev);  in dpaa_xdp_xmit_frame()
2549 	txq = netdev_get_tx_queue(net_dev, smp_processor_id());  in dpaa_xdp_xmit_frame()
2626 		if (dpaa_xdp_xmit_frame(priv->net_dev, xdpf))  in dpaa_run_xdp()
2635 		err = xdp_do_redirect(priv->net_dev, &xdp, xdp_prog);  in dpaa_run_xdp()
2637 			trace_xdp_exception(priv->net_dev, xdp_prog, xdp_act);  in dpaa_run_xdp()
2642 		bpf_warn_invalid_xdp_action(priv->net_dev, xdp_prog, xdp_act);  in dpaa_run_xdp()
2645 		trace_xdp_exception(priv->net_dev, xdp_prog, xdp_act);  in dpaa_run_xdp()
2670 	struct net_device *net_dev;  in rx_default_dqrr()  local
2686 	net_dev = dpaa_fq->net_dev;  in rx_default_dqrr()
2687 	priv = netdev_priv(net_dev);  in rx_default_dqrr()
2693 	trace_dpaa_rx_fd(net_dev, fq, &dq->fd);  in rx_default_dqrr()
2708 		dpaa_fd_release(net_dev, &dq->fd);  in rx_default_dqrr()
2714 			netif_warn(priv, hw, net_dev, "FD status = 0x%08x\n",  in rx_default_dqrr()
2718 		dpaa_fd_release(net_dev, fd);  in rx_default_dqrr()
2747 	if (net_dev->features & NETIF_F_RXHASH && priv->keygen_in_use &&  in rx_default_dqrr()
2790 	skb->protocol = eth_type_trans(skb, net_dev);  in rx_default_dqrr()
2821 	struct net_device *net_dev;  in conf_error_dqrr()  local
2824 	net_dev = ((struct dpaa_fq *)fq)->net_dev;  in conf_error_dqrr()
2825 	priv = netdev_priv(net_dev);  in conf_error_dqrr()
2832 	dpaa_tx_error(net_dev, priv, percpu_priv, &dq->fd, fq->fqid);  in conf_error_dqrr()
2843 	struct net_device *net_dev;  in conf_dflt_dqrr()  local
2846 	net_dev = ((struct dpaa_fq *)fq)->net_dev;  in conf_dflt_dqrr()
2847 	priv = netdev_priv(net_dev);  in conf_dflt_dqrr()
2850 	trace_dpaa_tx_conf_fd(net_dev, fq, &dq->fd);  in conf_dflt_dqrr()
2857 	dpaa_tx_conf(net_dev, priv, percpu_priv, &dq->fd, fq->fqid);  in conf_dflt_dqrr()
2869 	struct net_device *net_dev;  in egress_ern()  local
2872 	net_dev = ((struct dpaa_fq *)fq)->net_dev;  in egress_ern()
2873 	priv = netdev_priv(net_dev);  in egress_ern()
2918 static int dpaa_open(struct net_device *net_dev)  in dpaa_open()  argument
2924 	priv = netdev_priv(net_dev);  in dpaa_open()
2941 		netif_err(priv, ifup, net_dev, "mac_dev->enable() = %d\n", err);  in dpaa_open()
2946 	netif_tx_start_all_queues(net_dev);  in dpaa_open()
2961 static int dpaa_eth_stop(struct net_device *net_dev)  in dpaa_eth_stop()  argument
2966 	err = dpaa_stop(net_dev);  in dpaa_eth_stop()
2968 	priv = netdev_priv(net_dev);  in dpaa_eth_stop()
2982 		dev_warn(priv->net_dev->dev.parent,  in xdp_validate_mtu()
2991 static int dpaa_change_mtu(struct net_device *net_dev, int new_mtu)  in dpaa_change_mtu()  argument
2993 	struct dpaa_priv *priv = netdev_priv(net_dev);  in dpaa_change_mtu()
2998 	net_dev->mtu = new_mtu;  in dpaa_change_mtu()
3002 static int dpaa_setup_xdp(struct net_device *net_dev, struct netdev_bpf *bpf)  in dpaa_setup_xdp()  argument
3004 	struct dpaa_priv *priv = netdev_priv(net_dev);  in dpaa_setup_xdp()
3010 	if (bpf->prog && !xdp_validate_mtu(priv, net_dev->mtu)) {  in dpaa_setup_xdp()
3015 	up = netif_running(net_dev);  in dpaa_setup_xdp()
3018 		dpaa_eth_stop(net_dev);  in dpaa_setup_xdp()
3025 		err = dpaa_open(net_dev);  in dpaa_setup_xdp()
3035 static int dpaa_xdp(struct net_device *net_dev, struct netdev_bpf *xdp)  in dpaa_xdp()  argument
3039 		return dpaa_setup_xdp(net_dev, xdp);  in dpaa_xdp()
3045 static int dpaa_xdp_xmit(struct net_device *net_dev, int n,  in dpaa_xdp_xmit()  argument
3054 	if (!netif_running(net_dev))  in dpaa_xdp_xmit()
3059 		if (dpaa_xdp_xmit_frame(net_dev, xdpf))  in dpaa_xdp_xmit()
3106 static int dpaa_ioctl(struct net_device *net_dev, struct ifreq *rq, int cmd)  in dpaa_ioctl()  argument
3109 	struct dpaa_priv *priv = netdev_priv(net_dev);  in dpaa_ioctl()
3112 		if (net_dev->phydev)  in dpaa_ioctl()
3118 		return dpaa_ts_ioctl(net_dev, rq, cmd);  in dpaa_ioctl()
3140 static int dpaa_napi_add(struct net_device *net_dev)  in dpaa_napi_add()  argument
3142 	struct dpaa_priv *priv = netdev_priv(net_dev);  in dpaa_napi_add()
3149 		netif_napi_add(net_dev, &percpu_priv->np.napi, dpaa_eth_poll);  in dpaa_napi_add()
3155 static void dpaa_napi_del(struct net_device *net_dev)  in dpaa_napi_del()  argument
3157 	struct dpaa_priv *priv = netdev_priv(net_dev);  in dpaa_napi_del()
3282 	struct net_device *net_dev = NULL;  in dpaa_eth_probe()  local
3327 	net_dev = alloc_etherdev_mq(sizeof(*priv), DPAA_ETH_TXQ_NUM);  in dpaa_eth_probe()
3328 	if (!net_dev) {  in dpaa_eth_probe()
3334 	SET_NETDEV_DEV(net_dev, dev->parent);  in dpaa_eth_probe()
3335 	dev_set_drvdata(dev, net_dev);  in dpaa_eth_probe()
3337 	priv = netdev_priv(net_dev);  in dpaa_eth_probe()
3338 	priv->net_dev = net_dev;  in dpaa_eth_probe()
3344 		netdev_err(net_dev, "dpaa_mac_dev_get() failed\n");  in dpaa_eth_probe()
3357 		netdev_err(net_dev, "dma_coerce_mask_and_coherent() failed\n");  in dpaa_eth_probe()
3368 	net_dev->mtu = min(dpaa_get_max_mtu(), ETH_DATA_LEN);  in dpaa_eth_probe()
3370 	netdev_dbg(net_dev, "Setting initial MTU on net device: %d\n",  in dpaa_eth_probe()
3371 		   net_dev->mtu);  in dpaa_eth_probe()
3465 	netif_set_real_num_tx_queues(net_dev, priv->num_tc * DPAA_TC_TXQ_NUM);  in dpaa_eth_probe()
3468 	err = dpaa_napi_add(net_dev);  in dpaa_eth_probe()
3472 	err = dpaa_netdev_init(net_dev, &dpaa_ops, tx_timeout);  in dpaa_eth_probe()
3476 	dpaa_eth_sysfs_init(&net_dev->dev);  in dpaa_eth_probe()
3478 	netif_info(priv, probe, net_dev, "Probed interface %s\n",  in dpaa_eth_probe()
3479 		   net_dev->name);  in dpaa_eth_probe()
3484 	dpaa_napi_del(net_dev);  in dpaa_eth_probe()
3496 	free_netdev(net_dev);  in dpaa_eth_probe()
3503 	struct net_device *net_dev;  in dpaa_remove()  local
3509 	net_dev = dev_get_drvdata(dev);  in dpaa_remove()
3511 	priv = netdev_priv(net_dev);  in dpaa_remove()
3516 	unregister_netdev(net_dev);  in dpaa_remove()
3529 	dpaa_napi_del(net_dev);  in dpaa_remove()
3533 	free_netdev(net_dev);  in dpaa_remove()