1 // SPDX-License-Identifier: GPL-2.0-only
2 /*******************************************************************************
3 This is the driver for the ST MAC 10/100/1000 on-chip Ethernet controllers.
4 ST Ethernet IPs are built around a Synopsys IP Core.
5
6 Copyright(C) 2007-2011 STMicroelectronics Ltd
7
8
9 Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>
10
11 Documentation available at:
12 http://www.stlinux.com
13 Support available at:
14 https://bugzilla.stlinux.com/
15 *******************************************************************************/
16
17 #include <linux/clk.h>
18 #include <linux/kernel.h>
19 #include <linux/interrupt.h>
20 #include <linux/ip.h>
21 #include <linux/tcp.h>
22 #include <linux/skbuff.h>
23 #include <linux/ethtool.h>
24 #include <linux/if_ether.h>
25 #include <linux/crc32.h>
26 #include <linux/mii.h>
27 #include <linux/if.h>
28 #include <linux/if_vlan.h>
29 #include <linux/dma-mapping.h>
30 #include <linux/slab.h>
31 #include <linux/prefetch.h>
32 #include <linux/pinctrl/consumer.h>
33 #ifdef CONFIG_DEBUG_FS
34 #include <linux/debugfs.h>
35 #include <linux/seq_file.h>
36 #endif /* CONFIG_DEBUG_FS */
37 #include <linux/net_tstamp.h>
38 #include <linux/phylink.h>
39 #include <linux/udp.h>
40 #include <net/pkt_cls.h>
41 #include "stmmac_ptp.h"
42 #include "stmmac.h"
43 #include <linux/reset.h>
44 #include <linux/of_mdio.h>
45 #include "dwmac1000.h"
46 #include "dwxgmac2.h"
47 #include "hwif.h"
48
49 #define STMMAC_ALIGN(x) ALIGN(ALIGN(x, SMP_CACHE_BYTES), 16)
50 #define TSO_MAX_BUFF_SIZE (SZ_16K - 1)
51
52 /* Module parameters */
53 #define TX_TIMEO 5000
54 static int watchdog = TX_TIMEO;
55 module_param(watchdog, int, 0644);
56 MODULE_PARM_DESC(watchdog, "Transmit timeout in milliseconds (default 5s)");
57
58 static int debug = -1;
59 module_param(debug, int, 0644);
60 MODULE_PARM_DESC(debug, "Message Level (-1: default, 0: no output, 16: all)");
61
62 static int phyaddr = -1;
63 module_param(phyaddr, int, 0444);
64 MODULE_PARM_DESC(phyaddr, "Physical device address");
65
66 #define STMMAC_TX_THRESH(x) ((x)->dma_tx_size / 4)
67 #define STMMAC_RX_THRESH(x) ((x)->dma_rx_size / 4)
68
69 static int flow_ctrl = FLOW_AUTO;
70 module_param(flow_ctrl, int, 0644);
71 MODULE_PARM_DESC(flow_ctrl, "Flow control ability [on/off]");
72
73 static int pause = PAUSE_TIME;
74 module_param(pause, int, 0644);
75 MODULE_PARM_DESC(pause, "Flow Control Pause Time");
76
77 #define TC_DEFAULT 64
78 static int tc = TC_DEFAULT;
79 module_param(tc, int, 0644);
80 MODULE_PARM_DESC(tc, "DMA threshold control value");
81
82 #define DEFAULT_BUFSIZE 1536
83 static int buf_sz = DEFAULT_BUFSIZE;
84 module_param(buf_sz, int, 0644);
85 MODULE_PARM_DESC(buf_sz, "DMA buffer size");
86
87 #define STMMAC_RX_COPYBREAK 256
88
89 static const u32 default_msg_level = (NETIF_MSG_DRV | NETIF_MSG_PROBE |
90 NETIF_MSG_LINK | NETIF_MSG_IFUP |
91 NETIF_MSG_IFDOWN | NETIF_MSG_TIMER);
92
93 #define STMMAC_DEFAULT_LPI_TIMER 1000
94 static int eee_timer = STMMAC_DEFAULT_LPI_TIMER;
95 module_param(eee_timer, int, 0644);
96 MODULE_PARM_DESC(eee_timer, "LPI tx expiration time in msec");
97 #define STMMAC_LPI_T(x) (jiffies + usecs_to_jiffies(x))
98
99 /* By default the driver will use the ring mode to manage tx and rx descriptors,
100 * but allow user to force to use the chain instead of the ring
101 */
102 static unsigned int chain_mode;
103 module_param(chain_mode, int, 0444);
104 MODULE_PARM_DESC(chain_mode, "To use chain instead of ring mode");
105
106 static irqreturn_t stmmac_interrupt(int irq, void *dev_id);
107
108 #ifdef CONFIG_DEBUG_FS
109 static const struct net_device_ops stmmac_netdev_ops;
110 static void stmmac_init_fs(struct net_device *dev);
111 static void stmmac_exit_fs(struct net_device *dev);
112 #endif
113
114 #define STMMAC_COAL_TIMER(x) (jiffies + usecs_to_jiffies(x))
115
116 /**
117 * stmmac_verify_args - verify the driver parameters.
118 * Description: it checks the driver parameters and set a default in case of
119 * errors.
120 */
stmmac_verify_args(void)121 static void stmmac_verify_args(void)
122 {
123 if (unlikely(watchdog < 0))
124 watchdog = TX_TIMEO;
125 if (unlikely((buf_sz < DEFAULT_BUFSIZE) || (buf_sz > BUF_SIZE_16KiB)))
126 buf_sz = DEFAULT_BUFSIZE;
127 if (unlikely(flow_ctrl > 1))
128 flow_ctrl = FLOW_AUTO;
129 else if (likely(flow_ctrl < 0))
130 flow_ctrl = FLOW_OFF;
131 if (unlikely((pause < 0) || (pause > 0xffff)))
132 pause = PAUSE_TIME;
133 if (eee_timer < 0)
134 eee_timer = STMMAC_DEFAULT_LPI_TIMER;
135 }
136
137 /**
138 * stmmac_disable_all_queues - Disable all queues
139 * @priv: driver private structure
140 */
stmmac_disable_all_queues(struct stmmac_priv * priv)141 static void stmmac_disable_all_queues(struct stmmac_priv *priv)
142 {
143 u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
144 u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
145 u32 maxq = max(rx_queues_cnt, tx_queues_cnt);
146 u32 queue;
147
148 for (queue = 0; queue < maxq; queue++) {
149 struct stmmac_channel *ch = &priv->channel[queue];
150
151 if (queue < rx_queues_cnt)
152 napi_disable(&ch->rx_napi);
153 if (queue < tx_queues_cnt)
154 napi_disable(&ch->tx_napi);
155 }
156 }
157
158 /**
159 * stmmac_enable_all_queues - Enable all queues
160 * @priv: driver private structure
161 */
stmmac_enable_all_queues(struct stmmac_priv * priv)162 static void stmmac_enable_all_queues(struct stmmac_priv *priv)
163 {
164 u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
165 u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
166 u32 maxq = max(rx_queues_cnt, tx_queues_cnt);
167 u32 queue;
168
169 for (queue = 0; queue < maxq; queue++) {
170 struct stmmac_channel *ch = &priv->channel[queue];
171
172 if (queue < rx_queues_cnt)
173 napi_enable(&ch->rx_napi);
174 if (queue < tx_queues_cnt)
175 napi_enable(&ch->tx_napi);
176 }
177 }
178
stmmac_service_event_schedule(struct stmmac_priv * priv)179 static void stmmac_service_event_schedule(struct stmmac_priv *priv)
180 {
181 if (!test_bit(STMMAC_DOWN, &priv->state) &&
182 !test_and_set_bit(STMMAC_SERVICE_SCHED, &priv->state))
183 queue_work(priv->wq, &priv->service_task);
184 }
185
stmmac_global_err(struct stmmac_priv * priv)186 static void stmmac_global_err(struct stmmac_priv *priv)
187 {
188 netif_carrier_off(priv->dev);
189 set_bit(STMMAC_RESET_REQUESTED, &priv->state);
190 stmmac_service_event_schedule(priv);
191 }
192
193 /**
194 * stmmac_clk_csr_set - dynamically set the MDC clock
195 * @priv: driver private structure
196 * Description: this is to dynamically set the MDC clock according to the csr
197 * clock input.
198 * Note:
199 * If a specific clk_csr value is passed from the platform
200 * this means that the CSR Clock Range selection cannot be
201 * changed at run-time and it is fixed (as reported in the driver
202 * documentation). Viceversa the driver will try to set the MDC
203 * clock dynamically according to the actual clock input.
204 */
stmmac_clk_csr_set(struct stmmac_priv * priv)205 static void stmmac_clk_csr_set(struct stmmac_priv *priv)
206 {
207 u32 clk_rate;
208
209 clk_rate = clk_get_rate(priv->plat->stmmac_clk);
210
211 /* Platform provided default clk_csr would be assumed valid
212 * for all other cases except for the below mentioned ones.
213 * For values higher than the IEEE 802.3 specified frequency
214 * we can not estimate the proper divider as it is not known
215 * the frequency of clk_csr_i. So we do not change the default
216 * divider.
217 */
218 if (!(priv->clk_csr & MAC_CSR_H_FRQ_MASK)) {
219 if (clk_rate < CSR_F_35M)
220 priv->clk_csr = STMMAC_CSR_20_35M;
221 else if ((clk_rate >= CSR_F_35M) && (clk_rate < CSR_F_60M))
222 priv->clk_csr = STMMAC_CSR_35_60M;
223 else if ((clk_rate >= CSR_F_60M) && (clk_rate < CSR_F_100M))
224 priv->clk_csr = STMMAC_CSR_60_100M;
225 else if ((clk_rate >= CSR_F_100M) && (clk_rate < CSR_F_150M))
226 priv->clk_csr = STMMAC_CSR_100_150M;
227 else if ((clk_rate >= CSR_F_150M) && (clk_rate < CSR_F_250M))
228 priv->clk_csr = STMMAC_CSR_150_250M;
229 else if ((clk_rate >= CSR_F_250M) && (clk_rate < CSR_F_300M))
230 priv->clk_csr = STMMAC_CSR_250_300M;
231 }
232
233 if (priv->plat->has_sun8i) {
234 if (clk_rate > 160000000)
235 priv->clk_csr = 0x03;
236 else if (clk_rate > 80000000)
237 priv->clk_csr = 0x02;
238 else if (clk_rate > 40000000)
239 priv->clk_csr = 0x01;
240 else
241 priv->clk_csr = 0;
242 }
243
244 if (priv->plat->has_xgmac) {
245 if (clk_rate > 400000000)
246 priv->clk_csr = 0x5;
247 else if (clk_rate > 350000000)
248 priv->clk_csr = 0x4;
249 else if (clk_rate > 300000000)
250 priv->clk_csr = 0x3;
251 else if (clk_rate > 250000000)
252 priv->clk_csr = 0x2;
253 else if (clk_rate > 150000000)
254 priv->clk_csr = 0x1;
255 else
256 priv->clk_csr = 0x0;
257 }
258 }
259
print_pkt(unsigned char * buf,int len)260 static void print_pkt(unsigned char *buf, int len)
261 {
262 pr_debug("len = %d byte, buf addr: 0x%p\n", len, buf);
263 print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, buf, len);
264 }
265
stmmac_tx_avail(struct stmmac_priv * priv,u32 queue)266 static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue)
267 {
268 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
269 u32 avail;
270
271 if (tx_q->dirty_tx > tx_q->cur_tx)
272 avail = tx_q->dirty_tx - tx_q->cur_tx - 1;
273 else
274 avail = priv->dma_tx_size - tx_q->cur_tx + tx_q->dirty_tx - 1;
275
276 return avail;
277 }
278
279 /**
280 * stmmac_rx_dirty - Get RX queue dirty
281 * @priv: driver private structure
282 * @queue: RX queue index
283 */
stmmac_rx_dirty(struct stmmac_priv * priv,u32 queue)284 static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue)
285 {
286 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
287 u32 dirty;
288
289 if (rx_q->dirty_rx <= rx_q->cur_rx)
290 dirty = rx_q->cur_rx - rx_q->dirty_rx;
291 else
292 dirty = priv->dma_rx_size - rx_q->dirty_rx + rx_q->cur_rx;
293
294 return dirty;
295 }
296
297 /**
298 * stmmac_enable_eee_mode - check and enter in LPI mode
299 * @priv: driver private structure
300 * Description: this function is to verify and enter in LPI mode in case of
301 * EEE.
302 */
stmmac_enable_eee_mode(struct stmmac_priv * priv)303 static void stmmac_enable_eee_mode(struct stmmac_priv *priv)
304 {
305 u32 tx_cnt = priv->plat->tx_queues_to_use;
306 u32 queue;
307
308 /* check if all TX queues have the work finished */
309 for (queue = 0; queue < tx_cnt; queue++) {
310 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
311
312 if (tx_q->dirty_tx != tx_q->cur_tx)
313 return; /* still unfinished work */
314 }
315
316 /* Check and enter in LPI mode */
317 if (!priv->tx_path_in_lpi_mode)
318 stmmac_set_eee_mode(priv, priv->hw,
319 priv->plat->en_tx_lpi_clockgating);
320 }
321
322 /**
323 * stmmac_disable_eee_mode - disable and exit from LPI mode
324 * @priv: driver private structure
325 * Description: this function is to exit and disable EEE in case of
326 * LPI state is true. This is called by the xmit.
327 */
stmmac_disable_eee_mode(struct stmmac_priv * priv)328 void stmmac_disable_eee_mode(struct stmmac_priv *priv)
329 {
330 stmmac_reset_eee_mode(priv, priv->hw);
331 del_timer_sync(&priv->eee_ctrl_timer);
332 priv->tx_path_in_lpi_mode = false;
333 }
334
335 /**
336 * stmmac_eee_ctrl_timer - EEE TX SW timer.
337 * @t: timer_list struct containing private info
338 * Description:
339 * if there is no data transfer and if we are not in LPI state,
340 * then MAC Transmitter can be moved to LPI state.
341 */
stmmac_eee_ctrl_timer(struct timer_list * t)342 static void stmmac_eee_ctrl_timer(struct timer_list *t)
343 {
344 struct stmmac_priv *priv = from_timer(priv, t, eee_ctrl_timer);
345
346 stmmac_enable_eee_mode(priv);
347 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer));
348 }
349
350 /**
351 * stmmac_eee_init - init EEE
352 * @priv: driver private structure
353 * Description:
354 * if the GMAC supports the EEE (from the HW cap reg) and the phy device
355 * can also manage EEE, this function enable the LPI state and start related
356 * timer.
357 */
stmmac_eee_init(struct stmmac_priv * priv)358 bool stmmac_eee_init(struct stmmac_priv *priv)
359 {
360 int eee_tw_timer = priv->eee_tw_timer;
361
362 /* Using PCS we cannot dial with the phy registers at this stage
363 * so we do not support extra feature like EEE.
364 */
365 if (priv->hw->pcs == STMMAC_PCS_TBI ||
366 priv->hw->pcs == STMMAC_PCS_RTBI)
367 return false;
368
369 /* Check if MAC core supports the EEE feature. */
370 if (!priv->dma_cap.eee)
371 return false;
372
373 mutex_lock(&priv->lock);
374
375 /* Check if it needs to be deactivated */
376 if (!priv->eee_active) {
377 if (priv->eee_enabled) {
378 netdev_dbg(priv->dev, "disable EEE\n");
379 del_timer_sync(&priv->eee_ctrl_timer);
380 stmmac_set_eee_timer(priv, priv->hw, 0, eee_tw_timer);
381 }
382 mutex_unlock(&priv->lock);
383 return false;
384 }
385
386 if (priv->eee_active && !priv->eee_enabled) {
387 timer_setup(&priv->eee_ctrl_timer, stmmac_eee_ctrl_timer, 0);
388 stmmac_set_eee_timer(priv, priv->hw, STMMAC_DEFAULT_LIT_LS,
389 eee_tw_timer);
390 }
391
392 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer));
393
394 mutex_unlock(&priv->lock);
395 netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n");
396 return true;
397 }
398
399 /* stmmac_get_tx_hwtstamp - get HW TX timestamps
400 * @priv: driver private structure
401 * @p : descriptor pointer
402 * @skb : the socket buffer
403 * Description :
404 * This function will read timestamp from the descriptor & pass it to stack.
405 * and also perform some sanity checks.
406 */
stmmac_get_tx_hwtstamp(struct stmmac_priv * priv,struct dma_desc * p,struct sk_buff * skb)407 static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv,
408 struct dma_desc *p, struct sk_buff *skb)
409 {
410 struct skb_shared_hwtstamps shhwtstamp;
411 bool found = false;
412 u64 ns = 0;
413
414 if (!priv->hwts_tx_en)
415 return;
416
417 /* exit if skb doesn't support hw tstamp */
418 if (likely(!skb || !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)))
419 return;
420
421 /* check tx tstamp status */
422 if (stmmac_get_tx_timestamp_status(priv, p)) {
423 stmmac_get_timestamp(priv, p, priv->adv_ts, &ns);
424 found = true;
425 } else if (!stmmac_get_mac_tx_timestamp(priv, priv->hw, &ns)) {
426 found = true;
427 }
428
429 if (found) {
430 memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps));
431 shhwtstamp.hwtstamp = ns_to_ktime(ns);
432
433 netdev_dbg(priv->dev, "get valid TX hw timestamp %llu\n", ns);
434 /* pass tstamp to stack */
435 skb_tstamp_tx(skb, &shhwtstamp);
436 }
437 }
438
439 /* stmmac_get_rx_hwtstamp - get HW RX timestamps
440 * @priv: driver private structure
441 * @p : descriptor pointer
442 * @np : next descriptor pointer
443 * @skb : the socket buffer
444 * Description :
445 * This function will read received packet's timestamp from the descriptor
446 * and pass it to stack. It also perform some sanity checks.
447 */
stmmac_get_rx_hwtstamp(struct stmmac_priv * priv,struct dma_desc * p,struct dma_desc * np,struct sk_buff * skb)448 static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p,
449 struct dma_desc *np, struct sk_buff *skb)
450 {
451 struct skb_shared_hwtstamps *shhwtstamp = NULL;
452 struct dma_desc *desc = p;
453 u64 ns = 0;
454
455 if (!priv->hwts_rx_en)
456 return;
457 /* For GMAC4, the valid timestamp is from CTX next desc. */
458 if (priv->plat->has_gmac4 || priv->plat->has_xgmac)
459 desc = np;
460
461 /* Check if timestamp is available */
462 if (stmmac_get_rx_timestamp_status(priv, p, np, priv->adv_ts)) {
463 stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns);
464 netdev_dbg(priv->dev, "get valid RX hw timestamp %llu\n", ns);
465 shhwtstamp = skb_hwtstamps(skb);
466 memset(shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps));
467 shhwtstamp->hwtstamp = ns_to_ktime(ns);
468 } else {
469 netdev_dbg(priv->dev, "cannot get RX hw timestamp\n");
470 }
471 }
472
473 /**
474 * stmmac_hwtstamp_set - control hardware timestamping.
475 * @dev: device pointer.
476 * @ifr: An IOCTL specific structure, that can contain a pointer to
477 * a proprietary structure used to pass information to the driver.
478 * Description:
479 * This function configures the MAC to enable/disable both outgoing(TX)
480 * and incoming(RX) packets time stamping based on user input.
481 * Return Value:
482 * 0 on success and an appropriate -ve integer on failure.
483 */
stmmac_hwtstamp_set(struct net_device * dev,struct ifreq * ifr)484 static int stmmac_hwtstamp_set(struct net_device *dev, struct ifreq *ifr)
485 {
486 struct stmmac_priv *priv = netdev_priv(dev);
487 struct hwtstamp_config config;
488 struct timespec64 now;
489 u64 temp = 0;
490 u32 ptp_v2 = 0;
491 u32 tstamp_all = 0;
492 u32 ptp_over_ipv4_udp = 0;
493 u32 ptp_over_ipv6_udp = 0;
494 u32 ptp_over_ethernet = 0;
495 u32 snap_type_sel = 0;
496 u32 ts_master_en = 0;
497 u32 ts_event_en = 0;
498 u32 sec_inc = 0;
499 u32 value = 0;
500 bool xmac;
501
502 xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac;
503
504 if (!(priv->dma_cap.time_stamp || priv->adv_ts)) {
505 netdev_alert(priv->dev, "No support for HW time stamping\n");
506 priv->hwts_tx_en = 0;
507 priv->hwts_rx_en = 0;
508
509 return -EOPNOTSUPP;
510 }
511
512 if (copy_from_user(&config, ifr->ifr_data,
513 sizeof(config)))
514 return -EFAULT;
515
516 netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n",
517 __func__, config.flags, config.tx_type, config.rx_filter);
518
519 /* reserved for future extensions */
520 if (config.flags)
521 return -EINVAL;
522
523 if (config.tx_type != HWTSTAMP_TX_OFF &&
524 config.tx_type != HWTSTAMP_TX_ON)
525 return -ERANGE;
526
527 if (priv->adv_ts) {
528 switch (config.rx_filter) {
529 case HWTSTAMP_FILTER_NONE:
530 /* time stamp no incoming packet at all */
531 config.rx_filter = HWTSTAMP_FILTER_NONE;
532 break;
533
534 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
535 /* PTP v1, UDP, any kind of event packet */
536 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
537 /* 'xmac' hardware can support Sync, Pdelay_Req and
538 * Pdelay_resp by setting bit14 and bits17/16 to 01
539 * This leaves Delay_Req timestamps out.
540 * Enable all events *and* general purpose message
541 * timestamping
542 */
543 snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
544 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
545 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
546 break;
547
548 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
549 /* PTP v1, UDP, Sync packet */
550 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_SYNC;
551 /* take time stamp for SYNC messages only */
552 ts_event_en = PTP_TCR_TSEVNTENA;
553
554 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
555 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
556 break;
557
558 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
559 /* PTP v1, UDP, Delay_req packet */
560 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ;
561 /* take time stamp for Delay_Req messages only */
562 ts_master_en = PTP_TCR_TSMSTRENA;
563 ts_event_en = PTP_TCR_TSEVNTENA;
564
565 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
566 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
567 break;
568
569 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
570 /* PTP v2, UDP, any kind of event packet */
571 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT;
572 ptp_v2 = PTP_TCR_TSVER2ENA;
573 /* take time stamp for all event messages */
574 snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
575
576 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
577 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
578 break;
579
580 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
581 /* PTP v2, UDP, Sync packet */
582 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_SYNC;
583 ptp_v2 = PTP_TCR_TSVER2ENA;
584 /* take time stamp for SYNC messages only */
585 ts_event_en = PTP_TCR_TSEVNTENA;
586
587 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
588 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
589 break;
590
591 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
592 /* PTP v2, UDP, Delay_req packet */
593 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ;
594 ptp_v2 = PTP_TCR_TSVER2ENA;
595 /* take time stamp for Delay_Req messages only */
596 ts_master_en = PTP_TCR_TSMSTRENA;
597 ts_event_en = PTP_TCR_TSEVNTENA;
598
599 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
600 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
601 break;
602
603 case HWTSTAMP_FILTER_PTP_V2_EVENT:
604 /* PTP v2/802.AS1 any layer, any kind of event packet */
605 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
606 ptp_v2 = PTP_TCR_TSVER2ENA;
607 snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
608 if (priv->synopsys_id != DWMAC_CORE_5_10)
609 ts_event_en = PTP_TCR_TSEVNTENA;
610 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
611 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
612 ptp_over_ethernet = PTP_TCR_TSIPENA;
613 break;
614
615 case HWTSTAMP_FILTER_PTP_V2_SYNC:
616 /* PTP v2/802.AS1, any layer, Sync packet */
617 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_SYNC;
618 ptp_v2 = PTP_TCR_TSVER2ENA;
619 /* take time stamp for SYNC messages only */
620 ts_event_en = PTP_TCR_TSEVNTENA;
621
622 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
623 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
624 ptp_over_ethernet = PTP_TCR_TSIPENA;
625 break;
626
627 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
628 /* PTP v2/802.AS1, any layer, Delay_req packet */
629 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_DELAY_REQ;
630 ptp_v2 = PTP_TCR_TSVER2ENA;
631 /* take time stamp for Delay_Req messages only */
632 ts_master_en = PTP_TCR_TSMSTRENA;
633 ts_event_en = PTP_TCR_TSEVNTENA;
634
635 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
636 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
637 ptp_over_ethernet = PTP_TCR_TSIPENA;
638 break;
639
640 case HWTSTAMP_FILTER_NTP_ALL:
641 case HWTSTAMP_FILTER_ALL:
642 /* time stamp any incoming packet */
643 config.rx_filter = HWTSTAMP_FILTER_ALL;
644 tstamp_all = PTP_TCR_TSENALL;
645 break;
646
647 default:
648 return -ERANGE;
649 }
650 } else {
651 switch (config.rx_filter) {
652 case HWTSTAMP_FILTER_NONE:
653 config.rx_filter = HWTSTAMP_FILTER_NONE;
654 break;
655 default:
656 /* PTP v1, UDP, any kind of event packet */
657 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
658 break;
659 }
660 }
661 priv->hwts_rx_en = ((config.rx_filter == HWTSTAMP_FILTER_NONE) ? 0 : 1);
662 priv->hwts_tx_en = config.tx_type == HWTSTAMP_TX_ON;
663
664 if (!priv->hwts_tx_en && !priv->hwts_rx_en)
665 stmmac_config_hw_tstamping(priv, priv->ptpaddr, 0);
666 else {
667 value = (PTP_TCR_TSENA | PTP_TCR_TSCFUPDT | PTP_TCR_TSCTRLSSR |
668 tstamp_all | ptp_v2 | ptp_over_ethernet |
669 ptp_over_ipv6_udp | ptp_over_ipv4_udp | ts_event_en |
670 ts_master_en | snap_type_sel);
671 stmmac_config_hw_tstamping(priv, priv->ptpaddr, value);
672
673 /* program Sub Second Increment reg */
674 stmmac_config_sub_second_increment(priv,
675 priv->ptpaddr, priv->plat->clk_ptp_rate,
676 xmac, &sec_inc);
677 temp = div_u64(1000000000ULL, sec_inc);
678
679 /* Store sub second increment and flags for later use */
680 priv->sub_second_inc = sec_inc;
681 priv->systime_flags = value;
682
683 /* calculate default added value:
684 * formula is :
685 * addend = (2^32)/freq_div_ratio;
686 * where, freq_div_ratio = 1e9ns/sec_inc
687 */
688 temp = (u64)(temp << 32);
689 priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate);
690 stmmac_config_addend(priv, priv->ptpaddr, priv->default_addend);
691
692 /* initialize system time */
693 ktime_get_real_ts64(&now);
694
695 /* lower 32 bits of tv_sec are safe until y2106 */
696 stmmac_init_systime(priv, priv->ptpaddr,
697 (u32)now.tv_sec, now.tv_nsec);
698 }
699
700 memcpy(&priv->tstamp_config, &config, sizeof(config));
701
702 return copy_to_user(ifr->ifr_data, &config,
703 sizeof(config)) ? -EFAULT : 0;
704 }
705
706 /**
707 * stmmac_hwtstamp_get - read hardware timestamping.
708 * @dev: device pointer.
709 * @ifr: An IOCTL specific structure, that can contain a pointer to
710 * a proprietary structure used to pass information to the driver.
711 * Description:
712 * This function obtain the current hardware timestamping settings
713 * as requested.
714 */
stmmac_hwtstamp_get(struct net_device * dev,struct ifreq * ifr)715 static int stmmac_hwtstamp_get(struct net_device *dev, struct ifreq *ifr)
716 {
717 struct stmmac_priv *priv = netdev_priv(dev);
718 struct hwtstamp_config *config = &priv->tstamp_config;
719
720 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp))
721 return -EOPNOTSUPP;
722
723 return copy_to_user(ifr->ifr_data, config,
724 sizeof(*config)) ? -EFAULT : 0;
725 }
726
727 /**
728 * stmmac_init_ptp - init PTP
729 * @priv: driver private structure
730 * Description: this is to verify if the HW supports the PTPv1 or PTPv2.
731 * This is done by looking at the HW cap. register.
732 * This function also registers the ptp driver.
733 */
stmmac_init_ptp(struct stmmac_priv * priv)734 static int stmmac_init_ptp(struct stmmac_priv *priv)
735 {
736 bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac;
737
738 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp))
739 return -EOPNOTSUPP;
740
741 priv->adv_ts = 0;
742 /* Check if adv_ts can be enabled for dwmac 4.x / xgmac core */
743 if (xmac && priv->dma_cap.atime_stamp)
744 priv->adv_ts = 1;
745 /* Dwmac 3.x core with extend_desc can support adv_ts */
746 else if (priv->extend_desc && priv->dma_cap.atime_stamp)
747 priv->adv_ts = 1;
748
749 if (priv->dma_cap.time_stamp)
750 netdev_info(priv->dev, "IEEE 1588-2002 Timestamp supported\n");
751
752 if (priv->adv_ts)
753 netdev_info(priv->dev,
754 "IEEE 1588-2008 Advanced Timestamp supported\n");
755
756 priv->hwts_tx_en = 0;
757 priv->hwts_rx_en = 0;
758
759 stmmac_ptp_register(priv);
760
761 return 0;
762 }
763
stmmac_release_ptp(struct stmmac_priv * priv)764 static void stmmac_release_ptp(struct stmmac_priv *priv)
765 {
766 clk_disable_unprepare(priv->plat->clk_ptp_ref);
767 stmmac_ptp_unregister(priv);
768 }
769
770 /**
771 * stmmac_mac_flow_ctrl - Configure flow control in all queues
772 * @priv: driver private structure
773 * @duplex: duplex passed to the next function
774 * Description: It is used for configuring the flow control in all queues
775 */
stmmac_mac_flow_ctrl(struct stmmac_priv * priv,u32 duplex)776 static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex)
777 {
778 u32 tx_cnt = priv->plat->tx_queues_to_use;
779
780 stmmac_flow_ctrl(priv, priv->hw, duplex, priv->flow_ctrl,
781 priv->pause, tx_cnt);
782 }
783
stmmac_validate(struct phylink_config * config,unsigned long * supported,struct phylink_link_state * state)784 static void stmmac_validate(struct phylink_config *config,
785 unsigned long *supported,
786 struct phylink_link_state *state)
787 {
788 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
789 __ETHTOOL_DECLARE_LINK_MODE_MASK(mac_supported) = { 0, };
790 __ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, };
791 int tx_cnt = priv->plat->tx_queues_to_use;
792 int max_speed = priv->plat->max_speed;
793
794 phylink_set(mac_supported, 10baseT_Half);
795 phylink_set(mac_supported, 10baseT_Full);
796 phylink_set(mac_supported, 100baseT_Half);
797 phylink_set(mac_supported, 100baseT_Full);
798 phylink_set(mac_supported, 1000baseT_Half);
799 phylink_set(mac_supported, 1000baseT_Full);
800 phylink_set(mac_supported, 1000baseKX_Full);
801
802 phylink_set(mac_supported, Autoneg);
803 phylink_set(mac_supported, Pause);
804 phylink_set(mac_supported, Asym_Pause);
805 phylink_set_port_modes(mac_supported);
806
807 /* Cut down 1G if asked to */
808 if ((max_speed > 0) && (max_speed < 1000)) {
809 phylink_set(mask, 1000baseT_Full);
810 phylink_set(mask, 1000baseX_Full);
811 } else if (priv->plat->has_xgmac) {
812 if (!max_speed || (max_speed >= 2500)) {
813 phylink_set(mac_supported, 2500baseT_Full);
814 phylink_set(mac_supported, 2500baseX_Full);
815 }
816 if (!max_speed || (max_speed >= 5000)) {
817 phylink_set(mac_supported, 5000baseT_Full);
818 }
819 if (!max_speed || (max_speed >= 10000)) {
820 phylink_set(mac_supported, 10000baseSR_Full);
821 phylink_set(mac_supported, 10000baseLR_Full);
822 phylink_set(mac_supported, 10000baseER_Full);
823 phylink_set(mac_supported, 10000baseLRM_Full);
824 phylink_set(mac_supported, 10000baseT_Full);
825 phylink_set(mac_supported, 10000baseKX4_Full);
826 phylink_set(mac_supported, 10000baseKR_Full);
827 }
828 if (!max_speed || (max_speed >= 25000)) {
829 phylink_set(mac_supported, 25000baseCR_Full);
830 phylink_set(mac_supported, 25000baseKR_Full);
831 phylink_set(mac_supported, 25000baseSR_Full);
832 }
833 if (!max_speed || (max_speed >= 40000)) {
834 phylink_set(mac_supported, 40000baseKR4_Full);
835 phylink_set(mac_supported, 40000baseCR4_Full);
836 phylink_set(mac_supported, 40000baseSR4_Full);
837 phylink_set(mac_supported, 40000baseLR4_Full);
838 }
839 if (!max_speed || (max_speed >= 50000)) {
840 phylink_set(mac_supported, 50000baseCR2_Full);
841 phylink_set(mac_supported, 50000baseKR2_Full);
842 phylink_set(mac_supported, 50000baseSR2_Full);
843 phylink_set(mac_supported, 50000baseKR_Full);
844 phylink_set(mac_supported, 50000baseSR_Full);
845 phylink_set(mac_supported, 50000baseCR_Full);
846 phylink_set(mac_supported, 50000baseLR_ER_FR_Full);
847 phylink_set(mac_supported, 50000baseDR_Full);
848 }
849 if (!max_speed || (max_speed >= 100000)) {
850 phylink_set(mac_supported, 100000baseKR4_Full);
851 phylink_set(mac_supported, 100000baseSR4_Full);
852 phylink_set(mac_supported, 100000baseCR4_Full);
853 phylink_set(mac_supported, 100000baseLR4_ER4_Full);
854 phylink_set(mac_supported, 100000baseKR2_Full);
855 phylink_set(mac_supported, 100000baseSR2_Full);
856 phylink_set(mac_supported, 100000baseCR2_Full);
857 phylink_set(mac_supported, 100000baseLR2_ER2_FR2_Full);
858 phylink_set(mac_supported, 100000baseDR2_Full);
859 }
860 }
861
862 /* Half-Duplex can only work with single queue */
863 if (tx_cnt > 1) {
864 phylink_set(mask, 10baseT_Half);
865 phylink_set(mask, 100baseT_Half);
866 phylink_set(mask, 1000baseT_Half);
867 }
868
869 linkmode_and(supported, supported, mac_supported);
870 linkmode_andnot(supported, supported, mask);
871
872 linkmode_and(state->advertising, state->advertising, mac_supported);
873 linkmode_andnot(state->advertising, state->advertising, mask);
874
875 /* If PCS is supported, check which modes it supports. */
876 stmmac_xpcs_validate(priv, &priv->hw->xpcs_args, supported, state);
877 }
878
stmmac_mac_pcs_get_state(struct phylink_config * config,struct phylink_link_state * state)879 static void stmmac_mac_pcs_get_state(struct phylink_config *config,
880 struct phylink_link_state *state)
881 {
882 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
883
884 state->link = 0;
885 stmmac_xpcs_get_state(priv, &priv->hw->xpcs_args, state);
886 }
887
stmmac_mac_config(struct phylink_config * config,unsigned int mode,const struct phylink_link_state * state)888 static void stmmac_mac_config(struct phylink_config *config, unsigned int mode,
889 const struct phylink_link_state *state)
890 {
891 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
892
893 stmmac_xpcs_config(priv, &priv->hw->xpcs_args, state);
894 }
895
stmmac_mac_an_restart(struct phylink_config * config)896 static void stmmac_mac_an_restart(struct phylink_config *config)
897 {
898 /* Not Supported */
899 }
900
stmmac_mac_link_down(struct phylink_config * config,unsigned int mode,phy_interface_t interface)901 static void stmmac_mac_link_down(struct phylink_config *config,
902 unsigned int mode, phy_interface_t interface)
903 {
904 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
905
906 stmmac_mac_set(priv, priv->ioaddr, false);
907 priv->eee_active = false;
908 priv->tx_lpi_enabled = false;
909 stmmac_eee_init(priv);
910 stmmac_set_eee_pls(priv, priv->hw, false);
911 }
912
stmmac_mac_link_up(struct phylink_config * config,struct phy_device * phy,unsigned int mode,phy_interface_t interface,int speed,int duplex,bool tx_pause,bool rx_pause)913 static void stmmac_mac_link_up(struct phylink_config *config,
914 struct phy_device *phy,
915 unsigned int mode, phy_interface_t interface,
916 int speed, int duplex,
917 bool tx_pause, bool rx_pause)
918 {
919 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
920 u32 ctrl;
921
922 stmmac_xpcs_link_up(priv, &priv->hw->xpcs_args, speed, interface);
923
924 ctrl = readl(priv->ioaddr + MAC_CTRL_REG);
925 ctrl &= ~priv->hw->link.speed_mask;
926
927 if (interface == PHY_INTERFACE_MODE_USXGMII) {
928 switch (speed) {
929 case SPEED_10000:
930 ctrl |= priv->hw->link.xgmii.speed10000;
931 break;
932 case SPEED_5000:
933 ctrl |= priv->hw->link.xgmii.speed5000;
934 break;
935 case SPEED_2500:
936 ctrl |= priv->hw->link.xgmii.speed2500;
937 break;
938 default:
939 return;
940 }
941 } else if (interface == PHY_INTERFACE_MODE_XLGMII) {
942 switch (speed) {
943 case SPEED_100000:
944 ctrl |= priv->hw->link.xlgmii.speed100000;
945 break;
946 case SPEED_50000:
947 ctrl |= priv->hw->link.xlgmii.speed50000;
948 break;
949 case SPEED_40000:
950 ctrl |= priv->hw->link.xlgmii.speed40000;
951 break;
952 case SPEED_25000:
953 ctrl |= priv->hw->link.xlgmii.speed25000;
954 break;
955 case SPEED_10000:
956 ctrl |= priv->hw->link.xgmii.speed10000;
957 break;
958 case SPEED_2500:
959 ctrl |= priv->hw->link.speed2500;
960 break;
961 case SPEED_1000:
962 ctrl |= priv->hw->link.speed1000;
963 break;
964 default:
965 return;
966 }
967 } else {
968 switch (speed) {
969 case SPEED_2500:
970 ctrl |= priv->hw->link.speed2500;
971 break;
972 case SPEED_1000:
973 ctrl |= priv->hw->link.speed1000;
974 break;
975 case SPEED_100:
976 ctrl |= priv->hw->link.speed100;
977 break;
978 case SPEED_10:
979 ctrl |= priv->hw->link.speed10;
980 break;
981 default:
982 return;
983 }
984 }
985
986 priv->speed = speed;
987
988 if (priv->plat->fix_mac_speed)
989 priv->plat->fix_mac_speed(priv->plat->bsp_priv, speed);
990
991 if (!duplex)
992 ctrl &= ~priv->hw->link.duplex;
993 else
994 ctrl |= priv->hw->link.duplex;
995
996 /* Flow Control operation */
997 if (tx_pause && rx_pause)
998 stmmac_mac_flow_ctrl(priv, duplex);
999
1000 writel(ctrl, priv->ioaddr + MAC_CTRL_REG);
1001
1002 stmmac_mac_set(priv, priv->ioaddr, true);
1003 if (phy && priv->dma_cap.eee) {
1004 priv->eee_active = phy_init_eee(phy, 1) >= 0;
1005 priv->eee_enabled = stmmac_eee_init(priv);
1006 priv->tx_lpi_enabled = priv->eee_enabled;
1007 stmmac_set_eee_pls(priv, priv->hw, true);
1008 }
1009 }
1010
1011 static const struct phylink_mac_ops stmmac_phylink_mac_ops = {
1012 .validate = stmmac_validate,
1013 .mac_pcs_get_state = stmmac_mac_pcs_get_state,
1014 .mac_config = stmmac_mac_config,
1015 .mac_an_restart = stmmac_mac_an_restart,
1016 .mac_link_down = stmmac_mac_link_down,
1017 .mac_link_up = stmmac_mac_link_up,
1018 };
1019
1020 /**
1021 * stmmac_check_pcs_mode - verify if RGMII/SGMII is supported
1022 * @priv: driver private structure
1023 * Description: this is to verify if the HW supports the PCS.
1024 * Physical Coding Sublayer (PCS) interface that can be used when the MAC is
1025 * configured for the TBI, RTBI, or SGMII PHY interface.
1026 */
stmmac_check_pcs_mode(struct stmmac_priv * priv)1027 static void stmmac_check_pcs_mode(struct stmmac_priv *priv)
1028 {
1029 int interface = priv->plat->interface;
1030
1031 if (priv->dma_cap.pcs) {
1032 if ((interface == PHY_INTERFACE_MODE_RGMII) ||
1033 (interface == PHY_INTERFACE_MODE_RGMII_ID) ||
1034 (interface == PHY_INTERFACE_MODE_RGMII_RXID) ||
1035 (interface == PHY_INTERFACE_MODE_RGMII_TXID)) {
1036 netdev_dbg(priv->dev, "PCS RGMII support enabled\n");
1037 priv->hw->pcs = STMMAC_PCS_RGMII;
1038 } else if (interface == PHY_INTERFACE_MODE_SGMII) {
1039 netdev_dbg(priv->dev, "PCS SGMII support enabled\n");
1040 priv->hw->pcs = STMMAC_PCS_SGMII;
1041 }
1042 }
1043 }
1044
1045 /**
1046 * stmmac_init_phy - PHY initialization
1047 * @dev: net device structure
1048 * Description: it initializes the driver's PHY state, and attaches the PHY
1049 * to the mac driver.
1050 * Return value:
1051 * 0 on success
1052 */
stmmac_init_phy(struct net_device * dev)1053 static int stmmac_init_phy(struct net_device *dev)
1054 {
1055 struct ethtool_wolinfo wol = { .cmd = ETHTOOL_GWOL };
1056 struct stmmac_priv *priv = netdev_priv(dev);
1057 struct device_node *node;
1058 int ret;
1059
1060 node = priv->plat->phylink_node;
1061
1062 if (node)
1063 ret = phylink_of_phy_connect(priv->phylink, node, 0);
1064
1065 /* Some DT bindings do not set-up the PHY handle. Let's try to
1066 * manually parse it
1067 */
1068 if (!node || ret) {
1069 int addr = priv->plat->phy_addr;
1070 struct phy_device *phydev;
1071
1072 phydev = mdiobus_get_phy(priv->mii, addr);
1073 if (!phydev) {
1074 netdev_err(priv->dev, "no phy at addr %d\n", addr);
1075 return -ENODEV;
1076 }
1077
1078 ret = phylink_connect_phy(priv->phylink, phydev);
1079 }
1080
1081 phylink_ethtool_get_wol(priv->phylink, &wol);
1082 device_set_wakeup_capable(priv->device, !!wol.supported);
1083
1084 return ret;
1085 }
1086
stmmac_phy_setup(struct stmmac_priv * priv)1087 static int stmmac_phy_setup(struct stmmac_priv *priv)
1088 {
1089 struct fwnode_handle *fwnode = of_fwnode_handle(priv->plat->phylink_node);
1090 int mode = priv->plat->phy_interface;
1091 struct phylink *phylink;
1092
1093 priv->phylink_config.dev = &priv->dev->dev;
1094 priv->phylink_config.type = PHYLINK_NETDEV;
1095 priv->phylink_config.pcs_poll = true;
1096
1097 if (!fwnode)
1098 fwnode = dev_fwnode(priv->device);
1099
1100 phylink = phylink_create(&priv->phylink_config, fwnode,
1101 mode, &stmmac_phylink_mac_ops);
1102 if (IS_ERR(phylink))
1103 return PTR_ERR(phylink);
1104
1105 priv->phylink = phylink;
1106 return 0;
1107 }
1108
stmmac_display_rx_rings(struct stmmac_priv * priv)1109 static void stmmac_display_rx_rings(struct stmmac_priv *priv)
1110 {
1111 u32 rx_cnt = priv->plat->rx_queues_to_use;
1112 void *head_rx;
1113 u32 queue;
1114
1115 /* Display RX rings */
1116 for (queue = 0; queue < rx_cnt; queue++) {
1117 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1118
1119 pr_info("\tRX Queue %u rings\n", queue);
1120
1121 if (priv->extend_desc)
1122 head_rx = (void *)rx_q->dma_erx;
1123 else
1124 head_rx = (void *)rx_q->dma_rx;
1125
1126 /* Display RX ring */
1127 stmmac_display_ring(priv, head_rx, priv->dma_rx_size, true);
1128 }
1129 }
1130
stmmac_display_tx_rings(struct stmmac_priv * priv)1131 static void stmmac_display_tx_rings(struct stmmac_priv *priv)
1132 {
1133 u32 tx_cnt = priv->plat->tx_queues_to_use;
1134 void *head_tx;
1135 u32 queue;
1136
1137 /* Display TX rings */
1138 for (queue = 0; queue < tx_cnt; queue++) {
1139 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1140
1141 pr_info("\tTX Queue %d rings\n", queue);
1142
1143 if (priv->extend_desc)
1144 head_tx = (void *)tx_q->dma_etx;
1145 else if (tx_q->tbs & STMMAC_TBS_AVAIL)
1146 head_tx = (void *)tx_q->dma_entx;
1147 else
1148 head_tx = (void *)tx_q->dma_tx;
1149
1150 stmmac_display_ring(priv, head_tx, priv->dma_tx_size, false);
1151 }
1152 }
1153
stmmac_display_rings(struct stmmac_priv * priv)1154 static void stmmac_display_rings(struct stmmac_priv *priv)
1155 {
1156 /* Display RX ring */
1157 stmmac_display_rx_rings(priv);
1158
1159 /* Display TX ring */
1160 stmmac_display_tx_rings(priv);
1161 }
1162
stmmac_set_bfsize(int mtu,int bufsize)1163 static int stmmac_set_bfsize(int mtu, int bufsize)
1164 {
1165 int ret = bufsize;
1166
1167 if (mtu >= BUF_SIZE_8KiB)
1168 ret = BUF_SIZE_16KiB;
1169 else if (mtu >= BUF_SIZE_4KiB)
1170 ret = BUF_SIZE_8KiB;
1171 else if (mtu >= BUF_SIZE_2KiB)
1172 ret = BUF_SIZE_4KiB;
1173 else if (mtu > DEFAULT_BUFSIZE)
1174 ret = BUF_SIZE_2KiB;
1175 else
1176 ret = DEFAULT_BUFSIZE;
1177
1178 return ret;
1179 }
1180
1181 /**
1182 * stmmac_clear_rx_descriptors - clear RX descriptors
1183 * @priv: driver private structure
1184 * @queue: RX queue index
1185 * Description: this function is called to clear the RX descriptors
1186 * in case of both basic and extended descriptors are used.
1187 */
stmmac_clear_rx_descriptors(struct stmmac_priv * priv,u32 queue)1188 static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv, u32 queue)
1189 {
1190 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1191 int i;
1192
1193 /* Clear the RX descriptors */
1194 for (i = 0; i < priv->dma_rx_size; i++)
1195 if (priv->extend_desc)
1196 stmmac_init_rx_desc(priv, &rx_q->dma_erx[i].basic,
1197 priv->use_riwt, priv->mode,
1198 (i == priv->dma_rx_size - 1),
1199 priv->dma_buf_sz);
1200 else
1201 stmmac_init_rx_desc(priv, &rx_q->dma_rx[i],
1202 priv->use_riwt, priv->mode,
1203 (i == priv->dma_rx_size - 1),
1204 priv->dma_buf_sz);
1205 }
1206
1207 /**
1208 * stmmac_clear_tx_descriptors - clear tx descriptors
1209 * @priv: driver private structure
1210 * @queue: TX queue index.
1211 * Description: this function is called to clear the TX descriptors
1212 * in case of both basic and extended descriptors are used.
1213 */
stmmac_clear_tx_descriptors(struct stmmac_priv * priv,u32 queue)1214 static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, u32 queue)
1215 {
1216 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1217 int i;
1218
1219 /* Clear the TX descriptors */
1220 for (i = 0; i < priv->dma_tx_size; i++) {
1221 int last = (i == (priv->dma_tx_size - 1));
1222 struct dma_desc *p;
1223
1224 if (priv->extend_desc)
1225 p = &tx_q->dma_etx[i].basic;
1226 else if (tx_q->tbs & STMMAC_TBS_AVAIL)
1227 p = &tx_q->dma_entx[i].basic;
1228 else
1229 p = &tx_q->dma_tx[i];
1230
1231 stmmac_init_tx_desc(priv, p, priv->mode, last);
1232 }
1233 }
1234
1235 /**
1236 * stmmac_clear_descriptors - clear descriptors
1237 * @priv: driver private structure
1238 * Description: this function is called to clear the TX and RX descriptors
1239 * in case of both basic and extended descriptors are used.
1240 */
stmmac_clear_descriptors(struct stmmac_priv * priv)1241 static void stmmac_clear_descriptors(struct stmmac_priv *priv)
1242 {
1243 u32 rx_queue_cnt = priv->plat->rx_queues_to_use;
1244 u32 tx_queue_cnt = priv->plat->tx_queues_to_use;
1245 u32 queue;
1246
1247 /* Clear the RX descriptors */
1248 for (queue = 0; queue < rx_queue_cnt; queue++)
1249 stmmac_clear_rx_descriptors(priv, queue);
1250
1251 /* Clear the TX descriptors */
1252 for (queue = 0; queue < tx_queue_cnt; queue++)
1253 stmmac_clear_tx_descriptors(priv, queue);
1254 }
1255
1256 /**
1257 * stmmac_init_rx_buffers - init the RX descriptor buffer.
1258 * @priv: driver private structure
1259 * @p: descriptor pointer
1260 * @i: descriptor index
1261 * @flags: gfp flag
1262 * @queue: RX queue index
1263 * Description: this function is called to allocate a receive buffer, perform
1264 * the DMA mapping and init the descriptor.
1265 */
stmmac_init_rx_buffers(struct stmmac_priv * priv,struct dma_desc * p,int i,gfp_t flags,u32 queue)1266 static int stmmac_init_rx_buffers(struct stmmac_priv *priv, struct dma_desc *p,
1267 int i, gfp_t flags, u32 queue)
1268 {
1269 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1270 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i];
1271
1272 buf->page = page_pool_dev_alloc_pages(rx_q->page_pool);
1273 if (!buf->page)
1274 return -ENOMEM;
1275
1276 if (priv->sph) {
1277 buf->sec_page = page_pool_dev_alloc_pages(rx_q->page_pool);
1278 if (!buf->sec_page)
1279 return -ENOMEM;
1280
1281 buf->sec_addr = page_pool_get_dma_addr(buf->sec_page);
1282 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr);
1283 } else {
1284 buf->sec_page = NULL;
1285 }
1286
1287 buf->addr = page_pool_get_dma_addr(buf->page);
1288 stmmac_set_desc_addr(priv, p, buf->addr);
1289 if (priv->dma_buf_sz == BUF_SIZE_16KiB)
1290 stmmac_init_desc3(priv, p);
1291
1292 return 0;
1293 }
1294
1295 /**
1296 * stmmac_free_rx_buffer - free RX dma buffers
1297 * @priv: private structure
1298 * @queue: RX queue index
1299 * @i: buffer index.
1300 */
stmmac_free_rx_buffer(struct stmmac_priv * priv,u32 queue,int i)1301 static void stmmac_free_rx_buffer(struct stmmac_priv *priv, u32 queue, int i)
1302 {
1303 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1304 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i];
1305
1306 if (buf->page)
1307 page_pool_put_full_page(rx_q->page_pool, buf->page, false);
1308 buf->page = NULL;
1309
1310 if (buf->sec_page)
1311 page_pool_put_full_page(rx_q->page_pool, buf->sec_page, false);
1312 buf->sec_page = NULL;
1313 }
1314
1315 /**
1316 * stmmac_free_tx_buffer - free RX dma buffers
1317 * @priv: private structure
1318 * @queue: RX queue index
1319 * @i: buffer index.
1320 */
stmmac_free_tx_buffer(struct stmmac_priv * priv,u32 queue,int i)1321 static void stmmac_free_tx_buffer(struct stmmac_priv *priv, u32 queue, int i)
1322 {
1323 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1324
1325 if (tx_q->tx_skbuff_dma[i].buf) {
1326 if (tx_q->tx_skbuff_dma[i].map_as_page)
1327 dma_unmap_page(priv->device,
1328 tx_q->tx_skbuff_dma[i].buf,
1329 tx_q->tx_skbuff_dma[i].len,
1330 DMA_TO_DEVICE);
1331 else
1332 dma_unmap_single(priv->device,
1333 tx_q->tx_skbuff_dma[i].buf,
1334 tx_q->tx_skbuff_dma[i].len,
1335 DMA_TO_DEVICE);
1336 }
1337
1338 if (tx_q->tx_skbuff[i]) {
1339 dev_kfree_skb_any(tx_q->tx_skbuff[i]);
1340 tx_q->tx_skbuff[i] = NULL;
1341 tx_q->tx_skbuff_dma[i].buf = 0;
1342 tx_q->tx_skbuff_dma[i].map_as_page = false;
1343 }
1344 }
1345
1346 /**
1347 * init_dma_rx_desc_rings - init the RX descriptor rings
1348 * @dev: net device structure
1349 * @flags: gfp flag.
1350 * Description: this function initializes the DMA RX descriptors
1351 * and allocates the socket buffers. It supports the chained and ring
1352 * modes.
1353 */
init_dma_rx_desc_rings(struct net_device * dev,gfp_t flags)1354 static int init_dma_rx_desc_rings(struct net_device *dev, gfp_t flags)
1355 {
1356 struct stmmac_priv *priv = netdev_priv(dev);
1357 u32 rx_count = priv->plat->rx_queues_to_use;
1358 int ret = -ENOMEM;
1359 int queue;
1360 int i;
1361
1362 /* RX INITIALIZATION */
1363 netif_dbg(priv, probe, priv->dev,
1364 "SKB addresses:\nskb\t\tskb data\tdma data\n");
1365
1366 for (queue = 0; queue < rx_count; queue++) {
1367 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1368
1369 netif_dbg(priv, probe, priv->dev,
1370 "(%s) dma_rx_phy=0x%08x\n", __func__,
1371 (u32)rx_q->dma_rx_phy);
1372
1373 stmmac_clear_rx_descriptors(priv, queue);
1374
1375 for (i = 0; i < priv->dma_rx_size; i++) {
1376 struct dma_desc *p;
1377
1378 if (priv->extend_desc)
1379 p = &((rx_q->dma_erx + i)->basic);
1380 else
1381 p = rx_q->dma_rx + i;
1382
1383 ret = stmmac_init_rx_buffers(priv, p, i, flags,
1384 queue);
1385 if (ret)
1386 goto err_init_rx_buffers;
1387 }
1388
1389 rx_q->cur_rx = 0;
1390 rx_q->dirty_rx = (unsigned int)(i - priv->dma_rx_size);
1391
1392 /* Setup the chained descriptor addresses */
1393 if (priv->mode == STMMAC_CHAIN_MODE) {
1394 if (priv->extend_desc)
1395 stmmac_mode_init(priv, rx_q->dma_erx,
1396 rx_q->dma_rx_phy,
1397 priv->dma_rx_size, 1);
1398 else
1399 stmmac_mode_init(priv, rx_q->dma_rx,
1400 rx_q->dma_rx_phy,
1401 priv->dma_rx_size, 0);
1402 }
1403 }
1404
1405 return 0;
1406
1407 err_init_rx_buffers:
1408 while (queue >= 0) {
1409 while (--i >= 0)
1410 stmmac_free_rx_buffer(priv, queue, i);
1411
1412 if (queue == 0)
1413 break;
1414
1415 i = priv->dma_rx_size;
1416 queue--;
1417 }
1418
1419 return ret;
1420 }
1421
1422 /**
1423 * init_dma_tx_desc_rings - init the TX descriptor rings
1424 * @dev: net device structure.
1425 * Description: this function initializes the DMA TX descriptors
1426 * and allocates the socket buffers. It supports the chained and ring
1427 * modes.
1428 */
init_dma_tx_desc_rings(struct net_device * dev)1429 static int init_dma_tx_desc_rings(struct net_device *dev)
1430 {
1431 struct stmmac_priv *priv = netdev_priv(dev);
1432 u32 tx_queue_cnt = priv->plat->tx_queues_to_use;
1433 u32 queue;
1434 int i;
1435
1436 for (queue = 0; queue < tx_queue_cnt; queue++) {
1437 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1438
1439 netif_dbg(priv, probe, priv->dev,
1440 "(%s) dma_tx_phy=0x%08x\n", __func__,
1441 (u32)tx_q->dma_tx_phy);
1442
1443 /* Setup the chained descriptor addresses */
1444 if (priv->mode == STMMAC_CHAIN_MODE) {
1445 if (priv->extend_desc)
1446 stmmac_mode_init(priv, tx_q->dma_etx,
1447 tx_q->dma_tx_phy,
1448 priv->dma_tx_size, 1);
1449 else if (!(tx_q->tbs & STMMAC_TBS_AVAIL))
1450 stmmac_mode_init(priv, tx_q->dma_tx,
1451 tx_q->dma_tx_phy,
1452 priv->dma_tx_size, 0);
1453 }
1454
1455 for (i = 0; i < priv->dma_tx_size; i++) {
1456 struct dma_desc *p;
1457 if (priv->extend_desc)
1458 p = &((tx_q->dma_etx + i)->basic);
1459 else if (tx_q->tbs & STMMAC_TBS_AVAIL)
1460 p = &((tx_q->dma_entx + i)->basic);
1461 else
1462 p = tx_q->dma_tx + i;
1463
1464 stmmac_clear_desc(priv, p);
1465
1466 tx_q->tx_skbuff_dma[i].buf = 0;
1467 tx_q->tx_skbuff_dma[i].map_as_page = false;
1468 tx_q->tx_skbuff_dma[i].len = 0;
1469 tx_q->tx_skbuff_dma[i].last_segment = false;
1470 tx_q->tx_skbuff[i] = NULL;
1471 }
1472
1473 tx_q->dirty_tx = 0;
1474 tx_q->cur_tx = 0;
1475 tx_q->mss = 0;
1476
1477 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue));
1478 }
1479
1480 return 0;
1481 }
1482
1483 /**
1484 * init_dma_desc_rings - init the RX/TX descriptor rings
1485 * @dev: net device structure
1486 * @flags: gfp flag.
1487 * Description: this function initializes the DMA RX/TX descriptors
1488 * and allocates the socket buffers. It supports the chained and ring
1489 * modes.
1490 */
init_dma_desc_rings(struct net_device * dev,gfp_t flags)1491 static int init_dma_desc_rings(struct net_device *dev, gfp_t flags)
1492 {
1493 struct stmmac_priv *priv = netdev_priv(dev);
1494 int ret;
1495
1496 ret = init_dma_rx_desc_rings(dev, flags);
1497 if (ret)
1498 return ret;
1499
1500 ret = init_dma_tx_desc_rings(dev);
1501
1502 stmmac_clear_descriptors(priv);
1503
1504 if (netif_msg_hw(priv))
1505 stmmac_display_rings(priv);
1506
1507 return ret;
1508 }
1509
1510 /**
1511 * dma_free_rx_skbufs - free RX dma buffers
1512 * @priv: private structure
1513 * @queue: RX queue index
1514 */
dma_free_rx_skbufs(struct stmmac_priv * priv,u32 queue)1515 static void dma_free_rx_skbufs(struct stmmac_priv *priv, u32 queue)
1516 {
1517 int i;
1518
1519 for (i = 0; i < priv->dma_rx_size; i++)
1520 stmmac_free_rx_buffer(priv, queue, i);
1521 }
1522
1523 /**
1524 * dma_free_tx_skbufs - free TX dma buffers
1525 * @priv: private structure
1526 * @queue: TX queue index
1527 */
dma_free_tx_skbufs(struct stmmac_priv * priv,u32 queue)1528 static void dma_free_tx_skbufs(struct stmmac_priv *priv, u32 queue)
1529 {
1530 int i;
1531
1532 for (i = 0; i < priv->dma_tx_size; i++)
1533 stmmac_free_tx_buffer(priv, queue, i);
1534 }
1535
1536 /**
1537 * stmmac_free_tx_skbufs - free TX skb buffers
1538 * @priv: private structure
1539 */
stmmac_free_tx_skbufs(struct stmmac_priv * priv)1540 static void stmmac_free_tx_skbufs(struct stmmac_priv *priv)
1541 {
1542 u32 tx_queue_cnt = priv->plat->tx_queues_to_use;
1543 u32 queue;
1544
1545 for (queue = 0; queue < tx_queue_cnt; queue++)
1546 dma_free_tx_skbufs(priv, queue);
1547 }
1548
1549 /**
1550 * free_dma_rx_desc_resources - free RX dma desc resources
1551 * @priv: private structure
1552 */
free_dma_rx_desc_resources(struct stmmac_priv * priv)1553 static void free_dma_rx_desc_resources(struct stmmac_priv *priv)
1554 {
1555 u32 rx_count = priv->plat->rx_queues_to_use;
1556 u32 queue;
1557
1558 /* Free RX queue resources */
1559 for (queue = 0; queue < rx_count; queue++) {
1560 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1561
1562 /* Release the DMA RX socket buffers */
1563 dma_free_rx_skbufs(priv, queue);
1564
1565 /* Free DMA regions of consistent memory previously allocated */
1566 if (!priv->extend_desc)
1567 dma_free_coherent(priv->device, priv->dma_rx_size *
1568 sizeof(struct dma_desc),
1569 rx_q->dma_rx, rx_q->dma_rx_phy);
1570 else
1571 dma_free_coherent(priv->device, priv->dma_rx_size *
1572 sizeof(struct dma_extended_desc),
1573 rx_q->dma_erx, rx_q->dma_rx_phy);
1574
1575 kfree(rx_q->buf_pool);
1576 if (rx_q->page_pool)
1577 page_pool_destroy(rx_q->page_pool);
1578 }
1579 }
1580
1581 /**
1582 * free_dma_tx_desc_resources - free TX dma desc resources
1583 * @priv: private structure
1584 */
free_dma_tx_desc_resources(struct stmmac_priv * priv)1585 static void free_dma_tx_desc_resources(struct stmmac_priv *priv)
1586 {
1587 u32 tx_count = priv->plat->tx_queues_to_use;
1588 u32 queue;
1589
1590 /* Free TX queue resources */
1591 for (queue = 0; queue < tx_count; queue++) {
1592 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1593 size_t size;
1594 void *addr;
1595
1596 /* Release the DMA TX socket buffers */
1597 dma_free_tx_skbufs(priv, queue);
1598
1599 if (priv->extend_desc) {
1600 size = sizeof(struct dma_extended_desc);
1601 addr = tx_q->dma_etx;
1602 } else if (tx_q->tbs & STMMAC_TBS_AVAIL) {
1603 size = sizeof(struct dma_edesc);
1604 addr = tx_q->dma_entx;
1605 } else {
1606 size = sizeof(struct dma_desc);
1607 addr = tx_q->dma_tx;
1608 }
1609
1610 size *= priv->dma_tx_size;
1611
1612 dma_free_coherent(priv->device, size, addr, tx_q->dma_tx_phy);
1613
1614 kfree(tx_q->tx_skbuff_dma);
1615 kfree(tx_q->tx_skbuff);
1616 }
1617 }
1618
1619 /**
1620 * alloc_dma_rx_desc_resources - alloc RX resources.
1621 * @priv: private structure
1622 * Description: according to which descriptor can be used (extend or basic)
1623 * this function allocates the resources for TX and RX paths. In case of
1624 * reception, for example, it pre-allocated the RX socket buffer in order to
1625 * allow zero-copy mechanism.
1626 */
alloc_dma_rx_desc_resources(struct stmmac_priv * priv)1627 static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv)
1628 {
1629 u32 rx_count = priv->plat->rx_queues_to_use;
1630 int ret = -ENOMEM;
1631 u32 queue;
1632
1633 /* RX queues buffers and DMA */
1634 for (queue = 0; queue < rx_count; queue++) {
1635 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1636 struct page_pool_params pp_params = { 0 };
1637 unsigned int num_pages;
1638
1639 rx_q->queue_index = queue;
1640 rx_q->priv_data = priv;
1641
1642 pp_params.flags = PP_FLAG_DMA_MAP;
1643 pp_params.pool_size = priv->dma_rx_size;
1644 num_pages = DIV_ROUND_UP(priv->dma_buf_sz, PAGE_SIZE);
1645 pp_params.order = ilog2(num_pages);
1646 pp_params.nid = dev_to_node(priv->device);
1647 pp_params.dev = priv->device;
1648 pp_params.dma_dir = DMA_FROM_DEVICE;
1649
1650 rx_q->page_pool = page_pool_create(&pp_params);
1651 if (IS_ERR(rx_q->page_pool)) {
1652 ret = PTR_ERR(rx_q->page_pool);
1653 rx_q->page_pool = NULL;
1654 goto err_dma;
1655 }
1656
1657 rx_q->buf_pool = kcalloc(priv->dma_rx_size,
1658 sizeof(*rx_q->buf_pool),
1659 GFP_KERNEL);
1660 if (!rx_q->buf_pool)
1661 goto err_dma;
1662
1663 if (priv->extend_desc) {
1664 rx_q->dma_erx = dma_alloc_coherent(priv->device,
1665 priv->dma_rx_size *
1666 sizeof(struct dma_extended_desc),
1667 &rx_q->dma_rx_phy,
1668 GFP_KERNEL);
1669 if (!rx_q->dma_erx)
1670 goto err_dma;
1671
1672 } else {
1673 rx_q->dma_rx = dma_alloc_coherent(priv->device,
1674 priv->dma_rx_size *
1675 sizeof(struct dma_desc),
1676 &rx_q->dma_rx_phy,
1677 GFP_KERNEL);
1678 if (!rx_q->dma_rx)
1679 goto err_dma;
1680 }
1681 }
1682
1683 return 0;
1684
1685 err_dma:
1686 free_dma_rx_desc_resources(priv);
1687
1688 return ret;
1689 }
1690
1691 /**
1692 * alloc_dma_tx_desc_resources - alloc TX resources.
1693 * @priv: private structure
1694 * Description: according to which descriptor can be used (extend or basic)
1695 * this function allocates the resources for TX and RX paths. In case of
1696 * reception, for example, it pre-allocated the RX socket buffer in order to
1697 * allow zero-copy mechanism.
1698 */
alloc_dma_tx_desc_resources(struct stmmac_priv * priv)1699 static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv)
1700 {
1701 u32 tx_count = priv->plat->tx_queues_to_use;
1702 int ret = -ENOMEM;
1703 u32 queue;
1704
1705 /* TX queues buffers and DMA */
1706 for (queue = 0; queue < tx_count; queue++) {
1707 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1708 size_t size;
1709 void *addr;
1710
1711 tx_q->queue_index = queue;
1712 tx_q->priv_data = priv;
1713
1714 tx_q->tx_skbuff_dma = kcalloc(priv->dma_tx_size,
1715 sizeof(*tx_q->tx_skbuff_dma),
1716 GFP_KERNEL);
1717 if (!tx_q->tx_skbuff_dma)
1718 goto err_dma;
1719
1720 tx_q->tx_skbuff = kcalloc(priv->dma_tx_size,
1721 sizeof(struct sk_buff *),
1722 GFP_KERNEL);
1723 if (!tx_q->tx_skbuff)
1724 goto err_dma;
1725
1726 if (priv->extend_desc)
1727 size = sizeof(struct dma_extended_desc);
1728 else if (tx_q->tbs & STMMAC_TBS_AVAIL)
1729 size = sizeof(struct dma_edesc);
1730 else
1731 size = sizeof(struct dma_desc);
1732
1733 size *= priv->dma_tx_size;
1734
1735 addr = dma_alloc_coherent(priv->device, size,
1736 &tx_q->dma_tx_phy, GFP_KERNEL);
1737 if (!addr)
1738 goto err_dma;
1739
1740 if (priv->extend_desc)
1741 tx_q->dma_etx = addr;
1742 else if (tx_q->tbs & STMMAC_TBS_AVAIL)
1743 tx_q->dma_entx = addr;
1744 else
1745 tx_q->dma_tx = addr;
1746 }
1747
1748 return 0;
1749
1750 err_dma:
1751 free_dma_tx_desc_resources(priv);
1752 return ret;
1753 }
1754
1755 /**
1756 * alloc_dma_desc_resources - alloc TX/RX resources.
1757 * @priv: private structure
1758 * Description: according to which descriptor can be used (extend or basic)
1759 * this function allocates the resources for TX and RX paths. In case of
1760 * reception, for example, it pre-allocated the RX socket buffer in order to
1761 * allow zero-copy mechanism.
1762 */
alloc_dma_desc_resources(struct stmmac_priv * priv)1763 static int alloc_dma_desc_resources(struct stmmac_priv *priv)
1764 {
1765 /* RX Allocation */
1766 int ret = alloc_dma_rx_desc_resources(priv);
1767
1768 if (ret)
1769 return ret;
1770
1771 ret = alloc_dma_tx_desc_resources(priv);
1772
1773 return ret;
1774 }
1775
1776 /**
1777 * free_dma_desc_resources - free dma desc resources
1778 * @priv: private structure
1779 */
free_dma_desc_resources(struct stmmac_priv * priv)1780 static void free_dma_desc_resources(struct stmmac_priv *priv)
1781 {
1782 /* Release the DMA RX socket buffers */
1783 free_dma_rx_desc_resources(priv);
1784
1785 /* Release the DMA TX socket buffers */
1786 free_dma_tx_desc_resources(priv);
1787 }
1788
1789 /**
1790 * stmmac_mac_enable_rx_queues - Enable MAC rx queues
1791 * @priv: driver private structure
1792 * Description: It is used for enabling the rx queues in the MAC
1793 */
stmmac_mac_enable_rx_queues(struct stmmac_priv * priv)1794 static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv)
1795 {
1796 u32 rx_queues_count = priv->plat->rx_queues_to_use;
1797 int queue;
1798 u8 mode;
1799
1800 for (queue = 0; queue < rx_queues_count; queue++) {
1801 mode = priv->plat->rx_queues_cfg[queue].mode_to_use;
1802 stmmac_rx_queue_enable(priv, priv->hw, mode, queue);
1803 }
1804 }
1805
1806 /**
1807 * stmmac_start_rx_dma - start RX DMA channel
1808 * @priv: driver private structure
1809 * @chan: RX channel index
1810 * Description:
1811 * This starts a RX DMA channel
1812 */
stmmac_start_rx_dma(struct stmmac_priv * priv,u32 chan)1813 static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan)
1814 {
1815 netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan);
1816 stmmac_start_rx(priv, priv->ioaddr, chan);
1817 }
1818
1819 /**
1820 * stmmac_start_tx_dma - start TX DMA channel
1821 * @priv: driver private structure
1822 * @chan: TX channel index
1823 * Description:
1824 * This starts a TX DMA channel
1825 */
stmmac_start_tx_dma(struct stmmac_priv * priv,u32 chan)1826 static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan)
1827 {
1828 netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan);
1829 stmmac_start_tx(priv, priv->ioaddr, chan);
1830 }
1831
1832 /**
1833 * stmmac_stop_rx_dma - stop RX DMA channel
1834 * @priv: driver private structure
1835 * @chan: RX channel index
1836 * Description:
1837 * This stops a RX DMA channel
1838 */
stmmac_stop_rx_dma(struct stmmac_priv * priv,u32 chan)1839 static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan)
1840 {
1841 netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan);
1842 stmmac_stop_rx(priv, priv->ioaddr, chan);
1843 }
1844
1845 /**
1846 * stmmac_stop_tx_dma - stop TX DMA channel
1847 * @priv: driver private structure
1848 * @chan: TX channel index
1849 * Description:
1850 * This stops a TX DMA channel
1851 */
stmmac_stop_tx_dma(struct stmmac_priv * priv,u32 chan)1852 static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan)
1853 {
1854 netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan);
1855 stmmac_stop_tx(priv, priv->ioaddr, chan);
1856 }
1857
1858 /**
1859 * stmmac_start_all_dma - start all RX and TX DMA channels
1860 * @priv: driver private structure
1861 * Description:
1862 * This starts all the RX and TX DMA channels
1863 */
stmmac_start_all_dma(struct stmmac_priv * priv)1864 static void stmmac_start_all_dma(struct stmmac_priv *priv)
1865 {
1866 u32 rx_channels_count = priv->plat->rx_queues_to_use;
1867 u32 tx_channels_count = priv->plat->tx_queues_to_use;
1868 u32 chan = 0;
1869
1870 for (chan = 0; chan < rx_channels_count; chan++)
1871 stmmac_start_rx_dma(priv, chan);
1872
1873 for (chan = 0; chan < tx_channels_count; chan++)
1874 stmmac_start_tx_dma(priv, chan);
1875 }
1876
1877 /**
1878 * stmmac_stop_all_dma - stop all RX and TX DMA channels
1879 * @priv: driver private structure
1880 * Description:
1881 * This stops the RX and TX DMA channels
1882 */
stmmac_stop_all_dma(struct stmmac_priv * priv)1883 static void stmmac_stop_all_dma(struct stmmac_priv *priv)
1884 {
1885 u32 rx_channels_count = priv->plat->rx_queues_to_use;
1886 u32 tx_channels_count = priv->plat->tx_queues_to_use;
1887 u32 chan = 0;
1888
1889 for (chan = 0; chan < rx_channels_count; chan++)
1890 stmmac_stop_rx_dma(priv, chan);
1891
1892 for (chan = 0; chan < tx_channels_count; chan++)
1893 stmmac_stop_tx_dma(priv, chan);
1894 }
1895
1896 /**
1897 * stmmac_dma_operation_mode - HW DMA operation mode
1898 * @priv: driver private structure
1899 * Description: it is used for configuring the DMA operation mode register in
1900 * order to program the tx/rx DMA thresholds or Store-And-Forward mode.
1901 */
stmmac_dma_operation_mode(struct stmmac_priv * priv)1902 static void stmmac_dma_operation_mode(struct stmmac_priv *priv)
1903 {
1904 u32 rx_channels_count = priv->plat->rx_queues_to_use;
1905 u32 tx_channels_count = priv->plat->tx_queues_to_use;
1906 int rxfifosz = priv->plat->rx_fifo_size;
1907 int txfifosz = priv->plat->tx_fifo_size;
1908 u32 txmode = 0;
1909 u32 rxmode = 0;
1910 u32 chan = 0;
1911 u8 qmode = 0;
1912
1913 if (rxfifosz == 0)
1914 rxfifosz = priv->dma_cap.rx_fifo_size;
1915 if (txfifosz == 0)
1916 txfifosz = priv->dma_cap.tx_fifo_size;
1917
1918 /* Adjust for real per queue fifo size */
1919 rxfifosz /= rx_channels_count;
1920 txfifosz /= tx_channels_count;
1921
1922 if (priv->plat->force_thresh_dma_mode) {
1923 txmode = tc;
1924 rxmode = tc;
1925 } else if (priv->plat->force_sf_dma_mode || priv->plat->tx_coe) {
1926 /*
1927 * In case of GMAC, SF mode can be enabled
1928 * to perform the TX COE in HW. This depends on:
1929 * 1) TX COE if actually supported
1930 * 2) There is no bugged Jumbo frame support
1931 * that needs to not insert csum in the TDES.
1932 */
1933 txmode = SF_DMA_MODE;
1934 rxmode = SF_DMA_MODE;
1935 priv->xstats.threshold = SF_DMA_MODE;
1936 } else {
1937 txmode = tc;
1938 rxmode = SF_DMA_MODE;
1939 }
1940
1941 /* configure all channels */
1942 for (chan = 0; chan < rx_channels_count; chan++) {
1943 qmode = priv->plat->rx_queues_cfg[chan].mode_to_use;
1944
1945 stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan,
1946 rxfifosz, qmode);
1947 stmmac_set_dma_bfsize(priv, priv->ioaddr, priv->dma_buf_sz,
1948 chan);
1949 }
1950
1951 for (chan = 0; chan < tx_channels_count; chan++) {
1952 qmode = priv->plat->tx_queues_cfg[chan].mode_to_use;
1953
1954 stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan,
1955 txfifosz, qmode);
1956 }
1957 }
1958
1959 /**
1960 * stmmac_tx_clean - to manage the transmission completion
1961 * @priv: driver private structure
1962 * @budget: napi budget limiting this functions packet handling
1963 * @queue: TX queue index
1964 * Description: it reclaims the transmit resources after transmission completes.
1965 */
stmmac_tx_clean(struct stmmac_priv * priv,int budget,u32 queue)1966 static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue)
1967 {
1968 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1969 unsigned int bytes_compl = 0, pkts_compl = 0;
1970 unsigned int entry, count = 0;
1971
1972 __netif_tx_lock_bh(netdev_get_tx_queue(priv->dev, queue));
1973
1974 priv->xstats.tx_clean++;
1975
1976 entry = tx_q->dirty_tx;
1977 while ((entry != tx_q->cur_tx) && (count < budget)) {
1978 struct sk_buff *skb = tx_q->tx_skbuff[entry];
1979 struct dma_desc *p;
1980 int status;
1981
1982 if (priv->extend_desc)
1983 p = (struct dma_desc *)(tx_q->dma_etx + entry);
1984 else if (tx_q->tbs & STMMAC_TBS_AVAIL)
1985 p = &tx_q->dma_entx[entry].basic;
1986 else
1987 p = tx_q->dma_tx + entry;
1988
1989 status = stmmac_tx_status(priv, &priv->dev->stats,
1990 &priv->xstats, p, priv->ioaddr);
1991 /* Check if the descriptor is owned by the DMA */
1992 if (unlikely(status & tx_dma_own))
1993 break;
1994
1995 count++;
1996
1997 /* Make sure descriptor fields are read after reading
1998 * the own bit.
1999 */
2000 dma_rmb();
2001
2002 /* Just consider the last segment and ...*/
2003 if (likely(!(status & tx_not_ls))) {
2004 /* ... verify the status error condition */
2005 if (unlikely(status & tx_err)) {
2006 priv->dev->stats.tx_errors++;
2007 } else {
2008 priv->dev->stats.tx_packets++;
2009 priv->xstats.tx_pkt_n++;
2010 }
2011 stmmac_get_tx_hwtstamp(priv, p, skb);
2012 }
2013
2014 if (likely(tx_q->tx_skbuff_dma[entry].buf)) {
2015 if (tx_q->tx_skbuff_dma[entry].map_as_page)
2016 dma_unmap_page(priv->device,
2017 tx_q->tx_skbuff_dma[entry].buf,
2018 tx_q->tx_skbuff_dma[entry].len,
2019 DMA_TO_DEVICE);
2020 else
2021 dma_unmap_single(priv->device,
2022 tx_q->tx_skbuff_dma[entry].buf,
2023 tx_q->tx_skbuff_dma[entry].len,
2024 DMA_TO_DEVICE);
2025 tx_q->tx_skbuff_dma[entry].buf = 0;
2026 tx_q->tx_skbuff_dma[entry].len = 0;
2027 tx_q->tx_skbuff_dma[entry].map_as_page = false;
2028 }
2029
2030 stmmac_clean_desc3(priv, tx_q, p);
2031
2032 tx_q->tx_skbuff_dma[entry].last_segment = false;
2033 tx_q->tx_skbuff_dma[entry].is_jumbo = false;
2034
2035 if (likely(skb != NULL)) {
2036 pkts_compl++;
2037 bytes_compl += skb->len;
2038 dev_consume_skb_any(skb);
2039 tx_q->tx_skbuff[entry] = NULL;
2040 }
2041
2042 stmmac_release_tx_desc(priv, p, priv->mode);
2043
2044 entry = STMMAC_GET_ENTRY(entry, priv->dma_tx_size);
2045 }
2046 tx_q->dirty_tx = entry;
2047
2048 netdev_tx_completed_queue(netdev_get_tx_queue(priv->dev, queue),
2049 pkts_compl, bytes_compl);
2050
2051 if (unlikely(netif_tx_queue_stopped(netdev_get_tx_queue(priv->dev,
2052 queue))) &&
2053 stmmac_tx_avail(priv, queue) > STMMAC_TX_THRESH(priv)) {
2054
2055 netif_dbg(priv, tx_done, priv->dev,
2056 "%s: restart transmit\n", __func__);
2057 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, queue));
2058 }
2059
2060 if ((priv->eee_enabled) && (!priv->tx_path_in_lpi_mode)) {
2061 stmmac_enable_eee_mode(priv);
2062 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer));
2063 }
2064
2065 /* We still have pending packets, let's call for a new scheduling */
2066 if (tx_q->dirty_tx != tx_q->cur_tx)
2067 mod_timer(&tx_q->txtimer, STMMAC_COAL_TIMER(priv->tx_coal_timer));
2068
2069 __netif_tx_unlock_bh(netdev_get_tx_queue(priv->dev, queue));
2070
2071 return count;
2072 }
2073
2074 /**
2075 * stmmac_tx_err - to manage the tx error
2076 * @priv: driver private structure
2077 * @chan: channel index
2078 * Description: it cleans the descriptors and restarts the transmission
2079 * in case of transmission errors.
2080 */
stmmac_tx_err(struct stmmac_priv * priv,u32 chan)2081 static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan)
2082 {
2083 struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan];
2084
2085 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, chan));
2086
2087 stmmac_stop_tx_dma(priv, chan);
2088 dma_free_tx_skbufs(priv, chan);
2089 stmmac_clear_tx_descriptors(priv, chan);
2090 tx_q->dirty_tx = 0;
2091 tx_q->cur_tx = 0;
2092 tx_q->mss = 0;
2093 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, chan));
2094 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
2095 tx_q->dma_tx_phy, chan);
2096 stmmac_start_tx_dma(priv, chan);
2097
2098 priv->dev->stats.tx_errors++;
2099 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, chan));
2100 }
2101
2102 /**
2103 * stmmac_set_dma_operation_mode - Set DMA operation mode by channel
2104 * @priv: driver private structure
2105 * @txmode: TX operating mode
2106 * @rxmode: RX operating mode
2107 * @chan: channel index
2108 * Description: it is used for configuring of the DMA operation mode in
2109 * runtime in order to program the tx/rx DMA thresholds or Store-And-Forward
2110 * mode.
2111 */
stmmac_set_dma_operation_mode(struct stmmac_priv * priv,u32 txmode,u32 rxmode,u32 chan)2112 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode,
2113 u32 rxmode, u32 chan)
2114 {
2115 u8 rxqmode = priv->plat->rx_queues_cfg[chan].mode_to_use;
2116 u8 txqmode = priv->plat->tx_queues_cfg[chan].mode_to_use;
2117 u32 rx_channels_count = priv->plat->rx_queues_to_use;
2118 u32 tx_channels_count = priv->plat->tx_queues_to_use;
2119 int rxfifosz = priv->plat->rx_fifo_size;
2120 int txfifosz = priv->plat->tx_fifo_size;
2121
2122 if (rxfifosz == 0)
2123 rxfifosz = priv->dma_cap.rx_fifo_size;
2124 if (txfifosz == 0)
2125 txfifosz = priv->dma_cap.tx_fifo_size;
2126
2127 /* Adjust for real per queue fifo size */
2128 rxfifosz /= rx_channels_count;
2129 txfifosz /= tx_channels_count;
2130
2131 stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, rxfifosz, rxqmode);
2132 stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, txfifosz, txqmode);
2133 }
2134
stmmac_safety_feat_interrupt(struct stmmac_priv * priv)2135 static bool stmmac_safety_feat_interrupt(struct stmmac_priv *priv)
2136 {
2137 int ret;
2138
2139 ret = stmmac_safety_feat_irq_status(priv, priv->dev,
2140 priv->ioaddr, priv->dma_cap.asp, &priv->sstats);
2141 if (ret && (ret != -EINVAL)) {
2142 stmmac_global_err(priv);
2143 return true;
2144 }
2145
2146 return false;
2147 }
2148
stmmac_napi_check(struct stmmac_priv * priv,u32 chan)2149 static int stmmac_napi_check(struct stmmac_priv *priv, u32 chan)
2150 {
2151 int status = stmmac_dma_interrupt_status(priv, priv->ioaddr,
2152 &priv->xstats, chan);
2153 struct stmmac_channel *ch = &priv->channel[chan];
2154 unsigned long flags;
2155
2156 if ((status & handle_rx) && (chan < priv->plat->rx_queues_to_use)) {
2157 if (napi_schedule_prep(&ch->rx_napi)) {
2158 spin_lock_irqsave(&ch->lock, flags);
2159 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 0);
2160 spin_unlock_irqrestore(&ch->lock, flags);
2161 __napi_schedule_irqoff(&ch->rx_napi);
2162 }
2163 }
2164
2165 if ((status & handle_tx) && (chan < priv->plat->tx_queues_to_use)) {
2166 if (napi_schedule_prep(&ch->tx_napi)) {
2167 spin_lock_irqsave(&ch->lock, flags);
2168 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 0, 1);
2169 spin_unlock_irqrestore(&ch->lock, flags);
2170 __napi_schedule_irqoff(&ch->tx_napi);
2171 }
2172 }
2173
2174 return status;
2175 }
2176
2177 /**
2178 * stmmac_dma_interrupt - DMA ISR
2179 * @priv: driver private structure
2180 * Description: this is the DMA ISR. It is called by the main ISR.
2181 * It calls the dwmac dma routine and schedule poll method in case of some
2182 * work can be done.
2183 */
stmmac_dma_interrupt(struct stmmac_priv * priv)2184 static void stmmac_dma_interrupt(struct stmmac_priv *priv)
2185 {
2186 u32 tx_channel_count = priv->plat->tx_queues_to_use;
2187 u32 rx_channel_count = priv->plat->rx_queues_to_use;
2188 u32 channels_to_check = tx_channel_count > rx_channel_count ?
2189 tx_channel_count : rx_channel_count;
2190 u32 chan;
2191 int status[max_t(u32, MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES)];
2192
2193 /* Make sure we never check beyond our status buffer. */
2194 if (WARN_ON_ONCE(channels_to_check > ARRAY_SIZE(status)))
2195 channels_to_check = ARRAY_SIZE(status);
2196
2197 for (chan = 0; chan < channels_to_check; chan++)
2198 status[chan] = stmmac_napi_check(priv, chan);
2199
2200 for (chan = 0; chan < tx_channel_count; chan++) {
2201 if (unlikely(status[chan] & tx_hard_error_bump_tc)) {
2202 /* Try to bump up the dma threshold on this failure */
2203 if (unlikely(priv->xstats.threshold != SF_DMA_MODE) &&
2204 (tc <= 256)) {
2205 tc += 64;
2206 if (priv->plat->force_thresh_dma_mode)
2207 stmmac_set_dma_operation_mode(priv,
2208 tc,
2209 tc,
2210 chan);
2211 else
2212 stmmac_set_dma_operation_mode(priv,
2213 tc,
2214 SF_DMA_MODE,
2215 chan);
2216 priv->xstats.threshold = tc;
2217 }
2218 } else if (unlikely(status[chan] == tx_hard_error)) {
2219 stmmac_tx_err(priv, chan);
2220 }
2221 }
2222 }
2223
2224 /**
2225 * stmmac_mmc_setup: setup the Mac Management Counters (MMC)
2226 * @priv: driver private structure
2227 * Description: this masks the MMC irq, in fact, the counters are managed in SW.
2228 */
stmmac_mmc_setup(struct stmmac_priv * priv)2229 static void stmmac_mmc_setup(struct stmmac_priv *priv)
2230 {
2231 unsigned int mode = MMC_CNTRL_RESET_ON_READ | MMC_CNTRL_COUNTER_RESET |
2232 MMC_CNTRL_PRESET | MMC_CNTRL_FULL_HALF_PRESET;
2233
2234 stmmac_mmc_intr_all_mask(priv, priv->mmcaddr);
2235
2236 if (priv->dma_cap.rmon) {
2237 stmmac_mmc_ctrl(priv, priv->mmcaddr, mode);
2238 memset(&priv->mmc, 0, sizeof(struct stmmac_counters));
2239 } else
2240 netdev_info(priv->dev, "No MAC Management Counters available\n");
2241 }
2242
2243 /**
2244 * stmmac_get_hw_features - get MAC capabilities from the HW cap. register.
2245 * @priv: driver private structure
2246 * Description:
2247 * new GMAC chip generations have a new register to indicate the
2248 * presence of the optional feature/functions.
2249 * This can be also used to override the value passed through the
2250 * platform and necessary for old MAC10/100 and GMAC chips.
2251 */
stmmac_get_hw_features(struct stmmac_priv * priv)2252 static int stmmac_get_hw_features(struct stmmac_priv *priv)
2253 {
2254 return stmmac_get_hw_feature(priv, priv->ioaddr, &priv->dma_cap) == 0;
2255 }
2256
2257 /**
2258 * stmmac_check_ether_addr - check if the MAC addr is valid
2259 * @priv: driver private structure
2260 * Description:
2261 * it is to verify if the MAC address is valid, in case of failures it
2262 * generates a random MAC address
2263 */
stmmac_check_ether_addr(struct stmmac_priv * priv)2264 static void stmmac_check_ether_addr(struct stmmac_priv *priv)
2265 {
2266 if (!is_valid_ether_addr(priv->dev->dev_addr)) {
2267 stmmac_get_umac_addr(priv, priv->hw, priv->dev->dev_addr, 0);
2268 if (!is_valid_ether_addr(priv->dev->dev_addr))
2269 eth_hw_addr_random(priv->dev);
2270 dev_info(priv->device, "device MAC address %pM\n",
2271 priv->dev->dev_addr);
2272 }
2273 }
2274
2275 /**
2276 * stmmac_init_dma_engine - DMA init.
2277 * @priv: driver private structure
2278 * Description:
2279 * It inits the DMA invoking the specific MAC/GMAC callback.
2280 * Some DMA parameters can be passed from the platform;
2281 * in case of these are not passed a default is kept for the MAC or GMAC.
2282 */
stmmac_init_dma_engine(struct stmmac_priv * priv)2283 static int stmmac_init_dma_engine(struct stmmac_priv *priv)
2284 {
2285 u32 rx_channels_count = priv->plat->rx_queues_to_use;
2286 u32 tx_channels_count = priv->plat->tx_queues_to_use;
2287 u32 dma_csr_ch = max(rx_channels_count, tx_channels_count);
2288 struct stmmac_rx_queue *rx_q;
2289 struct stmmac_tx_queue *tx_q;
2290 u32 chan = 0;
2291 int atds = 0;
2292 int ret = 0;
2293
2294 if (!priv->plat->dma_cfg || !priv->plat->dma_cfg->pbl) {
2295 dev_err(priv->device, "Invalid DMA configuration\n");
2296 return -EINVAL;
2297 }
2298
2299 if (priv->extend_desc && (priv->mode == STMMAC_RING_MODE))
2300 atds = 1;
2301
2302 ret = stmmac_reset(priv, priv->ioaddr);
2303 if (ret) {
2304 dev_err(priv->device, "Failed to reset the dma\n");
2305 return ret;
2306 }
2307
2308 /* DMA Configuration */
2309 stmmac_dma_init(priv, priv->ioaddr, priv->plat->dma_cfg, atds);
2310
2311 if (priv->plat->axi)
2312 stmmac_axi(priv, priv->ioaddr, priv->plat->axi);
2313
2314 /* DMA CSR Channel configuration */
2315 for (chan = 0; chan < dma_csr_ch; chan++)
2316 stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan);
2317
2318 /* DMA RX Channel Configuration */
2319 for (chan = 0; chan < rx_channels_count; chan++) {
2320 rx_q = &priv->rx_queue[chan];
2321
2322 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
2323 rx_q->dma_rx_phy, chan);
2324
2325 rx_q->rx_tail_addr = rx_q->dma_rx_phy +
2326 (priv->dma_rx_size *
2327 sizeof(struct dma_desc));
2328 stmmac_set_rx_tail_ptr(priv, priv->ioaddr,
2329 rx_q->rx_tail_addr, chan);
2330 }
2331
2332 /* DMA TX Channel Configuration */
2333 for (chan = 0; chan < tx_channels_count; chan++) {
2334 tx_q = &priv->tx_queue[chan];
2335
2336 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
2337 tx_q->dma_tx_phy, chan);
2338
2339 tx_q->tx_tail_addr = tx_q->dma_tx_phy;
2340 stmmac_set_tx_tail_ptr(priv, priv->ioaddr,
2341 tx_q->tx_tail_addr, chan);
2342 }
2343
2344 return ret;
2345 }
2346
stmmac_tx_timer_arm(struct stmmac_priv * priv,u32 queue)2347 static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue)
2348 {
2349 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
2350
2351 mod_timer(&tx_q->txtimer, STMMAC_COAL_TIMER(priv->tx_coal_timer));
2352 }
2353
2354 /**
2355 * stmmac_tx_timer - mitigation sw timer for tx.
2356 * @t: data pointer
2357 * Description:
2358 * This is the timer handler to directly invoke the stmmac_tx_clean.
2359 */
stmmac_tx_timer(struct timer_list * t)2360 static void stmmac_tx_timer(struct timer_list *t)
2361 {
2362 struct stmmac_tx_queue *tx_q = from_timer(tx_q, t, txtimer);
2363 struct stmmac_priv *priv = tx_q->priv_data;
2364 struct stmmac_channel *ch;
2365
2366 ch = &priv->channel[tx_q->queue_index];
2367
2368 if (likely(napi_schedule_prep(&ch->tx_napi))) {
2369 unsigned long flags;
2370
2371 spin_lock_irqsave(&ch->lock, flags);
2372 stmmac_disable_dma_irq(priv, priv->ioaddr, ch->index, 0, 1);
2373 spin_unlock_irqrestore(&ch->lock, flags);
2374 __napi_schedule(&ch->tx_napi);
2375 }
2376 }
2377
2378 /**
2379 * stmmac_init_coalesce - init mitigation options.
2380 * @priv: driver private structure
2381 * Description:
2382 * This inits the coalesce parameters: i.e. timer rate,
2383 * timer handler and default threshold used for enabling the
2384 * interrupt on completion bit.
2385 */
stmmac_init_coalesce(struct stmmac_priv * priv)2386 static void stmmac_init_coalesce(struct stmmac_priv *priv)
2387 {
2388 u32 tx_channel_count = priv->plat->tx_queues_to_use;
2389 u32 chan;
2390
2391 priv->tx_coal_frames = STMMAC_TX_FRAMES;
2392 priv->tx_coal_timer = STMMAC_COAL_TX_TIMER;
2393 priv->rx_coal_frames = STMMAC_RX_FRAMES;
2394
2395 for (chan = 0; chan < tx_channel_count; chan++) {
2396 struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan];
2397
2398 timer_setup(&tx_q->txtimer, stmmac_tx_timer, 0);
2399 }
2400 }
2401
stmmac_set_rings_length(struct stmmac_priv * priv)2402 static void stmmac_set_rings_length(struct stmmac_priv *priv)
2403 {
2404 u32 rx_channels_count = priv->plat->rx_queues_to_use;
2405 u32 tx_channels_count = priv->plat->tx_queues_to_use;
2406 u32 chan;
2407
2408 /* set TX ring length */
2409 for (chan = 0; chan < tx_channels_count; chan++)
2410 stmmac_set_tx_ring_len(priv, priv->ioaddr,
2411 (priv->dma_tx_size - 1), chan);
2412
2413 /* set RX ring length */
2414 for (chan = 0; chan < rx_channels_count; chan++)
2415 stmmac_set_rx_ring_len(priv, priv->ioaddr,
2416 (priv->dma_rx_size - 1), chan);
2417 }
2418
2419 /**
2420 * stmmac_set_tx_queue_weight - Set TX queue weight
2421 * @priv: driver private structure
2422 * Description: It is used for setting TX queues weight
2423 */
stmmac_set_tx_queue_weight(struct stmmac_priv * priv)2424 static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv)
2425 {
2426 u32 tx_queues_count = priv->plat->tx_queues_to_use;
2427 u32 weight;
2428 u32 queue;
2429
2430 for (queue = 0; queue < tx_queues_count; queue++) {
2431 weight = priv->plat->tx_queues_cfg[queue].weight;
2432 stmmac_set_mtl_tx_queue_weight(priv, priv->hw, weight, queue);
2433 }
2434 }
2435
2436 /**
2437 * stmmac_configure_cbs - Configure CBS in TX queue
2438 * @priv: driver private structure
2439 * Description: It is used for configuring CBS in AVB TX queues
2440 */
stmmac_configure_cbs(struct stmmac_priv * priv)2441 static void stmmac_configure_cbs(struct stmmac_priv *priv)
2442 {
2443 u32 tx_queues_count = priv->plat->tx_queues_to_use;
2444 u32 mode_to_use;
2445 u32 queue;
2446
2447 /* queue 0 is reserved for legacy traffic */
2448 for (queue = 1; queue < tx_queues_count; queue++) {
2449 mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use;
2450 if (mode_to_use == MTL_QUEUE_DCB)
2451 continue;
2452
2453 stmmac_config_cbs(priv, priv->hw,
2454 priv->plat->tx_queues_cfg[queue].send_slope,
2455 priv->plat->tx_queues_cfg[queue].idle_slope,
2456 priv->plat->tx_queues_cfg[queue].high_credit,
2457 priv->plat->tx_queues_cfg[queue].low_credit,
2458 queue);
2459 }
2460 }
2461
2462 /**
2463 * stmmac_rx_queue_dma_chan_map - Map RX queue to RX dma channel
2464 * @priv: driver private structure
2465 * Description: It is used for mapping RX queues to RX dma channels
2466 */
stmmac_rx_queue_dma_chan_map(struct stmmac_priv * priv)2467 static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv)
2468 {
2469 u32 rx_queues_count = priv->plat->rx_queues_to_use;
2470 u32 queue;
2471 u32 chan;
2472
2473 for (queue = 0; queue < rx_queues_count; queue++) {
2474 chan = priv->plat->rx_queues_cfg[queue].chan;
2475 stmmac_map_mtl_to_dma(priv, priv->hw, queue, chan);
2476 }
2477 }
2478
2479 /**
2480 * stmmac_mac_config_rx_queues_prio - Configure RX Queue priority
2481 * @priv: driver private structure
2482 * Description: It is used for configuring the RX Queue Priority
2483 */
stmmac_mac_config_rx_queues_prio(struct stmmac_priv * priv)2484 static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv)
2485 {
2486 u32 rx_queues_count = priv->plat->rx_queues_to_use;
2487 u32 queue;
2488 u32 prio;
2489
2490 for (queue = 0; queue < rx_queues_count; queue++) {
2491 if (!priv->plat->rx_queues_cfg[queue].use_prio)
2492 continue;
2493
2494 prio = priv->plat->rx_queues_cfg[queue].prio;
2495 stmmac_rx_queue_prio(priv, priv->hw, prio, queue);
2496 }
2497 }
2498
2499 /**
2500 * stmmac_mac_config_tx_queues_prio - Configure TX Queue priority
2501 * @priv: driver private structure
2502 * Description: It is used for configuring the TX Queue Priority
2503 */
stmmac_mac_config_tx_queues_prio(struct stmmac_priv * priv)2504 static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv)
2505 {
2506 u32 tx_queues_count = priv->plat->tx_queues_to_use;
2507 u32 queue;
2508 u32 prio;
2509
2510 for (queue = 0; queue < tx_queues_count; queue++) {
2511 if (!priv->plat->tx_queues_cfg[queue].use_prio)
2512 continue;
2513
2514 prio = priv->plat->tx_queues_cfg[queue].prio;
2515 stmmac_tx_queue_prio(priv, priv->hw, prio, queue);
2516 }
2517 }
2518
2519 /**
2520 * stmmac_mac_config_rx_queues_routing - Configure RX Queue Routing
2521 * @priv: driver private structure
2522 * Description: It is used for configuring the RX queue routing
2523 */
stmmac_mac_config_rx_queues_routing(struct stmmac_priv * priv)2524 static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv)
2525 {
2526 u32 rx_queues_count = priv->plat->rx_queues_to_use;
2527 u32 queue;
2528 u8 packet;
2529
2530 for (queue = 0; queue < rx_queues_count; queue++) {
2531 /* no specific packet type routing specified for the queue */
2532 if (priv->plat->rx_queues_cfg[queue].pkt_route == 0x0)
2533 continue;
2534
2535 packet = priv->plat->rx_queues_cfg[queue].pkt_route;
2536 stmmac_rx_queue_routing(priv, priv->hw, packet, queue);
2537 }
2538 }
2539
stmmac_mac_config_rss(struct stmmac_priv * priv)2540 static void stmmac_mac_config_rss(struct stmmac_priv *priv)
2541 {
2542 if (!priv->dma_cap.rssen || !priv->plat->rss_en) {
2543 priv->rss.enable = false;
2544 return;
2545 }
2546
2547 if (priv->dev->features & NETIF_F_RXHASH)
2548 priv->rss.enable = true;
2549 else
2550 priv->rss.enable = false;
2551
2552 stmmac_rss_configure(priv, priv->hw, &priv->rss,
2553 priv->plat->rx_queues_to_use);
2554 }
2555
2556 /**
2557 * stmmac_mtl_configuration - Configure MTL
2558 * @priv: driver private structure
2559 * Description: It is used for configurring MTL
2560 */
stmmac_mtl_configuration(struct stmmac_priv * priv)2561 static void stmmac_mtl_configuration(struct stmmac_priv *priv)
2562 {
2563 u32 rx_queues_count = priv->plat->rx_queues_to_use;
2564 u32 tx_queues_count = priv->plat->tx_queues_to_use;
2565
2566 if (tx_queues_count > 1)
2567 stmmac_set_tx_queue_weight(priv);
2568
2569 /* Configure MTL RX algorithms */
2570 if (rx_queues_count > 1)
2571 stmmac_prog_mtl_rx_algorithms(priv, priv->hw,
2572 priv->plat->rx_sched_algorithm);
2573
2574 /* Configure MTL TX algorithms */
2575 if (tx_queues_count > 1)
2576 stmmac_prog_mtl_tx_algorithms(priv, priv->hw,
2577 priv->plat->tx_sched_algorithm);
2578
2579 /* Configure CBS in AVB TX queues */
2580 if (tx_queues_count > 1)
2581 stmmac_configure_cbs(priv);
2582
2583 /* Map RX MTL to DMA channels */
2584 stmmac_rx_queue_dma_chan_map(priv);
2585
2586 /* Enable MAC RX Queues */
2587 stmmac_mac_enable_rx_queues(priv);
2588
2589 /* Set RX priorities */
2590 if (rx_queues_count > 1)
2591 stmmac_mac_config_rx_queues_prio(priv);
2592
2593 /* Set TX priorities */
2594 if (tx_queues_count > 1)
2595 stmmac_mac_config_tx_queues_prio(priv);
2596
2597 /* Set RX routing */
2598 if (rx_queues_count > 1)
2599 stmmac_mac_config_rx_queues_routing(priv);
2600
2601 /* Receive Side Scaling */
2602 if (rx_queues_count > 1)
2603 stmmac_mac_config_rss(priv);
2604 }
2605
stmmac_safety_feat_configuration(struct stmmac_priv * priv)2606 static void stmmac_safety_feat_configuration(struct stmmac_priv *priv)
2607 {
2608 if (priv->dma_cap.asp) {
2609 netdev_info(priv->dev, "Enabling Safety Features\n");
2610 stmmac_safety_feat_config(priv, priv->ioaddr, priv->dma_cap.asp);
2611 } else {
2612 netdev_info(priv->dev, "No Safety Features support found\n");
2613 }
2614 }
2615
2616 /**
2617 * stmmac_hw_setup - setup mac in a usable state.
2618 * @dev : pointer to the device structure.
2619 * @init_ptp: initialize PTP if set
2620 * Description:
2621 * this is the main function to setup the HW in a usable state because the
2622 * dma engine is reset, the core registers are configured (e.g. AXI,
2623 * Checksum features, timers). The DMA is ready to start receiving and
2624 * transmitting.
2625 * Return value:
2626 * 0 on success and an appropriate (-)ve integer as defined in errno.h
2627 * file on failure.
2628 */
stmmac_hw_setup(struct net_device * dev,bool init_ptp)2629 static int stmmac_hw_setup(struct net_device *dev, bool init_ptp)
2630 {
2631 struct stmmac_priv *priv = netdev_priv(dev);
2632 u32 rx_cnt = priv->plat->rx_queues_to_use;
2633 u32 tx_cnt = priv->plat->tx_queues_to_use;
2634 u32 chan;
2635 int ret;
2636
2637 /* DMA initialization and SW reset */
2638 ret = stmmac_init_dma_engine(priv);
2639 if (ret < 0) {
2640 netdev_err(priv->dev, "%s: DMA engine initialization failed\n",
2641 __func__);
2642 return ret;
2643 }
2644
2645 /* Copy the MAC addr into the HW */
2646 stmmac_set_umac_addr(priv, priv->hw, dev->dev_addr, 0);
2647
2648 /* PS and related bits will be programmed according to the speed */
2649 if (priv->hw->pcs) {
2650 int speed = priv->plat->mac_port_sel_speed;
2651
2652 if ((speed == SPEED_10) || (speed == SPEED_100) ||
2653 (speed == SPEED_1000)) {
2654 priv->hw->ps = speed;
2655 } else {
2656 dev_warn(priv->device, "invalid port speed\n");
2657 priv->hw->ps = 0;
2658 }
2659 }
2660
2661 /* Initialize the MAC Core */
2662 stmmac_core_init(priv, priv->hw, dev);
2663
2664 /* Initialize MTL*/
2665 stmmac_mtl_configuration(priv);
2666
2667 /* Initialize Safety Features */
2668 stmmac_safety_feat_configuration(priv);
2669
2670 ret = stmmac_rx_ipc(priv, priv->hw);
2671 if (!ret) {
2672 netdev_warn(priv->dev, "RX IPC Checksum Offload disabled\n");
2673 priv->plat->rx_coe = STMMAC_RX_COE_NONE;
2674 priv->hw->rx_csum = 0;
2675 }
2676
2677 /* Enable the MAC Rx/Tx */
2678 stmmac_mac_set(priv, priv->ioaddr, true);
2679
2680 /* Set the HW DMA mode and the COE */
2681 stmmac_dma_operation_mode(priv);
2682
2683 stmmac_mmc_setup(priv);
2684
2685 if (init_ptp) {
2686 ret = clk_prepare_enable(priv->plat->clk_ptp_ref);
2687 if (ret < 0)
2688 netdev_warn(priv->dev, "failed to enable PTP reference clock: %d\n", ret);
2689
2690 ret = stmmac_init_ptp(priv);
2691 if (ret == -EOPNOTSUPP)
2692 netdev_warn(priv->dev, "PTP not supported by HW\n");
2693 else if (ret)
2694 netdev_warn(priv->dev, "PTP init failed\n");
2695 }
2696
2697 priv->eee_tw_timer = STMMAC_DEFAULT_TWT_LS;
2698
2699 /* Convert the timer from msec to usec */
2700 if (!priv->tx_lpi_timer)
2701 priv->tx_lpi_timer = eee_timer * 1000;
2702
2703 if (priv->use_riwt) {
2704 if (!priv->rx_riwt)
2705 priv->rx_riwt = DEF_DMA_RIWT;
2706
2707 ret = stmmac_rx_watchdog(priv, priv->ioaddr, priv->rx_riwt, rx_cnt);
2708 }
2709
2710 if (priv->hw->pcs)
2711 stmmac_pcs_ctrl_ane(priv, priv->ioaddr, 1, priv->hw->ps, 0);
2712
2713 /* set TX and RX rings length */
2714 stmmac_set_rings_length(priv);
2715
2716 /* Enable TSO */
2717 if (priv->tso) {
2718 for (chan = 0; chan < tx_cnt; chan++)
2719 stmmac_enable_tso(priv, priv->ioaddr, 1, chan);
2720 }
2721
2722 /* Enable Split Header */
2723 if (priv->sph && priv->hw->rx_csum) {
2724 for (chan = 0; chan < rx_cnt; chan++)
2725 stmmac_enable_sph(priv, priv->ioaddr, 1, chan);
2726 }
2727
2728 /* VLAN Tag Insertion */
2729 if (priv->dma_cap.vlins)
2730 stmmac_enable_vlan(priv, priv->hw, STMMAC_VLAN_INSERT);
2731
2732 /* TBS */
2733 for (chan = 0; chan < tx_cnt; chan++) {
2734 struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan];
2735 int enable = tx_q->tbs & STMMAC_TBS_AVAIL;
2736
2737 stmmac_enable_tbs(priv, priv->ioaddr, enable, chan);
2738 }
2739
2740 /* Configure real RX and TX queues */
2741 netif_set_real_num_rx_queues(dev, priv->plat->rx_queues_to_use);
2742 netif_set_real_num_tx_queues(dev, priv->plat->tx_queues_to_use);
2743
2744 /* Start the ball rolling... */
2745 stmmac_start_all_dma(priv);
2746
2747 return 0;
2748 }
2749
stmmac_hw_teardown(struct net_device * dev)2750 static void stmmac_hw_teardown(struct net_device *dev)
2751 {
2752 struct stmmac_priv *priv = netdev_priv(dev);
2753
2754 clk_disable_unprepare(priv->plat->clk_ptp_ref);
2755 }
2756
2757 /**
2758 * stmmac_open - open entry point of the driver
2759 * @dev : pointer to the device structure.
2760 * Description:
2761 * This function is the open entry point of the driver.
2762 * Return value:
2763 * 0 on success and an appropriate (-)ve integer as defined in errno.h
2764 * file on failure.
2765 */
stmmac_open(struct net_device * dev)2766 static int stmmac_open(struct net_device *dev)
2767 {
2768 struct stmmac_priv *priv = netdev_priv(dev);
2769 int bfsize = 0;
2770 u32 chan;
2771 int ret;
2772
2773 if (priv->hw->pcs != STMMAC_PCS_TBI &&
2774 priv->hw->pcs != STMMAC_PCS_RTBI &&
2775 priv->hw->xpcs == NULL) {
2776 ret = stmmac_init_phy(dev);
2777 if (ret) {
2778 netdev_err(priv->dev,
2779 "%s: Cannot attach to PHY (error: %d)\n",
2780 __func__, ret);
2781 return ret;
2782 }
2783 }
2784
2785 /* Extra statistics */
2786 memset(&priv->xstats, 0, sizeof(struct stmmac_extra_stats));
2787 priv->xstats.threshold = tc;
2788
2789 bfsize = stmmac_set_16kib_bfsize(priv, dev->mtu);
2790 if (bfsize < 0)
2791 bfsize = 0;
2792
2793 if (bfsize < BUF_SIZE_16KiB)
2794 bfsize = stmmac_set_bfsize(dev->mtu, priv->dma_buf_sz);
2795
2796 priv->dma_buf_sz = bfsize;
2797 buf_sz = bfsize;
2798
2799 priv->rx_copybreak = STMMAC_RX_COPYBREAK;
2800
2801 if (!priv->dma_tx_size)
2802 priv->dma_tx_size = DMA_DEFAULT_TX_SIZE;
2803 if (!priv->dma_rx_size)
2804 priv->dma_rx_size = DMA_DEFAULT_RX_SIZE;
2805
2806 /* Earlier check for TBS */
2807 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) {
2808 struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan];
2809 int tbs_en = priv->plat->tx_queues_cfg[chan].tbs_en;
2810
2811 tx_q->tbs |= tbs_en ? STMMAC_TBS_AVAIL : 0;
2812 if (stmmac_enable_tbs(priv, priv->ioaddr, tbs_en, chan))
2813 tx_q->tbs &= ~STMMAC_TBS_AVAIL;
2814 }
2815
2816 ret = alloc_dma_desc_resources(priv);
2817 if (ret < 0) {
2818 netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n",
2819 __func__);
2820 goto dma_desc_error;
2821 }
2822
2823 ret = init_dma_desc_rings(dev, GFP_KERNEL);
2824 if (ret < 0) {
2825 netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n",
2826 __func__);
2827 goto init_error;
2828 }
2829
2830 ret = stmmac_hw_setup(dev, true);
2831 if (ret < 0) {
2832 netdev_err(priv->dev, "%s: Hw setup failed\n", __func__);
2833 goto init_error;
2834 }
2835
2836 stmmac_init_coalesce(priv);
2837
2838 phylink_start(priv->phylink);
2839 /* We may have called phylink_speed_down before */
2840 phylink_speed_up(priv->phylink);
2841
2842 /* Request the IRQ lines */
2843 ret = request_irq(dev->irq, stmmac_interrupt,
2844 IRQF_SHARED, dev->name, dev);
2845 if (unlikely(ret < 0)) {
2846 netdev_err(priv->dev,
2847 "%s: ERROR: allocating the IRQ %d (error: %d)\n",
2848 __func__, dev->irq, ret);
2849 goto irq_error;
2850 }
2851
2852 /* Request the Wake IRQ in case of another line is used for WoL */
2853 if (priv->wol_irq != dev->irq) {
2854 ret = request_irq(priv->wol_irq, stmmac_interrupt,
2855 IRQF_SHARED, dev->name, dev);
2856 if (unlikely(ret < 0)) {
2857 netdev_err(priv->dev,
2858 "%s: ERROR: allocating the WoL IRQ %d (%d)\n",
2859 __func__, priv->wol_irq, ret);
2860 goto wolirq_error;
2861 }
2862 }
2863
2864 /* Request the IRQ lines */
2865 if (priv->lpi_irq > 0) {
2866 ret = request_irq(priv->lpi_irq, stmmac_interrupt, IRQF_SHARED,
2867 dev->name, dev);
2868 if (unlikely(ret < 0)) {
2869 netdev_err(priv->dev,
2870 "%s: ERROR: allocating the LPI IRQ %d (%d)\n",
2871 __func__, priv->lpi_irq, ret);
2872 goto lpiirq_error;
2873 }
2874 }
2875
2876 stmmac_enable_all_queues(priv);
2877 netif_tx_start_all_queues(priv->dev);
2878
2879 return 0;
2880
2881 lpiirq_error:
2882 if (priv->wol_irq != dev->irq)
2883 free_irq(priv->wol_irq, dev);
2884 wolirq_error:
2885 free_irq(dev->irq, dev);
2886 irq_error:
2887 phylink_stop(priv->phylink);
2888
2889 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)
2890 del_timer_sync(&priv->tx_queue[chan].txtimer);
2891
2892 stmmac_hw_teardown(dev);
2893 init_error:
2894 free_dma_desc_resources(priv);
2895 dma_desc_error:
2896 phylink_disconnect_phy(priv->phylink);
2897 return ret;
2898 }
2899
2900 /**
2901 * stmmac_release - close entry point of the driver
2902 * @dev : device pointer.
2903 * Description:
2904 * This is the stop entry point of the driver.
2905 */
stmmac_release(struct net_device * dev)2906 static int stmmac_release(struct net_device *dev)
2907 {
2908 struct stmmac_priv *priv = netdev_priv(dev);
2909 u32 chan;
2910
2911 if (device_may_wakeup(priv->device))
2912 phylink_speed_down(priv->phylink, false);
2913 /* Stop and disconnect the PHY */
2914 phylink_stop(priv->phylink);
2915 phylink_disconnect_phy(priv->phylink);
2916
2917 stmmac_disable_all_queues(priv);
2918
2919 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)
2920 del_timer_sync(&priv->tx_queue[chan].txtimer);
2921
2922 /* Free the IRQ lines */
2923 free_irq(dev->irq, dev);
2924 if (priv->wol_irq != dev->irq)
2925 free_irq(priv->wol_irq, dev);
2926 if (priv->lpi_irq > 0)
2927 free_irq(priv->lpi_irq, dev);
2928
2929 if (priv->eee_enabled) {
2930 priv->tx_path_in_lpi_mode = false;
2931 del_timer_sync(&priv->eee_ctrl_timer);
2932 }
2933
2934 /* Stop TX/RX DMA and clear the descriptors */
2935 stmmac_stop_all_dma(priv);
2936
2937 /* Release and free the Rx/Tx resources */
2938 free_dma_desc_resources(priv);
2939
2940 /* Disable the MAC Rx/Tx */
2941 stmmac_mac_set(priv, priv->ioaddr, false);
2942
2943 netif_carrier_off(dev);
2944
2945 stmmac_release_ptp(priv);
2946
2947 return 0;
2948 }
2949
stmmac_vlan_insert(struct stmmac_priv * priv,struct sk_buff * skb,struct stmmac_tx_queue * tx_q)2950 static bool stmmac_vlan_insert(struct stmmac_priv *priv, struct sk_buff *skb,
2951 struct stmmac_tx_queue *tx_q)
2952 {
2953 u16 tag = 0x0, inner_tag = 0x0;
2954 u32 inner_type = 0x0;
2955 struct dma_desc *p;
2956
2957 if (!priv->dma_cap.vlins)
2958 return false;
2959 if (!skb_vlan_tag_present(skb))
2960 return false;
2961 if (skb->vlan_proto == htons(ETH_P_8021AD)) {
2962 inner_tag = skb_vlan_tag_get(skb);
2963 inner_type = STMMAC_VLAN_INSERT;
2964 }
2965
2966 tag = skb_vlan_tag_get(skb);
2967
2968 if (tx_q->tbs & STMMAC_TBS_AVAIL)
2969 p = &tx_q->dma_entx[tx_q->cur_tx].basic;
2970 else
2971 p = &tx_q->dma_tx[tx_q->cur_tx];
2972
2973 if (stmmac_set_desc_vlan_tag(priv, p, tag, inner_tag, inner_type))
2974 return false;
2975
2976 stmmac_set_tx_owner(priv, p);
2977 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_tx_size);
2978 return true;
2979 }
2980
2981 /**
2982 * stmmac_tso_allocator - close entry point of the driver
2983 * @priv: driver private structure
2984 * @des: buffer start address
2985 * @total_len: total length to fill in descriptors
2986 * @last_segment: condition for the last descriptor
2987 * @queue: TX queue index
2988 * Description:
2989 * This function fills descriptor and request new descriptors according to
2990 * buffer length to fill
2991 */
stmmac_tso_allocator(struct stmmac_priv * priv,dma_addr_t des,int total_len,bool last_segment,u32 queue)2992 static void stmmac_tso_allocator(struct stmmac_priv *priv, dma_addr_t des,
2993 int total_len, bool last_segment, u32 queue)
2994 {
2995 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
2996 struct dma_desc *desc;
2997 u32 buff_size;
2998 int tmp_len;
2999
3000 tmp_len = total_len;
3001
3002 while (tmp_len > 0) {
3003 dma_addr_t curr_addr;
3004
3005 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx,
3006 priv->dma_tx_size);
3007 WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]);
3008
3009 if (tx_q->tbs & STMMAC_TBS_AVAIL)
3010 desc = &tx_q->dma_entx[tx_q->cur_tx].basic;
3011 else
3012 desc = &tx_q->dma_tx[tx_q->cur_tx];
3013
3014 curr_addr = des + (total_len - tmp_len);
3015 if (priv->dma_cap.addr64 <= 32)
3016 desc->des0 = cpu_to_le32(curr_addr);
3017 else
3018 stmmac_set_desc_addr(priv, desc, curr_addr);
3019
3020 buff_size = tmp_len >= TSO_MAX_BUFF_SIZE ?
3021 TSO_MAX_BUFF_SIZE : tmp_len;
3022
3023 stmmac_prepare_tso_tx_desc(priv, desc, 0, buff_size,
3024 0, 1,
3025 (last_segment) && (tmp_len <= TSO_MAX_BUFF_SIZE),
3026 0, 0);
3027
3028 tmp_len -= TSO_MAX_BUFF_SIZE;
3029 }
3030 }
3031
3032 /**
3033 * stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO)
3034 * @skb : the socket buffer
3035 * @dev : device pointer
3036 * Description: this is the transmit function that is called on TSO frames
3037 * (support available on GMAC4 and newer chips).
3038 * Diagram below show the ring programming in case of TSO frames:
3039 *
3040 * First Descriptor
3041 * --------
3042 * | DES0 |---> buffer1 = L2/L3/L4 header
3043 * | DES1 |---> TCP Payload (can continue on next descr...)
3044 * | DES2 |---> buffer 1 and 2 len
3045 * | DES3 |---> must set TSE, TCP hdr len-> [22:19]. TCP payload len [17:0]
3046 * --------
3047 * |
3048 * ...
3049 * |
3050 * --------
3051 * | DES0 | --| Split TCP Payload on Buffers 1 and 2
3052 * | DES1 | --|
3053 * | DES2 | --> buffer 1 and 2 len
3054 * | DES3 |
3055 * --------
3056 *
3057 * mss is fixed when enable tso, so w/o programming the TDES3 ctx field.
3058 */
stmmac_tso_xmit(struct sk_buff * skb,struct net_device * dev)3059 static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev)
3060 {
3061 struct dma_desc *desc, *first, *mss_desc = NULL;
3062 struct stmmac_priv *priv = netdev_priv(dev);
3063 int desc_size, tmp_pay_len = 0, first_tx;
3064 int nfrags = skb_shinfo(skb)->nr_frags;
3065 u32 queue = skb_get_queue_mapping(skb);
3066 unsigned int first_entry, tx_packets;
3067 struct stmmac_tx_queue *tx_q;
3068 bool has_vlan, set_ic;
3069 u8 proto_hdr_len, hdr;
3070 u32 pay_len, mss;
3071 dma_addr_t des;
3072 int i;
3073
3074 tx_q = &priv->tx_queue[queue];
3075 first_tx = tx_q->cur_tx;
3076
3077 /* Compute header lengths */
3078 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) {
3079 proto_hdr_len = skb_transport_offset(skb) + sizeof(struct udphdr);
3080 hdr = sizeof(struct udphdr);
3081 } else {
3082 proto_hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
3083 hdr = tcp_hdrlen(skb);
3084 }
3085
3086 /* Desc availability based on threshold should be enough safe */
3087 if (unlikely(stmmac_tx_avail(priv, queue) <
3088 (((skb->len - proto_hdr_len) / TSO_MAX_BUFF_SIZE + 1)))) {
3089 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) {
3090 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev,
3091 queue));
3092 /* This is a hard error, log it. */
3093 netdev_err(priv->dev,
3094 "%s: Tx Ring full when queue awake\n",
3095 __func__);
3096 }
3097 return NETDEV_TX_BUSY;
3098 }
3099
3100 pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */
3101
3102 mss = skb_shinfo(skb)->gso_size;
3103
3104 /* set new MSS value if needed */
3105 if (mss != tx_q->mss) {
3106 if (tx_q->tbs & STMMAC_TBS_AVAIL)
3107 mss_desc = &tx_q->dma_entx[tx_q->cur_tx].basic;
3108 else
3109 mss_desc = &tx_q->dma_tx[tx_q->cur_tx];
3110
3111 stmmac_set_mss(priv, mss_desc, mss);
3112 tx_q->mss = mss;
3113 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx,
3114 priv->dma_tx_size);
3115 WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]);
3116 }
3117
3118 if (netif_msg_tx_queued(priv)) {
3119 pr_info("%s: hdrlen %d, hdr_len %d, pay_len %d, mss %d\n",
3120 __func__, hdr, proto_hdr_len, pay_len, mss);
3121 pr_info("\tskb->len %d, skb->data_len %d\n", skb->len,
3122 skb->data_len);
3123 }
3124
3125 /* Check if VLAN can be inserted by HW */
3126 has_vlan = stmmac_vlan_insert(priv, skb, tx_q);
3127
3128 first_entry = tx_q->cur_tx;
3129 WARN_ON(tx_q->tx_skbuff[first_entry]);
3130
3131 if (tx_q->tbs & STMMAC_TBS_AVAIL)
3132 desc = &tx_q->dma_entx[first_entry].basic;
3133 else
3134 desc = &tx_q->dma_tx[first_entry];
3135 first = desc;
3136
3137 if (has_vlan)
3138 stmmac_set_desc_vlan(priv, first, STMMAC_VLAN_INSERT);
3139
3140 /* first descriptor: fill Headers on Buf1 */
3141 des = dma_map_single(priv->device, skb->data, skb_headlen(skb),
3142 DMA_TO_DEVICE);
3143 if (dma_mapping_error(priv->device, des))
3144 goto dma_map_err;
3145
3146 tx_q->tx_skbuff_dma[first_entry].buf = des;
3147 tx_q->tx_skbuff_dma[first_entry].len = skb_headlen(skb);
3148
3149 if (priv->dma_cap.addr64 <= 32) {
3150 first->des0 = cpu_to_le32(des);
3151
3152 /* Fill start of payload in buff2 of first descriptor */
3153 if (pay_len)
3154 first->des1 = cpu_to_le32(des + proto_hdr_len);
3155
3156 /* If needed take extra descriptors to fill the remaining payload */
3157 tmp_pay_len = pay_len - TSO_MAX_BUFF_SIZE;
3158 } else {
3159 stmmac_set_desc_addr(priv, first, des);
3160 tmp_pay_len = pay_len;
3161 des += proto_hdr_len;
3162 pay_len = 0;
3163 }
3164
3165 stmmac_tso_allocator(priv, des, tmp_pay_len, (nfrags == 0), queue);
3166
3167 /* Prepare fragments */
3168 for (i = 0; i < nfrags; i++) {
3169 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3170
3171 des = skb_frag_dma_map(priv->device, frag, 0,
3172 skb_frag_size(frag),
3173 DMA_TO_DEVICE);
3174 if (dma_mapping_error(priv->device, des))
3175 goto dma_map_err;
3176
3177 stmmac_tso_allocator(priv, des, skb_frag_size(frag),
3178 (i == nfrags - 1), queue);
3179
3180 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des;
3181 tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_frag_size(frag);
3182 tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = true;
3183 }
3184
3185 tx_q->tx_skbuff_dma[tx_q->cur_tx].last_segment = true;
3186
3187 /* Only the last descriptor gets to point to the skb. */
3188 tx_q->tx_skbuff[tx_q->cur_tx] = skb;
3189
3190 /* Manage tx mitigation */
3191 tx_packets = (tx_q->cur_tx + 1) - first_tx;
3192 tx_q->tx_count_frames += tx_packets;
3193
3194 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en)
3195 set_ic = true;
3196 else if (!priv->tx_coal_frames)
3197 set_ic = false;
3198 else if (tx_packets > priv->tx_coal_frames)
3199 set_ic = true;
3200 else if ((tx_q->tx_count_frames % priv->tx_coal_frames) < tx_packets)
3201 set_ic = true;
3202 else
3203 set_ic = false;
3204
3205 if (set_ic) {
3206 if (tx_q->tbs & STMMAC_TBS_AVAIL)
3207 desc = &tx_q->dma_entx[tx_q->cur_tx].basic;
3208 else
3209 desc = &tx_q->dma_tx[tx_q->cur_tx];
3210
3211 tx_q->tx_count_frames = 0;
3212 stmmac_set_tx_ic(priv, desc);
3213 priv->xstats.tx_set_ic_bit++;
3214 }
3215
3216 /* We've used all descriptors we need for this skb, however,
3217 * advance cur_tx so that it references a fresh descriptor.
3218 * ndo_start_xmit will fill this descriptor the next time it's
3219 * called and stmmac_tx_clean may clean up to this descriptor.
3220 */
3221 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_tx_size);
3222
3223 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) {
3224 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",
3225 __func__);
3226 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
3227 }
3228
3229 dev->stats.tx_bytes += skb->len;
3230 priv->xstats.tx_tso_frames++;
3231 priv->xstats.tx_tso_nfrags += nfrags;
3232
3233 if (priv->sarc_type)
3234 stmmac_set_desc_sarc(priv, first, priv->sarc_type);
3235
3236 skb_tx_timestamp(skb);
3237
3238 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
3239 priv->hwts_tx_en)) {
3240 /* declare that device is doing timestamping */
3241 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
3242 stmmac_enable_tx_timestamp(priv, first);
3243 }
3244
3245 /* Complete the first descriptor before granting the DMA */
3246 stmmac_prepare_tso_tx_desc(priv, first, 1,
3247 proto_hdr_len,
3248 pay_len,
3249 1, tx_q->tx_skbuff_dma[first_entry].last_segment,
3250 hdr / 4, (skb->len - proto_hdr_len));
3251
3252 /* If context desc is used to change MSS */
3253 if (mss_desc) {
3254 /* Make sure that first descriptor has been completely
3255 * written, including its own bit. This is because MSS is
3256 * actually before first descriptor, so we need to make
3257 * sure that MSS's own bit is the last thing written.
3258 */
3259 dma_wmb();
3260 stmmac_set_tx_owner(priv, mss_desc);
3261 }
3262
3263 /* The own bit must be the latest setting done when prepare the
3264 * descriptor and then barrier is needed to make sure that
3265 * all is coherent before granting the DMA engine.
3266 */
3267 wmb();
3268
3269 if (netif_msg_pktdata(priv)) {
3270 pr_info("%s: curr=%d dirty=%d f=%d, e=%d, f_p=%p, nfrags %d\n",
3271 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry,
3272 tx_q->cur_tx, first, nfrags);
3273 pr_info(">>> frame to be transmitted: ");
3274 print_pkt(skb->data, skb_headlen(skb));
3275 }
3276
3277 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len);
3278
3279 if (tx_q->tbs & STMMAC_TBS_AVAIL)
3280 desc_size = sizeof(struct dma_edesc);
3281 else
3282 desc_size = sizeof(struct dma_desc);
3283
3284 tx_q->tx_tail_addr = tx_q->dma_tx_phy + (tx_q->cur_tx * desc_size);
3285 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue);
3286 stmmac_tx_timer_arm(priv, queue);
3287
3288 return NETDEV_TX_OK;
3289
3290 dma_map_err:
3291 dev_err(priv->device, "Tx dma map failed\n");
3292 dev_kfree_skb(skb);
3293 priv->dev->stats.tx_dropped++;
3294 return NETDEV_TX_OK;
3295 }
3296
3297 /**
3298 * stmmac_xmit - Tx entry point of the driver
3299 * @skb : the socket buffer
3300 * @dev : device pointer
3301 * Description : this is the tx entry point of the driver.
3302 * It programs the chain or the ring and supports oversized frames
3303 * and SG feature.
3304 */
stmmac_xmit(struct sk_buff * skb,struct net_device * dev)3305 static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev)
3306 {
3307 unsigned int first_entry, tx_packets, enh_desc;
3308 struct stmmac_priv *priv = netdev_priv(dev);
3309 unsigned int nopaged_len = skb_headlen(skb);
3310 int i, csum_insertion = 0, is_jumbo = 0;
3311 u32 queue = skb_get_queue_mapping(skb);
3312 int nfrags = skb_shinfo(skb)->nr_frags;
3313 int gso = skb_shinfo(skb)->gso_type;
3314 struct dma_edesc *tbs_desc = NULL;
3315 int entry, desc_size, first_tx;
3316 struct dma_desc *desc, *first;
3317 struct stmmac_tx_queue *tx_q;
3318 bool has_vlan, set_ic;
3319 dma_addr_t des;
3320
3321 tx_q = &priv->tx_queue[queue];
3322 first_tx = tx_q->cur_tx;
3323
3324 if (priv->tx_path_in_lpi_mode)
3325 stmmac_disable_eee_mode(priv);
3326
3327 /* Manage oversized TCP frames for GMAC4 device */
3328 if (skb_is_gso(skb) && priv->tso) {
3329 if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))
3330 return stmmac_tso_xmit(skb, dev);
3331 if (priv->plat->has_gmac4 && (gso & SKB_GSO_UDP_L4))
3332 return stmmac_tso_xmit(skb, dev);
3333 }
3334
3335 if (unlikely(stmmac_tx_avail(priv, queue) < nfrags + 1)) {
3336 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) {
3337 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev,
3338 queue));
3339 /* This is a hard error, log it. */
3340 netdev_err(priv->dev,
3341 "%s: Tx Ring full when queue awake\n",
3342 __func__);
3343 }
3344 return NETDEV_TX_BUSY;
3345 }
3346
3347 /* Check if VLAN can be inserted by HW */
3348 has_vlan = stmmac_vlan_insert(priv, skb, tx_q);
3349
3350 entry = tx_q->cur_tx;
3351 first_entry = entry;
3352 WARN_ON(tx_q->tx_skbuff[first_entry]);
3353
3354 csum_insertion = (skb->ip_summed == CHECKSUM_PARTIAL);
3355
3356 if (likely(priv->extend_desc))
3357 desc = (struct dma_desc *)(tx_q->dma_etx + entry);
3358 else if (tx_q->tbs & STMMAC_TBS_AVAIL)
3359 desc = &tx_q->dma_entx[entry].basic;
3360 else
3361 desc = tx_q->dma_tx + entry;
3362
3363 first = desc;
3364
3365 if (has_vlan)
3366 stmmac_set_desc_vlan(priv, first, STMMAC_VLAN_INSERT);
3367
3368 enh_desc = priv->plat->enh_desc;
3369 /* To program the descriptors according to the size of the frame */
3370 if (enh_desc)
3371 is_jumbo = stmmac_is_jumbo_frm(priv, skb->len, enh_desc);
3372
3373 if (unlikely(is_jumbo)) {
3374 entry = stmmac_jumbo_frm(priv, tx_q, skb, csum_insertion);
3375 if (unlikely(entry < 0) && (entry != -EINVAL))
3376 goto dma_map_err;
3377 }
3378
3379 for (i = 0; i < nfrags; i++) {
3380 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3381 int len = skb_frag_size(frag);
3382 bool last_segment = (i == (nfrags - 1));
3383
3384 entry = STMMAC_GET_ENTRY(entry, priv->dma_tx_size);
3385 WARN_ON(tx_q->tx_skbuff[entry]);
3386
3387 if (likely(priv->extend_desc))
3388 desc = (struct dma_desc *)(tx_q->dma_etx + entry);
3389 else if (tx_q->tbs & STMMAC_TBS_AVAIL)
3390 desc = &tx_q->dma_entx[entry].basic;
3391 else
3392 desc = tx_q->dma_tx + entry;
3393
3394 des = skb_frag_dma_map(priv->device, frag, 0, len,
3395 DMA_TO_DEVICE);
3396 if (dma_mapping_error(priv->device, des))
3397 goto dma_map_err; /* should reuse desc w/o issues */
3398
3399 tx_q->tx_skbuff_dma[entry].buf = des;
3400
3401 stmmac_set_desc_addr(priv, desc, des);
3402
3403 tx_q->tx_skbuff_dma[entry].map_as_page = true;
3404 tx_q->tx_skbuff_dma[entry].len = len;
3405 tx_q->tx_skbuff_dma[entry].last_segment = last_segment;
3406
3407 /* Prepare the descriptor and set the own bit too */
3408 stmmac_prepare_tx_desc(priv, desc, 0, len, csum_insertion,
3409 priv->mode, 1, last_segment, skb->len);
3410 }
3411
3412 /* Only the last descriptor gets to point to the skb. */
3413 tx_q->tx_skbuff[entry] = skb;
3414
3415 /* According to the coalesce parameter the IC bit for the latest
3416 * segment is reset and the timer re-started to clean the tx status.
3417 * This approach takes care about the fragments: desc is the first
3418 * element in case of no SG.
3419 */
3420 tx_packets = (entry + 1) - first_tx;
3421 tx_q->tx_count_frames += tx_packets;
3422
3423 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en)
3424 set_ic = true;
3425 else if (!priv->tx_coal_frames)
3426 set_ic = false;
3427 else if (tx_packets > priv->tx_coal_frames)
3428 set_ic = true;
3429 else if ((tx_q->tx_count_frames % priv->tx_coal_frames) < tx_packets)
3430 set_ic = true;
3431 else
3432 set_ic = false;
3433
3434 if (set_ic) {
3435 if (likely(priv->extend_desc))
3436 desc = &tx_q->dma_etx[entry].basic;
3437 else if (tx_q->tbs & STMMAC_TBS_AVAIL)
3438 desc = &tx_q->dma_entx[entry].basic;
3439 else
3440 desc = &tx_q->dma_tx[entry];
3441
3442 tx_q->tx_count_frames = 0;
3443 stmmac_set_tx_ic(priv, desc);
3444 priv->xstats.tx_set_ic_bit++;
3445 }
3446
3447 /* We've used all descriptors we need for this skb, however,
3448 * advance cur_tx so that it references a fresh descriptor.
3449 * ndo_start_xmit will fill this descriptor the next time it's
3450 * called and stmmac_tx_clean may clean up to this descriptor.
3451 */
3452 entry = STMMAC_GET_ENTRY(entry, priv->dma_tx_size);
3453 tx_q->cur_tx = entry;
3454
3455 if (netif_msg_pktdata(priv)) {
3456 netdev_dbg(priv->dev,
3457 "%s: curr=%d dirty=%d f=%d, e=%d, first=%p, nfrags=%d",
3458 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry,
3459 entry, first, nfrags);
3460
3461 netdev_dbg(priv->dev, ">>> frame to be transmitted: ");
3462 print_pkt(skb->data, skb->len);
3463 }
3464
3465 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) {
3466 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",
3467 __func__);
3468 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
3469 }
3470
3471 dev->stats.tx_bytes += skb->len;
3472
3473 if (priv->sarc_type)
3474 stmmac_set_desc_sarc(priv, first, priv->sarc_type);
3475
3476 skb_tx_timestamp(skb);
3477
3478 /* Ready to fill the first descriptor and set the OWN bit w/o any
3479 * problems because all the descriptors are actually ready to be
3480 * passed to the DMA engine.
3481 */
3482 if (likely(!is_jumbo)) {
3483 bool last_segment = (nfrags == 0);
3484
3485 des = dma_map_single(priv->device, skb->data,
3486 nopaged_len, DMA_TO_DEVICE);
3487 if (dma_mapping_error(priv->device, des))
3488 goto dma_map_err;
3489
3490 tx_q->tx_skbuff_dma[first_entry].buf = des;
3491
3492 stmmac_set_desc_addr(priv, first, des);
3493
3494 tx_q->tx_skbuff_dma[first_entry].len = nopaged_len;
3495 tx_q->tx_skbuff_dma[first_entry].last_segment = last_segment;
3496
3497 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
3498 priv->hwts_tx_en)) {
3499 /* declare that device is doing timestamping */
3500 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
3501 stmmac_enable_tx_timestamp(priv, first);
3502 }
3503
3504 /* Prepare the first descriptor setting the OWN bit too */
3505 stmmac_prepare_tx_desc(priv, first, 1, nopaged_len,
3506 csum_insertion, priv->mode, 0, last_segment,
3507 skb->len);
3508 }
3509
3510 if (tx_q->tbs & STMMAC_TBS_EN) {
3511 struct timespec64 ts = ns_to_timespec64(skb->tstamp);
3512
3513 tbs_desc = &tx_q->dma_entx[first_entry];
3514 stmmac_set_desc_tbs(priv, tbs_desc, ts.tv_sec, ts.tv_nsec);
3515 }
3516
3517 stmmac_set_tx_owner(priv, first);
3518
3519 /* The own bit must be the latest setting done when prepare the
3520 * descriptor and then barrier is needed to make sure that
3521 * all is coherent before granting the DMA engine.
3522 */
3523 wmb();
3524
3525 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len);
3526
3527 stmmac_enable_dma_transmission(priv, priv->ioaddr);
3528
3529 if (likely(priv->extend_desc))
3530 desc_size = sizeof(struct dma_extended_desc);
3531 else if (tx_q->tbs & STMMAC_TBS_AVAIL)
3532 desc_size = sizeof(struct dma_edesc);
3533 else
3534 desc_size = sizeof(struct dma_desc);
3535
3536 tx_q->tx_tail_addr = tx_q->dma_tx_phy + (tx_q->cur_tx * desc_size);
3537 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue);
3538 stmmac_tx_timer_arm(priv, queue);
3539
3540 return NETDEV_TX_OK;
3541
3542 dma_map_err:
3543 netdev_err(priv->dev, "Tx DMA map failed\n");
3544 dev_kfree_skb(skb);
3545 priv->dev->stats.tx_dropped++;
3546 return NETDEV_TX_OK;
3547 }
3548
stmmac_rx_vlan(struct net_device * dev,struct sk_buff * skb)3549 static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb)
3550 {
3551 struct vlan_ethhdr *veth;
3552 __be16 vlan_proto;
3553 u16 vlanid;
3554
3555 veth = (struct vlan_ethhdr *)skb->data;
3556 vlan_proto = veth->h_vlan_proto;
3557
3558 if ((vlan_proto == htons(ETH_P_8021Q) &&
3559 dev->features & NETIF_F_HW_VLAN_CTAG_RX) ||
3560 (vlan_proto == htons(ETH_P_8021AD) &&
3561 dev->features & NETIF_F_HW_VLAN_STAG_RX)) {
3562 /* pop the vlan tag */
3563 vlanid = ntohs(veth->h_vlan_TCI);
3564 memmove(skb->data + VLAN_HLEN, veth, ETH_ALEN * 2);
3565 skb_pull(skb, VLAN_HLEN);
3566 __vlan_hwaccel_put_tag(skb, vlan_proto, vlanid);
3567 }
3568 }
3569
3570 /**
3571 * stmmac_rx_refill - refill used skb preallocated buffers
3572 * @priv: driver private structure
3573 * @queue: RX queue index
3574 * Description : this is to reallocate the skb for the reception process
3575 * that is based on zero-copy.
3576 */
stmmac_rx_refill(struct stmmac_priv * priv,u32 queue)3577 static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue)
3578 {
3579 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
3580 int len, dirty = stmmac_rx_dirty(priv, queue);
3581 unsigned int entry = rx_q->dirty_rx;
3582
3583 len = DIV_ROUND_UP(priv->dma_buf_sz, PAGE_SIZE) * PAGE_SIZE;
3584
3585 while (dirty-- > 0) {
3586 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry];
3587 struct dma_desc *p;
3588 bool use_rx_wd;
3589
3590 if (priv->extend_desc)
3591 p = (struct dma_desc *)(rx_q->dma_erx + entry);
3592 else
3593 p = rx_q->dma_rx + entry;
3594
3595 if (!buf->page) {
3596 buf->page = page_pool_dev_alloc_pages(rx_q->page_pool);
3597 if (!buf->page)
3598 break;
3599 }
3600
3601 if (priv->sph && !buf->sec_page) {
3602 buf->sec_page = page_pool_dev_alloc_pages(rx_q->page_pool);
3603 if (!buf->sec_page)
3604 break;
3605
3606 buf->sec_addr = page_pool_get_dma_addr(buf->sec_page);
3607
3608 dma_sync_single_for_device(priv->device, buf->sec_addr,
3609 len, DMA_FROM_DEVICE);
3610 }
3611
3612 buf->addr = page_pool_get_dma_addr(buf->page);
3613
3614 /* Sync whole allocation to device. This will invalidate old
3615 * data.
3616 */
3617 dma_sync_single_for_device(priv->device, buf->addr, len,
3618 DMA_FROM_DEVICE);
3619
3620 stmmac_set_desc_addr(priv, p, buf->addr);
3621 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr);
3622 stmmac_refill_desc3(priv, rx_q, p);
3623
3624 rx_q->rx_count_frames++;
3625 rx_q->rx_count_frames += priv->rx_coal_frames;
3626 if (rx_q->rx_count_frames > priv->rx_coal_frames)
3627 rx_q->rx_count_frames = 0;
3628
3629 use_rx_wd = !priv->rx_coal_frames;
3630 use_rx_wd |= rx_q->rx_count_frames > 0;
3631 if (!priv->use_riwt)
3632 use_rx_wd = false;
3633
3634 dma_wmb();
3635 stmmac_set_rx_owner(priv, p, use_rx_wd);
3636
3637 entry = STMMAC_GET_ENTRY(entry, priv->dma_rx_size);
3638 }
3639 rx_q->dirty_rx = entry;
3640 rx_q->rx_tail_addr = rx_q->dma_rx_phy +
3641 (rx_q->dirty_rx * sizeof(struct dma_desc));
3642 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue);
3643 }
3644
stmmac_rx_buf1_len(struct stmmac_priv * priv,struct dma_desc * p,int status,unsigned int len)3645 static unsigned int stmmac_rx_buf1_len(struct stmmac_priv *priv,
3646 struct dma_desc *p,
3647 int status, unsigned int len)
3648 {
3649 unsigned int plen = 0, hlen = 0;
3650 int coe = priv->hw->rx_csum;
3651
3652 /* Not first descriptor, buffer is always zero */
3653 if (priv->sph && len)
3654 return 0;
3655
3656 /* First descriptor, get split header length */
3657 stmmac_get_rx_header_len(priv, p, &hlen);
3658 if (priv->sph && hlen) {
3659 priv->xstats.rx_split_hdr_pkt_n++;
3660 return hlen;
3661 }
3662
3663 /* First descriptor, not last descriptor and not split header */
3664 if (status & rx_not_ls)
3665 return priv->dma_buf_sz;
3666
3667 plen = stmmac_get_rx_frame_len(priv, p, coe);
3668
3669 /* First descriptor and last descriptor and not split header */
3670 return min_t(unsigned int, priv->dma_buf_sz, plen);
3671 }
3672
stmmac_rx_buf2_len(struct stmmac_priv * priv,struct dma_desc * p,int status,unsigned int len)3673 static unsigned int stmmac_rx_buf2_len(struct stmmac_priv *priv,
3674 struct dma_desc *p,
3675 int status, unsigned int len)
3676 {
3677 int coe = priv->hw->rx_csum;
3678 unsigned int plen = 0;
3679
3680 /* Not split header, buffer is not available */
3681 if (!priv->sph)
3682 return 0;
3683
3684 /* Not last descriptor */
3685 if (status & rx_not_ls)
3686 return priv->dma_buf_sz;
3687
3688 plen = stmmac_get_rx_frame_len(priv, p, coe);
3689
3690 /* Last descriptor */
3691 return plen - len;
3692 }
3693
3694 /**
3695 * stmmac_rx - manage the receive process
3696 * @priv: driver private structure
3697 * @limit: napi bugget
3698 * @queue: RX queue index.
3699 * Description : this the function called by the napi poll method.
3700 * It gets all the frames inside the ring.
3701 */
stmmac_rx(struct stmmac_priv * priv,int limit,u32 queue)3702 static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)
3703 {
3704 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
3705 struct stmmac_channel *ch = &priv->channel[queue];
3706 unsigned int count = 0, error = 0, len = 0;
3707 int status = 0, coe = priv->hw->rx_csum;
3708 unsigned int next_entry = rx_q->cur_rx;
3709 struct sk_buff *skb = NULL;
3710
3711 if (netif_msg_rx_status(priv)) {
3712 void *rx_head;
3713
3714 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__);
3715 if (priv->extend_desc)
3716 rx_head = (void *)rx_q->dma_erx;
3717 else
3718 rx_head = (void *)rx_q->dma_rx;
3719
3720 stmmac_display_ring(priv, rx_head, priv->dma_rx_size, true);
3721 }
3722 while (count < limit) {
3723 unsigned int buf1_len = 0, buf2_len = 0;
3724 enum pkt_hash_types hash_type;
3725 struct stmmac_rx_buffer *buf;
3726 struct dma_desc *np, *p;
3727 int entry;
3728 u32 hash;
3729
3730 if (!count && rx_q->state_saved) {
3731 skb = rx_q->state.skb;
3732 error = rx_q->state.error;
3733 len = rx_q->state.len;
3734 } else {
3735 rx_q->state_saved = false;
3736 skb = NULL;
3737 error = 0;
3738 len = 0;
3739 }
3740
3741 if (count >= limit)
3742 break;
3743
3744 read_again:
3745 buf1_len = 0;
3746 buf2_len = 0;
3747 entry = next_entry;
3748 buf = &rx_q->buf_pool[entry];
3749
3750 if (priv->extend_desc)
3751 p = (struct dma_desc *)(rx_q->dma_erx + entry);
3752 else
3753 p = rx_q->dma_rx + entry;
3754
3755 /* read the status of the incoming frame */
3756 status = stmmac_rx_status(priv, &priv->dev->stats,
3757 &priv->xstats, p);
3758 /* check if managed by the DMA otherwise go ahead */
3759 if (unlikely(status & dma_own))
3760 break;
3761
3762 rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx,
3763 priv->dma_rx_size);
3764 next_entry = rx_q->cur_rx;
3765
3766 if (priv->extend_desc)
3767 np = (struct dma_desc *)(rx_q->dma_erx + next_entry);
3768 else
3769 np = rx_q->dma_rx + next_entry;
3770
3771 prefetch(np);
3772
3773 if (priv->extend_desc)
3774 stmmac_rx_extended_status(priv, &priv->dev->stats,
3775 &priv->xstats, rx_q->dma_erx + entry);
3776 if (unlikely(status == discard_frame)) {
3777 page_pool_recycle_direct(rx_q->page_pool, buf->page);
3778 buf->page = NULL;
3779 error = 1;
3780 if (!priv->hwts_rx_en)
3781 priv->dev->stats.rx_errors++;
3782 }
3783
3784 if (unlikely(error && (status & rx_not_ls)))
3785 goto read_again;
3786 if (unlikely(error)) {
3787 dev_kfree_skb(skb);
3788 skb = NULL;
3789 count++;
3790 continue;
3791 }
3792
3793 /* Buffer is good. Go on. */
3794
3795 prefetch(page_address(buf->page));
3796 if (buf->sec_page)
3797 prefetch(page_address(buf->sec_page));
3798
3799 buf1_len = stmmac_rx_buf1_len(priv, p, status, len);
3800 len += buf1_len;
3801 buf2_len = stmmac_rx_buf2_len(priv, p, status, len);
3802 len += buf2_len;
3803
3804 /* ACS is set; GMAC core strips PAD/FCS for IEEE 802.3
3805 * Type frames (LLC/LLC-SNAP)
3806 *
3807 * llc_snap is never checked in GMAC >= 4, so this ACS
3808 * feature is always disabled and packets need to be
3809 * stripped manually.
3810 */
3811 if (likely(!(status & rx_not_ls)) &&
3812 (likely(priv->synopsys_id >= DWMAC_CORE_4_00) ||
3813 unlikely(status != llc_snap))) {
3814 if (buf2_len)
3815 buf2_len -= ETH_FCS_LEN;
3816 else
3817 buf1_len -= ETH_FCS_LEN;
3818
3819 len -= ETH_FCS_LEN;
3820 }
3821
3822 if (!skb) {
3823 skb = napi_alloc_skb(&ch->rx_napi, buf1_len);
3824 if (!skb) {
3825 priv->dev->stats.rx_dropped++;
3826 count++;
3827 goto drain_data;
3828 }
3829
3830 dma_sync_single_for_cpu(priv->device, buf->addr,
3831 buf1_len, DMA_FROM_DEVICE);
3832 skb_copy_to_linear_data(skb, page_address(buf->page),
3833 buf1_len);
3834 skb_put(skb, buf1_len);
3835
3836 /* Data payload copied into SKB, page ready for recycle */
3837 page_pool_recycle_direct(rx_q->page_pool, buf->page);
3838 buf->page = NULL;
3839 } else if (buf1_len) {
3840 dma_sync_single_for_cpu(priv->device, buf->addr,
3841 buf1_len, DMA_FROM_DEVICE);
3842 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
3843 buf->page, 0, buf1_len,
3844 priv->dma_buf_sz);
3845
3846 /* Data payload appended into SKB */
3847 page_pool_release_page(rx_q->page_pool, buf->page);
3848 buf->page = NULL;
3849 }
3850
3851 if (buf2_len) {
3852 dma_sync_single_for_cpu(priv->device, buf->sec_addr,
3853 buf2_len, DMA_FROM_DEVICE);
3854 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
3855 buf->sec_page, 0, buf2_len,
3856 priv->dma_buf_sz);
3857
3858 /* Data payload appended into SKB */
3859 page_pool_release_page(rx_q->page_pool, buf->sec_page);
3860 buf->sec_page = NULL;
3861 }
3862
3863 drain_data:
3864 if (likely(status & rx_not_ls))
3865 goto read_again;
3866 if (!skb)
3867 continue;
3868
3869 /* Got entire packet into SKB. Finish it. */
3870
3871 stmmac_get_rx_hwtstamp(priv, p, np, skb);
3872 stmmac_rx_vlan(priv->dev, skb);
3873 skb->protocol = eth_type_trans(skb, priv->dev);
3874
3875 if (unlikely(!coe))
3876 skb_checksum_none_assert(skb);
3877 else
3878 skb->ip_summed = CHECKSUM_UNNECESSARY;
3879
3880 if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type))
3881 skb_set_hash(skb, hash, hash_type);
3882
3883 skb_record_rx_queue(skb, queue);
3884 napi_gro_receive(&ch->rx_napi, skb);
3885 skb = NULL;
3886
3887 priv->dev->stats.rx_packets++;
3888 priv->dev->stats.rx_bytes += len;
3889 count++;
3890 }
3891
3892 if (status & rx_not_ls || skb) {
3893 rx_q->state_saved = true;
3894 rx_q->state.skb = skb;
3895 rx_q->state.error = error;
3896 rx_q->state.len = len;
3897 }
3898
3899 stmmac_rx_refill(priv, queue);
3900
3901 priv->xstats.rx_pkt_n += count;
3902
3903 return count;
3904 }
3905
stmmac_napi_poll_rx(struct napi_struct * napi,int budget)3906 static int stmmac_napi_poll_rx(struct napi_struct *napi, int budget)
3907 {
3908 struct stmmac_channel *ch =
3909 container_of(napi, struct stmmac_channel, rx_napi);
3910 struct stmmac_priv *priv = ch->priv_data;
3911 u32 chan = ch->index;
3912 int work_done;
3913
3914 priv->xstats.napi_poll++;
3915
3916 work_done = stmmac_rx(priv, budget, chan);
3917 if (work_done < budget && napi_complete_done(napi, work_done)) {
3918 unsigned long flags;
3919
3920 spin_lock_irqsave(&ch->lock, flags);
3921 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 0);
3922 spin_unlock_irqrestore(&ch->lock, flags);
3923 }
3924
3925 return work_done;
3926 }
3927
stmmac_napi_poll_tx(struct napi_struct * napi,int budget)3928 static int stmmac_napi_poll_tx(struct napi_struct *napi, int budget)
3929 {
3930 struct stmmac_channel *ch =
3931 container_of(napi, struct stmmac_channel, tx_napi);
3932 struct stmmac_priv *priv = ch->priv_data;
3933 u32 chan = ch->index;
3934 int work_done;
3935
3936 priv->xstats.napi_poll++;
3937
3938 work_done = stmmac_tx_clean(priv, priv->dma_tx_size, chan);
3939 work_done = min(work_done, budget);
3940
3941 if (work_done < budget && napi_complete_done(napi, work_done)) {
3942 unsigned long flags;
3943
3944 spin_lock_irqsave(&ch->lock, flags);
3945 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 0, 1);
3946 spin_unlock_irqrestore(&ch->lock, flags);
3947 }
3948
3949 return work_done;
3950 }
3951
3952 /**
3953 * stmmac_tx_timeout
3954 * @dev : Pointer to net device structure
3955 * @txqueue: the index of the hanging transmit queue
3956 * Description: this function is called when a packet transmission fails to
3957 * complete within a reasonable time. The driver will mark the error in the
3958 * netdev structure and arrange for the device to be reset to a sane state
3959 * in order to transmit a new packet.
3960 */
stmmac_tx_timeout(struct net_device * dev,unsigned int txqueue)3961 static void stmmac_tx_timeout(struct net_device *dev, unsigned int txqueue)
3962 {
3963 struct stmmac_priv *priv = netdev_priv(dev);
3964
3965 stmmac_global_err(priv);
3966 }
3967
3968 /**
3969 * stmmac_set_rx_mode - entry point for multicast addressing
3970 * @dev : pointer to the device structure
3971 * Description:
3972 * This function is a driver entry point which gets called by the kernel
3973 * whenever multicast addresses must be enabled/disabled.
3974 * Return value:
3975 * void.
3976 */
stmmac_set_rx_mode(struct net_device * dev)3977 static void stmmac_set_rx_mode(struct net_device *dev)
3978 {
3979 struct stmmac_priv *priv = netdev_priv(dev);
3980
3981 stmmac_set_filter(priv, priv->hw, dev);
3982 }
3983
3984 /**
3985 * stmmac_change_mtu - entry point to change MTU size for the device.
3986 * @dev : device pointer.
3987 * @new_mtu : the new MTU size for the device.
3988 * Description: the Maximum Transfer Unit (MTU) is used by the network layer
3989 * to drive packet transmission. Ethernet has an MTU of 1500 octets
3990 * (ETH_DATA_LEN). This value can be changed with ifconfig.
3991 * Return value:
3992 * 0 on success and an appropriate (-)ve integer as defined in errno.h
3993 * file on failure.
3994 */
stmmac_change_mtu(struct net_device * dev,int new_mtu)3995 static int stmmac_change_mtu(struct net_device *dev, int new_mtu)
3996 {
3997 struct stmmac_priv *priv = netdev_priv(dev);
3998 int txfifosz = priv->plat->tx_fifo_size;
3999
4000 if (txfifosz == 0)
4001 txfifosz = priv->dma_cap.tx_fifo_size;
4002
4003 txfifosz /= priv->plat->tx_queues_to_use;
4004
4005 if (netif_running(dev)) {
4006 netdev_err(priv->dev, "must be stopped to change its MTU\n");
4007 return -EBUSY;
4008 }
4009
4010 new_mtu = STMMAC_ALIGN(new_mtu);
4011
4012 /* If condition true, FIFO is too small or MTU too large */
4013 if ((txfifosz < new_mtu) || (new_mtu > BUF_SIZE_16KiB))
4014 return -EINVAL;
4015
4016 dev->mtu = new_mtu;
4017
4018 netdev_update_features(dev);
4019
4020 return 0;
4021 }
4022
stmmac_fix_features(struct net_device * dev,netdev_features_t features)4023 static netdev_features_t stmmac_fix_features(struct net_device *dev,
4024 netdev_features_t features)
4025 {
4026 struct stmmac_priv *priv = netdev_priv(dev);
4027
4028 if (priv->plat->rx_coe == STMMAC_RX_COE_NONE)
4029 features &= ~NETIF_F_RXCSUM;
4030
4031 if (!priv->plat->tx_coe)
4032 features &= ~NETIF_F_CSUM_MASK;
4033
4034 /* Some GMAC devices have a bugged Jumbo frame support that
4035 * needs to have the Tx COE disabled for oversized frames
4036 * (due to limited buffer sizes). In this case we disable
4037 * the TX csum insertion in the TDES and not use SF.
4038 */
4039 if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN))
4040 features &= ~NETIF_F_CSUM_MASK;
4041
4042 /* Disable tso if asked by ethtool */
4043 if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) {
4044 if (features & NETIF_F_TSO)
4045 priv->tso = true;
4046 else
4047 priv->tso = false;
4048 }
4049
4050 return features;
4051 }
4052
stmmac_set_features(struct net_device * netdev,netdev_features_t features)4053 static int stmmac_set_features(struct net_device *netdev,
4054 netdev_features_t features)
4055 {
4056 struct stmmac_priv *priv = netdev_priv(netdev);
4057 bool sph_en;
4058 u32 chan;
4059
4060 /* Keep the COE Type in case of csum is supporting */
4061 if (features & NETIF_F_RXCSUM)
4062 priv->hw->rx_csum = priv->plat->rx_coe;
4063 else
4064 priv->hw->rx_csum = 0;
4065 /* No check needed because rx_coe has been set before and it will be
4066 * fixed in case of issue.
4067 */
4068 stmmac_rx_ipc(priv, priv->hw);
4069
4070 sph_en = (priv->hw->rx_csum > 0) && priv->sph;
4071 for (chan = 0; chan < priv->plat->rx_queues_to_use; chan++)
4072 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan);
4073
4074 return 0;
4075 }
4076
4077 /**
4078 * stmmac_interrupt - main ISR
4079 * @irq: interrupt number.
4080 * @dev_id: to pass the net device pointer (must be valid).
4081 * Description: this is the main driver interrupt service routine.
4082 * It can call:
4083 * o DMA service routine (to manage incoming frame reception and transmission
4084 * status)
4085 * o Core interrupts to manage: remote wake-up, management counter, LPI
4086 * interrupts.
4087 */
stmmac_interrupt(int irq,void * dev_id)4088 static irqreturn_t stmmac_interrupt(int irq, void *dev_id)
4089 {
4090 struct net_device *dev = (struct net_device *)dev_id;
4091 struct stmmac_priv *priv = netdev_priv(dev);
4092 u32 rx_cnt = priv->plat->rx_queues_to_use;
4093 u32 tx_cnt = priv->plat->tx_queues_to_use;
4094 u32 queues_count;
4095 u32 queue;
4096 bool xmac;
4097
4098 xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac;
4099 queues_count = (rx_cnt > tx_cnt) ? rx_cnt : tx_cnt;
4100
4101 if (priv->irq_wake)
4102 pm_wakeup_event(priv->device, 0);
4103
4104 /* Check if adapter is up */
4105 if (test_bit(STMMAC_DOWN, &priv->state))
4106 return IRQ_HANDLED;
4107 /* Check if a fatal error happened */
4108 if (stmmac_safety_feat_interrupt(priv))
4109 return IRQ_HANDLED;
4110
4111 /* To handle GMAC own interrupts */
4112 if ((priv->plat->has_gmac) || xmac) {
4113 int status = stmmac_host_irq_status(priv, priv->hw, &priv->xstats);
4114 int mtl_status;
4115
4116 if (unlikely(status)) {
4117 /* For LPI we need to save the tx status */
4118 if (status & CORE_IRQ_TX_PATH_IN_LPI_MODE)
4119 priv->tx_path_in_lpi_mode = true;
4120 if (status & CORE_IRQ_TX_PATH_EXIT_LPI_MODE)
4121 priv->tx_path_in_lpi_mode = false;
4122 }
4123
4124 for (queue = 0; queue < queues_count; queue++) {
4125 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
4126
4127 mtl_status = stmmac_host_mtl_irq_status(priv, priv->hw,
4128 queue);
4129 if (mtl_status != -EINVAL)
4130 status |= mtl_status;
4131
4132 if (status & CORE_IRQ_MTL_RX_OVERFLOW)
4133 stmmac_set_rx_tail_ptr(priv, priv->ioaddr,
4134 rx_q->rx_tail_addr,
4135 queue);
4136 }
4137
4138 /* PCS link status */
4139 if (priv->hw->pcs) {
4140 if (priv->xstats.pcs_link)
4141 netif_carrier_on(dev);
4142 else
4143 netif_carrier_off(dev);
4144 }
4145 }
4146
4147 /* To handle DMA interrupts */
4148 stmmac_dma_interrupt(priv);
4149
4150 return IRQ_HANDLED;
4151 }
4152
4153 #ifdef CONFIG_NET_POLL_CONTROLLER
4154 /* Polling receive - used by NETCONSOLE and other diagnostic tools
4155 * to allow network I/O with interrupts disabled.
4156 */
stmmac_poll_controller(struct net_device * dev)4157 static void stmmac_poll_controller(struct net_device *dev)
4158 {
4159 disable_irq(dev->irq);
4160 stmmac_interrupt(dev->irq, dev);
4161 enable_irq(dev->irq);
4162 }
4163 #endif
4164
4165 /**
4166 * stmmac_ioctl - Entry point for the Ioctl
4167 * @dev: Device pointer.
4168 * @rq: An IOCTL specefic structure, that can contain a pointer to
4169 * a proprietary structure used to pass information to the driver.
4170 * @cmd: IOCTL command
4171 * Description:
4172 * Currently it supports the phy_mii_ioctl(...) and HW time stamping.
4173 */
stmmac_ioctl(struct net_device * dev,struct ifreq * rq,int cmd)4174 static int stmmac_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
4175 {
4176 struct stmmac_priv *priv = netdev_priv (dev);
4177 int ret = -EOPNOTSUPP;
4178
4179 if (!netif_running(dev))
4180 return -EINVAL;
4181
4182 switch (cmd) {
4183 case SIOCGMIIPHY:
4184 case SIOCGMIIREG:
4185 case SIOCSMIIREG:
4186 ret = phylink_mii_ioctl(priv->phylink, rq, cmd);
4187 break;
4188 case SIOCSHWTSTAMP:
4189 ret = stmmac_hwtstamp_set(dev, rq);
4190 break;
4191 case SIOCGHWTSTAMP:
4192 ret = stmmac_hwtstamp_get(dev, rq);
4193 break;
4194 default:
4195 break;
4196 }
4197
4198 return ret;
4199 }
4200
stmmac_setup_tc_block_cb(enum tc_setup_type type,void * type_data,void * cb_priv)4201 static int stmmac_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
4202 void *cb_priv)
4203 {
4204 struct stmmac_priv *priv = cb_priv;
4205 int ret = -EOPNOTSUPP;
4206
4207 if (!tc_cls_can_offload_and_chain0(priv->dev, type_data))
4208 return ret;
4209
4210 stmmac_disable_all_queues(priv);
4211
4212 switch (type) {
4213 case TC_SETUP_CLSU32:
4214 ret = stmmac_tc_setup_cls_u32(priv, priv, type_data);
4215 break;
4216 case TC_SETUP_CLSFLOWER:
4217 ret = stmmac_tc_setup_cls(priv, priv, type_data);
4218 break;
4219 default:
4220 break;
4221 }
4222
4223 stmmac_enable_all_queues(priv);
4224 return ret;
4225 }
4226
4227 static LIST_HEAD(stmmac_block_cb_list);
4228
stmmac_setup_tc(struct net_device * ndev,enum tc_setup_type type,void * type_data)4229 static int stmmac_setup_tc(struct net_device *ndev, enum tc_setup_type type,
4230 void *type_data)
4231 {
4232 struct stmmac_priv *priv = netdev_priv(ndev);
4233
4234 switch (type) {
4235 case TC_SETUP_BLOCK:
4236 return flow_block_cb_setup_simple(type_data,
4237 &stmmac_block_cb_list,
4238 stmmac_setup_tc_block_cb,
4239 priv, priv, true);
4240 case TC_SETUP_QDISC_CBS:
4241 return stmmac_tc_setup_cbs(priv, priv, type_data);
4242 case TC_SETUP_QDISC_TAPRIO:
4243 return stmmac_tc_setup_taprio(priv, priv, type_data);
4244 case TC_SETUP_QDISC_ETF:
4245 return stmmac_tc_setup_etf(priv, priv, type_data);
4246 default:
4247 return -EOPNOTSUPP;
4248 }
4249 }
4250
stmmac_select_queue(struct net_device * dev,struct sk_buff * skb,struct net_device * sb_dev)4251 static u16 stmmac_select_queue(struct net_device *dev, struct sk_buff *skb,
4252 struct net_device *sb_dev)
4253 {
4254 int gso = skb_shinfo(skb)->gso_type;
4255
4256 if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6 | SKB_GSO_UDP_L4)) {
4257 /*
4258 * There is no way to determine the number of TSO/USO
4259 * capable Queues. Let's use always the Queue 0
4260 * because if TSO/USO is supported then at least this
4261 * one will be capable.
4262 */
4263 return 0;
4264 }
4265
4266 return netdev_pick_tx(dev, skb, NULL) % dev->real_num_tx_queues;
4267 }
4268
stmmac_set_mac_address(struct net_device * ndev,void * addr)4269 static int stmmac_set_mac_address(struct net_device *ndev, void *addr)
4270 {
4271 struct stmmac_priv *priv = netdev_priv(ndev);
4272 int ret = 0;
4273
4274 ret = eth_mac_addr(ndev, addr);
4275 if (ret)
4276 return ret;
4277
4278 stmmac_set_umac_addr(priv, priv->hw, ndev->dev_addr, 0);
4279
4280 return ret;
4281 }
4282
4283 #ifdef CONFIG_DEBUG_FS
4284 static struct dentry *stmmac_fs_dir;
4285
sysfs_display_ring(void * head,int size,int extend_desc,struct seq_file * seq)4286 static void sysfs_display_ring(void *head, int size, int extend_desc,
4287 struct seq_file *seq)
4288 {
4289 int i;
4290 struct dma_extended_desc *ep = (struct dma_extended_desc *)head;
4291 struct dma_desc *p = (struct dma_desc *)head;
4292
4293 for (i = 0; i < size; i++) {
4294 if (extend_desc) {
4295 seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n",
4296 i, (unsigned int)virt_to_phys(ep),
4297 le32_to_cpu(ep->basic.des0),
4298 le32_to_cpu(ep->basic.des1),
4299 le32_to_cpu(ep->basic.des2),
4300 le32_to_cpu(ep->basic.des3));
4301 ep++;
4302 } else {
4303 seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n",
4304 i, (unsigned int)virt_to_phys(p),
4305 le32_to_cpu(p->des0), le32_to_cpu(p->des1),
4306 le32_to_cpu(p->des2), le32_to_cpu(p->des3));
4307 p++;
4308 }
4309 seq_printf(seq, "\n");
4310 }
4311 }
4312
stmmac_rings_status_show(struct seq_file * seq,void * v)4313 static int stmmac_rings_status_show(struct seq_file *seq, void *v)
4314 {
4315 struct net_device *dev = seq->private;
4316 struct stmmac_priv *priv = netdev_priv(dev);
4317 u32 rx_count = priv->plat->rx_queues_to_use;
4318 u32 tx_count = priv->plat->tx_queues_to_use;
4319 u32 queue;
4320
4321 if ((dev->flags & IFF_UP) == 0)
4322 return 0;
4323
4324 for (queue = 0; queue < rx_count; queue++) {
4325 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
4326
4327 seq_printf(seq, "RX Queue %d:\n", queue);
4328
4329 if (priv->extend_desc) {
4330 seq_printf(seq, "Extended descriptor ring:\n");
4331 sysfs_display_ring((void *)rx_q->dma_erx,
4332 priv->dma_rx_size, 1, seq);
4333 } else {
4334 seq_printf(seq, "Descriptor ring:\n");
4335 sysfs_display_ring((void *)rx_q->dma_rx,
4336 priv->dma_rx_size, 0, seq);
4337 }
4338 }
4339
4340 for (queue = 0; queue < tx_count; queue++) {
4341 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
4342
4343 seq_printf(seq, "TX Queue %d:\n", queue);
4344
4345 if (priv->extend_desc) {
4346 seq_printf(seq, "Extended descriptor ring:\n");
4347 sysfs_display_ring((void *)tx_q->dma_etx,
4348 priv->dma_tx_size, 1, seq);
4349 } else if (!(tx_q->tbs & STMMAC_TBS_AVAIL)) {
4350 seq_printf(seq, "Descriptor ring:\n");
4351 sysfs_display_ring((void *)tx_q->dma_tx,
4352 priv->dma_tx_size, 0, seq);
4353 }
4354 }
4355
4356 return 0;
4357 }
4358 DEFINE_SHOW_ATTRIBUTE(stmmac_rings_status);
4359
stmmac_dma_cap_show(struct seq_file * seq,void * v)4360 static int stmmac_dma_cap_show(struct seq_file *seq, void *v)
4361 {
4362 struct net_device *dev = seq->private;
4363 struct stmmac_priv *priv = netdev_priv(dev);
4364
4365 if (!priv->hw_cap_support) {
4366 seq_printf(seq, "DMA HW features not supported\n");
4367 return 0;
4368 }
4369
4370 seq_printf(seq, "==============================\n");
4371 seq_printf(seq, "\tDMA HW features\n");
4372 seq_printf(seq, "==============================\n");
4373
4374 seq_printf(seq, "\t10/100 Mbps: %s\n",
4375 (priv->dma_cap.mbps_10_100) ? "Y" : "N");
4376 seq_printf(seq, "\t1000 Mbps: %s\n",
4377 (priv->dma_cap.mbps_1000) ? "Y" : "N");
4378 seq_printf(seq, "\tHalf duplex: %s\n",
4379 (priv->dma_cap.half_duplex) ? "Y" : "N");
4380 seq_printf(seq, "\tHash Filter: %s\n",
4381 (priv->dma_cap.hash_filter) ? "Y" : "N");
4382 seq_printf(seq, "\tMultiple MAC address registers: %s\n",
4383 (priv->dma_cap.multi_addr) ? "Y" : "N");
4384 seq_printf(seq, "\tPCS (TBI/SGMII/RTBI PHY interfaces): %s\n",
4385 (priv->dma_cap.pcs) ? "Y" : "N");
4386 seq_printf(seq, "\tSMA (MDIO) Interface: %s\n",
4387 (priv->dma_cap.sma_mdio) ? "Y" : "N");
4388 seq_printf(seq, "\tPMT Remote wake up: %s\n",
4389 (priv->dma_cap.pmt_remote_wake_up) ? "Y" : "N");
4390 seq_printf(seq, "\tPMT Magic Frame: %s\n",
4391 (priv->dma_cap.pmt_magic_frame) ? "Y" : "N");
4392 seq_printf(seq, "\tRMON module: %s\n",
4393 (priv->dma_cap.rmon) ? "Y" : "N");
4394 seq_printf(seq, "\tIEEE 1588-2002 Time Stamp: %s\n",
4395 (priv->dma_cap.time_stamp) ? "Y" : "N");
4396 seq_printf(seq, "\tIEEE 1588-2008 Advanced Time Stamp: %s\n",
4397 (priv->dma_cap.atime_stamp) ? "Y" : "N");
4398 seq_printf(seq, "\t802.3az - Energy-Efficient Ethernet (EEE): %s\n",
4399 (priv->dma_cap.eee) ? "Y" : "N");
4400 seq_printf(seq, "\tAV features: %s\n", (priv->dma_cap.av) ? "Y" : "N");
4401 seq_printf(seq, "\tChecksum Offload in TX: %s\n",
4402 (priv->dma_cap.tx_coe) ? "Y" : "N");
4403 if (priv->synopsys_id >= DWMAC_CORE_4_00) {
4404 seq_printf(seq, "\tIP Checksum Offload in RX: %s\n",
4405 (priv->dma_cap.rx_coe) ? "Y" : "N");
4406 } else {
4407 seq_printf(seq, "\tIP Checksum Offload (type1) in RX: %s\n",
4408 (priv->dma_cap.rx_coe_type1) ? "Y" : "N");
4409 seq_printf(seq, "\tIP Checksum Offload (type2) in RX: %s\n",
4410 (priv->dma_cap.rx_coe_type2) ? "Y" : "N");
4411 }
4412 seq_printf(seq, "\tRXFIFO > 2048bytes: %s\n",
4413 (priv->dma_cap.rxfifo_over_2048) ? "Y" : "N");
4414 seq_printf(seq, "\tNumber of Additional RX channel: %d\n",
4415 priv->dma_cap.number_rx_channel);
4416 seq_printf(seq, "\tNumber of Additional TX channel: %d\n",
4417 priv->dma_cap.number_tx_channel);
4418 seq_printf(seq, "\tNumber of Additional RX queues: %d\n",
4419 priv->dma_cap.number_rx_queues);
4420 seq_printf(seq, "\tNumber of Additional TX queues: %d\n",
4421 priv->dma_cap.number_tx_queues);
4422 seq_printf(seq, "\tEnhanced descriptors: %s\n",
4423 (priv->dma_cap.enh_desc) ? "Y" : "N");
4424 seq_printf(seq, "\tTX Fifo Size: %d\n", priv->dma_cap.tx_fifo_size);
4425 seq_printf(seq, "\tRX Fifo Size: %d\n", priv->dma_cap.rx_fifo_size);
4426 seq_printf(seq, "\tHash Table Size: %d\n", priv->dma_cap.hash_tb_sz);
4427 seq_printf(seq, "\tTSO: %s\n", priv->dma_cap.tsoen ? "Y" : "N");
4428 seq_printf(seq, "\tNumber of PPS Outputs: %d\n",
4429 priv->dma_cap.pps_out_num);
4430 seq_printf(seq, "\tSafety Features: %s\n",
4431 priv->dma_cap.asp ? "Y" : "N");
4432 seq_printf(seq, "\tFlexible RX Parser: %s\n",
4433 priv->dma_cap.frpsel ? "Y" : "N");
4434 seq_printf(seq, "\tEnhanced Addressing: %d\n",
4435 priv->dma_cap.addr64);
4436 seq_printf(seq, "\tReceive Side Scaling: %s\n",
4437 priv->dma_cap.rssen ? "Y" : "N");
4438 seq_printf(seq, "\tVLAN Hash Filtering: %s\n",
4439 priv->dma_cap.vlhash ? "Y" : "N");
4440 seq_printf(seq, "\tSplit Header: %s\n",
4441 priv->dma_cap.sphen ? "Y" : "N");
4442 seq_printf(seq, "\tVLAN TX Insertion: %s\n",
4443 priv->dma_cap.vlins ? "Y" : "N");
4444 seq_printf(seq, "\tDouble VLAN: %s\n",
4445 priv->dma_cap.dvlan ? "Y" : "N");
4446 seq_printf(seq, "\tNumber of L3/L4 Filters: %d\n",
4447 priv->dma_cap.l3l4fnum);
4448 seq_printf(seq, "\tARP Offloading: %s\n",
4449 priv->dma_cap.arpoffsel ? "Y" : "N");
4450 seq_printf(seq, "\tEnhancements to Scheduled Traffic (EST): %s\n",
4451 priv->dma_cap.estsel ? "Y" : "N");
4452 seq_printf(seq, "\tFrame Preemption (FPE): %s\n",
4453 priv->dma_cap.fpesel ? "Y" : "N");
4454 seq_printf(seq, "\tTime-Based Scheduling (TBS): %s\n",
4455 priv->dma_cap.tbssel ? "Y" : "N");
4456 return 0;
4457 }
4458 DEFINE_SHOW_ATTRIBUTE(stmmac_dma_cap);
4459
4460 /* Use network device events to rename debugfs file entries.
4461 */
stmmac_device_event(struct notifier_block * unused,unsigned long event,void * ptr)4462 static int stmmac_device_event(struct notifier_block *unused,
4463 unsigned long event, void *ptr)
4464 {
4465 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
4466 struct stmmac_priv *priv = netdev_priv(dev);
4467
4468 if (dev->netdev_ops != &stmmac_netdev_ops)
4469 goto done;
4470
4471 switch (event) {
4472 case NETDEV_CHANGENAME:
4473 if (priv->dbgfs_dir)
4474 priv->dbgfs_dir = debugfs_rename(stmmac_fs_dir,
4475 priv->dbgfs_dir,
4476 stmmac_fs_dir,
4477 dev->name);
4478 break;
4479 }
4480 done:
4481 return NOTIFY_DONE;
4482 }
4483
4484 static struct notifier_block stmmac_notifier = {
4485 .notifier_call = stmmac_device_event,
4486 };
4487
stmmac_init_fs(struct net_device * dev)4488 static void stmmac_init_fs(struct net_device *dev)
4489 {
4490 struct stmmac_priv *priv = netdev_priv(dev);
4491
4492 rtnl_lock();
4493
4494 /* Create per netdev entries */
4495 priv->dbgfs_dir = debugfs_create_dir(dev->name, stmmac_fs_dir);
4496
4497 /* Entry to report DMA RX/TX rings */
4498 debugfs_create_file("descriptors_status", 0444, priv->dbgfs_dir, dev,
4499 &stmmac_rings_status_fops);
4500
4501 /* Entry to report the DMA HW features */
4502 debugfs_create_file("dma_cap", 0444, priv->dbgfs_dir, dev,
4503 &stmmac_dma_cap_fops);
4504
4505 rtnl_unlock();
4506 }
4507
stmmac_exit_fs(struct net_device * dev)4508 static void stmmac_exit_fs(struct net_device *dev)
4509 {
4510 struct stmmac_priv *priv = netdev_priv(dev);
4511
4512 debugfs_remove_recursive(priv->dbgfs_dir);
4513 }
4514 #endif /* CONFIG_DEBUG_FS */
4515
stmmac_vid_crc32_le(__le16 vid_le)4516 static u32 stmmac_vid_crc32_le(__le16 vid_le)
4517 {
4518 unsigned char *data = (unsigned char *)&vid_le;
4519 unsigned char data_byte = 0;
4520 u32 crc = ~0x0;
4521 u32 temp = 0;
4522 int i, bits;
4523
4524 bits = get_bitmask_order(VLAN_VID_MASK);
4525 for (i = 0; i < bits; i++) {
4526 if ((i % 8) == 0)
4527 data_byte = data[i / 8];
4528
4529 temp = ((crc & 1) ^ data_byte) & 1;
4530 crc >>= 1;
4531 data_byte >>= 1;
4532
4533 if (temp)
4534 crc ^= 0xedb88320;
4535 }
4536
4537 return crc;
4538 }
4539
stmmac_vlan_update(struct stmmac_priv * priv,bool is_double)4540 static int stmmac_vlan_update(struct stmmac_priv *priv, bool is_double)
4541 {
4542 u32 crc, hash = 0;
4543 __le16 pmatch = 0;
4544 int count = 0;
4545 u16 vid = 0;
4546
4547 for_each_set_bit(vid, priv->active_vlans, VLAN_N_VID) {
4548 __le16 vid_le = cpu_to_le16(vid);
4549 crc = bitrev32(~stmmac_vid_crc32_le(vid_le)) >> 28;
4550 hash |= (1 << crc);
4551 count++;
4552 }
4553
4554 if (!priv->dma_cap.vlhash) {
4555 if (count > 2) /* VID = 0 always passes filter */
4556 return -EOPNOTSUPP;
4557
4558 pmatch = cpu_to_le16(vid);
4559 hash = 0;
4560 }
4561
4562 return stmmac_update_vlan_hash(priv, priv->hw, hash, pmatch, is_double);
4563 }
4564
stmmac_vlan_rx_add_vid(struct net_device * ndev,__be16 proto,u16 vid)4565 static int stmmac_vlan_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid)
4566 {
4567 struct stmmac_priv *priv = netdev_priv(ndev);
4568 bool is_double = false;
4569 int ret;
4570
4571 if (be16_to_cpu(proto) == ETH_P_8021AD)
4572 is_double = true;
4573
4574 set_bit(vid, priv->active_vlans);
4575 ret = stmmac_vlan_update(priv, is_double);
4576 if (ret) {
4577 clear_bit(vid, priv->active_vlans);
4578 return ret;
4579 }
4580
4581 if (priv->hw->num_vlan) {
4582 ret = stmmac_add_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid);
4583 if (ret)
4584 return ret;
4585 }
4586
4587 return 0;
4588 }
4589
stmmac_vlan_rx_kill_vid(struct net_device * ndev,__be16 proto,u16 vid)4590 static int stmmac_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto, u16 vid)
4591 {
4592 struct stmmac_priv *priv = netdev_priv(ndev);
4593 bool is_double = false;
4594 int ret;
4595
4596 if (be16_to_cpu(proto) == ETH_P_8021AD)
4597 is_double = true;
4598
4599 clear_bit(vid, priv->active_vlans);
4600
4601 if (priv->hw->num_vlan) {
4602 ret = stmmac_del_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid);
4603 if (ret)
4604 return ret;
4605 }
4606
4607 return stmmac_vlan_update(priv, is_double);
4608 }
4609
4610 static const struct net_device_ops stmmac_netdev_ops = {
4611 .ndo_open = stmmac_open,
4612 .ndo_start_xmit = stmmac_xmit,
4613 .ndo_stop = stmmac_release,
4614 .ndo_change_mtu = stmmac_change_mtu,
4615 .ndo_fix_features = stmmac_fix_features,
4616 .ndo_set_features = stmmac_set_features,
4617 .ndo_set_rx_mode = stmmac_set_rx_mode,
4618 .ndo_tx_timeout = stmmac_tx_timeout,
4619 .ndo_do_ioctl = stmmac_ioctl,
4620 .ndo_setup_tc = stmmac_setup_tc,
4621 .ndo_select_queue = stmmac_select_queue,
4622 #ifdef CONFIG_NET_POLL_CONTROLLER
4623 .ndo_poll_controller = stmmac_poll_controller,
4624 #endif
4625 .ndo_set_mac_address = stmmac_set_mac_address,
4626 .ndo_vlan_rx_add_vid = stmmac_vlan_rx_add_vid,
4627 .ndo_vlan_rx_kill_vid = stmmac_vlan_rx_kill_vid,
4628 };
4629
stmmac_reset_subtask(struct stmmac_priv * priv)4630 static void stmmac_reset_subtask(struct stmmac_priv *priv)
4631 {
4632 if (!test_and_clear_bit(STMMAC_RESET_REQUESTED, &priv->state))
4633 return;
4634 if (test_bit(STMMAC_DOWN, &priv->state))
4635 return;
4636
4637 netdev_err(priv->dev, "Reset adapter.\n");
4638
4639 rtnl_lock();
4640 netif_trans_update(priv->dev);
4641 while (test_and_set_bit(STMMAC_RESETING, &priv->state))
4642 usleep_range(1000, 2000);
4643
4644 set_bit(STMMAC_DOWN, &priv->state);
4645 dev_close(priv->dev);
4646 dev_open(priv->dev, NULL);
4647 clear_bit(STMMAC_DOWN, &priv->state);
4648 clear_bit(STMMAC_RESETING, &priv->state);
4649 rtnl_unlock();
4650 }
4651
stmmac_service_task(struct work_struct * work)4652 static void stmmac_service_task(struct work_struct *work)
4653 {
4654 struct stmmac_priv *priv = container_of(work, struct stmmac_priv,
4655 service_task);
4656
4657 stmmac_reset_subtask(priv);
4658 clear_bit(STMMAC_SERVICE_SCHED, &priv->state);
4659 }
4660
4661 /**
4662 * stmmac_hw_init - Init the MAC device
4663 * @priv: driver private structure
4664 * Description: this function is to configure the MAC device according to
4665 * some platform parameters or the HW capability register. It prepares the
4666 * driver to use either ring or chain modes and to setup either enhanced or
4667 * normal descriptors.
4668 */
stmmac_hw_init(struct stmmac_priv * priv)4669 static int stmmac_hw_init(struct stmmac_priv *priv)
4670 {
4671 int ret;
4672
4673 /* dwmac-sun8i only work in chain mode */
4674 if (priv->plat->has_sun8i)
4675 chain_mode = 1;
4676 priv->chain_mode = chain_mode;
4677
4678 /* Initialize HW Interface */
4679 ret = stmmac_hwif_init(priv);
4680 if (ret)
4681 return ret;
4682
4683 /* Get the HW capability (new GMAC newer than 3.50a) */
4684 priv->hw_cap_support = stmmac_get_hw_features(priv);
4685 if (priv->hw_cap_support) {
4686 dev_info(priv->device, "DMA HW capability register supported\n");
4687
4688 /* We can override some gmac/dma configuration fields: e.g.
4689 * enh_desc, tx_coe (e.g. that are passed through the
4690 * platform) with the values from the HW capability
4691 * register (if supported).
4692 */
4693 priv->plat->enh_desc = priv->dma_cap.enh_desc;
4694 priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up;
4695 priv->hw->pmt = priv->plat->pmt;
4696 if (priv->dma_cap.hash_tb_sz) {
4697 priv->hw->multicast_filter_bins =
4698 (BIT(priv->dma_cap.hash_tb_sz) << 5);
4699 priv->hw->mcast_bits_log2 =
4700 ilog2(priv->hw->multicast_filter_bins);
4701 }
4702
4703 /* TXCOE doesn't work in thresh DMA mode */
4704 if (priv->plat->force_thresh_dma_mode)
4705 priv->plat->tx_coe = 0;
4706 else
4707 priv->plat->tx_coe = priv->dma_cap.tx_coe;
4708
4709 /* In case of GMAC4 rx_coe is from HW cap register. */
4710 priv->plat->rx_coe = priv->dma_cap.rx_coe;
4711
4712 if (priv->dma_cap.rx_coe_type2)
4713 priv->plat->rx_coe = STMMAC_RX_COE_TYPE2;
4714 else if (priv->dma_cap.rx_coe_type1)
4715 priv->plat->rx_coe = STMMAC_RX_COE_TYPE1;
4716
4717 } else {
4718 dev_info(priv->device, "No HW DMA feature register supported\n");
4719 }
4720
4721 if (priv->plat->rx_coe) {
4722 priv->hw->rx_csum = priv->plat->rx_coe;
4723 dev_info(priv->device, "RX Checksum Offload Engine supported\n");
4724 if (priv->synopsys_id < DWMAC_CORE_4_00)
4725 dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum);
4726 }
4727 if (priv->plat->tx_coe)
4728 dev_info(priv->device, "TX Checksum insertion supported\n");
4729
4730 if (priv->plat->pmt) {
4731 dev_info(priv->device, "Wake-Up On Lan supported\n");
4732 device_set_wakeup_capable(priv->device, 1);
4733 }
4734
4735 if (priv->dma_cap.tsoen)
4736 dev_info(priv->device, "TSO supported\n");
4737
4738 priv->hw->vlan_fail_q_en = priv->plat->vlan_fail_q_en;
4739 priv->hw->vlan_fail_q = priv->plat->vlan_fail_q;
4740
4741 /* Run HW quirks, if any */
4742 if (priv->hwif_quirks) {
4743 ret = priv->hwif_quirks(priv);
4744 if (ret)
4745 return ret;
4746 }
4747
4748 /* Rx Watchdog is available in the COREs newer than the 3.40.
4749 * In some case, for example on bugged HW this feature
4750 * has to be disable and this can be done by passing the
4751 * riwt_off field from the platform.
4752 */
4753 if (((priv->synopsys_id >= DWMAC_CORE_3_50) ||
4754 (priv->plat->has_xgmac)) && (!priv->plat->riwt_off)) {
4755 priv->use_riwt = 1;
4756 dev_info(priv->device,
4757 "Enable RX Mitigation via HW Watchdog Timer\n");
4758 }
4759
4760 return 0;
4761 }
4762
stmmac_napi_add(struct net_device * dev)4763 static void stmmac_napi_add(struct net_device *dev)
4764 {
4765 struct stmmac_priv *priv = netdev_priv(dev);
4766 u32 queue, maxq;
4767
4768 maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use);
4769
4770 for (queue = 0; queue < maxq; queue++) {
4771 struct stmmac_channel *ch = &priv->channel[queue];
4772
4773 ch->priv_data = priv;
4774 ch->index = queue;
4775 spin_lock_init(&ch->lock);
4776
4777 if (queue < priv->plat->rx_queues_to_use) {
4778 netif_napi_add(dev, &ch->rx_napi, stmmac_napi_poll_rx,
4779 NAPI_POLL_WEIGHT);
4780 }
4781 if (queue < priv->plat->tx_queues_to_use) {
4782 netif_tx_napi_add(dev, &ch->tx_napi,
4783 stmmac_napi_poll_tx,
4784 NAPI_POLL_WEIGHT);
4785 }
4786 }
4787 }
4788
stmmac_napi_del(struct net_device * dev)4789 static void stmmac_napi_del(struct net_device *dev)
4790 {
4791 struct stmmac_priv *priv = netdev_priv(dev);
4792 u32 queue, maxq;
4793
4794 maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use);
4795
4796 for (queue = 0; queue < maxq; queue++) {
4797 struct stmmac_channel *ch = &priv->channel[queue];
4798
4799 if (queue < priv->plat->rx_queues_to_use)
4800 netif_napi_del(&ch->rx_napi);
4801 if (queue < priv->plat->tx_queues_to_use)
4802 netif_napi_del(&ch->tx_napi);
4803 }
4804 }
4805
stmmac_reinit_queues(struct net_device * dev,u32 rx_cnt,u32 tx_cnt)4806 int stmmac_reinit_queues(struct net_device *dev, u32 rx_cnt, u32 tx_cnt)
4807 {
4808 struct stmmac_priv *priv = netdev_priv(dev);
4809 int ret = 0;
4810
4811 if (netif_running(dev))
4812 stmmac_release(dev);
4813
4814 stmmac_napi_del(dev);
4815
4816 priv->plat->rx_queues_to_use = rx_cnt;
4817 priv->plat->tx_queues_to_use = tx_cnt;
4818
4819 stmmac_napi_add(dev);
4820
4821 if (netif_running(dev))
4822 ret = stmmac_open(dev);
4823
4824 return ret;
4825 }
4826
stmmac_reinit_ringparam(struct net_device * dev,u32 rx_size,u32 tx_size)4827 int stmmac_reinit_ringparam(struct net_device *dev, u32 rx_size, u32 tx_size)
4828 {
4829 struct stmmac_priv *priv = netdev_priv(dev);
4830 int ret = 0;
4831
4832 if (netif_running(dev))
4833 stmmac_release(dev);
4834
4835 priv->dma_rx_size = rx_size;
4836 priv->dma_tx_size = tx_size;
4837
4838 if (netif_running(dev))
4839 ret = stmmac_open(dev);
4840
4841 return ret;
4842 }
4843
4844 /**
4845 * stmmac_dvr_probe
4846 * @device: device pointer
4847 * @plat_dat: platform data pointer
4848 * @res: stmmac resource pointer
4849 * Description: this is the main probe function used to
4850 * call the alloc_etherdev, allocate the priv structure.
4851 * Return:
4852 * returns 0 on success, otherwise errno.
4853 */
stmmac_dvr_probe(struct device * device,struct plat_stmmacenet_data * plat_dat,struct stmmac_resources * res)4854 int stmmac_dvr_probe(struct device *device,
4855 struct plat_stmmacenet_data *plat_dat,
4856 struct stmmac_resources *res)
4857 {
4858 struct net_device *ndev = NULL;
4859 struct stmmac_priv *priv;
4860 u32 rxq;
4861 int i, ret = 0;
4862
4863 ndev = devm_alloc_etherdev_mqs(device, sizeof(struct stmmac_priv),
4864 MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES);
4865 if (!ndev)
4866 return -ENOMEM;
4867
4868 SET_NETDEV_DEV(ndev, device);
4869
4870 priv = netdev_priv(ndev);
4871 priv->device = device;
4872 priv->dev = ndev;
4873
4874 stmmac_set_ethtool_ops(ndev);
4875 priv->pause = pause;
4876 priv->plat = plat_dat;
4877 priv->ioaddr = res->addr;
4878 priv->dev->base_addr = (unsigned long)res->addr;
4879
4880 priv->dev->irq = res->irq;
4881 priv->wol_irq = res->wol_irq;
4882 priv->lpi_irq = res->lpi_irq;
4883
4884 if (!IS_ERR_OR_NULL(res->mac))
4885 memcpy(priv->dev->dev_addr, res->mac, ETH_ALEN);
4886
4887 dev_set_drvdata(device, priv->dev);
4888
4889 /* Verify driver arguments */
4890 stmmac_verify_args();
4891
4892 /* Allocate workqueue */
4893 priv->wq = create_singlethread_workqueue("stmmac_wq");
4894 if (!priv->wq) {
4895 dev_err(priv->device, "failed to create workqueue\n");
4896 return -ENOMEM;
4897 }
4898
4899 INIT_WORK(&priv->service_task, stmmac_service_task);
4900
4901 /* Override with kernel parameters if supplied XXX CRS XXX
4902 * this needs to have multiple instances
4903 */
4904 if ((phyaddr >= 0) && (phyaddr <= 31))
4905 priv->plat->phy_addr = phyaddr;
4906
4907 if (priv->plat->stmmac_rst) {
4908 ret = reset_control_assert(priv->plat->stmmac_rst);
4909 reset_control_deassert(priv->plat->stmmac_rst);
4910 /* Some reset controllers have only reset callback instead of
4911 * assert + deassert callbacks pair.
4912 */
4913 if (ret == -ENOTSUPP)
4914 reset_control_reset(priv->plat->stmmac_rst);
4915 }
4916
4917 /* Init MAC and get the capabilities */
4918 ret = stmmac_hw_init(priv);
4919 if (ret)
4920 goto error_hw_init;
4921
4922 stmmac_check_ether_addr(priv);
4923
4924 ndev->netdev_ops = &stmmac_netdev_ops;
4925
4926 ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
4927 NETIF_F_RXCSUM;
4928
4929 ret = stmmac_tc_init(priv, priv);
4930 if (!ret) {
4931 ndev->hw_features |= NETIF_F_HW_TC;
4932 }
4933
4934 if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) {
4935 ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6;
4936 if (priv->plat->has_gmac4)
4937 ndev->hw_features |= NETIF_F_GSO_UDP_L4;
4938 priv->tso = true;
4939 dev_info(priv->device, "TSO feature enabled\n");
4940 }
4941
4942 if (priv->dma_cap.sphen) {
4943 ndev->hw_features |= NETIF_F_GRO;
4944 priv->sph = true;
4945 dev_info(priv->device, "SPH feature enabled\n");
4946 }
4947
4948 /* The current IP register MAC_HW_Feature1[ADDR64] only define
4949 * 32/40/64 bit width, but some SOC support others like i.MX8MP
4950 * support 34 bits but it map to 40 bits width in MAC_HW_Feature1[ADDR64].
4951 * So overwrite dma_cap.addr64 according to HW real design.
4952 */
4953 if (priv->plat->addr64)
4954 priv->dma_cap.addr64 = priv->plat->addr64;
4955
4956 if (priv->dma_cap.addr64) {
4957 ret = dma_set_mask_and_coherent(device,
4958 DMA_BIT_MASK(priv->dma_cap.addr64));
4959 if (!ret) {
4960 dev_info(priv->device, "Using %d bits DMA width\n",
4961 priv->dma_cap.addr64);
4962
4963 /*
4964 * If more than 32 bits can be addressed, make sure to
4965 * enable enhanced addressing mode.
4966 */
4967 if (IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT))
4968 priv->plat->dma_cfg->eame = true;
4969 } else {
4970 ret = dma_set_mask_and_coherent(device, DMA_BIT_MASK(32));
4971 if (ret) {
4972 dev_err(priv->device, "Failed to set DMA Mask\n");
4973 goto error_hw_init;
4974 }
4975
4976 priv->dma_cap.addr64 = 32;
4977 }
4978 }
4979
4980 ndev->features |= ndev->hw_features | NETIF_F_HIGHDMA;
4981 ndev->watchdog_timeo = msecs_to_jiffies(watchdog);
4982 #ifdef STMMAC_VLAN_TAG_USED
4983 /* Both mac100 and gmac support receive VLAN tag detection */
4984 ndev->features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX;
4985 if (priv->dma_cap.vlhash) {
4986 ndev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
4987 ndev->features |= NETIF_F_HW_VLAN_STAG_FILTER;
4988 }
4989 if (priv->dma_cap.vlins) {
4990 ndev->features |= NETIF_F_HW_VLAN_CTAG_TX;
4991 if (priv->dma_cap.dvlan)
4992 ndev->features |= NETIF_F_HW_VLAN_STAG_TX;
4993 }
4994 #endif
4995 priv->msg_enable = netif_msg_init(debug, default_msg_level);
4996
4997 /* Initialize RSS */
4998 rxq = priv->plat->rx_queues_to_use;
4999 netdev_rss_key_fill(priv->rss.key, sizeof(priv->rss.key));
5000 for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++)
5001 priv->rss.table[i] = ethtool_rxfh_indir_default(i, rxq);
5002
5003 if (priv->dma_cap.rssen && priv->plat->rss_en)
5004 ndev->features |= NETIF_F_RXHASH;
5005
5006 /* MTU range: 46 - hw-specific max */
5007 ndev->min_mtu = ETH_ZLEN - ETH_HLEN;
5008 if (priv->plat->has_xgmac)
5009 ndev->max_mtu = XGMAC_JUMBO_LEN;
5010 else if ((priv->plat->enh_desc) || (priv->synopsys_id >= DWMAC_CORE_4_00))
5011 ndev->max_mtu = JUMBO_LEN;
5012 else
5013 ndev->max_mtu = SKB_MAX_HEAD(NET_SKB_PAD + NET_IP_ALIGN);
5014 /* Will not overwrite ndev->max_mtu if plat->maxmtu > ndev->max_mtu
5015 * as well as plat->maxmtu < ndev->min_mtu which is a invalid range.
5016 */
5017 if ((priv->plat->maxmtu < ndev->max_mtu) &&
5018 (priv->plat->maxmtu >= ndev->min_mtu))
5019 ndev->max_mtu = priv->plat->maxmtu;
5020 else if (priv->plat->maxmtu < ndev->min_mtu)
5021 dev_warn(priv->device,
5022 "%s: warning: maxmtu having invalid value (%d)\n",
5023 __func__, priv->plat->maxmtu);
5024
5025 if (flow_ctrl)
5026 priv->flow_ctrl = FLOW_AUTO; /* RX/TX pause on */
5027
5028 /* Setup channels NAPI */
5029 stmmac_napi_add(ndev);
5030
5031 mutex_init(&priv->lock);
5032
5033 /* If a specific clk_csr value is passed from the platform
5034 * this means that the CSR Clock Range selection cannot be
5035 * changed at run-time and it is fixed. Viceversa the driver'll try to
5036 * set the MDC clock dynamically according to the csr actual
5037 * clock input.
5038 */
5039 if (priv->plat->clk_csr >= 0)
5040 priv->clk_csr = priv->plat->clk_csr;
5041 else
5042 stmmac_clk_csr_set(priv);
5043
5044 stmmac_check_pcs_mode(priv);
5045
5046 if (priv->hw->pcs != STMMAC_PCS_TBI &&
5047 priv->hw->pcs != STMMAC_PCS_RTBI) {
5048 /* MDIO bus Registration */
5049 ret = stmmac_mdio_register(ndev);
5050 if (ret < 0) {
5051 dev_err(priv->device,
5052 "%s: MDIO bus (id: %d) registration failed",
5053 __func__, priv->plat->bus_id);
5054 goto error_mdio_register;
5055 }
5056 }
5057
5058 ret = stmmac_phy_setup(priv);
5059 if (ret) {
5060 netdev_err(ndev, "failed to setup phy (%d)\n", ret);
5061 goto error_phy_setup;
5062 }
5063
5064 ret = register_netdev(ndev);
5065 if (ret) {
5066 dev_err(priv->device, "%s: ERROR %i registering the device\n",
5067 __func__, ret);
5068 goto error_netdev_register;
5069 }
5070
5071 if (priv->plat->serdes_powerup) {
5072 ret = priv->plat->serdes_powerup(ndev,
5073 priv->plat->bsp_priv);
5074
5075 if (ret < 0)
5076 goto error_serdes_powerup;
5077 }
5078
5079 #ifdef CONFIG_DEBUG_FS
5080 stmmac_init_fs(ndev);
5081 #endif
5082
5083 return ret;
5084
5085 error_serdes_powerup:
5086 unregister_netdev(ndev);
5087 error_netdev_register:
5088 phylink_destroy(priv->phylink);
5089 error_phy_setup:
5090 if (priv->hw->pcs != STMMAC_PCS_TBI &&
5091 priv->hw->pcs != STMMAC_PCS_RTBI)
5092 stmmac_mdio_unregister(ndev);
5093 error_mdio_register:
5094 stmmac_napi_del(ndev);
5095 error_hw_init:
5096 destroy_workqueue(priv->wq);
5097
5098 return ret;
5099 }
5100 EXPORT_SYMBOL_GPL(stmmac_dvr_probe);
5101
5102 /**
5103 * stmmac_dvr_remove
5104 * @dev: device pointer
5105 * Description: this function resets the TX/RX processes, disables the MAC RX/TX
5106 * changes the link status, releases the DMA descriptor rings.
5107 */
stmmac_dvr_remove(struct device * dev)5108 int stmmac_dvr_remove(struct device *dev)
5109 {
5110 struct net_device *ndev = dev_get_drvdata(dev);
5111 struct stmmac_priv *priv = netdev_priv(ndev);
5112
5113 netdev_info(priv->dev, "%s: removing driver", __func__);
5114
5115 stmmac_stop_all_dma(priv);
5116
5117 if (priv->plat->serdes_powerdown)
5118 priv->plat->serdes_powerdown(ndev, priv->plat->bsp_priv);
5119
5120 stmmac_mac_set(priv, priv->ioaddr, false);
5121 netif_carrier_off(ndev);
5122 unregister_netdev(ndev);
5123 #ifdef CONFIG_DEBUG_FS
5124 stmmac_exit_fs(ndev);
5125 #endif
5126 phylink_destroy(priv->phylink);
5127 if (priv->plat->stmmac_rst)
5128 reset_control_assert(priv->plat->stmmac_rst);
5129 clk_disable_unprepare(priv->plat->pclk);
5130 clk_disable_unprepare(priv->plat->stmmac_clk);
5131 if (priv->hw->pcs != STMMAC_PCS_TBI &&
5132 priv->hw->pcs != STMMAC_PCS_RTBI)
5133 stmmac_mdio_unregister(ndev);
5134 destroy_workqueue(priv->wq);
5135 mutex_destroy(&priv->lock);
5136
5137 return 0;
5138 }
5139 EXPORT_SYMBOL_GPL(stmmac_dvr_remove);
5140
5141 /**
5142 * stmmac_suspend - suspend callback
5143 * @dev: device pointer
5144 * Description: this is the function to suspend the device and it is called
5145 * by the platform driver to stop the network queue, release the resources,
5146 * program the PMT register (for WoL), clean and release driver resources.
5147 */
stmmac_suspend(struct device * dev)5148 int stmmac_suspend(struct device *dev)
5149 {
5150 struct net_device *ndev = dev_get_drvdata(dev);
5151 struct stmmac_priv *priv = netdev_priv(ndev);
5152 u32 chan;
5153
5154 if (!ndev || !netif_running(ndev))
5155 return 0;
5156
5157 phylink_mac_change(priv->phylink, false);
5158
5159 mutex_lock(&priv->lock);
5160
5161 netif_device_detach(ndev);
5162
5163 stmmac_disable_all_queues(priv);
5164
5165 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)
5166 del_timer_sync(&priv->tx_queue[chan].txtimer);
5167
5168 if (priv->eee_enabled) {
5169 priv->tx_path_in_lpi_mode = false;
5170 del_timer_sync(&priv->eee_ctrl_timer);
5171 }
5172
5173 /* Stop TX/RX DMA */
5174 stmmac_stop_all_dma(priv);
5175
5176 if (priv->plat->serdes_powerdown)
5177 priv->plat->serdes_powerdown(ndev, priv->plat->bsp_priv);
5178
5179 /* Enable Power down mode by programming the PMT regs */
5180 if (device_may_wakeup(priv->device) && priv->plat->pmt) {
5181 stmmac_pmt(priv, priv->hw, priv->wolopts);
5182 priv->irq_wake = 1;
5183 } else {
5184 mutex_unlock(&priv->lock);
5185 rtnl_lock();
5186 if (device_may_wakeup(priv->device))
5187 phylink_speed_down(priv->phylink, false);
5188 phylink_stop(priv->phylink);
5189 rtnl_unlock();
5190 mutex_lock(&priv->lock);
5191
5192 stmmac_mac_set(priv, priv->ioaddr, false);
5193 pinctrl_pm_select_sleep_state(priv->device);
5194 /* Disable clock in case of PWM is off */
5195 clk_disable_unprepare(priv->plat->clk_ptp_ref);
5196 clk_disable_unprepare(priv->plat->pclk);
5197 clk_disable_unprepare(priv->plat->stmmac_clk);
5198 }
5199 mutex_unlock(&priv->lock);
5200
5201 priv->speed = SPEED_UNKNOWN;
5202 return 0;
5203 }
5204 EXPORT_SYMBOL_GPL(stmmac_suspend);
5205
5206 /**
5207 * stmmac_reset_queues_param - reset queue parameters
5208 * @priv: device pointer
5209 */
stmmac_reset_queues_param(struct stmmac_priv * priv)5210 static void stmmac_reset_queues_param(struct stmmac_priv *priv)
5211 {
5212 u32 rx_cnt = priv->plat->rx_queues_to_use;
5213 u32 tx_cnt = priv->plat->tx_queues_to_use;
5214 u32 queue;
5215
5216 for (queue = 0; queue < rx_cnt; queue++) {
5217 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
5218
5219 rx_q->cur_rx = 0;
5220 rx_q->dirty_rx = 0;
5221 }
5222
5223 for (queue = 0; queue < tx_cnt; queue++) {
5224 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
5225
5226 tx_q->cur_tx = 0;
5227 tx_q->dirty_tx = 0;
5228 tx_q->mss = 0;
5229 }
5230 }
5231
5232 /**
5233 * stmmac_resume - resume callback
5234 * @dev: device pointer
5235 * Description: when resume this function is invoked to setup the DMA and CORE
5236 * in a usable state.
5237 */
stmmac_resume(struct device * dev)5238 int stmmac_resume(struct device *dev)
5239 {
5240 struct net_device *ndev = dev_get_drvdata(dev);
5241 struct stmmac_priv *priv = netdev_priv(ndev);
5242 int ret;
5243
5244 if (!netif_running(ndev))
5245 return 0;
5246
5247 /* Power Down bit, into the PM register, is cleared
5248 * automatically as soon as a magic packet or a Wake-up frame
5249 * is received. Anyway, it's better to manually clear
5250 * this bit because it can generate problems while resuming
5251 * from another devices (e.g. serial console).
5252 */
5253 if (device_may_wakeup(priv->device) && priv->plat->pmt) {
5254 mutex_lock(&priv->lock);
5255 stmmac_pmt(priv, priv->hw, 0);
5256 mutex_unlock(&priv->lock);
5257 priv->irq_wake = 0;
5258 } else {
5259 pinctrl_pm_select_default_state(priv->device);
5260 /* enable the clk previously disabled */
5261 clk_prepare_enable(priv->plat->stmmac_clk);
5262 clk_prepare_enable(priv->plat->pclk);
5263 if (priv->plat->clk_ptp_ref)
5264 clk_prepare_enable(priv->plat->clk_ptp_ref);
5265 /* reset the phy so that it's ready */
5266 if (priv->mii)
5267 stmmac_mdio_reset(priv->mii);
5268 }
5269
5270 if (priv->plat->serdes_powerup) {
5271 ret = priv->plat->serdes_powerup(ndev,
5272 priv->plat->bsp_priv);
5273
5274 if (ret < 0)
5275 return ret;
5276 }
5277
5278 if (!device_may_wakeup(priv->device) || !priv->plat->pmt) {
5279 rtnl_lock();
5280 phylink_start(priv->phylink);
5281 /* We may have called phylink_speed_down before */
5282 phylink_speed_up(priv->phylink);
5283 rtnl_unlock();
5284 }
5285
5286 rtnl_lock();
5287 mutex_lock(&priv->lock);
5288
5289 stmmac_reset_queues_param(priv);
5290
5291 stmmac_free_tx_skbufs(priv);
5292 stmmac_clear_descriptors(priv);
5293
5294 stmmac_hw_setup(ndev, false);
5295 stmmac_init_coalesce(priv);
5296 stmmac_set_rx_mode(ndev);
5297
5298 stmmac_restore_hw_vlan_rx_fltr(priv, ndev, priv->hw);
5299
5300 stmmac_enable_all_queues(priv);
5301
5302 mutex_unlock(&priv->lock);
5303 rtnl_unlock();
5304
5305 phylink_mac_change(priv->phylink, true);
5306
5307 netif_device_attach(ndev);
5308
5309 return 0;
5310 }
5311 EXPORT_SYMBOL_GPL(stmmac_resume);
5312
5313 #ifndef MODULE
stmmac_cmdline_opt(char * str)5314 static int __init stmmac_cmdline_opt(char *str)
5315 {
5316 char *opt;
5317
5318 if (!str || !*str)
5319 return -EINVAL;
5320 while ((opt = strsep(&str, ",")) != NULL) {
5321 if (!strncmp(opt, "debug:", 6)) {
5322 if (kstrtoint(opt + 6, 0, &debug))
5323 goto err;
5324 } else if (!strncmp(opt, "phyaddr:", 8)) {
5325 if (kstrtoint(opt + 8, 0, &phyaddr))
5326 goto err;
5327 } else if (!strncmp(opt, "buf_sz:", 7)) {
5328 if (kstrtoint(opt + 7, 0, &buf_sz))
5329 goto err;
5330 } else if (!strncmp(opt, "tc:", 3)) {
5331 if (kstrtoint(opt + 3, 0, &tc))
5332 goto err;
5333 } else if (!strncmp(opt, "watchdog:", 9)) {
5334 if (kstrtoint(opt + 9, 0, &watchdog))
5335 goto err;
5336 } else if (!strncmp(opt, "flow_ctrl:", 10)) {
5337 if (kstrtoint(opt + 10, 0, &flow_ctrl))
5338 goto err;
5339 } else if (!strncmp(opt, "pause:", 6)) {
5340 if (kstrtoint(opt + 6, 0, &pause))
5341 goto err;
5342 } else if (!strncmp(opt, "eee_timer:", 10)) {
5343 if (kstrtoint(opt + 10, 0, &eee_timer))
5344 goto err;
5345 } else if (!strncmp(opt, "chain_mode:", 11)) {
5346 if (kstrtoint(opt + 11, 0, &chain_mode))
5347 goto err;
5348 }
5349 }
5350 return 0;
5351
5352 err:
5353 pr_err("%s: ERROR broken module parameter conversion", __func__);
5354 return -EINVAL;
5355 }
5356
5357 __setup("stmmaceth=", stmmac_cmdline_opt);
5358 #endif /* MODULE */
5359
stmmac_init(void)5360 static int __init stmmac_init(void)
5361 {
5362 #ifdef CONFIG_DEBUG_FS
5363 /* Create debugfs main directory if it doesn't exist yet */
5364 if (!stmmac_fs_dir)
5365 stmmac_fs_dir = debugfs_create_dir(STMMAC_RESOURCE_NAME, NULL);
5366 register_netdevice_notifier(&stmmac_notifier);
5367 #endif
5368
5369 return 0;
5370 }
5371
stmmac_exit(void)5372 static void __exit stmmac_exit(void)
5373 {
5374 #ifdef CONFIG_DEBUG_FS
5375 unregister_netdevice_notifier(&stmmac_notifier);
5376 debugfs_remove_recursive(stmmac_fs_dir);
5377 #endif
5378 }
5379
5380 module_init(stmmac_init)
5381 module_exit(stmmac_exit)
5382
5383 MODULE_DESCRIPTION("STMMAC 10/100/1000 Ethernet device driver");
5384 MODULE_AUTHOR("Giuseppe Cavallaro <peppe.cavallaro@st.com>");
5385 MODULE_LICENSE("GPL");
5386
