1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (C) 2021 Gerhard Engleder <gerhard@engleder-embedded.com> */
3
4 /* TSN endpoint Ethernet MAC driver
5 *
6 * The TSN endpoint Ethernet MAC is a FPGA based network device for real-time
7 * communication. It is designed for endpoints within TSN (Time Sensitive
8 * Networking) networks; e.g., for PLCs in the industrial automation case.
9 *
10 * It supports multiple TX/RX queue pairs. The first TX/RX queue pair is used
11 * by the driver.
12 *
13 * More information can be found here:
14 * - www.embedded-experts.at/tsn
15 * - www.engleder-embedded.com
16 */
17
18 #include "tsnep.h"
19 #include "tsnep_hw.h"
20
21 #include <linux/module.h>
22 #include <linux/of.h>
23 #include <linux/of_net.h>
24 #include <linux/of_mdio.h>
25 #include <linux/interrupt.h>
26 #include <linux/etherdevice.h>
27 #include <linux/phy.h>
28 #include <linux/iopoll.h>
29 #include <linux/bpf.h>
30 #include <linux/bpf_trace.h>
31 #include <net/page_pool/helpers.h>
32 #include <net/xdp_sock_drv.h>
33
34 #define TSNEP_RX_OFFSET (max(NET_SKB_PAD, XDP_PACKET_HEADROOM) + NET_IP_ALIGN)
35 #define TSNEP_HEADROOM ALIGN(TSNEP_RX_OFFSET, 4)
36 #define TSNEP_MAX_RX_BUF_SIZE (PAGE_SIZE - TSNEP_HEADROOM - \
37 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
38 /* XSK buffer shall store at least Q-in-Q frame */
39 #define TSNEP_XSK_RX_BUF_SIZE (ALIGN(TSNEP_RX_INLINE_METADATA_SIZE + \
40 ETH_FRAME_LEN + ETH_FCS_LEN + \
41 VLAN_HLEN * 2, 4))
42
43 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
44 #define DMA_ADDR_HIGH(dma_addr) ((u32)(((dma_addr) >> 32) & 0xFFFFFFFF))
45 #else
46 #define DMA_ADDR_HIGH(dma_addr) ((u32)(0))
47 #endif
48 #define DMA_ADDR_LOW(dma_addr) ((u32)((dma_addr) & 0xFFFFFFFF))
49
50 #define TSNEP_COALESCE_USECS_DEFAULT 64
51 #define TSNEP_COALESCE_USECS_MAX ((ECM_INT_DELAY_MASK >> ECM_INT_DELAY_SHIFT) * \
52 ECM_INT_DELAY_BASE_US + ECM_INT_DELAY_BASE_US - 1)
53
54 #define TSNEP_TX_TYPE_SKB BIT(0)
55 #define TSNEP_TX_TYPE_SKB_FRAG BIT(1)
56 #define TSNEP_TX_TYPE_XDP_TX BIT(2)
57 #define TSNEP_TX_TYPE_XDP_NDO BIT(3)
58 #define TSNEP_TX_TYPE_XDP (TSNEP_TX_TYPE_XDP_TX | TSNEP_TX_TYPE_XDP_NDO)
59 #define TSNEP_TX_TYPE_XSK BIT(4)
60
61 #define TSNEP_XDP_TX BIT(0)
62 #define TSNEP_XDP_REDIRECT BIT(1)
63
tsnep_enable_irq(struct tsnep_adapter * adapter,u32 mask)64 static void tsnep_enable_irq(struct tsnep_adapter *adapter, u32 mask)
65 {
66 iowrite32(mask, adapter->addr + ECM_INT_ENABLE);
67 }
68
tsnep_disable_irq(struct tsnep_adapter * adapter,u32 mask)69 static void tsnep_disable_irq(struct tsnep_adapter *adapter, u32 mask)
70 {
71 mask |= ECM_INT_DISABLE;
72 iowrite32(mask, adapter->addr + ECM_INT_ENABLE);
73 }
74
tsnep_irq(int irq,void * arg)75 static irqreturn_t tsnep_irq(int irq, void *arg)
76 {
77 struct tsnep_adapter *adapter = arg;
78 u32 active = ioread32(adapter->addr + ECM_INT_ACTIVE);
79
80 /* acknowledge interrupt */
81 if (active != 0)
82 iowrite32(active, adapter->addr + ECM_INT_ACKNOWLEDGE);
83
84 /* handle link interrupt */
85 if ((active & ECM_INT_LINK) != 0)
86 phy_mac_interrupt(adapter->netdev->phydev);
87
88 /* handle TX/RX queue 0 interrupt */
89 if ((active & adapter->queue[0].irq_mask) != 0) {
90 if (napi_schedule_prep(&adapter->queue[0].napi)) {
91 tsnep_disable_irq(adapter, adapter->queue[0].irq_mask);
92 /* schedule after masking to avoid races */
93 __napi_schedule(&adapter->queue[0].napi);
94 }
95 }
96
97 return IRQ_HANDLED;
98 }
99
tsnep_irq_txrx(int irq,void * arg)100 static irqreturn_t tsnep_irq_txrx(int irq, void *arg)
101 {
102 struct tsnep_queue *queue = arg;
103
104 /* handle TX/RX queue interrupt */
105 if (napi_schedule_prep(&queue->napi)) {
106 tsnep_disable_irq(queue->adapter, queue->irq_mask);
107 /* schedule after masking to avoid races */
108 __napi_schedule(&queue->napi);
109 }
110
111 return IRQ_HANDLED;
112 }
113
tsnep_set_irq_coalesce(struct tsnep_queue * queue,u32 usecs)114 int tsnep_set_irq_coalesce(struct tsnep_queue *queue, u32 usecs)
115 {
116 if (usecs > TSNEP_COALESCE_USECS_MAX)
117 return -ERANGE;
118
119 usecs /= ECM_INT_DELAY_BASE_US;
120 usecs <<= ECM_INT_DELAY_SHIFT;
121 usecs &= ECM_INT_DELAY_MASK;
122
123 queue->irq_delay &= ~ECM_INT_DELAY_MASK;
124 queue->irq_delay |= usecs;
125 iowrite8(queue->irq_delay, queue->irq_delay_addr);
126
127 return 0;
128 }
129
tsnep_get_irq_coalesce(struct tsnep_queue * queue)130 u32 tsnep_get_irq_coalesce(struct tsnep_queue *queue)
131 {
132 u32 usecs;
133
134 usecs = (queue->irq_delay & ECM_INT_DELAY_MASK);
135 usecs >>= ECM_INT_DELAY_SHIFT;
136 usecs *= ECM_INT_DELAY_BASE_US;
137
138 return usecs;
139 }
140
tsnep_mdiobus_read(struct mii_bus * bus,int addr,int regnum)141 static int tsnep_mdiobus_read(struct mii_bus *bus, int addr, int regnum)
142 {
143 struct tsnep_adapter *adapter = bus->priv;
144 u32 md;
145 int retval;
146
147 md = ECM_MD_READ;
148 if (!adapter->suppress_preamble)
149 md |= ECM_MD_PREAMBLE;
150 md |= (regnum << ECM_MD_ADDR_SHIFT) & ECM_MD_ADDR_MASK;
151 md |= (addr << ECM_MD_PHY_ADDR_SHIFT) & ECM_MD_PHY_ADDR_MASK;
152 iowrite32(md, adapter->addr + ECM_MD_CONTROL);
153 retval = readl_poll_timeout_atomic(adapter->addr + ECM_MD_STATUS, md,
154 !(md & ECM_MD_BUSY), 16, 1000);
155 if (retval != 0)
156 return retval;
157
158 return (md & ECM_MD_DATA_MASK) >> ECM_MD_DATA_SHIFT;
159 }
160
tsnep_mdiobus_write(struct mii_bus * bus,int addr,int regnum,u16 val)161 static int tsnep_mdiobus_write(struct mii_bus *bus, int addr, int regnum,
162 u16 val)
163 {
164 struct tsnep_adapter *adapter = bus->priv;
165 u32 md;
166 int retval;
167
168 md = ECM_MD_WRITE;
169 if (!adapter->suppress_preamble)
170 md |= ECM_MD_PREAMBLE;
171 md |= (regnum << ECM_MD_ADDR_SHIFT) & ECM_MD_ADDR_MASK;
172 md |= (addr << ECM_MD_PHY_ADDR_SHIFT) & ECM_MD_PHY_ADDR_MASK;
173 md |= ((u32)val << ECM_MD_DATA_SHIFT) & ECM_MD_DATA_MASK;
174 iowrite32(md, adapter->addr + ECM_MD_CONTROL);
175 retval = readl_poll_timeout_atomic(adapter->addr + ECM_MD_STATUS, md,
176 !(md & ECM_MD_BUSY), 16, 1000);
177 if (retval != 0)
178 return retval;
179
180 return 0;
181 }
182
tsnep_set_link_mode(struct tsnep_adapter * adapter)183 static void tsnep_set_link_mode(struct tsnep_adapter *adapter)
184 {
185 u32 mode;
186
187 switch (adapter->phydev->speed) {
188 case SPEED_100:
189 mode = ECM_LINK_MODE_100;
190 break;
191 case SPEED_1000:
192 mode = ECM_LINK_MODE_1000;
193 break;
194 default:
195 mode = ECM_LINK_MODE_OFF;
196 break;
197 }
198 iowrite32(mode, adapter->addr + ECM_STATUS);
199 }
200
tsnep_phy_link_status_change(struct net_device * netdev)201 static void tsnep_phy_link_status_change(struct net_device *netdev)
202 {
203 struct tsnep_adapter *adapter = netdev_priv(netdev);
204 struct phy_device *phydev = netdev->phydev;
205
206 if (phydev->link)
207 tsnep_set_link_mode(adapter);
208
209 phy_print_status(netdev->phydev);
210 }
211
tsnep_phy_loopback(struct tsnep_adapter * adapter,bool enable)212 static int tsnep_phy_loopback(struct tsnep_adapter *adapter, bool enable)
213 {
214 int retval;
215
216 retval = phy_loopback(adapter->phydev, enable);
217
218 /* PHY link state change is not signaled if loopback is enabled, it
219 * would delay a working loopback anyway, let's ensure that loopback
220 * is working immediately by setting link mode directly
221 */
222 if (!retval && enable)
223 tsnep_set_link_mode(adapter);
224
225 return retval;
226 }
227
tsnep_phy_open(struct tsnep_adapter * adapter)228 static int tsnep_phy_open(struct tsnep_adapter *adapter)
229 {
230 struct phy_device *phydev;
231 struct ethtool_eee ethtool_eee;
232 int retval;
233
234 retval = phy_connect_direct(adapter->netdev, adapter->phydev,
235 tsnep_phy_link_status_change,
236 adapter->phy_mode);
237 if (retval)
238 return retval;
239 phydev = adapter->netdev->phydev;
240
241 /* MAC supports only 100Mbps|1000Mbps full duplex
242 * SPE (Single Pair Ethernet) is also an option but not implemented yet
243 */
244 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_10baseT_Half_BIT);
245 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_10baseT_Full_BIT);
246 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_100baseT_Half_BIT);
247 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_1000baseT_Half_BIT);
248
249 /* disable EEE autoneg, EEE not supported by TSNEP */
250 memset(ðtool_eee, 0, sizeof(ethtool_eee));
251 phy_ethtool_set_eee(adapter->phydev, ðtool_eee);
252
253 adapter->phydev->irq = PHY_MAC_INTERRUPT;
254 phy_start(adapter->phydev);
255
256 return 0;
257 }
258
tsnep_phy_close(struct tsnep_adapter * adapter)259 static void tsnep_phy_close(struct tsnep_adapter *adapter)
260 {
261 phy_stop(adapter->netdev->phydev);
262 phy_disconnect(adapter->netdev->phydev);
263 }
264
tsnep_tx_ring_cleanup(struct tsnep_tx * tx)265 static void tsnep_tx_ring_cleanup(struct tsnep_tx *tx)
266 {
267 struct device *dmadev = tx->adapter->dmadev;
268 int i;
269
270 memset(tx->entry, 0, sizeof(tx->entry));
271
272 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) {
273 if (tx->page[i]) {
274 dma_free_coherent(dmadev, PAGE_SIZE, tx->page[i],
275 tx->page_dma[i]);
276 tx->page[i] = NULL;
277 tx->page_dma[i] = 0;
278 }
279 }
280 }
281
tsnep_tx_ring_create(struct tsnep_tx * tx)282 static int tsnep_tx_ring_create(struct tsnep_tx *tx)
283 {
284 struct device *dmadev = tx->adapter->dmadev;
285 struct tsnep_tx_entry *entry;
286 struct tsnep_tx_entry *next_entry;
287 int i, j;
288 int retval;
289
290 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) {
291 tx->page[i] =
292 dma_alloc_coherent(dmadev, PAGE_SIZE, &tx->page_dma[i],
293 GFP_KERNEL);
294 if (!tx->page[i]) {
295 retval = -ENOMEM;
296 goto alloc_failed;
297 }
298 for (j = 0; j < TSNEP_RING_ENTRIES_PER_PAGE; j++) {
299 entry = &tx->entry[TSNEP_RING_ENTRIES_PER_PAGE * i + j];
300 entry->desc_wb = (struct tsnep_tx_desc_wb *)
301 (((u8 *)tx->page[i]) + TSNEP_DESC_SIZE * j);
302 entry->desc = (struct tsnep_tx_desc *)
303 (((u8 *)entry->desc_wb) + TSNEP_DESC_OFFSET);
304 entry->desc_dma = tx->page_dma[i] + TSNEP_DESC_SIZE * j;
305 entry->owner_user_flag = false;
306 }
307 }
308 for (i = 0; i < TSNEP_RING_SIZE; i++) {
309 entry = &tx->entry[i];
310 next_entry = &tx->entry[(i + 1) & TSNEP_RING_MASK];
311 entry->desc->next = __cpu_to_le64(next_entry->desc_dma);
312 }
313
314 return 0;
315
316 alloc_failed:
317 tsnep_tx_ring_cleanup(tx);
318 return retval;
319 }
320
tsnep_tx_init(struct tsnep_tx * tx)321 static void tsnep_tx_init(struct tsnep_tx *tx)
322 {
323 dma_addr_t dma;
324
325 dma = tx->entry[0].desc_dma | TSNEP_RESET_OWNER_COUNTER;
326 iowrite32(DMA_ADDR_LOW(dma), tx->addr + TSNEP_TX_DESC_ADDR_LOW);
327 iowrite32(DMA_ADDR_HIGH(dma), tx->addr + TSNEP_TX_DESC_ADDR_HIGH);
328 tx->write = 0;
329 tx->read = 0;
330 tx->owner_counter = 1;
331 tx->increment_owner_counter = TSNEP_RING_SIZE - 1;
332 }
333
tsnep_tx_enable(struct tsnep_tx * tx)334 static void tsnep_tx_enable(struct tsnep_tx *tx)
335 {
336 struct netdev_queue *nq;
337
338 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index);
339
340 __netif_tx_lock_bh(nq);
341 netif_tx_wake_queue(nq);
342 __netif_tx_unlock_bh(nq);
343 }
344
tsnep_tx_disable(struct tsnep_tx * tx,struct napi_struct * napi)345 static void tsnep_tx_disable(struct tsnep_tx *tx, struct napi_struct *napi)
346 {
347 struct netdev_queue *nq;
348 u32 val;
349
350 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index);
351
352 __netif_tx_lock_bh(nq);
353 netif_tx_stop_queue(nq);
354 __netif_tx_unlock_bh(nq);
355
356 /* wait until TX is done in hardware */
357 readx_poll_timeout(ioread32, tx->addr + TSNEP_CONTROL, val,
358 ((val & TSNEP_CONTROL_TX_ENABLE) == 0), 10000,
359 1000000);
360
361 /* wait until TX is also done in software */
362 while (READ_ONCE(tx->read) != tx->write) {
363 napi_schedule(napi);
364 napi_synchronize(napi);
365 }
366 }
367
tsnep_tx_activate(struct tsnep_tx * tx,int index,int length,bool last)368 static void tsnep_tx_activate(struct tsnep_tx *tx, int index, int length,
369 bool last)
370 {
371 struct tsnep_tx_entry *entry = &tx->entry[index];
372
373 entry->properties = 0;
374 /* xdpf and zc are union with skb */
375 if (entry->skb) {
376 entry->properties = length & TSNEP_DESC_LENGTH_MASK;
377 entry->properties |= TSNEP_DESC_INTERRUPT_FLAG;
378 if ((entry->type & TSNEP_TX_TYPE_SKB) &&
379 (skb_shinfo(entry->skb)->tx_flags & SKBTX_IN_PROGRESS))
380 entry->properties |= TSNEP_DESC_EXTENDED_WRITEBACK_FLAG;
381
382 /* toggle user flag to prevent false acknowledge
383 *
384 * Only the first fragment is acknowledged. For all other
385 * fragments no acknowledge is done and the last written owner
386 * counter stays in the writeback descriptor. Therefore, it is
387 * possible that the last written owner counter is identical to
388 * the new incremented owner counter and a false acknowledge is
389 * detected before the real acknowledge has been done by
390 * hardware.
391 *
392 * The user flag is used to prevent this situation. The user
393 * flag is copied to the writeback descriptor by the hardware
394 * and is used as additional acknowledge data. By toggeling the
395 * user flag only for the first fragment (which is
396 * acknowledged), it is guaranteed that the last acknowledge
397 * done for this descriptor has used a different user flag and
398 * cannot be detected as false acknowledge.
399 */
400 entry->owner_user_flag = !entry->owner_user_flag;
401 }
402 if (last)
403 entry->properties |= TSNEP_TX_DESC_LAST_FRAGMENT_FLAG;
404 if (index == tx->increment_owner_counter) {
405 tx->owner_counter++;
406 if (tx->owner_counter == 4)
407 tx->owner_counter = 1;
408 tx->increment_owner_counter--;
409 if (tx->increment_owner_counter < 0)
410 tx->increment_owner_counter = TSNEP_RING_SIZE - 1;
411 }
412 entry->properties |=
413 (tx->owner_counter << TSNEP_DESC_OWNER_COUNTER_SHIFT) &
414 TSNEP_DESC_OWNER_COUNTER_MASK;
415 if (entry->owner_user_flag)
416 entry->properties |= TSNEP_TX_DESC_OWNER_USER_FLAG;
417 entry->desc->more_properties =
418 __cpu_to_le32(entry->len & TSNEP_DESC_LENGTH_MASK);
419
420 /* descriptor properties shall be written last, because valid data is
421 * signaled there
422 */
423 dma_wmb();
424
425 entry->desc->properties = __cpu_to_le32(entry->properties);
426 }
427
tsnep_tx_desc_available(struct tsnep_tx * tx)428 static int tsnep_tx_desc_available(struct tsnep_tx *tx)
429 {
430 if (tx->read <= tx->write)
431 return TSNEP_RING_SIZE - tx->write + tx->read - 1;
432 else
433 return tx->read - tx->write - 1;
434 }
435
tsnep_tx_map(struct sk_buff * skb,struct tsnep_tx * tx,int count)436 static int tsnep_tx_map(struct sk_buff *skb, struct tsnep_tx *tx, int count)
437 {
438 struct device *dmadev = tx->adapter->dmadev;
439 struct tsnep_tx_entry *entry;
440 unsigned int len;
441 dma_addr_t dma;
442 int map_len = 0;
443 int i;
444
445 for (i = 0; i < count; i++) {
446 entry = &tx->entry[(tx->write + i) & TSNEP_RING_MASK];
447
448 if (!i) {
449 len = skb_headlen(skb);
450 dma = dma_map_single(dmadev, skb->data, len,
451 DMA_TO_DEVICE);
452
453 entry->type = TSNEP_TX_TYPE_SKB;
454 } else {
455 len = skb_frag_size(&skb_shinfo(skb)->frags[i - 1]);
456 dma = skb_frag_dma_map(dmadev,
457 &skb_shinfo(skb)->frags[i - 1],
458 0, len, DMA_TO_DEVICE);
459
460 entry->type = TSNEP_TX_TYPE_SKB_FRAG;
461 }
462 if (dma_mapping_error(dmadev, dma))
463 return -ENOMEM;
464
465 entry->len = len;
466 dma_unmap_addr_set(entry, dma, dma);
467
468 entry->desc->tx = __cpu_to_le64(dma);
469
470 map_len += len;
471 }
472
473 return map_len;
474 }
475
tsnep_tx_unmap(struct tsnep_tx * tx,int index,int count)476 static int tsnep_tx_unmap(struct tsnep_tx *tx, int index, int count)
477 {
478 struct device *dmadev = tx->adapter->dmadev;
479 struct tsnep_tx_entry *entry;
480 int map_len = 0;
481 int i;
482
483 for (i = 0; i < count; i++) {
484 entry = &tx->entry[(index + i) & TSNEP_RING_MASK];
485
486 if (entry->len) {
487 if (entry->type & TSNEP_TX_TYPE_SKB)
488 dma_unmap_single(dmadev,
489 dma_unmap_addr(entry, dma),
490 dma_unmap_len(entry, len),
491 DMA_TO_DEVICE);
492 else if (entry->type &
493 (TSNEP_TX_TYPE_SKB_FRAG | TSNEP_TX_TYPE_XDP_NDO))
494 dma_unmap_page(dmadev,
495 dma_unmap_addr(entry, dma),
496 dma_unmap_len(entry, len),
497 DMA_TO_DEVICE);
498 map_len += entry->len;
499 entry->len = 0;
500 }
501 }
502
503 return map_len;
504 }
505
tsnep_xmit_frame_ring(struct sk_buff * skb,struct tsnep_tx * tx)506 static netdev_tx_t tsnep_xmit_frame_ring(struct sk_buff *skb,
507 struct tsnep_tx *tx)
508 {
509 int count = 1;
510 struct tsnep_tx_entry *entry;
511 int length;
512 int i;
513 int retval;
514
515 if (skb_shinfo(skb)->nr_frags > 0)
516 count += skb_shinfo(skb)->nr_frags;
517
518 if (tsnep_tx_desc_available(tx) < count) {
519 /* ring full, shall not happen because queue is stopped if full
520 * below
521 */
522 netif_stop_subqueue(tx->adapter->netdev, tx->queue_index);
523
524 return NETDEV_TX_BUSY;
525 }
526
527 entry = &tx->entry[tx->write];
528 entry->skb = skb;
529
530 retval = tsnep_tx_map(skb, tx, count);
531 if (retval < 0) {
532 tsnep_tx_unmap(tx, tx->write, count);
533 dev_kfree_skb_any(entry->skb);
534 entry->skb = NULL;
535
536 tx->dropped++;
537
538 return NETDEV_TX_OK;
539 }
540 length = retval;
541
542 if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)
543 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
544
545 for (i = 0; i < count; i++)
546 tsnep_tx_activate(tx, (tx->write + i) & TSNEP_RING_MASK, length,
547 i == count - 1);
548 tx->write = (tx->write + count) & TSNEP_RING_MASK;
549
550 skb_tx_timestamp(skb);
551
552 /* descriptor properties shall be valid before hardware is notified */
553 dma_wmb();
554
555 iowrite32(TSNEP_CONTROL_TX_ENABLE, tx->addr + TSNEP_CONTROL);
556
557 if (tsnep_tx_desc_available(tx) < (MAX_SKB_FRAGS + 1)) {
558 /* ring can get full with next frame */
559 netif_stop_subqueue(tx->adapter->netdev, tx->queue_index);
560 }
561
562 return NETDEV_TX_OK;
563 }
564
tsnep_xdp_tx_map(struct xdp_frame * xdpf,struct tsnep_tx * tx,struct skb_shared_info * shinfo,int count,u32 type)565 static int tsnep_xdp_tx_map(struct xdp_frame *xdpf, struct tsnep_tx *tx,
566 struct skb_shared_info *shinfo, int count, u32 type)
567 {
568 struct device *dmadev = tx->adapter->dmadev;
569 struct tsnep_tx_entry *entry;
570 struct page *page;
571 skb_frag_t *frag;
572 unsigned int len;
573 int map_len = 0;
574 dma_addr_t dma;
575 void *data;
576 int i;
577
578 frag = NULL;
579 len = xdpf->len;
580 for (i = 0; i < count; i++) {
581 entry = &tx->entry[(tx->write + i) & TSNEP_RING_MASK];
582 if (type & TSNEP_TX_TYPE_XDP_NDO) {
583 data = unlikely(frag) ? skb_frag_address(frag) :
584 xdpf->data;
585 dma = dma_map_single(dmadev, data, len, DMA_TO_DEVICE);
586 if (dma_mapping_error(dmadev, dma))
587 return -ENOMEM;
588
589 entry->type = TSNEP_TX_TYPE_XDP_NDO;
590 } else {
591 page = unlikely(frag) ? skb_frag_page(frag) :
592 virt_to_page(xdpf->data);
593 dma = page_pool_get_dma_addr(page);
594 if (unlikely(frag))
595 dma += skb_frag_off(frag);
596 else
597 dma += sizeof(*xdpf) + xdpf->headroom;
598 dma_sync_single_for_device(dmadev, dma, len,
599 DMA_BIDIRECTIONAL);
600
601 entry->type = TSNEP_TX_TYPE_XDP_TX;
602 }
603
604 entry->len = len;
605 dma_unmap_addr_set(entry, dma, dma);
606
607 entry->desc->tx = __cpu_to_le64(dma);
608
609 map_len += len;
610
611 if (i + 1 < count) {
612 frag = &shinfo->frags[i];
613 len = skb_frag_size(frag);
614 }
615 }
616
617 return map_len;
618 }
619
620 /* This function requires __netif_tx_lock is held by the caller. */
tsnep_xdp_xmit_frame_ring(struct xdp_frame * xdpf,struct tsnep_tx * tx,u32 type)621 static bool tsnep_xdp_xmit_frame_ring(struct xdp_frame *xdpf,
622 struct tsnep_tx *tx, u32 type)
623 {
624 struct skb_shared_info *shinfo = xdp_get_shared_info_from_frame(xdpf);
625 struct tsnep_tx_entry *entry;
626 int count, length, retval, i;
627
628 count = 1;
629 if (unlikely(xdp_frame_has_frags(xdpf)))
630 count += shinfo->nr_frags;
631
632 /* ensure that TX ring is not filled up by XDP, always MAX_SKB_FRAGS
633 * will be available for normal TX path and queue is stopped there if
634 * necessary
635 */
636 if (tsnep_tx_desc_available(tx) < (MAX_SKB_FRAGS + 1 + count))
637 return false;
638
639 entry = &tx->entry[tx->write];
640 entry->xdpf = xdpf;
641
642 retval = tsnep_xdp_tx_map(xdpf, tx, shinfo, count, type);
643 if (retval < 0) {
644 tsnep_tx_unmap(tx, tx->write, count);
645 entry->xdpf = NULL;
646
647 tx->dropped++;
648
649 return false;
650 }
651 length = retval;
652
653 for (i = 0; i < count; i++)
654 tsnep_tx_activate(tx, (tx->write + i) & TSNEP_RING_MASK, length,
655 i == count - 1);
656 tx->write = (tx->write + count) & TSNEP_RING_MASK;
657
658 /* descriptor properties shall be valid before hardware is notified */
659 dma_wmb();
660
661 return true;
662 }
663
tsnep_xdp_xmit_flush(struct tsnep_tx * tx)664 static void tsnep_xdp_xmit_flush(struct tsnep_tx *tx)
665 {
666 iowrite32(TSNEP_CONTROL_TX_ENABLE, tx->addr + TSNEP_CONTROL);
667 }
668
tsnep_xdp_xmit_back(struct tsnep_adapter * adapter,struct xdp_buff * xdp,struct netdev_queue * tx_nq,struct tsnep_tx * tx)669 static bool tsnep_xdp_xmit_back(struct tsnep_adapter *adapter,
670 struct xdp_buff *xdp,
671 struct netdev_queue *tx_nq, struct tsnep_tx *tx)
672 {
673 struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp);
674 bool xmit;
675
676 if (unlikely(!xdpf))
677 return false;
678
679 __netif_tx_lock(tx_nq, smp_processor_id());
680
681 xmit = tsnep_xdp_xmit_frame_ring(xdpf, tx, TSNEP_TX_TYPE_XDP_TX);
682
683 /* Avoid transmit queue timeout since we share it with the slow path */
684 if (xmit)
685 txq_trans_cond_update(tx_nq);
686
687 __netif_tx_unlock(tx_nq);
688
689 return xmit;
690 }
691
tsnep_xdp_tx_map_zc(struct xdp_desc * xdpd,struct tsnep_tx * tx)692 static int tsnep_xdp_tx_map_zc(struct xdp_desc *xdpd, struct tsnep_tx *tx)
693 {
694 struct tsnep_tx_entry *entry;
695 dma_addr_t dma;
696
697 entry = &tx->entry[tx->write];
698 entry->zc = true;
699
700 dma = xsk_buff_raw_get_dma(tx->xsk_pool, xdpd->addr);
701 xsk_buff_raw_dma_sync_for_device(tx->xsk_pool, dma, xdpd->len);
702
703 entry->type = TSNEP_TX_TYPE_XSK;
704 entry->len = xdpd->len;
705
706 entry->desc->tx = __cpu_to_le64(dma);
707
708 return xdpd->len;
709 }
710
tsnep_xdp_xmit_frame_ring_zc(struct xdp_desc * xdpd,struct tsnep_tx * tx)711 static void tsnep_xdp_xmit_frame_ring_zc(struct xdp_desc *xdpd,
712 struct tsnep_tx *tx)
713 {
714 int length;
715
716 length = tsnep_xdp_tx_map_zc(xdpd, tx);
717
718 tsnep_tx_activate(tx, tx->write, length, true);
719 tx->write = (tx->write + 1) & TSNEP_RING_MASK;
720 }
721
tsnep_xdp_xmit_zc(struct tsnep_tx * tx)722 static void tsnep_xdp_xmit_zc(struct tsnep_tx *tx)
723 {
724 int desc_available = tsnep_tx_desc_available(tx);
725 struct xdp_desc *descs = tx->xsk_pool->tx_descs;
726 int batch, i;
727
728 /* ensure that TX ring is not filled up by XDP, always MAX_SKB_FRAGS
729 * will be available for normal TX path and queue is stopped there if
730 * necessary
731 */
732 if (desc_available <= (MAX_SKB_FRAGS + 1))
733 return;
734 desc_available -= MAX_SKB_FRAGS + 1;
735
736 batch = xsk_tx_peek_release_desc_batch(tx->xsk_pool, desc_available);
737 for (i = 0; i < batch; i++)
738 tsnep_xdp_xmit_frame_ring_zc(&descs[i], tx);
739
740 if (batch) {
741 /* descriptor properties shall be valid before hardware is
742 * notified
743 */
744 dma_wmb();
745
746 tsnep_xdp_xmit_flush(tx);
747 }
748 }
749
tsnep_tx_poll(struct tsnep_tx * tx,int napi_budget)750 static bool tsnep_tx_poll(struct tsnep_tx *tx, int napi_budget)
751 {
752 struct tsnep_tx_entry *entry;
753 struct netdev_queue *nq;
754 int xsk_frames = 0;
755 int budget = 128;
756 int length;
757 int count;
758
759 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index);
760 __netif_tx_lock(nq, smp_processor_id());
761
762 do {
763 if (tx->read == tx->write)
764 break;
765
766 entry = &tx->entry[tx->read];
767 if ((__le32_to_cpu(entry->desc_wb->properties) &
768 TSNEP_TX_DESC_OWNER_MASK) !=
769 (entry->properties & TSNEP_TX_DESC_OWNER_MASK))
770 break;
771
772 /* descriptor properties shall be read first, because valid data
773 * is signaled there
774 */
775 dma_rmb();
776
777 count = 1;
778 if ((entry->type & TSNEP_TX_TYPE_SKB) &&
779 skb_shinfo(entry->skb)->nr_frags > 0)
780 count += skb_shinfo(entry->skb)->nr_frags;
781 else if ((entry->type & TSNEP_TX_TYPE_XDP) &&
782 xdp_frame_has_frags(entry->xdpf))
783 count += xdp_get_shared_info_from_frame(entry->xdpf)->nr_frags;
784
785 length = tsnep_tx_unmap(tx, tx->read, count);
786
787 if ((entry->type & TSNEP_TX_TYPE_SKB) &&
788 (skb_shinfo(entry->skb)->tx_flags & SKBTX_IN_PROGRESS) &&
789 (__le32_to_cpu(entry->desc_wb->properties) &
790 TSNEP_DESC_EXTENDED_WRITEBACK_FLAG)) {
791 struct skb_shared_hwtstamps hwtstamps;
792 u64 timestamp;
793
794 if (skb_shinfo(entry->skb)->tx_flags &
795 SKBTX_HW_TSTAMP_USE_CYCLES)
796 timestamp =
797 __le64_to_cpu(entry->desc_wb->counter);
798 else
799 timestamp =
800 __le64_to_cpu(entry->desc_wb->timestamp);
801
802 memset(&hwtstamps, 0, sizeof(hwtstamps));
803 hwtstamps.hwtstamp = ns_to_ktime(timestamp);
804
805 skb_tstamp_tx(entry->skb, &hwtstamps);
806 }
807
808 if (entry->type & TSNEP_TX_TYPE_SKB)
809 napi_consume_skb(entry->skb, napi_budget);
810 else if (entry->type & TSNEP_TX_TYPE_XDP)
811 xdp_return_frame_rx_napi(entry->xdpf);
812 else
813 xsk_frames++;
814 /* xdpf and zc are union with skb */
815 entry->skb = NULL;
816
817 tx->read = (tx->read + count) & TSNEP_RING_MASK;
818
819 tx->packets++;
820 tx->bytes += length + ETH_FCS_LEN;
821
822 budget--;
823 } while (likely(budget));
824
825 if (tx->xsk_pool) {
826 if (xsk_frames)
827 xsk_tx_completed(tx->xsk_pool, xsk_frames);
828 if (xsk_uses_need_wakeup(tx->xsk_pool))
829 xsk_set_tx_need_wakeup(tx->xsk_pool);
830 tsnep_xdp_xmit_zc(tx);
831 }
832
833 if ((tsnep_tx_desc_available(tx) >= ((MAX_SKB_FRAGS + 1) * 2)) &&
834 netif_tx_queue_stopped(nq)) {
835 netif_tx_wake_queue(nq);
836 }
837
838 __netif_tx_unlock(nq);
839
840 return budget != 0;
841 }
842
tsnep_tx_pending(struct tsnep_tx * tx)843 static bool tsnep_tx_pending(struct tsnep_tx *tx)
844 {
845 struct tsnep_tx_entry *entry;
846 struct netdev_queue *nq;
847 bool pending = false;
848
849 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index);
850 __netif_tx_lock(nq, smp_processor_id());
851
852 if (tx->read != tx->write) {
853 entry = &tx->entry[tx->read];
854 if ((__le32_to_cpu(entry->desc_wb->properties) &
855 TSNEP_TX_DESC_OWNER_MASK) ==
856 (entry->properties & TSNEP_TX_DESC_OWNER_MASK))
857 pending = true;
858 }
859
860 __netif_tx_unlock(nq);
861
862 return pending;
863 }
864
tsnep_tx_open(struct tsnep_tx * tx)865 static int tsnep_tx_open(struct tsnep_tx *tx)
866 {
867 int retval;
868
869 retval = tsnep_tx_ring_create(tx);
870 if (retval)
871 return retval;
872
873 tsnep_tx_init(tx);
874
875 return 0;
876 }
877
tsnep_tx_close(struct tsnep_tx * tx)878 static void tsnep_tx_close(struct tsnep_tx *tx)
879 {
880 tsnep_tx_ring_cleanup(tx);
881 }
882
tsnep_rx_ring_cleanup(struct tsnep_rx * rx)883 static void tsnep_rx_ring_cleanup(struct tsnep_rx *rx)
884 {
885 struct device *dmadev = rx->adapter->dmadev;
886 struct tsnep_rx_entry *entry;
887 int i;
888
889 for (i = 0; i < TSNEP_RING_SIZE; i++) {
890 entry = &rx->entry[i];
891 if (!rx->xsk_pool && entry->page)
892 page_pool_put_full_page(rx->page_pool, entry->page,
893 false);
894 if (rx->xsk_pool && entry->xdp)
895 xsk_buff_free(entry->xdp);
896 /* xdp is union with page */
897 entry->page = NULL;
898 }
899
900 if (rx->page_pool)
901 page_pool_destroy(rx->page_pool);
902
903 memset(rx->entry, 0, sizeof(rx->entry));
904
905 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) {
906 if (rx->page[i]) {
907 dma_free_coherent(dmadev, PAGE_SIZE, rx->page[i],
908 rx->page_dma[i]);
909 rx->page[i] = NULL;
910 rx->page_dma[i] = 0;
911 }
912 }
913 }
914
tsnep_rx_ring_create(struct tsnep_rx * rx)915 static int tsnep_rx_ring_create(struct tsnep_rx *rx)
916 {
917 struct device *dmadev = rx->adapter->dmadev;
918 struct tsnep_rx_entry *entry;
919 struct page_pool_params pp_params = { 0 };
920 struct tsnep_rx_entry *next_entry;
921 int i, j;
922 int retval;
923
924 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) {
925 rx->page[i] =
926 dma_alloc_coherent(dmadev, PAGE_SIZE, &rx->page_dma[i],
927 GFP_KERNEL);
928 if (!rx->page[i]) {
929 retval = -ENOMEM;
930 goto failed;
931 }
932 for (j = 0; j < TSNEP_RING_ENTRIES_PER_PAGE; j++) {
933 entry = &rx->entry[TSNEP_RING_ENTRIES_PER_PAGE * i + j];
934 entry->desc_wb = (struct tsnep_rx_desc_wb *)
935 (((u8 *)rx->page[i]) + TSNEP_DESC_SIZE * j);
936 entry->desc = (struct tsnep_rx_desc *)
937 (((u8 *)entry->desc_wb) + TSNEP_DESC_OFFSET);
938 entry->desc_dma = rx->page_dma[i] + TSNEP_DESC_SIZE * j;
939 }
940 }
941
942 pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV;
943 pp_params.order = 0;
944 pp_params.pool_size = TSNEP_RING_SIZE;
945 pp_params.nid = dev_to_node(dmadev);
946 pp_params.dev = dmadev;
947 pp_params.dma_dir = DMA_BIDIRECTIONAL;
948 pp_params.max_len = TSNEP_MAX_RX_BUF_SIZE;
949 pp_params.offset = TSNEP_RX_OFFSET;
950 rx->page_pool = page_pool_create(&pp_params);
951 if (IS_ERR(rx->page_pool)) {
952 retval = PTR_ERR(rx->page_pool);
953 rx->page_pool = NULL;
954 goto failed;
955 }
956
957 for (i = 0; i < TSNEP_RING_SIZE; i++) {
958 entry = &rx->entry[i];
959 next_entry = &rx->entry[(i + 1) & TSNEP_RING_MASK];
960 entry->desc->next = __cpu_to_le64(next_entry->desc_dma);
961 }
962
963 return 0;
964
965 failed:
966 tsnep_rx_ring_cleanup(rx);
967 return retval;
968 }
969
tsnep_rx_init(struct tsnep_rx * rx)970 static void tsnep_rx_init(struct tsnep_rx *rx)
971 {
972 dma_addr_t dma;
973
974 dma = rx->entry[0].desc_dma | TSNEP_RESET_OWNER_COUNTER;
975 iowrite32(DMA_ADDR_LOW(dma), rx->addr + TSNEP_RX_DESC_ADDR_LOW);
976 iowrite32(DMA_ADDR_HIGH(dma), rx->addr + TSNEP_RX_DESC_ADDR_HIGH);
977 rx->write = 0;
978 rx->read = 0;
979 rx->owner_counter = 1;
980 rx->increment_owner_counter = TSNEP_RING_SIZE - 1;
981 }
982
tsnep_rx_enable(struct tsnep_rx * rx)983 static void tsnep_rx_enable(struct tsnep_rx *rx)
984 {
985 /* descriptor properties shall be valid before hardware is notified */
986 dma_wmb();
987
988 iowrite32(TSNEP_CONTROL_RX_ENABLE, rx->addr + TSNEP_CONTROL);
989 }
990
tsnep_rx_disable(struct tsnep_rx * rx)991 static void tsnep_rx_disable(struct tsnep_rx *rx)
992 {
993 u32 val;
994
995 iowrite32(TSNEP_CONTROL_RX_DISABLE, rx->addr + TSNEP_CONTROL);
996 readx_poll_timeout(ioread32, rx->addr + TSNEP_CONTROL, val,
997 ((val & TSNEP_CONTROL_RX_ENABLE) == 0), 10000,
998 1000000);
999 }
1000
tsnep_rx_desc_available(struct tsnep_rx * rx)1001 static int tsnep_rx_desc_available(struct tsnep_rx *rx)
1002 {
1003 if (rx->read <= rx->write)
1004 return TSNEP_RING_SIZE - rx->write + rx->read - 1;
1005 else
1006 return rx->read - rx->write - 1;
1007 }
1008
tsnep_rx_free_page_buffer(struct tsnep_rx * rx)1009 static void tsnep_rx_free_page_buffer(struct tsnep_rx *rx)
1010 {
1011 struct page **page;
1012
1013 /* last entry of page_buffer is always zero, because ring cannot be
1014 * filled completely
1015 */
1016 page = rx->page_buffer;
1017 while (*page) {
1018 page_pool_put_full_page(rx->page_pool, *page, false);
1019 *page = NULL;
1020 page++;
1021 }
1022 }
1023
tsnep_rx_alloc_page_buffer(struct tsnep_rx * rx)1024 static int tsnep_rx_alloc_page_buffer(struct tsnep_rx *rx)
1025 {
1026 int i;
1027
1028 /* alloc for all ring entries except the last one, because ring cannot
1029 * be filled completely
1030 */
1031 for (i = 0; i < TSNEP_RING_SIZE - 1; i++) {
1032 rx->page_buffer[i] = page_pool_dev_alloc_pages(rx->page_pool);
1033 if (!rx->page_buffer[i]) {
1034 tsnep_rx_free_page_buffer(rx);
1035
1036 return -ENOMEM;
1037 }
1038 }
1039
1040 return 0;
1041 }
1042
tsnep_rx_set_page(struct tsnep_rx * rx,struct tsnep_rx_entry * entry,struct page * page)1043 static void tsnep_rx_set_page(struct tsnep_rx *rx, struct tsnep_rx_entry *entry,
1044 struct page *page)
1045 {
1046 entry->page = page;
1047 entry->len = TSNEP_MAX_RX_BUF_SIZE;
1048 entry->dma = page_pool_get_dma_addr(entry->page);
1049 entry->desc->rx = __cpu_to_le64(entry->dma + TSNEP_RX_OFFSET);
1050 }
1051
tsnep_rx_alloc_buffer(struct tsnep_rx * rx,int index)1052 static int tsnep_rx_alloc_buffer(struct tsnep_rx *rx, int index)
1053 {
1054 struct tsnep_rx_entry *entry = &rx->entry[index];
1055 struct page *page;
1056
1057 page = page_pool_dev_alloc_pages(rx->page_pool);
1058 if (unlikely(!page))
1059 return -ENOMEM;
1060 tsnep_rx_set_page(rx, entry, page);
1061
1062 return 0;
1063 }
1064
tsnep_rx_reuse_buffer(struct tsnep_rx * rx,int index)1065 static void tsnep_rx_reuse_buffer(struct tsnep_rx *rx, int index)
1066 {
1067 struct tsnep_rx_entry *entry = &rx->entry[index];
1068 struct tsnep_rx_entry *read = &rx->entry[rx->read];
1069
1070 tsnep_rx_set_page(rx, entry, read->page);
1071 read->page = NULL;
1072 }
1073
tsnep_rx_activate(struct tsnep_rx * rx,int index)1074 static void tsnep_rx_activate(struct tsnep_rx *rx, int index)
1075 {
1076 struct tsnep_rx_entry *entry = &rx->entry[index];
1077
1078 /* TSNEP_MAX_RX_BUF_SIZE and TSNEP_XSK_RX_BUF_SIZE are multiple of 4 */
1079 entry->properties = entry->len & TSNEP_DESC_LENGTH_MASK;
1080 entry->properties |= TSNEP_DESC_INTERRUPT_FLAG;
1081 if (index == rx->increment_owner_counter) {
1082 rx->owner_counter++;
1083 if (rx->owner_counter == 4)
1084 rx->owner_counter = 1;
1085 rx->increment_owner_counter--;
1086 if (rx->increment_owner_counter < 0)
1087 rx->increment_owner_counter = TSNEP_RING_SIZE - 1;
1088 }
1089 entry->properties |=
1090 (rx->owner_counter << TSNEP_DESC_OWNER_COUNTER_SHIFT) &
1091 TSNEP_DESC_OWNER_COUNTER_MASK;
1092
1093 /* descriptor properties shall be written last, because valid data is
1094 * signaled there
1095 */
1096 dma_wmb();
1097
1098 entry->desc->properties = __cpu_to_le32(entry->properties);
1099 }
1100
tsnep_rx_alloc(struct tsnep_rx * rx,int count,bool reuse)1101 static int tsnep_rx_alloc(struct tsnep_rx *rx, int count, bool reuse)
1102 {
1103 bool alloc_failed = false;
1104 int i, index;
1105
1106 for (i = 0; i < count && !alloc_failed; i++) {
1107 index = (rx->write + i) & TSNEP_RING_MASK;
1108
1109 if (unlikely(tsnep_rx_alloc_buffer(rx, index))) {
1110 rx->alloc_failed++;
1111 alloc_failed = true;
1112
1113 /* reuse only if no other allocation was successful */
1114 if (i == 0 && reuse)
1115 tsnep_rx_reuse_buffer(rx, index);
1116 else
1117 break;
1118 }
1119
1120 tsnep_rx_activate(rx, index);
1121 }
1122
1123 if (i)
1124 rx->write = (rx->write + i) & TSNEP_RING_MASK;
1125
1126 return i;
1127 }
1128
tsnep_rx_refill(struct tsnep_rx * rx,int count,bool reuse)1129 static int tsnep_rx_refill(struct tsnep_rx *rx, int count, bool reuse)
1130 {
1131 int desc_refilled;
1132
1133 desc_refilled = tsnep_rx_alloc(rx, count, reuse);
1134 if (desc_refilled)
1135 tsnep_rx_enable(rx);
1136
1137 return desc_refilled;
1138 }
1139
tsnep_rx_set_xdp(struct tsnep_rx * rx,struct tsnep_rx_entry * entry,struct xdp_buff * xdp)1140 static void tsnep_rx_set_xdp(struct tsnep_rx *rx, struct tsnep_rx_entry *entry,
1141 struct xdp_buff *xdp)
1142 {
1143 entry->xdp = xdp;
1144 entry->len = TSNEP_XSK_RX_BUF_SIZE;
1145 entry->dma = xsk_buff_xdp_get_dma(entry->xdp);
1146 entry->desc->rx = __cpu_to_le64(entry->dma);
1147 }
1148
tsnep_rx_reuse_buffer_zc(struct tsnep_rx * rx,int index)1149 static void tsnep_rx_reuse_buffer_zc(struct tsnep_rx *rx, int index)
1150 {
1151 struct tsnep_rx_entry *entry = &rx->entry[index];
1152 struct tsnep_rx_entry *read = &rx->entry[rx->read];
1153
1154 tsnep_rx_set_xdp(rx, entry, read->xdp);
1155 read->xdp = NULL;
1156 }
1157
tsnep_rx_alloc_zc(struct tsnep_rx * rx,int count,bool reuse)1158 static int tsnep_rx_alloc_zc(struct tsnep_rx *rx, int count, bool reuse)
1159 {
1160 u32 allocated;
1161 int i;
1162
1163 allocated = xsk_buff_alloc_batch(rx->xsk_pool, rx->xdp_batch, count);
1164 for (i = 0; i < allocated; i++) {
1165 int index = (rx->write + i) & TSNEP_RING_MASK;
1166 struct tsnep_rx_entry *entry = &rx->entry[index];
1167
1168 tsnep_rx_set_xdp(rx, entry, rx->xdp_batch[i]);
1169 tsnep_rx_activate(rx, index);
1170 }
1171 if (i == 0) {
1172 rx->alloc_failed++;
1173
1174 if (reuse) {
1175 tsnep_rx_reuse_buffer_zc(rx, rx->write);
1176 tsnep_rx_activate(rx, rx->write);
1177 }
1178 }
1179
1180 if (i)
1181 rx->write = (rx->write + i) & TSNEP_RING_MASK;
1182
1183 return i;
1184 }
1185
tsnep_rx_free_zc(struct tsnep_rx * rx)1186 static void tsnep_rx_free_zc(struct tsnep_rx *rx)
1187 {
1188 int i;
1189
1190 for (i = 0; i < TSNEP_RING_SIZE; i++) {
1191 struct tsnep_rx_entry *entry = &rx->entry[i];
1192
1193 if (entry->xdp)
1194 xsk_buff_free(entry->xdp);
1195 entry->xdp = NULL;
1196 }
1197 }
1198
tsnep_rx_refill_zc(struct tsnep_rx * rx,int count,bool reuse)1199 static int tsnep_rx_refill_zc(struct tsnep_rx *rx, int count, bool reuse)
1200 {
1201 int desc_refilled;
1202
1203 desc_refilled = tsnep_rx_alloc_zc(rx, count, reuse);
1204 if (desc_refilled)
1205 tsnep_rx_enable(rx);
1206
1207 return desc_refilled;
1208 }
1209
tsnep_xdp_run_prog(struct tsnep_rx * rx,struct bpf_prog * prog,struct xdp_buff * xdp,int * status,struct netdev_queue * tx_nq,struct tsnep_tx * tx)1210 static bool tsnep_xdp_run_prog(struct tsnep_rx *rx, struct bpf_prog *prog,
1211 struct xdp_buff *xdp, int *status,
1212 struct netdev_queue *tx_nq, struct tsnep_tx *tx)
1213 {
1214 unsigned int length;
1215 unsigned int sync;
1216 u32 act;
1217
1218 length = xdp->data_end - xdp->data_hard_start - XDP_PACKET_HEADROOM;
1219
1220 act = bpf_prog_run_xdp(prog, xdp);
1221 switch (act) {
1222 case XDP_PASS:
1223 return false;
1224 case XDP_TX:
1225 if (!tsnep_xdp_xmit_back(rx->adapter, xdp, tx_nq, tx))
1226 goto out_failure;
1227 *status |= TSNEP_XDP_TX;
1228 return true;
1229 case XDP_REDIRECT:
1230 if (xdp_do_redirect(rx->adapter->netdev, xdp, prog) < 0)
1231 goto out_failure;
1232 *status |= TSNEP_XDP_REDIRECT;
1233 return true;
1234 default:
1235 bpf_warn_invalid_xdp_action(rx->adapter->netdev, prog, act);
1236 fallthrough;
1237 case XDP_ABORTED:
1238 out_failure:
1239 trace_xdp_exception(rx->adapter->netdev, prog, act);
1240 fallthrough;
1241 case XDP_DROP:
1242 /* Due xdp_adjust_tail: DMA sync for_device cover max len CPU
1243 * touch
1244 */
1245 sync = xdp->data_end - xdp->data_hard_start -
1246 XDP_PACKET_HEADROOM;
1247 sync = max(sync, length);
1248 page_pool_put_page(rx->page_pool, virt_to_head_page(xdp->data),
1249 sync, true);
1250 return true;
1251 }
1252 }
1253
tsnep_xdp_run_prog_zc(struct tsnep_rx * rx,struct bpf_prog * prog,struct xdp_buff * xdp,int * status,struct netdev_queue * tx_nq,struct tsnep_tx * tx)1254 static bool tsnep_xdp_run_prog_zc(struct tsnep_rx *rx, struct bpf_prog *prog,
1255 struct xdp_buff *xdp, int *status,
1256 struct netdev_queue *tx_nq,
1257 struct tsnep_tx *tx)
1258 {
1259 u32 act;
1260
1261 act = bpf_prog_run_xdp(prog, xdp);
1262
1263 /* XDP_REDIRECT is the main action for zero-copy */
1264 if (likely(act == XDP_REDIRECT)) {
1265 if (xdp_do_redirect(rx->adapter->netdev, xdp, prog) < 0)
1266 goto out_failure;
1267 *status |= TSNEP_XDP_REDIRECT;
1268 return true;
1269 }
1270
1271 switch (act) {
1272 case XDP_PASS:
1273 return false;
1274 case XDP_TX:
1275 if (!tsnep_xdp_xmit_back(rx->adapter, xdp, tx_nq, tx))
1276 goto out_failure;
1277 *status |= TSNEP_XDP_TX;
1278 return true;
1279 default:
1280 bpf_warn_invalid_xdp_action(rx->adapter->netdev, prog, act);
1281 fallthrough;
1282 case XDP_ABORTED:
1283 out_failure:
1284 trace_xdp_exception(rx->adapter->netdev, prog, act);
1285 fallthrough;
1286 case XDP_DROP:
1287 xsk_buff_free(xdp);
1288 return true;
1289 }
1290 }
1291
tsnep_finalize_xdp(struct tsnep_adapter * adapter,int status,struct netdev_queue * tx_nq,struct tsnep_tx * tx)1292 static void tsnep_finalize_xdp(struct tsnep_adapter *adapter, int status,
1293 struct netdev_queue *tx_nq, struct tsnep_tx *tx)
1294 {
1295 if (status & TSNEP_XDP_TX) {
1296 __netif_tx_lock(tx_nq, smp_processor_id());
1297 tsnep_xdp_xmit_flush(tx);
1298 __netif_tx_unlock(tx_nq);
1299 }
1300
1301 if (status & TSNEP_XDP_REDIRECT)
1302 xdp_do_flush();
1303 }
1304
tsnep_build_skb(struct tsnep_rx * rx,struct page * page,int length)1305 static struct sk_buff *tsnep_build_skb(struct tsnep_rx *rx, struct page *page,
1306 int length)
1307 {
1308 struct sk_buff *skb;
1309
1310 skb = napi_build_skb(page_address(page), PAGE_SIZE);
1311 if (unlikely(!skb))
1312 return NULL;
1313
1314 /* update pointers within the skb to store the data */
1315 skb_reserve(skb, TSNEP_RX_OFFSET + TSNEP_RX_INLINE_METADATA_SIZE);
1316 __skb_put(skb, length - ETH_FCS_LEN);
1317
1318 if (rx->adapter->hwtstamp_config.rx_filter == HWTSTAMP_FILTER_ALL) {
1319 struct skb_shared_hwtstamps *hwtstamps = skb_hwtstamps(skb);
1320 struct tsnep_rx_inline *rx_inline =
1321 (struct tsnep_rx_inline *)(page_address(page) +
1322 TSNEP_RX_OFFSET);
1323
1324 skb_shinfo(skb)->tx_flags |=
1325 SKBTX_HW_TSTAMP_NETDEV;
1326 memset(hwtstamps, 0, sizeof(*hwtstamps));
1327 hwtstamps->netdev_data = rx_inline;
1328 }
1329
1330 skb_record_rx_queue(skb, rx->queue_index);
1331 skb->protocol = eth_type_trans(skb, rx->adapter->netdev);
1332
1333 return skb;
1334 }
1335
tsnep_rx_page(struct tsnep_rx * rx,struct napi_struct * napi,struct page * page,int length)1336 static void tsnep_rx_page(struct tsnep_rx *rx, struct napi_struct *napi,
1337 struct page *page, int length)
1338 {
1339 struct sk_buff *skb;
1340
1341 skb = tsnep_build_skb(rx, page, length);
1342 if (skb) {
1343 skb_mark_for_recycle(skb);
1344
1345 rx->packets++;
1346 rx->bytes += length;
1347 if (skb->pkt_type == PACKET_MULTICAST)
1348 rx->multicast++;
1349
1350 napi_gro_receive(napi, skb);
1351 } else {
1352 page_pool_recycle_direct(rx->page_pool, page);
1353
1354 rx->dropped++;
1355 }
1356 }
1357
tsnep_rx_poll(struct tsnep_rx * rx,struct napi_struct * napi,int budget)1358 static int tsnep_rx_poll(struct tsnep_rx *rx, struct napi_struct *napi,
1359 int budget)
1360 {
1361 struct device *dmadev = rx->adapter->dmadev;
1362 enum dma_data_direction dma_dir;
1363 struct tsnep_rx_entry *entry;
1364 struct netdev_queue *tx_nq;
1365 struct bpf_prog *prog;
1366 struct xdp_buff xdp;
1367 struct tsnep_tx *tx;
1368 int desc_available;
1369 int xdp_status = 0;
1370 int done = 0;
1371 int length;
1372
1373 desc_available = tsnep_rx_desc_available(rx);
1374 dma_dir = page_pool_get_dma_dir(rx->page_pool);
1375 prog = READ_ONCE(rx->adapter->xdp_prog);
1376 if (prog) {
1377 tx_nq = netdev_get_tx_queue(rx->adapter->netdev,
1378 rx->tx_queue_index);
1379 tx = &rx->adapter->tx[rx->tx_queue_index];
1380
1381 xdp_init_buff(&xdp, PAGE_SIZE, &rx->xdp_rxq);
1382 }
1383
1384 while (likely(done < budget) && (rx->read != rx->write)) {
1385 entry = &rx->entry[rx->read];
1386 if ((__le32_to_cpu(entry->desc_wb->properties) &
1387 TSNEP_DESC_OWNER_COUNTER_MASK) !=
1388 (entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK))
1389 break;
1390 done++;
1391
1392 if (desc_available >= TSNEP_RING_RX_REFILL) {
1393 bool reuse = desc_available >= TSNEP_RING_RX_REUSE;
1394
1395 desc_available -= tsnep_rx_refill(rx, desc_available,
1396 reuse);
1397 if (!entry->page) {
1398 /* buffer has been reused for refill to prevent
1399 * empty RX ring, thus buffer cannot be used for
1400 * RX processing
1401 */
1402 rx->read = (rx->read + 1) & TSNEP_RING_MASK;
1403 desc_available++;
1404
1405 rx->dropped++;
1406
1407 continue;
1408 }
1409 }
1410
1411 /* descriptor properties shall be read first, because valid data
1412 * is signaled there
1413 */
1414 dma_rmb();
1415
1416 prefetch(page_address(entry->page) + TSNEP_RX_OFFSET);
1417 length = __le32_to_cpu(entry->desc_wb->properties) &
1418 TSNEP_DESC_LENGTH_MASK;
1419 dma_sync_single_range_for_cpu(dmadev, entry->dma,
1420 TSNEP_RX_OFFSET, length, dma_dir);
1421
1422 /* RX metadata with timestamps is in front of actual data,
1423 * subtract metadata size to get length of actual data and
1424 * consider metadata size as offset of actual data during RX
1425 * processing
1426 */
1427 length -= TSNEP_RX_INLINE_METADATA_SIZE;
1428
1429 rx->read = (rx->read + 1) & TSNEP_RING_MASK;
1430 desc_available++;
1431
1432 if (prog) {
1433 bool consume;
1434
1435 xdp_prepare_buff(&xdp, page_address(entry->page),
1436 XDP_PACKET_HEADROOM + TSNEP_RX_INLINE_METADATA_SIZE,
1437 length, false);
1438
1439 consume = tsnep_xdp_run_prog(rx, prog, &xdp,
1440 &xdp_status, tx_nq, tx);
1441 if (consume) {
1442 rx->packets++;
1443 rx->bytes += length;
1444
1445 entry->page = NULL;
1446
1447 continue;
1448 }
1449 }
1450
1451 tsnep_rx_page(rx, napi, entry->page, length);
1452 entry->page = NULL;
1453 }
1454
1455 if (xdp_status)
1456 tsnep_finalize_xdp(rx->adapter, xdp_status, tx_nq, tx);
1457
1458 if (desc_available)
1459 tsnep_rx_refill(rx, desc_available, false);
1460
1461 return done;
1462 }
1463
tsnep_rx_poll_zc(struct tsnep_rx * rx,struct napi_struct * napi,int budget)1464 static int tsnep_rx_poll_zc(struct tsnep_rx *rx, struct napi_struct *napi,
1465 int budget)
1466 {
1467 struct tsnep_rx_entry *entry;
1468 struct netdev_queue *tx_nq;
1469 struct bpf_prog *prog;
1470 struct tsnep_tx *tx;
1471 int desc_available;
1472 int xdp_status = 0;
1473 struct page *page;
1474 int done = 0;
1475 int length;
1476
1477 desc_available = tsnep_rx_desc_available(rx);
1478 prog = READ_ONCE(rx->adapter->xdp_prog);
1479 if (prog) {
1480 tx_nq = netdev_get_tx_queue(rx->adapter->netdev,
1481 rx->tx_queue_index);
1482 tx = &rx->adapter->tx[rx->tx_queue_index];
1483 }
1484
1485 while (likely(done < budget) && (rx->read != rx->write)) {
1486 entry = &rx->entry[rx->read];
1487 if ((__le32_to_cpu(entry->desc_wb->properties) &
1488 TSNEP_DESC_OWNER_COUNTER_MASK) !=
1489 (entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK))
1490 break;
1491 done++;
1492
1493 if (desc_available >= TSNEP_RING_RX_REFILL) {
1494 bool reuse = desc_available >= TSNEP_RING_RX_REUSE;
1495
1496 desc_available -= tsnep_rx_refill_zc(rx, desc_available,
1497 reuse);
1498 if (!entry->xdp) {
1499 /* buffer has been reused for refill to prevent
1500 * empty RX ring, thus buffer cannot be used for
1501 * RX processing
1502 */
1503 rx->read = (rx->read + 1) & TSNEP_RING_MASK;
1504 desc_available++;
1505
1506 rx->dropped++;
1507
1508 continue;
1509 }
1510 }
1511
1512 /* descriptor properties shall be read first, because valid data
1513 * is signaled there
1514 */
1515 dma_rmb();
1516
1517 prefetch(entry->xdp->data);
1518 length = __le32_to_cpu(entry->desc_wb->properties) &
1519 TSNEP_DESC_LENGTH_MASK;
1520 xsk_buff_set_size(entry->xdp, length);
1521 xsk_buff_dma_sync_for_cpu(entry->xdp, rx->xsk_pool);
1522
1523 /* RX metadata with timestamps is in front of actual data,
1524 * subtract metadata size to get length of actual data and
1525 * consider metadata size as offset of actual data during RX
1526 * processing
1527 */
1528 length -= TSNEP_RX_INLINE_METADATA_SIZE;
1529
1530 rx->read = (rx->read + 1) & TSNEP_RING_MASK;
1531 desc_available++;
1532
1533 if (prog) {
1534 bool consume;
1535
1536 entry->xdp->data += TSNEP_RX_INLINE_METADATA_SIZE;
1537 entry->xdp->data_meta += TSNEP_RX_INLINE_METADATA_SIZE;
1538
1539 consume = tsnep_xdp_run_prog_zc(rx, prog, entry->xdp,
1540 &xdp_status, tx_nq, tx);
1541 if (consume) {
1542 rx->packets++;
1543 rx->bytes += length;
1544
1545 entry->xdp = NULL;
1546
1547 continue;
1548 }
1549 }
1550
1551 page = page_pool_dev_alloc_pages(rx->page_pool);
1552 if (page) {
1553 memcpy(page_address(page) + TSNEP_RX_OFFSET,
1554 entry->xdp->data - TSNEP_RX_INLINE_METADATA_SIZE,
1555 length + TSNEP_RX_INLINE_METADATA_SIZE);
1556 tsnep_rx_page(rx, napi, page, length);
1557 } else {
1558 rx->dropped++;
1559 }
1560 xsk_buff_free(entry->xdp);
1561 entry->xdp = NULL;
1562 }
1563
1564 if (xdp_status)
1565 tsnep_finalize_xdp(rx->adapter, xdp_status, tx_nq, tx);
1566
1567 if (desc_available)
1568 desc_available -= tsnep_rx_refill_zc(rx, desc_available, false);
1569
1570 if (xsk_uses_need_wakeup(rx->xsk_pool)) {
1571 if (desc_available)
1572 xsk_set_rx_need_wakeup(rx->xsk_pool);
1573 else
1574 xsk_clear_rx_need_wakeup(rx->xsk_pool);
1575
1576 return done;
1577 }
1578
1579 return desc_available ? budget : done;
1580 }
1581
tsnep_rx_pending(struct tsnep_rx * rx)1582 static bool tsnep_rx_pending(struct tsnep_rx *rx)
1583 {
1584 struct tsnep_rx_entry *entry;
1585
1586 if (rx->read != rx->write) {
1587 entry = &rx->entry[rx->read];
1588 if ((__le32_to_cpu(entry->desc_wb->properties) &
1589 TSNEP_DESC_OWNER_COUNTER_MASK) ==
1590 (entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK))
1591 return true;
1592 }
1593
1594 return false;
1595 }
1596
tsnep_rx_open(struct tsnep_rx * rx)1597 static int tsnep_rx_open(struct tsnep_rx *rx)
1598 {
1599 int desc_available;
1600 int retval;
1601
1602 retval = tsnep_rx_ring_create(rx);
1603 if (retval)
1604 return retval;
1605
1606 tsnep_rx_init(rx);
1607
1608 desc_available = tsnep_rx_desc_available(rx);
1609 if (rx->xsk_pool)
1610 retval = tsnep_rx_alloc_zc(rx, desc_available, false);
1611 else
1612 retval = tsnep_rx_alloc(rx, desc_available, false);
1613 if (retval != desc_available) {
1614 retval = -ENOMEM;
1615
1616 goto alloc_failed;
1617 }
1618
1619 /* prealloc pages to prevent allocation failures when XSK pool is
1620 * disabled at runtime
1621 */
1622 if (rx->xsk_pool) {
1623 retval = tsnep_rx_alloc_page_buffer(rx);
1624 if (retval)
1625 goto alloc_failed;
1626 }
1627
1628 return 0;
1629
1630 alloc_failed:
1631 tsnep_rx_ring_cleanup(rx);
1632 return retval;
1633 }
1634
tsnep_rx_close(struct tsnep_rx * rx)1635 static void tsnep_rx_close(struct tsnep_rx *rx)
1636 {
1637 if (rx->xsk_pool)
1638 tsnep_rx_free_page_buffer(rx);
1639
1640 tsnep_rx_ring_cleanup(rx);
1641 }
1642
tsnep_rx_reopen(struct tsnep_rx * rx)1643 static void tsnep_rx_reopen(struct tsnep_rx *rx)
1644 {
1645 struct page **page = rx->page_buffer;
1646 int i;
1647
1648 tsnep_rx_init(rx);
1649
1650 for (i = 0; i < TSNEP_RING_SIZE; i++) {
1651 struct tsnep_rx_entry *entry = &rx->entry[i];
1652
1653 /* defined initial values for properties are required for
1654 * correct owner counter checking
1655 */
1656 entry->desc->properties = 0;
1657 entry->desc_wb->properties = 0;
1658
1659 /* prevent allocation failures by reusing kept pages */
1660 if (*page) {
1661 tsnep_rx_set_page(rx, entry, *page);
1662 tsnep_rx_activate(rx, rx->write);
1663 rx->write++;
1664
1665 *page = NULL;
1666 page++;
1667 }
1668 }
1669 }
1670
tsnep_rx_reopen_xsk(struct tsnep_rx * rx)1671 static void tsnep_rx_reopen_xsk(struct tsnep_rx *rx)
1672 {
1673 struct page **page = rx->page_buffer;
1674 u32 allocated;
1675 int i;
1676
1677 tsnep_rx_init(rx);
1678
1679 /* alloc all ring entries except the last one, because ring cannot be
1680 * filled completely, as many buffers as possible is enough as wakeup is
1681 * done if new buffers are available
1682 */
1683 allocated = xsk_buff_alloc_batch(rx->xsk_pool, rx->xdp_batch,
1684 TSNEP_RING_SIZE - 1);
1685
1686 for (i = 0; i < TSNEP_RING_SIZE; i++) {
1687 struct tsnep_rx_entry *entry = &rx->entry[i];
1688
1689 /* keep pages to prevent allocation failures when xsk is
1690 * disabled
1691 */
1692 if (entry->page) {
1693 *page = entry->page;
1694 entry->page = NULL;
1695
1696 page++;
1697 }
1698
1699 /* defined initial values for properties are required for
1700 * correct owner counter checking
1701 */
1702 entry->desc->properties = 0;
1703 entry->desc_wb->properties = 0;
1704
1705 if (allocated) {
1706 tsnep_rx_set_xdp(rx, entry,
1707 rx->xdp_batch[allocated - 1]);
1708 tsnep_rx_activate(rx, rx->write);
1709 rx->write++;
1710
1711 allocated--;
1712 }
1713 }
1714 }
1715
tsnep_pending(struct tsnep_queue * queue)1716 static bool tsnep_pending(struct tsnep_queue *queue)
1717 {
1718 if (queue->tx && tsnep_tx_pending(queue->tx))
1719 return true;
1720
1721 if (queue->rx && tsnep_rx_pending(queue->rx))
1722 return true;
1723
1724 return false;
1725 }
1726
tsnep_poll(struct napi_struct * napi,int budget)1727 static int tsnep_poll(struct napi_struct *napi, int budget)
1728 {
1729 struct tsnep_queue *queue = container_of(napi, struct tsnep_queue,
1730 napi);
1731 bool complete = true;
1732 int done = 0;
1733
1734 if (queue->tx)
1735 complete = tsnep_tx_poll(queue->tx, budget);
1736
1737 /* handle case where we are called by netpoll with a budget of 0 */
1738 if (unlikely(budget <= 0))
1739 return budget;
1740
1741 if (queue->rx) {
1742 done = queue->rx->xsk_pool ?
1743 tsnep_rx_poll_zc(queue->rx, napi, budget) :
1744 tsnep_rx_poll(queue->rx, napi, budget);
1745 if (done >= budget)
1746 complete = false;
1747 }
1748
1749 /* if all work not completed, return budget and keep polling */
1750 if (!complete)
1751 return budget;
1752
1753 if (likely(napi_complete_done(napi, done))) {
1754 tsnep_enable_irq(queue->adapter, queue->irq_mask);
1755
1756 /* reschedule if work is already pending, prevent rotten packets
1757 * which are transmitted or received after polling but before
1758 * interrupt enable
1759 */
1760 if (tsnep_pending(queue)) {
1761 tsnep_disable_irq(queue->adapter, queue->irq_mask);
1762 napi_schedule(napi);
1763 }
1764 }
1765
1766 return min(done, budget - 1);
1767 }
1768
tsnep_request_irq(struct tsnep_queue * queue,bool first)1769 static int tsnep_request_irq(struct tsnep_queue *queue, bool first)
1770 {
1771 const char *name = netdev_name(queue->adapter->netdev);
1772 irq_handler_t handler;
1773 void *dev;
1774 int retval;
1775
1776 if (first) {
1777 sprintf(queue->name, "%s-mac", name);
1778 handler = tsnep_irq;
1779 dev = queue->adapter;
1780 } else {
1781 if (queue->tx && queue->rx)
1782 sprintf(queue->name, "%s-txrx-%d", name,
1783 queue->rx->queue_index);
1784 else if (queue->tx)
1785 sprintf(queue->name, "%s-tx-%d", name,
1786 queue->tx->queue_index);
1787 else
1788 sprintf(queue->name, "%s-rx-%d", name,
1789 queue->rx->queue_index);
1790 handler = tsnep_irq_txrx;
1791 dev = queue;
1792 }
1793
1794 retval = request_irq(queue->irq, handler, 0, queue->name, dev);
1795 if (retval) {
1796 /* if name is empty, then interrupt won't be freed */
1797 memset(queue->name, 0, sizeof(queue->name));
1798 }
1799
1800 return retval;
1801 }
1802
tsnep_free_irq(struct tsnep_queue * queue,bool first)1803 static void tsnep_free_irq(struct tsnep_queue *queue, bool first)
1804 {
1805 void *dev;
1806
1807 if (!strlen(queue->name))
1808 return;
1809
1810 if (first)
1811 dev = queue->adapter;
1812 else
1813 dev = queue;
1814
1815 free_irq(queue->irq, dev);
1816 memset(queue->name, 0, sizeof(queue->name));
1817 }
1818
tsnep_queue_close(struct tsnep_queue * queue,bool first)1819 static void tsnep_queue_close(struct tsnep_queue *queue, bool first)
1820 {
1821 struct tsnep_rx *rx = queue->rx;
1822
1823 tsnep_free_irq(queue, first);
1824
1825 if (rx) {
1826 if (xdp_rxq_info_is_reg(&rx->xdp_rxq))
1827 xdp_rxq_info_unreg(&rx->xdp_rxq);
1828 if (xdp_rxq_info_is_reg(&rx->xdp_rxq_zc))
1829 xdp_rxq_info_unreg(&rx->xdp_rxq_zc);
1830 }
1831
1832 netif_napi_del(&queue->napi);
1833 }
1834
tsnep_queue_open(struct tsnep_adapter * adapter,struct tsnep_queue * queue,bool first)1835 static int tsnep_queue_open(struct tsnep_adapter *adapter,
1836 struct tsnep_queue *queue, bool first)
1837 {
1838 struct tsnep_rx *rx = queue->rx;
1839 struct tsnep_tx *tx = queue->tx;
1840 int retval;
1841
1842 netif_napi_add(adapter->netdev, &queue->napi, tsnep_poll);
1843
1844 if (rx) {
1845 /* choose TX queue for XDP_TX */
1846 if (tx)
1847 rx->tx_queue_index = tx->queue_index;
1848 else if (rx->queue_index < adapter->num_tx_queues)
1849 rx->tx_queue_index = rx->queue_index;
1850 else
1851 rx->tx_queue_index = 0;
1852
1853 /* prepare both memory models to eliminate possible registration
1854 * errors when memory model is switched between page pool and
1855 * XSK pool during runtime
1856 */
1857 retval = xdp_rxq_info_reg(&rx->xdp_rxq, adapter->netdev,
1858 rx->queue_index, queue->napi.napi_id);
1859 if (retval)
1860 goto failed;
1861 retval = xdp_rxq_info_reg_mem_model(&rx->xdp_rxq,
1862 MEM_TYPE_PAGE_POOL,
1863 rx->page_pool);
1864 if (retval)
1865 goto failed;
1866 retval = xdp_rxq_info_reg(&rx->xdp_rxq_zc, adapter->netdev,
1867 rx->queue_index, queue->napi.napi_id);
1868 if (retval)
1869 goto failed;
1870 retval = xdp_rxq_info_reg_mem_model(&rx->xdp_rxq_zc,
1871 MEM_TYPE_XSK_BUFF_POOL,
1872 NULL);
1873 if (retval)
1874 goto failed;
1875 if (rx->xsk_pool)
1876 xsk_pool_set_rxq_info(rx->xsk_pool, &rx->xdp_rxq_zc);
1877 }
1878
1879 retval = tsnep_request_irq(queue, first);
1880 if (retval) {
1881 netif_err(adapter, drv, adapter->netdev,
1882 "can't get assigned irq %d.\n", queue->irq);
1883 goto failed;
1884 }
1885
1886 return 0;
1887
1888 failed:
1889 tsnep_queue_close(queue, first);
1890
1891 return retval;
1892 }
1893
tsnep_queue_enable(struct tsnep_queue * queue)1894 static void tsnep_queue_enable(struct tsnep_queue *queue)
1895 {
1896 napi_enable(&queue->napi);
1897 tsnep_enable_irq(queue->adapter, queue->irq_mask);
1898
1899 if (queue->tx)
1900 tsnep_tx_enable(queue->tx);
1901
1902 if (queue->rx)
1903 tsnep_rx_enable(queue->rx);
1904 }
1905
tsnep_queue_disable(struct tsnep_queue * queue)1906 static void tsnep_queue_disable(struct tsnep_queue *queue)
1907 {
1908 if (queue->tx)
1909 tsnep_tx_disable(queue->tx, &queue->napi);
1910
1911 napi_disable(&queue->napi);
1912 tsnep_disable_irq(queue->adapter, queue->irq_mask);
1913
1914 /* disable RX after NAPI polling has been disabled, because RX can be
1915 * enabled during NAPI polling
1916 */
1917 if (queue->rx)
1918 tsnep_rx_disable(queue->rx);
1919 }
1920
tsnep_netdev_open(struct net_device * netdev)1921 static int tsnep_netdev_open(struct net_device *netdev)
1922 {
1923 struct tsnep_adapter *adapter = netdev_priv(netdev);
1924 int i, retval;
1925
1926 for (i = 0; i < adapter->num_queues; i++) {
1927 if (adapter->queue[i].tx) {
1928 retval = tsnep_tx_open(adapter->queue[i].tx);
1929 if (retval)
1930 goto failed;
1931 }
1932 if (adapter->queue[i].rx) {
1933 retval = tsnep_rx_open(adapter->queue[i].rx);
1934 if (retval)
1935 goto failed;
1936 }
1937
1938 retval = tsnep_queue_open(adapter, &adapter->queue[i], i == 0);
1939 if (retval)
1940 goto failed;
1941 }
1942
1943 retval = netif_set_real_num_tx_queues(adapter->netdev,
1944 adapter->num_tx_queues);
1945 if (retval)
1946 goto failed;
1947 retval = netif_set_real_num_rx_queues(adapter->netdev,
1948 adapter->num_rx_queues);
1949 if (retval)
1950 goto failed;
1951
1952 tsnep_enable_irq(adapter, ECM_INT_LINK);
1953 retval = tsnep_phy_open(adapter);
1954 if (retval)
1955 goto phy_failed;
1956
1957 for (i = 0; i < adapter->num_queues; i++)
1958 tsnep_queue_enable(&adapter->queue[i]);
1959
1960 return 0;
1961
1962 phy_failed:
1963 tsnep_disable_irq(adapter, ECM_INT_LINK);
1964 failed:
1965 for (i = 0; i < adapter->num_queues; i++) {
1966 tsnep_queue_close(&adapter->queue[i], i == 0);
1967
1968 if (adapter->queue[i].rx)
1969 tsnep_rx_close(adapter->queue[i].rx);
1970 if (adapter->queue[i].tx)
1971 tsnep_tx_close(adapter->queue[i].tx);
1972 }
1973 return retval;
1974 }
1975
tsnep_netdev_close(struct net_device * netdev)1976 static int tsnep_netdev_close(struct net_device *netdev)
1977 {
1978 struct tsnep_adapter *adapter = netdev_priv(netdev);
1979 int i;
1980
1981 tsnep_disable_irq(adapter, ECM_INT_LINK);
1982 tsnep_phy_close(adapter);
1983
1984 for (i = 0; i < adapter->num_queues; i++) {
1985 tsnep_queue_disable(&adapter->queue[i]);
1986
1987 tsnep_queue_close(&adapter->queue[i], i == 0);
1988
1989 if (adapter->queue[i].rx)
1990 tsnep_rx_close(adapter->queue[i].rx);
1991 if (adapter->queue[i].tx)
1992 tsnep_tx_close(adapter->queue[i].tx);
1993 }
1994
1995 return 0;
1996 }
1997
tsnep_enable_xsk(struct tsnep_queue * queue,struct xsk_buff_pool * pool)1998 int tsnep_enable_xsk(struct tsnep_queue *queue, struct xsk_buff_pool *pool)
1999 {
2000 bool running = netif_running(queue->adapter->netdev);
2001 u32 frame_size;
2002
2003 frame_size = xsk_pool_get_rx_frame_size(pool);
2004 if (frame_size < TSNEP_XSK_RX_BUF_SIZE)
2005 return -EOPNOTSUPP;
2006
2007 queue->rx->page_buffer = kcalloc(TSNEP_RING_SIZE,
2008 sizeof(*queue->rx->page_buffer),
2009 GFP_KERNEL);
2010 if (!queue->rx->page_buffer)
2011 return -ENOMEM;
2012 queue->rx->xdp_batch = kcalloc(TSNEP_RING_SIZE,
2013 sizeof(*queue->rx->xdp_batch),
2014 GFP_KERNEL);
2015 if (!queue->rx->xdp_batch) {
2016 kfree(queue->rx->page_buffer);
2017 queue->rx->page_buffer = NULL;
2018
2019 return -ENOMEM;
2020 }
2021
2022 xsk_pool_set_rxq_info(pool, &queue->rx->xdp_rxq_zc);
2023
2024 if (running)
2025 tsnep_queue_disable(queue);
2026
2027 queue->tx->xsk_pool = pool;
2028 queue->rx->xsk_pool = pool;
2029
2030 if (running) {
2031 tsnep_rx_reopen_xsk(queue->rx);
2032 tsnep_queue_enable(queue);
2033 }
2034
2035 return 0;
2036 }
2037
tsnep_disable_xsk(struct tsnep_queue * queue)2038 void tsnep_disable_xsk(struct tsnep_queue *queue)
2039 {
2040 bool running = netif_running(queue->adapter->netdev);
2041
2042 if (running)
2043 tsnep_queue_disable(queue);
2044
2045 tsnep_rx_free_zc(queue->rx);
2046
2047 queue->rx->xsk_pool = NULL;
2048 queue->tx->xsk_pool = NULL;
2049
2050 if (running) {
2051 tsnep_rx_reopen(queue->rx);
2052 tsnep_queue_enable(queue);
2053 }
2054
2055 kfree(queue->rx->xdp_batch);
2056 queue->rx->xdp_batch = NULL;
2057 kfree(queue->rx->page_buffer);
2058 queue->rx->page_buffer = NULL;
2059 }
2060
tsnep_netdev_xmit_frame(struct sk_buff * skb,struct net_device * netdev)2061 static netdev_tx_t tsnep_netdev_xmit_frame(struct sk_buff *skb,
2062 struct net_device *netdev)
2063 {
2064 struct tsnep_adapter *adapter = netdev_priv(netdev);
2065 u16 queue_mapping = skb_get_queue_mapping(skb);
2066
2067 if (queue_mapping >= adapter->num_tx_queues)
2068 queue_mapping = 0;
2069
2070 return tsnep_xmit_frame_ring(skb, &adapter->tx[queue_mapping]);
2071 }
2072
tsnep_netdev_ioctl(struct net_device * netdev,struct ifreq * ifr,int cmd)2073 static int tsnep_netdev_ioctl(struct net_device *netdev, struct ifreq *ifr,
2074 int cmd)
2075 {
2076 if (!netif_running(netdev))
2077 return -EINVAL;
2078 if (cmd == SIOCSHWTSTAMP || cmd == SIOCGHWTSTAMP)
2079 return tsnep_ptp_ioctl(netdev, ifr, cmd);
2080 return phy_mii_ioctl(netdev->phydev, ifr, cmd);
2081 }
2082
tsnep_netdev_set_multicast(struct net_device * netdev)2083 static void tsnep_netdev_set_multicast(struct net_device *netdev)
2084 {
2085 struct tsnep_adapter *adapter = netdev_priv(netdev);
2086
2087 u16 rx_filter = 0;
2088
2089 /* configured MAC address and broadcasts are never filtered */
2090 if (netdev->flags & IFF_PROMISC) {
2091 rx_filter |= TSNEP_RX_FILTER_ACCEPT_ALL_MULTICASTS;
2092 rx_filter |= TSNEP_RX_FILTER_ACCEPT_ALL_UNICASTS;
2093 } else if (!netdev_mc_empty(netdev) || (netdev->flags & IFF_ALLMULTI)) {
2094 rx_filter |= TSNEP_RX_FILTER_ACCEPT_ALL_MULTICASTS;
2095 }
2096 iowrite16(rx_filter, adapter->addr + TSNEP_RX_FILTER);
2097 }
2098
tsnep_netdev_get_stats64(struct net_device * netdev,struct rtnl_link_stats64 * stats)2099 static void tsnep_netdev_get_stats64(struct net_device *netdev,
2100 struct rtnl_link_stats64 *stats)
2101 {
2102 struct tsnep_adapter *adapter = netdev_priv(netdev);
2103 u32 reg;
2104 u32 val;
2105 int i;
2106
2107 for (i = 0; i < adapter->num_tx_queues; i++) {
2108 stats->tx_packets += adapter->tx[i].packets;
2109 stats->tx_bytes += adapter->tx[i].bytes;
2110 stats->tx_dropped += adapter->tx[i].dropped;
2111 }
2112 for (i = 0; i < adapter->num_rx_queues; i++) {
2113 stats->rx_packets += adapter->rx[i].packets;
2114 stats->rx_bytes += adapter->rx[i].bytes;
2115 stats->rx_dropped += adapter->rx[i].dropped;
2116 stats->multicast += adapter->rx[i].multicast;
2117
2118 reg = ioread32(adapter->addr + TSNEP_QUEUE(i) +
2119 TSNEP_RX_STATISTIC);
2120 val = (reg & TSNEP_RX_STATISTIC_NO_DESC_MASK) >>
2121 TSNEP_RX_STATISTIC_NO_DESC_SHIFT;
2122 stats->rx_dropped += val;
2123 val = (reg & TSNEP_RX_STATISTIC_BUFFER_TOO_SMALL_MASK) >>
2124 TSNEP_RX_STATISTIC_BUFFER_TOO_SMALL_SHIFT;
2125 stats->rx_dropped += val;
2126 val = (reg & TSNEP_RX_STATISTIC_FIFO_OVERFLOW_MASK) >>
2127 TSNEP_RX_STATISTIC_FIFO_OVERFLOW_SHIFT;
2128 stats->rx_errors += val;
2129 stats->rx_fifo_errors += val;
2130 val = (reg & TSNEP_RX_STATISTIC_INVALID_FRAME_MASK) >>
2131 TSNEP_RX_STATISTIC_INVALID_FRAME_SHIFT;
2132 stats->rx_errors += val;
2133 stats->rx_frame_errors += val;
2134 }
2135
2136 reg = ioread32(adapter->addr + ECM_STAT);
2137 val = (reg & ECM_STAT_RX_ERR_MASK) >> ECM_STAT_RX_ERR_SHIFT;
2138 stats->rx_errors += val;
2139 val = (reg & ECM_STAT_INV_FRM_MASK) >> ECM_STAT_INV_FRM_SHIFT;
2140 stats->rx_errors += val;
2141 stats->rx_crc_errors += val;
2142 val = (reg & ECM_STAT_FWD_RX_ERR_MASK) >> ECM_STAT_FWD_RX_ERR_SHIFT;
2143 stats->rx_errors += val;
2144 }
2145
tsnep_mac_set_address(struct tsnep_adapter * adapter,u8 * addr)2146 static void tsnep_mac_set_address(struct tsnep_adapter *adapter, u8 *addr)
2147 {
2148 iowrite32(*(u32 *)addr, adapter->addr + TSNEP_MAC_ADDRESS_LOW);
2149 iowrite16(*(u16 *)(addr + sizeof(u32)),
2150 adapter->addr + TSNEP_MAC_ADDRESS_HIGH);
2151
2152 ether_addr_copy(adapter->mac_address, addr);
2153 netif_info(adapter, drv, adapter->netdev, "MAC address set to %pM\n",
2154 addr);
2155 }
2156
tsnep_netdev_set_mac_address(struct net_device * netdev,void * addr)2157 static int tsnep_netdev_set_mac_address(struct net_device *netdev, void *addr)
2158 {
2159 struct tsnep_adapter *adapter = netdev_priv(netdev);
2160 struct sockaddr *sock_addr = addr;
2161 int retval;
2162
2163 retval = eth_prepare_mac_addr_change(netdev, sock_addr);
2164 if (retval)
2165 return retval;
2166 eth_hw_addr_set(netdev, sock_addr->sa_data);
2167 tsnep_mac_set_address(adapter, sock_addr->sa_data);
2168
2169 return 0;
2170 }
2171
tsnep_netdev_set_features(struct net_device * netdev,netdev_features_t features)2172 static int tsnep_netdev_set_features(struct net_device *netdev,
2173 netdev_features_t features)
2174 {
2175 struct tsnep_adapter *adapter = netdev_priv(netdev);
2176 netdev_features_t changed = netdev->features ^ features;
2177 bool enable;
2178 int retval = 0;
2179
2180 if (changed & NETIF_F_LOOPBACK) {
2181 enable = !!(features & NETIF_F_LOOPBACK);
2182 retval = tsnep_phy_loopback(adapter, enable);
2183 }
2184
2185 return retval;
2186 }
2187
tsnep_netdev_get_tstamp(struct net_device * netdev,const struct skb_shared_hwtstamps * hwtstamps,bool cycles)2188 static ktime_t tsnep_netdev_get_tstamp(struct net_device *netdev,
2189 const struct skb_shared_hwtstamps *hwtstamps,
2190 bool cycles)
2191 {
2192 struct tsnep_rx_inline *rx_inline = hwtstamps->netdev_data;
2193 u64 timestamp;
2194
2195 if (cycles)
2196 timestamp = __le64_to_cpu(rx_inline->counter);
2197 else
2198 timestamp = __le64_to_cpu(rx_inline->timestamp);
2199
2200 return ns_to_ktime(timestamp);
2201 }
2202
tsnep_netdev_bpf(struct net_device * dev,struct netdev_bpf * bpf)2203 static int tsnep_netdev_bpf(struct net_device *dev, struct netdev_bpf *bpf)
2204 {
2205 struct tsnep_adapter *adapter = netdev_priv(dev);
2206
2207 switch (bpf->command) {
2208 case XDP_SETUP_PROG:
2209 return tsnep_xdp_setup_prog(adapter, bpf->prog, bpf->extack);
2210 case XDP_SETUP_XSK_POOL:
2211 return tsnep_xdp_setup_pool(adapter, bpf->xsk.pool,
2212 bpf->xsk.queue_id);
2213 default:
2214 return -EOPNOTSUPP;
2215 }
2216 }
2217
tsnep_xdp_get_tx(struct tsnep_adapter * adapter,u32 cpu)2218 static struct tsnep_tx *tsnep_xdp_get_tx(struct tsnep_adapter *adapter, u32 cpu)
2219 {
2220 if (cpu >= TSNEP_MAX_QUEUES)
2221 cpu &= TSNEP_MAX_QUEUES - 1;
2222
2223 while (cpu >= adapter->num_tx_queues)
2224 cpu -= adapter->num_tx_queues;
2225
2226 return &adapter->tx[cpu];
2227 }
2228
tsnep_netdev_xdp_xmit(struct net_device * dev,int n,struct xdp_frame ** xdp,u32 flags)2229 static int tsnep_netdev_xdp_xmit(struct net_device *dev, int n,
2230 struct xdp_frame **xdp, u32 flags)
2231 {
2232 struct tsnep_adapter *adapter = netdev_priv(dev);
2233 u32 cpu = smp_processor_id();
2234 struct netdev_queue *nq;
2235 struct tsnep_tx *tx;
2236 int nxmit;
2237 bool xmit;
2238
2239 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
2240 return -EINVAL;
2241
2242 tx = tsnep_xdp_get_tx(adapter, cpu);
2243 nq = netdev_get_tx_queue(adapter->netdev, tx->queue_index);
2244
2245 __netif_tx_lock(nq, cpu);
2246
2247 for (nxmit = 0; nxmit < n; nxmit++) {
2248 xmit = tsnep_xdp_xmit_frame_ring(xdp[nxmit], tx,
2249 TSNEP_TX_TYPE_XDP_NDO);
2250 if (!xmit)
2251 break;
2252
2253 /* avoid transmit queue timeout since we share it with the slow
2254 * path
2255 */
2256 txq_trans_cond_update(nq);
2257 }
2258
2259 if (flags & XDP_XMIT_FLUSH)
2260 tsnep_xdp_xmit_flush(tx);
2261
2262 __netif_tx_unlock(nq);
2263
2264 return nxmit;
2265 }
2266
tsnep_netdev_xsk_wakeup(struct net_device * dev,u32 queue_id,u32 flags)2267 static int tsnep_netdev_xsk_wakeup(struct net_device *dev, u32 queue_id,
2268 u32 flags)
2269 {
2270 struct tsnep_adapter *adapter = netdev_priv(dev);
2271 struct tsnep_queue *queue;
2272
2273 if (queue_id >= adapter->num_rx_queues ||
2274 queue_id >= adapter->num_tx_queues)
2275 return -EINVAL;
2276
2277 queue = &adapter->queue[queue_id];
2278
2279 if (!napi_if_scheduled_mark_missed(&queue->napi))
2280 napi_schedule(&queue->napi);
2281
2282 return 0;
2283 }
2284
2285 static const struct net_device_ops tsnep_netdev_ops = {
2286 .ndo_open = tsnep_netdev_open,
2287 .ndo_stop = tsnep_netdev_close,
2288 .ndo_start_xmit = tsnep_netdev_xmit_frame,
2289 .ndo_eth_ioctl = tsnep_netdev_ioctl,
2290 .ndo_set_rx_mode = tsnep_netdev_set_multicast,
2291 .ndo_get_stats64 = tsnep_netdev_get_stats64,
2292 .ndo_set_mac_address = tsnep_netdev_set_mac_address,
2293 .ndo_set_features = tsnep_netdev_set_features,
2294 .ndo_get_tstamp = tsnep_netdev_get_tstamp,
2295 .ndo_setup_tc = tsnep_tc_setup,
2296 .ndo_bpf = tsnep_netdev_bpf,
2297 .ndo_xdp_xmit = tsnep_netdev_xdp_xmit,
2298 .ndo_xsk_wakeup = tsnep_netdev_xsk_wakeup,
2299 };
2300
tsnep_mac_init(struct tsnep_adapter * adapter)2301 static int tsnep_mac_init(struct tsnep_adapter *adapter)
2302 {
2303 int retval;
2304
2305 /* initialize RX filtering, at least configured MAC address and
2306 * broadcast are not filtered
2307 */
2308 iowrite16(0, adapter->addr + TSNEP_RX_FILTER);
2309
2310 /* try to get MAC address in the following order:
2311 * - device tree
2312 * - valid MAC address already set
2313 * - MAC address register if valid
2314 * - random MAC address
2315 */
2316 retval = of_get_mac_address(adapter->pdev->dev.of_node,
2317 adapter->mac_address);
2318 if (retval == -EPROBE_DEFER)
2319 return retval;
2320 if (retval && !is_valid_ether_addr(adapter->mac_address)) {
2321 *(u32 *)adapter->mac_address =
2322 ioread32(adapter->addr + TSNEP_MAC_ADDRESS_LOW);
2323 *(u16 *)(adapter->mac_address + sizeof(u32)) =
2324 ioread16(adapter->addr + TSNEP_MAC_ADDRESS_HIGH);
2325 if (!is_valid_ether_addr(adapter->mac_address))
2326 eth_random_addr(adapter->mac_address);
2327 }
2328
2329 tsnep_mac_set_address(adapter, adapter->mac_address);
2330 eth_hw_addr_set(adapter->netdev, adapter->mac_address);
2331
2332 return 0;
2333 }
2334
tsnep_mdio_init(struct tsnep_adapter * adapter)2335 static int tsnep_mdio_init(struct tsnep_adapter *adapter)
2336 {
2337 struct device_node *np = adapter->pdev->dev.of_node;
2338 int retval;
2339
2340 if (np) {
2341 np = of_get_child_by_name(np, "mdio");
2342 if (!np)
2343 return 0;
2344
2345 adapter->suppress_preamble =
2346 of_property_read_bool(np, "suppress-preamble");
2347 }
2348
2349 adapter->mdiobus = devm_mdiobus_alloc(&adapter->pdev->dev);
2350 if (!adapter->mdiobus) {
2351 retval = -ENOMEM;
2352
2353 goto out;
2354 }
2355
2356 adapter->mdiobus->priv = (void *)adapter;
2357 adapter->mdiobus->parent = &adapter->pdev->dev;
2358 adapter->mdiobus->read = tsnep_mdiobus_read;
2359 adapter->mdiobus->write = tsnep_mdiobus_write;
2360 adapter->mdiobus->name = TSNEP "-mdiobus";
2361 snprintf(adapter->mdiobus->id, MII_BUS_ID_SIZE, "%s",
2362 adapter->pdev->name);
2363
2364 /* do not scan broadcast address */
2365 adapter->mdiobus->phy_mask = 0x0000001;
2366
2367 retval = of_mdiobus_register(adapter->mdiobus, np);
2368
2369 out:
2370 of_node_put(np);
2371
2372 return retval;
2373 }
2374
tsnep_phy_init(struct tsnep_adapter * adapter)2375 static int tsnep_phy_init(struct tsnep_adapter *adapter)
2376 {
2377 struct device_node *phy_node;
2378 int retval;
2379
2380 retval = of_get_phy_mode(adapter->pdev->dev.of_node,
2381 &adapter->phy_mode);
2382 if (retval)
2383 adapter->phy_mode = PHY_INTERFACE_MODE_GMII;
2384
2385 phy_node = of_parse_phandle(adapter->pdev->dev.of_node, "phy-handle",
2386 0);
2387 adapter->phydev = of_phy_find_device(phy_node);
2388 of_node_put(phy_node);
2389 if (!adapter->phydev && adapter->mdiobus)
2390 adapter->phydev = phy_find_first(adapter->mdiobus);
2391 if (!adapter->phydev)
2392 return -EIO;
2393
2394 return 0;
2395 }
2396
tsnep_queue_init(struct tsnep_adapter * adapter,int queue_count)2397 static int tsnep_queue_init(struct tsnep_adapter *adapter, int queue_count)
2398 {
2399 u32 irq_mask = ECM_INT_TX_0 | ECM_INT_RX_0;
2400 char name[8];
2401 int i;
2402 int retval;
2403
2404 /* one TX/RX queue pair for netdev is mandatory */
2405 if (platform_irq_count(adapter->pdev) == 1)
2406 retval = platform_get_irq(adapter->pdev, 0);
2407 else
2408 retval = platform_get_irq_byname(adapter->pdev, "mac");
2409 if (retval < 0)
2410 return retval;
2411 adapter->num_tx_queues = 1;
2412 adapter->num_rx_queues = 1;
2413 adapter->num_queues = 1;
2414 adapter->queue[0].adapter = adapter;
2415 adapter->queue[0].irq = retval;
2416 adapter->queue[0].tx = &adapter->tx[0];
2417 adapter->queue[0].tx->adapter = adapter;
2418 adapter->queue[0].tx->addr = adapter->addr + TSNEP_QUEUE(0);
2419 adapter->queue[0].tx->queue_index = 0;
2420 adapter->queue[0].rx = &adapter->rx[0];
2421 adapter->queue[0].rx->adapter = adapter;
2422 adapter->queue[0].rx->addr = adapter->addr + TSNEP_QUEUE(0);
2423 adapter->queue[0].rx->queue_index = 0;
2424 adapter->queue[0].irq_mask = irq_mask;
2425 adapter->queue[0].irq_delay_addr = adapter->addr + ECM_INT_DELAY;
2426 retval = tsnep_set_irq_coalesce(&adapter->queue[0],
2427 TSNEP_COALESCE_USECS_DEFAULT);
2428 if (retval < 0)
2429 return retval;
2430
2431 adapter->netdev->irq = adapter->queue[0].irq;
2432
2433 /* add additional TX/RX queue pairs only if dedicated interrupt is
2434 * available
2435 */
2436 for (i = 1; i < queue_count; i++) {
2437 sprintf(name, "txrx-%d", i);
2438 retval = platform_get_irq_byname_optional(adapter->pdev, name);
2439 if (retval < 0)
2440 break;
2441
2442 adapter->num_tx_queues++;
2443 adapter->num_rx_queues++;
2444 adapter->num_queues++;
2445 adapter->queue[i].adapter = adapter;
2446 adapter->queue[i].irq = retval;
2447 adapter->queue[i].tx = &adapter->tx[i];
2448 adapter->queue[i].tx->adapter = adapter;
2449 adapter->queue[i].tx->addr = adapter->addr + TSNEP_QUEUE(i);
2450 adapter->queue[i].tx->queue_index = i;
2451 adapter->queue[i].rx = &adapter->rx[i];
2452 adapter->queue[i].rx->adapter = adapter;
2453 adapter->queue[i].rx->addr = adapter->addr + TSNEP_QUEUE(i);
2454 adapter->queue[i].rx->queue_index = i;
2455 adapter->queue[i].irq_mask =
2456 irq_mask << (ECM_INT_TXRX_SHIFT * i);
2457 adapter->queue[i].irq_delay_addr =
2458 adapter->addr + ECM_INT_DELAY + ECM_INT_DELAY_OFFSET * i;
2459 retval = tsnep_set_irq_coalesce(&adapter->queue[i],
2460 TSNEP_COALESCE_USECS_DEFAULT);
2461 if (retval < 0)
2462 return retval;
2463 }
2464
2465 return 0;
2466 }
2467
tsnep_probe(struct platform_device * pdev)2468 static int tsnep_probe(struct platform_device *pdev)
2469 {
2470 struct tsnep_adapter *adapter;
2471 struct net_device *netdev;
2472 struct resource *io;
2473 u32 type;
2474 int revision;
2475 int version;
2476 int queue_count;
2477 int retval;
2478
2479 netdev = devm_alloc_etherdev_mqs(&pdev->dev,
2480 sizeof(struct tsnep_adapter),
2481 TSNEP_MAX_QUEUES, TSNEP_MAX_QUEUES);
2482 if (!netdev)
2483 return -ENODEV;
2484 SET_NETDEV_DEV(netdev, &pdev->dev);
2485 adapter = netdev_priv(netdev);
2486 platform_set_drvdata(pdev, adapter);
2487 adapter->pdev = pdev;
2488 adapter->dmadev = &pdev->dev;
2489 adapter->netdev = netdev;
2490 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE |
2491 NETIF_MSG_LINK | NETIF_MSG_IFUP |
2492 NETIF_MSG_IFDOWN | NETIF_MSG_TX_QUEUED;
2493
2494 netdev->min_mtu = ETH_MIN_MTU;
2495 netdev->max_mtu = TSNEP_MAX_FRAME_SIZE;
2496
2497 mutex_init(&adapter->gate_control_lock);
2498 mutex_init(&adapter->rxnfc_lock);
2499 INIT_LIST_HEAD(&adapter->rxnfc_rules);
2500
2501 io = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2502 adapter->addr = devm_ioremap_resource(&pdev->dev, io);
2503 if (IS_ERR(adapter->addr))
2504 return PTR_ERR(adapter->addr);
2505 netdev->mem_start = io->start;
2506 netdev->mem_end = io->end;
2507
2508 type = ioread32(adapter->addr + ECM_TYPE);
2509 revision = (type & ECM_REVISION_MASK) >> ECM_REVISION_SHIFT;
2510 version = (type & ECM_VERSION_MASK) >> ECM_VERSION_SHIFT;
2511 queue_count = (type & ECM_QUEUE_COUNT_MASK) >> ECM_QUEUE_COUNT_SHIFT;
2512 adapter->gate_control = type & ECM_GATE_CONTROL;
2513 adapter->rxnfc_max = TSNEP_RX_ASSIGN_ETHER_TYPE_COUNT;
2514
2515 tsnep_disable_irq(adapter, ECM_INT_ALL);
2516
2517 retval = tsnep_queue_init(adapter, queue_count);
2518 if (retval)
2519 return retval;
2520
2521 retval = dma_set_mask_and_coherent(&adapter->pdev->dev,
2522 DMA_BIT_MASK(64));
2523 if (retval) {
2524 dev_err(&adapter->pdev->dev, "no usable DMA configuration.\n");
2525 return retval;
2526 }
2527
2528 retval = tsnep_mac_init(adapter);
2529 if (retval)
2530 return retval;
2531
2532 retval = tsnep_mdio_init(adapter);
2533 if (retval)
2534 goto mdio_init_failed;
2535
2536 retval = tsnep_phy_init(adapter);
2537 if (retval)
2538 goto phy_init_failed;
2539
2540 retval = tsnep_ptp_init(adapter);
2541 if (retval)
2542 goto ptp_init_failed;
2543
2544 retval = tsnep_tc_init(adapter);
2545 if (retval)
2546 goto tc_init_failed;
2547
2548 retval = tsnep_rxnfc_init(adapter);
2549 if (retval)
2550 goto rxnfc_init_failed;
2551
2552 netdev->netdev_ops = &tsnep_netdev_ops;
2553 netdev->ethtool_ops = &tsnep_ethtool_ops;
2554 netdev->features = NETIF_F_SG;
2555 netdev->hw_features = netdev->features | NETIF_F_LOOPBACK;
2556
2557 netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
2558 NETDEV_XDP_ACT_NDO_XMIT |
2559 NETDEV_XDP_ACT_NDO_XMIT_SG |
2560 NETDEV_XDP_ACT_XSK_ZEROCOPY;
2561
2562 /* carrier off reporting is important to ethtool even BEFORE open */
2563 netif_carrier_off(netdev);
2564
2565 retval = register_netdev(netdev);
2566 if (retval)
2567 goto register_failed;
2568
2569 dev_info(&adapter->pdev->dev, "device version %d.%02d\n", version,
2570 revision);
2571 if (adapter->gate_control)
2572 dev_info(&adapter->pdev->dev, "gate control detected\n");
2573
2574 return 0;
2575
2576 register_failed:
2577 tsnep_rxnfc_cleanup(adapter);
2578 rxnfc_init_failed:
2579 tsnep_tc_cleanup(adapter);
2580 tc_init_failed:
2581 tsnep_ptp_cleanup(adapter);
2582 ptp_init_failed:
2583 phy_init_failed:
2584 if (adapter->mdiobus)
2585 mdiobus_unregister(adapter->mdiobus);
2586 mdio_init_failed:
2587 return retval;
2588 }
2589
tsnep_remove(struct platform_device * pdev)2590 static int tsnep_remove(struct platform_device *pdev)
2591 {
2592 struct tsnep_adapter *adapter = platform_get_drvdata(pdev);
2593
2594 unregister_netdev(adapter->netdev);
2595
2596 tsnep_rxnfc_cleanup(adapter);
2597
2598 tsnep_tc_cleanup(adapter);
2599
2600 tsnep_ptp_cleanup(adapter);
2601
2602 if (adapter->mdiobus)
2603 mdiobus_unregister(adapter->mdiobus);
2604
2605 tsnep_disable_irq(adapter, ECM_INT_ALL);
2606
2607 return 0;
2608 }
2609
2610 static const struct of_device_id tsnep_of_match[] = {
2611 { .compatible = "engleder,tsnep", },
2612 { },
2613 };
2614 MODULE_DEVICE_TABLE(of, tsnep_of_match);
2615
2616 static struct platform_driver tsnep_driver = {
2617 .driver = {
2618 .name = TSNEP,
2619 .of_match_table = tsnep_of_match,
2620 },
2621 .probe = tsnep_probe,
2622 .remove = tsnep_remove,
2623 };
2624 module_platform_driver(tsnep_driver);
2625
2626 MODULE_AUTHOR("Gerhard Engleder <gerhard@engleder-embedded.com>");
2627 MODULE_DESCRIPTION("TSN endpoint Ethernet MAC driver");
2628 MODULE_LICENSE("GPL");
2629
