1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
2 /* Copyright 2017-2019 NXP */
3
4 #include "enetc.h"
5 #include <linux/tcp.h>
6 #include <linux/udp.h>
7 #include <linux/vmalloc.h>
8
9 /* ENETC overhead: optional extension BD + 1 BD gap */
10 #define ENETC_TXBDS_NEEDED(val) ((val) + 2)
11 /* max # of chained Tx BDs is 15, including head and extension BD */
12 #define ENETC_MAX_SKB_FRAGS 13
13 #define ENETC_TXBDS_MAX_NEEDED ENETC_TXBDS_NEEDED(ENETC_MAX_SKB_FRAGS + 1)
14
15 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb,
16 int active_offloads);
17
enetc_xmit(struct sk_buff * skb,struct net_device * ndev)18 netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev)
19 {
20 struct enetc_ndev_priv *priv = netdev_priv(ndev);
21 struct enetc_bdr *tx_ring;
22 int count;
23
24 tx_ring = priv->tx_ring[skb->queue_mapping];
25
26 if (unlikely(skb_shinfo(skb)->nr_frags > ENETC_MAX_SKB_FRAGS))
27 if (unlikely(skb_linearize(skb)))
28 goto drop_packet_err;
29
30 count = skb_shinfo(skb)->nr_frags + 1; /* fragments + head */
31 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(count)) {
32 netif_stop_subqueue(ndev, tx_ring->index);
33 return NETDEV_TX_BUSY;
34 }
35
36 enetc_lock_mdio();
37 count = enetc_map_tx_buffs(tx_ring, skb, priv->active_offloads);
38 enetc_unlock_mdio();
39
40 if (unlikely(!count))
41 goto drop_packet_err;
42
43 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_MAX_NEEDED)
44 netif_stop_subqueue(ndev, tx_ring->index);
45
46 return NETDEV_TX_OK;
47
48 drop_packet_err:
49 dev_kfree_skb_any(skb);
50 return NETDEV_TX_OK;
51 }
52
enetc_tx_csum(struct sk_buff * skb,union enetc_tx_bd * txbd)53 static bool enetc_tx_csum(struct sk_buff *skb, union enetc_tx_bd *txbd)
54 {
55 int l3_start, l3_hsize;
56 u16 l3_flags, l4_flags;
57
58 if (skb->ip_summed != CHECKSUM_PARTIAL)
59 return false;
60
61 switch (skb->csum_offset) {
62 case offsetof(struct tcphdr, check):
63 l4_flags = ENETC_TXBD_L4_TCP;
64 break;
65 case offsetof(struct udphdr, check):
66 l4_flags = ENETC_TXBD_L4_UDP;
67 break;
68 default:
69 skb_checksum_help(skb);
70 return false;
71 }
72
73 l3_start = skb_network_offset(skb);
74 l3_hsize = skb_network_header_len(skb);
75
76 l3_flags = 0;
77 if (skb->protocol == htons(ETH_P_IPV6))
78 l3_flags = ENETC_TXBD_L3_IPV6;
79
80 /* write BD fields */
81 txbd->l3_csoff = enetc_txbd_l3_csoff(l3_start, l3_hsize, l3_flags);
82 txbd->l4_csoff = l4_flags;
83
84 return true;
85 }
86
enetc_unmap_tx_buff(struct enetc_bdr * tx_ring,struct enetc_tx_swbd * tx_swbd)87 static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring,
88 struct enetc_tx_swbd *tx_swbd)
89 {
90 if (tx_swbd->is_dma_page)
91 dma_unmap_page(tx_ring->dev, tx_swbd->dma,
92 tx_swbd->len, DMA_TO_DEVICE);
93 else
94 dma_unmap_single(tx_ring->dev, tx_swbd->dma,
95 tx_swbd->len, DMA_TO_DEVICE);
96 tx_swbd->dma = 0;
97 }
98
enetc_free_tx_skb(struct enetc_bdr * tx_ring,struct enetc_tx_swbd * tx_swbd)99 static void enetc_free_tx_skb(struct enetc_bdr *tx_ring,
100 struct enetc_tx_swbd *tx_swbd)
101 {
102 if (tx_swbd->dma)
103 enetc_unmap_tx_buff(tx_ring, tx_swbd);
104
105 if (tx_swbd->skb) {
106 dev_kfree_skb_any(tx_swbd->skb);
107 tx_swbd->skb = NULL;
108 }
109 }
110
enetc_map_tx_buffs(struct enetc_bdr * tx_ring,struct sk_buff * skb,int active_offloads)111 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb,
112 int active_offloads)
113 {
114 struct enetc_tx_swbd *tx_swbd;
115 skb_frag_t *frag;
116 int len = skb_headlen(skb);
117 union enetc_tx_bd temp_bd;
118 union enetc_tx_bd *txbd;
119 bool do_vlan, do_tstamp;
120 int i, count = 0;
121 unsigned int f;
122 dma_addr_t dma;
123 u8 flags = 0;
124
125 i = tx_ring->next_to_use;
126 txbd = ENETC_TXBD(*tx_ring, i);
127 prefetchw(txbd);
128
129 dma = dma_map_single(tx_ring->dev, skb->data, len, DMA_TO_DEVICE);
130 if (unlikely(dma_mapping_error(tx_ring->dev, dma)))
131 goto dma_err;
132
133 temp_bd.addr = cpu_to_le64(dma);
134 temp_bd.buf_len = cpu_to_le16(len);
135 temp_bd.lstatus = 0;
136
137 tx_swbd = &tx_ring->tx_swbd[i];
138 tx_swbd->dma = dma;
139 tx_swbd->len = len;
140 tx_swbd->is_dma_page = 0;
141 count++;
142
143 do_vlan = skb_vlan_tag_present(skb);
144 do_tstamp = (active_offloads & ENETC_F_TX_TSTAMP) &&
145 (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP);
146 tx_swbd->do_tstamp = do_tstamp;
147 tx_swbd->check_wb = tx_swbd->do_tstamp;
148
149 if (do_vlan || do_tstamp)
150 flags |= ENETC_TXBD_FLAGS_EX;
151
152 if (enetc_tx_csum(skb, &temp_bd))
153 flags |= ENETC_TXBD_FLAGS_CSUM | ENETC_TXBD_FLAGS_L4CS;
154 else if (tx_ring->tsd_enable)
155 flags |= ENETC_TXBD_FLAGS_TSE | ENETC_TXBD_FLAGS_TXSTART;
156
157 /* first BD needs frm_len and offload flags set */
158 temp_bd.frm_len = cpu_to_le16(skb->len);
159 temp_bd.flags = flags;
160
161 if (flags & ENETC_TXBD_FLAGS_TSE) {
162 u32 temp;
163
164 temp = (skb->skb_mstamp_ns >> 5 & ENETC_TXBD_TXSTART_MASK)
165 | (flags << ENETC_TXBD_FLAGS_OFFSET);
166 temp_bd.txstart = cpu_to_le32(temp);
167 }
168
169 if (flags & ENETC_TXBD_FLAGS_EX) {
170 u8 e_flags = 0;
171 *txbd = temp_bd;
172 enetc_clear_tx_bd(&temp_bd);
173
174 /* add extension BD for VLAN and/or timestamping */
175 flags = 0;
176 tx_swbd++;
177 txbd++;
178 i++;
179 if (unlikely(i == tx_ring->bd_count)) {
180 i = 0;
181 tx_swbd = tx_ring->tx_swbd;
182 txbd = ENETC_TXBD(*tx_ring, 0);
183 }
184 prefetchw(txbd);
185
186 if (do_vlan) {
187 temp_bd.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb));
188 temp_bd.ext.tpid = 0; /* < C-TAG */
189 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS;
190 }
191
192 if (do_tstamp) {
193 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
194 e_flags |= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP;
195 }
196
197 temp_bd.ext.e_flags = e_flags;
198 count++;
199 }
200
201 frag = &skb_shinfo(skb)->frags[0];
202 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) {
203 len = skb_frag_size(frag);
204 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len,
205 DMA_TO_DEVICE);
206 if (dma_mapping_error(tx_ring->dev, dma))
207 goto dma_err;
208
209 *txbd = temp_bd;
210 enetc_clear_tx_bd(&temp_bd);
211
212 flags = 0;
213 tx_swbd++;
214 txbd++;
215 i++;
216 if (unlikely(i == tx_ring->bd_count)) {
217 i = 0;
218 tx_swbd = tx_ring->tx_swbd;
219 txbd = ENETC_TXBD(*tx_ring, 0);
220 }
221 prefetchw(txbd);
222
223 temp_bd.addr = cpu_to_le64(dma);
224 temp_bd.buf_len = cpu_to_le16(len);
225
226 tx_swbd->dma = dma;
227 tx_swbd->len = len;
228 tx_swbd->is_dma_page = 1;
229 count++;
230 }
231
232 /* last BD needs 'F' bit set */
233 flags |= ENETC_TXBD_FLAGS_F;
234 temp_bd.flags = flags;
235 *txbd = temp_bd;
236
237 tx_ring->tx_swbd[i].skb = skb;
238
239 enetc_bdr_idx_inc(tx_ring, &i);
240 tx_ring->next_to_use = i;
241
242 skb_tx_timestamp(skb);
243
244 /* let H/W know BD ring has been updated */
245 enetc_wr_reg_hot(tx_ring->tpir, i); /* includes wmb() */
246
247 return count;
248
249 dma_err:
250 dev_err(tx_ring->dev, "DMA map error");
251
252 do {
253 tx_swbd = &tx_ring->tx_swbd[i];
254 enetc_free_tx_skb(tx_ring, tx_swbd);
255 if (i == 0)
256 i = tx_ring->bd_count;
257 i--;
258 } while (count--);
259
260 return 0;
261 }
262
enetc_msix(int irq,void * data)263 static irqreturn_t enetc_msix(int irq, void *data)
264 {
265 struct enetc_int_vector *v = data;
266 int i;
267
268 enetc_lock_mdio();
269
270 /* disable interrupts */
271 enetc_wr_reg_hot(v->rbier, 0);
272 enetc_wr_reg_hot(v->ricr1, v->rx_ictt);
273
274 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)
275 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 0);
276
277 enetc_unlock_mdio();
278
279 napi_schedule(&v->napi);
280
281 return IRQ_HANDLED;
282 }
283
284 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget);
285 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring,
286 struct napi_struct *napi, int work_limit);
287
enetc_rx_dim_work(struct work_struct * w)288 static void enetc_rx_dim_work(struct work_struct *w)
289 {
290 struct dim *dim = container_of(w, struct dim, work);
291 struct dim_cq_moder moder =
292 net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
293 struct enetc_int_vector *v =
294 container_of(dim, struct enetc_int_vector, rx_dim);
295
296 v->rx_ictt = enetc_usecs_to_cycles(moder.usec);
297 dim->state = DIM_START_MEASURE;
298 }
299
enetc_rx_net_dim(struct enetc_int_vector * v)300 static void enetc_rx_net_dim(struct enetc_int_vector *v)
301 {
302 struct dim_sample dim_sample;
303
304 v->comp_cnt++;
305
306 if (!v->rx_napi_work)
307 return;
308
309 dim_update_sample(v->comp_cnt,
310 v->rx_ring.stats.packets,
311 v->rx_ring.stats.bytes,
312 &dim_sample);
313 net_dim(&v->rx_dim, dim_sample);
314 }
315
enetc_poll(struct napi_struct * napi,int budget)316 static int enetc_poll(struct napi_struct *napi, int budget)
317 {
318 struct enetc_int_vector
319 *v = container_of(napi, struct enetc_int_vector, napi);
320 bool complete = true;
321 int work_done;
322 int i;
323
324 for (i = 0; i < v->count_tx_rings; i++)
325 if (!enetc_clean_tx_ring(&v->tx_ring[i], budget))
326 complete = false;
327
328 work_done = enetc_clean_rx_ring(&v->rx_ring, napi, budget);
329 if (work_done == budget)
330 complete = false;
331 if (work_done)
332 v->rx_napi_work = true;
333
334 if (!complete)
335 return budget;
336
337 napi_complete_done(napi, work_done);
338
339 if (likely(v->rx_dim_en))
340 enetc_rx_net_dim(v);
341
342 v->rx_napi_work = false;
343
344 enetc_lock_mdio();
345
346 /* enable interrupts */
347 enetc_wr_reg_hot(v->rbier, ENETC_RBIER_RXTIE);
348
349 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)
350 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i),
351 ENETC_TBIER_TXTIE);
352
353 enetc_unlock_mdio();
354
355 return work_done;
356 }
357
enetc_bd_ready_count(struct enetc_bdr * tx_ring,int ci)358 static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci)
359 {
360 int pi = enetc_rd_reg_hot(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK;
361
362 return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi;
363 }
364
enetc_get_tx_tstamp(struct enetc_hw * hw,union enetc_tx_bd * txbd,u64 * tstamp)365 static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd,
366 u64 *tstamp)
367 {
368 u32 lo, hi, tstamp_lo;
369
370 lo = enetc_rd(hw, ENETC_SICTR0);
371 hi = enetc_rd(hw, ENETC_SICTR1);
372 tstamp_lo = le32_to_cpu(txbd->wb.tstamp);
373 if (lo <= tstamp_lo)
374 hi -= 1;
375 *tstamp = (u64)hi << 32 | tstamp_lo;
376 }
377
enetc_tstamp_tx(struct sk_buff * skb,u64 tstamp)378 static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp)
379 {
380 struct skb_shared_hwtstamps shhwtstamps;
381
382 if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) {
383 memset(&shhwtstamps, 0, sizeof(shhwtstamps));
384 shhwtstamps.hwtstamp = ns_to_ktime(tstamp);
385 skb_tstamp_tx(skb, &shhwtstamps);
386 }
387 }
388
enetc_clean_tx_ring(struct enetc_bdr * tx_ring,int napi_budget)389 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget)
390 {
391 struct net_device *ndev = tx_ring->ndev;
392 int tx_frm_cnt = 0, tx_byte_cnt = 0;
393 struct enetc_tx_swbd *tx_swbd;
394 int i, bds_to_clean;
395 bool do_tstamp;
396 u64 tstamp = 0;
397
398 i = tx_ring->next_to_clean;
399 tx_swbd = &tx_ring->tx_swbd[i];
400
401 enetc_lock_mdio();
402 bds_to_clean = enetc_bd_ready_count(tx_ring, i);
403 enetc_unlock_mdio();
404
405 do_tstamp = false;
406
407 while (bds_to_clean && tx_frm_cnt < ENETC_DEFAULT_TX_WORK) {
408 bool is_eof = !!tx_swbd->skb;
409
410 if (unlikely(tx_swbd->check_wb)) {
411 struct enetc_ndev_priv *priv = netdev_priv(ndev);
412 union enetc_tx_bd *txbd;
413
414 txbd = ENETC_TXBD(*tx_ring, i);
415
416 if (txbd->flags & ENETC_TXBD_FLAGS_W &&
417 tx_swbd->do_tstamp) {
418 enetc_get_tx_tstamp(&priv->si->hw, txbd,
419 &tstamp);
420 do_tstamp = true;
421 }
422 }
423
424 if (likely(tx_swbd->dma))
425 enetc_unmap_tx_buff(tx_ring, tx_swbd);
426
427 if (is_eof) {
428 if (unlikely(do_tstamp)) {
429 enetc_tstamp_tx(tx_swbd->skb, tstamp);
430 do_tstamp = false;
431 }
432 napi_consume_skb(tx_swbd->skb, napi_budget);
433 tx_swbd->skb = NULL;
434 }
435
436 tx_byte_cnt += tx_swbd->len;
437
438 bds_to_clean--;
439 tx_swbd++;
440 i++;
441 if (unlikely(i == tx_ring->bd_count)) {
442 i = 0;
443 tx_swbd = tx_ring->tx_swbd;
444 }
445
446 enetc_lock_mdio();
447
448 /* BD iteration loop end */
449 if (is_eof) {
450 tx_frm_cnt++;
451 /* re-arm interrupt source */
452 enetc_wr_reg_hot(tx_ring->idr, BIT(tx_ring->index) |
453 BIT(16 + tx_ring->index));
454 }
455
456 if (unlikely(!bds_to_clean))
457 bds_to_clean = enetc_bd_ready_count(tx_ring, i);
458
459 enetc_unlock_mdio();
460 }
461
462 tx_ring->next_to_clean = i;
463 tx_ring->stats.packets += tx_frm_cnt;
464 tx_ring->stats.bytes += tx_byte_cnt;
465
466 if (unlikely(tx_frm_cnt && netif_carrier_ok(ndev) &&
467 __netif_subqueue_stopped(ndev, tx_ring->index) &&
468 (enetc_bd_unused(tx_ring) >= ENETC_TXBDS_MAX_NEEDED))) {
469 netif_wake_subqueue(ndev, tx_ring->index);
470 }
471
472 return tx_frm_cnt != ENETC_DEFAULT_TX_WORK;
473 }
474
enetc_new_page(struct enetc_bdr * rx_ring,struct enetc_rx_swbd * rx_swbd)475 static bool enetc_new_page(struct enetc_bdr *rx_ring,
476 struct enetc_rx_swbd *rx_swbd)
477 {
478 struct page *page;
479 dma_addr_t addr;
480
481 page = dev_alloc_page();
482 if (unlikely(!page))
483 return false;
484
485 addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, DMA_FROM_DEVICE);
486 if (unlikely(dma_mapping_error(rx_ring->dev, addr))) {
487 __free_page(page);
488
489 return false;
490 }
491
492 rx_swbd->dma = addr;
493 rx_swbd->page = page;
494 rx_swbd->page_offset = ENETC_RXB_PAD;
495
496 return true;
497 }
498
enetc_refill_rx_ring(struct enetc_bdr * rx_ring,const int buff_cnt)499 static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt)
500 {
501 struct enetc_rx_swbd *rx_swbd;
502 union enetc_rx_bd *rxbd;
503 int i, j;
504
505 i = rx_ring->next_to_use;
506 rx_swbd = &rx_ring->rx_swbd[i];
507 rxbd = enetc_rxbd(rx_ring, i);
508
509 for (j = 0; j < buff_cnt; j++) {
510 /* try reuse page */
511 if (unlikely(!rx_swbd->page)) {
512 if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) {
513 rx_ring->stats.rx_alloc_errs++;
514 break;
515 }
516 }
517
518 /* update RxBD */
519 rxbd->w.addr = cpu_to_le64(rx_swbd->dma +
520 rx_swbd->page_offset);
521 /* clear 'R" as well */
522 rxbd->r.lstatus = 0;
523
524 rxbd = enetc_rxbd_next(rx_ring, rxbd, i);
525 rx_swbd++;
526 i++;
527 if (unlikely(i == rx_ring->bd_count)) {
528 i = 0;
529 rx_swbd = rx_ring->rx_swbd;
530 }
531 }
532
533 if (likely(j)) {
534 rx_ring->next_to_alloc = i; /* keep track from page reuse */
535 rx_ring->next_to_use = i;
536 }
537
538 return j;
539 }
540
541 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
enetc_get_rx_tstamp(struct net_device * ndev,union enetc_rx_bd * rxbd,struct sk_buff * skb)542 static void enetc_get_rx_tstamp(struct net_device *ndev,
543 union enetc_rx_bd *rxbd,
544 struct sk_buff *skb)
545 {
546 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
547 struct enetc_ndev_priv *priv = netdev_priv(ndev);
548 struct enetc_hw *hw = &priv->si->hw;
549 u32 lo, hi, tstamp_lo;
550 u64 tstamp;
551
552 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) {
553 lo = enetc_rd_reg_hot(hw->reg + ENETC_SICTR0);
554 hi = enetc_rd_reg_hot(hw->reg + ENETC_SICTR1);
555 rxbd = enetc_rxbd_ext(rxbd);
556 tstamp_lo = le32_to_cpu(rxbd->ext.tstamp);
557 if (lo <= tstamp_lo)
558 hi -= 1;
559
560 tstamp = (u64)hi << 32 | tstamp_lo;
561 memset(shhwtstamps, 0, sizeof(*shhwtstamps));
562 shhwtstamps->hwtstamp = ns_to_ktime(tstamp);
563 }
564 }
565 #endif
566
enetc_get_offloads(struct enetc_bdr * rx_ring,union enetc_rx_bd * rxbd,struct sk_buff * skb)567 static void enetc_get_offloads(struct enetc_bdr *rx_ring,
568 union enetc_rx_bd *rxbd, struct sk_buff *skb)
569 {
570 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
571 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev);
572 #endif
573 /* TODO: hashing */
574 if (rx_ring->ndev->features & NETIF_F_RXCSUM) {
575 u16 inet_csum = le16_to_cpu(rxbd->r.inet_csum);
576
577 skb->csum = csum_unfold((__force __sum16)~htons(inet_csum));
578 skb->ip_summed = CHECKSUM_COMPLETE;
579 }
580
581 /* copy VLAN to skb, if one is extracted, for now we assume it's a
582 * standard TPID, but HW also supports custom values
583 */
584 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_VLAN)
585 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
586 le16_to_cpu(rxbd->r.vlan_opt));
587 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
588 if (priv->active_offloads & ENETC_F_RX_TSTAMP)
589 enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb);
590 #endif
591 }
592
enetc_process_skb(struct enetc_bdr * rx_ring,struct sk_buff * skb)593 static void enetc_process_skb(struct enetc_bdr *rx_ring,
594 struct sk_buff *skb)
595 {
596 skb_record_rx_queue(skb, rx_ring->index);
597 skb->protocol = eth_type_trans(skb, rx_ring->ndev);
598 }
599
enetc_page_reusable(struct page * page)600 static bool enetc_page_reusable(struct page *page)
601 {
602 return (!page_is_pfmemalloc(page) && page_ref_count(page) == 1);
603 }
604
enetc_reuse_page(struct enetc_bdr * rx_ring,struct enetc_rx_swbd * old)605 static void enetc_reuse_page(struct enetc_bdr *rx_ring,
606 struct enetc_rx_swbd *old)
607 {
608 struct enetc_rx_swbd *new;
609
610 new = &rx_ring->rx_swbd[rx_ring->next_to_alloc];
611
612 /* next buf that may reuse a page */
613 enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc);
614
615 /* copy page reference */
616 *new = *old;
617 }
618
enetc_get_rx_buff(struct enetc_bdr * rx_ring,int i,u16 size)619 static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring,
620 int i, u16 size)
621 {
622 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
623
624 dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma,
625 rx_swbd->page_offset,
626 size, DMA_FROM_DEVICE);
627 return rx_swbd;
628 }
629
enetc_put_rx_buff(struct enetc_bdr * rx_ring,struct enetc_rx_swbd * rx_swbd)630 static void enetc_put_rx_buff(struct enetc_bdr *rx_ring,
631 struct enetc_rx_swbd *rx_swbd)
632 {
633 if (likely(enetc_page_reusable(rx_swbd->page))) {
634 rx_swbd->page_offset ^= ENETC_RXB_TRUESIZE;
635 page_ref_inc(rx_swbd->page);
636
637 enetc_reuse_page(rx_ring, rx_swbd);
638
639 /* sync for use by the device */
640 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma,
641 rx_swbd->page_offset,
642 ENETC_RXB_DMA_SIZE,
643 DMA_FROM_DEVICE);
644 } else {
645 dma_unmap_page(rx_ring->dev, rx_swbd->dma,
646 PAGE_SIZE, DMA_FROM_DEVICE);
647 }
648
649 rx_swbd->page = NULL;
650 }
651
enetc_map_rx_buff_to_skb(struct enetc_bdr * rx_ring,int i,u16 size)652 static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring,
653 int i, u16 size)
654 {
655 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
656 struct sk_buff *skb;
657 void *ba;
658
659 ba = page_address(rx_swbd->page) + rx_swbd->page_offset;
660 skb = build_skb(ba - ENETC_RXB_PAD, ENETC_RXB_TRUESIZE);
661 if (unlikely(!skb)) {
662 rx_ring->stats.rx_alloc_errs++;
663 return NULL;
664 }
665
666 skb_reserve(skb, ENETC_RXB_PAD);
667 __skb_put(skb, size);
668
669 enetc_put_rx_buff(rx_ring, rx_swbd);
670
671 return skb;
672 }
673
enetc_add_rx_buff_to_skb(struct enetc_bdr * rx_ring,int i,u16 size,struct sk_buff * skb)674 static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i,
675 u16 size, struct sk_buff *skb)
676 {
677 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
678
679 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_swbd->page,
680 rx_swbd->page_offset, size, ENETC_RXB_TRUESIZE);
681
682 enetc_put_rx_buff(rx_ring, rx_swbd);
683 }
684
685 #define ENETC_RXBD_BUNDLE 16 /* # of BDs to update at once */
686
enetc_clean_rx_ring(struct enetc_bdr * rx_ring,struct napi_struct * napi,int work_limit)687 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring,
688 struct napi_struct *napi, int work_limit)
689 {
690 int rx_frm_cnt = 0, rx_byte_cnt = 0;
691 int cleaned_cnt, i;
692
693 cleaned_cnt = enetc_bd_unused(rx_ring);
694 /* next descriptor to process */
695 i = rx_ring->next_to_clean;
696
697 while (likely(rx_frm_cnt < work_limit)) {
698 union enetc_rx_bd *rxbd;
699 struct sk_buff *skb;
700 u32 bd_status;
701 u16 size;
702
703 enetc_lock_mdio();
704
705 if (cleaned_cnt >= ENETC_RXBD_BUNDLE) {
706 int count = enetc_refill_rx_ring(rx_ring, cleaned_cnt);
707
708 /* update ENETC's consumer index */
709 enetc_wr_reg_hot(rx_ring->rcir, rx_ring->next_to_use);
710 cleaned_cnt -= count;
711 }
712
713 rxbd = enetc_rxbd(rx_ring, i);
714 bd_status = le32_to_cpu(rxbd->r.lstatus);
715 if (!bd_status) {
716 enetc_unlock_mdio();
717 break;
718 }
719
720 enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index));
721 dma_rmb(); /* for reading other rxbd fields */
722 size = le16_to_cpu(rxbd->r.buf_len);
723 skb = enetc_map_rx_buff_to_skb(rx_ring, i, size);
724 if (!skb) {
725 enetc_unlock_mdio();
726 break;
727 }
728
729 enetc_get_offloads(rx_ring, rxbd, skb);
730
731 cleaned_cnt++;
732
733 rxbd = enetc_rxbd_next(rx_ring, rxbd, i);
734 if (unlikely(++i == rx_ring->bd_count))
735 i = 0;
736
737 if (unlikely(bd_status &
738 ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK))) {
739 enetc_unlock_mdio();
740 dev_kfree_skb(skb);
741 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
742 dma_rmb();
743 bd_status = le32_to_cpu(rxbd->r.lstatus);
744
745 rxbd = enetc_rxbd_next(rx_ring, rxbd, i);
746 if (unlikely(++i == rx_ring->bd_count))
747 i = 0;
748 }
749
750 rx_ring->ndev->stats.rx_dropped++;
751 rx_ring->ndev->stats.rx_errors++;
752
753 break;
754 }
755
756 /* not last BD in frame? */
757 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
758 bd_status = le32_to_cpu(rxbd->r.lstatus);
759 size = ENETC_RXB_DMA_SIZE;
760
761 if (bd_status & ENETC_RXBD_LSTATUS_F) {
762 dma_rmb();
763 size = le16_to_cpu(rxbd->r.buf_len);
764 }
765
766 enetc_add_rx_buff_to_skb(rx_ring, i, size, skb);
767
768 cleaned_cnt++;
769
770 rxbd = enetc_rxbd_next(rx_ring, rxbd, i);
771 if (unlikely(++i == rx_ring->bd_count))
772 i = 0;
773 }
774
775 rx_byte_cnt += skb->len;
776
777 enetc_process_skb(rx_ring, skb);
778
779 enetc_unlock_mdio();
780
781 napi_gro_receive(napi, skb);
782
783 rx_frm_cnt++;
784 }
785
786 rx_ring->next_to_clean = i;
787
788 rx_ring->stats.packets += rx_frm_cnt;
789 rx_ring->stats.bytes += rx_byte_cnt;
790
791 return rx_frm_cnt;
792 }
793
794 /* Probing and Init */
795 #define ENETC_MAX_RFS_SIZE 64
enetc_get_si_caps(struct enetc_si * si)796 void enetc_get_si_caps(struct enetc_si *si)
797 {
798 struct enetc_hw *hw = &si->hw;
799 u32 val;
800
801 /* find out how many of various resources we have to work with */
802 val = enetc_rd(hw, ENETC_SICAPR0);
803 si->num_rx_rings = (val >> 16) & 0xff;
804 si->num_tx_rings = val & 0xff;
805
806 val = enetc_rd(hw, ENETC_SIRFSCAPR);
807 si->num_fs_entries = ENETC_SIRFSCAPR_GET_NUM_RFS(val);
808 si->num_fs_entries = min(si->num_fs_entries, ENETC_MAX_RFS_SIZE);
809
810 si->num_rss = 0;
811 val = enetc_rd(hw, ENETC_SIPCAPR0);
812 if (val & ENETC_SIPCAPR0_RSS) {
813 u32 rss;
814
815 rss = enetc_rd(hw, ENETC_SIRSSCAPR);
816 si->num_rss = ENETC_SIRSSCAPR_GET_NUM_RSS(rss);
817 }
818
819 if (val & ENETC_SIPCAPR0_QBV)
820 si->hw_features |= ENETC_SI_F_QBV;
821
822 if (val & ENETC_SIPCAPR0_PSFP)
823 si->hw_features |= ENETC_SI_F_PSFP;
824 }
825
enetc_dma_alloc_bdr(struct enetc_bdr * r,size_t bd_size)826 static int enetc_dma_alloc_bdr(struct enetc_bdr *r, size_t bd_size)
827 {
828 r->bd_base = dma_alloc_coherent(r->dev, r->bd_count * bd_size,
829 &r->bd_dma_base, GFP_KERNEL);
830 if (!r->bd_base)
831 return -ENOMEM;
832
833 /* h/w requires 128B alignment */
834 if (!IS_ALIGNED(r->bd_dma_base, 128)) {
835 dma_free_coherent(r->dev, r->bd_count * bd_size, r->bd_base,
836 r->bd_dma_base);
837 return -EINVAL;
838 }
839
840 return 0;
841 }
842
enetc_alloc_txbdr(struct enetc_bdr * txr)843 static int enetc_alloc_txbdr(struct enetc_bdr *txr)
844 {
845 int err;
846
847 txr->tx_swbd = vzalloc(txr->bd_count * sizeof(struct enetc_tx_swbd));
848 if (!txr->tx_swbd)
849 return -ENOMEM;
850
851 err = enetc_dma_alloc_bdr(txr, sizeof(union enetc_tx_bd));
852 if (err) {
853 vfree(txr->tx_swbd);
854 return err;
855 }
856
857 txr->next_to_clean = 0;
858 txr->next_to_use = 0;
859
860 return 0;
861 }
862
enetc_free_txbdr(struct enetc_bdr * txr)863 static void enetc_free_txbdr(struct enetc_bdr *txr)
864 {
865 int size, i;
866
867 for (i = 0; i < txr->bd_count; i++)
868 enetc_free_tx_skb(txr, &txr->tx_swbd[i]);
869
870 size = txr->bd_count * sizeof(union enetc_tx_bd);
871
872 dma_free_coherent(txr->dev, size, txr->bd_base, txr->bd_dma_base);
873 txr->bd_base = NULL;
874
875 vfree(txr->tx_swbd);
876 txr->tx_swbd = NULL;
877 }
878
enetc_alloc_tx_resources(struct enetc_ndev_priv * priv)879 static int enetc_alloc_tx_resources(struct enetc_ndev_priv *priv)
880 {
881 int i, err;
882
883 for (i = 0; i < priv->num_tx_rings; i++) {
884 err = enetc_alloc_txbdr(priv->tx_ring[i]);
885
886 if (err)
887 goto fail;
888 }
889
890 return 0;
891
892 fail:
893 while (i-- > 0)
894 enetc_free_txbdr(priv->tx_ring[i]);
895
896 return err;
897 }
898
enetc_free_tx_resources(struct enetc_ndev_priv * priv)899 static void enetc_free_tx_resources(struct enetc_ndev_priv *priv)
900 {
901 int i;
902
903 for (i = 0; i < priv->num_tx_rings; i++)
904 enetc_free_txbdr(priv->tx_ring[i]);
905 }
906
enetc_alloc_rxbdr(struct enetc_bdr * rxr,bool extended)907 static int enetc_alloc_rxbdr(struct enetc_bdr *rxr, bool extended)
908 {
909 size_t size = sizeof(union enetc_rx_bd);
910 int err;
911
912 rxr->rx_swbd = vzalloc(rxr->bd_count * sizeof(struct enetc_rx_swbd));
913 if (!rxr->rx_swbd)
914 return -ENOMEM;
915
916 if (extended)
917 size *= 2;
918
919 err = enetc_dma_alloc_bdr(rxr, size);
920 if (err) {
921 vfree(rxr->rx_swbd);
922 return err;
923 }
924
925 rxr->next_to_clean = 0;
926 rxr->next_to_use = 0;
927 rxr->next_to_alloc = 0;
928 rxr->ext_en = extended;
929
930 return 0;
931 }
932
enetc_free_rxbdr(struct enetc_bdr * rxr)933 static void enetc_free_rxbdr(struct enetc_bdr *rxr)
934 {
935 int size;
936
937 size = rxr->bd_count * sizeof(union enetc_rx_bd);
938
939 dma_free_coherent(rxr->dev, size, rxr->bd_base, rxr->bd_dma_base);
940 rxr->bd_base = NULL;
941
942 vfree(rxr->rx_swbd);
943 rxr->rx_swbd = NULL;
944 }
945
enetc_alloc_rx_resources(struct enetc_ndev_priv * priv)946 static int enetc_alloc_rx_resources(struct enetc_ndev_priv *priv)
947 {
948 bool extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP);
949 int i, err;
950
951 for (i = 0; i < priv->num_rx_rings; i++) {
952 err = enetc_alloc_rxbdr(priv->rx_ring[i], extended);
953
954 if (err)
955 goto fail;
956 }
957
958 return 0;
959
960 fail:
961 while (i-- > 0)
962 enetc_free_rxbdr(priv->rx_ring[i]);
963
964 return err;
965 }
966
enetc_free_rx_resources(struct enetc_ndev_priv * priv)967 static void enetc_free_rx_resources(struct enetc_ndev_priv *priv)
968 {
969 int i;
970
971 for (i = 0; i < priv->num_rx_rings; i++)
972 enetc_free_rxbdr(priv->rx_ring[i]);
973 }
974
enetc_free_tx_ring(struct enetc_bdr * tx_ring)975 static void enetc_free_tx_ring(struct enetc_bdr *tx_ring)
976 {
977 int i;
978
979 if (!tx_ring->tx_swbd)
980 return;
981
982 for (i = 0; i < tx_ring->bd_count; i++) {
983 struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i];
984
985 enetc_free_tx_skb(tx_ring, tx_swbd);
986 }
987
988 tx_ring->next_to_clean = 0;
989 tx_ring->next_to_use = 0;
990 }
991
enetc_free_rx_ring(struct enetc_bdr * rx_ring)992 static void enetc_free_rx_ring(struct enetc_bdr *rx_ring)
993 {
994 int i;
995
996 if (!rx_ring->rx_swbd)
997 return;
998
999 for (i = 0; i < rx_ring->bd_count; i++) {
1000 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
1001
1002 if (!rx_swbd->page)
1003 continue;
1004
1005 dma_unmap_page(rx_ring->dev, rx_swbd->dma,
1006 PAGE_SIZE, DMA_FROM_DEVICE);
1007 __free_page(rx_swbd->page);
1008 rx_swbd->page = NULL;
1009 }
1010
1011 rx_ring->next_to_clean = 0;
1012 rx_ring->next_to_use = 0;
1013 rx_ring->next_to_alloc = 0;
1014 }
1015
enetc_free_rxtx_rings(struct enetc_ndev_priv * priv)1016 static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv)
1017 {
1018 int i;
1019
1020 for (i = 0; i < priv->num_rx_rings; i++)
1021 enetc_free_rx_ring(priv->rx_ring[i]);
1022
1023 for (i = 0; i < priv->num_tx_rings; i++)
1024 enetc_free_tx_ring(priv->tx_ring[i]);
1025 }
1026
enetc_alloc_cbdr(struct device * dev,struct enetc_cbdr * cbdr)1027 static int enetc_alloc_cbdr(struct device *dev, struct enetc_cbdr *cbdr)
1028 {
1029 int size = cbdr->bd_count * sizeof(struct enetc_cbd);
1030
1031 cbdr->bd_base = dma_alloc_coherent(dev, size, &cbdr->bd_dma_base,
1032 GFP_KERNEL);
1033 if (!cbdr->bd_base)
1034 return -ENOMEM;
1035
1036 /* h/w requires 128B alignment */
1037 if (!IS_ALIGNED(cbdr->bd_dma_base, 128)) {
1038 dma_free_coherent(dev, size, cbdr->bd_base, cbdr->bd_dma_base);
1039 return -EINVAL;
1040 }
1041
1042 cbdr->next_to_clean = 0;
1043 cbdr->next_to_use = 0;
1044
1045 return 0;
1046 }
1047
enetc_free_cbdr(struct device * dev,struct enetc_cbdr * cbdr)1048 static void enetc_free_cbdr(struct device *dev, struct enetc_cbdr *cbdr)
1049 {
1050 int size = cbdr->bd_count * sizeof(struct enetc_cbd);
1051
1052 dma_free_coherent(dev, size, cbdr->bd_base, cbdr->bd_dma_base);
1053 cbdr->bd_base = NULL;
1054 }
1055
enetc_setup_cbdr(struct enetc_hw * hw,struct enetc_cbdr * cbdr)1056 static void enetc_setup_cbdr(struct enetc_hw *hw, struct enetc_cbdr *cbdr)
1057 {
1058 /* set CBDR cache attributes */
1059 enetc_wr(hw, ENETC_SICAR2,
1060 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT);
1061
1062 enetc_wr(hw, ENETC_SICBDRBAR0, lower_32_bits(cbdr->bd_dma_base));
1063 enetc_wr(hw, ENETC_SICBDRBAR1, upper_32_bits(cbdr->bd_dma_base));
1064 enetc_wr(hw, ENETC_SICBDRLENR, ENETC_RTBLENR_LEN(cbdr->bd_count));
1065
1066 enetc_wr(hw, ENETC_SICBDRPIR, 0);
1067 enetc_wr(hw, ENETC_SICBDRCIR, 0);
1068
1069 /* enable ring */
1070 enetc_wr(hw, ENETC_SICBDRMR, BIT(31));
1071
1072 cbdr->pir = hw->reg + ENETC_SICBDRPIR;
1073 cbdr->cir = hw->reg + ENETC_SICBDRCIR;
1074 }
1075
enetc_clear_cbdr(struct enetc_hw * hw)1076 static void enetc_clear_cbdr(struct enetc_hw *hw)
1077 {
1078 enetc_wr(hw, ENETC_SICBDRMR, 0);
1079 }
1080
enetc_setup_default_rss_table(struct enetc_si * si,int num_groups)1081 static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups)
1082 {
1083 int *rss_table;
1084 int i;
1085
1086 rss_table = kmalloc_array(si->num_rss, sizeof(*rss_table), GFP_KERNEL);
1087 if (!rss_table)
1088 return -ENOMEM;
1089
1090 /* Set up RSS table defaults */
1091 for (i = 0; i < si->num_rss; i++)
1092 rss_table[i] = i % num_groups;
1093
1094 enetc_set_rss_table(si, rss_table, si->num_rss);
1095
1096 kfree(rss_table);
1097
1098 return 0;
1099 }
1100
enetc_configure_si(struct enetc_ndev_priv * priv)1101 static int enetc_configure_si(struct enetc_ndev_priv *priv)
1102 {
1103 struct enetc_si *si = priv->si;
1104 struct enetc_hw *hw = &si->hw;
1105 int err;
1106
1107 enetc_setup_cbdr(hw, &si->cbd_ring);
1108 /* set SI cache attributes */
1109 enetc_wr(hw, ENETC_SICAR0,
1110 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT);
1111 enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI);
1112 /* enable SI */
1113 enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN);
1114
1115 if (si->num_rss) {
1116 err = enetc_setup_default_rss_table(si, priv->num_rx_rings);
1117 if (err)
1118 return err;
1119 }
1120
1121 return 0;
1122 }
1123
enetc_init_si_rings_params(struct enetc_ndev_priv * priv)1124 void enetc_init_si_rings_params(struct enetc_ndev_priv *priv)
1125 {
1126 struct enetc_si *si = priv->si;
1127 int cpus = num_online_cpus();
1128
1129 priv->tx_bd_count = ENETC_TX_RING_DEFAULT_SIZE;
1130 priv->rx_bd_count = ENETC_RX_RING_DEFAULT_SIZE;
1131
1132 /* Enable all available TX rings in order to configure as many
1133 * priorities as possible, when needed.
1134 * TODO: Make # of TX rings run-time configurable
1135 */
1136 priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings);
1137 priv->num_tx_rings = si->num_tx_rings;
1138 priv->bdr_int_num = cpus;
1139 priv->ic_mode = ENETC_IC_RX_ADAPTIVE | ENETC_IC_TX_MANUAL;
1140 priv->tx_ictt = ENETC_TXIC_TIMETHR;
1141
1142 /* SI specific */
1143 si->cbd_ring.bd_count = ENETC_CBDR_DEFAULT_SIZE;
1144 }
1145
enetc_alloc_si_resources(struct enetc_ndev_priv * priv)1146 int enetc_alloc_si_resources(struct enetc_ndev_priv *priv)
1147 {
1148 struct enetc_si *si = priv->si;
1149 int err;
1150
1151 err = enetc_alloc_cbdr(priv->dev, &si->cbd_ring);
1152 if (err)
1153 return err;
1154
1155 priv->cls_rules = kcalloc(si->num_fs_entries, sizeof(*priv->cls_rules),
1156 GFP_KERNEL);
1157 if (!priv->cls_rules) {
1158 err = -ENOMEM;
1159 goto err_alloc_cls;
1160 }
1161
1162 err = enetc_configure_si(priv);
1163 if (err)
1164 goto err_config_si;
1165
1166 return 0;
1167
1168 err_config_si:
1169 kfree(priv->cls_rules);
1170 err_alloc_cls:
1171 enetc_clear_cbdr(&si->hw);
1172 enetc_free_cbdr(priv->dev, &si->cbd_ring);
1173
1174 return err;
1175 }
1176
enetc_free_si_resources(struct enetc_ndev_priv * priv)1177 void enetc_free_si_resources(struct enetc_ndev_priv *priv)
1178 {
1179 struct enetc_si *si = priv->si;
1180
1181 enetc_clear_cbdr(&si->hw);
1182 enetc_free_cbdr(priv->dev, &si->cbd_ring);
1183
1184 kfree(priv->cls_rules);
1185 }
1186
enetc_setup_txbdr(struct enetc_hw * hw,struct enetc_bdr * tx_ring)1187 static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
1188 {
1189 int idx = tx_ring->index;
1190 u32 tbmr;
1191
1192 enetc_txbdr_wr(hw, idx, ENETC_TBBAR0,
1193 lower_32_bits(tx_ring->bd_dma_base));
1194
1195 enetc_txbdr_wr(hw, idx, ENETC_TBBAR1,
1196 upper_32_bits(tx_ring->bd_dma_base));
1197
1198 WARN_ON(!IS_ALIGNED(tx_ring->bd_count, 64)); /* multiple of 64 */
1199 enetc_txbdr_wr(hw, idx, ENETC_TBLENR,
1200 ENETC_RTBLENR_LEN(tx_ring->bd_count));
1201
1202 /* clearing PI/CI registers for Tx not supported, adjust sw indexes */
1203 tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR);
1204 tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR);
1205
1206 /* enable Tx ints by setting pkt thr to 1 */
1207 enetc_txbdr_wr(hw, idx, ENETC_TBICR0, ENETC_TBICR0_ICEN | 0x1);
1208
1209 tbmr = ENETC_TBMR_EN;
1210 if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX)
1211 tbmr |= ENETC_TBMR_VIH;
1212
1213 /* enable ring */
1214 enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr);
1215
1216 tx_ring->tpir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBPIR);
1217 tx_ring->tcir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBCIR);
1218 tx_ring->idr = hw->reg + ENETC_SITXIDR;
1219 }
1220
enetc_setup_rxbdr(struct enetc_hw * hw,struct enetc_bdr * rx_ring)1221 static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
1222 {
1223 int idx = rx_ring->index;
1224 u32 rbmr;
1225
1226 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0,
1227 lower_32_bits(rx_ring->bd_dma_base));
1228
1229 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1,
1230 upper_32_bits(rx_ring->bd_dma_base));
1231
1232 WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64)); /* multiple of 64 */
1233 enetc_rxbdr_wr(hw, idx, ENETC_RBLENR,
1234 ENETC_RTBLENR_LEN(rx_ring->bd_count));
1235
1236 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE);
1237
1238 enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0);
1239
1240 /* enable Rx ints by setting pkt thr to 1 */
1241 enetc_rxbdr_wr(hw, idx, ENETC_RBICR0, ENETC_RBICR0_ICEN | 0x1);
1242
1243 rbmr = ENETC_RBMR_EN;
1244
1245 if (rx_ring->ext_en)
1246 rbmr |= ENETC_RBMR_BDS;
1247
1248 if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX)
1249 rbmr |= ENETC_RBMR_VTE;
1250
1251 rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR);
1252 rx_ring->idr = hw->reg + ENETC_SIRXIDR;
1253
1254 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring));
1255 enetc_wr(hw, ENETC_SIRXIDR, rx_ring->next_to_use);
1256
1257 /* enable ring */
1258 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr);
1259 }
1260
enetc_setup_bdrs(struct enetc_ndev_priv * priv)1261 static void enetc_setup_bdrs(struct enetc_ndev_priv *priv)
1262 {
1263 int i;
1264
1265 for (i = 0; i < priv->num_tx_rings; i++)
1266 enetc_setup_txbdr(&priv->si->hw, priv->tx_ring[i]);
1267
1268 for (i = 0; i < priv->num_rx_rings; i++)
1269 enetc_setup_rxbdr(&priv->si->hw, priv->rx_ring[i]);
1270 }
1271
enetc_clear_rxbdr(struct enetc_hw * hw,struct enetc_bdr * rx_ring)1272 static void enetc_clear_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
1273 {
1274 int idx = rx_ring->index;
1275
1276 /* disable EN bit on ring */
1277 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, 0);
1278 }
1279
enetc_clear_txbdr(struct enetc_hw * hw,struct enetc_bdr * tx_ring)1280 static void enetc_clear_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
1281 {
1282 int delay = 8, timeout = 100;
1283 int idx = tx_ring->index;
1284
1285 /* disable EN bit on ring */
1286 enetc_txbdr_wr(hw, idx, ENETC_TBMR, 0);
1287
1288 /* wait for busy to clear */
1289 while (delay < timeout &&
1290 enetc_txbdr_rd(hw, idx, ENETC_TBSR) & ENETC_TBSR_BUSY) {
1291 msleep(delay);
1292 delay *= 2;
1293 }
1294
1295 if (delay >= timeout)
1296 netdev_warn(tx_ring->ndev, "timeout for tx ring #%d clear\n",
1297 idx);
1298 }
1299
enetc_clear_bdrs(struct enetc_ndev_priv * priv)1300 static void enetc_clear_bdrs(struct enetc_ndev_priv *priv)
1301 {
1302 int i;
1303
1304 for (i = 0; i < priv->num_tx_rings; i++)
1305 enetc_clear_txbdr(&priv->si->hw, priv->tx_ring[i]);
1306
1307 for (i = 0; i < priv->num_rx_rings; i++)
1308 enetc_clear_rxbdr(&priv->si->hw, priv->rx_ring[i]);
1309
1310 udelay(1);
1311 }
1312
enetc_setup_irqs(struct enetc_ndev_priv * priv)1313 static int enetc_setup_irqs(struct enetc_ndev_priv *priv)
1314 {
1315 struct pci_dev *pdev = priv->si->pdev;
1316 cpumask_t cpu_mask;
1317 int i, j, err;
1318
1319 for (i = 0; i < priv->bdr_int_num; i++) {
1320 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
1321 struct enetc_int_vector *v = priv->int_vector[i];
1322 int entry = ENETC_BDR_INT_BASE_IDX + i;
1323 struct enetc_hw *hw = &priv->si->hw;
1324
1325 snprintf(v->name, sizeof(v->name), "%s-rxtx%d",
1326 priv->ndev->name, i);
1327 err = request_irq(irq, enetc_msix, 0, v->name, v);
1328 if (err) {
1329 dev_err(priv->dev, "request_irq() failed!\n");
1330 goto irq_err;
1331 }
1332 disable_irq(irq);
1333
1334 v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER);
1335 v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER);
1336 v->ricr1 = hw->reg + ENETC_BDR(RX, i, ENETC_RBICR1);
1337
1338 enetc_wr(hw, ENETC_SIMSIRRV(i), entry);
1339
1340 for (j = 0; j < v->count_tx_rings; j++) {
1341 int idx = v->tx_ring[j].index;
1342
1343 enetc_wr(hw, ENETC_SIMSITRV(idx), entry);
1344 }
1345 cpumask_clear(&cpu_mask);
1346 cpumask_set_cpu(i % num_online_cpus(), &cpu_mask);
1347 irq_set_affinity_hint(irq, &cpu_mask);
1348 }
1349
1350 return 0;
1351
1352 irq_err:
1353 while (i--) {
1354 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
1355
1356 irq_set_affinity_hint(irq, NULL);
1357 free_irq(irq, priv->int_vector[i]);
1358 }
1359
1360 return err;
1361 }
1362
enetc_free_irqs(struct enetc_ndev_priv * priv)1363 static void enetc_free_irqs(struct enetc_ndev_priv *priv)
1364 {
1365 struct pci_dev *pdev = priv->si->pdev;
1366 int i;
1367
1368 for (i = 0; i < priv->bdr_int_num; i++) {
1369 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
1370
1371 irq_set_affinity_hint(irq, NULL);
1372 free_irq(irq, priv->int_vector[i]);
1373 }
1374 }
1375
enetc_setup_interrupts(struct enetc_ndev_priv * priv)1376 static void enetc_setup_interrupts(struct enetc_ndev_priv *priv)
1377 {
1378 struct enetc_hw *hw = &priv->si->hw;
1379 u32 icpt, ictt;
1380 int i;
1381
1382 /* enable Tx & Rx event indication */
1383 if (priv->ic_mode &
1384 (ENETC_IC_RX_MANUAL | ENETC_IC_RX_ADAPTIVE)) {
1385 icpt = ENETC_RBICR0_SET_ICPT(ENETC_RXIC_PKTTHR);
1386 /* init to non-0 minimum, will be adjusted later */
1387 ictt = 0x1;
1388 } else {
1389 icpt = 0x1; /* enable Rx ints by setting pkt thr to 1 */
1390 ictt = 0;
1391 }
1392
1393 for (i = 0; i < priv->num_rx_rings; i++) {
1394 enetc_rxbdr_wr(hw, i, ENETC_RBICR1, ictt);
1395 enetc_rxbdr_wr(hw, i, ENETC_RBICR0, ENETC_RBICR0_ICEN | icpt);
1396 enetc_rxbdr_wr(hw, i, ENETC_RBIER, ENETC_RBIER_RXTIE);
1397 }
1398
1399 if (priv->ic_mode & ENETC_IC_TX_MANUAL)
1400 icpt = ENETC_TBICR0_SET_ICPT(ENETC_TXIC_PKTTHR);
1401 else
1402 icpt = 0x1; /* enable Tx ints by setting pkt thr to 1 */
1403
1404 for (i = 0; i < priv->num_tx_rings; i++) {
1405 enetc_txbdr_wr(hw, i, ENETC_TBICR1, priv->tx_ictt);
1406 enetc_txbdr_wr(hw, i, ENETC_TBICR0, ENETC_TBICR0_ICEN | icpt);
1407 enetc_txbdr_wr(hw, i, ENETC_TBIER, ENETC_TBIER_TXTIE);
1408 }
1409 }
1410
enetc_clear_interrupts(struct enetc_ndev_priv * priv)1411 static void enetc_clear_interrupts(struct enetc_ndev_priv *priv)
1412 {
1413 int i;
1414
1415 for (i = 0; i < priv->num_tx_rings; i++)
1416 enetc_txbdr_wr(&priv->si->hw, i, ENETC_TBIER, 0);
1417
1418 for (i = 0; i < priv->num_rx_rings; i++)
1419 enetc_rxbdr_wr(&priv->si->hw, i, ENETC_RBIER, 0);
1420 }
1421
enetc_phylink_connect(struct net_device * ndev)1422 static int enetc_phylink_connect(struct net_device *ndev)
1423 {
1424 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1425 struct ethtool_eee edata;
1426 int err;
1427
1428 if (!priv->phylink)
1429 return 0; /* phy-less mode */
1430
1431 err = phylink_of_phy_connect(priv->phylink, priv->dev->of_node, 0);
1432 if (err) {
1433 dev_err(&ndev->dev, "could not attach to PHY\n");
1434 return err;
1435 }
1436
1437 /* disable EEE autoneg, until ENETC driver supports it */
1438 memset(&edata, 0, sizeof(struct ethtool_eee));
1439 phylink_ethtool_set_eee(priv->phylink, &edata);
1440
1441 return 0;
1442 }
1443
enetc_start(struct net_device * ndev)1444 void enetc_start(struct net_device *ndev)
1445 {
1446 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1447 int i;
1448
1449 enetc_setup_interrupts(priv);
1450
1451 for (i = 0; i < priv->bdr_int_num; i++) {
1452 int irq = pci_irq_vector(priv->si->pdev,
1453 ENETC_BDR_INT_BASE_IDX + i);
1454
1455 napi_enable(&priv->int_vector[i]->napi);
1456 enable_irq(irq);
1457 }
1458
1459 if (priv->phylink)
1460 phylink_start(priv->phylink);
1461 else
1462 netif_carrier_on(ndev);
1463
1464 netif_tx_start_all_queues(ndev);
1465 }
1466
enetc_open(struct net_device * ndev)1467 int enetc_open(struct net_device *ndev)
1468 {
1469 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1470 int err;
1471
1472 err = enetc_setup_irqs(priv);
1473 if (err)
1474 return err;
1475
1476 err = enetc_phylink_connect(ndev);
1477 if (err)
1478 goto err_phy_connect;
1479
1480 err = enetc_alloc_tx_resources(priv);
1481 if (err)
1482 goto err_alloc_tx;
1483
1484 err = enetc_alloc_rx_resources(priv);
1485 if (err)
1486 goto err_alloc_rx;
1487
1488 err = netif_set_real_num_tx_queues(ndev, priv->num_tx_rings);
1489 if (err)
1490 goto err_set_queues;
1491
1492 err = netif_set_real_num_rx_queues(ndev, priv->num_rx_rings);
1493 if (err)
1494 goto err_set_queues;
1495
1496 enetc_setup_bdrs(priv);
1497 enetc_start(ndev);
1498
1499 return 0;
1500
1501 err_set_queues:
1502 enetc_free_rx_resources(priv);
1503 err_alloc_rx:
1504 enetc_free_tx_resources(priv);
1505 err_alloc_tx:
1506 if (priv->phylink)
1507 phylink_disconnect_phy(priv->phylink);
1508 err_phy_connect:
1509 enetc_free_irqs(priv);
1510
1511 return err;
1512 }
1513
enetc_stop(struct net_device * ndev)1514 void enetc_stop(struct net_device *ndev)
1515 {
1516 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1517 int i;
1518
1519 netif_tx_stop_all_queues(ndev);
1520
1521 for (i = 0; i < priv->bdr_int_num; i++) {
1522 int irq = pci_irq_vector(priv->si->pdev,
1523 ENETC_BDR_INT_BASE_IDX + i);
1524
1525 disable_irq(irq);
1526 napi_synchronize(&priv->int_vector[i]->napi);
1527 napi_disable(&priv->int_vector[i]->napi);
1528 }
1529
1530 if (priv->phylink)
1531 phylink_stop(priv->phylink);
1532 else
1533 netif_carrier_off(ndev);
1534
1535 enetc_clear_interrupts(priv);
1536 }
1537
enetc_close(struct net_device * ndev)1538 int enetc_close(struct net_device *ndev)
1539 {
1540 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1541
1542 enetc_stop(ndev);
1543 enetc_clear_bdrs(priv);
1544
1545 if (priv->phylink)
1546 phylink_disconnect_phy(priv->phylink);
1547 enetc_free_rxtx_rings(priv);
1548 enetc_free_rx_resources(priv);
1549 enetc_free_tx_resources(priv);
1550 enetc_free_irqs(priv);
1551
1552 return 0;
1553 }
1554
enetc_setup_tc_mqprio(struct net_device * ndev,void * type_data)1555 static int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data)
1556 {
1557 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1558 struct tc_mqprio_qopt *mqprio = type_data;
1559 struct enetc_bdr *tx_ring;
1560 u8 num_tc;
1561 int i;
1562
1563 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
1564 num_tc = mqprio->num_tc;
1565
1566 if (!num_tc) {
1567 netdev_reset_tc(ndev);
1568 netif_set_real_num_tx_queues(ndev, priv->num_tx_rings);
1569
1570 /* Reset all ring priorities to 0 */
1571 for (i = 0; i < priv->num_tx_rings; i++) {
1572 tx_ring = priv->tx_ring[i];
1573 enetc_set_bdr_prio(&priv->si->hw, tx_ring->index, 0);
1574 }
1575
1576 return 0;
1577 }
1578
1579 /* Check if we have enough BD rings available to accommodate all TCs */
1580 if (num_tc > priv->num_tx_rings) {
1581 netdev_err(ndev, "Max %d traffic classes supported\n",
1582 priv->num_tx_rings);
1583 return -EINVAL;
1584 }
1585
1586 /* For the moment, we use only one BD ring per TC.
1587 *
1588 * Configure num_tc BD rings with increasing priorities.
1589 */
1590 for (i = 0; i < num_tc; i++) {
1591 tx_ring = priv->tx_ring[i];
1592 enetc_set_bdr_prio(&priv->si->hw, tx_ring->index, i);
1593 }
1594
1595 /* Reset the number of netdev queues based on the TC count */
1596 netif_set_real_num_tx_queues(ndev, num_tc);
1597
1598 netdev_set_num_tc(ndev, num_tc);
1599
1600 /* Each TC is associated with one netdev queue */
1601 for (i = 0; i < num_tc; i++)
1602 netdev_set_tc_queue(ndev, i, 1, i);
1603
1604 return 0;
1605 }
1606
enetc_setup_tc(struct net_device * ndev,enum tc_setup_type type,void * type_data)1607 int enetc_setup_tc(struct net_device *ndev, enum tc_setup_type type,
1608 void *type_data)
1609 {
1610 switch (type) {
1611 case TC_SETUP_QDISC_MQPRIO:
1612 return enetc_setup_tc_mqprio(ndev, type_data);
1613 case TC_SETUP_QDISC_TAPRIO:
1614 return enetc_setup_tc_taprio(ndev, type_data);
1615 case TC_SETUP_QDISC_CBS:
1616 return enetc_setup_tc_cbs(ndev, type_data);
1617 case TC_SETUP_QDISC_ETF:
1618 return enetc_setup_tc_txtime(ndev, type_data);
1619 case TC_SETUP_BLOCK:
1620 return enetc_setup_tc_psfp(ndev, type_data);
1621 default:
1622 return -EOPNOTSUPP;
1623 }
1624 }
1625
enetc_get_stats(struct net_device * ndev)1626 struct net_device_stats *enetc_get_stats(struct net_device *ndev)
1627 {
1628 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1629 struct net_device_stats *stats = &ndev->stats;
1630 unsigned long packets = 0, bytes = 0;
1631 int i;
1632
1633 for (i = 0; i < priv->num_rx_rings; i++) {
1634 packets += priv->rx_ring[i]->stats.packets;
1635 bytes += priv->rx_ring[i]->stats.bytes;
1636 }
1637
1638 stats->rx_packets = packets;
1639 stats->rx_bytes = bytes;
1640 bytes = 0;
1641 packets = 0;
1642
1643 for (i = 0; i < priv->num_tx_rings; i++) {
1644 packets += priv->tx_ring[i]->stats.packets;
1645 bytes += priv->tx_ring[i]->stats.bytes;
1646 }
1647
1648 stats->tx_packets = packets;
1649 stats->tx_bytes = bytes;
1650
1651 return stats;
1652 }
1653
enetc_set_rss(struct net_device * ndev,int en)1654 static int enetc_set_rss(struct net_device *ndev, int en)
1655 {
1656 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1657 struct enetc_hw *hw = &priv->si->hw;
1658 u32 reg;
1659
1660 enetc_wr(hw, ENETC_SIRBGCR, priv->num_rx_rings);
1661
1662 reg = enetc_rd(hw, ENETC_SIMR);
1663 reg &= ~ENETC_SIMR_RSSE;
1664 reg |= (en) ? ENETC_SIMR_RSSE : 0;
1665 enetc_wr(hw, ENETC_SIMR, reg);
1666
1667 return 0;
1668 }
1669
enetc_set_psfp(struct net_device * ndev,int en)1670 static int enetc_set_psfp(struct net_device *ndev, int en)
1671 {
1672 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1673 int err;
1674
1675 if (en) {
1676 err = enetc_psfp_enable(priv);
1677 if (err)
1678 return err;
1679
1680 priv->active_offloads |= ENETC_F_QCI;
1681 return 0;
1682 }
1683
1684 err = enetc_psfp_disable(priv);
1685 if (err)
1686 return err;
1687
1688 priv->active_offloads &= ~ENETC_F_QCI;
1689
1690 return 0;
1691 }
1692
enetc_enable_rxvlan(struct net_device * ndev,bool en)1693 static void enetc_enable_rxvlan(struct net_device *ndev, bool en)
1694 {
1695 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1696 int i;
1697
1698 for (i = 0; i < priv->num_rx_rings; i++)
1699 enetc_bdr_enable_rxvlan(&priv->si->hw, i, en);
1700 }
1701
enetc_enable_txvlan(struct net_device * ndev,bool en)1702 static void enetc_enable_txvlan(struct net_device *ndev, bool en)
1703 {
1704 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1705 int i;
1706
1707 for (i = 0; i < priv->num_tx_rings; i++)
1708 enetc_bdr_enable_txvlan(&priv->si->hw, i, en);
1709 }
1710
enetc_set_features(struct net_device * ndev,netdev_features_t features)1711 int enetc_set_features(struct net_device *ndev,
1712 netdev_features_t features)
1713 {
1714 netdev_features_t changed = ndev->features ^ features;
1715 int err = 0;
1716
1717 if (changed & NETIF_F_RXHASH)
1718 enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH));
1719
1720 if (changed & NETIF_F_HW_VLAN_CTAG_RX)
1721 enetc_enable_rxvlan(ndev,
1722 !!(features & NETIF_F_HW_VLAN_CTAG_RX));
1723
1724 if (changed & NETIF_F_HW_VLAN_CTAG_TX)
1725 enetc_enable_txvlan(ndev,
1726 !!(features & NETIF_F_HW_VLAN_CTAG_TX));
1727
1728 if (changed & NETIF_F_HW_TC)
1729 err = enetc_set_psfp(ndev, !!(features & NETIF_F_HW_TC));
1730
1731 return err;
1732 }
1733
1734 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
enetc_hwtstamp_set(struct net_device * ndev,struct ifreq * ifr)1735 static int enetc_hwtstamp_set(struct net_device *ndev, struct ifreq *ifr)
1736 {
1737 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1738 struct hwtstamp_config config;
1739 int ao;
1740
1741 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
1742 return -EFAULT;
1743
1744 switch (config.tx_type) {
1745 case HWTSTAMP_TX_OFF:
1746 priv->active_offloads &= ~ENETC_F_TX_TSTAMP;
1747 break;
1748 case HWTSTAMP_TX_ON:
1749 priv->active_offloads |= ENETC_F_TX_TSTAMP;
1750 break;
1751 default:
1752 return -ERANGE;
1753 }
1754
1755 ao = priv->active_offloads;
1756 switch (config.rx_filter) {
1757 case HWTSTAMP_FILTER_NONE:
1758 priv->active_offloads &= ~ENETC_F_RX_TSTAMP;
1759 break;
1760 default:
1761 priv->active_offloads |= ENETC_F_RX_TSTAMP;
1762 config.rx_filter = HWTSTAMP_FILTER_ALL;
1763 }
1764
1765 if (netif_running(ndev) && ao != priv->active_offloads) {
1766 enetc_close(ndev);
1767 enetc_open(ndev);
1768 }
1769
1770 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
1771 -EFAULT : 0;
1772 }
1773
enetc_hwtstamp_get(struct net_device * ndev,struct ifreq * ifr)1774 static int enetc_hwtstamp_get(struct net_device *ndev, struct ifreq *ifr)
1775 {
1776 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1777 struct hwtstamp_config config;
1778
1779 config.flags = 0;
1780
1781 if (priv->active_offloads & ENETC_F_TX_TSTAMP)
1782 config.tx_type = HWTSTAMP_TX_ON;
1783 else
1784 config.tx_type = HWTSTAMP_TX_OFF;
1785
1786 config.rx_filter = (priv->active_offloads & ENETC_F_RX_TSTAMP) ?
1787 HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE;
1788
1789 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
1790 -EFAULT : 0;
1791 }
1792 #endif
1793
enetc_ioctl(struct net_device * ndev,struct ifreq * rq,int cmd)1794 int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd)
1795 {
1796 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1797 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
1798 if (cmd == SIOCSHWTSTAMP)
1799 return enetc_hwtstamp_set(ndev, rq);
1800 if (cmd == SIOCGHWTSTAMP)
1801 return enetc_hwtstamp_get(ndev, rq);
1802 #endif
1803
1804 if (!priv->phylink)
1805 return -EOPNOTSUPP;
1806
1807 return phylink_mii_ioctl(priv->phylink, rq, cmd);
1808 }
1809
enetc_alloc_msix(struct enetc_ndev_priv * priv)1810 int enetc_alloc_msix(struct enetc_ndev_priv *priv)
1811 {
1812 struct pci_dev *pdev = priv->si->pdev;
1813 int v_tx_rings;
1814 int i, n, err, nvec;
1815
1816 nvec = ENETC_BDR_INT_BASE_IDX + priv->bdr_int_num;
1817 /* allocate MSIX for both messaging and Rx/Tx interrupts */
1818 n = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX);
1819
1820 if (n < 0)
1821 return n;
1822
1823 if (n != nvec)
1824 return -EPERM;
1825
1826 /* # of tx rings per int vector */
1827 v_tx_rings = priv->num_tx_rings / priv->bdr_int_num;
1828
1829 for (i = 0; i < priv->bdr_int_num; i++) {
1830 struct enetc_int_vector *v;
1831 struct enetc_bdr *bdr;
1832 int j;
1833
1834 v = kzalloc(struct_size(v, tx_ring, v_tx_rings), GFP_KERNEL);
1835 if (!v) {
1836 err = -ENOMEM;
1837 goto fail;
1838 }
1839
1840 priv->int_vector[i] = v;
1841
1842 /* init defaults for adaptive IC */
1843 if (priv->ic_mode & ENETC_IC_RX_ADAPTIVE) {
1844 v->rx_ictt = 0x1;
1845 v->rx_dim_en = true;
1846 }
1847 INIT_WORK(&v->rx_dim.work, enetc_rx_dim_work);
1848 netif_napi_add(priv->ndev, &v->napi, enetc_poll,
1849 NAPI_POLL_WEIGHT);
1850 v->count_tx_rings = v_tx_rings;
1851
1852 for (j = 0; j < v_tx_rings; j++) {
1853 int idx;
1854
1855 /* default tx ring mapping policy */
1856 if (priv->bdr_int_num == ENETC_MAX_BDR_INT)
1857 idx = 2 * j + i; /* 2 CPUs */
1858 else
1859 idx = j + i * v_tx_rings; /* default */
1860
1861 __set_bit(idx, &v->tx_rings_map);
1862 bdr = &v->tx_ring[j];
1863 bdr->index = idx;
1864 bdr->ndev = priv->ndev;
1865 bdr->dev = priv->dev;
1866 bdr->bd_count = priv->tx_bd_count;
1867 priv->tx_ring[idx] = bdr;
1868 }
1869
1870 bdr = &v->rx_ring;
1871 bdr->index = i;
1872 bdr->ndev = priv->ndev;
1873 bdr->dev = priv->dev;
1874 bdr->bd_count = priv->rx_bd_count;
1875 priv->rx_ring[i] = bdr;
1876 }
1877
1878 return 0;
1879
1880 fail:
1881 while (i--) {
1882 netif_napi_del(&priv->int_vector[i]->napi);
1883 cancel_work_sync(&priv->int_vector[i]->rx_dim.work);
1884 kfree(priv->int_vector[i]);
1885 }
1886
1887 pci_free_irq_vectors(pdev);
1888
1889 return err;
1890 }
1891
enetc_free_msix(struct enetc_ndev_priv * priv)1892 void enetc_free_msix(struct enetc_ndev_priv *priv)
1893 {
1894 int i;
1895
1896 for (i = 0; i < priv->bdr_int_num; i++) {
1897 struct enetc_int_vector *v = priv->int_vector[i];
1898
1899 netif_napi_del(&v->napi);
1900 cancel_work_sync(&v->rx_dim.work);
1901 }
1902
1903 for (i = 0; i < priv->num_rx_rings; i++)
1904 priv->rx_ring[i] = NULL;
1905
1906 for (i = 0; i < priv->num_tx_rings; i++)
1907 priv->tx_ring[i] = NULL;
1908
1909 for (i = 0; i < priv->bdr_int_num; i++) {
1910 kfree(priv->int_vector[i]);
1911 priv->int_vector[i] = NULL;
1912 }
1913
1914 /* disable all MSIX for this device */
1915 pci_free_irq_vectors(priv->si->pdev);
1916 }
1917
enetc_kfree_si(struct enetc_si * si)1918 static void enetc_kfree_si(struct enetc_si *si)
1919 {
1920 char *p = (char *)si - si->pad;
1921
1922 kfree(p);
1923 }
1924
enetc_detect_errata(struct enetc_si * si)1925 static void enetc_detect_errata(struct enetc_si *si)
1926 {
1927 if (si->pdev->revision == ENETC_REV1)
1928 si->errata = ENETC_ERR_TXCSUM | ENETC_ERR_VLAN_ISOL |
1929 ENETC_ERR_UCMCSWP;
1930 }
1931
enetc_pci_probe(struct pci_dev * pdev,const char * name,int sizeof_priv)1932 int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv)
1933 {
1934 struct enetc_si *si, *p;
1935 struct enetc_hw *hw;
1936 size_t alloc_size;
1937 int err, len;
1938
1939 pcie_flr(pdev);
1940 err = pci_enable_device_mem(pdev);
1941 if (err) {
1942 dev_err(&pdev->dev, "device enable failed\n");
1943 return err;
1944 }
1945
1946 /* set up for high or low dma */
1947 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
1948 if (err) {
1949 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
1950 if (err) {
1951 dev_err(&pdev->dev,
1952 "DMA configuration failed: 0x%x\n", err);
1953 goto err_dma;
1954 }
1955 }
1956
1957 err = pci_request_mem_regions(pdev, name);
1958 if (err) {
1959 dev_err(&pdev->dev, "pci_request_regions failed err=%d\n", err);
1960 goto err_pci_mem_reg;
1961 }
1962
1963 pci_set_master(pdev);
1964
1965 alloc_size = sizeof(struct enetc_si);
1966 if (sizeof_priv) {
1967 /* align priv to 32B */
1968 alloc_size = ALIGN(alloc_size, ENETC_SI_ALIGN);
1969 alloc_size += sizeof_priv;
1970 }
1971 /* force 32B alignment for enetc_si */
1972 alloc_size += ENETC_SI_ALIGN - 1;
1973
1974 p = kzalloc(alloc_size, GFP_KERNEL);
1975 if (!p) {
1976 err = -ENOMEM;
1977 goto err_alloc_si;
1978 }
1979
1980 si = PTR_ALIGN(p, ENETC_SI_ALIGN);
1981 si->pad = (char *)si - (char *)p;
1982
1983 pci_set_drvdata(pdev, si);
1984 si->pdev = pdev;
1985 hw = &si->hw;
1986
1987 len = pci_resource_len(pdev, ENETC_BAR_REGS);
1988 hw->reg = ioremap(pci_resource_start(pdev, ENETC_BAR_REGS), len);
1989 if (!hw->reg) {
1990 err = -ENXIO;
1991 dev_err(&pdev->dev, "ioremap() failed\n");
1992 goto err_ioremap;
1993 }
1994 if (len > ENETC_PORT_BASE)
1995 hw->port = hw->reg + ENETC_PORT_BASE;
1996 if (len > ENETC_GLOBAL_BASE)
1997 hw->global = hw->reg + ENETC_GLOBAL_BASE;
1998
1999 enetc_detect_errata(si);
2000
2001 return 0;
2002
2003 err_ioremap:
2004 enetc_kfree_si(si);
2005 err_alloc_si:
2006 pci_release_mem_regions(pdev);
2007 err_pci_mem_reg:
2008 err_dma:
2009 pci_disable_device(pdev);
2010
2011 return err;
2012 }
2013
enetc_pci_remove(struct pci_dev * pdev)2014 void enetc_pci_remove(struct pci_dev *pdev)
2015 {
2016 struct enetc_si *si = pci_get_drvdata(pdev);
2017 struct enetc_hw *hw = &si->hw;
2018
2019 iounmap(hw->reg);
2020 enetc_kfree_si(si);
2021 pci_release_mem_regions(pdev);
2022 pci_disable_device(pdev);
2023 }
2024
