1 // SPDX-License-Identifier: GPL-2.0
2 /* Marvell Octeon EP (EndPoint) Ethernet Driver
3 *
4 * Copyright (C) 2020 Marvell.
5 *
6 */
7
8 #include <linux/types.h>
9 #include <linux/module.h>
10 #include <linux/pci.h>
11 #include <linux/aer.h>
12 #include <linux/netdevice.h>
13 #include <linux/etherdevice.h>
14 #include <linux/rtnetlink.h>
15 #include <linux/vmalloc.h>
16
17 #include "octep_config.h"
18 #include "octep_main.h"
19 #include "octep_ctrl_net.h"
20
21 struct workqueue_struct *octep_wq;
22
23 /* Supported Devices */
24 static const struct pci_device_id octep_pci_id_tbl[] = {
25 {PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CN93_PF)},
26 {0, },
27 };
28 MODULE_DEVICE_TABLE(pci, octep_pci_id_tbl);
29
30 MODULE_AUTHOR("Veerasenareddy Burru <vburru@marvell.com>");
31 MODULE_DESCRIPTION(OCTEP_DRV_STRING);
32 MODULE_LICENSE("GPL");
33
34 /**
35 * octep_alloc_ioq_vectors() - Allocate Tx/Rx Queue interrupt info.
36 *
37 * @oct: Octeon device private data structure.
38 *
39 * Allocate resources to hold per Tx/Rx queue interrupt info.
40 * This is the information passed to interrupt handler, from which napi poll
41 * is scheduled and includes quick access to private data of Tx/Rx queue
42 * corresponding to the interrupt being handled.
43 *
44 * Return: 0, on successful allocation of resources for all queue interrupts.
45 * -1, if failed to allocate any resource.
46 */
octep_alloc_ioq_vectors(struct octep_device * oct)47 static int octep_alloc_ioq_vectors(struct octep_device *oct)
48 {
49 int i;
50 struct octep_ioq_vector *ioq_vector;
51
52 for (i = 0; i < oct->num_oqs; i++) {
53 oct->ioq_vector[i] = vzalloc(sizeof(*oct->ioq_vector[i]));
54 if (!oct->ioq_vector[i])
55 goto free_ioq_vector;
56
57 ioq_vector = oct->ioq_vector[i];
58 ioq_vector->iq = oct->iq[i];
59 ioq_vector->oq = oct->oq[i];
60 ioq_vector->octep_dev = oct;
61 }
62
63 dev_info(&oct->pdev->dev, "Allocated %d IOQ vectors\n", oct->num_oqs);
64 return 0;
65
66 free_ioq_vector:
67 while (i) {
68 i--;
69 vfree(oct->ioq_vector[i]);
70 oct->ioq_vector[i] = NULL;
71 }
72 return -1;
73 }
74
75 /**
76 * octep_free_ioq_vectors() - Free Tx/Rx Queue interrupt vector info.
77 *
78 * @oct: Octeon device private data structure.
79 */
octep_free_ioq_vectors(struct octep_device * oct)80 static void octep_free_ioq_vectors(struct octep_device *oct)
81 {
82 int i;
83
84 for (i = 0; i < oct->num_oqs; i++) {
85 if (oct->ioq_vector[i]) {
86 vfree(oct->ioq_vector[i]);
87 oct->ioq_vector[i] = NULL;
88 }
89 }
90 netdev_info(oct->netdev, "Freed IOQ Vectors\n");
91 }
92
93 /**
94 * octep_enable_msix_range() - enable MSI-x interrupts.
95 *
96 * @oct: Octeon device private data structure.
97 *
98 * Allocate and enable all MSI-x interrupts (queue and non-queue interrupts)
99 * for the Octeon device.
100 *
101 * Return: 0, on successfully enabling all MSI-x interrupts.
102 * -1, if failed to enable any MSI-x interrupt.
103 */
octep_enable_msix_range(struct octep_device * oct)104 static int octep_enable_msix_range(struct octep_device *oct)
105 {
106 int num_msix, msix_allocated;
107 int i;
108
109 /* Generic interrupts apart from input/output queues */
110 num_msix = oct->num_oqs + CFG_GET_NON_IOQ_MSIX(oct->conf);
111 oct->msix_entries = kcalloc(num_msix,
112 sizeof(struct msix_entry), GFP_KERNEL);
113 if (!oct->msix_entries)
114 goto msix_alloc_err;
115
116 for (i = 0; i < num_msix; i++)
117 oct->msix_entries[i].entry = i;
118
119 msix_allocated = pci_enable_msix_range(oct->pdev, oct->msix_entries,
120 num_msix, num_msix);
121 if (msix_allocated != num_msix) {
122 dev_err(&oct->pdev->dev,
123 "Failed to enable %d msix irqs; got only %d\n",
124 num_msix, msix_allocated);
125 goto enable_msix_err;
126 }
127 oct->num_irqs = msix_allocated;
128 dev_info(&oct->pdev->dev, "MSI-X enabled successfully\n");
129
130 return 0;
131
132 enable_msix_err:
133 if (msix_allocated > 0)
134 pci_disable_msix(oct->pdev);
135 kfree(oct->msix_entries);
136 oct->msix_entries = NULL;
137 msix_alloc_err:
138 return -1;
139 }
140
141 /**
142 * octep_disable_msix() - disable MSI-x interrupts.
143 *
144 * @oct: Octeon device private data structure.
145 *
146 * Disable MSI-x on the Octeon device.
147 */
octep_disable_msix(struct octep_device * oct)148 static void octep_disable_msix(struct octep_device *oct)
149 {
150 pci_disable_msix(oct->pdev);
151 kfree(oct->msix_entries);
152 oct->msix_entries = NULL;
153 dev_info(&oct->pdev->dev, "Disabled MSI-X\n");
154 }
155
156 /**
157 * octep_non_ioq_intr_handler() - common handler for all generic interrupts.
158 *
159 * @irq: Interrupt number.
160 * @data: interrupt data.
161 *
162 * this is common handler for all non-queue (generic) interrupts.
163 */
octep_non_ioq_intr_handler(int irq,void * data)164 static irqreturn_t octep_non_ioq_intr_handler(int irq, void *data)
165 {
166 struct octep_device *oct = data;
167
168 return oct->hw_ops.non_ioq_intr_handler(oct);
169 }
170
171 /**
172 * octep_ioq_intr_handler() - handler for all Tx/Rx queue interrupts.
173 *
174 * @irq: Interrupt number.
175 * @data: interrupt data contains pointers to Tx/Rx queue private data
176 * and correspong NAPI context.
177 *
178 * this is common handler for all non-queue (generic) interrupts.
179 */
octep_ioq_intr_handler(int irq,void * data)180 static irqreturn_t octep_ioq_intr_handler(int irq, void *data)
181 {
182 struct octep_ioq_vector *ioq_vector = data;
183 struct octep_device *oct = ioq_vector->octep_dev;
184
185 return oct->hw_ops.ioq_intr_handler(ioq_vector);
186 }
187
188 /**
189 * octep_request_irqs() - Register interrupt handlers.
190 *
191 * @oct: Octeon device private data structure.
192 *
193 * Register handlers for all queue and non-queue interrupts.
194 *
195 * Return: 0, on successful registration of all interrupt handlers.
196 * -1, on any error.
197 */
octep_request_irqs(struct octep_device * oct)198 static int octep_request_irqs(struct octep_device *oct)
199 {
200 struct net_device *netdev = oct->netdev;
201 struct octep_ioq_vector *ioq_vector;
202 struct msix_entry *msix_entry;
203 char **non_ioq_msix_names;
204 int num_non_ioq_msix;
205 int ret, i, j;
206
207 num_non_ioq_msix = CFG_GET_NON_IOQ_MSIX(oct->conf);
208 non_ioq_msix_names = CFG_GET_NON_IOQ_MSIX_NAMES(oct->conf);
209
210 oct->non_ioq_irq_names = kcalloc(num_non_ioq_msix,
211 OCTEP_MSIX_NAME_SIZE, GFP_KERNEL);
212 if (!oct->non_ioq_irq_names)
213 goto alloc_err;
214
215 /* First few MSI-X interrupts are non-queue interrupts */
216 for (i = 0; i < num_non_ioq_msix; i++) {
217 char *irq_name;
218
219 irq_name = &oct->non_ioq_irq_names[i * OCTEP_MSIX_NAME_SIZE];
220 msix_entry = &oct->msix_entries[i];
221
222 snprintf(irq_name, OCTEP_MSIX_NAME_SIZE,
223 "%s-%s", netdev->name, non_ioq_msix_names[i]);
224 ret = request_irq(msix_entry->vector,
225 octep_non_ioq_intr_handler, 0,
226 irq_name, oct);
227 if (ret) {
228 netdev_err(netdev,
229 "request_irq failed for %s; err=%d",
230 irq_name, ret);
231 goto non_ioq_irq_err;
232 }
233 }
234
235 /* Request IRQs for Tx/Rx queues */
236 for (j = 0; j < oct->num_oqs; j++) {
237 ioq_vector = oct->ioq_vector[j];
238 msix_entry = &oct->msix_entries[j + num_non_ioq_msix];
239
240 snprintf(ioq_vector->name, sizeof(ioq_vector->name),
241 "%s-q%d", netdev->name, j);
242 ret = request_irq(msix_entry->vector,
243 octep_ioq_intr_handler, 0,
244 ioq_vector->name, ioq_vector);
245 if (ret) {
246 netdev_err(netdev,
247 "request_irq failed for Q-%d; err=%d",
248 j, ret);
249 goto ioq_irq_err;
250 }
251
252 cpumask_set_cpu(j % num_online_cpus(),
253 &ioq_vector->affinity_mask);
254 irq_set_affinity_hint(msix_entry->vector,
255 &ioq_vector->affinity_mask);
256 }
257
258 return 0;
259 ioq_irq_err:
260 while (j) {
261 --j;
262 ioq_vector = oct->ioq_vector[j];
263 msix_entry = &oct->msix_entries[j + num_non_ioq_msix];
264
265 irq_set_affinity_hint(msix_entry->vector, NULL);
266 free_irq(msix_entry->vector, ioq_vector);
267 }
268 non_ioq_irq_err:
269 while (i) {
270 --i;
271 free_irq(oct->msix_entries[i].vector, oct);
272 }
273 kfree(oct->non_ioq_irq_names);
274 oct->non_ioq_irq_names = NULL;
275 alloc_err:
276 return -1;
277 }
278
279 /**
280 * octep_free_irqs() - free all registered interrupts.
281 *
282 * @oct: Octeon device private data structure.
283 *
284 * Free all queue and non-queue interrupts of the Octeon device.
285 */
octep_free_irqs(struct octep_device * oct)286 static void octep_free_irqs(struct octep_device *oct)
287 {
288 int i;
289
290 /* First few MSI-X interrupts are non queue interrupts; free them */
291 for (i = 0; i < CFG_GET_NON_IOQ_MSIX(oct->conf); i++)
292 free_irq(oct->msix_entries[i].vector, oct);
293 kfree(oct->non_ioq_irq_names);
294
295 /* Free IRQs for Input/Output (Tx/Rx) queues */
296 for (i = CFG_GET_NON_IOQ_MSIX(oct->conf); i < oct->num_irqs; i++) {
297 irq_set_affinity_hint(oct->msix_entries[i].vector, NULL);
298 free_irq(oct->msix_entries[i].vector,
299 oct->ioq_vector[i - CFG_GET_NON_IOQ_MSIX(oct->conf)]);
300 }
301 netdev_info(oct->netdev, "IRQs freed\n");
302 }
303
304 /**
305 * octep_setup_irqs() - setup interrupts for the Octeon device.
306 *
307 * @oct: Octeon device private data structure.
308 *
309 * Allocate data structures to hold per interrupt information, allocate/enable
310 * MSI-x interrupt and register interrupt handlers.
311 *
312 * Return: 0, on successful allocation and registration of all interrupts.
313 * -1, on any error.
314 */
octep_setup_irqs(struct octep_device * oct)315 static int octep_setup_irqs(struct octep_device *oct)
316 {
317 if (octep_alloc_ioq_vectors(oct))
318 goto ioq_vector_err;
319
320 if (octep_enable_msix_range(oct))
321 goto enable_msix_err;
322
323 if (octep_request_irqs(oct))
324 goto request_irq_err;
325
326 return 0;
327
328 request_irq_err:
329 octep_disable_msix(oct);
330 enable_msix_err:
331 octep_free_ioq_vectors(oct);
332 ioq_vector_err:
333 return -1;
334 }
335
336 /**
337 * octep_clean_irqs() - free all interrupts and its resources.
338 *
339 * @oct: Octeon device private data structure.
340 */
octep_clean_irqs(struct octep_device * oct)341 static void octep_clean_irqs(struct octep_device *oct)
342 {
343 octep_free_irqs(oct);
344 octep_disable_msix(oct);
345 octep_free_ioq_vectors(oct);
346 }
347
348 /**
349 * octep_enable_ioq_irq() - Enable MSI-x interrupt of a Tx/Rx queue.
350 *
351 * @iq: Octeon Tx queue data structure.
352 * @oq: Octeon Rx queue data structure.
353 */
octep_enable_ioq_irq(struct octep_iq * iq,struct octep_oq * oq)354 static void octep_enable_ioq_irq(struct octep_iq *iq, struct octep_oq *oq)
355 {
356 u32 pkts_pend = oq->pkts_pending;
357
358 netdev_dbg(iq->netdev, "enabling intr for Q-%u\n", iq->q_no);
359 if (iq->pkts_processed) {
360 writel(iq->pkts_processed, iq->inst_cnt_reg);
361 iq->pkt_in_done -= iq->pkts_processed;
362 iq->pkts_processed = 0;
363 }
364 if (oq->last_pkt_count - pkts_pend) {
365 writel(oq->last_pkt_count - pkts_pend, oq->pkts_sent_reg);
366 oq->last_pkt_count = pkts_pend;
367 }
368
369 /* Flush the previous wrties before writing to RESEND bit */
370 wmb();
371 writeq(1UL << OCTEP_OQ_INTR_RESEND_BIT, oq->pkts_sent_reg);
372 writeq(1UL << OCTEP_IQ_INTR_RESEND_BIT, iq->inst_cnt_reg);
373 }
374
375 /**
376 * octep_napi_poll() - NAPI poll function for Tx/Rx.
377 *
378 * @napi: pointer to napi context.
379 * @budget: max number of packets to be processed in single invocation.
380 */
octep_napi_poll(struct napi_struct * napi,int budget)381 static int octep_napi_poll(struct napi_struct *napi, int budget)
382 {
383 struct octep_ioq_vector *ioq_vector =
384 container_of(napi, struct octep_ioq_vector, napi);
385 u32 tx_pending, rx_done;
386
387 tx_pending = octep_iq_process_completions(ioq_vector->iq, budget);
388 rx_done = octep_oq_process_rx(ioq_vector->oq, budget);
389
390 /* need more polling if tx completion processing is still pending or
391 * processed at least 'budget' number of rx packets.
392 */
393 if (tx_pending || rx_done >= budget)
394 return budget;
395
396 napi_complete(napi);
397 octep_enable_ioq_irq(ioq_vector->iq, ioq_vector->oq);
398 return rx_done;
399 }
400
401 /**
402 * octep_napi_add() - Add NAPI poll for all Tx/Rx queues.
403 *
404 * @oct: Octeon device private data structure.
405 */
octep_napi_add(struct octep_device * oct)406 static void octep_napi_add(struct octep_device *oct)
407 {
408 int i;
409
410 for (i = 0; i < oct->num_oqs; i++) {
411 netdev_dbg(oct->netdev, "Adding NAPI on Q-%d\n", i);
412 netif_napi_add(oct->netdev, &oct->ioq_vector[i]->napi,
413 octep_napi_poll);
414 oct->oq[i]->napi = &oct->ioq_vector[i]->napi;
415 }
416 }
417
418 /**
419 * octep_napi_delete() - delete NAPI poll callback for all Tx/Rx queues.
420 *
421 * @oct: Octeon device private data structure.
422 */
octep_napi_delete(struct octep_device * oct)423 static void octep_napi_delete(struct octep_device *oct)
424 {
425 int i;
426
427 for (i = 0; i < oct->num_oqs; i++) {
428 netdev_dbg(oct->netdev, "Deleting NAPI on Q-%d\n", i);
429 netif_napi_del(&oct->ioq_vector[i]->napi);
430 oct->oq[i]->napi = NULL;
431 }
432 }
433
434 /**
435 * octep_napi_enable() - enable NAPI for all Tx/Rx queues.
436 *
437 * @oct: Octeon device private data structure.
438 */
octep_napi_enable(struct octep_device * oct)439 static void octep_napi_enable(struct octep_device *oct)
440 {
441 int i;
442
443 for (i = 0; i < oct->num_oqs; i++) {
444 netdev_dbg(oct->netdev, "Enabling NAPI on Q-%d\n", i);
445 napi_enable(&oct->ioq_vector[i]->napi);
446 }
447 }
448
449 /**
450 * octep_napi_disable() - disable NAPI for all Tx/Rx queues.
451 *
452 * @oct: Octeon device private data structure.
453 */
octep_napi_disable(struct octep_device * oct)454 static void octep_napi_disable(struct octep_device *oct)
455 {
456 int i;
457
458 for (i = 0; i < oct->num_oqs; i++) {
459 netdev_dbg(oct->netdev, "Disabling NAPI on Q-%d\n", i);
460 napi_disable(&oct->ioq_vector[i]->napi);
461 }
462 }
463
octep_link_up(struct net_device * netdev)464 static void octep_link_up(struct net_device *netdev)
465 {
466 netif_carrier_on(netdev);
467 netif_tx_start_all_queues(netdev);
468 }
469
470 /**
471 * octep_open() - start the octeon network device.
472 *
473 * @netdev: pointer to kernel network device.
474 *
475 * setup Tx/Rx queues, interrupts and enable hardware operation of Tx/Rx queues
476 * and interrupts..
477 *
478 * Return: 0, on successfully setting up device and bring it up.
479 * -1, on any error.
480 */
octep_open(struct net_device * netdev)481 static int octep_open(struct net_device *netdev)
482 {
483 struct octep_device *oct = netdev_priv(netdev);
484 int err, ret;
485
486 netdev_info(netdev, "Starting netdev ...\n");
487 netif_carrier_off(netdev);
488
489 oct->hw_ops.reset_io_queues(oct);
490
491 if (octep_setup_iqs(oct))
492 goto setup_iq_err;
493 if (octep_setup_oqs(oct))
494 goto setup_oq_err;
495 if (octep_setup_irqs(oct))
496 goto setup_irq_err;
497
498 err = netif_set_real_num_tx_queues(netdev, oct->num_oqs);
499 if (err)
500 goto set_queues_err;
501 err = netif_set_real_num_rx_queues(netdev, oct->num_iqs);
502 if (err)
503 goto set_queues_err;
504
505 octep_napi_add(oct);
506 octep_napi_enable(oct);
507
508 oct->link_info.admin_up = 1;
509 octep_set_rx_state(oct, true);
510
511 ret = octep_get_link_status(oct);
512 if (!ret)
513 octep_set_link_status(oct, true);
514
515 /* Enable the input and output queues for this Octeon device */
516 oct->hw_ops.enable_io_queues(oct);
517
518 /* Enable Octeon device interrupts */
519 oct->hw_ops.enable_interrupts(oct);
520
521 octep_oq_dbell_init(oct);
522
523 ret = octep_get_link_status(oct);
524 if (ret > 0)
525 octep_link_up(netdev);
526
527 return 0;
528
529 set_queues_err:
530 octep_clean_irqs(oct);
531 setup_irq_err:
532 octep_free_oqs(oct);
533 setup_oq_err:
534 octep_free_iqs(oct);
535 setup_iq_err:
536 return -1;
537 }
538
539 /**
540 * octep_stop() - stop the octeon network device.
541 *
542 * @netdev: pointer to kernel network device.
543 *
544 * stop the device Tx/Rx operations, bring down the link and
545 * free up all resources allocated for Tx/Rx queues and interrupts.
546 */
octep_stop(struct net_device * netdev)547 static int octep_stop(struct net_device *netdev)
548 {
549 struct octep_device *oct = netdev_priv(netdev);
550
551 netdev_info(netdev, "Stopping the device ...\n");
552
553 /* Stop Tx from stack */
554 netif_tx_stop_all_queues(netdev);
555 netif_carrier_off(netdev);
556 netif_tx_disable(netdev);
557
558 octep_set_link_status(oct, false);
559 octep_set_rx_state(oct, false);
560
561 oct->link_info.admin_up = 0;
562 oct->link_info.oper_up = 0;
563
564 oct->hw_ops.disable_interrupts(oct);
565 octep_napi_disable(oct);
566 octep_napi_delete(oct);
567
568 octep_clean_irqs(oct);
569 octep_clean_iqs(oct);
570
571 oct->hw_ops.disable_io_queues(oct);
572 oct->hw_ops.reset_io_queues(oct);
573 octep_free_oqs(oct);
574 octep_free_iqs(oct);
575 netdev_info(netdev, "Device stopped !!\n");
576 return 0;
577 }
578
579 /**
580 * octep_iq_full_check() - check if a Tx queue is full.
581 *
582 * @iq: Octeon Tx queue data structure.
583 *
584 * Return: 0, if the Tx queue is not full.
585 * 1, if the Tx queue is full.
586 */
octep_iq_full_check(struct octep_iq * iq)587 static inline int octep_iq_full_check(struct octep_iq *iq)
588 {
589 if (likely((iq->max_count - atomic_read(&iq->instr_pending)) >=
590 OCTEP_WAKE_QUEUE_THRESHOLD))
591 return 0;
592
593 /* Stop the queue if unable to send */
594 netif_stop_subqueue(iq->netdev, iq->q_no);
595
596 /* check again and restart the queue, in case NAPI has just freed
597 * enough Tx ring entries.
598 */
599 if (unlikely((iq->max_count - atomic_read(&iq->instr_pending)) >=
600 OCTEP_WAKE_QUEUE_THRESHOLD)) {
601 netif_start_subqueue(iq->netdev, iq->q_no);
602 iq->stats.restart_cnt++;
603 return 0;
604 }
605
606 return 1;
607 }
608
609 /**
610 * octep_start_xmit() - Enqueue packet to Octoen hardware Tx Queue.
611 *
612 * @skb: packet skbuff pointer.
613 * @netdev: kernel network device.
614 *
615 * Return: NETDEV_TX_BUSY, if Tx Queue is full.
616 * NETDEV_TX_OK, if successfully enqueued to hardware Tx queue.
617 */
octep_start_xmit(struct sk_buff * skb,struct net_device * netdev)618 static netdev_tx_t octep_start_xmit(struct sk_buff *skb,
619 struct net_device *netdev)
620 {
621 struct octep_device *oct = netdev_priv(netdev);
622 struct octep_tx_sglist_desc *sglist;
623 struct octep_tx_buffer *tx_buffer;
624 struct octep_tx_desc_hw *hw_desc;
625 struct skb_shared_info *shinfo;
626 struct octep_instr_hdr *ih;
627 struct octep_iq *iq;
628 skb_frag_t *frag;
629 u16 nr_frags, si;
630 u16 q_no, wi;
631
632 q_no = skb_get_queue_mapping(skb);
633 if (q_no >= oct->num_iqs) {
634 netdev_err(netdev, "Invalid Tx skb->queue_mapping=%d\n", q_no);
635 q_no = q_no % oct->num_iqs;
636 }
637
638 iq = oct->iq[q_no];
639 if (octep_iq_full_check(iq)) {
640 iq->stats.tx_busy++;
641 return NETDEV_TX_BUSY;
642 }
643
644 shinfo = skb_shinfo(skb);
645 nr_frags = shinfo->nr_frags;
646
647 wi = iq->host_write_index;
648 hw_desc = &iq->desc_ring[wi];
649 hw_desc->ih64 = 0;
650
651 tx_buffer = iq->buff_info + wi;
652 tx_buffer->skb = skb;
653
654 ih = &hw_desc->ih;
655 ih->tlen = skb->len;
656 ih->pkind = oct->pkind;
657
658 if (!nr_frags) {
659 tx_buffer->gather = 0;
660 tx_buffer->dma = dma_map_single(iq->dev, skb->data,
661 skb->len, DMA_TO_DEVICE);
662 if (dma_mapping_error(iq->dev, tx_buffer->dma))
663 goto dma_map_err;
664 hw_desc->dptr = tx_buffer->dma;
665 } else {
666 /* Scatter/Gather */
667 dma_addr_t dma;
668 u16 len;
669
670 sglist = tx_buffer->sglist;
671
672 ih->gsz = nr_frags + 1;
673 ih->gather = 1;
674 tx_buffer->gather = 1;
675
676 len = skb_headlen(skb);
677 dma = dma_map_single(iq->dev, skb->data, len, DMA_TO_DEVICE);
678 if (dma_mapping_error(iq->dev, dma))
679 goto dma_map_err;
680
681 dma_sync_single_for_cpu(iq->dev, tx_buffer->sglist_dma,
682 OCTEP_SGLIST_SIZE_PER_PKT,
683 DMA_TO_DEVICE);
684 memset(sglist, 0, OCTEP_SGLIST_SIZE_PER_PKT);
685 sglist[0].len[3] = len;
686 sglist[0].dma_ptr[0] = dma;
687
688 si = 1; /* entry 0 is main skb, mapped above */
689 frag = &shinfo->frags[0];
690 while (nr_frags--) {
691 len = skb_frag_size(frag);
692 dma = skb_frag_dma_map(iq->dev, frag, 0,
693 len, DMA_TO_DEVICE);
694 if (dma_mapping_error(iq->dev, dma))
695 goto dma_map_sg_err;
696
697 sglist[si >> 2].len[3 - (si & 3)] = len;
698 sglist[si >> 2].dma_ptr[si & 3] = dma;
699
700 frag++;
701 si++;
702 }
703 dma_sync_single_for_device(iq->dev, tx_buffer->sglist_dma,
704 OCTEP_SGLIST_SIZE_PER_PKT,
705 DMA_TO_DEVICE);
706
707 hw_desc->dptr = tx_buffer->sglist_dma;
708 }
709
710 /* Flush the hw descriptor before writing to doorbell */
711 wmb();
712
713 /* Ring Doorbell to notify the NIC there is a new packet */
714 writel(1, iq->doorbell_reg);
715 atomic_inc(&iq->instr_pending);
716 wi++;
717 if (wi == iq->max_count)
718 wi = 0;
719 iq->host_write_index = wi;
720
721 netdev_tx_sent_queue(iq->netdev_q, skb->len);
722 iq->stats.instr_posted++;
723 skb_tx_timestamp(skb);
724 return NETDEV_TX_OK;
725
726 dma_map_sg_err:
727 if (si > 0) {
728 dma_unmap_single(iq->dev, sglist[0].dma_ptr[0],
729 sglist[0].len[0], DMA_TO_DEVICE);
730 sglist[0].len[0] = 0;
731 }
732 while (si > 1) {
733 dma_unmap_page(iq->dev, sglist[si >> 2].dma_ptr[si & 3],
734 sglist[si >> 2].len[si & 3], DMA_TO_DEVICE);
735 sglist[si >> 2].len[si & 3] = 0;
736 si--;
737 }
738 tx_buffer->gather = 0;
739 dma_map_err:
740 dev_kfree_skb_any(skb);
741 return NETDEV_TX_OK;
742 }
743
744 /**
745 * octep_get_stats64() - Get Octeon network device statistics.
746 *
747 * @netdev: kernel network device.
748 * @stats: pointer to stats structure to be filled in.
749 */
octep_get_stats64(struct net_device * netdev,struct rtnl_link_stats64 * stats)750 static void octep_get_stats64(struct net_device *netdev,
751 struct rtnl_link_stats64 *stats)
752 {
753 u64 tx_packets, tx_bytes, rx_packets, rx_bytes;
754 struct octep_device *oct = netdev_priv(netdev);
755 int q;
756
757 octep_get_if_stats(oct);
758 tx_packets = 0;
759 tx_bytes = 0;
760 rx_packets = 0;
761 rx_bytes = 0;
762 for (q = 0; q < oct->num_oqs; q++) {
763 struct octep_iq *iq = oct->iq[q];
764 struct octep_oq *oq = oct->oq[q];
765
766 tx_packets += iq->stats.instr_completed;
767 tx_bytes += iq->stats.bytes_sent;
768 rx_packets += oq->stats.packets;
769 rx_bytes += oq->stats.bytes;
770 }
771 stats->tx_packets = tx_packets;
772 stats->tx_bytes = tx_bytes;
773 stats->rx_packets = rx_packets;
774 stats->rx_bytes = rx_bytes;
775 stats->multicast = oct->iface_rx_stats.mcast_pkts;
776 stats->rx_errors = oct->iface_rx_stats.err_pkts;
777 stats->collisions = oct->iface_tx_stats.xscol;
778 stats->tx_fifo_errors = oct->iface_tx_stats.undflw;
779 }
780
781 /**
782 * octep_tx_timeout_task - work queue task to Handle Tx queue timeout.
783 *
784 * @work: pointer to Tx queue timeout work_struct
785 *
786 * Stop and start the device so that it frees up all queue resources
787 * and restarts the queues, that potentially clears a Tx queue timeout
788 * condition.
789 **/
octep_tx_timeout_task(struct work_struct * work)790 static void octep_tx_timeout_task(struct work_struct *work)
791 {
792 struct octep_device *oct = container_of(work, struct octep_device,
793 tx_timeout_task);
794 struct net_device *netdev = oct->netdev;
795
796 rtnl_lock();
797 if (netif_running(netdev)) {
798 octep_stop(netdev);
799 octep_open(netdev);
800 }
801 rtnl_unlock();
802 }
803
804 /**
805 * octep_tx_timeout() - Handle Tx Queue timeout.
806 *
807 * @netdev: pointer to kernel network device.
808 * @txqueue: Timed out Tx queue number.
809 *
810 * Schedule a work to handle Tx queue timeout.
811 */
octep_tx_timeout(struct net_device * netdev,unsigned int txqueue)812 static void octep_tx_timeout(struct net_device *netdev, unsigned int txqueue)
813 {
814 struct octep_device *oct = netdev_priv(netdev);
815
816 queue_work(octep_wq, &oct->tx_timeout_task);
817 }
818
octep_set_mac(struct net_device * netdev,void * p)819 static int octep_set_mac(struct net_device *netdev, void *p)
820 {
821 struct octep_device *oct = netdev_priv(netdev);
822 struct sockaddr *addr = (struct sockaddr *)p;
823 int err;
824
825 if (!is_valid_ether_addr(addr->sa_data))
826 return -EADDRNOTAVAIL;
827
828 err = octep_set_mac_addr(oct, addr->sa_data);
829 if (err)
830 return err;
831
832 memcpy(oct->mac_addr, addr->sa_data, ETH_ALEN);
833 eth_hw_addr_set(netdev, addr->sa_data);
834
835 return 0;
836 }
837
octep_change_mtu(struct net_device * netdev,int new_mtu)838 static int octep_change_mtu(struct net_device *netdev, int new_mtu)
839 {
840 struct octep_device *oct = netdev_priv(netdev);
841 struct octep_iface_link_info *link_info;
842 int err = 0;
843
844 link_info = &oct->link_info;
845 if (link_info->mtu == new_mtu)
846 return 0;
847
848 err = octep_set_mtu(oct, new_mtu);
849 if (!err) {
850 oct->link_info.mtu = new_mtu;
851 netdev->mtu = new_mtu;
852 }
853
854 return err;
855 }
856
857 static const struct net_device_ops octep_netdev_ops = {
858 .ndo_open = octep_open,
859 .ndo_stop = octep_stop,
860 .ndo_start_xmit = octep_start_xmit,
861 .ndo_get_stats64 = octep_get_stats64,
862 .ndo_tx_timeout = octep_tx_timeout,
863 .ndo_set_mac_address = octep_set_mac,
864 .ndo_change_mtu = octep_change_mtu,
865 };
866
867 /**
868 * octep_ctrl_mbox_task - work queue task to handle ctrl mbox messages.
869 *
870 * @work: pointer to ctrl mbox work_struct
871 *
872 * Poll ctrl mbox message queue and handle control messages from firmware.
873 **/
octep_ctrl_mbox_task(struct work_struct * work)874 static void octep_ctrl_mbox_task(struct work_struct *work)
875 {
876 struct octep_device *oct = container_of(work, struct octep_device,
877 ctrl_mbox_task);
878 struct net_device *netdev = oct->netdev;
879 struct octep_ctrl_net_f2h_req req = {};
880 struct octep_ctrl_mbox_msg msg;
881 int ret = 0;
882
883 msg.msg = &req;
884 while (true) {
885 ret = octep_ctrl_mbox_recv(&oct->ctrl_mbox, &msg);
886 if (ret)
887 break;
888
889 switch (req.hdr.cmd) {
890 case OCTEP_CTRL_NET_F2H_CMD_LINK_STATUS:
891 if (netif_running(netdev)) {
892 if (req.link.state) {
893 dev_info(&oct->pdev->dev, "netif_carrier_on\n");
894 netif_carrier_on(netdev);
895 } else {
896 dev_info(&oct->pdev->dev, "netif_carrier_off\n");
897 netif_carrier_off(netdev);
898 }
899 }
900 break;
901 default:
902 pr_info("Unknown mbox req : %u\n", req.hdr.cmd);
903 break;
904 }
905 }
906 }
907
908 /**
909 * octep_device_setup() - Setup Octeon Device.
910 *
911 * @oct: Octeon device private data structure.
912 *
913 * Setup Octeon device hardware operations, configuration, etc ...
914 */
octep_device_setup(struct octep_device * oct)915 int octep_device_setup(struct octep_device *oct)
916 {
917 struct octep_ctrl_mbox *ctrl_mbox;
918 struct pci_dev *pdev = oct->pdev;
919 int i, ret;
920
921 /* allocate memory for oct->conf */
922 oct->conf = kzalloc(sizeof(*oct->conf), GFP_KERNEL);
923 if (!oct->conf)
924 return -ENOMEM;
925
926 /* Map BAR regions */
927 for (i = 0; i < OCTEP_MMIO_REGIONS; i++) {
928 oct->mmio[i].hw_addr =
929 ioremap(pci_resource_start(oct->pdev, i * 2),
930 pci_resource_len(oct->pdev, i * 2));
931 oct->mmio[i].mapped = 1;
932 }
933
934 oct->chip_id = pdev->device;
935 oct->rev_id = pdev->revision;
936 dev_info(&pdev->dev, "chip_id = 0x%x\n", pdev->device);
937
938 switch (oct->chip_id) {
939 case OCTEP_PCI_DEVICE_ID_CN93_PF:
940 dev_info(&pdev->dev,
941 "Setting up OCTEON CN93XX PF PASS%d.%d\n",
942 OCTEP_MAJOR_REV(oct), OCTEP_MINOR_REV(oct));
943 octep_device_setup_cn93_pf(oct);
944 break;
945 default:
946 dev_err(&pdev->dev,
947 "%s: unsupported device\n", __func__);
948 goto unsupported_dev;
949 }
950
951 oct->pkind = CFG_GET_IQ_PKIND(oct->conf);
952
953 /* Initialize control mbox */
954 ctrl_mbox = &oct->ctrl_mbox;
955 ctrl_mbox->barmem = CFG_GET_CTRL_MBOX_MEM_ADDR(oct->conf);
956 ret = octep_ctrl_mbox_init(ctrl_mbox);
957 if (ret) {
958 dev_err(&pdev->dev, "Failed to initialize control mbox\n");
959 goto unsupported_dev;
960 }
961 oct->ctrl_mbox_ifstats_offset = OCTEP_CTRL_MBOX_SZ(ctrl_mbox->h2fq.elem_sz,
962 ctrl_mbox->h2fq.elem_cnt,
963 ctrl_mbox->f2hq.elem_sz,
964 ctrl_mbox->f2hq.elem_cnt);
965
966 return 0;
967
968 unsupported_dev:
969 for (i = 0; i < OCTEP_MMIO_REGIONS; i++)
970 iounmap(oct->mmio[i].hw_addr);
971
972 kfree(oct->conf);
973 return -1;
974 }
975
976 /**
977 * octep_device_cleanup() - Cleanup Octeon Device.
978 *
979 * @oct: Octeon device private data structure.
980 *
981 * Cleanup Octeon device allocated resources.
982 */
octep_device_cleanup(struct octep_device * oct)983 static void octep_device_cleanup(struct octep_device *oct)
984 {
985 int i;
986
987 dev_info(&oct->pdev->dev, "Cleaning up Octeon Device ...\n");
988
989 for (i = 0; i < OCTEP_MAX_VF; i++) {
990 vfree(oct->mbox[i]);
991 oct->mbox[i] = NULL;
992 }
993
994 octep_ctrl_mbox_uninit(&oct->ctrl_mbox);
995
996 oct->hw_ops.soft_reset(oct);
997 for (i = 0; i < OCTEP_MMIO_REGIONS; i++) {
998 if (oct->mmio[i].mapped)
999 iounmap(oct->mmio[i].hw_addr);
1000 }
1001
1002 kfree(oct->conf);
1003 oct->conf = NULL;
1004 }
1005
1006 /**
1007 * octep_probe() - Octeon PCI device probe handler.
1008 *
1009 * @pdev: PCI device structure.
1010 * @ent: entry in Octeon PCI device ID table.
1011 *
1012 * Initializes and enables the Octeon PCI device for network operations.
1013 * Initializes Octeon private data structure and registers a network device.
1014 */
octep_probe(struct pci_dev * pdev,const struct pci_device_id * ent)1015 static int octep_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
1016 {
1017 struct octep_device *octep_dev = NULL;
1018 struct net_device *netdev;
1019 int err;
1020
1021 err = pci_enable_device(pdev);
1022 if (err) {
1023 dev_err(&pdev->dev, "Failed to enable PCI device\n");
1024 return err;
1025 }
1026
1027 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
1028 if (err) {
1029 dev_err(&pdev->dev, "Failed to set DMA mask !!\n");
1030 goto err_dma_mask;
1031 }
1032
1033 err = pci_request_mem_regions(pdev, OCTEP_DRV_NAME);
1034 if (err) {
1035 dev_err(&pdev->dev, "Failed to map PCI memory regions\n");
1036 goto err_pci_regions;
1037 }
1038
1039 pci_enable_pcie_error_reporting(pdev);
1040 pci_set_master(pdev);
1041
1042 netdev = alloc_etherdev_mq(sizeof(struct octep_device),
1043 OCTEP_MAX_QUEUES);
1044 if (!netdev) {
1045 dev_err(&pdev->dev, "Failed to allocate netdev\n");
1046 err = -ENOMEM;
1047 goto err_alloc_netdev;
1048 }
1049 SET_NETDEV_DEV(netdev, &pdev->dev);
1050
1051 octep_dev = netdev_priv(netdev);
1052 octep_dev->netdev = netdev;
1053 octep_dev->pdev = pdev;
1054 octep_dev->dev = &pdev->dev;
1055 pci_set_drvdata(pdev, octep_dev);
1056
1057 err = octep_device_setup(octep_dev);
1058 if (err) {
1059 dev_err(&pdev->dev, "Device setup failed\n");
1060 goto err_octep_config;
1061 }
1062 INIT_WORK(&octep_dev->tx_timeout_task, octep_tx_timeout_task);
1063 INIT_WORK(&octep_dev->ctrl_mbox_task, octep_ctrl_mbox_task);
1064
1065 netdev->netdev_ops = &octep_netdev_ops;
1066 octep_set_ethtool_ops(netdev);
1067 netif_carrier_off(netdev);
1068
1069 netdev->hw_features = NETIF_F_SG;
1070 netdev->features |= netdev->hw_features;
1071 netdev->min_mtu = OCTEP_MIN_MTU;
1072 netdev->max_mtu = OCTEP_MAX_MTU;
1073 netdev->mtu = OCTEP_DEFAULT_MTU;
1074
1075 err = octep_get_mac_addr(octep_dev, octep_dev->mac_addr);
1076 if (err) {
1077 dev_err(&pdev->dev, "Failed to get mac address\n");
1078 goto register_dev_err;
1079 }
1080 eth_hw_addr_set(netdev, octep_dev->mac_addr);
1081
1082 err = register_netdev(netdev);
1083 if (err) {
1084 dev_err(&pdev->dev, "Failed to register netdev\n");
1085 goto register_dev_err;
1086 }
1087 dev_info(&pdev->dev, "Device probe successful\n");
1088 return 0;
1089
1090 register_dev_err:
1091 octep_device_cleanup(octep_dev);
1092 err_octep_config:
1093 free_netdev(netdev);
1094 err_alloc_netdev:
1095 pci_disable_pcie_error_reporting(pdev);
1096 pci_release_mem_regions(pdev);
1097 err_pci_regions:
1098 err_dma_mask:
1099 pci_disable_device(pdev);
1100 return err;
1101 }
1102
1103 /**
1104 * octep_remove() - Remove Octeon PCI device from driver control.
1105 *
1106 * @pdev: PCI device structure of the Octeon device.
1107 *
1108 * Cleanup all resources allocated for the Octeon device.
1109 * Unregister from network device and disable the PCI device.
1110 */
octep_remove(struct pci_dev * pdev)1111 static void octep_remove(struct pci_dev *pdev)
1112 {
1113 struct octep_device *oct = pci_get_drvdata(pdev);
1114 struct net_device *netdev;
1115
1116 if (!oct)
1117 return;
1118
1119 cancel_work_sync(&oct->tx_timeout_task);
1120 cancel_work_sync(&oct->ctrl_mbox_task);
1121 netdev = oct->netdev;
1122 if (netdev->reg_state == NETREG_REGISTERED)
1123 unregister_netdev(netdev);
1124
1125 octep_device_cleanup(oct);
1126 pci_release_mem_regions(pdev);
1127 free_netdev(netdev);
1128 pci_disable_pcie_error_reporting(pdev);
1129 pci_disable_device(pdev);
1130 }
1131
1132 static struct pci_driver octep_driver = {
1133 .name = OCTEP_DRV_NAME,
1134 .id_table = octep_pci_id_tbl,
1135 .probe = octep_probe,
1136 .remove = octep_remove,
1137 };
1138
1139 /**
1140 * octep_init_module() - Module initialiation.
1141 *
1142 * create common resource for the driver and register PCI driver.
1143 */
octep_init_module(void)1144 static int __init octep_init_module(void)
1145 {
1146 int ret;
1147
1148 pr_info("%s: Loading %s ...\n", OCTEP_DRV_NAME, OCTEP_DRV_STRING);
1149
1150 /* work queue for all deferred tasks */
1151 octep_wq = create_singlethread_workqueue(OCTEP_DRV_NAME);
1152 if (!octep_wq) {
1153 pr_err("%s: Failed to create common workqueue\n",
1154 OCTEP_DRV_NAME);
1155 return -ENOMEM;
1156 }
1157
1158 ret = pci_register_driver(&octep_driver);
1159 if (ret < 0) {
1160 pr_err("%s: Failed to register PCI driver; err=%d\n",
1161 OCTEP_DRV_NAME, ret);
1162 destroy_workqueue(octep_wq);
1163 return ret;
1164 }
1165
1166 pr_info("%s: Loaded successfully !\n", OCTEP_DRV_NAME);
1167
1168 return ret;
1169 }
1170
1171 /**
1172 * octep_exit_module() - Module exit routine.
1173 *
1174 * unregister the driver with PCI subsystem and cleanup common resources.
1175 */
octep_exit_module(void)1176 static void __exit octep_exit_module(void)
1177 {
1178 pr_info("%s: Unloading ...\n", OCTEP_DRV_NAME);
1179
1180 pci_unregister_driver(&octep_driver);
1181 destroy_workqueue(octep_wq);
1182
1183 pr_info("%s: Unloading complete\n", OCTEP_DRV_NAME);
1184 }
1185
1186 module_init(octep_init_module);
1187 module_exit(octep_exit_module);
1188
