1 // SPDX-License-Identifier: GPL-2.0-only
2 #include <linux/aer.h>
3 #include <linux/delay.h>
4 #include <linux/firmware.h>
5 #include <linux/list.h>
6 #include <linux/module.h>
7 #include <linux/mutex.h>
8 #include <linux/pci.h>
9 #include <linux/pci_ids.h>
10
11 #include "nitrox_dev.h"
12 #include "nitrox_common.h"
13 #include "nitrox_csr.h"
14 #include "nitrox_hal.h"
15 #include "nitrox_isr.h"
16 #include "nitrox_debugfs.h"
17
18 #define CNN55XX_DEV_ID 0x12
19 #define UCODE_HLEN 48
20 #define DEFAULT_SE_GROUP 0
21 #define DEFAULT_AE_GROUP 0
22
23 #define DRIVER_VERSION "1.2"
24 #define CNN55XX_UCD_BLOCK_SIZE 32768
25 #define CNN55XX_MAX_UCODE_SIZE (CNN55XX_UCD_BLOCK_SIZE * 2)
26 #define FW_DIR "cavium/"
27 /* SE microcode */
28 #define SE_FW FW_DIR "cnn55xx_se.fw"
29 /* AE microcode */
30 #define AE_FW FW_DIR "cnn55xx_ae.fw"
31
32 static const char nitrox_driver_name[] = "CNN55XX";
33
34 static LIST_HEAD(ndevlist);
35 static DEFINE_MUTEX(devlist_lock);
36 static unsigned int num_devices;
37
38 /**
39 * nitrox_pci_tbl - PCI Device ID Table
40 */
41 static const struct pci_device_id nitrox_pci_tbl[] = {
42 {PCI_VDEVICE(CAVIUM, CNN55XX_DEV_ID), 0},
43 /* required last entry */
44 {0, }
45 };
46 MODULE_DEVICE_TABLE(pci, nitrox_pci_tbl);
47
48 static unsigned int qlen = DEFAULT_CMD_QLEN;
49 module_param(qlen, uint, 0644);
50 MODULE_PARM_DESC(qlen, "Command queue length - default 2048");
51
52 #ifdef CONFIG_PCI_IOV
53 int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs);
54 #else
nitrox_sriov_configure(struct pci_dev * pdev,int num_vfs)55 int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs)
56 {
57 return 0;
58 }
59 #endif
60
61 /**
62 * struct ucode - Firmware Header
63 * @id: microcode ID
64 * @version: firmware version
65 * @code_size: code section size
66 * @raz: alignment
67 * @code: code section
68 */
69 struct ucode {
70 u8 id;
71 char version[VERSION_LEN - 1];
72 __be32 code_size;
73 u8 raz[12];
74 u64 code[0];
75 };
76
77 /**
78 * write_to_ucd_unit - Write Firmware to NITROX UCD unit
79 */
write_to_ucd_unit(struct nitrox_device * ndev,u32 ucode_size,u64 * ucode_data,int block_num)80 static void write_to_ucd_unit(struct nitrox_device *ndev, u32 ucode_size,
81 u64 *ucode_data, int block_num)
82 {
83 u32 code_size;
84 u64 offset, data;
85 int i = 0;
86
87 /*
88 * UCD structure
89 *
90 * -------------
91 * | BLK 7 |
92 * -------------
93 * | BLK 6 |
94 * -------------
95 * | ... |
96 * -------------
97 * | BLK 0 |
98 * -------------
99 * Total of 8 blocks, each size 32KB
100 */
101
102 /* set the block number */
103 offset = UCD_UCODE_LOAD_BLOCK_NUM;
104 nitrox_write_csr(ndev, offset, block_num);
105
106 code_size = ucode_size;
107 code_size = roundup(code_size, 8);
108 while (code_size) {
109 data = ucode_data[i];
110 /* write 8 bytes at a time */
111 offset = UCD_UCODE_LOAD_IDX_DATAX(i);
112 nitrox_write_csr(ndev, offset, data);
113 code_size -= 8;
114 i++;
115 }
116
117 usleep_range(300, 400);
118 }
119
nitrox_load_fw(struct nitrox_device * ndev)120 static int nitrox_load_fw(struct nitrox_device *ndev)
121 {
122 const struct firmware *fw;
123 const char *fw_name;
124 struct ucode *ucode;
125 u64 *ucode_data;
126 u64 offset;
127 union ucd_core_eid_ucode_block_num core_2_eid_val;
128 union aqm_grp_execmsk_lo aqm_grp_execmask_lo;
129 union aqm_grp_execmsk_hi aqm_grp_execmask_hi;
130 u32 ucode_size;
131 int ret, i = 0;
132
133 fw_name = SE_FW;
134 dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name);
135
136 ret = request_firmware(&fw, fw_name, DEV(ndev));
137 if (ret < 0) {
138 dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name);
139 return ret;
140 }
141
142 ucode = (struct ucode *)fw->data;
143
144 ucode_size = be32_to_cpu(ucode->code_size) * 2;
145 if (!ucode_size || ucode_size > CNN55XX_MAX_UCODE_SIZE) {
146 dev_err(DEV(ndev), "Invalid ucode size: %u for firmware %s\n",
147 ucode_size, fw_name);
148 release_firmware(fw);
149 return -EINVAL;
150 }
151 ucode_data = ucode->code;
152
153 /* copy the firmware version */
154 memcpy(&ndev->hw.fw_name[0][0], ucode->version, (VERSION_LEN - 2));
155 ndev->hw.fw_name[0][VERSION_LEN - 1] = '\0';
156
157 /* Load SE Firmware on UCD Block 0 */
158 write_to_ucd_unit(ndev, ucode_size, ucode_data, 0);
159
160 release_firmware(fw);
161
162 /* put all SE cores in DEFAULT_SE_GROUP */
163 offset = POM_GRP_EXECMASKX(DEFAULT_SE_GROUP);
164 nitrox_write_csr(ndev, offset, (~0ULL));
165
166 /* write block number and firmware length
167 * bit:<2:0> block number
168 * bit:3 is set SE uses 32KB microcode
169 * bit:3 is clear SE uses 64KB microcode
170 */
171 core_2_eid_val.value = 0ULL;
172 core_2_eid_val.ucode_blk = 0;
173 if (ucode_size <= CNN55XX_UCD_BLOCK_SIZE)
174 core_2_eid_val.ucode_len = 1;
175 else
176 core_2_eid_val.ucode_len = 0;
177
178 for (i = 0; i < ndev->hw.se_cores; i++) {
179 offset = UCD_SE_EID_UCODE_BLOCK_NUMX(i);
180 nitrox_write_csr(ndev, offset, core_2_eid_val.value);
181 }
182
183
184 fw_name = AE_FW;
185 dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name);
186
187 ret = request_firmware(&fw, fw_name, DEV(ndev));
188 if (ret < 0) {
189 dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name);
190 return ret;
191 }
192
193 ucode = (struct ucode *)fw->data;
194
195 ucode_size = be32_to_cpu(ucode->code_size) * 2;
196 if (!ucode_size || ucode_size > CNN55XX_MAX_UCODE_SIZE) {
197 dev_err(DEV(ndev), "Invalid ucode size: %u for firmware %s\n",
198 ucode_size, fw_name);
199 release_firmware(fw);
200 return -EINVAL;
201 }
202 ucode_data = ucode->code;
203
204 /* copy the firmware version */
205 memcpy(&ndev->hw.fw_name[1][0], ucode->version, (VERSION_LEN - 2));
206 ndev->hw.fw_name[1][VERSION_LEN - 1] = '\0';
207
208 /* Load AE Firmware on UCD Block 2 */
209 write_to_ucd_unit(ndev, ucode_size, ucode_data, 2);
210
211 release_firmware(fw);
212
213 /* put all AE cores in DEFAULT_AE_GROUP */
214 offset = AQM_GRP_EXECMSK_LOX(DEFAULT_AE_GROUP);
215 aqm_grp_execmask_lo.exec_0_to_39 = 0xFFFFFFFFFFULL;
216 nitrox_write_csr(ndev, offset, aqm_grp_execmask_lo.value);
217 offset = AQM_GRP_EXECMSK_HIX(DEFAULT_AE_GROUP);
218 aqm_grp_execmask_hi.exec_40_to_79 = 0xFFFFFFFFFFULL;
219 nitrox_write_csr(ndev, offset, aqm_grp_execmask_hi.value);
220
221 /* write block number and firmware length
222 * bit:<2:0> block number
223 * bit:3 is set SE uses 32KB microcode
224 * bit:3 is clear SE uses 64KB microcode
225 */
226 core_2_eid_val.value = 0ULL;
227 core_2_eid_val.ucode_blk = 0;
228 if (ucode_size <= CNN55XX_UCD_BLOCK_SIZE)
229 core_2_eid_val.ucode_len = 1;
230 else
231 core_2_eid_val.ucode_len = 0;
232
233 for (i = 0; i < ndev->hw.ae_cores; i++) {
234 offset = UCD_AE_EID_UCODE_BLOCK_NUMX(i);
235 nitrox_write_csr(ndev, offset, core_2_eid_val.value);
236 }
237
238 return 0;
239 }
240
241 /**
242 * nitrox_add_to_devlist - add NITROX device to global device list
243 * @ndev: NITROX device
244 */
nitrox_add_to_devlist(struct nitrox_device * ndev)245 static int nitrox_add_to_devlist(struct nitrox_device *ndev)
246 {
247 struct nitrox_device *dev;
248 int ret = 0;
249
250 INIT_LIST_HEAD(&ndev->list);
251 refcount_set(&ndev->refcnt, 1);
252
253 mutex_lock(&devlist_lock);
254 list_for_each_entry(dev, &ndevlist, list) {
255 if (dev == ndev) {
256 ret = -EEXIST;
257 goto unlock;
258 }
259 }
260 ndev->idx = num_devices++;
261 list_add_tail(&ndev->list, &ndevlist);
262 unlock:
263 mutex_unlock(&devlist_lock);
264 return ret;
265 }
266
267 /**
268 * nitrox_remove_from_devlist - remove NITROX device from
269 * global device list
270 * @ndev: NITROX device
271 */
nitrox_remove_from_devlist(struct nitrox_device * ndev)272 static void nitrox_remove_from_devlist(struct nitrox_device *ndev)
273 {
274 mutex_lock(&devlist_lock);
275 list_del(&ndev->list);
276 num_devices--;
277 mutex_unlock(&devlist_lock);
278 }
279
nitrox_get_first_device(void)280 struct nitrox_device *nitrox_get_first_device(void)
281 {
282 struct nitrox_device *ndev = NULL;
283
284 mutex_lock(&devlist_lock);
285 list_for_each_entry(ndev, &ndevlist, list) {
286 if (nitrox_ready(ndev))
287 break;
288 }
289 mutex_unlock(&devlist_lock);
290 if (!ndev)
291 return NULL;
292
293 refcount_inc(&ndev->refcnt);
294 /* barrier to sync with other cpus */
295 smp_mb__after_atomic();
296 return ndev;
297 }
298
nitrox_put_device(struct nitrox_device * ndev)299 void nitrox_put_device(struct nitrox_device *ndev)
300 {
301 if (!ndev)
302 return;
303
304 refcount_dec(&ndev->refcnt);
305 /* barrier to sync with other cpus */
306 smp_mb__after_atomic();
307 }
308
nitrox_device_flr(struct pci_dev * pdev)309 static int nitrox_device_flr(struct pci_dev *pdev)
310 {
311 int pos = 0;
312
313 pos = pci_save_state(pdev);
314 if (pos) {
315 dev_err(&pdev->dev, "Failed to save pci state\n");
316 return -ENOMEM;
317 }
318
319 /* check flr support */
320 if (pcie_has_flr(pdev))
321 pcie_flr(pdev);
322
323 pci_restore_state(pdev);
324
325 return 0;
326 }
327
nitrox_pf_sw_init(struct nitrox_device * ndev)328 static int nitrox_pf_sw_init(struct nitrox_device *ndev)
329 {
330 int err;
331
332 err = nitrox_common_sw_init(ndev);
333 if (err)
334 return err;
335
336 err = nitrox_register_interrupts(ndev);
337 if (err)
338 nitrox_common_sw_cleanup(ndev);
339
340 return err;
341 }
342
nitrox_pf_sw_cleanup(struct nitrox_device * ndev)343 static void nitrox_pf_sw_cleanup(struct nitrox_device *ndev)
344 {
345 nitrox_unregister_interrupts(ndev);
346 nitrox_common_sw_cleanup(ndev);
347 }
348
349 /**
350 * nitrox_bist_check - Check NITORX BIST registers status
351 * @ndev: NITROX device
352 */
nitrox_bist_check(struct nitrox_device * ndev)353 static int nitrox_bist_check(struct nitrox_device *ndev)
354 {
355 u64 value = 0;
356 int i;
357
358 for (i = 0; i < NR_CLUSTERS; i++) {
359 value += nitrox_read_csr(ndev, EMU_BIST_STATUSX(i));
360 value += nitrox_read_csr(ndev, EFL_CORE_BIST_REGX(i));
361 }
362 value += nitrox_read_csr(ndev, UCD_BIST_STATUS);
363 value += nitrox_read_csr(ndev, NPS_CORE_BIST_REG);
364 value += nitrox_read_csr(ndev, NPS_CORE_NPC_BIST_REG);
365 value += nitrox_read_csr(ndev, NPS_PKT_SLC_BIST_REG);
366 value += nitrox_read_csr(ndev, NPS_PKT_IN_BIST_REG);
367 value += nitrox_read_csr(ndev, POM_BIST_REG);
368 value += nitrox_read_csr(ndev, BMI_BIST_REG);
369 value += nitrox_read_csr(ndev, EFL_TOP_BIST_STAT);
370 value += nitrox_read_csr(ndev, BMO_BIST_REG);
371 value += nitrox_read_csr(ndev, LBC_BIST_STATUS);
372 value += nitrox_read_csr(ndev, PEM_BIST_STATUSX(0));
373 if (value)
374 return -EIO;
375 return 0;
376 }
377
nitrox_pf_hw_init(struct nitrox_device * ndev)378 static int nitrox_pf_hw_init(struct nitrox_device *ndev)
379 {
380 int err;
381
382 err = nitrox_bist_check(ndev);
383 if (err) {
384 dev_err(&ndev->pdev->dev, "BIST check failed\n");
385 return err;
386 }
387 /* get cores information */
388 nitrox_get_hwinfo(ndev);
389
390 nitrox_config_nps_core_unit(ndev);
391 nitrox_config_aqm_unit(ndev);
392 nitrox_config_nps_pkt_unit(ndev);
393 nitrox_config_pom_unit(ndev);
394 nitrox_config_efl_unit(ndev);
395 /* configure IO units */
396 nitrox_config_bmi_unit(ndev);
397 nitrox_config_bmo_unit(ndev);
398 /* configure Local Buffer Cache */
399 nitrox_config_lbc_unit(ndev);
400 nitrox_config_rand_unit(ndev);
401
402 /* load firmware on cores */
403 err = nitrox_load_fw(ndev);
404 if (err)
405 return err;
406
407 nitrox_config_emu_unit(ndev);
408
409 return 0;
410 }
411
412 /**
413 * nitrox_probe - NITROX Initialization function.
414 * @pdev: PCI device information struct
415 * @id: entry in nitrox_pci_tbl
416 *
417 * Return: 0, if the driver is bound to the device, or
418 * a negative error if there is failure.
419 */
nitrox_probe(struct pci_dev * pdev,const struct pci_device_id * id)420 static int nitrox_probe(struct pci_dev *pdev,
421 const struct pci_device_id *id)
422 {
423 struct nitrox_device *ndev;
424 int err;
425
426 dev_info_once(&pdev->dev, "%s driver version %s\n",
427 nitrox_driver_name, DRIVER_VERSION);
428
429 err = pci_enable_device_mem(pdev);
430 if (err)
431 return err;
432
433 /* do FLR */
434 err = nitrox_device_flr(pdev);
435 if (err) {
436 dev_err(&pdev->dev, "FLR failed\n");
437 pci_disable_device(pdev);
438 return err;
439 }
440
441 if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
442 dev_dbg(&pdev->dev, "DMA to 64-BIT address\n");
443 } else {
444 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
445 if (err) {
446 dev_err(&pdev->dev, "DMA configuration failed\n");
447 pci_disable_device(pdev);
448 return err;
449 }
450 }
451
452 err = pci_request_mem_regions(pdev, nitrox_driver_name);
453 if (err) {
454 pci_disable_device(pdev);
455 return err;
456 }
457 pci_set_master(pdev);
458
459 ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
460 if (!ndev) {
461 err = -ENOMEM;
462 goto ndev_fail;
463 }
464
465 pci_set_drvdata(pdev, ndev);
466 ndev->pdev = pdev;
467
468 /* add to device list */
469 nitrox_add_to_devlist(ndev);
470
471 ndev->hw.vendor_id = pdev->vendor;
472 ndev->hw.device_id = pdev->device;
473 ndev->hw.revision_id = pdev->revision;
474 /* command timeout in jiffies */
475 ndev->timeout = msecs_to_jiffies(CMD_TIMEOUT);
476 ndev->node = dev_to_node(&pdev->dev);
477 if (ndev->node == NUMA_NO_NODE)
478 ndev->node = 0;
479
480 ndev->bar_addr = ioremap(pci_resource_start(pdev, 0),
481 pci_resource_len(pdev, 0));
482 if (!ndev->bar_addr) {
483 err = -EIO;
484 goto ioremap_err;
485 }
486 /* allocate command queus based on cpus, max queues are 64 */
487 ndev->nr_queues = min_t(u32, MAX_PF_QUEUES, num_online_cpus());
488 ndev->qlen = qlen;
489
490 err = nitrox_pf_sw_init(ndev);
491 if (err)
492 goto ioremap_err;
493
494 err = nitrox_pf_hw_init(ndev);
495 if (err)
496 goto pf_hw_fail;
497
498 nitrox_debugfs_init(ndev);
499
500 /* clear the statistics */
501 atomic64_set(&ndev->stats.posted, 0);
502 atomic64_set(&ndev->stats.completed, 0);
503 atomic64_set(&ndev->stats.dropped, 0);
504
505 atomic_set(&ndev->state, __NDEV_READY);
506 /* barrier to sync with other cpus */
507 smp_mb__after_atomic();
508
509 err = nitrox_crypto_register();
510 if (err)
511 goto crypto_fail;
512
513 return 0;
514
515 crypto_fail:
516 nitrox_debugfs_exit(ndev);
517 atomic_set(&ndev->state, __NDEV_NOT_READY);
518 /* barrier to sync with other cpus */
519 smp_mb__after_atomic();
520 pf_hw_fail:
521 nitrox_pf_sw_cleanup(ndev);
522 ioremap_err:
523 nitrox_remove_from_devlist(ndev);
524 kfree(ndev);
525 pci_set_drvdata(pdev, NULL);
526 ndev_fail:
527 pci_release_mem_regions(pdev);
528 pci_disable_device(pdev);
529 return err;
530 }
531
532 /**
533 * nitrox_remove - Unbind the driver from the device.
534 * @pdev: PCI device information struct
535 */
nitrox_remove(struct pci_dev * pdev)536 static void nitrox_remove(struct pci_dev *pdev)
537 {
538 struct nitrox_device *ndev = pci_get_drvdata(pdev);
539
540 if (!ndev)
541 return;
542
543 if (!refcount_dec_and_test(&ndev->refcnt)) {
544 dev_err(DEV(ndev), "Device refcnt not zero (%d)\n",
545 refcount_read(&ndev->refcnt));
546 return;
547 }
548
549 dev_info(DEV(ndev), "Removing Device %x:%x\n",
550 ndev->hw.vendor_id, ndev->hw.device_id);
551
552 atomic_set(&ndev->state, __NDEV_NOT_READY);
553 /* barrier to sync with other cpus */
554 smp_mb__after_atomic();
555
556 nitrox_remove_from_devlist(ndev);
557
558 #ifdef CONFIG_PCI_IOV
559 /* disable SR-IOV */
560 nitrox_sriov_configure(pdev, 0);
561 #endif
562 nitrox_crypto_unregister();
563 nitrox_debugfs_exit(ndev);
564 nitrox_pf_sw_cleanup(ndev);
565
566 iounmap(ndev->bar_addr);
567 kfree(ndev);
568
569 pci_set_drvdata(pdev, NULL);
570 pci_release_mem_regions(pdev);
571 pci_disable_device(pdev);
572 }
573
nitrox_shutdown(struct pci_dev * pdev)574 static void nitrox_shutdown(struct pci_dev *pdev)
575 {
576 pci_set_drvdata(pdev, NULL);
577 pci_release_mem_regions(pdev);
578 pci_disable_device(pdev);
579 }
580
581 static struct pci_driver nitrox_driver = {
582 .name = nitrox_driver_name,
583 .id_table = nitrox_pci_tbl,
584 .probe = nitrox_probe,
585 .remove = nitrox_remove,
586 .shutdown = nitrox_shutdown,
587 #ifdef CONFIG_PCI_IOV
588 .sriov_configure = nitrox_sriov_configure,
589 #endif
590 };
591
592 module_pci_driver(nitrox_driver);
593
594 MODULE_AUTHOR("Srikanth Jampala <Jampala.Srikanth@cavium.com>");
595 MODULE_DESCRIPTION("Cavium CNN55XX PF Driver" DRIVER_VERSION " ");
596 MODULE_LICENSE("GPL");
597 MODULE_VERSION(DRIVER_VERSION);
598 MODULE_FIRMWARE(SE_FW);
599
