1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2023 Advanced Micro Devices, Inc */
3
4 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
5
6 #include <linux/pci.h>
7
8 #include <linux/pds/pds_common.h>
9
10 #include "core.h"
11
12 MODULE_DESCRIPTION(PDSC_DRV_DESCRIPTION);
13 MODULE_AUTHOR("Advanced Micro Devices, Inc");
14 MODULE_LICENSE("GPL");
15
16 /* Supported devices */
17 static const struct pci_device_id pdsc_id_table[] = {
18 { PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_CORE_PF) },
19 { PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_VDPA_VF) },
20 { 0, } /* end of table */
21 };
22 MODULE_DEVICE_TABLE(pci, pdsc_id_table);
23
pdsc_wdtimer_cb(struct timer_list * t)24 static void pdsc_wdtimer_cb(struct timer_list *t)
25 {
26 struct pdsc *pdsc = from_timer(pdsc, t, wdtimer);
27
28 dev_dbg(pdsc->dev, "%s: jiffies %ld\n", __func__, jiffies);
29 mod_timer(&pdsc->wdtimer,
30 round_jiffies(jiffies + pdsc->wdtimer_period));
31
32 queue_work(pdsc->wq, &pdsc->health_work);
33 }
34
pdsc_unmap_bars(struct pdsc * pdsc)35 static void pdsc_unmap_bars(struct pdsc *pdsc)
36 {
37 struct pdsc_dev_bar *bars = pdsc->bars;
38 unsigned int i;
39
40 for (i = 0; i < PDS_CORE_BARS_MAX; i++) {
41 if (bars[i].vaddr)
42 pci_iounmap(pdsc->pdev, bars[i].vaddr);
43 }
44 }
45
pdsc_map_bars(struct pdsc * pdsc)46 static int pdsc_map_bars(struct pdsc *pdsc)
47 {
48 struct pdsc_dev_bar *bar = pdsc->bars;
49 struct pci_dev *pdev = pdsc->pdev;
50 struct device *dev = pdsc->dev;
51 struct pdsc_dev_bar *bars;
52 unsigned int i, j;
53 int num_bars = 0;
54 int err;
55 u32 sig;
56
57 bars = pdsc->bars;
58
59 /* Since the PCI interface in the hardware is configurable,
60 * we need to poke into all the bars to find the set we're
61 * expecting.
62 */
63 for (i = 0, j = 0; i < PDS_CORE_BARS_MAX; i++) {
64 if (!(pci_resource_flags(pdev, i) & IORESOURCE_MEM))
65 continue;
66
67 bars[j].len = pci_resource_len(pdev, i);
68 bars[j].bus_addr = pci_resource_start(pdev, i);
69 bars[j].res_index = i;
70
71 /* only map the whole bar 0 */
72 if (j > 0) {
73 bars[j].vaddr = NULL;
74 } else {
75 bars[j].vaddr = pci_iomap(pdev, i, bars[j].len);
76 if (!bars[j].vaddr) {
77 dev_err(dev, "Cannot map BAR %d, aborting\n", i);
78 return -ENODEV;
79 }
80 }
81
82 j++;
83 }
84 num_bars = j;
85
86 /* BAR0: dev_cmd and interrupts */
87 if (num_bars < 1) {
88 dev_err(dev, "No bars found\n");
89 err = -EFAULT;
90 goto err_out;
91 }
92
93 if (bar->len < PDS_CORE_BAR0_SIZE) {
94 dev_err(dev, "Resource bar size %lu too small\n", bar->len);
95 err = -EFAULT;
96 goto err_out;
97 }
98
99 pdsc->info_regs = bar->vaddr + PDS_CORE_BAR0_DEV_INFO_REGS_OFFSET;
100 pdsc->cmd_regs = bar->vaddr + PDS_CORE_BAR0_DEV_CMD_REGS_OFFSET;
101 pdsc->intr_status = bar->vaddr + PDS_CORE_BAR0_INTR_STATUS_OFFSET;
102 pdsc->intr_ctrl = bar->vaddr + PDS_CORE_BAR0_INTR_CTRL_OFFSET;
103
104 sig = ioread32(&pdsc->info_regs->signature);
105 if (sig != PDS_CORE_DEV_INFO_SIGNATURE) {
106 dev_err(dev, "Incompatible firmware signature %x", sig);
107 err = -EFAULT;
108 goto err_out;
109 }
110
111 /* BAR1: doorbells */
112 bar++;
113 if (num_bars < 2) {
114 dev_err(dev, "Doorbell bar missing\n");
115 err = -EFAULT;
116 goto err_out;
117 }
118
119 pdsc->db_pages = bar->vaddr;
120 pdsc->phy_db_pages = bar->bus_addr;
121
122 return 0;
123
124 err_out:
125 pdsc_unmap_bars(pdsc);
126 return err;
127 }
128
pdsc_map_dbpage(struct pdsc * pdsc,int page_num)129 void __iomem *pdsc_map_dbpage(struct pdsc *pdsc, int page_num)
130 {
131 return pci_iomap_range(pdsc->pdev,
132 pdsc->bars[PDS_CORE_PCI_BAR_DBELL].res_index,
133 (u64)page_num << PAGE_SHIFT, PAGE_SIZE);
134 }
135
pdsc_sriov_configure(struct pci_dev * pdev,int num_vfs)136 static int pdsc_sriov_configure(struct pci_dev *pdev, int num_vfs)
137 {
138 struct pdsc *pdsc = pci_get_drvdata(pdev);
139 struct device *dev = pdsc->dev;
140 int ret = 0;
141
142 if (num_vfs > 0) {
143 pdsc->vfs = kcalloc(num_vfs, sizeof(struct pdsc_vf),
144 GFP_KERNEL);
145 if (!pdsc->vfs)
146 return -ENOMEM;
147 pdsc->num_vfs = num_vfs;
148
149 ret = pci_enable_sriov(pdev, num_vfs);
150 if (ret) {
151 dev_err(dev, "Cannot enable SRIOV: %pe\n",
152 ERR_PTR(ret));
153 goto no_vfs;
154 }
155
156 return num_vfs;
157 }
158
159 no_vfs:
160 pci_disable_sriov(pdev);
161
162 kfree(pdsc->vfs);
163 pdsc->vfs = NULL;
164 pdsc->num_vfs = 0;
165
166 return ret;
167 }
168
pdsc_init_vf(struct pdsc * vf)169 static int pdsc_init_vf(struct pdsc *vf)
170 {
171 struct devlink *dl;
172 struct pdsc *pf;
173 int err;
174
175 pf = pdsc_get_pf_struct(vf->pdev);
176 if (IS_ERR_OR_NULL(pf))
177 return PTR_ERR(pf) ?: -1;
178
179 vf->vf_id = pci_iov_vf_id(vf->pdev);
180
181 dl = priv_to_devlink(vf);
182 devl_lock(dl);
183 devl_register(dl);
184 devl_unlock(dl);
185
186 pf->vfs[vf->vf_id].vf = vf;
187 err = pdsc_auxbus_dev_add(vf, pf);
188 if (err) {
189 devl_lock(dl);
190 devl_unregister(dl);
191 devl_unlock(dl);
192 }
193
194 return err;
195 }
196
197 static const struct devlink_health_reporter_ops pdsc_fw_reporter_ops = {
198 .name = "fw",
199 .diagnose = pdsc_fw_reporter_diagnose,
200 };
201
202 static const struct devlink_param pdsc_dl_params[] = {
203 DEVLINK_PARAM_GENERIC(ENABLE_VNET,
204 BIT(DEVLINK_PARAM_CMODE_RUNTIME),
205 pdsc_dl_enable_get,
206 pdsc_dl_enable_set,
207 pdsc_dl_enable_validate),
208 };
209
210 #define PDSC_WQ_NAME_LEN 24
211
pdsc_init_pf(struct pdsc * pdsc)212 static int pdsc_init_pf(struct pdsc *pdsc)
213 {
214 struct devlink_health_reporter *hr;
215 char wq_name[PDSC_WQ_NAME_LEN];
216 struct devlink *dl;
217 int err;
218
219 pcie_print_link_status(pdsc->pdev);
220
221 err = pci_request_regions(pdsc->pdev, PDS_CORE_DRV_NAME);
222 if (err) {
223 dev_err(pdsc->dev, "Cannot request PCI regions: %pe\n",
224 ERR_PTR(err));
225 return err;
226 }
227
228 err = pdsc_map_bars(pdsc);
229 if (err)
230 goto err_out_release_regions;
231
232 /* General workqueue and timer, but don't start timer yet */
233 snprintf(wq_name, sizeof(wq_name), "%s.%d", PDS_CORE_DRV_NAME, pdsc->uid);
234 pdsc->wq = create_singlethread_workqueue(wq_name);
235 INIT_WORK(&pdsc->health_work, pdsc_health_thread);
236 timer_setup(&pdsc->wdtimer, pdsc_wdtimer_cb, 0);
237 pdsc->wdtimer_period = PDSC_WATCHDOG_SECS * HZ;
238
239 mutex_init(&pdsc->devcmd_lock);
240 mutex_init(&pdsc->config_lock);
241 spin_lock_init(&pdsc->adminq_lock);
242
243 mutex_lock(&pdsc->config_lock);
244 set_bit(PDSC_S_FW_DEAD, &pdsc->state);
245
246 err = pdsc_setup(pdsc, PDSC_SETUP_INIT);
247 if (err) {
248 mutex_unlock(&pdsc->config_lock);
249 goto err_out_unmap_bars;
250 }
251
252 err = pdsc_start(pdsc);
253 if (err) {
254 mutex_unlock(&pdsc->config_lock);
255 goto err_out_teardown;
256 }
257
258 mutex_unlock(&pdsc->config_lock);
259
260 dl = priv_to_devlink(pdsc);
261 devl_lock(dl);
262 err = devl_params_register(dl, pdsc_dl_params,
263 ARRAY_SIZE(pdsc_dl_params));
264 if (err) {
265 devl_unlock(dl);
266 dev_warn(pdsc->dev, "Failed to register devlink params: %pe\n",
267 ERR_PTR(err));
268 goto err_out_stop;
269 }
270
271 hr = devl_health_reporter_create(dl, &pdsc_fw_reporter_ops, 0, pdsc);
272 if (IS_ERR(hr)) {
273 devl_unlock(dl);
274 dev_warn(pdsc->dev, "Failed to create fw reporter: %pe\n", hr);
275 err = PTR_ERR(hr);
276 goto err_out_unreg_params;
277 }
278 pdsc->fw_reporter = hr;
279
280 devl_register(dl);
281 devl_unlock(dl);
282
283 /* Lastly, start the health check timer */
284 mod_timer(&pdsc->wdtimer, round_jiffies(jiffies + pdsc->wdtimer_period));
285
286 return 0;
287
288 err_out_unreg_params:
289 devlink_params_unregister(dl, pdsc_dl_params,
290 ARRAY_SIZE(pdsc_dl_params));
291 err_out_stop:
292 pdsc_stop(pdsc);
293 err_out_teardown:
294 pdsc_teardown(pdsc, PDSC_TEARDOWN_REMOVING);
295 err_out_unmap_bars:
296 del_timer_sync(&pdsc->wdtimer);
297 if (pdsc->wq)
298 destroy_workqueue(pdsc->wq);
299 mutex_destroy(&pdsc->config_lock);
300 mutex_destroy(&pdsc->devcmd_lock);
301 pci_free_irq_vectors(pdsc->pdev);
302 pdsc_unmap_bars(pdsc);
303 err_out_release_regions:
304 pci_release_regions(pdsc->pdev);
305
306 return err;
307 }
308
309 static const struct devlink_ops pdsc_dl_ops = {
310 .info_get = pdsc_dl_info_get,
311 .flash_update = pdsc_dl_flash_update,
312 };
313
314 static const struct devlink_ops pdsc_dl_vf_ops = {
315 };
316
317 static DEFINE_IDA(pdsc_ida);
318
pdsc_probe(struct pci_dev * pdev,const struct pci_device_id * ent)319 static int pdsc_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
320 {
321 struct device *dev = &pdev->dev;
322 const struct devlink_ops *ops;
323 struct devlink *dl;
324 struct pdsc *pdsc;
325 bool is_pf;
326 int err;
327
328 is_pf = !pdev->is_virtfn;
329 ops = is_pf ? &pdsc_dl_ops : &pdsc_dl_vf_ops;
330 dl = devlink_alloc(ops, sizeof(struct pdsc), dev);
331 if (!dl)
332 return -ENOMEM;
333 pdsc = devlink_priv(dl);
334
335 pdsc->pdev = pdev;
336 pdsc->dev = &pdev->dev;
337 set_bit(PDSC_S_INITING_DRIVER, &pdsc->state);
338 pci_set_drvdata(pdev, pdsc);
339 pdsc_debugfs_add_dev(pdsc);
340
341 err = ida_alloc(&pdsc_ida, GFP_KERNEL);
342 if (err < 0) {
343 dev_err(pdsc->dev, "%s: id alloc failed: %pe\n",
344 __func__, ERR_PTR(err));
345 goto err_out_free_devlink;
346 }
347 pdsc->uid = err;
348
349 /* Query system for DMA addressing limitation for the device. */
350 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(PDS_CORE_ADDR_LEN));
351 if (err) {
352 dev_err(dev, "Unable to obtain 64-bit DMA for consistent allocations, aborting: %pe\n",
353 ERR_PTR(err));
354 goto err_out_free_ida;
355 }
356
357 err = pci_enable_device(pdev);
358 if (err) {
359 dev_err(dev, "Cannot enable PCI device: %pe\n", ERR_PTR(err));
360 goto err_out_free_ida;
361 }
362 pci_set_master(pdev);
363
364 if (is_pf)
365 err = pdsc_init_pf(pdsc);
366 else
367 err = pdsc_init_vf(pdsc);
368 if (err) {
369 dev_err(dev, "Cannot init device: %pe\n", ERR_PTR(err));
370 goto err_out_disable_device;
371 }
372
373 clear_bit(PDSC_S_INITING_DRIVER, &pdsc->state);
374 return 0;
375
376 err_out_disable_device:
377 pci_disable_device(pdev);
378 err_out_free_ida:
379 ida_free(&pdsc_ida, pdsc->uid);
380 err_out_free_devlink:
381 pdsc_debugfs_del_dev(pdsc);
382 devlink_free(dl);
383
384 return err;
385 }
386
pdsc_remove(struct pci_dev * pdev)387 static void pdsc_remove(struct pci_dev *pdev)
388 {
389 struct pdsc *pdsc = pci_get_drvdata(pdev);
390 struct devlink *dl;
391
392 /* Unhook the registrations first to be sure there
393 * are no requests while we're stopping.
394 */
395 dl = priv_to_devlink(pdsc);
396 devl_lock(dl);
397 devl_unregister(dl);
398 if (!pdev->is_virtfn) {
399 if (pdsc->fw_reporter) {
400 devl_health_reporter_destroy(pdsc->fw_reporter);
401 pdsc->fw_reporter = NULL;
402 }
403 devl_params_unregister(dl, pdsc_dl_params,
404 ARRAY_SIZE(pdsc_dl_params));
405 }
406 devl_unlock(dl);
407
408 if (pdev->is_virtfn) {
409 struct pdsc *pf;
410
411 pf = pdsc_get_pf_struct(pdsc->pdev);
412 if (!IS_ERR(pf)) {
413 pdsc_auxbus_dev_del(pdsc, pf);
414 pf->vfs[pdsc->vf_id].vf = NULL;
415 }
416 } else {
417 /* Remove the VFs and their aux_bus connections before other
418 * cleanup so that the clients can use the AdminQ to cleanly
419 * shut themselves down.
420 */
421 pdsc_sriov_configure(pdev, 0);
422
423 del_timer_sync(&pdsc->wdtimer);
424 if (pdsc->wq)
425 destroy_workqueue(pdsc->wq);
426
427 mutex_lock(&pdsc->config_lock);
428 set_bit(PDSC_S_STOPPING_DRIVER, &pdsc->state);
429
430 pdsc_stop(pdsc);
431 pdsc_teardown(pdsc, PDSC_TEARDOWN_REMOVING);
432 mutex_unlock(&pdsc->config_lock);
433 mutex_destroy(&pdsc->config_lock);
434 mutex_destroy(&pdsc->devcmd_lock);
435
436 pci_free_irq_vectors(pdev);
437 pdsc_unmap_bars(pdsc);
438 pci_release_regions(pdev);
439 }
440
441 pci_disable_device(pdev);
442
443 ida_free(&pdsc_ida, pdsc->uid);
444 pdsc_debugfs_del_dev(pdsc);
445 devlink_free(dl);
446 }
447
448 static struct pci_driver pdsc_driver = {
449 .name = PDS_CORE_DRV_NAME,
450 .id_table = pdsc_id_table,
451 .probe = pdsc_probe,
452 .remove = pdsc_remove,
453 .sriov_configure = pdsc_sriov_configure,
454 };
455
pdsc_get_pf_struct(struct pci_dev * vf_pdev)456 void *pdsc_get_pf_struct(struct pci_dev *vf_pdev)
457 {
458 return pci_iov_get_pf_drvdata(vf_pdev, &pdsc_driver);
459 }
460 EXPORT_SYMBOL_GPL(pdsc_get_pf_struct);
461
pdsc_init_module(void)462 static int __init pdsc_init_module(void)
463 {
464 if (strcmp(KBUILD_MODNAME, PDS_CORE_DRV_NAME))
465 return -EINVAL;
466
467 pdsc_debugfs_create();
468 return pci_register_driver(&pdsc_driver);
469 }
470
pdsc_cleanup_module(void)471 static void __exit pdsc_cleanup_module(void)
472 {
473 pci_unregister_driver(&pdsc_driver);
474 pdsc_debugfs_destroy();
475 }
476
477 module_init(pdsc_init_module);
478 module_exit(pdsc_cleanup_module);
479
