1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
2 /* Copyright(c) 2020 Intel Corporation */
3 #include <linux/device.h>
4 #include <linux/module.h>
5 #include <linux/pci.h>
6
7 #include <adf_accel_devices.h>
8 #include <adf_cfg.h>
9 #include <adf_common_drv.h>
10
11 #include "adf_4xxx_hw_data.h"
12 #include "qat_crypto.h"
13 #include "adf_transport_access_macros.h"
14
15 static const struct pci_device_id adf_pci_tbl[] = {
16 { PCI_VDEVICE(INTEL, ADF_4XXX_PCI_DEVICE_ID), },
17 { PCI_VDEVICE(INTEL, ADF_401XX_PCI_DEVICE_ID), },
18 { }
19 };
20 MODULE_DEVICE_TABLE(pci, adf_pci_tbl);
21
adf_cleanup_accel(struct adf_accel_dev * accel_dev)22 static void adf_cleanup_accel(struct adf_accel_dev *accel_dev)
23 {
24 if (accel_dev->hw_device) {
25 adf_clean_hw_data_4xxx(accel_dev->hw_device);
26 accel_dev->hw_device = NULL;
27 }
28 adf_cfg_dev_remove(accel_dev);
29 debugfs_remove(accel_dev->debugfs_dir);
30 adf_devmgr_rm_dev(accel_dev, NULL);
31 }
32
adf_cfg_dev_init(struct adf_accel_dev * accel_dev)33 static int adf_cfg_dev_init(struct adf_accel_dev *accel_dev)
34 {
35 const char *config;
36 int ret;
37
38 config = accel_dev->accel_id % 2 ? ADF_CFG_DC : ADF_CFG_CY;
39
40 ret = adf_cfg_section_add(accel_dev, ADF_GENERAL_SEC);
41 if (ret)
42 return ret;
43
44 /* Default configuration is crypto only for even devices
45 * and compression for odd devices
46 */
47 ret = adf_cfg_add_key_value_param(accel_dev, ADF_GENERAL_SEC,
48 ADF_SERVICES_ENABLED, config,
49 ADF_STR);
50 if (ret)
51 return ret;
52
53 return 0;
54 }
55
adf_crypto_dev_config(struct adf_accel_dev * accel_dev)56 int adf_crypto_dev_config(struct adf_accel_dev *accel_dev)
57 {
58 char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES];
59 int banks = GET_MAX_BANKS(accel_dev);
60 int cpus = num_online_cpus();
61 unsigned long bank, val;
62 int instances;
63 int ret;
64 int i;
65
66 if (adf_hw_dev_has_crypto(accel_dev))
67 instances = min(cpus, banks / 2);
68 else
69 instances = 0;
70
71 ret = adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC);
72 if (ret)
73 goto err;
74
75 ret = adf_cfg_section_add(accel_dev, "Accelerator0");
76 if (ret)
77 goto err;
78
79 for (i = 0; i < instances; i++) {
80 val = i;
81 bank = i * 2;
82 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_BANK_NUM, i);
83 ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
84 key, &bank, ADF_DEC);
85 if (ret)
86 goto err;
87
88 bank += 1;
89 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_BANK_NUM, i);
90 ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
91 key, &bank, ADF_DEC);
92 if (ret)
93 goto err;
94
95 snprintf(key, sizeof(key), ADF_CY "%d" ADF_ETRMGR_CORE_AFFINITY,
96 i);
97 ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
98 key, &val, ADF_DEC);
99 if (ret)
100 goto err;
101
102 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, i);
103 val = 128;
104 ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
105 key, &val, ADF_DEC);
106 if (ret)
107 goto err;
108
109 val = 512;
110 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, i);
111 ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
112 key, &val, ADF_DEC);
113 if (ret)
114 goto err;
115
116 val = 0;
117 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i);
118 ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
119 key, &val, ADF_DEC);
120 if (ret)
121 goto err;
122
123 val = 0;
124 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, i);
125 ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
126 key, &val, ADF_DEC);
127 if (ret)
128 goto err;
129
130 val = 1;
131 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i);
132 ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
133 key, &val, ADF_DEC);
134 if (ret)
135 goto err;
136
137 val = 1;
138 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, i);
139 ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
140 key, &val, ADF_DEC);
141 if (ret)
142 goto err;
143
144 val = ADF_COALESCING_DEF_TIME;
145 snprintf(key, sizeof(key), ADF_ETRMGR_COALESCE_TIMER_FORMAT, i);
146 ret = adf_cfg_add_key_value_param(accel_dev, "Accelerator0",
147 key, &val, ADF_DEC);
148 if (ret)
149 goto err;
150 }
151
152 val = i;
153 ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, ADF_NUM_CY,
154 &val, ADF_DEC);
155 if (ret)
156 goto err;
157
158 set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status);
159 return 0;
160 err:
161 dev_err(&GET_DEV(accel_dev), "Failed to start QAT accel dev\n");
162 return ret;
163 }
164
adf_probe(struct pci_dev * pdev,const struct pci_device_id * ent)165 static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
166 {
167 struct adf_accel_dev *accel_dev;
168 struct adf_accel_pci *accel_pci_dev;
169 struct adf_hw_device_data *hw_data;
170 char name[ADF_DEVICE_NAME_LENGTH];
171 unsigned int i, bar_nr;
172 unsigned long bar_mask;
173 struct adf_bar *bar;
174 int ret;
175
176 if (num_possible_nodes() > 1 && dev_to_node(&pdev->dev) < 0) {
177 /*
178 * If the accelerator is connected to a node with no memory
179 * there is no point in using the accelerator since the remote
180 * memory transaction will be very slow.
181 */
182 dev_err(&pdev->dev, "Invalid NUMA configuration.\n");
183 return -EINVAL;
184 }
185
186 accel_dev = devm_kzalloc(&pdev->dev, sizeof(*accel_dev), GFP_KERNEL);
187 if (!accel_dev)
188 return -ENOMEM;
189
190 INIT_LIST_HEAD(&accel_dev->crypto_list);
191 accel_pci_dev = &accel_dev->accel_pci_dev;
192 accel_pci_dev->pci_dev = pdev;
193
194 /*
195 * Add accel device to accel table
196 * This should be called before adf_cleanup_accel is called
197 */
198 if (adf_devmgr_add_dev(accel_dev, NULL)) {
199 dev_err(&pdev->dev, "Failed to add new accelerator device.\n");
200 return -EFAULT;
201 }
202
203 accel_dev->owner = THIS_MODULE;
204 /* Allocate and initialise device hardware meta-data structure */
205 hw_data = devm_kzalloc(&pdev->dev, sizeof(*hw_data), GFP_KERNEL);
206 if (!hw_data) {
207 ret = -ENOMEM;
208 goto out_err;
209 }
210
211 accel_dev->hw_device = hw_data;
212 adf_init_hw_data_4xxx(accel_dev->hw_device);
213
214 pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid);
215 pci_read_config_dword(pdev, ADF_4XXX_FUSECTL4_OFFSET, &hw_data->fuses);
216
217 /* Get Accelerators and Accelerators Engines masks */
218 hw_data->accel_mask = hw_data->get_accel_mask(hw_data);
219 hw_data->ae_mask = hw_data->get_ae_mask(hw_data);
220 accel_pci_dev->sku = hw_data->get_sku(hw_data);
221 /* If the device has no acceleration engines then ignore it */
222 if (!hw_data->accel_mask || !hw_data->ae_mask ||
223 (~hw_data->ae_mask & 0x01)) {
224 dev_err(&pdev->dev, "No acceleration units found.\n");
225 ret = -EFAULT;
226 goto out_err;
227 }
228
229 /* Create dev top level debugfs entry */
230 snprintf(name, sizeof(name), "%s%s_%s", ADF_DEVICE_NAME_PREFIX,
231 hw_data->dev_class->name, pci_name(pdev));
232
233 accel_dev->debugfs_dir = debugfs_create_dir(name, NULL);
234
235 /* Create device configuration table */
236 ret = adf_cfg_dev_add(accel_dev);
237 if (ret)
238 goto out_err;
239
240 /* Enable PCI device */
241 ret = pcim_enable_device(pdev);
242 if (ret) {
243 dev_err(&pdev->dev, "Can't enable PCI device.\n");
244 goto out_err;
245 }
246
247 /* Set DMA identifier */
248 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
249 if (ret) {
250 dev_err(&pdev->dev, "No usable DMA configuration.\n");
251 goto out_err;
252 }
253
254 ret = adf_cfg_dev_init(accel_dev);
255 if (ret) {
256 dev_err(&pdev->dev, "Failed to initialize configuration.\n");
257 goto out_err;
258 }
259
260 /* Get accelerator capabilities mask */
261 hw_data->accel_capabilities_mask = hw_data->get_accel_cap(accel_dev);
262 if (!hw_data->accel_capabilities_mask) {
263 dev_err(&pdev->dev, "Failed to get capabilities mask.\n");
264 goto out_err;
265 }
266
267 /* Find and map all the device's BARS */
268 bar_mask = pci_select_bars(pdev, IORESOURCE_MEM) & ADF_4XXX_BAR_MASK;
269
270 ret = pcim_iomap_regions_request_all(pdev, bar_mask, pci_name(pdev));
271 if (ret) {
272 dev_err(&pdev->dev, "Failed to map pci regions.\n");
273 goto out_err;
274 }
275
276 i = 0;
277 for_each_set_bit(bar_nr, &bar_mask, PCI_STD_NUM_BARS) {
278 bar = &accel_pci_dev->pci_bars[i++];
279 bar->virt_addr = pcim_iomap_table(pdev)[bar_nr];
280 }
281
282 pci_set_master(pdev);
283
284 adf_enable_aer(accel_dev);
285
286 if (pci_save_state(pdev)) {
287 dev_err(&pdev->dev, "Failed to save pci state.\n");
288 ret = -ENOMEM;
289 goto out_err_disable_aer;
290 }
291
292 ret = adf_sysfs_init(accel_dev);
293 if (ret)
294 goto out_err_disable_aer;
295
296 ret = adf_crypto_dev_config(accel_dev);
297 if (ret)
298 goto out_err_disable_aer;
299
300 ret = adf_dev_init(accel_dev);
301 if (ret)
302 goto out_err_dev_shutdown;
303
304 ret = adf_dev_start(accel_dev);
305 if (ret)
306 goto out_err_dev_stop;
307
308 return ret;
309
310 out_err_dev_stop:
311 adf_dev_stop(accel_dev);
312 out_err_dev_shutdown:
313 adf_dev_shutdown(accel_dev);
314 out_err_disable_aer:
315 adf_disable_aer(accel_dev);
316 out_err:
317 adf_cleanup_accel(accel_dev);
318 return ret;
319 }
320
adf_remove(struct pci_dev * pdev)321 static void adf_remove(struct pci_dev *pdev)
322 {
323 struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev);
324
325 if (!accel_dev) {
326 pr_err("QAT: Driver removal failed\n");
327 return;
328 }
329 adf_dev_stop(accel_dev);
330 adf_dev_shutdown(accel_dev);
331 adf_disable_aer(accel_dev);
332 adf_cleanup_accel(accel_dev);
333 }
334
335 static struct pci_driver adf_driver = {
336 .id_table = adf_pci_tbl,
337 .name = ADF_4XXX_DEVICE_NAME,
338 .probe = adf_probe,
339 .remove = adf_remove,
340 .sriov_configure = adf_sriov_configure,
341 .err_handler = &adf_err_handler,
342 };
343
344 module_pci_driver(adf_driver);
345
346 MODULE_LICENSE("Dual BSD/GPL");
347 MODULE_AUTHOR("Intel");
348 MODULE_FIRMWARE(ADF_4XXX_FW);
349 MODULE_FIRMWARE(ADF_4XXX_MMP);
350 MODULE_DESCRIPTION("Intel(R) QuickAssist Technology");
351 MODULE_VERSION(ADF_DRV_VERSION);
352 MODULE_SOFTDEP("pre: crypto-intel_qat");
353
