1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (C) 2018 Google, Inc. */
3
4 #include "gasket_interrupt.h"
5
6 #include "gasket_constants.h"
7 #include "gasket_core.h"
8 #include "gasket_sysfs.h"
9 #include <linux/device.h>
10 #include <linux/interrupt.h>
11 #include <linux/printk.h>
12 #ifdef GASKET_KERNEL_TRACE_SUPPORT
13 #define CREATE_TRACE_POINTS
14 #include <trace/events/gasket_interrupt.h>
15 #else
16 #define trace_gasket_interrupt_event(x, ...)
17 #endif
18 /* Retry attempts if the requested number of interrupts aren't available. */
19 #define MSIX_RETRY_COUNT 3
20
21 /* Instance interrupt management data. */
22 struct gasket_interrupt_data {
23 /* The name associated with this interrupt data. */
24 const char *name;
25
26 /* Interrupt type. See gasket_interrupt_type in gasket_core.h */
27 int type;
28
29 /* The PCI device [if any] associated with the owning device. */
30 struct pci_dev *pci_dev;
31
32 /* Set to 1 if MSI-X has successfully been configred, 0 otherwise. */
33 int msix_configured;
34
35 /* The number of interrupts requested by the owning device. */
36 int num_interrupts;
37
38 /* A pointer to the interrupt descriptor struct for this device. */
39 const struct gasket_interrupt_desc *interrupts;
40
41 /* The index of the bar into which interrupts should be mapped. */
42 int interrupt_bar_index;
43
44 /* The width of a single interrupt in a packed interrupt register. */
45 int pack_width;
46
47 /*
48 * Design-wise, these elements should be bundled together, but
49 * pci_enable_msix's interface requires that they be managed
50 * individually (requires array of struct msix_entry).
51 */
52
53 /* The number of successfully configured interrupts. */
54 int num_configured;
55
56 /* The MSI-X data for each requested/configured interrupt. */
57 struct msix_entry *msix_entries;
58
59 /* The eventfd "callback" data for each interrupt. */
60 struct eventfd_ctx **eventfd_ctxs;
61
62 /* The number of times each interrupt has been called. */
63 ulong *interrupt_counts;
64
65 /* Linux IRQ number. */
66 int irq;
67 };
68
69 /* Structures to display interrupt counts in sysfs. */
70 enum interrupt_sysfs_attribute_type {
71 ATTR_INTERRUPT_COUNTS,
72 };
73
74 /* Set up device registers for interrupt handling. */
gasket_interrupt_setup(struct gasket_dev * gasket_dev)75 static void gasket_interrupt_setup(struct gasket_dev *gasket_dev)
76 {
77 int i;
78 int pack_shift;
79 ulong mask;
80 ulong value;
81 struct gasket_interrupt_data *interrupt_data =
82 gasket_dev->interrupt_data;
83
84 if (!interrupt_data) {
85 dev_dbg(gasket_dev->dev, "Interrupt data is not initialized\n");
86 return;
87 }
88
89 dev_dbg(gasket_dev->dev, "Running interrupt setup\n");
90
91 /* Setup the MSIX table. */
92
93 for (i = 0; i < interrupt_data->num_interrupts; i++) {
94 /*
95 * If the interrupt is not packed, we can write the index into
96 * the register directly. If not, we need to deal with a read-
97 * modify-write and shift based on the packing index.
98 */
99 dev_dbg(gasket_dev->dev,
100 "Setting up interrupt index %d with index 0x%llx and packing %d\n",
101 interrupt_data->interrupts[i].index,
102 interrupt_data->interrupts[i].reg,
103 interrupt_data->interrupts[i].packing);
104 if (interrupt_data->interrupts[i].packing == UNPACKED) {
105 value = interrupt_data->interrupts[i].index;
106 } else {
107 switch (interrupt_data->interrupts[i].packing) {
108 case PACK_0:
109 pack_shift = 0;
110 break;
111 case PACK_1:
112 pack_shift = interrupt_data->pack_width;
113 break;
114 case PACK_2:
115 pack_shift = 2 * interrupt_data->pack_width;
116 break;
117 case PACK_3:
118 pack_shift = 3 * interrupt_data->pack_width;
119 break;
120 default:
121 dev_dbg(gasket_dev->dev,
122 "Found interrupt description with unknown enum %d\n",
123 interrupt_data->interrupts[i].packing);
124 return;
125 }
126
127 mask = ~(0xFFFF << pack_shift);
128 value = gasket_dev_read_64(gasket_dev,
129 interrupt_data->interrupt_bar_index,
130 interrupt_data->interrupts[i].reg);
131 value &= mask;
132 value |= interrupt_data->interrupts[i].index
133 << pack_shift;
134 }
135 gasket_dev_write_64(gasket_dev, value,
136 interrupt_data->interrupt_bar_index,
137 interrupt_data->interrupts[i].reg);
138 }
139 }
140
141 static void
gasket_handle_interrupt(struct gasket_interrupt_data * interrupt_data,int interrupt_index)142 gasket_handle_interrupt(struct gasket_interrupt_data *interrupt_data,
143 int interrupt_index)
144 {
145 struct eventfd_ctx *ctx;
146
147 trace_gasket_interrupt_event(interrupt_data->name, interrupt_index);
148 ctx = interrupt_data->eventfd_ctxs[interrupt_index];
149 if (ctx)
150 eventfd_signal(ctx, 1);
151
152 ++(interrupt_data->interrupt_counts[interrupt_index]);
153 }
154
gasket_msix_interrupt_handler(int irq,void * dev_id)155 static irqreturn_t gasket_msix_interrupt_handler(int irq, void *dev_id)
156 {
157 struct gasket_interrupt_data *interrupt_data = dev_id;
158 int interrupt = -1;
159 int i;
160
161 /* If this linear lookup is a problem, we can maintain a map/hash. */
162 for (i = 0; i < interrupt_data->num_interrupts; i++) {
163 if (interrupt_data->msix_entries[i].vector == irq) {
164 interrupt = interrupt_data->msix_entries[i].entry;
165 break;
166 }
167 }
168 if (interrupt == -1) {
169 pr_err("Received unknown irq %d\n", irq);
170 return IRQ_HANDLED;
171 }
172 gasket_handle_interrupt(interrupt_data, interrupt);
173 return IRQ_HANDLED;
174 }
175
176 static int
gasket_interrupt_msix_init(struct gasket_interrupt_data * interrupt_data)177 gasket_interrupt_msix_init(struct gasket_interrupt_data *interrupt_data)
178 {
179 int ret = 1;
180 int i;
181
182 interrupt_data->msix_entries =
183 kcalloc(interrupt_data->num_interrupts,
184 sizeof(*interrupt_data->msix_entries), GFP_KERNEL);
185 if (!interrupt_data->msix_entries)
186 return -ENOMEM;
187
188 for (i = 0; i < interrupt_data->num_interrupts; i++) {
189 interrupt_data->msix_entries[i].entry = i;
190 interrupt_data->msix_entries[i].vector = 0;
191 interrupt_data->eventfd_ctxs[i] = NULL;
192 }
193
194 /* Retry MSIX_RETRY_COUNT times if not enough IRQs are available. */
195 for (i = 0; i < MSIX_RETRY_COUNT && ret > 0; i++)
196 ret = pci_enable_msix_exact(interrupt_data->pci_dev,
197 interrupt_data->msix_entries,
198 interrupt_data->num_interrupts);
199
200 if (ret)
201 return ret > 0 ? -EBUSY : ret;
202 interrupt_data->msix_configured = 1;
203
204 for (i = 0; i < interrupt_data->num_interrupts; i++) {
205 ret = request_irq(interrupt_data->msix_entries[i].vector,
206 gasket_msix_interrupt_handler, 0,
207 interrupt_data->name, interrupt_data);
208
209 if (ret) {
210 dev_err(&interrupt_data->pci_dev->dev,
211 "Cannot get IRQ for interrupt %d, vector %d; "
212 "%d\n",
213 i, interrupt_data->msix_entries[i].vector, ret);
214 return ret;
215 }
216
217 interrupt_data->num_configured++;
218 }
219
220 return 0;
221 }
222
223 /*
224 * On QCM DragonBoard, we exit gasket_interrupt_msix_init() and kernel interrupt
225 * setup code with MSIX vectors masked. This is wrong because nothing else in
226 * the driver will normally touch the MSIX vectors.
227 *
228 * As a temporary hack, force unmasking there.
229 *
230 * TODO: Figure out why QCM kernel doesn't unmask the MSIX vectors, after
231 * gasket_interrupt_msix_init(), and remove this code.
232 */
force_msix_interrupt_unmasking(struct gasket_dev * gasket_dev)233 static void force_msix_interrupt_unmasking(struct gasket_dev *gasket_dev)
234 {
235 int i;
236 #define MSIX_VECTOR_SIZE 16
237 #define MSIX_MASK_BIT_OFFSET 12
238 #define APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE 0x46800
239 for (i = 0; i < gasket_dev->interrupt_data->num_configured; i++) {
240 /* Check if the MSIX vector is unmasked */
241 ulong location = APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE +
242 MSIX_MASK_BIT_OFFSET + i * MSIX_VECTOR_SIZE;
243 u32 mask =
244 gasket_dev_read_32(gasket_dev,
245 gasket_dev->interrupt_data->interrupt_bar_index,
246 location);
247 if (!(mask & 1))
248 continue;
249 /* Unmask the msix vector (clear 32 bits) */
250 gasket_dev_write_32(gasket_dev, 0,
251 gasket_dev->interrupt_data->interrupt_bar_index,
252 location);
253 }
254 #undef MSIX_VECTOR_SIZE
255 #undef MSIX_MASK_BIT_OFFSET
256 #undef APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE
257 }
258
interrupt_sysfs_show(struct device * device,struct device_attribute * attr,char * buf)259 static ssize_t interrupt_sysfs_show(struct device *device,
260 struct device_attribute *attr, char *buf)
261 {
262 int i, ret;
263 ssize_t written = 0, total_written = 0;
264 struct gasket_interrupt_data *interrupt_data;
265 struct gasket_dev *gasket_dev;
266 struct gasket_sysfs_attribute *gasket_attr;
267 enum interrupt_sysfs_attribute_type sysfs_type;
268
269 gasket_dev = gasket_sysfs_get_device_data(device);
270 if (!gasket_dev) {
271 dev_dbg(device, "No sysfs mapping found for device\n");
272 return 0;
273 }
274
275 gasket_attr = gasket_sysfs_get_attr(device, attr);
276 if (!gasket_attr) {
277 dev_dbg(device, "No sysfs attr data found for device\n");
278 gasket_sysfs_put_device_data(device, gasket_dev);
279 return 0;
280 }
281
282 sysfs_type = (enum interrupt_sysfs_attribute_type)
283 gasket_attr->data.attr_type;
284 interrupt_data = gasket_dev->interrupt_data;
285 switch (sysfs_type) {
286 case ATTR_INTERRUPT_COUNTS:
287 for (i = 0; i < interrupt_data->num_interrupts; ++i) {
288 written =
289 scnprintf(buf, PAGE_SIZE - total_written,
290 "0x%02x: %ld\n", i,
291 interrupt_data->interrupt_counts[i]);
292 total_written += written;
293 buf += written;
294 }
295 ret = total_written;
296 break;
297 default:
298 dev_dbg(gasket_dev->dev, "Unknown attribute: %s\n",
299 attr->attr.name);
300 ret = 0;
301 break;
302 }
303
304 gasket_sysfs_put_attr(device, gasket_attr);
305 gasket_sysfs_put_device_data(device, gasket_dev);
306 return ret;
307 }
308
309 static struct gasket_sysfs_attribute interrupt_sysfs_attrs[] = {
310 GASKET_SYSFS_RO(interrupt_counts, interrupt_sysfs_show,
311 ATTR_INTERRUPT_COUNTS),
312 GASKET_END_OF_ATTR_ARRAY,
313 };
314
gasket_interrupt_init(struct gasket_dev * gasket_dev)315 int gasket_interrupt_init(struct gasket_dev *gasket_dev)
316 {
317 int ret;
318 struct gasket_interrupt_data *interrupt_data;
319 const struct gasket_driver_desc *driver_desc =
320 gasket_get_driver_desc(gasket_dev);
321
322 interrupt_data = kzalloc(sizeof(*interrupt_data), GFP_KERNEL);
323 if (!interrupt_data)
324 return -ENOMEM;
325 gasket_dev->interrupt_data = interrupt_data;
326 interrupt_data->name = driver_desc->name;
327 interrupt_data->type = driver_desc->interrupt_type;
328 interrupt_data->pci_dev = gasket_dev->pci_dev;
329 interrupt_data->num_interrupts = driver_desc->num_interrupts;
330 interrupt_data->interrupts = driver_desc->interrupts;
331 interrupt_data->interrupt_bar_index = driver_desc->interrupt_bar_index;
332 interrupt_data->pack_width = driver_desc->interrupt_pack_width;
333 interrupt_data->num_configured = 0;
334
335 interrupt_data->eventfd_ctxs =
336 kcalloc(driver_desc->num_interrupts,
337 sizeof(*interrupt_data->eventfd_ctxs), GFP_KERNEL);
338 if (!interrupt_data->eventfd_ctxs) {
339 kfree(interrupt_data);
340 return -ENOMEM;
341 }
342
343 interrupt_data->interrupt_counts =
344 kcalloc(driver_desc->num_interrupts,
345 sizeof(*interrupt_data->interrupt_counts), GFP_KERNEL);
346 if (!interrupt_data->interrupt_counts) {
347 kfree(interrupt_data->eventfd_ctxs);
348 kfree(interrupt_data);
349 return -ENOMEM;
350 }
351
352 switch (interrupt_data->type) {
353 case PCI_MSIX:
354 ret = gasket_interrupt_msix_init(interrupt_data);
355 if (ret)
356 break;
357 force_msix_interrupt_unmasking(gasket_dev);
358 break;
359
360 default:
361 ret = -EINVAL;
362 }
363
364 if (ret) {
365 /* Failing to setup interrupts will cause the device to report
366 * GASKET_STATUS_LAMED. But it is not fatal.
367 */
368 dev_warn(gasket_dev->dev,
369 "Couldn't initialize interrupts: %d\n", ret);
370 return 0;
371 }
372
373 gasket_interrupt_setup(gasket_dev);
374 gasket_sysfs_create_entries(gasket_dev->dev_info.device,
375 interrupt_sysfs_attrs);
376
377 return 0;
378 }
379
380 static void
gasket_interrupt_msix_cleanup(struct gasket_interrupt_data * interrupt_data)381 gasket_interrupt_msix_cleanup(struct gasket_interrupt_data *interrupt_data)
382 {
383 int i;
384
385 for (i = 0; i < interrupt_data->num_configured; i++)
386 free_irq(interrupt_data->msix_entries[i].vector,
387 interrupt_data);
388 interrupt_data->num_configured = 0;
389
390 if (interrupt_data->msix_configured)
391 pci_disable_msix(interrupt_data->pci_dev);
392 interrupt_data->msix_configured = 0;
393 kfree(interrupt_data->msix_entries);
394 }
395
gasket_interrupt_reinit(struct gasket_dev * gasket_dev)396 int gasket_interrupt_reinit(struct gasket_dev *gasket_dev)
397 {
398 int ret;
399
400 if (!gasket_dev->interrupt_data) {
401 dev_dbg(gasket_dev->dev,
402 "Attempted to reinit uninitialized interrupt data\n");
403 return -EINVAL;
404 }
405
406 switch (gasket_dev->interrupt_data->type) {
407 case PCI_MSIX:
408 gasket_interrupt_msix_cleanup(gasket_dev->interrupt_data);
409 ret = gasket_interrupt_msix_init(gasket_dev->interrupt_data);
410 if (ret)
411 break;
412 force_msix_interrupt_unmasking(gasket_dev);
413 break;
414
415 default:
416 ret = -EINVAL;
417 }
418
419 if (ret) {
420 /* Failing to setup interrupts will cause the device
421 * to report GASKET_STATUS_LAMED, but is not fatal.
422 */
423 dev_warn(gasket_dev->dev, "Couldn't reinit interrupts: %d\n",
424 ret);
425 return 0;
426 }
427
428 gasket_interrupt_setup(gasket_dev);
429
430 return 0;
431 }
432
433 /* See gasket_interrupt.h for description. */
gasket_interrupt_reset_counts(struct gasket_dev * gasket_dev)434 int gasket_interrupt_reset_counts(struct gasket_dev *gasket_dev)
435 {
436 dev_dbg(gasket_dev->dev, "Clearing interrupt counts\n");
437 memset(gasket_dev->interrupt_data->interrupt_counts, 0,
438 gasket_dev->interrupt_data->num_interrupts *
439 sizeof(*gasket_dev->interrupt_data->interrupt_counts));
440 return 0;
441 }
442
443 /* See gasket_interrupt.h for description. */
gasket_interrupt_cleanup(struct gasket_dev * gasket_dev)444 void gasket_interrupt_cleanup(struct gasket_dev *gasket_dev)
445 {
446 struct gasket_interrupt_data *interrupt_data =
447 gasket_dev->interrupt_data;
448 /*
449 * It is possible to get an error code from gasket_interrupt_init
450 * before interrupt_data has been allocated, so check it.
451 */
452 if (!interrupt_data)
453 return;
454
455 switch (interrupt_data->type) {
456 case PCI_MSIX:
457 gasket_interrupt_msix_cleanup(interrupt_data);
458 break;
459
460 default:
461 break;
462 }
463
464 kfree(interrupt_data->interrupt_counts);
465 kfree(interrupt_data->eventfd_ctxs);
466 kfree(interrupt_data);
467 gasket_dev->interrupt_data = NULL;
468 }
469
gasket_interrupt_system_status(struct gasket_dev * gasket_dev)470 int gasket_interrupt_system_status(struct gasket_dev *gasket_dev)
471 {
472 if (!gasket_dev->interrupt_data) {
473 dev_dbg(gasket_dev->dev, "Interrupt data is null\n");
474 return GASKET_STATUS_DEAD;
475 }
476
477 if (gasket_dev->interrupt_data->num_configured !=
478 gasket_dev->interrupt_data->num_interrupts) {
479 dev_dbg(gasket_dev->dev,
480 "Not all interrupts were configured\n");
481 return GASKET_STATUS_LAMED;
482 }
483
484 return GASKET_STATUS_ALIVE;
485 }
486
gasket_interrupt_set_eventfd(struct gasket_interrupt_data * interrupt_data,int interrupt,int event_fd)487 int gasket_interrupt_set_eventfd(struct gasket_interrupt_data *interrupt_data,
488 int interrupt, int event_fd)
489 {
490 struct eventfd_ctx *ctx = eventfd_ctx_fdget(event_fd);
491
492 if (IS_ERR(ctx))
493 return PTR_ERR(ctx);
494
495 if (interrupt < 0 || interrupt >= interrupt_data->num_interrupts)
496 return -EINVAL;
497
498 interrupt_data->eventfd_ctxs[interrupt] = ctx;
499 return 0;
500 }
501
gasket_interrupt_clear_eventfd(struct gasket_interrupt_data * interrupt_data,int interrupt)502 int gasket_interrupt_clear_eventfd(struct gasket_interrupt_data *interrupt_data,
503 int interrupt)
504 {
505 if (interrupt < 0 || interrupt >= interrupt_data->num_interrupts)
506 return -EINVAL;
507
508 interrupt_data->eventfd_ctxs[interrupt] = NULL;
509 return 0;
510 }
511
