1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (C) 2018 Google, Inc. */
3
4 #include "gasket_interrupt.h"
5
6 #include "gasket_constants.h"
7 #include "gasket_core.h"
8 #include "gasket_sysfs.h"
9 #include <linux/device.h>
10 #include <linux/interrupt.h>
11 #include <linux/printk.h>
12 #include <linux/version.h>
13 #ifdef GASKET_KERNEL_TRACE_SUPPORT
14 #define CREATE_TRACE_POINTS
15 #include <trace/events/gasket_interrupt.h>
16 #else
17 #define trace_gasket_interrupt_event(x, ...)
18 #endif
19 /* Retry attempts if the requested number of interrupts aren't available. */
20 #define MSIX_RETRY_COUNT 3
21
22 /* Instance interrupt management data. */
23 struct gasket_interrupt_data {
24 /* The name associated with this interrupt data. */
25 const char *name;
26
27 /* Interrupt type. See gasket_interrupt_type in gasket_core.h */
28 int type;
29
30 /* The PCI device [if any] associated with the owning device. */
31 struct pci_dev *pci_dev;
32
33 /* Set to 1 if MSI-X has successfully been configred, 0 otherwise. */
34 int msix_configured;
35
36 /* The number of interrupts requested by the owning device. */
37 int num_interrupts;
38
39 /* A pointer to the interrupt descriptor struct for this device. */
40 const struct gasket_interrupt_desc *interrupts;
41
42 /* The index of the bar into which interrupts should be mapped. */
43 int interrupt_bar_index;
44
45 /* The width of a single interrupt in a packed interrupt register. */
46 int pack_width;
47
48 /* offset of wire interrupt registers */
49 const struct gasket_wire_interrupt_offsets *wire_interrupt_offsets;
50
51 /*
52 * Design-wise, these elements should be bundled together, but
53 * pci_enable_msix's interface requires that they be managed
54 * individually (requires array of struct msix_entry).
55 */
56
57 /* The number of successfully configured interrupts. */
58 int num_configured;
59
60 /* The MSI-X data for each requested/configured interrupt. */
61 struct msix_entry *msix_entries;
62
63 /* The eventfd "callback" data for each interrupt. */
64 struct eventfd_ctx **eventfd_ctxs;
65
66 /* The number of times each interrupt has been called. */
67 ulong *interrupt_counts;
68
69 /* Linux IRQ number. */
70 int irq;
71 };
72
73 /* Structures to display interrupt counts in sysfs. */
74 enum interrupt_sysfs_attribute_type {
75 ATTR_INTERRUPT_COUNTS,
76 };
77
78 /* Set up device registers for interrupt handling. */
gasket_interrupt_setup(struct gasket_dev * gasket_dev)79 static void gasket_interrupt_setup(struct gasket_dev *gasket_dev)
80 {
81 int i;
82 int pack_shift;
83 ulong mask;
84 ulong value;
85 struct gasket_interrupt_data *interrupt_data =
86 gasket_dev->interrupt_data;
87
88 if (!interrupt_data) {
89 dev_dbg(gasket_dev->dev, "Interrupt data is not initialized\n");
90 return;
91 }
92
93 dev_dbg(gasket_dev->dev, "Running interrupt setup\n");
94
95 if (interrupt_data->type == PLATFORM_WIRE ||
96 interrupt_data->type == PCI_MSI) {
97 /* Nothing needs to be done for platform or PCI devices. */
98 return;
99 }
100
101 if (interrupt_data->type != PCI_MSIX) {
102 dev_dbg(gasket_dev->dev,
103 "Cannot handle unsupported interrupt type %d\n",
104 interrupt_data->type);
105 return;
106 }
107
108 /* Setup the MSIX table. */
109
110 for (i = 0; i < interrupt_data->num_interrupts; i++) {
111 /*
112 * If the interrupt is not packed, we can write the index into
113 * the register directly. If not, we need to deal with a read-
114 * modify-write and shift based on the packing index.
115 */
116 dev_dbg(gasket_dev->dev,
117 "Setting up interrupt index %d with index 0x%llx and "
118 "packing %d\n",
119 interrupt_data->interrupts[i].index,
120 interrupt_data->interrupts[i].reg,
121 interrupt_data->interrupts[i].packing);
122 if (interrupt_data->interrupts[i].packing == UNPACKED) {
123 value = interrupt_data->interrupts[i].index;
124 } else {
125 switch (interrupt_data->interrupts[i].packing) {
126 case PACK_0:
127 pack_shift = 0;
128 break;
129 case PACK_1:
130 pack_shift = interrupt_data->pack_width;
131 break;
132 case PACK_2:
133 pack_shift = 2 * interrupt_data->pack_width;
134 break;
135 case PACK_3:
136 pack_shift = 3 * interrupt_data->pack_width;
137 break;
138 default:
139 dev_dbg(gasket_dev->dev,
140 "Found interrupt description with "
141 "unknown enum %d\n",
142 interrupt_data->interrupts[i].packing);
143 return;
144 }
145
146 mask = ~(0xFFFF << pack_shift);
147 value = gasket_dev_read_64(gasket_dev,
148 interrupt_data->interrupt_bar_index,
149 interrupt_data->interrupts[i].reg);
150 value &= mask;
151 value |= interrupt_data->interrupts[i].index
152 << pack_shift;
153 }
154 gasket_dev_write_64(gasket_dev, value,
155 interrupt_data->interrupt_bar_index,
156 interrupt_data->interrupts[i].reg);
157 }
158 }
159
gasket_msix_interrupt_handler(int irq,void * dev_id)160 static irqreturn_t gasket_msix_interrupt_handler(int irq, void *dev_id)
161 {
162 struct eventfd_ctx *ctx;
163 struct gasket_interrupt_data *interrupt_data = dev_id;
164 int interrupt = -1;
165 int i;
166
167 /* If this linear lookup is a problem, we can maintain a map/hash. */
168 for (i = 0; i < interrupt_data->num_interrupts; i++) {
169 if (interrupt_data->msix_entries[i].vector == irq) {
170 interrupt = interrupt_data->msix_entries[i].entry;
171 break;
172 }
173 }
174 if (interrupt == -1) {
175 pr_err("Received unknown irq %d\n", irq);
176 return IRQ_HANDLED;
177 }
178 trace_gasket_interrupt_event(interrupt_data->name, interrupt);
179
180 ctx = interrupt_data->eventfd_ctxs[interrupt];
181 if (ctx)
182 eventfd_signal(ctx, 1);
183
184 ++(interrupt_data->interrupt_counts[interrupt]);
185
186 return IRQ_HANDLED;
187 }
188
189 static int
gasket_interrupt_msix_init(struct gasket_interrupt_data * interrupt_data)190 gasket_interrupt_msix_init(struct gasket_interrupt_data *interrupt_data)
191 {
192 int ret = 1;
193 int i;
194
195 for (i = 0; i < interrupt_data->num_interrupts; i++) {
196 interrupt_data->msix_entries[i].entry = i;
197 interrupt_data->msix_entries[i].vector = 0;
198 interrupt_data->eventfd_ctxs[i] = NULL;
199 }
200
201 /* Retry MSIX_RETRY_COUNT times if not enough IRQs are available. */
202 for (i = 0; i < MSIX_RETRY_COUNT && ret > 0; i++)
203 ret = pci_enable_msix_exact(interrupt_data->pci_dev,
204 interrupt_data->msix_entries,
205 interrupt_data->num_interrupts);
206
207 if (ret)
208 return ret > 0 ? -EBUSY : ret;
209 interrupt_data->msix_configured = 1;
210
211 for (i = 0; i < interrupt_data->num_interrupts; i++) {
212 ret = request_irq(interrupt_data->msix_entries[i].vector,
213 gasket_msix_interrupt_handler, 0,
214 interrupt_data->name, interrupt_data);
215
216 if (ret) {
217 dev_err(&interrupt_data->pci_dev->dev,
218 "Cannot get IRQ for interrupt %d, vector %d; "
219 "%d\n",
220 i, interrupt_data->msix_entries[i].vector, ret);
221 return ret;
222 }
223
224 interrupt_data->num_configured++;
225 }
226
227 return 0;
228 }
229
230 /*
231 * On QCM DragonBoard, we exit gasket_interrupt_msix_init() and kernel interrupt
232 * setup code with MSIX vectors masked. This is wrong because nothing else in
233 * the driver will normally touch the MSIX vectors.
234 *
235 * As a temporary hack, force unmasking there.
236 *
237 * TODO: Figure out why QCM kernel doesn't unmask the MSIX vectors, after
238 * gasket_interrupt_msix_init(), and remove this code.
239 */
force_msix_interrupt_unmasking(struct gasket_dev * gasket_dev)240 static void force_msix_interrupt_unmasking(struct gasket_dev *gasket_dev)
241 {
242 int i;
243 #define MSIX_VECTOR_SIZE 16
244 #define MSIX_MASK_BIT_OFFSET 12
245 #define APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE 0x46800
246 for (i = 0; i < gasket_dev->interrupt_data->num_configured; i++) {
247 /* Check if the MSIX vector is unmasked */
248 ulong location = APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE +
249 MSIX_MASK_BIT_OFFSET + i * MSIX_VECTOR_SIZE;
250 u32 mask =
251 gasket_dev_read_32(gasket_dev,
252 gasket_dev->interrupt_data->interrupt_bar_index,
253 location);
254 if (!(mask & 1))
255 continue;
256 /* Unmask the msix vector (clear 32 bits) */
257 gasket_dev_write_32(gasket_dev, 0,
258 gasket_dev->interrupt_data->interrupt_bar_index,
259 location);
260 }
261 #undef MSIX_VECTOR_SIZE
262 #undef MSIX_MASK_BIT_OFFSET
263 #undef APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE
264 }
265
interrupt_sysfs_show(struct device * device,struct device_attribute * attr,char * buf)266 static ssize_t interrupt_sysfs_show(struct device *device,
267 struct device_attribute *attr, char *buf)
268 {
269 int i, ret;
270 ssize_t written = 0, total_written = 0;
271 struct gasket_interrupt_data *interrupt_data;
272 struct gasket_dev *gasket_dev;
273 struct gasket_sysfs_attribute *gasket_attr;
274 enum interrupt_sysfs_attribute_type sysfs_type;
275
276 gasket_dev = gasket_sysfs_get_device_data(device);
277 if (!gasket_dev) {
278 dev_dbg(device, "No sysfs mapping found for device\n");
279 return 0;
280 }
281
282 gasket_attr = gasket_sysfs_get_attr(device, attr);
283 if (!gasket_attr) {
284 dev_dbg(device, "No sysfs attr data found for device\n");
285 gasket_sysfs_put_device_data(device, gasket_dev);
286 return 0;
287 }
288
289 sysfs_type = (enum interrupt_sysfs_attribute_type)
290 gasket_attr->data.attr_type;
291 interrupt_data = gasket_dev->interrupt_data;
292 switch (sysfs_type) {
293 case ATTR_INTERRUPT_COUNTS:
294 for (i = 0; i < interrupt_data->num_interrupts; ++i) {
295 written =
296 scnprintf(buf, PAGE_SIZE - total_written,
297 "0x%02x: %ld\n", i,
298 interrupt_data->interrupt_counts[i]);
299 total_written += written;
300 buf += written;
301 }
302 ret = total_written;
303 break;
304 default:
305 dev_dbg(gasket_dev->dev, "Unknown attribute: %s\n",
306 attr->attr.name);
307 ret = 0;
308 break;
309 }
310
311 gasket_sysfs_put_attr(device, gasket_attr);
312 gasket_sysfs_put_device_data(device, gasket_dev);
313 return ret;
314 }
315
316 static struct gasket_sysfs_attribute interrupt_sysfs_attrs[] = {
317 GASKET_SYSFS_RO(interrupt_counts, interrupt_sysfs_show,
318 ATTR_INTERRUPT_COUNTS),
319 GASKET_END_OF_ATTR_ARRAY,
320 };
321
gasket_interrupt_init(struct gasket_dev * gasket_dev,const char * name,int type,const struct gasket_interrupt_desc * interrupts,int num_interrupts,int pack_width,int bar_index,const struct gasket_wire_interrupt_offsets * wire_int_offsets)322 int gasket_interrupt_init(struct gasket_dev *gasket_dev, const char *name,
323 int type,
324 const struct gasket_interrupt_desc *interrupts,
325 int num_interrupts, int pack_width, int bar_index,
326 const struct gasket_wire_interrupt_offsets *wire_int_offsets)
327 {
328 int ret;
329 struct gasket_interrupt_data *interrupt_data;
330
331 interrupt_data = kzalloc(sizeof(struct gasket_interrupt_data),
332 GFP_KERNEL);
333 if (!interrupt_data)
334 return -ENOMEM;
335 gasket_dev->interrupt_data = interrupt_data;
336 interrupt_data->name = name;
337 interrupt_data->type = type;
338 interrupt_data->pci_dev = gasket_dev->pci_dev;
339 interrupt_data->num_interrupts = num_interrupts;
340 interrupt_data->interrupts = interrupts;
341 interrupt_data->interrupt_bar_index = bar_index;
342 interrupt_data->pack_width = pack_width;
343 interrupt_data->num_configured = 0;
344 interrupt_data->wire_interrupt_offsets = wire_int_offsets;
345
346 /* Allocate all dynamic structures. */
347 interrupt_data->msix_entries = kcalloc(num_interrupts,
348 sizeof(struct msix_entry),
349 GFP_KERNEL);
350 if (!interrupt_data->msix_entries) {
351 kfree(interrupt_data);
352 return -ENOMEM;
353 }
354
355 interrupt_data->eventfd_ctxs = kcalloc(num_interrupts,
356 sizeof(struct eventfd_ctx *),
357 GFP_KERNEL);
358 if (!interrupt_data->eventfd_ctxs) {
359 kfree(interrupt_data->msix_entries);
360 kfree(interrupt_data);
361 return -ENOMEM;
362 }
363
364 interrupt_data->interrupt_counts = kcalloc(num_interrupts,
365 sizeof(ulong),
366 GFP_KERNEL);
367 if (!interrupt_data->interrupt_counts) {
368 kfree(interrupt_data->eventfd_ctxs);
369 kfree(interrupt_data->msix_entries);
370 kfree(interrupt_data);
371 return -ENOMEM;
372 }
373
374 switch (interrupt_data->type) {
375 case PCI_MSIX:
376 ret = gasket_interrupt_msix_init(interrupt_data);
377 if (ret)
378 break;
379 force_msix_interrupt_unmasking(gasket_dev);
380 break;
381
382 case PCI_MSI:
383 case PLATFORM_WIRE:
384 default:
385 dev_err(gasket_dev->dev,
386 "Cannot handle unsupported interrupt type %d\n",
387 interrupt_data->type);
388 ret = -EINVAL;
389 }
390
391 if (ret) {
392 /* Failing to setup interrupts will cause the device to report
393 * GASKET_STATUS_LAMED. But it is not fatal.
394 */
395 dev_warn(gasket_dev->dev,
396 "Couldn't initialize interrupts: %d\n", ret);
397 return 0;
398 }
399
400 gasket_interrupt_setup(gasket_dev);
401 gasket_sysfs_create_entries(gasket_dev->dev_info.device,
402 interrupt_sysfs_attrs);
403
404 return 0;
405 }
406
407 static void
gasket_interrupt_msix_cleanup(struct gasket_interrupt_data * interrupt_data)408 gasket_interrupt_msix_cleanup(struct gasket_interrupt_data *interrupt_data)
409 {
410 int i;
411
412 for (i = 0; i < interrupt_data->num_configured; i++)
413 free_irq(interrupt_data->msix_entries[i].vector,
414 interrupt_data);
415 interrupt_data->num_configured = 0;
416
417 if (interrupt_data->msix_configured)
418 pci_disable_msix(interrupt_data->pci_dev);
419 interrupt_data->msix_configured = 0;
420 }
421
gasket_interrupt_reinit(struct gasket_dev * gasket_dev)422 int gasket_interrupt_reinit(struct gasket_dev *gasket_dev)
423 {
424 int ret;
425
426 if (!gasket_dev->interrupt_data) {
427 dev_dbg(gasket_dev->dev,
428 "Attempted to reinit uninitialized interrupt data\n");
429 return -EINVAL;
430 }
431
432 switch (gasket_dev->interrupt_data->type) {
433 case PCI_MSIX:
434 gasket_interrupt_msix_cleanup(gasket_dev->interrupt_data);
435 ret = gasket_interrupt_msix_init(gasket_dev->interrupt_data);
436 if (ret)
437 break;
438 force_msix_interrupt_unmasking(gasket_dev);
439 break;
440
441 case PCI_MSI:
442 case PLATFORM_WIRE:
443 default:
444 dev_dbg(gasket_dev->dev,
445 "Cannot handle unsupported interrupt type %d\n",
446 gasket_dev->interrupt_data->type);
447 ret = -EINVAL;
448 }
449
450 if (ret) {
451 /* Failing to setup MSIx will cause the device
452 * to report GASKET_STATUS_LAMED, but is not fatal.
453 */
454 dev_warn(gasket_dev->dev, "Couldn't init msix: %d\n", ret);
455 return 0;
456 }
457
458 gasket_interrupt_setup(gasket_dev);
459
460 return 0;
461 }
462
463 /* See gasket_interrupt.h for description. */
gasket_interrupt_reset_counts(struct gasket_dev * gasket_dev)464 int gasket_interrupt_reset_counts(struct gasket_dev *gasket_dev)
465 {
466 dev_dbg(gasket_dev->dev, "Clearing interrupt counts\n");
467 memset(gasket_dev->interrupt_data->interrupt_counts, 0,
468 gasket_dev->interrupt_data->num_interrupts *
469 sizeof(*gasket_dev->interrupt_data->interrupt_counts));
470 return 0;
471 }
472
473 /* See gasket_interrupt.h for description. */
gasket_interrupt_cleanup(struct gasket_dev * gasket_dev)474 void gasket_interrupt_cleanup(struct gasket_dev *gasket_dev)
475 {
476 struct gasket_interrupt_data *interrupt_data =
477 gasket_dev->interrupt_data;
478 /*
479 * It is possible to get an error code from gasket_interrupt_init
480 * before interrupt_data has been allocated, so check it.
481 */
482 if (!interrupt_data)
483 return;
484
485 switch (interrupt_data->type) {
486 case PCI_MSIX:
487 gasket_interrupt_msix_cleanup(interrupt_data);
488 break;
489
490 case PCI_MSI:
491 case PLATFORM_WIRE:
492 default:
493 dev_dbg(gasket_dev->dev,
494 "Cannot handle unsupported interrupt type %d\n",
495 interrupt_data->type);
496 }
497
498 kfree(interrupt_data->interrupt_counts);
499 kfree(interrupt_data->eventfd_ctxs);
500 kfree(interrupt_data->msix_entries);
501 kfree(interrupt_data);
502 gasket_dev->interrupt_data = NULL;
503 }
504
gasket_interrupt_system_status(struct gasket_dev * gasket_dev)505 int gasket_interrupt_system_status(struct gasket_dev *gasket_dev)
506 {
507 if (!gasket_dev->interrupt_data) {
508 dev_dbg(gasket_dev->dev, "Interrupt data is null\n");
509 return GASKET_STATUS_DEAD;
510 }
511
512 if (!gasket_dev->interrupt_data->msix_configured) {
513 dev_dbg(gasket_dev->dev, "Interrupt not initialized\n");
514 return GASKET_STATUS_LAMED;
515 }
516
517 if (gasket_dev->interrupt_data->num_configured !=
518 gasket_dev->interrupt_data->num_interrupts) {
519 dev_dbg(gasket_dev->dev,
520 "Not all interrupts were configured\n");
521 return GASKET_STATUS_LAMED;
522 }
523
524 return GASKET_STATUS_ALIVE;
525 }
526
gasket_interrupt_set_eventfd(struct gasket_interrupt_data * interrupt_data,int interrupt,int event_fd)527 int gasket_interrupt_set_eventfd(struct gasket_interrupt_data *interrupt_data,
528 int interrupt, int event_fd)
529 {
530 struct eventfd_ctx *ctx = eventfd_ctx_fdget(event_fd);
531
532 if (IS_ERR(ctx))
533 return PTR_ERR(ctx);
534
535 if (interrupt < 0 || interrupt >= interrupt_data->num_interrupts)
536 return -EINVAL;
537
538 interrupt_data->eventfd_ctxs[interrupt] = ctx;
539 return 0;
540 }
541
gasket_interrupt_clear_eventfd(struct gasket_interrupt_data * interrupt_data,int interrupt)542 int gasket_interrupt_clear_eventfd(struct gasket_interrupt_data *interrupt_data,
543 int interrupt)
544 {
545 if (interrupt < 0 || interrupt >= interrupt_data->num_interrupts)
546 return -EINVAL;
547
548 interrupt_data->eventfd_ctxs[interrupt] = NULL;
549 return 0;
550 }
551
