1 /*
2 * Copyright (c) 1997-1998, 2000-2002, 2004, 2006-2008, 2011-2015 Wind River
3 * Systems, Inc.
4 *
5 * SPDX-License-Identifier: Apache-2.0
6 */
7
8 #define DT_DRV_COMPAT intel_ioapic
9
10 /**
11 * @file
12 * @brief Intel IO APIC/xAPIC driver
13 *
14 * This module is a driver for the IO APIC/xAPIC (Advanced Programmable
15 * Interrupt Controller) for P6 (PentiumPro, II, III) family processors
16 * and P7 (Pentium4) family processors. The IO APIC/xAPIC is included
17 * in the Intel's system chip set, such as ICH2. Software intervention
18 * may be required to enable the IO APIC/xAPIC in some chip sets.
19 * The 8259A interrupt controller is intended for use in a uni-processor
20 * system, IO APIC can be used in either a uni-processor or multi-processor
21 * system. The IO APIC handles interrupts very differently than the 8259A.
22 * Briefly, these differences are:
23 * - Method of Interrupt Transmission. The IO APIC transmits interrupts
24 * through a 3-wire bus and interrupts are handled without the need for
25 * the processor to run an interrupt acknowledge cycle.
26 * - Interrupt Priority. The priority of interrupts in the IO APIC is
27 * independent of the interrupt number. For example, interrupt 10 can
28 * be given a higher priority than interrupt 3.
29 * - More Interrupts. The IO APIC supports a total of 24 interrupts.
30 *
31 * The IO APIC unit consists of a set of interrupt input signals, a 24-entry
32 * by 64-bit Interrupt Redirection Table, programmable registers, and a message
33 * unit for sending and receiving APIC messages over the APIC bus or the
34 * Front-Side (system) bus. IO devices inject interrupts into the system by
35 * asserting one of the interrupt lines to the IO APIC. The IO APIC selects the
36 * corresponding entry in the Redirection Table and uses the information in that
37 * entry to format an interrupt request message. Each entry in the Redirection
38 * Table can be individually programmed to indicate edge/level sensitive interrupt
39 * signals, the interrupt vector and priority, the destination processor, and how
40 * the processor is selected (statically and dynamically). The information in
41 * the table is used to transmit a message to other APIC units (via the APIC bus
42 * or the Front-Side (system) bus). IO APIC is used in the Symmetric IO Mode.
43 * The base address of IO APIC is determined in loapic_init() and stored in the
44 * global variable ioApicBase and ioApicData.
45 * The lower 32 bit value of the redirection table entries for IRQ 0
46 * to 15 are edge triggered positive high, and for IRQ 16 to 23 are level
47 * triggered positive low.
48 *
49 * This implementation doesn't support multiple IO APICs.
50 *
51 * INCLUDE FILES: ioapic.h loapic.h
52 *
53 */
54
55 #include <zephyr/kernel.h>
56 #include <zephyr/arch/cpu.h>
57
58 #include <zephyr/toolchain.h>
59 #include <zephyr/linker/sections.h>
60 #include <zephyr/device.h>
61 #include <zephyr/pm/device.h>
62 #include <string.h>
63
64 #include <zephyr/drivers/interrupt_controller/ioapic.h> /* public API declarations */
65 #include <zephyr/drivers/interrupt_controller/loapic.h> /* public API declarations and registers */
66 #include "intc_ioapic_priv.h"
67
68 DEVICE_MMIO_TOPLEVEL_STATIC(ioapic_regs, DT_DRV_INST(0));
69
70 #define IOAPIC_REG DEVICE_MMIO_TOPLEVEL_GET(ioapic_regs)
71
72 /*
73 * Destination field (bits[56:63]) defines a set of processors, which is
74 * used to be compared with local LDR to determine which local APICs accept
75 * the interrupt.
76 *
77 * XAPIC: in logical destination mode and flat model (determined by DFR).
78 * LDR bits[24:31] can accommodate up to 8 logical APIC IDs.
79 *
80 * X2APIC: in logical destination mode and cluster model.
81 * In this case, LDR is read-only to system software and supports up to 16
82 * logical IDs. (Cluster ID: don't care to IO APIC).
83 *
84 * In either case, regardless how many CPUs in the system, 0xff implies that
85 * it's intended to deliver to all possible 8 local APICs.
86 */
87 #define DEFAULT_RTE_DEST (0xFF << 24)
88
89 static __pinned_bss uint32_t ioapic_rtes;
90
91 #ifdef CONFIG_PM_DEVICE
92
93 #define BITS_PER_IRQ 4
94 #define IOAPIC_BITFIELD_HI_LO 0
95 #define IOAPIC_BITFIELD_LVL_EDGE 1
96 #define IOAPIC_BITFIELD_ENBL_DSBL 2
97 #define IOAPIC_BITFIELD_DELIV_MODE 3
98
99 #define BIT_POS_FOR_IRQ_OPTION(irq, option) ((irq) * BITS_PER_IRQ + (option))
100
101 /* Allocating up to 256 irq bits bufffer for RTEs, RTEs are dynamically found
102 * so let's just assume the maximum, it's only 128 bytes in total.
103 */
104 #define SUSPEND_BITS_REQD (ROUND_UP((256 * BITS_PER_IRQ), 32))
105
106 __pinned_bss
107 uint32_t ioapic_suspend_buf[SUSPEND_BITS_REQD / 32] = {0};
108 #endif
109
110 static uint32_t __IoApicGet(int32_t offset);
111 static void __IoApicSet(int32_t offset, uint32_t value);
112 static void ioApicRedSetHi(unsigned int irq, uint32_t upper32);
113 static void ioApicRedSetLo(unsigned int irq, uint32_t lower32);
114 static uint32_t ioApicRedGetLo(unsigned int irq);
115 static void IoApicRedUpdateLo(unsigned int irq, uint32_t value,
116 uint32_t mask);
117
118 #if defined(CONFIG_INTEL_VTD_ICTL) && \
119 !defined(CONFIG_INTEL_VTD_ICTL_XAPIC_PASSTHROUGH)
120
121 #include <zephyr/drivers/interrupt_controller/intel_vtd.h>
122 #include <zephyr/acpi/acpi.h>
123
124 static const struct device *const vtd =
125 DEVICE_DT_GET_OR_NULL(DT_INST(0, intel_vt_d));
126 static uint16_t ioapic_id;
127
get_vtd(void)128 static bool get_vtd(void)
129 {
130 if (!device_is_ready(vtd)) {
131 return false;
132 }
133
134 if (ioapic_id != 0) {
135 return true;
136 }
137
138 return acpi_dmar_ioapic_get(&ioapic_id) == 0;
139 }
140 #endif /* CONFIG_INTEL_VTD_ICTL && !INTEL_VTD_ICTL_XAPIC_PASSTHROUGH */
141
142 /*
143 * The functions irq_enable() and irq_disable() are implemented in the
144 * interrupt controller driver due to the IRQ virtualization imposed by
145 * the x86 architecture.
146 */
147
148 /**
149 * @brief Initialize the IO APIC or xAPIC
150 *
151 * This routine initializes the IO APIC or xAPIC.
152 *
153 * @retval 0 on success.
154 */
155 __boot_func
ioapic_init(const struct device * unused)156 int ioapic_init(const struct device *unused)
157 {
158 ARG_UNUSED(unused);
159
160 DEVICE_MMIO_TOPLEVEL_MAP(ioapic_regs, K_MEM_CACHE_NONE);
161
162 /* Reading MRE: this will give the number of RTEs available */
163 ioapic_rtes = ((__IoApicGet(IOAPIC_VERS) &
164 IOAPIC_MRE_MASK) >> IOAPIC_MRE_POS) + 1;
165
166 #ifdef CONFIG_IOAPIC_MASK_RTE
167 int32_t ix; /* redirection table index */
168 uint32_t rteValue; /* value to copy into redirection table entry */
169
170 rteValue = IOAPIC_EDGE | IOAPIC_HIGH | IOAPIC_FIXED | IOAPIC_INT_MASK |
171 IOAPIC_LOGICAL | 0 /* dummy vector */;
172
173 for (ix = 0; ix < ioapic_rtes; ix++) {
174 ioApicRedSetHi(ix, DEFAULT_RTE_DEST);
175 ioApicRedSetLo(ix, rteValue);
176 }
177 #endif
178 return 0;
179 }
180
181 __pinned_func
z_ioapic_num_rtes(void)182 uint32_t z_ioapic_num_rtes(void)
183 {
184 return ioapic_rtes;
185 }
186
187 /**
188 * @brief Enable a specified APIC interrupt input line
189 *
190 * This routine enables a specified APIC interrupt input line.
191 *
192 * @param irq IRQ number to enable
193 */
194 __pinned_func
z_ioapic_irq_enable(unsigned int irq)195 void z_ioapic_irq_enable(unsigned int irq)
196 {
197 IoApicRedUpdateLo(irq, 0, IOAPIC_INT_MASK);
198 }
199
200 /**
201 * @brief Disable a specified APIC interrupt input line
202 *
203 * This routine disables a specified APIC interrupt input line.
204 * @param irq IRQ number to disable
205 */
206 __pinned_func
z_ioapic_irq_disable(unsigned int irq)207 void z_ioapic_irq_disable(unsigned int irq)
208 {
209 IoApicRedUpdateLo(irq, IOAPIC_INT_MASK, IOAPIC_INT_MASK);
210 }
211
212
213 #ifdef CONFIG_PM_DEVICE
214
215 __pinned_func
store_flags(unsigned int irq,uint32_t flags)216 void store_flags(unsigned int irq, uint32_t flags)
217 {
218 /* Currently only the following four flags are modified */
219 if (flags & IOAPIC_LOW) {
220 sys_bitfield_set_bit((mem_addr_t) ioapic_suspend_buf,
221 BIT_POS_FOR_IRQ_OPTION(irq, IOAPIC_BITFIELD_HI_LO));
222 }
223
224 if (flags & IOAPIC_LEVEL) {
225 sys_bitfield_set_bit((mem_addr_t) ioapic_suspend_buf,
226 BIT_POS_FOR_IRQ_OPTION(irq, IOAPIC_BITFIELD_LVL_EDGE));
227 }
228
229 if (flags & IOAPIC_INT_MASK) {
230 sys_bitfield_set_bit((mem_addr_t) ioapic_suspend_buf,
231 BIT_POS_FOR_IRQ_OPTION(irq, IOAPIC_BITFIELD_ENBL_DSBL));
232 }
233
234 /*
235 * We support lowest priority and fixed mode only, so only one bit
236 * needs to be saved.
237 */
238 if (flags & IOAPIC_LOWEST) {
239 sys_bitfield_set_bit((mem_addr_t) ioapic_suspend_buf,
240 BIT_POS_FOR_IRQ_OPTION(irq, IOAPIC_BITFIELD_DELIV_MODE));
241 }
242 }
243
244 __pinned_func
restore_flags(unsigned int irq)245 uint32_t restore_flags(unsigned int irq)
246 {
247 uint32_t flags = 0U;
248
249 if (sys_bitfield_test_bit((mem_addr_t) ioapic_suspend_buf,
250 BIT_POS_FOR_IRQ_OPTION(irq, IOAPIC_BITFIELD_HI_LO))) {
251 flags |= IOAPIC_LOW;
252 }
253
254 if (sys_bitfield_test_bit((mem_addr_t) ioapic_suspend_buf,
255 BIT_POS_FOR_IRQ_OPTION(irq, IOAPIC_BITFIELD_LVL_EDGE))) {
256 flags |= IOAPIC_LEVEL;
257 }
258
259 if (sys_bitfield_test_bit((mem_addr_t) ioapic_suspend_buf,
260 BIT_POS_FOR_IRQ_OPTION(irq, IOAPIC_BITFIELD_ENBL_DSBL))) {
261 flags |= IOAPIC_INT_MASK;
262 }
263
264 if (sys_bitfield_test_bit((mem_addr_t) ioapic_suspend_buf,
265 BIT_POS_FOR_IRQ_OPTION(irq, IOAPIC_BITFIELD_DELIV_MODE))) {
266 flags |= IOAPIC_LOWEST;
267 }
268
269 return flags;
270 }
271
272
273 __pinned_func
ioapic_suspend(const struct device * port)274 int ioapic_suspend(const struct device *port)
275 {
276 int irq;
277 uint32_t rte_lo;
278
279 ARG_UNUSED(port);
280 (void)memset(ioapic_suspend_buf, 0, (SUSPEND_BITS_REQD >> 3));
281 for (irq = 0; irq < ioapic_rtes; irq++) {
282 /*
283 * The following check is to figure out the registered
284 * IRQ lines, so as to limit ourselves to saving the
285 * flags for them only.
286 */
287 if (_irq_to_interrupt_vector[irq]) {
288 rte_lo = ioApicRedGetLo(irq);
289 store_flags(irq, rte_lo);
290 }
291 }
292 return 0;
293 }
294
295 __pinned_func
ioapic_resume_from_suspend(const struct device * port)296 int ioapic_resume_from_suspend(const struct device *port)
297 {
298 int irq;
299 uint32_t flags;
300 uint32_t rteValue;
301
302 ARG_UNUSED(port);
303
304 for (irq = 0; irq < ioapic_rtes; irq++) {
305 if (_irq_to_interrupt_vector[irq]) {
306 /* Get the saved flags */
307 flags = restore_flags(irq);
308 /* Appending the flags that are never modified */
309 flags = flags | IOAPIC_LOGICAL;
310
311 rteValue = (_irq_to_interrupt_vector[irq] &
312 IOAPIC_VEC_MASK) | flags;
313 } else {
314 /* Initialize the other RTEs to sane values */
315 rteValue = IOAPIC_EDGE | IOAPIC_HIGH |
316 IOAPIC_FIXED | IOAPIC_INT_MASK |
317 IOAPIC_LOGICAL | 0 ; /* dummy vector*/
318 }
319 ioApicRedSetHi(irq, DEFAULT_RTE_DEST);
320 ioApicRedSetLo(irq, rteValue);
321 }
322 return 0;
323 }
324
325 /*
326 * Implements the driver control management functionality
327 * the *context may include IN data or/and OUT data
328 */
329 __pinned_func
ioapic_pm_action(const struct device * dev,enum pm_device_action action)330 static int ioapic_pm_action(const struct device *dev,
331 enum pm_device_action action)
332 {
333 int ret = 0;
334
335 switch (action) {
336 case PM_DEVICE_ACTION_RESUME:
337 ret = ioapic_resume_from_suspend(dev);
338 break;
339 case PM_DEVICE_ACTION_SUSPEND:
340 ret = ioapic_suspend(dev);
341 break;
342 default:
343 ret = -ENOTSUP;
344 }
345
346 return ret;
347 }
348
349 #endif /*CONFIG_PM_DEVICE*/
350
351 /**
352 * @brief Programs the interrupt redirection table
353 *
354 * This routine sets up the redirection table entry for the specified IRQ
355 * @param irq Virtualized IRQ
356 * @param vector Vector number
357 * @param flags Interrupt flags
358 */
359 __boot_func
z_ioapic_irq_set(unsigned int irq,unsigned int vector,uint32_t flags)360 void z_ioapic_irq_set(unsigned int irq, unsigned int vector, uint32_t flags)
361 {
362 uint32_t rteValue; /* value to copy into redirection table entry */
363 #if defined(CONFIG_INTEL_VTD_ICTL) && \
364 !defined(CONFIG_INTEL_VTD_ICTL_XAPIC_PASSTHROUGH)
365 int irte_idx;
366
367 if (!get_vtd()) {
368 goto no_vtd;
369 }
370
371 irte_idx = vtd_get_irte_by_vector(vtd, vector);
372 if (irte_idx < 0) {
373 irte_idx = vtd_get_irte_by_irq(vtd, irq);
374 }
375
376 if (irte_idx >= 0 && !vtd_irte_is_msi(vtd, irte_idx)) {
377 /* Enable interrupt remapping format and set the irte index */
378 rteValue = IOAPIC_VTD_REMAP_FORMAT |
379 IOAPIC_VTD_INDEX(irte_idx);
380 ioApicRedSetHi(irq, rteValue);
381
382 /* Remapped: delivery mode is Fixed (000) and
383 * destination mode is no longer present as it is replaced by
384 * the 15th bit of irte index, which is always 0 in our case.
385 */
386 rteValue = IOAPIC_INT_MASK |
387 (vector & IOAPIC_VEC_MASK) |
388 (flags & IOAPIC_TRIGGER_MASK) |
389 (flags & IOAPIC_POLARITY_MASK);
390 ioApicRedSetLo(irq, rteValue);
391
392 vtd_remap(vtd, irte_idx, vector, flags, ioapic_id);
393 } else {
394 no_vtd:
395 #else
396 {
397 #endif /* CONFIG_INTEL_VTD_ICTL && !CONFIG_INTEL_VTD_ICTL_XAPIC_PASSTHROUGH */
398 /* the delivery mode is determined by the flags
399 * passed from drivers
400 */
401 rteValue = IOAPIC_INT_MASK | IOAPIC_LOGICAL |
402 (vector & IOAPIC_VEC_MASK) | flags;
403 ioApicRedSetHi(irq, DEFAULT_RTE_DEST);
404 ioApicRedSetLo(irq, rteValue);
405 }
406 }
407
408 /**
409 * @brief Program interrupt vector for specified irq
410 *
411 * The routine writes the interrupt vector in the Interrupt Redirection
412 * Table for specified irq number
413 *
414 * @param irq Interrupt number
415 * @param vector Vector number
416 */
417 __boot_func
418 void z_ioapic_int_vec_set(unsigned int irq, unsigned int vector)
419 {
420 IoApicRedUpdateLo(irq, vector, IOAPIC_VEC_MASK);
421 }
422
423 /**
424 * @brief Read a 32 bit IO APIC register
425 *
426 * This routine reads the specified IO APIC register using indirect addressing.
427 * @param offset Register offset (8 bits)
428 *
429 * @return register value
430 */
431 __pinned_func
432 static uint32_t __IoApicGet(int32_t offset)
433 {
434 uint32_t value; /* value */
435 unsigned int key; /* interrupt lock level */
436
437 /* lock interrupts to ensure indirect addressing works "atomically" */
438
439 key = irq_lock();
440
441 *((volatile uint32_t *) (IOAPIC_REG + IOAPIC_IND)) = (unsigned char)offset;
442 value = *((volatile uint32_t *)(IOAPIC_REG + IOAPIC_DATA));
443
444 irq_unlock(key);
445
446 return value;
447 }
448
449 /**
450 * @brief Write a 32 bit IO APIC register
451 *
452 * This routine writes the specified IO APIC register using indirect addressing.
453 *
454 * @param offset Register offset (8 bits)
455 * @param value Value to set the register
456 */
457 __pinned_func
458 static void __IoApicSet(int32_t offset, uint32_t value)
459 {
460 unsigned int key; /* interrupt lock level */
461
462 /* lock interrupts to ensure indirect addressing works "atomically" */
463
464 key = irq_lock();
465
466 *(volatile uint32_t *)(IOAPIC_REG + IOAPIC_IND) = (unsigned char)offset;
467 *((volatile uint32_t *)(IOAPIC_REG + IOAPIC_DATA)) = value;
468
469 irq_unlock(key);
470 }
471
472 /**
473 * @brief Get low 32 bits of Redirection Table entry
474 *
475 * This routine reads the low-order 32 bits of a Redirection Table entry.
476 *
477 * @param irq INTIN number
478 * @return 32 low-order bits
479 */
480 __pinned_func
481 static uint32_t ioApicRedGetLo(unsigned int irq)
482 {
483 int32_t offset = IOAPIC_REDTBL + (irq << 1); /* register offset */
484
485 return __IoApicGet(offset);
486 }
487
488 /**
489 * @brief Set low 32 bits of Redirection Table entry
490 *
491 * This routine writes the low-order 32 bits of a Redirection Table entry.
492 *
493 * @param irq INTIN number
494 * @param lower32 Value to be written
495 */
496 __pinned_func
497 static void ioApicRedSetLo(unsigned int irq, uint32_t lower32)
498 {
499 int32_t offset = IOAPIC_REDTBL + (irq << 1); /* register offset */
500
501 __IoApicSet(offset, lower32);
502 }
503
504 /**
505 * @brief Set high 32 bits of Redirection Table entry
506 *
507 * This routine writes the high-order 32 bits of a Redirection Table entry.
508 *
509 * @param irq INTIN number
510 * @param upper32 Value to be written
511 */
512 __pinned_func
513 static void ioApicRedSetHi(unsigned int irq, uint32_t upper32)
514 {
515 int32_t offset = IOAPIC_REDTBL + (irq << 1) + 1; /* register offset */
516
517 __IoApicSet(offset, upper32);
518 }
519
520 /**
521 * @brief Modify low 32 bits of Redirection Table entry
522 *
523 * This routine modifies selected portions of the low-order 32 bits of a
524 * Redirection Table entry, as indicated by the associate bit mask.
525 *
526 * @param irq INTIN number
527 * @param value Value to be written
528 * @param mask Mask of bits to be modified
529 */
530 __pinned_func
531 static void IoApicRedUpdateLo(unsigned int irq,
532 uint32_t value,
533 uint32_t mask)
534 {
535 ioApicRedSetLo(irq, (ioApicRedGetLo(irq) & ~mask) | (value & mask));
536 }
537
538 PM_DEVICE_DT_INST_DEFINE(0, ioapic_pm_action);
539
540 DEVICE_DT_INST_DEFINE(0, ioapic_init, PM_DEVICE_DT_INST_GET(0), NULL, NULL,
541 PRE_KERNEL_1, CONFIG_INTC_INIT_PRIORITY, NULL);
542
