1 /*
2 * Copyright (c) 1984-2008, 2011-2015 Wind River Systems, Inc.
3 * SPDX-License-Identifier: Apache-2.0
4 */
5
6 /*
7 * driver for x86 CPU local APIC (as an interrupt controller)
8 */
9
10 #include <kernel.h>
11 #include <kernel_structs.h>
12 #include <arch/cpu.h>
13 #include <zephyr/types.h>
14 #include <string.h>
15 #include <sys/__assert.h>
16 #include <arch/x86/msr.h>
17
18 #include <toolchain.h>
19 #include <linker/sections.h>
20 #include <drivers/interrupt_controller/loapic.h> /* public API declarations */
21 #include <device.h>
22 #include <drivers/interrupt_controller/sysapic.h>
23 #include <drivers/interrupt_controller/ioapic.h>
24
25 /* Local APIC Version Register Bits */
26
27 #define LOAPIC_VERSION_MASK 0x000000ff /* LO APIC Version mask */
28 #define LOAPIC_MAXLVT_MASK 0x00ff0000 /* LO APIC Max LVT mask */
29 #define LOAPIC_PENTIUM4 0x00000014 /* LO APIC in Pentium4 */
30 #define LOAPIC_LVT_PENTIUM4 5 /* LO APIC LVT - Pentium4 */
31 #define LOAPIC_LVT_P6 4 /* LO APIC LVT - P6 */
32 #define LOAPIC_LVT_P5 3 /* LO APIC LVT - P5 */
33
34 /* Local APIC Vector Table Bits */
35
36 #define LOAPIC_VECTOR 0x000000ff /* vectorNo */
37 #define LOAPIC_MODE 0x00000700 /* delivery mode */
38 #define LOAPIC_FIXED 0x00000000 /* delivery mode: FIXED */
39 #define LOAPIC_SMI 0x00000200 /* delivery mode: SMI */
40 #define LOAPIC_NMI 0x00000400 /* delivery mode: NMI */
41 #define LOAPIC_EXT 0x00000700 /* delivery mode: ExtINT */
42 #define LOAPIC_IDLE 0x00000000 /* delivery status: Idle */
43 #define LOAPIC_PEND 0x00001000 /* delivery status: Pend */
44 #define LOAPIC_HIGH 0x00000000 /* polarity: High */
45 #define LOAPIC_LOW 0x00002000 /* polarity: Low */
46 #define LOAPIC_REMOTE 0x00004000 /* remote IRR */
47 #define LOAPIC_EDGE 0x00000000 /* trigger mode: Edge */
48 #define LOAPIC_LEVEL 0x00008000 /* trigger mode: Level */
49
50 /* Local APIC Spurious-Interrupt Register Bits */
51
52 #define LOAPIC_ENABLE 0x100 /* APIC Enabled */
53 #define LOAPIC_FOCUS_DISABLE 0x200 /* Focus Processor Checking */
54
55 #if CONFIG_LOAPIC_SPURIOUS_VECTOR_ID == -1
56 #define LOAPIC_SPURIOUS_VECTOR_ID (CONFIG_IDT_NUM_VECTORS - 1)
57 #else
58 #define LOAPIC_SPURIOUS_VECTOR_ID CONFIG_LOAPIC_SPURIOUS_VECTOR_ID
59 #endif
60
61 #define LOPIC_SSPND_BITS_PER_IRQ 1 /* Just the one for enable disable*/
62 #define LOPIC_SUSPEND_BITS_REQD (ROUND_UP((LOAPIC_IRQ_COUNT * LOPIC_SSPND_BITS_PER_IRQ), 32))
63 #ifdef CONFIG_PM_DEVICE
64 #include <pm/device.h>
65 __pinned_bss
66 uint32_t loapic_suspend_buf[LOPIC_SUSPEND_BITS_REQD / 32] = {0};
67 #endif
68
69 #ifdef DEVICE_MMIO_IS_IN_RAM
70 __pinned_bss
71 mm_reg_t z_loapic_regs;
72 #endif
73
74 __pinned_func
send_eoi(void)75 void send_eoi(void)
76 {
77 x86_write_xapic(LOAPIC_EOI, 0);
78 }
79
80 /**
81 * @brief Enable and initialize the local APIC.
82 *
83 * Called from early assembly layer (e.g., crt0.S).
84 */
85 __pinned_func
z_loapic_enable(unsigned char cpu_number)86 void z_loapic_enable(unsigned char cpu_number)
87 {
88 int32_t loApicMaxLvt; /* local APIC Max LVT */
89
90 #ifdef DEVICE_MMIO_IS_IN_RAM
91 device_map(&z_loapic_regs, CONFIG_LOAPIC_BASE_ADDRESS, 0x1000,
92 K_MEM_CACHE_NONE);
93 #endif /* DEVICE_MMIO_IS_IN_RAM */
94 #ifndef CONFIG_X2APIC
95 /*
96 * in xAPIC and flat model, bits 24-31 in LDR (Logical APIC ID) are
97 * bitmap of target logical APIC ID and it supports maximum 8 local
98 * APICs.
99 *
100 * The logical APIC ID could be arbitrarily selected by system software
101 * and is different from local APIC ID in local APIC ID register.
102 *
103 * We choose 0 for BSP, and the index to x86_cpuboot[] for secondary
104 * CPUs.
105 *
106 * in X2APIC, LDR is read-only.
107 */
108 x86_write_xapic(LOAPIC_LDR, 1 << (cpu_number + 24));
109 #endif
110
111 /*
112 * enable the local APIC. note that we use xAPIC mode here, since
113 * x2APIC access is not enabled until the next step (if at all).
114 */
115
116 x86_write_xapic(LOAPIC_SVR,
117 x86_read_xapic(LOAPIC_SVR) | LOAPIC_ENABLE);
118
119 #ifdef CONFIG_X2APIC
120 /*
121 * turn on x2APIC mode. we trust the config option, so
122 * we don't check CPUID to see if x2APIC is supported.
123 */
124
125 uint64_t msr = z_x86_msr_read(X86_APIC_BASE_MSR);
126 msr |= X86_APIC_BASE_MSR_X2APIC;
127 z_x86_msr_write(X86_APIC_BASE_MSR, msr);
128 #endif
129
130 loApicMaxLvt = (x86_read_loapic(LOAPIC_VER) & LOAPIC_MAXLVT_MASK) >> 16;
131
132 /* reset the DFR, TPR, TIMER_CONFIG, and TIMER_ICR */
133
134 #ifndef CONFIG_X2APIC
135 /* Flat model */
136 x86_write_loapic(LOAPIC_DFR, 0xffffffff); /* no DFR in x2APIC mode */
137 #endif
138
139 x86_write_loapic(LOAPIC_TPR, 0x0);
140 x86_write_loapic(LOAPIC_TIMER_CONFIG, 0x0);
141 x86_write_loapic(LOAPIC_TIMER_ICR, 0x0);
142
143 /* program Local Vector Table for the Virtual Wire Mode */
144
145 /* skip LINT0/LINT1 for Jailhouse guest case, because we won't
146 * ever be waiting for interrupts on those
147 */
148 /* set LINT0: extInt, high-polarity, edge-trigger, not-masked */
149
150 x86_write_loapic(LOAPIC_LINT0, (x86_read_loapic(LOAPIC_LINT0) &
151 ~(LOAPIC_MODE | LOAPIC_LOW |
152 LOAPIC_LEVEL | LOAPIC_LVT_MASKED)) |
153 (LOAPIC_EXT | LOAPIC_HIGH | LOAPIC_EDGE));
154
155 /* set LINT1: NMI, high-polarity, edge-trigger, not-masked */
156
157 x86_write_loapic(LOAPIC_LINT1, (x86_read_loapic(LOAPIC_LINT1) &
158 ~(LOAPIC_MODE | LOAPIC_LOW |
159 LOAPIC_LEVEL | LOAPIC_LVT_MASKED)) |
160 (LOAPIC_NMI | LOAPIC_HIGH | LOAPIC_EDGE));
161
162 /* lock the Local APIC interrupts */
163
164 x86_write_loapic(LOAPIC_TIMER, LOAPIC_LVT_MASKED);
165 x86_write_loapic(LOAPIC_ERROR, LOAPIC_LVT_MASKED);
166
167 if (loApicMaxLvt >= LOAPIC_LVT_P6) {
168 x86_write_loapic(LOAPIC_PMC, LOAPIC_LVT_MASKED);
169 }
170
171 if (loApicMaxLvt >= LOAPIC_LVT_PENTIUM4) {
172 x86_write_loapic(LOAPIC_THERMAL, LOAPIC_LVT_MASKED);
173 }
174
175 #if CONFIG_LOAPIC_SPURIOUS_VECTOR
176 x86_write_loapic(LOAPIC_SVR, (x86_read_loapic(LOAPIC_SVR) & 0xFFFFFF00) |
177 (LOAPIC_SPURIOUS_VECTOR_ID & 0xFF));
178 #endif
179
180 /* discard a pending interrupt if any */
181 x86_write_loapic(LOAPIC_EOI, 0);
182 }
183
184 /**
185 *
186 * @brief Dummy initialization function.
187 *
188 * The local APIC is initialized via z_loapic_enable() long before the
189 * kernel runs through its device initializations, so this is unneeded.
190 */
191 __boot_func
loapic_init(const struct device * unused)192 static int loapic_init(const struct device *unused)
193 {
194 ARG_UNUSED(unused);
195 return 0;
196 }
197
198
199 __pinned_func
z_loapic_irq_base(void)200 uint32_t z_loapic_irq_base(void)
201 {
202 return z_ioapic_num_rtes();
203 }
204
205 /**
206 *
207 * @brief Set the vector field in the specified RTE
208 *
209 * This associates an IRQ with the desired vector in the IDT.
210 *
211 * @return N/A
212 */
213 __boot_func
z_loapic_int_vec_set(unsigned int irq,unsigned int vector)214 void z_loapic_int_vec_set(unsigned int irq, /* IRQ number of the interrupt */
215 unsigned int vector /* vector to copy into the LVT */
216 )
217 {
218 unsigned int oldLevel; /* previous interrupt lock level */
219
220 /*
221 * The following mappings are used:
222 *
223 * IRQ0 -> LOAPIC_TIMER
224 * IRQ1 -> LOAPIC_THERMAL
225 * IRQ2 -> LOAPIC_PMC
226 * IRQ3 -> LOAPIC_LINT0
227 * IRQ4 -> LOAPIC_LINT1
228 * IRQ5 -> LOAPIC_ERROR
229 *
230 * It's assumed that LVTs are spaced by 0x10 bytes
231 */
232
233 /* update the 'vector' bits in the LVT */
234
235 oldLevel = irq_lock();
236 x86_write_loapic(LOAPIC_TIMER + (irq * 0x10),
237 (x86_read_loapic(LOAPIC_TIMER + (irq * 0x10)) &
238 ~LOAPIC_VECTOR) | vector);
239 irq_unlock(oldLevel);
240 }
241
242 /**
243 *
244 * @brief Enable an individual LOAPIC interrupt (IRQ)
245 *
246 * @param irq the IRQ number of the interrupt
247 *
248 * This routine clears the interrupt mask bit in the LVT for the specified IRQ
249 *
250 * @return N/A
251 */
252 __pinned_func
z_loapic_irq_enable(unsigned int irq)253 void z_loapic_irq_enable(unsigned int irq)
254 {
255 unsigned int oldLevel; /* previous interrupt lock level */
256
257 /*
258 * See the comments in _LoApicLvtVecSet() regarding IRQ to LVT mappings
259 * and ths assumption concerning LVT spacing.
260 */
261
262 /* clear the mask bit in the LVT */
263
264 oldLevel = irq_lock();
265 x86_write_loapic(LOAPIC_TIMER + (irq * 0x10),
266 x86_read_loapic(LOAPIC_TIMER + (irq * 0x10)) &
267 ~LOAPIC_LVT_MASKED);
268 irq_unlock(oldLevel);
269 }
270
271 /**
272 *
273 * @brief Disable an individual LOAPIC interrupt (IRQ)
274 *
275 * @param irq the IRQ number of the interrupt
276 *
277 * This routine clears the interrupt mask bit in the LVT for the specified IRQ
278 *
279 * @return N/A
280 */
281 __pinned_func
z_loapic_irq_disable(unsigned int irq)282 void z_loapic_irq_disable(unsigned int irq)
283 {
284 unsigned int oldLevel; /* previous interrupt lock level */
285
286 /*
287 * See the comments in _LoApicLvtVecSet() regarding IRQ to LVT mappings
288 * and ths assumption concerning LVT spacing.
289 */
290
291 /* set the mask bit in the LVT */
292
293 oldLevel = irq_lock();
294 x86_write_loapic(LOAPIC_TIMER + (irq * 0x10),
295 x86_read_loapic(LOAPIC_TIMER + (irq * 0x10)) |
296 LOAPIC_LVT_MASKED);
297 irq_unlock(oldLevel);
298 }
299
300
301 /**
302 * @brief Find the currently executing interrupt vector, if any
303 *
304 * This routine finds the vector of the interrupt that is being processed.
305 * The ISR (In-Service Register) register contain the vectors of the interrupts
306 * in service. And the higher vector is the identification of the interrupt
307 * being currently processed.
308 *
309 * This function must be called with interrupts locked in interrupt context.
310 *
311 * ISR registers' offsets:
312 * --------------------
313 * | Offset | bits |
314 * --------------------
315 * | 0100H | 0:31 |
316 * | 0110H | 32:63 |
317 * | 0120H | 64:95 |
318 * | 0130H | 96:127 |
319 * | 0140H | 128:159 |
320 * | 0150H | 160:191 |
321 * | 0160H | 192:223 |
322 * | 0170H | 224:255 |
323 * --------------------
324 *
325 * @return The vector of the interrupt that is currently being processed, or -1
326 * if no IRQ is being serviced.
327 */
328 __pinned_func
z_irq_controller_isr_vector_get(void)329 int z_irq_controller_isr_vector_get(void)
330 {
331 int pReg, block;
332
333 /* Block 0 bits never lit up as these are all exception or reserved
334 * vectors
335 */
336 for (block = 7; likely(block > 0); block--) {
337 pReg = x86_read_loapic(LOAPIC_ISR + (block * 0x10));
338 if (pReg) {
339 return (block * 32) + (find_msb_set(pReg) - 1);
340 }
341
342 }
343 return -1;
344 }
345
346 #ifdef CONFIG_PM_DEVICE
347 __pinned_func
loapic_suspend(const struct device * port)348 static int loapic_suspend(const struct device *port)
349 {
350 volatile uint32_t lvt; /* local vector table entry value */
351 int loapic_irq;
352
353 ARG_UNUSED(port);
354
355 (void)memset(loapic_suspend_buf, 0, (LOPIC_SUSPEND_BITS_REQD >> 3));
356
357 for (loapic_irq = 0; loapic_irq < LOAPIC_IRQ_COUNT; loapic_irq++) {
358
359 if (_irq_to_interrupt_vector[z_loapic_irq_base() + loapic_irq]) {
360
361 /* Since vector numbers are already present in RAM/ROM,
362 * We save only the mask bits here.
363 */
364 lvt = x86_read_loapic(LOAPIC_TIMER + (loapic_irq * 0x10));
365
366 if ((lvt & LOAPIC_LVT_MASKED) == 0U) {
367 sys_bitfield_set_bit((mem_addr_t)loapic_suspend_buf,
368 loapic_irq);
369 }
370 }
371 }
372
373 return 0;
374 }
375
376 __pinned_func
loapic_resume(const struct device * port)377 int loapic_resume(const struct device *port)
378 {
379 int loapic_irq;
380
381 ARG_UNUSED(port);
382
383 /* Assuming all loapic device registers lose their state, the call to
384 * z_loapic_init(), should bring all the registers to a sane state.
385 */
386 loapic_init(NULL);
387
388 for (loapic_irq = 0; loapic_irq < LOAPIC_IRQ_COUNT; loapic_irq++) {
389
390 if (_irq_to_interrupt_vector[z_loapic_irq_base() + loapic_irq]) {
391 /* Configure vector and enable the required ones*/
392 z_loapic_int_vec_set(loapic_irq,
393 _irq_to_interrupt_vector[z_loapic_irq_base() +
394 loapic_irq]);
395
396 if (sys_bitfield_test_bit((mem_addr_t) loapic_suspend_buf,
397 loapic_irq)) {
398 z_loapic_irq_enable(loapic_irq);
399 }
400 }
401 }
402
403 return 0;
404 }
405
406 /*
407 * Implements the driver control management functionality
408 * the *context may include IN data or/and OUT data
409 */
410 __pinned_func
loapic_device_ctrl(const struct device * dev,enum pm_device_action action)411 static int loapic_device_ctrl(const struct device *dev,
412 enum pm_device_action action)
413 {
414 int ret = 0;
415
416 switch (action) {
417 case PM_DEVICE_ACTION_SUSPEND:
418 ret = loapic_suspend(dev);
419 break;
420 case PM_DEVICE_ACTION_RESUME:
421 ret = loapic_resume(dev);
422 break;
423 default:
424 return -ENOTSUP;
425 }
426
427 return ret;
428 }
429
430 SYS_DEVICE_DEFINE("loapic", loapic_init, loapic_device_ctrl, PRE_KERNEL_1,
431 CONFIG_KERNEL_INIT_PRIORITY_DEFAULT);
432 #else
433 SYS_INIT(loapic_init, PRE_KERNEL_1, CONFIG_KERNEL_INIT_PRIORITY_DEFAULT);
434 #endif /* CONFIG_PM_DEVICE */
435
436
437 #if CONFIG_LOAPIC_SPURIOUS_VECTOR
438 extern void z_loapic_spurious_handler(void);
439
440 NANO_CPU_INT_REGISTER(z_loapic_spurious_handler, NANO_SOFT_IRQ,
441 LOAPIC_SPURIOUS_VECTOR_ID >> 4,
442 LOAPIC_SPURIOUS_VECTOR_ID, 0);
443 #endif
444
