1 /*
2 * Copyright (c) 2010-2014 Wind River Systems, Inc.
3 *
4 * SPDX-License-Identifier: Apache-2.0
5 */
6
7 /**
8 * @file
9 * @brief IA-32 specific kernel interface header
10 * This header contains the IA-32 specific kernel interface. It is included
11 * by the generic kernel interface header (include/arch/cpu.h)
12 */
13
14 #ifndef ZEPHYR_INCLUDE_ARCH_X86_IA32_ARCH_H_
15 #define ZEPHYR_INCLUDE_ARCH_X86_IA32_ARCH_H_
16
17 #include "sys_io.h"
18 #include <stdbool.h>
19 #include <zephyr/kernel_structs.h>
20 #include <zephyr/arch/common/ffs.h>
21 #include <zephyr/sys/util.h>
22 #include <zephyr/arch/x86/ia32/exception.h>
23 #include <zephyr/arch/x86/ia32/gdbstub.h>
24 #include <zephyr/arch/x86/ia32/thread.h>
25 #include <zephyr/arch/x86/ia32/syscall.h>
26
27 #ifndef _ASMLANGUAGE
28 #include <stddef.h> /* for size_t */
29
30 #include <zephyr/arch/common/addr_types.h>
31 #include <zephyr/arch/x86/ia32/segmentation.h>
32 #include <zephyr/pm/pm.h>
33
34 #endif /* _ASMLANGUAGE */
35
36 /* GDT layout */
37 #define CODE_SEG 0x08
38 #define DATA_SEG 0x10
39 #define MAIN_TSS 0x18
40 #define DF_TSS 0x20
41
42 /*
43 * Use for thread local storage.
44 * Match these to gen_gdt.py.
45 * The 0x03 is added to limit privilege.
46 */
47 #if defined(CONFIG_USERSPACE)
48 #define GS_TLS_SEG (0x38 | 0x03)
49 #elif defined(CONFIG_X86_STACK_PROTECTION)
50 #define GS_TLS_SEG (0x28 | 0x03)
51 #else
52 #define GS_TLS_SEG (0x18 | 0x03)
53 #endif
54
55 /**
56 * Macro used internally by NANO_CPU_INT_REGISTER and NANO_CPU_INT_REGISTER_ASM.
57 * Not meant to be used explicitly by platform, driver or application code.
58 */
59 #define MK_ISR_NAME(x) __isr__##x
60
61 #define Z_DYN_STUB_SIZE 4
62 #define Z_DYN_STUB_OFFSET 0
63 #define Z_DYN_STUB_LONG_JMP_EXTRA_SIZE 3
64 #define Z_DYN_STUB_PER_BLOCK 32
65
66
67 #ifndef _ASMLANGUAGE
68
69 #ifdef __cplusplus
70 extern "C" {
71 #endif
72
73 /* interrupt/exception/error related definitions */
74
75 typedef struct s_isrList {
76 /** Address of ISR/stub */
77 void *fnc;
78 /** IRQ associated with the ISR/stub, or -1 if this is not
79 * associated with a real interrupt; in this case vec must
80 * not be -1
81 */
82 unsigned int irq;
83 /** Priority associated with the IRQ. Ignored if vec is not -1 */
84 unsigned int priority;
85 /** Vector number associated with ISR/stub, or -1 to assign based
86 * on priority
87 */
88 unsigned int vec;
89 /** Privilege level associated with ISR/stub */
90 unsigned int dpl;
91
92 /** If nonzero, specifies a TSS segment selector. Will configure
93 * a task gate instead of an interrupt gate. fnc parameter will be
94 * ignored
95 */
96 unsigned int tss;
97 } ISR_LIST;
98
99
100 /**
101 * @brief Connect a routine to an interrupt vector
102 *
103 * This macro "connects" the specified routine, @a r, to the specified interrupt
104 * vector, @a v using the descriptor privilege level @a d. On the IA-32
105 * architecture, an interrupt vector is a value from 0 to 255. This macro
106 * populates the special intList section with the address of the routine, the
107 * vector number and the descriptor privilege level. The genIdt tool then picks
108 * up this information and generates an actual IDT entry with this information
109 * properly encoded.
110 *
111 * The @a d argument specifies the privilege level for the interrupt-gate
112 * descriptor; (hardware) interrupts and exceptions should specify a level of 0,
113 * whereas handlers for user-mode software generated interrupts should specify 3.
114 * @param r Routine to be connected
115 * @param n IRQ number
116 * @param p IRQ priority
117 * @param v Interrupt Vector
118 * @param d Descriptor Privilege Level
119 */
120
121 #define NANO_CPU_INT_REGISTER(r, n, p, v, d) \
122 static ISR_LIST __attribute__((section(".intList"))) \
123 __attribute__((used)) MK_ISR_NAME(r) = \
124 { \
125 .fnc = &(r), \
126 .irq = (n), \
127 .priority = (p), \
128 .vec = (v), \
129 .dpl = (d), \
130 .tss = 0 \
131 }
132
133 /**
134 * @brief Connect an IA hardware task to an interrupt vector
135 *
136 * This is very similar to NANO_CPU_INT_REGISTER but instead of connecting
137 * a handler function, the interrupt will induce an IA hardware task
138 * switch to another hardware task instead.
139 *
140 * @param tss_p GDT/LDT segment selector for the TSS representing the task
141 * @param irq_p IRQ number
142 * @param priority_p IRQ priority
143 * @param vec_p Interrupt vector
144 * @param dpl_p Descriptor privilege level
145 */
146 #define _X86_IDT_TSS_REGISTER(tss_p, irq_p, priority_p, vec_p, dpl_p) \
147 static ISR_LIST __attribute__((section(".intList"))) \
148 __attribute__((used)) MK_ISR_NAME(vec_p) = \
149 { \
150 .fnc = NULL, \
151 .irq = (irq_p), \
152 .priority = (priority_p), \
153 .vec = (vec_p), \
154 .dpl = (dpl_p), \
155 .tss = (tss_p) \
156 }
157
158 /**
159 * Code snippets for populating the vector ID and priority into the intList
160 *
161 * The 'magic' of static interrupts is accomplished by building up an array
162 * 'intList' at compile time, and the gen_idt tool uses this to create the
163 * actual IDT data structure.
164 *
165 * For controllers like APIC, the vectors in the IDT are not normally assigned
166 * at build time; instead the sentinel value -1 is saved, and gen_idt figures
167 * out the right vector to use based on our priority scheme. Groups of 16
168 * vectors starting at 32 correspond to each priority level.
169 *
170 * These macros are only intended to be used by IRQ_CONNECT() macro.
171 */
172 #define _VECTOR_ARG(irq_p) (-1)
173
174 #ifdef CONFIG_LINKER_USE_PINNED_SECTION
175 #define IRQSTUBS_TEXT_SECTION ".pinned_text.irqstubs"
176 #else
177 #define IRQSTUBS_TEXT_SECTION ".text.irqstubs"
178 #endif
179
180 /* Internally this function does a few things:
181 *
182 * 1. There is a declaration of the interrupt parameters in the .intList
183 * section, used by gen_idt to create the IDT. This does the same thing
184 * as the NANO_CPU_INT_REGISTER() macro, but is done in assembly as we
185 * need to populate the .fnc member with the address of the assembly
186 * IRQ stub that we generate immediately afterwards.
187 *
188 * 2. The IRQ stub itself is declared. The code will go in its own named
189 * section .text.irqstubs section (which eventually gets linked into 'text')
190 * and the stub shall be named (isr_name)_irq(irq_line)_stub
191 *
192 * 3. The IRQ stub pushes the ISR routine and its argument onto the stack
193 * and then jumps to the common interrupt handling code in _interrupt_enter().
194 *
195 * 4. z_irq_controller_irq_config() is called at runtime to set the mapping
196 * between the vector and the IRQ line as well as triggering flags
197 */
198 #define ARCH_IRQ_CONNECT(irq_p, priority_p, isr_p, isr_param_p, flags_p) \
199 { \
200 __asm__ __volatile__( \
201 ".pushsection .intList\n\t" \
202 ".long %c[isr]_irq%c[irq]_stub\n\t" /* ISR_LIST.fnc */ \
203 ".long %c[irq]\n\t" /* ISR_LIST.irq */ \
204 ".long %c[priority]\n\t" /* ISR_LIST.priority */ \
205 ".long %c[vector]\n\t" /* ISR_LIST.vec */ \
206 ".long 0\n\t" /* ISR_LIST.dpl */ \
207 ".long 0\n\t" /* ISR_LIST.tss */ \
208 ".popsection\n\t" \
209 ".pushsection " IRQSTUBS_TEXT_SECTION "\n\t" \
210 ".global %c[isr]_irq%c[irq]_stub\n\t" \
211 "%c[isr]_irq%c[irq]_stub:\n\t" \
212 "pushl %[isr_param]\n\t" \
213 "pushl %[isr]\n\t" \
214 "jmp _interrupt_enter\n\t" \
215 ".popsection\n\t" \
216 : \
217 : [isr] "i" (isr_p), \
218 [isr_param] "i" (isr_param_p), \
219 [priority] "i" (priority_p), \
220 [vector] "i" _VECTOR_ARG(irq_p), \
221 [irq] "i" (irq_p)); \
222 z_irq_controller_irq_config(Z_IRQ_TO_INTERRUPT_VECTOR(irq_p), (irq_p), \
223 (flags_p)); \
224 }
225
226 #ifdef CONFIG_PCIE
227
228 #define ARCH_PCIE_IRQ_CONNECT(bdf_p, irq_p, priority_p, \
229 isr_p, isr_param_p, flags_p) \
230 ARCH_IRQ_CONNECT(irq_p, priority_p, isr_p, isr_param_p, flags_p)
231
232 #endif /* CONFIG_PCIE */
233
234 /* Direct interrupts won't work as expected with KPTI turned on, because
235 * all non-user accessible pages in the page table are marked non-present.
236 * It's likely possible to add logic to ARCH_ISR_DIRECT_HEADER/FOOTER to do
237 * the necessary trampolining to switch page tables / stacks, but this
238 * probably loses all the latency benefits that direct interrupts provide
239 * and one might as well use a regular interrupt anyway.
240 */
241 #ifndef CONFIG_X86_KPTI
242 #define ARCH_IRQ_DIRECT_CONNECT(irq_p, priority_p, isr_p, flags_p) \
243 { \
244 NANO_CPU_INT_REGISTER(isr_p, irq_p, priority_p, -1, 0); \
245 z_irq_controller_irq_config(Z_IRQ_TO_INTERRUPT_VECTOR(irq_p), (irq_p), \
246 (flags_p)); \
247 }
248
249 #ifdef CONFIG_PM
arch_irq_direct_pm(void)250 static inline void arch_irq_direct_pm(void)
251 {
252 if (_kernel.idle) {
253 _kernel.idle = 0;
254 pm_system_resume();
255 }
256 }
257
258 #define ARCH_ISR_DIRECT_PM() arch_irq_direct_pm()
259 #else
260 #define ARCH_ISR_DIRECT_PM() do { } while (false)
261 #endif
262
263 #define ARCH_ISR_DIRECT_HEADER() arch_isr_direct_header()
264 #define ARCH_ISR_DIRECT_FOOTER(swap) arch_isr_direct_footer(swap)
265
266 /* FIXME:
267 * tracing/tracing.h cannot be included here due to circular dependency
268 */
269 #if defined(CONFIG_TRACING)
270 void sys_trace_isr_enter(void);
271 void sys_trace_isr_exit(void);
272 #endif
273
arch_isr_direct_header(void)274 static inline void arch_isr_direct_header(void)
275 {
276 #if defined(CONFIG_TRACING)
277 sys_trace_isr_enter();
278 #endif
279
280 /* We're not going to unlock IRQs, but we still need to increment this
281 * so that arch_is_in_isr() works
282 */
283 ++_kernel.cpus[0].nested;
284 }
285
286 /*
287 * FIXME: z_swap_irqlock is an inline function declared in a private header and
288 * cannot be referenced from a public header, so we move it to an
289 * external function.
290 */
291 void arch_isr_direct_footer_swap(unsigned int key);
292
arch_isr_direct_footer(int swap)293 static inline void arch_isr_direct_footer(int swap)
294 {
295 z_irq_controller_eoi();
296 #if defined(CONFIG_TRACING)
297 sys_trace_isr_exit();
298 #endif
299 --_kernel.cpus[0].nested;
300
301 /* Call swap if all the following is true:
302 *
303 * 1) swap argument was enabled to this function
304 * 2) We are not in a nested interrupt
305 * 3) Next thread to run in the ready queue is not this thread
306 */
307 if (swap != 0 && _kernel.cpus[0].nested == 0 &&
308 _kernel.ready_q.cache != arch_current_thread()) {
309 unsigned int flags;
310
311 /* Fetch EFLAGS argument to z_swap() */
312 __asm__ volatile (
313 "pushfl\n\t"
314 "popl %0\n\t"
315 : "=g" (flags)
316 :
317 : "memory"
318 );
319
320 arch_isr_direct_footer_swap(flags);
321 }
322 }
323
324 #define ARCH_ISR_DIRECT_DECLARE(name) \
325 static inline int name##_body(void); \
326 __attribute__ ((interrupt)) void name(void *stack_frame) \
327 { \
328 ARG_UNUSED(stack_frame); \
329 int check_reschedule; \
330 ISR_DIRECT_HEADER(); \
331 check_reschedule = name##_body(); \
332 ISR_DIRECT_FOOTER(check_reschedule); \
333 } \
334 static inline int name##_body(void)
335 #endif /* !CONFIG_X86_KPTI */
336
arch_irq_lock(void)337 static ALWAYS_INLINE unsigned int arch_irq_lock(void)
338 {
339 unsigned int key;
340
341 __asm__ volatile ("pushfl; cli; popl %0" : "=g" (key) :: "memory");
342
343 return key;
344 }
345
346
347 /**
348 * The NANO_SOFT_IRQ macro must be used as the value for the @a irq parameter
349 * to NANO_CPU_INT_REGISTER when connecting to an interrupt that does not
350 * correspond to any IRQ line (such as spurious vector or SW IRQ)
351 */
352 #define NANO_SOFT_IRQ ((unsigned int) (-1))
353
354 #ifdef CONFIG_X86_ENABLE_TSS
355 extern struct task_state_segment _main_tss;
356 #endif
357
358 #define ARCH_EXCEPT(reason_p) do { \
359 __asm__ volatile( \
360 "push %[reason]\n\t" \
361 "int %[vector]\n\t" \
362 : \
363 : [vector] "i" (Z_X86_OOPS_VECTOR), \
364 [reason] "i" (reason_p)); \
365 CODE_UNREACHABLE; /* LCOV_EXCL_LINE */ \
366 } while (false)
367
368 /*
369 * Dynamic thread object memory alignment.
370 *
371 * If support for SSEx extensions is enabled a 16 byte boundary is required,
372 * since the 'fxsave' and 'fxrstor' instructions require this. In all other
373 * cases a 4 byte boundary is sufficient.
374 */
375 #if defined(CONFIG_EAGER_FPU_SHARING) || defined(CONFIG_LAZY_FPU_SHARING)
376 #ifdef CONFIG_SSE
377 #define ARCH_DYNAMIC_OBJ_K_THREAD_ALIGNMENT 16
378 #else
379 #define ARCH_DYNAMIC_OBJ_K_THREAD_ALIGNMENT (sizeof(void *))
380 #endif
381 #else
382 /* No special alignment requirements, simply align on pointer size. */
383 #define ARCH_DYNAMIC_OBJ_K_THREAD_ALIGNMENT (sizeof(void *))
384 #endif /* CONFIG_*_FP_SHARING */
385
386
387 #ifdef __cplusplus
388 }
389 #endif
390
391 #endif /* !_ASMLANGUAGE */
392
393 #endif /* ZEPHYR_INCLUDE_ARCH_X86_IA32_ARCH_H_ */
394
