1/*
2 * Copyright (c) 2019 Intel Corporation
3 * SPDX-License-Identifier: Apache-2.0
4 */
5
6#include <toolchain.h>
7#include <arch/x86/multiboot.h>
8#include <sys/util.h>
9#include <arch/x86/msr.h>
10#include <kernel_arch_data.h>
11#include <offsets_short.h>
12#include <drivers/interrupt_controller/loapic.h>
13#include <arch/cpu.h>
14#include <sys/mem_manage.h>
15
16/*
17 * Definitions/macros for enabling paging
18 */
19
20/* Long mode, no-execute, syscall */
21#define EFER_BITS (X86_EFER_MSR_LME | X86_EFER_MSR_NXE | X86_EFER_MSR_SCE)
22
23/* Paging, write-protect */
24#define CR0_BITS (CR0_PG | CR0_WP)
25
26/* PAE, SSE */
27#define CR4_BITS (CR4_PAE | CR4_OSFXSR)
28
29.macro set_efer
30	movl $X86_EFER_MSR, %ecx
31	rdmsr
32	orl $EFER_BITS, %eax
33	wrmsr
34.endm
35
36.macro install_pagetables_32
37	movl %cr4, %eax
38	orl $CR4_BITS, %eax
39	movl %eax, %cr4
40	clts
41
42	/* Page tables created at build time by gen_mmu.py
43	 * NOTE: Presumes phys=virt
44	 */
45	movl $Z_MEM_PHYS_ADDR(z_x86_kernel_ptables), %eax
46	movl %eax, %cr3
47
48	set_efer
49
50	movl %cr0, %eax
51	orl $CR0_BITS, %eax
52	movl %eax, %cr0
53.endm
54
55.macro install_pagetables_64
56	/* Here, we are already in long mode with paging enabled and
57	 * just need to switch to our own page tables, but let's be
58	 * paranoid and ensure CR4, CR0, and EFER_MSR are set up
59	 * exactly how we expect. Logic is the same as install_pagetables_32
60	 */
61	movq %cr4, %rax
62	orq $CR4_BITS, %rax
63	movq %rax, %cr4
64	clts
65
66	/* NOTE: Presumes phys=virt */
67	movq $Z_MEM_PHYS_ADDR(z_x86_kernel_ptables), %rax
68	movq %rax, %cr3
69
70	set_efer
71
72	movq %cr0, %rax
73	/* Use 32-bit instructions due to assembler fussiness with large
74	 * immediate values with `orq`, CR0_PG is bit 31. We don't ever set any
75	 * high bits in cr0 anyway.
76	 */
77	orl $CR0_BITS, %eax
78	movq %rax, %cr0
79.endm
80
81/* The .locore section begins the page-aligned initialization region
82 * of low memory.  The first address is used as the architectural
83 * entry point for auxiliary CPUs being brought up (in real mode!)
84 * via a startup IPI.  It's is ALSO used by some loaders (well,
85 * ACRN...) who hard-coded the address by inspecting _start on a
86 * non-SMP build.
87 *
88 *               === OUTRAGEOUS HACK FOLLOWS ===
89 *
90 * Therefore it needs to start at OS entry with a 32 bit jump to the
91 * 32 bit entry point, and gets clobbered later (see the beginning of
92 * __start32) with NOP bytes such that the next CPU will fall through
93 * to the 16 bit SMP entry.
94 *
95 * We write out the JMP followed by 8 NOPs for simplicity.  No i386
96 * JMP encodes with more than 8 bytes, so we can come back later and
97 * scribble over it with 8 0x90 bytes (which is the 1-byte NOP) and be
98 * sure to get all of it without overwriting anything.
99 */
100.section .locore,"ax"
101.code32
102.globl __start
103__start:
104	jmp __start32
105	nop
106	nop
107	nop
108	nop
109	nop
110	nop
111	nop
112	nop
113
114#if CONFIG_MP_NUM_CPUS > 1
115
116.code16
117.global x86_ap_start
118x86_ap_start:
119
120	/*
121	 * First, we move to 32-bit protected mode, and set up the
122	 * same flat environment that the BSP gets from the loader.
123	 */
124
125	lgdt gdt48
126	lidt idt48
127	movl %cr0, %eax
128	or $1, %eax
129	movl %eax, %cr0
130
131	jmpl $X86_KERNEL_CS_32, $1f
132.code32
1331:	movw $X86_KERNEL_DS_32, %ax
134	movw %ax, %ds
135	movw %ax, %es
136	movw %ax, %ss
137	movw %ax, %fs
138
139	/*
140	 * Now, reverse-map our local APIC ID to our logical CPU ID
141	 * so we can locate our x86_cpuboot[] bundle. Put it in EBP.
142	 */
143
144	movl CONFIG_LOAPIC_BASE_ADDRESS+LOAPIC_ID, %eax
145	shrl $24, %eax
146	andl $0x0F, %eax		/* local APIC ID -> EAX */
147
148	movl $x86_cpuboot, %ebp
149	xorl %ebx, %ebx
1501:	cmpl $CONFIG_MP_NUM_CPUS, %ebx
151	jz unknown_loapic_id
152	cmpb %al, x86_cpu_loapics(%ebx)
153	je go64				/* proceed to 64-bit mode */
154	incl %ebx
155	addl $__X86_CPUBOOT_SIZEOF, %ebp
156	jmp 1b
157
158unknown_loapic_id:
159	jmp unknown_loapic_id
160
161#endif /* CONFIG_MP_NUM_CPUS > 1 */
162
163.code32
164.globl __start32
165__start32:
166	/*
167	 * kernel execution begins here in 32-bit mode, with flat-mode
168	 * descriptors in all segment registers, interrupts disabled.
169	 */
170
171	/* See note above, re: OUTRAGEOUS HACK */
172	movl $__start, %ebp
173	movb $0x90, 0(%ebp)
174	movb $0x90, 1(%ebp)
175	movb $0x90, 2(%ebp)
176	movb $0x90, 3(%ebp)
177	movb $0x90, 4(%ebp)
178	movb $0x90, 5(%ebp)
179	movb $0x90, 6(%ebp)
180	movb $0x90, 7(%ebp)
181	wbinvd
182
183	lgdt gdt48
184	lidt idt48
185
186#include "../common.S"
187
188	/*
189	 * N.B.: if multiboot info struct is present, "common.S"
190	 * has left a pointer to it in EBX.
191	 */
192
193	movl $x86_cpuboot, %ebp		/* BSP is always logical CPU id 0 */
194	movl %ebx, __x86_cpuboot_t_arg_OFFSET(%ebp)	/* multiboot info */
195
196
197go64:	/* Install page tables and transition to long mode */
198	install_pagetables_32
199	jmpl $X86_KERNEL_CS, $enter_code64
200
201	/* Long mode entry point.  Arrive here from the code
202	 * immediately above (shared between main CPU startup and AP
203	 * startup), or from EFI entry in __start64.
204	 *
205	 * Here we reload the segment registers,
206	 * and configure per-CPU stuff: GS, task register, stack.
207	 */
208	.code64
209enter_code64:
210	movl $X86_KERNEL_DS, %eax
211	movw %ax, %ds
212	movw %ax, %es
213	movw %ax, %ss
214	movw %ax, %fs
215
216	/* On Intel processors, if GS is not zero and is being set to
217	 * zero, GS_BASE is also being set to zero. This would interfere
218	 * with the actual use of GS_BASE for usespace. To avoid accidentally
219	 * clearing GS_BASE, simply set GS to 0 at boot, so any subsequent
220	 * clearing of GS will not clear GS_BASE.
221	 */
222	mov $0, %eax
223	movw %ax, %gs
224
225	movw __x86_cpuboot_t_tr_OFFSET(%rbp), %ax
226	ltr %ax
227
228	/* Set up MSRs for GS / KERNEL_GS base */
229	movq __x86_cpuboot_t_gs_base_OFFSET(%rbp), %rax
230	movq %rax, %rdx
231	shrq $32, %rdx
232	/* X86_KERNEL_GS_BASE and X86_GS_BASE are swapped by the 'swapgs'
233	 * instruction.
234	 */
235	movl $X86_KERNEL_GS_BASE, %ecx
236	wrmsr
237	/* X86_GS_BASE shadows base fields of %gs, effectively setting %gs */
238	movl $X86_GS_BASE, %ecx
239	wrmsr
240
241	movq __x86_cpuboot_t_sp_OFFSET(%rbp), %rsp
242	movq %rsp, %gs:__x86_tss64_t_ist1_OFFSET
243
244	/* finally, complete environment for the C runtime and go. */
245	cld	/* GCC presumes a clear direction flag */
246
247#ifdef CONFIG_INIT_STACKS
248	movq $0xAAAAAAAAAAAAAAAA, %rax
249	movq %rsp, %rdi
250	subq __x86_cpuboot_t_stack_size_OFFSET(%rbp), %rdi
251	movq __x86_cpuboot_t_stack_size_OFFSET(%rbp), %rcx
252	shr $3, %rcx /* moving 8 bytes a time, so fewer repeats */
253	rep stosq
254#endif
255
256	/* Enter C domain now that we have a stack set up, never to return */
257	movq %rbp, %rdi
258	call z_x86_cpu_init
259
260	/* 64 bit OS entry point, used by EFI support.  UEFI
261	 * guarantees an identity-mapped page table that covers
262	 * physical memory, and the loader stub already used it to
263	 * write all of the Zephyr image, so we know it works for what
264	 * we need.  Other things need fixups to match what multiboot
265	 * 32 bit startup does.
266	 */
267.globl __start64
268__start64:
269	/* Zero the TSC */
270	xorq %rax, %rax
271	xorq %rdx, %rdx
272	movq $X86_TIME_STAMP_COUNTER_MSR, %rcx
273	wrmsr
274
275	lidt idt80
276	lgdt gdt80
277
278	install_pagetables_64
279
280	/* Disable 8259 PIT.  Almost certainly not needed on modern
281	 * UEFI platforms taking this code path, but...
282	 */
283	movb $0xff, %al
284	outb %al, $0x21
285	outb %al, $0xA1
286
287	/* Far call into the Zephyr code segment */
288	movq $x86_cpuboot, %rbp
289	mov jmpdesc, %rax
290	jmp *%rax
291jmpdesc:
292	.quad enter_code64
293	.short X86_KERNEL_CS
294
295/*
296 * void x86_sse_init(struct k_thread *thread);
297 *
298 * Initialize floating-point state to something sane. If 'thread' is
299 * not NULL, then the resulting FP state is saved to thread->arch.sse.
300 */
301
302.global x86_sse_init
303x86_sse_init:
304	fninit
305	ldmxcsr mxcsr
306	testq %rdi, %rdi
307	jz 1f
308	fxsave _thread_offset_to_sse(%rdi)
3091:	retq
310
311mxcsr:	.long X86_MXCSR_SANE
312
313/*
314 * void z_x86_switch(void *switch_to, void **switched_from);
315 *
316 * Note that switch_handle for us is simply a pointer to the containing
317 * 'struct k_thread', thus:
318 *
319 * RDI = (struct k_thread *) switch_to
320 * RSI = (struct k_thread **) address of output thread switch_handle field
321 */
322
323.globl z_x86_switch
324z_x86_switch:
325	/* RSI contains the switch_handle field to which we are
326	 * notionally supposed to store.  Offset it to get back to the
327	 * thread handle instead.
328	 */
329	subq $___thread_t_switch_handle_OFFSET, %rsi
330
331	andb $~X86_THREAD_FLAG_ALL, _thread_offset_to_flags(%rsi)
332
333	popq %rax
334	movq %rax, _thread_offset_to_rip(%rsi)
335	pushfq
336	popq %rax
337	movq %rax, _thread_offset_to_rflags(%rsi)
338	movq %rsp, _thread_offset_to_rsp(%rsi)
339	movq %rbx, _thread_offset_to_rbx(%rsi)
340	movq %rbp, _thread_offset_to_rbp(%rsi)
341	movq %r12, _thread_offset_to_r12(%rsi)
342	movq %r13, _thread_offset_to_r13(%rsi)
343	movq %r14, _thread_offset_to_r14(%rsi)
344	movq %r15, _thread_offset_to_r15(%rsi)
345#ifdef CONFIG_USERSPACE
346	/* We're always in supervisor mode if we get here, the other case
347	 * is when __resume is invoked from irq_dispatch
348	 */
349	movq $X86_KERNEL_CS, _thread_offset_to_cs(%rsi)
350	movq $X86_KERNEL_DS, _thread_offset_to_ss(%rsi)
351#endif
352	/* Store the handle (i.e. our thread struct address) into the
353	 * switch handle field, this is a synchronization signal that
354	 * must occur after the last data from the old context is
355	 * saved.
356	 */
357	movq %rsi, ___thread_t_switch_handle_OFFSET(%rsi)
358
359	movq %gs:__x86_tss64_t_ist1_OFFSET, %rsp
360
361	/* fall through to __resume */
362
363/*
364 * Entry:
365 *   RSP = top of CPU interrupt stack
366 *   RDI = (struct k_thread *) thread to resume
367 */
368
369__resume:
370#if (!defined(CONFIG_X86_KPTI) && defined(CONFIG_USERSPACE)) \
371		|| defined(CONFIG_INSTRUMENT_THREAD_SWITCHING)
372	pushq %rdi	/* Caller-saved, stash it */
373#if !defined(CONFIG_X86_KPTI) && defined(CONFIG_USERSPACE)
374	/* If KPTI is enabled we're always on the kernel's page tables in
375	 * this context and the appropriate page table switch takes place
376	 * when trampolining back to user mode
377	 */
378	call z_x86_swap_update_page_tables
379#endif
380#ifdef CONFIG_INSTRUMENT_THREAD_SWITCHING
381	call z_thread_mark_switched_in
382#endif
383	popq %rdi
384#endif /* (!CONFIG_X86_KPTI && CONFIG_USERSPACE) || \
385	   CONFIG_INSTRUMENT_THREAD_SWITCHING */
386
387#ifdef CONFIG_USERSPACE
388	/* Set up exception return stack frame */
389	pushq _thread_offset_to_ss(%rdi)	/* SS */
390#else
391	pushq $X86_KERNEL_DS			/* SS */
392#endif /* CONFIG_USERSPACE */
393	pushq _thread_offset_to_rsp(%rdi)	/* RSP */
394	pushq _thread_offset_to_rflags(%rdi)	/* RFLAGS */
395#ifdef CONFIG_USERSPACE
396	pushq _thread_offset_to_cs(%rdi)	/* CS */
397#else
398	pushq $X86_KERNEL_CS			/* CS */
399#endif
400	pushq _thread_offset_to_rip(%rdi)	/* RIP */
401
402#ifdef CONFIG_ASSERT
403	/* Poison the old thread's saved RIP pointer with a
404	 * recognizable value near NULL, to easily catch reuse of the
405	 * thread object across CPUs in SMP.  Strictly speaking this
406	 * is not an assertion, but it's very cheap and worth having
407	 * on during routine testing.
408	 */
409	movq $0xB9, _thread_offset_to_rip(%rdi)
410#endif
411
412#ifdef CONFIG_THREAD_LOCAL_STORAGE
413	/*
414	 * Write the TLS base pointer to FS_BASE MSR,
415	 * where GCC emits code to access TLS data via
416	 * offset to FS.
417	 * Since wrmsr write EDX:EAX to MSR indicated by
418	 * ECX, the high 32-bit needs to be loaded into
419	 * RDX and right shifted by 32 bits so EDX has
420	 * the higher 32-bit value.
421	 */
422	movl $X86_FS_BASE, %ecx
423	movq _thread_offset_to_tls(%rdi), %rax
424	movq _thread_offset_to_tls(%rdi), %rdx
425	shrq $32, %rdx
426	wrmsr
427#endif
428
429	movq _thread_offset_to_rbx(%rdi), %rbx
430	movq _thread_offset_to_rbp(%rdi), %rbp
431	movq _thread_offset_to_r12(%rdi), %r12
432	movq _thread_offset_to_r13(%rdi), %r13
433	movq _thread_offset_to_r14(%rdi), %r14
434	movq _thread_offset_to_r15(%rdi), %r15
435#ifdef CONFIG_USERSPACE
436	/* Set correct privilege elevation stack to manually switch to in
437	 * z_x86_syscall_entry_stub()
438	 */
439	movq _thread_offset_to_psp(%rdi), %rax
440	movq %rax, %gs:__x86_tss64_t_psp_OFFSET
441#endif
442
443	testb $X86_THREAD_FLAG_ALL, _thread_offset_to_flags(%rdi)
444	jz 1f
445
446	fxrstor _thread_offset_to_sse(%rdi)
447	movq _thread_offset_to_rax(%rdi), %rax
448	movq _thread_offset_to_rcx(%rdi), %rcx
449	movq _thread_offset_to_rdx(%rdi), %rdx
450	movq _thread_offset_to_rsi(%rdi), %rsi
451	movq _thread_offset_to_r8(%rdi), %r8
452	movq _thread_offset_to_r9(%rdi), %r9
453	movq _thread_offset_to_r10(%rdi), %r10
454	movq _thread_offset_to_r11(%rdi), %r11
455	movq _thread_offset_to_rdi(%rdi), %rdi  /* do last :-) */
456
457#ifdef CONFIG_USERSPACE
458	/* Swap GS register values if we are returning to user mode */
459	testb $0x3, 8(%rsp)
460	jz 1f
461#ifdef CONFIG_X86_KPTI
462	jmp z_x86_trampoline_to_user
463#else
464	swapgs
465#endif /* CONFIG_X86_KPTI */
466#endif /* CONFIG_USERSPACE */
4671:
468#ifdef CONFIG_X86_BOUNDS_CHECK_BYPASS_MITIGATION
469	/* swapgs variant of Spectre V1. Disable speculation past this point */
470	lfence
471#endif /* CONFIG_X86_BOUNDS_CHECK_BYPASS_MITIGATION */
472	iretq
473
474
475#ifdef CONFIG_X86_KPTI
476#define EXCEPT_CODE(nr, ist)						\
477	vector_ ## nr: pushq %gs:__x86_tss64_t_ist ## ist ## _OFFSET;	\
478	pushq $nr;							\
479	jmp except
480#define EXCEPT(nr, ist)							\
481	vector_ ## nr: pushq $0;					\
482	pushq %gs:__x86_tss64_t_ist ## ist ## _OFFSET;			\
483	pushq $nr; \
484	jmp except
485#else
486#define EXCEPT_CODE(nr) vector_ ## nr: pushq $nr; jmp except
487#define EXCEPT(nr)	vector_ ## nr: pushq $0; pushq $nr; jmp except
488#endif
489
490/*
491 * When we arrive at 'except' from one of the EXCEPT(X) stubs,
492 * we're on the exception stack with irqs unlocked (or the trampoline stack
493 * with irqs locked if KPTI is enabled) and it contains:
494 *
495 *   SS
496 *   RSP
497 *   RFLAGS
498 *   CS
499 *   RIP
500 *   Error Code if pushed by CPU, else 0
501 *   IST index in TSS
502 *   Vector number <- RSP points here
503 *
504 */
505
506except: /*
507	 * finish struct NANO_ESF on stack. 'vector' .. 'ss' are
508	 * already there from hardware trap and EXCEPT_*() stub.
509	 */
510
511	pushq %r11
512
513#ifdef CONFIG_USERSPACE
514	/* Swap GS register values and page tables if we came from user mode */
515	testb $0x3, 40(%rsp)
516	jz 1f
517	swapgs
518#ifdef CONFIG_X86_KPTI
519	/* Load kernel's page table. NOTE: Presumes phys=virt */
520	movq $z_x86_kernel_ptables, %r11
521	movq %r11, %cr3
522#endif /* CONFIG_X86_KPTI */
5231:
524#ifdef CONFIG_X86_BOUNDS_CHECK_BYPASS_MITIGATION
525	/* swapgs variant of Spectre V1. Disable speculation past this point */
526	lfence
527#endif /* CONFIG_X86_BOUNDS_CHECK_BYPASS_MITIGATION */
528#ifdef CONFIG_X86_KPTI
529	/* Save old trampoline stack pointer in R11 */
530	movq %rsp, %r11
531
532	/* Switch to the correct stack */
533	movq 16(%r11), %rsp
534
535	/* Transplant trampoline stack contents */
536	pushq 64(%r11)	/* SS */
537	pushq 56(%r11)	/* RSP */
538	pushq 48(%r11)	/* RFLAGS */
539	pushq 40(%r11)	/* CS */
540	pushq 32(%r11)	/* RIP */
541	pushq 24(%r11)	/* Error code */
542	pushq 8(%r11)	/* Vector */
543	pushq (%r11)	/* Stashed R11 */
544	movq $0, (%r11) /* Cover our tracks */
545
546	/* We're done, it's safe to re-enable interrupts. */
547	sti
548#endif /* CONFIG_X86_KPTI */
549#endif /* CONFIG_USERSPACE */
550
551	/* In addition to r11, push the rest of the caller-saved regs */
552	/* Positioning of this fxsave is important, RSP must be 16-byte
553	 * aligned
554	 */
555	subq $X86_FXSAVE_SIZE, %rsp
556	fxsave (%rsp)
557	pushq %r10
558	pushq %r9
559	pushq %r8
560	pushq %rdi
561	pushq %rsi
562	pushq %rdx
563	pushq %rcx
564	pushq %rax
565	pushq %rbp
566#ifdef CONFIG_EXCEPTION_DEBUG
567	/* Callee saved regs */
568	pushq %r15
569	pushq %r14
570	pushq %r13
571	pushq %r12
572	pushq %rbx
573#endif /* CONFIG_EXCEPTION_DEBUG */
574	movq %rsp, %rdi
575
576	call z_x86_exception
577
578	/* If we returned, the exception was handled successfully and the
579	 * thread may resume (the pushed RIP may have been modified)
580	 */
581#ifdef CONFIG_EXCEPTION_DEBUG
582	popq %rbx
583	popq %r12
584	popq %r13
585	popq %r14
586	popq %r15
587#endif /* CONFIG_EXCEPTION_DEBUG */
588	popq %rbp
589	popq %rax
590	popq %rcx
591	popq %rdx
592	popq %rsi
593	popq %rdi
594	popq %r8
595	popq %r9
596	popq %r10
597	fxrstor (%rsp)
598	addq $X86_FXSAVE_SIZE, %rsp
599	popq %r11
600
601	/* Drop the vector/err code pushed by the HW or EXCEPT_*() stub */
602	add $16, %rsp
603
604#ifdef CONFIG_USERSPACE
605	/* Swap GS register values if we are returning to user mode */
606	testb $0x3, 8(%rsp)
607	jz 1f
608	cli
609#ifdef CONFIG_X86_KPTI
610	jmp z_x86_trampoline_to_user
611#else
612	swapgs
613#endif /* CONFIG_X86_KPTI */
6141:
615#endif /* CONFIG_USERSPACE */
616
617	iretq
618
619#ifdef CONFIG_X86_KPTI
620EXCEPT      ( 0, 7); EXCEPT      ( 1, 7); EXCEPT       (2, 6); EXCEPT      ( 3, 7)
621EXCEPT      ( 4, 7); EXCEPT      ( 5, 7); EXCEPT       (6, 7); EXCEPT      ( 7, 7)
622EXCEPT_CODE ( 8, 7); EXCEPT      ( 9, 7); EXCEPT_CODE (10, 7); EXCEPT_CODE (11, 7)
623EXCEPT_CODE (12, 7); EXCEPT_CODE (13, 7); EXCEPT_CODE (14, 7); EXCEPT      (15, 7)
624EXCEPT      (16, 7); EXCEPT_CODE (17, 7); EXCEPT      (18, 7); EXCEPT      (19, 7)
625EXCEPT      (20, 7); EXCEPT      (21, 7); EXCEPT      (22, 7); EXCEPT      (23, 7)
626EXCEPT      (24, 7); EXCEPT      (25, 7); EXCEPT      (26, 7); EXCEPT      (27, 7)
627EXCEPT      (28, 7); EXCEPT      (29, 7); EXCEPT      (30, 7); EXCEPT      (31, 7)
628
629/* Vector reserved for handling a kernel oops; treat as an exception
630 * and not an interrupt
631 */
632EXCEPT(Z_X86_OOPS_VECTOR, 7);
633#else
634EXCEPT      ( 0); EXCEPT      ( 1); EXCEPT      ( 2); EXCEPT      ( 3)
635EXCEPT      ( 4); EXCEPT      ( 5); EXCEPT      ( 6); EXCEPT      ( 7)
636EXCEPT_CODE ( 8); EXCEPT      ( 9); EXCEPT_CODE (10); EXCEPT_CODE (11)
637EXCEPT_CODE (12); EXCEPT_CODE (13); EXCEPT_CODE (14); EXCEPT      (15)
638EXCEPT      (16); EXCEPT_CODE (17); EXCEPT      (18); EXCEPT      (19)
639EXCEPT      (20); EXCEPT      (21); EXCEPT      (22); EXCEPT      (23)
640EXCEPT      (24); EXCEPT      (25); EXCEPT      (26); EXCEPT      (27)
641EXCEPT      (28); EXCEPT      (29); EXCEPT      (30); EXCEPT      (31)
642
643/* Vector reserved for handling a kernel oops; treat as an exception
644 * and not an interrupt
645 */
646EXCEPT(Z_X86_OOPS_VECTOR);
647#endif /* CONFIG_X86_KPTI */
648
649/*
650 * When we arrive at 'irq' from one of the IRQ(X) stubs,
651 * we're on the "freshest" IRQ stack (or the trampoline stack if we came from
652 * user mode and KPTI is enabled) and it contains:
653 *
654 *   SS
655 *   RSP
656 *   RFLAGS
657 *   CS
658 *   RIP
659 *   (vector number - IV_IRQS) <-- RSP points here
660 */
661
662.globl x86_irq_funcs	/* see irq_manage.c .. */
663.globl x86_irq_args	/* .. for these definitions */
664
665irq:
666	pushq %rsi
667
668#ifdef CONFIG_USERSPACE
669	/* Swap GS register values if we came in from user mode */
670	testb $0x3, 24(%rsp)
671	jz 1f
672	swapgs
673#ifdef CONFIG_X86_KPTI
674	/* Load kernel's page table. NOTE: presumes phys=virt */
675	movq $z_x86_kernel_ptables, %rsi
676	movq %rsi, %cr3
677#endif /* CONFIG_X86_KPTI */
6781:
679#ifdef CONFIG_X86_BOUNDS_CHECK_BYPASS_MITIGATION
680	/* swapgs variant of Spectre V1. Disable speculation past this point */
681	lfence
682#endif /* CONFIG_X86_BOUNDS_CHECK_BYPASS_MITIGATION */
683#ifdef CONFIG_X86_KPTI
684	/* Save old trampoline stack pointer in RSI */
685	movq %rsp, %rsi
686
687	/* Switch to the interrupt stack stack */
688	movq %gs:__x86_tss64_t_ist1_OFFSET, %rsp
689
690	/* Transplant trampoline stack contents */
691	pushq 48(%rsi)	/* SS */
692	pushq 40(%rsi)	/* RSP */
693	pushq 32(%rsi)	/* RFLAGS */
694	pushq 24(%rsi)	/* CS */
695	pushq 16(%rsi)	/* RIP */
696	pushq 8(%rsi)	/* Vector */
697	pushq (%rsi)	/* Stashed RSI value */
698	movq $0, (%rsi) /* Cover our tracks, stashed RSI might be sensitive */
699#endif /* CONFIG_X86_KPTI */
700#endif /* CONFIG_USERSPACE */
701
702	movq %gs:__x86_tss64_t_cpu_OFFSET, %rsi
703
704	/*
705	 * Bump the IRQ nesting count and move to the next IRQ stack.
706	 * That's sufficient to safely re-enable interrupts, so if we
707	 * haven't reached the maximum nesting depth yet, do it.
708	 */
709
710	incl ___cpu_t_nested_OFFSET(%rsi)
711	subq $CONFIG_ISR_SUBSTACK_SIZE, %gs:__x86_tss64_t_ist1_OFFSET
712	cmpl $CONFIG_ISR_DEPTH, ___cpu_t_nested_OFFSET(%rsi)
713	jz 1f
714	sti
7151:	cmpl $1, ___cpu_t_nested_OFFSET(%rsi)
716	je irq_enter_unnested
717
718	/*
719	 * if we're a nested interrupt, we have to dump the state to the
720	 * stack. we play some games here to re-arrange the stack thusly:
721	 *
722	 * SS RSP RFLAGS CS RIP RAX RSI
723	 * RCX RDX RDI R8 R9 R10 R11
724	 * X86_FXSAVE_SIZE bytes of SSE data <-- RSP points here
725	 *
726	 * note that the final value of RSP must be 16-byte aligned here,
727	 * both to satisfy FXSAVE/FXRSTOR but also to honor the C ABI.
728	 */
729
730irq_enter_nested: /* Nested IRQ: dump register state to stack. */
731	pushq %rcx
732	movq 16(%rsp), %rcx /* RCX = vector */
733	movq %rax, 16(%rsp) /* looks like we pushed RAX, not the vector */
734	pushq %rdx
735	pushq %rdi
736	pushq %r8
737	pushq %r9
738	pushq %r10
739	pushq %r11
740	subq $X86_FXSAVE_SIZE, %rsp
741	fxsave (%rsp)
742	jmp irq_dispatch
743
744irq_enter_unnested: /* Not nested: dump state to thread struct for __resume */
745	movq ___cpu_t_current_OFFSET(%rsi), %rsi
746	orb $X86_THREAD_FLAG_ALL, _thread_offset_to_flags(%rsi)
747	fxsave _thread_offset_to_sse(%rsi)
748	movq %rbx, _thread_offset_to_rbx(%rsi)
749	movq %rbp, _thread_offset_to_rbp(%rsi)
750	movq %r12, _thread_offset_to_r12(%rsi)
751	movq %r13, _thread_offset_to_r13(%rsi)
752	movq %r14, _thread_offset_to_r14(%rsi)
753	movq %r15, _thread_offset_to_r15(%rsi)
754	movq %rax, _thread_offset_to_rax(%rsi)
755	movq %rcx, _thread_offset_to_rcx(%rsi)
756	movq %rdx, _thread_offset_to_rdx(%rsi)
757	movq %rdi, _thread_offset_to_rdi(%rsi)
758	movq %r8, _thread_offset_to_r8(%rsi)
759	movq %r9, _thread_offset_to_r9(%rsi)
760	movq %r10, _thread_offset_to_r10(%rsi)
761	movq %r11, _thread_offset_to_r11(%rsi)
762	popq %rax /* RSI */
763	movq %rax, _thread_offset_to_rsi(%rsi)
764	popq %rcx /* vector number */
765	popq %rax /* RIP */
766	movq %rax, _thread_offset_to_rip(%rsi)
767	popq %rax /* CS */
768#ifdef CONFIG_USERSPACE
769	movq %rax, _thread_offset_to_cs(%rsi)
770#endif
771	popq %rax /* RFLAGS */
772	movq %rax, _thread_offset_to_rflags(%rsi)
773	popq %rax /* RSP */
774	movq %rax, _thread_offset_to_rsp(%rsi)
775	popq %rax /* SS */
776#ifdef CONFIG_USERSPACE
777	movq %rax, _thread_offset_to_ss(%rsi)
778#endif
779
780irq_dispatch:
781	movq x86_irq_funcs(,%rcx,8), %rax
782	movq x86_irq_args(,%rcx,8), %rdi
783	call *%rax
784
785	xorq %rax, %rax
786#ifdef CONFIG_X2APIC
787	xorl %edx, %edx
788	movl $(X86_X2APIC_BASE_MSR + (LOAPIC_EOI >> 4)), %ecx
789	wrmsr
790#else /* xAPIC */
791	movq z_loapic_regs, %rdx
792	movl %eax, LOAPIC_EOI(%rdx)
793#endif /* CONFIG_X2APIC */
794
795	movq %gs:__x86_tss64_t_cpu_OFFSET, %rsi
796
797	cli
798	addq $CONFIG_ISR_SUBSTACK_SIZE, %gs:__x86_tss64_t_ist1_OFFSET
799	decl ___cpu_t_nested_OFFSET(%rsi)
800	jnz irq_exit_nested
801
802	/* not nested; ask the scheduler who's up next and resume it */
803
804	movq ___cpu_t_current_OFFSET(%rsi), %rdi
805	call z_get_next_switch_handle
806	movq %rax, %rdi
807	jmp __resume
808
809irq_exit_nested:
810	fxrstor (%rsp)
811	addq $X86_FXSAVE_SIZE, %rsp
812	popq %r11
813	popq %r10
814	popq %r9
815	popq %r8
816	popq %rdi
817	popq %rdx
818	popq %rcx
819	popq %rsi
820	popq %rax
821	iretq
822
823#define IRQ(nr) vector_ ## nr: pushq $(nr - IV_IRQS); jmp irq
824
825IRQ( 33); IRQ( 34); IRQ( 35); IRQ( 36); IRQ( 37); IRQ( 38); IRQ( 39)
826IRQ( 40); IRQ( 41); IRQ( 42); IRQ( 43); IRQ( 44); IRQ( 45); IRQ( 46); IRQ( 47)
827IRQ( 48); IRQ( 49); IRQ( 50); IRQ( 51); IRQ( 52); IRQ( 53); IRQ( 54); IRQ( 55)
828IRQ( 56); IRQ( 57); IRQ( 58); IRQ( 59); IRQ( 60); IRQ( 61); IRQ( 62); IRQ( 63)
829IRQ( 64); IRQ( 65); IRQ( 66); IRQ( 67); IRQ( 68); IRQ( 69); IRQ( 70); IRQ( 71)
830IRQ( 72); IRQ( 73); IRQ( 74); IRQ( 75); IRQ( 76); IRQ( 77); IRQ( 78); IRQ( 79)
831IRQ( 80); IRQ( 81); IRQ( 82); IRQ( 83); IRQ( 84); IRQ( 85); IRQ( 86); IRQ( 87)
832IRQ( 88); IRQ( 89); IRQ( 90); IRQ( 91); IRQ( 92); IRQ( 93); IRQ( 94); IRQ( 95)
833IRQ( 96); IRQ( 97); IRQ( 98); IRQ( 99); IRQ(100); IRQ(101); IRQ(102); IRQ(103)
834IRQ(104); IRQ(105); IRQ(106); IRQ(107); IRQ(108); IRQ(109); IRQ(110); IRQ(111)
835IRQ(112); IRQ(113); IRQ(114); IRQ(115); IRQ(116); IRQ(117); IRQ(118); IRQ(119)
836IRQ(120); IRQ(121); IRQ(122); IRQ(123); IRQ(124); IRQ(125); IRQ(126); IRQ(127)
837IRQ(128); IRQ(129); IRQ(130); IRQ(131); IRQ(132); IRQ(133); IRQ(134); IRQ(135)
838IRQ(136); IRQ(137); IRQ(138); IRQ(139); IRQ(140); IRQ(141); IRQ(142); IRQ(143)
839IRQ(144); IRQ(145); IRQ(146); IRQ(147); IRQ(148); IRQ(149); IRQ(150); IRQ(151)
840IRQ(152); IRQ(153); IRQ(154); IRQ(155); IRQ(156); IRQ(157); IRQ(158); IRQ(159)
841IRQ(160); IRQ(161); IRQ(162); IRQ(163); IRQ(164); IRQ(165); IRQ(166); IRQ(167)
842IRQ(168); IRQ(169); IRQ(170); IRQ(171); IRQ(172); IRQ(173); IRQ(174); IRQ(175)
843IRQ(176); IRQ(177); IRQ(178); IRQ(179); IRQ(180); IRQ(181); IRQ(182); IRQ(183)
844IRQ(184); IRQ(185); IRQ(186); IRQ(187); IRQ(188); IRQ(189); IRQ(190); IRQ(191)
845IRQ(192); IRQ(193); IRQ(194); IRQ(195); IRQ(196); IRQ(197); IRQ(198); IRQ(199)
846IRQ(200); IRQ(201); IRQ(202); IRQ(203); IRQ(204); IRQ(205); IRQ(206); IRQ(207)
847IRQ(208); IRQ(209); IRQ(210); IRQ(211); IRQ(212); IRQ(213); IRQ(214); IRQ(215)
848IRQ(216); IRQ(217); IRQ(218); IRQ(219); IRQ(220); IRQ(221); IRQ(222); IRQ(223)
849IRQ(224); IRQ(225); IRQ(226); IRQ(227); IRQ(228); IRQ(229); IRQ(230); IRQ(231)
850IRQ(232); IRQ(233); IRQ(234); IRQ(235); IRQ(236); IRQ(237); IRQ(238); IRQ(239)
851IRQ(240); IRQ(241); IRQ(242); IRQ(243); IRQ(244); IRQ(245); IRQ(246); IRQ(247)
852IRQ(248); IRQ(249); IRQ(250); IRQ(251); IRQ(252); IRQ(253); IRQ(254); IRQ(255)
853
854.section .lorodata,"a"
855
856/*
857 * IDT.
858 */
859
860/* Descriptor type. Traps don't implicitly disable interrupts. User variants
861 * can be invoked by software running in user mode (ring 3).
862 *
863 * For KPTI everything lands on the trampoline stack and we must get off of
864 * it before re-enabling interrupts; use interrupt gates for everything.
865 */
866#define INTR		0x8e
867#define USER_INTR	0xee
868#ifdef CONFIG_X86_KPTI
869#define TRAP		INTR
870#define USER_TRAP	UINTR
871#else
872#define TRAP		0x8f
873#define USER_TRAP	0xef
874#endif
875
876#define IDT(nr, type, ist) \
877	.word vector_ ## nr, X86_KERNEL_CS; \
878	.byte ist, type; \
879	.word 0, 0, 0, 0, 0
880
881/* Which IST entry in TSS to use for automatic stack switching, or 0 if
882 * no automatic switch is to take place. Stack page must be present in
883 * the current page tables, if KPTI is on only the trampoline stack and
884 * the current user stack can be accessed.
885 */
886#ifdef CONFIG_X86_KPTI
887/* Everything lands on ist2, which is set to the trampoline stack.
888 * Interrupt/exception entry updates page tables and manually switches to
889 * the irq/exception stacks stored in ist1/ist7
890 */
891#define	IRQ_STACK	2
892#define EXC_STACK	2
893#define BAD_STACK	2
894#define NMI_STACK	2
895#else
896#define	IRQ_STACK	1
897#define NMI_STACK	6 /* NMI stack */
898#define EXC_STACK	7
899#define BAD_STACK	7 /* Horrible things: double faults, MCEs */
900#endif
901
902.align 16
903idt:
904	IDT(  0, TRAP, EXC_STACK); IDT(  1, TRAP, EXC_STACK)
905	IDT(  2, TRAP, NMI_STACK); IDT(  3, TRAP, EXC_STACK)
906	IDT(  4, TRAP, EXC_STACK); IDT(  5, TRAP, EXC_STACK)
907	IDT(  6, TRAP, EXC_STACK); IDT(  7, TRAP, EXC_STACK)
908	IDT(  8, TRAP, BAD_STACK); IDT(  9, TRAP, EXC_STACK)
909	IDT( 10, TRAP, EXC_STACK); IDT( 11, TRAP, EXC_STACK)
910	IDT( 12, TRAP, EXC_STACK); IDT( 13, TRAP, EXC_STACK)
911	IDT( 14, TRAP, EXC_STACK); IDT( 15, TRAP, EXC_STACK)
912	IDT( 16, TRAP, EXC_STACK); IDT( 17, TRAP, EXC_STACK)
913	IDT( 18, TRAP, BAD_STACK); IDT( 19, TRAP, EXC_STACK)
914	IDT( 20, TRAP, EXC_STACK); IDT( 21, TRAP, EXC_STACK)
915	IDT( 22, TRAP, EXC_STACK); IDT( 23, TRAP, EXC_STACK)
916	IDT( 24, TRAP, EXC_STACK); IDT( 25, TRAP, EXC_STACK)
917	IDT( 26, TRAP, EXC_STACK); IDT( 27, TRAP, EXC_STACK)
918	IDT( 28, TRAP, EXC_STACK); IDT( 29, TRAP, EXC_STACK)
919	IDT( 30, TRAP, EXC_STACK); IDT( 31, TRAP, EXC_STACK)
920
921	/* Oops vector can be invoked from Ring 3 and runs on exception stack */
922	IDT(Z_X86_OOPS_VECTOR, USER_INTR, EXC_STACK); IDT( 33, INTR, IRQ_STACK)
923	IDT( 34, INTR, IRQ_STACK); IDT( 35, INTR, IRQ_STACK)
924	IDT( 36, INTR, IRQ_STACK); IDT( 37, INTR, IRQ_STACK)
925	IDT( 38, INTR, IRQ_STACK); IDT( 39, INTR, IRQ_STACK)
926	IDT( 40, INTR, IRQ_STACK); IDT( 41, INTR, IRQ_STACK)
927	IDT( 42, INTR, IRQ_STACK); IDT( 43, INTR, IRQ_STACK)
928	IDT( 44, INTR, IRQ_STACK); IDT( 45, INTR, IRQ_STACK)
929	IDT( 46, INTR, IRQ_STACK); IDT( 47, INTR, IRQ_STACK)
930	IDT( 48, INTR, IRQ_STACK); IDT( 49, INTR, IRQ_STACK)
931	IDT( 50, INTR, IRQ_STACK); IDT( 51, INTR, IRQ_STACK)
932	IDT( 52, INTR, IRQ_STACK); IDT( 53, INTR, IRQ_STACK)
933	IDT( 54, INTR, IRQ_STACK); IDT( 55, INTR, IRQ_STACK)
934	IDT( 56, INTR, IRQ_STACK); IDT( 57, INTR, IRQ_STACK)
935	IDT( 58, INTR, IRQ_STACK); IDT( 59, INTR, IRQ_STACK)
936	IDT( 60, INTR, IRQ_STACK); IDT( 61, INTR, IRQ_STACK)
937	IDT( 62, INTR, IRQ_STACK); IDT( 63, INTR, IRQ_STACK)
938	IDT( 64, INTR, IRQ_STACK); IDT( 65, INTR, IRQ_STACK)
939	IDT( 66, INTR, IRQ_STACK); IDT( 67, INTR, IRQ_STACK)
940	IDT( 68, INTR, IRQ_STACK); IDT( 69, INTR, IRQ_STACK)
941	IDT( 70, INTR, IRQ_STACK); IDT( 71, INTR, IRQ_STACK)
942	IDT( 72, INTR, IRQ_STACK); IDT( 73, INTR, IRQ_STACK)
943	IDT( 74, INTR, IRQ_STACK); IDT( 75, INTR, IRQ_STACK)
944	IDT( 76, INTR, IRQ_STACK); IDT( 77, INTR, IRQ_STACK)
945	IDT( 78, INTR, IRQ_STACK); IDT( 79, INTR, IRQ_STACK)
946	IDT( 80, INTR, IRQ_STACK); IDT( 81, INTR, IRQ_STACK)
947	IDT( 82, INTR, IRQ_STACK); IDT( 83, INTR, IRQ_STACK)
948	IDT( 84, INTR, IRQ_STACK); IDT( 85, INTR, IRQ_STACK)
949	IDT( 86, INTR, IRQ_STACK); IDT( 87, INTR, IRQ_STACK)
950	IDT( 88, INTR, IRQ_STACK); IDT( 89, INTR, IRQ_STACK)
951	IDT( 90, INTR, IRQ_STACK); IDT( 91, INTR, IRQ_STACK)
952	IDT( 92, INTR, IRQ_STACK); IDT( 93, INTR, IRQ_STACK)
953	IDT( 94, INTR, IRQ_STACK); IDT( 95, INTR, IRQ_STACK)
954	IDT( 96, INTR, IRQ_STACK); IDT( 97, INTR, IRQ_STACK)
955	IDT( 98, INTR, IRQ_STACK); IDT( 99, INTR, IRQ_STACK)
956	IDT(100, INTR, IRQ_STACK); IDT(101, INTR, IRQ_STACK)
957	IDT(102, INTR, IRQ_STACK); IDT(103, INTR, IRQ_STACK)
958	IDT(104, INTR, IRQ_STACK); IDT(105, INTR, IRQ_STACK)
959	IDT(106, INTR, IRQ_STACK); IDT(107, INTR, IRQ_STACK)
960	IDT(108, INTR, IRQ_STACK); IDT(109, INTR, IRQ_STACK)
961	IDT(110, INTR, IRQ_STACK); IDT(111, INTR, IRQ_STACK)
962	IDT(112, INTR, IRQ_STACK); IDT(113, INTR, IRQ_STACK)
963	IDT(114, INTR, IRQ_STACK); IDT(115, INTR, IRQ_STACK)
964	IDT(116, INTR, IRQ_STACK); IDT(117, INTR, IRQ_STACK)
965	IDT(118, INTR, IRQ_STACK); IDT(119, INTR, IRQ_STACK)
966	IDT(120, INTR, IRQ_STACK); IDT(121, INTR, IRQ_STACK)
967	IDT(122, INTR, IRQ_STACK); IDT(123, INTR, IRQ_STACK)
968	IDT(124, INTR, IRQ_STACK); IDT(125, INTR, IRQ_STACK)
969	IDT(126, INTR, IRQ_STACK); IDT(127, INTR, IRQ_STACK)
970	IDT(128, INTR, IRQ_STACK); IDT(129, INTR, IRQ_STACK)
971	IDT(130, INTR, IRQ_STACK); IDT(131, INTR, IRQ_STACK)
972	IDT(132, INTR, IRQ_STACK); IDT(133, INTR, IRQ_STACK)
973	IDT(134, INTR, IRQ_STACK); IDT(135, INTR, IRQ_STACK)
974	IDT(136, INTR, IRQ_STACK); IDT(137, INTR, IRQ_STACK)
975	IDT(138, INTR, IRQ_STACK); IDT(139, INTR, IRQ_STACK)
976	IDT(140, INTR, IRQ_STACK); IDT(141, INTR, IRQ_STACK)
977	IDT(142, INTR, IRQ_STACK); IDT(143, INTR, IRQ_STACK)
978	IDT(144, INTR, IRQ_STACK); IDT(145, INTR, IRQ_STACK)
979	IDT(146, INTR, IRQ_STACK); IDT(147, INTR, IRQ_STACK)
980	IDT(148, INTR, IRQ_STACK); IDT(149, INTR, IRQ_STACK)
981	IDT(150, INTR, IRQ_STACK); IDT(151, INTR, IRQ_STACK)
982	IDT(152, INTR, IRQ_STACK); IDT(153, INTR, IRQ_STACK)
983	IDT(154, INTR, IRQ_STACK); IDT(155, INTR, IRQ_STACK)
984	IDT(156, INTR, IRQ_STACK); IDT(157, INTR, IRQ_STACK)
985	IDT(158, INTR, IRQ_STACK); IDT(159, INTR, IRQ_STACK)
986	IDT(160, INTR, IRQ_STACK); IDT(161, INTR, IRQ_STACK)
987	IDT(162, INTR, IRQ_STACK); IDT(163, INTR, IRQ_STACK)
988	IDT(164, INTR, IRQ_STACK); IDT(165, INTR, IRQ_STACK)
989	IDT(166, INTR, IRQ_STACK); IDT(167, INTR, IRQ_STACK)
990	IDT(168, INTR, IRQ_STACK); IDT(169, INTR, IRQ_STACK)
991	IDT(170, INTR, IRQ_STACK); IDT(171, INTR, IRQ_STACK)
992	IDT(172, INTR, IRQ_STACK); IDT(173, INTR, IRQ_STACK)
993	IDT(174, INTR, IRQ_STACK); IDT(175, INTR, IRQ_STACK)
994	IDT(176, INTR, IRQ_STACK); IDT(177, INTR, IRQ_STACK)
995	IDT(178, INTR, IRQ_STACK); IDT(179, INTR, IRQ_STACK)
996	IDT(180, INTR, IRQ_STACK); IDT(181, INTR, IRQ_STACK)
997	IDT(182, INTR, IRQ_STACK); IDT(183, INTR, IRQ_STACK)
998	IDT(184, INTR, IRQ_STACK); IDT(185, INTR, IRQ_STACK)
999	IDT(186, INTR, IRQ_STACK); IDT(187, INTR, IRQ_STACK)
1000	IDT(188, INTR, IRQ_STACK); IDT(189, INTR, IRQ_STACK)
1001	IDT(190, INTR, IRQ_STACK); IDT(191, INTR, IRQ_STACK)
1002	IDT(192, INTR, IRQ_STACK); IDT(193, INTR, IRQ_STACK)
1003	IDT(194, INTR, IRQ_STACK); IDT(195, INTR, IRQ_STACK)
1004	IDT(196, INTR, IRQ_STACK); IDT(197, INTR, IRQ_STACK)
1005	IDT(198, INTR, IRQ_STACK); IDT(199, INTR, IRQ_STACK)
1006	IDT(200, INTR, IRQ_STACK); IDT(201, INTR, IRQ_STACK)
1007	IDT(202, INTR, IRQ_STACK); IDT(203, INTR, IRQ_STACK)
1008	IDT(204, INTR, IRQ_STACK); IDT(205, INTR, IRQ_STACK)
1009	IDT(206, INTR, IRQ_STACK); IDT(207, INTR, IRQ_STACK)
1010	IDT(208, INTR, IRQ_STACK); IDT(209, INTR, IRQ_STACK)
1011	IDT(210, INTR, IRQ_STACK); IDT(211, INTR, IRQ_STACK)
1012	IDT(212, INTR, IRQ_STACK); IDT(213, INTR, IRQ_STACK)
1013	IDT(214, INTR, IRQ_STACK); IDT(215, INTR, IRQ_STACK)
1014	IDT(216, INTR, IRQ_STACK); IDT(217, INTR, IRQ_STACK)
1015	IDT(218, INTR, IRQ_STACK); IDT(219, INTR, IRQ_STACK)
1016	IDT(220, INTR, IRQ_STACK); IDT(221, INTR, IRQ_STACK)
1017	IDT(222, INTR, IRQ_STACK); IDT(223, INTR, IRQ_STACK)
1018	IDT(224, INTR, IRQ_STACK); IDT(225, INTR, IRQ_STACK)
1019	IDT(226, INTR, IRQ_STACK); IDT(227, INTR, IRQ_STACK)
1020	IDT(228, INTR, IRQ_STACK); IDT(229, INTR, IRQ_STACK)
1021	IDT(230, INTR, IRQ_STACK); IDT(231, INTR, IRQ_STACK)
1022	IDT(232, INTR, IRQ_STACK); IDT(233, INTR, IRQ_STACK)
1023	IDT(234, INTR, IRQ_STACK); IDT(235, INTR, IRQ_STACK)
1024	IDT(236, INTR, IRQ_STACK); IDT(237, INTR, IRQ_STACK)
1025	IDT(238, INTR, IRQ_STACK); IDT(239, INTR, IRQ_STACK)
1026	IDT(240, INTR, IRQ_STACK); IDT(241, INTR, IRQ_STACK)
1027	IDT(242, INTR, IRQ_STACK); IDT(243, INTR, IRQ_STACK)
1028	IDT(244, INTR, IRQ_STACK); IDT(245, INTR, IRQ_STACK)
1029	IDT(246, INTR, IRQ_STACK); IDT(247, INTR, IRQ_STACK)
1030	IDT(248, INTR, IRQ_STACK); IDT(249, INTR, IRQ_STACK)
1031	IDT(250, INTR, IRQ_STACK); IDT(251, INTR, IRQ_STACK)
1032	IDT(252, INTR, IRQ_STACK); IDT(253, INTR, IRQ_STACK)
1033	IDT(254, INTR, IRQ_STACK); IDT(255, INTR, IRQ_STACK)
1034idt_end:
1035
1036idt48:  /* LIDT descriptor for 32 bit mode */
1037	.word (idt_end - idt - 1)
1038	.long idt
1039
1040idt80:  /* LIDT descriptor for 64 bit mode */
1041	.word (idt_end - idt - 1)
1042	.quad idt
1043
1044.section .gdt,"ad"
1045
1046/*
1047 * GDT - a single GDT is shared by all threads (and, eventually, all CPUs).
1048 * This layout must agree with the selectors in
1049 * include/arch/x86/intel64/thread.h.
1050 *
1051 * The 64-bit kernel code and data segment descriptors must be in sequence as
1052 * required by 'syscall'
1053 *
1054 * The 32-bit user code, 64-bit user code, and 64-bit user data segment
1055 * descriptors must be in sequence as required by 'sysret'
1056 */
1057.align 8
1058
1059gdt:
1060	.word 0,      0, 0,      0	/* 0x00: null descriptor */
1061	.word 0xFFFF, 0, 0x9A00, 0x00CF	/* 0x08: 32-bit kernel code */
1062	.word 0xFFFF, 0, 0x9200, 0x00CF	/* 0x10: 32-bit kernel data */
1063	.word 0,      0, 0x9800, 0x0020	/* 0x18: 64-bit kernel code */
1064	.word 0,      0, 0x9200, 0x0000	/* 0x20: 64-bit kernel data */
1065	.word 0xFFFF, 0, 0xFA00, 0x00CF /* 0x28: 32-bit user code (unused) */
1066	.word 0,      0, 0xF200, 0x0000	/* 0x30: 64-bit user data */
1067	.word 0,      0, 0xF800, 0x0020	/* 0x38: 64-bit user code */
1068
1069	/* Remaining entries are TSS for each enabled CPU */
1070
1071	.word __X86_TSS64_SIZEOF-1	/* 0x40: 64-bit TSS (16-byte entry) */
1072	.word tss0
1073	.word 0x8900
1074	.word 0, 0, 0, 0, 0
1075
1076#if CONFIG_MP_NUM_CPUS > 1
1077	.word __X86_TSS64_SIZEOF-1	/* 0x50: 64-bit TSS (16-byte entry) */
1078	.word tss1
1079	.word 0x8900
1080	.word 0, 0, 0, 0, 0
1081#endif
1082
1083#if CONFIG_MP_NUM_CPUS > 2
1084	.word __X86_TSS64_SIZEOF-1	/* 0x60: 64-bit TSS (16-byte entry) */
1085	.word tss2
1086	.word 0x8900
1087	.word 0, 0, 0, 0, 0
1088#endif
1089
1090#if CONFIG_MP_NUM_CPUS > 3
1091	.word __X86_TSS64_SIZEOF-1	/* 0x70: 64-bit TSS (16-byte entry) */
1092	.word tss3
1093	.word 0x8900
1094	.word 0, 0, 0, 0, 0
1095#endif
1096
1097gdt_end:
1098
1099gdt48:  /* LGDT descriptor for 32 bit mode */
1100	.word (gdt_end - gdt - 1)
1101	.long gdt
1102
1103gdt80:  /* LGDT descriptor for long mode */
1104	.word (gdt_end - gdt - 1)
1105	.quad gdt
1106.section .lodata,"ad"
1107
1108/*
1109 * Known-good stack for handling CPU exceptions.
1110 */
1111
1112.global z_x86_exception_stack
1113.align 16
1114z_x86_exception_stack:
1115	.fill CONFIG_X86_EXCEPTION_STACK_SIZE, 1, 0xAA
1116.global z_x86_nmi_stack
1117.align 16
1118z_x86_nmi_stack:
1119	.fill CONFIG_X86_EXCEPTION_STACK_SIZE, 1, 0xAA
1120
1121#if CONFIG_MP_NUM_CPUS > 1
1122.global z_x86_exception_stack1
1123.align 16
1124z_x86_exception_stack1:
1125	.fill CONFIG_X86_EXCEPTION_STACK_SIZE, 1, 0xAA
1126.global z_x86_nmi_stack1
1127.align 16
1128z_x86_nmi_stack1:
1129	.fill CONFIG_X86_EXCEPTION_STACK_SIZE, 1, 0xAA
1130#endif
1131
1132#if CONFIG_MP_NUM_CPUS > 2
1133.global z_x86_exception_stack2
1134.align 16
1135z_x86_exception_stack2:
1136	.fill CONFIG_X86_EXCEPTION_STACK_SIZE, 1, 0xAA
1137.global z_x86_nmi_stack2
1138.align 16
1139z_x86_nmi_stack2:
1140	.fill CONFIG_X86_EXCEPTION_STACK_SIZE, 1, 0xAA
1141#endif
1142
1143#if CONFIG_MP_NUM_CPUS > 3
1144.global z_x86_exception_stack3
1145.align 16
1146z_x86_exception_stack3:
1147	.fill CONFIG_X86_EXCEPTION_STACK_SIZE, 1, 0xAA
1148.global z_x86_nmi_stack3
1149.align 16
1150z_x86_nmi_stack3:
1151	.fill CONFIG_X86_EXCEPTION_STACK_SIZE, 1, 0xAA
1152#endif
1153
1154#ifdef CONFIG_X86_KPTI
1155.section .trampolines,"ad"
1156
1157.global z_x86_trampoline_stack
1158.align 16
1159z_x86_trampoline_stack:
1160	.fill Z_X86_TRAMPOLINE_STACK_SIZE, 1, 0xAA
1161
1162#if CONFIG_MP_NUM_CPUS > 1
1163.global z_x86_trampoline_stack1
1164.align 16
1165z_x86_trampoline_stack1:
1166	.fill Z_X86_TRAMPOLINE_STACK_SIZE, 1, 0xAA
1167#endif
1168
1169#if CONFIG_MP_NUM_CPUS > 2
1170.global z_x86_trampoline_stack2
1171.align 16
1172z_x86_trampoline_stack2:
1173	.fill Z_X86_TRAMPOLINE_STACK_SIZE, 1, 0xAA
1174#endif
1175
1176#if CONFIG_MP_NUM_CPUS > 3
1177.global z_x86_trampoline_stack3
1178.align 16
1179z_x86_trampoline_stack3:
1180	.fill Z_X86_TRAMPOLINE_STACK_SIZE, 1, 0xAA
1181#endif
1182#endif /* CONFIG_X86_KPTI */
1183