1 /*
2 * Copyright (c) 2011-2014 Wind River Systems, Inc.
3 * Copyright (c) 2017-2020 Intel Corporation
4 *
5 * SPDX-License-Identifier: Apache-2.0
6 *
7 * Internal memory management interfaces implemented in x86_mmu.c.
8 * None of these are application-facing, use only if you know what you are
9 * doing!
10 */
11
12 #ifndef ZEPHYR_ARCH_X86_INCLUDE_X86_MMU_H
13 #define ZEPHYR_ARCH_X86_INCLUDE_X86_MMU_H
14
15 #include <kernel.h>
16 #include <arch/x86/mmustructs.h>
17 #include <sys/mem_manage.h>
18
19 #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
20 #define XD_SUPPORTED
21 #define BITL BIT64
22 #define PRI_ENTRY "0x%016llx"
23 #else
24 #define BITL BIT
25 #define PRI_ENTRY "0x%08x"
26 #endif
27
28 /*
29 * Common flags in the same bit position regardless of which structure level,
30 * although not every flag is supported at every level, and some may be
31 * ignored depending on the state of other bits (such as P or PS)
32 *
33 * These flags indicate bit position, and can be used for setting flags or
34 * masks as needed.
35 */
36
37 #define MMU_P BITL(0) /** Present */
38 #define MMU_RW BITL(1) /** Read-Write */
39 #define MMU_US BITL(2) /** User-Supervisor */
40 #define MMU_PWT BITL(3) /** Page Write Through */
41 #define MMU_PCD BITL(4) /** Page Cache Disable */
42 #define MMU_A BITL(5) /** Accessed */
43 #define MMU_D BITL(6) /** Dirty */
44 #define MMU_PS BITL(7) /** Page Size (non PTE)*/
45 #define MMU_PAT BITL(7) /** Page Attribute (PTE) */
46 #define MMU_G BITL(8) /** Global */
47 #ifdef XD_SUPPORTED
48 #define MMU_XD BITL(63) /** Execute Disable */
49 #else
50 #define MMU_XD 0
51 #endif
52
53 /* Unused PTE bits ignored by the CPU, which we use for our own OS purposes.
54 * These bits ignored for all paging modes.
55 */
56 #define MMU_IGNORED0 BITL(9)
57 #define MMU_IGNORED1 BITL(10)
58 #define MMU_IGNORED2 BITL(11)
59
60 /* Page fault error code flags. See Chapter 4.7 of the Intel SDM vol. 3A. */
61 #define PF_P BIT(0) /* 0 Non-present page 1 Protection violation */
62 #define PF_WR BIT(1) /* 0 Read 1 Write */
63 #define PF_US BIT(2) /* 0 Supervisor mode 1 User mode */
64 #define PF_RSVD BIT(3) /* 1 reserved bit set */
65 #define PF_ID BIT(4) /* 1 instruction fetch */
66 #define PF_PK BIT(5) /* 1 protection-key violation */
67 #define PF_SGX BIT(15) /* 1 SGX-specific access control requirements */
68
69 #ifndef _ASMLANGUAGE
70
71 #ifdef CONFIG_EXCEPTION_DEBUG
72 /**
73 * Dump out page table entries for a particular virtual memory address
74 *
75 * For the provided memory address, dump out interesting information about
76 * its mapping to the error log
77 *
78 * @param ptables Page tables to walk
79 * @param virt Virtual address to inspect
80 */
81 void z_x86_dump_mmu_flags(pentry_t *ptables, void *virt);
82
83 /**
84 * Fetch the page table entry for a virtual memory address
85 *
86 * @param paging_level [out] what paging level the entry was found at.
87 * 0=toplevel
88 * @param val Value stored in page table entry, with address and flags
89 * @param ptables Toplevel pointer to page tables
90 * @param virt Virtual address to lookup
91 */
92 void z_x86_pentry_get(int *paging_level, pentry_t *val, pentry_t *ptables,
93 void *virt);
94
95 /**
96 * Debug function for dumping out page tables
97 *
98 * Iterates through the entire linked set of page table structures,
99 * dumping out codes for the configuration of each table entry.
100 *
101 * Entry codes:
102 *
103 * . - not present
104 * w - present, writable, not executable
105 * a - present, writable, executable
106 * r - present, read-only, not executable
107 * x - present, read-only, executable
108 *
109 * Entry codes in uppercase indicate that user mode may access.
110 *
111 * Color is used to indicate the physical mapping characteristics:
112 *
113 * yellow - Identity mapping (virt = phys)
114 * green - Fixed virtual memory mapping (virt = phys + constant)
115 * magenta - entry is child page table
116 * cyan - General mapped memory
117 *
118 * @param ptables Top-level pointer to the page tables, as programmed in CR3
119 */
120 void z_x86_dump_page_tables(pentry_t *ptables);
121 #endif /* CONFIG_EXCEPTION_DEBUG */
122
123 #ifdef CONFIG_HW_STACK_PROTECTION
124 /* Legacy function - set identity-mapped MMU stack guard page to RO in the
125 * kernel's page tables to prevent writes and generate an exception
126 */
127 void z_x86_set_stack_guard(k_thread_stack_t *stack);
128 #endif
129
130 #ifdef CONFIG_USERSPACE
131 #ifdef CONFIG_X86_KPTI
132 /* Defined in linker script. Contains all the data that must be mapped
133 * in a KPTI table even though US bit is not set (trampoline stack, GDT,
134 * IDT, etc)
135 */
136 extern uint8_t z_shared_kernel_page_start;
137
138 #ifdef CONFIG_DEMAND_PAGING
139 /* Called from page fault handler. ptables here is the ptage tables for the
140 * faulting user thread and not the current set of page tables
141 */
z_x86_kpti_is_access_ok(void * virt,pentry_t * ptables)142 extern bool z_x86_kpti_is_access_ok(void *virt, pentry_t *ptables)
143 #endif /* CONFIG_DEMAND_PAGING */
144 #endif /* CONFIG_X86_KPTI */
145 #endif /* CONFIG_USERSPACE */
146
147 #ifdef CONFIG_X86_PAE
148 #define PTABLES_ALIGN 0x1fU
149 #else
150 #define PTABLES_ALIGN 0xfffU
151 #endif
152
153 /* Set CR3 to a physical address. There must be a valid top-level paging
154 * structure here or the CPU will triple fault. The incoming page tables must
155 * have the same kernel mappings wrt supervisor mode. Don't use this function
156 * unless you know exactly what you are doing.
157 */
158 static inline void z_x86_cr3_set(uintptr_t phys)
159 {
160 __ASSERT((phys & PTABLES_ALIGN) == 0U, "unaligned page tables");
161 #ifdef CONFIG_X86_64
162 __asm__ volatile("movq %0, %%cr3\n\t" : : "r" (phys) : "memory");
163 #else
164 __asm__ volatile("movl %0, %%cr3\n\t" : : "r" (phys) : "memory");
165 #endif
166 }
167
168 /* Return cr3 value, which is the physical (not virtual) address of the
169 * current set of page tables
170 */
z_x86_cr3_get(void)171 static inline uintptr_t z_x86_cr3_get(void)
172 {
173 uintptr_t cr3;
174 #ifdef CONFIG_X86_64
175 __asm__ volatile("movq %%cr3, %0\n\t" : "=r" (cr3));
176 #else
177 __asm__ volatile("movl %%cr3, %0\n\t" : "=r" (cr3));
178 #endif
179 return cr3;
180 }
181
182 /* Return the virtual address of the page tables installed in this CPU in CR3 */
z_x86_page_tables_get(void)183 static inline pentry_t *z_x86_page_tables_get(void)
184 {
185 return z_mem_virt_addr(z_x86_cr3_get());
186 }
187
188 /* Return cr2 value, which contains the page fault linear address.
189 * See Section 6.15 of the IA32 Software Developer's Manual vol 3.
190 * Used by page fault handling code.
191 */
z_x86_cr2_get(void)192 static inline void *z_x86_cr2_get(void)
193 {
194 void *cr2;
195 #ifdef CONFIG_X86_64
196 __asm__ volatile("movq %%cr2, %0\n\t" : "=r" (cr2));
197 #else
198 __asm__ volatile("movl %%cr2, %0\n\t" : "=r" (cr2));
199 #endif
200 return cr2;
201 }
202
203 /* Kernel's page table. This is in CR3 for all supervisor threads.
204 * if KPTI is enabled, we switch to this when handling exceptions or syscalls
205 */
206 extern pentry_t z_x86_kernel_ptables[];
207
208 /* Get the page tables used by this thread during normal execution */
z_x86_thread_page_tables_get(struct k_thread * thread)209 static inline pentry_t *z_x86_thread_page_tables_get(struct k_thread *thread)
210 {
211 #if defined(CONFIG_USERSPACE) && !defined(CONFIG_X86_COMMON_PAGE_TABLE)
212 if (!IS_ENABLED(CONFIG_X86_KPTI) ||
213 (thread->base.user_options & K_USER) != 0U) {
214 /* If KPTI is enabled, supervisor threads always use
215 * the kernel's page tables and not the page tables associated
216 * with their memory domain.
217 */
218 return z_mem_virt_addr(thread->arch.ptables);
219 }
220 #endif
221 return z_x86_kernel_ptables;
222 }
223
224 #ifdef CONFIG_SMP
225 /* Handling function for TLB shootdown inter-processor interrupts. */
226 void z_x86_tlb_ipi(const void *arg);
227 #endif
228
229 #ifdef CONFIG_X86_COMMON_PAGE_TABLE
230 void z_x86_swap_update_common_page_table(struct k_thread *incoming);
231 #endif
232
233 /* Early-boot paging setup tasks, called from prep_c */
234 void z_x86_mmu_init(void);
235 #endif /* _ASMLANGUAGE */
236 #endif /* ZEPHYR_ARCH_X86_INCLUDE_X86_MMU_H */
237
