1 /*
2 * Copyright (c) 2022 Intel Corporation
3 * SPDX-License-Identifier: Apache-2.0
4 */
5 #include <zephyr/kernel.h>
6 #include <zephyr/cache.h>
7 #include <zephyr/arch/xtensa/arch.h>
8 #include <zephyr/arch/xtensa/xtensa_mmu.h>
9 #include <zephyr/linker/linker-defs.h>
10 #include <zephyr/logging/log.h>
11 #include <zephyr/kernel/mm.h>
12 #include <zephyr/toolchain.h>
13 #include <xtensa/corebits.h>
14 #include <xtensa_mmu_priv.h>
15
16 #include <kernel_arch_func.h>
17 #include <mmu.h>
18
19 /* Skip TLB IPI when updating page tables.
20 * This allows us to send IPI only after the last
21 * changes of a series.
22 */
23 #define OPTION_NO_TLB_IPI BIT(0)
24
25 /* Level 1 contains page table entries
26 * necessary to map the page table itself.
27 */
28 #define XTENSA_L1_PAGE_TABLE_ENTRIES 1024U
29
30 /* Size of level 1 page table.
31 */
32 #define XTENSA_L1_PAGE_TABLE_SIZE (XTENSA_L1_PAGE_TABLE_ENTRIES * sizeof(uint32_t))
33
34 /* Level 2 contains page table entries
35 * necessary to map the page table itself.
36 */
37 #define XTENSA_L2_PAGE_TABLE_ENTRIES 1024U
38
39 /* Size of level 2 page table.
40 */
41 #define XTENSA_L2_PAGE_TABLE_SIZE (XTENSA_L2_PAGE_TABLE_ENTRIES * sizeof(uint32_t))
42
43 LOG_MODULE_DECLARE(os, CONFIG_KERNEL_LOG_LEVEL);
44
45 BUILD_ASSERT(CONFIG_MMU_PAGE_SIZE == 0x1000,
46 "MMU_PAGE_SIZE value is invalid, only 4 kB pages are supported\n");
47
48 /*
49 * Level 1 page table has to be 4Kb to fit into one of the wired entries.
50 * All entries are initialized as INVALID, so an attempt to read an unmapped
51 * area will cause a double exception.
52 *
53 * Each memory domain contains its own l1 page table. The kernel l1 page table is
54 * located at the index 0.
55 */
56 static uint32_t l1_page_table[CONFIG_XTENSA_MMU_NUM_L1_TABLES][XTENSA_L1_PAGE_TABLE_ENTRIES]
57 __aligned(KB(4));
58
59
60 /*
61 * That is an alias for the page tables set used by the kernel.
62 */
63 uint32_t *xtensa_kernel_ptables = (uint32_t *)l1_page_table[0];
64
65 /*
66 * Each table in the level 2 maps a 4Mb memory range. It consists of 1024 entries each one
67 * covering a 4Kb page.
68 */
69 static uint32_t l2_page_tables[CONFIG_XTENSA_MMU_NUM_L2_TABLES][XTENSA_L2_PAGE_TABLE_ENTRIES]
70 __aligned(KB(4));
71
72 /*
73 * This additional variable tracks which l1 tables are in use. This is kept separated from
74 * the tables to keep alignment easier.
75 *
76 * @note: The first bit is set because it is used for the kernel page tables.
77 */
78 static ATOMIC_DEFINE(l1_page_table_track, CONFIG_XTENSA_MMU_NUM_L1_TABLES);
79
80 /*
81 * This additional variable tracks which l2 tables are in use. This is kept separated from
82 * the tables to keep alignment easier.
83 */
84 static ATOMIC_DEFINE(l2_page_tables_track, CONFIG_XTENSA_MMU_NUM_L2_TABLES);
85
86 /*
87 * Protects xtensa_domain_list and serializes access to page tables.
88 */
89 static struct k_spinlock xtensa_mmu_lock;
90
91 #ifdef CONFIG_USERSPACE
92
93 /*
94 * Each domain has its own ASID. ASID can go through 1 (kernel) to 255.
95 * When a TLB entry matches, the hw will check the ASID in the entry and finds
96 * the correspondent position in the RASID register. This position will then be
97 * compared with the current ring (CRING) to check the permission.
98 */
99 static uint8_t asid_count = 3;
100
101 /*
102 * List with all active and initialized memory domains.
103 */
104 static sys_slist_t xtensa_domain_list;
105 #endif /* CONFIG_USERSPACE */
106
107 extern char _heap_end[];
108 extern char _heap_start[];
109 /*
110 * Static definition of all code & data memory regions of the
111 * current Zephyr image. This information must be available &
112 * processed upon MMU initialization.
113 */
114
115 static const struct xtensa_mmu_range mmu_zephyr_ranges[] = {
116 /*
117 * Mark the zephyr execution regions (data, bss, noinit, etc.)
118 * cacheable, read / write and non-executable
119 */
120 {
121 /* This includes .data, .bss and various kobject sections. */
122 .start = (uint32_t)_image_ram_start,
123 .end = (uint32_t)_image_ram_end,
124 #ifdef CONFIG_XTENSA_RPO_CACHE
125 .attrs = XTENSA_MMU_PERM_W,
126 #else
127 .attrs = XTENSA_MMU_PERM_W | XTENSA_MMU_CACHED_WB,
128 #endif
129 .name = "data",
130 },
131 #if K_HEAP_MEM_POOL_SIZE > 0
132 /* System heap memory */
133 {
134 .start = (uint32_t)_heap_start,
135 .end = (uint32_t)_heap_end,
136 #ifdef CONFIG_XTENSA_RPO_CACHE
137 .attrs = XTENSA_MMU_PERM_W,
138 #else
139 .attrs = XTENSA_MMU_PERM_W | XTENSA_MMU_CACHED_WB,
140 #endif
141 .name = "heap",
142 },
143 #endif
144 /* Mark text segment cacheable, read only and executable */
145 {
146 .start = (uint32_t)__text_region_start,
147 .end = (uint32_t)__text_region_end,
148 .attrs = XTENSA_MMU_PERM_X | XTENSA_MMU_CACHED_WB | XTENSA_MMU_MAP_SHARED,
149 .name = "text",
150 },
151 /* Mark rodata segment cacheable, read only and non-executable */
152 {
153 .start = (uint32_t)__rodata_region_start,
154 .end = (uint32_t)__rodata_region_end,
155 .attrs = XTENSA_MMU_CACHED_WB | XTENSA_MMU_MAP_SHARED,
156 .name = "rodata",
157 },
158 };
159
thread_page_tables_get(const struct k_thread * thread)160 static inline uint32_t *thread_page_tables_get(const struct k_thread *thread)
161 {
162 #ifdef CONFIG_USERSPACE
163 if ((thread->base.user_options & K_USER) != 0U) {
164 return thread->arch.ptables;
165 }
166 #endif
167
168 return xtensa_kernel_ptables;
169 }
170
171 /**
172 * @brief Check if the page table entry is illegal.
173 *
174 * @param[in] Page table entry.
175 */
is_pte_illegal(uint32_t pte)176 static inline bool is_pte_illegal(uint32_t pte)
177 {
178 uint32_t attr = pte & XTENSA_MMU_PTE_ATTR_MASK;
179
180 /*
181 * The ISA manual states only 12 and 14 are illegal values.
182 * 13 and 15 are not. So we need to be specific than simply
183 * testing if bits 2 and 3 are set.
184 */
185 return (attr == 12) || (attr == 14);
186 }
187
188 /*
189 * @brief Initialize all page table entries to be illegal.
190 *
191 * @param[in] Pointer to page table.
192 * @param[in] Number of page table entries in the page table.
193 */
init_page_table(uint32_t * ptable,size_t num_entries)194 static void init_page_table(uint32_t *ptable, size_t num_entries)
195 {
196 int i;
197
198 for (i = 0; i < num_entries; i++) {
199 ptable[i] = XTENSA_MMU_PTE_ILLEGAL;
200 }
201 }
202
alloc_l2_table(void)203 static inline uint32_t *alloc_l2_table(void)
204 {
205 uint16_t idx;
206
207 for (idx = 0; idx < CONFIG_XTENSA_MMU_NUM_L2_TABLES; idx++) {
208 if (!atomic_test_and_set_bit(l2_page_tables_track, idx)) {
209 return (uint32_t *)&l2_page_tables[idx];
210 }
211 }
212
213 return NULL;
214 }
215
map_memory_range(const uint32_t start,const uint32_t end,const uint32_t attrs)216 static void map_memory_range(const uint32_t start, const uint32_t end,
217 const uint32_t attrs)
218 {
219 uint32_t page, *table;
220 bool shared = !!(attrs & XTENSA_MMU_MAP_SHARED);
221 uint32_t sw_attrs = (attrs & XTENSA_MMU_PTE_ATTR_ORIGINAL) == XTENSA_MMU_PTE_ATTR_ORIGINAL ?
222 attrs : 0;
223
224 for (page = start; page < end; page += CONFIG_MMU_PAGE_SIZE) {
225 uint32_t pte = XTENSA_MMU_PTE(page,
226 shared ? XTENSA_MMU_SHARED_RING :
227 XTENSA_MMU_KERNEL_RING,
228 sw_attrs, attrs);
229 uint32_t l2_pos = XTENSA_MMU_L2_POS(page);
230 uint32_t l1_pos = XTENSA_MMU_L1_POS(page);
231
232 if (is_pte_illegal(xtensa_kernel_ptables[l1_pos])) {
233 table = alloc_l2_table();
234
235 __ASSERT(table != NULL, "There is no l2 page table available to "
236 "map 0x%08x\n", page);
237
238 init_page_table(table, XTENSA_L2_PAGE_TABLE_ENTRIES);
239
240 xtensa_kernel_ptables[l1_pos] =
241 XTENSA_MMU_PTE((uint32_t)table, XTENSA_MMU_KERNEL_RING,
242 sw_attrs, XTENSA_MMU_PAGE_TABLE_ATTR);
243 }
244
245 table = (uint32_t *)(xtensa_kernel_ptables[l1_pos] & XTENSA_MMU_PTE_PPN_MASK);
246 table[l2_pos] = pte;
247 }
248 }
249
map_memory(const uint32_t start,const uint32_t end,const uint32_t attrs)250 static void map_memory(const uint32_t start, const uint32_t end,
251 const uint32_t attrs)
252 {
253 map_memory_range(start, end, attrs);
254
255 #ifdef CONFIG_XTENSA_MMU_DOUBLE_MAP
256 if (sys_cache_is_ptr_uncached((void *)start)) {
257 map_memory_range(POINTER_TO_UINT(sys_cache_cached_ptr_get((void *)start)),
258 POINTER_TO_UINT(sys_cache_cached_ptr_get((void *)end)),
259 attrs | XTENSA_MMU_CACHED_WB);
260 } else if (sys_cache_is_ptr_cached((void *)start)) {
261 map_memory_range(POINTER_TO_UINT(sys_cache_uncached_ptr_get((void *)start)),
262 POINTER_TO_UINT(sys_cache_uncached_ptr_get((void *)end)), attrs);
263 }
264 #endif
265 }
266
xtensa_init_page_tables(void)267 static void xtensa_init_page_tables(void)
268 {
269 volatile uint8_t entry;
270
271 init_page_table(xtensa_kernel_ptables, XTENSA_L1_PAGE_TABLE_ENTRIES);
272 atomic_set_bit(l1_page_table_track, 0);
273
274 for (entry = 0; entry < ARRAY_SIZE(mmu_zephyr_ranges); entry++) {
275 const struct xtensa_mmu_range *range = &mmu_zephyr_ranges[entry];
276
277 map_memory(range->start, range->end, range->attrs | XTENSA_MMU_PTE_ATTR_ORIGINAL);
278 }
279
280 for (entry = 0; entry < xtensa_soc_mmu_ranges_num; entry++) {
281 const struct xtensa_mmu_range *range = &xtensa_soc_mmu_ranges[entry];
282
283 map_memory(range->start, range->end, range->attrs | XTENSA_MMU_PTE_ATTR_ORIGINAL);
284 }
285
286 /* Finally, the direct-mapped pages used in the page tables
287 * must be fixed up to use the same cache attribute (but these
288 * must be writable, obviously). They shouldn't be left at
289 * the default.
290 */
291 map_memory_range((uint32_t) &l1_page_table[0],
292 (uint32_t) &l1_page_table[CONFIG_XTENSA_MMU_NUM_L1_TABLES],
293 XTENSA_MMU_PAGE_TABLE_ATTR | XTENSA_MMU_PERM_W);
294 map_memory_range((uint32_t) &l2_page_tables[0],
295 (uint32_t) &l2_page_tables[CONFIG_XTENSA_MMU_NUM_L2_TABLES],
296 XTENSA_MMU_PAGE_TABLE_ATTR | XTENSA_MMU_PERM_W);
297
298 sys_cache_data_flush_all();
299 }
300
arch_xtensa_mmu_post_init(bool is_core0)301 __weak void arch_xtensa_mmu_post_init(bool is_core0)
302 {
303 ARG_UNUSED(is_core0);
304 }
305
xtensa_mmu_init(void)306 void xtensa_mmu_init(void)
307 {
308 if (_current_cpu->id == 0) {
309 /* This is normally done via arch_kernel_init() inside z_cstart().
310 * However, before that is called, we go through the sys_init of
311 * INIT_LEVEL_EARLY, which is going to result in TLB misses.
312 * So setup whatever necessary so the exception handler can work
313 * properly.
314 */
315 xtensa_init_page_tables();
316 }
317
318 xtensa_init_paging(xtensa_kernel_ptables);
319
320 /*
321 * This is used to determine whether we are faulting inside double
322 * exception if this is not zero. Sometimes SoC starts with this not
323 * being set to zero. So clear it during boot.
324 */
325 XTENSA_WSR(ZSR_DEPC_SAVE_STR, 0);
326
327 arch_xtensa_mmu_post_init(_current_cpu->id == 0);
328 }
329
xtensa_mmu_reinit(void)330 void xtensa_mmu_reinit(void)
331 {
332 /* First initialize the hardware */
333 xtensa_init_paging(xtensa_kernel_ptables);
334
335 #ifdef CONFIG_USERSPACE
336 struct k_thread *thread = _current_cpu->current;
337 struct arch_mem_domain *domain =
338 &(thread->mem_domain_info.mem_domain->arch);
339
340
341 /* Set the page table for current context */
342 xtensa_set_paging(domain->asid, domain->ptables);
343 #endif /* CONFIG_USERSPACE */
344
345 arch_xtensa_mmu_post_init(_current_cpu->id == 0);
346 }
347
348 #ifdef CONFIG_ARCH_HAS_RESERVED_PAGE_FRAMES
349 /* Zephyr's linker scripts for Xtensa usually puts
350 * something before z_mapped_start (aka .text),
351 * i.e. vecbase, so that we need to reserve those
352 * space or else k_mem_map() would be mapping those,
353 * resulting in faults.
354 */
arch_reserved_pages_update(void)355 __weak void arch_reserved_pages_update(void)
356 {
357 uintptr_t page;
358 int idx;
359
360 for (page = CONFIG_SRAM_BASE_ADDRESS, idx = 0;
361 page < (uintptr_t)z_mapped_start;
362 page += CONFIG_MMU_PAGE_SIZE, idx++) {
363 k_mem_page_frame_set(&k_mem_page_frames[idx], K_MEM_PAGE_FRAME_RESERVED);
364 }
365 }
366 #endif /* CONFIG_ARCH_HAS_RESERVED_PAGE_FRAMES */
367
l2_page_table_map(uint32_t * l1_table,void * vaddr,uintptr_t phys,uint32_t flags,bool is_user)368 static bool l2_page_table_map(uint32_t *l1_table, void *vaddr, uintptr_t phys,
369 uint32_t flags, bool is_user)
370 {
371 uint32_t l1_pos = XTENSA_MMU_L1_POS((uint32_t)vaddr);
372 uint32_t l2_pos = XTENSA_MMU_L2_POS((uint32_t)vaddr);
373 uint32_t *table;
374
375 sys_cache_data_invd_range((void *)&l1_table[l1_pos], sizeof(l1_table[0]));
376
377 if (is_pte_illegal(l1_table[l1_pos])) {
378 table = alloc_l2_table();
379
380 if (table == NULL) {
381 return false;
382 }
383
384 init_page_table(table, XTENSA_L2_PAGE_TABLE_ENTRIES);
385
386 l1_table[l1_pos] = XTENSA_MMU_PTE((uint32_t)table, XTENSA_MMU_KERNEL_RING,
387 0, XTENSA_MMU_PAGE_TABLE_ATTR);
388
389 sys_cache_data_flush_range((void *)&l1_table[l1_pos], sizeof(l1_table[0]));
390 }
391
392 table = (uint32_t *)(l1_table[l1_pos] & XTENSA_MMU_PTE_PPN_MASK);
393 table[l2_pos] = XTENSA_MMU_PTE(phys, is_user ? XTENSA_MMU_USER_RING :
394 XTENSA_MMU_KERNEL_RING,
395 0, flags);
396
397 sys_cache_data_flush_range((void *)&table[l2_pos], sizeof(table[0]));
398 xtensa_tlb_autorefill_invalidate();
399
400 return true;
401 }
402
__arch_mem_map(void * va,uintptr_t pa,uint32_t xtensa_flags,bool is_user)403 static inline void __arch_mem_map(void *va, uintptr_t pa, uint32_t xtensa_flags, bool is_user)
404 {
405 bool ret;
406 void *vaddr, *vaddr_uc;
407 uintptr_t paddr, paddr_uc;
408 uint32_t flags, flags_uc;
409
410 if (IS_ENABLED(CONFIG_XTENSA_MMU_DOUBLE_MAP)) {
411 if (sys_cache_is_ptr_cached(va)) {
412 vaddr = va;
413 vaddr_uc = sys_cache_uncached_ptr_get(va);
414 } else {
415 vaddr = sys_cache_cached_ptr_get(va);
416 vaddr_uc = va;
417 }
418
419 if (sys_cache_is_ptr_cached((void *)pa)) {
420 paddr = pa;
421 paddr_uc = (uintptr_t)sys_cache_uncached_ptr_get((void *)pa);
422 } else {
423 paddr = (uintptr_t)sys_cache_cached_ptr_get((void *)pa);
424 paddr_uc = pa;
425 }
426
427 flags_uc = (xtensa_flags & ~XTENSA_MMU_PTE_ATTR_CACHED_MASK);
428 flags = flags_uc | XTENSA_MMU_CACHED_WB;
429 } else {
430 vaddr = va;
431 paddr = pa;
432 flags = xtensa_flags;
433 }
434
435 ret = l2_page_table_map(xtensa_kernel_ptables, (void *)vaddr, paddr,
436 flags, is_user);
437 __ASSERT(ret, "Virtual address (%p) already mapped", va);
438
439 if (IS_ENABLED(CONFIG_XTENSA_MMU_DOUBLE_MAP) && ret) {
440 ret = l2_page_table_map(xtensa_kernel_ptables, (void *)vaddr_uc, paddr_uc,
441 flags_uc, is_user);
442 __ASSERT(ret, "Virtual address (%p) already mapped", vaddr_uc);
443 }
444
445 #ifndef CONFIG_USERSPACE
446 ARG_UNUSED(ret);
447 #else
448 if (ret) {
449 sys_snode_t *node;
450 struct arch_mem_domain *domain;
451 k_spinlock_key_t key;
452
453 key = k_spin_lock(&z_mem_domain_lock);
454 SYS_SLIST_FOR_EACH_NODE(&xtensa_domain_list, node) {
455 domain = CONTAINER_OF(node, struct arch_mem_domain, node);
456
457 ret = l2_page_table_map(domain->ptables, (void *)vaddr, paddr,
458 flags, is_user);
459 __ASSERT(ret, "Virtual address (%p) already mapped for domain %p",
460 vaddr, domain);
461
462 if (IS_ENABLED(CONFIG_XTENSA_MMU_DOUBLE_MAP) && ret) {
463 ret = l2_page_table_map(domain->ptables,
464 (void *)vaddr_uc, paddr_uc,
465 flags_uc, is_user);
466 __ASSERT(ret, "Virtual address (%p) already mapped for domain %p",
467 vaddr_uc, domain);
468 }
469 }
470 k_spin_unlock(&z_mem_domain_lock, key);
471 }
472 #endif /* CONFIG_USERSPACE */
473 }
474
arch_mem_map(void * virt,uintptr_t phys,size_t size,uint32_t flags)475 void arch_mem_map(void *virt, uintptr_t phys, size_t size, uint32_t flags)
476 {
477 uint32_t va = (uint32_t)virt;
478 uint32_t pa = (uint32_t)phys;
479 uint32_t rem_size = (uint32_t)size;
480 uint32_t xtensa_flags = 0;
481 k_spinlock_key_t key;
482 bool is_user;
483
484 if (size == 0) {
485 LOG_ERR("Cannot map physical memory at 0x%08X: invalid "
486 "zero size", (uint32_t)phys);
487 k_panic();
488 }
489
490 switch (flags & K_MEM_CACHE_MASK) {
491
492 case K_MEM_CACHE_WB:
493 xtensa_flags |= XTENSA_MMU_CACHED_WB;
494 break;
495 case K_MEM_CACHE_WT:
496 xtensa_flags |= XTENSA_MMU_CACHED_WT;
497 break;
498 case K_MEM_CACHE_NONE:
499 __fallthrough;
500 default:
501 break;
502 }
503
504 if ((flags & K_MEM_PERM_RW) == K_MEM_PERM_RW) {
505 xtensa_flags |= XTENSA_MMU_PERM_W;
506 }
507 if ((flags & K_MEM_PERM_EXEC) == K_MEM_PERM_EXEC) {
508 xtensa_flags |= XTENSA_MMU_PERM_X;
509 }
510
511 is_user = (flags & K_MEM_PERM_USER) == K_MEM_PERM_USER;
512
513 key = k_spin_lock(&xtensa_mmu_lock);
514
515 while (rem_size > 0) {
516 __arch_mem_map((void *)va, pa, xtensa_flags, is_user);
517
518 rem_size -= (rem_size >= KB(4)) ? KB(4) : rem_size;
519 va += KB(4);
520 pa += KB(4);
521 }
522
523 #if CONFIG_MP_MAX_NUM_CPUS > 1
524 xtensa_mmu_tlb_ipi();
525 #endif
526
527 sys_cache_data_flush_and_invd_all();
528 k_spin_unlock(&xtensa_mmu_lock, key);
529 }
530
531 /**
532 * @return True if page is executable (thus need to invalidate ITLB),
533 * false if not.
534 */
l2_page_table_unmap(uint32_t * l1_table,void * vaddr)535 static bool l2_page_table_unmap(uint32_t *l1_table, void *vaddr)
536 {
537 uint32_t l1_pos = XTENSA_MMU_L1_POS((uint32_t)vaddr);
538 uint32_t l2_pos = XTENSA_MMU_L2_POS((uint32_t)vaddr);
539 uint32_t *l2_table;
540 uint32_t table_pos;
541 bool exec;
542
543 sys_cache_data_invd_range((void *)&l1_table[l1_pos], sizeof(l1_table[0]));
544
545 if (is_pte_illegal(l1_table[l1_pos])) {
546 /* We shouldn't be unmapping an illegal entry.
547 * Return true so that we can invalidate ITLB too.
548 */
549 return true;
550 }
551
552 exec = l1_table[l1_pos] & XTENSA_MMU_PERM_X;
553
554 l2_table = (uint32_t *)(l1_table[l1_pos] & XTENSA_MMU_PTE_PPN_MASK);
555
556 sys_cache_data_invd_range((void *)&l2_table[l2_pos], sizeof(l2_table[0]));
557
558 l2_table[l2_pos] = XTENSA_MMU_PTE_ILLEGAL;
559
560 sys_cache_data_flush_range((void *)&l2_table[l2_pos], sizeof(l2_table[0]));
561
562 for (l2_pos = 0; l2_pos < XTENSA_L2_PAGE_TABLE_ENTRIES; l2_pos++) {
563 if (!is_pte_illegal(l2_table[l2_pos])) {
564 goto end;
565 }
566 }
567
568 l1_table[l1_pos] = XTENSA_MMU_PTE_ILLEGAL;
569 sys_cache_data_flush_range((void *)&l1_table[l1_pos], sizeof(l1_table[0]));
570
571 table_pos = (l2_table - (uint32_t *)l2_page_tables) / (XTENSA_L2_PAGE_TABLE_ENTRIES);
572 atomic_clear_bit(l2_page_tables_track, table_pos);
573
574 end:
575 /* Need to invalidate L2 page table as it is no longer valid. */
576 xtensa_tlb_autorefill_invalidate();
577 return exec;
578 }
579
__arch_mem_unmap(void * va)580 static inline void __arch_mem_unmap(void *va)
581 {
582 bool is_exec;
583 void *vaddr, *vaddr_uc;
584
585 if (IS_ENABLED(CONFIG_XTENSA_MMU_DOUBLE_MAP)) {
586 if (sys_cache_is_ptr_cached(va)) {
587 vaddr = va;
588 vaddr_uc = sys_cache_uncached_ptr_get(va);
589 } else {
590 vaddr = sys_cache_cached_ptr_get(va);
591 vaddr_uc = va;
592 }
593 } else {
594 vaddr = va;
595 }
596
597 is_exec = l2_page_table_unmap(xtensa_kernel_ptables, (void *)vaddr);
598
599 if (IS_ENABLED(CONFIG_XTENSA_MMU_DOUBLE_MAP)) {
600 (void)l2_page_table_unmap(xtensa_kernel_ptables, (void *)vaddr_uc);
601 }
602
603 #ifdef CONFIG_USERSPACE
604 sys_snode_t *node;
605 struct arch_mem_domain *domain;
606 k_spinlock_key_t key;
607
608 key = k_spin_lock(&z_mem_domain_lock);
609 SYS_SLIST_FOR_EACH_NODE(&xtensa_domain_list, node) {
610 domain = CONTAINER_OF(node, struct arch_mem_domain, node);
611
612 (void)l2_page_table_unmap(domain->ptables, (void *)vaddr);
613
614 if (IS_ENABLED(CONFIG_XTENSA_MMU_DOUBLE_MAP)) {
615 (void)l2_page_table_unmap(domain->ptables, (void *)vaddr_uc);
616 }
617 }
618 k_spin_unlock(&z_mem_domain_lock, key);
619 #endif /* CONFIG_USERSPACE */
620 }
621
arch_mem_unmap(void * addr,size_t size)622 void arch_mem_unmap(void *addr, size_t size)
623 {
624 uint32_t va = (uint32_t)addr;
625 uint32_t rem_size = (uint32_t)size;
626 k_spinlock_key_t key;
627
628 if (addr == NULL) {
629 LOG_ERR("Cannot unmap NULL pointer");
630 return;
631 }
632
633 if (size == 0) {
634 LOG_ERR("Cannot unmap virtual memory with zero size");
635 return;
636 }
637
638 key = k_spin_lock(&xtensa_mmu_lock);
639
640 while (rem_size > 0) {
641 __arch_mem_unmap((void *)va);
642
643 rem_size -= (rem_size >= KB(4)) ? KB(4) : rem_size;
644 va += KB(4);
645 }
646
647 #if CONFIG_MP_MAX_NUM_CPUS > 1
648 xtensa_mmu_tlb_ipi();
649 #endif
650
651 sys_cache_data_flush_and_invd_all();
652 k_spin_unlock(&xtensa_mmu_lock, key);
653 }
654
655 /* This should be implemented in the SoC layer.
656 * This weak version is here to avoid build errors.
657 */
xtensa_mmu_tlb_ipi(void)658 void __weak xtensa_mmu_tlb_ipi(void)
659 {
660 }
661
xtensa_mmu_tlb_shootdown(void)662 void xtensa_mmu_tlb_shootdown(void)
663 {
664 unsigned int key;
665
666 /* Need to lock interrupts to prevent any context
667 * switching until all the page tables are updated.
668 * Or else we would be switching to another thread
669 * and running that with incorrect page tables
670 * which would result in permission issues.
671 */
672 key = arch_irq_lock();
673
674 K_SPINLOCK(&xtensa_mmu_lock) {
675 /* We don't have information on which page tables have changed,
676 * so we just invalidate the cache for all L1 page tables.
677 */
678 sys_cache_data_invd_range((void *)l1_page_table, sizeof(l1_page_table));
679 sys_cache_data_invd_range((void *)l2_page_tables, sizeof(l2_page_tables));
680 }
681
682 #ifdef CONFIG_USERSPACE
683 struct k_thread *thread = _current_cpu->current;
684
685 /* If current thread is a user thread, we need to see if it has
686 * been migrated to another memory domain as the L1 page table
687 * is different from the currently used one.
688 */
689 if ((thread->base.user_options & K_USER) == K_USER) {
690 uint32_t ptevaddr_entry, ptevaddr,
691 thread_ptables, current_ptables;
692
693 /* Need to read the currently used L1 page table.
694 * We know that L1 page table is always mapped at way
695 * MMU_PTE_WAY, so we can skip the probing step by
696 * generating the query entry directly.
697 */
698 ptevaddr = (uint32_t)xtensa_ptevaddr_get();
699 ptevaddr_entry = XTENSA_MMU_PTE_ENTRY_VADDR(ptevaddr, ptevaddr)
700 | XTENSA_MMU_PTE_WAY;
701 current_ptables = xtensa_dtlb_paddr_read(ptevaddr_entry);
702 thread_ptables = (uint32_t)thread->arch.ptables;
703
704 if (thread_ptables != current_ptables) {
705 /* Need to remap the thread page tables if the ones
706 * indicated by the current thread are different
707 * than the current mapped page table.
708 */
709 struct arch_mem_domain *domain =
710 &(thread->mem_domain_info.mem_domain->arch);
711 xtensa_set_paging(domain->asid, (uint32_t *)thread_ptables);
712 }
713
714 }
715 #endif /* CONFIG_USERSPACE */
716
717 /* L2 are done via autofill, so invalidate autofill TLBs
718 * would refresh the L2 page tables.
719 *
720 * L1 will be refreshed during context switch so no need
721 * to do anything here.
722 */
723 xtensa_tlb_autorefill_invalidate();
724
725 arch_irq_unlock(key);
726 }
727
728 #ifdef CONFIG_USERSPACE
729
alloc_l1_table(void)730 static inline uint32_t *alloc_l1_table(void)
731 {
732 uint16_t idx;
733
734 for (idx = 0; idx < CONFIG_XTENSA_MMU_NUM_L1_TABLES; idx++) {
735 if (!atomic_test_and_set_bit(l1_page_table_track, idx)) {
736 return (uint32_t *)&l1_page_table[idx];
737 }
738 }
739
740 return NULL;
741 }
742
dup_table(void)743 static uint32_t *dup_table(void)
744 {
745 uint16_t i, j;
746 uint32_t *dst_table = alloc_l1_table();
747
748 if (!dst_table) {
749 return NULL;
750 }
751
752 for (i = 0; i < XTENSA_L1_PAGE_TABLE_ENTRIES; i++) {
753 uint32_t *l2_table, *src_l2_table;
754
755 if (is_pte_illegal(xtensa_kernel_ptables[i]) ||
756 (i == XTENSA_MMU_L1_POS(XTENSA_MMU_PTEVADDR))) {
757 dst_table[i] = XTENSA_MMU_PTE_ILLEGAL;
758 continue;
759 }
760
761 src_l2_table = (uint32_t *)(xtensa_kernel_ptables[i] & XTENSA_MMU_PTE_PPN_MASK);
762 l2_table = alloc_l2_table();
763 if (l2_table == NULL) {
764 goto err;
765 }
766
767 for (j = 0; j < XTENSA_L2_PAGE_TABLE_ENTRIES; j++) {
768 uint32_t original_attr = XTENSA_MMU_PTE_SW_GET(src_l2_table[j]);
769
770 l2_table[j] = src_l2_table[j];
771 if (original_attr != 0x0) {
772 uint8_t ring;
773
774 ring = XTENSA_MMU_PTE_RING_GET(l2_table[j]);
775 l2_table[j] = XTENSA_MMU_PTE_ATTR_SET(l2_table[j], original_attr);
776 l2_table[j] = XTENSA_MMU_PTE_RING_SET(l2_table[j],
777 ring == XTENSA_MMU_SHARED_RING ?
778 XTENSA_MMU_SHARED_RING : XTENSA_MMU_KERNEL_RING);
779 }
780 }
781
782 /* The page table is using kernel ASID because we don't
783 * user thread manipulate it.
784 */
785 dst_table[i] = XTENSA_MMU_PTE((uint32_t)l2_table, XTENSA_MMU_KERNEL_RING,
786 0, XTENSA_MMU_PAGE_TABLE_ATTR);
787
788 sys_cache_data_flush_range((void *)l2_table, XTENSA_L2_PAGE_TABLE_SIZE);
789 }
790
791 sys_cache_data_flush_range((void *)dst_table, XTENSA_L1_PAGE_TABLE_SIZE);
792
793 return dst_table;
794
795 err:
796 /* TODO: Cleanup failed allocation*/
797 return NULL;
798 }
799
arch_mem_domain_init(struct k_mem_domain * domain)800 int arch_mem_domain_init(struct k_mem_domain *domain)
801 {
802 uint32_t *ptables;
803 k_spinlock_key_t key;
804 int ret;
805
806 /*
807 * For now, lets just assert if we have reached the maximum number
808 * of asid we assert.
809 */
810 __ASSERT(asid_count < (XTENSA_MMU_SHARED_ASID), "Reached maximum of ASID available");
811
812 key = k_spin_lock(&xtensa_mmu_lock);
813 /* If this is the default domain, we don't need
814 * to create a new set of page tables. We can just
815 * use the kernel page tables and save memory.
816 */
817
818 if (domain == &k_mem_domain_default) {
819 domain->arch.ptables = xtensa_kernel_ptables;
820 domain->arch.asid = asid_count;
821 goto end;
822 }
823
824
825 ptables = dup_table();
826
827 if (ptables == NULL) {
828 ret = -ENOMEM;
829 goto err;
830 }
831
832 domain->arch.ptables = ptables;
833 domain->arch.asid = ++asid_count;
834
835 sys_slist_append(&xtensa_domain_list, &domain->arch.node);
836
837 end:
838 ret = 0;
839
840 err:
841 k_spin_unlock(&xtensa_mmu_lock, key);
842
843 return ret;
844 }
845
region_map_update(uint32_t * ptables,uintptr_t start,size_t size,uint32_t ring,uint32_t flags)846 static int region_map_update(uint32_t *ptables, uintptr_t start,
847 size_t size, uint32_t ring, uint32_t flags)
848 {
849 int ret = 0;
850
851 for (size_t offset = 0; offset < size; offset += CONFIG_MMU_PAGE_SIZE) {
852 uint32_t *l2_table, pte;
853 uint32_t page = start + offset;
854 uint32_t l1_pos = XTENSA_MMU_L1_POS(page);
855 uint32_t l2_pos = XTENSA_MMU_L2_POS(page);
856 /* Make sure we grab a fresh copy of L1 page table */
857 sys_cache_data_invd_range((void *)&ptables[l1_pos], sizeof(ptables[0]));
858
859 l2_table = (uint32_t *)(ptables[l1_pos] & XTENSA_MMU_PTE_PPN_MASK);
860
861 sys_cache_data_invd_range((void *)&l2_table[l2_pos], sizeof(l2_table[0]));
862
863 pte = XTENSA_MMU_PTE_RING_SET(l2_table[l2_pos], ring);
864 pte = XTENSA_MMU_PTE_ATTR_SET(pte, flags);
865
866 l2_table[l2_pos] = pte;
867
868 sys_cache_data_flush_range((void *)&l2_table[l2_pos], sizeof(l2_table[0]));
869
870 xtensa_dtlb_vaddr_invalidate((void *)page);
871 }
872
873 return ret;
874 }
875
update_region(uint32_t * ptables,uintptr_t start,size_t size,uint32_t ring,uint32_t flags,uint32_t option)876 static inline int update_region(uint32_t *ptables, uintptr_t start,
877 size_t size, uint32_t ring, uint32_t flags,
878 uint32_t option)
879 {
880 int ret;
881 k_spinlock_key_t key;
882
883 key = k_spin_lock(&xtensa_mmu_lock);
884
885 #ifdef CONFIG_XTENSA_MMU_DOUBLE_MAP
886 uintptr_t va, va_uc;
887 uint32_t new_flags, new_flags_uc;
888
889 if (sys_cache_is_ptr_cached((void *)start)) {
890 va = start;
891 va_uc = (uintptr_t)sys_cache_uncached_ptr_get((void *)start);
892 } else {
893 va = (uintptr_t)sys_cache_cached_ptr_get((void *)start);
894 va_uc = start;
895 }
896
897 new_flags_uc = (flags & ~XTENSA_MMU_PTE_ATTR_CACHED_MASK);
898 new_flags = new_flags_uc | XTENSA_MMU_CACHED_WB;
899
900 ret = region_map_update(ptables, va, size, ring, new_flags);
901
902 if (ret == 0) {
903 ret = region_map_update(ptables, va_uc, size, ring, new_flags_uc);
904 }
905 #else
906 ret = region_map_update(ptables, start, size, ring, flags);
907 #endif /* CONFIG_XTENSA_MMU_DOUBLE_MAP */
908
909 #if CONFIG_MP_MAX_NUM_CPUS > 1
910 if ((option & OPTION_NO_TLB_IPI) != OPTION_NO_TLB_IPI) {
911 xtensa_mmu_tlb_ipi();
912 }
913 #endif
914
915 sys_cache_data_flush_and_invd_all();
916 k_spin_unlock(&xtensa_mmu_lock, key);
917
918 return ret;
919 }
920
reset_region(uint32_t * ptables,uintptr_t start,size_t size,uint32_t option)921 static inline int reset_region(uint32_t *ptables, uintptr_t start, size_t size, uint32_t option)
922 {
923 return update_region(ptables, start, size,
924 XTENSA_MMU_KERNEL_RING, XTENSA_MMU_PERM_W, option);
925 }
926
xtensa_user_stack_perms(struct k_thread * thread)927 void xtensa_user_stack_perms(struct k_thread *thread)
928 {
929 (void)memset((void *)thread->stack_info.start,
930 (IS_ENABLED(CONFIG_INIT_STACKS)) ? 0xAA : 0x00,
931 thread->stack_info.size - thread->stack_info.delta);
932
933 update_region(thread_page_tables_get(thread),
934 thread->stack_info.start, thread->stack_info.size,
935 XTENSA_MMU_USER_RING, XTENSA_MMU_PERM_W | XTENSA_MMU_CACHED_WB, 0);
936 }
937
arch_mem_domain_max_partitions_get(void)938 int arch_mem_domain_max_partitions_get(void)
939 {
940 return CONFIG_MAX_DOMAIN_PARTITIONS;
941 }
942
arch_mem_domain_partition_remove(struct k_mem_domain * domain,uint32_t partition_id)943 int arch_mem_domain_partition_remove(struct k_mem_domain *domain,
944 uint32_t partition_id)
945 {
946 struct k_mem_partition *partition = &domain->partitions[partition_id];
947
948 /* Reset the partition's region back to defaults */
949 return reset_region(domain->arch.ptables, partition->start,
950 partition->size, 0);
951 }
952
arch_mem_domain_partition_add(struct k_mem_domain * domain,uint32_t partition_id)953 int arch_mem_domain_partition_add(struct k_mem_domain *domain,
954 uint32_t partition_id)
955 {
956 struct k_mem_partition *partition = &domain->partitions[partition_id];
957 uint32_t ring = K_MEM_PARTITION_IS_USER(partition->attr) ? XTENSA_MMU_USER_RING :
958 XTENSA_MMU_KERNEL_RING;
959
960 return update_region(domain->arch.ptables, partition->start,
961 partition->size, ring, partition->attr, 0);
962 }
963
964 /* These APIs don't need to do anything */
arch_mem_domain_thread_add(struct k_thread * thread)965 int arch_mem_domain_thread_add(struct k_thread *thread)
966 {
967 int ret = 0;
968 bool is_user, is_migration;
969 uint32_t *old_ptables;
970 struct k_mem_domain *domain;
971
972 old_ptables = thread->arch.ptables;
973 domain = thread->mem_domain_info.mem_domain;
974 thread->arch.ptables = domain->arch.ptables;
975
976 is_user = (thread->base.user_options & K_USER) != 0;
977 is_migration = (old_ptables != NULL) && is_user;
978
979 if (is_migration) {
980 /* Give access to the thread's stack in its new
981 * memory domain if it is migrating.
982 */
983 update_region(thread_page_tables_get(thread),
984 thread->stack_info.start, thread->stack_info.size,
985 XTENSA_MMU_USER_RING,
986 XTENSA_MMU_PERM_W | XTENSA_MMU_CACHED_WB,
987 OPTION_NO_TLB_IPI);
988 /* and reset thread's stack permission in
989 * the old page tables.
990 */
991 ret = reset_region(old_ptables,
992 thread->stack_info.start,
993 thread->stack_info.size, 0);
994 }
995
996 /* Need to switch to new page tables if this is
997 * the current thread running.
998 */
999 if (thread == _current_cpu->current) {
1000 xtensa_set_paging(domain->arch.asid, thread->arch.ptables);
1001 }
1002
1003 #if CONFIG_MP_MAX_NUM_CPUS > 1
1004 /* Need to tell other CPUs to switch to the new page table
1005 * in case the thread is running on one of them.
1006 *
1007 * Note that there is no need to send TLB IPI if this is
1008 * migration as it was sent above during reset_region().
1009 */
1010 if ((thread != _current_cpu->current) && !is_migration) {
1011 xtensa_mmu_tlb_ipi();
1012 }
1013 #endif
1014
1015 return ret;
1016 }
1017
arch_mem_domain_thread_remove(struct k_thread * thread)1018 int arch_mem_domain_thread_remove(struct k_thread *thread)
1019 {
1020 struct k_mem_domain *domain = thread->mem_domain_info.mem_domain;
1021
1022 if ((thread->base.user_options & K_USER) == 0) {
1023 return 0;
1024 }
1025
1026 if ((thread->base.thread_state & _THREAD_DEAD) == 0) {
1027 /* Thread is migrating to another memory domain and not
1028 * exiting for good; we weren't called from
1029 * z_thread_abort(). Resetting the stack region will
1030 * take place in the forthcoming thread_add() call.
1031 */
1032 return 0;
1033 }
1034
1035 /* Restore permissions on the thread's stack area since it is no
1036 * longer a member of the domain.
1037 *
1038 * Note that, since every thread must have an associated memory
1039 * domain, removing a thread from domain will be followed by
1040 * adding it back to another. So there is no need to send TLB IPI
1041 * at this point.
1042 */
1043 return reset_region(domain->arch.ptables,
1044 thread->stack_info.start,
1045 thread->stack_info.size, OPTION_NO_TLB_IPI);
1046 }
1047
page_validate(uint32_t * ptables,uint32_t page,uint8_t ring,bool write)1048 static bool page_validate(uint32_t *ptables, uint32_t page, uint8_t ring, bool write)
1049 {
1050 uint8_t asid_ring;
1051 uint32_t rasid, pte, *l2_table;
1052 uint32_t l1_pos = XTENSA_MMU_L1_POS(page);
1053 uint32_t l2_pos = XTENSA_MMU_L2_POS(page);
1054
1055 if (is_pte_illegal(ptables[l1_pos])) {
1056 return false;
1057 }
1058
1059 l2_table = (uint32_t *)(ptables[l1_pos] & XTENSA_MMU_PTE_PPN_MASK);
1060 pte = l2_table[l2_pos];
1061
1062 if (is_pte_illegal(pte)) {
1063 return false;
1064 }
1065
1066 asid_ring = 0;
1067 rasid = xtensa_rasid_get();
1068 for (uint32_t i = 0; i < 4; i++) {
1069 if (XTENSA_MMU_PTE_ASID_GET(pte, rasid) == XTENSA_MMU_RASID_ASID_GET(rasid, i)) {
1070 asid_ring = i;
1071 break;
1072 }
1073 }
1074
1075 if (ring > asid_ring) {
1076 return false;
1077 }
1078
1079 if (write) {
1080 return (XTENSA_MMU_PTE_ATTR_GET((pte)) & XTENSA_MMU_PERM_W) != 0;
1081 }
1082
1083 return true;
1084 }
1085
mem_buffer_validate(const void * addr,size_t size,int write,int ring)1086 static int mem_buffer_validate(const void *addr, size_t size, int write, int ring)
1087 {
1088 int ret = 0;
1089 uint8_t *virt;
1090 size_t aligned_size;
1091 const struct k_thread *thread = _current;
1092 uint32_t *ptables = thread_page_tables_get(thread);
1093
1094 /* addr/size arbitrary, fix this up into an aligned region */
1095 k_mem_region_align((uintptr_t *)&virt, &aligned_size,
1096 (uintptr_t)addr, size, CONFIG_MMU_PAGE_SIZE);
1097
1098 for (size_t offset = 0; offset < aligned_size;
1099 offset += CONFIG_MMU_PAGE_SIZE) {
1100 if (!page_validate(ptables, (uint32_t)(virt + offset), ring, write)) {
1101 ret = -1;
1102 break;
1103 }
1104 }
1105
1106 return ret;
1107 }
1108
xtensa_mem_kernel_has_access(void * addr,size_t size,int write)1109 bool xtensa_mem_kernel_has_access(void *addr, size_t size, int write)
1110 {
1111 return mem_buffer_validate(addr, size, write, XTENSA_MMU_KERNEL_RING) == 0;
1112 }
1113
arch_buffer_validate(const void * addr,size_t size,int write)1114 int arch_buffer_validate(const void *addr, size_t size, int write)
1115 {
1116 return mem_buffer_validate(addr, size, write, XTENSA_MMU_USER_RING);
1117 }
1118
xtensa_swap_update_page_tables(struct k_thread * incoming)1119 void xtensa_swap_update_page_tables(struct k_thread *incoming)
1120 {
1121 uint32_t *ptables = incoming->arch.ptables;
1122 struct arch_mem_domain *domain =
1123 &(incoming->mem_domain_info.mem_domain->arch);
1124
1125 xtensa_set_paging(domain->asid, ptables);
1126
1127 #ifdef CONFIG_XTENSA_INVALIDATE_MEM_DOMAIN_TLB_ON_SWAP
1128 struct k_mem_domain *mem_domain = incoming->mem_domain_info.mem_domain;
1129
1130 for (int idx = 0; idx < mem_domain->num_partitions; idx++) {
1131 struct k_mem_partition *part = &mem_domain->partitions[idx];
1132 uintptr_t end = part->start + part->size;
1133
1134 for (uintptr_t addr = part->start; addr < end; addr += CONFIG_MMU_PAGE_SIZE) {
1135 xtensa_dtlb_vaddr_invalidate((void *)addr);
1136 }
1137 }
1138 #endif
1139 }
1140
1141 #endif /* CONFIG_USERSPACE */
1142
