1 /*
2 * Copyright (c) 2017, Intel Corporation
3 *
4 * SPDX-License-Identifier: Apache-2.0
5 */
6 #include <string.h>
7 #include <xtensa-asm2.h>
8 #include <zephyr/kernel.h>
9 #include <ksched.h>
10 #include <zephyr/kernel_structs.h>
11 #include <kernel_internal.h>
12 #include <kswap.h>
13 #include <_soc_inthandlers.h>
14 #include <zephyr/toolchain.h>
15 #include <zephyr/logging/log.h>
16 #include <offsets.h>
17 #include <zsr.h>
18
19 LOG_MODULE_DECLARE(os, CONFIG_KERNEL_LOG_LEVEL);
20
21 extern char xtensa_arch_except_epc[];
22
xtensa_init_stack(struct k_thread * thread,int * stack_top,void (* entry)(void *,void *,void *),void * arg1,void * arg2,void * arg3)23 void *xtensa_init_stack(struct k_thread *thread, int *stack_top,
24 void (*entry)(void *, void *, void *),
25 void *arg1, void *arg2, void *arg3)
26 {
27 void *ret;
28 _xtensa_irq_stack_frame_a11_t *frame;
29
30 /* Not-a-cpu ID Ensures that the first time this is run, the
31 * stack will be invalidated. That covers the edge case of
32 * restarting a thread on a stack that had previously been run
33 * on one CPU, but then initialized on this one, and
34 * potentially run THERE and not HERE.
35 */
36 thread->arch.last_cpu = -1;
37
38 /* We cheat and shave 16 bytes off, the top four words are the
39 * A0-A3 spill area for the caller of the entry function,
40 * which doesn't exist. It will never be touched, so we
41 * arrange to enter the function with a CALLINC of 1 and a
42 * stack pointer 16 bytes above the top, so its ENTRY at the
43 * start will decrement the stack pointer by 16.
44 */
45 const int bsasz = sizeof(*frame) - 16;
46
47 frame = (void *)(((char *) stack_top) - bsasz);
48
49 (void)memset(frame, 0, bsasz);
50
51 frame->bsa.pc = (uintptr_t)z_thread_entry;
52 frame->bsa.ps = PS_WOE | PS_UM | PS_CALLINC(1);
53
54 #if XCHAL_HAVE_THREADPTR && defined(CONFIG_THREAD_LOCAL_STORAGE)
55 frame->bsa.threadptr = thread->tls;
56 #endif
57
58 /* Arguments to z_thread_entry(). Remember these start at A6,
59 * which will be rotated into A2 by the ENTRY instruction that
60 * begins the C function. And A4-A7 and A8-A11 are optional
61 * quads that live below the BSA!
62 */
63 frame->a7 = (uintptr_t)arg1; /* a7 */
64 frame->a6 = (uintptr_t)entry; /* a6 */
65 frame->a5 = 0; /* a5 */
66 frame->a4 = 0; /* a4 */
67
68 frame->a11 = 0; /* a11 */
69 frame->a10 = 0; /* a10 */
70 frame->a9 = (uintptr_t)arg3; /* a9 */
71 frame->a8 = (uintptr_t)arg2; /* a8 */
72
73 /* Finally push the BSA pointer and return the stack pointer
74 * as the handle
75 */
76 frame->ptr_to_bsa = (void *)&frame->bsa;
77 ret = &frame->ptr_to_bsa;
78
79 return ret;
80 }
81
arch_new_thread(struct k_thread * thread,k_thread_stack_t * stack,char * stack_ptr,k_thread_entry_t entry,void * p1,void * p2,void * p3)82 void arch_new_thread(struct k_thread *thread, k_thread_stack_t *stack,
83 char *stack_ptr, k_thread_entry_t entry,
84 void *p1, void *p2, void *p3)
85 {
86 thread->switch_handle = xtensa_init_stack(thread,
87 (int *)stack_ptr, entry,
88 p1, p2, p3);
89 #ifdef CONFIG_KERNEL_COHERENCE
90 __ASSERT((((size_t)stack) % XCHAL_DCACHE_LINESIZE) == 0, "");
91 __ASSERT((((size_t)stack_ptr) % XCHAL_DCACHE_LINESIZE) == 0, "");
92 sys_cache_data_flush_and_invd_range(stack, (char *)stack_ptr - (char *)stack);
93 #endif
94 }
95
z_irq_spurious(const void * arg)96 void z_irq_spurious(const void *arg)
97 {
98 int irqs, ie;
99
100 ARG_UNUSED(arg);
101
102 __asm__ volatile("rsr.interrupt %0" : "=r"(irqs));
103 __asm__ volatile("rsr.intenable %0" : "=r"(ie));
104 LOG_ERR(" ** Spurious INTERRUPT(s) %p, INTENABLE = %p",
105 (void *)irqs, (void *)ie);
106 z_xtensa_fatal_error(K_ERR_SPURIOUS_IRQ, NULL);
107 }
108
z_xtensa_dump_stack(const z_arch_esf_t * stack)109 void z_xtensa_dump_stack(const z_arch_esf_t *stack)
110 {
111 _xtensa_irq_stack_frame_raw_t *frame = (void *)stack;
112 _xtensa_irq_bsa_t *bsa = frame->ptr_to_bsa;
113 uintptr_t num_high_regs;
114 int reg_blks_remaining;
115
116 /* Calculate number of high registers. */
117 num_high_regs = (uint8_t *)bsa - (uint8_t *)frame + sizeof(void *);
118 num_high_regs /= sizeof(uintptr_t);
119
120 /* And high registers are always comes in 4 in a block. */
121 reg_blks_remaining = (int)num_high_regs / 4;
122
123 LOG_ERR(" ** A0 %p SP %p A2 %p A3 %p",
124 (void *)bsa->a0,
125 (void *)((char *)bsa + sizeof(*bsa)),
126 (void *)bsa->a2, (void *)bsa->a3);
127
128 if (reg_blks_remaining > 0) {
129 reg_blks_remaining--;
130
131 LOG_ERR(" ** A4 %p A5 %p A6 %p A7 %p",
132 (void *)frame->blks[reg_blks_remaining].r0,
133 (void *)frame->blks[reg_blks_remaining].r1,
134 (void *)frame->blks[reg_blks_remaining].r2,
135 (void *)frame->blks[reg_blks_remaining].r3);
136 }
137
138 if (reg_blks_remaining > 0) {
139 reg_blks_remaining--;
140
141 LOG_ERR(" ** A8 %p A9 %p A10 %p A11 %p",
142 (void *)frame->blks[reg_blks_remaining].r0,
143 (void *)frame->blks[reg_blks_remaining].r1,
144 (void *)frame->blks[reg_blks_remaining].r2,
145 (void *)frame->blks[reg_blks_remaining].r3);
146 }
147
148 if (reg_blks_remaining > 0) {
149 reg_blks_remaining--;
150
151 LOG_ERR(" ** A12 %p A13 %p A14 %p A15 %p",
152 (void *)frame->blks[reg_blks_remaining].r0,
153 (void *)frame->blks[reg_blks_remaining].r1,
154 (void *)frame->blks[reg_blks_remaining].r2,
155 (void *)frame->blks[reg_blks_remaining].r3);
156 }
157
158 #if XCHAL_HAVE_LOOPS
159 LOG_ERR(" ** LBEG %p LEND %p LCOUNT %p",
160 (void *)bsa->lbeg,
161 (void *)bsa->lend,
162 (void *)bsa->lcount);
163 #endif
164
165 LOG_ERR(" ** SAR %p", (void *)bsa->sar);
166 }
167
get_bits(int offset,int num_bits,unsigned int val)168 static inline unsigned int get_bits(int offset, int num_bits, unsigned int val)
169 {
170 int mask;
171
172 mask = BIT(num_bits) - 1;
173 val = val >> offset;
174 return val & mask;
175 }
176
usage_stop(void)177 static ALWAYS_INLINE void usage_stop(void)
178 {
179 #ifdef CONFIG_SCHED_THREAD_USAGE
180 z_sched_usage_stop();
181 #endif
182 }
183
184 #ifdef CONFIG_MULTITHREADING
z_arch_get_next_switch_handle(struct k_thread * interrupted)185 void *z_arch_get_next_switch_handle(struct k_thread *interrupted)
186 {
187 return _current_cpu->nested <= 1 ?
188 z_get_next_switch_handle(interrupted) : interrupted;
189 }
190 #else
z_arch_get_next_switch_handle(struct k_thread * interrupted)191 void *z_arch_get_next_switch_handle(struct k_thread *interrupted)
192 {
193 return interrupted;
194 }
195 #endif /* CONFIG_MULTITHREADING */
196
return_to(void * interrupted)197 static inline void *return_to(void *interrupted)
198 {
199 return z_arch_get_next_switch_handle(interrupted);
200 }
201
202 #if defined(CONFIG_FPU) && defined(CONFIG_FPU_SHARING)
arch_float_disable(struct k_thread * thread)203 int arch_float_disable(struct k_thread *thread)
204 {
205 /* xtensa always has FPU enabled so cannot be disabled */
206 return -ENOTSUP;
207 }
208
arch_float_enable(struct k_thread * thread,unsigned int options)209 int arch_float_enable(struct k_thread *thread, unsigned int options)
210 {
211 /* xtensa always has FPU enabled so nothing to do here */
212 return 0;
213 }
214 #endif /* CONFIG_FPU && CONFIG_FPU_SHARING */
215
216 /* The wrapper code lives here instead of in the python script that
217 * generates _xtensa_handle_one_int*(). Seems cleaner, still kind of
218 * ugly.
219 *
220 * This may be unused depending on number of interrupt levels
221 * supported by the SoC.
222 */
223 #define DEF_INT_C_HANDLER(l) \
224 __unused void *xtensa_int##l##_c(void *interrupted_stack) \
225 { \
226 uint32_t irqs, intenable, m; \
227 usage_stop(); \
228 __asm__ volatile("rsr.interrupt %0" : "=r"(irqs)); \
229 __asm__ volatile("rsr.intenable %0" : "=r"(intenable)); \
230 irqs &= intenable; \
231 while ((m = _xtensa_handle_one_int##l(irqs))) { \
232 irqs ^= m; \
233 __asm__ volatile("wsr.intclear %0" : : "r"(m)); \
234 } \
235 return return_to(interrupted_stack); \
236 }
237
238 #if XCHAL_NMILEVEL >= 2
239 DEF_INT_C_HANDLER(2)
240 #endif
241
242 #if XCHAL_NMILEVEL >= 3
243 DEF_INT_C_HANDLER(3)
244 #endif
245
246 #if XCHAL_NMILEVEL >= 4
247 DEF_INT_C_HANDLER(4)
248 #endif
249
250 #if XCHAL_NMILEVEL >= 5
251 DEF_INT_C_HANDLER(5)
252 #endif
253
254 #if XCHAL_NMILEVEL >= 6
255 DEF_INT_C_HANDLER(6)
256 #endif
257
258 #if XCHAL_NMILEVEL >= 7
259 DEF_INT_C_HANDLER(7)
260 #endif
261
262 static inline DEF_INT_C_HANDLER(1)
263
264 /* C handler for level 1 exceptions/interrupts. Hooked from the
265 * DEF_EXCINT 1 vector declaration in assembly code. This one looks
266 * different because exceptions and interrupts land at the same
267 * vector; other interrupt levels have their own vectors.
268 */
xtensa_excint1_c(int * interrupted_stack)269 void *xtensa_excint1_c(int *interrupted_stack)
270 {
271 int cause, vaddr;
272 _xtensa_irq_bsa_t *bsa = (void *)*(int **)interrupted_stack;
273 bool is_fatal_error = false;
274 uint32_t ps;
275 void *pc;
276
277 __asm__ volatile("rsr.exccause %0" : "=r"(cause));
278
279 #ifdef CONFIG_XTENSA_MMU
280 /* TLB miss exception comes through level 1 interrupt also.
281 * We need to preserve execution context after we have handled
282 * the TLB miss, so we cannot unconditionally unmask interrupts.
283 * For other cause, we can unmask interrupts so this would act
284 * the same as if there is no MMU.
285 */
286 switch (cause) {
287 case EXCCAUSE_ITLB_MISS:
288 /* Instruction TLB miss */
289 __fallthrough;
290 case EXCCAUSE_DTLB_MISS:
291 /* Data TLB miss */
292
293 /* Do not unmask interrupt while handling TLB misses. */
294 break;
295 default:
296 /* For others, we can unmask interrupts. */
297 bsa->ps &= ~PS_INTLEVEL_MASK;
298 break;
299 }
300 #endif /* CONFIG_XTENSA_MMU */
301
302 switch (cause) {
303 case EXCCAUSE_LEVEL1_INTERRUPT:
304 return xtensa_int1_c(interrupted_stack);
305 case EXCCAUSE_SYSCALL:
306 /* Just report it to the console for now */
307 LOG_ERR(" ** SYSCALL PS %p PC %p",
308 (void *)bsa->ps, (void *)bsa->pc);
309 z_xtensa_dump_stack(interrupted_stack);
310
311 /* Xtensa exceptions don't automatically advance PC,
312 * have to skip the SYSCALL instruction manually or
313 * else it will just loop forever
314 */
315 bsa->pc += 3;
316 break;
317 #ifdef CONFIG_XTENSA_MMU
318 case EXCCAUSE_ITLB_MISS:
319 /* Instruction TLB miss */
320 __fallthrough;
321 case EXCCAUSE_DTLB_MISS:
322 /* Data TLB miss */
323
324 /**
325 * The way it works is, when we try to access an address
326 * that is not mapped, we will have a miss. The HW then
327 * will try to get the correspondent memory in the page
328 * table. As the page table is not mapped in memory we will
329 * have a second miss, which will trigger an exception.
330 * In the exception (here) what we do is to exploit this
331 * hardware capability just trying to load the page table
332 * (not mapped address), which will cause a miss, but then
333 * the hardware will automatically map it again from
334 * the page table. This time it will work since the page
335 * necessary to map the page table itself are wired map.
336 */
337 __asm__ volatile("wsr a0, " ZSR_EXTRA0_STR "\n\t"
338 "rsr.ptevaddr a0\n\t"
339 "l32i a0, a0, 0\n\t"
340 "rsr a0, " ZSR_EXTRA0_STR "\n\t"
341 "rsync"
342 : : : "a0", "memory");
343
344 /* Since we are dealing with TLB misses, we will probably not
345 * want to switch to another thread.
346 */
347 return interrupted_stack;
348 #endif /* CONFIG_XTENSA_MMU */
349 default:
350 ps = bsa->ps;
351 pc = (void *)bsa->pc;
352
353 __asm__ volatile("rsr.excvaddr %0" : "=r"(vaddr));
354
355 /* Default for exception */
356 int reason = K_ERR_CPU_EXCEPTION;
357
358 /* We need to distinguish between an ill in xtensa_arch_except,
359 * e.g for k_panic, and any other ill. For exceptions caused by
360 * xtensa_arch_except calls, we also need to pass the reason_p
361 * to z_xtensa_fatal_error. Since the ARCH_EXCEPT frame is in the
362 * BSA, the first arg reason_p is stored at the A2 offset.
363 * We assign EXCCAUSE the unused, reserved code 63; this may be
364 * problematic if the app or new boards also decide to repurpose
365 * this code.
366 */
367 if ((pc == (void *) &xtensa_arch_except_epc) && (cause == 0)) {
368 cause = 63;
369 __asm__ volatile("wsr.exccause %0" : : "r"(cause));
370 reason = bsa->a2;
371 }
372
373 LOG_ERR(" ** FATAL EXCEPTION");
374 LOG_ERR(" ** CPU %d EXCCAUSE %d (%s)",
375 arch_curr_cpu()->id, cause,
376 z_xtensa_exccause(cause));
377 LOG_ERR(" ** PC %p VADDR %p",
378 pc, (void *)vaddr);
379 LOG_ERR(" ** PS %p", (void *)bsa->ps);
380 LOG_ERR(" ** (INTLEVEL:%d EXCM: %d UM:%d RING:%d WOE:%d OWB:%d CALLINC:%d)",
381 get_bits(0, 4, ps), get_bits(4, 1, ps),
382 get_bits(5, 1, ps), get_bits(6, 2, ps),
383 get_bits(18, 1, ps),
384 get_bits(8, 4, ps), get_bits(16, 2, ps));
385
386 /* FIXME: legacy xtensa port reported "HW" exception
387 * for all unhandled exceptions, which seems incorrect
388 * as these are software errors. Should clean this
389 * up.
390 */
391 z_xtensa_fatal_error(reason,
392 (void *)interrupted_stack);
393 break;
394 }
395
396
397 switch (cause) {
398 case EXCCAUSE_SYSCALL:
399 case EXCCAUSE_LEVEL1_INTERRUPT:
400 case EXCCAUSE_ALLOCA:
401 case EXCCAUSE_ITLB_MISS:
402 case EXCCAUSE_DTLB_MISS:
403 is_fatal_error = false;
404 break;
405 default:
406 is_fatal_error = true;
407 break;
408 }
409
410 if (is_fatal_error) {
411 uint32_t ignore;
412
413 /* We are going to manipulate _current_cpu->nested manually.
414 * Since the error is fatal, for recoverable errors, code
415 * execution must not return back to the current thread as
416 * it is being terminated (via above z_xtensa_fatal_error()).
417 * So we need to prevent more interrupts coming in which
418 * will affect the nested value as we are going outside of
419 * normal interrupt handling procedure.
420 *
421 * Setting nested to 1 has two effects:
422 * 1. Force return_to() to choose a new thread.
423 * Since the current thread is being terminated, it will
424 * not be chosen again.
425 * 2. When context switches to the newly chosen thread,
426 * nested must be zero for normal code execution,
427 * as that is not in interrupt context at all.
428 * After returning from this function, the rest of
429 * interrupt handling code will decrement nested,
430 * resulting it being zero before switching to another
431 * thread.
432 */
433 __asm__ volatile("rsil %0, %1"
434 : "=r" (ignore) : "i"(XCHAL_NMILEVEL));
435
436 _current_cpu->nested = 1;
437 }
438
439 return return_to(interrupted_stack);
440 }
441
442 #if defined(CONFIG_GDBSTUB)
xtensa_debugint_c(int * interrupted_stack)443 void *xtensa_debugint_c(int *interrupted_stack)
444 {
445 extern void z_gdb_isr(z_arch_esf_t *esf);
446
447 z_gdb_isr((void *)interrupted_stack);
448
449 return return_to(interrupted_stack);
450 }
451 #endif
452
z_xtensa_irq_is_enabled(unsigned int irq)453 int z_xtensa_irq_is_enabled(unsigned int irq)
454 {
455 uint32_t ie;
456
457 __asm__ volatile("rsr.intenable %0" : "=r"(ie));
458
459 return (ie & (1 << irq)) != 0U;
460 }
461
462 #ifdef CONFIG_XTENSA_MORE_SPIN_RELAX_NOPS
463 /* Some compilers might "optimize out" (i.e. remove) continuous NOPs.
464 * So force no optimization to avoid that.
465 */
466 __no_optimization
arch_spin_relax(void)467 void arch_spin_relax(void)
468 {
469 #define NOP1(_, __) __asm__ volatile("nop.n;");
470 LISTIFY(CONFIG_XTENSA_NUM_SPIN_RELAX_NOPS, NOP1, (;))
471 #undef NOP1
472 }
473 #endif /* CONFIG_XTENSA_MORE_SPIN_RELAX_NOPS */
474
