1 /*
2 * Copyright (c) 2010-2014 Wind River Systems, Inc.
3 *
4 * SPDX-License-Identifier: Apache-2.0
5 */
6
7 /**
8 * @file
9 * @brief Kernel initialization module
10 *
11 * This module contains routines that are used to initialize the kernel.
12 */
13
14 #include <offsets_short.h>
15 #include <zephyr/kernel.h>
16 #include <zephyr/sys/printk.h>
17 #include <zephyr/debug/stack.h>
18 #include <zephyr/random/rand32.h>
19 #include <zephyr/linker/sections.h>
20 #include <zephyr/toolchain.h>
21 #include <zephyr/kernel_structs.h>
22 #include <zephyr/device.h>
23 #include <zephyr/init.h>
24 #include <zephyr/linker/linker-defs.h>
25 #include <ksched.h>
26 #include <string.h>
27 #include <zephyr/sys/dlist.h>
28 #include <kernel_internal.h>
29 #include <zephyr/drivers/entropy.h>
30 #include <zephyr/logging/log_ctrl.h>
31 #include <zephyr/tracing/tracing.h>
32 #include <stdbool.h>
33 #include <zephyr/debug/gcov.h>
34 #include <kswap.h>
35 #include <zephyr/timing/timing.h>
36 #include <zephyr/logging/log.h>
37 #include <zephyr/pm/device_runtime.h>
38 LOG_MODULE_REGISTER(os, CONFIG_KERNEL_LOG_LEVEL);
39
40
41 BUILD_ASSERT(CONFIG_MP_NUM_CPUS == CONFIG_MP_MAX_NUM_CPUS,
42 "CONFIG_MP_NUM_CPUS and CONFIG_MP_MAX_NUM_CPUS need to be set the same");
43
44 /* the only struct z_kernel instance */
45 __pinned_bss
46 struct z_kernel _kernel;
47
48 __pinned_bss
49 atomic_t _cpus_active;
50
51 /* init/main and idle threads */
52 K_THREAD_PINNED_STACK_DEFINE(z_main_stack, CONFIG_MAIN_STACK_SIZE);
53 struct k_thread z_main_thread;
54
55 #ifdef CONFIG_MULTITHREADING
56 __pinned_bss
57 struct k_thread z_idle_threads[CONFIG_MP_MAX_NUM_CPUS];
58
59 static K_KERNEL_PINNED_STACK_ARRAY_DEFINE(z_idle_stacks,
60 CONFIG_MP_MAX_NUM_CPUS,
61 CONFIG_IDLE_STACK_SIZE);
62 #endif /* CONFIG_MULTITHREADING */
63
64 extern const struct init_entry __init_start[];
65 extern const struct init_entry __init_EARLY_start[];
66 extern const struct init_entry __init_PRE_KERNEL_1_start[];
67 extern const struct init_entry __init_PRE_KERNEL_2_start[];
68 extern const struct init_entry __init_POST_KERNEL_start[];
69 extern const struct init_entry __init_APPLICATION_start[];
70 extern const struct init_entry __init_end[];
71
72 enum init_level {
73 INIT_LEVEL_EARLY = 0,
74 INIT_LEVEL_PRE_KERNEL_1,
75 INIT_LEVEL_PRE_KERNEL_2,
76 INIT_LEVEL_POST_KERNEL,
77 INIT_LEVEL_APPLICATION,
78 #ifdef CONFIG_SMP
79 INIT_LEVEL_SMP,
80 #endif
81 };
82
83 #ifdef CONFIG_SMP
84 extern const struct init_entry __init_SMP_start[];
85 #endif
86
87 /*
88 * storage space for the interrupt stack
89 *
90 * Note: This area is used as the system stack during kernel initialization,
91 * since the kernel hasn't yet set up its own stack areas. The dual purposing
92 * of this area is safe since interrupts are disabled until the kernel context
93 * switches to the init thread.
94 */
95 K_KERNEL_PINNED_STACK_ARRAY_DEFINE(z_interrupt_stacks,
96 CONFIG_MP_MAX_NUM_CPUS,
97 CONFIG_ISR_STACK_SIZE);
98
99 extern void idle(void *unused1, void *unused2, void *unused3);
100
101
102 /* LCOV_EXCL_START
103 *
104 * This code is called so early in the boot process that code coverage
105 * doesn't work properly. In addition, not all arches call this code,
106 * some like x86 do this with optimized assembly
107 */
108
109 /**
110 * @brief equivalent of memset() for early boot usage
111 *
112 * Architectures that can't safely use the regular (optimized) memset very
113 * early during boot because e.g. hardware isn't yet sufficiently initialized
114 * may override this with their own safe implementation.
115 */
116 __boot_func
z_early_memset(void * dst,int c,size_t n)117 void __weak z_early_memset(void *dst, int c, size_t n)
118 {
119 (void) memset(dst, c, n);
120 }
121
122 /**
123 * @brief equivalent of memcpy() for early boot usage
124 *
125 * Architectures that can't safely use the regular (optimized) memcpy very
126 * early during boot because e.g. hardware isn't yet sufficiently initialized
127 * may override this with their own safe implementation.
128 */
129 __boot_func
z_early_memcpy(void * dst,const void * src,size_t n)130 void __weak z_early_memcpy(void *dst, const void *src, size_t n)
131 {
132 (void) memcpy(dst, src, n);
133 }
134
135 /**
136 * @brief Clear BSS
137 *
138 * This routine clears the BSS region, so all bytes are 0.
139 */
140 __boot_func
z_bss_zero(void)141 void z_bss_zero(void)
142 {
143 if (IS_ENABLED(CONFIG_ARCH_POSIX)) {
144 /* native_posix gets its memory cleared on entry by
145 * the host OS, and in any case the host clang/lld
146 * doesn't emit the __bss_end symbol this code expects
147 * to see
148 */
149 return;
150 }
151
152 z_early_memset(__bss_start, 0, __bss_end - __bss_start);
153 #if DT_NODE_HAS_STATUS(DT_CHOSEN(zephyr_ccm), okay)
154 z_early_memset(&__ccm_bss_start, 0,
155 (uintptr_t) &__ccm_bss_end
156 - (uintptr_t) &__ccm_bss_start);
157 #endif
158 #if DT_NODE_HAS_STATUS(DT_CHOSEN(zephyr_dtcm), okay)
159 z_early_memset(&__dtcm_bss_start, 0,
160 (uintptr_t) &__dtcm_bss_end
161 - (uintptr_t) &__dtcm_bss_start);
162 #endif
163 #if DT_NODE_HAS_STATUS(DT_CHOSEN(zephyr_ocm), okay)
164 z_early_memset(&__ocm_bss_start, 0,
165 (uintptr_t) &__ocm_bss_end
166 - (uintptr_t) &__ocm_bss_start);
167 #endif
168 #ifdef CONFIG_CODE_DATA_RELOCATION
169 extern void bss_zeroing_relocation(void);
170
171 bss_zeroing_relocation();
172 #endif /* CONFIG_CODE_DATA_RELOCATION */
173 #ifdef CONFIG_COVERAGE_GCOV
174 z_early_memset(&__gcov_bss_start, 0,
175 ((uintptr_t) &__gcov_bss_end - (uintptr_t) &__gcov_bss_start));
176 #endif
177 }
178
179 #ifdef CONFIG_LINKER_USE_BOOT_SECTION
180 /**
181 * @brief Clear BSS within the bot region
182 *
183 * This routine clears the BSS within the boot region.
184 * This is separate from z_bss_zero() as boot region may
185 * contain symbols required for the boot process before
186 * paging is initialized.
187 */
188 __boot_func
z_bss_zero_boot(void)189 void z_bss_zero_boot(void)
190 {
191 z_early_memset(&lnkr_boot_bss_start, 0,
192 (uintptr_t)&lnkr_boot_bss_end
193 - (uintptr_t)&lnkr_boot_bss_start);
194 }
195 #endif /* CONFIG_LINKER_USE_BOOT_SECTION */
196
197 #ifdef CONFIG_LINKER_USE_PINNED_SECTION
198 /**
199 * @brief Clear BSS within the pinned region
200 *
201 * This routine clears the BSS within the pinned region.
202 * This is separate from z_bss_zero() as pinned region may
203 * contain symbols required for the boot process before
204 * paging is initialized.
205 */
206 #ifdef CONFIG_LINKER_USE_BOOT_SECTION
207 __boot_func
208 #else
209 __pinned_func
210 #endif
z_bss_zero_pinned(void)211 void z_bss_zero_pinned(void)
212 {
213 z_early_memset(&lnkr_pinned_bss_start, 0,
214 (uintptr_t)&lnkr_pinned_bss_end
215 - (uintptr_t)&lnkr_pinned_bss_start);
216 }
217 #endif /* CONFIG_LINKER_USE_PINNED_SECTION */
218
219 #ifdef CONFIG_STACK_CANARIES
220 extern volatile uintptr_t __stack_chk_guard;
221 #endif /* CONFIG_STACK_CANARIES */
222
223 /* LCOV_EXCL_STOP */
224
225 __pinned_bss
226 bool z_sys_post_kernel;
227
228 /**
229 * @brief Execute all the init entry initialization functions at a given level
230 *
231 * @details Invokes the initialization routine for each init entry object
232 * created by the INIT_ENTRY_DEFINE() macro using the specified level.
233 * The linker script places the init entry objects in memory in the order
234 * they need to be invoked, with symbols indicating where one level leaves
235 * off and the next one begins.
236 *
237 * @param level init level to run.
238 */
z_sys_init_run_level(enum init_level level)239 static void z_sys_init_run_level(enum init_level level)
240 {
241 static const struct init_entry *levels[] = {
242 __init_EARLY_start,
243 __init_PRE_KERNEL_1_start,
244 __init_PRE_KERNEL_2_start,
245 __init_POST_KERNEL_start,
246 __init_APPLICATION_start,
247 #ifdef CONFIG_SMP
248 __init_SMP_start,
249 #endif
250 /* End marker */
251 __init_end,
252 };
253 const struct init_entry *entry;
254
255 for (entry = levels[level]; entry < levels[level+1]; entry++) {
256 const struct device *dev = entry->dev;
257
258 if (dev != NULL) {
259 int rc = 0;
260
261 if (entry->init_fn.dev != NULL) {
262 rc = entry->init_fn.dev(dev);
263 /* Mark device initialized. If initialization
264 * failed, record the error condition.
265 */
266 if (rc != 0) {
267 if (rc < 0) {
268 rc = -rc;
269 }
270 if (rc > UINT8_MAX) {
271 rc = UINT8_MAX;
272 }
273 dev->state->init_res = rc;
274 }
275 }
276
277 dev->state->initialized = true;
278
279 if (rc == 0) {
280 /* Run automatic device runtime enablement */
281 (void)pm_device_runtime_auto_enable(dev);
282 }
283 } else {
284 (void)entry->init_fn.sys();
285 }
286 }
287 }
288
289 extern void boot_banner(void);
290
291 /**
292 * @brief Mainline for kernel's background thread
293 *
294 * This routine completes kernel initialization by invoking the remaining
295 * init functions, then invokes application's main() routine.
296 */
297 __boot_func
bg_thread_main(void * unused1,void * unused2,void * unused3)298 static void bg_thread_main(void *unused1, void *unused2, void *unused3)
299 {
300 ARG_UNUSED(unused1);
301 ARG_UNUSED(unused2);
302 ARG_UNUSED(unused3);
303
304 #ifdef CONFIG_MMU
305 /* Invoked here such that backing store or eviction algorithms may
306 * initialize kernel objects, and that all POST_KERNEL and later tasks
307 * may perform memory management tasks (except for z_phys_map() which
308 * is allowed at any time)
309 */
310 z_mem_manage_init();
311 #endif /* CONFIG_MMU */
312 z_sys_post_kernel = true;
313
314 z_sys_init_run_level(INIT_LEVEL_POST_KERNEL);
315 #if CONFIG_STACK_POINTER_RANDOM
316 z_stack_adjust_initialized = 1;
317 #endif
318 boot_banner();
319
320 #if defined(CONFIG_CPP)
321 void z_cpp_init_static(void);
322 z_cpp_init_static();
323 #endif
324
325 /* Final init level before app starts */
326 z_sys_init_run_level(INIT_LEVEL_APPLICATION);
327
328 z_init_static_threads();
329
330 #ifdef CONFIG_KERNEL_COHERENCE
331 __ASSERT_NO_MSG(arch_mem_coherent(&_kernel));
332 #endif
333
334 #ifdef CONFIG_SMP
335 if (!IS_ENABLED(CONFIG_SMP_BOOT_DELAY)) {
336 z_smp_init();
337 }
338 z_sys_init_run_level(INIT_LEVEL_SMP);
339 #endif
340
341 #ifdef CONFIG_MMU
342 z_mem_manage_boot_finish();
343 #endif /* CONFIG_MMU */
344
345 extern int main(void);
346
347 (void)main();
348
349 /* Mark nonessential since main() has no more work to do */
350 z_main_thread.base.user_options &= ~K_ESSENTIAL;
351
352 #ifdef CONFIG_COVERAGE_DUMP
353 /* Dump coverage data once the main() has exited. */
354 gcov_coverage_dump();
355 #endif
356 } /* LCOV_EXCL_LINE ... because we just dumped final coverage data */
357
358 #if defined(CONFIG_MULTITHREADING)
359 __boot_func
init_idle_thread(int i)360 static void init_idle_thread(int i)
361 {
362 struct k_thread *thread = &z_idle_threads[i];
363 k_thread_stack_t *stack = z_idle_stacks[i];
364
365 #ifdef CONFIG_THREAD_NAME
366
367 #if CONFIG_MP_MAX_NUM_CPUS > 1
368 char tname[8];
369 snprintk(tname, 8, "idle %02d", i);
370 #else
371 char *tname = "idle";
372 #endif
373
374 #else
375 char *tname = NULL;
376 #endif /* CONFIG_THREAD_NAME */
377
378 z_setup_new_thread(thread, stack,
379 CONFIG_IDLE_STACK_SIZE, idle, &_kernel.cpus[i],
380 NULL, NULL, K_IDLE_PRIO, K_ESSENTIAL,
381 tname);
382 z_mark_thread_as_started(thread);
383
384 #ifdef CONFIG_SMP
385 thread->base.is_idle = 1U;
386 #endif
387 }
388
z_init_cpu(int id)389 void z_init_cpu(int id)
390 {
391 init_idle_thread(id);
392 _kernel.cpus[id].idle_thread = &z_idle_threads[id];
393 _kernel.cpus[id].id = id;
394 _kernel.cpus[id].irq_stack =
395 (Z_KERNEL_STACK_BUFFER(z_interrupt_stacks[id]) +
396 K_KERNEL_STACK_SIZEOF(z_interrupt_stacks[id]));
397 #ifdef CONFIG_SCHED_THREAD_USAGE_ALL
398 _kernel.cpus[id].usage.track_usage =
399 CONFIG_SCHED_THREAD_USAGE_AUTO_ENABLE;
400 #endif
401
402 /*
403 * Increment number of CPUs active. The pm subsystem
404 * will keep track of this from here.
405 */
406 atomic_inc(&_cpus_active);
407 }
408
409 /**
410 *
411 * @brief Initializes kernel data structures
412 *
413 * This routine initializes various kernel data structures, including
414 * the init and idle threads and any architecture-specific initialization.
415 *
416 * Note that all fields of "_kernel" are set to zero on entry, which may
417 * be all the initialization many of them require.
418 *
419 * @return initial stack pointer for the main thread
420 */
421 __boot_func
prepare_multithreading(void)422 static char *prepare_multithreading(void)
423 {
424 char *stack_ptr;
425
426 /* _kernel.ready_q is all zeroes */
427 z_sched_init();
428
429 #ifndef CONFIG_SMP
430 /*
431 * prime the cache with the main thread since:
432 *
433 * - the cache can never be NULL
434 * - the main thread will be the one to run first
435 * - no other thread is initialized yet and thus their priority fields
436 * contain garbage, which would prevent the cache loading algorithm
437 * to work as intended
438 */
439 _kernel.ready_q.cache = &z_main_thread;
440 #endif
441 stack_ptr = z_setup_new_thread(&z_main_thread, z_main_stack,
442 CONFIG_MAIN_STACK_SIZE, bg_thread_main,
443 NULL, NULL, NULL,
444 CONFIG_MAIN_THREAD_PRIORITY,
445 K_ESSENTIAL, "main");
446 z_mark_thread_as_started(&z_main_thread);
447 z_ready_thread(&z_main_thread);
448
449 z_init_cpu(0);
450
451 return stack_ptr;
452 }
453
454 __boot_func
switch_to_main_thread(char * stack_ptr)455 static FUNC_NORETURN void switch_to_main_thread(char *stack_ptr)
456 {
457 #ifdef CONFIG_ARCH_HAS_CUSTOM_SWAP_TO_MAIN
458 arch_switch_to_main_thread(&z_main_thread, stack_ptr, bg_thread_main);
459 #else
460 ARG_UNUSED(stack_ptr);
461 /*
462 * Context switch to main task (entry function is _main()): the
463 * current fake thread is not on a wait queue or ready queue, so it
464 * will never be rescheduled in.
465 */
466 z_swap_unlocked();
467 #endif
468 CODE_UNREACHABLE; /* LCOV_EXCL_LINE */
469 }
470 #endif /* CONFIG_MULTITHREADING */
471
472 #if defined(CONFIG_ENTROPY_HAS_DRIVER) || defined(CONFIG_TEST_RANDOM_GENERATOR)
473 __boot_func
z_early_boot_rand_get(uint8_t * buf,size_t length)474 void z_early_boot_rand_get(uint8_t *buf, size_t length)
475 {
476 #ifdef CONFIG_ENTROPY_HAS_DRIVER
477 const struct device *const entropy = DEVICE_DT_GET_OR_NULL(DT_CHOSEN(zephyr_entropy));
478 int rc;
479
480 if (!device_is_ready(entropy)) {
481 goto sys_rand_fallback;
482 }
483
484 /* Try to see if driver provides an ISR-specific API */
485 rc = entropy_get_entropy_isr(entropy, buf, length, ENTROPY_BUSYWAIT);
486 if (rc == -ENOTSUP) {
487 /* Driver does not provide an ISR-specific API, assume it can
488 * be called from ISR context
489 */
490 rc = entropy_get_entropy(entropy, buf, length);
491 }
492
493 if (rc >= 0) {
494 return;
495 }
496
497 /* Fall through to fallback */
498
499 sys_rand_fallback:
500 #endif
501
502 /* FIXME: this assumes sys_rand32_get() won't use any synchronization
503 * primitive, like semaphores or mutexes. It's too early in the boot
504 * process to use any of them. Ideally, only the path where entropy
505 * devices are available should be built, this is only a fallback for
506 * those devices without a HWRNG entropy driver.
507 */
508 sys_rand_get(buf, length);
509 }
510 /* defined(CONFIG_ENTROPY_HAS_DRIVER) || defined(CONFIG_TEST_RANDOM_GENERATOR) */
511 #endif
512
513 /**
514 *
515 * @brief Initialize kernel
516 *
517 * This routine is invoked when the system is ready to run C code. The
518 * processor must be running in 32-bit mode, and the BSS must have been
519 * cleared/zeroed.
520 *
521 * @return Does not return
522 */
523 __boot_func
524 FUNC_NO_STACK_PROTECTOR
z_cstart(void)525 FUNC_NORETURN void z_cstart(void)
526 {
527 /* gcov hook needed to get the coverage report.*/
528 gcov_static_init();
529
530 /* initialize early init calls */
531 z_sys_init_run_level(INIT_LEVEL_EARLY);
532
533 /* perform any architecture-specific initialization */
534 arch_kernel_init();
535
536 LOG_CORE_INIT();
537
538 #if defined(CONFIG_MULTITHREADING)
539 /* Note: The z_ready_thread() call in prepare_multithreading() requires
540 * a dummy thread even if CONFIG_ARCH_HAS_CUSTOM_SWAP_TO_MAIN=y
541 */
542 struct k_thread dummy_thread;
543
544 z_dummy_thread_init(&dummy_thread);
545 #endif
546 /* do any necessary initialization of static devices */
547 z_device_state_init();
548
549 /* perform basic hardware initialization */
550 z_sys_init_run_level(INIT_LEVEL_PRE_KERNEL_1);
551 z_sys_init_run_level(INIT_LEVEL_PRE_KERNEL_2);
552
553 #ifdef CONFIG_STACK_CANARIES
554 uintptr_t stack_guard;
555
556 z_early_boot_rand_get((uint8_t *)&stack_guard, sizeof(stack_guard));
557 __stack_chk_guard = stack_guard;
558 __stack_chk_guard <<= 8;
559 #endif /* CONFIG_STACK_CANARIES */
560
561 #ifdef CONFIG_TIMING_FUNCTIONS_NEED_AT_BOOT
562 timing_init();
563 timing_start();
564 #endif
565
566 #ifdef CONFIG_MULTITHREADING
567 switch_to_main_thread(prepare_multithreading());
568 #else
569 #ifdef ARCH_SWITCH_TO_MAIN_NO_MULTITHREADING
570 /* Custom ARCH-specific routine to switch to main()
571 * in the case of no multi-threading.
572 */
573 ARCH_SWITCH_TO_MAIN_NO_MULTITHREADING(bg_thread_main,
574 NULL, NULL, NULL);
575 #else
576 bg_thread_main(NULL, NULL, NULL);
577
578 /* LCOV_EXCL_START
579 * We've already dumped coverage data at this point.
580 */
581 irq_lock();
582 while (true) {
583 }
584 /* LCOV_EXCL_STOP */
585 #endif
586 #endif /* CONFIG_MULTITHREADING */
587
588 /*
589 * Compiler can't tell that the above routines won't return and issues
590 * a warning unless we explicitly tell it that control never gets this
591 * far.
592 */
593
594 CODE_UNREACHABLE; /* LCOV_EXCL_LINE */
595 }
596
