1 /*
2 * Copyright (c) 2020 Intel Corporation
3 *
4 * SPDX-License-Identifier: Apache-2.0
5 */
6
7 #include <zephyr/kernel.h>
8 #include <ksched.h>
9 #include <zephyr/wait_q.h>
10 #include <zephyr/init.h>
11 #include <zephyr/linker/linker-defs.h>
12 #include <zephyr/sys/iterable_sections.h>
13
k_heap_init(struct k_heap * h,void * mem,size_t bytes)14 void k_heap_init(struct k_heap *h, void *mem, size_t bytes)
15 {
16 z_waitq_init(&h->wait_q);
17 sys_heap_init(&h->heap, mem, bytes);
18
19 SYS_PORT_TRACING_OBJ_INIT(k_heap, h);
20 }
21
statics_init(void)22 static int statics_init(void)
23 {
24 STRUCT_SECTION_FOREACH(k_heap, h) {
25 #if defined(CONFIG_DEMAND_PAGING) && !defined(CONFIG_LINKER_GENERIC_SECTIONS_PRESENT_AT_BOOT)
26 /* Some heaps may not present at boot, so we need to wait for
27 * paging mechanism to be initialized before we can initialize
28 * each heap.
29 */
30 extern bool z_sys_post_kernel;
31 bool do_clear = z_sys_post_kernel;
32
33 /* During pre-kernel init, z_sys_post_kernel == false,
34 * initialize if within pinned region. Otherwise skip.
35 * In post-kernel init, z_sys_post_kernel == true, skip those in
36 * pinned region as they have already been initialized and
37 * possibly already in use. Otherwise initialize.
38 */
39 if (lnkr_is_pinned((uint8_t *)h) &&
40 lnkr_is_pinned((uint8_t *)&h->wait_q) &&
41 lnkr_is_region_pinned((uint8_t *)h->heap.init_mem,
42 h->heap.init_bytes)) {
43 do_clear = !do_clear;
44 }
45
46 if (do_clear)
47 #endif /* CONFIG_DEMAND_PAGING && !CONFIG_LINKER_GENERIC_SECTIONS_PRESENT_AT_BOOT */
48 {
49 k_heap_init(h, h->heap.init_mem, h->heap.init_bytes);
50 }
51 }
52 return 0;
53 }
54
55 SYS_INIT_NAMED(statics_init_pre, statics_init, PRE_KERNEL_1, CONFIG_KERNEL_INIT_PRIORITY_OBJECTS);
56
57 #if defined(CONFIG_DEMAND_PAGING) && !defined(CONFIG_LINKER_GENERIC_SECTIONS_PRESENT_AT_BOOT)
58 /* Need to wait for paging mechanism to be initialized before
59 * heaps that are not in pinned sections can be initialized.
60 */
61 SYS_INIT_NAMED(statics_init_post, statics_init, POST_KERNEL, 0);
62 #endif /* CONFIG_DEMAND_PAGING && !CONFIG_LINKER_GENERIC_SECTIONS_PRESENT_AT_BOOT */
63
k_heap_aligned_alloc(struct k_heap * h,size_t align,size_t bytes,k_timeout_t timeout)64 void *k_heap_aligned_alloc(struct k_heap *h, size_t align, size_t bytes,
65 k_timeout_t timeout)
66 {
67 int64_t now, end = sys_clock_timeout_end_calc(timeout);
68 void *ret = NULL;
69
70 end = K_TIMEOUT_EQ(timeout, K_FOREVER) ? INT64_MAX : end;
71
72 k_spinlock_key_t key = k_spin_lock(&h->lock);
73
74 SYS_PORT_TRACING_OBJ_FUNC_ENTER(k_heap, aligned_alloc, h, timeout);
75
76 __ASSERT(!arch_is_in_isr() || K_TIMEOUT_EQ(timeout, K_NO_WAIT), "");
77
78 bool blocked_alloc = false;
79
80 while (ret == NULL) {
81 ret = sys_heap_aligned_alloc(&h->heap, align, bytes);
82
83 now = sys_clock_tick_get();
84 if (!IS_ENABLED(CONFIG_MULTITHREADING) ||
85 (ret != NULL) || ((end - now) <= 0)) {
86 break;
87 }
88
89 if (!blocked_alloc) {
90 blocked_alloc = true;
91
92 SYS_PORT_TRACING_OBJ_FUNC_BLOCKING(k_heap, aligned_alloc, h, timeout);
93 } else {
94 /**
95 * @todo Trace attempt to avoid empty trace segments
96 */
97 }
98
99 (void) z_pend_curr(&h->lock, key, &h->wait_q,
100 K_TICKS(end - now));
101 key = k_spin_lock(&h->lock);
102 }
103
104 SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_heap, aligned_alloc, h, timeout, ret);
105
106 k_spin_unlock(&h->lock, key);
107 return ret;
108 }
109
k_heap_alloc(struct k_heap * h,size_t bytes,k_timeout_t timeout)110 void *k_heap_alloc(struct k_heap *h, size_t bytes, k_timeout_t timeout)
111 {
112 SYS_PORT_TRACING_OBJ_FUNC_ENTER(k_heap, alloc, h, timeout);
113
114 void *ret = k_heap_aligned_alloc(h, sizeof(void *), bytes, timeout);
115
116 SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_heap, alloc, h, timeout, ret);
117
118 return ret;
119 }
120
k_heap_free(struct k_heap * h,void * mem)121 void k_heap_free(struct k_heap *h, void *mem)
122 {
123 k_spinlock_key_t key = k_spin_lock(&h->lock);
124
125 sys_heap_free(&h->heap, mem);
126
127 SYS_PORT_TRACING_OBJ_FUNC(k_heap, free, h);
128 if (IS_ENABLED(CONFIG_MULTITHREADING) && z_unpend_all(&h->wait_q) != 0) {
129 z_reschedule(&h->lock, key);
130 } else {
131 k_spin_unlock(&h->lock, key);
132 }
133 }
134
