1 /*
2 * Copyright (c) 2020 Intel Corporation
3 *
4 * SPDX-License-Identifier: Apache-2.0
5 */
6
7 #include <zephyr/kernel.h>
8 #include <zephyr/init.h>
9 #include <zephyr/linker/linker-defs.h>
10 #include <zephyr/sys/iterable_sections.h>
11 /* private kernel APIs */
12 #include <ksched.h>
13 #include <wait_q.h>
14
k_heap_init(struct k_heap * heap,void * mem,size_t bytes)15 void k_heap_init(struct k_heap *heap, void *mem, size_t bytes)
16 {
17 z_waitq_init(&heap->wait_q);
18 heap->lock = (struct k_spinlock) {};
19 sys_heap_init(&heap->heap, mem, bytes);
20
21 SYS_PORT_TRACING_OBJ_INIT(k_heap, heap);
22 }
23
statics_init(void)24 static int statics_init(void)
25 {
26 STRUCT_SECTION_FOREACH(k_heap, heap) {
27 #if defined(CONFIG_DEMAND_PAGING) && !defined(CONFIG_LINKER_GENERIC_SECTIONS_PRESENT_AT_BOOT)
28 /* Some heaps may not present at boot, so we need to wait for
29 * paging mechanism to be initialized before we can initialize
30 * each heap.
31 */
32 extern bool z_sys_post_kernel;
33 bool do_clear = z_sys_post_kernel;
34
35 /* During pre-kernel init, z_sys_post_kernel == false,
36 * initialize if within pinned region. Otherwise skip.
37 * In post-kernel init, z_sys_post_kernel == true, skip those in
38 * pinned region as they have already been initialized and
39 * possibly already in use. Otherwise initialize.
40 */
41 if (lnkr_is_pinned((uint8_t *)heap) &&
42 lnkr_is_pinned((uint8_t *)&heap->wait_q) &&
43 lnkr_is_region_pinned((uint8_t *)heap->heap.init_mem,
44 heap->heap.init_bytes)) {
45 do_clear = !do_clear;
46 }
47
48 if (do_clear)
49 #endif /* CONFIG_DEMAND_PAGING && !CONFIG_LINKER_GENERIC_SECTIONS_PRESENT_AT_BOOT */
50 {
51 k_heap_init(heap, heap->heap.init_mem, heap->heap.init_bytes);
52 }
53 }
54 return 0;
55 }
56
57 SYS_INIT_NAMED(statics_init_pre, statics_init, PRE_KERNEL_1, CONFIG_KERNEL_INIT_PRIORITY_OBJECTS);
58
59 #if defined(CONFIG_DEMAND_PAGING) && !defined(CONFIG_LINKER_GENERIC_SECTIONS_PRESENT_AT_BOOT)
60 /* Need to wait for paging mechanism to be initialized before
61 * heaps that are not in pinned sections can be initialized.
62 */
63 SYS_INIT_NAMED(statics_init_post, statics_init, POST_KERNEL, 0);
64 #endif /* CONFIG_DEMAND_PAGING && !CONFIG_LINKER_GENERIC_SECTIONS_PRESENT_AT_BOOT */
65
k_heap_aligned_alloc(struct k_heap * heap,size_t align,size_t bytes,k_timeout_t timeout)66 void *k_heap_aligned_alloc(struct k_heap *heap, size_t align, size_t bytes,
67 k_timeout_t timeout)
68 {
69 k_timepoint_t end = sys_timepoint_calc(timeout);
70 void *ret = NULL;
71
72 k_spinlock_key_t key = k_spin_lock(&heap->lock);
73
74 SYS_PORT_TRACING_OBJ_FUNC_ENTER(k_heap, aligned_alloc, heap, timeout);
75
76 __ASSERT(!arch_is_in_isr() || K_TIMEOUT_EQ(timeout, K_NO_WAIT), "");
77
78 bool blocked_alloc = false;
79
80 while (ret == NULL) {
81 ret = sys_heap_aligned_alloc(&heap->heap, align, bytes);
82
83 if (!IS_ENABLED(CONFIG_MULTITHREADING) ||
84 (ret != NULL) || K_TIMEOUT_EQ(timeout, K_NO_WAIT)) {
85 break;
86 }
87
88 if (!blocked_alloc) {
89 blocked_alloc = true;
90
91 SYS_PORT_TRACING_OBJ_FUNC_BLOCKING(k_heap, aligned_alloc, heap, timeout);
92 } else {
93 /**
94 * @todo Trace attempt to avoid empty trace segments
95 */
96 }
97
98 timeout = sys_timepoint_timeout(end);
99 (void) z_pend_curr(&heap->lock, key, &heap->wait_q, timeout);
100 key = k_spin_lock(&heap->lock);
101 }
102
103 SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_heap, aligned_alloc, heap, timeout, ret);
104
105 k_spin_unlock(&heap->lock, key);
106 return ret;
107 }
108
k_heap_alloc(struct k_heap * heap,size_t bytes,k_timeout_t timeout)109 void *k_heap_alloc(struct k_heap *heap, size_t bytes, k_timeout_t timeout)
110 {
111 SYS_PORT_TRACING_OBJ_FUNC_ENTER(k_heap, alloc, heap, timeout);
112
113 void *ret = k_heap_aligned_alloc(heap, sizeof(void *), bytes, timeout);
114
115 SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_heap, alloc, heap, timeout, ret);
116
117 return ret;
118 }
119
k_heap_calloc(struct k_heap * heap,size_t num,size_t size,k_timeout_t timeout)120 void *k_heap_calloc(struct k_heap *heap, size_t num, size_t size, k_timeout_t timeout)
121 {
122 SYS_PORT_TRACING_OBJ_FUNC_ENTER(k_heap, calloc, heap, timeout);
123
124 void *ret = NULL;
125 size_t bounds = 0U;
126
127 if (!size_mul_overflow(num, size, &bounds)) {
128 ret = k_heap_alloc(heap, bounds, timeout);
129 }
130 if (ret != NULL) {
131 (void)memset(ret, 0, bounds);
132 }
133
134 SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_heap, calloc, heap, timeout, ret);
135
136 return ret;
137 }
138
k_heap_realloc(struct k_heap * heap,void * ptr,size_t bytes,k_timeout_t timeout)139 void *k_heap_realloc(struct k_heap *heap, void *ptr, size_t bytes, k_timeout_t timeout)
140 {
141 k_timepoint_t end = sys_timepoint_calc(timeout);
142 void *ret = NULL;
143
144 k_spinlock_key_t key = k_spin_lock(&heap->lock);
145
146 SYS_PORT_TRACING_OBJ_FUNC_ENTER(k_heap, realloc, heap, ptr, bytes, timeout);
147
148 __ASSERT(!arch_is_in_isr() || K_TIMEOUT_EQ(timeout, K_NO_WAIT), "");
149
150 while (ret == NULL) {
151 ret = sys_heap_aligned_realloc(&heap->heap, ptr, sizeof(void *), bytes);
152
153 if (!IS_ENABLED(CONFIG_MULTITHREADING) ||
154 (ret != NULL) || K_TIMEOUT_EQ(timeout, K_NO_WAIT)) {
155 break;
156 }
157
158 timeout = sys_timepoint_timeout(end);
159 (void) z_pend_curr(&heap->lock, key, &heap->wait_q, timeout);
160 key = k_spin_lock(&heap->lock);
161 }
162
163 SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_heap, realloc, heap, ptr, bytes, timeout, ret);
164
165 k_spin_unlock(&heap->lock, key);
166 return ret;
167 }
168
k_heap_free(struct k_heap * heap,void * mem)169 void k_heap_free(struct k_heap *heap, void *mem)
170 {
171 k_spinlock_key_t key = k_spin_lock(&heap->lock);
172
173 sys_heap_free(&heap->heap, mem);
174
175 SYS_PORT_TRACING_OBJ_FUNC(k_heap, free, heap);
176 if (IS_ENABLED(CONFIG_MULTITHREADING) && (z_unpend_all(&heap->wait_q) != 0)) {
177 z_reschedule(&heap->lock, key);
178 } else {
179 k_spin_unlock(&heap->lock, key);
180 }
181 }
182
