1 /*
2 * Copyright (c) 2020 Intel Corporation
3 *
4 * SPDX-License-Identifier: Apache-2.0
5 */
6
7 #include <zephyr/kernel.h>
8 #include <zephyr/init.h>
9 #include <zephyr/linker/linker-defs.h>
10 #include <zephyr/sys/iterable_sections.h>
11 /* private kernel APIs */
12 #include <ksched.h>
13 #include <wait_q.h>
14
k_heap_init(struct k_heap * heap,void * mem,size_t bytes)15 void k_heap_init(struct k_heap *heap, void *mem, size_t bytes)
16 {
17 z_waitq_init(&heap->wait_q);
18 sys_heap_init(&heap->heap, mem, bytes);
19
20 SYS_PORT_TRACING_OBJ_INIT(k_heap, heap);
21 }
22
statics_init(void)23 static int statics_init(void)
24 {
25 STRUCT_SECTION_FOREACH(k_heap, heap) {
26 #if defined(CONFIG_DEMAND_PAGING) && !defined(CONFIG_LINKER_GENERIC_SECTIONS_PRESENT_AT_BOOT)
27 /* Some heaps may not present at boot, so we need to wait for
28 * paging mechanism to be initialized before we can initialize
29 * each heap.
30 */
31 extern bool z_sys_post_kernel;
32 bool do_clear = z_sys_post_kernel;
33
34 /* During pre-kernel init, z_sys_post_kernel == false,
35 * initialize if within pinned region. Otherwise skip.
36 * In post-kernel init, z_sys_post_kernel == true, skip those in
37 * pinned region as they have already been initialized and
38 * possibly already in use. Otherwise initialize.
39 */
40 if (lnkr_is_pinned((uint8_t *)heap) &&
41 lnkr_is_pinned((uint8_t *)&heap->wait_q) &&
42 lnkr_is_region_pinned((uint8_t *)heap->heap.init_mem,
43 heap->heap.init_bytes)) {
44 do_clear = !do_clear;
45 }
46
47 if (do_clear)
48 #endif /* CONFIG_DEMAND_PAGING && !CONFIG_LINKER_GENERIC_SECTIONS_PRESENT_AT_BOOT */
49 {
50 k_heap_init(heap, heap->heap.init_mem, heap->heap.init_bytes);
51 }
52 }
53 return 0;
54 }
55
56 SYS_INIT_NAMED(statics_init_pre, statics_init, PRE_KERNEL_1, CONFIG_KERNEL_INIT_PRIORITY_OBJECTS);
57
58 #if defined(CONFIG_DEMAND_PAGING) && !defined(CONFIG_LINKER_GENERIC_SECTIONS_PRESENT_AT_BOOT)
59 /* Need to wait for paging mechanism to be initialized before
60 * heaps that are not in pinned sections can be initialized.
61 */
62 SYS_INIT_NAMED(statics_init_post, statics_init, POST_KERNEL, 0);
63 #endif /* CONFIG_DEMAND_PAGING && !CONFIG_LINKER_GENERIC_SECTIONS_PRESENT_AT_BOOT */
64
k_heap_aligned_alloc(struct k_heap * heap,size_t align,size_t bytes,k_timeout_t timeout)65 void *k_heap_aligned_alloc(struct k_heap *heap, size_t align, size_t bytes,
66 k_timeout_t timeout)
67 {
68 k_timepoint_t end = sys_timepoint_calc(timeout);
69 void *ret = NULL;
70
71 k_spinlock_key_t key = k_spin_lock(&heap->lock);
72
73 SYS_PORT_TRACING_OBJ_FUNC_ENTER(k_heap, aligned_alloc, heap, timeout);
74
75 __ASSERT(!arch_is_in_isr() || K_TIMEOUT_EQ(timeout, K_NO_WAIT), "");
76
77 bool blocked_alloc = false;
78
79 while (ret == NULL) {
80 ret = sys_heap_aligned_alloc(&heap->heap, align, bytes);
81
82 if (!IS_ENABLED(CONFIG_MULTITHREADING) ||
83 (ret != NULL) || K_TIMEOUT_EQ(timeout, K_NO_WAIT)) {
84 break;
85 }
86
87 if (!blocked_alloc) {
88 blocked_alloc = true;
89
90 SYS_PORT_TRACING_OBJ_FUNC_BLOCKING(k_heap, aligned_alloc, heap, timeout);
91 } else {
92 /**
93 * @todo Trace attempt to avoid empty trace segments
94 */
95 }
96
97 timeout = sys_timepoint_timeout(end);
98 (void) z_pend_curr(&heap->lock, key, &heap->wait_q, timeout);
99 key = k_spin_lock(&heap->lock);
100 }
101
102 SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_heap, aligned_alloc, heap, timeout, ret);
103
104 k_spin_unlock(&heap->lock, key);
105 return ret;
106 }
107
k_heap_alloc(struct k_heap * heap,size_t bytes,k_timeout_t timeout)108 void *k_heap_alloc(struct k_heap *heap, size_t bytes, k_timeout_t timeout)
109 {
110 SYS_PORT_TRACING_OBJ_FUNC_ENTER(k_heap, alloc, heap, timeout);
111
112 void *ret = k_heap_aligned_alloc(heap, sizeof(void *), bytes, timeout);
113
114 SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_heap, alloc, heap, timeout, ret);
115
116 return ret;
117 }
118
k_heap_calloc(struct k_heap * heap,size_t num,size_t size,k_timeout_t timeout)119 void *k_heap_calloc(struct k_heap *heap, size_t num, size_t size, k_timeout_t timeout)
120 {
121 SYS_PORT_TRACING_OBJ_FUNC_ENTER(k_heap, calloc, heap, timeout);
122
123 void *ret = NULL;
124 size_t bounds = 0U;
125
126 if (!size_mul_overflow(num, size, &bounds)) {
127 ret = k_heap_alloc(heap, bounds, timeout);
128 }
129 if (ret != NULL) {
130 (void)memset(ret, 0, bounds);
131 }
132
133 SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_heap, calloc, heap, timeout, ret);
134
135 return ret;
136 }
137
k_heap_realloc(struct k_heap * heap,void * ptr,size_t bytes,k_timeout_t timeout)138 void *k_heap_realloc(struct k_heap *heap, void *ptr, size_t bytes, k_timeout_t timeout)
139 {
140 k_timepoint_t end = sys_timepoint_calc(timeout);
141 void *ret = NULL;
142
143 k_spinlock_key_t key = k_spin_lock(&heap->lock);
144
145 SYS_PORT_TRACING_OBJ_FUNC_ENTER(k_heap, realloc, heap, ptr, bytes, timeout);
146
147 __ASSERT(!arch_is_in_isr() || K_TIMEOUT_EQ(timeout, K_NO_WAIT), "");
148
149 while (ret == NULL) {
150 ret = sys_heap_aligned_realloc(&heap->heap, ptr, sizeof(void *), bytes);
151
152 if (!IS_ENABLED(CONFIG_MULTITHREADING) ||
153 (ret != NULL) || K_TIMEOUT_EQ(timeout, K_NO_WAIT)) {
154 break;
155 }
156
157 timeout = sys_timepoint_timeout(end);
158 (void) z_pend_curr(&heap->lock, key, &heap->wait_q, timeout);
159 key = k_spin_lock(&heap->lock);
160 }
161
162 SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_heap, realloc, heap, ptr, bytes, timeout, ret);
163
164 k_spin_unlock(&heap->lock, key);
165 return ret;
166 }
167
k_heap_free(struct k_heap * heap,void * mem)168 void k_heap_free(struct k_heap *heap, void *mem)
169 {
170 k_spinlock_key_t key = k_spin_lock(&heap->lock);
171
172 sys_heap_free(&heap->heap, mem);
173
174 SYS_PORT_TRACING_OBJ_FUNC(k_heap, free, heap);
175 if (IS_ENABLED(CONFIG_MULTITHREADING) && (z_unpend_all(&heap->wait_q) != 0)) {
176 z_reschedule(&heap->lock, key);
177 } else {
178 k_spin_unlock(&heap->lock, key);
179 }
180 }
181
