1 /*
2 * Copyright (c) 2024 Nordic Semiconductor ASA
3 * SPDX-License-Identifier: Apache-2.0
4 */
5
6 #include <string.h>
7 #include <zephyr/cache.h>
8 #include <zephyr/kernel.h>
9 #include <zephyr/sys/sys_heap.h>
10 #include <zephyr/mem_mgmt/mem_attr.h>
11 #include "dmm.h"
12
13 #define _FILTER_MEM(node_id, fn) \
14 COND_CODE_1(DT_NODE_HAS_PROP(node_id, zephyr_memory_attr), (fn(node_id)), ())
15 #define DT_MEMORY_REGION_FOREACH_STATUS_OKAY_NODE(fn) \
16 DT_FOREACH_STATUS_OKAY_NODE_VARGS(_FILTER_MEM, fn)
17
18 #define __BUILD_LINKER_END_VAR(_name) DT_CAT3(__, _name, _end)
19 #define _BUILD_LINKER_END_VAR(node_id) \
20 __BUILD_LINKER_END_VAR(DT_STRING_UNQUOTED(node_id, zephyr_memory_region))
21
22 #define _BUILD_MEM_REGION(node_id) \
23 {.dt_addr = DT_REG_ADDR(node_id), \
24 .dt_size = DT_REG_SIZE(node_id), \
25 .dt_attr = DT_PROP(node_id, zephyr_memory_attr), \
26 .dt_align = DMM_REG_ALIGN_SIZE(node_id), \
27 .dt_allc = &_BUILD_LINKER_END_VAR(node_id)},
28
29 /* Generate declarations of linker variables used to determine size of preallocated variables
30 * stored in memory sections spanning over memory regions.
31 * These are used to determine memory left for dynamic bounce buffer allocator to work with.
32 */
33 #define _DECLARE_LINKER_VARS(node_id) extern uint32_t _BUILD_LINKER_END_VAR(node_id);
34 DT_MEMORY_REGION_FOREACH_STATUS_OKAY_NODE(_DECLARE_LINKER_VARS);
35
36 struct dmm_region {
37 uintptr_t dt_addr;
38 size_t dt_size;
39 uint32_t dt_attr;
40 uint32_t dt_align;
41 void *dt_allc;
42 };
43
44 struct dmm_heap {
45 struct sys_heap heap;
46 const struct dmm_region *region;
47 };
48
49 static const struct dmm_region dmm_regions[] = {
50 DT_MEMORY_REGION_FOREACH_STATUS_OKAY_NODE(_BUILD_MEM_REGION)
51 };
52
53 struct {
54 struct dmm_heap dmm_heaps[ARRAY_SIZE(dmm_regions)];
55 } dmm_heaps_data;
56
dmm_heap_find(void * region)57 static struct dmm_heap *dmm_heap_find(void *region)
58 {
59 struct dmm_heap *dh;
60
61 for (size_t idx = 0; idx < ARRAY_SIZE(dmm_heaps_data.dmm_heaps); idx++) {
62 dh = &dmm_heaps_data.dmm_heaps[idx];
63 if (dh->region->dt_addr == (uintptr_t)region) {
64 return dh;
65 }
66 }
67
68 return NULL;
69 }
70
is_region_cacheable(const struct dmm_region * region)71 static bool is_region_cacheable(const struct dmm_region *region)
72 {
73 return (IS_ENABLED(CONFIG_DCACHE) && (region->dt_attr & DT_MEM_CACHEABLE));
74 }
75
is_buffer_within_region(uintptr_t start,size_t size,uintptr_t reg_start,size_t reg_size)76 static bool is_buffer_within_region(uintptr_t start, size_t size,
77 uintptr_t reg_start, size_t reg_size)
78 {
79 return ((start >= reg_start) && ((start + size) <= (reg_start + reg_size)));
80 }
81
is_user_buffer_correctly_preallocated(void const * user_buffer,size_t user_length,const struct dmm_region * region)82 static bool is_user_buffer_correctly_preallocated(void const *user_buffer, size_t user_length,
83 const struct dmm_region *region)
84 {
85 uintptr_t addr = (uintptr_t)user_buffer;
86
87 if (!is_buffer_within_region(addr, user_length, region->dt_addr, region->dt_size)) {
88 return false;
89 }
90
91 if (!is_region_cacheable(region)) {
92 /* Buffer is contained within non-cacheable region - use it as it is. */
93 return true;
94 }
95
96 if (IS_ALIGNED(addr, region->dt_align)) {
97 /* If buffer is in cacheable region it must be aligned to data cache line size. */
98 return true;
99 }
100
101 return false;
102 }
103
dmm_heap_start_get(struct dmm_heap * dh)104 static size_t dmm_heap_start_get(struct dmm_heap *dh)
105 {
106 return ROUND_UP(dh->region->dt_allc, dh->region->dt_align);
107 }
108
dmm_heap_size_get(struct dmm_heap * dh)109 static size_t dmm_heap_size_get(struct dmm_heap *dh)
110 {
111 return (dh->region->dt_size - (dmm_heap_start_get(dh) - dh->region->dt_addr));
112 }
113
dmm_buffer_alloc(struct dmm_heap * dh,size_t length)114 static void *dmm_buffer_alloc(struct dmm_heap *dh, size_t length)
115 {
116 length = ROUND_UP(length, dh->region->dt_align);
117 return sys_heap_aligned_alloc(&dh->heap, dh->region->dt_align, length);
118 }
119
dmm_buffer_free(struct dmm_heap * dh,void * buffer)120 static void dmm_buffer_free(struct dmm_heap *dh, void *buffer)
121 {
122 sys_heap_free(&dh->heap, buffer);
123 }
124
dmm_buffer_out_prepare(void * region,void const * user_buffer,size_t user_length,void ** buffer_out)125 int dmm_buffer_out_prepare(void *region, void const *user_buffer, size_t user_length,
126 void **buffer_out)
127 {
128 struct dmm_heap *dh;
129
130 if (user_length == 0) {
131 /* Assume that zero-length buffers are correct as they are. */
132 *buffer_out = (void *)user_buffer;
133 return 0;
134 }
135
136 /* Get memory region that specified device can perform DMA transfers from */
137 dh = dmm_heap_find(region);
138 if (dh == NULL) {
139 return -EINVAL;
140 }
141
142 /* Check if:
143 * - provided user buffer is already in correct memory region,
144 * - provided user buffer is aligned and padded to cache line,
145 * if it is located in cacheable region.
146 */
147 if (is_user_buffer_correctly_preallocated(user_buffer, user_length, dh->region)) {
148 /* If yes, assign buffer_out to user_buffer*/
149 *buffer_out = (void *)user_buffer;
150 } else {
151 /* If no:
152 * - dynamically allocate buffer in correct memory region that respects cache line
153 * alignment and padding
154 */
155 *buffer_out = dmm_buffer_alloc(dh, user_length);
156 /* Return error if dynamic allocation fails */
157 if (*buffer_out == NULL) {
158 return -ENOMEM;
159 }
160 /* - copy user buffer contents into allocated buffer */
161 memcpy(*buffer_out, user_buffer, user_length);
162 }
163
164 /* Check if device memory region is cacheable
165 * If yes, writeback all cache lines associated with output buffer
166 * (either user or allocated)
167 */
168 if (is_region_cacheable(dh->region)) {
169 sys_cache_data_flush_range(*buffer_out, user_length);
170 }
171 /* If no, no action is needed */
172
173 return 0;
174 }
175
dmm_buffer_out_release(void * region,void * buffer_out)176 int dmm_buffer_out_release(void *region, void *buffer_out)
177 {
178 struct dmm_heap *dh;
179 uintptr_t addr = (uintptr_t)buffer_out;
180
181 /* Get memory region that specified device can perform DMA transfers from */
182 dh = dmm_heap_find(region);
183 if (dh == NULL) {
184 return -EINVAL;
185 }
186
187 /* Check if output buffer is contained within memory area
188 * managed by dynamic memory allocator
189 */
190 if (is_buffer_within_region(addr, 0, dmm_heap_start_get(dh), dmm_heap_size_get(dh))) {
191 /* If yes, free the buffer */
192 dmm_buffer_free(dh, buffer_out);
193 }
194 /* If no, no action is needed */
195
196 return 0;
197 }
198
dmm_buffer_in_prepare(void * region,void * user_buffer,size_t user_length,void ** buffer_in)199 int dmm_buffer_in_prepare(void *region, void *user_buffer, size_t user_length, void **buffer_in)
200 {
201 struct dmm_heap *dh;
202
203 if (user_length == 0) {
204 /* Assume that zero-length buffers are correct as they are. */
205 *buffer_in = (void *)user_buffer;
206 return 0;
207 }
208
209 /* Get memory region that specified device can perform DMA transfers to */
210 dh = dmm_heap_find(region);
211 if (dh == NULL) {
212 return -EINVAL;
213 }
214
215 /* Check if:
216 * - provided user buffer is already in correct memory region,
217 * - provided user buffer is aligned and padded to cache line,
218 * if it is located in cacheable region.
219 */
220 if (is_user_buffer_correctly_preallocated(user_buffer, user_length, dh->region)) {
221 /* If yes, assign buffer_in to user_buffer */
222 *buffer_in = user_buffer;
223 } else {
224 /* If no, dynamically allocate buffer in correct memory region that respects cache
225 * line alignment and padding
226 */
227 *buffer_in = dmm_buffer_alloc(dh, user_length);
228 /* Return error if dynamic allocation fails */
229 if (*buffer_in == NULL) {
230 return -ENOMEM;
231 }
232 }
233
234 /* Check if device memory region is cacheable
235 * If yes, invalidate all cache lines associated with input buffer
236 * (either user or allocated) to clear potential dirty bits.
237 */
238 if (is_region_cacheable(dh->region)) {
239 sys_cache_data_invd_range(*buffer_in, user_length);
240 }
241 /* If no, no action is needed */
242
243 return 0;
244 }
245
dmm_buffer_in_release(void * region,void * user_buffer,size_t user_length,void * buffer_in)246 int dmm_buffer_in_release(void *region, void *user_buffer, size_t user_length, void *buffer_in)
247 {
248 struct dmm_heap *dh;
249 uintptr_t addr = (uintptr_t)buffer_in;
250
251 /* Get memory region that specified device can perform DMA transfers to, using devicetree */
252 dh = dmm_heap_find(region);
253 if (dh == NULL) {
254 return -EINVAL;
255 }
256
257 /* Check if device memory region is cacheable
258 * If yes, invalidate all cache lines associated with input buffer
259 * (either user or allocated)
260 */
261 if (is_region_cacheable(dh->region)) {
262 sys_cache_data_invd_range(buffer_in, user_length);
263 }
264 /* If no, no action is needed */
265
266 /* Check if user buffer and allocated buffer points to the same memory location
267 * If no, copy allocated buffer to the user buffer
268 */
269 if (buffer_in != user_buffer) {
270 memcpy(user_buffer, buffer_in, user_length);
271 }
272 /* If yes, no action is needed */
273
274 /* Check if input buffer is contained within memory area
275 * managed by dynamic memory allocator
276 */
277 if (is_buffer_within_region(addr, 0, dmm_heap_start_get(dh), dmm_heap_size_get(dh))) {
278 /* If yes, free the buffer */
279 dmm_buffer_free(dh, buffer_in);
280 }
281 /* If no, no action is needed */
282
283 return 0;
284 }
285
dmm_init(void)286 int dmm_init(void)
287 {
288 struct dmm_heap *dh;
289
290 for (size_t idx = 0; idx < ARRAY_SIZE(dmm_regions); idx++) {
291 dh = &dmm_heaps_data.dmm_heaps[idx];
292 dh->region = &dmm_regions[idx];
293 sys_heap_init(&dh->heap, (void *)dmm_heap_start_get(dh), dmm_heap_size_get(dh));
294 }
295
296 return 0;
297 }
298
