1 /*
2  * SPDX-FileCopyrightText: 2020-2022 Espressif Systems (Shanghai) CO LTD
3  *
4  * SPDX-License-Identifier: Apache-2.0
5  */
6 
7 #include <sys/param.h>
8 #include "freertos/FreeRTOS.h"
9 #include "freertos/semphr.h"
10 #include "hal/dma_types.h"
11 #include "esp_check.h"
12 #include "esp_heap_caps.h"
13 #include "esp_log.h"
14 #include "esp_async_memcpy.h"
15 #include "esp_async_memcpy_impl.h"
16 
17 static const char *TAG = "async_memcpy";
18 
19 #define ALIGN_DOWN(val, align)  ((val) & ~((align) - 1))
20 
21 /**
22  * @brief Type of async mcp stream
23  *        mcp stream inherits DMA descriptor, besides that, it has a callback function member
24  */
25 typedef struct {
26     dma_descriptor_t desc;
27     async_memcpy_isr_cb_t cb;
28     void *cb_args;
29 } async_memcpy_stream_t;
30 
31 /**
32  * @brief Type of async mcp driver context
33  */
34 typedef struct async_memcpy_context_t {
35     async_memcpy_impl_t mcp_impl; // implementation layer
36     portMUX_TYPE spinlock;     // spinlock, prevent operating descriptors concurrently
37     intr_handle_t intr_hdl;    // interrupt handle
38     uint32_t flags;            // extra driver flags
39     dma_descriptor_t *tx_desc; // pointer to the next free TX descriptor
40     dma_descriptor_t *rx_desc; // pointer to the next free RX descriptor
41     dma_descriptor_t *next_rx_desc_to_check; // pointer to the next RX descriptor to recycle
42     uint32_t max_stream_num;    // maximum number of streams
43     size_t max_dma_buffer_size; // maximum DMA buffer size
44     async_memcpy_stream_t *out_streams;    // pointer to the first TX stream
45     async_memcpy_stream_t *in_streams;     // pointer to the first RX stream
46     async_memcpy_stream_t streams_pool[0]; // stream pool (TX + RX), the size is configured during driver installation
47 } async_memcpy_context_t;
48 
esp_async_memcpy_install(const async_memcpy_config_t * config,async_memcpy_t * asmcp)49 esp_err_t esp_async_memcpy_install(const async_memcpy_config_t *config, async_memcpy_t *asmcp)
50 {
51     esp_err_t ret = ESP_OK;
52     async_memcpy_context_t *mcp_hdl = NULL;
53 
54     ESP_GOTO_ON_FALSE(config, ESP_ERR_INVALID_ARG, err, TAG, "configuration can't be null");
55     ESP_GOTO_ON_FALSE(asmcp, ESP_ERR_INVALID_ARG, err, TAG, "can't assign mcp handle to null");
56 
57     // context memory size + stream pool size
58     size_t total_malloc_size = sizeof(async_memcpy_context_t) + sizeof(async_memcpy_stream_t) * config->backlog * 2;
59     // to work when cache is disabled, the driver handle should located in SRAM
60     mcp_hdl = heap_caps_calloc(1, total_malloc_size, MALLOC_CAP_8BIT | MALLOC_CAP_DMA | MALLOC_CAP_INTERNAL);
61     ESP_GOTO_ON_FALSE(mcp_hdl, ESP_ERR_NO_MEM, err, TAG, "allocate context memory failed");
62 
63     mcp_hdl->flags = config->flags;
64     mcp_hdl->out_streams = mcp_hdl->streams_pool;
65     mcp_hdl->in_streams = mcp_hdl->streams_pool + config->backlog;
66     mcp_hdl->max_stream_num = config->backlog;
67 
68     // circle TX/RX descriptors
69     for (size_t i = 0; i < mcp_hdl->max_stream_num; i++) {
70         mcp_hdl->out_streams[i].desc.dw0.owner = DMA_DESCRIPTOR_BUFFER_OWNER_CPU;
71         mcp_hdl->out_streams[i].desc.next = &mcp_hdl->out_streams[i + 1].desc;
72         mcp_hdl->in_streams[i].desc.dw0.owner = DMA_DESCRIPTOR_BUFFER_OWNER_CPU;
73         mcp_hdl->in_streams[i].desc.next = &mcp_hdl->in_streams[i + 1].desc;
74     }
75     mcp_hdl->out_streams[mcp_hdl->max_stream_num - 1].desc.next = &mcp_hdl->out_streams[0].desc;
76     mcp_hdl->in_streams[mcp_hdl->max_stream_num - 1].desc.next = &mcp_hdl->in_streams[0].desc;
77 
78     mcp_hdl->tx_desc = &mcp_hdl->out_streams[0].desc;
79     mcp_hdl->rx_desc = &mcp_hdl->in_streams[0].desc;
80     mcp_hdl->next_rx_desc_to_check = &mcp_hdl->in_streams[0].desc;
81     mcp_hdl->spinlock = (portMUX_TYPE)portMUX_INITIALIZER_UNLOCKED;
82     mcp_hdl->mcp_impl.sram_trans_align = config->sram_trans_align;
83     mcp_hdl->mcp_impl.psram_trans_align = config->psram_trans_align;
84     size_t trans_align = MAX(config->sram_trans_align, config->psram_trans_align);
85     mcp_hdl->max_dma_buffer_size = trans_align ? ALIGN_DOWN(DMA_DESCRIPTOR_BUFFER_MAX_SIZE, trans_align) : DMA_DESCRIPTOR_BUFFER_MAX_SIZE;
86 
87     // initialize implementation layer
88     ret = async_memcpy_impl_init(&mcp_hdl->mcp_impl);
89     ESP_GOTO_ON_ERROR(ret, err, TAG, "DMA M2M init failed");
90 
91     ESP_LOGD(TAG, "installed memory to memory copy channel at %p", mcp_hdl);
92 
93     *asmcp = mcp_hdl;
94 
95     async_memcpy_impl_start(&mcp_hdl->mcp_impl, (intptr_t)&mcp_hdl->out_streams[0].desc, (intptr_t)&mcp_hdl->in_streams[0].desc);
96 
97     return ESP_OK;
98 err:
99     if (mcp_hdl) {
100         free(mcp_hdl);
101     }
102     if (asmcp) {
103         *asmcp = NULL;
104     }
105     return ret;
106 }
107 
esp_async_memcpy_uninstall(async_memcpy_t asmcp)108 esp_err_t esp_async_memcpy_uninstall(async_memcpy_t asmcp)
109 {
110     esp_err_t ret = ESP_OK;
111     ESP_GOTO_ON_FALSE(asmcp, ESP_ERR_INVALID_ARG, err, TAG, "mcp handle can't be null");
112 
113     async_memcpy_impl_stop(&asmcp->mcp_impl);
114     async_memcpy_impl_deinit(&asmcp->mcp_impl);
115     free(asmcp);
116 err:
117     return ret;
118 }
119 
esp_async_memcpy_new_etm_event(async_memcpy_t asmcp,async_memcpy_etm_event_t event_type,esp_etm_event_handle_t * out_event)120 esp_err_t esp_async_memcpy_new_etm_event(async_memcpy_t asmcp, async_memcpy_etm_event_t event_type, esp_etm_event_handle_t *out_event)
121 {
122     ESP_RETURN_ON_FALSE(asmcp, ESP_ERR_INVALID_ARG, TAG, "mcp handle can't be null");
123     return async_memcpy_impl_new_etm_event(&asmcp->mcp_impl, event_type, out_event);
124 }
125 
async_memcpy_prepare_receive(async_memcpy_t asmcp,void * buffer,size_t size,dma_descriptor_t ** start_desc,dma_descriptor_t ** end_desc)126 static int async_memcpy_prepare_receive(async_memcpy_t asmcp, void *buffer, size_t size, dma_descriptor_t **start_desc, dma_descriptor_t **end_desc)
127 {
128     uint32_t prepared_length = 0;
129     uint8_t *buf = (uint8_t *)buffer;
130     dma_descriptor_t *desc = asmcp->rx_desc; // descriptor iterator
131     dma_descriptor_t *start = desc;
132     dma_descriptor_t *end = desc;
133 
134     while (size > asmcp->max_dma_buffer_size) {
135         if (desc->dw0.owner != DMA_DESCRIPTOR_BUFFER_OWNER_DMA) {
136             desc->dw0.suc_eof = 0;
137             desc->dw0.size = asmcp->max_dma_buffer_size;
138             desc->buffer = &buf[prepared_length];
139             desc = desc->next; // move to next descriptor
140             prepared_length += asmcp->max_dma_buffer_size;
141             size -= asmcp->max_dma_buffer_size;
142         } else {
143             // out of RX descriptors
144             goto _exit;
145         }
146     }
147     if (size) {
148         if (desc->dw0.owner != DMA_DESCRIPTOR_BUFFER_OWNER_DMA) {
149             end = desc; // the last descriptor used
150             desc->dw0.suc_eof = 0;
151             desc->dw0.size = size;
152             desc->buffer = &buf[prepared_length];
153             desc = desc->next; // move to next descriptor
154             prepared_length += size;
155         } else {
156             // out of RX descriptors
157             goto _exit;
158         }
159     }
160 
161 _exit:
162     *start_desc = start;
163     *end_desc = end;
164     return prepared_length;
165 }
166 
async_memcpy_prepare_transmit(async_memcpy_t asmcp,void * buffer,size_t len,dma_descriptor_t ** start_desc,dma_descriptor_t ** end_desc)167 static int async_memcpy_prepare_transmit(async_memcpy_t asmcp, void *buffer, size_t len, dma_descriptor_t **start_desc, dma_descriptor_t **end_desc)
168 {
169     uint32_t prepared_length = 0;
170     uint8_t *buf = (uint8_t *)buffer;
171     dma_descriptor_t *desc = asmcp->tx_desc; // descriptor iterator
172     dma_descriptor_t *start = desc;
173     dma_descriptor_t *end = desc;
174 
175     while (len > asmcp->max_dma_buffer_size) {
176         if (desc->dw0.owner != DMA_DESCRIPTOR_BUFFER_OWNER_DMA) {
177             desc->dw0.suc_eof = 0; // not the end of the transaction
178             desc->dw0.size = asmcp->max_dma_buffer_size;
179             desc->dw0.length = asmcp->max_dma_buffer_size;
180             desc->buffer = &buf[prepared_length];
181             desc = desc->next; // move to next descriptor
182             prepared_length += asmcp->max_dma_buffer_size;
183             len -= asmcp->max_dma_buffer_size;
184         } else {
185             // out of TX descriptors
186             goto _exit;
187         }
188     }
189     if (len) {
190         if (desc->dw0.owner != DMA_DESCRIPTOR_BUFFER_OWNER_DMA) {
191             end = desc;            // the last descriptor used
192             desc->dw0.suc_eof = 1; // end of the transaction
193             desc->dw0.size = len;
194             desc->dw0.length = len;
195             desc->buffer = &buf[prepared_length];
196             desc = desc->next; // move to next descriptor
197             prepared_length += len;
198         } else {
199             // out of TX descriptors
200             goto _exit;
201         }
202     }
203 
204     *start_desc = start;
205     *end_desc = end;
206 _exit:
207     return prepared_length;
208 }
209 
async_memcpy_get_next_rx_descriptor(async_memcpy_t asmcp,dma_descriptor_t * eof_desc,dma_descriptor_t ** next_desc)210 static bool async_memcpy_get_next_rx_descriptor(async_memcpy_t asmcp, dma_descriptor_t *eof_desc, dma_descriptor_t **next_desc)
211 {
212     dma_descriptor_t *next = asmcp->next_rx_desc_to_check;
213     // additional check, to avoid potential interrupt got triggered by mistake
214     if (next->dw0.owner == DMA_DESCRIPTOR_BUFFER_OWNER_CPU) {
215         asmcp->next_rx_desc_to_check = asmcp->next_rx_desc_to_check->next;
216         *next_desc = next;
217         // return if we need to continue
218         return eof_desc == next ? false : true;
219     }
220 
221     *next_desc = NULL;
222     return false;
223 }
224 
esp_async_memcpy(async_memcpy_t asmcp,void * dst,void * src,size_t n,async_memcpy_isr_cb_t cb_isr,void * cb_args)225 esp_err_t esp_async_memcpy(async_memcpy_t asmcp, void *dst, void *src, size_t n, async_memcpy_isr_cb_t cb_isr, void *cb_args)
226 {
227     esp_err_t ret = ESP_OK;
228     dma_descriptor_t *rx_start_desc = NULL;
229     dma_descriptor_t *rx_end_desc = NULL;
230     dma_descriptor_t *tx_start_desc = NULL;
231     dma_descriptor_t *tx_end_desc = NULL;
232     size_t rx_prepared_size = 0;
233     size_t tx_prepared_size = 0;
234     ESP_GOTO_ON_FALSE(asmcp, ESP_ERR_INVALID_ARG, err, TAG, "mcp handle can't be null");
235     ESP_GOTO_ON_FALSE(async_memcpy_impl_is_buffer_address_valid(&asmcp->mcp_impl, src, dst), ESP_ERR_INVALID_ARG, err, TAG, "buffer address not valid: %p -> %p", src, dst);
236     ESP_GOTO_ON_FALSE(n <= asmcp->max_dma_buffer_size * asmcp->max_stream_num, ESP_ERR_INVALID_ARG, err, TAG, "buffer size too large");
237     if (asmcp->mcp_impl.sram_trans_align) {
238         ESP_GOTO_ON_FALSE(((n & (asmcp->mcp_impl.sram_trans_align - 1)) == 0), ESP_ERR_INVALID_ARG, err, TAG, "copy size should align to %d bytes", asmcp->mcp_impl.sram_trans_align);
239     }
240     if (asmcp->mcp_impl.psram_trans_align) {
241         ESP_GOTO_ON_FALSE(((n & (asmcp->mcp_impl.psram_trans_align - 1)) == 0), ESP_ERR_INVALID_ARG, err, TAG, "copy size should align to %d bytes", asmcp->mcp_impl.psram_trans_align);
242     }
243 
244     // Prepare TX and RX descriptor
245     portENTER_CRITICAL_SAFE(&asmcp->spinlock);
246     rx_prepared_size = async_memcpy_prepare_receive(asmcp, dst, n, &rx_start_desc, &rx_end_desc);
247     tx_prepared_size = async_memcpy_prepare_transmit(asmcp, src, n, &tx_start_desc, &tx_end_desc);
248     if (rx_start_desc && tx_start_desc && (rx_prepared_size == n) && (tx_prepared_size == n)) {
249         // register user callback to the last descriptor
250         async_memcpy_stream_t *mcp_stream = __containerof(rx_end_desc, async_memcpy_stream_t, desc);
251         mcp_stream->cb = cb_isr;
252         mcp_stream->cb_args = cb_args;
253         // restart RX firstly
254         dma_descriptor_t *desc = rx_start_desc;
255         while (desc != rx_end_desc) {
256             desc->dw0.owner = DMA_DESCRIPTOR_BUFFER_OWNER_DMA;
257             desc = desc->next;
258         }
259         desc->dw0.owner = DMA_DESCRIPTOR_BUFFER_OWNER_DMA;
260         asmcp->rx_desc = desc->next;
261         // restart TX secondly
262         desc = tx_start_desc;
263         while (desc != tx_end_desc) {
264             desc->dw0.owner = DMA_DESCRIPTOR_BUFFER_OWNER_DMA;
265             desc = desc->next;
266         }
267         desc->dw0.owner = DMA_DESCRIPTOR_BUFFER_OWNER_DMA;
268         asmcp->tx_desc = desc->next;
269         async_memcpy_impl_restart(&asmcp->mcp_impl);
270     }
271     portEXIT_CRITICAL_SAFE(&asmcp->spinlock);
272 
273     // It's unlikely that we have space for rx descriptor but no space for tx descriptor
274     // Both tx and rx descriptor should move in the same pace
275     ESP_GOTO_ON_FALSE(rx_prepared_size == n, ESP_FAIL, err, TAG, "out of rx descriptor");
276     ESP_GOTO_ON_FALSE(tx_prepared_size == n, ESP_FAIL, err, TAG, "out of tx descriptor");
277 
278 err:
279     return ret;
280 }
281 
async_memcpy_isr_on_rx_done_event(async_memcpy_impl_t * impl)282 IRAM_ATTR void async_memcpy_isr_on_rx_done_event(async_memcpy_impl_t *impl)
283 {
284     bool to_continue = false;
285     async_memcpy_stream_t *in_stream = NULL;
286     dma_descriptor_t *next_desc = NULL;
287     async_memcpy_context_t *asmcp = __containerof(impl, async_memcpy_context_t, mcp_impl);
288 
289     // get the RX eof descriptor address
290     dma_descriptor_t *eof = (dma_descriptor_t *)impl->rx_eof_addr;
291     // traversal all unchecked descriptors
292     do {
293         portENTER_CRITICAL_ISR(&asmcp->spinlock);
294         // There is an assumption that the usage of rx descriptors are in the same pace as tx descriptors (this is determined by M2M DMA working mechanism)
295         // And once the rx descriptor is recycled, the corresponding tx desc is guaranteed to be returned by DMA
296         to_continue = async_memcpy_get_next_rx_descriptor(asmcp, eof, &next_desc);
297         portEXIT_CRITICAL_ISR(&asmcp->spinlock);
298         if (next_desc) {
299             in_stream = __containerof(next_desc, async_memcpy_stream_t, desc);
300             // invoke user registered callback if available
301             if (in_stream->cb) {
302                 async_memcpy_event_t e = {0};
303                 if (in_stream->cb(asmcp, &e, in_stream->cb_args)) {
304                     impl->isr_need_yield = true;
305                 }
306                 in_stream->cb = NULL;
307                 in_stream->cb_args = NULL;
308             }
309         }
310     } while (to_continue);
311 }
312