1 /*
2 Abstraction layer for spi-ram. For now, it's no more than a stub for the spiram_psram functions, but if
3 we add more types of external RAM memory, this can be made into a more intelligent dispatcher.
4 */
5 
6 /*
7  * SPDX-FileCopyrightText: 2015-2021 Espressif Systems (Shanghai) CO LTD
8  *
9  * SPDX-License-Identifier: Apache-2.0
10  */
11 
12 #if defined(__ZEPHYR__)
13 #include <zephyr/kernel.h>
14 #include <zephyr/logging/log.h>
15 #include "esp_rom_uart.h"
16 #endif
17 #include <stdint.h>
18 #include <string.h>
19 #include <sys/param.h>
20 #include "sdkconfig.h"
21 #include "esp_attr.h"
22 #include "esp_err.h"
23 #include "esp32s3/spiram.h"
24 #include "spiram_psram.h"
25 #include "esp_log.h"
26 #include "soc/soc.h"
27 #if !defined(__ZEPHYR__)
28 #include "freertos/FreeRTOS.h"
29 #include "freertos/xtensa_api.h"
30 #include "esp_heap_caps_init.h"
31 #endif
32 #include "soc/soc_memory_layout.h"
33 #include "soc/dport_reg.h"
34 #include "esp32s3/rom/cache.h"
35 #include "soc/cache_memory.h"
36 #include "soc/extmem_reg.h"
37 
38 #define PSRAM_MODE PSRAM_VADDR_MODE_NORMAL
39 
40 #if CONFIG_SPIRAM
41 
42 static const char *TAG = "spiram";
43 
44 #if CONFIG_SPIRAM_SPEED_40M
45 #define PSRAM_SPEED PSRAM_CACHE_S40M
46 #else  //#if CONFIG_SPIRAM_SPEED_80M
47 #define PSRAM_SPEED PSRAM_CACHE_S80M
48 #endif
49 
50 static bool s_spiram_inited = false;
51 
52 
53 /*
54  Simple RAM test. Writes a word every 32 bytes. Takes about a second to complete for 4MiB. Returns
55  true when RAM seems OK, false when test fails. WARNING: Do not run this before the 2nd cpu has been
56  initialized (in a two-core system) or after the heap allocator has taken ownership of the memory.
57 */
esp_spiram_test(void)58 bool esp_spiram_test(void)
59 {
60     size_t spiram_size = esp_spiram_get_size();
61     volatile int *spiram = (volatile int *)(SOC_EXTRAM_DATA_HIGH - spiram_size);
62     size_t p;
63     size_t s = spiram_size;
64     int errct = 0;
65     int initial_err = -1;
66 
67     if (SOC_EXTRAM_DATA_SIZE < spiram_size) {
68         ESP_EARLY_LOGW(TAG, "Only test spiram from %08x to %08x\n", SOC_EXTRAM_DATA_LOW, SOC_EXTRAM_DATA_HIGH);
69         spiram = (volatile int *)SOC_EXTRAM_DATA_LOW;
70         s = SOC_EXTRAM_DATA_HIGH - SOC_EXTRAM_DATA_LOW;
71     }
72     for (p = 0; p < (s / sizeof(int)); p += 8) {
73         spiram[p] = p ^ 0xAAAAAAAA;
74     }
75     for (p = 0; p < (s / sizeof(int)); p += 8) {
76         if (spiram[p] != (p ^ 0xAAAAAAAA)) {
77             errct++;
78             if (errct == 1) {
79                 initial_err = p * 4;
80             }
81             if (errct < 4) {
82                 ESP_EARLY_LOGE(TAG, "SPI SRAM error@%08x:%08x/%08x \n", &spiram[p], spiram[p], p ^ 0xAAAAAAAA);
83             }
84         }
85     }
86     if (errct) {
87         ESP_EARLY_LOGE(TAG, "SPI SRAM memory test fail. %d/%d writes failed, first @ %X\n", errct, s / 32, initial_err + SOC_EXTRAM_DATA_LOW);
88         return false;
89     } else {
90         ESP_EARLY_LOGI(TAG, "SPI SRAM memory test OK");
91         return true;
92     }
93 }
94 
esp_spiram_init_cache(void)95 void IRAM_ATTR esp_spiram_init_cache(void)
96 {
97     size_t spiram_size = esp_spiram_get_size();
98     Cache_Suspend_DCache();
99     if ((SOC_EXTRAM_DATA_HIGH - SOC_EXTRAM_DATA_LOW) >= spiram_size) {
100         Cache_Dbus_MMU_Set(MMU_ACCESS_SPIRAM, SOC_EXTRAM_DATA_HIGH - spiram_size, 0, 64, spiram_size >> 16, 0);
101     } else {
102         Cache_Dbus_MMU_Set(MMU_ACCESS_SPIRAM, SOC_EXTRAM_DATA_HIGH - spiram_size, 0, 64, (SOC_EXTRAM_DATA_HIGH - SOC_EXTRAM_DATA_LOW) >> 16, 0);
103     }
104     REG_CLR_BIT(EXTMEM_DCACHE_CTRL1_REG, EXTMEM_DCACHE_SHUT_CORE0_BUS);
105 #if !CONFIG_FREERTOS_UNICORE
106     REG_CLR_BIT(EXTMEM_DCACHE_CTRL1_REG, EXTMEM_DCACHE_SHUT_CORE1_BUS);
107 #endif
108     Cache_Resume_DCache(0);
109 }
110 
111 static uint32_t pages_for_flash = 0;
112 static uint32_t instruction_in_spiram = 0;
113 static uint32_t rodata_in_spiram = 0;
114 
115 #if CONFIG_SPIRAM_FETCH_INSTRUCTIONS
116 static int instr_flash2spiram_offs = 0;
117 static uint32_t instr_start_page = 0;
118 static uint32_t instr_end_page = 0;
119 #endif
120 
121 #if CONFIG_SPIRAM_RODATA
122 static int rodata_flash2spiram_offs = 0;
123 static uint32_t rodata_start_page = 0;
124 static uint32_t rodata_end_page = 0;
125 #endif
126 
127 #if CONFIG_SPIRAM_FETCH_INSTRUCTIONS || CONFIG_SPIRAM_RODATA
128 static uint32_t page0_mapped = 0;
129 static uint32_t page0_page = INVALID_PHY_PAGE;
130 #endif
131 
esp_spiram_instruction_access_enabled(void)132 uint32_t esp_spiram_instruction_access_enabled(void)
133 {
134     return instruction_in_spiram;
135 }
136 
esp_spiram_rodata_access_enabled(void)137 uint32_t esp_spiram_rodata_access_enabled(void)
138 {
139     return rodata_in_spiram;
140 }
141 
142 #if CONFIG_SPIRAM_FETCH_INSTRUCTIONS
esp_spiram_enable_instruction_access(void)143 esp_err_t esp_spiram_enable_instruction_access(void)
144 {
145     size_t spiram_size = esp_spiram_get_size();
146     uint32_t pages_in_flash = 0;
147     pages_in_flash += Cache_Count_Flash_Pages(CACHE_IBUS, &page0_mapped);
148     if ((pages_in_flash + pages_for_flash) > (spiram_size >> 16)) {
149         ESP_EARLY_LOGE(TAG, "SPI RAM space not enough for the instructions, has %d pages, need %d pages.", (spiram_size >> 16), (pages_in_flash + pages_for_flash));
150         return ESP_FAIL;
151     }
152     ESP_EARLY_LOGI(TAG, "Instructions copied and mapped to SPIRAM");
153     uint32_t mmu_value = *(volatile uint32_t *)(DR_REG_MMU_TABLE + CACHE_IROM_MMU_START);
154     instr_flash2spiram_offs = mmu_value - pages_for_flash;
155     ESP_EARLY_LOGV(TAG, "Instructions from flash page%d copy to SPIRAM page%d, Offset: %d", mmu_value, pages_for_flash, instr_flash2spiram_offs);
156     pages_for_flash = Cache_Flash_To_SPIRAM_Copy(CACHE_IBUS, IRAM0_CACHE_ADDRESS_LOW, pages_for_flash, &page0_page);
157     instruction_in_spiram = 1;
158     return ESP_OK;
159 }
160 #endif
161 
162 #if CONFIG_SPIRAM_RODATA
esp_spiram_enable_rodata_access(void)163 esp_err_t esp_spiram_enable_rodata_access(void)
164 {
165     size_t spiram_size = esp_spiram_get_size();
166     uint32_t pages_in_flash = 0;
167     pages_in_flash += Cache_Count_Flash_Pages(CACHE_DBUS, &page0_mapped);
168 
169     if ((pages_in_flash + pages_for_flash) > (spiram_size >> 16)) {
170         ESP_EARLY_LOGE(TAG, "SPI RAM space not enough for the read only data.");
171         return ESP_FAIL;
172     }
173     ESP_EARLY_LOGI(TAG, "Read only data copied and mapped to SPIRAM");
174     uint32_t mmu_value = *(volatile uint32_t *)(DR_REG_MMU_TABLE + CACHE_DROM_MMU_START);
175     rodata_flash2spiram_offs = mmu_value - pages_for_flash;
176     ESP_EARLY_LOGV(TAG, "Rodata from flash page%d copy to SPIRAM page%d, Offset: %d", mmu_value, pages_for_flash, rodata_flash2spiram_offs);
177     pages_for_flash = Cache_Flash_To_SPIRAM_Copy(CACHE_DBUS, DRAM0_CACHE_ADDRESS_LOW, pages_for_flash, &page0_page);
178     rodata_in_spiram = 1;
179     return ESP_OK;
180 }
181 #endif
182 
183 #if CONFIG_SPIRAM_FETCH_INSTRUCTIONS
instruction_flash_page_info_init(void)184 void instruction_flash_page_info_init(void)
185 {
186     uint32_t instr_page_cnt = ((uint32_t)&_instruction_reserved_end - SOC_IROM_LOW + MMU_PAGE_SIZE - 1) / MMU_PAGE_SIZE;
187 
188     instr_start_page = *(volatile uint32_t *)(DR_REG_MMU_TABLE + CACHE_IROM_MMU_START);
189     instr_start_page &= MMU_ADDRESS_MASK;
190     instr_end_page = instr_start_page + instr_page_cnt - 1;
191 }
192 
instruction_flash_start_page_get(void)193 uint32_t IRAM_ATTR instruction_flash_start_page_get(void)
194 {
195     return instr_start_page;
196 }
197 
instruction_flash_end_page_get(void)198 uint32_t IRAM_ATTR instruction_flash_end_page_get(void)
199 {
200     return instr_end_page;
201 }
202 
instruction_flash2spiram_offset(void)203 int IRAM_ATTR instruction_flash2spiram_offset(void)
204 {
205     return instr_flash2spiram_offs;
206 }
207 #endif
208 
209 #if CONFIG_SPIRAM_RODATA
rodata_flash_page_info_init(void)210 void rodata_flash_page_info_init(void)
211 {
212     uint32_t rodata_page_cnt = ((uint32_t)&_rodata_reserved_end - ((uint32_t)&_rodata_reserved_start & ~ (MMU_PAGE_SIZE - 1)) + MMU_PAGE_SIZE - 1) / MMU_PAGE_SIZE;
213 
214     rodata_start_page = *(volatile uint32_t *)(DR_REG_MMU_TABLE + CACHE_DROM_MMU_START);
215     rodata_start_page &= MMU_ADDRESS_MASK;
216     rodata_end_page = rodata_start_page + rodata_page_cnt - 1;
217 }
218 
rodata_flash_start_page_get(void)219 uint32_t IRAM_ATTR rodata_flash_start_page_get(void)
220 {
221     return rodata_start_page;
222 }
223 
rodata_flash_end_page_get(void)224 uint32_t IRAM_ATTR rodata_flash_end_page_get(void)
225 {
226     return rodata_end_page;
227 }
228 
rodata_flash2spiram_offset(void)229 int IRAM_ATTR rodata_flash2spiram_offset(void)
230 {
231     return rodata_flash2spiram_offs;
232 }
233 #endif
234 
esp_spiram_init(void)235 esp_err_t esp_spiram_init(void)
236 {
237     esp_err_t r;
238     r = psram_enable(PSRAM_SPEED, PSRAM_MODE);
239     if (r != ESP_OK) {
240 #if CONFIG_SPIRAM_IGNORE_NOTFOUND
241         ESP_EARLY_LOGE(TAG, "SPI RAM enabled but initialization failed. Bailing out.");
242 #endif
243         return r;
244     }
245     s_spiram_inited = true;
246 #if (CONFIG_SPIRAM_SIZE != -1)
247     if (esp_spiram_get_size() != CONFIG_SPIRAM_SIZE) {
248         ESP_EARLY_LOGE(TAG, "Expected %dKiB chip but found %dKiB chip. Bailing out..", CONFIG_SPIRAM_SIZE / 1024, esp_spiram_get_size() / 1024);
249         return ESP_ERR_INVALID_SIZE;
250     }
251 #endif
252 
253     ESP_EARLY_LOGI(TAG, "Found %dMBit SPI RAM device",
254                    (esp_spiram_get_size() * 8) / (1024 * 1024));
255     ESP_EARLY_LOGI(TAG, "SPI RAM mode: %s", PSRAM_SPEED == PSRAM_CACHE_S40M ? "sram 40m" : "sram 80m");
256     ESP_EARLY_LOGI(TAG, "PSRAM initialized, cache is in %s mode.", \
257                    (PSRAM_MODE == PSRAM_VADDR_MODE_EVENODD) ? "even/odd (2-core)" : \
258                    (PSRAM_MODE == PSRAM_VADDR_MODE_LOWHIGH) ? "low/high (2-core)" : \
259                    (PSRAM_MODE == PSRAM_VADDR_MODE_NORMAL) ? "normal (1-core)" : "ERROR");
260     return ESP_OK;
261 }
262 
263 #if !defined(__ZEPHYR__)
esp_spiram_add_to_heapalloc(void)264 esp_err_t esp_spiram_add_to_heapalloc(void)
265 {
266     size_t spiram_size = esp_spiram_get_size();
267     uint32_t size_for_flash = (pages_for_flash << 16);
268     ESP_EARLY_LOGI(TAG, "Adding pool of %dK of external SPI memory to heap allocator", (spiram_size - (pages_for_flash << 16)) / 1024);
269     //Add entire external RAM region to heap allocator. Heap allocator knows the capabilities of this type of memory, so there's
270     //no need to explicitly specify them.
271 
272     return heap_caps_add_region((intptr_t)SOC_EXTRAM_DATA_HIGH - spiram_size + size_for_flash, (intptr_t)SOC_EXTRAM_DATA_HIGH - 1);
273 }
274 
275 
276 static uint8_t *dma_heap;
277 
esp_spiram_reserve_dma_pool(size_t size)278 esp_err_t esp_spiram_reserve_dma_pool(size_t size)
279 {
280     if (size == 0) {
281         return ESP_OK;    //no-op
282     }
283     ESP_EARLY_LOGI(TAG, "Reserving pool of %dK of internal memory for DMA/internal allocations", size / 1024);
284     dma_heap = heap_caps_malloc(size, MALLOC_CAP_DMA | MALLOC_CAP_INTERNAL);
285     if (!dma_heap) {
286         return ESP_ERR_NO_MEM;
287     }
288     uint32_t caps[] = {MALLOC_CAP_DMA | MALLOC_CAP_INTERNAL, 0, MALLOC_CAP_8BIT | MALLOC_CAP_32BIT};
289     return heap_caps_add_region_with_caps(caps, (intptr_t) dma_heap, (intptr_t) dma_heap + size - 1);
290 }
291 
292 #endif /*__ZEPHYR__*/
293 
esp_spiram_get_size(void)294 size_t esp_spiram_get_size(void)
295 {
296     if (!s_spiram_inited) {
297         ESP_EARLY_LOGE(TAG, "SPI RAM not initialized");
298         abort();
299     }
300 
301     psram_size_t size = psram_get_size();
302     if (size == PSRAM_SIZE_16MBITS) {
303         return 2 * 1024 * 1024;
304     }
305     if (size == PSRAM_SIZE_32MBITS) {
306         return 4 * 1024 * 1024;
307     }
308     if (size == PSRAM_SIZE_64MBITS) {
309         return 8 * 1024 * 1024;
310     }
311     if (size == PSRAM_SIZE_128MBITS) {
312         return 16 * 1024 * 1024;
313     }
314     if (size == PSRAM_SIZE_256MBITS) {
315         return 32 * 1024 * 1024;
316     }
317     return CONFIG_SPIRAM_SIZE;
318 }
319 
320 /*
321  Before flushing the cache, if psram is enabled as a memory-mapped thing, we need to write back the data in the cache to the psram first,
322  otherwise it will get lost. For now, we just read 64/128K of random PSRAM memory to do this.
323 */
esp_spiram_writeback_cache(void)324 void IRAM_ATTR esp_spiram_writeback_cache(void)
325 {
326     extern void Cache_WriteBack_All(void);
327     Cache_WriteBack_All();
328 }
329 
330 /**
331  * @brief If SPI RAM(PSRAM) has been initialized
332  *
333  * @return true SPI RAM has been initialized successfully
334  * @return false SPI RAM hasn't been initialized or initialized failed
335  */
esp_spiram_is_initialized(void)336 bool esp_spiram_is_initialized(void)
337 {
338     return s_spiram_inited;
339 }
340 
esp_spiram_get_cs_io(void)341 uint8_t esp_spiram_get_cs_io(void)
342 {
343     return psram_get_cs_io();
344 }
345 
346 #endif
347