1 /*
2 Abstraction layer for spi-ram. For now, it's no more than a stub for the spiram_psram functions, but if
3 we add more types of external RAM memory, this can be made into a more intelligent dispatcher.
4 */
5 
6 // Copyright 2015-2017 Espressif Systems (Shanghai) PTE LTD
7 //
8 // Licensed under the Apache License, Version 2.0 (the "License");
9 // you may not use this file except in compliance with the License.
10 // You may obtain a copy of the License at
11 //
12 //     http://www.apache.org/licenses/LICENSE-2.0
13 //
14 // Unless required by applicable law or agreed to in writing, software
15 // distributed under the License is distributed on an "AS IS" BASIS,
16 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 // See the License for the specific language governing permissions and
18 // limitations under the License.
19 
20 #include <stdint.h>
21 #include <string.h>
22 #include <sys/param.h>
23 #include "sdkconfig.h"
24 #include "esp_attr.h"
25 #include "esp_err.h"
26 #include "esp32s3/spiram.h"
27 #include "spiram_psram.h"
28 #include "esp_log.h"
29 #include "freertos/FreeRTOS.h"
30 #include "freertos/xtensa_api.h"
31 #include "soc/soc.h"
32 #include "esp_heap_caps_init.h"
33 #include "soc/soc_memory_layout.h"
34 #include "soc/dport_reg.h"
35 #include "esp32s3/rom/cache.h"
36 #include "soc/cache_memory.h"
37 #include "soc/extmem_reg.h"
38 
39 #define PSRAM_MODE PSRAM_VADDR_MODE_NORMAL
40 
41 #if CONFIG_SPIRAM
42 
43 static const char *TAG = "spiram";
44 
45 #if CONFIG_SPIRAM_SPEED_40M
46 #define PSRAM_SPEED PSRAM_CACHE_S40M
47 #elif CONFIG_SPIRAM_SPEED_80M
48 #define PSRAM_SPEED PSRAM_CACHE_S80M
49 #else
50 #define PSRAM_SPEED PSRAM_CACHE_S20M
51 #endif
52 
53 static bool spiram_inited = false;
54 
55 
56 /*
57  Simple RAM test. Writes a word every 32 bytes. Takes about a second to complete for 4MiB. Returns
58  true when RAM seems OK, false when test fails. WARNING: Do not run this before the 2nd cpu has been
59  initialized (in a two-core system) or after the heap allocator has taken ownership of the memory.
60 */
esp_spiram_test(void)61 bool esp_spiram_test(void)
62 {
63     size_t spiram_size = esp_spiram_get_size();
64     volatile int *spiram = (volatile int *)(SOC_EXTRAM_DATA_HIGH - spiram_size);
65     size_t p;
66     size_t s = spiram_size;
67     int errct = 0;
68     int initial_err = -1;
69 
70     if (SOC_EXTRAM_DATA_SIZE < spiram_size) {
71         ESP_EARLY_LOGW(TAG, "Only test spiram from %08x to %08x\n", SOC_EXTRAM_DATA_LOW, SOC_EXTRAM_DATA_HIGH);
72         spiram = (volatile int *)SOC_EXTRAM_DATA_LOW;
73         s = SOC_EXTRAM_DATA_HIGH - SOC_EXTRAM_DATA_LOW;
74     }
75     for (p = 0; p < (s / sizeof(int)); p += 8) {
76         spiram[p] = p ^ 0xAAAAAAAA;
77     }
78     for (p = 0; p < (s / sizeof(int)); p += 8) {
79         if (spiram[p] != (p ^ 0xAAAAAAAA)) {
80             errct++;
81             if (errct == 1) {
82                 initial_err = p * 4;
83             }
84             if (errct < 4) {
85                 ESP_EARLY_LOGE(TAG, "SPI SRAM error@%08x:%08x/%08x \n", &spiram[p], spiram[p], p ^ 0xAAAAAAAA);
86             }
87         }
88     }
89     if (errct) {
90         ESP_EARLY_LOGE(TAG, "SPI SRAM memory test fail. %d/%d writes failed, first @ %X\n", errct, s / 32, initial_err + SOC_EXTRAM_DATA_LOW);
91         return false;
92     } else {
93         ESP_EARLY_LOGI(TAG, "SPI SRAM memory test OK");
94         return true;
95     }
96 }
97 
esp_spiram_init_cache(void)98 void IRAM_ATTR esp_spiram_init_cache(void)
99 {
100     size_t spiram_size = esp_spiram_get_size();
101     Cache_Suspend_DCache();
102     if ((SOC_EXTRAM_DATA_HIGH - SOC_EXTRAM_DATA_LOW) >= spiram_size) {
103         Cache_Dbus_MMU_Set(MMU_ACCESS_SPIRAM, SOC_EXTRAM_DATA_HIGH - spiram_size, 0, 64, spiram_size >> 16, 0);
104     } else {
105         Cache_Dbus_MMU_Set(MMU_ACCESS_SPIRAM, SOC_EXTRAM_DATA_HIGH - spiram_size, 0, 64, (SOC_EXTRAM_DATA_HIGH - SOC_EXTRAM_DATA_LOW) >> 16, 0);
106     }
107     REG_CLR_BIT(EXTMEM_DCACHE_CTRL1_REG, EXTMEM_DCACHE_SHUT_CORE0_BUS);
108 #if !CONFIG_FREERTOS_UNICORE
109     REG_CLR_BIT(EXTMEM_DCACHE_CTRL1_REG, EXTMEM_DCACHE_SHUT_CORE1_BUS);
110 #endif
111     Cache_Resume_DCache(0);
112 }
113 
114 static uint32_t pages_for_flash = 0;
115 static uint32_t instrcution_in_spiram = 0;
116 static uint32_t rodata_in_spiram = 0;
117 
118 #if CONFIG_SPIRAM_FETCH_INSTRUCTIONS
119 static int instr_flash2spiram_offs = 0;
120 static uint32_t instr_start_page = 0;
121 static uint32_t instr_end_page = 0;
122 #endif
123 
124 #if CONFIG_SPIRAM_RODATA
125 static int rodata_flash2spiram_offs = 0;
126 static uint32_t rodata_start_page = 0;
127 static uint32_t rodata_end_page = 0;
128 #endif
129 
130 #if CONFIG_SPIRAM_FETCH_INSTRUCTIONS || CONFIG_SPIRAM_RODATA
131 static uint32_t page0_mapped = 0;
132 static uint32_t page0_page = INVALID_PHY_PAGE;
133 #endif
134 
esp_spiram_instruction_access_enabled(void)135 uint32_t esp_spiram_instruction_access_enabled(void)
136 {
137     return instrcution_in_spiram;
138 }
139 
esp_spiram_rodata_access_enabled(void)140 uint32_t esp_spiram_rodata_access_enabled(void)
141 {
142     return rodata_in_spiram;
143 }
144 
145 #if CONFIG_SPIRAM_FETCH_INSTRUCTIONS
esp_spiram_enable_instruction_access(void)146 esp_err_t esp_spiram_enable_instruction_access(void)
147 {
148     size_t spiram_size = esp_spiram_get_size();
149     uint32_t pages_in_flash = 0;
150     pages_in_flash += Cache_Count_Flash_Pages(CACHE_IBUS, &page0_mapped);
151     if ((pages_in_flash + pages_for_flash) > (spiram_size >> 16)) {
152         ESP_EARLY_LOGE(TAG, "SPI RAM space not enough for the instructions, has %d pages, need %d pages.", (spiram_size >> 16), (pages_in_flash + pages_for_flash));
153         return ESP_FAIL;
154     }
155     ESP_EARLY_LOGI(TAG, "Instructions copied and mapped to SPIRAM");
156     uint32_t mmu_value = *(volatile uint32_t *)(DR_REG_MMU_TABLE + CACHE_IROM_MMU_START);
157     instr_flash2spiram_offs = mmu_value - pages_for_flash;
158     ESP_EARLY_LOGV(TAG, "Instructions from flash page%d copy to SPIRAM page%d, Offset: %d", mmu_value, pages_for_flash, instr_flash2spiram_offs);
159     pages_for_flash = Cache_Flash_To_SPIRAM_Copy(CACHE_IBUS, IRAM0_CACHE_ADDRESS_LOW, pages_for_flash, &page0_page);
160     instrcution_in_spiram = 1;
161     return ESP_OK;
162 }
163 #endif
164 
165 #if CONFIG_SPIRAM_RODATA
esp_spiram_enable_rodata_access(void)166 esp_err_t esp_spiram_enable_rodata_access(void)
167 {
168     size_t spiram_size = esp_spiram_get_size();
169     uint32_t pages_in_flash = 0;
170     pages_in_flash += Cache_Count_Flash_Pages(CACHE_DBUS, &page0_mapped);
171 
172     if ((pages_in_flash + pages_for_flash) > (spiram_size >> 16)) {
173         ESP_EARLY_LOGE(TAG, "SPI RAM space not enough for the read only data.");
174         return ESP_FAIL;
175     }
176     ESP_EARLY_LOGI(TAG, "Read only data copied and mapped to SPIRAM");
177     uint32_t mmu_value = *(volatile uint32_t *)(DR_REG_MMU_TABLE + CACHE_DROM_MMU_START);
178     rodata_flash2spiram_offs = mmu_value - pages_for_flash;
179     ESP_EARLY_LOGV(TAG, "Rodata from flash page%d copy to SPIRAM page%d, Offset: %d", mmu_value, pages_for_flash, rodata_flash2spiram_offs);
180     pages_for_flash = Cache_Flash_To_SPIRAM_Copy(CACHE_DBUS, DRAM0_CACHE_ADDRESS_LOW, pages_for_flash, &page0_page);
181     rodata_in_spiram = 1;
182     return ESP_OK;
183 }
184 #endif
185 
186 #if CONFIG_SPIRAM_FETCH_INSTRUCTIONS
instruction_flash_page_info_init(void)187 void instruction_flash_page_info_init(void)
188 {
189     uint32_t instr_page_cnt = ((uint32_t)&_instruction_reserved_end - SOC_IROM_LOW + MMU_PAGE_SIZE - 1) / MMU_PAGE_SIZE;
190 
191     instr_start_page = *(volatile uint32_t *)(DR_REG_MMU_TABLE + CACHE_IROM_MMU_START);
192     instr_start_page &= MMU_ADDRESS_MASK;
193     instr_end_page = instr_start_page + instr_page_cnt - 1;
194 }
195 
instruction_flash_start_page_get(void)196 uint32_t IRAM_ATTR instruction_flash_start_page_get(void)
197 {
198     return instr_start_page;
199 }
200 
instruction_flash_end_page_get(void)201 uint32_t IRAM_ATTR instruction_flash_end_page_get(void)
202 {
203     return instr_end_page;
204 }
205 
instruction_flash2spiram_offset(void)206 int IRAM_ATTR instruction_flash2spiram_offset(void)
207 {
208     return instr_flash2spiram_offs;
209 }
210 #endif
211 
212 #if CONFIG_SPIRAM_RODATA
rodata_flash_page_info_init(void)213 void rodata_flash_page_info_init(void)
214 {
215     uint32_t rodata_page_cnt = ((uint32_t)&_rodata_reserved_end - ((uint32_t)&_rodata_reserved_start & ~ (MMU_PAGE_SIZE - 1)) + MMU_PAGE_SIZE - 1) / MMU_PAGE_SIZE;
216 
217     rodata_start_page = *(volatile uint32_t *)(DR_REG_MMU_TABLE + CACHE_DROM_MMU_START);
218     rodata_start_page &= MMU_ADDRESS_MASK;
219     rodata_end_page = rodata_start_page + rodata_page_cnt - 1;
220 }
221 
rodata_flash_start_page_get(void)222 uint32_t IRAM_ATTR rodata_flash_start_page_get(void)
223 {
224     return rodata_start_page;
225 }
226 
rodata_flash_end_page_get(void)227 uint32_t IRAM_ATTR rodata_flash_end_page_get(void)
228 {
229     return rodata_end_page;
230 }
231 
rodata_flash2spiram_offset(void)232 int IRAM_ATTR rodata_flash2spiram_offset(void)
233 {
234     return rodata_flash2spiram_offs;
235 }
236 #endif
237 
esp_spiram_init(void)238 esp_err_t esp_spiram_init(void)
239 {
240     esp_err_t r;
241     r = psram_enable(PSRAM_SPEED, PSRAM_MODE);
242     if (r != ESP_OK) {
243 #if CONFIG_SPIRAM_IGNORE_NOTFOUND
244         ESP_EARLY_LOGE(TAG, "SPI RAM enabled but initialization failed. Bailing out.");
245 #endif
246         return r;
247     }
248 
249     spiram_inited = true;
250 #if (CONFIG_SPIRAM_SIZE != -1)
251     if (esp_spiram_get_size() != CONFIG_SPIRAM_SIZE) {
252         ESP_EARLY_LOGE(TAG, "Expected %dKiB chip but found %dKiB chip. Bailing out..", CONFIG_SPIRAM_SIZE / 1024, esp_spiram_get_size() / 1024);
253         return ESP_ERR_INVALID_SIZE;
254     }
255 #endif
256 
257     ESP_EARLY_LOGI(TAG, "Found %dMBit SPI RAM device",
258                    (esp_spiram_get_size() * 8) / (1024 * 1024));
259     ESP_EARLY_LOGI(TAG, "SPI RAM mode: %s", PSRAM_SPEED == PSRAM_CACHE_S40M ? "sram 40m" : \
260                    PSRAM_SPEED == PSRAM_CACHE_S80M ? "sram 80m" : "sram 20m");
261     ESP_EARLY_LOGI(TAG, "PSRAM initialized, cache is in %s mode.", \
262                    (PSRAM_MODE == PSRAM_VADDR_MODE_EVENODD) ? "even/odd (2-core)" : \
263                    (PSRAM_MODE == PSRAM_VADDR_MODE_LOWHIGH) ? "low/high (2-core)" : \
264                    (PSRAM_MODE == PSRAM_VADDR_MODE_NORMAL) ? "normal (1-core)" : "ERROR");
265     return ESP_OK;
266 }
267 
268 
esp_spiram_add_to_heapalloc(void)269 esp_err_t esp_spiram_add_to_heapalloc(void)
270 {
271     size_t spiram_size = esp_spiram_get_size();
272     uint32_t size_for_flash = (pages_for_flash << 16);
273     ESP_EARLY_LOGI(TAG, "Adding pool of %dK of external SPI memory to heap allocator", (spiram_size - (pages_for_flash << 16)) / 1024);
274     //Add entire external RAM region to heap allocator. Heap allocator knows the capabilities of this type of memory, so there's
275     //no need to explicitly specify them.
276 
277     return heap_caps_add_region((intptr_t)SOC_EXTRAM_DATA_HIGH - spiram_size + size_for_flash, (intptr_t)SOC_EXTRAM_DATA_HIGH - 1);
278 }
279 
280 
281 static uint8_t *dma_heap;
282 
esp_spiram_reserve_dma_pool(size_t size)283 esp_err_t esp_spiram_reserve_dma_pool(size_t size)
284 {
285     if (size == 0) {
286         return ESP_OK;    //no-op
287     }
288     ESP_EARLY_LOGI(TAG, "Reserving pool of %dK of internal memory for DMA/internal allocations", size / 1024);
289     dma_heap = heap_caps_malloc(size, MALLOC_CAP_DMA | MALLOC_CAP_INTERNAL);
290     if (!dma_heap) {
291         return ESP_ERR_NO_MEM;
292     }
293     uint32_t caps[] = {MALLOC_CAP_DMA | MALLOC_CAP_INTERNAL, 0, MALLOC_CAP_8BIT | MALLOC_CAP_32BIT};
294     return heap_caps_add_region_with_caps(caps, (intptr_t) dma_heap, (intptr_t) dma_heap + size - 1);
295 }
296 
esp_spiram_get_size(void)297 size_t esp_spiram_get_size(void)
298 {
299     if (!spiram_inited) {
300         ESP_EARLY_LOGE(TAG, "SPI RAM not initialized");
301         abort();
302     }
303 
304     psram_size_t size = psram_get_size();
305     if (size == PSRAM_SIZE_16MBITS) {
306         return 2 * 1024 * 1024;
307     }
308     if (size == PSRAM_SIZE_32MBITS) {
309         return 4 * 1024 * 1024;
310     }
311     if (size == PSRAM_SIZE_64MBITS) {
312         return 8 * 1024 * 1024;
313     }
314     return CONFIG_SPIRAM_SIZE;
315 }
316 
317 /*
318  Before flushing the cache, if psram is enabled as a memory-mapped thing, we need to write back the data in the cache to the psram first,
319  otherwise it will get lost. For now, we just read 64/128K of random PSRAM memory to do this.
320 */
esp_spiram_writeback_cache(void)321 void IRAM_ATTR esp_spiram_writeback_cache(void)
322 {
323     extern void Cache_WriteBack_All(void);
324     Cache_WriteBack_All();
325 }
326 
327 #endif
328