1 /*
2 * SPDX-FileCopyrightText: 2019-2021 Espressif Systems (Shanghai) CO LTD
3 *
4 * SPDX-License-Identifier: Apache-2.0
5 */
6
7 #include <stdlib.h>
8 #include <assert.h>
9 #include <string.h>
10 #include <stdio.h>
11 #include <sys/param.h> // For MIN/MAX(a, b)
12
13 #if !defined(__ZEPHYR__)
14 #include <freertos/FreeRTOS.h>
15 #include <freertos/task.h>
16 #include <freertos/semphr.h>
17 #endif
18
19 #include <soc/soc.h>
20 #include <soc/soc_memory_layout.h>
21 #include "sdkconfig.h"
22 #include "esp_attr.h"
23 #include "esp_spi_flash.h"
24 #include "esp_log.h"
25 #include "esp_private/system_internal.h"
26 #include "esp_private/spi_flash_os.h"
27 #if CONFIG_IDF_TARGET_ESP32
28 #include "esp32/rom/cache.h"
29 #include "esp32/rom/spi_flash.h"
30 #include "esp32/clk.h"
31 #elif CONFIG_IDF_TARGET_ESP32S2
32 #include "esp32s2/rom/cache.h"
33 #include "esp32s2/rom/spi_flash.h"
34 #include "esp32s2/clk.h"
35 #elif CONFIG_IDF_TARGET_ESP32S3
36 #include "soc/spi_mem_reg.h"
37 #include "esp32s3/rom/spi_flash.h"
38 #include "esp32s3/rom/opi_flash.h"
39 #include "esp32s3/rom/cache.h"
40 #include "esp32s3/clk.h"
41 #include "esp32s3/clk.h"
42 #include "esp32s3/opi_flash_private.h"
43 #elif CONFIG_IDF_TARGET_ESP32C3
44 #include "esp32c3/rom/cache.h"
45 #include "esp32c3/rom/spi_flash.h"
46 #include "esp32c3/clk.h"
47 #elif CONFIG_IDF_TARGET_ESP32H2
48 #include "esp32h2/rom/cache.h"
49 #include "esp32h2/rom/spi_flash.h"
50 #include "esp32h2/clk.h"
51 #endif
52 #include "esp_flash_partitions.h"
53 #include "cache_utils.h"
54 #include "esp_flash.h"
55 #include "esp_attr.h"
56 #include "bootloader_flash.h"
57 #include "esp_compiler.h"
58
59 #if defined(__ZEPHYR__)
60 #include "host_flash/cache_utils.h"
61 #endif
62
63 esp_rom_spiflash_result_t spi_flash_write_encrypted_chip(size_t dest_addr, const void *src, size_t size);
64
65 /* bytes erased by SPIEraseBlock() ROM function */
66 #define BLOCK_ERASE_SIZE 65536
67
68 /* Limit number of bytes written/read in a single SPI operation,
69 as these operations disable all higher priority tasks from running.
70 */
71 #ifdef CONFIG_SPI_FLASH_WRITE_CHUNK_SIZE
72 #define MAX_WRITE_CHUNK CONFIG_SPI_FLASH_WRITE_CHUNK_SIZE
73 #else
74 #define MAX_WRITE_CHUNK 8192
75 #endif // CONFIG_SPI_FLASH_WRITE_CHUNK_SIZE
76
77 #define MAX_READ_CHUNK 16384
78
79 static const char *TAG __attribute__((unused)) = "spi_flash";
80
81 #if CONFIG_SPI_FLASH_ENABLE_COUNTERS
82 static spi_flash_counters_t s_flash_stats;
83
84 #define COUNTER_START() uint32_t ts_begin = cpu_hal_get_cycle_count()
85 #define COUNTER_STOP(counter) \
86 do{ \
87 s_flash_stats.counter.count++; \
88 s_flash_stats.counter.time += (cpu_hal_get_cycle_count() - ts_begin) / (esp_clk_cpu_freq() / 1000000); \
89 } while(0)
90
91 #define COUNTER_ADD_BYTES(counter, size) \
92 do { \
93 s_flash_stats.counter.bytes += size; \
94 } while (0)
95
96 #else
97 #define COUNTER_START()
98 #define COUNTER_STOP(counter)
99 #define COUNTER_ADD_BYTES(counter, size)
100
101 #endif //CONFIG_SPI_FLASH_ENABLE_COUNTERS
102
103 #if CONFIG_SPI_FLASH_USE_LEGACY_IMPL
104 static esp_err_t spi_flash_translate_rc(esp_rom_spiflash_result_t rc);
105 #endif //CONFIG_SPI_FLASH_USE_LEGACY_IMPL
106 static bool is_safe_write_address(size_t addr, size_t size);
107 #if !defined(__ZEPHYR__)
108 static void spi_flash_os_yield(void);
109 #endif
110
111 const DRAM_ATTR spi_flash_guard_funcs_t g_flash_guard_default_ops = {
112 #if defined(__ZEPHYR__)
113 .start = spi_flash_disable_interrupts_caches_and_other_cpu,
114 .end = spi_flash_enable_interrupts_caches_and_other_cpu,
115 .op_lock = 0,
116 .op_unlock = 0,
117 #if !CONFIG_SPI_FLASH_DANGEROUS_WRITE_ALLOWED
118 .is_safe_write_address = 0
119 #endif
120 #else
121 .start = spi_flash_disable_interrupts_caches_and_other_cpu,
122 .end = spi_flash_enable_interrupts_caches_and_other_cpu,
123 .op_lock = spi_flash_op_lock,
124 .op_unlock = spi_flash_op_unlock,
125 #if !CONFIG_SPI_FLASH_DANGEROUS_WRITE_ALLOWED
126 .is_safe_write_address = is_safe_write_address,
127 #endif
128 .yield = spi_flash_os_yield,
129 #endif // __ZEPHYR__
130 };
131
132 const DRAM_ATTR spi_flash_guard_funcs_t g_flash_guard_no_os_ops = {
133 .start = spi_flash_disable_interrupts_caches_and_other_cpu_no_os,
134 .end = spi_flash_enable_interrupts_caches_no_os,
135 .op_lock = NULL,
136 .op_unlock = NULL,
137 #if !CONFIG_SPI_FLASH_DANGEROUS_WRITE_ALLOWED
138 .is_safe_write_address = NULL,
139 #endif
140 .yield = NULL,
141 };
142
143 #ifdef CONFIG_SPI_FLASH_DANGEROUS_WRITE_ABORTS
144 #define UNSAFE_WRITE_ADDRESS abort()
145 #else
146 #define UNSAFE_WRITE_ADDRESS return false
147 #endif
148
149
150 /* CHECK_WRITE_ADDRESS macro to fail writes which land in the
151 bootloader, partition table, or running application region.
152 */
153 #if CONFIG_SPI_FLASH_DANGEROUS_WRITE_ALLOWED
154 #define CHECK_WRITE_ADDRESS(ADDR, SIZE)
155 #else /* FAILS or ABORTS */
156 #define CHECK_WRITE_ADDRESS(ADDR, SIZE) do { \
157 if (guard && guard->is_safe_write_address && !guard->is_safe_write_address(ADDR, SIZE)) { \
158 return ESP_ERR_INVALID_ARG; \
159 } \
160 } while(0)
161 #endif // CONFIG_SPI_FLASH_DANGEROUS_WRITE_ALLOWED
162
is_safe_write_address(size_t addr,size_t size)163 static __attribute__((unused)) bool is_safe_write_address(size_t addr, size_t size)
164 {
165 if (!esp_partition_main_flash_region_safe(addr, size)) {
166 UNSAFE_WRITE_ADDRESS;
167 }
168 return true;
169 }
170
171 #if CONFIG_SPI_FLASH_ROM_IMPL
172 #include "esp_heap_caps.h"
173 typedef void *(*malloc_internal_cb_t)(size_t size);
174
spi_flash_malloc_internal(size_t size)175 void IRAM_ATTR *spi_flash_malloc_internal(size_t size)
176 {
177 return heap_caps_malloc(size, MALLOC_CAP_8BIT|MALLOC_CAP_INTERNAL);
178 }
179 #endif
180
esp_mspi_pin_init(void)181 void IRAM_ATTR esp_mspi_pin_init(void)
182 {
183 #if CONFIG_ESPTOOLPY_OCT_FLASH || CONFIG_SPIRAM_MODE_OCT
184 esp_rom_opiflash_pin_config();
185 extern void spi_timing_set_pin_drive_strength(void);
186 spi_timing_set_pin_drive_strength();
187 #else
188 //Set F4R4 board pin drive strength. TODO: IDF-3663
189 #endif
190 }
191
spi_flash_init_chip_state(void)192 esp_err_t IRAM_ATTR spi_flash_init_chip_state(void)
193 {
194 #if CONFIG_ESPTOOLPY_OCT_FLASH
195 return esp_opiflash_init(rom_spiflash_legacy_data->chip.device_id);
196 #else
197 //currently we don't need other setup for initialising Quad Flash
198 return ESP_OK;
199 #endif
200 }
201
spi_flash_init(void)202 void spi_flash_init(void)
203 {
204 spi_flash_init_lock();
205 #if CONFIG_SPI_FLASH_ENABLE_COUNTERS
206 spi_flash_reset_counters();
207 #endif
208
209 #if CONFIG_SPI_FLASH_ROM_IMPL
210 spi_flash_guard_set(&g_flash_guard_default_ops);
211
212 /* These two functions are in ROM only */
213 extern void spi_flash_mmap_os_func_set(void *(*func1)(size_t size), void (*func2)(void *p));
214 spi_flash_mmap_os_func_set(spi_flash_malloc_internal, heap_caps_free);
215
216 extern esp_err_t spi_flash_mmap_page_num_init(uint32_t page_num);
217 spi_flash_mmap_page_num_init(128);
218 #endif
219 }
220
221 #if !CONFIG_SPI_FLASH_ROM_IMPL
222
223 static const DRAM_ATTR spi_flash_guard_funcs_t *s_flash_guard_ops;
224
spi_flash_guard_set(const spi_flash_guard_funcs_t * funcs)225 void IRAM_ATTR spi_flash_guard_set(const spi_flash_guard_funcs_t *funcs)
226 {
227 s_flash_guard_ops = funcs;
228 }
229
spi_flash_guard_get(void)230 const spi_flash_guard_funcs_t *IRAM_ATTR spi_flash_guard_get(void)
231 {
232 return s_flash_guard_ops;
233 }
234
235 #endif
236
spi_flash_get_chip_size(void)237 size_t IRAM_ATTR spi_flash_get_chip_size(void)
238 {
239 return g_rom_flashchip.chip_size;
240 }
241
spi_flash_guard_start(void)242 static inline void IRAM_ATTR spi_flash_guard_start(void)
243 {
244 const spi_flash_guard_funcs_t *guard = spi_flash_guard_get();
245 if (guard && guard->start) {
246 guard->start();
247 }
248 }
249
spi_flash_guard_end(void)250 static inline void IRAM_ATTR spi_flash_guard_end(void)
251 {
252 const spi_flash_guard_funcs_t *guard = spi_flash_guard_get();
253 if (guard && guard->end) {
254 guard->end();
255 }
256 }
257
spi_flash_guard_op_lock(void)258 static inline void IRAM_ATTR spi_flash_guard_op_lock(void)
259 {
260 const spi_flash_guard_funcs_t *guard = spi_flash_guard_get();
261 if (guard && guard->op_lock) {
262 guard->op_lock();
263 }
264 }
265
spi_flash_guard_op_unlock(void)266 static inline void IRAM_ATTR spi_flash_guard_op_unlock(void)
267 {
268 const spi_flash_guard_funcs_t *guard = spi_flash_guard_get();
269 if (guard && guard->op_unlock) {
270 guard->op_unlock();
271 }
272 }
273
274 #if !defined(__ZEPHYR__)
spi_flash_os_yield(void)275 static void IRAM_ATTR spi_flash_os_yield(void)
276 {
277 #ifdef CONFIG_SPI_FLASH_YIELD_DURING_ERASE
278 if (likely(xTaskGetSchedulerState() == taskSCHEDULER_RUNNING)) {
279 vTaskDelay(CONFIG_SPI_FLASH_ERASE_YIELD_TICKS);
280 }
281 #endif
282 }
283 #endif
284
285 #ifdef CONFIG_SPI_FLASH_USE_LEGACY_IMPL
spi_flash_unlock(void)286 static esp_rom_spiflash_result_t IRAM_ATTR spi_flash_unlock(void)
287 {
288 static bool unlocked = false;
289 if (!unlocked) {
290 spi_flash_guard_start();
291 esp_rom_spiflash_result_t rc = esp_rom_spiflash_unlock();
292 spi_flash_guard_end();
293 if (rc != ESP_ROM_SPIFLASH_RESULT_OK) {
294 return rc;
295 }
296 unlocked = true;
297 }
298 return ESP_ROM_SPIFLASH_RESULT_OK;
299 }
300 #endif // CONFIG_SPI_FLASH_USE_LEGACY_IMPL
301
spi_flash_erase_sector(size_t sec)302 esp_err_t IRAM_ATTR spi_flash_erase_sector(size_t sec)
303 {
304 const spi_flash_guard_funcs_t *guard = spi_flash_guard_get();
305 CHECK_WRITE_ADDRESS(sec * SPI_FLASH_SEC_SIZE, SPI_FLASH_SEC_SIZE);
306 return spi_flash_erase_range(sec * SPI_FLASH_SEC_SIZE, SPI_FLASH_SEC_SIZE);
307 }
308
309 #ifdef CONFIG_SPI_FLASH_USE_LEGACY_IMPL
310 //deprecated, only used in compatible mode
spi_flash_erase_range(size_t start_addr,size_t size)311 esp_err_t IRAM_ATTR spi_flash_erase_range(size_t start_addr, size_t size)
312 {
313 const spi_flash_guard_funcs_t *guard = spi_flash_guard_get();
314 CHECK_WRITE_ADDRESS(start_addr, size);
315 if (start_addr % SPI_FLASH_SEC_SIZE != 0) {
316 return ESP_ERR_INVALID_ARG;
317 }
318 if (size % SPI_FLASH_SEC_SIZE != 0) {
319 return ESP_ERR_INVALID_SIZE;
320 }
321 if (size + start_addr > spi_flash_get_chip_size()) {
322 return ESP_ERR_INVALID_SIZE;
323 }
324 size_t start = start_addr / SPI_FLASH_SEC_SIZE;
325 size_t end = start + size / SPI_FLASH_SEC_SIZE;
326 const size_t sectors_per_block = BLOCK_ERASE_SIZE / SPI_FLASH_SEC_SIZE;
327 COUNTER_START();
328 esp_rom_spiflash_result_t rc;
329 rc = spi_flash_unlock();
330 if (rc == ESP_ROM_SPIFLASH_RESULT_OK) {
331 #ifdef CONFIG_SPI_FLASH_YIELD_DURING_ERASE
332 int64_t no_yield_time_us = 0;
333 #endif
334 for (size_t sector = start; sector != end && rc == ESP_ROM_SPIFLASH_RESULT_OK; ) {
335 #ifdef CONFIG_SPI_FLASH_YIELD_DURING_ERASE
336 int64_t start_time_us = esp_system_get_time();
337 #endif
338 spi_flash_guard_start();
339 #ifndef CONFIG_SPI_FLASH_BYPASS_BLOCK_ERASE
340 if (sector % sectors_per_block == 0 && end - sector >= sectors_per_block) {
341 rc = esp_rom_spiflash_erase_block(sector / sectors_per_block);
342 sector += sectors_per_block;
343 COUNTER_ADD_BYTES(erase, sectors_per_block * SPI_FLASH_SEC_SIZE);
344 } else
345 #endif
346 {
347 rc = esp_rom_spiflash_erase_sector(sector);
348 ++sector;
349 COUNTER_ADD_BYTES(erase, SPI_FLASH_SEC_SIZE);
350 }
351 spi_flash_guard_end();
352 #ifdef CONFIG_SPI_FLASH_YIELD_DURING_ERASE
353 no_yield_time_us += (esp_system_get_time() - start_time_us);
354 if (no_yield_time_us / 1000 >= CONFIG_SPI_FLASH_ERASE_YIELD_DURATION_MS) {
355 no_yield_time_us = 0;
356 if (s_flash_guard_ops && s_flash_guard_ops->yield) {
357 s_flash_guard_ops->yield();
358 }
359 }
360 #endif
361 }
362 }
363 COUNTER_STOP(erase);
364
365 spi_flash_guard_start();
366 // Ensure WEL is 0 after the operation, even if the erase failed.
367 esp_rom_spiflash_write_disable();
368 spi_flash_check_and_flush_cache(start_addr, size);
369 spi_flash_guard_end();
370
371 return spi_flash_translate_rc(rc);
372 }
373
374 /* Wrapper around esp_rom_spiflash_write() that verifies data as written if CONFIG_SPI_FLASH_VERIFY_WRITE is set.
375
376 If CONFIG_SPI_FLASH_VERIFY_WRITE is not set, this is esp_rom_spiflash_write().
377 */
spi_flash_write_inner(uint32_t target,const uint32_t * src_addr,int32_t len)378 static IRAM_ATTR esp_rom_spiflash_result_t spi_flash_write_inner(uint32_t target, const uint32_t *src_addr, int32_t len)
379 {
380 #ifndef CONFIG_SPI_FLASH_VERIFY_WRITE
381 return esp_rom_spiflash_write(target, src_addr, len);
382 #else // CONFIG_SPI_FLASH_VERIFY_WRITE
383 esp_rom_spiflash_result_t res = ESP_ROM_SPIFLASH_RESULT_OK;
384 assert(len % sizeof(uint32_t) == 0);
385
386 uint32_t before_buf[ESP_ROM_SPIFLASH_BUFF_BYTE_READ_NUM / sizeof(uint32_t)];
387 uint32_t after_buf[ESP_ROM_SPIFLASH_BUFF_BYTE_READ_NUM / sizeof(uint32_t)];
388 uint32_t *expected_buf = before_buf;
389 int32_t remaining = len;
390 for(int i = 0; i < len; i += sizeof(before_buf)) {
391 int i_w = i / sizeof(uint32_t); // index in words (i is an index in bytes)
392 int32_t read_len = MIN(sizeof(before_buf), remaining);
393
394 // Read "before" contents from flash
395 res = esp_rom_spiflash_read(target + i, before_buf, read_len);
396 if (res != ESP_ROM_SPIFLASH_RESULT_OK) {
397 break;
398 }
399
400 for (int r = 0; r < read_len; r += sizeof(uint32_t)) {
401 int r_w = r / sizeof(uint32_t); // index in words (r is index in bytes)
402
403 uint32_t write = src_addr[i_w + r_w];
404 uint32_t before = before_buf[r_w];
405 uint32_t expected = write & before;
406 #ifdef CONFIG_SPI_FLASH_WARN_SETTING_ZERO_TO_ONE
407 if ((before & write) != write) {
408 spi_flash_guard_end();
409 ESP_LOGW(TAG, "Write at offset 0x%x requests 0x%08x but will write 0x%08x -> 0x%08x",
410 target + i + r, write, before, before & write);
411 spi_flash_guard_start();
412 }
413 #endif
414 expected_buf[r_w] = expected;
415 }
416
417 res = esp_rom_spiflash_write(target + i, &src_addr[i_w], read_len);
418 if (res != ESP_ROM_SPIFLASH_RESULT_OK) {
419 break;
420 }
421
422 res = esp_rom_spiflash_read(target + i, after_buf, read_len);
423 if (res != ESP_ROM_SPIFLASH_RESULT_OK) {
424 break;
425 }
426
427 for (int r = 0; r < read_len; r += sizeof(uint32_t)) {
428 int r_w = r / sizeof(uint32_t); // index in words (r is index in bytes)
429
430 uint32_t expected = expected_buf[r_w];
431 uint32_t actual = after_buf[r_w];
432 if (expected != actual) {
433 #ifdef CONFIG_SPI_FLASH_LOG_FAILED_WRITE
434 spi_flash_guard_end();
435 ESP_LOGE(TAG, "Bad write at offset 0x%x expected 0x%08x readback 0x%08x", target + i + r, expected, actual);
436 spi_flash_guard_start();
437 #endif
438 res = ESP_ROM_SPIFLASH_RESULT_ERR;
439 }
440 }
441 if (res != ESP_ROM_SPIFLASH_RESULT_OK) {
442 break;
443 }
444 remaining -= read_len;
445 }
446 return res;
447 #endif // CONFIG_SPI_FLASH_VERIFY_WRITE
448 }
449
450
spi_flash_write(size_t dst,const void * srcv,size_t size)451 esp_err_t IRAM_ATTR spi_flash_write(size_t dst, const void *srcv, size_t size)
452 {
453 const spi_flash_guard_funcs_t *guard = spi_flash_guard_get();
454 CHECK_WRITE_ADDRESS(dst, size);
455 // Out of bound writes are checked in ROM code, but we can give better
456 // error code here
457 if (dst + size > g_rom_flashchip.chip_size) {
458 return ESP_ERR_INVALID_SIZE;
459 }
460 if (size == 0) {
461 return ESP_OK;
462 }
463
464 esp_rom_spiflash_result_t rc = ESP_ROM_SPIFLASH_RESULT_OK;
465 COUNTER_START();
466 const uint8_t *srcc = (const uint8_t *) srcv;
467 /*
468 * Large operations are split into (up to) 3 parts:
469 * - Left padding: 4 bytes up to the first 4-byte aligned destination offset.
470 * - Middle part
471 * - Right padding: 4 bytes from the last 4-byte aligned offset covered.
472 */
473 size_t left_off = dst & ~3U;
474 size_t left_size = MIN(((dst + 3) & ~3U) - dst, size);
475 size_t mid_off = left_size;
476 size_t mid_size = (size - left_size) & ~3U;
477 size_t right_off = left_size + mid_size;
478 size_t right_size = size - mid_size - left_size;
479
480 rc = spi_flash_unlock();
481 if (rc != ESP_ROM_SPIFLASH_RESULT_OK) {
482 goto out;
483 }
484 if (left_size > 0) {
485 uint32_t t = 0xffffffff;
486 memcpy(((uint8_t *) &t) + (dst - left_off), srcc, left_size);
487 spi_flash_guard_start();
488 rc = spi_flash_write_inner(left_off, &t, 4);
489 spi_flash_guard_end();
490 if (rc != ESP_ROM_SPIFLASH_RESULT_OK) {
491 goto out;
492 }
493 COUNTER_ADD_BYTES(write, 4);
494 }
495 if (mid_size > 0) {
496 /* If src buffer is 4-byte aligned as well and is not in a region that requires cache access to be enabled, we
497 * can write directly without buffering in RAM. */
498 #ifdef ESP_PLATFORM
499 bool direct_write = esp_ptr_internal(srcc)
500 && esp_ptr_byte_accessible(srcc)
501 && ((uintptr_t) srcc + mid_off) % 4 == 0;
502 #else
503 bool direct_write = true;
504 #endif
505 while(mid_size > 0 && rc == ESP_ROM_SPIFLASH_RESULT_OK) {
506 uint32_t write_buf[8];
507 uint32_t write_size = MIN(mid_size, MAX_WRITE_CHUNK);
508 const uint8_t *write_src = srcc + mid_off;
509 if (!direct_write) {
510 write_size = MIN(write_size, sizeof(write_buf));
511 memcpy(write_buf, write_src, write_size);
512 write_src = (const uint8_t *)write_buf;
513 }
514 spi_flash_guard_start();
515 rc = spi_flash_write_inner(dst + mid_off, (const uint32_t *) write_src, write_size);
516 spi_flash_guard_end();
517 COUNTER_ADD_BYTES(write, write_size);
518 mid_size -= write_size;
519 mid_off += write_size;
520 }
521 if (rc != ESP_ROM_SPIFLASH_RESULT_OK) {
522 goto out;
523 }
524 }
525
526 if (right_size > 0) {
527 uint32_t t = 0xffffffff;
528 memcpy(&t, srcc + right_off, right_size);
529 spi_flash_guard_start();
530 rc = spi_flash_write_inner(dst + right_off, &t, 4);
531 spi_flash_guard_end();
532 if (rc != ESP_ROM_SPIFLASH_RESULT_OK) {
533 goto out;
534 }
535 COUNTER_ADD_BYTES(write, 4);
536 }
537 out:
538 COUNTER_STOP(write);
539
540 spi_flash_guard_start();
541 // Ensure WEL is 0 after the operation, even if the write failed.
542 esp_rom_spiflash_write_disable();
543 spi_flash_check_and_flush_cache(dst, size);
544 spi_flash_guard_end();
545
546 return spi_flash_translate_rc(rc);
547 }
548 #endif // CONFIG_SPI_FLASH_USE_LEGACY_IMPL
549
550 #if !CONFIG_SPI_FLASH_USE_LEGACY_IMPL
551 #if !CONFIG_ESPTOOLPY_OCT_FLASH // Test for encryption on opi flash, IDF-3852.
552 extern void spi_common_set_dummy_output(esp_rom_spiflash_read_mode_t mode);
553 extern void spi_dummy_len_fix(uint8_t spi, uint8_t freqdiv);
flash_rom_init(void)554 void IRAM_ATTR flash_rom_init(void)
555 {
556 uint32_t freqdiv = 0;
557
558 #if CONFIG_IDF_TARGET_ESP32
559 uint32_t dummy_bit = 0;
560 #if CONFIG_ESPTOOLPY_FLASHFREQ_80M
561 dummy_bit = ESP_ROM_SPIFLASH_DUMMY_LEN_PLUS_80M;
562 #elif CONFIG_ESPTOOLPY_FLASHFREQ_40M
563 dummy_bit = ESP_ROM_SPIFLASH_DUMMY_LEN_PLUS_40M;
564 #elif CONFIG_ESPTOOLPY_FLASHFREQ_26M
565 dummy_bit = ESP_ROM_SPIFLASH_DUMMY_LEN_PLUS_26M;
566 #elif CONFIG_ESPTOOLPY_FLASHFREQ_20M
567 dummy_bit = ESP_ROM_SPIFLASH_DUMMY_LEN_PLUS_20M;
568 #endif
569 #endif//CONFIG_IDF_TARGET_ESP32
570
571 #if CONFIG_ESPTOOLPY_FLASHFREQ_80M
572 freqdiv = 1;
573 #elif CONFIG_ESPTOOLPY_FLASHFREQ_40M
574 freqdiv = 2;
575 #elif CONFIG_ESPTOOLPY_FLASHFREQ_26M
576 freqdiv = 3;
577 #elif CONFIG_ESPTOOLPY_FLASHFREQ_20M
578 freqdiv = 4;
579 #endif
580
581 #if !CONFIG_IDF_TARGET_ESP32S2 && !CONFIG_IDF_TARGET_ESP32
582 esp_rom_spiflash_read_mode_t read_mode;
583 #if CONFIG_ESPTOOLPY_FLASHMODE_QIO
584 read_mode = ESP_ROM_SPIFLASH_QIO_MODE;
585 #elif CONFIG_ESPTOOLPY_FLASHMODE_QOUT
586 read_mode = ESP_ROM_SPIFLASH_QOUT_MODE;
587 #elif CONFIG_ESPTOOLPY_FLASHMODE_DIO
588 read_mode = ESP_ROM_SPIFLASH_DIO_MODE;
589 #elif CONFIG_ESPTOOLPY_FLASHMODE_DOUT
590 read_mode = ESP_ROM_SPIFLASH_DOUT_MODE;
591 #endif
592 #endif //!CONFIG_IDF_TARGET_ESP32S2 && !CONFIG_IDF_TARGET_ESP32
593
594 #if CONFIG_IDF_TARGET_ESP32
595 g_rom_spiflash_dummy_len_plus[1] = dummy_bit;
596 #else
597 spi_dummy_len_fix(1, freqdiv);
598 #endif //CONFIG_IDF_TARGET_ESP32
599
600 #if !CONFIG_IDF_TARGET_ESP32S2 && !CONFIG_IDF_TARGET_ESP32
601 spi_common_set_dummy_output(read_mode);
602 #endif //!CONFIG_IDF_TARGET_ESP32S2
603 esp_rom_spiflash_config_clk(freqdiv, 1);
604 }
605 #endif //CONFIG_ESPTOOLPY_OCT_FLASH
606 #else
flash_rom_init(void)607 void IRAM_ATTR flash_rom_init(void)
608 {
609 return;
610 }
611
spi_flash_write_encrypted(size_t dest_addr,const void * src,size_t size)612 esp_err_t IRAM_ATTR spi_flash_write_encrypted(size_t dest_addr, const void *src, size_t size)
613 {
614 esp_err_t err = ESP_OK;
615 const spi_flash_guard_funcs_t *guard = spi_flash_guard_get();
616 CHECK_WRITE_ADDRESS(dest_addr, size);
617 if ((dest_addr % 16) != 0) {
618 return ESP_ERR_INVALID_ARG;
619 }
620 if ((size % 16) != 0) {
621 return ESP_ERR_INVALID_SIZE;
622 }
623
624 COUNTER_START();
625 esp_rom_spiflash_result_t rc = spi_flash_unlock();
626 err = spi_flash_translate_rc(rc);
627 if (err != ESP_OK) {
628 goto fail;
629 }
630
631 #ifndef CONFIG_SPI_FLASH_VERIFY_WRITE
632 err = spi_flash_write_encrypted_chip(dest_addr, src, size);
633 COUNTER_ADD_BYTES(write, size);
634 spi_flash_guard_start();
635 esp_rom_spiflash_write_disable();
636 spi_flash_check_and_flush_cache(dest_addr, size);
637 spi_flash_guard_end();
638 #else
639 const uint32_t* src_w = (const uint32_t*)src;
640 uint32_t read_buf[ESP_ROM_SPIFLASH_BUFF_BYTE_READ_NUM / sizeof(uint32_t)];
641 int32_t remaining = size;
642 for(int i = 0; i < size; i += sizeof(read_buf)) {
643 int i_w = i / sizeof(uint32_t); // index in words (i is an index in bytes)
644 int32_t read_len = MIN(sizeof(read_buf), remaining);
645
646 // Read "before" contents from flash
647 esp_err_t err = spi_flash_read(dest_addr + i, read_buf, read_len);
648 if (err != ESP_OK) {
649 break;
650 }
651
652 #ifdef CONFIG_SPI_FLASH_WARN_SETTING_ZERO_TO_ONE
653 //The written data cannot be predicted, so warning is shown if any of the bits is not 1.
654 for (int r = 0; r < read_len; r += sizeof(uint32_t)) {
655 uint32_t before = read_buf[r / sizeof(uint32_t)];
656 if (before != 0xFFFFFFFF) {
657 ESP_LOGW(TAG, "Encrypted write at offset 0x%x but not erased (0x%08x)",
658 dest_addr + i + r, before);
659 }
660 }
661 #endif
662
663 err = spi_flash_write_encrypted_chip(dest_addr + i, src + i, read_len);
664 if (err != ESP_OK) {
665 break;
666 }
667 COUNTER_ADD_BYTES(write, size);
668
669 spi_flash_guard_start();
670 esp_rom_spiflash_write_disable();
671 spi_flash_check_and_flush_cache(dest_addr, size);
672 spi_flash_guard_end();
673
674 err = spi_flash_read_encrypted(dest_addr + i, read_buf, read_len);
675 if (err != ESP_OK) {
676 break;
677 }
678
679 for (int r = 0; r < read_len; r += sizeof(uint32_t)) {
680 int r_w = r / sizeof(uint32_t); // index in words (r is index in bytes)
681
682 uint32_t expected = src_w[i_w + r_w];
683 uint32_t actual = read_buf[r_w];
684 if (expected != actual) {
685 #ifdef CONFIG_SPI_FLASH_LOG_FAILED_WRITE
686 ESP_LOGE(TAG, "Bad write at offset 0x%x expected 0x%08x readback 0x%08x", dest_addr + i + r, expected, actual);
687 #endif
688 err = ESP_FAIL;
689 }
690 }
691 if (err != ESP_OK) {
692 break;
693 }
694 remaining -= read_len;
695 }
696 #endif // CONFIG_SPI_FLASH_VERIFY_WRITE
697
698 fail:
699
700 COUNTER_STOP(write);
701 return err;
702 }
703
spi_flash_read(size_t src,void * dstv,size_t size)704 esp_err_t IRAM_ATTR spi_flash_read(size_t src, void *dstv, size_t size)
705 {
706 // Out of bound reads are checked in ROM code, but we can give better
707 // error code here
708 if (src + size > g_rom_flashchip.chip_size) {
709 return ESP_ERR_INVALID_SIZE;
710 }
711 if (size == 0) {
712 return ESP_OK;
713 }
714
715 esp_rom_spiflash_result_t rc = ESP_ROM_SPIFLASH_RESULT_OK;
716 COUNTER_START();
717 spi_flash_guard_start();
718 /* To simplify boundary checks below, we handle small reads separately. */
719 if (size < 16) {
720 uint32_t t[6]; /* Enough for 16 bytes + 4 on either side for padding. */
721 uint32_t read_src = src & ~3U;
722 uint32_t left_off = src & 3U;
723 uint32_t read_size = (left_off + size + 3) & ~3U;
724 rc = esp_rom_spiflash_read(read_src, t, read_size);
725 if (rc != ESP_ROM_SPIFLASH_RESULT_OK) {
726 goto out;
727 }
728 COUNTER_ADD_BYTES(read, read_size);
729 #ifdef ESP_PLATFORM
730 if (esp_ptr_external_ram(dstv)) {
731 spi_flash_guard_end();
732 memcpy(dstv, ((uint8_t *) t) + left_off, size);
733 spi_flash_guard_start();
734 } else {
735 memcpy(dstv, ((uint8_t *) t) + left_off, size);
736 }
737 #else
738 memcpy(dstv, ((uint8_t *) t) + left_off, size);
739 #endif
740 goto out;
741 }
742 uint8_t *dstc = (uint8_t *) dstv;
743 intptr_t dsti = (intptr_t) dstc;
744 /*
745 * Large operations are split into (up to) 3 parts:
746 * - The middle part: from the first 4-aligned position in src to the first
747 * 4-aligned position in dst.
748 */
749 size_t src_mid_off = (src % 4 == 0 ? 0 : 4 - (src % 4));
750 size_t dst_mid_off = (dsti % 4 == 0 ? 0 : 4 - (dsti % 4));
751 size_t mid_size = (size - MAX(src_mid_off, dst_mid_off)) & ~3U;
752 /*
753 * - Once the middle part is in place, src_mid_off bytes from the preceding
754 * 4-aligned source location are added on the left.
755 */
756 size_t pad_left_src = src & ~3U;
757 size_t pad_left_size = src_mid_off;
758 /*
759 * - Finally, the right part is added: from the end of the middle part to
760 * the end. Depending on the alignment of source and destination, this may
761 * be a 4 or 8 byte read from pad_right_src.
762 */
763 size_t pad_right_src = (src + pad_left_size + mid_size) & ~3U;
764 size_t pad_right_off = (pad_right_src - src);
765 size_t pad_right_size = (size - pad_right_off);
766
767 #ifdef ESP_PLATFORM
768 bool direct_read = esp_ptr_internal(dstc)
769 && esp_ptr_byte_accessible(dstc)
770 && ((uintptr_t) dstc + dst_mid_off) % 4 == 0;
771 #else
772 bool direct_read = true;
773 #endif
774 if (mid_size > 0) {
775 uint32_t mid_remaining = mid_size;
776 uint32_t mid_read = 0;
777 while (mid_remaining > 0) {
778 uint32_t read_size = MIN(mid_remaining, MAX_READ_CHUNK);
779 uint32_t read_buf[8];
780 uint8_t *read_dst_final = dstc + dst_mid_off + mid_read;
781 uint8_t *read_dst = read_dst_final;
782 if (!direct_read) {
783 read_size = MIN(read_size, sizeof(read_buf));
784 read_dst = (uint8_t *) read_buf;
785 }
786 rc = esp_rom_spiflash_read(src + src_mid_off + mid_read,
787 (uint32_t *) read_dst, read_size);
788 if (rc != ESP_ROM_SPIFLASH_RESULT_OK) {
789 goto out;
790 }
791 mid_remaining -= read_size;
792 mid_read += read_size;
793 if (!direct_read) {
794 spi_flash_guard_end();
795 memcpy(read_dst_final, read_buf, read_size);
796 spi_flash_guard_start();
797 } else if (mid_remaining > 0) {
798 /* Drop guard momentarily, allows other tasks to preempt */
799 spi_flash_guard_end();
800 spi_flash_guard_start();
801 }
802 }
803 COUNTER_ADD_BYTES(read, mid_size);
804 /*
805 * If offsets in src and dst are different, perform an in-place shift
806 * to put destination data into its final position.
807 * Note that the shift can be left (src_mid_off < dst_mid_off) or right.
808 */
809 if (src_mid_off != dst_mid_off) {
810 if (!direct_read) {
811 spi_flash_guard_end();
812 }
813 memmove(dstc + src_mid_off, dstc + dst_mid_off, mid_size);
814 if (!direct_read) {
815 spi_flash_guard_start();
816 }
817 }
818 }
819 if (pad_left_size > 0) {
820 uint32_t t;
821 rc = esp_rom_spiflash_read(pad_left_src, &t, 4);
822 if (rc != ESP_ROM_SPIFLASH_RESULT_OK) {
823 goto out;
824 }
825 COUNTER_ADD_BYTES(read, 4);
826 if (!direct_read) {
827 spi_flash_guard_end();
828 }
829 memcpy(dstc, ((uint8_t *) &t) + (4 - pad_left_size), pad_left_size);
830 if (!direct_read) {
831 spi_flash_guard_start();
832 }
833 }
834 if (pad_right_size > 0) {
835 uint32_t t[2];
836 int32_t read_size = (pad_right_size <= 4 ? 4 : 8);
837 rc = esp_rom_spiflash_read(pad_right_src, t, read_size);
838 if (rc != ESP_ROM_SPIFLASH_RESULT_OK) {
839 goto out;
840 }
841 COUNTER_ADD_BYTES(read, read_size);
842 if (!direct_read) {
843 spi_flash_guard_end();
844 }
845 memcpy(dstc + pad_right_off, t, pad_right_size);
846 if (!direct_read) {
847 spi_flash_guard_start();
848 }
849 }
850 out:
851 spi_flash_guard_end();
852 COUNTER_STOP(read);
853 return spi_flash_translate_rc(rc);
854 }
855 #endif // !CONFIG_SPI_FLASH_USE_LEGACY_IMPL
856
spi_flash_read_encrypted(size_t src,void * dstv,size_t size)857 esp_err_t IRAM_ATTR spi_flash_read_encrypted(size_t src, void *dstv, size_t size)
858 {
859 if (src + size > g_rom_flashchip.chip_size) {
860 return ESP_ERR_INVALID_SIZE;
861 }
862 if (size == 0) {
863 return ESP_OK;
864 }
865
866 esp_err_t err;
867 const uint8_t *map;
868 spi_flash_mmap_handle_t map_handle;
869 size_t map_src = src & ~(SPI_FLASH_MMU_PAGE_SIZE - 1);
870 size_t map_size = size + (src - map_src);
871
872 err = spi_flash_mmap(map_src, map_size, SPI_FLASH_MMAP_DATA, (const void **)&map, &map_handle);
873 if (err != ESP_OK) {
874 return err;
875 }
876 memcpy(dstv, map + (src - map_src), size);
877 spi_flash_munmap(map_handle);
878 return err;
879 }
880
881 #if CONFIG_SPI_FLASH_USE_LEGACY_IMPL
spi_flash_translate_rc(esp_rom_spiflash_result_t rc)882 static esp_err_t IRAM_ATTR spi_flash_translate_rc(esp_rom_spiflash_result_t rc)
883 {
884 switch (rc) {
885 case ESP_ROM_SPIFLASH_RESULT_OK:
886 return ESP_OK;
887 case ESP_ROM_SPIFLASH_RESULT_TIMEOUT:
888 return ESP_ERR_FLASH_OP_TIMEOUT;
889 case ESP_ROM_SPIFLASH_RESULT_ERR:
890 default:
891 return ESP_ERR_FLASH_OP_FAIL;
892 }
893 }
894 #endif //CONFIG_SPI_FLASH_USE_LEGACY_IMPL
895
896 #if CONFIG_SPI_FLASH_ENABLE_COUNTERS
897
dump_counter(spi_flash_counter_t * counter,const char * name)898 static inline void dump_counter(spi_flash_counter_t *counter, const char *name)
899 {
900 ESP_LOGI(TAG, "%s count=%8d time=%8dus bytes=%8d\n", name,
901 counter->count, counter->time, counter->bytes);
902 }
903
spi_flash_get_counters(void)904 const spi_flash_counters_t *spi_flash_get_counters(void)
905 {
906 return &s_flash_stats;
907 }
908
spi_flash_reset_counters(void)909 void spi_flash_reset_counters(void)
910 {
911 memset(&s_flash_stats, 0, sizeof(s_flash_stats));
912 }
913
spi_flash_dump_counters(void)914 void spi_flash_dump_counters(void)
915 {
916 dump_counter(&s_flash_stats.read, "read ");
917 dump_counter(&s_flash_stats.write, "write");
918 dump_counter(&s_flash_stats.erase, "erase");
919 }
920
921 #endif //CONFIG_SPI_FLASH_ENABLE_COUNTERS
922
923 #if CONFIG_SPI_FLASH_USE_LEGACY_IMPL && !CONFIG_IDF_TARGET_ESP32
924 // TODO esp32s2: Remove once ESP32-S2 & later chips has new SPI Flash API support
925 esp_flash_t *esp_flash_default_chip = NULL;
926 #endif
927
spi_flash_set_rom_required_regs(void)928 void IRAM_ATTR spi_flash_set_rom_required_regs(void)
929 {
930 #if CONFIG_ESPTOOLPY_OCT_FLASH
931 //Disable the variable dummy mode when doing timing tuning
932 CLEAR_PERI_REG_MASK(SPI_MEM_DDR_REG(1), SPI_MEM_SPI_FMEM_VAR_DUMMY);
933 /**
934 * STR /DTR mode setting is done every time when `esp_rom_opiflash_exec_cmd` is called
935 *
936 * Add any registers that are not set in ROM SPI flash functions here in the future
937 */
938 #endif
939 }
940
spi_flash_set_vendor_required_regs(void)941 void IRAM_ATTR spi_flash_set_vendor_required_regs(void)
942 {
943 #if CONFIG_ESPTOOLPY_OCT_FLASH
944 //Flash chip requires MSPI specifically, call this function to set them
945 esp_opiflash_set_required_regs();
946 #else
947 //currently we don't need to set other MSPI registers for Quad Flash
948 #endif
949 }
950
