1 // Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 #include <stdlib.h>
16 #include <assert.h>
17 #include <string.h>
18 #include <stdio.h>
19 #include <sys/param.h> // For MIN/MAX(a, b)
20
21 #include <freertos/FreeRTOS.h>
22 #include <freertos/task.h>
23 #include <freertos/semphr.h>
24 #include <soc/soc.h>
25 #include <soc/soc_memory_layout.h>
26 #include "sdkconfig.h"
27 #include "esp_attr.h"
28 #include "esp_spi_flash.h"
29 #include "esp_log.h"
30 #include "esp_private/system_internal.h"
31 #if CONFIG_IDF_TARGET_ESP32
32 #include "esp32/rom/cache.h"
33 #include "esp32/rom/spi_flash.h"
34 #include "esp32/clk.h"
35 #elif CONFIG_IDF_TARGET_ESP32S2
36 #include "esp32s2/rom/cache.h"
37 #include "esp32s2/rom/spi_flash.h"
38 #include "esp32s2/clk.h"
39 #elif CONFIG_IDF_TARGET_ESP32S3
40 #include "esp32s3/rom/spi_flash.h"
41 #include "esp32s3/rom/cache.h"
42 #include "esp32s3/clk.h"
43 #elif CONFIG_IDF_TARGET_ESP32C3
44 #include "esp32c3/rom/cache.h"
45 #include "esp32c3/rom/spi_flash.h"
46 #include "esp32c3/clk.h"
47 #endif
48 #include "esp_flash_partitions.h"
49 #include "cache_utils.h"
50 #include "esp_flash.h"
51 #include "esp_attr.h"
52
53 #if defined(__ZEPHYR__)
54 #include "common/cache_utils.h"
55 #endif
56
57 esp_rom_spiflash_result_t IRAM_ATTR spi_flash_write_encrypted_chip(size_t dest_addr, const void *src, size_t size);
58
59 /* bytes erased by SPIEraseBlock() ROM function */
60 #define BLOCK_ERASE_SIZE 65536
61
62 /* Limit number of bytes written/read in a single SPI operation,
63 as these operations disable all higher priority tasks from running.
64 */
65 #ifdef CONFIG_SPI_FLASH_WRITE_CHUNK_SIZE
66 #define MAX_WRITE_CHUNK CONFIG_SPI_FLASH_WRITE_CHUNK_SIZE
67 #else
68 #define MAX_WRITE_CHUNK 8192
69 #endif // CONFIG_SPI_FLASH_WRITE_CHUNK_SIZE
70
71 #define MAX_READ_CHUNK 16384
72
73 static const char *TAG __attribute__((unused)) = "spi_flash";
74
75 #if CONFIG_SPI_FLASH_ENABLE_COUNTERS
76 static spi_flash_counters_t s_flash_stats;
77
78 #define COUNTER_START() uint32_t ts_begin = cpu_hal_get_cycle_count()
79 #define COUNTER_STOP(counter) \
80 do{ \
81 s_flash_stats.counter.count++; \
82 s_flash_stats.counter.time += (cpu_hal_get_cycle_count() - ts_begin) / (esp_clk_cpu_freq() / 1000000); \
83 } while(0)
84
85 #define COUNTER_ADD_BYTES(counter, size) \
86 do { \
87 s_flash_stats.counter.bytes += size; \
88 } while (0)
89
90 #else
91 #define COUNTER_START()
92 #define COUNTER_STOP(counter)
93 #define COUNTER_ADD_BYTES(counter, size)
94
95 #endif //CONFIG_SPI_FLASH_ENABLE_COUNTERS
96
97 static esp_err_t spi_flash_translate_rc(esp_rom_spiflash_result_t rc);
98 static bool is_safe_write_address(size_t addr, size_t size);
99 #if !defined(__ZEPHYR__)
100 static void spi_flash_os_yield(void);
101 #endif
102
103 const DRAM_ATTR spi_flash_guard_funcs_t g_flash_guard_default_ops = {
104 #if defined(__ZEPHYR__)
105 .start = esp32_spiflash_start,
106 .end = esp32_spiflash_end,
107 .op_lock = 0,
108 .op_unlock = 0,
109 #if !CONFIG_SPI_FLASH_DANGEROUS_WRITE_ALLOWED
110 .is_safe_write_address = 0
111 #endif
112 #else
113 .start = spi_flash_disable_interrupts_caches_and_other_cpu,
114 .end = spi_flash_enable_interrupts_caches_and_other_cpu,
115 .op_lock = spi_flash_op_lock,
116 .op_unlock = spi_flash_op_unlock,
117 #if !CONFIG_SPI_FLASH_DANGEROUS_WRITE_ALLOWED
118 .is_safe_write_address = is_safe_write_address,
119 #endif
120 .yield = spi_flash_os_yield,
121 #endif // __ZEPHYR__
122 };
123
124 const DRAM_ATTR spi_flash_guard_funcs_t g_flash_guard_no_os_ops = {
125 .start = spi_flash_disable_interrupts_caches_and_other_cpu_no_os,
126 .end = spi_flash_enable_interrupts_caches_no_os,
127 .op_lock = NULL,
128 .op_unlock = NULL,
129 #if !CONFIG_SPI_FLASH_DANGEROUS_WRITE_ALLOWED
130 .is_safe_write_address = NULL,
131 #endif
132 .yield = NULL,
133 };
134
135 #ifdef CONFIG_SPI_FLASH_DANGEROUS_WRITE_ABORTS
136 #define UNSAFE_WRITE_ADDRESS abort()
137 #else
138 #define UNSAFE_WRITE_ADDRESS return false
139 #endif
140
141
142 /* CHECK_WRITE_ADDRESS macro to fail writes which land in the
143 bootloader, partition table, or running application region.
144 */
145 #if CONFIG_SPI_FLASH_DANGEROUS_WRITE_ALLOWED
146 #define CHECK_WRITE_ADDRESS(ADDR, SIZE)
147 #else /* FAILS or ABORTS */
148 #define CHECK_WRITE_ADDRESS(ADDR, SIZE) do { \
149 if (guard && guard->is_safe_write_address && !guard->is_safe_write_address(ADDR, SIZE)) { \
150 return ESP_ERR_INVALID_ARG; \
151 } \
152 } while(0)
153 #endif // CONFIG_SPI_FLASH_DANGEROUS_WRITE_ALLOWED
154
is_safe_write_address(size_t addr,size_t size)155 static __attribute__((unused)) bool is_safe_write_address(size_t addr, size_t size)
156 {
157 if (!esp_partition_main_flash_region_safe(addr, size)) {
158 UNSAFE_WRITE_ADDRESS;
159 }
160 return true;
161 }
162
163 #if CONFIG_SPI_FLASH_ROM_IMPL
164 #include "esp_heap_caps.h"
165 typedef void *(*malloc_internal_cb_t)(size_t size);
166
spi_flash_malloc_internal(size_t size)167 void IRAM_ATTR *spi_flash_malloc_internal(size_t size)
168 {
169 return heap_caps_malloc(size, MALLOC_CAP_8BIT|MALLOC_CAP_INTERNAL);
170 }
171 #endif
172
spi_flash_init(void)173 void spi_flash_init(void)
174 {
175 spi_flash_init_lock();
176 #if CONFIG_SPI_FLASH_ENABLE_COUNTERS
177 spi_flash_reset_counters();
178 #endif
179
180 #if CONFIG_SPI_FLASH_ROM_IMPL
181 spi_flash_guard_set(&g_flash_guard_default_ops);
182
183 /* These two functions are in ROM only */
184 extern void spi_flash_mmap_os_func_set(void *(*func1)(size_t size), void (*func2)(void *p));
185 spi_flash_mmap_os_func_set(spi_flash_malloc_internal, heap_caps_free);
186
187 extern esp_err_t spi_flash_mmap_page_num_init(uint32_t page_num);
188 spi_flash_mmap_page_num_init(128);
189 #endif
190 }
191
192 #if !CONFIG_SPI_FLASH_ROM_IMPL
193
194 static const spi_flash_guard_funcs_t *s_flash_guard_ops;
195
spi_flash_guard_set(const spi_flash_guard_funcs_t * funcs)196 void IRAM_ATTR spi_flash_guard_set(const spi_flash_guard_funcs_t *funcs)
197 {
198 s_flash_guard_ops = funcs;
199 }
200
spi_flash_guard_get(void)201 const spi_flash_guard_funcs_t *IRAM_ATTR spi_flash_guard_get(void)
202 {
203 return s_flash_guard_ops;
204 }
205
206 #endif
207
spi_flash_get_chip_size(void)208 size_t IRAM_ATTR spi_flash_get_chip_size(void)
209 {
210 return g_rom_flashchip.chip_size;
211 }
212
spi_flash_guard_start(void)213 static inline void IRAM_ATTR spi_flash_guard_start(void)
214 {
215 const spi_flash_guard_funcs_t *guard = spi_flash_guard_get();
216 if (guard && guard->start) {
217 guard->start();
218 }
219 }
220
spi_flash_guard_end(void)221 static inline void IRAM_ATTR spi_flash_guard_end(void)
222 {
223 const spi_flash_guard_funcs_t *guard = spi_flash_guard_get();
224 if (guard && guard->end) {
225 guard->end();
226 }
227 }
228
spi_flash_guard_op_lock(void)229 static inline void IRAM_ATTR spi_flash_guard_op_lock(void)
230 {
231 const spi_flash_guard_funcs_t *guard = spi_flash_guard_get();
232 if (guard && guard->op_lock) {
233 guard->op_lock();
234 }
235 }
236
spi_flash_guard_op_unlock(void)237 static inline void IRAM_ATTR spi_flash_guard_op_unlock(void)
238 {
239 const spi_flash_guard_funcs_t *guard = spi_flash_guard_get();
240 if (guard && guard->op_unlock) {
241 guard->op_unlock();
242 }
243 }
244
245 #if !defined(__ZEPHYR__)
spi_flash_os_yield(void)246 static void IRAM_ATTR spi_flash_os_yield(void)
247 {
248 #ifdef CONFIG_SPI_FLASH_YIELD_DURING_ERASE
249 vTaskDelay(CONFIG_SPI_FLASH_ERASE_YIELD_TICKS);
250 #endif
251 }
252 #endif
253
254 #ifdef CONFIG_SPI_FLASH_USE_LEGACY_IMPL
spi_flash_unlock(void)255 static esp_rom_spiflash_result_t IRAM_ATTR spi_flash_unlock(void)
256 {
257 static bool unlocked = false;
258 if (!unlocked) {
259 spi_flash_guard_start();
260 esp_rom_spiflash_result_t rc = esp_rom_spiflash_unlock();
261 spi_flash_guard_end();
262 if (rc != ESP_ROM_SPIFLASH_RESULT_OK) {
263 return rc;
264 }
265 unlocked = true;
266 }
267 return ESP_ROM_SPIFLASH_RESULT_OK;
268 }
269 #else
spi_flash_unlock(void)270 static esp_rom_spiflash_result_t IRAM_ATTR spi_flash_unlock(void)
271 {
272 esp_err_t err = esp_flash_set_chip_write_protect(NULL, false);
273 if (err != ESP_OK) {
274 return ESP_ROM_SPIFLASH_RESULT_ERR;
275 }
276 return ESP_ROM_SPIFLASH_RESULT_OK;
277 }
278 #endif // CONFIG_SPI_FLASH_USE_LEGACY_IMPL
279
spi_flash_erase_sector(size_t sec)280 esp_err_t IRAM_ATTR spi_flash_erase_sector(size_t sec)
281 {
282 const spi_flash_guard_funcs_t *guard = spi_flash_guard_get();
283 CHECK_WRITE_ADDRESS(sec * SPI_FLASH_SEC_SIZE, SPI_FLASH_SEC_SIZE);
284 return spi_flash_erase_range(sec * SPI_FLASH_SEC_SIZE, SPI_FLASH_SEC_SIZE);
285 }
286
287 #ifdef CONFIG_SPI_FLASH_USE_LEGACY_IMPL
288 //deprecated, only used in compatible mode
spi_flash_erase_range(size_t start_addr,size_t size)289 esp_err_t IRAM_ATTR spi_flash_erase_range(size_t start_addr, size_t size)
290 {
291 const spi_flash_guard_funcs_t *guard = spi_flash_guard_get();
292 CHECK_WRITE_ADDRESS(start_addr, size);
293 if (start_addr % SPI_FLASH_SEC_SIZE != 0) {
294 return ESP_ERR_INVALID_ARG;
295 }
296 if (size % SPI_FLASH_SEC_SIZE != 0) {
297 return ESP_ERR_INVALID_SIZE;
298 }
299 if (size + start_addr > spi_flash_get_chip_size()) {
300 return ESP_ERR_INVALID_SIZE;
301 }
302 size_t start = start_addr / SPI_FLASH_SEC_SIZE;
303 size_t end = start + size / SPI_FLASH_SEC_SIZE;
304 const size_t sectors_per_block = BLOCK_ERASE_SIZE / SPI_FLASH_SEC_SIZE;
305 COUNTER_START();
306 esp_rom_spiflash_result_t rc;
307 rc = spi_flash_unlock();
308 if (rc == ESP_ROM_SPIFLASH_RESULT_OK) {
309 #ifdef CONFIG_SPI_FLASH_YIELD_DURING_ERASE
310 int64_t no_yield_time_us = 0;
311 #endif
312 for (size_t sector = start; sector != end && rc == ESP_ROM_SPIFLASH_RESULT_OK; ) {
313 #ifdef CONFIG_SPI_FLASH_YIELD_DURING_ERASE
314 int64_t start_time_us = esp_system_get_time();
315 #endif
316 spi_flash_guard_start();
317 #ifndef CONFIG_SPI_FLASH_BYPASS_BLOCK_ERASE
318 if (sector % sectors_per_block == 0 && end - sector >= sectors_per_block) {
319 rc = esp_rom_spiflash_erase_block(sector / sectors_per_block);
320 sector += sectors_per_block;
321 COUNTER_ADD_BYTES(erase, sectors_per_block * SPI_FLASH_SEC_SIZE);
322 } else
323 #endif
324 {
325 rc = esp_rom_spiflash_erase_sector(sector);
326 ++sector;
327 COUNTER_ADD_BYTES(erase, SPI_FLASH_SEC_SIZE);
328 }
329 spi_flash_guard_end();
330 #ifdef CONFIG_SPI_FLASH_YIELD_DURING_ERASE
331 no_yield_time_us += (esp_system_get_time() - start_time_us);
332 if (no_yield_time_us / 1000 >= CONFIG_SPI_FLASH_ERASE_YIELD_DURATION_MS) {
333 no_yield_time_us = 0;
334 if (s_flash_guard_ops && s_flash_guard_ops->yield) {
335 s_flash_guard_ops->yield();
336 }
337 }
338 #endif
339 }
340 }
341 COUNTER_STOP(erase);
342
343 spi_flash_guard_start();
344 // Ensure WEL is 0 after the operation, even if the erase failed.
345 esp_rom_spiflash_write_disable();
346 spi_flash_check_and_flush_cache(start_addr, size);
347 spi_flash_guard_end();
348
349 return spi_flash_translate_rc(rc);
350 }
351
352 /* Wrapper around esp_rom_spiflash_write() that verifies data as written if CONFIG_SPI_FLASH_VERIFY_WRITE is set.
353
354 If CONFIG_SPI_FLASH_VERIFY_WRITE is not set, this is esp_rom_spiflash_write().
355 */
spi_flash_write_inner(uint32_t target,const uint32_t * src_addr,int32_t len)356 static IRAM_ATTR esp_rom_spiflash_result_t spi_flash_write_inner(uint32_t target, const uint32_t *src_addr, int32_t len)
357 {
358 #ifndef CONFIG_SPI_FLASH_VERIFY_WRITE
359 return esp_rom_spiflash_write(target, src_addr, len);
360 #else // CONFIG_SPI_FLASH_VERIFY_WRITE
361 esp_rom_spiflash_result_t res = ESP_ROM_SPIFLASH_RESULT_OK;
362 assert(len % sizeof(uint32_t) == 0);
363
364 uint32_t before_buf[ESP_ROM_SPIFLASH_BUFF_BYTE_READ_NUM / sizeof(uint32_t)];
365 uint32_t after_buf[ESP_ROM_SPIFLASH_BUFF_BYTE_READ_NUM / sizeof(uint32_t)];
366 uint32_t *expected_buf = before_buf;
367 int32_t remaining = len;
368 for(int i = 0; i < len; i += sizeof(before_buf)) {
369 int i_w = i / sizeof(uint32_t); // index in words (i is an index in bytes)
370 int32_t read_len = MIN(sizeof(before_buf), remaining);
371
372 // Read "before" contents from flash
373 res = esp_rom_spiflash_read(target + i, before_buf, read_len);
374 if (res != ESP_ROM_SPIFLASH_RESULT_OK) {
375 break;
376 }
377
378 for (int r = 0; r < read_len; r += sizeof(uint32_t)) {
379 int r_w = r / sizeof(uint32_t); // index in words (r is index in bytes)
380
381 uint32_t write = src_addr[i_w + r_w];
382 uint32_t before = before_buf[r_w];
383 uint32_t expected = write & before;
384 #ifdef CONFIG_SPI_FLASH_WARN_SETTING_ZERO_TO_ONE
385 if ((before & write) != write) {
386 spi_flash_guard_end();
387 ESP_LOGW(TAG, "Write at offset 0x%x requests 0x%08x but will write 0x%08x -> 0x%08x",
388 target + i + r, write, before, before & write);
389 spi_flash_guard_start();
390 }
391 #endif
392 expected_buf[r_w] = expected;
393 }
394
395 res = esp_rom_spiflash_write(target + i, &src_addr[i_w], read_len);
396 if (res != ESP_ROM_SPIFLASH_RESULT_OK) {
397 break;
398 }
399
400 res = esp_rom_spiflash_read(target + i, after_buf, read_len);
401 if (res != ESP_ROM_SPIFLASH_RESULT_OK) {
402 break;
403 }
404
405 for (int r = 0; r < read_len; r += sizeof(uint32_t)) {
406 int r_w = r / sizeof(uint32_t); // index in words (r is index in bytes)
407
408 uint32_t expected = expected_buf[r_w];
409 uint32_t actual = after_buf[r_w];
410 if (expected != actual) {
411 #ifdef CONFIG_SPI_FLASH_LOG_FAILED_WRITE
412 spi_flash_guard_end();
413 ESP_LOGE(TAG, "Bad write at offset 0x%x expected 0x%08x readback 0x%08x", target + i + r, expected, actual);
414 spi_flash_guard_start();
415 #endif
416 res = ESP_ROM_SPIFLASH_RESULT_ERR;
417 }
418 }
419 if (res != ESP_ROM_SPIFLASH_RESULT_OK) {
420 break;
421 }
422 remaining -= read_len;
423 }
424 return res;
425 #endif // CONFIG_SPI_FLASH_VERIFY_WRITE
426 }
427
428
spi_flash_write(size_t dst,const void * srcv,size_t size)429 esp_err_t IRAM_ATTR spi_flash_write(size_t dst, const void *srcv, size_t size)
430 {
431 const spi_flash_guard_funcs_t *guard = spi_flash_guard_get();
432 CHECK_WRITE_ADDRESS(dst, size);
433 // Out of bound writes are checked in ROM code, but we can give better
434 // error code here
435 if (dst + size > g_rom_flashchip.chip_size) {
436 return ESP_ERR_INVALID_SIZE;
437 }
438 if (size == 0) {
439 return ESP_OK;
440 }
441
442 esp_rom_spiflash_result_t rc = ESP_ROM_SPIFLASH_RESULT_OK;
443 COUNTER_START();
444 const uint8_t *srcc = (const uint8_t *) srcv;
445 /*
446 * Large operations are split into (up to) 3 parts:
447 * - Left padding: 4 bytes up to the first 4-byte aligned destination offset.
448 * - Middle part
449 * - Right padding: 4 bytes from the last 4-byte aligned offset covered.
450 */
451 size_t left_off = dst & ~3U;
452 size_t left_size = MIN(((dst + 3) & ~3U) - dst, size);
453 size_t mid_off = left_size;
454 size_t mid_size = (size - left_size) & ~3U;
455 size_t right_off = left_size + mid_size;
456 size_t right_size = size - mid_size - left_size;
457
458 rc = spi_flash_unlock();
459 if (rc != ESP_ROM_SPIFLASH_RESULT_OK) {
460 goto out;
461 }
462 if (left_size > 0) {
463 uint32_t t = 0xffffffff;
464 memcpy(((uint8_t *) &t) + (dst - left_off), srcc, left_size);
465 spi_flash_guard_start();
466 rc = spi_flash_write_inner(left_off, &t, 4);
467 spi_flash_guard_end();
468 if (rc != ESP_ROM_SPIFLASH_RESULT_OK) {
469 goto out;
470 }
471 COUNTER_ADD_BYTES(write, 4);
472 }
473 if (mid_size > 0) {
474 /* If src buffer is 4-byte aligned as well and is not in a region that requires cache access to be enabled, we
475 * can write directly without buffering in RAM. */
476 #ifdef ESP_PLATFORM
477 bool direct_write = esp_ptr_internal(srcc)
478 && esp_ptr_byte_accessible(srcc)
479 && ((uintptr_t) srcc + mid_off) % 4 == 0;
480 #else
481 bool direct_write = true;
482 #endif
483 while(mid_size > 0 && rc == ESP_ROM_SPIFLASH_RESULT_OK) {
484 uint32_t write_buf[8];
485 uint32_t write_size = MIN(mid_size, MAX_WRITE_CHUNK);
486 const uint8_t *write_src = srcc + mid_off;
487 if (!direct_write) {
488 write_size = MIN(write_size, sizeof(write_buf));
489 memcpy(write_buf, write_src, write_size);
490 write_src = (const uint8_t *)write_buf;
491 }
492 spi_flash_guard_start();
493 rc = spi_flash_write_inner(dst + mid_off, (const uint32_t *) write_src, write_size);
494 spi_flash_guard_end();
495 COUNTER_ADD_BYTES(write, write_size);
496 mid_size -= write_size;
497 mid_off += write_size;
498 }
499 if (rc != ESP_ROM_SPIFLASH_RESULT_OK) {
500 goto out;
501 }
502 }
503
504 if (right_size > 0) {
505 uint32_t t = 0xffffffff;
506 memcpy(&t, srcc + right_off, right_size);
507 spi_flash_guard_start();
508 rc = spi_flash_write_inner(dst + right_off, &t, 4);
509 spi_flash_guard_end();
510 if (rc != ESP_ROM_SPIFLASH_RESULT_OK) {
511 goto out;
512 }
513 COUNTER_ADD_BYTES(write, 4);
514 }
515 out:
516 COUNTER_STOP(write);
517
518 spi_flash_guard_start();
519 // Ensure WEL is 0 after the operation, even if the write failed.
520 esp_rom_spiflash_write_disable();
521 spi_flash_check_and_flush_cache(dst, size);
522 spi_flash_guard_end();
523
524 return spi_flash_translate_rc(rc);
525 }
526 #endif // CONFIG_SPI_FLASH_USE_LEGACY_IMPL
527
528 #if !CONFIG_SPI_FLASH_USE_LEGACY_IMPL
529 extern void spi_common_set_dummy_output(esp_rom_spiflash_read_mode_t mode);
530 extern void spi_dummy_len_fix(uint8_t spi, uint8_t freqdiv);
flash_rom_init(void)531 void IRAM_ATTR flash_rom_init(void)
532 {
533 uint32_t freqdiv = 0;
534
535 #if CONFIG_IDF_TARGET_ESP32
536 uint32_t dummy_bit = 0;
537 #if CONFIG_ESPTOOLPY_FLASHFREQ_80M
538 dummy_bit = ESP_ROM_SPIFLASH_DUMMY_LEN_PLUS_80M;
539 #elif CONFIG_ESPTOOLPY_FLASHFREQ_40M
540 dummy_bit = ESP_ROM_SPIFLASH_DUMMY_LEN_PLUS_40M;
541 #elif CONFIG_ESPTOOLPY_FLASHFREQ_26M
542 dummy_bit = ESP_ROM_SPIFLASH_DUMMY_LEN_PLUS_26M;
543 #elif CONFIG_ESPTOOLPY_FLASHFREQ_20M
544 dummy_bit = ESP_ROM_SPIFLASH_DUMMY_LEN_PLUS_20M;
545 #endif
546 #endif//CONFIG_IDF_TARGET_ESP32
547
548 #if CONFIG_ESPTOOLPY_FLASHFREQ_80M
549 freqdiv = 1;
550 #elif CONFIG_ESPTOOLPY_FLASHFREQ_40M
551 freqdiv = 2;
552 #elif CONFIG_ESPTOOLPY_FLASHFREQ_26M
553 freqdiv = 3;
554 #elif CONFIG_ESPTOOLPY_FLASHFREQ_20M
555 freqdiv = 4;
556 #endif
557
558 #if !CONFIG_IDF_TARGET_ESP32S2 && !CONFIG_IDF_TARGET_ESP32
559 esp_rom_spiflash_read_mode_t read_mode;
560 #if CONFIG_ESPTOOLPY_FLASHMODE_QIO
561 read_mode = ESP_ROM_SPIFLASH_QIO_MODE;
562 #elif CONFIG_ESPTOOLPY_FLASHMODE_QOUT
563 read_mode = ESP_ROM_SPIFLASH_QOUT_MODE;
564 #elif CONFIG_ESPTOOLPY_FLASHMODE_DIO
565 read_mode = ESP_ROM_SPIFLASH_DIO_MODE;
566 #elif CONFIG_ESPTOOLPY_FLASHMODE_DOUT
567 read_mode = ESP_ROM_SPIFLASH_DOUT_MODE;
568 #endif
569 #endif //!CONFIG_IDF_TARGET_ESP32S2 && !CONFIG_IDF_TARGET_ESP32
570
571 #if CONFIG_IDF_TARGET_ESP32
572 g_rom_spiflash_dummy_len_plus[1] = dummy_bit;
573 #else
574 spi_dummy_len_fix(1, freqdiv);
575 #endif //CONFIG_IDF_TARGET_ESP32
576
577 #if !CONFIG_IDF_TARGET_ESP32S2 && !CONFIG_IDF_TARGET_ESP32
578 spi_common_set_dummy_output(read_mode);
579 #endif //!CONFIG_IDF_TARGET_ESP32S2
580 esp_rom_spiflash_config_clk(freqdiv, 1);
581 }
582 #else
flash_rom_init(void)583 void IRAM_ATTR flash_rom_init(void)
584 {
585 return;
586 }
587 #endif // !CONFIG_SPI_FLASH_USE_LEGACY_IMPL
588
spi_flash_write_encrypted(size_t dest_addr,const void * src,size_t size)589 esp_err_t IRAM_ATTR spi_flash_write_encrypted(size_t dest_addr, const void *src, size_t size)
590 {
591 esp_err_t err = ESP_OK;
592 const spi_flash_guard_funcs_t *guard = spi_flash_guard_get();
593 CHECK_WRITE_ADDRESS(dest_addr, size);
594 if ((dest_addr % 16) != 0) {
595 return ESP_ERR_INVALID_ARG;
596 }
597 if ((size % 16) != 0) {
598 return ESP_ERR_INVALID_SIZE;
599 }
600
601 COUNTER_START();
602 esp_rom_spiflash_result_t rc = spi_flash_unlock();
603 err = spi_flash_translate_rc(rc);
604 if (err != ESP_OK) {
605 goto fail;
606 }
607
608 #ifndef CONFIG_SPI_FLASH_VERIFY_WRITE
609 err = spi_flash_write_encrypted_chip(dest_addr, src, size);
610 COUNTER_ADD_BYTES(write, size);
611 spi_flash_guard_start();
612 esp_rom_spiflash_write_disable();
613 spi_flash_check_and_flush_cache(dest_addr, size);
614 spi_flash_guard_end();
615 #else
616 const uint32_t* src_w = (const uint32_t*)src;
617 uint32_t read_buf[ESP_ROM_SPIFLASH_BUFF_BYTE_READ_NUM / sizeof(uint32_t)];
618 int32_t remaining = size;
619 for(int i = 0; i < size; i += sizeof(read_buf)) {
620 int i_w = i / sizeof(uint32_t); // index in words (i is an index in bytes)
621 int32_t read_len = MIN(sizeof(read_buf), remaining);
622
623 // Read "before" contents from flash
624 esp_err_t err = spi_flash_read(dest_addr + i, read_buf, read_len);
625 if (err != ESP_OK) {
626 break;
627 }
628
629 #ifdef CONFIG_SPI_FLASH_WARN_SETTING_ZERO_TO_ONE
630 //The written data cannot be predicted, so warning is shown if any of the bits is not 1.
631 for (int r = 0; r < read_len; r += sizeof(uint32_t)) {
632 uint32_t before = read_buf[r / sizeof(uint32_t)];
633 if (before != 0xFFFFFFFF) {
634 ESP_LOGW(TAG, "Encrypted write at offset 0x%x but not erased (0x%08x)",
635 dest_addr + i + r, before);
636 }
637 }
638 #endif
639
640 err = spi_flash_write_encrypted_chip(dest_addr + i, src + i, read_len);
641 if (err != ESP_OK) {
642 break;
643 }
644 COUNTER_ADD_BYTES(write, size);
645
646 spi_flash_guard_start();
647 esp_rom_spiflash_write_disable();
648 spi_flash_check_and_flush_cache(dest_addr, size);
649 spi_flash_guard_end();
650
651 err = spi_flash_read_encrypted(dest_addr + i, read_buf, read_len);
652 if (err != ESP_OK) {
653 break;
654 }
655
656 for (int r = 0; r < read_len; r += sizeof(uint32_t)) {
657 int r_w = r / sizeof(uint32_t); // index in words (r is index in bytes)
658
659 uint32_t expected = src_w[i_w + r_w];
660 uint32_t actual = read_buf[r_w];
661 if (expected != actual) {
662 #ifdef CONFIG_SPI_FLASH_LOG_FAILED_WRITE
663 ESP_LOGE(TAG, "Bad write at offset 0x%x expected 0x%08x readback 0x%08x", dest_addr + i + r, expected, actual);
664 #endif
665 err = ESP_FAIL;
666 }
667 }
668 if (err != ESP_OK) {
669 break;
670 }
671 remaining -= read_len;
672 }
673 #endif // CONFIG_SPI_FLASH_VERIFY_WRITE
674
675 fail:
676
677 COUNTER_STOP(write);
678 return err;
679 }
680
681
682 #ifdef CONFIG_SPI_FLASH_USE_LEGACY_IMPL
spi_flash_read(size_t src,void * dstv,size_t size)683 esp_err_t IRAM_ATTR spi_flash_read(size_t src, void *dstv, size_t size)
684 {
685 // Out of bound reads are checked in ROM code, but we can give better
686 // error code here
687 if (src + size > g_rom_flashchip.chip_size) {
688 return ESP_ERR_INVALID_SIZE;
689 }
690 if (size == 0) {
691 return ESP_OK;
692 }
693
694 esp_rom_spiflash_result_t rc = ESP_ROM_SPIFLASH_RESULT_OK;
695 COUNTER_START();
696 spi_flash_guard_start();
697 /* To simplify boundary checks below, we handle small reads separately. */
698 if (size < 16) {
699 uint32_t t[6]; /* Enough for 16 bytes + 4 on either side for padding. */
700 uint32_t read_src = src & ~3U;
701 uint32_t left_off = src & 3U;
702 uint32_t read_size = (left_off + size + 3) & ~3U;
703 rc = esp_rom_spiflash_read(read_src, t, read_size);
704 if (rc != ESP_ROM_SPIFLASH_RESULT_OK) {
705 goto out;
706 }
707 COUNTER_ADD_BYTES(read, read_size);
708 #ifdef ESP_PLATFORM
709 if (esp_ptr_external_ram(dstv)) {
710 spi_flash_guard_end();
711 memcpy(dstv, ((uint8_t *) t) + left_off, size);
712 spi_flash_guard_start();
713 } else {
714 memcpy(dstv, ((uint8_t *) t) + left_off, size);
715 }
716 #else
717 memcpy(dstv, ((uint8_t *) t) + left_off, size);
718 #endif
719 goto out;
720 }
721 uint8_t *dstc = (uint8_t *) dstv;
722 intptr_t dsti = (intptr_t) dstc;
723 /*
724 * Large operations are split into (up to) 3 parts:
725 * - The middle part: from the first 4-aligned position in src to the first
726 * 4-aligned position in dst.
727 */
728 size_t src_mid_off = (src % 4 == 0 ? 0 : 4 - (src % 4));
729 size_t dst_mid_off = (dsti % 4 == 0 ? 0 : 4 - (dsti % 4));
730 size_t mid_size = (size - MAX(src_mid_off, dst_mid_off)) & ~3U;
731 /*
732 * - Once the middle part is in place, src_mid_off bytes from the preceding
733 * 4-aligned source location are added on the left.
734 */
735 size_t pad_left_src = src & ~3U;
736 size_t pad_left_size = src_mid_off;
737 /*
738 * - Finally, the right part is added: from the end of the middle part to
739 * the end. Depending on the alignment of source and destination, this may
740 * be a 4 or 8 byte read from pad_right_src.
741 */
742 size_t pad_right_src = (src + pad_left_size + mid_size) & ~3U;
743 size_t pad_right_off = (pad_right_src - src);
744 size_t pad_right_size = (size - pad_right_off);
745
746 #ifdef ESP_PLATFORM
747 bool direct_read = esp_ptr_internal(dstc)
748 && esp_ptr_byte_accessible(dstc)
749 && ((uintptr_t) dstc + dst_mid_off) % 4 == 0;
750 #else
751 bool direct_read = true;
752 #endif
753 if (mid_size > 0) {
754 uint32_t mid_remaining = mid_size;
755 uint32_t mid_read = 0;
756 while (mid_remaining > 0) {
757 uint32_t read_size = MIN(mid_remaining, MAX_READ_CHUNK);
758 uint32_t read_buf[8];
759 uint8_t *read_dst_final = dstc + dst_mid_off + mid_read;
760 uint8_t *read_dst = read_dst_final;
761 if (!direct_read) {
762 read_size = MIN(read_size, sizeof(read_buf));
763 read_dst = (uint8_t *) read_buf;
764 }
765 rc = esp_rom_spiflash_read(src + src_mid_off + mid_read,
766 (uint32_t *) read_dst, read_size);
767 if (rc != ESP_ROM_SPIFLASH_RESULT_OK) {
768 goto out;
769 }
770 mid_remaining -= read_size;
771 mid_read += read_size;
772 if (!direct_read) {
773 spi_flash_guard_end();
774 memcpy(read_dst_final, read_buf, read_size);
775 spi_flash_guard_start();
776 } else if (mid_remaining > 0) {
777 /* Drop guard momentarily, allows other tasks to preempt */
778 spi_flash_guard_end();
779 spi_flash_guard_start();
780 }
781 }
782 COUNTER_ADD_BYTES(read, mid_size);
783 /*
784 * If offsets in src and dst are different, perform an in-place shift
785 * to put destination data into its final position.
786 * Note that the shift can be left (src_mid_off < dst_mid_off) or right.
787 */
788 if (src_mid_off != dst_mid_off) {
789 if (!direct_read) {
790 spi_flash_guard_end();
791 }
792 memmove(dstc + src_mid_off, dstc + dst_mid_off, mid_size);
793 if (!direct_read) {
794 spi_flash_guard_start();
795 }
796 }
797 }
798 if (pad_left_size > 0) {
799 uint32_t t;
800 rc = esp_rom_spiflash_read(pad_left_src, &t, 4);
801 if (rc != ESP_ROM_SPIFLASH_RESULT_OK) {
802 goto out;
803 }
804 COUNTER_ADD_BYTES(read, 4);
805 if (!direct_read) {
806 spi_flash_guard_end();
807 }
808 memcpy(dstc, ((uint8_t *) &t) + (4 - pad_left_size), pad_left_size);
809 if (!direct_read) {
810 spi_flash_guard_start();
811 }
812 }
813 if (pad_right_size > 0) {
814 uint32_t t[2];
815 int32_t read_size = (pad_right_size <= 4 ? 4 : 8);
816 rc = esp_rom_spiflash_read(pad_right_src, t, read_size);
817 if (rc != ESP_ROM_SPIFLASH_RESULT_OK) {
818 goto out;
819 }
820 COUNTER_ADD_BYTES(read, read_size);
821 if (!direct_read) {
822 spi_flash_guard_end();
823 }
824 memcpy(dstc + pad_right_off, t, pad_right_size);
825 if (!direct_read) {
826 spi_flash_guard_start();
827 }
828 }
829 out:
830 spi_flash_guard_end();
831 COUNTER_STOP(read);
832 return spi_flash_translate_rc(rc);
833 }
834 #endif
835
spi_flash_read_encrypted(size_t src,void * dstv,size_t size)836 esp_err_t IRAM_ATTR spi_flash_read_encrypted(size_t src, void *dstv, size_t size)
837 {
838 if (src + size > g_rom_flashchip.chip_size) {
839 return ESP_ERR_INVALID_SIZE;
840 }
841 if (size == 0) {
842 return ESP_OK;
843 }
844
845 esp_err_t err;
846 const uint8_t *map;
847 spi_flash_mmap_handle_t map_handle;
848 size_t map_src = src & ~(SPI_FLASH_MMU_PAGE_SIZE - 1);
849 size_t map_size = size + (src - map_src);
850
851 err = spi_flash_mmap(map_src, map_size, SPI_FLASH_MMAP_DATA, (const void **)&map, &map_handle);
852 if (err != ESP_OK) {
853 return err;
854 }
855 memcpy(dstv, map + (src - map_src), size);
856 spi_flash_munmap(map_handle);
857 return err;
858 }
859
860
spi_flash_translate_rc(esp_rom_spiflash_result_t rc)861 static esp_err_t IRAM_ATTR spi_flash_translate_rc(esp_rom_spiflash_result_t rc)
862 {
863 switch (rc) {
864 case ESP_ROM_SPIFLASH_RESULT_OK:
865 return ESP_OK;
866 case ESP_ROM_SPIFLASH_RESULT_TIMEOUT:
867 return ESP_ERR_FLASH_OP_TIMEOUT;
868 case ESP_ROM_SPIFLASH_RESULT_ERR:
869 default:
870 return ESP_ERR_FLASH_OP_FAIL;
871 }
872 }
873
874 #if CONFIG_SPI_FLASH_ENABLE_COUNTERS
875
dump_counter(spi_flash_counter_t * counter,const char * name)876 static inline void dump_counter(spi_flash_counter_t *counter, const char *name)
877 {
878 ESP_LOGI(TAG, "%s count=%8d time=%8dus bytes=%8d\n", name,
879 counter->count, counter->time, counter->bytes);
880 }
881
spi_flash_get_counters(void)882 const spi_flash_counters_t *spi_flash_get_counters(void)
883 {
884 return &s_flash_stats;
885 }
886
spi_flash_reset_counters(void)887 void spi_flash_reset_counters(void)
888 {
889 memset(&s_flash_stats, 0, sizeof(s_flash_stats));
890 }
891
spi_flash_dump_counters(void)892 void spi_flash_dump_counters(void)
893 {
894 dump_counter(&s_flash_stats.read, "read ");
895 dump_counter(&s_flash_stats.write, "write");
896 dump_counter(&s_flash_stats.erase, "erase");
897 }
898
899 #endif //CONFIG_SPI_FLASH_ENABLE_COUNTERS
900
901 #if CONFIG_SPI_FLASH_USE_LEGACY_IMPL && !CONFIG_IDF_TARGET_ESP32
902 // TODO esp32s2: Remove once ESP32-S2 & later chips has new SPI Flash API support
903 esp_flash_t *esp_flash_default_chip = NULL;
904 #endif
905
