1 #include "hardware/xip_cache.h"
2 #include "hardware/structs/xip.h"
3 // For barrier macros:
4 #include "hardware/sync.h"
5
6 // Implementation-private constants (exporting these would create a compatibility headache as they
7 // don't exist on all platforms; all of these operations are exposed through APIs anyways)
8
9 #if !PICO_RP2040
10 typedef enum {
11 XIP_CACHE_INVALIDATE_BY_SET_WAY = 0,
12 XIP_CACHE_CLEAN_BY_SET_WAY = 1,
13 XIP_CACHE_INVALIDATE_BY_ADDRESS = 2,
14 XIP_CACHE_CLEAN_BY_ADDRESS = 3,
15 XIP_CACHE_PIN_AT_ADDRESS = 7,
16 XIP_CACHE_OP_MAX = 7
17 } cache_op_t;
18 #endif
19
20 // Used to ensure subsequent accesses observe the new state of the maintained cache lines
21 #define __post_maintenance_barrier() do {__dsb(); __isb();} while (0)
22
23 // All functions in this file are marked non-flash, even though they themselves may be executed
24 // safely from flash, because they are likely to be called during a flash programming operation
25 // (which makes flash execution momentarily unsafe)
26
check_xip_offset_range(uintptr_t start_offset,uintptr_t size_bytes)27 __force_inline static void check_xip_offset_range(uintptr_t start_offset, uintptr_t size_bytes) {
28 // We use offsets, not addresses, for consistency with the flash API. This means the range of
29 // valid inputs starts at 0.
30 (void)start_offset;
31 (void)size_bytes;
32 valid_params_if(HARDWARE_XIP_CACHE, start_offset <= XIP_CACHE_ADDRESS_SPACE_SIZE);
33 valid_params_if(HARDWARE_XIP_CACHE, start_offset + size_bytes <= XIP_CACHE_ADDRESS_SPACE_SIZE);
34 // Check for unsigned wrapping too:
35 valid_params_if(HARDWARE_XIP_CACHE, start_offset + size_bytes >= start_offset);
36 }
37
38 #if !PICO_RP2040
39 // Generic code for RP2350-style caches: apply a maintenance operation to a range of offsets
__no_inline_not_in_flash_func(xip_cache_maintain)40 static void __no_inline_not_in_flash_func(xip_cache_maintain)(uintptr_t start_offset, uintptr_t size_bytes, cache_op_t op) {
41 check_xip_offset_range(start_offset, size_bytes);
42 valid_params_if(HARDWARE_XIP_CACHE, (start_offset & (XIP_CACHE_LINE_SIZE - 1u)) == 0);
43 valid_params_if(HARDWARE_XIP_CACHE, (size_bytes & (XIP_CACHE_LINE_SIZE - 1u)) == 0);
44 valid_params_if(HARDWARE_XIP_CACHE, (uint)op <= (uint)XIP_CACHE_OP_MAX);
45
46 uintptr_t end = start_offset + size_bytes;
47 for (uintptr_t offset = start_offset; offset < end; offset += XIP_CACHE_LINE_SIZE) {
48 *(io_wo_8 *) (XIP_MAINTENANCE_BASE + offset + (uintptr_t)op) = 0;
49 }
50 __post_maintenance_barrier();
51 }
52 #endif
53
__no_inline_not_in_flash_func(xip_cache_invalidate_all)54 void __no_inline_not_in_flash_func(xip_cache_invalidate_all)(void) {
55 #if PICO_RP2040
56 xip_ctrl_hw->flush = 1;
57 // Read back to wait for completion
58 (void)xip_ctrl_hw->flush;
59 __post_maintenance_barrier();
60 #else
61 xip_cache_maintain(XIP_CACHE_ADDRESS_SPACE_SIZE - XIP_CACHE_SIZE, XIP_CACHE_SIZE, XIP_CACHE_INVALIDATE_BY_SET_WAY);
62 #endif
63 }
64
__no_inline_not_in_flash_func(xip_cache_invalidate_range)65 void __no_inline_not_in_flash_func(xip_cache_invalidate_range)(uintptr_t start_offset, uintptr_t size_bytes) {
66 #if PICO_RP2040
67 // Accsses are at intervals of one half cache line (so 4 bytes) because RP2040's cache has two
68 // valid flags per cache line, and we need to clear both.
69 check_xip_offset_range(start_offset, size_bytes);
70 valid_params_if(HARDWARE_XIP_CACHE, (start_offset & 3u) == 0);
71 valid_params_if(HARDWARE_XIP_CACHE, (size_bytes & 3u) == 0);
72
73 uintptr_t end = start_offset + size_bytes;
74 // On RP2040 you can invalidate a sector (half-line) by writing to its normal cached+allocating address
75 for (uintptr_t offset = start_offset; offset < end; offset += 4u) {
76 *(io_wo_32 *)(offset + XIP_BASE) = 0;
77 }
78 __post_maintenance_barrier();
79
80 #else
81
82 xip_cache_maintain(start_offset, size_bytes, XIP_CACHE_INVALIDATE_BY_ADDRESS);
83
84 #endif
85 }
86
87 #if !XIP_CACHE_IS_READ_ONLY
__no_inline_not_in_flash_func(xip_cache_clean_all)88 void __no_inline_not_in_flash_func(xip_cache_clean_all)(void) {
89 // Use addresses outside of the downstream QMI address range to work around RP2350-E11; this
90 // effectively performs a clean+invalidate (except being a no-op on pinned lines) due to the
91 // erroneous update of the tag. Consequently you will take a miss on the next access to the
92 // cleaned address.
93 xip_cache_maintain(XIP_END - XIP_BASE - XIP_CACHE_SIZE, XIP_CACHE_SIZE, XIP_CACHE_CLEAN_BY_SET_WAY);
94 }
95 #endif
96
97 #if !XIP_CACHE_IS_READ_ONLY
__no_inline_not_in_flash_func(xip_cache_clean_range)98 void __no_inline_not_in_flash_func(xip_cache_clean_range)(uintptr_t start_offset, uintptr_t size_bytes) {
99 xip_cache_maintain(start_offset, size_bytes, XIP_CACHE_CLEAN_BY_ADDRESS);
100 }
101 #endif
102
103 #if !PICO_RP2040
__no_inline_not_in_flash_func(xip_cache_pin_range)104 void __no_inline_not_in_flash_func(xip_cache_pin_range)(uintptr_t start_offset, uintptr_t size_bytes) {
105 valid_params_if(HARDWARE_XIP_CACHE, size_bytes <= XIP_CACHE_SIZE);
106 xip_cache_maintain(start_offset, size_bytes, XIP_CACHE_PIN_AT_ADDRESS);
107 }
108 #endif
109
110
