1 /*
2 * Copyright (c) 2024 Raspberry Pi (Trading) Ltd.
3 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6
7 #ifndef _HARDWARE_SYNC_SPIN_LOCK_H
8 #define _HARDWARE_SYNC_SPIN_LOCK_H
9
10 #include "pico.h"
11 #include "hardware/sync.h"
12
13 // PICO_CONFIG: PICO_USE_SW_SPIN_LOCKS, Use software implementation for spin locks, type=bool, default=1 on RP2350 due to errata, group=hardware_sync
14 #ifndef PICO_USE_SW_SPIN_LOCKS
15 #if PICO_RP2350
16 #define PICO_USE_SW_SPIN_LOCKS 1
17 #endif
18 #endif
19
20 // PICO_CONFIG: PICO_SPINLOCK_ID_IRQ, Spinlock ID for IRQ protection, min=0, max=31, default=9, group=hardware_sync
21 #ifndef PICO_SPINLOCK_ID_IRQ
22 #define PICO_SPINLOCK_ID_IRQ 9
23 #endif
24
25 // PICO_CONFIG: PICO_SPINLOCK_ID_TIMER, Spinlock ID for Timer protection, min=0, max=31, default=10, group=hardware_sync
26 #ifndef PICO_SPINLOCK_ID_TIMER
27 #define PICO_SPINLOCK_ID_TIMER 10
28 #endif
29
30 // PICO_CONFIG: PICO_SPINLOCK_ID_HARDWARE_CLAIM, Spinlock ID for Hardware claim protection, min=0, max=31, default=11, group=hardware_sync
31 #ifndef PICO_SPINLOCK_ID_HARDWARE_CLAIM
32 #define PICO_SPINLOCK_ID_HARDWARE_CLAIM 11
33 #endif
34
35 // PICO_CONFIG: PICO_SPINLOCK_ID_RAND, Spinlock ID for Random Number Generator, min=0, max=31, default=12, group=hardware_sync
36 #ifndef PICO_SPINLOCK_ID_RAND
37 #define PICO_SPINLOCK_ID_RAND 12
38 #endif
39
40 // PICO_CONFIG: PICO_SPINLOCK_ID_ATOMIC, Spinlock ID for atomics, min=0, max=31, default=13, group=hardware_sync
41 #ifndef PICO_SPINLOCK_ID_ATOMIC
42 #define PICO_SPINLOCK_ID_ATOMIC 13
43 #endif
44
45 // PICO_CONFIG: PICO_SPINLOCK_ID_OS1, First Spinlock ID reserved for use by low level OS style software, min=0, max=31, default=14, group=hardware_sync
46 #ifndef PICO_SPINLOCK_ID_OS1
47 #define PICO_SPINLOCK_ID_OS1 14
48 #endif
49
50 // PICO_CONFIG: PICO_SPINLOCK_ID_OS2, Second Spinlock ID reserved for use by low level OS style software, min=0, max=31, default=15, group=hardware_sync
51 #ifndef PICO_SPINLOCK_ID_OS2
52 #define PICO_SPINLOCK_ID_OS2 15
53 #endif
54
55 // PICO_CONFIG: PICO_SPINLOCK_ID_STRIPED_FIRST, Lowest Spinlock ID in the 'striped' range, min=0, max=31, default=16, group=hardware_sync
56 #ifndef PICO_SPINLOCK_ID_STRIPED_FIRST
57 #define PICO_SPINLOCK_ID_STRIPED_FIRST 16
58 #endif
59
60 // PICO_CONFIG: PICO_SPINLOCK_ID_STRIPED_LAST, Highest Spinlock ID in the 'striped' range, min=0, max=31, default=23, group=hardware_sync
61 #ifndef PICO_SPINLOCK_ID_STRIPED_LAST
62 #define PICO_SPINLOCK_ID_STRIPED_LAST 23
63 #endif
64
65 // PICO_CONFIG: PICO_SPINLOCK_ID_CLAIM_FREE_FIRST, Lowest Spinlock ID in the 'claim free' range, min=0, max=31, default=24, group=hardware_sync
66 #ifndef PICO_SPINLOCK_ID_CLAIM_FREE_FIRST
67 #define PICO_SPINLOCK_ID_CLAIM_FREE_FIRST 24
68 #endif
69
70 #ifdef PICO_SPINLOCK_ID_CLAIM_FREE_END
71 #warning PICO_SPINLOCK_ID_CLAIM_FREE_END has been renamed to PICO_SPINLOCK_ID_CLAIM_FREE_LAST
72 #endif
73
74 // PICO_CONFIG: PICO_SPINLOCK_ID_CLAIM_FREE_LAST, Highest Spinlock ID in the 'claim free' range, min=0, max=31, default=31, group=hardware_sync
75 #ifndef PICO_SPINLOCK_ID_CLAIM_FREE_LAST
76 #define PICO_SPINLOCK_ID_CLAIM_FREE_LAST 31
77 #endif
78
79 /** \brief A spin lock identifier
80 * \ingroup hardware_sync
81 */
82 #if !PICO_USE_SW_SPIN_LOCKS
83 // Hardware lock flag in SIO:
84 typedef io_rw_32 spin_lock_t;
85 #else
86 #ifndef SW_SPIN_LOCK_TYPE
87 // Byte flag in memory:
88 #define SW_SPIN_LOCK_TYPE volatile uint8_t
89 #endif
90 typedef SW_SPIN_LOCK_TYPE spin_lock_t;
91 #endif
92
93 #if PICO_USE_SW_SPIN_LOCKS
94 #ifndef SW_SPIN_LOCK_INSTANCE
95 #define SW_SPIN_LOCK_INSTANCE(lock_num) ({ \
96 extern spin_lock_t _sw_spin_locks[NUM_SPIN_LOCKS]; \
97 &_sw_spin_locks[lock_num]; \
98 })
99 #endif
100
101 #ifndef SW_SPIN_LOCK_NUM
102 #define SW_SPIN_LOCK_NUM(lock) ({ \
103 extern spin_lock_t _sw_spin_locks[NUM_SPIN_LOCKS]; \
104 (lock) - _sw_spin_locks; \
105 })
106 #endif
107
108 #ifndef SW_SPIN_LOCK_IS_LOCKED
109 #define SW_SPIN_LOCK_IS_LOCKED(lock) ((bool) *(lock))
110 #endif
111
112 #ifndef SW_SPIN_LOCK_LOCK
113 #if __ARM_ARCH_8M_MAIN__
114 #define SW_SPIN_LOCK_LOCK(lock) ({ \
115 uint32_t _tmp0, _tmp1; \
116 pico_default_asm_volatile ( \
117 "1:\n" \
118 "ldaexb %1, [%2]\n" \
119 "movs %0, #1\n" /* fill dependency slot */ \
120 "cmp %1, #0\n" \
121 /* Immediately retry if lock is seen to be taken */ \
122 "bne 1b\n" \
123 /* Attempt to claim */ \
124 "strexb %1, %0, [%2]\n" \
125 "cmp %1, #0\n" \
126 /* Claim failed due to intervening write, so retry */ \
127 "bne 1b\n" \
128 : "=&r" (_tmp0), "=&r" (_tmp1) : "r" (lock) \
129 ); \
130 __mem_fence_acquire(); \
131 })
132 #elif __riscv && (defined(__riscv_a) || defined(__riscv_zaamo))
133 #define SW_SPIN_LOCK_LOCK(lock) ({ \
134 uint32_t _tmp0, _tmp1; \
135 pico_default_asm_volatile ( \
136 /* Get word address, and bit mask for LSB of the */ \
137 /* correct byte within that word -- note shamt is modulo xlen: */ \
138 "slli %1, %0, 3\n" \
139 "bset %1, zero, %1\n" \
140 "andi %0, %0, -4\n" \
141 /* Repeatedly set the bit until we see that it was clear at the */ \
142 /* point we set it. A set from 0 -> 1 is a successful lock take. */ \
143 "1:" \
144 "amoor.w.aq %2, %1, (%0)\n" \
145 "and %2, %2, %1\n" \
146 "bnez %2, 1b\n" \
147 : "+r" (lock), "=r" (_tmp0), "=r" (_tmp1) \
148 ); \
149 __mem_fence_acquire(); \
150 })
151 #else
152 #error no SW_SPIN_TRY_LOCK available for PICO_USE_SW_SPIN_LOCK on this platform
153 #endif
154 #endif
155
156 #ifndef SW_SPIN_TRY_LOCK
157 #if __ARM_ARCH_8M_MAIN__
158 #define SW_SPIN_TRY_LOCK(lock) ({ \
159 uint32_t _tmp0, _tmp1; \
160 pico_default_asm_volatile ( \
161 "ldaexb %1, [%2]\n" \
162 "movs %0, #1\n" /* fill dependency slot */ \
163 "cmp %1, #0\n" \
164 /* Immediately give up if lock is seen to be taken */ \
165 "bne 1f\n" \
166 /* Otherwise attempt to claim, once. */ \
167 "strexb %1, %0, [%2]\n" \
168 "1:\n" \
169 : "=&r" (_tmp0), "=&r" (_tmp1) : "r" (lock) \
170 ); \
171 __mem_fence_acquire(); \
172 !_tmp1; \
173 })
174 #elif __riscv && (defined(__riscv_a) || defined(__riscv_zaamo))
175 #define SW_SPIN_TRY_LOCK(lock) ({ \
176 uint32_t _tmp0; \
177 pico_default_asm_volatile ( \
178 /* Get word address, and bit mask for LSB of the */ \
179 /* correct byte within that word -- note shamt is modulo xlen: */ \
180 "slli %1, %0, 3\n" \
181 "bset %1, zero, %1\n" \
182 "andi %0, %0, -4\n" \
183 /* Set the bit. If it was clear at the point we set it, then we took */ \
184 /* the lock. Otherwise the lock was already held, and we give up. */ \
185 "amoor.w.aq %0, %1, (%0)\n" \
186 "and %1, %1, %0\n" \
187 : "+r" (lock), "=r" (_tmp0) \
188 ); \
189 __mem_fence_acquire(); \
190 !_tmp0; \
191 })
192 #else
193 #error no SW_SPIN_TRY_LOCK available for PICO_USE_SW_SPIN_LOCK on this platform
194 #endif
195 #endif
196
197 #ifndef SW_SPIN_LOCK_UNLOCK
198 #if __ARM_ARCH_8M_MAIN__
199 #define SW_SPIN_LOCK_UNLOCK(lock) ({ \
200 /* Release-ordered store is available: use instead of separate fence */ \
201 uint32_t zero = 0; \
202 pico_default_asm_volatile( \
203 "stlb %0, [%1]\n" \
204 : : "r" (zero), "r" (lock) \
205 ); \
206 })
207 #elif __riscv
208 #define SW_SPIN_LOCK_UNLOCK(lock) ({ \
209 __mem_fence_release(); \
210 *(lock) = 0; /* write to spinlock register (release lock) */ \
211 })
212 #else
213 #error no SW_SPIN_TRY_LOCK available for PICO_USE_SW_SPIN_LOCK on this platform
214 #endif
215 #endif
216
217 #endif
218
219 /*! \brief Get HW Spinlock instance from number
220 * \ingroup hardware_sync
221 *
222 * \param lock_num Spinlock ID
223 * \return The spinlock instance
224 */
spin_lock_instance(uint lock_num)225 __force_inline static spin_lock_t *spin_lock_instance(uint lock_num) {
226 invalid_params_if(HARDWARE_SYNC, lock_num >= NUM_SPIN_LOCKS);
227 #if PICO_USE_SW_SPIN_LOCKS
228 return SW_SPIN_LOCK_INSTANCE(lock_num);
229 #else
230 return (spin_lock_t *) (SIO_BASE + SIO_SPINLOCK0_OFFSET + lock_num * 4);
231 #endif
232 }
233
234 /*! \brief Get HW Spinlock number from instance
235 * \ingroup hardware_sync
236 *
237 * \param lock The Spinlock instance
238 * \return The Spinlock ID
239 */
spin_lock_get_num(spin_lock_t * lock)240 __force_inline static uint spin_lock_get_num(spin_lock_t *lock) {
241 #if PICO_USE_SW_SPIN_LOCKS
242 uint lock_num = SW_SPIN_LOCK_NUM(lock);
243 invalid_params_if(HARDWARE_SYNC, lock_num >= (uint)NUM_SPIN_LOCKS);
244 return lock_num;
245 #else
246 invalid_params_if(HARDWARE_SYNC, (uint) lock < SIO_BASE + SIO_SPINLOCK0_OFFSET ||
247 (uint) lock >= NUM_SPIN_LOCKS * sizeof(spin_lock_t) + SIO_BASE + SIO_SPINLOCK0_OFFSET ||
248 ((uint) lock - SIO_BASE + SIO_SPINLOCK0_OFFSET) % sizeof(spin_lock_t) != 0);
249 return (uint) (lock - (spin_lock_t *) (SIO_BASE + SIO_SPINLOCK0_OFFSET));
250 #endif
251 }
252
253 /*! \brief Acquire a spin lock without disabling interrupts (hence unsafe)
254 * \ingroup hardware_sync
255 *
256 * \param lock Spinlock instance
257 */
spin_lock_unsafe_blocking(spin_lock_t * lock)258 __force_inline static void spin_lock_unsafe_blocking(spin_lock_t *lock) {
259 // Note we don't do a wfe or anything, because by convention these spin_locks are VERY SHORT LIVED and NEVER BLOCK and run
260 // with INTERRUPTS disabled (to ensure that)... therefore nothing on our core could be blocking us, so we just need to wait on another core
261 // anyway which should be finished soon
262 #if PICO_USE_SW_SPIN_LOCKS
263 SW_SPIN_LOCK_LOCK(lock);
264 #else
265 while (__builtin_expect(!*lock, 0)) { // read from spinlock register (tries to acquire the lock)
266 tight_loop_contents();
267 }
268 __mem_fence_acquire();
269 #endif
270 }
271
spin_try_lock_unsafe(spin_lock_t * lock)272 __force_inline static bool spin_try_lock_unsafe(spin_lock_t *lock) {
273 #if PICO_USE_SW_SPIN_LOCKS
274 return SW_SPIN_TRY_LOCK(lock);
275 #else
276 return *lock;
277 #endif
278 }
279 /*! \brief Release a spin lock without re-enabling interrupts
280 * \ingroup hardware_sync
281 *
282 * \param lock Spinlock instance
283 */
spin_unlock_unsafe(spin_lock_t * lock)284 __force_inline static void spin_unlock_unsafe(spin_lock_t *lock) {
285 #if PICO_USE_SW_SPIN_LOCKS
286 SW_SPIN_LOCK_UNLOCK(lock);
287 #else
288 __mem_fence_release();
289 *lock = 0; // write to spinlock register (release lock)
290 #endif
291 }
292
293 /*! \brief Acquire a spin lock safely
294 * \ingroup hardware_sync
295 *
296 * This function will disable interrupts prior to acquiring the spinlock
297 *
298 * \param lock Spinlock instance
299 * \return interrupt status to be used when unlocking, to restore to original state
300 */
spin_lock_blocking(spin_lock_t * lock)301 __force_inline static uint32_t spin_lock_blocking(spin_lock_t *lock) {
302 uint32_t save = save_and_disable_interrupts();
303 spin_lock_unsafe_blocking(lock);
304 return save;
305 }
306
307 /*! \brief Check to see if a spinlock is currently acquired elsewhere.
308 * \ingroup hardware_sync
309 *
310 * \param lock Spinlock instance
311 */
is_spin_locked(spin_lock_t * lock)312 inline static bool is_spin_locked(spin_lock_t *lock) {
313 #if PICO_USE_SW_SPIN_LOCKS
314 return SW_SPIN_LOCK_IS_LOCKED(lock);
315 #else
316 check_hw_size(spin_lock_t, 4);
317 uint lock_num = spin_lock_get_num(lock);
318 return 0 != (*(io_ro_32 *) (SIO_BASE + SIO_SPINLOCK_ST_OFFSET) & (1u << lock_num));
319 #endif
320 }
321
322 /*! \brief Release a spin lock safely
323 * \ingroup hardware_sync
324 *
325 * This function will re-enable interrupts according to the parameters.
326 *
327 * \param lock Spinlock instance
328 * \param saved_irq Return value from the \ref spin_lock_blocking() function.
329 *
330 * \sa spin_lock_blocking()
331 */
spin_unlock(spin_lock_t * lock,uint32_t saved_irq)332 __force_inline static void spin_unlock(spin_lock_t *lock, uint32_t saved_irq) {
333 spin_unlock_unsafe(lock);
334 restore_interrupts_from_disabled(saved_irq);
335 }
336
337 /*! \brief Initialise a spin lock
338 * \ingroup hardware_sync
339 *
340 * The spin lock is initially unlocked
341 *
342 * \param lock_num The spin lock number
343 * \return The spin lock instance
344 */
345 spin_lock_t *spin_lock_init(uint lock_num);
346
347 /*! \brief Release all spin locks
348 * \ingroup hardware_sync
349 */
350 void spin_locks_reset(void);
351
352 #endif
353
