1 /*
2 * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 */
8
9 #ifndef _ASM_ARC_ATOMIC_H
10 #define _ASM_ARC_ATOMIC_H
11
12 #ifndef __ASSEMBLY__
13
14 #include <linux/types.h>
15 #include <linux/compiler.h>
16 #include <asm/cmpxchg.h>
17 #include <asm/barrier.h>
18 #include <asm/smp.h>
19
20 #define ATOMIC_INIT(i) { (i) }
21
22 #ifndef CONFIG_ARC_PLAT_EZNPS
23
24 #define atomic_read(v) READ_ONCE((v)->counter)
25
26 #ifdef CONFIG_ARC_HAS_LLSC
27
28 #define atomic_set(v, i) WRITE_ONCE(((v)->counter), (i))
29
30 #define ATOMIC_OP(op, c_op, asm_op) \
31 static inline void atomic_##op(int i, atomic_t *v) \
32 { \
33 unsigned int val; \
34 \
35 __asm__ __volatile__( \
36 "1: llock %[val], [%[ctr]] \n" \
37 " " #asm_op " %[val], %[val], %[i] \n" \
38 " scond %[val], [%[ctr]] \n" \
39 " bnz 1b \n" \
40 : [val] "=&r" (val) /* Early clobber to prevent reg reuse */ \
41 : [ctr] "r" (&v->counter), /* Not "m": llock only supports reg direct addr mode */ \
42 [i] "ir" (i) \
43 : "cc"); \
44 } \
45
46 #define ATOMIC_OP_RETURN(op, c_op, asm_op) \
47 static inline int atomic_##op##_return(int i, atomic_t *v) \
48 { \
49 unsigned int val; \
50 \
51 /* \
52 * Explicit full memory barrier needed before/after as \
53 * LLOCK/SCOND thmeselves don't provide any such semantics \
54 */ \
55 smp_mb(); \
56 \
57 __asm__ __volatile__( \
58 "1: llock %[val], [%[ctr]] \n" \
59 " " #asm_op " %[val], %[val], %[i] \n" \
60 " scond %[val], [%[ctr]] \n" \
61 " bnz 1b \n" \
62 : [val] "=&r" (val) \
63 : [ctr] "r" (&v->counter), \
64 [i] "ir" (i) \
65 : "cc"); \
66 \
67 smp_mb(); \
68 \
69 return val; \
70 }
71
72 #define ATOMIC_FETCH_OP(op, c_op, asm_op) \
73 static inline int atomic_fetch_##op(int i, atomic_t *v) \
74 { \
75 unsigned int val, orig; \
76 \
77 /* \
78 * Explicit full memory barrier needed before/after as \
79 * LLOCK/SCOND thmeselves don't provide any such semantics \
80 */ \
81 smp_mb(); \
82 \
83 __asm__ __volatile__( \
84 "1: llock %[orig], [%[ctr]] \n" \
85 " " #asm_op " %[val], %[orig], %[i] \n" \
86 " scond %[val], [%[ctr]] \n" \
87 " bnz 1b \n" \
88 : [val] "=&r" (val), \
89 [orig] "=&r" (orig) \
90 : [ctr] "r" (&v->counter), \
91 [i] "ir" (i) \
92 : "cc"); \
93 \
94 smp_mb(); \
95 \
96 return orig; \
97 }
98
99 #else /* !CONFIG_ARC_HAS_LLSC */
100
101 #ifndef CONFIG_SMP
102
103 /* violating atomic_xxx API locking protocol in UP for optimization sake */
104 #define atomic_set(v, i) WRITE_ONCE(((v)->counter), (i))
105
106 #else
107
atomic_set(atomic_t * v,int i)108 static inline void atomic_set(atomic_t *v, int i)
109 {
110 /*
111 * Independent of hardware support, all of the atomic_xxx() APIs need
112 * to follow the same locking rules to make sure that a "hardware"
113 * atomic insn (e.g. LD) doesn't clobber an "emulated" atomic insn
114 * sequence
115 *
116 * Thus atomic_set() despite being 1 insn (and seemingly atomic)
117 * requires the locking.
118 */
119 unsigned long flags;
120
121 atomic_ops_lock(flags);
122 WRITE_ONCE(v->counter, i);
123 atomic_ops_unlock(flags);
124 }
125
126 #define atomic_set_release(v, i) atomic_set((v), (i))
127
128 #endif
129
130 /*
131 * Non hardware assisted Atomic-R-M-W
132 * Locking would change to irq-disabling only (UP) and spinlocks (SMP)
133 */
134
135 #define ATOMIC_OP(op, c_op, asm_op) \
136 static inline void atomic_##op(int i, atomic_t *v) \
137 { \
138 unsigned long flags; \
139 \
140 atomic_ops_lock(flags); \
141 v->counter c_op i; \
142 atomic_ops_unlock(flags); \
143 }
144
145 #define ATOMIC_OP_RETURN(op, c_op, asm_op) \
146 static inline int atomic_##op##_return(int i, atomic_t *v) \
147 { \
148 unsigned long flags; \
149 unsigned long temp; \
150 \
151 /* \
152 * spin lock/unlock provides the needed smp_mb() before/after \
153 */ \
154 atomic_ops_lock(flags); \
155 temp = v->counter; \
156 temp c_op i; \
157 v->counter = temp; \
158 atomic_ops_unlock(flags); \
159 \
160 return temp; \
161 }
162
163 #define ATOMIC_FETCH_OP(op, c_op, asm_op) \
164 static inline int atomic_fetch_##op(int i, atomic_t *v) \
165 { \
166 unsigned long flags; \
167 unsigned long orig; \
168 \
169 /* \
170 * spin lock/unlock provides the needed smp_mb() before/after \
171 */ \
172 atomic_ops_lock(flags); \
173 orig = v->counter; \
174 v->counter c_op i; \
175 atomic_ops_unlock(flags); \
176 \
177 return orig; \
178 }
179
180 #endif /* !CONFIG_ARC_HAS_LLSC */
181
182 #define ATOMIC_OPS(op, c_op, asm_op) \
183 ATOMIC_OP(op, c_op, asm_op) \
184 ATOMIC_OP_RETURN(op, c_op, asm_op) \
185 ATOMIC_FETCH_OP(op, c_op, asm_op)
186
187 ATOMIC_OPS(add, +=, add)
188 ATOMIC_OPS(sub, -=, sub)
189
190 #define atomic_andnot atomic_andnot
191 #define atomic_fetch_andnot atomic_fetch_andnot
192
193 #undef ATOMIC_OPS
194 #define ATOMIC_OPS(op, c_op, asm_op) \
195 ATOMIC_OP(op, c_op, asm_op) \
196 ATOMIC_FETCH_OP(op, c_op, asm_op)
197
198 ATOMIC_OPS(and, &=, and)
199 ATOMIC_OPS(andnot, &= ~, bic)
200 ATOMIC_OPS(or, |=, or)
201 ATOMIC_OPS(xor, ^=, xor)
202
203 #else /* CONFIG_ARC_PLAT_EZNPS */
204
205 static inline int atomic_read(const atomic_t *v)
206 {
207 int temp;
208
209 __asm__ __volatile__(
210 " ld.di %0, [%1]"
211 : "=r"(temp)
212 : "r"(&v->counter)
213 : "memory");
214 return temp;
215 }
216
217 static inline void atomic_set(atomic_t *v, int i)
218 {
219 __asm__ __volatile__(
220 " st.di %0,[%1]"
221 :
222 : "r"(i), "r"(&v->counter)
223 : "memory");
224 }
225
226 #define ATOMIC_OP(op, c_op, asm_op) \
227 static inline void atomic_##op(int i, atomic_t *v) \
228 { \
229 __asm__ __volatile__( \
230 " mov r2, %0\n" \
231 " mov r3, %1\n" \
232 " .word %2\n" \
233 : \
234 : "r"(i), "r"(&v->counter), "i"(asm_op) \
235 : "r2", "r3", "memory"); \
236 } \
237
238 #define ATOMIC_OP_RETURN(op, c_op, asm_op) \
239 static inline int atomic_##op##_return(int i, atomic_t *v) \
240 { \
241 unsigned int temp = i; \
242 \
243 /* Explicit full memory barrier needed before/after */ \
244 smp_mb(); \
245 \
246 __asm__ __volatile__( \
247 " mov r2, %0\n" \
248 " mov r3, %1\n" \
249 " .word %2\n" \
250 " mov %0, r2" \
251 : "+r"(temp) \
252 : "r"(&v->counter), "i"(asm_op) \
253 : "r2", "r3", "memory"); \
254 \
255 smp_mb(); \
256 \
257 temp c_op i; \
258 \
259 return temp; \
260 }
261
262 #define ATOMIC_FETCH_OP(op, c_op, asm_op) \
263 static inline int atomic_fetch_##op(int i, atomic_t *v) \
264 { \
265 unsigned int temp = i; \
266 \
267 /* Explicit full memory barrier needed before/after */ \
268 smp_mb(); \
269 \
270 __asm__ __volatile__( \
271 " mov r2, %0\n" \
272 " mov r3, %1\n" \
273 " .word %2\n" \
274 " mov %0, r2" \
275 : "+r"(temp) \
276 : "r"(&v->counter), "i"(asm_op) \
277 : "r2", "r3", "memory"); \
278 \
279 smp_mb(); \
280 \
281 return temp; \
282 }
283
284 #define ATOMIC_OPS(op, c_op, asm_op) \
285 ATOMIC_OP(op, c_op, asm_op) \
286 ATOMIC_OP_RETURN(op, c_op, asm_op) \
287 ATOMIC_FETCH_OP(op, c_op, asm_op)
288
289 ATOMIC_OPS(add, +=, CTOP_INST_AADD_DI_R2_R2_R3)
290 #define atomic_sub(i, v) atomic_add(-(i), (v))
291 #define atomic_sub_return(i, v) atomic_add_return(-(i), (v))
292 #define atomic_fetch_sub(i, v) atomic_fetch_add(-(i), (v))
293
294 #undef ATOMIC_OPS
295 #define ATOMIC_OPS(op, c_op, asm_op) \
296 ATOMIC_OP(op, c_op, asm_op) \
297 ATOMIC_FETCH_OP(op, c_op, asm_op)
298
299 ATOMIC_OPS(and, &=, CTOP_INST_AAND_DI_R2_R2_R3)
300 ATOMIC_OPS(or, |=, CTOP_INST_AOR_DI_R2_R2_R3)
301 ATOMIC_OPS(xor, ^=, CTOP_INST_AXOR_DI_R2_R2_R3)
302
303 #endif /* CONFIG_ARC_PLAT_EZNPS */
304
305 #undef ATOMIC_OPS
306 #undef ATOMIC_FETCH_OP
307 #undef ATOMIC_OP_RETURN
308 #undef ATOMIC_OP
309
310 #ifdef CONFIG_GENERIC_ATOMIC64
311
312 #include <asm-generic/atomic64.h>
313
314 #else /* Kconfig ensures this is only enabled with needed h/w assist */
315
316 /*
317 * ARCv2 supports 64-bit exclusive load (LLOCKD) / store (SCONDD)
318 * - The address HAS to be 64-bit aligned
319 * - There are 2 semantics involved here:
320 * = exclusive implies no interim update between load/store to same addr
321 * = both words are observed/updated together: this is guaranteed even
322 * for regular 64-bit load (LDD) / store (STD). Thus atomic64_set()
323 * is NOT required to use LLOCKD+SCONDD, STD suffices
324 */
325
326 typedef struct {
327 aligned_u64 counter;
328 } atomic64_t;
329
330 #define ATOMIC64_INIT(a) { (a) }
331
332 static inline long long atomic64_read(const atomic64_t *v)
333 {
334 unsigned long long val;
335
336 __asm__ __volatile__(
337 " ldd %0, [%1] \n"
338 : "=r"(val)
339 : "r"(&v->counter));
340
341 return val;
342 }
343
344 static inline void atomic64_set(atomic64_t *v, long long a)
345 {
346 /*
347 * This could have been a simple assignment in "C" but would need
348 * explicit volatile. Otherwise gcc optimizers could elide the store
349 * which borked atomic64 self-test
350 * In the inline asm version, memory clobber needed for exact same
351 * reason, to tell gcc about the store.
352 *
353 * This however is not needed for sibling atomic64_add() etc since both
354 * load/store are explicitly done in inline asm. As long as API is used
355 * for each access, gcc has no way to optimize away any load/store
356 */
357 __asm__ __volatile__(
358 " std %0, [%1] \n"
359 :
360 : "r"(a), "r"(&v->counter)
361 : "memory");
362 }
363
364 #define ATOMIC64_OP(op, op1, op2) \
365 static inline void atomic64_##op(long long a, atomic64_t *v) \
366 { \
367 unsigned long long val; \
368 \
369 __asm__ __volatile__( \
370 "1: \n" \
371 " llockd %0, [%1] \n" \
372 " " #op1 " %L0, %L0, %L2 \n" \
373 " " #op2 " %H0, %H0, %H2 \n" \
374 " scondd %0, [%1] \n" \
375 " bnz 1b \n" \
376 : "=&r"(val) \
377 : "r"(&v->counter), "ir"(a) \
378 : "cc"); \
379 } \
380
381 #define ATOMIC64_OP_RETURN(op, op1, op2) \
382 static inline long long atomic64_##op##_return(long long a, atomic64_t *v) \
383 { \
384 unsigned long long val; \
385 \
386 smp_mb(); \
387 \
388 __asm__ __volatile__( \
389 "1: \n" \
390 " llockd %0, [%1] \n" \
391 " " #op1 " %L0, %L0, %L2 \n" \
392 " " #op2 " %H0, %H0, %H2 \n" \
393 " scondd %0, [%1] \n" \
394 " bnz 1b \n" \
395 : [val] "=&r"(val) \
396 : "r"(&v->counter), "ir"(a) \
397 : "cc"); /* memory clobber comes from smp_mb() */ \
398 \
399 smp_mb(); \
400 \
401 return val; \
402 }
403
404 #define ATOMIC64_FETCH_OP(op, op1, op2) \
405 static inline long long atomic64_fetch_##op(long long a, atomic64_t *v) \
406 { \
407 unsigned long long val, orig; \
408 \
409 smp_mb(); \
410 \
411 __asm__ __volatile__( \
412 "1: \n" \
413 " llockd %0, [%2] \n" \
414 " " #op1 " %L1, %L0, %L3 \n" \
415 " " #op2 " %H1, %H0, %H3 \n" \
416 " scondd %1, [%2] \n" \
417 " bnz 1b \n" \
418 : "=&r"(orig), "=&r"(val) \
419 : "r"(&v->counter), "ir"(a) \
420 : "cc"); /* memory clobber comes from smp_mb() */ \
421 \
422 smp_mb(); \
423 \
424 return orig; \
425 }
426
427 #define ATOMIC64_OPS(op, op1, op2) \
428 ATOMIC64_OP(op, op1, op2) \
429 ATOMIC64_OP_RETURN(op, op1, op2) \
430 ATOMIC64_FETCH_OP(op, op1, op2)
431
432 #define atomic64_andnot atomic64_andnot
433 #define atomic64_fetch_andnot atomic64_fetch_andnot
434
435 ATOMIC64_OPS(add, add.f, adc)
436 ATOMIC64_OPS(sub, sub.f, sbc)
437 ATOMIC64_OPS(and, and, and)
438 ATOMIC64_OPS(andnot, bic, bic)
439 ATOMIC64_OPS(or, or, or)
440 ATOMIC64_OPS(xor, xor, xor)
441
442 #undef ATOMIC64_OPS
443 #undef ATOMIC64_FETCH_OP
444 #undef ATOMIC64_OP_RETURN
445 #undef ATOMIC64_OP
446
447 static inline long long
448 atomic64_cmpxchg(atomic64_t *ptr, long long expected, long long new)
449 {
450 long long prev;
451
452 smp_mb();
453
454 __asm__ __volatile__(
455 "1: llockd %0, [%1] \n"
456 " brne %L0, %L2, 2f \n"
457 " brne %H0, %H2, 2f \n"
458 " scondd %3, [%1] \n"
459 " bnz 1b \n"
460 "2: \n"
461 : "=&r"(prev)
462 : "r"(ptr), "ir"(expected), "r"(new)
463 : "cc"); /* memory clobber comes from smp_mb() */
464
465 smp_mb();
466
467 return prev;
468 }
469
470 static inline long long atomic64_xchg(atomic64_t *ptr, long long new)
471 {
472 long long prev;
473
474 smp_mb();
475
476 __asm__ __volatile__(
477 "1: llockd %0, [%1] \n"
478 " scondd %2, [%1] \n"
479 " bnz 1b \n"
480 "2: \n"
481 : "=&r"(prev)
482 : "r"(ptr), "r"(new)
483 : "cc"); /* memory clobber comes from smp_mb() */
484
485 smp_mb();
486
487 return prev;
488 }
489
490 /**
491 * atomic64_dec_if_positive - decrement by 1 if old value positive
492 * @v: pointer of type atomic64_t
493 *
494 * The function returns the old value of *v minus 1, even if
495 * the atomic variable, v, was not decremented.
496 */
497
498 static inline long long atomic64_dec_if_positive(atomic64_t *v)
499 {
500 long long val;
501
502 smp_mb();
503
504 __asm__ __volatile__(
505 "1: llockd %0, [%1] \n"
506 " sub.f %L0, %L0, 1 # w0 - 1, set C on borrow\n"
507 " sub.c %H0, %H0, 1 # if C set, w1 - 1\n"
508 " brlt %H0, 0, 2f \n"
509 " scondd %0, [%1] \n"
510 " bnz 1b \n"
511 "2: \n"
512 : "=&r"(val)
513 : "r"(&v->counter)
514 : "cc"); /* memory clobber comes from smp_mb() */
515
516 smp_mb();
517
518 return val;
519 }
520 #define atomic64_dec_if_positive atomic64_dec_if_positive
521
522 /**
523 * atomic64_fetch_add_unless - add unless the number is a given value
524 * @v: pointer of type atomic64_t
525 * @a: the amount to add to v...
526 * @u: ...unless v is equal to u.
527 *
528 * Atomically adds @a to @v, if it was not @u.
529 * Returns the old value of @v
530 */
531 static inline long long atomic64_fetch_add_unless(atomic64_t *v, long long a,
532 long long u)
533 {
534 long long old, temp;
535
536 smp_mb();
537
538 __asm__ __volatile__(
539 "1: llockd %0, [%2] \n"
540 " brne %L0, %L4, 2f # continue to add since v != u \n"
541 " breq.d %H0, %H4, 3f # return since v == u \n"
542 "2: \n"
543 " add.f %L1, %L0, %L3 \n"
544 " adc %H1, %H0, %H3 \n"
545 " scondd %1, [%2] \n"
546 " bnz 1b \n"
547 "3: \n"
548 : "=&r"(old), "=&r" (temp)
549 : "r"(&v->counter), "r"(a), "r"(u)
550 : "cc"); /* memory clobber comes from smp_mb() */
551
552 smp_mb();
553
554 return old;
555 }
556 #define atomic64_fetch_add_unless atomic64_fetch_add_unless
557
558 #endif /* !CONFIG_GENERIC_ATOMIC64 */
559
560 #endif /* !__ASSEMBLY__ */
561
562 #endif
563
