47 #define rseq_smp_store_release(p, v)					\  argument
50 	RSEQ_WRITE_ONCE(*p, v);						\
116 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)  in rseq_cmpeqv_storev()  argument
118 	RSEQ_INJECT_C(9)  in rseq_cmpeqv_storev()
131 		"cmpq %[v], %[expect]\n\t"  in rseq_cmpeqv_storev()
136 		"cmpq %[v], %[expect]\n\t"  in rseq_cmpeqv_storev()
140 		"movq %[newv], %[v]\n\t"  in rseq_cmpeqv_storev()
147 		  [v]			"m" (*v),  in rseq_cmpeqv_storev()
177  * Compare @v against @expectnot. When it does _not_ match, load @v
178  * into @load, and store the content of *@v + voffp into @v.
181 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,  in rseq_cmpnev_storeoffp_load()  argument
184 	RSEQ_INJECT_C(9)  in rseq_cmpnev_storeoffp_load()
197 		"movq %[v], %%rbx\n\t"  in rseq_cmpnev_storeoffp_load()
203 		"movq %[v], %%rbx\n\t"  in rseq_cmpnev_storeoffp_load()
211 		"movq %%rbx, %[v]\n\t"  in rseq_cmpnev_storeoffp_load()
219 		  [v]			"m" (*v),  in rseq_cmpnev_storeoffp_load()
250 int rseq_addv(intptr_t *v, intptr_t count, int cpu)  in rseq_addv()  argument
252 	RSEQ_INJECT_C(9)  in rseq_addv()
267 		"addq %[count], %[v]\n\t"  in rseq_addv()
275 		  [v]			"m" (*v),  in rseq_addv()
306 	RSEQ_INJECT_C(9)  in rseq_offset_deref_addv()
320 		/* get p+v */  in rseq_offset_deref_addv()
355 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,  in rseq_cmpeqv_trystorev_storev()  argument
359 	RSEQ_INJECT_C(9)  in rseq_cmpeqv_trystorev_storev()
372 		"cmpq %[v], %[expect]\n\t"  in rseq_cmpeqv_trystorev_storev()
377 		"cmpq %[v], %[expect]\n\t"  in rseq_cmpeqv_trystorev_storev()
384 		"movq %[newv], %[v]\n\t"  in rseq_cmpeqv_trystorev_storev()
395 		  [v]			"m" (*v),  in rseq_cmpeqv_trystorev_storev()
426 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,  in rseq_cmpeqv_trystorev_storev_release()  argument
430 	return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu);  in rseq_cmpeqv_trystorev_storev_release()
434 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,  in rseq_cmpeqv_cmpeqv_storev()  argument
438 	RSEQ_INJECT_C(9)  in rseq_cmpeqv_cmpeqv_storev()
452 		"cmpq %[v], %[expect]\n\t"  in rseq_cmpeqv_cmpeqv_storev()
460 		"cmpq %[v], %[expect]\n\t"  in rseq_cmpeqv_cmpeqv_storev()
466 		"movq %[newv], %[v]\n\t"  in rseq_cmpeqv_cmpeqv_storev()
477 		  [v]			"m" (*v),  in rseq_cmpeqv_cmpeqv_storev()
510 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,  in rseq_cmpeqv_trymemcpy_storev()  argument
516 	RSEQ_INJECT_C(9)  in rseq_cmpeqv_trymemcpy_storev()
532 		"cmpq %[v], %[expect]\n\t"  in rseq_cmpeqv_trymemcpy_storev()
537 		"cmpq %[v], %[expect]\n\t"  in rseq_cmpeqv_trymemcpy_storev()
553 		"movq %[newv], %[v]\n\t"  in rseq_cmpeqv_trymemcpy_storev()
586 		  [v]			"m" (*v),  in rseq_cmpeqv_trymemcpy_storev()
624 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,  in rseq_cmpeqv_trymemcpy_storev_release()  argument
628 	return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len,  in rseq_cmpeqv_trymemcpy_storev_release()
654 #define rseq_smp_store_release(p, v)					\  argument
657 	RSEQ_WRITE_ONCE(*p, v);						\
725 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)  in rseq_cmpeqv_storev()  argument
727 	RSEQ_INJECT_C(9)  in rseq_cmpeqv_storev()
740 		"cmpl %[v], %[expect]\n\t"  in rseq_cmpeqv_storev()
745 		"cmpl %[v], %[expect]\n\t"  in rseq_cmpeqv_storev()
749 		"movl %[newv], %[v]\n\t"  in rseq_cmpeqv_storev()
756 		  [v]			"m" (*v),  in rseq_cmpeqv_storev()
786  * Compare @v against @expectnot. When it does _not_ match, load @v
787  * into @load, and store the content of *@v + voffp into @v.
790 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,  in rseq_cmpnev_storeoffp_load()  argument
793 	RSEQ_INJECT_C(9)  in rseq_cmpnev_storeoffp_load()
806 		"movl %[v], %%ebx\n\t"  in rseq_cmpnev_storeoffp_load()
812 		"movl %[v], %%ebx\n\t"  in rseq_cmpnev_storeoffp_load()
820 		"movl %%ebx, %[v]\n\t"  in rseq_cmpnev_storeoffp_load()
828 		  [v]			"m" (*v),  in rseq_cmpnev_storeoffp_load()
859 int rseq_addv(intptr_t *v, intptr_t count, int cpu)  in rseq_addv()  argument
861 	RSEQ_INJECT_C(9)  in rseq_addv()
876 		"addl %[count], %[v]\n\t"  in rseq_addv()
884 		  [v]			"m" (*v),  in rseq_addv()
907 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,  in rseq_cmpeqv_trystorev_storev()  argument
911 	RSEQ_INJECT_C(9)  in rseq_cmpeqv_trystorev_storev()
924 		"cmpl %[v], %[expect]\n\t"  in rseq_cmpeqv_trystorev_storev()
929 		"cmpl %[v], %[expect]\n\t"  in rseq_cmpeqv_trystorev_storev()
937 		"movl %[newv], %[v]\n\t"  in rseq_cmpeqv_trystorev_storev()
948 		  [v]			"m" (*v),  in rseq_cmpeqv_trystorev_storev()
978 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,  in rseq_cmpeqv_trystorev_storev_release()  argument
982 	RSEQ_INJECT_C(9)  in rseq_cmpeqv_trystorev_storev_release()
996 		"cmpl %[v], %%eax\n\t"  in rseq_cmpeqv_trystorev_storev_release()
1002 		"cmpl %[v], %%eax\n\t"  in rseq_cmpeqv_trystorev_storev_release()
1010 		"movl %[newv], %[v]\n\t"  in rseq_cmpeqv_trystorev_storev_release()
1021 		  [v]			"m" (*v),  in rseq_cmpeqv_trystorev_storev_release()
1052 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,  in rseq_cmpeqv_cmpeqv_storev()  argument
1056 	RSEQ_INJECT_C(9)  in rseq_cmpeqv_cmpeqv_storev()
1070 		"cmpl %[v], %[expect]\n\t"  in rseq_cmpeqv_cmpeqv_storev()
1078 		"cmpl %[v], %[expect]\n\t"  in rseq_cmpeqv_cmpeqv_storev()
1085 		"movl %%eax, %[v]\n\t"  in rseq_cmpeqv_cmpeqv_storev()
1096 		  [v]			"m" (*v),  in rseq_cmpeqv_cmpeqv_storev()
1130 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,  in rseq_cmpeqv_trymemcpy_storev()  argument
1136 	RSEQ_INJECT_C(9)  in rseq_cmpeqv_trymemcpy_storev()
1153 		"cmpl %%eax, %[v]\n\t"  in rseq_cmpeqv_trymemcpy_storev()
1159 		"cmpl %%eax, %[v]\n\t"  in rseq_cmpeqv_trymemcpy_storev()
1176 		"movl %%eax, %[v]\n\t"  in rseq_cmpeqv_trymemcpy_storev()
1209 		  [v]			"m" (*v),  in rseq_cmpeqv_trymemcpy_storev()
1247 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,  in rseq_cmpeqv_trymemcpy_storev_release()  argument
1253 	RSEQ_INJECT_C(9)  in rseq_cmpeqv_trymemcpy_storev_release()
1270 		"cmpl %%eax, %[v]\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
1276 		"cmpl %%eax, %[v]\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
1294 		"movl %%eax, %[v]\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
1327 		  [v]			"m" (*v),  in rseq_cmpeqv_trymemcpy_storev_release()