88 		".pushsection __rseq_cs, \"aw\"\n\t"			\
89 		".balign 32\n\t"					\
90 		__rseq_str(label) ":\n\t"					\
91 		".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
92 …str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
93 		".popsection\n\t"					\
94 		".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"		\
95 		".word " __rseq_str(label) "b, 0x0\n\t"			\
96 		".popsection\n\t"
111 		".pushsection __rseq_exit_point_array, \"aw\"\n\t"	\
112 		".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \
113 		".popsection\n\t"
117 		"adr r0, " __rseq_str(cs_label) "\n\t"			\
118 		"str r0, %[" __rseq_str(rseq_cs) "]\n\t"		\
119 		__rseq_str(label) ":\n\t"
123 		"ldr r0, %[" __rseq_str(current_cpu_id) "]\n\t"	\
124 		"cmp %[" __rseq_str(cpu_id) "], r0\n\t"		\
125 		"bne " __rseq_str(label) "\n\t"
130 		".balign 32\n\t"					\
131 		__rseq_str(table_label) ":\n\t"				\
132 		".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
133 …str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
134 		".word " __rseq_str(RSEQ_SIG) "\n\t"			\
135 		__rseq_str(label) ":\n\t"				\
137 		"b %l[" __rseq_str(abort_label) "]\n\t"
146 		__rseq_str(label) ":\n\t"				\
148 		"b %l[" __rseq_str(cmpfail_label) "]\n\t"
166 		"ldr r0, %[v]\n\t"  in rseq_cmpeqv_storev()
167 		"cmp %[expect], r0\n\t"  in rseq_cmpeqv_storev()
168 		"bne %l[cmpfail]\n\t"  in rseq_cmpeqv_storev()
172 		"ldr r0, %[v]\n\t"  in rseq_cmpeqv_storev()
173 		"cmp %[expect], r0\n\t"  in rseq_cmpeqv_storev()
174 		"bne %l[error2]\n\t"  in rseq_cmpeqv_storev()
177 		"str %[newv], %[v]\n\t"  in rseq_cmpeqv_storev()
178 		"2:\n\t"  in rseq_cmpeqv_storev()
180 		"b 5f\n\t"  in rseq_cmpeqv_storev()
182 		"5:\n\t"  in rseq_cmpeqv_storev()
234 		"ldr r0, %[v]\n\t"  in rseq_cmpnev_storeoffp_load()
235 		"cmp %[expectnot], r0\n\t"  in rseq_cmpnev_storeoffp_load()
236 		"beq %l[cmpfail]\n\t"  in rseq_cmpnev_storeoffp_load()
240 		"ldr r0, %[v]\n\t"  in rseq_cmpnev_storeoffp_load()
241 		"cmp %[expectnot], r0\n\t"  in rseq_cmpnev_storeoffp_load()
242 		"beq %l[error2]\n\t"  in rseq_cmpnev_storeoffp_load()
244 		"str r0, %[load]\n\t"  in rseq_cmpnev_storeoffp_load()
245 		"add r0, %[voffp]\n\t"  in rseq_cmpnev_storeoffp_load()
246 		"ldr r0, [r0]\n\t"  in rseq_cmpnev_storeoffp_load()
248 		"str r0, %[v]\n\t"  in rseq_cmpnev_storeoffp_load()
249 		"2:\n\t"  in rseq_cmpnev_storeoffp_load()
251 		"b 5f\n\t"  in rseq_cmpnev_storeoffp_load()
253 		"5:\n\t"  in rseq_cmpnev_storeoffp_load()
307 		"ldr r0, %[v]\n\t"  in rseq_addv()
308 		"add r0, %[count]\n\t"  in rseq_addv()
310 		"str r0, %[v]\n\t"  in rseq_addv()
311 		"2:\n\t"  in rseq_addv()
313 		"b 5f\n\t"  in rseq_addv()
315 		"5:\n\t"  in rseq_addv()
361 		"ldr r0, %[v]\n\t"  in rseq_cmpeqv_trystorev_storev()
362 		"cmp %[expect], r0\n\t"  in rseq_cmpeqv_trystorev_storev()
363 		"bne %l[cmpfail]\n\t"  in rseq_cmpeqv_trystorev_storev()
367 		"ldr r0, %[v]\n\t"  in rseq_cmpeqv_trystorev_storev()
368 		"cmp %[expect], r0\n\t"  in rseq_cmpeqv_trystorev_storev()
369 		"bne %l[error2]\n\t"  in rseq_cmpeqv_trystorev_storev()
372 		"str %[newv2], %[v2]\n\t"  in rseq_cmpeqv_trystorev_storev()
375 		"str %[newv], %[v]\n\t"  in rseq_cmpeqv_trystorev_storev()
376 		"2:\n\t"  in rseq_cmpeqv_trystorev_storev()
378 		"b 5f\n\t"  in rseq_cmpeqv_trystorev_storev()
380 		"5:\n\t"  in rseq_cmpeqv_trystorev_storev()
437 		"ldr r0, %[v]\n\t"  in rseq_cmpeqv_trystorev_storev_release()
438 		"cmp %[expect], r0\n\t"  in rseq_cmpeqv_trystorev_storev_release()
439 		"bne %l[cmpfail]\n\t"  in rseq_cmpeqv_trystorev_storev_release()
443 		"ldr r0, %[v]\n\t"  in rseq_cmpeqv_trystorev_storev_release()
444 		"cmp %[expect], r0\n\t"  in rseq_cmpeqv_trystorev_storev_release()
445 		"bne %l[error2]\n\t"  in rseq_cmpeqv_trystorev_storev_release()
448 		"str %[newv2], %[v2]\n\t"  in rseq_cmpeqv_trystorev_storev_release()
450 		"dmb\n\t"	/* full mb provides store-release */  in rseq_cmpeqv_trystorev_storev_release()
452 		"str %[newv], %[v]\n\t"  in rseq_cmpeqv_trystorev_storev_release()
453 		"2:\n\t"  in rseq_cmpeqv_trystorev_storev_release()
455 		"b 5f\n\t"  in rseq_cmpeqv_trystorev_storev_release()
457 		"5:\n\t"  in rseq_cmpeqv_trystorev_storev_release()
515 		"ldr r0, %[v]\n\t"  in rseq_cmpeqv_cmpeqv_storev()
516 		"cmp %[expect], r0\n\t"  in rseq_cmpeqv_cmpeqv_storev()
517 		"bne %l[cmpfail]\n\t"  in rseq_cmpeqv_cmpeqv_storev()
519 		"ldr r0, %[v2]\n\t"  in rseq_cmpeqv_cmpeqv_storev()
520 		"cmp %[expect2], r0\n\t"  in rseq_cmpeqv_cmpeqv_storev()
521 		"bne %l[cmpfail]\n\t"  in rseq_cmpeqv_cmpeqv_storev()
525 		"ldr r0, %[v]\n\t"  in rseq_cmpeqv_cmpeqv_storev()
526 		"cmp %[expect], r0\n\t"  in rseq_cmpeqv_cmpeqv_storev()
527 		"bne %l[error2]\n\t"  in rseq_cmpeqv_cmpeqv_storev()
528 		"ldr r0, %[v2]\n\t"  in rseq_cmpeqv_cmpeqv_storev()
529 		"cmp %[expect2], r0\n\t"  in rseq_cmpeqv_cmpeqv_storev()
530 		"bne %l[error3]\n\t"  in rseq_cmpeqv_cmpeqv_storev()
533 		"str %[newv], %[v]\n\t"  in rseq_cmpeqv_cmpeqv_storev()
534 		"2:\n\t"  in rseq_cmpeqv_cmpeqv_storev()
536 		"b 5f\n\t"  in rseq_cmpeqv_cmpeqv_storev()
538 		"5:\n\t"  in rseq_cmpeqv_cmpeqv_storev()
596 		"str %[src], %[rseq_scratch0]\n\t"  in rseq_cmpeqv_trymemcpy_storev()
597 		"str %[dst], %[rseq_scratch1]\n\t"  in rseq_cmpeqv_trymemcpy_storev()
598 		"str %[len], %[rseq_scratch2]\n\t"  in rseq_cmpeqv_trymemcpy_storev()
603 		"ldr r0, %[v]\n\t"  in rseq_cmpeqv_trymemcpy_storev()
604 		"cmp %[expect], r0\n\t"  in rseq_cmpeqv_trymemcpy_storev()
605 		"bne 5f\n\t"  in rseq_cmpeqv_trymemcpy_storev()
609 		"ldr r0, %[v]\n\t"  in rseq_cmpeqv_trymemcpy_storev()
610 		"cmp %[expect], r0\n\t"  in rseq_cmpeqv_trymemcpy_storev()
611 		"bne 7f\n\t"  in rseq_cmpeqv_trymemcpy_storev()
614 		"cmp %[len], #0\n\t" \  in rseq_cmpeqv_trymemcpy_storev()
615 		"beq 333f\n\t" \  in rseq_cmpeqv_trymemcpy_storev()
616 		"222:\n\t" \  in rseq_cmpeqv_trymemcpy_storev()
617 		"ldrb %%r0, [%[src]]\n\t" \  in rseq_cmpeqv_trymemcpy_storev()
618 		"strb %%r0, [%[dst]]\n\t" \  in rseq_cmpeqv_trymemcpy_storev()
619 		"adds %[src], #1\n\t" \  in rseq_cmpeqv_trymemcpy_storev()
620 		"adds %[dst], #1\n\t" \  in rseq_cmpeqv_trymemcpy_storev()
621 		"subs %[len], #1\n\t" \  in rseq_cmpeqv_trymemcpy_storev()
622 		"bne 222b\n\t" \  in rseq_cmpeqv_trymemcpy_storev()
623 		"333:\n\t" \  in rseq_cmpeqv_trymemcpy_storev()
626 		"str %[newv], %[v]\n\t"  in rseq_cmpeqv_trymemcpy_storev()
627 		"2:\n\t"  in rseq_cmpeqv_trymemcpy_storev()
630 		"ldr %[len], %[rseq_scratch2]\n\t"  in rseq_cmpeqv_trymemcpy_storev()
631 		"ldr %[dst], %[rseq_scratch1]\n\t"  in rseq_cmpeqv_trymemcpy_storev()
632 		"ldr %[src], %[rseq_scratch0]\n\t"  in rseq_cmpeqv_trymemcpy_storev()
633 		"b 8f\n\t"  in rseq_cmpeqv_trymemcpy_storev()
636 				      "ldr %[len], %[rseq_scratch2]\n\t"  in rseq_cmpeqv_trymemcpy_storev()
637 				      "ldr %[dst], %[rseq_scratch1]\n\t"  in rseq_cmpeqv_trymemcpy_storev()
638 				      "ldr %[src], %[rseq_scratch0]\n\t",  in rseq_cmpeqv_trymemcpy_storev()
642 					"ldr %[len], %[rseq_scratch2]\n\t"  in rseq_cmpeqv_trymemcpy_storev()
643 					"ldr %[dst], %[rseq_scratch1]\n\t"  in rseq_cmpeqv_trymemcpy_storev()
644 					"ldr %[src], %[rseq_scratch0]\n\t",  in rseq_cmpeqv_trymemcpy_storev()
649 					"ldr %[len], %[rseq_scratch2]\n\t"  in rseq_cmpeqv_trymemcpy_storev()
650 					"ldr %[dst], %[rseq_scratch1]\n\t"  in rseq_cmpeqv_trymemcpy_storev()
651 					"ldr %[src], %[rseq_scratch0]\n\t",  in rseq_cmpeqv_trymemcpy_storev()
655 					"ldr %[len], %[rseq_scratch2]\n\t"  in rseq_cmpeqv_trymemcpy_storev()
656 					"ldr %[dst], %[rseq_scratch1]\n\t"  in rseq_cmpeqv_trymemcpy_storev()
657 					"ldr %[src], %[rseq_scratch0]\n\t",  in rseq_cmpeqv_trymemcpy_storev()
660 		"8:\n\t"  in rseq_cmpeqv_trymemcpy_storev()
719 		"str %[src], %[rseq_scratch0]\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
720 		"str %[dst], %[rseq_scratch1]\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
721 		"str %[len], %[rseq_scratch2]\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
726 		"ldr r0, %[v]\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
727 		"cmp %[expect], r0\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
728 		"bne 5f\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
732 		"ldr r0, %[v]\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
733 		"cmp %[expect], r0\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
734 		"bne 7f\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
737 		"cmp %[len], #0\n\t" \  in rseq_cmpeqv_trymemcpy_storev_release()
738 		"beq 333f\n\t" \  in rseq_cmpeqv_trymemcpy_storev_release()
739 		"222:\n\t" \  in rseq_cmpeqv_trymemcpy_storev_release()
740 		"ldrb %%r0, [%[src]]\n\t" \  in rseq_cmpeqv_trymemcpy_storev_release()
741 		"strb %%r0, [%[dst]]\n\t" \  in rseq_cmpeqv_trymemcpy_storev_release()
742 		"adds %[src], #1\n\t" \  in rseq_cmpeqv_trymemcpy_storev_release()
743 		"adds %[dst], #1\n\t" \  in rseq_cmpeqv_trymemcpy_storev_release()
744 		"subs %[len], #1\n\t" \  in rseq_cmpeqv_trymemcpy_storev_release()
745 		"bne 222b\n\t" \  in rseq_cmpeqv_trymemcpy_storev_release()
746 		"333:\n\t" \  in rseq_cmpeqv_trymemcpy_storev_release()
748 		"dmb\n\t"	/* full mb provides store-release */  in rseq_cmpeqv_trymemcpy_storev_release()
750 		"str %[newv], %[v]\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
751 		"2:\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
754 		"ldr %[len], %[rseq_scratch2]\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
755 		"ldr %[dst], %[rseq_scratch1]\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
756 		"ldr %[src], %[rseq_scratch0]\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
757 		"b 8f\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
760 				      "ldr %[len], %[rseq_scratch2]\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
761 				      "ldr %[dst], %[rseq_scratch1]\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
762 				      "ldr %[src], %[rseq_scratch0]\n\t",  in rseq_cmpeqv_trymemcpy_storev_release()
766 					"ldr %[len], %[rseq_scratch2]\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
767 					"ldr %[dst], %[rseq_scratch1]\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
768 					"ldr %[src], %[rseq_scratch0]\n\t",  in rseq_cmpeqv_trymemcpy_storev_release()
773 					"ldr %[len], %[rseq_scratch2]\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
774 					"ldr %[dst], %[rseq_scratch1]\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
775 					"ldr %[src], %[rseq_scratch0]\n\t",  in rseq_cmpeqv_trymemcpy_storev_release()
779 					"ldr %[len], %[rseq_scratch2]\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
780 					"ldr %[dst], %[rseq_scratch1]\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
781 					"ldr %[src], %[rseq_scratch0]\n\t",  in rseq_cmpeqv_trymemcpy_storev_release()
784 		"8:\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()