1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
3  * rseq-arm.h
5  * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
9  * - ARM little endian
12  * value 0x5de3. This traps if user-space reaches this instruction by mistake,
14  * pointer to attacker-controlled code on rseq abort.
27  * - ARMv6+ big endian (BE8):
29  * ARMv6+ -mbig-endian generates mixed endianness code vs data: little-endian
30  * code and big-endian data. The data value of the signature needs to have its
44  * - Prior to ARMv6 big endian (BE32):
46  * Prior to ARMv6, -mbig-endian generates big-endian code and data
52  * data (not a trap instruction) prior to ARMv6 on big endian. This is
83 #include "rseq-skip.h"
100 				(post_commit_ip - start_ip), abort_ip)
103  * Exit points of a rseq critical section consist of all instructions outside
104  * of the critical section where a critical section can either branch to or
106  * post-commit IP are already part of the __rseq_cs section and should not be
116 		RSEQ_INJECT_ASM(1)					\
143 				(post_commit_ip - start_ip), abort_ip)
155 	RSEQ_INJECT_C(9)  in rseq_cmpeqv_storev()
159 		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */  in rseq_cmpeqv_storev()
160 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])  in rseq_cmpeqv_storev()
162 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])  in rseq_cmpeqv_storev()
163 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])  in rseq_cmpeqv_storev()
166 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)  in rseq_cmpeqv_storev()
167 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)  in rseq_cmpeqv_storev()
183 		"b 5f\n\t"  in rseq_cmpeqv_storev()
184 		RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)  in rseq_cmpeqv_storev()
206 	return -1;  in rseq_cmpeqv_storev()
209 	return 1;  in rseq_cmpeqv_storev()
222 	RSEQ_INJECT_C(9)  in rseq_cmpnev_storeoffp_load()
226 		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */  in rseq_cmpnev_storeoffp_load()
227 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])  in rseq_cmpnev_storeoffp_load()
229 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])  in rseq_cmpnev_storeoffp_load()
230 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])  in rseq_cmpnev_storeoffp_load()
233 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)  in rseq_cmpnev_storeoffp_load()
234 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)  in rseq_cmpnev_storeoffp_load()
253 		"b 5f\n\t"  in rseq_cmpnev_storeoffp_load()
254 		RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)  in rseq_cmpnev_storeoffp_load()
278 	return -1;  in rseq_cmpnev_storeoffp_load()
281 	return 1;  in rseq_cmpnev_storeoffp_load()
293 	RSEQ_INJECT_C(9)  in rseq_addv()
297 		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */  in rseq_addv()
299 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])  in rseq_addv()
302 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)  in rseq_addv()
303 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)  in rseq_addv()
314 		"b 5f\n\t"  in rseq_addv()
315 		RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)  in rseq_addv()
336 	return -1;  in rseq_addv()
348 	RSEQ_INJECT_C(9)  in rseq_cmpeqv_trystorev_storev()
352 		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */  in rseq_cmpeqv_trystorev_storev()
353 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])  in rseq_cmpeqv_trystorev_storev()
355 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])  in rseq_cmpeqv_trystorev_storev()
356 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])  in rseq_cmpeqv_trystorev_storev()
359 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)  in rseq_cmpeqv_trystorev_storev()
360 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)  in rseq_cmpeqv_trystorev_storev()
379 		"b 5f\n\t"  in rseq_cmpeqv_trystorev_storev()
380 		RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)  in rseq_cmpeqv_trystorev_storev()
406 	return -1;  in rseq_cmpeqv_trystorev_storev()
409 	return 1;  in rseq_cmpeqv_trystorev_storev()
423 	RSEQ_INJECT_C(9)  in rseq_cmpeqv_trystorev_storev_release()
427 		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */  in rseq_cmpeqv_trystorev_storev_release()
428 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])  in rseq_cmpeqv_trystorev_storev_release()
430 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])  in rseq_cmpeqv_trystorev_storev_release()
431 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])  in rseq_cmpeqv_trystorev_storev_release()
434 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)  in rseq_cmpeqv_trystorev_storev_release()
435 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)  in rseq_cmpeqv_trystorev_storev_release()
450 		"dmb\n\t"	/* full mb provides store-release */  in rseq_cmpeqv_trystorev_storev_release()
455 		"b 5f\n\t"  in rseq_cmpeqv_trystorev_storev_release()
456 		RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)  in rseq_cmpeqv_trystorev_storev_release()
482 	return -1;  in rseq_cmpeqv_trystorev_storev_release()
485 	return 1;  in rseq_cmpeqv_trystorev_storev_release()
499 	RSEQ_INJECT_C(9)  in rseq_cmpeqv_cmpeqv_storev()
503 		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */  in rseq_cmpeqv_cmpeqv_storev()
504 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])  in rseq_cmpeqv_cmpeqv_storev()
506 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])  in rseq_cmpeqv_cmpeqv_storev()
507 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])  in rseq_cmpeqv_cmpeqv_storev()
508 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])  in rseq_cmpeqv_cmpeqv_storev()
511 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)  in rseq_cmpeqv_cmpeqv_storev()
512 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)  in rseq_cmpeqv_cmpeqv_storev()
535 		"b 5f\n\t"  in rseq_cmpeqv_cmpeqv_storev()
536 		RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)  in rseq_cmpeqv_cmpeqv_storev()
562 	return -1;  in rseq_cmpeqv_cmpeqv_storev()
565 	return 1;  in rseq_cmpeqv_cmpeqv_storev()
570 	rseq_bug("1st expected value comparison failed");  in rseq_cmpeqv_cmpeqv_storev()
583 	RSEQ_INJECT_C(9)  in rseq_cmpeqv_trymemcpy_storev()
587 		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */  in rseq_cmpeqv_trymemcpy_storev()
588 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])  in rseq_cmpeqv_trymemcpy_storev()
590 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])  in rseq_cmpeqv_trymemcpy_storev()
591 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])  in rseq_cmpeqv_trymemcpy_storev()
597 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)  in rseq_cmpeqv_trymemcpy_storev()
598 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)  in rseq_cmpeqv_trymemcpy_storev()
602 		"bne 5f\n\t"  in rseq_cmpeqv_trymemcpy_storev()
605 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)  in rseq_cmpeqv_trymemcpy_storev()
608 		"bne 7f\n\t"  in rseq_cmpeqv_trymemcpy_storev()
612 		"beq 333f\n\t" \  in rseq_cmpeqv_trymemcpy_storev()
616 		"adds %[src], #1\n\t" \  in rseq_cmpeqv_trymemcpy_storev()
617 		"adds %[dst], #1\n\t" \  in rseq_cmpeqv_trymemcpy_storev()
618 		"subs %[len], #1\n\t" \  in rseq_cmpeqv_trymemcpy_storev()
630 		"b 8f\n\t"  in rseq_cmpeqv_trymemcpy_storev()
636 				      abort, 1b, 2b, 4f)  in rseq_cmpeqv_trymemcpy_storev()
671 		  [rseq_scratch1]	"m" (rseq_scratch[1]),  in rseq_cmpeqv_trymemcpy_storev()
686 	return -1;  in rseq_cmpeqv_trymemcpy_storev()
689 	return 1;  in rseq_cmpeqv_trymemcpy_storev()
707 	RSEQ_INJECT_C(9)  in rseq_cmpeqv_trymemcpy_storev_release()
711 		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */  in rseq_cmpeqv_trymemcpy_storev_release()
712 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])  in rseq_cmpeqv_trymemcpy_storev_release()
714 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])  in rseq_cmpeqv_trymemcpy_storev_release()
715 		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])  in rseq_cmpeqv_trymemcpy_storev_release()
721 		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)  in rseq_cmpeqv_trymemcpy_storev_release()
722 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)  in rseq_cmpeqv_trymemcpy_storev_release()
726 		"bne 5f\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
729 		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)  in rseq_cmpeqv_trymemcpy_storev_release()
732 		"bne 7f\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
736 		"beq 333f\n\t" \  in rseq_cmpeqv_trymemcpy_storev_release()
740 		"adds %[src], #1\n\t" \  in rseq_cmpeqv_trymemcpy_storev_release()
741 		"adds %[dst], #1\n\t" \  in rseq_cmpeqv_trymemcpy_storev_release()
742 		"subs %[len], #1\n\t" \  in rseq_cmpeqv_trymemcpy_storev_release()
746 		"dmb\n\t"	/* full mb provides store-release */  in rseq_cmpeqv_trymemcpy_storev_release()
755 		"b 8f\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
761 				      abort, 1b, 2b, 4f)  in rseq_cmpeqv_trymemcpy_storev_release()
796 		  [rseq_scratch1]	"m" (rseq_scratch[1]),  in rseq_cmpeqv_trymemcpy_storev_release()
811 	return -1;  in rseq_cmpeqv_trymemcpy_storev_release()
814 	return 1;  in rseq_cmpeqv_trymemcpy_storev_release()