24 static void noinstr enter_from_kernel_mode(struct pt_regs *regs)  in enter_from_kernel_mode()  argument
26 	regs->exit_rcu = false;  in enter_from_kernel_mode()
33 		regs->exit_rcu = true;  in enter_from_kernel_mode()
46 static void noinstr exit_to_kernel_mode(struct pt_regs *regs)  in exit_to_kernel_mode()  argument
50 	if (interrupts_enabled(regs)) {  in exit_to_kernel_mode()
51 		if (regs->exit_rcu) {  in exit_to_kernel_mode()
61 		if (regs->exit_rcu)  in exit_to_kernel_mode()
66 void noinstr arm64_enter_nmi(struct pt_regs *regs)  in arm64_enter_nmi()  argument
68 	regs->lockdep_hardirqs = lockdep_hardirqs_enabled();  in arm64_enter_nmi()
79 void noinstr arm64_exit_nmi(struct pt_regs *regs)  in arm64_exit_nmi()  argument
81 	bool restore = regs->lockdep_hardirqs;  in arm64_exit_nmi()
96 asmlinkage void noinstr enter_el1_irq_or_nmi(struct pt_regs *regs)  in enter_el1_irq_or_nmi()  argument
98 	if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs))  in enter_el1_irq_or_nmi()
99 		arm64_enter_nmi(regs);  in enter_el1_irq_or_nmi()
101 		enter_from_kernel_mode(regs);  in enter_el1_irq_or_nmi()
104 asmlinkage void noinstr exit_el1_irq_or_nmi(struct pt_regs *regs)  in exit_el1_irq_or_nmi()  argument
106 	if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs))  in exit_el1_irq_or_nmi()
107 		arm64_exit_nmi(regs);  in exit_el1_irq_or_nmi()
109 		exit_to_kernel_mode(regs);  in exit_el1_irq_or_nmi()
112 static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr)  in el1_abort()  argument
116 	enter_from_kernel_mode(regs);  in el1_abort()
117 	local_daif_inherit(regs);  in el1_abort()
119 	do_mem_abort(far, esr, regs);  in el1_abort()
121 	exit_to_kernel_mode(regs);  in el1_abort()
124 static void noinstr el1_pc(struct pt_regs *regs, unsigned long esr)  in el1_pc()  argument
128 	enter_from_kernel_mode(regs);  in el1_pc()
129 	local_daif_inherit(regs);  in el1_pc()
130 	do_sp_pc_abort(far, esr, regs);  in el1_pc()
132 	exit_to_kernel_mode(regs);  in el1_pc()
135 static void noinstr el1_undef(struct pt_regs *regs)  in el1_undef()  argument
137 	enter_from_kernel_mode(regs);  in el1_undef()
138 	local_daif_inherit(regs);  in el1_undef()
139 	do_undefinstr(regs);  in el1_undef()
141 	exit_to_kernel_mode(regs);  in el1_undef()
144 static void noinstr el1_inv(struct pt_regs *regs, unsigned long esr)  in el1_inv()  argument
146 	enter_from_kernel_mode(regs);  in el1_inv()
147 	local_daif_inherit(regs);  in el1_inv()
148 	bad_mode(regs, 0, esr);  in el1_inv()
150 	exit_to_kernel_mode(regs);  in el1_inv()
153 static void noinstr arm64_enter_el1_dbg(struct pt_regs *regs)  in arm64_enter_el1_dbg()  argument
155 	regs->lockdep_hardirqs = lockdep_hardirqs_enabled();  in arm64_enter_el1_dbg()
163 static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs)  in arm64_exit_el1_dbg()  argument
165 	bool restore = regs->lockdep_hardirqs;  in arm64_exit_el1_dbg()
177 static void noinstr el1_dbg(struct pt_regs *regs, unsigned long esr)  in el1_dbg()  argument
189 	arm64_enter_el1_dbg(regs);  in el1_dbg()
190 	do_debug_exception(far, esr, regs);  in el1_dbg()
191 	arm64_exit_el1_dbg(regs);  in el1_dbg()
194 static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr)  in el1_fpac()  argument
196 	enter_from_kernel_mode(regs);  in el1_fpac()
197 	local_daif_inherit(regs);  in el1_fpac()
198 	do_ptrauth_fault(regs, esr);  in el1_fpac()
200 	exit_to_kernel_mode(regs);  in el1_fpac()
203 asmlinkage void noinstr el1_sync_handler(struct pt_regs *regs)  in el1_sync_handler()  argument
210 		el1_abort(regs, esr);  in el1_sync_handler()
217 		el1_pc(regs, esr);  in el1_sync_handler()
221 		el1_undef(regs);  in el1_sync_handler()
227 		el1_dbg(regs, esr);  in el1_sync_handler()
230 		el1_fpac(regs, esr);  in el1_sync_handler()
233 		el1_inv(regs, esr);  in el1_sync_handler()
253 static void noinstr el0_da(struct pt_regs *regs, unsigned long esr)  in el0_da()  argument
260 	do_mem_abort(far, esr, regs);  in el0_da()
263 static void noinstr el0_ia(struct pt_regs *regs, unsigned long esr)  in el0_ia()  argument
277 	do_mem_abort(far, esr, regs);  in el0_ia()
280 static void noinstr el0_fpsimd_acc(struct pt_regs *regs, unsigned long esr)  in el0_fpsimd_acc()  argument
284 	do_fpsimd_acc(esr, regs);  in el0_fpsimd_acc()
287 static void noinstr el0_sve_acc(struct pt_regs *regs, unsigned long esr)  in el0_sve_acc()  argument
291 	do_sve_acc(esr, regs);  in el0_sve_acc()
294 static void noinstr el0_fpsimd_exc(struct pt_regs *regs, unsigned long esr)  in el0_fpsimd_exc()  argument
298 	do_fpsimd_exc(esr, regs);  in el0_fpsimd_exc()
301 static void noinstr el0_sys(struct pt_regs *regs, unsigned long esr)  in el0_sys()  argument
305 	do_sysinstr(esr, regs);  in el0_sys()
308 static void noinstr el0_pc(struct pt_regs *regs, unsigned long esr)  in el0_pc()  argument
312 	if (!is_ttbr0_addr(instruction_pointer(regs)))  in el0_pc()
317 	do_sp_pc_abort(far, esr, regs);  in el0_pc()
320 static void noinstr el0_sp(struct pt_regs *regs, unsigned long esr)  in el0_sp()  argument
324 	do_sp_pc_abort(regs->sp, esr, regs);  in el0_sp()
327 static void noinstr el0_undef(struct pt_regs *regs)  in el0_undef()  argument
331 	do_undefinstr(regs);  in el0_undef()
334 static void noinstr el0_bti(struct pt_regs *regs)  in el0_bti()  argument
338 	do_bti(regs);  in el0_bti()
341 static void noinstr el0_inv(struct pt_regs *regs, unsigned long esr)  in el0_inv()  argument
345 	bad_el0_sync(regs, 0, esr);  in el0_inv()
348 static void noinstr el0_dbg(struct pt_regs *regs, unsigned long esr)  in el0_dbg()  argument
357 	do_debug_exception(far, esr, regs);  in el0_dbg()
361 static void noinstr el0_svc(struct pt_regs *regs)  in el0_svc()  argument
367 	do_el0_svc(regs);  in el0_svc()
370 static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr)  in el0_fpac()  argument
374 	do_ptrauth_fault(regs, esr);  in el0_fpac()
377 asmlinkage void noinstr el0_sync_handler(struct pt_regs *regs)  in el0_sync_handler()  argument
383 		el0_svc(regs);  in el0_sync_handler()
386 		el0_da(regs, esr);  in el0_sync_handler()
389 		el0_ia(regs, esr);  in el0_sync_handler()
392 		el0_fpsimd_acc(regs, esr);  in el0_sync_handler()
395 		el0_sve_acc(regs, esr);  in el0_sync_handler()
398 		el0_fpsimd_exc(regs, esr);  in el0_sync_handler()
402 		el0_sys(regs, esr);  in el0_sync_handler()
405 		el0_sp(regs, esr);  in el0_sync_handler()
408 		el0_pc(regs, esr);  in el0_sync_handler()
411 		el0_undef(regs);  in el0_sync_handler()
414 		el0_bti(regs);  in el0_sync_handler()
420 		el0_dbg(regs, esr);  in el0_sync_handler()
423 		el0_fpac(regs, esr);  in el0_sync_handler()
426 		el0_inv(regs, esr);  in el0_sync_handler()
431 static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr)  in el0_cp15()  argument
435 	do_cp15instr(esr, regs);  in el0_cp15()
438 static void noinstr el0_svc_compat(struct pt_regs *regs)  in el0_svc_compat()  argument
444 	do_el0_svc_compat(regs);  in el0_svc_compat()
447 asmlinkage void noinstr el0_sync_compat_handler(struct pt_regs *regs)  in el0_sync_compat_handler()  argument
453 		el0_svc_compat(regs);  in el0_sync_compat_handler()
456 		el0_da(regs, esr);  in el0_sync_compat_handler()
459 		el0_ia(regs, esr);  in el0_sync_compat_handler()
462 		el0_fpsimd_acc(regs, esr);  in el0_sync_compat_handler()
465 		el0_fpsimd_exc(regs, esr);  in el0_sync_compat_handler()
468 		el0_pc(regs, esr);  in el0_sync_compat_handler()
474 		el0_undef(regs);  in el0_sync_compat_handler()
478 		el0_cp15(regs, esr);  in el0_sync_compat_handler()
484 		el0_dbg(regs, esr);  in el0_sync_compat_handler()
487 		el0_inv(regs, esr);  in el0_sync_compat_handler()