36 static __always_inline void __enter_from_kernel_mode(struct pt_regs *regs)  in __enter_from_kernel_mode()  argument
38 	regs->exit_rcu = false;  in __enter_from_kernel_mode()
45 		regs->exit_rcu = true;  in __enter_from_kernel_mode()
54 static void noinstr enter_from_kernel_mode(struct pt_regs *regs)  in enter_from_kernel_mode()  argument
56 	__enter_from_kernel_mode(regs);  in enter_from_kernel_mode()
68 static __always_inline void __exit_to_kernel_mode(struct pt_regs *regs)  in __exit_to_kernel_mode()  argument
72 	if (interrupts_enabled(regs)) {  in __exit_to_kernel_mode()
73 		if (regs->exit_rcu) {  in __exit_to_kernel_mode()
83 		if (regs->exit_rcu)  in __exit_to_kernel_mode()
88 static void noinstr exit_to_kernel_mode(struct pt_regs *regs)  in exit_to_kernel_mode()  argument
91 	__exit_to_kernel_mode(regs);  in exit_to_kernel_mode()
107 static __always_inline void enter_from_user_mode(struct pt_regs *regs)  in enter_from_user_mode()  argument
125 static __always_inline void prepare_exit_to_user_mode(struct pt_regs *regs)  in prepare_exit_to_user_mode()  argument
133 		do_notify_resume(regs, flags);  in prepare_exit_to_user_mode()
136 static __always_inline void exit_to_user_mode(struct pt_regs *regs)  in exit_to_user_mode()  argument
138 	prepare_exit_to_user_mode(regs);  in exit_to_user_mode()
143 asmlinkage void noinstr asm_exit_to_user_mode(struct pt_regs *regs)  in asm_exit_to_user_mode()  argument
145 	exit_to_user_mode(regs);  in asm_exit_to_user_mode()
153 static void noinstr arm64_enter_nmi(struct pt_regs *regs)  in arm64_enter_nmi()  argument
155 	regs->lockdep_hardirqs = lockdep_hardirqs_enabled();  in arm64_enter_nmi()
171 static void noinstr arm64_exit_nmi(struct pt_regs *regs)  in arm64_exit_nmi()  argument
173 	bool restore = regs->lockdep_hardirqs;  in arm64_exit_nmi()
193 static void noinstr arm64_enter_el1_dbg(struct pt_regs *regs)  in arm64_enter_el1_dbg()  argument
195 	regs->lockdep_hardirqs = lockdep_hardirqs_enabled();  in arm64_enter_el1_dbg()
208 static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs)  in arm64_exit_el1_dbg()  argument
210 	bool restore = regs->lockdep_hardirqs;  in arm64_exit_el1_dbg()
222 static void noinstr enter_el1_irq_or_nmi(struct pt_regs *regs)  in enter_el1_irq_or_nmi()  argument
224 	if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs))  in enter_el1_irq_or_nmi()
225 		arm64_enter_nmi(regs);  in enter_el1_irq_or_nmi()
227 		enter_from_kernel_mode(regs);  in enter_el1_irq_or_nmi()
230 static void noinstr exit_el1_irq_or_nmi(struct pt_regs *regs)  in exit_el1_irq_or_nmi()  argument
232 	if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs))  in exit_el1_irq_or_nmi()
233 		arm64_exit_nmi(regs);  in exit_el1_irq_or_nmi()
235 		exit_to_kernel_mode(regs);  in exit_el1_irq_or_nmi()
263 static void do_interrupt_handler(struct pt_regs *regs,  in do_interrupt_handler()  argument
267 		call_on_irq_stack(regs, handler);  in do_interrupt_handler()
269 		handler(regs);  in do_interrupt_handler()
275 static void noinstr __panic_unhandled(struct pt_regs *regs, const char *vector,  in __panic_unhandled()  argument
278 	arm64_enter_nmi(regs);  in __panic_unhandled()
286 	__show_regs(regs);  in __panic_unhandled()
291 asmlinkage void noinstr el##_##regsize##_##vector##_handler(struct pt_regs *regs)	\
294 	__panic_unhandled(regs, desc, read_sysreg(esr_el1));				\
323 static bool cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs)  in cortex_a76_erratum_1463225_debug_handler()  argument
335 	regs->pstate |= PSR_D_BIT;  in cortex_a76_erratum_1463225_debug_handler()
340 static bool cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs)  in cortex_a76_erratum_1463225_debug_handler()  argument
351 static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr)  in el1_abort()  argument
355 	enter_from_kernel_mode(regs);  in el1_abort()
356 	local_daif_inherit(regs);  in el1_abort()
357 	do_mem_abort(far, esr, regs);  in el1_abort()
359 	exit_to_kernel_mode(regs);  in el1_abort()
362 static void noinstr el1_pc(struct pt_regs *regs, unsigned long esr)  in el1_pc()  argument
366 	enter_from_kernel_mode(regs);  in el1_pc()
367 	local_daif_inherit(regs);  in el1_pc()
368 	do_sp_pc_abort(far, esr, regs);  in el1_pc()
370 	exit_to_kernel_mode(regs);  in el1_pc()
373 static void noinstr el1_undef(struct pt_regs *regs)  in el1_undef()  argument
375 	enter_from_kernel_mode(regs);  in el1_undef()
376 	local_daif_inherit(regs);  in el1_undef()
377 	do_undefinstr(regs);  in el1_undef()
379 	exit_to_kernel_mode(regs);  in el1_undef()
382 static void noinstr el1_dbg(struct pt_regs *regs, unsigned long esr)  in el1_dbg()  argument
386 	arm64_enter_el1_dbg(regs);  in el1_dbg()
387 	if (!cortex_a76_erratum_1463225_debug_handler(regs))  in el1_dbg()
388 		do_debug_exception(far, esr, regs);  in el1_dbg()
389 	arm64_exit_el1_dbg(regs);  in el1_dbg()
392 static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr)  in el1_fpac()  argument
394 	enter_from_kernel_mode(regs);  in el1_fpac()
395 	local_daif_inherit(regs);  in el1_fpac()
396 	do_ptrauth_fault(regs, esr);  in el1_fpac()
398 	exit_to_kernel_mode(regs);  in el1_fpac()
401 asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs)  in el1h_64_sync_handler()  argument
408 		el1_abort(regs, esr);  in el1h_64_sync_handler()
415 		el1_pc(regs, esr);  in el1h_64_sync_handler()
419 		el1_undef(regs);  in el1h_64_sync_handler()
425 		el1_dbg(regs, esr);  in el1h_64_sync_handler()
428 		el1_fpac(regs, esr);  in el1h_64_sync_handler()
431 		__panic_unhandled(regs, "64-bit el1h sync", esr);  in el1h_64_sync_handler()
435 static void noinstr el1_interrupt(struct pt_regs *regs,  in el1_interrupt()  argument
440 	enter_el1_irq_or_nmi(regs);  in el1_interrupt()
441 	do_interrupt_handler(regs, handler);  in el1_interrupt()
452 	exit_el1_irq_or_nmi(regs);  in el1_interrupt()
455 asmlinkage void noinstr el1h_64_irq_handler(struct pt_regs *regs)  in el1h_64_irq_handler()  argument
457 	el1_interrupt(regs, handle_arch_irq);  in el1h_64_irq_handler()
460 asmlinkage void noinstr el1h_64_fiq_handler(struct pt_regs *regs)  in el1h_64_fiq_handler()  argument
462 	el1_interrupt(regs, handle_arch_fiq);  in el1h_64_fiq_handler()
465 asmlinkage void noinstr el1h_64_error_handler(struct pt_regs *regs)  in el1h_64_error_handler()  argument
470 	arm64_enter_nmi(regs);  in el1h_64_error_handler()
471 	do_serror(regs, esr);  in el1h_64_error_handler()
472 	arm64_exit_nmi(regs);  in el1h_64_error_handler()
475 static void noinstr el0_da(struct pt_regs *regs, unsigned long esr)  in el0_da()  argument
479 	enter_from_user_mode(regs);  in el0_da()
481 	do_mem_abort(far, esr, regs);  in el0_da()
482 	exit_to_user_mode(regs);  in el0_da()
485 static void noinstr el0_ia(struct pt_regs *regs, unsigned long esr)  in el0_ia()  argument
497 	enter_from_user_mode(regs);  in el0_ia()
499 	do_mem_abort(far, esr, regs);  in el0_ia()
500 	exit_to_user_mode(regs);  in el0_ia()
503 static void noinstr el0_fpsimd_acc(struct pt_regs *regs, unsigned long esr)  in el0_fpsimd_acc()  argument
505 	enter_from_user_mode(regs);  in el0_fpsimd_acc()
507 	do_fpsimd_acc(esr, regs);  in el0_fpsimd_acc()
508 	exit_to_user_mode(regs);  in el0_fpsimd_acc()
511 static void noinstr el0_sve_acc(struct pt_regs *regs, unsigned long esr)  in el0_sve_acc()  argument
513 	enter_from_user_mode(regs);  in el0_sve_acc()
515 	do_sve_acc(esr, regs);  in el0_sve_acc()
516 	exit_to_user_mode(regs);  in el0_sve_acc()
519 static void noinstr el0_fpsimd_exc(struct pt_regs *regs, unsigned long esr)  in el0_fpsimd_exc()  argument
521 	enter_from_user_mode(regs);  in el0_fpsimd_exc()
523 	do_fpsimd_exc(esr, regs);  in el0_fpsimd_exc()
524 	exit_to_user_mode(regs);  in el0_fpsimd_exc()
527 static void noinstr el0_sys(struct pt_regs *regs, unsigned long esr)  in el0_sys()  argument
529 	enter_from_user_mode(regs);  in el0_sys()
531 	do_sysinstr(esr, regs);  in el0_sys()
532 	exit_to_user_mode(regs);  in el0_sys()
535 static void noinstr el0_pc(struct pt_regs *regs, unsigned long esr)  in el0_pc()  argument
539 	if (!is_ttbr0_addr(instruction_pointer(regs)))  in el0_pc()
542 	enter_from_user_mode(regs);  in el0_pc()
544 	do_sp_pc_abort(far, esr, regs);  in el0_pc()
545 	exit_to_user_mode(regs);  in el0_pc()
548 static void noinstr el0_sp(struct pt_regs *regs, unsigned long esr)  in el0_sp()  argument
550 	enter_from_user_mode(regs);  in el0_sp()
552 	do_sp_pc_abort(regs->sp, esr, regs);  in el0_sp()
553 	exit_to_user_mode(regs);  in el0_sp()
556 static void noinstr el0_undef(struct pt_regs *regs)  in el0_undef()  argument
558 	enter_from_user_mode(regs);  in el0_undef()
560 	do_undefinstr(regs);  in el0_undef()
561 	exit_to_user_mode(regs);  in el0_undef()
564 static void noinstr el0_bti(struct pt_regs *regs)  in el0_bti()  argument
566 	enter_from_user_mode(regs);  in el0_bti()
568 	do_bti(regs);  in el0_bti()
569 	exit_to_user_mode(regs);  in el0_bti()
572 static void noinstr el0_inv(struct pt_regs *regs, unsigned long esr)  in el0_inv()  argument
574 	enter_from_user_mode(regs);  in el0_inv()
576 	bad_el0_sync(regs, 0, esr);  in el0_inv()
577 	exit_to_user_mode(regs);  in el0_inv()
580 static void noinstr el0_dbg(struct pt_regs *regs, unsigned long esr)  in el0_dbg()  argument
585 	enter_from_user_mode(regs);  in el0_dbg()
586 	do_debug_exception(far, esr, regs);  in el0_dbg()
588 	exit_to_user_mode(regs);  in el0_dbg()
591 static void noinstr el0_svc(struct pt_regs *regs)  in el0_svc()  argument
593 	enter_from_user_mode(regs);  in el0_svc()
595 	do_el0_svc(regs);  in el0_svc()
596 	exit_to_user_mode(regs);  in el0_svc()
599 static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr)  in el0_fpac()  argument
601 	enter_from_user_mode(regs);  in el0_fpac()
603 	do_ptrauth_fault(regs, esr);  in el0_fpac()
604 	exit_to_user_mode(regs);  in el0_fpac()
607 asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs)  in el0t_64_sync_handler()  argument
613 		el0_svc(regs);  in el0t_64_sync_handler()
616 		el0_da(regs, esr);  in el0t_64_sync_handler()
619 		el0_ia(regs, esr);  in el0t_64_sync_handler()
622 		el0_fpsimd_acc(regs, esr);  in el0t_64_sync_handler()
625 		el0_sve_acc(regs, esr);  in el0t_64_sync_handler()
628 		el0_fpsimd_exc(regs, esr);  in el0t_64_sync_handler()
632 		el0_sys(regs, esr);  in el0t_64_sync_handler()
635 		el0_sp(regs, esr);  in el0t_64_sync_handler()
638 		el0_pc(regs, esr);  in el0t_64_sync_handler()
641 		el0_undef(regs);  in el0t_64_sync_handler()
644 		el0_bti(regs);  in el0t_64_sync_handler()
650 		el0_dbg(regs, esr);  in el0t_64_sync_handler()
653 		el0_fpac(regs, esr);  in el0t_64_sync_handler()
656 		el0_inv(regs, esr);  in el0t_64_sync_handler()
660 static void noinstr el0_interrupt(struct pt_regs *regs,  in el0_interrupt()  argument
663 	enter_from_user_mode(regs);  in el0_interrupt()
667 	if (regs->pc & BIT(55))  in el0_interrupt()
670 	do_interrupt_handler(regs, handler);  in el0_interrupt()
672 	exit_to_user_mode(regs);  in el0_interrupt()
675 static void noinstr __el0_irq_handler_common(struct pt_regs *regs)  in __el0_irq_handler_common()  argument
677 	el0_interrupt(regs, handle_arch_irq);  in __el0_irq_handler_common()
680 asmlinkage void noinstr el0t_64_irq_handler(struct pt_regs *regs)  in el0t_64_irq_handler()  argument
682 	__el0_irq_handler_common(regs);  in el0t_64_irq_handler()
685 static void noinstr __el0_fiq_handler_common(struct pt_regs *regs)  in __el0_fiq_handler_common()  argument
687 	el0_interrupt(regs, handle_arch_fiq);  in __el0_fiq_handler_common()
690 asmlinkage void noinstr el0t_64_fiq_handler(struct pt_regs *regs)  in el0t_64_fiq_handler()  argument
692 	__el0_fiq_handler_common(regs);  in el0t_64_fiq_handler()
695 static void noinstr __el0_error_handler_common(struct pt_regs *regs)  in __el0_error_handler_common()  argument
699 	enter_from_user_mode(regs);  in __el0_error_handler_common()
701 	arm64_enter_nmi(regs);  in __el0_error_handler_common()
702 	do_serror(regs, esr);  in __el0_error_handler_common()
703 	arm64_exit_nmi(regs);  in __el0_error_handler_common()
705 	exit_to_user_mode(regs);  in __el0_error_handler_common()
708 asmlinkage void noinstr el0t_64_error_handler(struct pt_regs *regs)  in el0t_64_error_handler()  argument
710 	__el0_error_handler_common(regs);  in el0t_64_error_handler()
714 static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr)  in el0_cp15()  argument
716 	enter_from_user_mode(regs);  in el0_cp15()
718 	do_cp15instr(esr, regs);  in el0_cp15()
719 	exit_to_user_mode(regs);  in el0_cp15()
722 static void noinstr el0_svc_compat(struct pt_regs *regs)  in el0_svc_compat()  argument
724 	enter_from_user_mode(regs);  in el0_svc_compat()
726 	do_el0_svc_compat(regs);  in el0_svc_compat()
727 	exit_to_user_mode(regs);  in el0_svc_compat()
730 asmlinkage void noinstr el0t_32_sync_handler(struct pt_regs *regs)  in el0t_32_sync_handler()  argument
736 		el0_svc_compat(regs);  in el0t_32_sync_handler()
739 		el0_da(regs, esr);  in el0t_32_sync_handler()
742 		el0_ia(regs, esr);  in el0t_32_sync_handler()
745 		el0_fpsimd_acc(regs, esr);  in el0t_32_sync_handler()
748 		el0_fpsimd_exc(regs, esr);  in el0t_32_sync_handler()
751 		el0_pc(regs, esr);  in el0t_32_sync_handler()
757 		el0_undef(regs);  in el0t_32_sync_handler()
761 		el0_cp15(regs, esr);  in el0t_32_sync_handler()
767 		el0_dbg(regs, esr);  in el0t_32_sync_handler()
770 		el0_inv(regs, esr);  in el0t_32_sync_handler()
774 asmlinkage void noinstr el0t_32_irq_handler(struct pt_regs *regs)  in el0t_32_irq_handler()  argument
776 	__el0_irq_handler_common(regs);  in el0t_32_irq_handler()
779 asmlinkage void noinstr el0t_32_fiq_handler(struct pt_regs *regs)  in el0t_32_fiq_handler()  argument
781 	__el0_fiq_handler_common(regs);  in el0t_32_fiq_handler()
784 asmlinkage void noinstr el0t_32_error_handler(struct pt_regs *regs)  in el0t_32_error_handler()  argument
786 	__el0_error_handler_common(regs);  in el0t_32_error_handler()
796 asmlinkage void noinstr handle_bad_stack(struct pt_regs *regs)  in handle_bad_stack()  argument
801 	arm64_enter_nmi(regs);  in handle_bad_stack()
802 	panic_bad_stack(regs, esr, far);  in handle_bad_stack()
808 __sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg)  in __sdei_handler()  argument
834 	arm64_enter_nmi(regs);  in __sdei_handler()
835 	ret = do_sdei_event(regs, arg);  in __sdei_handler()
836 	arm64_exit_nmi(regs);  in __sdei_handler()