38 static __always_inline void __enter_from_kernel_mode(struct pt_regs *regs)  in __enter_from_kernel_mode()  argument
40 	regs->exit_rcu = false;  in __enter_from_kernel_mode()
47 		regs->exit_rcu = true;  in __enter_from_kernel_mode()
56 static void noinstr enter_from_kernel_mode(struct pt_regs *regs)  in enter_from_kernel_mode()  argument
58 	__enter_from_kernel_mode(regs);  in enter_from_kernel_mode()
71 static __always_inline void __exit_to_kernel_mode(struct pt_regs *regs)  in __exit_to_kernel_mode()  argument
75 	if (interrupts_enabled(regs)) {  in __exit_to_kernel_mode()
76 		if (regs->exit_rcu) {  in __exit_to_kernel_mode()
86 		if (regs->exit_rcu)  in __exit_to_kernel_mode()
91 static void noinstr exit_to_kernel_mode(struct pt_regs *regs)  in exit_to_kernel_mode()  argument
94 	__exit_to_kernel_mode(regs);  in exit_to_kernel_mode()
111 static __always_inline void enter_from_user_mode(struct pt_regs *regs)  in enter_from_user_mode()  argument
129 static __always_inline void prepare_exit_to_user_mode(struct pt_regs *regs)  in prepare_exit_to_user_mode()  argument
137 		do_notify_resume(regs, flags);  in prepare_exit_to_user_mode()
140 static __always_inline void exit_to_user_mode(struct pt_regs *regs)  in exit_to_user_mode()  argument
142 	prepare_exit_to_user_mode(regs);  in exit_to_user_mode()
147 asmlinkage void noinstr asm_exit_to_user_mode(struct pt_regs *regs)  in asm_exit_to_user_mode()  argument
149 	exit_to_user_mode(regs);  in asm_exit_to_user_mode()
157 static void noinstr arm64_enter_nmi(struct pt_regs *regs)  in arm64_enter_nmi()  argument
159 	regs->lockdep_hardirqs = lockdep_hardirqs_enabled();  in arm64_enter_nmi()
175 static void noinstr arm64_exit_nmi(struct pt_regs *regs)  in arm64_exit_nmi()  argument
177 	bool restore = regs->lockdep_hardirqs;  in arm64_exit_nmi()
197 static void noinstr arm64_enter_el1_dbg(struct pt_regs *regs)  in arm64_enter_el1_dbg()  argument
199 	regs->lockdep_hardirqs = lockdep_hardirqs_enabled();  in arm64_enter_el1_dbg()
212 static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs)  in arm64_exit_el1_dbg()  argument
214 	bool restore = regs->lockdep_hardirqs;  in arm64_exit_el1_dbg()
268 static void do_interrupt_handler(struct pt_regs *regs,  in do_interrupt_handler()  argument
271 	struct pt_regs *old_regs = set_irq_regs(regs);  in do_interrupt_handler()
274 		call_on_irq_stack(regs, handler);  in do_interrupt_handler()
276 		handler(regs);  in do_interrupt_handler()
284 static void noinstr __panic_unhandled(struct pt_regs *regs, const char *vector,  in __panic_unhandled()  argument
287 	arm64_enter_nmi(regs);  in __panic_unhandled()
295 	__show_regs(regs);  in __panic_unhandled()
300 asmlinkage void noinstr el##_##regsize##_##vector##_handler(struct pt_regs *regs)	\
303 	__panic_unhandled(regs, desc, read_sysreg(esr_el1));				\
333 cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs)  in cortex_a76_erratum_1463225_debug_handler()  argument
345 	regs->pstate |= PSR_D_BIT;  in cortex_a76_erratum_1463225_debug_handler()
350 static bool cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs)  in cortex_a76_erratum_1463225_debug_handler()  argument
361 static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr)  in el1_abort()  argument
365 	enter_from_kernel_mode(regs);  in el1_abort()
366 	local_daif_inherit(regs);  in el1_abort()
367 	do_mem_abort(far, esr, regs);  in el1_abort()
369 	exit_to_kernel_mode(regs);  in el1_abort()
372 static void noinstr el1_pc(struct pt_regs *regs, unsigned long esr)  in el1_pc()  argument
376 	enter_from_kernel_mode(regs);  in el1_pc()
377 	local_daif_inherit(regs);  in el1_pc()
378 	do_sp_pc_abort(far, esr, regs);  in el1_pc()
380 	exit_to_kernel_mode(regs);  in el1_pc()
383 static void noinstr el1_undef(struct pt_regs *regs, unsigned long esr)  in el1_undef()  argument
385 	enter_from_kernel_mode(regs);  in el1_undef()
386 	local_daif_inherit(regs);  in el1_undef()
387 	do_undefinstr(regs, esr);  in el1_undef()
389 	exit_to_kernel_mode(regs);  in el1_undef()
392 static void noinstr el1_bti(struct pt_regs *regs, unsigned long esr)  in el1_bti()  argument
394 	enter_from_kernel_mode(regs);  in el1_bti()
395 	local_daif_inherit(regs);  in el1_bti()
396 	do_el1_bti(regs, esr);  in el1_bti()
398 	exit_to_kernel_mode(regs);  in el1_bti()
401 static void noinstr el1_dbg(struct pt_regs *regs, unsigned long esr)  in el1_dbg()  argument
405 	arm64_enter_el1_dbg(regs);  in el1_dbg()
406 	if (!cortex_a76_erratum_1463225_debug_handler(regs))  in el1_dbg()
407 		do_debug_exception(far, esr, regs);  in el1_dbg()
408 	arm64_exit_el1_dbg(regs);  in el1_dbg()
411 static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr)  in el1_fpac()  argument
413 	enter_from_kernel_mode(regs);  in el1_fpac()
414 	local_daif_inherit(regs);  in el1_fpac()
415 	do_el1_fpac(regs, esr);  in el1_fpac()
417 	exit_to_kernel_mode(regs);  in el1_fpac()
420 asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs)  in el1h_64_sync_handler()  argument
427 		el1_abort(regs, esr);  in el1h_64_sync_handler()
434 		el1_pc(regs, esr);  in el1h_64_sync_handler()
438 		el1_undef(regs, esr);  in el1h_64_sync_handler()
441 		el1_bti(regs, esr);  in el1h_64_sync_handler()
447 		el1_dbg(regs, esr);  in el1h_64_sync_handler()
450 		el1_fpac(regs, esr);  in el1h_64_sync_handler()
453 		__panic_unhandled(regs, "64-bit el1h sync", esr);  in el1h_64_sync_handler()
457 static __always_inline void __el1_pnmi(struct pt_regs *regs,  in __el1_pnmi()  argument
460 	arm64_enter_nmi(regs);  in __el1_pnmi()
461 	do_interrupt_handler(regs, handler);  in __el1_pnmi()
462 	arm64_exit_nmi(regs);  in __el1_pnmi()
465 static __always_inline void __el1_irq(struct pt_regs *regs,  in __el1_irq()  argument
468 	enter_from_kernel_mode(regs);  in __el1_irq()
471 	do_interrupt_handler(regs, handler);  in __el1_irq()
476 	exit_to_kernel_mode(regs);  in __el1_irq()
478 static void noinstr el1_interrupt(struct pt_regs *regs,  in el1_interrupt()  argument
483 	if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs))  in el1_interrupt()
484 		__el1_pnmi(regs, handler);  in el1_interrupt()
486 		__el1_irq(regs, handler);  in el1_interrupt()
489 asmlinkage void noinstr el1h_64_irq_handler(struct pt_regs *regs)  in el1h_64_irq_handler()  argument
491 	el1_interrupt(regs, handle_arch_irq);  in el1h_64_irq_handler()
494 asmlinkage void noinstr el1h_64_fiq_handler(struct pt_regs *regs)  in el1h_64_fiq_handler()  argument
496 	el1_interrupt(regs, handle_arch_fiq);  in el1h_64_fiq_handler()
499 asmlinkage void noinstr el1h_64_error_handler(struct pt_regs *regs)  in el1h_64_error_handler()  argument
504 	arm64_enter_nmi(regs);  in el1h_64_error_handler()
505 	do_serror(regs, esr);  in el1h_64_error_handler()
506 	arm64_exit_nmi(regs);  in el1h_64_error_handler()
509 static void noinstr el0_da(struct pt_regs *regs, unsigned long esr)  in el0_da()  argument
513 	enter_from_user_mode(regs);  in el0_da()
515 	do_mem_abort(far, esr, regs);  in el0_da()
516 	exit_to_user_mode(regs);  in el0_da()
519 static void noinstr el0_ia(struct pt_regs *regs, unsigned long esr)  in el0_ia()  argument
531 	enter_from_user_mode(regs);  in el0_ia()
533 	do_mem_abort(far, esr, regs);  in el0_ia()
534 	exit_to_user_mode(regs);  in el0_ia()
537 static void noinstr el0_fpsimd_acc(struct pt_regs *regs, unsigned long esr)  in el0_fpsimd_acc()  argument
539 	enter_from_user_mode(regs);  in el0_fpsimd_acc()
541 	do_fpsimd_acc(esr, regs);  in el0_fpsimd_acc()
542 	exit_to_user_mode(regs);  in el0_fpsimd_acc()
545 static void noinstr el0_sve_acc(struct pt_regs *regs, unsigned long esr)  in el0_sve_acc()  argument
547 	enter_from_user_mode(regs);  in el0_sve_acc()
549 	do_sve_acc(esr, regs);  in el0_sve_acc()
550 	exit_to_user_mode(regs);  in el0_sve_acc()
553 static void noinstr el0_sme_acc(struct pt_regs *regs, unsigned long esr)  in el0_sme_acc()  argument
555 	enter_from_user_mode(regs);  in el0_sme_acc()
557 	do_sme_acc(esr, regs);  in el0_sme_acc()
558 	exit_to_user_mode(regs);  in el0_sme_acc()
561 static void noinstr el0_fpsimd_exc(struct pt_regs *regs, unsigned long esr)  in el0_fpsimd_exc()  argument
563 	enter_from_user_mode(regs);  in el0_fpsimd_exc()
565 	do_fpsimd_exc(esr, regs);  in el0_fpsimd_exc()
566 	exit_to_user_mode(regs);  in el0_fpsimd_exc()
569 static void noinstr el0_sys(struct pt_regs *regs, unsigned long esr)  in el0_sys()  argument
571 	enter_from_user_mode(regs);  in el0_sys()
573 	do_sysinstr(esr, regs);  in el0_sys()
574 	exit_to_user_mode(regs);  in el0_sys()
577 static void noinstr el0_pc(struct pt_regs *regs, unsigned long esr)  in el0_pc()  argument
581 	if (!is_ttbr0_addr(instruction_pointer(regs)))  in el0_pc()
584 	enter_from_user_mode(regs);  in el0_pc()
586 	do_sp_pc_abort(far, esr, regs);  in el0_pc()
587 	exit_to_user_mode(regs);  in el0_pc()
590 static void noinstr el0_sp(struct pt_regs *regs, unsigned long esr)  in el0_sp()  argument
592 	enter_from_user_mode(regs);  in el0_sp()
594 	do_sp_pc_abort(regs->sp, esr, regs);  in el0_sp()
595 	exit_to_user_mode(regs);  in el0_sp()
598 static void noinstr el0_undef(struct pt_regs *regs, unsigned long esr)  in el0_undef()  argument
600 	enter_from_user_mode(regs);  in el0_undef()
602 	do_undefinstr(regs, esr);  in el0_undef()
603 	exit_to_user_mode(regs);  in el0_undef()
606 static void noinstr el0_bti(struct pt_regs *regs)  in el0_bti()  argument
608 	enter_from_user_mode(regs);  in el0_bti()
610 	do_el0_bti(regs);  in el0_bti()
611 	exit_to_user_mode(regs);  in el0_bti()
614 static void noinstr el0_inv(struct pt_regs *regs, unsigned long esr)  in el0_inv()  argument
616 	enter_from_user_mode(regs);  in el0_inv()
618 	bad_el0_sync(regs, 0, esr);  in el0_inv()
619 	exit_to_user_mode(regs);  in el0_inv()
622 static void noinstr el0_dbg(struct pt_regs *regs, unsigned long esr)  in el0_dbg()  argument
627 	enter_from_user_mode(regs);  in el0_dbg()
628 	do_debug_exception(far, esr, regs);  in el0_dbg()
630 	exit_to_user_mode(regs);  in el0_dbg()
633 static void noinstr el0_svc(struct pt_regs *regs)  in el0_svc()  argument
635 	enter_from_user_mode(regs);  in el0_svc()
637 	do_el0_svc(regs);  in el0_svc()
638 	exit_to_user_mode(regs);  in el0_svc()
641 static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr)  in el0_fpac()  argument
643 	enter_from_user_mode(regs);  in el0_fpac()
645 	do_el0_fpac(regs, esr);  in el0_fpac()
646 	exit_to_user_mode(regs);  in el0_fpac()
649 asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs)  in el0t_64_sync_handler()  argument
655 		el0_svc(regs);  in el0t_64_sync_handler()
658 		el0_da(regs, esr);  in el0t_64_sync_handler()
661 		el0_ia(regs, esr);  in el0t_64_sync_handler()
664 		el0_fpsimd_acc(regs, esr);  in el0t_64_sync_handler()
667 		el0_sve_acc(regs, esr);  in el0t_64_sync_handler()
670 		el0_sme_acc(regs, esr);  in el0t_64_sync_handler()
673 		el0_fpsimd_exc(regs, esr);  in el0t_64_sync_handler()
677 		el0_sys(regs, esr);  in el0t_64_sync_handler()
680 		el0_sp(regs, esr);  in el0t_64_sync_handler()
683 		el0_pc(regs, esr);  in el0t_64_sync_handler()
686 		el0_undef(regs, esr);  in el0t_64_sync_handler()
689 		el0_bti(regs);  in el0t_64_sync_handler()
695 		el0_dbg(regs, esr);  in el0t_64_sync_handler()
698 		el0_fpac(regs, esr);  in el0t_64_sync_handler()
701 		el0_inv(regs, esr);  in el0t_64_sync_handler()
705 static void noinstr el0_interrupt(struct pt_regs *regs,  in el0_interrupt()  argument
708 	enter_from_user_mode(regs);  in el0_interrupt()
712 	if (regs->pc & BIT(55))  in el0_interrupt()
716 	do_interrupt_handler(regs, handler);  in el0_interrupt()
719 	exit_to_user_mode(regs);  in el0_interrupt()
722 static void noinstr __el0_irq_handler_common(struct pt_regs *regs)  in __el0_irq_handler_common()  argument
724 	el0_interrupt(regs, handle_arch_irq);  in __el0_irq_handler_common()
727 asmlinkage void noinstr el0t_64_irq_handler(struct pt_regs *regs)  in el0t_64_irq_handler()  argument
729 	__el0_irq_handler_common(regs);  in el0t_64_irq_handler()
732 static void noinstr __el0_fiq_handler_common(struct pt_regs *regs)  in __el0_fiq_handler_common()  argument
734 	el0_interrupt(regs, handle_arch_fiq);  in __el0_fiq_handler_common()
737 asmlinkage void noinstr el0t_64_fiq_handler(struct pt_regs *regs)  in el0t_64_fiq_handler()  argument
739 	__el0_fiq_handler_common(regs);  in el0t_64_fiq_handler()
742 static void noinstr __el0_error_handler_common(struct pt_regs *regs)  in __el0_error_handler_common()  argument
746 	enter_from_user_mode(regs);  in __el0_error_handler_common()
748 	arm64_enter_nmi(regs);  in __el0_error_handler_common()
749 	do_serror(regs, esr);  in __el0_error_handler_common()
750 	arm64_exit_nmi(regs);  in __el0_error_handler_common()
752 	exit_to_user_mode(regs);  in __el0_error_handler_common()
755 asmlinkage void noinstr el0t_64_error_handler(struct pt_regs *regs)  in el0t_64_error_handler()  argument
757 	__el0_error_handler_common(regs);  in el0t_64_error_handler()
761 static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr)  in el0_cp15()  argument
763 	enter_from_user_mode(regs);  in el0_cp15()
765 	do_cp15instr(esr, regs);  in el0_cp15()
766 	exit_to_user_mode(regs);  in el0_cp15()
769 static void noinstr el0_svc_compat(struct pt_regs *regs)  in el0_svc_compat()  argument
771 	enter_from_user_mode(regs);  in el0_svc_compat()
773 	do_el0_svc_compat(regs);  in el0_svc_compat()
774 	exit_to_user_mode(regs);  in el0_svc_compat()
777 asmlinkage void noinstr el0t_32_sync_handler(struct pt_regs *regs)  in el0t_32_sync_handler()  argument
783 		el0_svc_compat(regs);  in el0t_32_sync_handler()
786 		el0_da(regs, esr);  in el0t_32_sync_handler()
789 		el0_ia(regs, esr);  in el0t_32_sync_handler()
792 		el0_fpsimd_acc(regs, esr);  in el0t_32_sync_handler()
795 		el0_fpsimd_exc(regs, esr);  in el0t_32_sync_handler()
798 		el0_pc(regs, esr);  in el0t_32_sync_handler()
804 		el0_undef(regs, esr);  in el0t_32_sync_handler()
808 		el0_cp15(regs, esr);  in el0t_32_sync_handler()
814 		el0_dbg(regs, esr);  in el0t_32_sync_handler()
817 		el0_inv(regs, esr);  in el0t_32_sync_handler()
821 asmlinkage void noinstr el0t_32_irq_handler(struct pt_regs *regs)  in el0t_32_irq_handler()  argument
823 	__el0_irq_handler_common(regs);  in el0t_32_irq_handler()
826 asmlinkage void noinstr el0t_32_fiq_handler(struct pt_regs *regs)  in el0t_32_fiq_handler()  argument
828 	__el0_fiq_handler_common(regs);  in el0t_32_fiq_handler()
831 asmlinkage void noinstr el0t_32_error_handler(struct pt_regs *regs)  in el0t_32_error_handler()  argument
833 	__el0_error_handler_common(regs);  in el0t_32_error_handler()
843 asmlinkage void noinstr handle_bad_stack(struct pt_regs *regs)  in handle_bad_stack()  argument
848 	arm64_enter_nmi(regs);  in handle_bad_stack()
849 	panic_bad_stack(regs, esr, far);  in handle_bad_stack()
855 __sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg)  in __sdei_handler()  argument
881 	arm64_enter_nmi(regs);  in __sdei_handler()
882 	ret = do_sdei_event(regs, arg);  in __sdei_handler()
883 	arm64_exit_nmi(regs);  in __sdei_handler()