1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Exception handling code
4  *
5  * Copyright (C) 2019 ARM Ltd.
6  */
7 
8 #include <linux/context_tracking.h>
9 #include <linux/ptrace.h>
10 #include <linux/thread_info.h>
11 
12 #include <asm/cpufeature.h>
13 #include <asm/daifflags.h>
14 #include <asm/esr.h>
15 #include <asm/exception.h>
16 #include <asm/kprobes.h>
17 #include <asm/mmu.h>
18 #include <asm/sysreg.h>
19 
20 /*
21  * This is intended to match the logic in irqentry_enter(), handling the kernel
22  * mode transitions only.
23  */
enter_from_kernel_mode(struct pt_regs * regs)24 static void noinstr enter_from_kernel_mode(struct pt_regs *regs)
25 {
26 	regs->exit_rcu = false;
27 
28 	if (!IS_ENABLED(CONFIG_TINY_RCU) && is_idle_task(current)) {
29 		lockdep_hardirqs_off(CALLER_ADDR0);
30 		rcu_irq_enter();
31 		trace_hardirqs_off_finish();
32 
33 		regs->exit_rcu = true;
34 		return;
35 	}
36 
37 	lockdep_hardirqs_off(CALLER_ADDR0);
38 	rcu_irq_enter_check_tick();
39 	trace_hardirqs_off_finish();
40 }
41 
42 /*
43  * This is intended to match the logic in irqentry_exit(), handling the kernel
44  * mode transitions only, and with preemption handled elsewhere.
45  */
exit_to_kernel_mode(struct pt_regs * regs)46 static void noinstr exit_to_kernel_mode(struct pt_regs *regs)
47 {
48 	lockdep_assert_irqs_disabled();
49 
50 	if (interrupts_enabled(regs)) {
51 		if (regs->exit_rcu) {
52 			trace_hardirqs_on_prepare();
53 			lockdep_hardirqs_on_prepare(CALLER_ADDR0);
54 			rcu_irq_exit();
55 			lockdep_hardirqs_on(CALLER_ADDR0);
56 			return;
57 		}
58 
59 		trace_hardirqs_on();
60 	} else {
61 		if (regs->exit_rcu)
62 			rcu_irq_exit();
63 	}
64 }
65 
arm64_enter_nmi(struct pt_regs * regs)66 void noinstr arm64_enter_nmi(struct pt_regs *regs)
67 {
68 	regs->lockdep_hardirqs = lockdep_hardirqs_enabled();
69 
70 	__nmi_enter();
71 	lockdep_hardirqs_off(CALLER_ADDR0);
72 	lockdep_hardirq_enter();
73 	rcu_nmi_enter();
74 
75 	trace_hardirqs_off_finish();
76 	ftrace_nmi_enter();
77 }
78 
arm64_exit_nmi(struct pt_regs * regs)79 void noinstr arm64_exit_nmi(struct pt_regs *regs)
80 {
81 	bool restore = regs->lockdep_hardirqs;
82 
83 	ftrace_nmi_exit();
84 	if (restore) {
85 		trace_hardirqs_on_prepare();
86 		lockdep_hardirqs_on_prepare(CALLER_ADDR0);
87 	}
88 
89 	rcu_nmi_exit();
90 	lockdep_hardirq_exit();
91 	if (restore)
92 		lockdep_hardirqs_on(CALLER_ADDR0);
93 	__nmi_exit();
94 }
95 
enter_el1_irq_or_nmi(struct pt_regs * regs)96 asmlinkage void noinstr enter_el1_irq_or_nmi(struct pt_regs *regs)
97 {
98 	if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs))
99 		arm64_enter_nmi(regs);
100 	else
101 		enter_from_kernel_mode(regs);
102 }
103 
exit_el1_irq_or_nmi(struct pt_regs * regs)104 asmlinkage void noinstr exit_el1_irq_or_nmi(struct pt_regs *regs)
105 {
106 	if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs))
107 		arm64_exit_nmi(regs);
108 	else
109 		exit_to_kernel_mode(regs);
110 }
111 
el1_abort(struct pt_regs * regs,unsigned long esr)112 static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr)
113 {
114 	unsigned long far = read_sysreg(far_el1);
115 
116 	enter_from_kernel_mode(regs);
117 	local_daif_inherit(regs);
118 	far = untagged_addr(far);
119 	do_mem_abort(far, esr, regs);
120 	local_daif_mask();
121 	exit_to_kernel_mode(regs);
122 }
123 
el1_pc(struct pt_regs * regs,unsigned long esr)124 static void noinstr el1_pc(struct pt_regs *regs, unsigned long esr)
125 {
126 	unsigned long far = read_sysreg(far_el1);
127 
128 	enter_from_kernel_mode(regs);
129 	local_daif_inherit(regs);
130 	do_sp_pc_abort(far, esr, regs);
131 	local_daif_mask();
132 	exit_to_kernel_mode(regs);
133 }
134 
el1_undef(struct pt_regs * regs)135 static void noinstr el1_undef(struct pt_regs *regs)
136 {
137 	enter_from_kernel_mode(regs);
138 	local_daif_inherit(regs);
139 	do_undefinstr(regs);
140 	local_daif_mask();
141 	exit_to_kernel_mode(regs);
142 }
143 
el1_inv(struct pt_regs * regs,unsigned long esr)144 static void noinstr el1_inv(struct pt_regs *regs, unsigned long esr)
145 {
146 	enter_from_kernel_mode(regs);
147 	local_daif_inherit(regs);
148 	bad_mode(regs, 0, esr);
149 	local_daif_mask();
150 	exit_to_kernel_mode(regs);
151 }
152 
arm64_enter_el1_dbg(struct pt_regs * regs)153 static void noinstr arm64_enter_el1_dbg(struct pt_regs *regs)
154 {
155 	regs->lockdep_hardirqs = lockdep_hardirqs_enabled();
156 
157 	lockdep_hardirqs_off(CALLER_ADDR0);
158 	rcu_nmi_enter();
159 
160 	trace_hardirqs_off_finish();
161 }
162 
arm64_exit_el1_dbg(struct pt_regs * regs)163 static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs)
164 {
165 	bool restore = regs->lockdep_hardirqs;
166 
167 	if (restore) {
168 		trace_hardirqs_on_prepare();
169 		lockdep_hardirqs_on_prepare(CALLER_ADDR0);
170 	}
171 
172 	rcu_nmi_exit();
173 	if (restore)
174 		lockdep_hardirqs_on(CALLER_ADDR0);
175 }
176 
el1_dbg(struct pt_regs * regs,unsigned long esr)177 static void noinstr el1_dbg(struct pt_regs *regs, unsigned long esr)
178 {
179 	unsigned long far = read_sysreg(far_el1);
180 
181 	/*
182 	 * The CPU masked interrupts, and we are leaving them masked during
183 	 * do_debug_exception(). Update PMR as if we had called
184 	 * local_daif_mask().
185 	 */
186 	if (system_uses_irq_prio_masking())
187 		gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);
188 
189 	arm64_enter_el1_dbg(regs);
190 	do_debug_exception(far, esr, regs);
191 	arm64_exit_el1_dbg(regs);
192 }
193 
el1_fpac(struct pt_regs * regs,unsigned long esr)194 static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr)
195 {
196 	enter_from_kernel_mode(regs);
197 	local_daif_inherit(regs);
198 	do_ptrauth_fault(regs, esr);
199 	local_daif_mask();
200 	exit_to_kernel_mode(regs);
201 }
202 
el1_sync_handler(struct pt_regs * regs)203 asmlinkage void noinstr el1_sync_handler(struct pt_regs *regs)
204 {
205 	unsigned long esr = read_sysreg(esr_el1);
206 
207 	switch (ESR_ELx_EC(esr)) {
208 	case ESR_ELx_EC_DABT_CUR:
209 	case ESR_ELx_EC_IABT_CUR:
210 		el1_abort(regs, esr);
211 		break;
212 	/*
213 	 * We don't handle ESR_ELx_EC_SP_ALIGN, since we will have hit a
214 	 * recursive exception when trying to push the initial pt_regs.
215 	 */
216 	case ESR_ELx_EC_PC_ALIGN:
217 		el1_pc(regs, esr);
218 		break;
219 	case ESR_ELx_EC_SYS64:
220 	case ESR_ELx_EC_UNKNOWN:
221 		el1_undef(regs);
222 		break;
223 	case ESR_ELx_EC_BREAKPT_CUR:
224 	case ESR_ELx_EC_SOFTSTP_CUR:
225 	case ESR_ELx_EC_WATCHPT_CUR:
226 	case ESR_ELx_EC_BRK64:
227 		el1_dbg(regs, esr);
228 		break;
229 	case ESR_ELx_EC_FPAC:
230 		el1_fpac(regs, esr);
231 		break;
232 	default:
233 		el1_inv(regs, esr);
234 	}
235 }
236 
enter_from_user_mode(void)237 asmlinkage void noinstr enter_from_user_mode(void)
238 {
239 	lockdep_hardirqs_off(CALLER_ADDR0);
240 	CT_WARN_ON(ct_state() != CONTEXT_USER);
241 	user_exit_irqoff();
242 	trace_hardirqs_off_finish();
243 }
244 
exit_to_user_mode(void)245 asmlinkage void noinstr exit_to_user_mode(void)
246 {
247 	trace_hardirqs_on_prepare();
248 	lockdep_hardirqs_on_prepare(CALLER_ADDR0);
249 	user_enter_irqoff();
250 	lockdep_hardirqs_on(CALLER_ADDR0);
251 }
252 
el0_da(struct pt_regs * regs,unsigned long esr)253 static void noinstr el0_da(struct pt_regs *regs, unsigned long esr)
254 {
255 	unsigned long far = read_sysreg(far_el1);
256 
257 	enter_from_user_mode();
258 	local_daif_restore(DAIF_PROCCTX);
259 	far = untagged_addr(far);
260 	do_mem_abort(far, esr, regs);
261 }
262 
el0_ia(struct pt_regs * regs,unsigned long esr)263 static void noinstr el0_ia(struct pt_regs *regs, unsigned long esr)
264 {
265 	unsigned long far = read_sysreg(far_el1);
266 
267 	/*
268 	 * We've taken an instruction abort from userspace and not yet
269 	 * re-enabled IRQs. If the address is a kernel address, apply
270 	 * BP hardening prior to enabling IRQs and pre-emption.
271 	 */
272 	if (!is_ttbr0_addr(far))
273 		arm64_apply_bp_hardening();
274 
275 	enter_from_user_mode();
276 	local_daif_restore(DAIF_PROCCTX);
277 	do_mem_abort(far, esr, regs);
278 }
279 
el0_fpsimd_acc(struct pt_regs * regs,unsigned long esr)280 static void noinstr el0_fpsimd_acc(struct pt_regs *regs, unsigned long esr)
281 {
282 	enter_from_user_mode();
283 	local_daif_restore(DAIF_PROCCTX);
284 	do_fpsimd_acc(esr, regs);
285 }
286 
el0_sve_acc(struct pt_regs * regs,unsigned long esr)287 static void noinstr el0_sve_acc(struct pt_regs *regs, unsigned long esr)
288 {
289 	enter_from_user_mode();
290 	local_daif_restore(DAIF_PROCCTX);
291 	do_sve_acc(esr, regs);
292 }
293 
el0_fpsimd_exc(struct pt_regs * regs,unsigned long esr)294 static void noinstr el0_fpsimd_exc(struct pt_regs *regs, unsigned long esr)
295 {
296 	enter_from_user_mode();
297 	local_daif_restore(DAIF_PROCCTX);
298 	do_fpsimd_exc(esr, regs);
299 }
300 
el0_sys(struct pt_regs * regs,unsigned long esr)301 static void noinstr el0_sys(struct pt_regs *regs, unsigned long esr)
302 {
303 	enter_from_user_mode();
304 	local_daif_restore(DAIF_PROCCTX);
305 	do_sysinstr(esr, regs);
306 }
307 
el0_pc(struct pt_regs * regs,unsigned long esr)308 static void noinstr el0_pc(struct pt_regs *regs, unsigned long esr)
309 {
310 	unsigned long far = read_sysreg(far_el1);
311 
312 	if (!is_ttbr0_addr(instruction_pointer(regs)))
313 		arm64_apply_bp_hardening();
314 
315 	enter_from_user_mode();
316 	local_daif_restore(DAIF_PROCCTX);
317 	do_sp_pc_abort(far, esr, regs);
318 }
319 
el0_sp(struct pt_regs * regs,unsigned long esr)320 static void noinstr el0_sp(struct pt_regs *regs, unsigned long esr)
321 {
322 	enter_from_user_mode();
323 	local_daif_restore(DAIF_PROCCTX);
324 	do_sp_pc_abort(regs->sp, esr, regs);
325 }
326 
el0_undef(struct pt_regs * regs)327 static void noinstr el0_undef(struct pt_regs *regs)
328 {
329 	enter_from_user_mode();
330 	local_daif_restore(DAIF_PROCCTX);
331 	do_undefinstr(regs);
332 }
333 
el0_bti(struct pt_regs * regs)334 static void noinstr el0_bti(struct pt_regs *regs)
335 {
336 	enter_from_user_mode();
337 	local_daif_restore(DAIF_PROCCTX);
338 	do_bti(regs);
339 }
340 
el0_inv(struct pt_regs * regs,unsigned long esr)341 static void noinstr el0_inv(struct pt_regs *regs, unsigned long esr)
342 {
343 	enter_from_user_mode();
344 	local_daif_restore(DAIF_PROCCTX);
345 	bad_el0_sync(regs, 0, esr);
346 }
347 
el0_dbg(struct pt_regs * regs,unsigned long esr)348 static void noinstr el0_dbg(struct pt_regs *regs, unsigned long esr)
349 {
350 	/* Only watchpoints write FAR_EL1, otherwise its UNKNOWN */
351 	unsigned long far = read_sysreg(far_el1);
352 
353 	if (system_uses_irq_prio_masking())
354 		gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);
355 
356 	enter_from_user_mode();
357 	do_debug_exception(far, esr, regs);
358 	local_daif_restore(DAIF_PROCCTX_NOIRQ);
359 }
360 
el0_svc(struct pt_regs * regs)361 static void noinstr el0_svc(struct pt_regs *regs)
362 {
363 	if (system_uses_irq_prio_masking())
364 		gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);
365 
366 	enter_from_user_mode();
367 	do_el0_svc(regs);
368 }
369 
el0_fpac(struct pt_regs * regs,unsigned long esr)370 static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr)
371 {
372 	enter_from_user_mode();
373 	local_daif_restore(DAIF_PROCCTX);
374 	do_ptrauth_fault(regs, esr);
375 }
376 
el0_sync_handler(struct pt_regs * regs)377 asmlinkage void noinstr el0_sync_handler(struct pt_regs *regs)
378 {
379 	unsigned long esr = read_sysreg(esr_el1);
380 
381 	switch (ESR_ELx_EC(esr)) {
382 	case ESR_ELx_EC_SVC64:
383 		el0_svc(regs);
384 		break;
385 	case ESR_ELx_EC_DABT_LOW:
386 		el0_da(regs, esr);
387 		break;
388 	case ESR_ELx_EC_IABT_LOW:
389 		el0_ia(regs, esr);
390 		break;
391 	case ESR_ELx_EC_FP_ASIMD:
392 		el0_fpsimd_acc(regs, esr);
393 		break;
394 	case ESR_ELx_EC_SVE:
395 		el0_sve_acc(regs, esr);
396 		break;
397 	case ESR_ELx_EC_FP_EXC64:
398 		el0_fpsimd_exc(regs, esr);
399 		break;
400 	case ESR_ELx_EC_SYS64:
401 	case ESR_ELx_EC_WFx:
402 		el0_sys(regs, esr);
403 		break;
404 	case ESR_ELx_EC_SP_ALIGN:
405 		el0_sp(regs, esr);
406 		break;
407 	case ESR_ELx_EC_PC_ALIGN:
408 		el0_pc(regs, esr);
409 		break;
410 	case ESR_ELx_EC_UNKNOWN:
411 		el0_undef(regs);
412 		break;
413 	case ESR_ELx_EC_BTI:
414 		el0_bti(regs);
415 		break;
416 	case ESR_ELx_EC_BREAKPT_LOW:
417 	case ESR_ELx_EC_SOFTSTP_LOW:
418 	case ESR_ELx_EC_WATCHPT_LOW:
419 	case ESR_ELx_EC_BRK64:
420 		el0_dbg(regs, esr);
421 		break;
422 	case ESR_ELx_EC_FPAC:
423 		el0_fpac(regs, esr);
424 		break;
425 	default:
426 		el0_inv(regs, esr);
427 	}
428 }
429 
430 #ifdef CONFIG_COMPAT
el0_cp15(struct pt_regs * regs,unsigned long esr)431 static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr)
432 {
433 	enter_from_user_mode();
434 	local_daif_restore(DAIF_PROCCTX);
435 	do_cp15instr(esr, regs);
436 }
437 
el0_svc_compat(struct pt_regs * regs)438 static void noinstr el0_svc_compat(struct pt_regs *regs)
439 {
440 	if (system_uses_irq_prio_masking())
441 		gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);
442 
443 	enter_from_user_mode();
444 	do_el0_svc_compat(regs);
445 }
446 
el0_sync_compat_handler(struct pt_regs * regs)447 asmlinkage void noinstr el0_sync_compat_handler(struct pt_regs *regs)
448 {
449 	unsigned long esr = read_sysreg(esr_el1);
450 
451 	switch (ESR_ELx_EC(esr)) {
452 	case ESR_ELx_EC_SVC32:
453 		el0_svc_compat(regs);
454 		break;
455 	case ESR_ELx_EC_DABT_LOW:
456 		el0_da(regs, esr);
457 		break;
458 	case ESR_ELx_EC_IABT_LOW:
459 		el0_ia(regs, esr);
460 		break;
461 	case ESR_ELx_EC_FP_ASIMD:
462 		el0_fpsimd_acc(regs, esr);
463 		break;
464 	case ESR_ELx_EC_FP_EXC32:
465 		el0_fpsimd_exc(regs, esr);
466 		break;
467 	case ESR_ELx_EC_PC_ALIGN:
468 		el0_pc(regs, esr);
469 		break;
470 	case ESR_ELx_EC_UNKNOWN:
471 	case ESR_ELx_EC_CP14_MR:
472 	case ESR_ELx_EC_CP14_LS:
473 	case ESR_ELx_EC_CP14_64:
474 		el0_undef(regs);
475 		break;
476 	case ESR_ELx_EC_CP15_32:
477 	case ESR_ELx_EC_CP15_64:
478 		el0_cp15(regs, esr);
479 		break;
480 	case ESR_ELx_EC_BREAKPT_LOW:
481 	case ESR_ELx_EC_SOFTSTP_LOW:
482 	case ESR_ELx_EC_WATCHPT_LOW:
483 	case ESR_ELx_EC_BKPT32:
484 		el0_dbg(regs, esr);
485 		break;
486 	default:
487 		el0_inv(regs, esr);
488 	}
489 }
490 #endif /* CONFIG_COMPAT */
491