1 /*
2 * Copyright (c) 2017-2024, Arm Limited and Contributors. All rights reserved.
3 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6
7 /*
8 * Exception handlers at EL3, their priority levels, and management.
9 */
10
11 #include <assert.h>
12 #include <stdbool.h>
13
14 #include <bl31/ehf.h>
15 #include <bl31/interrupt_mgmt.h>
16 #include <context.h>
17 #include <common/debug.h>
18 #include <drivers/arm/gic_common.h>
19 #include <lib/el3_runtime/context_mgmt.h>
20 #include <lib/el3_runtime/cpu_data.h>
21 #include <lib/el3_runtime/pubsub_events.h>
22 #include <plat/common/platform.h>
23
24 /* Output EHF logs as verbose */
25 #define EHF_LOG(...) VERBOSE("EHF: " __VA_ARGS__)
26
27 #define EHF_INVALID_IDX (-1)
28
29 /* For a valid handler, return the actual function pointer; otherwise, 0. */
30 #define RAW_HANDLER(h) \
31 ((ehf_handler_t) ((((h) & EHF_PRI_VALID_) != 0U) ? \
32 ((h) & ~EHF_PRI_VALID_) : 0U))
33
34 #define PRI_BIT(idx) (((ehf_pri_bits_t) 1u) << (idx))
35
36 /*
37 * Convert index into secure priority using the platform-defined priority bits
38 * field.
39 */
40 #define IDX_TO_PRI(idx) \
41 ((((unsigned) idx) << (7u - exception_data.pri_bits)) & 0x7fU)
42
43 /* Check whether a given index is valid */
44 #define IS_IDX_VALID(idx) \
45 ((exception_data.ehf_priorities[idx].ehf_handler & EHF_PRI_VALID_) != 0U)
46
47 /* Returns whether given priority is in secure priority range */
48 #define IS_PRI_SECURE(pri) (((pri) & 0x80U) == 0U)
49
50 /* To be defined by the platform */
51 extern const ehf_priorities_t exception_data;
52
53 /* Translate priority to the index in the priority array */
pri_to_idx(unsigned int priority)54 static unsigned int pri_to_idx(unsigned int priority)
55 {
56 unsigned int idx;
57
58 idx = EHF_PRI_TO_IDX(priority, exception_data.pri_bits);
59 assert(idx < exception_data.num_priorities);
60 assert(IS_IDX_VALID(idx));
61
62 return idx;
63 }
64
65 /* Return whether there are outstanding priority activation */
has_valid_pri_activations(pe_exc_data_t * pe_data)66 static bool has_valid_pri_activations(pe_exc_data_t *pe_data)
67 {
68 return pe_data->active_pri_bits != 0U;
69 }
70
this_cpu_data(void)71 static pe_exc_data_t *this_cpu_data(void)
72 {
73 return &get_cpu_data(ehf_data);
74 }
75
76 /*
77 * Return the current priority index of this CPU. If no priority is active,
78 * return EHF_INVALID_IDX.
79 */
get_pe_highest_active_idx(pe_exc_data_t * pe_data)80 static int get_pe_highest_active_idx(pe_exc_data_t *pe_data)
81 {
82 if (!has_valid_pri_activations(pe_data))
83 return EHF_INVALID_IDX;
84
85 /* Current priority is the right-most bit */
86 return (int) __builtin_ctz(pe_data->active_pri_bits);
87 }
88
89 /*
90 * Mark priority active by setting the corresponding bit in active_pri_bits and
91 * programming the priority mask.
92 *
93 * This API is to be used as part of delegating to lower ELs other than for
94 * interrupts; e.g. while handling synchronous exceptions.
95 *
96 * This API is expected to be invoked before restoring context (Secure or
97 * Non-secure) in preparation for the respective dispatch.
98 */
ehf_activate_priority(unsigned int priority)99 void ehf_activate_priority(unsigned int priority)
100 {
101 int cur_pri_idx;
102 unsigned int old_mask, run_pri, idx;
103 pe_exc_data_t *pe_data = this_cpu_data();
104
105 /*
106 * Query interrupt controller for the running priority, or idle priority
107 * if no interrupts are being handled. The requested priority must be
108 * less (higher priority) than the active running priority.
109 */
110 run_pri = plat_ic_get_running_priority();
111 if (priority >= run_pri) {
112 ERROR("Running priority higher (0x%x) than requested (0x%x)\n",
113 run_pri, priority);
114 panic();
115 }
116
117 /*
118 * If there were priority activations already, the requested priority
119 * must be less (higher priority) than the current highest priority
120 * activation so far.
121 */
122 cur_pri_idx = get_pe_highest_active_idx(pe_data);
123 idx = pri_to_idx(priority);
124 if ((cur_pri_idx != EHF_INVALID_IDX) &&
125 (idx >= ((unsigned int) cur_pri_idx))) {
126 ERROR("Activation priority mismatch: req=0x%x current=0x%x\n",
127 priority, IDX_TO_PRI(cur_pri_idx));
128 panic();
129 }
130
131 /* Set the bit corresponding to the requested priority */
132 pe_data->active_pri_bits |= PRI_BIT(idx);
133
134 /*
135 * Program priority mask for the activated level. Check that the new
136 * priority mask is setting a higher priority level than the existing
137 * mask.
138 */
139 old_mask = plat_ic_set_priority_mask(priority);
140 if (priority >= old_mask) {
141 ERROR("Requested priority (0x%x) lower than Priority Mask (0x%x)\n",
142 priority, old_mask);
143 panic();
144 }
145
146 /*
147 * If this is the first activation, save the priority mask. This will be
148 * restored after the last deactivation.
149 */
150 if (cur_pri_idx == EHF_INVALID_IDX)
151 pe_data->init_pri_mask = (uint8_t) old_mask;
152
153 EHF_LOG("activate prio=%d\n", get_pe_highest_active_idx(pe_data));
154 }
155
156 /*
157 * Mark priority inactive by clearing the corresponding bit in active_pri_bits,
158 * and programming the priority mask.
159 *
160 * This API is expected to be used as part of delegating to to lower ELs other
161 * than for interrupts; e.g. while handling synchronous exceptions.
162 *
163 * This API is expected to be invoked after saving context (Secure or
164 * Non-secure), having concluded the respective dispatch.
165 */
ehf_deactivate_priority(unsigned int priority)166 void ehf_deactivate_priority(unsigned int priority)
167 {
168 int cur_pri_idx;
169 pe_exc_data_t *pe_data = this_cpu_data();
170 unsigned int old_mask, run_pri, idx;
171
172 /*
173 * Query interrupt controller for the running priority, or idle priority
174 * if no interrupts are being handled. The requested priority must be
175 * less (higher priority) than the active running priority.
176 */
177 run_pri = plat_ic_get_running_priority();
178 if (priority >= run_pri) {
179 ERROR("Running priority higher (0x%x) than requested (0x%x)\n",
180 run_pri, priority);
181 panic();
182 }
183
184 /*
185 * Deactivation is allowed only when there are priority activations, and
186 * the deactivation priority level must match the current activated
187 * priority.
188 */
189 cur_pri_idx = get_pe_highest_active_idx(pe_data);
190 idx = pri_to_idx(priority);
191 if ((cur_pri_idx == EHF_INVALID_IDX) ||
192 (idx != ((unsigned int) cur_pri_idx))) {
193 ERROR("Deactivation priority mismatch: req=0x%x current=0x%x\n",
194 priority, IDX_TO_PRI(cur_pri_idx));
195 panic();
196 }
197
198 /* Clear bit corresponding to highest priority */
199 pe_data->active_pri_bits &= (pe_data->active_pri_bits - 1u);
200
201 /*
202 * Restore priority mask corresponding to the next priority, or the
203 * one stashed earlier if there are no more to deactivate.
204 */
205 cur_pri_idx = get_pe_highest_active_idx(pe_data);
206
207 #if GIC600_ERRATA_WA_2384374
208 if (cur_pri_idx == EHF_INVALID_IDX) {
209 old_mask = plat_ic_deactivate_priority(pe_data->init_pri_mask);
210 } else {
211 old_mask = plat_ic_deactivate_priority(priority);
212 }
213 #else
214 if (cur_pri_idx == EHF_INVALID_IDX) {
215 old_mask = plat_ic_set_priority_mask(pe_data->init_pri_mask);
216 } else {
217 old_mask = plat_ic_set_priority_mask(priority);
218 }
219 #endif
220
221 if (old_mask > priority) {
222 ERROR("Deactivation priority (0x%x) lower than Priority Mask (0x%x)\n",
223 priority, old_mask);
224 panic();
225 }
226
227 EHF_LOG("deactivate prio=%d\n", get_pe_highest_active_idx(pe_data));
228 }
229
230 /*
231 * After leaving Non-secure world, stash current Non-secure Priority Mask, and
232 * set Priority Mask to the highest Non-secure priority so that Non-secure
233 * interrupts cannot preempt Secure execution.
234 *
235 * If the current running priority is in the secure range, or if there are
236 * outstanding priority activations, this function does nothing.
237 *
238 * This function subscribes to the 'cm_exited_normal_world' event published by
239 * the Context Management Library.
240 */
ehf_exited_normal_world(const void * arg)241 static void *ehf_exited_normal_world(const void *arg)
242 {
243 unsigned int run_pri;
244 pe_exc_data_t *pe_data = this_cpu_data();
245
246 /* If the running priority is in the secure range, do nothing */
247 run_pri = plat_ic_get_running_priority();
248 if (IS_PRI_SECURE(run_pri))
249 return NULL;
250
251 /* Do nothing if there are explicit activations */
252 if (has_valid_pri_activations(pe_data))
253 return NULL;
254
255 assert(pe_data->ns_pri_mask == 0u);
256
257 pe_data->ns_pri_mask =
258 (uint8_t) plat_ic_set_priority_mask(GIC_HIGHEST_NS_PRIORITY);
259
260 /* The previous Priority Mask is not expected to be in secure range */
261 if (IS_PRI_SECURE(pe_data->ns_pri_mask)) {
262 ERROR("Priority Mask (0x%x) already in secure range\n",
263 pe_data->ns_pri_mask);
264 panic();
265 }
266
267 EHF_LOG("Priority Mask: 0x%x => 0x%x\n", pe_data->ns_pri_mask,
268 GIC_HIGHEST_NS_PRIORITY);
269
270 return NULL;
271 }
272
273 /*
274 * Conclude Secure execution and prepare for return to Non-secure world. Restore
275 * the Non-secure Priority Mask previously stashed upon leaving Non-secure
276 * world.
277 *
278 * If there the current running priority is in the secure range, or if there are
279 * outstanding priority activations, this function does nothing.
280 *
281 * This function subscribes to the 'cm_entering_normal_world' event published by
282 * the Context Management Library.
283 */
ehf_entering_normal_world(const void * arg)284 static void *ehf_entering_normal_world(const void *arg)
285 {
286 unsigned int old_pmr, run_pri;
287 pe_exc_data_t *pe_data = this_cpu_data();
288
289 /* If the running priority is in the secure range, do nothing */
290 run_pri = plat_ic_get_running_priority();
291 if (IS_PRI_SECURE(run_pri))
292 return NULL;
293
294 /*
295 * If there are explicit activations, do nothing. The Priority Mask will
296 * be restored upon the last deactivation.
297 */
298 if (has_valid_pri_activations(pe_data))
299 return NULL;
300
301 /* Do nothing if we don't have a valid Priority Mask to restore */
302 if (pe_data->ns_pri_mask == 0U)
303 return NULL;
304
305 old_pmr = plat_ic_set_priority_mask(pe_data->ns_pri_mask);
306
307 /*
308 * When exiting secure world, the current Priority Mask must be
309 * GIC_HIGHEST_NS_PRIORITY (as set during entry), or the Non-secure
310 * priority mask set upon calling ehf_allow_ns_preemption()
311 */
312 if ((old_pmr != GIC_HIGHEST_NS_PRIORITY) &&
313 (old_pmr != pe_data->ns_pri_mask)) {
314 ERROR("Invalid Priority Mask (0x%x) restored\n", old_pmr);
315 panic();
316 }
317
318 EHF_LOG("Priority Mask: 0x%x => 0x%x\n", old_pmr, pe_data->ns_pri_mask);
319
320 pe_data->ns_pri_mask = 0;
321
322 return NULL;
323 }
324
325 /*
326 * Program Priority Mask to the original Non-secure priority such that
327 * Non-secure interrupts may preempt Secure execution (for example, during
328 * Yielding SMC calls). The 'preempt_ret_code' parameter indicates the Yielding
329 * SMC's return value in case the call was preempted.
330 *
331 * This API is expected to be invoked before delegating a yielding SMC to Secure
332 * EL1. I.e. within the window of secure execution after Non-secure context is
333 * saved (after entry into EL3) and Secure context is restored (before entering
334 * Secure EL1).
335 */
ehf_allow_ns_preemption(uint64_t preempt_ret_code)336 void ehf_allow_ns_preemption(uint64_t preempt_ret_code)
337 {
338 cpu_context_t *ns_ctx;
339 unsigned int old_pmr __unused;
340 pe_exc_data_t *pe_data = this_cpu_data();
341
342 /*
343 * We should have been notified earlier of entering secure world, and
344 * therefore have stashed the Non-secure priority mask.
345 */
346 assert(pe_data->ns_pri_mask != 0U);
347
348 /* Make sure no priority levels are active when requesting this */
349 if (has_valid_pri_activations(pe_data)) {
350 ERROR("PE %lx has priority activations: 0x%x\n",
351 read_mpidr_el1(), pe_data->active_pri_bits);
352 panic();
353 }
354
355 /*
356 * Program preempted return code to x0 right away so that, if the
357 * Yielding SMC was indeed preempted before a dispatcher gets a chance
358 * to populate it, the caller would find the correct return value.
359 */
360 ns_ctx = cm_get_context(NON_SECURE);
361 assert(ns_ctx != NULL);
362 write_ctx_reg(get_gpregs_ctx(ns_ctx), CTX_GPREG_X0, preempt_ret_code);
363
364 old_pmr = plat_ic_set_priority_mask(pe_data->ns_pri_mask);
365
366 EHF_LOG("Priority Mask: 0x%x => 0x%x\n", old_pmr, pe_data->ns_pri_mask);
367
368 pe_data->ns_pri_mask = 0;
369 }
370
371 /*
372 * Return whether Secure execution has explicitly allowed Non-secure interrupts
373 * to preempt itself (for example, during Yielding SMC calls).
374 */
ehf_is_ns_preemption_allowed(void)375 unsigned int ehf_is_ns_preemption_allowed(void)
376 {
377 unsigned int run_pri;
378 pe_exc_data_t *pe_data = this_cpu_data();
379
380 /* If running priority is in secure range, return false */
381 run_pri = plat_ic_get_running_priority();
382 if (IS_PRI_SECURE(run_pri))
383 return 0;
384
385 /*
386 * If Non-secure preemption was permitted by calling
387 * ehf_allow_ns_preemption() earlier:
388 *
389 * - There wouldn't have been priority activations;
390 * - We would have cleared the stashed the Non-secure Priority Mask.
391 */
392 if (has_valid_pri_activations(pe_data))
393 return 0;
394 if (pe_data->ns_pri_mask != 0U)
395 return 0;
396
397 return 1;
398 }
399
400 /*
401 * Top-level EL3 interrupt handler.
402 */
ehf_el3_interrupt_handler(uint32_t id,uint32_t flags,void * handle,void * cookie)403 static uint64_t ehf_el3_interrupt_handler(uint32_t id, uint32_t flags,
404 void *handle, void *cookie)
405 {
406 int ret = 0;
407 uint32_t intr_raw;
408 unsigned int intr, pri, idx;
409 ehf_handler_t handler;
410
411 /*
412 * Top-level interrupt type handler from Interrupt Management Framework
413 * doesn't acknowledge the interrupt; so the interrupt ID must be
414 * invalid.
415 */
416 assert(id == INTR_ID_UNAVAILABLE);
417
418 /*
419 * Acknowledge interrupt. Proceed with handling only for valid interrupt
420 * IDs. This situation may arise because of Interrupt Management
421 * Framework identifying an EL3 interrupt, but before it's been
422 * acknowledged here, the interrupt was either deasserted, or there was
423 * a higher-priority interrupt of another type.
424 */
425 intr_raw = plat_ic_acknowledge_interrupt();
426 intr = plat_ic_get_interrupt_id(intr_raw);
427 if (intr == INTR_ID_UNAVAILABLE)
428 return 0;
429
430 /* Having acknowledged the interrupt, get the running priority */
431 pri = plat_ic_get_running_priority();
432
433 /* Check EL3 interrupt priority is in secure range */
434 assert(IS_PRI_SECURE(pri));
435
436 /*
437 * Translate the priority to a descriptor index. We do this by masking
438 * and shifting the running priority value (platform-supplied).
439 */
440 idx = pri_to_idx(pri);
441
442 /* Validate priority */
443 assert(pri == IDX_TO_PRI(idx));
444
445 handler = (ehf_handler_t) RAW_HANDLER(
446 exception_data.ehf_priorities[idx].ehf_handler);
447 if (handler == NULL) {
448 ERROR("No EL3 exception handler for priority 0x%x\n",
449 IDX_TO_PRI(idx));
450 panic();
451 }
452
453 /*
454 * Call registered handler. Pass the raw interrupt value to registered
455 * handlers.
456 */
457 ret = handler(intr_raw, flags, handle, cookie);
458
459 return (uint64_t) ret;
460 }
461
462 /*
463 * Initialize the EL3 exception handling.
464 */
ehf_init(void)465 void __init ehf_init(void)
466 {
467 unsigned int flags = 0;
468 int ret __unused;
469
470 /* Ensure EL3 interrupts are supported */
471 assert(plat_ic_has_interrupt_type(INTR_TYPE_EL3));
472
473 /*
474 * Make sure that priority water mark has enough bits to represent the
475 * whole priority array.
476 */
477 assert(exception_data.num_priorities <= (sizeof(ehf_pri_bits_t) * 8U));
478
479 assert(exception_data.ehf_priorities != NULL);
480
481 /*
482 * Bit 7 of GIC priority must be 0 for secure interrupts. This means
483 * platforms must use at least 1 of the remaining 7 bits.
484 */
485 assert((exception_data.pri_bits >= 1U) ||
486 (exception_data.pri_bits < 8U));
487
488 /* Route EL3 interrupts when in Non-secure. */
489 set_interrupt_rm_flag(flags, NON_SECURE);
490
491 /*
492 * Route EL3 interrupts when in secure, only when SPMC is not present
493 * in S-EL2.
494 */
495 #if !(defined(SPD_spmd) && (SPMD_SPM_AT_SEL2 == 1))
496 set_interrupt_rm_flag(flags, SECURE);
497 #endif /* !(defined(SPD_spmd) && (SPMD_SPM_AT_SEL2 == 1)) */
498
499 /* Register handler for EL3 interrupts */
500 ret = register_interrupt_type_handler(INTR_TYPE_EL3,
501 ehf_el3_interrupt_handler, flags);
502 assert(ret == 0);
503 }
504
505 /*
506 * Register a handler at the supplied priority. Registration is allowed only if
507 * a handler hasn't been registered before, or one wasn't provided at build
508 * time. The priority for which the handler is being registered must also accord
509 * with the platform-supplied data.
510 */
ehf_register_priority_handler(unsigned int pri,ehf_handler_t handler)511 void ehf_register_priority_handler(unsigned int pri, ehf_handler_t handler)
512 {
513 unsigned int idx;
514
515 /* Sanity check for handler */
516 assert(handler != NULL);
517
518 /* Handler ought to be 4-byte aligned */
519 assert((((uintptr_t) handler) & 3U) == 0U);
520
521 /* Ensure we register for valid priority */
522 idx = pri_to_idx(pri);
523 assert(idx < exception_data.num_priorities);
524 assert(IDX_TO_PRI(idx) == pri);
525
526 /* Return failure if a handler was already registered */
527 if (exception_data.ehf_priorities[idx].ehf_handler != EHF_NO_HANDLER_) {
528 ERROR("Handler already registered for priority 0x%x\n", pri);
529 panic();
530 }
531
532 /*
533 * Install handler, and retain the valid bit. We assume that the handler
534 * is 4-byte aligned, which is usually the case.
535 */
536 exception_data.ehf_priorities[idx].ehf_handler =
537 (((uintptr_t) handler) | EHF_PRI_VALID_);
538
539 EHF_LOG("register pri=0x%x handler=%p\n", pri, handler);
540 }
541
542 SUBSCRIBE_TO_EVENT(cm_entering_normal_world, ehf_entering_normal_world);
543 SUBSCRIBE_TO_EVENT(cm_exited_normal_world, ehf_exited_normal_world);
544
