1 /*
2 * Copyright (c) 2013-2022, Arm Limited and Contributors. All rights reserved.
3 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6
7 #include <assert.h>
8 #include <stdbool.h>
9 #include <string.h>
10
11 #include <platform_def.h>
12
13 #include <arch.h>
14 #include <arch_helpers.h>
15 #include <arch_features.h>
16 #include <bl31/interrupt_mgmt.h>
17 #include <common/bl_common.h>
18 #include <context.h>
19 #include <drivers/arm/gicv3.h>
20 #include <lib/el3_runtime/context_mgmt.h>
21 #include <lib/el3_runtime/pubsub_events.h>
22 #include <lib/extensions/amu.h>
23 #include <lib/extensions/brbe.h>
24 #include <lib/extensions/mpam.h>
25 #include <lib/extensions/sme.h>
26 #include <lib/extensions/spe.h>
27 #include <lib/extensions/sve.h>
28 #include <lib/extensions/sys_reg_trace.h>
29 #include <lib/extensions/trbe.h>
30 #include <lib/extensions/trf.h>
31 #include <lib/utils.h>
32
33 #if ENABLE_FEAT_TWED
34 /* Make sure delay value fits within the range(0-15) */
35 CASSERT(((TWED_DELAY & ~SCR_TWEDEL_MASK) == 0U), assert_twed_delay_value_check);
36 #endif /* ENABLE_FEAT_TWED */
37
38 static void manage_extensions_secure(cpu_context_t *ctx);
39
setup_el1_context(cpu_context_t * ctx,const struct entry_point_info * ep)40 static void setup_el1_context(cpu_context_t *ctx, const struct entry_point_info *ep)
41 {
42 u_register_t sctlr_elx, actlr_elx;
43
44 /*
45 * Initialise SCTLR_EL1 to the reset value corresponding to the target
46 * execution state setting all fields rather than relying on the hw.
47 * Some fields have architecturally UNKNOWN reset values and these are
48 * set to zero.
49 *
50 * SCTLR.EE: Endianness is taken from the entrypoint attributes.
51 *
52 * SCTLR.M, SCTLR.C and SCTLR.I: These fields must be zero (as
53 * required by PSCI specification)
54 */
55 sctlr_elx = (EP_GET_EE(ep->h.attr) != 0U) ? SCTLR_EE_BIT : 0UL;
56 if (GET_RW(ep->spsr) == MODE_RW_64) {
57 sctlr_elx |= SCTLR_EL1_RES1;
58 } else {
59 /*
60 * If the target execution state is AArch32 then the following
61 * fields need to be set.
62 *
63 * SCTRL_EL1.nTWE: Set to one so that EL0 execution of WFE
64 * instructions are not trapped to EL1.
65 *
66 * SCTLR_EL1.nTWI: Set to one so that EL0 execution of WFI
67 * instructions are not trapped to EL1.
68 *
69 * SCTLR_EL1.CP15BEN: Set to one to enable EL0 execution of the
70 * CP15DMB, CP15DSB, and CP15ISB instructions.
71 */
72 sctlr_elx |= SCTLR_AARCH32_EL1_RES1 | SCTLR_CP15BEN_BIT
73 | SCTLR_NTWI_BIT | SCTLR_NTWE_BIT;
74 }
75
76 #if ERRATA_A75_764081
77 /*
78 * If workaround of errata 764081 for Cortex-A75 is used then set
79 * SCTLR_EL1.IESB to enable Implicit Error Synchronization Barrier.
80 */
81 sctlr_elx |= SCTLR_IESB_BIT;
82 #endif
83 /* Store the initialised SCTLR_EL1 value in the cpu_context */
84 write_ctx_reg(get_el1_sysregs_ctx(ctx), CTX_SCTLR_EL1, sctlr_elx);
85
86 /*
87 * Base the context ACTLR_EL1 on the current value, as it is
88 * implementation defined. The context restore process will write
89 * the value from the context to the actual register and can cause
90 * problems for processor cores that don't expect certain bits to
91 * be zero.
92 */
93 actlr_elx = read_actlr_el1();
94 write_ctx_reg((get_el1_sysregs_ctx(ctx)), (CTX_ACTLR_EL1), (actlr_elx));
95 }
96
97 /******************************************************************************
98 * This function performs initializations that are specific to SECURE state
99 * and updates the cpu context specified by 'ctx'.
100 *****************************************************************************/
setup_secure_context(cpu_context_t * ctx,const struct entry_point_info * ep)101 static void setup_secure_context(cpu_context_t *ctx, const struct entry_point_info *ep)
102 {
103 u_register_t scr_el3;
104 el3_state_t *state;
105
106 state = get_el3state_ctx(ctx);
107 scr_el3 = read_ctx_reg(state, CTX_SCR_EL3);
108
109 #if defined(IMAGE_BL31) && !defined(SPD_spmd)
110 /*
111 * SCR_EL3.IRQ, SCR_EL3.FIQ: Enable the physical FIQ and IRQ routing as
112 * indicated by the interrupt routing model for BL31.
113 */
114 scr_el3 |= get_scr_el3_from_routing_model(SECURE);
115 #endif
116
117 #if !CTX_INCLUDE_MTE_REGS || ENABLE_ASSERTIONS
118 /* Get Memory Tagging Extension support level */
119 unsigned int mte = get_armv8_5_mte_support();
120 #endif
121 /*
122 * Allow access to Allocation Tags when CTX_INCLUDE_MTE_REGS
123 * is set, or when MTE is only implemented at EL0.
124 */
125 #if CTX_INCLUDE_MTE_REGS
126 assert((mte == MTE_IMPLEMENTED_ELX) || (mte == MTE_IMPLEMENTED_ASY));
127 scr_el3 |= SCR_ATA_BIT;
128 #else
129 if (mte == MTE_IMPLEMENTED_EL0) {
130 scr_el3 |= SCR_ATA_BIT;
131 }
132 #endif /* CTX_INCLUDE_MTE_REGS */
133
134 /* Enable S-EL2 if the next EL is EL2 and S-EL2 is present */
135 if ((GET_EL(ep->spsr) == MODE_EL2) && is_armv8_4_sel2_present()) {
136 if (GET_RW(ep->spsr) != MODE_RW_64) {
137 ERROR("S-EL2 can not be used in AArch32\n.");
138 panic();
139 }
140
141 scr_el3 |= SCR_EEL2_BIT;
142 }
143
144 write_ctx_reg(state, CTX_SCR_EL3, scr_el3);
145
146 /*
147 * Initialize EL1 context registers unless SPMC is running
148 * at S-EL2.
149 */
150 #if !SPMD_SPM_AT_SEL2
151 setup_el1_context(ctx, ep);
152 #endif
153
154 manage_extensions_secure(ctx);
155 }
156
157 #if ENABLE_RME
158 /******************************************************************************
159 * This function performs initializations that are specific to REALM state
160 * and updates the cpu context specified by 'ctx'.
161 *****************************************************************************/
setup_realm_context(cpu_context_t * ctx,const struct entry_point_info * ep)162 static void setup_realm_context(cpu_context_t *ctx, const struct entry_point_info *ep)
163 {
164 u_register_t scr_el3;
165 el3_state_t *state;
166
167 state = get_el3state_ctx(ctx);
168 scr_el3 = read_ctx_reg(state, CTX_SCR_EL3);
169
170 scr_el3 |= SCR_NS_BIT | SCR_NSE_BIT | SCR_EnSCXT_BIT;
171
172 write_ctx_reg(state, CTX_SCR_EL3, scr_el3);
173 }
174 #endif /* ENABLE_RME */
175
176 /******************************************************************************
177 * This function performs initializations that are specific to NON-SECURE state
178 * and updates the cpu context specified by 'ctx'.
179 *****************************************************************************/
setup_ns_context(cpu_context_t * ctx,const struct entry_point_info * ep)180 static void setup_ns_context(cpu_context_t *ctx, const struct entry_point_info *ep)
181 {
182 u_register_t scr_el3;
183 el3_state_t *state;
184
185 state = get_el3state_ctx(ctx);
186 scr_el3 = read_ctx_reg(state, CTX_SCR_EL3);
187
188 /* SCR_NS: Set the NS bit */
189 scr_el3 |= SCR_NS_BIT;
190
191 #if !CTX_INCLUDE_PAUTH_REGS
192 /*
193 * If the pointer authentication registers aren't saved during world
194 * switches the value of the registers can be leaked from the Secure to
195 * the Non-secure world. To prevent this, rather than enabling pointer
196 * authentication everywhere, we only enable it in the Non-secure world.
197 *
198 * If the Secure world wants to use pointer authentication,
199 * CTX_INCLUDE_PAUTH_REGS must be set to 1.
200 */
201 scr_el3 |= SCR_API_BIT | SCR_APK_BIT;
202 #endif /* !CTX_INCLUDE_PAUTH_REGS */
203
204 /* Allow access to Allocation Tags when MTE is implemented. */
205 scr_el3 |= SCR_ATA_BIT;
206
207 #ifdef IMAGE_BL31
208 /*
209 * SCR_EL3.IRQ, SCR_EL3.FIQ: Enable the physical FIQ and IRQ routing as
210 * indicated by the interrupt routing model for BL31.
211 */
212 scr_el3 |= get_scr_el3_from_routing_model(NON_SECURE);
213 #endif
214 write_ctx_reg(state, CTX_SCR_EL3, scr_el3);
215
216 /* Initialize EL1 context registers */
217 setup_el1_context(ctx, ep);
218
219 /* Initialize EL2 context registers */
220 #if CTX_INCLUDE_EL2_REGS
221
222 /*
223 * Initialize SCTLR_EL2 context register using Endianness value
224 * taken from the entrypoint attribute.
225 */
226 u_register_t sctlr_el2 = (EP_GET_EE(ep->h.attr) != 0U) ? SCTLR_EE_BIT : 0UL;
227 sctlr_el2 |= SCTLR_EL2_RES1;
228 write_ctx_reg(get_el2_sysregs_ctx(ctx), CTX_SCTLR_EL2,
229 sctlr_el2);
230
231 /*
232 * The GICv3 driver initializes the ICC_SRE_EL2 register during
233 * platform setup. Use the same setting for the corresponding
234 * context register to make sure the correct bits are set when
235 * restoring NS context.
236 */
237 u_register_t icc_sre_el2 = read_icc_sre_el2();
238 icc_sre_el2 |= (ICC_SRE_DIB_BIT | ICC_SRE_DFB_BIT);
239 icc_sre_el2 |= (ICC_SRE_EN_BIT | ICC_SRE_SRE_BIT);
240 write_ctx_reg(get_el2_sysregs_ctx(ctx), CTX_ICC_SRE_EL2,
241 icc_sre_el2);
242 #endif /* CTX_INCLUDE_EL2_REGS */
243 }
244
245 /*******************************************************************************
246 * The following function performs initialization of the cpu_context 'ctx'
247 * for first use that is common to all security states, and sets the
248 * initial entrypoint state as specified by the entry_point_info structure.
249 *
250 * The EE and ST attributes are used to configure the endianness and secure
251 * timer availability for the new execution context.
252 ******************************************************************************/
setup_context_common(cpu_context_t * ctx,const entry_point_info_t * ep)253 static void setup_context_common(cpu_context_t *ctx, const entry_point_info_t *ep)
254 {
255 u_register_t scr_el3;
256 el3_state_t *state;
257 gp_regs_t *gp_regs;
258
259 /* Clear any residual register values from the context */
260 zeromem(ctx, sizeof(*ctx));
261
262 /*
263 * SCR_EL3 was initialised during reset sequence in macro
264 * el3_arch_init_common. This code modifies the SCR_EL3 fields that
265 * affect the next EL.
266 *
267 * The following fields are initially set to zero and then updated to
268 * the required value depending on the state of the SPSR_EL3 and the
269 * Security state and entrypoint attributes of the next EL.
270 */
271 scr_el3 = read_scr();
272 scr_el3 &= ~(SCR_NS_BIT | SCR_RW_BIT | SCR_FIQ_BIT | SCR_IRQ_BIT |
273 SCR_ST_BIT | SCR_HCE_BIT | SCR_NSE_BIT);
274
275 /*
276 * SCR_EL3.RW: Set the execution state, AArch32 or AArch64, for next
277 * Exception level as specified by SPSR.
278 */
279 if (GET_RW(ep->spsr) == MODE_RW_64) {
280 scr_el3 |= SCR_RW_BIT;
281 }
282
283 /*
284 * SCR_EL3.ST: Traps Secure EL1 accesses to the Counter-timer Physical
285 * Secure timer registers to EL3, from AArch64 state only, if specified
286 * by the entrypoint attributes. If SEL2 is present and enabled, the ST
287 * bit always behaves as 1 (i.e. secure physical timer register access
288 * is not trapped)
289 */
290 if (EP_GET_ST(ep->h.attr) != 0U) {
291 scr_el3 |= SCR_ST_BIT;
292 }
293
294 /*
295 * If FEAT_HCX is enabled, enable access to HCRX_EL2 by setting
296 * SCR_EL3.HXEn.
297 */
298 #if ENABLE_FEAT_HCX
299 scr_el3 |= SCR_HXEn_BIT;
300 #endif
301
302 #if RAS_TRAP_LOWER_EL_ERR_ACCESS
303 /*
304 * SCR_EL3.TERR: Trap Error record accesses. Accesses to the RAS ERR
305 * and RAS ERX registers from EL1 and EL2 are trapped to EL3.
306 */
307 scr_el3 |= SCR_TERR_BIT;
308 #endif
309
310 #if !HANDLE_EA_EL3_FIRST
311 /*
312 * SCR_EL3.EA: Do not route External Abort and SError Interrupt External
313 * to EL3 when executing at a lower EL. When executing at EL3, External
314 * Aborts are taken to EL3.
315 */
316 scr_el3 &= ~SCR_EA_BIT;
317 #endif
318
319 #if FAULT_INJECTION_SUPPORT
320 /* Enable fault injection from lower ELs */
321 scr_el3 |= SCR_FIEN_BIT;
322 #endif
323
324 /*
325 * CPTR_EL3 was initialized out of reset, copy that value to the
326 * context register.
327 */
328 write_ctx_reg(get_el3state_ctx(ctx), CTX_CPTR_EL3, read_cptr_el3());
329
330 /*
331 * SCR_EL3.HCE: Enable HVC instructions if next execution state is
332 * AArch64 and next EL is EL2, or if next execution state is AArch32 and
333 * next mode is Hyp.
334 * SCR_EL3.FGTEn: Enable Fine Grained Virtualization Traps under the
335 * same conditions as HVC instructions and when the processor supports
336 * ARMv8.6-FGT.
337 * SCR_EL3.ECVEn: Enable Enhanced Counter Virtualization (ECV)
338 * CNTPOFF_EL2 register under the same conditions as HVC instructions
339 * and when the processor supports ECV.
340 */
341 if (((GET_RW(ep->spsr) == MODE_RW_64) && (GET_EL(ep->spsr) == MODE_EL2))
342 || ((GET_RW(ep->spsr) != MODE_RW_64)
343 && (GET_M32(ep->spsr) == MODE32_hyp))) {
344 scr_el3 |= SCR_HCE_BIT;
345
346 if (is_armv8_6_fgt_present()) {
347 scr_el3 |= SCR_FGTEN_BIT;
348 }
349
350 if (get_armv8_6_ecv_support()
351 == ID_AA64MMFR0_EL1_ECV_SELF_SYNCH) {
352 scr_el3 |= SCR_ECVEN_BIT;
353 }
354 }
355
356 #if ENABLE_FEAT_TWED
357 /* Enable WFE trap delay in SCR_EL3 if supported and configured */
358 /* Set delay in SCR_EL3 */
359 scr_el3 &= ~(SCR_TWEDEL_MASK << SCR_TWEDEL_SHIFT);
360 scr_el3 |= ((TWED_DELAY & SCR_TWEDEL_MASK)
361 << SCR_TWEDEL_SHIFT);
362
363 /* Enable WFE delay */
364 scr_el3 |= SCR_TWEDEn_BIT;
365 #endif /* ENABLE_FEAT_TWED */
366
367 /*
368 * Populate EL3 state so that we've the right context
369 * before doing ERET
370 */
371 state = get_el3state_ctx(ctx);
372 write_ctx_reg(state, CTX_SCR_EL3, scr_el3);
373 write_ctx_reg(state, CTX_ELR_EL3, ep->pc);
374 write_ctx_reg(state, CTX_SPSR_EL3, ep->spsr);
375
376 /*
377 * Store the X0-X7 value from the entrypoint into the context
378 * Use memcpy as we are in control of the layout of the structures
379 */
380 gp_regs = get_gpregs_ctx(ctx);
381 memcpy(gp_regs, (void *)&ep->args, sizeof(aapcs64_params_t));
382 }
383
384 /*******************************************************************************
385 * Context management library initialization routine. This library is used by
386 * runtime services to share pointers to 'cpu_context' structures for secure
387 * non-secure and realm states. Management of the structures and their associated
388 * memory is not done by the context management library e.g. the PSCI service
389 * manages the cpu context used for entry from and exit to the non-secure state.
390 * The Secure payload dispatcher service manages the context(s) corresponding to
391 * the secure state. It also uses this library to get access to the non-secure
392 * state cpu context pointers.
393 * Lastly, this library provides the API to make SP_EL3 point to the cpu context
394 * which will be used for programming an entry into a lower EL. The same context
395 * will be used to save state upon exception entry from that EL.
396 ******************************************************************************/
cm_init(void)397 void __init cm_init(void)
398 {
399 /*
400 * The context management library has only global data to intialize, but
401 * that will be done when the BSS is zeroed out.
402 */
403 }
404
405 /*******************************************************************************
406 * This is the high-level function used to initialize the cpu_context 'ctx' for
407 * first use. It performs initializations that are common to all security states
408 * and initializations specific to the security state specified in 'ep'
409 ******************************************************************************/
cm_setup_context(cpu_context_t * ctx,const entry_point_info_t * ep)410 void cm_setup_context(cpu_context_t *ctx, const entry_point_info_t *ep)
411 {
412 unsigned int security_state;
413
414 assert(ctx != NULL);
415
416 /*
417 * Perform initializations that are common
418 * to all security states
419 */
420 setup_context_common(ctx, ep);
421
422 security_state = GET_SECURITY_STATE(ep->h.attr);
423
424 /* Perform security state specific initializations */
425 switch (security_state) {
426 case SECURE:
427 setup_secure_context(ctx, ep);
428 break;
429 #if ENABLE_RME
430 case REALM:
431 setup_realm_context(ctx, ep);
432 break;
433 #endif
434 case NON_SECURE:
435 setup_ns_context(ctx, ep);
436 break;
437 default:
438 ERROR("Invalid security state\n");
439 panic();
440 break;
441 }
442 }
443
444 /*******************************************************************************
445 * Enable architecture extensions on first entry to Non-secure world.
446 * When EL2 is implemented but unused `el2_unused` is non-zero, otherwise
447 * it is zero.
448 ******************************************************************************/
manage_extensions_nonsecure(bool el2_unused,cpu_context_t * ctx)449 static void manage_extensions_nonsecure(bool el2_unused, cpu_context_t *ctx)
450 {
451 #if IMAGE_BL31
452 #if ENABLE_SPE_FOR_LOWER_ELS
453 spe_enable(el2_unused);
454 #endif
455
456 #if ENABLE_AMU
457 amu_enable(el2_unused, ctx);
458 #endif
459
460 #if ENABLE_SME_FOR_NS
461 /* Enable SME, SVE, and FPU/SIMD for non-secure world. */
462 sme_enable(ctx);
463 #elif ENABLE_SVE_FOR_NS
464 /* Enable SVE and FPU/SIMD for non-secure world. */
465 sve_enable(ctx);
466 #endif
467
468 #if ENABLE_MPAM_FOR_LOWER_ELS
469 mpam_enable(el2_unused);
470 #endif
471
472 #if ENABLE_TRBE_FOR_NS
473 trbe_enable();
474 #endif /* ENABLE_TRBE_FOR_NS */
475
476 #if ENABLE_BRBE_FOR_NS
477 brbe_enable();
478 #endif /* ENABLE_BRBE_FOR_NS */
479
480 #if ENABLE_SYS_REG_TRACE_FOR_NS
481 sys_reg_trace_enable(ctx);
482 #endif /* ENABLE_SYS_REG_TRACE_FOR_NS */
483
484 #if ENABLE_TRF_FOR_NS
485 trf_enable();
486 #endif /* ENABLE_TRF_FOR_NS */
487 #endif
488 }
489
490 /*******************************************************************************
491 * Enable architecture extensions on first entry to Secure world.
492 ******************************************************************************/
manage_extensions_secure(cpu_context_t * ctx)493 static void manage_extensions_secure(cpu_context_t *ctx)
494 {
495 #if IMAGE_BL31
496 #if ENABLE_SME_FOR_NS
497 #if ENABLE_SME_FOR_SWD
498 /*
499 * Enable SME, SVE, FPU/SIMD in secure context, secure manager must
500 * ensure SME, SVE, and FPU/SIMD context properly managed.
501 */
502 sme_enable(ctx);
503 #else /* ENABLE_SME_FOR_SWD */
504 /*
505 * Disable SME, SVE, FPU/SIMD in secure context so non-secure world can
506 * safely use the associated registers.
507 */
508 sme_disable(ctx);
509 #endif /* ENABLE_SME_FOR_SWD */
510 #elif ENABLE_SVE_FOR_NS
511 #if ENABLE_SVE_FOR_SWD
512 /*
513 * Enable SVE and FPU in secure context, secure manager must ensure that
514 * the SVE and FPU register contexts are properly managed.
515 */
516 sve_enable(ctx);
517 #else /* ENABLE_SVE_FOR_SWD */
518 /*
519 * Disable SVE and FPU in secure context so non-secure world can safely
520 * use them.
521 */
522 sve_disable(ctx);
523 #endif /* ENABLE_SVE_FOR_SWD */
524 #endif /* ENABLE_SVE_FOR_NS */
525 #endif /* IMAGE_BL31 */
526 }
527
528 /*******************************************************************************
529 * The following function initializes the cpu_context for a CPU specified by
530 * its `cpu_idx` for first use, and sets the initial entrypoint state as
531 * specified by the entry_point_info structure.
532 ******************************************************************************/
cm_init_context_by_index(unsigned int cpu_idx,const entry_point_info_t * ep)533 void cm_init_context_by_index(unsigned int cpu_idx,
534 const entry_point_info_t *ep)
535 {
536 cpu_context_t *ctx;
537 ctx = cm_get_context_by_index(cpu_idx, GET_SECURITY_STATE(ep->h.attr));
538 cm_setup_context(ctx, ep);
539 }
540
541 /*******************************************************************************
542 * The following function initializes the cpu_context for the current CPU
543 * for first use, and sets the initial entrypoint state as specified by the
544 * entry_point_info structure.
545 ******************************************************************************/
cm_init_my_context(const entry_point_info_t * ep)546 void cm_init_my_context(const entry_point_info_t *ep)
547 {
548 cpu_context_t *ctx;
549 ctx = cm_get_context(GET_SECURITY_STATE(ep->h.attr));
550 cm_setup_context(ctx, ep);
551 }
552
553 /*******************************************************************************
554 * Prepare the CPU system registers for first entry into realm, secure, or
555 * normal world.
556 *
557 * If execution is requested to EL2 or hyp mode, SCTLR_EL2 is initialized
558 * If execution is requested to non-secure EL1 or svc mode, and the CPU supports
559 * EL2 then EL2 is disabled by configuring all necessary EL2 registers.
560 * For all entries, the EL1 registers are initialized from the cpu_context
561 ******************************************************************************/
cm_prepare_el3_exit(uint32_t security_state)562 void cm_prepare_el3_exit(uint32_t security_state)
563 {
564 u_register_t sctlr_elx, scr_el3, mdcr_el2;
565 cpu_context_t *ctx = cm_get_context(security_state);
566 bool el2_unused = false;
567 uint64_t hcr_el2 = 0U;
568
569 assert(ctx != NULL);
570
571 if (security_state == NON_SECURE) {
572 scr_el3 = read_ctx_reg(get_el3state_ctx(ctx),
573 CTX_SCR_EL3);
574 if ((scr_el3 & SCR_HCE_BIT) != 0U) {
575 /* Use SCTLR_EL1.EE value to initialise sctlr_el2 */
576 sctlr_elx = read_ctx_reg(get_el1_sysregs_ctx(ctx),
577 CTX_SCTLR_EL1);
578 sctlr_elx &= SCTLR_EE_BIT;
579 sctlr_elx |= SCTLR_EL2_RES1;
580 #if ERRATA_A75_764081
581 /*
582 * If workaround of errata 764081 for Cortex-A75 is used
583 * then set SCTLR_EL2.IESB to enable Implicit Error
584 * Synchronization Barrier.
585 */
586 sctlr_elx |= SCTLR_IESB_BIT;
587 #endif
588 write_sctlr_el2(sctlr_elx);
589 } else if (el_implemented(2) != EL_IMPL_NONE) {
590 el2_unused = true;
591
592 /*
593 * EL2 present but unused, need to disable safely.
594 * SCTLR_EL2 can be ignored in this case.
595 *
596 * Set EL2 register width appropriately: Set HCR_EL2
597 * field to match SCR_EL3.RW.
598 */
599 if ((scr_el3 & SCR_RW_BIT) != 0U)
600 hcr_el2 |= HCR_RW_BIT;
601
602 /*
603 * For Armv8.3 pointer authentication feature, disable
604 * traps to EL2 when accessing key registers or using
605 * pointer authentication instructions from lower ELs.
606 */
607 hcr_el2 |= (HCR_API_BIT | HCR_APK_BIT);
608
609 write_hcr_el2(hcr_el2);
610
611 /*
612 * Initialise CPTR_EL2 setting all fields rather than
613 * relying on the hw. All fields have architecturally
614 * UNKNOWN reset values.
615 *
616 * CPTR_EL2.TCPAC: Set to zero so that Non-secure EL1
617 * accesses to the CPACR_EL1 or CPACR from both
618 * Execution states do not trap to EL2.
619 *
620 * CPTR_EL2.TTA: Set to zero so that Non-secure System
621 * register accesses to the trace registers from both
622 * Execution states do not trap to EL2.
623 * If PE trace unit System registers are not implemented
624 * then this bit is reserved, and must be set to zero.
625 *
626 * CPTR_EL2.TFP: Set to zero so that Non-secure accesses
627 * to SIMD and floating-point functionality from both
628 * Execution states do not trap to EL2.
629 */
630 write_cptr_el2(CPTR_EL2_RESET_VAL &
631 ~(CPTR_EL2_TCPAC_BIT | CPTR_EL2_TTA_BIT
632 | CPTR_EL2_TFP_BIT));
633
634 /*
635 * Initialise CNTHCTL_EL2. All fields are
636 * architecturally UNKNOWN on reset and are set to zero
637 * except for field(s) listed below.
638 *
639 * CNTHCTL_EL2.EL1PTEN: Set to one to disable traps to
640 * Hyp mode of Non-secure EL0 and EL1 accesses to the
641 * physical timer registers.
642 *
643 * CNTHCTL_EL2.EL1PCTEN: Set to one to disable traps to
644 * Hyp mode of Non-secure EL0 and EL1 accesses to the
645 * physical counter registers.
646 */
647 write_cnthctl_el2(CNTHCTL_RESET_VAL |
648 EL1PCEN_BIT | EL1PCTEN_BIT);
649
650 /*
651 * Initialise CNTVOFF_EL2 to zero as it resets to an
652 * architecturally UNKNOWN value.
653 */
654 write_cntvoff_el2(0);
655
656 /*
657 * Set VPIDR_EL2 and VMPIDR_EL2 to match MIDR_EL1 and
658 * MPIDR_EL1 respectively.
659 */
660 write_vpidr_el2(read_midr_el1());
661 write_vmpidr_el2(read_mpidr_el1());
662
663 /*
664 * Initialise VTTBR_EL2. All fields are architecturally
665 * UNKNOWN on reset.
666 *
667 * VTTBR_EL2.VMID: Set to zero. Even though EL1&0 stage
668 * 2 address translation is disabled, cache maintenance
669 * operations depend on the VMID.
670 *
671 * VTTBR_EL2.BADDR: Set to zero as EL1&0 stage 2 address
672 * translation is disabled.
673 */
674 write_vttbr_el2(VTTBR_RESET_VAL &
675 ~((VTTBR_VMID_MASK << VTTBR_VMID_SHIFT)
676 | (VTTBR_BADDR_MASK << VTTBR_BADDR_SHIFT)));
677
678 /*
679 * Initialise MDCR_EL2, setting all fields rather than
680 * relying on hw. Some fields are architecturally
681 * UNKNOWN on reset.
682 *
683 * MDCR_EL2.HLP: Set to one so that event counter
684 * overflow, that is recorded in PMOVSCLR_EL0[0-30],
685 * occurs on the increment that changes
686 * PMEVCNTR<n>_EL0[63] from 1 to 0, when ARMv8.5-PMU is
687 * implemented. This bit is RES0 in versions of the
688 * architecture earlier than ARMv8.5, setting it to 1
689 * doesn't have any effect on them.
690 *
691 * MDCR_EL2.TTRF: Set to zero so that access to Trace
692 * Filter Control register TRFCR_EL1 at EL1 is not
693 * trapped to EL2. This bit is RES0 in versions of
694 * the architecture earlier than ARMv8.4.
695 *
696 * MDCR_EL2.HPMD: Set to one so that event counting is
697 * prohibited at EL2. This bit is RES0 in versions of
698 * the architecture earlier than ARMv8.1, setting it
699 * to 1 doesn't have any effect on them.
700 *
701 * MDCR_EL2.TPMS: Set to zero so that accesses to
702 * Statistical Profiling control registers from EL1
703 * do not trap to EL2. This bit is RES0 when SPE is
704 * not implemented.
705 *
706 * MDCR_EL2.TDRA: Set to zero so that Non-secure EL0 and
707 * EL1 System register accesses to the Debug ROM
708 * registers are not trapped to EL2.
709 *
710 * MDCR_EL2.TDOSA: Set to zero so that Non-secure EL1
711 * System register accesses to the powerdown debug
712 * registers are not trapped to EL2.
713 *
714 * MDCR_EL2.TDA: Set to zero so that System register
715 * accesses to the debug registers do not trap to EL2.
716 *
717 * MDCR_EL2.TDE: Set to zero so that debug exceptions
718 * are not routed to EL2.
719 *
720 * MDCR_EL2.HPME: Set to zero to disable EL2 Performance
721 * Monitors.
722 *
723 * MDCR_EL2.TPM: Set to zero so that Non-secure EL0 and
724 * EL1 accesses to all Performance Monitors registers
725 * are not trapped to EL2.
726 *
727 * MDCR_EL2.TPMCR: Set to zero so that Non-secure EL0
728 * and EL1 accesses to the PMCR_EL0 or PMCR are not
729 * trapped to EL2.
730 *
731 * MDCR_EL2.HPMN: Set to value of PMCR_EL0.N which is the
732 * architecturally-defined reset value.
733 *
734 * MDCR_EL2.E2TB: Set to zero so that the trace Buffer
735 * owning exception level is NS-EL1 and, tracing is
736 * prohibited at NS-EL2. These bits are RES0 when
737 * FEAT_TRBE is not implemented.
738 */
739 mdcr_el2 = ((MDCR_EL2_RESET_VAL | MDCR_EL2_HLP |
740 MDCR_EL2_HPMD) |
741 ((read_pmcr_el0() & PMCR_EL0_N_BITS)
742 >> PMCR_EL0_N_SHIFT)) &
743 ~(MDCR_EL2_TTRF | MDCR_EL2_TPMS |
744 MDCR_EL2_TDRA_BIT | MDCR_EL2_TDOSA_BIT |
745 MDCR_EL2_TDA_BIT | MDCR_EL2_TDE_BIT |
746 MDCR_EL2_HPME_BIT | MDCR_EL2_TPM_BIT |
747 MDCR_EL2_TPMCR_BIT |
748 MDCR_EL2_E2TB(MDCR_EL2_E2TB_EL1));
749
750 write_mdcr_el2(mdcr_el2);
751
752 /*
753 * Initialise HSTR_EL2. All fields are architecturally
754 * UNKNOWN on reset.
755 *
756 * HSTR_EL2.T<n>: Set all these fields to zero so that
757 * Non-secure EL0 or EL1 accesses to System registers
758 * do not trap to EL2.
759 */
760 write_hstr_el2(HSTR_EL2_RESET_VAL & ~(HSTR_EL2_T_MASK));
761 /*
762 * Initialise CNTHP_CTL_EL2. All fields are
763 * architecturally UNKNOWN on reset.
764 *
765 * CNTHP_CTL_EL2:ENABLE: Set to zero to disable the EL2
766 * physical timer and prevent timer interrupts.
767 */
768 write_cnthp_ctl_el2(CNTHP_CTL_RESET_VAL &
769 ~(CNTHP_CTL_ENABLE_BIT));
770 }
771 manage_extensions_nonsecure(el2_unused, ctx);
772 }
773
774 cm_el1_sysregs_context_restore(security_state);
775 cm_set_next_eret_context(security_state);
776 }
777
778 #if CTX_INCLUDE_EL2_REGS
779 /*******************************************************************************
780 * Save EL2 sysreg context
781 ******************************************************************************/
cm_el2_sysregs_context_save(uint32_t security_state)782 void cm_el2_sysregs_context_save(uint32_t security_state)
783 {
784 u_register_t scr_el3 = read_scr();
785
786 /*
787 * Always save the non-secure and realm EL2 context, only save the
788 * S-EL2 context if S-EL2 is enabled.
789 */
790 if ((security_state != SECURE) ||
791 ((security_state == SECURE) && ((scr_el3 & SCR_EEL2_BIT) != 0U))) {
792 cpu_context_t *ctx;
793 el2_sysregs_t *el2_sysregs_ctx;
794
795 ctx = cm_get_context(security_state);
796 assert(ctx != NULL);
797
798 el2_sysregs_ctx = get_el2_sysregs_ctx(ctx);
799
800 el2_sysregs_context_save_common(el2_sysregs_ctx);
801 #if ENABLE_SPE_FOR_LOWER_ELS
802 el2_sysregs_context_save_spe(el2_sysregs_ctx);
803 #endif
804 #if CTX_INCLUDE_MTE_REGS
805 el2_sysregs_context_save_mte(el2_sysregs_ctx);
806 #endif
807 #if ENABLE_MPAM_FOR_LOWER_ELS
808 el2_sysregs_context_save_mpam(el2_sysregs_ctx);
809 #endif
810 #if ENABLE_FEAT_FGT
811 el2_sysregs_context_save_fgt(el2_sysregs_ctx);
812 #endif
813 #if ENABLE_FEAT_ECV
814 el2_sysregs_context_save_ecv(el2_sysregs_ctx);
815 #endif
816 #if ENABLE_FEAT_VHE
817 el2_sysregs_context_save_vhe(el2_sysregs_ctx);
818 #endif
819 #if RAS_EXTENSION
820 el2_sysregs_context_save_ras(el2_sysregs_ctx);
821 #endif
822 #if CTX_INCLUDE_NEVE_REGS
823 el2_sysregs_context_save_nv2(el2_sysregs_ctx);
824 #endif
825 #if ENABLE_TRF_FOR_NS
826 el2_sysregs_context_save_trf(el2_sysregs_ctx);
827 #endif
828 #if ENABLE_FEAT_CSV2_2
829 el2_sysregs_context_save_csv2(el2_sysregs_ctx);
830 #endif
831 #if ENABLE_FEAT_HCX
832 el2_sysregs_context_save_hcx(el2_sysregs_ctx);
833 #endif
834 }
835 }
836
837 /*******************************************************************************
838 * Restore EL2 sysreg context
839 ******************************************************************************/
cm_el2_sysregs_context_restore(uint32_t security_state)840 void cm_el2_sysregs_context_restore(uint32_t security_state)
841 {
842 u_register_t scr_el3 = read_scr();
843
844 /*
845 * Always restore the non-secure and realm EL2 context, only restore the
846 * S-EL2 context if S-EL2 is enabled.
847 */
848 if ((security_state != SECURE) ||
849 ((security_state == SECURE) && ((scr_el3 & SCR_EEL2_BIT) != 0U))) {
850 cpu_context_t *ctx;
851 el2_sysregs_t *el2_sysregs_ctx;
852
853 ctx = cm_get_context(security_state);
854 assert(ctx != NULL);
855
856 el2_sysregs_ctx = get_el2_sysregs_ctx(ctx);
857
858 el2_sysregs_context_restore_common(el2_sysregs_ctx);
859 #if ENABLE_SPE_FOR_LOWER_ELS
860 el2_sysregs_context_restore_spe(el2_sysregs_ctx);
861 #endif
862 #if CTX_INCLUDE_MTE_REGS
863 el2_sysregs_context_restore_mte(el2_sysregs_ctx);
864 #endif
865 #if ENABLE_MPAM_FOR_LOWER_ELS
866 el2_sysregs_context_restore_mpam(el2_sysregs_ctx);
867 #endif
868 #if ENABLE_FEAT_FGT
869 el2_sysregs_context_restore_fgt(el2_sysregs_ctx);
870 #endif
871 #if ENABLE_FEAT_ECV
872 el2_sysregs_context_restore_ecv(el2_sysregs_ctx);
873 #endif
874 #if ENABLE_FEAT_VHE
875 el2_sysregs_context_restore_vhe(el2_sysregs_ctx);
876 #endif
877 #if RAS_EXTENSION
878 el2_sysregs_context_restore_ras(el2_sysregs_ctx);
879 #endif
880 #if CTX_INCLUDE_NEVE_REGS
881 el2_sysregs_context_restore_nv2(el2_sysregs_ctx);
882 #endif
883 #if ENABLE_TRF_FOR_NS
884 el2_sysregs_context_restore_trf(el2_sysregs_ctx);
885 #endif
886 #if ENABLE_FEAT_CSV2_2
887 el2_sysregs_context_restore_csv2(el2_sysregs_ctx);
888 #endif
889 #if ENABLE_FEAT_HCX
890 el2_sysregs_context_restore_hcx(el2_sysregs_ctx);
891 #endif
892 }
893 }
894 #endif /* CTX_INCLUDE_EL2_REGS */
895
896 /*******************************************************************************
897 * This function is used to exit to Non-secure world. If CTX_INCLUDE_EL2_REGS
898 * is enabled, it restores EL1 and EL2 sysreg contexts instead of directly
899 * updating EL1 and EL2 registers. Otherwise, it calls the generic
900 * cm_prepare_el3_exit function.
901 ******************************************************************************/
cm_prepare_el3_exit_ns(void)902 void cm_prepare_el3_exit_ns(void)
903 {
904 #if CTX_INCLUDE_EL2_REGS
905 cpu_context_t *ctx = cm_get_context(NON_SECURE);
906 assert(ctx != NULL);
907
908 /* Assert that EL2 is used. */
909 #if ENABLE_ASSERTIONS
910 el3_state_t *state = get_el3state_ctx(ctx);
911 u_register_t scr_el3 = read_ctx_reg(state, CTX_SCR_EL3);
912 #endif
913 assert(((scr_el3 & SCR_HCE_BIT) != 0UL) &&
914 (el_implemented(2U) != EL_IMPL_NONE));
915
916 /*
917 * Currently some extensions are configured using
918 * direct register updates. Therefore, do this here
919 * instead of when setting up context.
920 */
921 manage_extensions_nonsecure(0, ctx);
922
923 /*
924 * Set the NS bit to be able to access the ICC_SRE_EL2
925 * register when restoring context.
926 */
927 write_scr_el3(read_scr_el3() | SCR_NS_BIT);
928
929 /*
930 * Ensure the NS bit change is committed before the EL2/EL1
931 * state restoration.
932 */
933 isb();
934
935 /* Restore EL2 and EL1 sysreg contexts */
936 cm_el2_sysregs_context_restore(NON_SECURE);
937 cm_el1_sysregs_context_restore(NON_SECURE);
938 cm_set_next_eret_context(NON_SECURE);
939 #else
940 cm_prepare_el3_exit(NON_SECURE);
941 #endif /* CTX_INCLUDE_EL2_REGS */
942 }
943
944 /*******************************************************************************
945 * The next four functions are used by runtime services to save and restore
946 * EL1 context on the 'cpu_context' structure for the specified security
947 * state.
948 ******************************************************************************/
cm_el1_sysregs_context_save(uint32_t security_state)949 void cm_el1_sysregs_context_save(uint32_t security_state)
950 {
951 cpu_context_t *ctx;
952
953 ctx = cm_get_context(security_state);
954 assert(ctx != NULL);
955
956 el1_sysregs_context_save(get_el1_sysregs_ctx(ctx));
957
958 #if IMAGE_BL31
959 if (security_state == SECURE)
960 PUBLISH_EVENT(cm_exited_secure_world);
961 else
962 PUBLISH_EVENT(cm_exited_normal_world);
963 #endif
964 }
965
cm_el1_sysregs_context_restore(uint32_t security_state)966 void cm_el1_sysregs_context_restore(uint32_t security_state)
967 {
968 cpu_context_t *ctx;
969
970 ctx = cm_get_context(security_state);
971 assert(ctx != NULL);
972
973 el1_sysregs_context_restore(get_el1_sysregs_ctx(ctx));
974
975 #if IMAGE_BL31
976 if (security_state == SECURE)
977 PUBLISH_EVENT(cm_entering_secure_world);
978 else
979 PUBLISH_EVENT(cm_entering_normal_world);
980 #endif
981 }
982
983 /*******************************************************************************
984 * This function populates ELR_EL3 member of 'cpu_context' pertaining to the
985 * given security state with the given entrypoint
986 ******************************************************************************/
cm_set_elr_el3(uint32_t security_state,uintptr_t entrypoint)987 void cm_set_elr_el3(uint32_t security_state, uintptr_t entrypoint)
988 {
989 cpu_context_t *ctx;
990 el3_state_t *state;
991
992 ctx = cm_get_context(security_state);
993 assert(ctx != NULL);
994
995 /* Populate EL3 state so that ERET jumps to the correct entry */
996 state = get_el3state_ctx(ctx);
997 write_ctx_reg(state, CTX_ELR_EL3, entrypoint);
998 }
999
1000 /*******************************************************************************
1001 * This function populates ELR_EL3 and SPSR_EL3 members of 'cpu_context'
1002 * pertaining to the given security state
1003 ******************************************************************************/
cm_set_elr_spsr_el3(uint32_t security_state,uintptr_t entrypoint,uint32_t spsr)1004 void cm_set_elr_spsr_el3(uint32_t security_state,
1005 uintptr_t entrypoint, uint32_t spsr)
1006 {
1007 cpu_context_t *ctx;
1008 el3_state_t *state;
1009
1010 ctx = cm_get_context(security_state);
1011 assert(ctx != NULL);
1012
1013 /* Populate EL3 state so that ERET jumps to the correct entry */
1014 state = get_el3state_ctx(ctx);
1015 write_ctx_reg(state, CTX_ELR_EL3, entrypoint);
1016 write_ctx_reg(state, CTX_SPSR_EL3, spsr);
1017 }
1018
1019 /*******************************************************************************
1020 * This function updates a single bit in the SCR_EL3 member of the 'cpu_context'
1021 * pertaining to the given security state using the value and bit position
1022 * specified in the parameters. It preserves all other bits.
1023 ******************************************************************************/
cm_write_scr_el3_bit(uint32_t security_state,uint32_t bit_pos,uint32_t value)1024 void cm_write_scr_el3_bit(uint32_t security_state,
1025 uint32_t bit_pos,
1026 uint32_t value)
1027 {
1028 cpu_context_t *ctx;
1029 el3_state_t *state;
1030 u_register_t scr_el3;
1031
1032 ctx = cm_get_context(security_state);
1033 assert(ctx != NULL);
1034
1035 /* Ensure that the bit position is a valid one */
1036 assert(((1UL << bit_pos) & SCR_VALID_BIT_MASK) != 0U);
1037
1038 /* Ensure that the 'value' is only a bit wide */
1039 assert(value <= 1U);
1040
1041 /*
1042 * Get the SCR_EL3 value from the cpu context, clear the desired bit
1043 * and set it to its new value.
1044 */
1045 state = get_el3state_ctx(ctx);
1046 scr_el3 = read_ctx_reg(state, CTX_SCR_EL3);
1047 scr_el3 &= ~(1UL << bit_pos);
1048 scr_el3 |= (u_register_t)value << bit_pos;
1049 write_ctx_reg(state, CTX_SCR_EL3, scr_el3);
1050 }
1051
1052 /*******************************************************************************
1053 * This function retrieves SCR_EL3 member of 'cpu_context' pertaining to the
1054 * given security state.
1055 ******************************************************************************/
cm_get_scr_el3(uint32_t security_state)1056 u_register_t cm_get_scr_el3(uint32_t security_state)
1057 {
1058 cpu_context_t *ctx;
1059 el3_state_t *state;
1060
1061 ctx = cm_get_context(security_state);
1062 assert(ctx != NULL);
1063
1064 /* Populate EL3 state so that ERET jumps to the correct entry */
1065 state = get_el3state_ctx(ctx);
1066 return read_ctx_reg(state, CTX_SCR_EL3);
1067 }
1068
1069 /*******************************************************************************
1070 * This function is used to program the context that's used for exception
1071 * return. This initializes the SP_EL3 to a pointer to a 'cpu_context' set for
1072 * the required security state
1073 ******************************************************************************/
cm_set_next_eret_context(uint32_t security_state)1074 void cm_set_next_eret_context(uint32_t security_state)
1075 {
1076 cpu_context_t *ctx;
1077
1078 ctx = cm_get_context(security_state);
1079 assert(ctx != NULL);
1080
1081 cm_set_next_context(ctx);
1082 }
1083