1 /*
2  * Copyright (c) 2013-2022, Arm Limited and Contributors. All rights reserved.
3  *
4  * SPDX-License-Identifier: BSD-3-Clause
5  */
6 
7 #include <assert.h>
8 #include <stdbool.h>
9 #include <string.h>
10 
11 #include <platform_def.h>
12 
13 #include <arch.h>
14 #include <arch_helpers.h>
15 #include <arch_features.h>
16 #include <bl31/interrupt_mgmt.h>
17 #include <common/bl_common.h>
18 #include <context.h>
19 #include <drivers/arm/gicv3.h>
20 #include <lib/el3_runtime/context_mgmt.h>
21 #include <lib/el3_runtime/pubsub_events.h>
22 #include <lib/extensions/amu.h>
23 #include <lib/extensions/brbe.h>
24 #include <lib/extensions/mpam.h>
25 #include <lib/extensions/sme.h>
26 #include <lib/extensions/spe.h>
27 #include <lib/extensions/sve.h>
28 #include <lib/extensions/sys_reg_trace.h>
29 #include <lib/extensions/trbe.h>
30 #include <lib/extensions/trf.h>
31 #include <lib/utils.h>
32 
33 #if ENABLE_FEAT_TWED
34 /* Make sure delay value fits within the range(0-15) */
35 CASSERT(((TWED_DELAY & ~SCR_TWEDEL_MASK) == 0U), assert_twed_delay_value_check);
36 #endif /* ENABLE_FEAT_TWED */
37 
38 static void manage_extensions_secure(cpu_context_t *ctx);
39 
setup_el1_context(cpu_context_t * ctx,const struct entry_point_info * ep)40 static void setup_el1_context(cpu_context_t *ctx, const struct entry_point_info *ep)
41 {
42 	u_register_t sctlr_elx, actlr_elx;
43 
44 	/*
45 	 * Initialise SCTLR_EL1 to the reset value corresponding to the target
46 	 * execution state setting all fields rather than relying on the hw.
47 	 * Some fields have architecturally UNKNOWN reset values and these are
48 	 * set to zero.
49 	 *
50 	 * SCTLR.EE: Endianness is taken from the entrypoint attributes.
51 	 *
52 	 * SCTLR.M, SCTLR.C and SCTLR.I: These fields must be zero (as
53 	 * required by PSCI specification)
54 	 */
55 	sctlr_elx = (EP_GET_EE(ep->h.attr) != 0U) ? SCTLR_EE_BIT : 0UL;
56 	if (GET_RW(ep->spsr) == MODE_RW_64) {
57 		sctlr_elx |= SCTLR_EL1_RES1;
58 	} else {
59 		/*
60 		 * If the target execution state is AArch32 then the following
61 		 * fields need to be set.
62 		 *
63 		 * SCTRL_EL1.nTWE: Set to one so that EL0 execution of WFE
64 		 *  instructions are not trapped to EL1.
65 		 *
66 		 * SCTLR_EL1.nTWI: Set to one so that EL0 execution of WFI
67 		 *  instructions are not trapped to EL1.
68 		 *
69 		 * SCTLR_EL1.CP15BEN: Set to one to enable EL0 execution of the
70 		 *  CP15DMB, CP15DSB, and CP15ISB instructions.
71 		 */
72 		sctlr_elx |= SCTLR_AARCH32_EL1_RES1 | SCTLR_CP15BEN_BIT
73 					| SCTLR_NTWI_BIT | SCTLR_NTWE_BIT;
74 	}
75 
76 #if ERRATA_A75_764081
77 	/*
78 	 * If workaround of errata 764081 for Cortex-A75 is used then set
79 	 * SCTLR_EL1.IESB to enable Implicit Error Synchronization Barrier.
80 	 */
81 	sctlr_elx |= SCTLR_IESB_BIT;
82 #endif
83 	/* Store the initialised SCTLR_EL1 value in the cpu_context */
84 	write_ctx_reg(get_el1_sysregs_ctx(ctx), CTX_SCTLR_EL1, sctlr_elx);
85 
86 	/*
87 	 * Base the context ACTLR_EL1 on the current value, as it is
88 	 * implementation defined. The context restore process will write
89 	 * the value from the context to the actual register and can cause
90 	 * problems for processor cores that don't expect certain bits to
91 	 * be zero.
92 	 */
93 	actlr_elx = read_actlr_el1();
94 	write_ctx_reg((get_el1_sysregs_ctx(ctx)), (CTX_ACTLR_EL1), (actlr_elx));
95 }
96 
97 /******************************************************************************
98  * This function performs initializations that are specific to SECURE state
99  * and updates the cpu context specified by 'ctx'.
100  *****************************************************************************/
setup_secure_context(cpu_context_t * ctx,const struct entry_point_info * ep)101 static void setup_secure_context(cpu_context_t *ctx, const struct entry_point_info *ep)
102 {
103 	u_register_t scr_el3;
104 	el3_state_t *state;
105 
106 	state = get_el3state_ctx(ctx);
107 	scr_el3 = read_ctx_reg(state, CTX_SCR_EL3);
108 
109 #if defined(IMAGE_BL31) && !defined(SPD_spmd)
110 	/*
111 	 * SCR_EL3.IRQ, SCR_EL3.FIQ: Enable the physical FIQ and IRQ routing as
112 	 * indicated by the interrupt routing model for BL31.
113 	 */
114 	scr_el3 |= get_scr_el3_from_routing_model(SECURE);
115 #endif
116 
117 #if !CTX_INCLUDE_MTE_REGS || ENABLE_ASSERTIONS
118 	/* Get Memory Tagging Extension support level */
119 	unsigned int mte = get_armv8_5_mte_support();
120 #endif
121 	/*
122 	 * Allow access to Allocation Tags when CTX_INCLUDE_MTE_REGS
123 	 * is set, or when MTE is only implemented at EL0.
124 	 */
125 #if CTX_INCLUDE_MTE_REGS
126 	assert((mte == MTE_IMPLEMENTED_ELX) || (mte == MTE_IMPLEMENTED_ASY));
127 	scr_el3 |= SCR_ATA_BIT;
128 #else
129 	if (mte == MTE_IMPLEMENTED_EL0) {
130 		scr_el3 |= SCR_ATA_BIT;
131 	}
132 #endif /* CTX_INCLUDE_MTE_REGS */
133 
134 	/* Enable S-EL2 if the next EL is EL2 and S-EL2 is present */
135 	if ((GET_EL(ep->spsr) == MODE_EL2) && is_armv8_4_sel2_present()) {
136 		if (GET_RW(ep->spsr) != MODE_RW_64) {
137 			ERROR("S-EL2 can not be used in AArch32\n.");
138 			panic();
139 		}
140 
141 		scr_el3 |= SCR_EEL2_BIT;
142 	}
143 
144 	write_ctx_reg(state, CTX_SCR_EL3, scr_el3);
145 
146 	/*
147 	 * Initialize EL1 context registers unless SPMC is running
148 	 * at S-EL2.
149 	 */
150 #if !SPMD_SPM_AT_SEL2
151 	setup_el1_context(ctx, ep);
152 #endif
153 
154 	manage_extensions_secure(ctx);
155 }
156 
157 #if ENABLE_RME
158 /******************************************************************************
159  * This function performs initializations that are specific to REALM state
160  * and updates the cpu context specified by 'ctx'.
161  *****************************************************************************/
setup_realm_context(cpu_context_t * ctx,const struct entry_point_info * ep)162 static void setup_realm_context(cpu_context_t *ctx, const struct entry_point_info *ep)
163 {
164 	u_register_t scr_el3;
165 	el3_state_t *state;
166 
167 	state = get_el3state_ctx(ctx);
168 	scr_el3 = read_ctx_reg(state, CTX_SCR_EL3);
169 
170 	scr_el3 |= SCR_NS_BIT | SCR_NSE_BIT | SCR_EnSCXT_BIT;
171 
172 	write_ctx_reg(state, CTX_SCR_EL3, scr_el3);
173 }
174 #endif /* ENABLE_RME */
175 
176 /******************************************************************************
177  * This function performs initializations that are specific to NON-SECURE state
178  * and updates the cpu context specified by 'ctx'.
179  *****************************************************************************/
setup_ns_context(cpu_context_t * ctx,const struct entry_point_info * ep)180 static void setup_ns_context(cpu_context_t *ctx, const struct entry_point_info *ep)
181 {
182 	u_register_t scr_el3;
183 	el3_state_t *state;
184 
185 	state = get_el3state_ctx(ctx);
186 	scr_el3 = read_ctx_reg(state, CTX_SCR_EL3);
187 
188 	/* SCR_NS: Set the NS bit */
189 	scr_el3 |= SCR_NS_BIT;
190 
191 #if !CTX_INCLUDE_PAUTH_REGS
192 	/*
193 	 * If the pointer authentication registers aren't saved during world
194 	 * switches the value of the registers can be leaked from the Secure to
195 	 * the Non-secure world. To prevent this, rather than enabling pointer
196 	 * authentication everywhere, we only enable it in the Non-secure world.
197 	 *
198 	 * If the Secure world wants to use pointer authentication,
199 	 * CTX_INCLUDE_PAUTH_REGS must be set to 1.
200 	 */
201 	scr_el3 |= SCR_API_BIT | SCR_APK_BIT;
202 #endif /* !CTX_INCLUDE_PAUTH_REGS */
203 
204 	/* Allow access to Allocation Tags when MTE is implemented. */
205 	scr_el3 |= SCR_ATA_BIT;
206 
207 #ifdef IMAGE_BL31
208 	/*
209 	 * SCR_EL3.IRQ, SCR_EL3.FIQ: Enable the physical FIQ and IRQ routing as
210 	 *  indicated by the interrupt routing model for BL31.
211 	 */
212 	scr_el3 |= get_scr_el3_from_routing_model(NON_SECURE);
213 #endif
214 	write_ctx_reg(state, CTX_SCR_EL3, scr_el3);
215 
216 	/* Initialize EL1 context registers */
217 	setup_el1_context(ctx, ep);
218 
219 	/* Initialize EL2 context registers */
220 #if CTX_INCLUDE_EL2_REGS
221 
222 	/*
223 	 * Initialize SCTLR_EL2 context register using Endianness value
224 	 * taken from the entrypoint attribute.
225 	 */
226 	u_register_t sctlr_el2 = (EP_GET_EE(ep->h.attr) != 0U) ? SCTLR_EE_BIT : 0UL;
227 	sctlr_el2 |= SCTLR_EL2_RES1;
228 	write_ctx_reg(get_el2_sysregs_ctx(ctx), CTX_SCTLR_EL2,
229 			sctlr_el2);
230 
231 	/*
232 	 * The GICv3 driver initializes the ICC_SRE_EL2 register during
233 	 * platform setup. Use the same setting for the corresponding
234 	 * context register to make sure the correct bits are set when
235 	 * restoring NS context.
236 	 */
237 	u_register_t icc_sre_el2 = read_icc_sre_el2();
238 	icc_sre_el2 |= (ICC_SRE_DIB_BIT | ICC_SRE_DFB_BIT);
239 	icc_sre_el2 |= (ICC_SRE_EN_BIT | ICC_SRE_SRE_BIT);
240 	write_ctx_reg(get_el2_sysregs_ctx(ctx), CTX_ICC_SRE_EL2,
241 			icc_sre_el2);
242 #endif /* CTX_INCLUDE_EL2_REGS */
243 }
244 
245 /*******************************************************************************
246  * The following function performs initialization of the cpu_context 'ctx'
247  * for first use that is common to all security states, and sets the
248  * initial entrypoint state as specified by the entry_point_info structure.
249  *
250  * The EE and ST attributes are used to configure the endianness and secure
251  * timer availability for the new execution context.
252  ******************************************************************************/
setup_context_common(cpu_context_t * ctx,const entry_point_info_t * ep)253 static void setup_context_common(cpu_context_t *ctx, const entry_point_info_t *ep)
254 {
255 	u_register_t scr_el3;
256 	el3_state_t *state;
257 	gp_regs_t *gp_regs;
258 
259 	/* Clear any residual register values from the context */
260 	zeromem(ctx, sizeof(*ctx));
261 
262 	/*
263 	 * SCR_EL3 was initialised during reset sequence in macro
264 	 * el3_arch_init_common. This code modifies the SCR_EL3 fields that
265 	 * affect the next EL.
266 	 *
267 	 * The following fields are initially set to zero and then updated to
268 	 * the required value depending on the state of the SPSR_EL3 and the
269 	 * Security state and entrypoint attributes of the next EL.
270 	 */
271 	scr_el3 = read_scr();
272 	scr_el3 &= ~(SCR_NS_BIT | SCR_RW_BIT | SCR_FIQ_BIT | SCR_IRQ_BIT |
273 			SCR_ST_BIT | SCR_HCE_BIT | SCR_NSE_BIT);
274 
275 	/*
276 	 * SCR_EL3.RW: Set the execution state, AArch32 or AArch64, for next
277 	 *  Exception level as specified by SPSR.
278 	 */
279 	if (GET_RW(ep->spsr) == MODE_RW_64) {
280 		scr_el3 |= SCR_RW_BIT;
281 	}
282 
283 	/*
284 	 * SCR_EL3.ST: Traps Secure EL1 accesses to the Counter-timer Physical
285 	 * Secure timer registers to EL3, from AArch64 state only, if specified
286 	 * by the entrypoint attributes. If SEL2 is present and enabled, the ST
287 	 * bit always behaves as 1 (i.e. secure physical timer register access
288 	 * is not trapped)
289 	 */
290 	if (EP_GET_ST(ep->h.attr) != 0U) {
291 		scr_el3 |= SCR_ST_BIT;
292 	}
293 
294 	/*
295 	 * If FEAT_HCX is enabled, enable access to HCRX_EL2 by setting
296 	 * SCR_EL3.HXEn.
297 	 */
298 #if ENABLE_FEAT_HCX
299 	scr_el3 |= SCR_HXEn_BIT;
300 #endif
301 
302 #if RAS_TRAP_LOWER_EL_ERR_ACCESS
303 	/*
304 	 * SCR_EL3.TERR: Trap Error record accesses. Accesses to the RAS ERR
305 	 * and RAS ERX registers from EL1 and EL2 are trapped to EL3.
306 	 */
307 	scr_el3 |= SCR_TERR_BIT;
308 #endif
309 
310 #if !HANDLE_EA_EL3_FIRST
311 	/*
312 	 * SCR_EL3.EA: Do not route External Abort and SError Interrupt External
313 	 * to EL3 when executing at a lower EL. When executing at EL3, External
314 	 * Aborts are taken to EL3.
315 	 */
316 	scr_el3 &= ~SCR_EA_BIT;
317 #endif
318 
319 #if FAULT_INJECTION_SUPPORT
320 	/* Enable fault injection from lower ELs */
321 	scr_el3 |= SCR_FIEN_BIT;
322 #endif
323 
324 	/*
325 	 * CPTR_EL3 was initialized out of reset, copy that value to the
326 	 * context register.
327 	 */
328 	write_ctx_reg(get_el3state_ctx(ctx), CTX_CPTR_EL3, read_cptr_el3());
329 
330 	/*
331 	 * SCR_EL3.HCE: Enable HVC instructions if next execution state is
332 	 * AArch64 and next EL is EL2, or if next execution state is AArch32 and
333 	 * next mode is Hyp.
334 	 * SCR_EL3.FGTEn: Enable Fine Grained Virtualization Traps under the
335 	 * same conditions as HVC instructions and when the processor supports
336 	 * ARMv8.6-FGT.
337 	 * SCR_EL3.ECVEn: Enable Enhanced Counter Virtualization (ECV)
338 	 * CNTPOFF_EL2 register under the same conditions as HVC instructions
339 	 * and when the processor supports ECV.
340 	 */
341 	if (((GET_RW(ep->spsr) == MODE_RW_64) && (GET_EL(ep->spsr) == MODE_EL2))
342 	    || ((GET_RW(ep->spsr) != MODE_RW_64)
343 		&& (GET_M32(ep->spsr) == MODE32_hyp))) {
344 		scr_el3 |= SCR_HCE_BIT;
345 
346 		if (is_armv8_6_fgt_present()) {
347 			scr_el3 |= SCR_FGTEN_BIT;
348 		}
349 
350 		if (get_armv8_6_ecv_support()
351 		    == ID_AA64MMFR0_EL1_ECV_SELF_SYNCH) {
352 			scr_el3 |= SCR_ECVEN_BIT;
353 		}
354 	}
355 
356 #if ENABLE_FEAT_TWED
357 	/* Enable WFE trap delay in SCR_EL3 if supported and configured */
358 	/* Set delay in SCR_EL3 */
359 	scr_el3 &= ~(SCR_TWEDEL_MASK << SCR_TWEDEL_SHIFT);
360 	scr_el3 |= ((TWED_DELAY & SCR_TWEDEL_MASK)
361 			<< SCR_TWEDEL_SHIFT);
362 
363 	/* Enable WFE delay */
364 	scr_el3 |= SCR_TWEDEn_BIT;
365 #endif /* ENABLE_FEAT_TWED */
366 
367 	/*
368 	 * Populate EL3 state so that we've the right context
369 	 * before doing ERET
370 	 */
371 	state = get_el3state_ctx(ctx);
372 	write_ctx_reg(state, CTX_SCR_EL3, scr_el3);
373 	write_ctx_reg(state, CTX_ELR_EL3, ep->pc);
374 	write_ctx_reg(state, CTX_SPSR_EL3, ep->spsr);
375 
376 	/*
377 	 * Store the X0-X7 value from the entrypoint into the context
378 	 * Use memcpy as we are in control of the layout of the structures
379 	 */
380 	gp_regs = get_gpregs_ctx(ctx);
381 	memcpy(gp_regs, (void *)&ep->args, sizeof(aapcs64_params_t));
382 }
383 
384 /*******************************************************************************
385  * Context management library initialization routine. This library is used by
386  * runtime services to share pointers to 'cpu_context' structures for secure
387  * non-secure and realm states. Management of the structures and their associated
388  * memory is not done by the context management library e.g. the PSCI service
389  * manages the cpu context used for entry from and exit to the non-secure state.
390  * The Secure payload dispatcher service manages the context(s) corresponding to
391  * the secure state. It also uses this library to get access to the non-secure
392  * state cpu context pointers.
393  * Lastly, this library provides the API to make SP_EL3 point to the cpu context
394  * which will be used for programming an entry into a lower EL. The same context
395  * will be used to save state upon exception entry from that EL.
396  ******************************************************************************/
cm_init(void)397 void __init cm_init(void)
398 {
399 	/*
400 	 * The context management library has only global data to intialize, but
401 	 * that will be done when the BSS is zeroed out.
402 	 */
403 }
404 
405 /*******************************************************************************
406  * This is the high-level function used to initialize the cpu_context 'ctx' for
407  * first use. It performs initializations that are common to all security states
408  * and initializations specific to the security state specified in 'ep'
409  ******************************************************************************/
cm_setup_context(cpu_context_t * ctx,const entry_point_info_t * ep)410 void cm_setup_context(cpu_context_t *ctx, const entry_point_info_t *ep)
411 {
412 	unsigned int security_state;
413 
414 	assert(ctx != NULL);
415 
416 	/*
417 	 * Perform initializations that are common
418 	 * to all security states
419 	 */
420 	setup_context_common(ctx, ep);
421 
422 	security_state = GET_SECURITY_STATE(ep->h.attr);
423 
424 	/* Perform security state specific initializations */
425 	switch (security_state) {
426 	case SECURE:
427 		setup_secure_context(ctx, ep);
428 		break;
429 #if ENABLE_RME
430 	case REALM:
431 		setup_realm_context(ctx, ep);
432 		break;
433 #endif
434 	case NON_SECURE:
435 		setup_ns_context(ctx, ep);
436 		break;
437 	default:
438 		ERROR("Invalid security state\n");
439 		panic();
440 		break;
441 	}
442 }
443 
444 /*******************************************************************************
445  * Enable architecture extensions on first entry to Non-secure world.
446  * When EL2 is implemented but unused `el2_unused` is non-zero, otherwise
447  * it is zero.
448  ******************************************************************************/
manage_extensions_nonsecure(bool el2_unused,cpu_context_t * ctx)449 static void manage_extensions_nonsecure(bool el2_unused, cpu_context_t *ctx)
450 {
451 #if IMAGE_BL31
452 #if ENABLE_SPE_FOR_LOWER_ELS
453 	spe_enable(el2_unused);
454 #endif
455 
456 #if ENABLE_AMU
457 	amu_enable(el2_unused, ctx);
458 #endif
459 
460 #if ENABLE_SME_FOR_NS
461 	/* Enable SME, SVE, and FPU/SIMD for non-secure world. */
462 	sme_enable(ctx);
463 #elif ENABLE_SVE_FOR_NS
464 	/* Enable SVE and FPU/SIMD for non-secure world. */
465 	sve_enable(ctx);
466 #endif
467 
468 #if ENABLE_MPAM_FOR_LOWER_ELS
469 	mpam_enable(el2_unused);
470 #endif
471 
472 #if ENABLE_TRBE_FOR_NS
473 	trbe_enable();
474 #endif /* ENABLE_TRBE_FOR_NS */
475 
476 #if ENABLE_BRBE_FOR_NS
477 	brbe_enable();
478 #endif /* ENABLE_BRBE_FOR_NS */
479 
480 #if ENABLE_SYS_REG_TRACE_FOR_NS
481 	sys_reg_trace_enable(ctx);
482 #endif /* ENABLE_SYS_REG_TRACE_FOR_NS */
483 
484 #if ENABLE_TRF_FOR_NS
485 	trf_enable();
486 #endif /* ENABLE_TRF_FOR_NS */
487 #endif
488 }
489 
490 /*******************************************************************************
491  * Enable architecture extensions on first entry to Secure world.
492  ******************************************************************************/
manage_extensions_secure(cpu_context_t * ctx)493 static void manage_extensions_secure(cpu_context_t *ctx)
494 {
495 #if IMAGE_BL31
496  #if ENABLE_SME_FOR_NS
497   #if ENABLE_SME_FOR_SWD
498 	/*
499 	 * Enable SME, SVE, FPU/SIMD in secure context, secure manager must
500 	 * ensure SME, SVE, and FPU/SIMD context properly managed.
501 	 */
502 	sme_enable(ctx);
503   #else /* ENABLE_SME_FOR_SWD */
504 	/*
505 	 * Disable SME, SVE, FPU/SIMD in secure context so non-secure world can
506 	 * safely use the associated registers.
507 	 */
508 	sme_disable(ctx);
509   #endif /* ENABLE_SME_FOR_SWD */
510  #elif ENABLE_SVE_FOR_NS
511   #if ENABLE_SVE_FOR_SWD
512 	/*
513 	 * Enable SVE and FPU in secure context, secure manager must ensure that
514 	 * the SVE and FPU register contexts are properly managed.
515 	 */
516 	sve_enable(ctx);
517  #else /* ENABLE_SVE_FOR_SWD */
518 	/*
519 	 * Disable SVE and FPU in secure context so non-secure world can safely
520 	 * use them.
521 	 */
522 	sve_disable(ctx);
523   #endif /* ENABLE_SVE_FOR_SWD */
524  #endif /* ENABLE_SVE_FOR_NS */
525 #endif /* IMAGE_BL31 */
526 }
527 
528 /*******************************************************************************
529  * The following function initializes the cpu_context for a CPU specified by
530  * its `cpu_idx` for first use, and sets the initial entrypoint state as
531  * specified by the entry_point_info structure.
532  ******************************************************************************/
cm_init_context_by_index(unsigned int cpu_idx,const entry_point_info_t * ep)533 void cm_init_context_by_index(unsigned int cpu_idx,
534 			      const entry_point_info_t *ep)
535 {
536 	cpu_context_t *ctx;
537 	ctx = cm_get_context_by_index(cpu_idx, GET_SECURITY_STATE(ep->h.attr));
538 	cm_setup_context(ctx, ep);
539 }
540 
541 /*******************************************************************************
542  * The following function initializes the cpu_context for the current CPU
543  * for first use, and sets the initial entrypoint state as specified by the
544  * entry_point_info structure.
545  ******************************************************************************/
cm_init_my_context(const entry_point_info_t * ep)546 void cm_init_my_context(const entry_point_info_t *ep)
547 {
548 	cpu_context_t *ctx;
549 	ctx = cm_get_context(GET_SECURITY_STATE(ep->h.attr));
550 	cm_setup_context(ctx, ep);
551 }
552 
553 /*******************************************************************************
554  * Prepare the CPU system registers for first entry into realm, secure, or
555  * normal world.
556  *
557  * If execution is requested to EL2 or hyp mode, SCTLR_EL2 is initialized
558  * If execution is requested to non-secure EL1 or svc mode, and the CPU supports
559  * EL2 then EL2 is disabled by configuring all necessary EL2 registers.
560  * For all entries, the EL1 registers are initialized from the cpu_context
561  ******************************************************************************/
cm_prepare_el3_exit(uint32_t security_state)562 void cm_prepare_el3_exit(uint32_t security_state)
563 {
564 	u_register_t sctlr_elx, scr_el3, mdcr_el2;
565 	cpu_context_t *ctx = cm_get_context(security_state);
566 	bool el2_unused = false;
567 	uint64_t hcr_el2 = 0U;
568 
569 	assert(ctx != NULL);
570 
571 	if (security_state == NON_SECURE) {
572 		scr_el3 = read_ctx_reg(get_el3state_ctx(ctx),
573 						 CTX_SCR_EL3);
574 		if ((scr_el3 & SCR_HCE_BIT) != 0U) {
575 			/* Use SCTLR_EL1.EE value to initialise sctlr_el2 */
576 			sctlr_elx = read_ctx_reg(get_el1_sysregs_ctx(ctx),
577 							   CTX_SCTLR_EL1);
578 			sctlr_elx &= SCTLR_EE_BIT;
579 			sctlr_elx |= SCTLR_EL2_RES1;
580 #if ERRATA_A75_764081
581 			/*
582 			 * If workaround of errata 764081 for Cortex-A75 is used
583 			 * then set SCTLR_EL2.IESB to enable Implicit Error
584 			 * Synchronization Barrier.
585 			 */
586 			sctlr_elx |= SCTLR_IESB_BIT;
587 #endif
588 			write_sctlr_el2(sctlr_elx);
589 		} else if (el_implemented(2) != EL_IMPL_NONE) {
590 			el2_unused = true;
591 
592 			/*
593 			 * EL2 present but unused, need to disable safely.
594 			 * SCTLR_EL2 can be ignored in this case.
595 			 *
596 			 * Set EL2 register width appropriately: Set HCR_EL2
597 			 * field to match SCR_EL3.RW.
598 			 */
599 			if ((scr_el3 & SCR_RW_BIT) != 0U)
600 				hcr_el2 |= HCR_RW_BIT;
601 
602 			/*
603 			 * For Armv8.3 pointer authentication feature, disable
604 			 * traps to EL2 when accessing key registers or using
605 			 * pointer authentication instructions from lower ELs.
606 			 */
607 			hcr_el2 |= (HCR_API_BIT | HCR_APK_BIT);
608 
609 			write_hcr_el2(hcr_el2);
610 
611 			/*
612 			 * Initialise CPTR_EL2 setting all fields rather than
613 			 * relying on the hw. All fields have architecturally
614 			 * UNKNOWN reset values.
615 			 *
616 			 * CPTR_EL2.TCPAC: Set to zero so that Non-secure EL1
617 			 *  accesses to the CPACR_EL1 or CPACR from both
618 			 *  Execution states do not trap to EL2.
619 			 *
620 			 * CPTR_EL2.TTA: Set to zero so that Non-secure System
621 			 *  register accesses to the trace registers from both
622 			 *  Execution states do not trap to EL2.
623 			 *  If PE trace unit System registers are not implemented
624 			 *  then this bit is reserved, and must be set to zero.
625 			 *
626 			 * CPTR_EL2.TFP: Set to zero so that Non-secure accesses
627 			 *  to SIMD and floating-point functionality from both
628 			 *  Execution states do not trap to EL2.
629 			 */
630 			write_cptr_el2(CPTR_EL2_RESET_VAL &
631 					~(CPTR_EL2_TCPAC_BIT | CPTR_EL2_TTA_BIT
632 					| CPTR_EL2_TFP_BIT));
633 
634 			/*
635 			 * Initialise CNTHCTL_EL2. All fields are
636 			 * architecturally UNKNOWN on reset and are set to zero
637 			 * except for field(s) listed below.
638 			 *
639 			 * CNTHCTL_EL2.EL1PTEN: Set to one to disable traps to
640 			 *  Hyp mode of Non-secure EL0 and EL1 accesses to the
641 			 *  physical timer registers.
642 			 *
643 			 * CNTHCTL_EL2.EL1PCTEN: Set to one to disable traps to
644 			 *  Hyp mode of  Non-secure EL0 and EL1 accesses to the
645 			 *  physical counter registers.
646 			 */
647 			write_cnthctl_el2(CNTHCTL_RESET_VAL |
648 						EL1PCEN_BIT | EL1PCTEN_BIT);
649 
650 			/*
651 			 * Initialise CNTVOFF_EL2 to zero as it resets to an
652 			 * architecturally UNKNOWN value.
653 			 */
654 			write_cntvoff_el2(0);
655 
656 			/*
657 			 * Set VPIDR_EL2 and VMPIDR_EL2 to match MIDR_EL1 and
658 			 * MPIDR_EL1 respectively.
659 			 */
660 			write_vpidr_el2(read_midr_el1());
661 			write_vmpidr_el2(read_mpidr_el1());
662 
663 			/*
664 			 * Initialise VTTBR_EL2. All fields are architecturally
665 			 * UNKNOWN on reset.
666 			 *
667 			 * VTTBR_EL2.VMID: Set to zero. Even though EL1&0 stage
668 			 *  2 address translation is disabled, cache maintenance
669 			 *  operations depend on the VMID.
670 			 *
671 			 * VTTBR_EL2.BADDR: Set to zero as EL1&0 stage 2 address
672 			 *  translation is disabled.
673 			 */
674 			write_vttbr_el2(VTTBR_RESET_VAL &
675 				~((VTTBR_VMID_MASK << VTTBR_VMID_SHIFT)
676 				| (VTTBR_BADDR_MASK << VTTBR_BADDR_SHIFT)));
677 
678 			/*
679 			 * Initialise MDCR_EL2, setting all fields rather than
680 			 * relying on hw. Some fields are architecturally
681 			 * UNKNOWN on reset.
682 			 *
683 			 * MDCR_EL2.HLP: Set to one so that event counter
684 			 *  overflow, that is recorded in PMOVSCLR_EL0[0-30],
685 			 *  occurs on the increment that changes
686 			 *  PMEVCNTR<n>_EL0[63] from 1 to 0, when ARMv8.5-PMU is
687 			 *  implemented. This bit is RES0 in versions of the
688 			 *  architecture earlier than ARMv8.5, setting it to 1
689 			 *  doesn't have any effect on them.
690 			 *
691 			 * MDCR_EL2.TTRF: Set to zero so that access to Trace
692 			 *  Filter Control register TRFCR_EL1 at EL1 is not
693 			 *  trapped to EL2. This bit is RES0 in versions of
694 			 *  the architecture earlier than ARMv8.4.
695 			 *
696 			 * MDCR_EL2.HPMD: Set to one so that event counting is
697 			 *  prohibited at EL2. This bit is RES0 in versions of
698 			 *  the architecture earlier than ARMv8.1, setting it
699 			 *  to 1 doesn't have any effect on them.
700 			 *
701 			 * MDCR_EL2.TPMS: Set to zero so that accesses to
702 			 *  Statistical Profiling control registers from EL1
703 			 *  do not trap to EL2. This bit is RES0 when SPE is
704 			 *  not implemented.
705 			 *
706 			 * MDCR_EL2.TDRA: Set to zero so that Non-secure EL0 and
707 			 *  EL1 System register accesses to the Debug ROM
708 			 *  registers are not trapped to EL2.
709 			 *
710 			 * MDCR_EL2.TDOSA: Set to zero so that Non-secure EL1
711 			 *  System register accesses to the powerdown debug
712 			 *  registers are not trapped to EL2.
713 			 *
714 			 * MDCR_EL2.TDA: Set to zero so that System register
715 			 *  accesses to the debug registers do not trap to EL2.
716 			 *
717 			 * MDCR_EL2.TDE: Set to zero so that debug exceptions
718 			 *  are not routed to EL2.
719 			 *
720 			 * MDCR_EL2.HPME: Set to zero to disable EL2 Performance
721 			 *  Monitors.
722 			 *
723 			 * MDCR_EL2.TPM: Set to zero so that Non-secure EL0 and
724 			 *  EL1 accesses to all Performance Monitors registers
725 			 *  are not trapped to EL2.
726 			 *
727 			 * MDCR_EL2.TPMCR: Set to zero so that Non-secure EL0
728 			 *  and EL1 accesses to the PMCR_EL0 or PMCR are not
729 			 *  trapped to EL2.
730 			 *
731 			 * MDCR_EL2.HPMN: Set to value of PMCR_EL0.N which is the
732 			 *  architecturally-defined reset value.
733 			 *
734 			 * MDCR_EL2.E2TB: Set to zero so that the trace Buffer
735 			 *  owning exception level is NS-EL1 and, tracing is
736 			 *  prohibited at NS-EL2. These bits are RES0 when
737 			 *  FEAT_TRBE is not implemented.
738 			 */
739 			mdcr_el2 = ((MDCR_EL2_RESET_VAL | MDCR_EL2_HLP |
740 				     MDCR_EL2_HPMD) |
741 				   ((read_pmcr_el0() & PMCR_EL0_N_BITS)
742 				   >> PMCR_EL0_N_SHIFT)) &
743 				   ~(MDCR_EL2_TTRF | MDCR_EL2_TPMS |
744 				     MDCR_EL2_TDRA_BIT | MDCR_EL2_TDOSA_BIT |
745 				     MDCR_EL2_TDA_BIT | MDCR_EL2_TDE_BIT |
746 				     MDCR_EL2_HPME_BIT | MDCR_EL2_TPM_BIT |
747 				     MDCR_EL2_TPMCR_BIT |
748 				     MDCR_EL2_E2TB(MDCR_EL2_E2TB_EL1));
749 
750 			write_mdcr_el2(mdcr_el2);
751 
752 			/*
753 			 * Initialise HSTR_EL2. All fields are architecturally
754 			 * UNKNOWN on reset.
755 			 *
756 			 * HSTR_EL2.T<n>: Set all these fields to zero so that
757 			 *  Non-secure EL0 or EL1 accesses to System registers
758 			 *  do not trap to EL2.
759 			 */
760 			write_hstr_el2(HSTR_EL2_RESET_VAL & ~(HSTR_EL2_T_MASK));
761 			/*
762 			 * Initialise CNTHP_CTL_EL2. All fields are
763 			 * architecturally UNKNOWN on reset.
764 			 *
765 			 * CNTHP_CTL_EL2:ENABLE: Set to zero to disable the EL2
766 			 *  physical timer and prevent timer interrupts.
767 			 */
768 			write_cnthp_ctl_el2(CNTHP_CTL_RESET_VAL &
769 						~(CNTHP_CTL_ENABLE_BIT));
770 		}
771 		manage_extensions_nonsecure(el2_unused, ctx);
772 	}
773 
774 	cm_el1_sysregs_context_restore(security_state);
775 	cm_set_next_eret_context(security_state);
776 }
777 
778 #if CTX_INCLUDE_EL2_REGS
779 /*******************************************************************************
780  * Save EL2 sysreg context
781  ******************************************************************************/
cm_el2_sysregs_context_save(uint32_t security_state)782 void cm_el2_sysregs_context_save(uint32_t security_state)
783 {
784 	u_register_t scr_el3 = read_scr();
785 
786 	/*
787 	 * Always save the non-secure and realm EL2 context, only save the
788 	 * S-EL2 context if S-EL2 is enabled.
789 	 */
790 	if ((security_state != SECURE) ||
791 	    ((security_state == SECURE) && ((scr_el3 & SCR_EEL2_BIT) != 0U))) {
792 		cpu_context_t *ctx;
793 		el2_sysregs_t *el2_sysregs_ctx;
794 
795 		ctx = cm_get_context(security_state);
796 		assert(ctx != NULL);
797 
798 		el2_sysregs_ctx = get_el2_sysregs_ctx(ctx);
799 
800 		el2_sysregs_context_save_common(el2_sysregs_ctx);
801 #if ENABLE_SPE_FOR_LOWER_ELS
802 		el2_sysregs_context_save_spe(el2_sysregs_ctx);
803 #endif
804 #if CTX_INCLUDE_MTE_REGS
805 		el2_sysregs_context_save_mte(el2_sysregs_ctx);
806 #endif
807 #if ENABLE_MPAM_FOR_LOWER_ELS
808 		el2_sysregs_context_save_mpam(el2_sysregs_ctx);
809 #endif
810 #if ENABLE_FEAT_FGT
811 		el2_sysregs_context_save_fgt(el2_sysregs_ctx);
812 #endif
813 #if ENABLE_FEAT_ECV
814 		el2_sysregs_context_save_ecv(el2_sysregs_ctx);
815 #endif
816 #if ENABLE_FEAT_VHE
817 		el2_sysregs_context_save_vhe(el2_sysregs_ctx);
818 #endif
819 #if RAS_EXTENSION
820 		el2_sysregs_context_save_ras(el2_sysregs_ctx);
821 #endif
822 #if CTX_INCLUDE_NEVE_REGS
823 		el2_sysregs_context_save_nv2(el2_sysregs_ctx);
824 #endif
825 #if ENABLE_TRF_FOR_NS
826 		el2_sysregs_context_save_trf(el2_sysregs_ctx);
827 #endif
828 #if ENABLE_FEAT_CSV2_2
829 		el2_sysregs_context_save_csv2(el2_sysregs_ctx);
830 #endif
831 #if ENABLE_FEAT_HCX
832 		el2_sysregs_context_save_hcx(el2_sysregs_ctx);
833 #endif
834 	}
835 }
836 
837 /*******************************************************************************
838  * Restore EL2 sysreg context
839  ******************************************************************************/
cm_el2_sysregs_context_restore(uint32_t security_state)840 void cm_el2_sysregs_context_restore(uint32_t security_state)
841 {
842 	u_register_t scr_el3 = read_scr();
843 
844 	/*
845 	 * Always restore the non-secure and realm EL2 context, only restore the
846 	 * S-EL2 context if S-EL2 is enabled.
847 	 */
848 	if ((security_state != SECURE) ||
849 	    ((security_state == SECURE) && ((scr_el3 & SCR_EEL2_BIT) != 0U))) {
850 		cpu_context_t *ctx;
851 		el2_sysregs_t *el2_sysregs_ctx;
852 
853 		ctx = cm_get_context(security_state);
854 		assert(ctx != NULL);
855 
856 		el2_sysregs_ctx = get_el2_sysregs_ctx(ctx);
857 
858 		el2_sysregs_context_restore_common(el2_sysregs_ctx);
859 #if ENABLE_SPE_FOR_LOWER_ELS
860 		el2_sysregs_context_restore_spe(el2_sysregs_ctx);
861 #endif
862 #if CTX_INCLUDE_MTE_REGS
863 		el2_sysregs_context_restore_mte(el2_sysregs_ctx);
864 #endif
865 #if ENABLE_MPAM_FOR_LOWER_ELS
866 		el2_sysregs_context_restore_mpam(el2_sysregs_ctx);
867 #endif
868 #if ENABLE_FEAT_FGT
869 		el2_sysregs_context_restore_fgt(el2_sysregs_ctx);
870 #endif
871 #if ENABLE_FEAT_ECV
872 		el2_sysregs_context_restore_ecv(el2_sysregs_ctx);
873 #endif
874 #if ENABLE_FEAT_VHE
875 		el2_sysregs_context_restore_vhe(el2_sysregs_ctx);
876 #endif
877 #if RAS_EXTENSION
878 		el2_sysregs_context_restore_ras(el2_sysregs_ctx);
879 #endif
880 #if CTX_INCLUDE_NEVE_REGS
881 		el2_sysregs_context_restore_nv2(el2_sysregs_ctx);
882 #endif
883 #if ENABLE_TRF_FOR_NS
884 		el2_sysregs_context_restore_trf(el2_sysregs_ctx);
885 #endif
886 #if ENABLE_FEAT_CSV2_2
887 		el2_sysregs_context_restore_csv2(el2_sysregs_ctx);
888 #endif
889 #if ENABLE_FEAT_HCX
890 		el2_sysregs_context_restore_hcx(el2_sysregs_ctx);
891 #endif
892 	}
893 }
894 #endif /* CTX_INCLUDE_EL2_REGS */
895 
896 /*******************************************************************************
897  * This function is used to exit to Non-secure world. If CTX_INCLUDE_EL2_REGS
898  * is enabled, it restores EL1 and EL2 sysreg contexts instead of directly
899  * updating EL1 and EL2 registers. Otherwise, it calls the generic
900  * cm_prepare_el3_exit function.
901  ******************************************************************************/
cm_prepare_el3_exit_ns(void)902 void cm_prepare_el3_exit_ns(void)
903 {
904 #if CTX_INCLUDE_EL2_REGS
905 	cpu_context_t *ctx = cm_get_context(NON_SECURE);
906 	assert(ctx != NULL);
907 
908 	/* Assert that EL2 is used. */
909 #if ENABLE_ASSERTIONS
910 	el3_state_t *state = get_el3state_ctx(ctx);
911 	u_register_t scr_el3 = read_ctx_reg(state, CTX_SCR_EL3);
912 #endif
913 	assert(((scr_el3 & SCR_HCE_BIT) != 0UL) &&
914 			(el_implemented(2U) != EL_IMPL_NONE));
915 
916 	/*
917 	 * Currently some extensions are configured using
918 	 * direct register updates. Therefore, do this here
919 	 * instead of when setting up context.
920 	 */
921 	manage_extensions_nonsecure(0, ctx);
922 
923 	/*
924 	 * Set the NS bit to be able to access the ICC_SRE_EL2
925 	 * register when restoring context.
926 	 */
927 	write_scr_el3(read_scr_el3() | SCR_NS_BIT);
928 
929 	/*
930 	 * Ensure the NS bit change is committed before the EL2/EL1
931 	 * state restoration.
932 	 */
933 	isb();
934 
935 	/* Restore EL2 and EL1 sysreg contexts */
936 	cm_el2_sysregs_context_restore(NON_SECURE);
937 	cm_el1_sysregs_context_restore(NON_SECURE);
938 	cm_set_next_eret_context(NON_SECURE);
939 #else
940 	cm_prepare_el3_exit(NON_SECURE);
941 #endif /* CTX_INCLUDE_EL2_REGS */
942 }
943 
944 /*******************************************************************************
945  * The next four functions are used by runtime services to save and restore
946  * EL1 context on the 'cpu_context' structure for the specified security
947  * state.
948  ******************************************************************************/
cm_el1_sysregs_context_save(uint32_t security_state)949 void cm_el1_sysregs_context_save(uint32_t security_state)
950 {
951 	cpu_context_t *ctx;
952 
953 	ctx = cm_get_context(security_state);
954 	assert(ctx != NULL);
955 
956 	el1_sysregs_context_save(get_el1_sysregs_ctx(ctx));
957 
958 #if IMAGE_BL31
959 	if (security_state == SECURE)
960 		PUBLISH_EVENT(cm_exited_secure_world);
961 	else
962 		PUBLISH_EVENT(cm_exited_normal_world);
963 #endif
964 }
965 
cm_el1_sysregs_context_restore(uint32_t security_state)966 void cm_el1_sysregs_context_restore(uint32_t security_state)
967 {
968 	cpu_context_t *ctx;
969 
970 	ctx = cm_get_context(security_state);
971 	assert(ctx != NULL);
972 
973 	el1_sysregs_context_restore(get_el1_sysregs_ctx(ctx));
974 
975 #if IMAGE_BL31
976 	if (security_state == SECURE)
977 		PUBLISH_EVENT(cm_entering_secure_world);
978 	else
979 		PUBLISH_EVENT(cm_entering_normal_world);
980 #endif
981 }
982 
983 /*******************************************************************************
984  * This function populates ELR_EL3 member of 'cpu_context' pertaining to the
985  * given security state with the given entrypoint
986  ******************************************************************************/
cm_set_elr_el3(uint32_t security_state,uintptr_t entrypoint)987 void cm_set_elr_el3(uint32_t security_state, uintptr_t entrypoint)
988 {
989 	cpu_context_t *ctx;
990 	el3_state_t *state;
991 
992 	ctx = cm_get_context(security_state);
993 	assert(ctx != NULL);
994 
995 	/* Populate EL3 state so that ERET jumps to the correct entry */
996 	state = get_el3state_ctx(ctx);
997 	write_ctx_reg(state, CTX_ELR_EL3, entrypoint);
998 }
999 
1000 /*******************************************************************************
1001  * This function populates ELR_EL3 and SPSR_EL3 members of 'cpu_context'
1002  * pertaining to the given security state
1003  ******************************************************************************/
cm_set_elr_spsr_el3(uint32_t security_state,uintptr_t entrypoint,uint32_t spsr)1004 void cm_set_elr_spsr_el3(uint32_t security_state,
1005 			uintptr_t entrypoint, uint32_t spsr)
1006 {
1007 	cpu_context_t *ctx;
1008 	el3_state_t *state;
1009 
1010 	ctx = cm_get_context(security_state);
1011 	assert(ctx != NULL);
1012 
1013 	/* Populate EL3 state so that ERET jumps to the correct entry */
1014 	state = get_el3state_ctx(ctx);
1015 	write_ctx_reg(state, CTX_ELR_EL3, entrypoint);
1016 	write_ctx_reg(state, CTX_SPSR_EL3, spsr);
1017 }
1018 
1019 /*******************************************************************************
1020  * This function updates a single bit in the SCR_EL3 member of the 'cpu_context'
1021  * pertaining to the given security state using the value and bit position
1022  * specified in the parameters. It preserves all other bits.
1023  ******************************************************************************/
cm_write_scr_el3_bit(uint32_t security_state,uint32_t bit_pos,uint32_t value)1024 void cm_write_scr_el3_bit(uint32_t security_state,
1025 			  uint32_t bit_pos,
1026 			  uint32_t value)
1027 {
1028 	cpu_context_t *ctx;
1029 	el3_state_t *state;
1030 	u_register_t scr_el3;
1031 
1032 	ctx = cm_get_context(security_state);
1033 	assert(ctx != NULL);
1034 
1035 	/* Ensure that the bit position is a valid one */
1036 	assert(((1UL << bit_pos) & SCR_VALID_BIT_MASK) != 0U);
1037 
1038 	/* Ensure that the 'value' is only a bit wide */
1039 	assert(value <= 1U);
1040 
1041 	/*
1042 	 * Get the SCR_EL3 value from the cpu context, clear the desired bit
1043 	 * and set it to its new value.
1044 	 */
1045 	state = get_el3state_ctx(ctx);
1046 	scr_el3 = read_ctx_reg(state, CTX_SCR_EL3);
1047 	scr_el3 &= ~(1UL << bit_pos);
1048 	scr_el3 |= (u_register_t)value << bit_pos;
1049 	write_ctx_reg(state, CTX_SCR_EL3, scr_el3);
1050 }
1051 
1052 /*******************************************************************************
1053  * This function retrieves SCR_EL3 member of 'cpu_context' pertaining to the
1054  * given security state.
1055  ******************************************************************************/
cm_get_scr_el3(uint32_t security_state)1056 u_register_t cm_get_scr_el3(uint32_t security_state)
1057 {
1058 	cpu_context_t *ctx;
1059 	el3_state_t *state;
1060 
1061 	ctx = cm_get_context(security_state);
1062 	assert(ctx != NULL);
1063 
1064 	/* Populate EL3 state so that ERET jumps to the correct entry */
1065 	state = get_el3state_ctx(ctx);
1066 	return read_ctx_reg(state, CTX_SCR_EL3);
1067 }
1068 
1069 /*******************************************************************************
1070  * This function is used to program the context that's used for exception
1071  * return. This initializes the SP_EL3 to a pointer to a 'cpu_context' set for
1072  * the required security state
1073  ******************************************************************************/
cm_set_next_eret_context(uint32_t security_state)1074 void cm_set_next_eret_context(uint32_t security_state)
1075 {
1076 	cpu_context_t *ctx;
1077 
1078 	ctx = cm_get_context(security_state);
1079 	assert(ctx != NULL);
1080 
1081 	cm_set_next_context(ctx);
1082 }
1083