1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * AMD Encrypted Register State Support
4 *
5 * Author: Joerg Roedel <jroedel@suse.de>
6 */
7
8 /*
9 * misc.h needs to be first because it knows how to include the other kernel
10 * headers in the pre-decompression code in a way that does not break
11 * compilation.
12 */
13 #include "misc.h"
14
15 #include <asm/pgtable_types.h>
16 #include <asm/sev.h>
17 #include <asm/trapnr.h>
18 #include <asm/trap_pf.h>
19 #include <asm/msr-index.h>
20 #include <asm/fpu/xcr.h>
21 #include <asm/ptrace.h>
22 #include <asm/svm.h>
23 #include <asm/cpuid.h>
24
25 #include "error.h"
26 #include "../msr.h"
27
28 struct ghcb boot_ghcb_page __aligned(PAGE_SIZE);
29 struct ghcb *boot_ghcb;
30
31 /*
32 * Copy a version of this function here - insn-eval.c can't be used in
33 * pre-decompression code.
34 */
insn_has_rep_prefix(struct insn * insn)35 static bool insn_has_rep_prefix(struct insn *insn)
36 {
37 insn_byte_t p;
38 int i;
39
40 insn_get_prefixes(insn);
41
42 for_each_insn_prefix(insn, i, p) {
43 if (p == 0xf2 || p == 0xf3)
44 return true;
45 }
46
47 return false;
48 }
49
50 /*
51 * Only a dummy for insn_get_seg_base() - Early boot-code is 64bit only and
52 * doesn't use segments.
53 */
insn_get_seg_base(struct pt_regs * regs,int seg_reg_idx)54 static unsigned long insn_get_seg_base(struct pt_regs *regs, int seg_reg_idx)
55 {
56 return 0UL;
57 }
58
sev_es_rd_ghcb_msr(void)59 static inline u64 sev_es_rd_ghcb_msr(void)
60 {
61 struct msr m;
62
63 boot_rdmsr(MSR_AMD64_SEV_ES_GHCB, &m);
64
65 return m.q;
66 }
67
sev_es_wr_ghcb_msr(u64 val)68 static inline void sev_es_wr_ghcb_msr(u64 val)
69 {
70 struct msr m;
71
72 m.q = val;
73 boot_wrmsr(MSR_AMD64_SEV_ES_GHCB, &m);
74 }
75
vc_decode_insn(struct es_em_ctxt * ctxt)76 static enum es_result vc_decode_insn(struct es_em_ctxt *ctxt)
77 {
78 char buffer[MAX_INSN_SIZE];
79 int ret;
80
81 memcpy(buffer, (unsigned char *)ctxt->regs->ip, MAX_INSN_SIZE);
82
83 ret = insn_decode(&ctxt->insn, buffer, MAX_INSN_SIZE, INSN_MODE_64);
84 if (ret < 0)
85 return ES_DECODE_FAILED;
86
87 return ES_OK;
88 }
89
vc_write_mem(struct es_em_ctxt * ctxt,void * dst,char * buf,size_t size)90 static enum es_result vc_write_mem(struct es_em_ctxt *ctxt,
91 void *dst, char *buf, size_t size)
92 {
93 memcpy(dst, buf, size);
94
95 return ES_OK;
96 }
97
vc_read_mem(struct es_em_ctxt * ctxt,void * src,char * buf,size_t size)98 static enum es_result vc_read_mem(struct es_em_ctxt *ctxt,
99 void *src, char *buf, size_t size)
100 {
101 memcpy(buf, src, size);
102
103 return ES_OK;
104 }
105
106 #undef __init
107 #undef __pa
108 #define __init
109 #define __pa(x) ((unsigned long)(x))
110
111 #define __BOOT_COMPRESSED
112
113 /* Basic instruction decoding support needed */
114 #include "../../lib/inat.c"
115 #include "../../lib/insn.c"
116
117 /* Include code for early handlers */
118 #include "../../kernel/sev-shared.c"
119
sev_snp_enabled(void)120 static inline bool sev_snp_enabled(void)
121 {
122 return sev_status & MSR_AMD64_SEV_SNP_ENABLED;
123 }
124
__page_state_change(unsigned long paddr,enum psc_op op)125 static void __page_state_change(unsigned long paddr, enum psc_op op)
126 {
127 u64 val;
128
129 if (!sev_snp_enabled())
130 return;
131
132 /*
133 * If private -> shared then invalidate the page before requesting the
134 * state change in the RMP table.
135 */
136 if (op == SNP_PAGE_STATE_SHARED && pvalidate(paddr, RMP_PG_SIZE_4K, 0))
137 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PVALIDATE);
138
139 /* Issue VMGEXIT to change the page state in RMP table. */
140 sev_es_wr_ghcb_msr(GHCB_MSR_PSC_REQ_GFN(paddr >> PAGE_SHIFT, op));
141 VMGEXIT();
142
143 /* Read the response of the VMGEXIT. */
144 val = sev_es_rd_ghcb_msr();
145 if ((GHCB_RESP_CODE(val) != GHCB_MSR_PSC_RESP) || GHCB_MSR_PSC_RESP_VAL(val))
146 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC);
147
148 /*
149 * Now that page state is changed in the RMP table, validate it so that it is
150 * consistent with the RMP entry.
151 */
152 if (op == SNP_PAGE_STATE_PRIVATE && pvalidate(paddr, RMP_PG_SIZE_4K, 1))
153 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PVALIDATE);
154 }
155
snp_set_page_private(unsigned long paddr)156 void snp_set_page_private(unsigned long paddr)
157 {
158 __page_state_change(paddr, SNP_PAGE_STATE_PRIVATE);
159 }
160
snp_set_page_shared(unsigned long paddr)161 void snp_set_page_shared(unsigned long paddr)
162 {
163 __page_state_change(paddr, SNP_PAGE_STATE_SHARED);
164 }
165
early_setup_ghcb(void)166 static bool early_setup_ghcb(void)
167 {
168 if (set_page_decrypted((unsigned long)&boot_ghcb_page))
169 return false;
170
171 /* Page is now mapped decrypted, clear it */
172 memset(&boot_ghcb_page, 0, sizeof(boot_ghcb_page));
173
174 boot_ghcb = &boot_ghcb_page;
175
176 /* Initialize lookup tables for the instruction decoder */
177 inat_init_tables();
178
179 /* SNP guest requires the GHCB GPA must be registered */
180 if (sev_snp_enabled())
181 snp_register_ghcb_early(__pa(&boot_ghcb_page));
182
183 return true;
184 }
185
sev_es_shutdown_ghcb(void)186 void sev_es_shutdown_ghcb(void)
187 {
188 if (!boot_ghcb)
189 return;
190
191 if (!sev_es_check_cpu_features())
192 error("SEV-ES CPU Features missing.");
193
194 /*
195 * GHCB Page must be flushed from the cache and mapped encrypted again.
196 * Otherwise the running kernel will see strange cache effects when
197 * trying to use that page.
198 */
199 if (set_page_encrypted((unsigned long)&boot_ghcb_page))
200 error("Can't map GHCB page encrypted");
201
202 /*
203 * GHCB page is mapped encrypted again and flushed from the cache.
204 * Mark it non-present now to catch bugs when #VC exceptions trigger
205 * after this point.
206 */
207 if (set_page_non_present((unsigned long)&boot_ghcb_page))
208 error("Can't unmap GHCB page");
209 }
210
sev_es_check_ghcb_fault(unsigned long address)211 bool sev_es_check_ghcb_fault(unsigned long address)
212 {
213 /* Check whether the fault was on the GHCB page */
214 return ((address & PAGE_MASK) == (unsigned long)&boot_ghcb_page);
215 }
216
do_boot_stage2_vc(struct pt_regs * regs,unsigned long exit_code)217 void do_boot_stage2_vc(struct pt_regs *regs, unsigned long exit_code)
218 {
219 struct es_em_ctxt ctxt;
220 enum es_result result;
221
222 if (!boot_ghcb && !early_setup_ghcb())
223 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
224
225 vc_ghcb_invalidate(boot_ghcb);
226 result = vc_init_em_ctxt(&ctxt, regs, exit_code);
227 if (result != ES_OK)
228 goto finish;
229
230 switch (exit_code) {
231 case SVM_EXIT_RDTSC:
232 case SVM_EXIT_RDTSCP:
233 result = vc_handle_rdtsc(boot_ghcb, &ctxt, exit_code);
234 break;
235 case SVM_EXIT_IOIO:
236 result = vc_handle_ioio(boot_ghcb, &ctxt);
237 break;
238 case SVM_EXIT_CPUID:
239 result = vc_handle_cpuid(boot_ghcb, &ctxt);
240 break;
241 default:
242 result = ES_UNSUPPORTED;
243 break;
244 }
245
246 finish:
247 if (result == ES_OK)
248 vc_finish_insn(&ctxt);
249 else if (result != ES_RETRY)
250 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
251 }
252
enforce_vmpl0(void)253 static void enforce_vmpl0(void)
254 {
255 u64 attrs;
256 int err;
257
258 /*
259 * RMPADJUST modifies RMP permissions of a lesser-privileged (numerically
260 * higher) privilege level. Here, clear the VMPL1 permission mask of the
261 * GHCB page. If the guest is not running at VMPL0, this will fail.
262 *
263 * If the guest is running at VMPL0, it will succeed. Even if that operation
264 * modifies permission bits, it is still ok to do so currently because Linux
265 * SNP guests are supported only on VMPL0 so VMPL1 or higher permission masks
266 * changing is a don't-care.
267 */
268 attrs = 1;
269 if (rmpadjust((unsigned long)&boot_ghcb_page, RMP_PG_SIZE_4K, attrs))
270 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_NOT_VMPL0);
271 }
272
sev_enable(struct boot_params * bp)273 void sev_enable(struct boot_params *bp)
274 {
275 unsigned int eax, ebx, ecx, edx;
276 struct msr m;
277 bool snp;
278
279 /*
280 * bp->cc_blob_address should only be set by boot/compressed kernel.
281 * Initialize it to 0 to ensure that uninitialized values from
282 * buggy bootloaders aren't propagated.
283 */
284 if (bp)
285 bp->cc_blob_address = 0;
286
287 /*
288 * Setup/preliminary detection of SNP. This will be sanity-checked
289 * against CPUID/MSR values later.
290 */
291 snp = snp_init(bp);
292
293 /* Check for the SME/SEV support leaf */
294 eax = 0x80000000;
295 ecx = 0;
296 native_cpuid(&eax, &ebx, &ecx, &edx);
297 if (eax < 0x8000001f)
298 return;
299
300 /*
301 * Check for the SME/SEV feature:
302 * CPUID Fn8000_001F[EAX]
303 * - Bit 0 - Secure Memory Encryption support
304 * - Bit 1 - Secure Encrypted Virtualization support
305 * CPUID Fn8000_001F[EBX]
306 * - Bits 5:0 - Pagetable bit position used to indicate encryption
307 */
308 eax = 0x8000001f;
309 ecx = 0;
310 native_cpuid(&eax, &ebx, &ecx, &edx);
311 /* Check whether SEV is supported */
312 if (!(eax & BIT(1))) {
313 if (snp)
314 error("SEV-SNP support indicated by CC blob, but not CPUID.");
315 return;
316 }
317
318 /* Set the SME mask if this is an SEV guest. */
319 boot_rdmsr(MSR_AMD64_SEV, &m);
320 sev_status = m.q;
321 if (!(sev_status & MSR_AMD64_SEV_ENABLED))
322 return;
323
324 /* Negotiate the GHCB protocol version. */
325 if (sev_status & MSR_AMD64_SEV_ES_ENABLED) {
326 if (!sev_es_negotiate_protocol())
327 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_PROT_UNSUPPORTED);
328 }
329
330 /*
331 * SNP is supported in v2 of the GHCB spec which mandates support for HV
332 * features.
333 */
334 if (sev_status & MSR_AMD64_SEV_SNP_ENABLED) {
335 if (!(get_hv_features() & GHCB_HV_FT_SNP))
336 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
337
338 enforce_vmpl0();
339 }
340
341 if (snp && !(sev_status & MSR_AMD64_SEV_SNP_ENABLED))
342 error("SEV-SNP supported indicated by CC blob, but not SEV status MSR.");
343
344 sme_me_mask = BIT_ULL(ebx & 0x3f);
345 }
346
347 /* Search for Confidential Computing blob in the EFI config table. */
find_cc_blob_efi(struct boot_params * bp)348 static struct cc_blob_sev_info *find_cc_blob_efi(struct boot_params *bp)
349 {
350 unsigned long cfg_table_pa;
351 unsigned int cfg_table_len;
352 int ret;
353
354 ret = efi_get_conf_table(bp, &cfg_table_pa, &cfg_table_len);
355 if (ret)
356 return NULL;
357
358 return (struct cc_blob_sev_info *)efi_find_vendor_table(bp, cfg_table_pa,
359 cfg_table_len,
360 EFI_CC_BLOB_GUID);
361 }
362
363 /*
364 * Initial set up of SNP relies on information provided by the
365 * Confidential Computing blob, which can be passed to the boot kernel
366 * by firmware/bootloader in the following ways:
367 *
368 * - via an entry in the EFI config table
369 * - via a setup_data structure, as defined by the Linux Boot Protocol
370 *
371 * Scan for the blob in that order.
372 */
find_cc_blob(struct boot_params * bp)373 static struct cc_blob_sev_info *find_cc_blob(struct boot_params *bp)
374 {
375 struct cc_blob_sev_info *cc_info;
376
377 cc_info = find_cc_blob_efi(bp);
378 if (cc_info)
379 goto found_cc_info;
380
381 cc_info = find_cc_blob_setup_data(bp);
382 if (!cc_info)
383 return NULL;
384
385 found_cc_info:
386 if (cc_info->magic != CC_BLOB_SEV_HDR_MAGIC)
387 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
388
389 return cc_info;
390 }
391
392 /*
393 * Indicate SNP based on presence of SNP-specific CC blob. Subsequent checks
394 * will verify the SNP CPUID/MSR bits.
395 */
snp_init(struct boot_params * bp)396 bool snp_init(struct boot_params *bp)
397 {
398 struct cc_blob_sev_info *cc_info;
399
400 if (!bp)
401 return false;
402
403 cc_info = find_cc_blob(bp);
404 if (!cc_info)
405 return false;
406
407 /*
408 * If a SNP-specific Confidential Computing blob is present, then
409 * firmware/bootloader have indicated SNP support. Verifying this
410 * involves CPUID checks which will be more reliable if the SNP
411 * CPUID table is used. See comments over snp_setup_cpuid_table() for
412 * more details.
413 */
414 setup_cpuid_table(cc_info);
415
416 /*
417 * Pass run-time kernel a pointer to CC info via boot_params so EFI
418 * config table doesn't need to be searched again during early startup
419 * phase.
420 */
421 bp->cc_blob_address = (u32)(unsigned long)cc_info;
422
423 return true;
424 }
425
sev_prep_identity_maps(unsigned long top_level_pgt)426 void sev_prep_identity_maps(unsigned long top_level_pgt)
427 {
428 /*
429 * The Confidential Computing blob is used very early in uncompressed
430 * kernel to find the in-memory CPUID table to handle CPUID
431 * instructions. Make sure an identity-mapping exists so it can be
432 * accessed after switchover.
433 */
434 if (sev_snp_enabled()) {
435 unsigned long cc_info_pa = boot_params->cc_blob_address;
436 struct cc_blob_sev_info *cc_info;
437
438 kernel_add_identity_map(cc_info_pa, cc_info_pa + sizeof(*cc_info));
439
440 cc_info = (struct cc_blob_sev_info *)cc_info_pa;
441 kernel_add_identity_map(cc_info->cpuid_phys, cc_info->cpuid_phys + cc_info->cpuid_len);
442 }
443
444 sev_verify_cbit(top_level_pgt);
445 }
446
