1 // SPDX-License-Identifier: GPL-2.0
7  * This file is not compiled stand-alone. It contains code shared
8  * between the pre-decompression boot code and the running Linux kernel
9  * and is included directly into both code-bases.
17 /* I/O parameters for CPUID-related helpers */
45  * Section 8.14.2.6. Also noted there is the SNP firmware-enforced limit
70  * These will be initialized based on CPUID table so that non-present
71  * all-zero leaves (for sparse tables) can be differentiated from
72  * invalid/out-of-range leaves. This is needed since all-zero leaves
73  * still need to be post-processed.
82 		error("RDRAND instruction not supported - no trusted source of randomness available\n");  in sev_es_check_cpu_features()
163 	ghcb->save.sw_exit_code = 0;  in vc_ghcb_invalidate()
164 	__builtin_memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap));  in vc_ghcb_invalidate()
181 	ctxt->regs = regs;  in vc_init_em_ctxt()
191 	ctxt->regs->ip += ctxt->insn.length;  in vc_finish_insn()
198 	ret = ghcb->save.sw_exit_info_1 & GENMASK_ULL(31, 0);  in verify_exception_info()
203 		u64 info = ghcb->save.sw_exit_info_2;  in verify_exception_info()
210 			ctxt->fi.vector = v;  in verify_exception_info()
213 				ctxt->fi.error_code = info >> 32;  in verify_exception_info()
228 	ghcb->protocol_version = ghcb_version;  in sev_es_ghcb_hv_call()
229 	ghcb->ghcb_usage       = GHCB_DEFAULT_USAGE;  in sev_es_ghcb_hv_call()
249 		return -EIO;  in __sev_cpuid_hv()
261 	 * MSR protocol does not support fetching non-zero subfunctions, but is  in sev_cpuid_hv()
262 	 * sufficient to handle current early-boot cases. Should that change,  in sev_cpuid_hv()
265 	 * can be added here to use GHCB-page protocol for cases that occur late  in sev_cpuid_hv()
268 	if (cpuid_function_is_indexed(leaf->fn) && leaf->subfn)  in sev_cpuid_hv()
269 		return -EINVAL;  in sev_cpuid_hv()
271 	ret =         __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EAX, &leaf->eax);  in sev_cpuid_hv()
272 	ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EBX, &leaf->ebx);  in sev_cpuid_hv()
273 	ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_ECX, &leaf->ecx);  in sev_cpuid_hv()
274 	ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EDX, &leaf->edx);  in sev_cpuid_hv()
281  * mapping. Use RIP-relative addressing to obtain the correct address
283  * switch-over to kernel virtual addresses later.
323 	for (i = 0; i < cpuid_table->count; i++) {  in snp_cpuid_calc_xsave_size()
324 		const struct snp_cpuid_fn *e = &cpuid_table->fn[i];  in snp_cpuid_calc_xsave_size()
326 		if (!(e->eax_in == 0xD && e->ecx_in > 1 && e->ecx_in < 64))  in snp_cpuid_calc_xsave_size()
328 		if (!(xfeatures_en & (BIT_ULL(e->ecx_in))))  in snp_cpuid_calc_xsave_size()
330 		if (xfeatures_found & (BIT_ULL(e->ecx_in)))  in snp_cpuid_calc_xsave_size()
333 		xfeatures_found |= (BIT_ULL(e->ecx_in));  in snp_cpuid_calc_xsave_size()
336 			xsave_size += e->eax;  in snp_cpuid_calc_xsave_size()
338 			xsave_size = max(xsave_size, e->eax + e->ebx);  in snp_cpuid_calc_xsave_size()
358 	for (i = 0; i < cpuid_table->count; i++) {  in snp_cpuid_get_validated_func()
359 		const struct snp_cpuid_fn *e = &cpuid_table->fn[i];  in snp_cpuid_get_validated_func()
361 		if (e->eax_in != leaf->fn)  in snp_cpuid_get_validated_func()
364 		if (cpuid_function_is_indexed(leaf->fn) && e->ecx_in != leaf->subfn)  in snp_cpuid_get_validated_func()
373 		if (e->eax_in == 0xD && (e->ecx_in == 0 || e->ecx_in == 1))  in snp_cpuid_get_validated_func()
374 			if (!(e->xcr0_in == 1 || e->xcr0_in == 3) || e->xss_in)  in snp_cpuid_get_validated_func()
377 		leaf->eax = e->eax;  in snp_cpuid_get_validated_func()
378 		leaf->ebx = e->ebx;  in snp_cpuid_get_validated_func()
379 		leaf->ecx = e->ecx;  in snp_cpuid_get_validated_func()
380 		leaf->edx = e->edx;  in snp_cpuid_get_validated_func()
398 	switch (leaf->fn) {  in snp_cpuid_postprocess()
403 		leaf->ebx = (leaf_hv.ebx & GENMASK(31, 24)) | (leaf->ebx & GENMASK(23, 0));  in snp_cpuid_postprocess()
405 		leaf->edx = (leaf_hv.edx & BIT(9)) | (leaf->edx & ~BIT(9));  in snp_cpuid_postprocess()
409 			leaf->ecx |= BIT(27);  in snp_cpuid_postprocess()
413 		leaf->ecx &= ~BIT(4);  in snp_cpuid_postprocess()
415 			leaf->ecx |= BIT(4);  in snp_cpuid_postprocess()
422 		leaf->edx = leaf_hv.edx;  in snp_cpuid_postprocess()
429 		if (leaf->subfn != 0 && leaf->subfn != 1)  in snp_cpuid_postprocess()
434 		if (leaf->subfn == 1) {  in snp_cpuid_postprocess()
436 			if (leaf->eax & BIT(3)) {  in snp_cpuid_postprocess()
448 			 * bit 3) since SNP-capable hardware has these feature  in snp_cpuid_postprocess()
453 			if (!(leaf->eax & (BIT(1) | BIT(3))))  in snp_cpuid_postprocess()
454 				return -EINVAL;  in snp_cpuid_postprocess()
461 			return -EINVAL;  in snp_cpuid_postprocess()
463 		leaf->ebx = xsave_size;  in snp_cpuid_postprocess()
470 		leaf->eax = leaf_hv.eax;  in snp_cpuid_postprocess()
472 		leaf->ebx = (leaf->ebx & GENMASK(31, 8)) | (leaf_hv.ebx & GENMASK(7, 0));  in snp_cpuid_postprocess()
474 		leaf->ecx = (leaf->ecx & GENMASK(31, 8)) | (leaf_hv.ecx & GENMASK(7, 0));  in snp_cpuid_postprocess()
477 		/* No fix-ups needed, use values as-is. */  in snp_cpuid_postprocess()
485  * Returns -EOPNOTSUPP if feature not enabled. Any other non-zero return value
492 	if (!cpuid_table->count)  in snp_cpuid()
493 		return -EOPNOTSUPP;  in snp_cpuid()
499 		 * same as out-of-range values (all-zero). This is useful here  in snp_cpuid()
504 		 * out-of-range entries and in-range zero entries, since the  in snp_cpuid()
507 		 * CPU-specific information during post-processing. So if it's  in snp_cpuid()
509 		 * within a valid CPUID range, proceed with post-processing  in snp_cpuid()
511 		 * post-processing and just return zeros immediately.  in snp_cpuid()
513 		leaf->eax = leaf->ebx = leaf->ecx = leaf->edx = 0;  in snp_cpuid()
515 		/* Skip post-processing for out-of-range zero leafs. */  in snp_cpuid()
516 		if (!(leaf->fn <= cpuid_std_range_max ||  in snp_cpuid()
517 		      (leaf->fn >= 0x40000000 && leaf->fn <= cpuid_hyp_range_max) ||  in snp_cpuid()
518 		      (leaf->fn >= 0x80000000 && leaf->fn <= cpuid_ext_range_max)))  in snp_cpuid()
526  * Boot VC Handler - This is the first VC handler during boot, there is no GHCB
528  * hypervisor and only the CPUID exit-code.
532 	unsigned int subfn = lower_bits(regs->cx, 32);  in do_vc_no_ghcb()
533 	unsigned int fn = lower_bits(regs->ax, 32);  in do_vc_no_ghcb()
548 	if (ret != -EOPNOTSUPP)  in do_vc_no_ghcb()
555 	regs->ax = leaf.eax;  in do_vc_no_ghcb()
556 	regs->bx = leaf.ebx;  in do_vc_no_ghcb()
557 	regs->cx = leaf.ecx;  in do_vc_no_ghcb()
558 	regs->dx = leaf.edx;  in do_vc_no_ghcb()
561 	 * This is a VC handler and the #VC is only raised when SEV-ES is  in do_vc_no_ghcb()
564 	 * into the no-sev path. This could map sensitive data unencrypted and  in do_vc_no_ghcb()
568 	 *	- Availability of CPUID leaf 0x8000001f  in do_vc_no_ghcb()
569 	 *	- SEV CPUID bit.  in do_vc_no_ghcb()
571 	 * The hypervisor might still report the wrong C-bit position, but this  in do_vc_no_ghcb()
575 	if (fn == 0x80000000 && (regs->ax < 0x8000001f))  in do_vc_no_ghcb()
578 	else if ((fn == 0x8000001f && !(regs->ax & BIT(1))))  in do_vc_no_ghcb()
582 	/* Skip over the CPUID two-byte opcode */  in do_vc_no_ghcb()
583 	regs->ip += 2;  in do_vc_no_ghcb()
598 	int i, b = backwards ? -1 : 1;  in vc_insn_string_read()
619 	int i, s = backwards ? -1 : 1;  in vc_insn_string_write()
655 	struct insn *insn = &ctxt->insn;  in vc_ioio_exitinfo()
658 	switch (insn->opcode.bytes[0]) {  in vc_ioio_exitinfo()
664 		*exitinfo |= (ctxt->regs->dx & 0xffff) << 16;  in vc_ioio_exitinfo()
672 		*exitinfo |= (ctxt->regs->dx & 0xffff) << 16;  in vc_ioio_exitinfo()
679 		*exitinfo |= (u8)insn->immediate.value << 16;  in vc_ioio_exitinfo()
686 		*exitinfo |= (u8)insn->immediate.value << 16;  in vc_ioio_exitinfo()
693 		*exitinfo |= (ctxt->regs->dx & 0xffff) << 16;  in vc_ioio_exitinfo()
700 		*exitinfo |= (ctxt->regs->dx & 0xffff) << 16;  in vc_ioio_exitinfo()
707 	switch (insn->opcode.bytes[0]) {  in vc_ioio_exitinfo()
719 		*exitinfo |= (insn->opnd_bytes == 2) ? IOIO_DATA_16  in vc_ioio_exitinfo()
722 	switch (insn->addr_bytes) {  in vc_ioio_exitinfo()
742 	struct pt_regs *regs = ctxt->regs;  in vc_handle_ioio()
754 		bool df = ((regs->flags & X86_EFLAGS_DF) == X86_EFLAGS_DF);  in vc_handle_ioio()
763 		 * has a chance to take interrupts and re-schedule while the  in vc_handle_ioio()
767 		ghcb_count = sizeof(ghcb->shared_buffer) / io_bytes;  in vc_handle_ioio()
769 		op_count    = (exit_info_1 & IOIO_REP) ? regs->cx : 1;  in vc_handle_ioio()
773 		es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES);  in vc_handle_ioio()
778 					       (void *)(es_base + regs->si),  in vc_handle_ioio()
779 					       ghcb->shared_buffer, io_bytes,  in vc_handle_ioio()
800 						   (void *)(es_base + regs->di),  in vc_handle_ioio()
801 						   ghcb->shared_buffer, io_bytes,  in vc_handle_ioio()
807 				regs->di -= exit_bytes;  in vc_handle_ioio()
809 				regs->di += exit_bytes;  in vc_handle_ioio()
812 				regs->si -= exit_bytes;  in vc_handle_ioio()
814 				regs->si += exit_bytes;  in vc_handle_ioio()
818 			regs->cx -= exit_info_2;  in vc_handle_ioio()
820 		ret = regs->cx ? ES_RETRY : ES_OK;  in vc_handle_ioio()
830 			rax = lower_bits(regs->ax, bits);  in vc_handle_ioio()
841 			regs->ax = lower_bits(ghcb->save.rax, bits);  in vc_handle_ioio()
853 	leaf.fn = regs->ax;  in vc_handle_cpuid_snp()
854 	leaf.subfn = regs->cx;  in vc_handle_cpuid_snp()
857 		regs->ax = leaf.eax;  in vc_handle_cpuid_snp()
858 		regs->bx = leaf.ebx;  in vc_handle_cpuid_snp()
859 		regs->cx = leaf.ecx;  in vc_handle_cpuid_snp()
860 		regs->dx = leaf.edx;  in vc_handle_cpuid_snp()
869 	struct pt_regs *regs = ctxt->regs;  in vc_handle_cpuid()
877 	if (snp_cpuid_ret != -EOPNOTSUPP)  in vc_handle_cpuid()
880 	ghcb_set_rax(ghcb, regs->ax);  in vc_handle_cpuid()
881 	ghcb_set_rcx(ghcb, regs->cx);  in vc_handle_cpuid()
887 		/* xgetbv will cause #GP - use reset value for xcr0 */  in vc_handle_cpuid()
900 	regs->ax = ghcb->save.rax;  in vc_handle_cpuid()
901 	regs->bx = ghcb->save.rbx;  in vc_handle_cpuid()
902 	regs->cx = ghcb->save.rcx;  in vc_handle_cpuid()
903 	regs->dx = ghcb->save.rdx;  in vc_handle_cpuid()
923 	ctxt->regs->ax = ghcb->save.rax;  in vc_handle_rdtsc()
924 	ctxt->regs->dx = ghcb->save.rdx;  in vc_handle_rdtsc()
926 		ctxt->regs->cx = ghcb->save.rcx;  in vc_handle_rdtsc()
945 	hdr = (struct setup_data *)bp->hdr.setup_data;  in find_cc_blob_setup_data()
948 		if (hdr->type == SETUP_CC_BLOB) {  in find_cc_blob_setup_data()
950 			return (struct cc_blob_sev_info *)(unsigned long)sd->cc_blob_address;  in find_cc_blob_setup_data()
952 		hdr = (struct setup_data *)hdr->next;  in find_cc_blob_setup_data()
972 	if (!cc_info || !cc_info->cpuid_phys || cc_info->cpuid_len < PAGE_SIZE)  in setup_cpuid_table()
975 	cpuid_table_fw = (const struct snp_cpuid_table *)cc_info->cpuid_phys;  in setup_cpuid_table()
976 	if (!cpuid_table_fw->count || cpuid_table_fw->count > SNP_CPUID_COUNT_MAX)  in setup_cpuid_table()
982 	/* Initialize CPUID ranges for range-checking. */  in setup_cpuid_table()
983 	for (i = 0; i < cpuid_table->count; i++) {  in setup_cpuid_table()
984 		const struct snp_cpuid_fn *fn = &cpuid_table->fn[i];  in setup_cpuid_table()
986 		if (fn->eax_in == 0x0)  in setup_cpuid_table()
987 			cpuid_std_range_max = fn->eax;  in setup_cpuid_table()
988 		else if (fn->eax_in == 0x40000000)  in setup_cpuid_table()
989 			cpuid_hyp_range_max = fn->eax;  in setup_cpuid_table()
990 		else if (fn->eax_in == 0x80000000)  in setup_cpuid_table()
991 			cpuid_ext_range_max = fn->eax;  in setup_cpuid_table()