24 struct vsie_page {  struct
63 static void prefix_unmapped(struct vsie_page *vsie_page)  in prefix_unmapped()  argument
65 	atomic_or(PROG_REQUEST, &vsie_page->scb_s.prog20);  in prefix_unmapped()
69 static void prefix_unmapped_sync(struct vsie_page *vsie_page)  in prefix_unmapped_sync()  argument
71 	prefix_unmapped(vsie_page);  in prefix_unmapped_sync()
72 	if (vsie_page->scb_s.prog0c & PROG_IN_SIE)  in prefix_unmapped_sync()
73 		atomic_or(CPUSTAT_STOP_INT, &vsie_page->scb_s.cpuflags);  in prefix_unmapped_sync()
74 	while (vsie_page->scb_s.prog0c & PROG_IN_SIE)  in prefix_unmapped_sync()
79 static void prefix_mapped(struct vsie_page *vsie_page)  in prefix_mapped()  argument
81 	atomic_andnot(PROG_REQUEST, &vsie_page->scb_s.prog20);  in prefix_mapped()
85 static int prefix_is_mapped(struct vsie_page *vsie_page)  in prefix_is_mapped()  argument
87 	return !(atomic_read(&vsie_page->scb_s.prog20) & PROG_REQUEST);  in prefix_is_mapped()
91 static void update_intervention_requests(struct vsie_page *vsie_page)  in update_intervention_requests()  argument
96 	cpuflags = atomic_read(&vsie_page->scb_o->cpuflags);  in update_intervention_requests()
97 	atomic_andnot(bits, &vsie_page->scb_s.cpuflags);  in update_intervention_requests()
98 	atomic_or(cpuflags & bits, &vsie_page->scb_s.cpuflags);  in update_intervention_requests()
102 static int prepare_cpuflags(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)  in prepare_cpuflags()  argument
104 	struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;  in prepare_cpuflags()
105 	struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;  in prepare_cpuflags()
283 static int shadow_crycb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)  in shadow_crycb()  argument
285 	struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;  in shadow_crycb()
286 	struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;  in shadow_crycb()
315 		ret = setup_apcb(vcpu, &vsie_page->crycb, crycb_addr,  in shadow_crycb()
332 			    vsie_page->crycb.dea_wrapping_key_mask, 56))  in shadow_crycb()
339 	b1 = (unsigned long *) vsie_page->crycb.dea_wrapping_key_mask;  in shadow_crycb()
353 	scb_s->crycbd = ((__u32)(__u64) &vsie_page->crycb) | CRYCB_FORMAT2;  in shadow_crycb()
358 static void prepare_ibc(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)  in prepare_ibc()  argument
360 	struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;  in prepare_ibc()
361 	struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;  in prepare_ibc()
381 static void unshadow_scb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)  in unshadow_scb()  argument
383 	struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;  in unshadow_scb()
384 	struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;  in unshadow_scb()
437 static int shadow_scb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)  in shadow_scb()  argument
439 	struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;  in shadow_scb()
440 	struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;  in shadow_scb()
459 	rc = prepare_cpuflags(vcpu, vsie_page);  in shadow_scb()
495 		prefix_unmapped(vsie_page);  in shadow_scb()
514 			prefix_unmapped(vsie_page);  in shadow_scb()
553 	prepare_ibc(vcpu, vsie_page);  in shadow_scb()
554 	rc = shadow_crycb(vcpu, vsie_page);  in shadow_scb()
557 		unshadow_scb(vcpu, vsie_page);  in shadow_scb()
565 	struct vsie_page *cur;  in kvm_s390_vsie_gmap_notifier()
607 static int map_prefix(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)  in map_prefix()  argument
609 	struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;  in map_prefix()
613 	if (prefix_is_mapped(vsie_page))  in map_prefix()
617 	prefix_mapped(vsie_page);  in map_prefix()
622 	rc = kvm_s390_shadow_fault(vcpu, vsie_page->gmap, prefix);  in map_prefix()
624 		rc = kvm_s390_shadow_fault(vcpu, vsie_page->gmap,  in map_prefix()
631 		prefix_unmapped(vsie_page);  in map_prefix()
664 static void unpin_blocks(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)  in unpin_blocks()  argument
666 	struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;  in unpin_blocks()
671 		unpin_guest_page(vcpu->kvm, vsie_page->sca_gpa, hpa);  in unpin_blocks()
672 		vsie_page->sca_gpa = 0;  in unpin_blocks()
679 		unpin_guest_page(vcpu->kvm, vsie_page->itdba_gpa, hpa);  in unpin_blocks()
680 		vsie_page->itdba_gpa = 0;  in unpin_blocks()
686 		unpin_guest_page(vcpu->kvm, vsie_page->gvrd_gpa, hpa);  in unpin_blocks()
687 		vsie_page->gvrd_gpa = 0;  in unpin_blocks()
693 		unpin_guest_page(vcpu->kvm, vsie_page->riccbd_gpa, hpa);  in unpin_blocks()
694 		vsie_page->riccbd_gpa = 0;  in unpin_blocks()
700 		unpin_guest_page(vcpu->kvm, vsie_page->sdnx_gpa, hpa);  in unpin_blocks()
701 		vsie_page->sdnx_gpa = 0;  in unpin_blocks()
720 static int pin_blocks(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)  in pin_blocks()  argument
722 	struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;  in pin_blocks()
723 	struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;  in pin_blocks()
746 		vsie_page->sca_gpa = gpa;  in pin_blocks()
763 		vsie_page->itdba_gpa = gpa;  in pin_blocks()
782 		vsie_page->gvrd_gpa = gpa;  in pin_blocks()
799 		vsie_page->riccbd_gpa = gpa;  in pin_blocks()
828 		vsie_page->sdnx_gpa = gpa;  in pin_blocks()
833 	unpin_blocks(vcpu, vsie_page);  in pin_blocks()
838 static void unpin_scb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page,  in unpin_scb()  argument
841 	hpa_t hpa = (hpa_t) vsie_page->scb_o;  in unpin_scb()
845 	vsie_page->scb_o = NULL;  in unpin_scb()
854 static int pin_scb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page,  in pin_scb()  argument
866 	vsie_page->scb_o = (struct kvm_s390_sie_block *) hpa;  in pin_scb()
906 static int handle_fault(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)  in handle_fault()  argument
915 	rc = kvm_s390_shadow_fault(vcpu, vsie_page->gmap,  in handle_fault()
922 			vsie_page->fault_addr = current->thread.gmap_addr;  in handle_fault()
934 			      struct vsie_page *vsie_page)  in handle_last_fault()  argument
936 	if (vsie_page->fault_addr)  in handle_last_fault()
937 		kvm_s390_shadow_fault(vcpu, vsie_page->gmap,  in handle_last_fault()
938 				      vsie_page->fault_addr);  in handle_last_fault()
939 	vsie_page->fault_addr = 0;  in handle_last_fault()
942 static inline void clear_vsie_icpt(struct vsie_page *vsie_page)  in clear_vsie_icpt()  argument
944 	vsie_page->scb_s.icptcode = 0;  in clear_vsie_icpt()
948 static void retry_vsie_icpt(struct vsie_page *vsie_page)  in retry_vsie_icpt()  argument
950 	struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;  in retry_vsie_icpt()
960 	clear_vsie_icpt(vsie_page);  in retry_vsie_icpt()
970 static int handle_stfle(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)  in handle_stfle()  argument
972 	struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;  in handle_stfle()
973 	__u32 fac = READ_ONCE(vsie_page->scb_o->fac) & 0x7ffffff8U;  in handle_stfle()
976 		retry_vsie_icpt(vsie_page);  in handle_stfle()
977 		if (read_guest_real(vcpu, fac, &vsie_page->fac,  in handle_stfle()
978 				    sizeof(vsie_page->fac)))  in handle_stfle()
980 		scb_s->fac = (__u32)(__u64) &vsie_page->fac;  in handle_stfle()
993 static int do_vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)  in do_vsie_run()  argument
997 	struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;  in do_vsie_run()
998 	struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;  in do_vsie_run()
1002 	handle_last_fault(vcpu, vsie_page);  in do_vsie_run()
1048 		kvm_s390_reinject_machine_check(vcpu, &vsie_page->mcck_info);  in do_vsie_run()
1055 		return handle_fault(vcpu, vsie_page);  in do_vsie_run()
1060 			rc = handle_stfle(vcpu, vsie_page);  in do_vsie_run()
1065 			clear_vsie_icpt(vsie_page);  in do_vsie_run()
1075 static void release_gmap_shadow(struct vsie_page *vsie_page)  in release_gmap_shadow()  argument
1077 	if (vsie_page->gmap)  in release_gmap_shadow()
1078 		gmap_put(vsie_page->gmap);  in release_gmap_shadow()
1079 	WRITE_ONCE(vsie_page->gmap, NULL);  in release_gmap_shadow()
1080 	prefix_unmapped(vsie_page);  in release_gmap_shadow()
1084 			       struct vsie_page *vsie_page)  in acquire_gmap_shadow()  argument
1101 	if (vsie_page->gmap && gmap_shadow_valid(vsie_page->gmap, asce, edat))  in acquire_gmap_shadow()
1105 	release_gmap_shadow(vsie_page);  in acquire_gmap_shadow()
1110 	WRITE_ONCE(vsie_page->gmap, gmap);  in acquire_gmap_shadow()
1118 				struct vsie_page *vsie_page)  in register_shadow_scb()  argument
1120 	struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;  in register_shadow_scb()
1122 	WRITE_ONCE(vcpu->arch.vsie_block, &vsie_page->scb_s);  in register_shadow_scb()
1161 static int vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)  in vsie_run()  argument
1163 	struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;  in vsie_run()
1167 		rc = acquire_gmap_shadow(vcpu, vsie_page);  in vsie_run()
1169 			rc = map_prefix(vcpu, vsie_page);  in vsie_run()
1171 			gmap_enable(vsie_page->gmap);  in vsie_run()
1172 			update_intervention_requests(vsie_page);  in vsie_run()
1173 			rc = do_vsie_run(vcpu, vsie_page);  in vsie_run()
1214 static struct vsie_page *get_vsie_page(struct kvm *kvm, unsigned long addr)  in get_vsie_page()
1216 	struct vsie_page *vsie_page;  in get_vsie_page()  local
1266 	vsie_page = page_to_virt(page);  in get_vsie_page()
1267 	memset(&vsie_page->scb_s, 0, sizeof(struct kvm_s390_sie_block));  in get_vsie_page()
1268 	release_gmap_shadow(vsie_page);  in get_vsie_page()
1269 	vsie_page->fault_addr = 0;  in get_vsie_page()
1270 	vsie_page->scb_s.ihcpu = 0xffffU;  in get_vsie_page()
1271 	return vsie_page;  in get_vsie_page()
1275 static void put_vsie_page(struct kvm *kvm, struct vsie_page *vsie_page)  in put_vsie_page()  argument
1277 	struct page *page = pfn_to_page(__pa(vsie_page) >> PAGE_SHIFT);  in put_vsie_page()
1284 	struct vsie_page *vsie_page;  in kvm_s390_handle_vsie()  local
1294 	BUILD_BUG_ON(sizeof(struct vsie_page) != PAGE_SIZE);  in kvm_s390_handle_vsie()
1305 	vsie_page = get_vsie_page(vcpu->kvm, scb_addr);  in kvm_s390_handle_vsie()
1306 	if (IS_ERR(vsie_page))  in kvm_s390_handle_vsie()
1307 		return PTR_ERR(vsie_page);  in kvm_s390_handle_vsie()
1308 	else if (!vsie_page)  in kvm_s390_handle_vsie()
1312 	rc = pin_scb(vcpu, vsie_page, scb_addr);  in kvm_s390_handle_vsie()
1315 	rc = shadow_scb(vcpu, vsie_page);  in kvm_s390_handle_vsie()
1318 	rc = pin_blocks(vcpu, vsie_page);  in kvm_s390_handle_vsie()
1321 	register_shadow_scb(vcpu, vsie_page);  in kvm_s390_handle_vsie()
1322 	rc = vsie_run(vcpu, vsie_page);  in kvm_s390_handle_vsie()
1324 	unpin_blocks(vcpu, vsie_page);  in kvm_s390_handle_vsie()
1326 	unshadow_scb(vcpu, vsie_page);  in kvm_s390_handle_vsie()
1328 	unpin_scb(vcpu, vsie_page, scb_addr);  in kvm_s390_handle_vsie()
1330 	put_vsie_page(vcpu->kvm, vsie_page);  in kvm_s390_handle_vsie()
1345 	struct vsie_page *vsie_page;  in kvm_s390_vsie_destroy()  local
1353 		vsie_page = page_to_virt(page);  in kvm_s390_vsie_destroy()
1354 		release_gmap_shadow(vsie_page);  in kvm_s390_vsie_destroy()