25 struct vsie_page {  struct
64 static void prefix_unmapped(struct vsie_page *vsie_page)  in prefix_unmapped()  argument
66 	atomic_or(PROG_REQUEST, &vsie_page->scb_s.prog20);  in prefix_unmapped()
70 static void prefix_unmapped_sync(struct vsie_page *vsie_page)  in prefix_unmapped_sync()  argument
72 	prefix_unmapped(vsie_page);  in prefix_unmapped_sync()
73 	if (vsie_page->scb_s.prog0c & PROG_IN_SIE)  in prefix_unmapped_sync()
74 		atomic_or(CPUSTAT_STOP_INT, &vsie_page->scb_s.cpuflags);  in prefix_unmapped_sync()
75 	while (vsie_page->scb_s.prog0c & PROG_IN_SIE)  in prefix_unmapped_sync()
80 static void prefix_mapped(struct vsie_page *vsie_page)  in prefix_mapped()  argument
82 	atomic_andnot(PROG_REQUEST, &vsie_page->scb_s.prog20);  in prefix_mapped()
86 static int prefix_is_mapped(struct vsie_page *vsie_page)  in prefix_is_mapped()  argument
88 	return !(atomic_read(&vsie_page->scb_s.prog20) & PROG_REQUEST);  in prefix_is_mapped()
92 static void update_intervention_requests(struct vsie_page *vsie_page)  in update_intervention_requests()  argument
97 	cpuflags = atomic_read(&vsie_page->scb_o->cpuflags);  in update_intervention_requests()
98 	atomic_andnot(bits, &vsie_page->scb_s.cpuflags);  in update_intervention_requests()
99 	atomic_or(cpuflags & bits, &vsie_page->scb_s.cpuflags);  in update_intervention_requests()
103 static int prepare_cpuflags(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)  in prepare_cpuflags()  argument
105 	struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;  in prepare_cpuflags()
106 	struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;  in prepare_cpuflags()
294 static int shadow_crycb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)  in shadow_crycb()  argument
296 	struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;  in shadow_crycb()
297 	struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;  in shadow_crycb()
326 		ret = setup_apcb(vcpu, &vsie_page->crycb, crycb_addr,  in shadow_crycb()
343 			    vsie_page->crycb.dea_wrapping_key_mask, 56))  in shadow_crycb()
350 	b1 = (unsigned long *) vsie_page->crycb.dea_wrapping_key_mask;  in shadow_crycb()
364 	scb_s->crycbd = ((__u32)(__u64) &vsie_page->crycb) | CRYCB_FORMAT2;  in shadow_crycb()
369 static void prepare_ibc(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)  in prepare_ibc()  argument
371 	struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;  in prepare_ibc()
372 	struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;  in prepare_ibc()
392 static void unshadow_scb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)  in unshadow_scb()  argument
394 	struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;  in unshadow_scb()
395 	struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;  in unshadow_scb()
443 static int shadow_scb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)  in shadow_scb()  argument
445 	struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;  in shadow_scb()
446 	struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;  in shadow_scb()
465 	rc = prepare_cpuflags(vcpu, vsie_page);  in shadow_scb()
501 		prefix_unmapped(vsie_page);  in shadow_scb()
528 			prefix_unmapped(vsie_page);  in shadow_scb()
571 	prepare_ibc(vcpu, vsie_page);  in shadow_scb()
572 	rc = shadow_crycb(vcpu, vsie_page);  in shadow_scb()
575 		unshadow_scb(vcpu, vsie_page);  in shadow_scb()
583 	struct vsie_page *cur;  in kvm_s390_vsie_gmap_notifier()
625 static int map_prefix(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)  in map_prefix()  argument
627 	struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;  in map_prefix()
631 	if (prefix_is_mapped(vsie_page))  in map_prefix()
635 	prefix_mapped(vsie_page);  in map_prefix()
640 	rc = kvm_s390_shadow_fault(vcpu, vsie_page->gmap, prefix, NULL);  in map_prefix()
642 		rc = kvm_s390_shadow_fault(vcpu, vsie_page->gmap,  in map_prefix()
649 		prefix_unmapped(vsie_page);  in map_prefix()
682 static void unpin_blocks(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)  in unpin_blocks()  argument
684 	struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;  in unpin_blocks()
689 		unpin_guest_page(vcpu->kvm, vsie_page->sca_gpa, hpa);  in unpin_blocks()
690 		vsie_page->sca_gpa = 0;  in unpin_blocks()
697 		unpin_guest_page(vcpu->kvm, vsie_page->itdba_gpa, hpa);  in unpin_blocks()
698 		vsie_page->itdba_gpa = 0;  in unpin_blocks()
704 		unpin_guest_page(vcpu->kvm, vsie_page->gvrd_gpa, hpa);  in unpin_blocks()
705 		vsie_page->gvrd_gpa = 0;  in unpin_blocks()
711 		unpin_guest_page(vcpu->kvm, vsie_page->riccbd_gpa, hpa);  in unpin_blocks()
712 		vsie_page->riccbd_gpa = 0;  in unpin_blocks()
718 		unpin_guest_page(vcpu->kvm, vsie_page->sdnx_gpa, hpa);  in unpin_blocks()
719 		vsie_page->sdnx_gpa = 0;  in unpin_blocks()
738 static int pin_blocks(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)  in pin_blocks()  argument
740 	struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;  in pin_blocks()
741 	struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;  in pin_blocks()
764 		vsie_page->sca_gpa = gpa;  in pin_blocks()
781 		vsie_page->itdba_gpa = gpa;  in pin_blocks()
800 		vsie_page->gvrd_gpa = gpa;  in pin_blocks()
817 		vsie_page->riccbd_gpa = gpa;  in pin_blocks()
846 		vsie_page->sdnx_gpa = gpa;  in pin_blocks()
851 	unpin_blocks(vcpu, vsie_page);  in pin_blocks()
856 static void unpin_scb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page,  in unpin_scb()  argument
859 	hpa_t hpa = (hpa_t) vsie_page->scb_o;  in unpin_scb()
863 	vsie_page->scb_o = NULL;  in unpin_scb()
872 static int pin_scb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page,  in pin_scb()  argument
884 	vsie_page->scb_o = phys_to_virt(hpa);  in pin_scb()
924 static int handle_fault(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)  in handle_fault()  argument
933 	rc = kvm_s390_shadow_fault(vcpu, vsie_page->gmap,  in handle_fault()
940 			vsie_page->fault_addr = current->thread.gmap_addr;  in handle_fault()
952 			      struct vsie_page *vsie_page)  in handle_last_fault()  argument
954 	if (vsie_page->fault_addr)  in handle_last_fault()
955 		kvm_s390_shadow_fault(vcpu, vsie_page->gmap,  in handle_last_fault()
956 				      vsie_page->fault_addr, NULL);  in handle_last_fault()
957 	vsie_page->fault_addr = 0;  in handle_last_fault()
960 static inline void clear_vsie_icpt(struct vsie_page *vsie_page)  in clear_vsie_icpt()  argument
962 	vsie_page->scb_s.icptcode = 0;  in clear_vsie_icpt()
966 static void retry_vsie_icpt(struct vsie_page *vsie_page)  in retry_vsie_icpt()  argument
968 	struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;  in retry_vsie_icpt()
978 	clear_vsie_icpt(vsie_page);  in retry_vsie_icpt()
988 static int handle_stfle(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)  in handle_stfle()  argument
990 	struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;  in handle_stfle()
991 	__u32 fac = READ_ONCE(vsie_page->scb_o->fac) & 0x7ffffff8U;  in handle_stfle()
994 		retry_vsie_icpt(vsie_page);  in handle_stfle()
995 		if (read_guest_real(vcpu, fac, &vsie_page->fac,  in handle_stfle()
996 				    sizeof(vsie_page->fac)))  in handle_stfle()
998 		scb_s->fac = (__u32)(__u64) &vsie_page->fac;  in handle_stfle()
1010 static u64 vsie_get_register(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, u8 reg)  in vsie_get_register()  argument
1016 		return vsie_page->scb_s.gg15;  in vsie_get_register()
1018 		return vsie_page->scb_s.gg14;  in vsie_get_register()
1024 static int vsie_handle_mvpg(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)  in vsie_handle_mvpg()  argument
1026 	struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;  in vsie_handle_mvpg()
1028 	u64 *pei_block = &vsie_page->scb_o->mcic;  in vsie_handle_mvpg()
1037 	dest = vsie_get_register(vcpu, vsie_page, scb_s->ipb >> 20) & mask;  in vsie_handle_mvpg()
1039 	src = vsie_get_register(vcpu, vsie_page, scb_s->ipb >> 16) & mask;  in vsie_handle_mvpg()
1042 	rc_dest = kvm_s390_shadow_fault(vcpu, vsie_page->gmap, dest, &pei_dest);  in vsie_handle_mvpg()
1043 	rc_src = kvm_s390_shadow_fault(vcpu, vsie_page->gmap, src, &pei_src);  in vsie_handle_mvpg()
1049 		retry_vsie_icpt(vsie_page);  in vsie_handle_mvpg()
1060 		clear_vsie_icpt(vsie_page);  in vsie_handle_mvpg()
1083 	retry_vsie_icpt(vsie_page);  in vsie_handle_mvpg()
1103 static int do_vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)  in do_vsie_run()  argument
1107 	struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;  in do_vsie_run()
1108 	struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;  in do_vsie_run()
1112 	handle_last_fault(vcpu, vsie_page);  in do_vsie_run()
1160 		kvm_s390_reinject_machine_check(vcpu, &vsie_page->mcck_info);  in do_vsie_run()
1167 		return handle_fault(vcpu, vsie_page);  in do_vsie_run()
1172 			rc = handle_stfle(vcpu, vsie_page);  in do_vsie_run()
1177 			clear_vsie_icpt(vsie_page);  in do_vsie_run()
1185 			rc = vsie_handle_mvpg(vcpu, vsie_page);  in do_vsie_run()
1191 static void release_gmap_shadow(struct vsie_page *vsie_page)  in release_gmap_shadow()  argument
1193 	if (vsie_page->gmap)  in release_gmap_shadow()
1194 		gmap_put(vsie_page->gmap);  in release_gmap_shadow()
1195 	WRITE_ONCE(vsie_page->gmap, NULL);  in release_gmap_shadow()
1196 	prefix_unmapped(vsie_page);  in release_gmap_shadow()
1200 			       struct vsie_page *vsie_page)  in acquire_gmap_shadow()  argument
1217 	if (vsie_page->gmap && gmap_shadow_valid(vsie_page->gmap, asce, edat))  in acquire_gmap_shadow()
1221 	release_gmap_shadow(vsie_page);  in acquire_gmap_shadow()
1226 	WRITE_ONCE(vsie_page->gmap, gmap);  in acquire_gmap_shadow()
1234 				struct vsie_page *vsie_page)  in register_shadow_scb()  argument
1236 	struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;  in register_shadow_scb()
1238 	WRITE_ONCE(vcpu->arch.vsie_block, &vsie_page->scb_s);  in register_shadow_scb()
1277 static int vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)  in vsie_run()  argument
1279 	struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;  in vsie_run()
1283 		rc = acquire_gmap_shadow(vcpu, vsie_page);  in vsie_run()
1285 			rc = map_prefix(vcpu, vsie_page);  in vsie_run()
1287 			gmap_enable(vsie_page->gmap);  in vsie_run()
1288 			update_intervention_requests(vsie_page);  in vsie_run()
1289 			rc = do_vsie_run(vcpu, vsie_page);  in vsie_run()
1330 static struct vsie_page *get_vsie_page(struct kvm *kvm, unsigned long addr)  in get_vsie_page()
1332 	struct vsie_page *vsie_page;  in get_vsie_page()  local
1382 	vsie_page = page_to_virt(page);  in get_vsie_page()
1383 	memset(&vsie_page->scb_s, 0, sizeof(struct kvm_s390_sie_block));  in get_vsie_page()
1384 	release_gmap_shadow(vsie_page);  in get_vsie_page()
1385 	vsie_page->fault_addr = 0;  in get_vsie_page()
1386 	vsie_page->scb_s.ihcpu = 0xffffU;  in get_vsie_page()
1387 	return vsie_page;  in get_vsie_page()
1391 static void put_vsie_page(struct kvm *kvm, struct vsie_page *vsie_page)  in put_vsie_page()  argument
1393 	struct page *page = pfn_to_page(__pa(vsie_page) >> PAGE_SHIFT);  in put_vsie_page()
1400 	struct vsie_page *vsie_page;  in kvm_s390_handle_vsie()  local
1410 	BUILD_BUG_ON(sizeof(struct vsie_page) != PAGE_SIZE);  in kvm_s390_handle_vsie()
1421 	vsie_page = get_vsie_page(vcpu->kvm, scb_addr);  in kvm_s390_handle_vsie()
1422 	if (IS_ERR(vsie_page))  in kvm_s390_handle_vsie()
1423 		return PTR_ERR(vsie_page);  in kvm_s390_handle_vsie()
1424 	else if (!vsie_page)  in kvm_s390_handle_vsie()
1428 	rc = pin_scb(vcpu, vsie_page, scb_addr);  in kvm_s390_handle_vsie()
1431 	rc = shadow_scb(vcpu, vsie_page);  in kvm_s390_handle_vsie()
1434 	rc = pin_blocks(vcpu, vsie_page);  in kvm_s390_handle_vsie()
1437 	register_shadow_scb(vcpu, vsie_page);  in kvm_s390_handle_vsie()
1438 	rc = vsie_run(vcpu, vsie_page);  in kvm_s390_handle_vsie()
1440 	unpin_blocks(vcpu, vsie_page);  in kvm_s390_handle_vsie()
1442 	unshadow_scb(vcpu, vsie_page);  in kvm_s390_handle_vsie()
1444 	unpin_scb(vcpu, vsie_page, scb_addr);  in kvm_s390_handle_vsie()
1446 	put_vsie_page(vcpu->kvm, vsie_page);  in kvm_s390_handle_vsie()
1461 	struct vsie_page *vsie_page;  in kvm_s390_vsie_destroy()  local
1469 		vsie_page = page_to_virt(page);  in kvm_s390_vsie_destroy()
1470 		release_gmap_shadow(vsie_page);  in kvm_s390_vsie_destroy()