25 static void kvmhv_update_ptbl_cache(struct kvm_nested_guest *gp);
424 static void kvmhv_set_nested_ptbl(struct kvm_nested_guest *gp)  in kvmhv_set_nested_ptbl()  argument
429 		__pa(gp->shadow_pgtable) | RADIX_PGD_INDEX_SIZE;  in kvmhv_set_nested_ptbl()
430 	kvmhv_set_ptbl_entry(gp->shadow_lpid, dw0, gp->process_table);  in kvmhv_set_nested_ptbl()
475 	struct kvm_nested_guest *gp;  in kvmhv_copy_tofrom_guest_nested()  local
496 	gp = kvmhv_get_nested(vcpu->kvm, l1_lpid, false);  in kvmhv_copy_tofrom_guest_nested()
497 	if (!gp) {  in kvmhv_copy_tofrom_guest_nested()
502 	mutex_lock(&gp->tlb_lock);  in kvmhv_copy_tofrom_guest_nested()
506 		rc = __kvmhv_copy_tofrom_guest_radix(gp->shadow_lpid, pid,  in kvmhv_copy_tofrom_guest_nested()
526 		rc = __kvmhv_copy_tofrom_guest_radix(gp->shadow_lpid, pid,  in kvmhv_copy_tofrom_guest_nested()
533 	mutex_unlock(&gp->tlb_lock);  in kvmhv_copy_tofrom_guest_nested()
534 	kvmhv_put_nested(gp);  in kvmhv_copy_tofrom_guest_nested()
545  * Caller must hold gp->tlb_lock.
547 static void kvmhv_update_ptbl_cache(struct kvm_nested_guest *gp)  in kvmhv_update_ptbl_cache()  argument
552 	struct kvm *kvm = gp->l1_host;  in kvmhv_update_ptbl_cache()
555 	ptbl_addr = (kvm->arch.l1_ptcr & PRTB_MASK) + (gp->l1_lpid << 4);  in kvmhv_update_ptbl_cache()
556 	if (gp->l1_lpid < (1ul << ((kvm->arch.l1_ptcr & PRTS_MASK) + 8))) {  in kvmhv_update_ptbl_cache()
563 		gp->l1_gr_to_hr = 0;  in kvmhv_update_ptbl_cache()
564 		gp->process_table = 0;  in kvmhv_update_ptbl_cache()
566 		gp->l1_gr_to_hr = be64_to_cpu(ptbl_entry.patb0);  in kvmhv_update_ptbl_cache()
567 		gp->process_table = be64_to_cpu(ptbl_entry.patb1);  in kvmhv_update_ptbl_cache()
569 	kvmhv_set_nested_ptbl(gp);  in kvmhv_update_ptbl_cache()
574 	struct kvm_nested_guest *gp;  in kvmhv_alloc_nested()  local
577 	gp = kzalloc(sizeof(*gp), GFP_KERNEL);  in kvmhv_alloc_nested()
578 	if (!gp)  in kvmhv_alloc_nested()
580 	gp->l1_host = kvm;  in kvmhv_alloc_nested()
581 	gp->l1_lpid = lpid;  in kvmhv_alloc_nested()
582 	mutex_init(&gp->tlb_lock);  in kvmhv_alloc_nested()
583 	gp->shadow_pgtable = pgd_alloc(kvm->mm);  in kvmhv_alloc_nested()
584 	if (!gp->shadow_pgtable)  in kvmhv_alloc_nested()
589 	gp->shadow_lpid = shadow_lpid;  in kvmhv_alloc_nested()
590 	gp->radix = 1;  in kvmhv_alloc_nested()
592 	memset(gp->prev_cpu, -1, sizeof(gp->prev_cpu));  in kvmhv_alloc_nested()
594 	return gp;  in kvmhv_alloc_nested()
597 	pgd_free(kvm->mm, gp->shadow_pgtable);  in kvmhv_alloc_nested()
599 	kfree(gp);  in kvmhv_alloc_nested()
606 static void kvmhv_release_nested(struct kvm_nested_guest *gp)  in kvmhv_release_nested()  argument
608 	struct kvm *kvm = gp->l1_host;  in kvmhv_release_nested()
610 	if (gp->shadow_pgtable) {  in kvmhv_release_nested()
616 		kvmppc_free_pgtable_radix(kvm, gp->shadow_pgtable,  in kvmhv_release_nested()
617 					  gp->shadow_lpid);  in kvmhv_release_nested()
618 		pgd_free(kvm->mm, gp->shadow_pgtable);  in kvmhv_release_nested()
620 	kvmhv_set_ptbl_entry(gp->shadow_lpid, 0, 0);  in kvmhv_release_nested()
621 	kvmppc_free_lpid(gp->shadow_lpid);  in kvmhv_release_nested()
622 	kfree(gp);  in kvmhv_release_nested()
625 static void kvmhv_remove_nested(struct kvm_nested_guest *gp)  in kvmhv_remove_nested()  argument
627 	struct kvm *kvm = gp->l1_host;  in kvmhv_remove_nested()
628 	int lpid = gp->l1_lpid;  in kvmhv_remove_nested()
632 	if (gp == kvm->arch.nested_guests[lpid]) {  in kvmhv_remove_nested()
639 		--gp->refcnt;  in kvmhv_remove_nested()
641 	ref = gp->refcnt;  in kvmhv_remove_nested()
644 		kvmhv_release_nested(gp);  in kvmhv_remove_nested()
656 	struct kvm_nested_guest *gp;  in kvmhv_release_all_nested()  local
663 		gp = kvm->arch.nested_guests[i];  in kvmhv_release_all_nested()
664 		if (!gp)  in kvmhv_release_all_nested()
667 		if (--gp->refcnt == 0) {  in kvmhv_release_all_nested()
668 			gp->next = freelist;  in kvmhv_release_all_nested()
669 			freelist = gp;  in kvmhv_release_all_nested()
674 	while ((gp = freelist) != NULL) {  in kvmhv_release_all_nested()
675 		freelist = gp->next;  in kvmhv_release_all_nested()
676 		kvmhv_release_nested(gp);  in kvmhv_release_all_nested()
685 /* caller must hold gp->tlb_lock */
686 static void kvmhv_flush_nested(struct kvm_nested_guest *gp)  in kvmhv_flush_nested()  argument
688 	struct kvm *kvm = gp->l1_host;  in kvmhv_flush_nested()
691 	kvmppc_free_pgtable_radix(kvm, gp->shadow_pgtable, gp->shadow_lpid);  in kvmhv_flush_nested()
693 	kvmhv_flush_lpid(gp->shadow_lpid);  in kvmhv_flush_nested()
694 	kvmhv_update_ptbl_cache(gp);  in kvmhv_flush_nested()
695 	if (gp->l1_gr_to_hr == 0)  in kvmhv_flush_nested()
696 		kvmhv_remove_nested(gp);  in kvmhv_flush_nested()
702 	struct kvm_nested_guest *gp, *newgp;  in kvmhv_get_nested()  local
709 	gp = kvm->arch.nested_guests[l1_lpid];  in kvmhv_get_nested()
710 	if (gp)  in kvmhv_get_nested()
711 		++gp->refcnt;  in kvmhv_get_nested()
714 	if (gp || !create)  in kvmhv_get_nested()
715 		return gp;  in kvmhv_get_nested()
723 		gp = kvm->arch.nested_guests[l1_lpid];  in kvmhv_get_nested()
727 		gp = newgp;  in kvmhv_get_nested()
732 	++gp->refcnt;  in kvmhv_get_nested()
738 	return gp;  in kvmhv_get_nested()
741 void kvmhv_put_nested(struct kvm_nested_guest *gp)  in kvmhv_put_nested()  argument
743 	struct kvm *kvm = gp->l1_host;  in kvmhv_put_nested()
747 	ref = --gp->refcnt;  in kvmhv_put_nested()
750 		kvmhv_release_nested(gp);  in kvmhv_put_nested()
763 	struct kvm_nested_guest *gp;  in find_kvm_nested_guest_pte()  local
766 	gp = kvmhv_find_nested(kvm, lpid);  in find_kvm_nested_guest_pte()
767 	if (!gp)  in find_kvm_nested_guest_pte()
772 	pte = __find_linux_pte(gp->shadow_pgtable, ea, NULL, hshift);  in find_kvm_nested_guest_pte()
865 	struct kvm_nested_guest *gp;  in kvmhv_remove_nest_rmap()  local
872 	gp = kvmhv_find_nested(kvm, lpid);  in kvmhv_remove_nest_rmap()
873 	if (!gp)  in kvmhv_remove_nest_rmap()
880 		kvmppc_unmap_pte(kvm, ptep, gpa, shift, NULL, gp->shadow_lpid);  in kvmhv_remove_nest_rmap()
935 					struct kvm_nested_guest *gp,  in kvmhv_invalidate_shadow_pte()  argument
944 	ptep = find_kvm_nested_guest_pte(kvm, gp->l1_lpid, gpa, &shift);  in kvmhv_invalidate_shadow_pte()
948 		kvmppc_unmap_pte(kvm, ptep, gpa, shift, NULL, gp->shadow_lpid);  in kvmhv_invalidate_shadow_pte()
997 	struct kvm_nested_guest *gp;  in kvmhv_emulate_tlbie_tlb_addr()  local
1011 	gp = kvmhv_get_nested(kvm, lpid, false);  in kvmhv_emulate_tlbie_tlb_addr()
1012 	if (!gp) /* No such guest -> nothing to do */  in kvmhv_emulate_tlbie_tlb_addr()
1014 	mutex_lock(&gp->tlb_lock);  in kvmhv_emulate_tlbie_tlb_addr()
1018 		kvmhv_invalidate_shadow_pte(vcpu, gp, addr, &shadow_shift);  in kvmhv_emulate_tlbie_tlb_addr()
1024 	mutex_unlock(&gp->tlb_lock);  in kvmhv_emulate_tlbie_tlb_addr()
1025 	kvmhv_put_nested(gp);  in kvmhv_emulate_tlbie_tlb_addr()
1030 				     struct kvm_nested_guest *gp, int ric)  in kvmhv_emulate_tlbie_lpid()  argument
1034 	mutex_lock(&gp->tlb_lock);  in kvmhv_emulate_tlbie_lpid()
1039 		kvmppc_free_pgtable_radix(kvm, gp->shadow_pgtable,  in kvmhv_emulate_tlbie_lpid()
1040 					  gp->shadow_lpid);  in kvmhv_emulate_tlbie_lpid()
1041 		kvmhv_flush_lpid(gp->shadow_lpid);  in kvmhv_emulate_tlbie_lpid()
1052 		kvmhv_flush_nested(gp);  in kvmhv_emulate_tlbie_lpid()
1057 	mutex_unlock(&gp->tlb_lock);  in kvmhv_emulate_tlbie_lpid()
1063 	struct kvm_nested_guest *gp;  in kvmhv_emulate_tlbie_all_lpid()  local
1068 		gp = kvm->arch.nested_guests[i];  in kvmhv_emulate_tlbie_all_lpid()
1069 		if (gp) {  in kvmhv_emulate_tlbie_all_lpid()
1071 			kvmhv_emulate_tlbie_lpid(vcpu, gp, ric);  in kvmhv_emulate_tlbie_all_lpid()
1082 	struct kvm_nested_guest *gp;  in kvmhv_emulate_priv_tlbie()  local
1118 		gp = kvmhv_get_nested(kvm, lpid, false);  in kvmhv_emulate_priv_tlbie()
1119 		if (gp) {  in kvmhv_emulate_priv_tlbie()
1120 			kvmhv_emulate_tlbie_lpid(vcpu, gp, ric);  in kvmhv_emulate_priv_tlbie()
1121 			kvmhv_put_nested(gp);  in kvmhv_emulate_priv_tlbie()
1154 				       struct kvm_nested_guest *gp,  in kvmhv_translate_addr_nested()  argument
1161 	ret = kvmppc_mmu_walk_radix_tree(vcpu, n_gpa, gpte_p, gp->l1_gr_to_hr,  in kvmhv_translate_addr_nested()
1216 				       struct kvm_nested_guest *gp,  in kvmhv_handle_nested_set_rc()  argument
1244 				      n_gpa, gp->l1_lpid);  in kvmhv_handle_nested_set_rc()
1279 /* called with gp->tlb_lock held */
1281 					  struct kvm_nested_guest *gp)  in __kvmhv_nested_page_fault()  argument
1298 	if (!gp->l1_gr_to_hr) {  in __kvmhv_nested_page_fault()
1299 		kvmhv_update_ptbl_cache(gp);  in __kvmhv_nested_page_fault()
1300 		if (!gp->l1_gr_to_hr)  in __kvmhv_nested_page_fault()
1309 	ret = kvmhv_translate_addr_nested(vcpu, gp, n_gpa, dsisr, &gpte);  in __kvmhv_nested_page_fault()
1325 		ret = kvmhv_handle_nested_set_rc(vcpu, gp, n_gpa, gpte, dsisr);  in __kvmhv_nested_page_fault()
1434 		(((unsigned long) gp->l1_lpid) << RMAP_NESTED_LPID_SHIFT);  in __kvmhv_nested_page_fault()
1436 	ret = kvmppc_create_pte(kvm, gp->shadow_pgtable, pte, n_gpa, level,  in __kvmhv_nested_page_fault()
1437 				mmu_seq, gp->shadow_lpid, rmapp, &n_rmap);  in __kvmhv_nested_page_fault()
1445 	kvmhv_invalidate_shadow_pte(vcpu, gp, n_gpa, NULL);  in __kvmhv_nested_page_fault()
1451 	struct kvm_nested_guest *gp = vcpu->arch.nested;  in kvmhv_nested_page_fault()  local
1454 	mutex_lock(&gp->tlb_lock);  in kvmhv_nested_page_fault()
1455 	ret = __kvmhv_nested_page_fault(vcpu, gp);  in kvmhv_nested_page_fault()
1456 	mutex_unlock(&gp->tlb_lock);  in kvmhv_nested_page_fault()