27 static void kvmhv_update_ptbl_cache(struct kvm_nested_guest *gp);
516 static void kvmhv_set_nested_ptbl(struct kvm_nested_guest *gp)  in kvmhv_set_nested_ptbl()  argument
521 		__pa(gp->shadow_pgtable) | RADIX_PGD_INDEX_SIZE;  in kvmhv_set_nested_ptbl()
522 	kvmhv_set_ptbl_entry(gp->shadow_lpid, dw0, gp->process_table);  in kvmhv_set_nested_ptbl()
560 	struct kvm_nested_guest *gp;  in kvmhv_copy_tofrom_guest_nested()  local
581 	gp = kvmhv_get_nested(vcpu->kvm, l1_lpid, false);  in kvmhv_copy_tofrom_guest_nested()
582 	if (!gp) {  in kvmhv_copy_tofrom_guest_nested()
587 	mutex_lock(&gp->tlb_lock);  in kvmhv_copy_tofrom_guest_nested()
591 		rc = __kvmhv_copy_tofrom_guest_radix(gp->shadow_lpid, pid,  in kvmhv_copy_tofrom_guest_nested()
611 		rc = __kvmhv_copy_tofrom_guest_radix(gp->shadow_lpid, pid,  in kvmhv_copy_tofrom_guest_nested()
618 	mutex_unlock(&gp->tlb_lock);  in kvmhv_copy_tofrom_guest_nested()
619 	kvmhv_put_nested(gp);  in kvmhv_copy_tofrom_guest_nested()
630  * Caller must hold gp->tlb_lock.
632 static void kvmhv_update_ptbl_cache(struct kvm_nested_guest *gp)  in kvmhv_update_ptbl_cache()  argument
637 	struct kvm *kvm = gp->l1_host;  in kvmhv_update_ptbl_cache()
640 	ptbl_addr = (kvm->arch.l1_ptcr & PRTB_MASK) + (gp->l1_lpid << 4);  in kvmhv_update_ptbl_cache()
641 	if (gp->l1_lpid < (1ul << ((kvm->arch.l1_ptcr & PRTS_MASK) + 12 - 4))) {  in kvmhv_update_ptbl_cache()
648 		gp->l1_gr_to_hr = 0;  in kvmhv_update_ptbl_cache()
649 		gp->process_table = 0;  in kvmhv_update_ptbl_cache()
651 		gp->l1_gr_to_hr = be64_to_cpu(ptbl_entry.patb0);  in kvmhv_update_ptbl_cache()
652 		gp->process_table = be64_to_cpu(ptbl_entry.patb1);  in kvmhv_update_ptbl_cache()
654 	kvmhv_set_nested_ptbl(gp);  in kvmhv_update_ptbl_cache()
675 static void __add_nested(struct kvm *kvm, int lpid, struct kvm_nested_guest *gp)  in __add_nested()  argument
677 	if (idr_replace(&kvm->arch.kvm_nested_guest_idr, gp, lpid))  in __add_nested()
688 	struct kvm_nested_guest *gp;  in kvmhv_alloc_nested()  local
691 	gp = kzalloc(sizeof(*gp), GFP_KERNEL);  in kvmhv_alloc_nested()
692 	if (!gp)  in kvmhv_alloc_nested()
694 	gp->l1_host = kvm;  in kvmhv_alloc_nested()
695 	gp->l1_lpid = lpid;  in kvmhv_alloc_nested()
696 	mutex_init(&gp->tlb_lock);  in kvmhv_alloc_nested()
697 	gp->shadow_pgtable = pgd_alloc(kvm->mm);  in kvmhv_alloc_nested()
698 	if (!gp->shadow_pgtable)  in kvmhv_alloc_nested()
703 	gp->shadow_lpid = shadow_lpid;  in kvmhv_alloc_nested()
704 	gp->radix = 1;  in kvmhv_alloc_nested()
706 	memset(gp->prev_cpu, -1, sizeof(gp->prev_cpu));  in kvmhv_alloc_nested()
708 	return gp;  in kvmhv_alloc_nested()
711 	pgd_free(kvm->mm, gp->shadow_pgtable);  in kvmhv_alloc_nested()
713 	kfree(gp);  in kvmhv_alloc_nested()
720 static void kvmhv_release_nested(struct kvm_nested_guest *gp)  in kvmhv_release_nested()  argument
722 	struct kvm *kvm = gp->l1_host;  in kvmhv_release_nested()
724 	if (gp->shadow_pgtable) {  in kvmhv_release_nested()
730 		kvmppc_free_pgtable_radix(kvm, gp->shadow_pgtable,  in kvmhv_release_nested()
731 					  gp->shadow_lpid);  in kvmhv_release_nested()
732 		pgd_free(kvm->mm, gp->shadow_pgtable);  in kvmhv_release_nested()
734 	kvmhv_set_ptbl_entry(gp->shadow_lpid, 0, 0);  in kvmhv_release_nested()
735 	kvmppc_free_lpid(gp->shadow_lpid);  in kvmhv_release_nested()
736 	kfree(gp);  in kvmhv_release_nested()
739 static void kvmhv_remove_nested(struct kvm_nested_guest *gp)  in kvmhv_remove_nested()  argument
741 	struct kvm *kvm = gp->l1_host;  in kvmhv_remove_nested()
742 	int lpid = gp->l1_lpid;  in kvmhv_remove_nested()
746 	if (gp == __find_nested(kvm, lpid)) {  in kvmhv_remove_nested()
748 		--gp->refcnt;  in kvmhv_remove_nested()
750 	ref = gp->refcnt;  in kvmhv_remove_nested()
753 		kvmhv_release_nested(gp);  in kvmhv_remove_nested()
765 	struct kvm_nested_guest *gp;  in kvmhv_release_all_nested()  local
771 	idr_for_each_entry(&kvm->arch.kvm_nested_guest_idr, gp, lpid) {  in kvmhv_release_all_nested()
773 		if (--gp->refcnt == 0) {  in kvmhv_release_all_nested()
774 			gp->next = freelist;  in kvmhv_release_all_nested()
775 			freelist = gp;  in kvmhv_release_all_nested()
781 	while ((gp = freelist) != NULL) {  in kvmhv_release_all_nested()
782 		freelist = gp->next;  in kvmhv_release_all_nested()
783 		kvmhv_release_nested(gp);  in kvmhv_release_all_nested()
792 /* caller must hold gp->tlb_lock */
793 static void kvmhv_flush_nested(struct kvm_nested_guest *gp)  in kvmhv_flush_nested()  argument
795 	struct kvm *kvm = gp->l1_host;  in kvmhv_flush_nested()
798 	kvmppc_free_pgtable_radix(kvm, gp->shadow_pgtable, gp->shadow_lpid);  in kvmhv_flush_nested()
800 	kvmhv_flush_lpid(gp->shadow_lpid);  in kvmhv_flush_nested()
801 	kvmhv_update_ptbl_cache(gp);  in kvmhv_flush_nested()
802 	if (gp->l1_gr_to_hr == 0)  in kvmhv_flush_nested()
803 		kvmhv_remove_nested(gp);  in kvmhv_flush_nested()
809 	struct kvm_nested_guest *gp, *newgp;  in kvmhv_get_nested()  local
815 	gp = __find_nested(kvm, l1_lpid);  in kvmhv_get_nested()
816 	if (gp)  in kvmhv_get_nested()
817 		++gp->refcnt;  in kvmhv_get_nested()
820 	if (gp || !create)  in kvmhv_get_nested()
821 		return gp;  in kvmhv_get_nested()
833 	gp = __find_nested(kvm, l1_lpid);  in kvmhv_get_nested()
834 	if (!gp) {  in kvmhv_get_nested()
837 		gp = newgp;  in kvmhv_get_nested()
840 	++gp->refcnt;  in kvmhv_get_nested()
846 	return gp;  in kvmhv_get_nested()
849 void kvmhv_put_nested(struct kvm_nested_guest *gp)  in kvmhv_put_nested()  argument
851 	struct kvm *kvm = gp->l1_host;  in kvmhv_put_nested()
855 	ref = --gp->refcnt;  in kvmhv_put_nested()
858 		kvmhv_release_nested(gp);  in kvmhv_put_nested()
864 	struct kvm_nested_guest *gp;  in find_kvm_nested_guest_pte()  local
867 	gp = __find_nested(kvm, lpid);  in find_kvm_nested_guest_pte()
868 	if (!gp)  in find_kvm_nested_guest_pte()
873 	pte = __find_linux_pte(gp->shadow_pgtable, ea, NULL, hshift);  in find_kvm_nested_guest_pte()
966 	struct kvm_nested_guest *gp;  in kvmhv_remove_nest_rmap()  local
973 	gp = __find_nested(kvm, lpid);  in kvmhv_remove_nest_rmap()
974 	if (!gp)  in kvmhv_remove_nest_rmap()
981 		kvmppc_unmap_pte(kvm, ptep, gpa, shift, NULL, gp->shadow_lpid);  in kvmhv_remove_nest_rmap()
1036 					struct kvm_nested_guest *gp,  in kvmhv_invalidate_shadow_pte()  argument
1045 	ptep = find_kvm_nested_guest_pte(kvm, gp->l1_lpid, gpa, &shift);  in kvmhv_invalidate_shadow_pte()
1049 		kvmppc_unmap_pte(kvm, ptep, gpa, shift, NULL, gp->shadow_lpid);  in kvmhv_invalidate_shadow_pte()
1098 	struct kvm_nested_guest *gp;  in kvmhv_emulate_tlbie_tlb_addr()  local
1112 	gp = kvmhv_get_nested(kvm, lpid, false);  in kvmhv_emulate_tlbie_tlb_addr()
1113 	if (!gp) /* No such guest -> nothing to do */  in kvmhv_emulate_tlbie_tlb_addr()
1115 	mutex_lock(&gp->tlb_lock);  in kvmhv_emulate_tlbie_tlb_addr()
1119 		kvmhv_invalidate_shadow_pte(vcpu, gp, addr, &shadow_shift);  in kvmhv_emulate_tlbie_tlb_addr()
1125 	mutex_unlock(&gp->tlb_lock);  in kvmhv_emulate_tlbie_tlb_addr()
1126 	kvmhv_put_nested(gp);  in kvmhv_emulate_tlbie_tlb_addr()
1131 				     struct kvm_nested_guest *gp, int ric)  in kvmhv_emulate_tlbie_lpid()  argument
1135 	mutex_lock(&gp->tlb_lock);  in kvmhv_emulate_tlbie_lpid()
1140 		kvmppc_free_pgtable_radix(kvm, gp->shadow_pgtable,  in kvmhv_emulate_tlbie_lpid()
1141 					  gp->shadow_lpid);  in kvmhv_emulate_tlbie_lpid()
1142 		kvmhv_flush_lpid(gp->shadow_lpid);  in kvmhv_emulate_tlbie_lpid()
1153 		kvmhv_flush_nested(gp);  in kvmhv_emulate_tlbie_lpid()
1158 	mutex_unlock(&gp->tlb_lock);  in kvmhv_emulate_tlbie_lpid()
1164 	struct kvm_nested_guest *gp;  in kvmhv_emulate_tlbie_all_lpid()  local
1168 	idr_for_each_entry(&kvm->arch.kvm_nested_guest_idr, gp, lpid) {  in kvmhv_emulate_tlbie_all_lpid()
1170 		kvmhv_emulate_tlbie_lpid(vcpu, gp, ric);  in kvmhv_emulate_tlbie_all_lpid()
1180 	struct kvm_nested_guest *gp;  in kvmhv_emulate_priv_tlbie()  local
1216 		gp = kvmhv_get_nested(kvm, lpid, false);  in kvmhv_emulate_priv_tlbie()
1217 		if (gp) {  in kvmhv_emulate_priv_tlbie()
1218 			kvmhv_emulate_tlbie_lpid(vcpu, gp, ric);  in kvmhv_emulate_priv_tlbie()
1219 			kvmhv_put_nested(gp);  in kvmhv_emulate_priv_tlbie()
1254 	struct kvm_nested_guest *gp;  in do_tlb_invalidate_nested_all()  local
1256 	gp = kvmhv_get_nested(kvm, lpid, false);  in do_tlb_invalidate_nested_all()
1257 	if (gp) {  in do_tlb_invalidate_nested_all()
1258 		kvmhv_emulate_tlbie_lpid(vcpu, gp, ric);  in do_tlb_invalidate_nested_all()
1259 		kvmhv_put_nested(gp);  in do_tlb_invalidate_nested_all()
1359 				       struct kvm_nested_guest *gp,  in kvmhv_translate_addr_nested()  argument
1366 	ret = kvmppc_mmu_walk_radix_tree(vcpu, n_gpa, gpte_p, gp->l1_gr_to_hr,  in kvmhv_translate_addr_nested()
1421 				       struct kvm_nested_guest *gp,  in kvmhv_handle_nested_set_rc()  argument
1449 				      n_gpa, gp->l1_lpid);  in kvmhv_handle_nested_set_rc()
1484 /* called with gp->tlb_lock held */
1486 					  struct kvm_nested_guest *gp)  in __kvmhv_nested_page_fault()  argument
1503 	if (!gp->l1_gr_to_hr) {  in __kvmhv_nested_page_fault()
1504 		kvmhv_update_ptbl_cache(gp);  in __kvmhv_nested_page_fault()
1505 		if (!gp->l1_gr_to_hr)  in __kvmhv_nested_page_fault()
1514 	ret = kvmhv_translate_addr_nested(vcpu, gp, n_gpa, dsisr, &gpte);  in __kvmhv_nested_page_fault()
1530 		ret = kvmhv_handle_nested_set_rc(vcpu, gp, n_gpa, gpte, dsisr);  in __kvmhv_nested_page_fault()
1639 		(((unsigned long) gp->l1_lpid) << RMAP_NESTED_LPID_SHIFT);  in __kvmhv_nested_page_fault()
1641 	ret = kvmppc_create_pte(kvm, gp->shadow_pgtable, pte, n_gpa, level,  in __kvmhv_nested_page_fault()
1642 				mmu_seq, gp->shadow_lpid, rmapp, &n_rmap);  in __kvmhv_nested_page_fault()
1650 	kvmhv_invalidate_shadow_pte(vcpu, gp, n_gpa, NULL);  in __kvmhv_nested_page_fault()
1656 	struct kvm_nested_guest *gp = vcpu->arch.nested;  in kvmhv_nested_page_fault()  local
1659 	mutex_lock(&gp->tlb_lock);  in kvmhv_nested_page_fault()
1660 	ret = __kvmhv_nested_page_fault(vcpu, gp);  in kvmhv_nested_page_fault()
1661 	mutex_unlock(&gp->tlb_lock);  in kvmhv_nested_page_fault()