72 	struct kvmppc_spapr_tce_table *stt;  in kvm_spapr_tce_release_iommu_group()  local
77 	list_for_each_entry_rcu(stt, &kvm->arch.spapr_tce_tables, list) {  in kvm_spapr_tce_release_iommu_group()
83 		list_for_each_entry_safe(stit, tmp, &stt->iommu_tables, next) {  in kvm_spapr_tce_release_iommu_group()
99 	struct kvmppc_spapr_tce_table *stt = NULL;  in kvm_spapr_tce_attach_iommu_group()  local
112 	list_for_each_entry_rcu(stt, &kvm->arch.spapr_tce_tables, list) {  in kvm_spapr_tce_attach_iommu_group()
113 		if (stt == f.file->private_data) {  in kvm_spapr_tce_attach_iommu_group()
135 		if ((tbltmp->it_page_shift <= stt->page_shift) &&  in kvm_spapr_tce_attach_iommu_group()
137 				 stt->offset << stt->page_shift) &&  in kvm_spapr_tce_attach_iommu_group()
139 				 stt->size << stt->page_shift)) {  in kvm_spapr_tce_attach_iommu_group()
152 	list_for_each_entry_rcu(stit, &stt->iommu_tables, next) {  in kvm_spapr_tce_attach_iommu_group()
180 	list_add_rcu(&stit->next, &stt->iommu_tables);  in kvm_spapr_tce_attach_iommu_group()
187 	struct kvmppc_spapr_tce_table *stt = container_of(head,  in release_spapr_tce_table()  local
189 	unsigned long i, npages = kvmppc_tce_pages(stt->size);  in release_spapr_tce_table()
192 		if (stt->pages[i])  in release_spapr_tce_table()
193 			__free_page(stt->pages[i]);  in release_spapr_tce_table()
195 	kfree(stt);  in release_spapr_tce_table()
198 static struct page *kvm_spapr_get_tce_page(struct kvmppc_spapr_tce_table *stt,  in kvm_spapr_get_tce_page()  argument
201 	struct page *page = stt->pages[sttpage];  in kvm_spapr_get_tce_page()
206 	mutex_lock(&stt->alloc_lock);  in kvm_spapr_get_tce_page()
207 	page = stt->pages[sttpage];  in kvm_spapr_get_tce_page()
212 			stt->pages[sttpage] = page;  in kvm_spapr_get_tce_page()
214 	mutex_unlock(&stt->alloc_lock);  in kvm_spapr_get_tce_page()
221 	struct kvmppc_spapr_tce_table *stt = vmf->vma->vm_file->private_data;  in kvm_spapr_tce_fault()  local
224 	if (vmf->pgoff >= kvmppc_tce_pages(stt->size))  in kvm_spapr_tce_fault()
227 	page = kvm_spapr_get_tce_page(stt, vmf->pgoff);  in kvm_spapr_tce_fault()
248 	struct kvmppc_spapr_tce_table *stt = filp->private_data;  in kvm_spapr_tce_release()  local
250 	struct kvm *kvm = stt->kvm;  in kvm_spapr_tce_release()
253 	list_del_rcu(&stt->list);  in kvm_spapr_tce_release()
256 	list_for_each_entry_safe(stit, tmp, &stt->iommu_tables, next) {  in kvm_spapr_tce_release()
265 		kvmppc_stt_pages(kvmppc_tce_pages(stt->size)), false);  in kvm_spapr_tce_release()
267 	kvm_put_kvm(stt->kvm);  in kvm_spapr_tce_release()
269 	call_rcu(&stt->rcu, release_spapr_tce_table);  in kvm_spapr_tce_release()
282 	struct kvmppc_spapr_tce_table *stt = NULL;  in kvm_vm_ioctl_create_spapr_tce()  local
298 	stt = kzalloc(sizeof(*stt) + npages * sizeof(struct page *),  in kvm_vm_ioctl_create_spapr_tce()
300 	if (!stt)  in kvm_vm_ioctl_create_spapr_tce()
303 	stt->liobn = args->liobn;  in kvm_vm_ioctl_create_spapr_tce()
304 	stt->page_shift = args->page_shift;  in kvm_vm_ioctl_create_spapr_tce()
305 	stt->offset = args->offset;  in kvm_vm_ioctl_create_spapr_tce()
306 	stt->size = size;  in kvm_vm_ioctl_create_spapr_tce()
307 	stt->kvm = kvm;  in kvm_vm_ioctl_create_spapr_tce()
308 	mutex_init(&stt->alloc_lock);  in kvm_vm_ioctl_create_spapr_tce()
309 	INIT_LIST_HEAD_RCU(&stt->iommu_tables);  in kvm_vm_ioctl_create_spapr_tce()
325 				       stt, O_RDWR | O_CLOEXEC);  in kvm_vm_ioctl_create_spapr_tce()
328 		list_add_rcu(&stt->list, &kvm->arch.spapr_tce_tables);  in kvm_vm_ioctl_create_spapr_tce()
337 	kfree(stt);  in kvm_vm_ioctl_create_spapr_tce()
359 static long kvmppc_tce_validate(struct kvmppc_spapr_tce_table *stt,  in kvmppc_tce_validate()  argument
371 	if (iommu_tce_check_gpa(stt->page_shift, gpa))  in kvmppc_tce_validate()
374 	if (kvmppc_tce_to_ua(stt->kvm, tce, &ua))  in kvmppc_tce_validate()
378 	list_for_each_entry_rcu(stit, &stt->iommu_tables, next) {  in kvmppc_tce_validate()
383 		mem = mm_iommu_lookup(stt->kvm->mm, ua, 1ULL << shift);  in kvmppc_tce_validate()
399 static void kvmppc_tce_put(struct kvmppc_spapr_tce_table *stt,  in kvmppc_tce_put()  argument
406 	idx -= stt->offset;  in kvmppc_tce_put()
408 	page = stt->pages[sttpage];  in kvmppc_tce_put()
415 		page = kvm_spapr_get_tce_page(stt, sttpage);  in kvmppc_tce_put()
476 		struct kvmppc_spapr_tce_table *stt, struct iommu_table *tbl,  in kvmppc_tce_iommu_unmap()  argument
480 	unsigned long subpages = 1ULL << (stt->page_shift - tbl->it_page_shift);  in kvmppc_tce_iommu_unmap()
531 		struct kvmppc_spapr_tce_table *stt, struct iommu_table *tbl,  in kvmppc_tce_iommu_map()  argument
536 	unsigned long subpages = 1ULL << (stt->page_shift - tbl->it_page_shift);  in kvmppc_tce_iommu_map()
554 	struct kvmppc_spapr_tce_table *stt;  in kvmppc_h_put_tce()  local
563 	stt = kvmppc_find_table(vcpu->kvm, liobn);  in kvmppc_h_put_tce()
564 	if (!stt)  in kvmppc_h_put_tce()
567 	ret = kvmppc_ioba_validate(stt, ioba, 1);  in kvmppc_h_put_tce()
573 	ret = kvmppc_tce_validate(stt, tce);  in kvmppc_h_put_tce()
584 	entry = ioba >> stt->page_shift;  in kvmppc_h_put_tce()
586 	list_for_each_entry_lockless(stit, &stt->iommu_tables, next) {  in kvmppc_h_put_tce()
588 			ret = kvmppc_tce_iommu_unmap(vcpu->kvm, stt,  in kvmppc_h_put_tce()
591 			ret = kvmppc_tce_iommu_map(vcpu->kvm, stt, stit->tbl,  in kvmppc_h_put_tce()
602 	kvmppc_tce_put(stt, entry, tce);  in kvmppc_h_put_tce()
615 	struct kvmppc_spapr_tce_table *stt;  in kvmppc_h_put_tce_indirect()  local
622 	stt = kvmppc_find_table(vcpu->kvm, liobn);  in kvmppc_h_put_tce_indirect()
623 	if (!stt)  in kvmppc_h_put_tce_indirect()
626 	entry = ioba >> stt->page_shift;  in kvmppc_h_put_tce_indirect()
637 	ret = kvmppc_ioba_validate(stt, ioba, npages);  in kvmppc_h_put_tce_indirect()
655 		ret = kvmppc_tce_validate(stt, tce);  in kvmppc_h_put_tce_indirect()
682 		list_for_each_entry_lockless(stit, &stt->iommu_tables, next) {  in kvmppc_h_put_tce_indirect()
683 			ret = kvmppc_tce_iommu_map(vcpu->kvm, stt,  in kvmppc_h_put_tce_indirect()
694 		kvmppc_tce_put(stt, entry + i, tce);  in kvmppc_h_put_tce_indirect()
698 	list_for_each_entry_lockless(stit, &stt->iommu_tables, next)  in kvmppc_h_put_tce_indirect()
712 	struct kvmppc_spapr_tce_table *stt;  in kvmppc_h_stuff_tce()  local
716 	stt = kvmppc_find_table(vcpu->kvm, liobn);  in kvmppc_h_stuff_tce()
717 	if (!stt)  in kvmppc_h_stuff_tce()
720 	ret = kvmppc_ioba_validate(stt, ioba, npages);  in kvmppc_h_stuff_tce()
728 	list_for_each_entry_lockless(stit, &stt->iommu_tables, next) {  in kvmppc_h_stuff_tce()
729 		unsigned long entry = ioba >> stt->page_shift;  in kvmppc_h_stuff_tce()
732 			ret = kvmppc_tce_iommu_unmap(vcpu->kvm, stt,  in kvmppc_h_stuff_tce()
746 	for (i = 0; i < npages; ++i, ioba += (1ULL << stt->page_shift))  in kvmppc_h_stuff_tce()
747 		kvmppc_tce_put(stt, ioba >> stt->page_shift, tce_value);  in kvmppc_h_stuff_tce()
750 	list_for_each_entry_lockless(stit, &stt->iommu_tables, next)  in kvmppc_h_stuff_tce()
751 		iommu_tce_kill(stit->tbl, ioba >> stt->page_shift, npages);  in kvmppc_h_stuff_tce()