48 svm_lookup_device_by_dev(struct intel_svm *svm, struct device *dev)  in svm_lookup_device_by_dev()  argument
53 	list_for_each_entry_rcu(t, &svm->devs, list) {  in svm_lookup_device_by_dev()
172 static void __flush_svm_range_dev(struct intel_svm *svm,  in __flush_svm_range_dev()  argument
182 	qi_flush_piotlb(sdev->iommu, sdev->did, svm->pasid, address, pages, ih);  in __flush_svm_range_dev()
185 					 svm->pasid, sdev->qdep, address,  in __flush_svm_range_dev()
188 					  svm->pasid, sdev->qdep);  in __flush_svm_range_dev()
192 static void intel_flush_svm_range_dev(struct intel_svm *svm,  in intel_flush_svm_range_dev()  argument
203 		__flush_svm_range_dev(svm, sdev, start, align >> VTD_PAGE_SHIFT, ih);  in intel_flush_svm_range_dev()
208 static void intel_flush_svm_range(struct intel_svm *svm, unsigned long address,  in intel_flush_svm_range()  argument
214 	list_for_each_entry_rcu(sdev, &svm->devs, list)  in intel_flush_svm_range()
215 		intel_flush_svm_range_dev(svm, sdev, address, pages, ih);  in intel_flush_svm_range()
224 	struct intel_svm *svm = container_of(mn, struct intel_svm, notifier);  in intel_arch_invalidate_secondary_tlbs()  local
226 	intel_flush_svm_range(svm, start,  in intel_arch_invalidate_secondary_tlbs()
232 	struct intel_svm *svm = container_of(mn, struct intel_svm, notifier);  in intel_mm_release()  local
248 	list_for_each_entry_rcu(sdev, &svm->devs, list)  in intel_mm_release()
250 					    svm->pasid, true);  in intel_mm_release()
265 	struct intel_svm *svm;  in pasid_to_svm_sdev()  local
270 	svm = pasid_private_find(pasid);  in pasid_to_svm_sdev()
271 	if (IS_ERR(svm))  in pasid_to_svm_sdev()
272 		return PTR_ERR(svm);  in pasid_to_svm_sdev()
274 	if (!svm)  in pasid_to_svm_sdev()
281 	if (WARN_ON(list_empty(&svm->devs)))  in pasid_to_svm_sdev()
283 	sdev = svm_lookup_device_by_dev(svm, dev);  in pasid_to_svm_sdev()
286 	*rsvm = svm;  in pasid_to_svm_sdev()
297 	struct intel_svm *svm;  in intel_svm_bind_mm()  local
301 	svm = pasid_private_find(mm->pasid);  in intel_svm_bind_mm()
302 	if (!svm) {  in intel_svm_bind_mm()
303 		svm = kzalloc(sizeof(*svm), GFP_KERNEL);  in intel_svm_bind_mm()
304 		if (!svm)  in intel_svm_bind_mm()
307 		svm->pasid = mm->pasid;  in intel_svm_bind_mm()
308 		svm->mm = mm;  in intel_svm_bind_mm()
309 		INIT_LIST_HEAD_RCU(&svm->devs);  in intel_svm_bind_mm()
311 		svm->notifier.ops = &intel_mmuops;  in intel_svm_bind_mm()
312 		ret = mmu_notifier_register(&svm->notifier, mm);  in intel_svm_bind_mm()
314 			kfree(svm);  in intel_svm_bind_mm()
318 		ret = pasid_private_add(svm->pasid, svm);  in intel_svm_bind_mm()
320 			mmu_notifier_unregister(&svm->notifier, mm);  in intel_svm_bind_mm()
321 			kfree(svm);  in intel_svm_bind_mm()
350 	list_add_rcu(&sdev->list, &svm->devs);  in intel_svm_bind_mm()
357 	if (list_empty(&svm->devs)) {  in intel_svm_bind_mm()
358 		mmu_notifier_unregister(&svm->notifier, mm);  in intel_svm_bind_mm()
360 		kfree(svm);  in intel_svm_bind_mm()
370 	struct intel_svm *svm;  in intel_svm_remove_dev_pasid()  local
377 	if (pasid_to_svm_sdev(dev, pasid, &svm, &sdev))  in intel_svm_remove_dev_pasid()
379 	mm = svm->mm;  in intel_svm_remove_dev_pasid()
385 		if (list_empty(&svm->devs)) {  in intel_svm_remove_dev_pasid()
386 			if (svm->notifier.ops)  in intel_svm_remove_dev_pasid()
387 				mmu_notifier_unregister(&svm->notifier, mm);  in intel_svm_remove_dev_pasid()
388 			pasid_private_remove(svm->pasid);  in intel_svm_remove_dev_pasid()
395 			memset(svm, 0x6b, sizeof(*svm));  in intel_svm_remove_dev_pasid()
396 			kfree(svm);  in intel_svm_remove_dev_pasid()