102 static void intel_flush_svm_range_dev (struct intel_svm *svm, struct intel_svm_dev *sdev,  in intel_flush_svm_range_dev()  argument
111 	if (pages == -1 || !cap_pgsel_inv(svm->iommu->cap)) {  in intel_flush_svm_range_dev()
112 		desc.qw0 = QI_EIOTLB_PASID(svm->pasid) |  in intel_flush_svm_range_dev()
120 		desc.qw0 = QI_EIOTLB_PASID(svm->pasid) |  in intel_flush_svm_range_dev()
130 	qi_submit_sync(&desc, svm->iommu);  in intel_flush_svm_range_dev()
133 		desc.qw0 = QI_DEV_EIOTLB_PASID(svm->pasid) |  in intel_flush_svm_range_dev()
154 		qi_submit_sync(&desc, svm->iommu);  in intel_flush_svm_range_dev()
158 static void intel_flush_svm_range(struct intel_svm *svm, unsigned long address,  in intel_flush_svm_range()  argument
164 	list_for_each_entry_rcu(sdev, &svm->devs, list)  in intel_flush_svm_range()
165 		intel_flush_svm_range_dev(svm, sdev, address, pages, ih);  in intel_flush_svm_range()
174 	struct intel_svm *svm = container_of(mn, struct intel_svm, notifier);  in intel_invalidate_range()  local
176 	intel_flush_svm_range(svm, start,  in intel_invalidate_range()
182 	struct intel_svm *svm = container_of(mn, struct intel_svm, notifier);  in intel_mm_release()  local
198 	list_for_each_entry_rcu(sdev, &svm->devs, list) {  in intel_mm_release()
199 		intel_pasid_tear_down_entry(svm->iommu, sdev->dev, svm->pasid);  in intel_mm_release()
200 		intel_flush_svm_range_dev(svm, sdev, 0, -1, 0);  in intel_mm_release()
219 	struct intel_svm *svm = NULL;  in intel_svm_bind_mm()  local
250 			svm = t;  in intel_svm_bind_mm()
251 			if (svm->pasid >= pasid_max) {  in intel_svm_bind_mm()
254 					 svm->pasid);  in intel_svm_bind_mm()
259 			list_for_each_entry(sdev, &svm->devs, list) {  in intel_svm_bind_mm()
309 	if (!svm) {  in intel_svm_bind_mm()
310 		svm = kzalloc(sizeof(*svm), GFP_KERNEL);  in intel_svm_bind_mm()
311 		if (!svm) {  in intel_svm_bind_mm()
316 		svm->iommu = iommu;  in intel_svm_bind_mm()
322 		ret = intel_pasid_alloc_id(svm,  in intel_svm_bind_mm()
326 			kfree(svm);  in intel_svm_bind_mm()
330 		svm->pasid = ret;  in intel_svm_bind_mm()
331 		svm->notifier.ops = &intel_mmuops;  in intel_svm_bind_mm()
332 		svm->mm = mm;  in intel_svm_bind_mm()
333 		svm->flags = flags;  in intel_svm_bind_mm()
334 		INIT_LIST_HEAD_RCU(&svm->devs);  in intel_svm_bind_mm()
335 		INIT_LIST_HEAD(&svm->list);  in intel_svm_bind_mm()
338 			ret = mmu_notifier_register(&svm->notifier, mm);  in intel_svm_bind_mm()
340 				intel_pasid_free_id(svm->pasid);  in intel_svm_bind_mm()
341 				kfree(svm);  in intel_svm_bind_mm()
350 				svm->pasid, FLPT_DEFAULT_DID,  in intel_svm_bind_mm()
355 				mmu_notifier_unregister(&svm->notifier, mm);  in intel_svm_bind_mm()
356 			intel_pasid_free_id(svm->pasid);  in intel_svm_bind_mm()
357 			kfree(svm);  in intel_svm_bind_mm()
362 		list_add_tail(&svm->list, &global_svm_list);  in intel_svm_bind_mm()
371 						svm->pasid, FLPT_DEFAULT_DID,  in intel_svm_bind_mm()
379 	list_add_rcu(&sdev->list, &svm->devs);  in intel_svm_bind_mm()
382 	*pasid = svm->pasid;  in intel_svm_bind_mm()
396 	struct intel_svm *svm;  in intel_svm_unbind_mm()  local
404 	svm = intel_pasid_lookup_id(pasid);  in intel_svm_unbind_mm()
405 	if (!svm)  in intel_svm_unbind_mm()
408 	list_for_each_entry(sdev, &svm->devs, list) {  in intel_svm_unbind_mm()
421 				intel_pasid_tear_down_entry(iommu, dev, svm->pasid);  in intel_svm_unbind_mm()
422 				intel_flush_svm_range_dev(svm, sdev, 0, -1, 0);  in intel_svm_unbind_mm()
425 				if (list_empty(&svm->devs)) {  in intel_svm_unbind_mm()
426 					intel_pasid_free_id(svm->pasid);  in intel_svm_unbind_mm()
427 					if (svm->mm)  in intel_svm_unbind_mm()
428 						mmu_notifier_unregister(&svm->notifier, svm->mm);  in intel_svm_unbind_mm()
430 					list_del(&svm->list);  in intel_svm_unbind_mm()
436 					memset(svm, 0x6b, sizeof(*svm));  in intel_svm_unbind_mm()
437 					kfree(svm);  in intel_svm_unbind_mm()
453 	struct intel_svm *svm;  in intel_svm_is_pasid_valid()  local
461 	svm = intel_pasid_lookup_id(pasid);  in intel_svm_is_pasid_valid()
462 	if (!svm)  in intel_svm_is_pasid_valid()
466 	if (!svm->mm)  in intel_svm_is_pasid_valid()
468 	else if (atomic_read(&svm->mm->mm_users) > 0)  in intel_svm_is_pasid_valid()
538 	struct intel_svm *svm = NULL;  in prq_event_thread()  local
569 		if (!svm || svm->pasid != req->pasid) {  in prq_event_thread()
571 			svm = intel_pasid_lookup_id(req->pasid);  in prq_event_thread()
577 			if (!svm) {  in prq_event_thread()
588 		if (!svm->mm)  in prq_event_thread()
591 		if (!mmget_not_zero(svm->mm))  in prq_event_thread()
598 		down_read(&svm->mm->mmap_sem);  in prq_event_thread()
599 		vma = find_extend_vma(svm->mm, address);  in prq_event_thread()
613 		up_read(&svm->mm->mmap_sem);  in prq_event_thread()
614 		mmput(svm->mm);  in prq_event_thread()
618 		list_for_each_entry_rcu(sdev, &svm->devs, list) {  in prq_event_thread()
627 		if (WARN_ON(&sdev->list == &svm->devs))  in prq_event_thread()
639 		svm = NULL;  in prq_event_thread()