156 static void intel_flush_svm_range_dev (struct intel_svm *svm, struct intel_svm_dev *sdev,  in intel_flush_svm_range_dev()  argument
166 			desc.low = QI_EIOTLB_PASID(svm->pasid) | QI_EIOTLB_DID(sdev->did) |  in intel_flush_svm_range_dev()
169 			desc.low = QI_EIOTLB_PASID(svm->pasid) | QI_EIOTLB_DID(sdev->did) |  in intel_flush_svm_range_dev()
175 		desc.low = QI_EIOTLB_PASID(svm->pasid) | QI_EIOTLB_DID(sdev->did) |  in intel_flush_svm_range_dev()
180 	qi_submit_sync(&desc, svm->iommu);  in intel_flush_svm_range_dev()
183 		desc.low = QI_DEV_EIOTLB_PASID(svm->pasid) | QI_DEV_EIOTLB_SID(sdev->sid) |  in intel_flush_svm_range_dev()
198 		qi_submit_sync(&desc, svm->iommu);  in intel_flush_svm_range_dev()
202 static void intel_flush_svm_range(struct intel_svm *svm, unsigned long address,  in intel_flush_svm_range()  argument
208 	if (svm->iommu->pasid_state_table &&  in intel_flush_svm_range()
209 	    !cmpxchg64(&svm->iommu->pasid_state_table[svm->pasid].val, 0, 1ULL << 63))  in intel_flush_svm_range()
213 	list_for_each_entry_rcu(sdev, &svm->devs, list)  in intel_flush_svm_range()
214 		intel_flush_svm_range_dev(svm, sdev, address, pages, ih, gl);  in intel_flush_svm_range()
221 	struct intel_svm *svm = container_of(mn, struct intel_svm, notifier);  in intel_change_pte()  local
223 	intel_flush_svm_range(svm, address, 1, 1, 0);  in intel_change_pte()
231 	struct intel_svm *svm = container_of(mn, struct intel_svm, notifier);  in intel_invalidate_range()  local
233 	intel_flush_svm_range(svm, start,  in intel_invalidate_range()
238 static void intel_flush_pasid_dev(struct intel_svm *svm, struct intel_svm_dev *sdev, int pasid)  in intel_flush_pasid_dev()  argument
245 	qi_submit_sync(&desc, svm->iommu);  in intel_flush_pasid_dev()
250 	struct intel_svm *svm = container_of(mn, struct intel_svm, notifier);  in intel_mm_release()  local
266 	list_for_each_entry_rcu(sdev, &svm->devs, list) {  in intel_mm_release()
267 		intel_pasid_clear_entry(sdev->dev, svm->pasid);  in intel_mm_release()
268 		intel_flush_pasid_dev(svm, sdev, svm->pasid);  in intel_mm_release()
269 		intel_flush_svm_range_dev(svm, sdev, 0, -1, 0, !svm->mm);  in intel_mm_release()
290 	struct intel_svm *svm = NULL;  in intel_svm_bind_mm()  local
322 			svm = t;  in intel_svm_bind_mm()
323 			if (svm->pasid >= pasid_max) {  in intel_svm_bind_mm()
326 					 svm->pasid);  in intel_svm_bind_mm()
331 			list_for_each_entry(sdev, &svm->devs, list) {  in intel_svm_bind_mm()
365 	if (!svm) {  in intel_svm_bind_mm()
366 		svm = kzalloc(sizeof(*svm), GFP_KERNEL);  in intel_svm_bind_mm()
367 		if (!svm) {  in intel_svm_bind_mm()
372 		svm->iommu = iommu;  in intel_svm_bind_mm()
378 		ret = intel_pasid_alloc_id(svm,  in intel_svm_bind_mm()
382 			kfree(svm);  in intel_svm_bind_mm()
386 		svm->pasid = ret;  in intel_svm_bind_mm()
387 		svm->notifier.ops = &intel_mmuops;  in intel_svm_bind_mm()
388 		svm->mm = mm;  in intel_svm_bind_mm()
389 		svm->flags = flags;  in intel_svm_bind_mm()
390 		INIT_LIST_HEAD_RCU(&svm->devs);  in intel_svm_bind_mm()
391 		INIT_LIST_HEAD(&svm->list);  in intel_svm_bind_mm()
394 			ret = mmu_notifier_register(&svm->notifier, mm);  in intel_svm_bind_mm()
396 				intel_pasid_free_id(svm->pasid);  in intel_svm_bind_mm()
397 				kfree(svm);  in intel_svm_bind_mm()
408 		entry = intel_pasid_get_entry(dev, svm->pasid);  in intel_svm_bind_mm()
418 			intel_flush_pasid_dev(svm, sdev, svm->pasid);  in intel_svm_bind_mm()
420 		list_add_tail(&svm->list, &global_svm_list);  in intel_svm_bind_mm()
422 	list_add_rcu(&sdev->list, &svm->devs);  in intel_svm_bind_mm()
425 	*pasid = svm->pasid;  in intel_svm_bind_mm()
439 	struct intel_svm *svm;  in intel_svm_unbind_mm()  local
447 	svm = intel_pasid_lookup_id(pasid);  in intel_svm_unbind_mm()
448 	if (!svm)  in intel_svm_unbind_mm()
451 	list_for_each_entry(sdev, &svm->devs, list) {  in intel_svm_unbind_mm()
464 				intel_flush_pasid_dev(svm, sdev, svm->pasid);  in intel_svm_unbind_mm()
465 				intel_flush_svm_range_dev(svm, sdev, 0, -1, 0, !svm->mm);  in intel_svm_unbind_mm()
467 				intel_pasid_clear_entry(dev, svm->pasid);  in intel_svm_unbind_mm()
469 				if (list_empty(&svm->devs)) {  in intel_svm_unbind_mm()
470 					intel_pasid_free_id(svm->pasid);  in intel_svm_unbind_mm()
471 					if (svm->mm)  in intel_svm_unbind_mm()
472 						mmu_notifier_unregister(&svm->notifier, svm->mm);  in intel_svm_unbind_mm()
474 					list_del(&svm->list);  in intel_svm_unbind_mm()
480 					memset(svm, 0x6b, sizeof(*svm));  in intel_svm_unbind_mm()
481 					kfree(svm);  in intel_svm_unbind_mm()
497 	struct intel_svm *svm;  in intel_svm_is_pasid_valid()  local
505 	svm = intel_pasid_lookup_id(pasid);  in intel_svm_is_pasid_valid()
506 	if (!svm)  in intel_svm_is_pasid_valid()
510 	if (!svm->mm)  in intel_svm_is_pasid_valid()
512 	else if (atomic_read(&svm->mm->mm_users) > 0)  in intel_svm_is_pasid_valid()
571 	struct intel_svm *svm = NULL;  in prq_event_thread()  local
602 		if (!svm || svm->pasid != req->pasid) {  in prq_event_thread()
604 			svm = intel_pasid_lookup_id(req->pasid);  in prq_event_thread()
610 			if (!svm) {  in prq_event_thread()
621 		if (!svm->mm)  in prq_event_thread()
624 		if (!mmget_not_zero(svm->mm))  in prq_event_thread()
631 		down_read(&svm->mm->mmap_sem);  in prq_event_thread()
632 		vma = find_extend_vma(svm->mm, address);  in prq_event_thread()
646 		up_read(&svm->mm->mmap_sem);  in prq_event_thread()
647 		mmput(svm->mm);  in prq_event_thread()
651 		list_for_each_entry_rcu(sdev, &svm->devs, list) {  in prq_event_thread()
660 		if (WARN_ON(&sdev->list == &svm->devs))  in prq_event_thread()
671 		svm = NULL;  in prq_event_thread()