31 static void intel_svm_drain_prq(struct device *dev, u32 pasid);
35 static int pasid_private_add(ioasid_t pasid, void *priv)  in pasid_private_add()  argument
37 	return xa_alloc(&pasid_private_array, &pasid, priv,  in pasid_private_add()
38 			XA_LIMIT(pasid, pasid), GFP_ATOMIC);  in pasid_private_add()
41 static void pasid_private_remove(ioasid_t pasid)  in pasid_private_remove()  argument
43 	xa_erase(&pasid_private_array, pasid);  in pasid_private_remove()
46 static void *pasid_private_find(ioasid_t pasid)  in pasid_private_find()  argument
48 	return xa_load(&pasid_private_array, pasid);  in pasid_private_find()
208 	qi_flush_piotlb(sdev->iommu, sdev->did, svm->pasid, address, pages, ih);  in __flush_svm_range_dev()
211 					 svm->pasid, sdev->qdep, address,  in __flush_svm_range_dev()
273 					    svm->pasid, true);  in intel_mm_release()
285 static int pasid_to_svm_sdev(struct device *dev, unsigned int pasid,  in pasid_to_svm_sdev()  argument
296 	if (pasid == INVALID_IOASID || pasid >= PASID_MAX)  in pasid_to_svm_sdev()
299 	svm = pasid_private_find(pasid);  in pasid_to_svm_sdev()
378 				     svm->pasid);  in intel_svm_bind_gpasid()
396 		svm->pasid = data->hpasid;  in intel_svm_bind_gpasid()
463 int intel_svm_unbind_gpasid(struct device *dev, u32 pasid)  in intel_svm_unbind_gpasid()  argument
474 	ret = pasid_to_svm_sdev(dev, pasid, &svm, &sdev);  in intel_svm_unbind_gpasid()
484 						    svm->pasid, false);  in intel_svm_unbind_gpasid()
485 			intel_svm_drain_prq(dev, svm->pasid);  in intel_svm_unbind_gpasid()
498 				pasid_private_remove(pasid);  in intel_svm_unbind_gpasid()
513 static void load_pasid(struct mm_struct *mm, u32 pasid)  in load_pasid()  argument
548 	svm = pasid_private_find(mm->pasid);  in intel_svm_bind_mm()
554 		svm->pasid = mm->pasid;  in intel_svm_bind_mm()
568 		ret = pasid_private_add(svm->pasid, svm);  in intel_svm_bind_mm()
595 	sdev->pasid = svm->pasid;  in intel_svm_bind_mm()
610 	ret = intel_pasid_setup_first_level(iommu, dev, mm->pgd, mm->pasid,  in intel_svm_bind_mm()
619 		load_pasid(mm, svm->pasid);  in intel_svm_bind_mm()
631 		pasid_private_remove(mm->pasid);  in intel_svm_bind_mm()
639 static int intel_svm_unbind_mm(struct device *dev, u32 pasid)  in intel_svm_unbind_mm()  argument
651 	ret = pasid_to_svm_sdev(dev, pasid, &svm, &sdev);  in intel_svm_unbind_mm()
668 						    svm->pasid, false);  in intel_svm_unbind_mm()
669 			intel_svm_drain_prq(dev, svm->pasid);  in intel_svm_unbind_mm()
678 				pasid_private_remove(svm->pasid);  in intel_svm_unbind_mm()
703 			u64 pasid:20;  member
746 static void intel_svm_drain_prq(struct device *dev, u32 pasid)  in intel_svm_drain_prq()  argument
783 		if (!req->pasid_present || req->pasid != pasid) {  in intel_svm_drain_prq()
814 	desc[1].qw0 = QI_EIOTLB_PASID(pasid) |  in intel_svm_drain_prq()
818 	desc[2].qw0 = QI_DEV_EIOTLB_PASID(pasid) |  in intel_svm_drain_prq()
860 	event.fault.prm.pasid = desc->pasid;  in intel_svm_prq_report()
911 	desc.qw0 = QI_PGRP_PASID(req->pasid) |  in handle_bad_prq_event()
981 		if (!svm || svm->pasid != req->pasid) {  in prq_event_thread()
986 			svm = pasid_private_find(req->pasid);  in prq_event_thread()
1084 	intel_svm_unbind_mm(sdev->dev, sdev->pasid);  in intel_svm_unbind()
1091 	u32 pasid;  in intel_svm_get_pasid()  local
1095 	pasid = sdev->pasid;  in intel_svm_get_pasid()
1098 	return pasid;  in intel_svm_get_pasid()
1139 	if (prm->pasid == 0 || prm->pasid >= PASID_MAX) {  in intel_svm_page_response()
1144 	ret = pasid_to_svm_sdev(dev, prm->pasid, &svm, &sdev);  in intel_svm_page_response()
1181 		desc.qw0 = QI_PGRP_PASID(prm->pasid) | QI_PGRP_DID(sid) |  in intel_svm_page_response()