156 static void intel_flush_svm_range_dev (struct intel_svm *svm, struct intel_svm_dev *sdev,  in intel_flush_svm_range_dev()  argument
166 			desc.low = QI_EIOTLB_PASID(svm->pasid) | QI_EIOTLB_DID(sdev->did) |  in intel_flush_svm_range_dev()
169 			desc.low = QI_EIOTLB_PASID(svm->pasid) | QI_EIOTLB_DID(sdev->did) |  in intel_flush_svm_range_dev()
175 		desc.low = QI_EIOTLB_PASID(svm->pasid) | QI_EIOTLB_DID(sdev->did) |  in intel_flush_svm_range_dev()
182 	if (sdev->dev_iotlb) {  in intel_flush_svm_range_dev()
183 		desc.low = QI_DEV_EIOTLB_PASID(svm->pasid) | QI_DEV_EIOTLB_SID(sdev->sid) |  in intel_flush_svm_range_dev()
184 			QI_DEV_EIOTLB_QDEP(sdev->qdep) | QI_DEIOTLB_TYPE;  in intel_flush_svm_range_dev()
205 	struct intel_svm_dev *sdev;  in intel_flush_svm_range()  local
213 	list_for_each_entry_rcu(sdev, &svm->devs, list)  in intel_flush_svm_range()
214 		intel_flush_svm_range_dev(svm, sdev, address, pages, ih, gl);  in intel_flush_svm_range()
238 static void intel_flush_pasid_dev(struct intel_svm *svm, struct intel_svm_dev *sdev, int pasid)  in intel_flush_pasid_dev()  argument
243 	desc.low = QI_PC_TYPE | QI_PC_DID(sdev->did) | QI_PC_PASID_SEL | QI_PC_PASID(pasid);  in intel_flush_pasid_dev()
251 	struct intel_svm_dev *sdev;  in intel_mm_release()  local
266 	list_for_each_entry_rcu(sdev, &svm->devs, list) {  in intel_mm_release()
267 		intel_pasid_clear_entry(sdev->dev, svm->pasid);  in intel_mm_release()
268 		intel_flush_pasid_dev(svm, sdev, svm->pasid);  in intel_mm_release()
269 		intel_flush_svm_range_dev(svm, sdev, 0, -1, 0, !svm->mm);  in intel_mm_release()
289 	struct intel_svm_dev *sdev;  in intel_svm_bind_mm()  local
331 			list_for_each_entry(sdev, &svm->devs, list) {  in intel_svm_bind_mm()
332 				if (dev == sdev->dev) {  in intel_svm_bind_mm()
333 					if (sdev->ops != ops) {  in intel_svm_bind_mm()
337 					sdev->users++;  in intel_svm_bind_mm()
346 	sdev = kzalloc(sizeof(*sdev), GFP_KERNEL);  in intel_svm_bind_mm()
347 	if (!sdev) {  in intel_svm_bind_mm()
351 	sdev->dev = dev;  in intel_svm_bind_mm()
353 	ret = intel_iommu_enable_pasid(iommu, sdev);  in intel_svm_bind_mm()
357 		kfree(sdev);  in intel_svm_bind_mm()
361 	sdev->users = 1;  in intel_svm_bind_mm()
362 	sdev->ops = ops;  in intel_svm_bind_mm()
363 	init_rcu_head(&sdev->rcu);  in intel_svm_bind_mm()
369 			kfree(sdev);  in intel_svm_bind_mm()
383 			kfree(sdev);  in intel_svm_bind_mm()
398 				kfree(sdev);  in intel_svm_bind_mm()
418 			intel_flush_pasid_dev(svm, sdev, svm->pasid);  in intel_svm_bind_mm()
422 	list_add_rcu(&sdev->list, &svm->devs);  in intel_svm_bind_mm()
437 	struct intel_svm_dev *sdev;  in intel_svm_unbind_mm()  local
451 	list_for_each_entry(sdev, &svm->devs, list) {  in intel_svm_unbind_mm()
452 		if (dev == sdev->dev) {  in intel_svm_unbind_mm()
454 			sdev->users--;  in intel_svm_unbind_mm()
455 			if (!sdev->users) {  in intel_svm_unbind_mm()
456 				list_del_rcu(&sdev->list);  in intel_svm_unbind_mm()
464 				intel_flush_pasid_dev(svm, sdev, svm->pasid);  in intel_svm_unbind_mm()
465 				intel_flush_svm_range_dev(svm, sdev, 0, -1, 0, !svm->mm);  in intel_svm_unbind_mm()
466 				kfree_rcu(sdev, rcu);  in intel_svm_unbind_mm()
581 		struct intel_svm_dev *sdev;  in prq_event_thread()  local
651 		list_for_each_entry_rcu(sdev, &svm->devs, list) {  in prq_event_thread()
652 			if (sdev->sid == PCI_DEVID(req->bus, req->devfn))  in prq_event_thread()
660 		if (WARN_ON(&sdev->list == &svm->devs))  in prq_event_thread()
661 			sdev = NULL;  in prq_event_thread()
663 		if (sdev && sdev->ops && sdev->ops->fault_cb) {  in prq_event_thread()
666 			sdev->ops->fault_cb(sdev->dev, req->pasid, req->addr, req->private, rwxp, result);  in prq_event_thread()
670 		sdev = NULL;  in prq_event_thread()