31 int intel_svm_enable_prq(struct intel_iommu *iommu)  in intel_svm_enable_prq()  argument
39 			iommu->name);  in intel_svm_enable_prq()
42 	iommu->prq = page_address(pages);  in intel_svm_enable_prq()
44 	irq = dmar_alloc_hwirq(DMAR_UNITS_SUPPORTED + iommu->seq_id, iommu->node, iommu);  in intel_svm_enable_prq()
47 		       iommu->name);  in intel_svm_enable_prq()
50 		free_pages((unsigned long)iommu->prq, PRQ_ORDER);  in intel_svm_enable_prq()
51 		iommu->prq = NULL;  in intel_svm_enable_prq()
54 	iommu->pr_irq = irq;  in intel_svm_enable_prq()
56 	snprintf(iommu->prq_name, sizeof(iommu->prq_name), "dmar%d-prq", iommu->seq_id);  in intel_svm_enable_prq()
59 				   iommu->prq_name, iommu);  in intel_svm_enable_prq()
62 		       iommu->name);  in intel_svm_enable_prq()
64 		iommu->pr_irq = 0;  in intel_svm_enable_prq()
67 	dmar_writeq(iommu->reg + DMAR_PQH_REG, 0ULL);  in intel_svm_enable_prq()
68 	dmar_writeq(iommu->reg + DMAR_PQT_REG, 0ULL);  in intel_svm_enable_prq()
69 	dmar_writeq(iommu->reg + DMAR_PQA_REG, virt_to_phys(iommu->prq) | PRQ_ORDER);  in intel_svm_enable_prq()
71 	init_completion(&iommu->prq_complete);  in intel_svm_enable_prq()
76 int intel_svm_finish_prq(struct intel_iommu *iommu)  in intel_svm_finish_prq()  argument
78 	dmar_writeq(iommu->reg + DMAR_PQH_REG, 0ULL);  in intel_svm_finish_prq()
79 	dmar_writeq(iommu->reg + DMAR_PQT_REG, 0ULL);  in intel_svm_finish_prq()
80 	dmar_writeq(iommu->reg + DMAR_PQA_REG, 0ULL);  in intel_svm_finish_prq()
82 	if (iommu->pr_irq) {  in intel_svm_finish_prq()
83 		free_irq(iommu->pr_irq, iommu);  in intel_svm_finish_prq()
84 		dmar_free_hwirq(iommu->pr_irq);  in intel_svm_finish_prq()
85 		iommu->pr_irq = 0;  in intel_svm_finish_prq()
88 	free_pages((unsigned long)iommu->prq, PRQ_ORDER);  in intel_svm_finish_prq()
89 	iommu->prq = NULL;  in intel_svm_finish_prq()
94 static inline bool intel_svm_capable(struct intel_iommu *iommu)  in intel_svm_capable()  argument
96 	return iommu->flags & VTD_FLAG_SVM_CAPABLE;  in intel_svm_capable()
99 void intel_svm_check(struct intel_iommu *iommu)  in intel_svm_check()  argument
101 	if (!pasid_supported(iommu))  in intel_svm_check()
105 	    !cap_fl1gp_support(iommu->cap)) {  in intel_svm_check()
107 		       iommu->name);  in intel_svm_check()
112 	    !cap_5lp_support(iommu->cap)) {  in intel_svm_check()
114 		       iommu->name);  in intel_svm_check()
118 	iommu->flags |= VTD_FLAG_SVM_CAPABLE;  in intel_svm_check()
145 	qi_submit_sync(svm->iommu, &desc, 1, 0);  in intel_flush_svm_range_dev()
169 		qi_submit_sync(svm->iommu, &desc, 1, 0);  in intel_flush_svm_range_dev()
214 		intel_pasid_tear_down_entry(svm->iommu, sdev->dev,  in intel_mm_release()
279 	struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL);  in intel_svm_bind_gpasid()  local
286 	if (WARN_ON(!iommu) || !data)  in intel_svm_bind_gpasid()
372 	ret = intel_iommu_enable_pasid(iommu, sdev->dev);  in intel_svm_bind_gpasid()
384 	spin_lock(&iommu->lock);  in intel_svm_bind_gpasid()
385 	ret = intel_pasid_setup_nested(iommu, dev,  in intel_svm_bind_gpasid()
389 	spin_unlock(&iommu->lock);  in intel_svm_bind_gpasid()
418 	struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL);  in intel_svm_unbind_gpasid()  local
423 	if (WARN_ON(!iommu))  in intel_svm_unbind_gpasid()
436 			intel_pasid_tear_down_entry(iommu, dev,  in intel_svm_unbind_gpasid()
485 	struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL);  in intel_svm_bind_mm()  local
492 	if (!iommu || dmar_disabled)  in intel_svm_bind_mm()
495 	if (!intel_svm_capable(iommu))  in intel_svm_bind_mm()
507 		if (!ecap_srs(iommu->ecap) || mm) {  in intel_svm_bind_mm()
550 	ret = intel_iommu_enable_pasid(iommu, dev);  in intel_svm_bind_mm()
578 		svm->iommu = iommu;  in intel_svm_bind_mm()
608 		spin_lock(&iommu->lock);  in intel_svm_bind_mm()
609 		ret = intel_pasid_setup_first_level(iommu, dev,  in intel_svm_bind_mm()
615 		spin_unlock(&iommu->lock);  in intel_svm_bind_mm()
635 		spin_lock(&iommu->lock);  in intel_svm_bind_mm()
636 		ret = intel_pasid_setup_first_level(iommu, dev,  in intel_svm_bind_mm()
642 		spin_unlock(&iommu->lock);  in intel_svm_bind_mm()
663 	struct intel_iommu *iommu;  in intel_svm_unbind_mm()  local
667 	iommu = device_to_iommu(dev, NULL, NULL);  in intel_svm_unbind_mm()
668 	if (!iommu)  in intel_svm_unbind_mm()
686 			intel_pasid_tear_down_entry(iommu, dev,  in intel_svm_unbind_mm()
786 	struct intel_iommu *iommu;  in intel_svm_drain_prq()  local
800 	iommu = info->iommu;  in intel_svm_drain_prq()
804 	did = domain->iommu_did[iommu->seq_id];  in intel_svm_drain_prq()
812 	reinit_completion(&iommu->prq_complete);  in intel_svm_drain_prq()
813 	tail = dmar_readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK;  in intel_svm_drain_prq()
814 	head = dmar_readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK;  in intel_svm_drain_prq()
818 		req = &iommu->prq[head / sizeof(*req)];  in intel_svm_drain_prq()
824 		wait_for_completion(&iommu->prq_complete);  in intel_svm_drain_prq()
846 	reinit_completion(&iommu->prq_complete);  in intel_svm_drain_prq()
847 	qi_submit_sync(iommu, desc, 3, QI_OPT_WAIT_DRAIN);  in intel_svm_drain_prq()
848 	if (readl(iommu->reg + DMAR_PRS_REG) & DMA_PRS_PRO) {  in intel_svm_drain_prq()
849 		wait_for_completion(&iommu->prq_complete);  in intel_svm_drain_prq()
911 	struct intel_iommu *iommu = d;  in prq_event_thread()  local
917 	writel(DMA_PRS_PPR, iommu->reg + DMAR_PRS_REG);  in prq_event_thread()
919 	tail = dmar_readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK;  in prq_event_thread()
920 	head = dmar_readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK;  in prq_event_thread()
931 		req = &iommu->prq[head / sizeof(*req)];  in prq_event_thread()
937 			       iommu->name, ((unsigned long long *)req)[0],  in prq_event_thread()
951 				       iommu->name, req->pasid, ((unsigned long long *)req)[0],  in prq_event_thread()
1049 			qi_submit_sync(iommu, &resp, 1, 0);  in prq_event_thread()
1055 	dmar_writeq(iommu->reg + DMAR_PQH_REG, tail);  in prq_event_thread()
1061 	if (readl(iommu->reg + DMAR_PRS_REG) & DMA_PRS_PRO)  in prq_event_thread()
1062 		writel(DMA_PRS_PRO, iommu->reg + DMAR_PRS_REG);  in prq_event_thread()
1064 	if (!completion_done(&iommu->prq_complete))  in prq_event_thread()
1065 		complete(&iommu->prq_complete);  in prq_event_thread()
1130 	struct intel_iommu *iommu;  in intel_svm_page_response()  local
1141 	iommu = device_to_iommu(dev, &bus, &devfn);  in intel_svm_page_response()
1142 	if (!iommu)  in intel_svm_page_response()
1215 		qi_submit_sync(iommu, &desc, 1, 0);  in intel_svm_page_response()