15 #include <linux/intel-iommu.h>
16 #include <linux/iommu.h>
25  * Intel IOMMU system wide PASID name space:
29 int vcmd_alloc_pasid(struct intel_iommu *iommu, u32 *pasid)  in vcmd_alloc_pasid()  argument
36 	raw_spin_lock_irqsave(&iommu->register_lock, flags);  in vcmd_alloc_pasid()
37 	dmar_writeq(iommu->reg + DMAR_VCMD_REG, VCMD_CMD_ALLOC);  in vcmd_alloc_pasid()
38 	IOMMU_WAIT_OP(iommu, DMAR_VCRSP_REG, dmar_readq,  in vcmd_alloc_pasid()
40 	raw_spin_unlock_irqrestore(&iommu->register_lock, flags);  in vcmd_alloc_pasid()
48 		pr_info("IOMMU: %s: No PASID available\n", iommu->name);  in vcmd_alloc_pasid()
53 		pr_warn("IOMMU: %s: Unexpected error code %d\n",  in vcmd_alloc_pasid()
54 			iommu->name, status_code);  in vcmd_alloc_pasid()
60 void vcmd_free_pasid(struct intel_iommu *iommu, u32 pasid)  in vcmd_free_pasid()  argument
66 	raw_spin_lock_irqsave(&iommu->register_lock, flags);  in vcmd_free_pasid()
67 	dmar_writeq(iommu->reg + DMAR_VCMD_REG,  in vcmd_free_pasid()
69 	IOMMU_WAIT_OP(iommu, DMAR_VCRSP_REG, dmar_readq,  in vcmd_free_pasid()
71 	raw_spin_unlock_irqrestore(&iommu->register_lock, flags);  in vcmd_free_pasid()
78 		pr_info("IOMMU: %s: Invalid PASID\n", iommu->name);  in vcmd_free_pasid()
81 		pr_warn("IOMMU: %s: Unexpected error code %d\n",  in vcmd_free_pasid()
82 			iommu->name, status_code);  in vcmd_free_pasid()
116 	if (info->iommu->segment == data->segment &&  in search_pasid_table()
175 	pages = alloc_pages_node(info->iommu->node,  in intel_pasid_alloc_table()
264 		entries = alloc_pgtable_page(info->iommu->node);  in intel_pasid_get_entry()
469 pasid_cache_invalidation_with_pasid(struct intel_iommu *iommu,  in pasid_cache_invalidation_with_pasid()  argument
480 	qi_submit_sync(iommu, &desc, 1, 0);  in pasid_cache_invalidation_with_pasid()
484 devtlb_invalidation_with_pasid(struct intel_iommu *iommu,  in devtlb_invalidation_with_pasid()  argument
505 		qi_flush_dev_iotlb(iommu, sid, pfsid, qdep, 0, 64 - VTD_PAGE_SHIFT);  in devtlb_invalidation_with_pasid()
507 		qi_flush_dev_iotlb_pasid(iommu, sid, pfsid, pasid, qdep, 0, 64 - VTD_PAGE_SHIFT);  in devtlb_invalidation_with_pasid()
510 void intel_pasid_tear_down_entry(struct intel_iommu *iommu, struct device *dev,  in intel_pasid_tear_down_entry()  argument
528 	if (!ecap_coherent(iommu->ecap))  in intel_pasid_tear_down_entry()
531 	pasid_cache_invalidation_with_pasid(iommu, did, pasid);  in intel_pasid_tear_down_entry()
534 		qi_flush_piotlb(iommu, did, pasid, 0, -1, 0);  in intel_pasid_tear_down_entry()
536 		iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);  in intel_pasid_tear_down_entry()
539 	if (!cap_caching_mode(iommu->cap))  in intel_pasid_tear_down_entry()
540 		devtlb_invalidation_with_pasid(iommu, dev, pasid);  in intel_pasid_tear_down_entry()
547 static void pasid_flush_caches(struct intel_iommu *iommu,  in pasid_flush_caches()  argument
551 	if (!ecap_coherent(iommu->ecap))  in pasid_flush_caches()
554 	if (cap_caching_mode(iommu->cap)) {  in pasid_flush_caches()
555 		pasid_cache_invalidation_with_pasid(iommu, did, pasid);  in pasid_flush_caches()
556 		qi_flush_piotlb(iommu, did, pasid, 0, -1, 0);  in pasid_flush_caches()
558 		iommu_flush_write_buffer(iommu);  in pasid_flush_caches()
582 int intel_pasid_setup_first_level(struct intel_iommu *iommu,  in intel_pasid_setup_first_level()  argument
588 	if (!ecap_flts(iommu->ecap)) {  in intel_pasid_setup_first_level()
590 		       iommu->name);  in intel_pasid_setup_first_level()
607 		if (!ecap_srs(iommu->ecap)) {  in intel_pasid_setup_first_level()
609 			       iommu->name);  in intel_pasid_setup_first_level()
619 		if (cap_5lp_support(iommu->cap)) {  in intel_pasid_setup_first_level()
632 	pasid_set_address_width(pte, iommu->agaw);  in intel_pasid_setup_first_level()
633 	pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap));  in intel_pasid_setup_first_level()
638 	pasid_flush_caches(iommu, pte, pasid, did);  in intel_pasid_setup_first_level()
644  * Skip top levels of page tables for iommu which has less agaw
648 				  struct intel_iommu *iommu,  in iommu_skip_agaw()  argument
653 	for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) {  in iommu_skip_agaw()
665 int intel_pasid_setup_second_level(struct intel_iommu *iommu,  in intel_pasid_setup_second_level()  argument
679 	if (!ecap_slts(iommu->ecap)) {  in intel_pasid_setup_second_level()
681 		       iommu->name);  in intel_pasid_setup_second_level()
686 	agaw = iommu_skip_agaw(domain, iommu, &pgd);  in intel_pasid_setup_second_level()
693 	did = domain->iommu_did[iommu->seq_id];  in intel_pasid_setup_second_level()
711 	pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap));  in intel_pasid_setup_second_level()
723 	pasid_flush_caches(iommu, pte, pasid, did);  in intel_pasid_setup_second_level()
731 int intel_pasid_setup_pass_through(struct intel_iommu *iommu,  in intel_pasid_setup_pass_through()  argument
750 	pasid_set_address_width(pte, iommu->agaw);  in intel_pasid_setup_pass_through()
753 	pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap));  in intel_pasid_setup_pass_through()
761 	pasid_flush_caches(iommu, pte, pasid, did);  in intel_pasid_setup_pass_through()
767 intel_pasid_setup_bind_data(struct intel_iommu *iommu, struct pasid_entry *pte,  in intel_pasid_setup_bind_data()  argument
778 		if (!ecap_srs(iommu->ecap)) {  in intel_pasid_setup_bind_data()
780 					   iommu->name);  in intel_pasid_setup_bind_data()
790 		if (!ecap_eafs(iommu->ecap)) {  in intel_pasid_setup_bind_data()
792 					   iommu->name);  in intel_pasid_setup_bind_data()
804 				    iommu->name);  in intel_pasid_setup_bind_data()
817  * @iommu:      IOMMU which the device belong to
825 int intel_pasid_setup_nested(struct intel_iommu *iommu, struct device *dev,  in intel_pasid_setup_nested()  argument
837 	if (!ecap_nest(iommu->ecap)) {  in intel_pasid_setup_nested()
838 		pr_err_ratelimited("IOMMU: %s: No nested translation support\n",  in intel_pasid_setup_nested()
839 				   iommu->name);  in intel_pasid_setup_nested()
864 	 * 1. CPU vs. IOMMU  in intel_pasid_setup_nested()
871 		    !cap_5lp_support(iommu->cap)) {  in intel_pasid_setup_nested()
898 	ret = intel_pasid_setup_bind_data(iommu, pte, pasid_data);  in intel_pasid_setup_nested()
905 	agaw = iommu_skip_agaw(domain, iommu, &pgd);  in intel_pasid_setup_nested()
914 	did = domain->iommu_did[iommu->seq_id];  in intel_pasid_setup_nested()
918 	pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap));  in intel_pasid_setup_nested()
922 	pasid_flush_caches(iommu, pte, pasid, did);  in intel_pasid_setup_nested()