15 #include <linux/intel-iommu.h>
16 #include <linux/iommu.h>
25  * Intel IOMMU system wide PASID name space:
30 int vcmd_alloc_pasid(struct intel_iommu *iommu, u32 *pasid)  in vcmd_alloc_pasid()  argument
37 	raw_spin_lock_irqsave(&iommu->register_lock, flags);  in vcmd_alloc_pasid()
38 	dmar_writeq(iommu->reg + DMAR_VCMD_REG, VCMD_CMD_ALLOC);  in vcmd_alloc_pasid()
39 	IOMMU_WAIT_OP(iommu, DMAR_VCRSP_REG, dmar_readq,  in vcmd_alloc_pasid()
41 	raw_spin_unlock_irqrestore(&iommu->register_lock, flags);  in vcmd_alloc_pasid()
49 		pr_info("IOMMU: %s: No PASID available\n", iommu->name);  in vcmd_alloc_pasid()
54 		pr_warn("IOMMU: %s: Unexpected error code %d\n",  in vcmd_alloc_pasid()
55 			iommu->name, status_code);  in vcmd_alloc_pasid()
61 void vcmd_free_pasid(struct intel_iommu *iommu, u32 pasid)  in vcmd_free_pasid()  argument
67 	raw_spin_lock_irqsave(&iommu->register_lock, flags);  in vcmd_free_pasid()
68 	dmar_writeq(iommu->reg + DMAR_VCMD_REG,  in vcmd_free_pasid()
70 	IOMMU_WAIT_OP(iommu, DMAR_VCRSP_REG, dmar_readq,  in vcmd_free_pasid()
72 	raw_spin_unlock_irqrestore(&iommu->register_lock, flags);  in vcmd_free_pasid()
79 		pr_info("IOMMU: %s: Invalid PASID\n", iommu->name);  in vcmd_free_pasid()
82 		pr_warn("IOMMU: %s: Unexpected error code %d\n",  in vcmd_free_pasid()
83 			iommu->name, status_code);  in vcmd_free_pasid()
117 	if (info->iommu->segment == data->segment &&  in search_pasid_table()
176 	pages = alloc_pages_node(info->iommu->node,  in intel_pasid_alloc_table()
265 		entries = alloc_pgtable_page(info->iommu->node);  in intel_pasid_get_entry()
445 pasid_cache_invalidation_with_pasid(struct intel_iommu *iommu,  in pasid_cache_invalidation_with_pasid()  argument
456 	qi_submit_sync(iommu, &desc, 1, 0);  in pasid_cache_invalidation_with_pasid()
460 iotlb_invalidation_with_pasid(struct intel_iommu *iommu, u16 did, u32 pasid)  in iotlb_invalidation_with_pasid()  argument
470 	qi_submit_sync(iommu, &desc, 1, 0);  in iotlb_invalidation_with_pasid()
474 devtlb_invalidation_with_pasid(struct intel_iommu *iommu,  in devtlb_invalidation_with_pasid()  argument
495 		qi_flush_dev_iotlb(iommu, sid, pfsid, qdep, 0, 64 - VTD_PAGE_SHIFT);  in devtlb_invalidation_with_pasid()
497 		qi_flush_dev_iotlb_pasid(iommu, sid, pfsid, pasid, qdep, 0, 64 - VTD_PAGE_SHIFT);  in devtlb_invalidation_with_pasid()
500 void intel_pasid_tear_down_entry(struct intel_iommu *iommu, struct device *dev,  in intel_pasid_tear_down_entry()  argument
513 	if (!ecap_coherent(iommu->ecap))  in intel_pasid_tear_down_entry()
516 	pasid_cache_invalidation_with_pasid(iommu, did, pasid);  in intel_pasid_tear_down_entry()
517 	iotlb_invalidation_with_pasid(iommu, did, pasid);  in intel_pasid_tear_down_entry()
520 	if (!cap_caching_mode(iommu->cap))  in intel_pasid_tear_down_entry()
521 		devtlb_invalidation_with_pasid(iommu, dev, pasid);  in intel_pasid_tear_down_entry()
524 static void pasid_flush_caches(struct intel_iommu *iommu,  in pasid_flush_caches()  argument
528 	if (!ecap_coherent(iommu->ecap))  in pasid_flush_caches()
531 	if (cap_caching_mode(iommu->cap)) {  in pasid_flush_caches()
532 		pasid_cache_invalidation_with_pasid(iommu, did, pasid);  in pasid_flush_caches()
533 		iotlb_invalidation_with_pasid(iommu, did, pasid);  in pasid_flush_caches()
535 		iommu_flush_write_buffer(iommu);  in pasid_flush_caches()
543 int intel_pasid_setup_first_level(struct intel_iommu *iommu,  in intel_pasid_setup_first_level()  argument
549 	if (!ecap_flts(iommu->ecap)) {  in intel_pasid_setup_first_level()
551 		       iommu->name);  in intel_pasid_setup_first_level()
564 		if (!ecap_srs(iommu->ecap)) {  in intel_pasid_setup_first_level()
566 			       iommu->name);  in intel_pasid_setup_first_level()
573 		if (cap_5lp_support(iommu->cap)) {  in intel_pasid_setup_first_level()
583 	pasid_set_address_width(pte, iommu->agaw);  in intel_pasid_setup_first_level()
584 	pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap));  in intel_pasid_setup_first_level()
589 	pasid_flush_caches(iommu, pte, pasid, did);  in intel_pasid_setup_first_level()
595  * Skip top levels of page tables for iommu which has less agaw
599 				  struct intel_iommu *iommu,  in iommu_skip_agaw()  argument
604 	for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) {  in iommu_skip_agaw()
616 int intel_pasid_setup_second_level(struct intel_iommu *iommu,  in intel_pasid_setup_second_level()  argument
630 	if (!ecap_slts(iommu->ecap)) {  in intel_pasid_setup_second_level()
632 		       iommu->name);  in intel_pasid_setup_second_level()
637 	agaw = iommu_skip_agaw(domain, iommu, &pgd);  in intel_pasid_setup_second_level()
644 	did = domain->iommu_did[iommu->seq_id];  in intel_pasid_setup_second_level()
658 	pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap));  in intel_pasid_setup_second_level()
666 	pasid_flush_caches(iommu, pte, pasid, did);  in intel_pasid_setup_second_level()
674 int intel_pasid_setup_pass_through(struct intel_iommu *iommu,  in intel_pasid_setup_pass_through()  argument
689 	pasid_set_address_width(pte, iommu->agaw);  in intel_pasid_setup_pass_through()
692 	pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap));  in intel_pasid_setup_pass_through()
700 	pasid_flush_caches(iommu, pte, pasid, did);  in intel_pasid_setup_pass_through()
706 intel_pasid_setup_bind_data(struct intel_iommu *iommu, struct pasid_entry *pte,  in intel_pasid_setup_bind_data()  argument
717 		if (!ecap_srs(iommu->ecap)) {  in intel_pasid_setup_bind_data()
719 					   iommu->name);  in intel_pasid_setup_bind_data()
726 		if (!ecap_eafs(iommu->ecap)) {  in intel_pasid_setup_bind_data()
728 					   iommu->name);  in intel_pasid_setup_bind_data()
740 				    iommu->name);  in intel_pasid_setup_bind_data()
753  * @iommu:      IOMMU which the device belong to
761 int intel_pasid_setup_nested(struct intel_iommu *iommu, struct device *dev,  in intel_pasid_setup_nested()  argument
773 	if (!ecap_nest(iommu->ecap)) {  in intel_pasid_setup_nested()
774 		pr_err_ratelimited("IOMMU: %s: No nested translation support\n",  in intel_pasid_setup_nested()
775 				   iommu->name);  in intel_pasid_setup_nested()
800 	 * 1. CPU vs. IOMMU  in intel_pasid_setup_nested()
807 		    !cap_5lp_support(iommu->cap)) {  in intel_pasid_setup_nested()
834 	ret = intel_pasid_setup_bind_data(iommu, pte, pasid_data);  in intel_pasid_setup_nested()
841 	agaw = iommu_skip_agaw(domain, iommu, &pgd);  in intel_pasid_setup_nested()
850 	did = domain->iommu_did[iommu->seq_id];  in intel_pasid_setup_nested()
854 	pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap));  in intel_pasid_setup_nested()
858 	pasid_flush_caches(iommu, pte, pasid, did);  in intel_pasid_setup_nested()