15 #include <linux/iommu.h>
21 #include "iommu.h"
25  * Intel IOMMU system wide PASID name space:
29 int vcmd_alloc_pasid(struct intel_iommu *iommu, u32 *pasid)  in vcmd_alloc_pasid()  argument
36 	raw_spin_lock_irqsave(&iommu->register_lock, flags);  in vcmd_alloc_pasid()
37 	dmar_writeq(iommu->reg + DMAR_VCMD_REG, VCMD_CMD_ALLOC);  in vcmd_alloc_pasid()
38 	IOMMU_WAIT_OP(iommu, DMAR_VCRSP_REG, dmar_readq,  in vcmd_alloc_pasid()
40 	raw_spin_unlock_irqrestore(&iommu->register_lock, flags);  in vcmd_alloc_pasid()
48 		pr_info("IOMMU: %s: No PASID available\n", iommu->name);  in vcmd_alloc_pasid()
53 		pr_warn("IOMMU: %s: Unexpected error code %d\n",  in vcmd_alloc_pasid()
54 			iommu->name, status_code);  in vcmd_alloc_pasid()
60 void vcmd_free_pasid(struct intel_iommu *iommu, u32 pasid)  in vcmd_free_pasid()  argument
66 	raw_spin_lock_irqsave(&iommu->register_lock, flags);  in vcmd_free_pasid()
67 	dmar_writeq(iommu->reg + DMAR_VCMD_REG,  in vcmd_free_pasid()
69 	IOMMU_WAIT_OP(iommu, DMAR_VCRSP_REG, dmar_readq,  in vcmd_free_pasid()
71 	raw_spin_unlock_irqrestore(&iommu->register_lock, flags);  in vcmd_free_pasid()
78 		pr_info("IOMMU: %s: Invalid PASID\n", iommu->name);  in vcmd_free_pasid()
81 		pr_warn("IOMMU: %s: Unexpected error code %d\n",  in vcmd_free_pasid()
82 			iommu->name, status_code);  in vcmd_free_pasid()
117 	pages = alloc_pages_node(info->iommu->node,  in intel_pasid_alloc_table()
201 		entries = alloc_pgtable_page(info->iommu->node);  in intel_pasid_get_entry()
396 pasid_cache_invalidation_with_pasid(struct intel_iommu *iommu,  in pasid_cache_invalidation_with_pasid()  argument
407 	qi_submit_sync(iommu, &desc, 1, 0);  in pasid_cache_invalidation_with_pasid()
411 devtlb_invalidation_with_pasid(struct intel_iommu *iommu,  in devtlb_invalidation_with_pasid()  argument
432 		qi_flush_dev_iotlb(iommu, sid, pfsid, qdep, 0, 64 - VTD_PAGE_SHIFT);  in devtlb_invalidation_with_pasid()
434 		qi_flush_dev_iotlb_pasid(iommu, sid, pfsid, pasid, qdep, 0, 64 - VTD_PAGE_SHIFT);  in devtlb_invalidation_with_pasid()
437 void intel_pasid_tear_down_entry(struct intel_iommu *iommu, struct device *dev,  in intel_pasid_tear_down_entry()  argument
443 	spin_lock(&iommu->lock);  in intel_pasid_tear_down_entry()
446 		spin_unlock(&iommu->lock);  in intel_pasid_tear_down_entry()
453 	spin_unlock(&iommu->lock);  in intel_pasid_tear_down_entry()
455 	if (!ecap_coherent(iommu->ecap))  in intel_pasid_tear_down_entry()
458 	pasid_cache_invalidation_with_pasid(iommu, did, pasid);  in intel_pasid_tear_down_entry()
461 		qi_flush_piotlb(iommu, did, pasid, 0, -1, 0);  in intel_pasid_tear_down_entry()
463 		iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);  in intel_pasid_tear_down_entry()
466 	if (!cap_caching_mode(iommu->cap))  in intel_pasid_tear_down_entry()
467 		devtlb_invalidation_with_pasid(iommu, dev, pasid);  in intel_pasid_tear_down_entry()
474 static void pasid_flush_caches(struct intel_iommu *iommu,  in pasid_flush_caches()  argument
478 	if (!ecap_coherent(iommu->ecap))  in pasid_flush_caches()
481 	if (cap_caching_mode(iommu->cap)) {  in pasid_flush_caches()
482 		pasid_cache_invalidation_with_pasid(iommu, did, pasid);  in pasid_flush_caches()
483 		qi_flush_piotlb(iommu, did, pasid, 0, -1, 0);  in pasid_flush_caches()
485 		iommu_flush_write_buffer(iommu);  in pasid_flush_caches()
493 int intel_pasid_setup_first_level(struct intel_iommu *iommu,  in intel_pasid_setup_first_level()  argument
499 	if (!ecap_flts(iommu->ecap)) {  in intel_pasid_setup_first_level()
501 		       iommu->name);  in intel_pasid_setup_first_level()
515 		if (!ecap_srs(iommu->ecap)) {  in intel_pasid_setup_first_level()
517 			       iommu->name);  in intel_pasid_setup_first_level()
522 	if ((flags & PASID_FLAG_FL5LP) && !cap_fl5lp_support(iommu->cap)) {  in intel_pasid_setup_first_level()
524 		       iommu->name);  in intel_pasid_setup_first_level()
528 	spin_lock(&iommu->lock);  in intel_pasid_setup_first_level()
531 		spin_unlock(&iommu->lock);  in intel_pasid_setup_first_level()
536 		spin_unlock(&iommu->lock);  in intel_pasid_setup_first_level()
556 	pasid_set_address_width(pte, iommu->agaw);  in intel_pasid_setup_first_level()
557 	pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap));  in intel_pasid_setup_first_level()
562 	spin_unlock(&iommu->lock);  in intel_pasid_setup_first_level()
564 	pasid_flush_caches(iommu, pte, pasid, did);  in intel_pasid_setup_first_level()
570  * Skip top levels of page tables for iommu which has less agaw
574 				  struct intel_iommu *iommu,  in iommu_skip_agaw()  argument
579 	for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) {  in iommu_skip_agaw()
591 int intel_pasid_setup_second_level(struct intel_iommu *iommu,  in intel_pasid_setup_second_level()  argument
605 	if (!ecap_slts(iommu->ecap)) {  in intel_pasid_setup_second_level()
607 		       iommu->name);  in intel_pasid_setup_second_level()
612 	agaw = iommu_skip_agaw(domain, iommu, &pgd);  in intel_pasid_setup_second_level()
619 	did = domain_id_iommu(domain, iommu);  in intel_pasid_setup_second_level()
621 	spin_lock(&iommu->lock);  in intel_pasid_setup_second_level()
624 		spin_unlock(&iommu->lock);  in intel_pasid_setup_second_level()
629 		spin_unlock(&iommu->lock);  in intel_pasid_setup_second_level()
639 	pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap));  in intel_pasid_setup_second_level()
645 	if (pasid != PASID_RID2PASID && ecap_srs(iommu->ecap))  in intel_pasid_setup_second_level()
648 	spin_unlock(&iommu->lock);  in intel_pasid_setup_second_level()
650 	pasid_flush_caches(iommu, pte, pasid, did);  in intel_pasid_setup_second_level()
658 int intel_pasid_setup_pass_through(struct intel_iommu *iommu,  in intel_pasid_setup_pass_through()  argument
665 	spin_lock(&iommu->lock);  in intel_pasid_setup_pass_through()
668 		spin_unlock(&iommu->lock);  in intel_pasid_setup_pass_through()
673 		spin_unlock(&iommu->lock);  in intel_pasid_setup_pass_through()
679 	pasid_set_address_width(pte, iommu->agaw);  in intel_pasid_setup_pass_through()
682 	pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap));  in intel_pasid_setup_pass_through()
688 	if (ecap_srs(iommu->ecap))  in intel_pasid_setup_pass_through()
691 	spin_unlock(&iommu->lock);  in intel_pasid_setup_pass_through()
693 	pasid_flush_caches(iommu, pte, pasid, did);  in intel_pasid_setup_pass_through()
701 void intel_pasid_setup_page_snoop_control(struct intel_iommu *iommu,  in intel_pasid_setup_page_snoop_control()  argument
707 	spin_lock(&iommu->lock);  in intel_pasid_setup_page_snoop_control()
710 		spin_unlock(&iommu->lock);  in intel_pasid_setup_page_snoop_control()
716 	spin_unlock(&iommu->lock);  in intel_pasid_setup_page_snoop_control()
718 	if (!ecap_coherent(iommu->ecap))  in intel_pasid_setup_page_snoop_control()
732 	pasid_cache_invalidation_with_pasid(iommu, did, pasid);  in intel_pasid_setup_page_snoop_control()
733 	qi_flush_piotlb(iommu, did, pasid, 0, -1, 0);  in intel_pasid_setup_page_snoop_control()
736 	if (!cap_caching_mode(iommu->cap))  in intel_pasid_setup_page_snoop_control()
737 		devtlb_invalidation_with_pasid(iommu, dev, pasid);  in intel_pasid_setup_page_snoop_control()