13 #include <linux/intel-iommu.h>
33 	struct intel_iommu *iommu;  member
40 	struct intel_iommu *iommu;  member
47 	struct intel_iommu *iommu;  member
74  *	->iommu->register_lock
83 static void iommu_disable_irq_remapping(struct intel_iommu *iommu);
86 static bool ir_pre_enabled(struct intel_iommu *iommu)  in ir_pre_enabled()  argument
88 	return (iommu->flags & VTD_FLAG_IRQ_REMAP_PRE_ENABLED);  in ir_pre_enabled()
91 static void clear_ir_pre_enabled(struct intel_iommu *iommu)  in clear_ir_pre_enabled()  argument
93 	iommu->flags &= ~VTD_FLAG_IRQ_REMAP_PRE_ENABLED;  in clear_ir_pre_enabled()
96 static void init_ir_status(struct intel_iommu *iommu)  in init_ir_status()  argument
100 	gsts = readl(iommu->reg + DMAR_GSTS_REG);  in init_ir_status()
102 		iommu->flags |= VTD_FLAG_IRQ_REMAP_PRE_ENABLED;  in init_ir_status()
105 static int alloc_irte(struct intel_iommu *iommu,  in alloc_irte()  argument
108 	struct ir_table *table = iommu->ir_table;  in alloc_irte()
121 	if (mask > ecap_max_handle_mask(iommu->ecap)) {  in alloc_irte()
124 		       ecap_max_handle_mask(iommu->ecap));  in alloc_irte()
132 		pr_warn("IR%d: can't allocate an IRTE\n", iommu->seq_id);  in alloc_irte()
134 		irq_iommu->iommu = iommu;  in alloc_irte()
145 static int qi_flush_iec(struct intel_iommu *iommu, int index, int mask)  in qi_flush_iec()  argument
155 	return qi_submit_sync(iommu, &desc, 1, 0);  in qi_flush_iec()
161 	struct intel_iommu *iommu;  in modify_irte()  local
171 	iommu = irq_iommu->iommu;  in modify_irte()
174 	irte = &iommu->ir_table->base[index];  in modify_irte()
196 	__iommu_flush_cache(iommu, irte, sizeof(*irte));  in modify_irte()
198 	rc = qi_flush_iec(iommu, index, 0);  in modify_irte()
200 	/* Update iommu mode according to the IRTE mode */  in modify_irte()
212 		if (ir_hpet[i].id == hpet_id && ir_hpet[i].iommu)  in map_hpet_to_ir()
213 			return ir_hpet[i].iommu->ir_domain;  in map_hpet_to_ir()
223 		if (ir_ioapic[i].id == apic && ir_ioapic[i].iommu)  in map_ioapic_to_iommu()
224 			return ir_ioapic[i].iommu;  in map_ioapic_to_iommu()
231 	struct intel_iommu *iommu = map_ioapic_to_iommu(apic);  in map_ioapic_to_ir()  local
233 	return iommu ? iommu->ir_domain : NULL;  in map_ioapic_to_ir()
240 	return drhd ? drhd->iommu->ir_msi_domain : NULL;  in map_dev_to_ir()
246 	struct intel_iommu *iommu;  in clear_entries()  local
252 	iommu = irq_iommu->iommu;  in clear_entries()
255 	start = iommu->ir_table->base + index;  in clear_entries()
262 	bitmap_release_region(iommu->ir_table->bitmap, index,  in clear_entries()
265 	return qi_flush_iec(iommu, index, irq_iommu->irte_mask);  in clear_entries()
325 		if (ir_ioapic[i].iommu && ir_ioapic[i].id == apic) {  in set_ioapic_sid()
352 		if (ir_hpet[i].iommu && ir_hpet[i].id == id) {  in set_hpet_sid()
438 static int iommu_load_old_irte(struct intel_iommu *iommu)  in iommu_load_old_irte()  argument
447 	irta = dmar_readq(iommu->reg + DMAR_IRTA_REG);  in iommu_load_old_irte()
461 	memcpy(iommu->ir_table->base, old_ir_table, size);  in iommu_load_old_irte()
463 	__iommu_flush_cache(iommu, iommu->ir_table->base, size);  in iommu_load_old_irte()
470 		if (iommu->ir_table->base[i].present)  in iommu_load_old_irte()
471 			bitmap_set(iommu->ir_table->bitmap, i, 1);  in iommu_load_old_irte()
480 static void iommu_set_irq_remapping(struct intel_iommu *iommu, int mode)  in iommu_set_irq_remapping()  argument
486 	addr = virt_to_phys((void *)iommu->ir_table->base);  in iommu_set_irq_remapping()
488 	raw_spin_lock_irqsave(&iommu->register_lock, flags);  in iommu_set_irq_remapping()
490 	dmar_writeq(iommu->reg + DMAR_IRTA_REG,  in iommu_set_irq_remapping()
494 	writel(iommu->gcmd | DMA_GCMD_SIRTP, iommu->reg + DMAR_GCMD_REG);  in iommu_set_irq_remapping()
496 	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,  in iommu_set_irq_remapping()
498 	raw_spin_unlock_irqrestore(&iommu->register_lock, flags);  in iommu_set_irq_remapping()
504 	qi_global_iec(iommu);  in iommu_set_irq_remapping()
507 static void iommu_enable_irq_remapping(struct intel_iommu *iommu)  in iommu_enable_irq_remapping()  argument
512 	raw_spin_lock_irqsave(&iommu->register_lock, flags);  in iommu_enable_irq_remapping()
515 	iommu->gcmd |= DMA_GCMD_IRE;  in iommu_enable_irq_remapping()
516 	writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);  in iommu_enable_irq_remapping()
517 	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,  in iommu_enable_irq_remapping()
522 		iommu->gcmd &= ~DMA_GCMD_CFI;  in iommu_enable_irq_remapping()
523 		writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);  in iommu_enable_irq_remapping()
524 		IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,  in iommu_enable_irq_remapping()
538 	raw_spin_unlock_irqrestore(&iommu->register_lock, flags);  in iommu_enable_irq_remapping()
541 static int intel_setup_irq_remapping(struct intel_iommu *iommu)  in intel_setup_irq_remapping()  argument
548 	if (iommu->ir_table)  in intel_setup_irq_remapping()
555 	pages = alloc_pages_node(iommu->node, GFP_KERNEL | __GFP_ZERO,  in intel_setup_irq_remapping()
559 		       iommu->seq_id, INTR_REMAP_PAGE_ORDER);  in intel_setup_irq_remapping()
565 		pr_err("IR%d: failed to allocate bitmap\n", iommu->seq_id);  in intel_setup_irq_remapping()
569 	fn = irq_domain_alloc_named_id_fwnode("INTEL-IR", iommu->seq_id);  in intel_setup_irq_remapping()
573 	iommu->ir_domain =  in intel_setup_irq_remapping()
577 					    iommu);  in intel_setup_irq_remapping()
578 	if (!iommu->ir_domain) {  in intel_setup_irq_remapping()
580 		pr_err("IR%d: failed to allocate irqdomain\n", iommu->seq_id);  in intel_setup_irq_remapping()
583 	iommu->ir_msi_domain =  in intel_setup_irq_remapping()
584 		arch_create_remap_msi_irq_domain(iommu->ir_domain,  in intel_setup_irq_remapping()
586 						 iommu->seq_id);  in intel_setup_irq_remapping()
590 	iommu->ir_table = ir_table;  in intel_setup_irq_remapping()
596 	if (!iommu->qi) {  in intel_setup_irq_remapping()
600 		dmar_fault(-1, iommu);  in intel_setup_irq_remapping()
601 		dmar_disable_qi(iommu);  in intel_setup_irq_remapping()
603 		if (dmar_enable_qi(iommu)) {  in intel_setup_irq_remapping()
609 	init_ir_status(iommu);  in intel_setup_irq_remapping()
611 	if (ir_pre_enabled(iommu)) {  in intel_setup_irq_remapping()
614 				iommu->name);  in intel_setup_irq_remapping()
615 			clear_ir_pre_enabled(iommu);  in intel_setup_irq_remapping()
616 			iommu_disable_irq_remapping(iommu);  in intel_setup_irq_remapping()
617 		} else if (iommu_load_old_irte(iommu))  in intel_setup_irq_remapping()
619 			       iommu->name);  in intel_setup_irq_remapping()
622 				iommu->name);  in intel_setup_irq_remapping()
625 	iommu_set_irq_remapping(iommu, eim_mode);  in intel_setup_irq_remapping()
636 	iommu->ir_table  = NULL;  in intel_setup_irq_remapping()
641 static void intel_teardown_irq_remapping(struct intel_iommu *iommu)  in intel_teardown_irq_remapping()  argument
645 	if (iommu && iommu->ir_table) {  in intel_teardown_irq_remapping()
646 		if (iommu->ir_msi_domain) {  in intel_teardown_irq_remapping()
647 			fn = iommu->ir_msi_domain->fwnode;  in intel_teardown_irq_remapping()
649 			irq_domain_remove(iommu->ir_msi_domain);  in intel_teardown_irq_remapping()
651 			iommu->ir_msi_domain = NULL;  in intel_teardown_irq_remapping()
653 		if (iommu->ir_domain) {  in intel_teardown_irq_remapping()
654 			fn = iommu->ir_domain->fwnode;  in intel_teardown_irq_remapping()
656 			irq_domain_remove(iommu->ir_domain);  in intel_teardown_irq_remapping()
658 			iommu->ir_domain = NULL;  in intel_teardown_irq_remapping()
660 		free_pages((unsigned long)iommu->ir_table->base,  in intel_teardown_irq_remapping()
662 		bitmap_free(iommu->ir_table->bitmap);  in intel_teardown_irq_remapping()
663 		kfree(iommu->ir_table);  in intel_teardown_irq_remapping()
664 		iommu->ir_table = NULL;  in intel_teardown_irq_remapping()
671 static void iommu_disable_irq_remapping(struct intel_iommu *iommu)  in iommu_disable_irq_remapping()  argument
676 	if (!ecap_ir_support(iommu->ecap))  in iommu_disable_irq_remapping()
683 	qi_global_iec(iommu);  in iommu_disable_irq_remapping()
685 	raw_spin_lock_irqsave(&iommu->register_lock, flags);  in iommu_disable_irq_remapping()
687 	sts = readl(iommu->reg + DMAR_GSTS_REG);  in iommu_disable_irq_remapping()
691 	iommu->gcmd &= ~DMA_GCMD_IRE;  in iommu_disable_irq_remapping()
692 	writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);  in iommu_disable_irq_remapping()
694 	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,  in iommu_disable_irq_remapping()
698 	raw_spin_unlock_irqrestore(&iommu->register_lock, flags);  in iommu_disable_irq_remapping()
713 	struct intel_iommu *iommu;  in intel_cleanup_irq_remapping()  local
715 	for_each_iommu(iommu, drhd) {  in intel_cleanup_irq_remapping()
716 		if (ecap_ir_support(iommu->ecap)) {  in intel_cleanup_irq_remapping()
717 			iommu_disable_irq_remapping(iommu);  in intel_cleanup_irq_remapping()
718 			intel_teardown_irq_remapping(iommu);  in intel_cleanup_irq_remapping()
729 	struct intel_iommu *iommu;  in intel_prepare_irq_remapping()  local
754 	for_each_iommu(iommu, drhd)  in intel_prepare_irq_remapping()
755 		if (!ecap_ir_support(iommu->ecap))  in intel_prepare_irq_remapping()
767 	for_each_iommu(iommu, drhd) {  in intel_prepare_irq_remapping()
768 		if (eim && !ecap_eim_support(iommu->ecap)) {  in intel_prepare_irq_remapping()
769 			pr_info("%s does not support EIM\n", iommu->name);  in intel_prepare_irq_remapping()
779 	for_each_iommu(iommu, drhd) {  in intel_prepare_irq_remapping()
780 		if (intel_setup_irq_remapping(iommu)) {  in intel_prepare_irq_remapping()
782 			       iommu->name);  in intel_prepare_irq_remapping()
800 	struct intel_iommu *iommu;  in set_irq_posting_cap()  local
814 		for_each_iommu(iommu, drhd)  in set_irq_posting_cap()
815 			if (!cap_pi_support(iommu->cap)) {  in set_irq_posting_cap()
826 	struct intel_iommu *iommu;  in intel_enable_irq_remapping()  local
832 	for_each_iommu(iommu, drhd) {  in intel_enable_irq_remapping()
833 		if (!ir_pre_enabled(iommu))  in intel_enable_irq_remapping()
834 			iommu_enable_irq_remapping(iommu);  in intel_enable_irq_remapping()
855 				   struct intel_iommu *iommu,  in ir_parse_one_hpet_scope()  argument
878 		if (ir_hpet[count].iommu == iommu &&  in ir_parse_one_hpet_scope()
881 		else if (ir_hpet[count].iommu == NULL && free == -1)  in ir_parse_one_hpet_scope()
889 	ir_hpet[free].iommu = iommu;  in ir_parse_one_hpet_scope()
900 				     struct intel_iommu *iommu,  in ir_parse_one_ioapic_scope()  argument
923 		if (ir_ioapic[count].iommu == iommu &&  in ir_parse_one_ioapic_scope()
926 		else if (ir_ioapic[count].iommu == NULL && free == -1)  in ir_parse_one_ioapic_scope()
936 	ir_ioapic[free].iommu = iommu;  in ir_parse_one_ioapic_scope()
938 	pr_info("IOAPIC id %d under DRHD base  0x%Lx IOMMU %d\n",  in ir_parse_one_ioapic_scope()
939 		scope->enumeration_id, drhd->address, iommu->seq_id);  in ir_parse_one_ioapic_scope()
945 				      struct intel_iommu *iommu)  in ir_parse_ioapic_hpet_scope()  argument
959 			ret = ir_parse_one_ioapic_scope(scope, iommu, drhd);  in ir_parse_ioapic_hpet_scope()
961 			ret = ir_parse_one_hpet_scope(scope, iommu, drhd);  in ir_parse_ioapic_hpet_scope()
968 static void ir_remove_ioapic_hpet_scope(struct intel_iommu *iommu)  in ir_remove_ioapic_hpet_scope()  argument
973 		if (ir_hpet[i].iommu == iommu)  in ir_remove_ioapic_hpet_scope()
974 			ir_hpet[i].iommu = NULL;  in ir_remove_ioapic_hpet_scope()
977 		if (ir_ioapic[i].iommu == iommu)  in ir_remove_ioapic_hpet_scope()
978 			ir_ioapic[i].iommu = NULL;  in ir_remove_ioapic_hpet_scope()
988 	struct intel_iommu *iommu;  in parse_ioapics_under_ir()  local
992 	for_each_iommu(iommu, drhd) {  in parse_ioapics_under_ir()
995 		if (!ecap_ir_support(iommu->ecap))  in parse_ioapics_under_ir()
998 		ret = ir_parse_ioapic_hpet_scope(drhd->hdr, iommu);  in parse_ioapics_under_ir()
1011 			pr_err(FW_BUG "ioapic %d has no mapping iommu, "  in parse_ioapics_under_ir()
1039 	struct intel_iommu *iommu = NULL;  in disable_irq_remapping()  local
1044 	for_each_iommu(iommu, drhd) {  in disable_irq_remapping()
1045 		if (!ecap_ir_support(iommu->ecap))  in disable_irq_remapping()
1048 		iommu_disable_irq_remapping(iommu);  in disable_irq_remapping()
1062 	struct intel_iommu *iommu = NULL;  in reenable_irq_remapping()  local
1064 	for_each_iommu(iommu, drhd)  in reenable_irq_remapping()
1065 		if (iommu->qi)  in reenable_irq_remapping()
1066 			dmar_reenable_qi(iommu);  in reenable_irq_remapping()
1071 	for_each_iommu(iommu, drhd) {  in reenable_irq_remapping()
1072 		if (!ecap_ir_support(iommu->ecap))  in reenable_irq_remapping()
1075 		/* Set up interrupt remapping for iommu.*/  in reenable_irq_remapping()
1076 		iommu_set_irq_remapping(iommu, eim);  in reenable_irq_remapping()
1077 		iommu_enable_irq_remapping(iommu);  in reenable_irq_remapping()
1351 	struct intel_iommu *iommu = domain->host_data;  in intel_irq_remapping_alloc()  local
1358 	if (!info || !iommu)  in intel_irq_remapping_alloc()
1381 	index = alloc_irte(iommu, &data->irq_2_iommu, nr_irqs);  in intel_irq_remapping_alloc()
1457 static int dmar_ir_add(struct dmar_drhd_unit *dmaru, struct intel_iommu *iommu)  in dmar_ir_add()  argument
1462 	if (eim && !ecap_eim_support(iommu->ecap)) {  in dmar_ir_add()
1464 			iommu->reg_phys, iommu->ecap);  in dmar_ir_add()
1468 	if (ir_parse_ioapic_hpet_scope(dmaru->hdr, iommu)) {  in dmar_ir_add()
1470 			iommu->reg_phys);  in dmar_ir_add()
1474 	/* TODO: check all IOAPICs are covered by IOMMU */  in dmar_ir_add()
1477 	ret = intel_setup_irq_remapping(iommu);  in dmar_ir_add()
1480 		       iommu->name);  in dmar_ir_add()
1481 		intel_teardown_irq_remapping(iommu);  in dmar_ir_add()
1482 		ir_remove_ioapic_hpet_scope(iommu);  in dmar_ir_add()
1484 		iommu_enable_irq_remapping(iommu);  in dmar_ir_add()
1493 	struct intel_iommu *iommu = dmaru->iommu;  in dmar_ir_hotplug()  local
1497 	if (iommu == NULL)  in dmar_ir_hotplug()
1499 	if (!ecap_ir_support(iommu->ecap))  in dmar_ir_hotplug()
1502 	    !cap_pi_support(iommu->cap))  in dmar_ir_hotplug()
1506 		if (!iommu->ir_table)  in dmar_ir_hotplug()
1507 			ret = dmar_ir_add(dmaru, iommu);  in dmar_ir_hotplug()
1509 		if (iommu->ir_table) {  in dmar_ir_hotplug()
1510 			if (!bitmap_empty(iommu->ir_table->bitmap,  in dmar_ir_hotplug()
1514 				iommu_disable_irq_remapping(iommu);  in dmar_ir_hotplug()
1515 				intel_teardown_irq_remapping(iommu);  in dmar_ir_hotplug()
1516 				ir_remove_ioapic_hpet_scope(iommu);  in dmar_ir_hotplug()