23 #include "iommu.h"
33 	struct intel_iommu *iommu;  member
40 	struct intel_iommu *iommu;  member
47 	struct intel_iommu *iommu;  member
74  *	->iommu->register_lock
83 static void iommu_disable_irq_remapping(struct intel_iommu *iommu);
86 static bool ir_pre_enabled(struct intel_iommu *iommu)  in ir_pre_enabled()  argument
88 	return (iommu->flags & VTD_FLAG_IRQ_REMAP_PRE_ENABLED);  in ir_pre_enabled()
91 static void clear_ir_pre_enabled(struct intel_iommu *iommu)  in clear_ir_pre_enabled()  argument
93 	iommu->flags &= ~VTD_FLAG_IRQ_REMAP_PRE_ENABLED;  in clear_ir_pre_enabled()
96 static void init_ir_status(struct intel_iommu *iommu)  in init_ir_status()  argument
100 	gsts = readl(iommu->reg + DMAR_GSTS_REG);  in init_ir_status()
102 		iommu->flags |= VTD_FLAG_IRQ_REMAP_PRE_ENABLED;  in init_ir_status()
105 static int alloc_irte(struct intel_iommu *iommu,  in alloc_irte()  argument
108 	struct ir_table *table = iommu->ir_table;  in alloc_irte()
121 	if (mask > ecap_max_handle_mask(iommu->ecap)) {  in alloc_irte()
124 		       ecap_max_handle_mask(iommu->ecap));  in alloc_irte()
132 		pr_warn("IR%d: can't allocate an IRTE\n", iommu->seq_id);  in alloc_irte()
134 		irq_iommu->iommu = iommu;  in alloc_irte()
145 static int qi_flush_iec(struct intel_iommu *iommu, int index, int mask)  in qi_flush_iec()  argument
155 	return qi_submit_sync(iommu, &desc, 1, 0);  in qi_flush_iec()
161 	struct intel_iommu *iommu;  in modify_irte()  local
171 	iommu = irq_iommu->iommu;  in modify_irte()
174 	irte = &iommu->ir_table->base[index];  in modify_irte()
196 	__iommu_flush_cache(iommu, irte, sizeof(*irte));  in modify_irte()
198 	rc = qi_flush_iec(iommu, index, 0);  in modify_irte()
200 	/* Update iommu mode according to the IRTE mode */  in modify_irte()
212 		if (ir_hpet[i].id == hpet_id && ir_hpet[i].iommu)  in map_hpet_to_iommu()
213 			return ir_hpet[i].iommu;  in map_hpet_to_iommu()
223 		if (ir_ioapic[i].id == apic && ir_ioapic[i].iommu)  in map_ioapic_to_iommu()
224 			return ir_ioapic[i].iommu;  in map_ioapic_to_iommu()
233 	return drhd ? drhd->iommu->ir_msi_domain : NULL;  in map_dev_to_ir()
239 	struct intel_iommu *iommu;  in clear_entries()  local
245 	iommu = irq_iommu->iommu;  in clear_entries()
248 	start = iommu->ir_table->base + index;  in clear_entries()
255 	bitmap_release_region(iommu->ir_table->bitmap, index,  in clear_entries()
258 	return qi_flush_iec(iommu, index, irq_iommu->irte_mask);  in clear_entries()
318 		if (ir_ioapic[i].iommu && ir_ioapic[i].id == apic) {  in set_ioapic_sid()
345 		if (ir_hpet[i].iommu && ir_hpet[i].id == id) {  in set_hpet_sid()
431 static int iommu_load_old_irte(struct intel_iommu *iommu)  in iommu_load_old_irte()  argument
440 	irta = dmar_readq(iommu->reg + DMAR_IRTA_REG);  in iommu_load_old_irte()
454 	memcpy(iommu->ir_table->base, old_ir_table, size);  in iommu_load_old_irte()
456 	__iommu_flush_cache(iommu, iommu->ir_table->base, size);  in iommu_load_old_irte()
463 		if (iommu->ir_table->base[i].present)  in iommu_load_old_irte()
464 			bitmap_set(iommu->ir_table->bitmap, i, 1);  in iommu_load_old_irte()
473 static void iommu_set_irq_remapping(struct intel_iommu *iommu, int mode)  in iommu_set_irq_remapping()  argument
479 	addr = virt_to_phys((void *)iommu->ir_table->base);  in iommu_set_irq_remapping()
481 	raw_spin_lock_irqsave(&iommu->register_lock, flags);  in iommu_set_irq_remapping()
483 	dmar_writeq(iommu->reg + DMAR_IRTA_REG,  in iommu_set_irq_remapping()
487 	writel(iommu->gcmd | DMA_GCMD_SIRTP, iommu->reg + DMAR_GCMD_REG);  in iommu_set_irq_remapping()
489 	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,  in iommu_set_irq_remapping()
491 	raw_spin_unlock_irqrestore(&iommu->register_lock, flags);  in iommu_set_irq_remapping()
497 	if (!cap_esirtps(iommu->cap))  in iommu_set_irq_remapping()
498 		qi_global_iec(iommu);  in iommu_set_irq_remapping()
501 static void iommu_enable_irq_remapping(struct intel_iommu *iommu)  in iommu_enable_irq_remapping()  argument
506 	raw_spin_lock_irqsave(&iommu->register_lock, flags);  in iommu_enable_irq_remapping()
509 	iommu->gcmd |= DMA_GCMD_IRE;  in iommu_enable_irq_remapping()
510 	writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);  in iommu_enable_irq_remapping()
511 	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,  in iommu_enable_irq_remapping()
516 		iommu->gcmd &= ~DMA_GCMD_CFI;  in iommu_enable_irq_remapping()
517 		writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);  in iommu_enable_irq_remapping()
518 		IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,  in iommu_enable_irq_remapping()
532 	raw_spin_unlock_irqrestore(&iommu->register_lock, flags);  in iommu_enable_irq_remapping()
535 static int intel_setup_irq_remapping(struct intel_iommu *iommu)  in intel_setup_irq_remapping()  argument
542 	if (iommu->ir_table)  in intel_setup_irq_remapping()
549 	pages = alloc_pages_node(iommu->node, GFP_KERNEL | __GFP_ZERO,  in intel_setup_irq_remapping()
553 		       iommu->seq_id, INTR_REMAP_PAGE_ORDER);  in intel_setup_irq_remapping()
559 		pr_err("IR%d: failed to allocate bitmap\n", iommu->seq_id);  in intel_setup_irq_remapping()
563 	fn = irq_domain_alloc_named_id_fwnode("INTEL-IR", iommu->seq_id);  in intel_setup_irq_remapping()
567 	iommu->ir_domain =  in intel_setup_irq_remapping()
571 					    iommu);  in intel_setup_irq_remapping()
572 	if (!iommu->ir_domain) {  in intel_setup_irq_remapping()
573 		pr_err("IR%d: failed to allocate irqdomain\n", iommu->seq_id);  in intel_setup_irq_remapping()
576 	iommu->ir_msi_domain =  in intel_setup_irq_remapping()
577 		arch_create_remap_msi_irq_domain(iommu->ir_domain,  in intel_setup_irq_remapping()
579 						 iommu->seq_id);  in intel_setup_irq_remapping()
583 	iommu->ir_table = ir_table;  in intel_setup_irq_remapping()
589 	if (!iommu->qi) {  in intel_setup_irq_remapping()
593 		dmar_fault(-1, iommu);  in intel_setup_irq_remapping()
594 		dmar_disable_qi(iommu);  in intel_setup_irq_remapping()
596 		if (dmar_enable_qi(iommu)) {  in intel_setup_irq_remapping()
602 	init_ir_status(iommu);  in intel_setup_irq_remapping()
604 	if (ir_pre_enabled(iommu)) {  in intel_setup_irq_remapping()
607 				iommu->name);  in intel_setup_irq_remapping()
608 			clear_ir_pre_enabled(iommu);  in intel_setup_irq_remapping()
609 			iommu_disable_irq_remapping(iommu);  in intel_setup_irq_remapping()
610 		} else if (iommu_load_old_irte(iommu))  in intel_setup_irq_remapping()
612 			       iommu->name);  in intel_setup_irq_remapping()
615 				iommu->name);  in intel_setup_irq_remapping()
618 	iommu_set_irq_remapping(iommu, eim_mode);  in intel_setup_irq_remapping()
623 	if (iommu->ir_msi_domain)  in intel_setup_irq_remapping()
624 		irq_domain_remove(iommu->ir_msi_domain);  in intel_setup_irq_remapping()
625 	iommu->ir_msi_domain = NULL;  in intel_setup_irq_remapping()
626 	irq_domain_remove(iommu->ir_domain);  in intel_setup_irq_remapping()
627 	iommu->ir_domain = NULL;  in intel_setup_irq_remapping()
637 	iommu->ir_table  = NULL;  in intel_setup_irq_remapping()
642 static void intel_teardown_irq_remapping(struct intel_iommu *iommu)  in intel_teardown_irq_remapping()  argument
646 	if (iommu && iommu->ir_table) {  in intel_teardown_irq_remapping()
647 		if (iommu->ir_msi_domain) {  in intel_teardown_irq_remapping()
648 			fn = iommu->ir_msi_domain->fwnode;  in intel_teardown_irq_remapping()
650 			irq_domain_remove(iommu->ir_msi_domain);  in intel_teardown_irq_remapping()
652 			iommu->ir_msi_domain = NULL;  in intel_teardown_irq_remapping()
654 		if (iommu->ir_domain) {  in intel_teardown_irq_remapping()
655 			fn = iommu->ir_domain->fwnode;  in intel_teardown_irq_remapping()
657 			irq_domain_remove(iommu->ir_domain);  in intel_teardown_irq_remapping()
659 			iommu->ir_domain = NULL;  in intel_teardown_irq_remapping()
661 		free_pages((unsigned long)iommu->ir_table->base,  in intel_teardown_irq_remapping()
663 		bitmap_free(iommu->ir_table->bitmap);  in intel_teardown_irq_remapping()
664 		kfree(iommu->ir_table);  in intel_teardown_irq_remapping()
665 		iommu->ir_table = NULL;  in intel_teardown_irq_remapping()
672 static void iommu_disable_irq_remapping(struct intel_iommu *iommu)  in iommu_disable_irq_remapping()  argument
677 	if (!ecap_ir_support(iommu->ecap))  in iommu_disable_irq_remapping()
684 	if (!cap_esirtps(iommu->cap))  in iommu_disable_irq_remapping()
685 		qi_global_iec(iommu);  in iommu_disable_irq_remapping()
687 	raw_spin_lock_irqsave(&iommu->register_lock, flags);  in iommu_disable_irq_remapping()
689 	sts = readl(iommu->reg + DMAR_GSTS_REG);  in iommu_disable_irq_remapping()
693 	iommu->gcmd &= ~DMA_GCMD_IRE;  in iommu_disable_irq_remapping()
694 	writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);  in iommu_disable_irq_remapping()
696 	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,  in iommu_disable_irq_remapping()
700 	raw_spin_unlock_irqrestore(&iommu->register_lock, flags);  in iommu_disable_irq_remapping()
715 	struct intel_iommu *iommu;  in intel_cleanup_irq_remapping()  local
717 	for_each_iommu(iommu, drhd) {  in intel_cleanup_irq_remapping()
718 		if (ecap_ir_support(iommu->ecap)) {  in intel_cleanup_irq_remapping()
719 			iommu_disable_irq_remapping(iommu);  in intel_cleanup_irq_remapping()
720 			intel_teardown_irq_remapping(iommu);  in intel_cleanup_irq_remapping()
731 	struct intel_iommu *iommu;  in intel_prepare_irq_remapping()  local
759 	for_each_iommu(iommu, drhd)  in intel_prepare_irq_remapping()
760 		if (!ecap_ir_support(iommu->ecap))  in intel_prepare_irq_remapping()
772 	for_each_iommu(iommu, drhd) {  in intel_prepare_irq_remapping()
773 		if (eim && !ecap_eim_support(iommu->ecap)) {  in intel_prepare_irq_remapping()
774 			pr_info("%s does not support EIM\n", iommu->name);  in intel_prepare_irq_remapping()
784 	for_each_iommu(iommu, drhd) {  in intel_prepare_irq_remapping()
785 		if (intel_setup_irq_remapping(iommu)) {  in intel_prepare_irq_remapping()
787 			       iommu->name);  in intel_prepare_irq_remapping()
805 	struct intel_iommu *iommu;  in set_irq_posting_cap()  local
819 		for_each_iommu(iommu, drhd)  in set_irq_posting_cap()
820 			if (!cap_pi_support(iommu->cap)) {  in set_irq_posting_cap()
831 	struct intel_iommu *iommu;  in intel_enable_irq_remapping()  local
837 	for_each_iommu(iommu, drhd) {  in intel_enable_irq_remapping()
838 		if (!ir_pre_enabled(iommu))  in intel_enable_irq_remapping()
839 			iommu_enable_irq_remapping(iommu);  in intel_enable_irq_remapping()
860 				   struct intel_iommu *iommu,  in ir_parse_one_hpet_scope()  argument
883 		if (ir_hpet[count].iommu == iommu &&  in ir_parse_one_hpet_scope()
886 		else if (ir_hpet[count].iommu == NULL && free == -1)  in ir_parse_one_hpet_scope()
894 	ir_hpet[free].iommu = iommu;  in ir_parse_one_hpet_scope()
905 				     struct intel_iommu *iommu,  in ir_parse_one_ioapic_scope()  argument
928 		if (ir_ioapic[count].iommu == iommu &&  in ir_parse_one_ioapic_scope()
931 		else if (ir_ioapic[count].iommu == NULL && free == -1)  in ir_parse_one_ioapic_scope()
941 	ir_ioapic[free].iommu = iommu;  in ir_parse_one_ioapic_scope()
943 	pr_info("IOAPIC id %d under DRHD base  0x%Lx IOMMU %d\n",  in ir_parse_one_ioapic_scope()
944 		scope->enumeration_id, drhd->address, iommu->seq_id);  in ir_parse_one_ioapic_scope()
950 				      struct intel_iommu *iommu)  in ir_parse_ioapic_hpet_scope()  argument
964 			ret = ir_parse_one_ioapic_scope(scope, iommu, drhd);  in ir_parse_ioapic_hpet_scope()
966 			ret = ir_parse_one_hpet_scope(scope, iommu, drhd);  in ir_parse_ioapic_hpet_scope()
973 static void ir_remove_ioapic_hpet_scope(struct intel_iommu *iommu)  in ir_remove_ioapic_hpet_scope()  argument
978 		if (ir_hpet[i].iommu == iommu)  in ir_remove_ioapic_hpet_scope()
979 			ir_hpet[i].iommu = NULL;  in ir_remove_ioapic_hpet_scope()
982 		if (ir_ioapic[i].iommu == iommu)  in ir_remove_ioapic_hpet_scope()
983 			ir_ioapic[i].iommu = NULL;  in ir_remove_ioapic_hpet_scope()
993 	struct intel_iommu *iommu;  in parse_ioapics_under_ir()  local
997 	for_each_iommu(iommu, drhd) {  in parse_ioapics_under_ir()
1000 		if (!ecap_ir_support(iommu->ecap))  in parse_ioapics_under_ir()
1003 		ret = ir_parse_ioapic_hpet_scope(drhd->hdr, iommu);  in parse_ioapics_under_ir()
1016 			pr_err(FW_BUG "ioapic %d has no mapping iommu, "  in parse_ioapics_under_ir()
1044 	struct intel_iommu *iommu = NULL;  in disable_irq_remapping()  local
1049 	for_each_iommu(iommu, drhd) {  in disable_irq_remapping()
1050 		if (!ecap_ir_support(iommu->ecap))  in disable_irq_remapping()
1053 		iommu_disable_irq_remapping(iommu);  in disable_irq_remapping()
1067 	struct intel_iommu *iommu = NULL;  in reenable_irq_remapping()  local
1069 	for_each_iommu(iommu, drhd)  in reenable_irq_remapping()
1070 		if (iommu->qi)  in reenable_irq_remapping()
1071 			dmar_reenable_qi(iommu);  in reenable_irq_remapping()
1076 	for_each_iommu(iommu, drhd) {  in reenable_irq_remapping()
1077 		if (!ecap_ir_support(iommu->ecap))  in reenable_irq_remapping()
1080 		/* Set up interrupt remapping for iommu.*/  in reenable_irq_remapping()
1081 		iommu_set_irq_remapping(iommu, eim);  in reenable_irq_remapping()
1082 		iommu_enable_irq_remapping(iommu);  in reenable_irq_remapping()
1328 	struct intel_iommu *iommu = domain->host_data;  in intel_irq_remapping_alloc()  local
1335 	if (!info || !iommu)  in intel_irq_remapping_alloc()
1358 	index = alloc_irte(iommu, &data->irq_2_iommu, nr_irqs);  in intel_irq_remapping_alloc()
1430 	struct intel_iommu *iommu = NULL;  in intel_irq_remapping_select()  local
1433 		iommu = map_ioapic_to_iommu(fwspec->param[0]);  in intel_irq_remapping_select()
1435 		iommu = map_hpet_to_iommu(fwspec->param[0]);  in intel_irq_remapping_select()
1437 	return iommu && d == iommu->ir_domain;  in intel_irq_remapping_select()
1451 static int dmar_ir_add(struct dmar_drhd_unit *dmaru, struct intel_iommu *iommu)  in dmar_ir_add()  argument
1456 	ret = intel_cap_audit(CAP_AUDIT_HOTPLUG_IRQR, iommu);  in dmar_ir_add()
1460 	if (eim && !ecap_eim_support(iommu->ecap)) {  in dmar_ir_add()
1462 			iommu->reg_phys, iommu->ecap);  in dmar_ir_add()
1466 	if (ir_parse_ioapic_hpet_scope(dmaru->hdr, iommu)) {  in dmar_ir_add()
1468 			iommu->reg_phys);  in dmar_ir_add()
1472 	/* TODO: check all IOAPICs are covered by IOMMU */  in dmar_ir_add()
1475 	ret = intel_setup_irq_remapping(iommu);  in dmar_ir_add()
1478 		       iommu->name);  in dmar_ir_add()
1479 		intel_teardown_irq_remapping(iommu);  in dmar_ir_add()
1480 		ir_remove_ioapic_hpet_scope(iommu);  in dmar_ir_add()
1482 		iommu_enable_irq_remapping(iommu);  in dmar_ir_add()
1491 	struct intel_iommu *iommu = dmaru->iommu;  in dmar_ir_hotplug()  local
1495 	if (iommu == NULL)  in dmar_ir_hotplug()
1497 	if (!ecap_ir_support(iommu->ecap))  in dmar_ir_hotplug()
1500 	    !cap_pi_support(iommu->cap))  in dmar_ir_hotplug()
1504 		if (!iommu->ir_table)  in dmar_ir_hotplug()
1505 			ret = dmar_ir_add(dmaru, iommu);  in dmar_ir_hotplug()
1507 		if (iommu->ir_table) {  in dmar_ir_hotplug()
1508 			if (!bitmap_empty(iommu->ir_table->bitmap,  in dmar_ir_hotplug()
1512 				iommu_disable_irq_remapping(iommu);  in dmar_ir_hotplug()
1513 				intel_teardown_irq_remapping(iommu);  in dmar_ir_hotplug()
1514 				ir_remove_ioapic_hpet_scope(iommu);  in dmar_ir_hotplug()