13 #include <linux/intel-iommu.h>
33 	struct intel_iommu *iommu;  member
40 	struct intel_iommu *iommu;  member
47 	struct intel_iommu *iommu;  member
74  *	->iommu->register_lock
83 static void iommu_disable_irq_remapping(struct intel_iommu *iommu);
86 static bool ir_pre_enabled(struct intel_iommu *iommu)  in ir_pre_enabled()  argument
88 	return (iommu->flags & VTD_FLAG_IRQ_REMAP_PRE_ENABLED);  in ir_pre_enabled()
91 static void clear_ir_pre_enabled(struct intel_iommu *iommu)  in clear_ir_pre_enabled()  argument
93 	iommu->flags &= ~VTD_FLAG_IRQ_REMAP_PRE_ENABLED;  in clear_ir_pre_enabled()
96 static void init_ir_status(struct intel_iommu *iommu)  in init_ir_status()  argument
100 	gsts = readl(iommu->reg + DMAR_GSTS_REG);  in init_ir_status()
102 		iommu->flags |= VTD_FLAG_IRQ_REMAP_PRE_ENABLED;  in init_ir_status()
105 static int alloc_irte(struct intel_iommu *iommu,  in alloc_irte()  argument
108 	struct ir_table *table = iommu->ir_table;  in alloc_irte()
121 	if (mask > ecap_max_handle_mask(iommu->ecap)) {  in alloc_irte()
124 		       ecap_max_handle_mask(iommu->ecap));  in alloc_irte()
132 		pr_warn("IR%d: can't allocate an IRTE\n", iommu->seq_id);  in alloc_irte()
134 		irq_iommu->iommu = iommu;  in alloc_irte()
145 static int qi_flush_iec(struct intel_iommu *iommu, int index, int mask)  in qi_flush_iec()  argument
155 	return qi_submit_sync(iommu, &desc, 1, 0);  in qi_flush_iec()
161 	struct intel_iommu *iommu;  in modify_irte()  local
171 	iommu = irq_iommu->iommu;  in modify_irte()
174 	irte = &iommu->ir_table->base[index];  in modify_irte()
196 	__iommu_flush_cache(iommu, irte, sizeof(*irte));  in modify_irte()
198 	rc = qi_flush_iec(iommu, index, 0);  in modify_irte()
200 	/* Update iommu mode according to the IRTE mode */  in modify_irte()
212 		if (ir_hpet[i].id == hpet_id && ir_hpet[i].iommu)  in map_hpet_to_iommu()
213 			return ir_hpet[i].iommu;  in map_hpet_to_iommu()
223 		if (ir_ioapic[i].id == apic && ir_ioapic[i].iommu)  in map_ioapic_to_iommu()
224 			return ir_ioapic[i].iommu;  in map_ioapic_to_iommu()
233 	return drhd ? drhd->iommu->ir_msi_domain : NULL;  in map_dev_to_ir()
239 	struct intel_iommu *iommu;  in clear_entries()  local
245 	iommu = irq_iommu->iommu;  in clear_entries()
248 	start = iommu->ir_table->base + index;  in clear_entries()
255 	bitmap_release_region(iommu->ir_table->bitmap, index,  in clear_entries()
258 	return qi_flush_iec(iommu, index, irq_iommu->irte_mask);  in clear_entries()
318 		if (ir_ioapic[i].iommu && ir_ioapic[i].id == apic) {  in set_ioapic_sid()
345 		if (ir_hpet[i].iommu && ir_hpet[i].id == id) {  in set_hpet_sid()
431 static int iommu_load_old_irte(struct intel_iommu *iommu)  in iommu_load_old_irte()  argument
440 	irta = dmar_readq(iommu->reg + DMAR_IRTA_REG);  in iommu_load_old_irte()
454 	memcpy(iommu->ir_table->base, old_ir_table, size);  in iommu_load_old_irte()
456 	__iommu_flush_cache(iommu, iommu->ir_table->base, size);  in iommu_load_old_irte()
463 		if (iommu->ir_table->base[i].present)  in iommu_load_old_irte()
464 			bitmap_set(iommu->ir_table->bitmap, i, 1);  in iommu_load_old_irte()
473 static void iommu_set_irq_remapping(struct intel_iommu *iommu, int mode)  in iommu_set_irq_remapping()  argument
479 	addr = virt_to_phys((void *)iommu->ir_table->base);  in iommu_set_irq_remapping()
481 	raw_spin_lock_irqsave(&iommu->register_lock, flags);  in iommu_set_irq_remapping()
483 	dmar_writeq(iommu->reg + DMAR_IRTA_REG,  in iommu_set_irq_remapping()
487 	writel(iommu->gcmd | DMA_GCMD_SIRTP, iommu->reg + DMAR_GCMD_REG);  in iommu_set_irq_remapping()
489 	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,  in iommu_set_irq_remapping()
491 	raw_spin_unlock_irqrestore(&iommu->register_lock, flags);  in iommu_set_irq_remapping()
497 	qi_global_iec(iommu);  in iommu_set_irq_remapping()
500 static void iommu_enable_irq_remapping(struct intel_iommu *iommu)  in iommu_enable_irq_remapping()  argument
505 	raw_spin_lock_irqsave(&iommu->register_lock, flags);  in iommu_enable_irq_remapping()
508 	iommu->gcmd |= DMA_GCMD_IRE;  in iommu_enable_irq_remapping()
509 	writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);  in iommu_enable_irq_remapping()
510 	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,  in iommu_enable_irq_remapping()
515 		iommu->gcmd &= ~DMA_GCMD_CFI;  in iommu_enable_irq_remapping()
516 		writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);  in iommu_enable_irq_remapping()
517 		IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,  in iommu_enable_irq_remapping()
531 	raw_spin_unlock_irqrestore(&iommu->register_lock, flags);  in iommu_enable_irq_remapping()
534 static int intel_setup_irq_remapping(struct intel_iommu *iommu)  in intel_setup_irq_remapping()  argument
541 	if (iommu->ir_table)  in intel_setup_irq_remapping()
548 	pages = alloc_pages_node(iommu->node, GFP_KERNEL | __GFP_ZERO,  in intel_setup_irq_remapping()
552 		       iommu->seq_id, INTR_REMAP_PAGE_ORDER);  in intel_setup_irq_remapping()
558 		pr_err("IR%d: failed to allocate bitmap\n", iommu->seq_id);  in intel_setup_irq_remapping()
562 	fn = irq_domain_alloc_named_id_fwnode("INTEL-IR", iommu->seq_id);  in intel_setup_irq_remapping()
566 	iommu->ir_domain =  in intel_setup_irq_remapping()
570 					    iommu);  in intel_setup_irq_remapping()
571 	if (!iommu->ir_domain) {  in intel_setup_irq_remapping()
573 		pr_err("IR%d: failed to allocate irqdomain\n", iommu->seq_id);  in intel_setup_irq_remapping()
576 	iommu->ir_msi_domain =  in intel_setup_irq_remapping()
577 		arch_create_remap_msi_irq_domain(iommu->ir_domain,  in intel_setup_irq_remapping()
579 						 iommu->seq_id);  in intel_setup_irq_remapping()
583 	iommu->ir_table = ir_table;  in intel_setup_irq_remapping()
589 	if (!iommu->qi) {  in intel_setup_irq_remapping()
593 		dmar_fault(-1, iommu);  in intel_setup_irq_remapping()
594 		dmar_disable_qi(iommu);  in intel_setup_irq_remapping()
596 		if (dmar_enable_qi(iommu)) {  in intel_setup_irq_remapping()
602 	init_ir_status(iommu);  in intel_setup_irq_remapping()
604 	if (ir_pre_enabled(iommu)) {  in intel_setup_irq_remapping()
607 				iommu->name);  in intel_setup_irq_remapping()
608 			clear_ir_pre_enabled(iommu);  in intel_setup_irq_remapping()
609 			iommu_disable_irq_remapping(iommu);  in intel_setup_irq_remapping()
610 		} else if (iommu_load_old_irte(iommu))  in intel_setup_irq_remapping()
612 			       iommu->name);  in intel_setup_irq_remapping()
615 				iommu->name);  in intel_setup_irq_remapping()
618 	iommu_set_irq_remapping(iommu, eim_mode);  in intel_setup_irq_remapping()
629 	iommu->ir_table  = NULL;  in intel_setup_irq_remapping()
634 static void intel_teardown_irq_remapping(struct intel_iommu *iommu)  in intel_teardown_irq_remapping()  argument
638 	if (iommu && iommu->ir_table) {  in intel_teardown_irq_remapping()
639 		if (iommu->ir_msi_domain) {  in intel_teardown_irq_remapping()
640 			fn = iommu->ir_msi_domain->fwnode;  in intel_teardown_irq_remapping()
642 			irq_domain_remove(iommu->ir_msi_domain);  in intel_teardown_irq_remapping()
644 			iommu->ir_msi_domain = NULL;  in intel_teardown_irq_remapping()
646 		if (iommu->ir_domain) {  in intel_teardown_irq_remapping()
647 			fn = iommu->ir_domain->fwnode;  in intel_teardown_irq_remapping()
649 			irq_domain_remove(iommu->ir_domain);  in intel_teardown_irq_remapping()
651 			iommu->ir_domain = NULL;  in intel_teardown_irq_remapping()
653 		free_pages((unsigned long)iommu->ir_table->base,  in intel_teardown_irq_remapping()
655 		bitmap_free(iommu->ir_table->bitmap);  in intel_teardown_irq_remapping()
656 		kfree(iommu->ir_table);  in intel_teardown_irq_remapping()
657 		iommu->ir_table = NULL;  in intel_teardown_irq_remapping()
664 static void iommu_disable_irq_remapping(struct intel_iommu *iommu)  in iommu_disable_irq_remapping()  argument
669 	if (!ecap_ir_support(iommu->ecap))  in iommu_disable_irq_remapping()
676 	qi_global_iec(iommu);  in iommu_disable_irq_remapping()
678 	raw_spin_lock_irqsave(&iommu->register_lock, flags);  in iommu_disable_irq_remapping()
680 	sts = readl(iommu->reg + DMAR_GSTS_REG);  in iommu_disable_irq_remapping()
684 	iommu->gcmd &= ~DMA_GCMD_IRE;  in iommu_disable_irq_remapping()
685 	writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);  in iommu_disable_irq_remapping()
687 	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,  in iommu_disable_irq_remapping()
691 	raw_spin_unlock_irqrestore(&iommu->register_lock, flags);  in iommu_disable_irq_remapping()
706 	struct intel_iommu *iommu;  in intel_cleanup_irq_remapping()  local
708 	for_each_iommu(iommu, drhd) {  in intel_cleanup_irq_remapping()
709 		if (ecap_ir_support(iommu->ecap)) {  in intel_cleanup_irq_remapping()
710 			iommu_disable_irq_remapping(iommu);  in intel_cleanup_irq_remapping()
711 			intel_teardown_irq_remapping(iommu);  in intel_cleanup_irq_remapping()
722 	struct intel_iommu *iommu;  in intel_prepare_irq_remapping()  local
750 	for_each_iommu(iommu, drhd)  in intel_prepare_irq_remapping()
751 		if (!ecap_ir_support(iommu->ecap))  in intel_prepare_irq_remapping()
763 	for_each_iommu(iommu, drhd) {  in intel_prepare_irq_remapping()
764 		if (eim && !ecap_eim_support(iommu->ecap)) {  in intel_prepare_irq_remapping()
765 			pr_info("%s does not support EIM\n", iommu->name);  in intel_prepare_irq_remapping()
775 	for_each_iommu(iommu, drhd) {  in intel_prepare_irq_remapping()
776 		if (intel_setup_irq_remapping(iommu)) {  in intel_prepare_irq_remapping()
778 			       iommu->name);  in intel_prepare_irq_remapping()
796 	struct intel_iommu *iommu;  in set_irq_posting_cap()  local
810 		for_each_iommu(iommu, drhd)  in set_irq_posting_cap()
811 			if (!cap_pi_support(iommu->cap)) {  in set_irq_posting_cap()
822 	struct intel_iommu *iommu;  in intel_enable_irq_remapping()  local
828 	for_each_iommu(iommu, drhd) {  in intel_enable_irq_remapping()
829 		if (!ir_pre_enabled(iommu))  in intel_enable_irq_remapping()
830 			iommu_enable_irq_remapping(iommu);  in intel_enable_irq_remapping()
851 				   struct intel_iommu *iommu,  in ir_parse_one_hpet_scope()  argument
874 		if (ir_hpet[count].iommu == iommu &&  in ir_parse_one_hpet_scope()
877 		else if (ir_hpet[count].iommu == NULL && free == -1)  in ir_parse_one_hpet_scope()
885 	ir_hpet[free].iommu = iommu;  in ir_parse_one_hpet_scope()
896 				     struct intel_iommu *iommu,  in ir_parse_one_ioapic_scope()  argument
919 		if (ir_ioapic[count].iommu == iommu &&  in ir_parse_one_ioapic_scope()
922 		else if (ir_ioapic[count].iommu == NULL && free == -1)  in ir_parse_one_ioapic_scope()
932 	ir_ioapic[free].iommu = iommu;  in ir_parse_one_ioapic_scope()
934 	pr_info("IOAPIC id %d under DRHD base  0x%Lx IOMMU %d\n",  in ir_parse_one_ioapic_scope()
935 		scope->enumeration_id, drhd->address, iommu->seq_id);  in ir_parse_one_ioapic_scope()
941 				      struct intel_iommu *iommu)  in ir_parse_ioapic_hpet_scope()  argument
955 			ret = ir_parse_one_ioapic_scope(scope, iommu, drhd);  in ir_parse_ioapic_hpet_scope()
957 			ret = ir_parse_one_hpet_scope(scope, iommu, drhd);  in ir_parse_ioapic_hpet_scope()
964 static void ir_remove_ioapic_hpet_scope(struct intel_iommu *iommu)  in ir_remove_ioapic_hpet_scope()  argument
969 		if (ir_hpet[i].iommu == iommu)  in ir_remove_ioapic_hpet_scope()
970 			ir_hpet[i].iommu = NULL;  in ir_remove_ioapic_hpet_scope()
973 		if (ir_ioapic[i].iommu == iommu)  in ir_remove_ioapic_hpet_scope()
974 			ir_ioapic[i].iommu = NULL;  in ir_remove_ioapic_hpet_scope()
984 	struct intel_iommu *iommu;  in parse_ioapics_under_ir()  local
988 	for_each_iommu(iommu, drhd) {  in parse_ioapics_under_ir()
991 		if (!ecap_ir_support(iommu->ecap))  in parse_ioapics_under_ir()
994 		ret = ir_parse_ioapic_hpet_scope(drhd->hdr, iommu);  in parse_ioapics_under_ir()
1007 			pr_err(FW_BUG "ioapic %d has no mapping iommu, "  in parse_ioapics_under_ir()
1035 	struct intel_iommu *iommu = NULL;  in disable_irq_remapping()  local
1040 	for_each_iommu(iommu, drhd) {  in disable_irq_remapping()
1041 		if (!ecap_ir_support(iommu->ecap))  in disable_irq_remapping()
1044 		iommu_disable_irq_remapping(iommu);  in disable_irq_remapping()
1058 	struct intel_iommu *iommu = NULL;  in reenable_irq_remapping()  local
1060 	for_each_iommu(iommu, drhd)  in reenable_irq_remapping()
1061 		if (iommu->qi)  in reenable_irq_remapping()
1062 			dmar_reenable_qi(iommu);  in reenable_irq_remapping()
1067 	for_each_iommu(iommu, drhd) {  in reenable_irq_remapping()
1068 		if (!ecap_ir_support(iommu->ecap))  in reenable_irq_remapping()
1071 		/* Set up interrupt remapping for iommu.*/  in reenable_irq_remapping()
1072 		iommu_set_irq_remapping(iommu, eim);  in reenable_irq_remapping()
1073 		iommu_enable_irq_remapping(iommu);  in reenable_irq_remapping()
1319 	struct intel_iommu *iommu = domain->host_data;  in intel_irq_remapping_alloc()  local
1326 	if (!info || !iommu)  in intel_irq_remapping_alloc()
1349 	index = alloc_irte(iommu, &data->irq_2_iommu, nr_irqs);  in intel_irq_remapping_alloc()
1421 	struct intel_iommu *iommu = NULL;  in intel_irq_remapping_select()  local
1424 		iommu = map_ioapic_to_iommu(fwspec->param[0]);  in intel_irq_remapping_select()
1426 		iommu = map_hpet_to_iommu(fwspec->param[0]);  in intel_irq_remapping_select()
1428 	return iommu && d == iommu->ir_domain;  in intel_irq_remapping_select()
1442 static int dmar_ir_add(struct dmar_drhd_unit *dmaru, struct intel_iommu *iommu)  in dmar_ir_add()  argument
1447 	ret = intel_cap_audit(CAP_AUDIT_HOTPLUG_IRQR, iommu);  in dmar_ir_add()
1451 	if (eim && !ecap_eim_support(iommu->ecap)) {  in dmar_ir_add()
1453 			iommu->reg_phys, iommu->ecap);  in dmar_ir_add()
1457 	if (ir_parse_ioapic_hpet_scope(dmaru->hdr, iommu)) {  in dmar_ir_add()
1459 			iommu->reg_phys);  in dmar_ir_add()
1463 	/* TODO: check all IOAPICs are covered by IOMMU */  in dmar_ir_add()
1466 	ret = intel_setup_irq_remapping(iommu);  in dmar_ir_add()
1469 		       iommu->name);  in dmar_ir_add()
1470 		intel_teardown_irq_remapping(iommu);  in dmar_ir_add()
1471 		ir_remove_ioapic_hpet_scope(iommu);  in dmar_ir_add()
1473 		iommu_enable_irq_remapping(iommu);  in dmar_ir_add()
1482 	struct intel_iommu *iommu = dmaru->iommu;  in dmar_ir_hotplug()  local
1486 	if (iommu == NULL)  in dmar_ir_hotplug()
1488 	if (!ecap_ir_support(iommu->ecap))  in dmar_ir_hotplug()
1491 	    !cap_pi_support(iommu->cap))  in dmar_ir_hotplug()
1495 		if (!iommu->ir_table)  in dmar_ir_hotplug()
1496 			ret = dmar_ir_add(dmaru, iommu);  in dmar_ir_hotplug()
1498 		if (iommu->ir_table) {  in dmar_ir_hotplug()
1499 			if (!bitmap_empty(iommu->ir_table->bitmap,  in dmar_ir_hotplug()
1503 				iommu_disable_irq_remapping(iommu);  in dmar_ir_hotplug()
1504 				intel_teardown_irq_remapping(iommu);  in dmar_ir_hotplug()
1505 				ir_remove_ioapic_hpet_scope(iommu);  in dmar_ir_hotplug()