67 static void free_iommu(struct intel_iommu *iommu);
461 	if (dmaru->iommu)  in dmar_free_drhd()
462 		free_iommu(dmaru->iommu);  in dmar_free_drhd()
501 			drhd->iommu->node = node;  in dmar_parse_one_rhsa()
939 		x86_init.iommu.iommu_init = intel_iommu_init;  in detect_intel_iommu()
952 static void unmap_iommu(struct intel_iommu *iommu)  in unmap_iommu()  argument
954 	iounmap(iommu->reg);  in unmap_iommu()
955 	release_mem_region(iommu->reg_phys, iommu->reg_size);  in unmap_iommu()
966 static int map_iommu(struct intel_iommu *iommu, struct dmar_drhd_unit *drhd)  in map_iommu()  argument
971 	iommu->reg_phys = phys_addr;  in map_iommu()
972 	iommu->reg_size = drhd->reg_size;  in map_iommu()
974 	if (!request_mem_region(iommu->reg_phys, iommu->reg_size, iommu->name)) {  in map_iommu()
980 	iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);  in map_iommu()
981 	if (!iommu->reg) {  in map_iommu()
987 	iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);  in map_iommu()
988 	iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);  in map_iommu()
990 	if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {  in map_iommu()
997 	map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),  in map_iommu()
998 			 cap_max_fault_reg_offset(iommu->cap));  in map_iommu()
1000 	if (map_size > iommu->reg_size) {  in map_iommu()
1001 		iounmap(iommu->reg);  in map_iommu()
1002 		release_mem_region(iommu->reg_phys, iommu->reg_size);  in map_iommu()
1003 		iommu->reg_size = map_size;  in map_iommu()
1004 		if (!request_mem_region(iommu->reg_phys, iommu->reg_size,  in map_iommu()
1005 					iommu->name)) {  in map_iommu()
1010 		iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);  in map_iommu()
1011 		if (!iommu->reg) {  in map_iommu()
1018 	if (cap_ecmds(iommu->cap)) {  in map_iommu()
1022 			iommu->ecmdcap[i] = dmar_readq(iommu->reg + DMAR_ECCAP_REG +  in map_iommu()
1031 	iounmap(iommu->reg);  in map_iommu()
1033 	release_mem_region(iommu->reg_phys, iommu->reg_size);  in map_iommu()
1040 	struct intel_iommu *iommu;  in alloc_iommu()  local
1051 	iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);  in alloc_iommu()
1052 	if (!iommu)  in alloc_iommu()
1055 	iommu->seq_id = ida_alloc_range(&dmar_seq_ids, 0,  in alloc_iommu()
1057 	if (iommu->seq_id < 0) {  in alloc_iommu()
1059 		err = iommu->seq_id;  in alloc_iommu()
1062 	sprintf(iommu->name, "dmar%d", iommu->seq_id);  in alloc_iommu()
1064 	err = map_iommu(iommu, drhd);  in alloc_iommu()
1066 		pr_err("Failed to map %s\n", iommu->name);  in alloc_iommu()
1071 	if (!cap_sagaw(iommu->cap) &&  in alloc_iommu()
1072 	    (!ecap_smts(iommu->ecap) || ecap_slts(iommu->ecap))) {  in alloc_iommu()
1074 			iommu->name);  in alloc_iommu()
1079 		agaw = iommu_calculate_agaw(iommu);  in alloc_iommu()
1082 			       iommu->seq_id);  in alloc_iommu()
1087 		msagaw = iommu_calculate_max_sagaw(iommu);  in alloc_iommu()
1090 			       iommu->seq_id);  in alloc_iommu()
1095 	iommu->agaw = agaw;  in alloc_iommu()
1096 	iommu->msagaw = msagaw;  in alloc_iommu()
1097 	iommu->segment = drhd->segment;  in alloc_iommu()
1099 	iommu->node = NUMA_NO_NODE;  in alloc_iommu()
1101 	ver = readl(iommu->reg + DMAR_VER_REG);  in alloc_iommu()
1103 		iommu->name,  in alloc_iommu()
1106 		(unsigned long long)iommu->cap,  in alloc_iommu()
1107 		(unsigned long long)iommu->ecap);  in alloc_iommu()
1110 	sts = readl(iommu->reg + DMAR_GSTS_REG);  in alloc_iommu()
1112 		iommu->gcmd |= DMA_GCMD_IRE;  in alloc_iommu()
1114 		iommu->gcmd |= DMA_GCMD_TE;  in alloc_iommu()
1116 		iommu->gcmd |= DMA_GCMD_QIE;  in alloc_iommu()
1118 	if (alloc_iommu_pmu(iommu))  in alloc_iommu()
1119 		pr_debug("Cannot alloc PMU for iommu (seq_id = %d)\n", iommu->seq_id);  in alloc_iommu()
1121 	raw_spin_lock_init(&iommu->register_lock);  in alloc_iommu()
1127 	if (pasid_supported(iommu))  in alloc_iommu()
1128 		iommu->iommu.max_pasids = 2UL << ecap_pss(iommu->ecap);  in alloc_iommu()
1136 		err = iommu_device_sysfs_add(&iommu->iommu, NULL,  in alloc_iommu()
1138 					     "%s", iommu->name);  in alloc_iommu()
1142 		err = iommu_device_register(&iommu->iommu, &intel_iommu_ops, NULL);  in alloc_iommu()
1146 		iommu_pmu_register(iommu);  in alloc_iommu()
1149 	drhd->iommu = iommu;  in alloc_iommu()
1150 	iommu->drhd = drhd;  in alloc_iommu()
1155 	iommu_device_sysfs_remove(&iommu->iommu);  in alloc_iommu()
1157 	free_iommu_pmu(iommu);  in alloc_iommu()
1158 	unmap_iommu(iommu);  in alloc_iommu()
1160 	ida_free(&dmar_seq_ids, iommu->seq_id);  in alloc_iommu()
1162 	kfree(iommu);  in alloc_iommu()
1166 static void free_iommu(struct intel_iommu *iommu)  in free_iommu()  argument
1168 	if (intel_iommu_enabled && !iommu->drhd->ignored) {  in free_iommu()
1169 		iommu_pmu_unregister(iommu);  in free_iommu()
1170 		iommu_device_unregister(&iommu->iommu);  in free_iommu()
1171 		iommu_device_sysfs_remove(&iommu->iommu);  in free_iommu()
1174 	free_iommu_pmu(iommu);  in free_iommu()
1176 	if (iommu->irq) {  in free_iommu()
1177 		if (iommu->pr_irq) {  in free_iommu()
1178 			free_irq(iommu->pr_irq, iommu);  in free_iommu()
1179 			dmar_free_hwirq(iommu->pr_irq);  in free_iommu()
1180 			iommu->pr_irq = 0;  in free_iommu()
1182 		free_irq(iommu->irq, iommu);  in free_iommu()
1183 		dmar_free_hwirq(iommu->irq);  in free_iommu()
1184 		iommu->irq = 0;  in free_iommu()
1187 	if (iommu->qi) {  in free_iommu()
1188 		free_page((unsigned long)iommu->qi->desc);  in free_iommu()
1189 		kfree(iommu->qi->desc_status);  in free_iommu()
1190 		kfree(iommu->qi);  in free_iommu()
1193 	if (iommu->reg)  in free_iommu()
1194 		unmap_iommu(iommu);  in free_iommu()
1196 	ida_free(&dmar_seq_ids, iommu->seq_id);  in free_iommu()
1197 	kfree(iommu);  in free_iommu()
1239 static void qi_dump_fault(struct intel_iommu *iommu, u32 fault)  in qi_dump_fault()  argument
1241 	unsigned int head = dmar_readl(iommu->reg + DMAR_IQH_REG);  in qi_dump_fault()
1242 	u64 iqe_err = dmar_readq(iommu->reg + DMAR_IQER_REG);  in qi_dump_fault()
1243 	struct qi_desc *desc = iommu->qi->desc + head;  in qi_dump_fault()
1260 	head = ((head >> qi_shift(iommu)) + QI_LENGTH - 1) % QI_LENGTH;  in qi_dump_fault()
1261 	head <<= qi_shift(iommu);  in qi_dump_fault()
1262 	desc = iommu->qi->desc + head;  in qi_dump_fault()
1270 static int qi_check_fault(struct intel_iommu *iommu, int index, int wait_index)  in qi_check_fault()  argument
1274 	struct q_inval *qi = iommu->qi;  in qi_check_fault()
1275 	int shift = qi_shift(iommu);  in qi_check_fault()
1280 	fault = readl(iommu->reg + DMAR_FSTS_REG);  in qi_check_fault()
1282 		qi_dump_fault(iommu, fault);  in qi_check_fault()
1290 		head = readl(iommu->reg + DMAR_IQH_REG);  in qi_check_fault()
1301 			writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);  in qi_check_fault()
1312 		head = readl(iommu->reg + DMAR_IQH_REG);  in qi_check_fault()
1315 		tail = readl(iommu->reg + DMAR_IQT_REG);  in qi_check_fault()
1318 		writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);  in qi_check_fault()
1332 		writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);  in qi_check_fault()
1346 int qi_submit_sync(struct intel_iommu *iommu, struct qi_desc *desc,  in qi_submit_sync()  argument
1349 	struct q_inval *qi = iommu->qi;  in qi_submit_sync()
1366 	    dmar_latency_enabled(iommu, DMAR_LATENCY_INV_IOTLB))  in qi_submit_sync()
1370 	    dmar_latency_enabled(iommu, DMAR_LATENCY_INV_DEVTLB))  in qi_submit_sync()
1374 	    dmar_latency_enabled(iommu, DMAR_LATENCY_INV_IEC))  in qi_submit_sync()
1394 	shift = qi_shift(iommu);  in qi_submit_sync()
1400 		trace_qi_submit(iommu, desc[i].qw0, desc[i].qw1,  in qi_submit_sync()
1423 	writel(qi->free_head << shift, iommu->reg + DMAR_IQT_REG);  in qi_submit_sync()
1433 		rc = qi_check_fault(iommu, index, wait_index);  in qi_submit_sync()
1452 		dmar_latency_update(iommu, DMAR_LATENCY_INV_IOTLB,  in qi_submit_sync()
1456 		dmar_latency_update(iommu, DMAR_LATENCY_INV_DEVTLB,  in qi_submit_sync()
1460 		dmar_latency_update(iommu, DMAR_LATENCY_INV_IEC,  in qi_submit_sync()
1469 void qi_global_iec(struct intel_iommu *iommu)  in qi_global_iec()  argument
1479 	qi_submit_sync(iommu, &desc, 1, 0);  in qi_global_iec()
1482 void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,  in qi_flush_context()  argument
1493 	qi_submit_sync(iommu, &desc, 1, 0);  in qi_flush_context()
1496 void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,  in qi_flush_iotlb()  argument
1504 	if (cap_write_drain(iommu->cap))  in qi_flush_iotlb()
1507 	if (cap_read_drain(iommu->cap))  in qi_flush_iotlb()
1517 	qi_submit_sync(iommu, &desc, 1, 0);  in qi_flush_iotlb()
1520 void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid,  in qi_flush_dev_iotlb()  argument
1539 	qi_submit_sync(iommu, &desc, 1, 0);  in qi_flush_dev_iotlb()
1543 void qi_flush_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr,  in qi_flush_piotlb()  argument
1580 	qi_submit_sync(iommu, &desc, 1, 0);  in qi_flush_piotlb()
1584 void qi_flush_dev_iotlb_pasid(struct intel_iommu *iommu, u16 sid, u16 pfsid,  in qi_flush_dev_iotlb_pasid()  argument
1624 	qi_submit_sync(iommu, &desc, 1, 0);  in qi_flush_dev_iotlb_pasid()
1627 void qi_flush_pasid_cache(struct intel_iommu *iommu, u16 did,  in qi_flush_pasid_cache()  argument
1634 	qi_submit_sync(iommu, &desc, 1, 0);  in qi_flush_pasid_cache()
1640 void dmar_disable_qi(struct intel_iommu *iommu)  in dmar_disable_qi()  argument
1646 	if (!ecap_qis(iommu->ecap))  in dmar_disable_qi()
1649 	raw_spin_lock_irqsave(&iommu->register_lock, flags);  in dmar_disable_qi()
1651 	sts =  readl(iommu->reg + DMAR_GSTS_REG);  in dmar_disable_qi()
1658 	while ((readl(iommu->reg + DMAR_IQT_REG) !=  in dmar_disable_qi()
1659 		readl(iommu->reg + DMAR_IQH_REG)) &&  in dmar_disable_qi()
1663 	iommu->gcmd &= ~DMA_GCMD_QIE;  in dmar_disable_qi()
1664 	writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);  in dmar_disable_qi()
1666 	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,  in dmar_disable_qi()
1669 	raw_spin_unlock_irqrestore(&iommu->register_lock, flags);  in dmar_disable_qi()
1675 static void __dmar_enable_qi(struct intel_iommu *iommu)  in __dmar_enable_qi()  argument
1679 	struct q_inval *qi = iommu->qi;  in __dmar_enable_qi()
1689 	if (ecap_smts(iommu->ecap))  in __dmar_enable_qi()
1692 	raw_spin_lock_irqsave(&iommu->register_lock, flags);  in __dmar_enable_qi()
1695 	writel(0, iommu->reg + DMAR_IQT_REG);  in __dmar_enable_qi()
1697 	dmar_writeq(iommu->reg + DMAR_IQA_REG, val);  in __dmar_enable_qi()
1699 	iommu->gcmd |= DMA_GCMD_QIE;  in __dmar_enable_qi()
1700 	writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);  in __dmar_enable_qi()
1703 	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);  in __dmar_enable_qi()
1705 	raw_spin_unlock_irqrestore(&iommu->register_lock, flags);  in __dmar_enable_qi()
1713 int dmar_enable_qi(struct intel_iommu *iommu)  in dmar_enable_qi()  argument
1718 	if (!ecap_qis(iommu->ecap))  in dmar_enable_qi()
1724 	if (iommu->qi)  in dmar_enable_qi()
1727 	iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);  in dmar_enable_qi()
1728 	if (!iommu->qi)  in dmar_enable_qi()
1731 	qi = iommu->qi;  in dmar_enable_qi()
1737 	desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO,  in dmar_enable_qi()
1738 				     !!ecap_smts(iommu->ecap));  in dmar_enable_qi()
1741 		iommu->qi = NULL;  in dmar_enable_qi()
1751 		iommu->qi = NULL;  in dmar_enable_qi()
1757 	__dmar_enable_qi(iommu);  in dmar_enable_qi()
1877 static inline int dmar_msi_reg(struct intel_iommu *iommu, int irq)  in dmar_msi_reg()  argument
1879 	if (iommu->irq == irq)  in dmar_msi_reg()
1881 	else if (iommu->pr_irq == irq)  in dmar_msi_reg()
1883 	else if (iommu->perf_irq == irq)  in dmar_msi_reg()
1891 	struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);  in dmar_msi_unmask()  local
1892 	int reg = dmar_msi_reg(iommu, data->irq);  in dmar_msi_unmask()
1896 	raw_spin_lock_irqsave(&iommu->register_lock, flag);  in dmar_msi_unmask()
1897 	writel(0, iommu->reg + reg);  in dmar_msi_unmask()
1899 	readl(iommu->reg + reg);  in dmar_msi_unmask()
1900 	raw_spin_unlock_irqrestore(&iommu->register_lock, flag);  in dmar_msi_unmask()
1905 	struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);  in dmar_msi_mask()  local
1906 	int reg = dmar_msi_reg(iommu, data->irq);  in dmar_msi_mask()
1910 	raw_spin_lock_irqsave(&iommu->register_lock, flag);  in dmar_msi_mask()
1911 	writel(DMA_FECTL_IM, iommu->reg + reg);  in dmar_msi_mask()
1913 	readl(iommu->reg + reg);  in dmar_msi_mask()
1914 	raw_spin_unlock_irqrestore(&iommu->register_lock, flag);  in dmar_msi_mask()
1919 	struct intel_iommu *iommu = irq_get_handler_data(irq);  in dmar_msi_write()  local
1920 	int reg = dmar_msi_reg(iommu, irq);  in dmar_msi_write()
1923 	raw_spin_lock_irqsave(&iommu->register_lock, flag);  in dmar_msi_write()
1924 	writel(msg->data, iommu->reg + reg + 4);  in dmar_msi_write()
1925 	writel(msg->address_lo, iommu->reg + reg + 8);  in dmar_msi_write()
1926 	writel(msg->address_hi, iommu->reg + reg + 12);  in dmar_msi_write()
1927 	raw_spin_unlock_irqrestore(&iommu->register_lock, flag);  in dmar_msi_write()
1932 	struct intel_iommu *iommu = irq_get_handler_data(irq);  in dmar_msi_read()  local
1933 	int reg = dmar_msi_reg(iommu, irq);  in dmar_msi_read()
1936 	raw_spin_lock_irqsave(&iommu->register_lock, flag);  in dmar_msi_read()
1937 	msg->data = readl(iommu->reg + reg + 4);  in dmar_msi_read()
1938 	msg->address_lo = readl(iommu->reg + reg + 8);  in dmar_msi_read()
1939 	msg->address_hi = readl(iommu->reg + reg + 12);  in dmar_msi_read()
1940 	raw_spin_unlock_irqrestore(&iommu->register_lock, flag);  in dmar_msi_read()
1943 static int dmar_fault_do_one(struct intel_iommu *iommu, int type,  in dmar_fault_do_one()  argument
1974 	dmar_fault_dump_ptes(iommu, source_id, addr, pasid);  in dmar_fault_do_one()
1982 	struct intel_iommu *iommu = dev_id;  in dmar_fault()  local
1990 	raw_spin_lock_irqsave(&iommu->register_lock, flag);  in dmar_fault()
1991 	fault_status = readl(iommu->reg + DMAR_FSTS_REG);  in dmar_fault()
2000 	reg = cap_fault_reg_offset(iommu->cap);  in dmar_fault()
2013 		data = readl(iommu->reg + reg +  in dmar_fault()
2023 			data = readl(iommu->reg + reg +  in dmar_fault()
2028 			guest_addr = dmar_readq(iommu->reg + reg +  in dmar_fault()
2034 		writel(DMA_FRCD_F, iommu->reg + reg +  in dmar_fault()
2037 		raw_spin_unlock_irqrestore(&iommu->register_lock, flag);  in dmar_fault()
2041 			dmar_fault_do_one(iommu, type, fault_reason,  in dmar_fault()
2046 		if (fault_index >= cap_num_fault_regs(iommu->cap))  in dmar_fault()
2048 		raw_spin_lock_irqsave(&iommu->register_lock, flag);  in dmar_fault()
2052 	       iommu->reg + DMAR_FSTS_REG);  in dmar_fault()
2055 	raw_spin_unlock_irqrestore(&iommu->register_lock, flag);  in dmar_fault()
2059 int dmar_set_interrupt(struct intel_iommu *iommu)  in dmar_set_interrupt()  argument
2066 	if (iommu->irq)  in dmar_set_interrupt()
2069 	irq = dmar_alloc_hwirq(iommu->seq_id, iommu->node, iommu);  in dmar_set_interrupt()
2071 		iommu->irq = irq;  in dmar_set_interrupt()
2077 	ret = request_irq(irq, dmar_fault, IRQF_NO_THREAD, iommu->name, iommu);  in dmar_set_interrupt()
2086 	struct intel_iommu *iommu;  in enable_drhd_fault_handling()  local
2091 	for_each_iommu(iommu, drhd) {  in enable_drhd_fault_handling()
2093 		int ret = dmar_set_interrupt(iommu);  in enable_drhd_fault_handling()
2104 		dmar_fault(iommu->irq, iommu);  in enable_drhd_fault_handling()
2105 		fault_status = readl(iommu->reg + DMAR_FSTS_REG);  in enable_drhd_fault_handling()
2106 		writel(fault_status, iommu->reg + DMAR_FSTS_REG);  in enable_drhd_fault_handling()
2115 int dmar_reenable_qi(struct intel_iommu *iommu)  in dmar_reenable_qi()  argument
2117 	if (!ecap_qis(iommu->ecap))  in dmar_reenable_qi()
2120 	if (!iommu->qi)  in dmar_reenable_qi()
2126 	dmar_disable_qi(iommu);  in dmar_reenable_qi()
2132 	__dmar_enable_qi(iommu);  in dmar_reenable_qi()