239 bool translation_pre_enabled(struct amd_iommu *iommu)  in translation_pre_enabled()  argument
241 	return (iommu->flags & AMD_IOMMU_FLAG_TRANS_PRE_ENABLED);  in translation_pre_enabled()
244 static void clear_translation_pre_enabled(struct amd_iommu *iommu)  in clear_translation_pre_enabled()  argument
246 	iommu->flags &= ~AMD_IOMMU_FLAG_TRANS_PRE_ENABLED;  in clear_translation_pre_enabled()
249 static void init_translation_status(struct amd_iommu *iommu)  in init_translation_status()  argument
253 	ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET);  in init_translation_status()
255 		iommu->flags |= AMD_IOMMU_FLAG_TRANS_PRE_ENABLED;  in init_translation_status()
277 	struct amd_iommu *iommu;  in get_global_efr()  local
279 	for_each_iommu(iommu) {  in get_global_efr()
280 		u64 tmp = iommu->features;  in get_global_efr()
281 		u64 tmp2 = iommu->features2;  in get_global_efr()
283 		if (list_is_first(&iommu->list, &amd_iommu_list)) {  in get_global_efr()
296 		       iommu->index, iommu->pci_seg->id,  in get_global_efr()
297 		       PCI_BUS_NUM(iommu->devid), PCI_SLOT(iommu->devid),  in get_global_efr()
298 		       PCI_FUNC(iommu->devid));  in get_global_efr()
322 static void __init early_iommu_features_init(struct amd_iommu *iommu,  in early_iommu_features_init()  argument
326 		iommu->features = h->efr_reg;  in early_iommu_features_init()
327 		iommu->features2 = h->efr_reg2;  in early_iommu_features_init()
335 static u32 iommu_read_l1(struct amd_iommu *iommu, u16 l1, u8 address)  in iommu_read_l1()  argument
339 	pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16));  in iommu_read_l1()
340 	pci_read_config_dword(iommu->dev, 0xfc, &val);  in iommu_read_l1()
344 static void iommu_write_l1(struct amd_iommu *iommu, u16 l1, u8 address, u32 val)  in iommu_write_l1()  argument
346 	pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16 | 1 << 31));  in iommu_write_l1()
347 	pci_write_config_dword(iommu->dev, 0xfc, val);  in iommu_write_l1()
348 	pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16));  in iommu_write_l1()
351 static u32 iommu_read_l2(struct amd_iommu *iommu, u8 address)  in iommu_read_l2()  argument
355 	pci_write_config_dword(iommu->dev, 0xf0, address);  in iommu_read_l2()
356 	pci_read_config_dword(iommu->dev, 0xf4, &val);  in iommu_read_l2()
360 static void iommu_write_l2(struct amd_iommu *iommu, u8 address, u32 val)  in iommu_write_l2()  argument
362 	pci_write_config_dword(iommu->dev, 0xf0, (address | 1 << 8));  in iommu_write_l2()
363 	pci_write_config_dword(iommu->dev, 0xf4, val);  in iommu_write_l2()
379 static void iommu_set_exclusion_range(struct amd_iommu *iommu)  in iommu_set_exclusion_range()  argument
381 	u64 start = iommu->exclusion_start & PAGE_MASK;  in iommu_set_exclusion_range()
382 	u64 limit = (start + iommu->exclusion_length - 1) & PAGE_MASK;  in iommu_set_exclusion_range()
385 	if (!iommu->exclusion_start)  in iommu_set_exclusion_range()
389 	memcpy_toio(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET,  in iommu_set_exclusion_range()
393 	memcpy_toio(iommu->mmio_base + MMIO_EXCL_LIMIT_OFFSET,  in iommu_set_exclusion_range()
397 static void iommu_set_cwwb_range(struct amd_iommu *iommu)  in iommu_set_cwwb_range()  argument
399 	u64 start = iommu_virt_to_phys((void *)iommu->cmd_sem);  in iommu_set_cwwb_range()
409 	memcpy_toio(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET,  in iommu_set_cwwb_range()
416 	memcpy_toio(iommu->mmio_base + MMIO_EXCL_LIMIT_OFFSET,  in iommu_set_cwwb_range()
421 static void iommu_set_device_table(struct amd_iommu *iommu)  in iommu_set_device_table()  argument
424 	u32 dev_table_size = iommu->pci_seg->dev_table_size;  in iommu_set_device_table()
425 	void *dev_table = (void *)get_dev_table(iommu);  in iommu_set_device_table()
427 	BUG_ON(iommu->mmio_base == NULL);  in iommu_set_device_table()
431 	memcpy_toio(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET,  in iommu_set_device_table()
436 static void iommu_feature_enable(struct amd_iommu *iommu, u8 bit)  in iommu_feature_enable()  argument
440 	ctrl = readq(iommu->mmio_base +  MMIO_CONTROL_OFFSET);  in iommu_feature_enable()
442 	writeq(ctrl, iommu->mmio_base +  MMIO_CONTROL_OFFSET);  in iommu_feature_enable()
445 static void iommu_feature_disable(struct amd_iommu *iommu, u8 bit)  in iommu_feature_disable()  argument
449 	ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET);  in iommu_feature_disable()
451 	writeq(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);  in iommu_feature_disable()
454 static void iommu_set_inv_tlb_timeout(struct amd_iommu *iommu, int timeout)  in iommu_set_inv_tlb_timeout()  argument
458 	ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET);  in iommu_set_inv_tlb_timeout()
461 	writeq(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);  in iommu_set_inv_tlb_timeout()
465 static void iommu_enable(struct amd_iommu *iommu)  in iommu_enable()  argument
467 	iommu_feature_enable(iommu, CONTROL_IOMMU_EN);  in iommu_enable()
470 static void iommu_disable(struct amd_iommu *iommu)  in iommu_disable()  argument
472 	if (!iommu->mmio_base)  in iommu_disable()
476 	iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);  in iommu_disable()
479 	iommu_feature_disable(iommu, CONTROL_EVT_INT_EN);  in iommu_disable()
480 	iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN);  in iommu_disable()
483 	iommu_feature_disable(iommu, CONTROL_GALOG_EN);  in iommu_disable()
484 	iommu_feature_disable(iommu, CONTROL_GAINT_EN);  in iommu_disable()
487 	iommu_feature_disable(iommu, CONTROL_PPRLOG_EN);  in iommu_disable()
488 	iommu_feature_disable(iommu, CONTROL_PPRINT_EN);  in iommu_disable()
491 	iommu_feature_disable(iommu, CONTROL_IOMMU_EN);  in iommu_disable()
494 	iommu_feature_disable(iommu, CONTROL_IRTCACHEDIS);  in iommu_disable()
513 static void __init iommu_unmap_mmio_space(struct amd_iommu *iommu)  in iommu_unmap_mmio_space()  argument
515 	if (iommu->mmio_base)  in iommu_unmap_mmio_space()
516 		iounmap(iommu->mmio_base);  in iommu_unmap_mmio_space()
517 	release_mem_region(iommu->mmio_phys, iommu->mmio_phys_end);  in iommu_unmap_mmio_space()
751 static int __init alloc_command_buffer(struct amd_iommu *iommu)  in alloc_command_buffer()  argument
753 	iommu->cmd_buf = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,  in alloc_command_buffer()
756 	return iommu->cmd_buf ? 0 : -ENOMEM;  in alloc_command_buffer()
763 static void amd_iommu_restart_log(struct amd_iommu *iommu, const char *evt_type,  in amd_iommu_restart_log()  argument
769 	status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);  in amd_iommu_restart_log()
775 	iommu_feature_disable(iommu, cntrl_log);  in amd_iommu_restart_log()
776 	iommu_feature_disable(iommu, cntrl_intr);  in amd_iommu_restart_log()
778 	writel(status_overflow_mask, iommu->mmio_base + MMIO_STATUS_OFFSET);  in amd_iommu_restart_log()
780 	iommu_feature_enable(iommu, cntrl_intr);  in amd_iommu_restart_log()
781 	iommu_feature_enable(iommu, cntrl_log);  in amd_iommu_restart_log()
788 void amd_iommu_restart_event_logging(struct amd_iommu *iommu)  in amd_iommu_restart_event_logging()  argument
790 	amd_iommu_restart_log(iommu, "Event", CONTROL_EVT_INT_EN,  in amd_iommu_restart_event_logging()
799 void amd_iommu_restart_ga_log(struct amd_iommu *iommu)  in amd_iommu_restart_ga_log()  argument
801 	amd_iommu_restart_log(iommu, "GA", CONTROL_GAINT_EN,  in amd_iommu_restart_ga_log()
810 void amd_iommu_restart_ppr_log(struct amd_iommu *iommu)  in amd_iommu_restart_ppr_log()  argument
812 	amd_iommu_restart_log(iommu, "PPR", CONTROL_PPRINT_EN,  in amd_iommu_restart_ppr_log()
821 static void amd_iommu_reset_cmd_buffer(struct amd_iommu *iommu)  in amd_iommu_reset_cmd_buffer()  argument
823 	iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);  in amd_iommu_reset_cmd_buffer()
825 	writel(0x00, iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);  in amd_iommu_reset_cmd_buffer()
826 	writel(0x00, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);  in amd_iommu_reset_cmd_buffer()
827 	iommu->cmd_buf_head = 0;  in amd_iommu_reset_cmd_buffer()
828 	iommu->cmd_buf_tail = 0;  in amd_iommu_reset_cmd_buffer()
830 	iommu_feature_enable(iommu, CONTROL_CMDBUF_EN);  in amd_iommu_reset_cmd_buffer()
837 static void iommu_enable_command_buffer(struct amd_iommu *iommu)  in iommu_enable_command_buffer()  argument
841 	BUG_ON(iommu->cmd_buf == NULL);  in iommu_enable_command_buffer()
843 	entry = iommu_virt_to_phys(iommu->cmd_buf);  in iommu_enable_command_buffer()
846 	memcpy_toio(iommu->mmio_base + MMIO_CMD_BUF_OFFSET,  in iommu_enable_command_buffer()
849 	amd_iommu_reset_cmd_buffer(iommu);  in iommu_enable_command_buffer()
855 static void iommu_disable_command_buffer(struct amd_iommu *iommu)  in iommu_disable_command_buffer()  argument
857 	iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);  in iommu_disable_command_buffer()
860 static void __init free_command_buffer(struct amd_iommu *iommu)  in free_command_buffer()  argument
862 	free_pages((unsigned long)iommu->cmd_buf, get_order(CMD_BUFFER_SIZE));  in free_command_buffer()
865 static void *__init iommu_alloc_4k_pages(struct amd_iommu *iommu,  in iommu_alloc_4k_pages()  argument
882 static int __init alloc_event_buffer(struct amd_iommu *iommu)  in alloc_event_buffer()  argument
884 	iommu->evt_buf = iommu_alloc_4k_pages(iommu, GFP_KERNEL | __GFP_ZERO,  in alloc_event_buffer()
887 	return iommu->evt_buf ? 0 : -ENOMEM;  in alloc_event_buffer()
890 static void iommu_enable_event_buffer(struct amd_iommu *iommu)  in iommu_enable_event_buffer()  argument
894 	BUG_ON(iommu->evt_buf == NULL);  in iommu_enable_event_buffer()
896 	entry = iommu_virt_to_phys(iommu->evt_buf) | EVT_LEN_MASK;  in iommu_enable_event_buffer()
898 	memcpy_toio(iommu->mmio_base + MMIO_EVT_BUF_OFFSET,  in iommu_enable_event_buffer()
902 	writel(0x00, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);  in iommu_enable_event_buffer()
903 	writel(0x00, iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);  in iommu_enable_event_buffer()
905 	iommu_feature_enable(iommu, CONTROL_EVT_LOG_EN);  in iommu_enable_event_buffer()
911 static void iommu_disable_event_buffer(struct amd_iommu *iommu)  in iommu_disable_event_buffer()  argument
913 	iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN);  in iommu_disable_event_buffer()
916 static void __init free_event_buffer(struct amd_iommu *iommu)  in free_event_buffer()  argument
918 	free_pages((unsigned long)iommu->evt_buf, get_order(EVT_BUFFER_SIZE));  in free_event_buffer()
922 static int __init alloc_ppr_log(struct amd_iommu *iommu)  in alloc_ppr_log()  argument
924 	iommu->ppr_log = iommu_alloc_4k_pages(iommu, GFP_KERNEL | __GFP_ZERO,  in alloc_ppr_log()
927 	return iommu->ppr_log ? 0 : -ENOMEM;  in alloc_ppr_log()
930 static void iommu_enable_ppr_log(struct amd_iommu *iommu)  in iommu_enable_ppr_log()  argument
934 	if (iommu->ppr_log == NULL)  in iommu_enable_ppr_log()
937 	iommu_feature_enable(iommu, CONTROL_PPR_EN);  in iommu_enable_ppr_log()
939 	entry = iommu_virt_to_phys(iommu->ppr_log) | PPR_LOG_SIZE_512;  in iommu_enable_ppr_log()
941 	memcpy_toio(iommu->mmio_base + MMIO_PPR_LOG_OFFSET,  in iommu_enable_ppr_log()
945 	writel(0x00, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);  in iommu_enable_ppr_log()
946 	writel(0x00, iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);  in iommu_enable_ppr_log()
948 	iommu_feature_enable(iommu, CONTROL_PPRLOG_EN);  in iommu_enable_ppr_log()
949 	iommu_feature_enable(iommu, CONTROL_PPRINT_EN);  in iommu_enable_ppr_log()
952 static void __init free_ppr_log(struct amd_iommu *iommu)  in free_ppr_log()  argument
954 	free_pages((unsigned long)iommu->ppr_log, get_order(PPR_LOG_SIZE));  in free_ppr_log()
957 static void free_ga_log(struct amd_iommu *iommu)  in free_ga_log()  argument
960 	free_pages((unsigned long)iommu->ga_log, get_order(GA_LOG_SIZE));  in free_ga_log()
961 	free_pages((unsigned long)iommu->ga_log_tail, get_order(8));  in free_ga_log()
966 static int iommu_ga_log_enable(struct amd_iommu *iommu)  in iommu_ga_log_enable()  argument
971 	if (!iommu->ga_log)  in iommu_ga_log_enable()
974 	entry = iommu_virt_to_phys(iommu->ga_log) | GA_LOG_SIZE_512;  in iommu_ga_log_enable()
975 	memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_BASE_OFFSET,  in iommu_ga_log_enable()
977 	entry = (iommu_virt_to_phys(iommu->ga_log_tail) &  in iommu_ga_log_enable()
979 	memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_TAIL_OFFSET,  in iommu_ga_log_enable()
981 	writel(0x00, iommu->mmio_base + MMIO_GA_HEAD_OFFSET);  in iommu_ga_log_enable()
982 	writel(0x00, iommu->mmio_base + MMIO_GA_TAIL_OFFSET);  in iommu_ga_log_enable()
985 	iommu_feature_enable(iommu, CONTROL_GAINT_EN);  in iommu_ga_log_enable()
986 	iommu_feature_enable(iommu, CONTROL_GALOG_EN);  in iommu_ga_log_enable()
989 		status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);  in iommu_ga_log_enable()
1001 static int iommu_init_ga_log(struct amd_iommu *iommu)  in iommu_init_ga_log()  argument
1006 	iommu->ga_log = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,  in iommu_init_ga_log()
1008 	if (!iommu->ga_log)  in iommu_init_ga_log()
1011 	iommu->ga_log_tail = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,  in iommu_init_ga_log()
1013 	if (!iommu->ga_log_tail)  in iommu_init_ga_log()
1018 	free_ga_log(iommu);  in iommu_init_ga_log()
1023 static int __init alloc_cwwb_sem(struct amd_iommu *iommu)  in alloc_cwwb_sem()  argument
1025 	iommu->cmd_sem = iommu_alloc_4k_pages(iommu, GFP_KERNEL | __GFP_ZERO, 1);  in alloc_cwwb_sem()
1027 	return iommu->cmd_sem ? 0 : -ENOMEM;  in alloc_cwwb_sem()
1030 static void __init free_cwwb_sem(struct amd_iommu *iommu)  in free_cwwb_sem()  argument
1032 	if (iommu->cmd_sem)  in free_cwwb_sem()
1033 		free_page((unsigned long)iommu->cmd_sem);  in free_cwwb_sem()
1036 static void iommu_enable_xt(struct amd_iommu *iommu)  in iommu_enable_xt()  argument
1045 		iommu_feature_enable(iommu, CONTROL_XT_EN);  in iommu_enable_xt()
1049 static void iommu_enable_gt(struct amd_iommu *iommu)  in iommu_enable_gt()  argument
1051 	if (!iommu_feature(iommu, FEATURE_GT))  in iommu_enable_gt()
1054 	iommu_feature_enable(iommu, CONTROL_GT_EN);  in iommu_enable_gt()
1067 static void set_dev_entry_bit(struct amd_iommu *iommu, u16 devid, u8 bit)  in set_dev_entry_bit()  argument
1069 	struct dev_table_entry *dev_table = get_dev_table(iommu);  in set_dev_entry_bit()
1083 static int get_dev_entry_bit(struct amd_iommu *iommu, u16 devid, u8 bit)  in get_dev_entry_bit()  argument
1085 	struct dev_table_entry *dev_table = get_dev_table(iommu);  in get_dev_entry_bit()
1090 static bool __copy_device_table(struct amd_iommu *iommu)  in __copy_device_table()  argument
1093 	struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;  in __copy_device_table()
1102 	lo = readl(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET);  in __copy_device_table()
1103 	hi = readl(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET + 4);  in __copy_device_table()
1109 			iommu->index);  in __copy_device_table()
1182 	struct amd_iommu *iommu;  in copy_device_table()  local
1195 		for_each_iommu(iommu) {  in copy_device_table()
1196 			if (pci_seg->id != iommu->pci_seg->id)  in copy_device_table()
1198 			if (!__copy_device_table(iommu))  in copy_device_table()
1207 void amd_iommu_apply_erratum_63(struct amd_iommu *iommu, u16 devid)  in amd_iommu_apply_erratum_63()  argument
1211 	sysmgt = get_dev_entry_bit(iommu, devid, DEV_ENTRY_SYSMGT1) |  in amd_iommu_apply_erratum_63()
1212 		 (get_dev_entry_bit(iommu, devid, DEV_ENTRY_SYSMGT2) << 1);  in amd_iommu_apply_erratum_63()
1215 		set_dev_entry_bit(iommu, devid, DEV_ENTRY_IW);  in amd_iommu_apply_erratum_63()
1222 static void __init set_dev_entry_from_acpi(struct amd_iommu *iommu,  in set_dev_entry_from_acpi()  argument
1226 		set_dev_entry_bit(iommu, devid, DEV_ENTRY_INIT_PASS);  in set_dev_entry_from_acpi()
1228 		set_dev_entry_bit(iommu, devid, DEV_ENTRY_EINT_PASS);  in set_dev_entry_from_acpi()
1230 		set_dev_entry_bit(iommu, devid, DEV_ENTRY_NMI_PASS);  in set_dev_entry_from_acpi()
1232 		set_dev_entry_bit(iommu, devid, DEV_ENTRY_SYSMGT1);  in set_dev_entry_from_acpi()
1234 		set_dev_entry_bit(iommu, devid, DEV_ENTRY_SYSMGT2);  in set_dev_entry_from_acpi()
1236 		set_dev_entry_bit(iommu, devid, DEV_ENTRY_LINT0_PASS);  in set_dev_entry_from_acpi()
1238 		set_dev_entry_bit(iommu, devid, DEV_ENTRY_LINT1_PASS);  in set_dev_entry_from_acpi()
1240 	amd_iommu_apply_erratum_63(iommu, devid);  in set_dev_entry_from_acpi()
1242 	amd_iommu_set_rlookup_table(iommu, devid);  in set_dev_entry_from_acpi()
1356 static int __init init_iommu_from_acpi(struct amd_iommu *iommu,  in init_iommu_from_acpi()  argument
1365 	struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;  in init_iommu_from_acpi()
1379 	iommu->acpi_flags = h->flags;  in init_iommu_from_acpi()
1405 				set_dev_entry_from_acpi(iommu, dev_i, e->flags, 0);  in init_iommu_from_acpi()
1417 			set_dev_entry_from_acpi(iommu, devid, e->flags, 0);  in init_iommu_from_acpi()
1447 			set_dev_entry_from_acpi(iommu, devid   , e->flags, 0);  in init_iommu_from_acpi()
1448 			set_dev_entry_from_acpi(iommu, devid_to, e->flags, 0);  in init_iommu_from_acpi()
1480 			set_dev_entry_from_acpi(iommu, devid, e->flags,  in init_iommu_from_acpi()
1508 					set_dev_entry_from_acpi(iommu,  in init_iommu_from_acpi()
1511 				set_dev_entry_from_acpi(iommu, dev_i,  in init_iommu_from_acpi()
1547 			set_dev_entry_from_acpi(iommu, devid, e->flags, 0);  in init_iommu_from_acpi()
1613 			set_dev_entry_from_acpi(iommu, devid, e->flags, 0);  in init_iommu_from_acpi()
1695 static void __init free_iommu_one(struct amd_iommu *iommu)  in free_iommu_one()  argument
1697 	free_cwwb_sem(iommu);  in free_iommu_one()
1698 	free_command_buffer(iommu);  in free_iommu_one()
1699 	free_event_buffer(iommu);  in free_iommu_one()
1700 	free_ppr_log(iommu);  in free_iommu_one()
1701 	free_ga_log(iommu);  in free_iommu_one()
1702 	iommu_unmap_mmio_space(iommu);  in free_iommu_one()
1707 	struct amd_iommu *iommu, *next;  in free_iommu_all()  local
1709 	for_each_iommu_safe(iommu, next) {  in free_iommu_all()
1710 		list_del(&iommu->list);  in free_iommu_all()
1711 		free_iommu_one(iommu);  in free_iommu_all()
1712 		kfree(iommu);  in free_iommu_all()
1722 static void amd_iommu_erratum_746_workaround(struct amd_iommu *iommu)  in amd_iommu_erratum_746_workaround()  argument
1731 	pci_write_config_dword(iommu->dev, 0xf0, 0x90);  in amd_iommu_erratum_746_workaround()
1732 	pci_read_config_dword(iommu->dev, 0xf4, &value);  in amd_iommu_erratum_746_workaround()
1738 	pci_write_config_dword(iommu->dev, 0xf0, 0x90 | (1 << 8));  in amd_iommu_erratum_746_workaround()
1740 	pci_write_config_dword(iommu->dev, 0xf4, value | 0x4);  in amd_iommu_erratum_746_workaround()
1741 	pci_info(iommu->dev, "Applying erratum 746 workaround\n");  in amd_iommu_erratum_746_workaround()
1744 	pci_write_config_dword(iommu->dev, 0xf0, 0x90);  in amd_iommu_erratum_746_workaround()
1753 static void amd_iommu_ats_write_check_workaround(struct amd_iommu *iommu)  in amd_iommu_ats_write_check_workaround()  argument
1763 	value = iommu_read_l2(iommu, 0x47);  in amd_iommu_ats_write_check_workaround()
1769 	iommu_write_l2(iommu, 0x47, value | BIT(0));  in amd_iommu_ats_write_check_workaround()
1771 	pci_info(iommu->dev, "Applying ATS write check workaround\n");  in amd_iommu_ats_write_check_workaround()
1779 static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h,  in init_iommu_one()  argument
1787 	iommu->pci_seg = pci_seg;  in init_iommu_one()
1789 	raw_spin_lock_init(&iommu->lock);  in init_iommu_one()
1790 	atomic64_set(&iommu->cmd_sem_val, 0);  in init_iommu_one()
1793 	list_add_tail(&iommu->list, &amd_iommu_list);  in init_iommu_one()
1794 	iommu->index = amd_iommus_present++;  in init_iommu_one()
1796 	if (unlikely(iommu->index >= MAX_IOMMUS)) {  in init_iommu_one()
1802 	amd_iommus[iommu->index] = iommu;  in init_iommu_one()
1807 	iommu->devid   = h->devid;  in init_iommu_one()
1808 	iommu->cap_ptr = h->cap_ptr;  in init_iommu_one()
1809 	iommu->mmio_phys = h->mmio_phys;  in init_iommu_one()
1817 			iommu->mmio_phys_end = MMIO_REG_END_OFFSET;  in init_iommu_one()
1819 			iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET;  in init_iommu_one()
1833 			iommu->mmio_phys_end = MMIO_REG_END_OFFSET;  in init_iommu_one()
1835 			iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET;  in init_iommu_one()
1851 		early_iommu_features_init(iommu, h);  in init_iommu_one()
1858 	iommu->mmio_base = iommu_map_mmio_space(iommu->mmio_phys,  in init_iommu_one()
1859 						iommu->mmio_phys_end);  in init_iommu_one()
1860 	if (!iommu->mmio_base)  in init_iommu_one()
1863 	return init_iommu_from_acpi(iommu, h);  in init_iommu_one()
1866 static int __init init_iommu_one_late(struct amd_iommu *iommu)  in init_iommu_one_late()  argument
1870 	if (alloc_cwwb_sem(iommu))  in init_iommu_one_late()
1873 	if (alloc_command_buffer(iommu))  in init_iommu_one_late()
1876 	if (alloc_event_buffer(iommu))  in init_iommu_one_late()
1879 	iommu->int_enabled = false;  in init_iommu_one_late()
1881 	init_translation_status(iommu);  in init_iommu_one_late()
1882 	if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {  in init_iommu_one_late()
1883 		iommu_disable(iommu);  in init_iommu_one_late()
1884 		clear_translation_pre_enabled(iommu);  in init_iommu_one_late()
1886 			iommu->index);  in init_iommu_one_late()
1889 		amd_iommu_pre_enabled = translation_pre_enabled(iommu);  in init_iommu_one_late()
1892 		ret = amd_iommu_create_irq_domain(iommu);  in init_iommu_one_late()
1901 	iommu->pci_seg->rlookup_table[iommu->devid] = NULL;  in init_iommu_one_late()
1940 	struct amd_iommu *iommu;  in init_iommu_all()  local
1959 			iommu = kzalloc(sizeof(struct amd_iommu), GFP_KERNEL);  in init_iommu_all()
1960 			if (iommu == NULL)  in init_iommu_all()
1963 			ret = init_iommu_one(iommu, h, table);  in init_iommu_all()
1976 	for_each_iommu(iommu) {  in init_iommu_all()
1977 		ret = init_iommu_one_late(iommu);  in init_iommu_all()
1985 static void init_iommu_perf_ctr(struct amd_iommu *iommu)  in init_iommu_perf_ctr()  argument
1988 	struct pci_dev *pdev = iommu->dev;  in init_iommu_perf_ctr()
1990 	if (!iommu_feature(iommu, FEATURE_PC))  in init_iommu_perf_ctr()
1997 	val = readl(iommu->mmio_base + MMIO_CNTR_CONF_OFFSET);  in init_iommu_perf_ctr()
1998 	iommu->max_banks = (u8) ((val >> 12) & 0x3f);  in init_iommu_perf_ctr()
1999 	iommu->max_counters = (u8) ((val >> 7) & 0xf);  in init_iommu_perf_ctr()
2008 	struct amd_iommu *iommu = dev_to_amd_iommu(dev);  in amd_iommu_show_cap()  local
2009 	return sysfs_emit(buf, "%x\n", iommu->cap);  in amd_iommu_show_cap()
2017 	struct amd_iommu *iommu = dev_to_amd_iommu(dev);  in amd_iommu_show_features()  local
2018 	return sysfs_emit(buf, "%llx:%llx\n", iommu->features2, iommu->features);  in amd_iommu_show_features()
2043 static void __init late_iommu_features_init(struct amd_iommu *iommu)  in late_iommu_features_init()  argument
2047 	if (!(iommu->cap & (1 << IOMMU_CAP_EFR)))  in late_iommu_features_init()
2051 	features = readq(iommu->mmio_base + MMIO_EXT_FEATURES);  in late_iommu_features_init()
2052 	features2 = readq(iommu->mmio_base + MMIO_EXT_FEATURES2);  in late_iommu_features_init()
2054 	if (!iommu->features) {  in late_iommu_features_init()
2055 		iommu->features = features;  in late_iommu_features_init()
2056 		iommu->features2 = features2;  in late_iommu_features_init()
2064 	if (features != iommu->features ||  in late_iommu_features_init()
2065 	    features2 != iommu->features2) {  in late_iommu_features_init()
2068 			features, iommu->features,  in late_iommu_features_init()
2069 			features2, iommu->features2);  in late_iommu_features_init()
2073 static int __init iommu_init_pci(struct amd_iommu *iommu)  in iommu_init_pci()  argument
2075 	int cap_ptr = iommu->cap_ptr;  in iommu_init_pci()
2078 	iommu->dev = pci_get_domain_bus_and_slot(iommu->pci_seg->id,  in iommu_init_pci()
2079 						 PCI_BUS_NUM(iommu->devid),  in iommu_init_pci()
2080 						 iommu->devid & 0xff);  in iommu_init_pci()
2081 	if (!iommu->dev)  in iommu_init_pci()
2085 	iommu->dev->match_driver = false;  in iommu_init_pci()
2087 	pci_read_config_dword(iommu->dev, cap_ptr + MMIO_CAP_HDR_OFFSET,  in iommu_init_pci()
2088 			      &iommu->cap);  in iommu_init_pci()
2090 	if (!(iommu->cap & (1 << IOMMU_CAP_IOTLB)))  in iommu_init_pci()
2093 	late_iommu_features_init(iommu);  in iommu_init_pci()
2095 	if (iommu_feature(iommu, FEATURE_GT)) {  in iommu_init_pci()
2100 		pasmax = iommu->features & FEATURE_PASID_MASK;  in iommu_init_pci()
2108 		glxval   = iommu->features & FEATURE_GLXVAL_MASK;  in iommu_init_pci()
2117 	if (iommu_feature(iommu, FEATURE_GT) &&  in iommu_init_pci()
2118 	    iommu_feature(iommu, FEATURE_PPR)) {  in iommu_init_pci()
2119 		iommu->is_iommu_v2   = true;  in iommu_init_pci()
2123 	if (iommu_feature(iommu, FEATURE_PPR) && alloc_ppr_log(iommu))  in iommu_init_pci()
2126 	if (iommu->cap & (1UL << IOMMU_CAP_NPCACHE)) {  in iommu_init_pci()
2132 	init_iommu_perf_ctr(iommu);  in iommu_init_pci()
2135 		if (!iommu_feature(iommu, FEATURE_GIOSUP) ||  in iommu_init_pci()
2136 		    !iommu_feature(iommu, FEATURE_GT)) {  in iommu_init_pci()
2145 	if (is_rd890_iommu(iommu->dev)) {  in iommu_init_pci()
2148 		iommu->root_pdev =  in iommu_init_pci()
2149 			pci_get_domain_bus_and_slot(iommu->pci_seg->id,  in iommu_init_pci()
2150 						    iommu->dev->bus->number,  in iommu_init_pci()
2158 		pci_read_config_dword(iommu->dev, iommu->cap_ptr + 4,  in iommu_init_pci()
2159 				&iommu->stored_addr_lo);  in iommu_init_pci()
2160 		pci_read_config_dword(iommu->dev, iommu->cap_ptr + 8,  in iommu_init_pci()
2161 				&iommu->stored_addr_hi);  in iommu_init_pci()
2164 		iommu->stored_addr_lo &= ~1;  in iommu_init_pci()
2168 				iommu->stored_l1[i][j] = iommu_read_l1(iommu, i, j);  in iommu_init_pci()
2171 			iommu->stored_l2[i] = iommu_read_l2(iommu, i);  in iommu_init_pci()
2174 	amd_iommu_erratum_746_workaround(iommu);  in iommu_init_pci()
2175 	amd_iommu_ats_write_check_workaround(iommu);  in iommu_init_pci()
2177 	ret = iommu_device_sysfs_add(&iommu->iommu, &iommu->dev->dev,  in iommu_init_pci()
2178 			       amd_iommu_groups, "ivhd%d", iommu->index);  in iommu_init_pci()
2182 	iommu_device_register(&iommu->iommu, &amd_iommu_ops, NULL);  in iommu_init_pci()
2184 	return pci_enable_device(iommu->dev);  in iommu_init_pci()
2193 	struct amd_iommu *iommu;  in print_iommu_info()  local
2195 	for_each_iommu(iommu) {  in print_iommu_info()
2196 		struct pci_dev *pdev = iommu->dev;  in print_iommu_info()
2199 		pci_info(pdev, "Found IOMMU cap 0x%x\n", iommu->cap_ptr);  in print_iommu_info()
2201 		if (iommu->cap & (1 << IOMMU_CAP_EFR)) {  in print_iommu_info()
2202 			pr_info("Extended features (%#llx, %#llx):", iommu->features, iommu->features2);  in print_iommu_info()
2205 				if (iommu_feature(iommu, (1ULL << i)))  in print_iommu_info()
2209 			if (iommu->features & FEATURE_GAM_VAPIC)  in print_iommu_info()
2212 			if (iommu->features & FEATURE_SNP)  in print_iommu_info()
2231 	struct amd_iommu *iommu;  in amd_iommu_init_pci()  local
2235 	for_each_iommu(iommu) {  in amd_iommu_init_pci()
2236 		ret = iommu_init_pci(iommu);  in amd_iommu_init_pci()
2239 			       iommu->index, ret);  in amd_iommu_init_pci()
2243 		iommu_set_cwwb_range(iommu);  in amd_iommu_init_pci()
2259 	for_each_iommu(iommu)  in amd_iommu_init_pci()
2260 		iommu_flush_all_caches(iommu);  in amd_iommu_init_pci()
2277 static int iommu_setup_msi(struct amd_iommu *iommu)  in iommu_setup_msi()  argument
2281 	r = pci_enable_msi(iommu->dev);  in iommu_setup_msi()
2285 	r = request_threaded_irq(iommu->dev->irq,  in iommu_setup_msi()
2289 				 iommu);  in iommu_setup_msi()
2292 		pci_disable_msi(iommu->dev);  in iommu_setup_msi()
2361 	struct amd_iommu *iommu = irqd->chip_data;  in intcapxt_unmask_irq()  local
2371 	writeq(xt.capxt, iommu->mmio_base + irqd->hwirq);  in intcapxt_unmask_irq()
2376 	struct amd_iommu *iommu = irqd->chip_data;  in intcapxt_mask_irq()  local
2378 	writeq(0, iommu->mmio_base + irqd->hwirq);  in intcapxt_mask_irq()
2441 static int __iommu_setup_intcapxt(struct amd_iommu *iommu, const char *devname,  in __iommu_setup_intcapxt()  argument
2447 	int node = dev_to_node(&iommu->dev->dev);  in __iommu_setup_intcapxt()
2455 	info.data = iommu;  in __iommu_setup_intcapxt()
2465 				   thread_fn, 0, devname, iommu);  in __iommu_setup_intcapxt()
2475 static int iommu_setup_intcapxt(struct amd_iommu *iommu)  in iommu_setup_intcapxt()  argument
2479 	snprintf(iommu->evt_irq_name, sizeof(iommu->evt_irq_name),  in iommu_setup_intcapxt()
2480 		 "AMD-Vi%d-Evt", iommu->index);  in iommu_setup_intcapxt()
2481 	ret = __iommu_setup_intcapxt(iommu, iommu->evt_irq_name,  in iommu_setup_intcapxt()
2487 	snprintf(iommu->ppr_irq_name, sizeof(iommu->ppr_irq_name),  in iommu_setup_intcapxt()
2488 		 "AMD-Vi%d-PPR", iommu->index);  in iommu_setup_intcapxt()
2489 	ret = __iommu_setup_intcapxt(iommu, iommu->ppr_irq_name,  in iommu_setup_intcapxt()
2496 	snprintf(iommu->ga_irq_name, sizeof(iommu->ga_irq_name),  in iommu_setup_intcapxt()
2497 		 "AMD-Vi%d-GA", iommu->index);  in iommu_setup_intcapxt()
2498 	ret = __iommu_setup_intcapxt(iommu, iommu->ga_irq_name,  in iommu_setup_intcapxt()
2506 static int iommu_init_irq(struct amd_iommu *iommu)  in iommu_init_irq()  argument
2510 	if (iommu->int_enabled)  in iommu_init_irq()
2514 		ret = iommu_setup_intcapxt(iommu);  in iommu_init_irq()
2515 	else if (iommu->dev->msi_cap)  in iommu_init_irq()
2516 		ret = iommu_setup_msi(iommu);  in iommu_init_irq()
2523 	iommu->int_enabled = true;  in iommu_init_irq()
2527 		iommu_feature_enable(iommu, CONTROL_INTCAPXT_EN);  in iommu_init_irq()
2529 	iommu_feature_enable(iommu, CONTROL_EVT_INT_EN);  in iommu_init_irq()
2685 static void iommu_init_flags(struct amd_iommu *iommu)  in iommu_init_flags()  argument
2687 	iommu->acpi_flags & IVHD_FLAG_HT_TUN_EN_MASK ?  in iommu_init_flags()
2688 		iommu_feature_enable(iommu, CONTROL_HT_TUN_EN) :  in iommu_init_flags()
2689 		iommu_feature_disable(iommu, CONTROL_HT_TUN_EN);  in iommu_init_flags()
2691 	iommu->acpi_flags & IVHD_FLAG_PASSPW_EN_MASK ?  in iommu_init_flags()
2692 		iommu_feature_enable(iommu, CONTROL_PASSPW_EN) :  in iommu_init_flags()
2693 		iommu_feature_disable(iommu, CONTROL_PASSPW_EN);  in iommu_init_flags()
2695 	iommu->acpi_flags & IVHD_FLAG_RESPASSPW_EN_MASK ?  in iommu_init_flags()
2696 		iommu_feature_enable(iommu, CONTROL_RESPASSPW_EN) :  in iommu_init_flags()
2697 		iommu_feature_disable(iommu, CONTROL_RESPASSPW_EN);  in iommu_init_flags()
2699 	iommu->acpi_flags & IVHD_FLAG_ISOC_EN_MASK ?  in iommu_init_flags()
2700 		iommu_feature_enable(iommu, CONTROL_ISOC_EN) :  in iommu_init_flags()
2701 		iommu_feature_disable(iommu, CONTROL_ISOC_EN);  in iommu_init_flags()
2706 	iommu_feature_enable(iommu, CONTROL_COHERENT_EN);  in iommu_init_flags()
2709 	iommu_set_inv_tlb_timeout(iommu, CTRL_INV_TO_1S);  in iommu_init_flags()
2712 static void iommu_apply_resume_quirks(struct amd_iommu *iommu)  in iommu_apply_resume_quirks()  argument
2716 	struct pci_dev *pdev = iommu->root_pdev;  in iommu_apply_resume_quirks()
2719 	if (!is_rd890_iommu(iommu->dev) || !pdev)  in iommu_apply_resume_quirks()
2736 	pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4,  in iommu_apply_resume_quirks()
2737 			       iommu->stored_addr_lo);  in iommu_apply_resume_quirks()
2738 	pci_write_config_dword(iommu->dev, iommu->cap_ptr + 8,  in iommu_apply_resume_quirks()
2739 			       iommu->stored_addr_hi);  in iommu_apply_resume_quirks()
2744 			iommu_write_l1(iommu, i, j, iommu->stored_l1[i][j]);  in iommu_apply_resume_quirks()
2748 		iommu_write_l2(iommu, i, iommu->stored_l2[i]);  in iommu_apply_resume_quirks()
2751 	pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4,  in iommu_apply_resume_quirks()
2752 			       iommu->stored_addr_lo | 1);  in iommu_apply_resume_quirks()
2755 static void iommu_enable_ga(struct amd_iommu *iommu)  in iommu_enable_ga()  argument
2761 		iommu_feature_enable(iommu, CONTROL_GA_EN);  in iommu_enable_ga()
2762 		iommu->irte_ops = &irte_128_ops;  in iommu_enable_ga()
2765 		iommu->irte_ops = &irte_32_ops;  in iommu_enable_ga()
2771 static void iommu_disable_irtcachedis(struct amd_iommu *iommu)  in iommu_disable_irtcachedis()  argument
2773 	iommu_feature_disable(iommu, CONTROL_IRTCACHEDIS);  in iommu_disable_irtcachedis()
2776 static void iommu_enable_irtcachedis(struct amd_iommu *iommu)  in iommu_enable_irtcachedis()  argument
2788 	iommu_feature_enable(iommu, CONTROL_IRTCACHEDIS);  in iommu_enable_irtcachedis()
2789 	ctrl = readq(iommu->mmio_base +  MMIO_CONTROL_OFFSET);  in iommu_enable_irtcachedis()
2792 		iommu->irtcachedis_enabled = true;  in iommu_enable_irtcachedis()
2794 		iommu->index, iommu->devid,  in iommu_enable_irtcachedis()
2795 		iommu->irtcachedis_enabled ? "disabled" : "enabled");  in iommu_enable_irtcachedis()
2798 static void early_enable_iommu(struct amd_iommu *iommu)  in early_enable_iommu()  argument
2800 	iommu_disable(iommu);  in early_enable_iommu()
2801 	iommu_init_flags(iommu);  in early_enable_iommu()
2802 	iommu_set_device_table(iommu);  in early_enable_iommu()
2803 	iommu_enable_command_buffer(iommu);  in early_enable_iommu()
2804 	iommu_enable_event_buffer(iommu);  in early_enable_iommu()
2805 	iommu_set_exclusion_range(iommu);  in early_enable_iommu()
2806 	iommu_enable_ga(iommu);  in early_enable_iommu()
2807 	iommu_enable_xt(iommu);  in early_enable_iommu()
2808 	iommu_enable_irtcachedis(iommu);  in early_enable_iommu()
2809 	iommu_enable(iommu);  in early_enable_iommu()
2810 	iommu_flush_all_caches(iommu);  in early_enable_iommu()
2823 	struct amd_iommu *iommu;  in early_enable_iommus()  local
2843 		for_each_iommu(iommu) {  in early_enable_iommus()
2844 			clear_translation_pre_enabled(iommu);  in early_enable_iommus()
2845 			early_enable_iommu(iommu);  in early_enable_iommus()
2856 		for_each_iommu(iommu) {  in early_enable_iommus()
2857 			iommu_disable_command_buffer(iommu);  in early_enable_iommus()
2858 			iommu_disable_event_buffer(iommu);  in early_enable_iommus()
2859 			iommu_disable_irtcachedis(iommu);  in early_enable_iommus()
2860 			iommu_enable_command_buffer(iommu);  in early_enable_iommus()
2861 			iommu_enable_event_buffer(iommu);  in early_enable_iommus()
2862 			iommu_enable_ga(iommu);  in early_enable_iommus()
2863 			iommu_enable_xt(iommu);  in early_enable_iommus()
2864 			iommu_enable_irtcachedis(iommu);  in early_enable_iommus()
2865 			iommu_set_device_table(iommu);  in early_enable_iommus()
2866 			iommu_flush_all_caches(iommu);  in early_enable_iommus()
2873 	struct amd_iommu *iommu;  in enable_iommus_v2()  local
2875 	for_each_iommu(iommu) {  in enable_iommus_v2()
2876 		iommu_enable_ppr_log(iommu);  in enable_iommus_v2()
2877 		iommu_enable_gt(iommu);  in enable_iommus_v2()
2885 	struct amd_iommu *iommu;  in enable_iommus_vapic()  local
2887 	for_each_iommu(iommu) {  in enable_iommus_vapic()
2892 		status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);  in enable_iommus_vapic()
2896 		iommu_feature_disable(iommu, CONTROL_GALOG_EN);  in enable_iommus_vapic()
2897 		iommu_feature_disable(iommu, CONTROL_GAINT_EN);  in enable_iommus_vapic()
2904 			status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);  in enable_iommus_vapic()
2928 	for_each_iommu(iommu) {  in enable_iommus_vapic()
2929 		if (iommu_init_ga_log(iommu) ||  in enable_iommus_vapic()
2930 		    iommu_ga_log_enable(iommu))  in enable_iommus_vapic()
2933 		iommu_feature_enable(iommu, CONTROL_GAM_EN);  in enable_iommus_vapic()
2935 			iommu_feature_enable(iommu, CONTROL_SNPAVIC_EN);  in enable_iommus_vapic()
2950 	struct amd_iommu *iommu;  in disable_iommus()  local
2952 	for_each_iommu(iommu)  in disable_iommus()
2953 		iommu_disable(iommu);  in disable_iommus()
2968 	struct amd_iommu *iommu;  in amd_iommu_resume()  local
2970 	for_each_iommu(iommu)  in amd_iommu_resume()
2971 		iommu_apply_resume_quirks(iommu);  in amd_iommu_resume()
3200 	struct amd_iommu *iommu;  in amd_iommu_enable_interrupts()  local
3203 	for_each_iommu(iommu) {  in amd_iommu_enable_interrupts()
3204 		ret = iommu_init_irq(iommu);  in amd_iommu_enable_interrupts()
3323 			struct amd_iommu *iommu;  in state_next()  local
3329 			for_each_iommu(iommu)  in state_next()
3330 				iommu_flush_all_caches(iommu);  in state_next()
3405 	struct amd_iommu *iommu;  in amd_iommu_init()  local
3419 	for_each_iommu(iommu)  in amd_iommu_init()
3420 		amd_iommu_debugfs_setup(iommu);  in amd_iommu_init()
3467 	x86_init.iommu.iommu_init = amd_iommu_init;  in amd_iommu_detect()
3708 	struct amd_iommu *iommu;  in get_amd_iommu()  local
3710 	for_each_iommu(iommu)  in get_amd_iommu()
3712 			return iommu;  in get_amd_iommu()
3725 	struct amd_iommu *iommu = get_amd_iommu(idx);  in amd_iommu_pc_get_max_banks()  local
3727 	if (iommu)  in amd_iommu_pc_get_max_banks()
3728 		return iommu->max_banks;  in amd_iommu_pc_get_max_banks()
3742 	struct amd_iommu *iommu = get_amd_iommu(idx);  in amd_iommu_pc_get_max_counters()  local
3744 	if (iommu)  in amd_iommu_pc_get_max_counters()
3745 		return iommu->max_counters;  in amd_iommu_pc_get_max_counters()
3751 static int iommu_pc_get_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr,  in iommu_pc_get_set_reg()  argument
3762 	if (WARN_ON(!iommu || (fxn > 0x28) || (fxn & 7)))  in iommu_pc_get_set_reg()
3768 	max_offset_lim = (u32)(((0x40 | iommu->max_banks) << 12) |  in iommu_pc_get_set_reg()
3769 				(iommu->max_counters << 8) | 0x28);  in iommu_pc_get_set_reg()
3777 		writel((u32)val, iommu->mmio_base + offset);  in iommu_pc_get_set_reg()
3778 		writel((val >> 32), iommu->mmio_base + offset + 4);  in iommu_pc_get_set_reg()
3780 		*value = readl(iommu->mmio_base + offset + 4);  in iommu_pc_get_set_reg()
3782 		*value |= readl(iommu->mmio_base + offset);  in iommu_pc_get_set_reg()
3789 int amd_iommu_pc_get_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, u8 fxn, u64 *value)  in amd_iommu_pc_get_reg()  argument
3791 	if (!iommu)  in amd_iommu_pc_get_reg()
3794 	return iommu_pc_get_set_reg(iommu, bank, cntr, fxn, value, false);  in amd_iommu_pc_get_reg()
3797 int amd_iommu_pc_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, u8 fxn, u64 *value)  in amd_iommu_pc_set_reg()  argument
3799 	if (!iommu)  in amd_iommu_pc_set_reg()
3802 	return iommu_pc_get_set_reg(iommu, bank, cntr, fxn, value, true);  in amd_iommu_pc_set_reg()