109 	u16 pci_seg;  member
146 	u16 pci_seg;  member
230 static void init_device_table_dma(struct amd_iommu_pci_seg *pci_seg);
293 		       iommu->index, iommu->pci_seg->id,  in get_global_efr()
416 	u32 dev_table_size = iommu->pci_seg->dev_table_size;  in iommu_set_device_table()
612 static int __init find_last_devid_acpi(struct acpi_table_header *table, u16 pci_seg)  in find_last_devid_acpi()  argument
623 		if (h->pci_seg == pci_seg &&  in find_last_devid_acpi()
649 static inline int __init alloc_dev_table(struct amd_iommu_pci_seg *pci_seg)  in alloc_dev_table()  argument
651 	pci_seg->dev_table = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO | GFP_DMA32,  in alloc_dev_table()
652 						      get_order(pci_seg->dev_table_size));  in alloc_dev_table()
653 	if (!pci_seg->dev_table)  in alloc_dev_table()
659 static inline void free_dev_table(struct amd_iommu_pci_seg *pci_seg)  in free_dev_table()  argument
661 	free_pages((unsigned long)pci_seg->dev_table,  in free_dev_table()
662 		    get_order(pci_seg->dev_table_size));  in free_dev_table()
663 	pci_seg->dev_table = NULL;  in free_dev_table()
667 static inline int __init alloc_rlookup_table(struct amd_iommu_pci_seg *pci_seg)  in alloc_rlookup_table()  argument
669 	pci_seg->rlookup_table = (void *)__get_free_pages(  in alloc_rlookup_table()
671 						get_order(pci_seg->rlookup_table_size));  in alloc_rlookup_table()
672 	if (pci_seg->rlookup_table == NULL)  in alloc_rlookup_table()
678 static inline void free_rlookup_table(struct amd_iommu_pci_seg *pci_seg)  in free_rlookup_table()  argument
680 	free_pages((unsigned long)pci_seg->rlookup_table,  in free_rlookup_table()
681 		   get_order(pci_seg->rlookup_table_size));  in free_rlookup_table()
682 	pci_seg->rlookup_table = NULL;  in free_rlookup_table()
685 static inline int __init alloc_irq_lookup_table(struct amd_iommu_pci_seg *pci_seg)  in alloc_irq_lookup_table()  argument
687 	pci_seg->irq_lookup_table = (void *)__get_free_pages(  in alloc_irq_lookup_table()
689 					     get_order(pci_seg->rlookup_table_size));  in alloc_irq_lookup_table()
690 	kmemleak_alloc(pci_seg->irq_lookup_table,  in alloc_irq_lookup_table()
691 		       pci_seg->rlookup_table_size, 1, GFP_KERNEL);  in alloc_irq_lookup_table()
692 	if (pci_seg->irq_lookup_table == NULL)  in alloc_irq_lookup_table()
698 static inline void free_irq_lookup_table(struct amd_iommu_pci_seg *pci_seg)  in free_irq_lookup_table()  argument
700 	kmemleak_free(pci_seg->irq_lookup_table);  in free_irq_lookup_table()
701 	free_pages((unsigned long)pci_seg->irq_lookup_table,  in free_irq_lookup_table()
702 		   get_order(pci_seg->rlookup_table_size));  in free_irq_lookup_table()
703 	pci_seg->irq_lookup_table = NULL;  in free_irq_lookup_table()
706 static int __init alloc_alias_table(struct amd_iommu_pci_seg *pci_seg)  in alloc_alias_table()  argument
710 	pci_seg->alias_table = (void *)__get_free_pages(GFP_KERNEL,  in alloc_alias_table()
711 					get_order(pci_seg->alias_table_size));  in alloc_alias_table()
712 	if (!pci_seg->alias_table)  in alloc_alias_table()
718 	for (i = 0; i <= pci_seg->last_bdf; ++i)  in alloc_alias_table()
719 		pci_seg->alias_table[i] = i;  in alloc_alias_table()
724 static void __init free_alias_table(struct amd_iommu_pci_seg *pci_seg)  in free_alias_table()  argument
726 	free_pages((unsigned long)pci_seg->alias_table,  in free_alias_table()
727 		   get_order(pci_seg->alias_table_size));  in free_alias_table()
728 	pci_seg->alias_table = NULL;  in free_alias_table()
1028 	struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;  in __copy_device_table()  local
1042 	if (old_devtb_size != pci_seg->dev_table_size) {  in __copy_device_table()
1061 							pci_seg->dev_table_size)  in __copy_device_table()
1062 		    : memremap(old_devtb_phys, pci_seg->dev_table_size, MEMREMAP_WB);  in __copy_device_table()
1068 	pci_seg->old_dev_tbl_cpy = (void *)__get_free_pages(gfp_flag,  in __copy_device_table()
1069 						    get_order(pci_seg->dev_table_size));  in __copy_device_table()
1070 	if (pci_seg->old_dev_tbl_cpy == NULL) {  in __copy_device_table()
1076 	for (devid = 0; devid <= pci_seg->last_bdf; ++devid) {  in __copy_device_table()
1077 		pci_seg->old_dev_tbl_cpy[devid] = old_devtb[devid];  in __copy_device_table()
1082 			pci_seg->old_dev_tbl_cpy[devid].data[0] = old_devtb[devid].data[0];  in __copy_device_table()
1083 			pci_seg->old_dev_tbl_cpy[devid].data[1] = old_devtb[devid].data[1];  in __copy_device_table()
1089 				pci_seg->old_dev_tbl_cpy[devid].data[1] &= ~tmp;  in __copy_device_table()
1092 				pci_seg->old_dev_tbl_cpy[devid].data[0] &= ~tmp;  in __copy_device_table()
1107 			pci_seg->old_dev_tbl_cpy[devid].data[2] = old_devtb[devid].data[2];  in __copy_device_table()
1118 	struct amd_iommu_pci_seg *pci_seg;  in copy_device_table()  local
1129 	for_each_pci_segment(pci_seg) {  in copy_device_table()
1131 			if (pci_seg->id != iommu->pci_seg->id)  in copy_device_table()
1300 	struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;  in init_iommu_from_acpi()  local
1332 		seg_id = pci_seg->id;  in init_iommu_from_acpi()
1339 			for (dev_i = 0; dev_i <= pci_seg->last_bdf; ++dev_i)  in init_iommu_from_acpi()
1384 			pci_seg->alias_table[devid] = devid_to;  in init_iommu_from_acpi()
1442 					pci_seg->alias_table[dev_i] = devid_to;  in init_iommu_from_acpi()
1566 	struct amd_iommu_pci_seg *pci_seg;  in alloc_pci_segment()  local
1578 	pci_seg = kzalloc(sizeof(struct amd_iommu_pci_seg), GFP_KERNEL);  in alloc_pci_segment()
1579 	if (pci_seg == NULL)  in alloc_pci_segment()
1582 	pci_seg->last_bdf = last_bdf;  in alloc_pci_segment()
1584 	pci_seg->dev_table_size     = tbl_size(DEV_TABLE_ENTRY_SIZE, last_bdf);  in alloc_pci_segment()
1585 	pci_seg->alias_table_size   = tbl_size(ALIAS_TABLE_ENTRY_SIZE, last_bdf);  in alloc_pci_segment()
1586 	pci_seg->rlookup_table_size = tbl_size(RLOOKUP_TABLE_ENTRY_SIZE, last_bdf);  in alloc_pci_segment()
1588 	pci_seg->id = id;  in alloc_pci_segment()
1589 	init_llist_head(&pci_seg->dev_data_list);  in alloc_pci_segment()
1590 	INIT_LIST_HEAD(&pci_seg->unity_map);  in alloc_pci_segment()
1591 	list_add_tail(&pci_seg->list, &amd_iommu_pci_seg_list);  in alloc_pci_segment()
1593 	if (alloc_dev_table(pci_seg))  in alloc_pci_segment()
1595 	if (alloc_alias_table(pci_seg))  in alloc_pci_segment()
1597 	if (alloc_rlookup_table(pci_seg))  in alloc_pci_segment()
1600 	return pci_seg;  in alloc_pci_segment()
1606 	struct amd_iommu_pci_seg *pci_seg;  in get_pci_segment()  local
1608 	for_each_pci_segment(pci_seg) {  in get_pci_segment()
1609 		if (pci_seg->id == id)  in get_pci_segment()
1610 			return pci_seg;  in get_pci_segment()
1618 	struct amd_iommu_pci_seg *pci_seg, *next;  in free_pci_segments()  local
1620 	for_each_pci_segment_safe(pci_seg, next) {  in free_pci_segments()
1621 		list_del(&pci_seg->list);  in free_pci_segments()
1622 		free_irq_lookup_table(pci_seg);  in free_pci_segments()
1623 		free_rlookup_table(pci_seg);  in free_pci_segments()
1624 		free_alias_table(pci_seg);  in free_pci_segments()
1625 		free_dev_table(pci_seg);  in free_pci_segments()
1626 		kfree(pci_seg);  in free_pci_segments()
1717 	struct amd_iommu_pci_seg *pci_seg;  in init_iommu_one()  local
1719 	pci_seg = get_pci_segment(h->pci_seg, ivrs_base);  in init_iommu_one()
1720 	if (pci_seg == NULL)  in init_iommu_one()
1722 	iommu->pci_seg = pci_seg;  in init_iommu_one()
1836 	iommu->pci_seg->rlookup_table[iommu->devid] = NULL;  in init_iommu_one_late()
1888 				    h->pci_seg, PCI_BUS_NUM(h->devid),  in init_iommu_all()
2013 	iommu->dev = pci_get_domain_bus_and_slot(iommu->pci_seg->id,  in iommu_init_pci()
2084 			pci_get_domain_bus_and_slot(iommu->pci_seg->id,  in iommu_init_pci()
2165 	struct amd_iommu_pci_seg *pci_seg;  in amd_iommu_init_pci()  local
2189 	for_each_pci_segment(pci_seg)  in amd_iommu_init_pci()
2190 		init_device_table_dma(pci_seg);  in amd_iommu_init_pci()
2453 	struct amd_iommu_pci_seg *p, *pci_seg;  in free_unity_maps()  local
2455 	for_each_pci_segment_safe(pci_seg, p) {  in free_unity_maps()
2456 		list_for_each_entry_safe(entry, next, &pci_seg->unity_map, list) {  in free_unity_maps()
2468 	struct amd_iommu_pci_seg *pci_seg;  in init_unity_map_range()  local
2471 	pci_seg = get_pci_segment(m->pci_seg, ivrs_base);  in init_unity_map_range()
2472 	if (pci_seg == NULL)  in init_unity_map_range()
2490 		e->devid_end = pci_seg->last_bdf;  in init_unity_map_range()
2514 		    " flags: %x\n", s, m->pci_seg,  in init_unity_map_range()
2516 		    PCI_FUNC(e->devid_start), m->pci_seg,  in init_unity_map_range()
2521 	list_add_tail(&e->list, &pci_seg->unity_map);  in init_unity_map_range()
2549 static void init_device_table_dma(struct amd_iommu_pci_seg *pci_seg)  in init_device_table_dma()  argument
2552 	struct dev_table_entry *dev_table = pci_seg->dev_table;  in init_device_table_dma()
2557 	for (devid = 0; devid <= pci_seg->last_bdf; ++devid) {  in init_device_table_dma()
2564 static void __init uninit_device_table_dma(struct amd_iommu_pci_seg *pci_seg)  in uninit_device_table_dma()  argument
2567 	struct dev_table_entry *dev_table = pci_seg->dev_table;  in uninit_device_table_dma()
2572 	for (devid = 0; devid <= pci_seg->last_bdf; ++devid) {  in uninit_device_table_dma()
2580 	struct amd_iommu_pci_seg *pci_seg;  in init_device_table()  local
2586 	for_each_pci_segment(pci_seg) {  in init_device_table()
2587 		for (devid = 0; devid <= pci_seg->last_bdf; ++devid)  in init_device_table()
2588 			__set_dev_entry_bit(pci_seg->dev_table,  in init_device_table()
2704 	struct amd_iommu_pci_seg *pci_seg;  in early_enable_iommus()  local
2715 		for_each_pci_segment(pci_seg) {  in early_enable_iommus()
2716 			if (pci_seg->old_dev_tbl_cpy != NULL) {  in early_enable_iommus()
2717 				free_pages((unsigned long)pci_seg->old_dev_tbl_cpy,  in early_enable_iommus()
2718 						get_order(pci_seg->dev_table_size));  in early_enable_iommus()
2719 				pci_seg->old_dev_tbl_cpy = NULL;  in early_enable_iommus()
2730 		for_each_pci_segment(pci_seg) {  in early_enable_iommus()
2731 			free_pages((unsigned long)pci_seg->dev_table,  in early_enable_iommus()
2732 				   get_order(pci_seg->dev_table_size));  in early_enable_iommus()
2733 			pci_seg->dev_table = pci_seg->old_dev_tbl_cpy;  in early_enable_iommus()
3036 		struct amd_iommu_pci_seg *pci_seg;  in early_amd_iommu_init()  local
3053 		for_each_pci_segment(pci_seg) {  in early_amd_iommu_init()
3054 			if (alloc_irq_lookup_table(pci_seg))  in early_amd_iommu_init()
3194 			struct amd_iommu_pci_seg *pci_seg;  in state_next()  local
3196 			for_each_pci_segment(pci_seg)  in state_next()
3197 				uninit_device_table_dma(pci_seg);  in state_next()