1438 static long pnv_pci_ioda2_unset_window(struct iommu_table_group *table_group,
1446 	tbl = pe->table_group.tables[0];  in pnv_pci_ioda2_release_dma_pe()
1447 	rc = pnv_pci_ioda2_unset_window(&pe->table_group, 0);  in pnv_pci_ioda2_release_dma_pe()
1452 	if (pe->table_group.group) {  in pnv_pci_ioda2_release_dma_pe()
1453 		iommu_group_put(pe->table_group.group);  in pnv_pci_ioda2_release_dma_pe()
1454 		BUG_ON(pe->table_group.group);  in pnv_pci_ioda2_release_dma_pe()
1740 	set_iommu_table_base(&pdev->dev, pe->table_group.tables[0]);  in pnv_pci_ioda_dma_dev_setup()
1934 		set_iommu_table_base(&dev->dev, pe->table_group.tables[0]);  in pnv_ioda_setup_bus_dma()
1958 	struct pnv_ioda_pe *pe = container_of(tgl->table_group,  in pnv_pci_p7ioc_tce_invalidate()
1959 			struct pnv_ioda_pe, table_group);  in pnv_pci_p7ioc_tce_invalidate()
2116 		struct pnv_ioda_pe *pe = container_of(tgl->table_group,  in pnv_pci_ioda2_tce_invalidate()
2117 				struct pnv_ioda_pe, table_group);  in pnv_pci_ioda2_tce_invalidate()
2313 	iommu_register_group(&pe->table_group, phb->hose->global_number,  in pnv_pci_ioda1_setup_dma_pe()
2315 	pnv_pci_link_table_and_group(phb->hose->node, 0, tbl, &pe->table_group);  in pnv_pci_ioda1_setup_dma_pe()
2366 	pe->table_group.tce32_start = tbl->it_offset << tbl->it_page_shift;  in pnv_pci_ioda1_setup_dma_pe()
2367 	pe->table_group.tce32_size = tbl->it_size << tbl->it_page_shift;  in pnv_pci_ioda1_setup_dma_pe()
2387 		pnv_pci_unlink_table_and_group(tbl, &pe->table_group);  in pnv_pci_ioda1_setup_dma_pe()
2392 static long pnv_pci_ioda2_set_window(struct iommu_table_group *table_group,  in pnv_pci_ioda2_set_window()  argument
2395 	struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,  in pnv_pci_ioda2_set_window()
2396 			table_group);  in pnv_pci_ioda2_set_window()
2425 			tbl, &pe->table_group);  in pnv_pci_ioda2_set_window()
2459 static long pnv_pci_ioda2_create_table(struct iommu_table_group *table_group,  in pnv_pci_ioda2_create_table()  argument
2463 	struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,  in pnv_pci_ioda2_create_table()
2464 			table_group);  in pnv_pci_ioda2_create_table()
2466 	__u64 bus_offset = num ? pe->tce_bypass_base : table_group->tce32_start;  in pnv_pci_ioda2_create_table()
2506 	const u64 window_size = min((u64)pe->table_group.tce32_size, max_memory);  in pnv_pci_ioda2_setup_default_config()
2508 	rc = pnv_pci_ioda2_create_table(&pe->table_group, 0,  in pnv_pci_ioda2_setup_default_config()
2520 	rc = pnv_pci_ioda2_set_window(&pe->table_group, 0, tbl);  in pnv_pci_ioda2_setup_default_config()
2543 static long pnv_pci_ioda2_unset_window(struct iommu_table_group *table_group,  in pnv_pci_ioda2_unset_window()  argument
2546 	struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,  in pnv_pci_ioda2_unset_window()
2547 			table_group);  in pnv_pci_ioda2_unset_window()
2562 	pnv_pci_unlink_table_and_group(table_group->tables[num], table_group);  in pnv_pci_ioda2_unset_window()
2602 		struct iommu_table_group *table_group,  in pnv_pci_ioda2_create_table_userspace()  argument
2606 	return pnv_pci_ioda2_create_table(table_group,  in pnv_pci_ioda2_create_table_userspace()
2610 static void pnv_ioda2_take_ownership(struct iommu_table_group *table_group)  in pnv_ioda2_take_ownership()  argument
2612 	struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,  in pnv_ioda2_take_ownership()
2613 						table_group);  in pnv_ioda2_take_ownership()
2615 	struct iommu_table *tbl = pe->table_group.tables[0];  in pnv_ioda2_take_ownership()
2618 	pnv_pci_ioda2_unset_window(&pe->table_group, 0);  in pnv_ioda2_take_ownership()
2624 static void pnv_ioda2_release_ownership(struct iommu_table_group *table_group)  in pnv_ioda2_release_ownership()  argument
2626 	struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,  in pnv_ioda2_release_ownership()
2627 						table_group);  in pnv_ioda2_release_ownership()
2670 		struct iommu_table_group *table_group)  in gpe_table_group_to_npe()  argument
2673 	int ret = iommu_group_for_each_dev(table_group->group, &npe,  in gpe_table_group_to_npe()
2681 static long pnv_pci_ioda2_npu_set_window(struct iommu_table_group *table_group,  in pnv_pci_ioda2_npu_set_window()  argument
2684 	struct pnv_ioda_pe *npe = gpe_table_group_to_npe(table_group);  in pnv_pci_ioda2_npu_set_window()
2686 	long ret = pnv_pci_ioda2_set_window(table_group, num, tbl);  in pnv_pci_ioda2_npu_set_window()
2691 	if (table_group->tables[num2])  in pnv_pci_ioda2_npu_set_window()
2696 		pnv_pci_ioda2_unset_window(table_group, num);  in pnv_pci_ioda2_npu_set_window()
2697 		if (table_group->tables[num2])  in pnv_pci_ioda2_npu_set_window()
2699 					table_group->tables[num2]);  in pnv_pci_ioda2_npu_set_window()
2706 		struct iommu_table_group *table_group,  in pnv_pci_ioda2_npu_unset_window()  argument
2709 	struct pnv_ioda_pe *npe = gpe_table_group_to_npe(table_group);  in pnv_pci_ioda2_npu_unset_window()
2711 	long ret = pnv_pci_ioda2_unset_window(table_group, num);  in pnv_pci_ioda2_npu_unset_window()
2716 	if (!npe->table_group.tables[num])  in pnv_pci_ioda2_npu_unset_window()
2723 	if (table_group->tables[num2])  in pnv_pci_ioda2_npu_unset_window()
2724 		ret = pnv_npu_set_window(npe, num2, table_group->tables[num2]);  in pnv_pci_ioda2_npu_unset_window()
2729 static void pnv_ioda2_npu_take_ownership(struct iommu_table_group *table_group)  in pnv_ioda2_npu_take_ownership()  argument
2735 	pnv_npu_take_ownership(gpe_table_group_to_npe(table_group));  in pnv_ioda2_npu_take_ownership()
2736 	pnv_ioda2_take_ownership(table_group);  in pnv_ioda2_npu_take_ownership()
2767 				gpe->table_group.ops = &pnv_pci_ioda2_npu_ops;  in pnv_pci_ioda_setup_iommu_api()
2814 	iommu_register_group(&pe->table_group, phb->hose->global_number,  in pnv_pci_ioda2_setup_dma_pe()
2822 	pe->table_group.tce32_start = 0;  in pnv_pci_ioda2_setup_dma_pe()
2823 	pe->table_group.tce32_size = phb->ioda.m32_pci_base;  in pnv_pci_ioda2_setup_dma_pe()
2824 	pe->table_group.max_dynamic_windows_supported =  in pnv_pci_ioda2_setup_dma_pe()
2826 	pe->table_group.max_levels = POWERNV_IOMMU_MAX_LEVELS;  in pnv_pci_ioda2_setup_dma_pe()
2827 	pe->table_group.pgsizes = pnv_ioda_parse_tce_sizes(phb);  in pnv_pci_ioda2_setup_dma_pe()
2829 	pe->table_group.ops = &pnv_pci_ioda2_ops;  in pnv_pci_ioda2_setup_dma_pe()
3497 static long pnv_pci_ioda1_unset_window(struct iommu_table_group *table_group,  in pnv_pci_ioda1_unset_window()  argument
3500 	struct pnv_ioda_pe *pe = container_of(table_group,  in pnv_pci_ioda1_unset_window()
3501 					      struct pnv_ioda_pe, table_group);  in pnv_pci_ioda1_unset_window()
3522 	pnv_pci_unlink_table_and_group(table_group->tables[num], table_group);  in pnv_pci_ioda1_unset_window()
3529 	struct iommu_table *tbl = pe->table_group.tables[0];  in pnv_pci_ioda1_release_pe_dma()
3535 	rc = pnv_pci_ioda1_unset_window(&pe->table_group, 0);  in pnv_pci_ioda1_release_pe_dma()
3540 	if (pe->table_group.group) {  in pnv_pci_ioda1_release_pe_dma()
3541 		iommu_group_put(pe->table_group.group);  in pnv_pci_ioda1_release_pe_dma()
3542 		WARN_ON(pe->table_group.group);  in pnv_pci_ioda1_release_pe_dma()
3551 	struct iommu_table *tbl = pe->table_group.tables[0];  in pnv_pci_ioda2_release_pe_dma()
3561 	rc = pnv_pci_ioda2_unset_window(&pe->table_group, 0);  in pnv_pci_ioda2_release_pe_dma()
3567 	if (pe->table_group.group) {  in pnv_pci_ioda2_release_pe_dma()
3568 		iommu_group_put(pe->table_group.group);  in pnv_pci_ioda2_release_pe_dma()
3569 		WARN_ON(pe->table_group.group);  in pnv_pci_ioda2_release_pe_dma()