103 					  struct domain_pgtable *pgtable);
156 					 struct domain_pgtable *pgtable)  in amd_iommu_domain_get_pgtable()  argument
160 	pgtable->root = (u64 *)(pt_root & PAGE_MASK);  in amd_iommu_domain_get_pgtable()
161 	pgtable->mode = pt_root & 7; /* lowest 3 bits encode pgtable mode */  in amd_iommu_domain_get_pgtable()
1475 static void free_pagetable(struct domain_pgtable *pgtable)  in free_pagetable()  argument
1480 	if (pgtable->mode == PAGE_MODE_NONE)  in free_pagetable()
1483 	BUG_ON(pgtable->mode < PAGE_MODE_NONE ||  in free_pagetable()
1484 	       pgtable->mode > PAGE_MODE_6_LEVEL);  in free_pagetable()
1486 	root = (unsigned long)pgtable->root;  in free_pagetable()
1487 	freelist = free_sub_pt(root, pgtable->mode, freelist);  in free_pagetable()
1501 	struct domain_pgtable pgtable;  in increase_address_space()  local
1508 	amd_iommu_domain_get_pgtable(domain, &pgtable);  in increase_address_space()
1510 	if (address <= PM_LEVEL_SIZE(pgtable.mode))  in increase_address_space()
1514 	if (WARN_ON_ONCE(pgtable.mode == PAGE_MODE_6_LEVEL))  in increase_address_space()
1521 	*pte = PM_LEVEL_PDE(pgtable.mode, iommu_virt_to_phys(pgtable.root));  in increase_address_space()
1523 	pgtable.root  = pte;  in increase_address_space()
1524 	pgtable.mode += 1;  in increase_address_space()
1525 	update_and_flush_device_table(domain, &pgtable);  in increase_address_space()
1532 	amd_iommu_domain_set_pgtable(domain, pte, pgtable.mode);  in increase_address_space()
1549 	struct domain_pgtable pgtable;  in alloc_pte()  local
1555 	amd_iommu_domain_get_pgtable(domain, &pgtable);  in alloc_pte()
1557 	while (address > PM_LEVEL_SIZE(pgtable.mode)) {  in alloc_pte()
1566 		amd_iommu_domain_get_pgtable(domain, &pgtable);  in alloc_pte()
1570 	level   = pgtable.mode - 1;  in alloc_pte()
1571 	pte     = &pgtable.root[PM_LEVEL_INDEX(level, address)];  in alloc_pte()
1647 	struct domain_pgtable pgtable;  in fetch_pte()  local
1653 	amd_iommu_domain_get_pgtable(domain, &pgtable);  in fetch_pte()
1655 	if (address > PM_LEVEL_SIZE(pgtable.mode))  in fetch_pte()
1658 	level	   =  pgtable.mode - 1;  in fetch_pte()
1659 	pte	   = &pgtable.root[PM_LEVEL_INDEX(level, address)];  in fetch_pte()
1900 			  struct domain_pgtable *pgtable,  in set_dte_entry()  argument
1907 	if (pgtable->mode != PAGE_MODE_NONE)  in set_dte_entry()
1908 		pte_root = iommu_virt_to_phys(pgtable->root);  in set_dte_entry()
1910 	pte_root |= (pgtable->mode & DEV_ENTRY_MODE_MASK)  in set_dte_entry()
1983 	struct domain_pgtable pgtable;  in do_attach()  local
1999 	amd_iommu_domain_get_pgtable(domain, &pgtable);  in do_attach()
2000 	set_dte_entry(dev_data->devid, domain, &pgtable,  in do_attach()
2308 				struct domain_pgtable *pgtable)  in update_device_table()  argument
2313 		set_dte_entry(dev_data->devid, domain, pgtable,  in update_device_table()
2320 					  struct domain_pgtable *pgtable)  in update_and_flush_device_table()  argument
2322 	update_device_table(domain, pgtable);  in update_and_flush_device_table()
2328 	struct domain_pgtable pgtable;  in update_domain()  local
2331 	amd_iommu_domain_get_pgtable(domain, &pgtable);  in update_domain()
2332 	update_and_flush_device_table(domain, &pgtable);  in update_domain()
2404 	struct domain_pgtable pgtable;  in protection_domain_free()  local
2412 	amd_iommu_domain_get_pgtable(domain, &pgtable);  in protection_domain_free()
2414 	free_pagetable(&pgtable);  in protection_domain_free()
2584 	struct domain_pgtable pgtable;  in amd_iommu_map()  local
2588 	amd_iommu_domain_get_pgtable(domain, &pgtable);  in amd_iommu_map()
2589 	if (pgtable.mode == PAGE_MODE_NONE)  in amd_iommu_map()
2609 	struct domain_pgtable pgtable;  in amd_iommu_unmap()  local
2611 	amd_iommu_domain_get_pgtable(domain, &pgtable);  in amd_iommu_unmap()
2612 	if (pgtable.mode == PAGE_MODE_NONE)  in amd_iommu_unmap()
2623 	struct domain_pgtable pgtable;  in amd_iommu_iova_to_phys()  local
2626 	amd_iommu_domain_get_pgtable(domain, &pgtable);  in amd_iommu_iova_to_phys()
2627 	if (pgtable.mode == PAGE_MODE_NONE)  in amd_iommu_iova_to_phys()
2803 	struct domain_pgtable pgtable;  in amd_iommu_domain_direct_map()  local
2809 	amd_iommu_domain_get_pgtable(domain, &pgtable);  in amd_iommu_domain_direct_map()
2818 	free_pagetable(&pgtable);  in amd_iommu_domain_direct_map()
3007 	struct domain_pgtable pgtable;  in __set_gcr3()  local
3010 	amd_iommu_domain_get_pgtable(domain, &pgtable);  in __set_gcr3()
3011 	if (pgtable.mode != PAGE_MODE_NONE)  in __set_gcr3()
3025 	struct domain_pgtable pgtable;  in __clear_gcr3()  local
3028 	amd_iommu_domain_get_pgtable(domain, &pgtable);  in __clear_gcr3()
3029 	if (pgtable.mode != PAGE_MODE_NONE)  in __clear_gcr3()