295 			       unsigned long iova, size_t size, int lvl,
319 			     unsigned long iova, phys_addr_t paddr,  in arm_lpae_init_pte()  argument
337 		tblp = ptep - ARM_LPAE_LVL_IDX(iova, lvl, data);  in arm_lpae_init_pte()
338 		if (WARN_ON(__arm_lpae_unmap(data, iova, sz, lvl, tblp) != sz))  in arm_lpae_init_pte()
378 static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova,  in __arm_lpae_map()  argument
388 	ptep += ARM_LPAE_LVL_IDX(iova, lvl, data);  in __arm_lpae_map()
392 		return arm_lpae_init_pte(data, iova, paddr, prot, lvl, ptep);  in __arm_lpae_map()
422 	return __arm_lpae_map(data, iova, paddr, size, prot, lvl + 1, cptep);  in __arm_lpae_map()
466 static int arm_lpae_map(struct io_pgtable_ops *ops, unsigned long iova,  in arm_lpae_map()  argument
478 	if (WARN_ON(iova >= (1ULL << data->iop.cfg.ias) ||  in arm_lpae_map()
483 	ret = __arm_lpae_map(data, iova, paddr, size, prot, lvl, ptep);  in arm_lpae_map()
533 				       unsigned long iova, size_t size,  in arm_lpae_split_blk_unmap()  argument
552 		unmap_idx = ARM_LPAE_LVL_IDX(iova, lvl, data);  in arm_lpae_split_blk_unmap()
580 		return __arm_lpae_unmap(data, iova, size, lvl, tablep);  in arm_lpae_split_blk_unmap()
582 	io_pgtable_tlb_add_flush(&data->iop, iova, size, size, true);  in arm_lpae_split_blk_unmap()
587 			       unsigned long iova, size_t size, int lvl,  in __arm_lpae_unmap()  argument
597 	ptep += ARM_LPAE_LVL_IDX(iova, lvl, data);  in __arm_lpae_unmap()
608 			io_pgtable_tlb_add_flush(iop, iova, size,  in __arm_lpae_unmap()
614 			io_pgtable_tlb_add_flush(iop, iova, size, size, true);  in __arm_lpae_unmap()
623 		return arm_lpae_split_blk_unmap(data, iova, size, pte,  in __arm_lpae_unmap()
629 	return __arm_lpae_unmap(data, iova, size, lvl + 1, ptep);  in __arm_lpae_unmap()
632 static size_t arm_lpae_unmap(struct io_pgtable_ops *ops, unsigned long iova,  in arm_lpae_unmap()  argument
639 	if (WARN_ON(iova >= (1ULL << data->iop.cfg.ias)))  in arm_lpae_unmap()
642 	return __arm_lpae_unmap(data, iova, size, lvl, ptep);  in arm_lpae_unmap()
646 					 unsigned long iova)  in arm_lpae_iova_to_phys()  argument
658 		ptep += ARM_LPAE_LVL_IDX(iova, lvl, data);  in arm_lpae_iova_to_phys()
677 	iova &= (ARM_LPAE_BLOCK_SIZE(lvl, data) - 1);  in arm_lpae_iova_to_phys()
678 	return iopte_to_paddr(pte, data) | iova;  in arm_lpae_iova_to_phys()
1019 static void dummy_tlb_add_flush(unsigned long iova, size_t size,  in dummy_tlb_add_flush()  argument
1064 	unsigned long iova;  in arm_lpae_run_tests()  local
1094 		iova = 0;  in arm_lpae_run_tests()
1098 			if (ops->map(ops, iova, iova, size, IOMMU_READ |  in arm_lpae_run_tests()
1105 			if (!ops->map(ops, iova, iova + size, size,  in arm_lpae_run_tests()
1109 			if (ops->iova_to_phys(ops, iova + 42) != (iova + 42))  in arm_lpae_run_tests()
1112 			iova += SZ_1G;  in arm_lpae_run_tests()
1128 		iova = 0;  in arm_lpae_run_tests()
1132 			if (ops->unmap(ops, iova, size) != size)  in arm_lpae_run_tests()
1135 			if (ops->iova_to_phys(ops, iova + 42))  in arm_lpae_run_tests()
1139 			if (ops->map(ops, iova, iova, size, IOMMU_WRITE))  in arm_lpae_run_tests()
1142 			if (ops->iova_to_phys(ops, iova + 42) != (iova + 42))  in arm_lpae_run_tests()
1145 			iova += SZ_1G;  in arm_lpae_run_tests()