295 			       unsigned long iova, size_t size, int lvl,
300 				int lvl, arm_lpae_iopte *ptep)  in __arm_lpae_init_pte()  argument
307 	if (lvl == ARM_LPAE_MAX_LEVELS - 1)  in __arm_lpae_init_pte()
320 			     arm_lpae_iopte prot, int lvl,  in arm_lpae_init_pte()  argument
325 	if (iopte_leaf(pte, lvl)) {  in arm_lpae_init_pte()
329 	} else if (iopte_type(pte, lvl) == ARM_LPAE_PTE_TYPE_TABLE) {  in arm_lpae_init_pte()
335 		size_t sz = ARM_LPAE_BLOCK_SIZE(lvl, data);  in arm_lpae_init_pte()
337 		tblp = ptep - ARM_LPAE_LVL_IDX(iova, lvl, data);  in arm_lpae_init_pte()
338 		if (WARN_ON(__arm_lpae_unmap(data, iova, sz, lvl, tblp) != sz))  in arm_lpae_init_pte()
342 	__arm_lpae_init_pte(data, paddr, prot, lvl, ptep);  in arm_lpae_init_pte()
380 			  int lvl, arm_lpae_iopte *ptep)  in __arm_lpae_map()  argument
383 	size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data);  in __arm_lpae_map()
388 	ptep += ARM_LPAE_LVL_IDX(iova, lvl, data);  in __arm_lpae_map()
392 		return arm_lpae_init_pte(data, iova, paddr, prot, lvl, ptep);  in __arm_lpae_map()
395 	if (WARN_ON(lvl >= ARM_LPAE_MAX_LEVELS - 1))  in __arm_lpae_map()
413 	if (pte && !iopte_leaf(pte, lvl)) {  in __arm_lpae_map()
422 	return __arm_lpae_map(data, iova, paddr, size, prot, lvl + 1, cptep);  in __arm_lpae_map()
471 	int ret, lvl = ARM_LPAE_START_LVL(data);  in arm_lpae_map()  local
483 	ret = __arm_lpae_map(data, iova, paddr, size, prot, lvl, ptep);  in arm_lpae_map()
493 static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl,  in __arm_lpae_free_pgtable()  argument
499 	if (lvl == ARM_LPAE_START_LVL(data))  in __arm_lpae_free_pgtable()
507 	if (lvl == ARM_LPAE_MAX_LEVELS - 1)  in __arm_lpae_free_pgtable()
515 		if (!pte || iopte_leaf(pte, lvl))  in __arm_lpae_free_pgtable()
518 		__arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data));  in __arm_lpae_free_pgtable()
534 				       arm_lpae_iopte blk_pte, int lvl,  in arm_lpae_split_blk_unmap()  argument
541 	size_t split_sz = ARM_LPAE_BLOCK_SIZE(lvl, data);  in arm_lpae_split_blk_unmap()
544 	if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS))  in arm_lpae_split_blk_unmap()
552 		unmap_idx = ARM_LPAE_LVL_IDX(iova, lvl, data);  in arm_lpae_split_blk_unmap()
562 		__arm_lpae_init_pte(data, blk_paddr, pte, lvl, &tablep[i]);  in arm_lpae_split_blk_unmap()
573 		if (iopte_type(pte, lvl - 1) != ARM_LPAE_PTE_TYPE_TABLE)  in arm_lpae_split_blk_unmap()
580 		return __arm_lpae_unmap(data, iova, size, lvl, tablep);  in arm_lpae_split_blk_unmap()
587 			       unsigned long iova, size_t size, int lvl,  in __arm_lpae_unmap()  argument
594 	if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS))  in __arm_lpae_unmap()
597 	ptep += ARM_LPAE_LVL_IDX(iova, lvl, data);  in __arm_lpae_unmap()
603 	if (size == ARM_LPAE_BLOCK_SIZE(lvl, data)) {  in __arm_lpae_unmap()
606 		if (!iopte_leaf(pte, lvl)) {  in __arm_lpae_unmap()
612 			__arm_lpae_free_pgtable(data, lvl + 1, ptep);  in __arm_lpae_unmap()
618 	} else if (iopte_leaf(pte, lvl)) {  in __arm_lpae_unmap()
624 						lvl + 1, ptep);  in __arm_lpae_unmap()
629 	return __arm_lpae_unmap(data, iova, size, lvl + 1, ptep);  in __arm_lpae_unmap()
637 	int lvl = ARM_LPAE_START_LVL(data);  in arm_lpae_unmap()  local
642 	return __arm_lpae_unmap(data, iova, size, lvl, ptep);  in arm_lpae_unmap()
650 	int lvl = ARM_LPAE_START_LVL(data);  in arm_lpae_iova_to_phys()  local
658 		ptep += ARM_LPAE_LVL_IDX(iova, lvl, data);  in arm_lpae_iova_to_phys()
666 		if (iopte_leaf(pte,lvl))  in arm_lpae_iova_to_phys()
671 	} while (++lvl < ARM_LPAE_MAX_LEVELS);  in arm_lpae_iova_to_phys()
677 	iova &= (ARM_LPAE_BLOCK_SIZE(lvl, data) - 1);  in arm_lpae_iova_to_phys()