278 static void __arm_lpae_sync_pte(arm_lpae_iopte *ptep,  in __arm_lpae_sync_pte()  argument
281 	dma_sync_single_for_device(cfg->iommu_dev, __arm_lpae_dma_addr(ptep),  in __arm_lpae_sync_pte()
282 				   sizeof(*ptep), DMA_TO_DEVICE);  in __arm_lpae_sync_pte()
285 static void __arm_lpae_set_pte(arm_lpae_iopte *ptep, arm_lpae_iopte pte,  in __arm_lpae_set_pte()  argument
288 	*ptep = pte;  in __arm_lpae_set_pte()
291 		__arm_lpae_sync_pte(ptep, cfg);  in __arm_lpae_set_pte()
297 			       arm_lpae_iopte *ptep);
301 				int lvl, arm_lpae_iopte *ptep)  in __arm_lpae_init_pte()  argument
318 	__arm_lpae_set_pte(ptep, pte, &data->iop.cfg);  in __arm_lpae_init_pte()
324 			     arm_lpae_iopte *ptep)  in arm_lpae_init_pte()  argument
326 	arm_lpae_iopte pte = *ptep;  in arm_lpae_init_pte()
340 		tblp = ptep - ARM_LPAE_LVL_IDX(iova, lvl, data);  in arm_lpae_init_pte()
347 	__arm_lpae_init_pte(data, paddr, prot, lvl, ptep);  in arm_lpae_init_pte()
352 					     arm_lpae_iopte *ptep,  in arm_lpae_install_table()  argument
369 	old = cmpxchg64_relaxed(ptep, curr, new);  in arm_lpae_install_table()
375 	__arm_lpae_sync_pte(ptep, cfg);  in arm_lpae_install_table()
377 		WRITE_ONCE(*ptep, new | ARM_LPAE_PTE_SW_SYNC);  in arm_lpae_install_table()
384 			  int lvl, arm_lpae_iopte *ptep)  in __arm_lpae_map()  argument
392 	ptep += ARM_LPAE_LVL_IDX(iova, lvl, data);  in __arm_lpae_map()
396 		return arm_lpae_init_pte(data, iova, paddr, prot, lvl, ptep);  in __arm_lpae_map()
403 	pte = READ_ONCE(*ptep);  in __arm_lpae_map()
409 		pte = arm_lpae_install_table(cptep, ptep, 0, cfg);  in __arm_lpae_map()
413 		__arm_lpae_sync_pte(ptep, cfg);  in __arm_lpae_map()
482 	arm_lpae_iopte *ptep = data->pgd;  in arm_lpae_map()  local
495 	ret = __arm_lpae_map(data, iova, paddr, size, prot, lvl, ptep);  in arm_lpae_map()
506 				    arm_lpae_iopte *ptep)  in __arm_lpae_free_pgtable()  argument
516 	start = ptep;  in __arm_lpae_free_pgtable()
520 		end = ptep;  in __arm_lpae_free_pgtable()
522 		end = (void *)ptep + table_size;  in __arm_lpae_free_pgtable()
524 	while (ptep != end) {  in __arm_lpae_free_pgtable()
525 		arm_lpae_iopte pte = *ptep++;  in __arm_lpae_free_pgtable()
548 				       arm_lpae_iopte *ptep)  in arm_lpae_split_blk_unmap()  argument
578 	pte = arm_lpae_install_table(tablep, ptep, blk_pte, cfg);  in arm_lpae_split_blk_unmap()
601 			       arm_lpae_iopte *ptep)  in __arm_lpae_unmap()  argument
610 	ptep += ARM_LPAE_LVL_IDX(iova, lvl, data);  in __arm_lpae_unmap()
611 	pte = READ_ONCE(*ptep);  in __arm_lpae_unmap()
617 		__arm_lpae_set_pte(ptep, 0, &iop->cfg);  in __arm_lpae_unmap()
623 			ptep = iopte_deref(pte, data);  in __arm_lpae_unmap()
624 			__arm_lpae_free_pgtable(data, lvl + 1, ptep);  in __arm_lpae_unmap()
643 						lvl + 1, ptep);  in __arm_lpae_unmap()
647 	ptep = iopte_deref(pte, data);  in __arm_lpae_unmap()
648 	return __arm_lpae_unmap(data, gather, iova, size, lvl + 1, ptep);  in __arm_lpae_unmap()
655 	arm_lpae_iopte *ptep = data->pgd;  in arm_lpae_unmap()  local
661 	return __arm_lpae_unmap(data, gather, iova, size, lvl, ptep);  in arm_lpae_unmap()
668 	arm_lpae_iopte pte, *ptep = data->pgd;  in arm_lpae_iova_to_phys()  local
673 		if (!ptep)  in arm_lpae_iova_to_phys()
677 		ptep += ARM_LPAE_LVL_IDX(iova, lvl, data);  in arm_lpae_iova_to_phys()
678 		pte = READ_ONCE(*ptep);  in arm_lpae_iova_to_phys()
689 		ptep = iopte_deref(pte, data);  in arm_lpae_iova_to_phys()