278 static void __arm_lpae_sync_pte(arm_lpae_iopte *ptep,  in __arm_lpae_sync_pte()  argument
281 	dma_sync_single_for_device(cfg->iommu_dev, __arm_lpae_dma_addr(ptep),  in __arm_lpae_sync_pte()
282 				   sizeof(*ptep), DMA_TO_DEVICE);  in __arm_lpae_sync_pte()
285 static void __arm_lpae_set_pte(arm_lpae_iopte *ptep, arm_lpae_iopte pte,  in __arm_lpae_set_pte()  argument
288 	*ptep = pte;  in __arm_lpae_set_pte()
291 		__arm_lpae_sync_pte(ptep, cfg);  in __arm_lpae_set_pte()
296 			       arm_lpae_iopte *ptep);
300 				int lvl, arm_lpae_iopte *ptep)  in __arm_lpae_init_pte()  argument
315 	__arm_lpae_set_pte(ptep, pte, &data->iop.cfg);  in __arm_lpae_init_pte()
321 			     arm_lpae_iopte *ptep)  in arm_lpae_init_pte()  argument
323 	arm_lpae_iopte pte = *ptep;  in arm_lpae_init_pte()
337 		tblp = ptep - ARM_LPAE_LVL_IDX(iova, lvl, data);  in arm_lpae_init_pte()
342 	__arm_lpae_init_pte(data, paddr, prot, lvl, ptep);  in arm_lpae_init_pte()
347 					     arm_lpae_iopte *ptep,  in arm_lpae_install_table()  argument
364 	old = cmpxchg64_relaxed(ptep, curr, new);  in arm_lpae_install_table()
371 	__arm_lpae_sync_pte(ptep, cfg);  in arm_lpae_install_table()
373 		WRITE_ONCE(*ptep, new | ARM_LPAE_PTE_SW_SYNC);  in arm_lpae_install_table()
380 			  int lvl, arm_lpae_iopte *ptep)  in __arm_lpae_map()  argument
388 	ptep += ARM_LPAE_LVL_IDX(iova, lvl, data);  in __arm_lpae_map()
392 		return arm_lpae_init_pte(data, iova, paddr, prot, lvl, ptep);  in __arm_lpae_map()
399 	pte = READ_ONCE(*ptep);  in __arm_lpae_map()
405 		pte = arm_lpae_install_table(cptep, ptep, 0, cfg);  in __arm_lpae_map()
410 		__arm_lpae_sync_pte(ptep, cfg);  in __arm_lpae_map()
470 	arm_lpae_iopte *ptep = data->pgd;  in arm_lpae_map()  local
483 	ret = __arm_lpae_map(data, iova, paddr, size, prot, lvl, ptep);  in arm_lpae_map()
494 				    arm_lpae_iopte *ptep)  in __arm_lpae_free_pgtable()  argument
504 	start = ptep;  in __arm_lpae_free_pgtable()
508 		end = ptep;  in __arm_lpae_free_pgtable()
510 		end = (void *)ptep + table_size;  in __arm_lpae_free_pgtable()
512 	while (ptep != end) {  in __arm_lpae_free_pgtable()
513 		arm_lpae_iopte pte = *ptep++;  in __arm_lpae_free_pgtable()
535 				       arm_lpae_iopte *ptep)  in arm_lpae_split_blk_unmap()  argument
565 	pte = arm_lpae_install_table(tablep, ptep, blk_pte, cfg);  in arm_lpae_split_blk_unmap()
588 			       arm_lpae_iopte *ptep)  in __arm_lpae_unmap()  argument
597 	ptep += ARM_LPAE_LVL_IDX(iova, lvl, data);  in __arm_lpae_unmap()
598 	pte = READ_ONCE(*ptep);  in __arm_lpae_unmap()
604 		__arm_lpae_set_pte(ptep, 0, &iop->cfg);  in __arm_lpae_unmap()
611 			ptep = iopte_deref(pte, data);  in __arm_lpae_unmap()
612 			__arm_lpae_free_pgtable(data, lvl + 1, ptep);  in __arm_lpae_unmap()
624 						lvl + 1, ptep);  in __arm_lpae_unmap()
628 	ptep = iopte_deref(pte, data);  in __arm_lpae_unmap()
629 	return __arm_lpae_unmap(data, iova, size, lvl + 1, ptep);  in __arm_lpae_unmap()
636 	arm_lpae_iopte *ptep = data->pgd;  in arm_lpae_unmap()  local
642 	return __arm_lpae_unmap(data, iova, size, lvl, ptep);  in arm_lpae_unmap()
649 	arm_lpae_iopte pte, *ptep = data->pgd;  in arm_lpae_iova_to_phys()  local
654 		if (!ptep)  in arm_lpae_iova_to_phys()
658 		ptep += ARM_LPAE_LVL_IDX(iova, lvl, data);  in arm_lpae_iova_to_phys()
659 		pte = READ_ONCE(*ptep);  in arm_lpae_iova_to_phys()
670 		ptep = iopte_deref(pte, data);  in arm_lpae_iova_to_phys()