282 	struct tegra_smmu_as *as;  in tegra_smmu_domain_alloc()  local
287 	as = kzalloc(sizeof(*as), GFP_KERNEL);  in tegra_smmu_domain_alloc()
288 	if (!as)  in tegra_smmu_domain_alloc()
291 	as->attr = SMMU_PD_READABLE | SMMU_PD_WRITABLE | SMMU_PD_NONSECURE;  in tegra_smmu_domain_alloc()
293 	as->pd = alloc_page(GFP_KERNEL | __GFP_DMA | __GFP_ZERO);  in tegra_smmu_domain_alloc()
294 	if (!as->pd) {  in tegra_smmu_domain_alloc()
295 		kfree(as);  in tegra_smmu_domain_alloc()
299 	as->count = kcalloc(SMMU_NUM_PDE, sizeof(u32), GFP_KERNEL);  in tegra_smmu_domain_alloc()
300 	if (!as->count) {  in tegra_smmu_domain_alloc()
301 		__free_page(as->pd);  in tegra_smmu_domain_alloc()
302 		kfree(as);  in tegra_smmu_domain_alloc()
306 	as->pts = kcalloc(SMMU_NUM_PDE, sizeof(*as->pts), GFP_KERNEL);  in tegra_smmu_domain_alloc()
307 	if (!as->pts) {  in tegra_smmu_domain_alloc()
308 		kfree(as->count);  in tegra_smmu_domain_alloc()
309 		__free_page(as->pd);  in tegra_smmu_domain_alloc()
310 		kfree(as);  in tegra_smmu_domain_alloc()
314 	spin_lock_init(&as->lock);  in tegra_smmu_domain_alloc()
317 	as->domain.geometry.aperture_start = 0;  in tegra_smmu_domain_alloc()
318 	as->domain.geometry.aperture_end = 0xffffffff;  in tegra_smmu_domain_alloc()
319 	as->domain.geometry.force_aperture = true;  in tegra_smmu_domain_alloc()
321 	return &as->domain;  in tegra_smmu_domain_alloc()
326 	struct tegra_smmu_as *as = to_smmu_as(domain);  in tegra_smmu_domain_free()  local
330 	WARN_ON_ONCE(as->use_count);  in tegra_smmu_domain_free()
331 	kfree(as->count);  in tegra_smmu_domain_free()
332 	kfree(as->pts);  in tegra_smmu_domain_free()
333 	kfree(as);  in tegra_smmu_domain_free()
414 				 struct tegra_smmu_as *as)  in tegra_smmu_as_prepare()  argument
421 	if (as->use_count > 0) {  in tegra_smmu_as_prepare()
422 		as->use_count++;  in tegra_smmu_as_prepare()
426 	as->pd_dma = dma_map_page(smmu->dev, as->pd, 0, SMMU_SIZE_PD,  in tegra_smmu_as_prepare()
428 	if (dma_mapping_error(smmu->dev, as->pd_dma)) {  in tegra_smmu_as_prepare()
434 	if (!smmu_dma_addr_valid(smmu, as->pd_dma)) {  in tegra_smmu_as_prepare()
439 	err = tegra_smmu_alloc_asid(smmu, &as->id);  in tegra_smmu_as_prepare()
443 	smmu_flush_ptc(smmu, as->pd_dma, 0);  in tegra_smmu_as_prepare()
444 	smmu_flush_tlb_asid(smmu, as->id);  in tegra_smmu_as_prepare()
446 	smmu_writel(smmu, as->id & 0x7f, SMMU_PTB_ASID);  in tegra_smmu_as_prepare()
447 	value = SMMU_PTB_DATA_VALUE(as->pd_dma, as->attr);  in tegra_smmu_as_prepare()
451 	as->smmu = smmu;  in tegra_smmu_as_prepare()
452 	as->use_count++;  in tegra_smmu_as_prepare()
459 	dma_unmap_page(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE);  in tegra_smmu_as_prepare()
467 				    struct tegra_smmu_as *as)  in tegra_smmu_as_unprepare()  argument
471 	if (--as->use_count > 0) {  in tegra_smmu_as_unprepare()
476 	tegra_smmu_free_asid(smmu, as->id);  in tegra_smmu_as_unprepare()
478 	dma_unmap_page(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE);  in tegra_smmu_as_unprepare()
480 	as->smmu = NULL;  in tegra_smmu_as_unprepare()
490 	struct tegra_smmu_as *as = to_smmu_as(domain);  in tegra_smmu_attach_dev()  local
498 		err = tegra_smmu_as_prepare(smmu, as);  in tegra_smmu_attach_dev()
502 		tegra_smmu_enable(smmu, fwspec->ids[index], as->id);  in tegra_smmu_attach_dev()
512 		tegra_smmu_disable(smmu, fwspec->ids[index], as->id);  in tegra_smmu_attach_dev()
513 		tegra_smmu_as_unprepare(smmu, as);  in tegra_smmu_attach_dev()
522 	struct tegra_smmu_as *as = to_smmu_as(domain);  in tegra_smmu_detach_dev()  local
523 	struct tegra_smmu *smmu = as->smmu;  in tegra_smmu_detach_dev()
530 		tegra_smmu_disable(smmu, fwspec->ids[index], as->id);  in tegra_smmu_detach_dev()
531 		tegra_smmu_as_unprepare(smmu, as);  in tegra_smmu_detach_dev()
535 static void tegra_smmu_set_pde(struct tegra_smmu_as *as, unsigned long iova,  in tegra_smmu_set_pde()  argument
539 	struct tegra_smmu *smmu = as->smmu;  in tegra_smmu_set_pde()
540 	u32 *pd = page_address(as->pd);  in tegra_smmu_set_pde()
547 	dma_sync_single_range_for_device(smmu->dev, as->pd_dma, offset,  in tegra_smmu_set_pde()
551 	smmu_flush_ptc(smmu, as->pd_dma, offset);  in tegra_smmu_set_pde()
552 	smmu_flush_tlb_section(smmu, as->id, iova);  in tegra_smmu_set_pde()
563 static u32 *tegra_smmu_pte_lookup(struct tegra_smmu_as *as, unsigned long iova,  in tegra_smmu_pte_lookup()  argument
567 	struct tegra_smmu *smmu = as->smmu;  in tegra_smmu_pte_lookup()
571 	pt_page = as->pts[pd_index];  in tegra_smmu_pte_lookup()
575 	pd = page_address(as->pd);  in tegra_smmu_pte_lookup()
581 static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova,  in as_get_pte()  argument
585 	struct tegra_smmu *smmu = as->smmu;  in as_get_pte()
587 	if (!as->pts[pde]) {  in as_get_pte()
604 		as->pts[pde] = page;  in as_get_pte()
606 		tegra_smmu_set_pde(as, iova, SMMU_MK_PDE(dma, SMMU_PDE_ATTR |  in as_get_pte()
611 		u32 *pd = page_address(as->pd);  in as_get_pte()
616 	return tegra_smmu_pte_offset(as->pts[pde], iova);  in as_get_pte()
619 static void tegra_smmu_pte_get_use(struct tegra_smmu_as *as, unsigned long iova)  in tegra_smmu_pte_get_use()  argument
623 	as->count[pd_index]++;  in tegra_smmu_pte_get_use()
626 static void tegra_smmu_pte_put_use(struct tegra_smmu_as *as, unsigned long iova)  in tegra_smmu_pte_put_use()  argument
629 	struct page *page = as->pts[pde];  in tegra_smmu_pte_put_use()
635 	if (--as->count[pde] == 0) {  in tegra_smmu_pte_put_use()
636 		struct tegra_smmu *smmu = as->smmu;  in tegra_smmu_pte_put_use()
637 		u32 *pd = page_address(as->pd);  in tegra_smmu_pte_put_use()
640 		tegra_smmu_set_pde(as, iova, 0);  in tegra_smmu_pte_put_use()
644 		as->pts[pde] = NULL;  in tegra_smmu_pte_put_use()
648 static void tegra_smmu_set_pte(struct tegra_smmu_as *as, unsigned long iova,  in tegra_smmu_set_pte()  argument
651 	struct tegra_smmu *smmu = as->smmu;  in tegra_smmu_set_pte()
659 	smmu_flush_tlb_group(smmu, as->id, iova);  in tegra_smmu_set_pte()
663 static struct page *as_get_pde_page(struct tegra_smmu_as *as,  in as_get_pde_page()  argument
668 	struct page *page = as->pts[pde];  in as_get_pde_page()
680 		spin_unlock_irqrestore(&as->lock, *flags);  in as_get_pde_page()
685 		spin_lock_irqsave(&as->lock, *flags);  in as_get_pde_page()
692 	if (as->pts[pde]) {  in as_get_pde_page()
696 		page = as->pts[pde];  in as_get_pde_page()
707 	struct tegra_smmu_as *as = to_smmu_as(domain);  in __tegra_smmu_map()  local
713 	page = as_get_pde_page(as, iova, gfp, flags);  in __tegra_smmu_map()
717 	pte = as_get_pte(as, iova, &pte_dma, page);  in __tegra_smmu_map()
723 		tegra_smmu_pte_get_use(as, iova);  in __tegra_smmu_map()
733 	tegra_smmu_set_pte(as, iova, pte, pte_dma,  in __tegra_smmu_map()
743 	struct tegra_smmu_as *as = to_smmu_as(domain);  in __tegra_smmu_unmap()  local
747 	pte = tegra_smmu_pte_lookup(as, iova, &pte_dma);  in __tegra_smmu_unmap()
751 	tegra_smmu_set_pte(as, iova, pte, pte_dma, 0);  in __tegra_smmu_unmap()
752 	tegra_smmu_pte_put_use(as, iova);  in __tegra_smmu_unmap()
760 	struct tegra_smmu_as *as = to_smmu_as(domain);  in tegra_smmu_map()  local
764 	spin_lock_irqsave(&as->lock, flags);  in tegra_smmu_map()
766 	spin_unlock_irqrestore(&as->lock, flags);  in tegra_smmu_map()
774 	struct tegra_smmu_as *as = to_smmu_as(domain);  in tegra_smmu_unmap()  local
777 	spin_lock_irqsave(&as->lock, flags);  in tegra_smmu_unmap()
779 	spin_unlock_irqrestore(&as->lock, flags);  in tegra_smmu_unmap()
787 	struct tegra_smmu_as *as = to_smmu_as(domain);  in tegra_smmu_iova_to_phys()  local
792 	pte = tegra_smmu_pte_lookup(as, iova, &pte_dma);  in tegra_smmu_iova_to_phys()
796 	pfn = *pte & as->smmu->pfn_mask;  in tegra_smmu_iova_to_phys()
959 	 * the SMMU parent device is the same as the MC, so the reference count  in tegra_smmu_of_xlate()