31 	/* Wait for the MMU status to indicate there is no active command, in  in wait_ready()
46 	/* write AS_COMMAND when MMU is ready to accept another command */  in write_cmd()
92 	/* Run the MMU operation */  in mmu_hw_do_operation_locked()
100 			       struct panfrost_mmu *mmu,  in mmu_hw_do_operation()  argument
106 	ret = mmu_hw_do_operation_locked(pfdev, mmu->as, iova, size, op);  in mmu_hw_do_operation()
111 static void panfrost_mmu_enable(struct panfrost_device *pfdev, struct panfrost_mmu *mmu)  in panfrost_mmu_enable()  argument
113 	int as_nr = mmu->as;  in panfrost_mmu_enable()
114 	struct io_pgtable_cfg *cfg = &mmu->pgtbl_cfg;  in panfrost_mmu_enable()
145 u32 panfrost_mmu_as_get(struct panfrost_device *pfdev, struct panfrost_mmu *mmu)  in panfrost_mmu_as_get()  argument
151 	as = mmu->as;  in panfrost_mmu_as_get()
153 		int en = atomic_inc_return(&mmu->as_count);  in panfrost_mmu_as_get()
161 		list_move(&mmu->list, &pfdev->as_lru_list);  in panfrost_mmu_as_get()
184 	mmu->as = as;  in panfrost_mmu_as_get()
186 	atomic_set(&mmu->as_count, 1);  in panfrost_mmu_as_get()
187 	list_add(&mmu->list, &pfdev->as_lru_list);  in panfrost_mmu_as_get()
189 	dev_dbg(pfdev->dev, "Assigned AS%d to mmu %p, alloc_mask=%lx", as, mmu, pfdev->as_alloc_mask);  in panfrost_mmu_as_get()
191 	panfrost_mmu_enable(pfdev, mmu);  in panfrost_mmu_as_get()
198 void panfrost_mmu_as_put(struct panfrost_device *pfdev, struct panfrost_mmu *mmu)  in panfrost_mmu_as_put()  argument
200 	atomic_dec(&mmu->as_count);  in panfrost_mmu_as_put()
201 	WARN_ON(atomic_read(&mmu->as_count) < 0);  in panfrost_mmu_as_put()
206 	struct panfrost_mmu *mmu, *mmu_tmp;  in panfrost_mmu_reset()  local
212 	list_for_each_entry_safe(mmu, mmu_tmp, &pfdev->as_lru_list, list) {  in panfrost_mmu_reset()
213 		mmu->as = -1;  in panfrost_mmu_reset()
214 		atomic_set(&mmu->as_count, 0);  in panfrost_mmu_reset()
215 		list_del_init(&mmu->list);  in panfrost_mmu_reset()
233 				     struct panfrost_mmu *mmu,  in panfrost_mmu_flush_range()  argument
236 	if (mmu->as < 0)  in panfrost_mmu_flush_range()
243 		mmu_hw_do_operation(pfdev, mmu, iova, size, AS_COMMAND_FLUSH_PT);  in panfrost_mmu_flush_range()
248 static int mmu_map_sg(struct panfrost_device *pfdev, struct panfrost_mmu *mmu,  in mmu_map_sg()  argument
253 	struct io_pgtable_ops *ops = mmu->pgtbl_ops;  in mmu_map_sg()
260 		dev_dbg(pfdev->dev, "map: as=%d, iova=%llx, paddr=%lx, len=%zx", mmu->as, iova, paddr, len);  in mmu_map_sg()
272 	panfrost_mmu_flush_range(pfdev, mmu, start_iova, iova - start_iova);  in mmu_map_sg()
295 	mmu_map_sg(pfdev, mapping->mmu, mapping->mmnode.start << PAGE_SHIFT,  in panfrost_mmu_map()
307 	struct io_pgtable_ops *ops = mapping->mmu->pgtbl_ops;  in panfrost_mmu_unmap()
316 		mapping->mmu->as, iova, len);  in panfrost_mmu_unmap()
330 	panfrost_mmu_flush_range(pfdev, mapping->mmu,  in panfrost_mmu_unmap()
364 	struct panfrost_mmu *mmu = &priv->mmu;  in panfrost_mmu_pgtable_alloc()  local
367 	INIT_LIST_HEAD(&mmu->list);  in panfrost_mmu_pgtable_alloc()
368 	mmu->as = -1;  in panfrost_mmu_pgtable_alloc()
370 	mmu->pgtbl_cfg = (struct io_pgtable_cfg) {  in panfrost_mmu_pgtable_alloc()
378 	mmu->pgtbl_ops = alloc_io_pgtable_ops(ARM_MALI_LPAE, &mmu->pgtbl_cfg,  in panfrost_mmu_pgtable_alloc()
380 	if (!mmu->pgtbl_ops)  in panfrost_mmu_pgtable_alloc()
389 	struct panfrost_mmu *mmu = &priv->mmu;  in panfrost_mmu_pgtable_free()  local
392 	if (mmu->as >= 0) {  in panfrost_mmu_pgtable_free()
395 			panfrost_mmu_disable(pfdev, mmu->as);  in panfrost_mmu_pgtable_free()
398 		clear_bit(mmu->as, &pfdev->as_alloc_mask);  in panfrost_mmu_pgtable_free()
399 		clear_bit(mmu->as, &pfdev->as_in_use_mask);  in panfrost_mmu_pgtable_free()
400 		list_del(&mmu->list);  in panfrost_mmu_pgtable_free()
404 	free_io_pgtable_ops(mmu->pgtbl_ops);  in panfrost_mmu_pgtable_free()
414 	struct panfrost_mmu *mmu;  in addr_to_mapping()  local
417 	list_for_each_entry(mmu, &pfdev->as_lru_list, list) {  in addr_to_mapping()
418 		if (as == mmu->as)  in addr_to_mapping()
424 	priv = container_of(mmu, struct panfrost_file_priv, mmu);  in addr_to_mapping()
468 	WARN_ON(bomapping->mmu->as != as);  in panfrost_mmu_map_fault_addr()
524 	mmu_map_sg(pfdev, bomapping->mmu, addr,  in panfrost_mmu_map_fault_addr()
638 	irq = platform_get_irq_byname(to_platform_device(pfdev->dev), "mmu");  in panfrost_mmu_init()
645 					IRQF_SHARED, KBUILD_MODNAME "-mmu",  in panfrost_mmu_init()
649 		dev_err(pfdev->dev, "failed to request mmu irq");  in panfrost_mmu_init()