18 static int rockchip_gem_iommu_map(struct rockchip_gem_object *rk_obj)  in rockchip_gem_iommu_map()  argument
20 	struct drm_device *drm = rk_obj->base.dev;  in rockchip_gem_iommu_map()
26 	ret = drm_mm_insert_node_generic(&private->mm, &rk_obj->mm,  in rockchip_gem_iommu_map()
27 					 rk_obj->base.size, PAGE_SIZE,  in rockchip_gem_iommu_map()
36 	rk_obj->dma_addr = rk_obj->mm.start;  in rockchip_gem_iommu_map()
38 	ret = iommu_map_sg(private->domain, rk_obj->dma_addr, rk_obj->sgt->sgl,  in rockchip_gem_iommu_map()
39 			   rk_obj->sgt->nents, prot);  in rockchip_gem_iommu_map()
40 	if (ret < rk_obj->base.size) {  in rockchip_gem_iommu_map()
42 			  ret, rk_obj->base.size);  in rockchip_gem_iommu_map()
47 	rk_obj->size = ret;  in rockchip_gem_iommu_map()
53 	drm_mm_remove_node(&rk_obj->mm);  in rockchip_gem_iommu_map()
59 static int rockchip_gem_iommu_unmap(struct rockchip_gem_object *rk_obj)  in rockchip_gem_iommu_unmap()  argument
61 	struct drm_device *drm = rk_obj->base.dev;  in rockchip_gem_iommu_unmap()
64 	iommu_unmap(private->domain, rk_obj->dma_addr, rk_obj->size);  in rockchip_gem_iommu_unmap()
68 	drm_mm_remove_node(&rk_obj->mm);  in rockchip_gem_iommu_unmap()
75 static int rockchip_gem_get_pages(struct rockchip_gem_object *rk_obj)  in rockchip_gem_get_pages()  argument
77 	struct drm_device *drm = rk_obj->base.dev;  in rockchip_gem_get_pages()
81 	rk_obj->pages = drm_gem_get_pages(&rk_obj->base);  in rockchip_gem_get_pages()
82 	if (IS_ERR(rk_obj->pages))  in rockchip_gem_get_pages()
83 		return PTR_ERR(rk_obj->pages);  in rockchip_gem_get_pages()
85 	rk_obj->num_pages = rk_obj->base.size >> PAGE_SHIFT;  in rockchip_gem_get_pages()
87 	rk_obj->sgt = drm_prime_pages_to_sg(rk_obj->pages, rk_obj->num_pages);  in rockchip_gem_get_pages()
88 	if (IS_ERR(rk_obj->sgt)) {  in rockchip_gem_get_pages()
89 		ret = PTR_ERR(rk_obj->sgt);  in rockchip_gem_get_pages()
100 	for_each_sg(rk_obj->sgt->sgl, s, rk_obj->sgt->nents, i)  in rockchip_gem_get_pages()
103 	dma_sync_sg_for_device(drm->dev, rk_obj->sgt->sgl, rk_obj->sgt->nents,  in rockchip_gem_get_pages()
109 	drm_gem_put_pages(&rk_obj->base, rk_obj->pages, false, false);  in rockchip_gem_get_pages()
113 static void rockchip_gem_put_pages(struct rockchip_gem_object *rk_obj)  in rockchip_gem_put_pages()  argument
115 	sg_free_table(rk_obj->sgt);  in rockchip_gem_put_pages()
116 	kfree(rk_obj->sgt);  in rockchip_gem_put_pages()
117 	drm_gem_put_pages(&rk_obj->base, rk_obj->pages, true, true);  in rockchip_gem_put_pages()
120 static int rockchip_gem_alloc_iommu(struct rockchip_gem_object *rk_obj,  in rockchip_gem_alloc_iommu()  argument
125 	ret = rockchip_gem_get_pages(rk_obj);  in rockchip_gem_alloc_iommu()
129 	ret = rockchip_gem_iommu_map(rk_obj);  in rockchip_gem_alloc_iommu()
134 		rk_obj->kvaddr = vmap(rk_obj->pages, rk_obj->num_pages, VM_MAP,  in rockchip_gem_alloc_iommu()
136 		if (!rk_obj->kvaddr) {  in rockchip_gem_alloc_iommu()
146 	rockchip_gem_iommu_unmap(rk_obj);  in rockchip_gem_alloc_iommu()
148 	rockchip_gem_put_pages(rk_obj);  in rockchip_gem_alloc_iommu()
153 static int rockchip_gem_alloc_dma(struct rockchip_gem_object *rk_obj,  in rockchip_gem_alloc_dma()  argument
156 	struct drm_gem_object *obj = &rk_obj->base;  in rockchip_gem_alloc_dma()
159 	rk_obj->dma_attrs = DMA_ATTR_WRITE_COMBINE;  in rockchip_gem_alloc_dma()
162 		rk_obj->dma_attrs |= DMA_ATTR_NO_KERNEL_MAPPING;  in rockchip_gem_alloc_dma()
164 	rk_obj->kvaddr = dma_alloc_attrs(drm->dev, obj->size,  in rockchip_gem_alloc_dma()
165 					 &rk_obj->dma_addr, GFP_KERNEL,  in rockchip_gem_alloc_dma()
166 					 rk_obj->dma_attrs);  in rockchip_gem_alloc_dma()
167 	if (!rk_obj->kvaddr) {  in rockchip_gem_alloc_dma()
175 static int rockchip_gem_alloc_buf(struct rockchip_gem_object *rk_obj,  in rockchip_gem_alloc_buf()  argument
178 	struct drm_gem_object *obj = &rk_obj->base;  in rockchip_gem_alloc_buf()
183 		return rockchip_gem_alloc_iommu(rk_obj, alloc_kmap);  in rockchip_gem_alloc_buf()
185 		return rockchip_gem_alloc_dma(rk_obj, alloc_kmap);  in rockchip_gem_alloc_buf()
188 static void rockchip_gem_free_iommu(struct rockchip_gem_object *rk_obj)  in rockchip_gem_free_iommu()  argument
190 	vunmap(rk_obj->kvaddr);  in rockchip_gem_free_iommu()
191 	rockchip_gem_iommu_unmap(rk_obj);  in rockchip_gem_free_iommu()
192 	rockchip_gem_put_pages(rk_obj);  in rockchip_gem_free_iommu()
195 static void rockchip_gem_free_dma(struct rockchip_gem_object *rk_obj)  in rockchip_gem_free_dma()  argument
197 	struct drm_gem_object *obj = &rk_obj->base;  in rockchip_gem_free_dma()
200 	dma_free_attrs(drm->dev, obj->size, rk_obj->kvaddr, rk_obj->dma_addr,  in rockchip_gem_free_dma()
201 		       rk_obj->dma_attrs);  in rockchip_gem_free_dma()
204 static void rockchip_gem_free_buf(struct rockchip_gem_object *rk_obj)  in rockchip_gem_free_buf()  argument
206 	if (rk_obj->pages)  in rockchip_gem_free_buf()
207 		rockchip_gem_free_iommu(rk_obj);  in rockchip_gem_free_buf()
209 		rockchip_gem_free_dma(rk_obj);  in rockchip_gem_free_buf()
215 	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);  in rockchip_drm_gem_object_mmap_iommu()  local
222 	return vm_map_pages(vma, rk_obj->pages, count);  in rockchip_drm_gem_object_mmap_iommu()
228 	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);  in rockchip_drm_gem_object_mmap_dma()  local
231 	return dma_mmap_attrs(drm->dev, vma, rk_obj->kvaddr, rk_obj->dma_addr,  in rockchip_drm_gem_object_mmap_dma()
232 			      obj->size, rk_obj->dma_attrs);  in rockchip_drm_gem_object_mmap_dma()
239 	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);  in rockchip_drm_gem_object_mmap()  local
247 	if (rk_obj->pages)  in rockchip_drm_gem_object_mmap()
291 static void rockchip_gem_release_object(struct rockchip_gem_object *rk_obj)  in rockchip_gem_release_object()  argument
293 	drm_gem_object_release(&rk_obj->base);  in rockchip_gem_release_object()
294 	kfree(rk_obj);  in rockchip_gem_release_object()
300 	struct rockchip_gem_object *rk_obj;  in rockchip_gem_alloc_object()  local
305 	rk_obj = kzalloc(sizeof(*rk_obj), GFP_KERNEL);  in rockchip_gem_alloc_object()
306 	if (!rk_obj)  in rockchip_gem_alloc_object()
309 	obj = &rk_obj->base;  in rockchip_gem_alloc_object()
313 	return rk_obj;  in rockchip_gem_alloc_object()
320 	struct rockchip_gem_object *rk_obj;  in rockchip_gem_create_object()  local
323 	rk_obj = rockchip_gem_alloc_object(drm, size);  in rockchip_gem_create_object()
324 	if (IS_ERR(rk_obj))  in rockchip_gem_create_object()
325 		return rk_obj;  in rockchip_gem_create_object()
327 	ret = rockchip_gem_alloc_buf(rk_obj, alloc_kmap);  in rockchip_gem_create_object()
331 	return rk_obj;  in rockchip_gem_create_object()
334 	rockchip_gem_release_object(rk_obj);  in rockchip_gem_create_object()
346 	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);  in rockchip_gem_free_object()  local
350 			rockchip_gem_iommu_unmap(rk_obj);  in rockchip_gem_free_object()
352 			dma_unmap_sg(drm->dev, rk_obj->sgt->sgl,  in rockchip_gem_free_object()
353 				     rk_obj->sgt->nents, DMA_BIDIRECTIONAL);  in rockchip_gem_free_object()
355 		drm_prime_gem_destroy(obj, rk_obj->sgt);  in rockchip_gem_free_object()
357 		rockchip_gem_free_buf(rk_obj);  in rockchip_gem_free_object()
360 	rockchip_gem_release_object(rk_obj);  in rockchip_gem_free_object()
375 	struct rockchip_gem_object *rk_obj;  in rockchip_gem_create_with_handle()  local
379 	rk_obj = rockchip_gem_create_object(drm, size, false);  in rockchip_gem_create_with_handle()
380 	if (IS_ERR(rk_obj))  in rockchip_gem_create_with_handle()
381 		return ERR_CAST(rk_obj);  in rockchip_gem_create_with_handle()
383 	obj = &rk_obj->base;  in rockchip_gem_create_with_handle()
396 	return rk_obj;  in rockchip_gem_create_with_handle()
415 	struct rockchip_gem_object *rk_obj;  in rockchip_gem_dumb_create()  local
424 	rk_obj = rockchip_gem_create_with_handle(file_priv, dev, args->size,  in rockchip_gem_dumb_create()
427 	return PTR_ERR_OR_ZERO(rk_obj);  in rockchip_gem_dumb_create()
438 	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);  in rockchip_gem_prime_get_sg_table()  local
443 	if (rk_obj->pages)  in rockchip_gem_prime_get_sg_table()
444 		return drm_prime_pages_to_sg(rk_obj->pages, rk_obj->num_pages);  in rockchip_gem_prime_get_sg_table()
450 	ret = dma_get_sgtable_attrs(drm->dev, sgt, rk_obj->kvaddr,  in rockchip_gem_prime_get_sg_table()
451 				    rk_obj->dma_addr, obj->size,  in rockchip_gem_prime_get_sg_table()
452 				    rk_obj->dma_attrs);  in rockchip_gem_prime_get_sg_table()
483 			  struct rockchip_gem_object *rk_obj)  in rockchip_gem_iommu_map_sg()  argument
485 	rk_obj->sgt = sg;  in rockchip_gem_iommu_map_sg()
486 	return rockchip_gem_iommu_map(rk_obj);  in rockchip_gem_iommu_map_sg()
493 			struct rockchip_gem_object *rk_obj)  in rockchip_gem_dma_map_sg()  argument
507 	rk_obj->dma_addr = sg_dma_address(sg->sgl);  in rockchip_gem_dma_map_sg()
508 	rk_obj->sgt = sg;  in rockchip_gem_dma_map_sg()
518 	struct rockchip_gem_object *rk_obj;  in rockchip_gem_prime_import_sg_table()  local
521 	rk_obj = rockchip_gem_alloc_object(drm, attach->dmabuf->size);  in rockchip_gem_prime_import_sg_table()
522 	if (IS_ERR(rk_obj))  in rockchip_gem_prime_import_sg_table()
523 		return ERR_CAST(rk_obj);  in rockchip_gem_prime_import_sg_table()
526 		ret = rockchip_gem_iommu_map_sg(drm, attach, sg, rk_obj);  in rockchip_gem_prime_import_sg_table()
528 		ret = rockchip_gem_dma_map_sg(drm, attach, sg, rk_obj);  in rockchip_gem_prime_import_sg_table()
535 	return &rk_obj->base;  in rockchip_gem_prime_import_sg_table()
538 	rockchip_gem_release_object(rk_obj);  in rockchip_gem_prime_import_sg_table()
544 	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);  in rockchip_gem_prime_vmap()  local
546 	if (rk_obj->pages)  in rockchip_gem_prime_vmap()
547 		return vmap(rk_obj->pages, rk_obj->num_pages, VM_MAP,  in rockchip_gem_prime_vmap()
550 	if (rk_obj->dma_attrs & DMA_ATTR_NO_KERNEL_MAPPING)  in rockchip_gem_prime_vmap()
553 	return rk_obj->kvaddr;  in rockchip_gem_prime_vmap()
558 	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);  in rockchip_gem_prime_vunmap()  local
560 	if (rk_obj->pages) {  in rockchip_gem_prime_vunmap()