20 static int rockchip_gem_iommu_map(struct rockchip_gem_object *rk_obj)  in rockchip_gem_iommu_map()  argument
22 	struct drm_device *drm = rk_obj->base.dev;  in rockchip_gem_iommu_map()
28 	ret = drm_mm_insert_node_generic(&private->mm, &rk_obj->mm,  in rockchip_gem_iommu_map()
29 					 rk_obj->base.size, PAGE_SIZE,  in rockchip_gem_iommu_map()
38 	rk_obj->dma_addr = rk_obj->mm.start;  in rockchip_gem_iommu_map()
40 	ret = iommu_map_sgtable(private->domain, rk_obj->dma_addr, rk_obj->sgt,  in rockchip_gem_iommu_map()
42 	if (ret < rk_obj->base.size) {  in rockchip_gem_iommu_map()
44 			  ret, rk_obj->base.size);  in rockchip_gem_iommu_map()
49 	rk_obj->size = ret;  in rockchip_gem_iommu_map()
55 	drm_mm_remove_node(&rk_obj->mm);  in rockchip_gem_iommu_map()
61 static int rockchip_gem_iommu_unmap(struct rockchip_gem_object *rk_obj)  in rockchip_gem_iommu_unmap()  argument
63 	struct drm_device *drm = rk_obj->base.dev;  in rockchip_gem_iommu_unmap()
66 	iommu_unmap(private->domain, rk_obj->dma_addr, rk_obj->size);  in rockchip_gem_iommu_unmap()
70 	drm_mm_remove_node(&rk_obj->mm);  in rockchip_gem_iommu_unmap()
77 static int rockchip_gem_get_pages(struct rockchip_gem_object *rk_obj)  in rockchip_gem_get_pages()  argument
79 	struct drm_device *drm = rk_obj->base.dev;  in rockchip_gem_get_pages()
83 	rk_obj->pages = drm_gem_get_pages(&rk_obj->base);  in rockchip_gem_get_pages()
84 	if (IS_ERR(rk_obj->pages))  in rockchip_gem_get_pages()
85 		return PTR_ERR(rk_obj->pages);  in rockchip_gem_get_pages()
87 	rk_obj->num_pages = rk_obj->base.size >> PAGE_SHIFT;  in rockchip_gem_get_pages()
89 	rk_obj->sgt = drm_prime_pages_to_sg(rk_obj->base.dev,  in rockchip_gem_get_pages()
90 					    rk_obj->pages, rk_obj->num_pages);  in rockchip_gem_get_pages()
91 	if (IS_ERR(rk_obj->sgt)) {  in rockchip_gem_get_pages()
92 		ret = PTR_ERR(rk_obj->sgt);  in rockchip_gem_get_pages()
103 	for_each_sgtable_sg(rk_obj->sgt, s, i)  in rockchip_gem_get_pages()
106 	dma_sync_sgtable_for_device(drm->dev, rk_obj->sgt, DMA_TO_DEVICE);  in rockchip_gem_get_pages()
111 	drm_gem_put_pages(&rk_obj->base, rk_obj->pages, false, false);  in rockchip_gem_get_pages()
115 static void rockchip_gem_put_pages(struct rockchip_gem_object *rk_obj)  in rockchip_gem_put_pages()  argument
117 	sg_free_table(rk_obj->sgt);  in rockchip_gem_put_pages()
118 	kfree(rk_obj->sgt);  in rockchip_gem_put_pages()
119 	drm_gem_put_pages(&rk_obj->base, rk_obj->pages, true, true);  in rockchip_gem_put_pages()
122 static int rockchip_gem_alloc_iommu(struct rockchip_gem_object *rk_obj,  in rockchip_gem_alloc_iommu()  argument
127 	ret = rockchip_gem_get_pages(rk_obj);  in rockchip_gem_alloc_iommu()
131 	ret = rockchip_gem_iommu_map(rk_obj);  in rockchip_gem_alloc_iommu()
136 		rk_obj->kvaddr = vmap(rk_obj->pages, rk_obj->num_pages, VM_MAP,  in rockchip_gem_alloc_iommu()
138 		if (!rk_obj->kvaddr) {  in rockchip_gem_alloc_iommu()
148 	rockchip_gem_iommu_unmap(rk_obj);  in rockchip_gem_alloc_iommu()
150 	rockchip_gem_put_pages(rk_obj);  in rockchip_gem_alloc_iommu()
155 static int rockchip_gem_alloc_dma(struct rockchip_gem_object *rk_obj,  in rockchip_gem_alloc_dma()  argument
158 	struct drm_gem_object *obj = &rk_obj->base;  in rockchip_gem_alloc_dma()
161 	rk_obj->dma_attrs = DMA_ATTR_WRITE_COMBINE;  in rockchip_gem_alloc_dma()
164 		rk_obj->dma_attrs |= DMA_ATTR_NO_KERNEL_MAPPING;  in rockchip_gem_alloc_dma()
166 	rk_obj->kvaddr = dma_alloc_attrs(drm->dev, obj->size,  in rockchip_gem_alloc_dma()
167 					 &rk_obj->dma_addr, GFP_KERNEL,  in rockchip_gem_alloc_dma()
168 					 rk_obj->dma_attrs);  in rockchip_gem_alloc_dma()
169 	if (!rk_obj->kvaddr) {  in rockchip_gem_alloc_dma()
177 static int rockchip_gem_alloc_buf(struct rockchip_gem_object *rk_obj,  in rockchip_gem_alloc_buf()  argument
180 	struct drm_gem_object *obj = &rk_obj->base;  in rockchip_gem_alloc_buf()
185 		return rockchip_gem_alloc_iommu(rk_obj, alloc_kmap);  in rockchip_gem_alloc_buf()
187 		return rockchip_gem_alloc_dma(rk_obj, alloc_kmap);  in rockchip_gem_alloc_buf()
190 static void rockchip_gem_free_iommu(struct rockchip_gem_object *rk_obj)  in rockchip_gem_free_iommu()  argument
192 	vunmap(rk_obj->kvaddr);  in rockchip_gem_free_iommu()
193 	rockchip_gem_iommu_unmap(rk_obj);  in rockchip_gem_free_iommu()
194 	rockchip_gem_put_pages(rk_obj);  in rockchip_gem_free_iommu()
197 static void rockchip_gem_free_dma(struct rockchip_gem_object *rk_obj)  in rockchip_gem_free_dma()  argument
199 	struct drm_gem_object *obj = &rk_obj->base;  in rockchip_gem_free_dma()
202 	dma_free_attrs(drm->dev, obj->size, rk_obj->kvaddr, rk_obj->dma_addr,  in rockchip_gem_free_dma()
203 		       rk_obj->dma_attrs);  in rockchip_gem_free_dma()
206 static void rockchip_gem_free_buf(struct rockchip_gem_object *rk_obj)  in rockchip_gem_free_buf()  argument
208 	if (rk_obj->pages)  in rockchip_gem_free_buf()
209 		rockchip_gem_free_iommu(rk_obj);  in rockchip_gem_free_buf()
211 		rockchip_gem_free_dma(rk_obj);  in rockchip_gem_free_buf()
217 	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);  in rockchip_drm_gem_object_mmap_iommu()  local
224 	return vm_map_pages(vma, rk_obj->pages, count);  in rockchip_drm_gem_object_mmap_iommu()
230 	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);  in rockchip_drm_gem_object_mmap_dma()  local
233 	return dma_mmap_attrs(drm->dev, vma, rk_obj->kvaddr, rk_obj->dma_addr,  in rockchip_drm_gem_object_mmap_dma()
234 			      obj->size, rk_obj->dma_attrs);  in rockchip_drm_gem_object_mmap_dma()
241 	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);  in rockchip_drm_gem_object_mmap()  local
259 	if (rk_obj->pages)  in rockchip_drm_gem_object_mmap()
270 static void rockchip_gem_release_object(struct rockchip_gem_object *rk_obj)  in rockchip_gem_release_object()  argument
272 	drm_gem_object_release(&rk_obj->base);  in rockchip_gem_release_object()
273 	kfree(rk_obj);  in rockchip_gem_release_object()
288 	struct rockchip_gem_object *rk_obj;  in rockchip_gem_alloc_object()  local
293 	rk_obj = kzalloc(sizeof(*rk_obj), GFP_KERNEL);  in rockchip_gem_alloc_object()
294 	if (!rk_obj)  in rockchip_gem_alloc_object()
297 	obj = &rk_obj->base;  in rockchip_gem_alloc_object()
303 	return rk_obj;  in rockchip_gem_alloc_object()
310 	struct rockchip_gem_object *rk_obj;  in rockchip_gem_create_object()  local
313 	rk_obj = rockchip_gem_alloc_object(drm, size);  in rockchip_gem_create_object()
314 	if (IS_ERR(rk_obj))  in rockchip_gem_create_object()
315 		return rk_obj;  in rockchip_gem_create_object()
317 	ret = rockchip_gem_alloc_buf(rk_obj, alloc_kmap);  in rockchip_gem_create_object()
321 	return rk_obj;  in rockchip_gem_create_object()
324 	rockchip_gem_release_object(rk_obj);  in rockchip_gem_create_object()
336 	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);  in rockchip_gem_free_object()  local
340 			rockchip_gem_iommu_unmap(rk_obj);  in rockchip_gem_free_object()
342 			dma_unmap_sgtable(drm->dev, rk_obj->sgt,  in rockchip_gem_free_object()
345 		drm_prime_gem_destroy(obj, rk_obj->sgt);  in rockchip_gem_free_object()
347 		rockchip_gem_free_buf(rk_obj);  in rockchip_gem_free_object()
350 	rockchip_gem_release_object(rk_obj);  in rockchip_gem_free_object()
365 	struct rockchip_gem_object *rk_obj;  in rockchip_gem_create_with_handle()  local
372 	rk_obj = rockchip_gem_create_object(drm, size, is_framebuffer);  in rockchip_gem_create_with_handle()
373 	if (IS_ERR(rk_obj))  in rockchip_gem_create_with_handle()
374 		return ERR_CAST(rk_obj);  in rockchip_gem_create_with_handle()
376 	obj = &rk_obj->base;  in rockchip_gem_create_with_handle()
389 	return rk_obj;  in rockchip_gem_create_with_handle()
408 	struct rockchip_gem_object *rk_obj;  in rockchip_gem_dumb_create()  local
417 	rk_obj = rockchip_gem_create_with_handle(file_priv, dev, args->size,  in rockchip_gem_dumb_create()
420 	return PTR_ERR_OR_ZERO(rk_obj);  in rockchip_gem_dumb_create()
431 	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);  in rockchip_gem_prime_get_sg_table()  local
436 	if (rk_obj->pages)  in rockchip_gem_prime_get_sg_table()
437 		return drm_prime_pages_to_sg(obj->dev, rk_obj->pages, rk_obj->num_pages);  in rockchip_gem_prime_get_sg_table()
443 	ret = dma_get_sgtable_attrs(drm->dev, sgt, rk_obj->kvaddr,  in rockchip_gem_prime_get_sg_table()
444 				    rk_obj->dma_addr, obj->size,  in rockchip_gem_prime_get_sg_table()
445 				    rk_obj->dma_attrs);  in rockchip_gem_prime_get_sg_table()
459 			  struct rockchip_gem_object *rk_obj)  in rockchip_gem_iommu_map_sg()  argument
461 	rk_obj->sgt = sg;  in rockchip_gem_iommu_map_sg()
462 	return rockchip_gem_iommu_map(rk_obj);  in rockchip_gem_iommu_map_sg()
469 			struct rockchip_gem_object *rk_obj)  in rockchip_gem_dma_map_sg()  argument
481 	rk_obj->dma_addr = sg_dma_address(sg->sgl);  in rockchip_gem_dma_map_sg()
482 	rk_obj->sgt = sg;  in rockchip_gem_dma_map_sg()
492 	struct rockchip_gem_object *rk_obj;  in rockchip_gem_prime_import_sg_table()  local
495 	rk_obj = rockchip_gem_alloc_object(drm, attach->dmabuf->size);  in rockchip_gem_prime_import_sg_table()
496 	if (IS_ERR(rk_obj))  in rockchip_gem_prime_import_sg_table()
497 		return ERR_CAST(rk_obj);  in rockchip_gem_prime_import_sg_table()
500 		ret = rockchip_gem_iommu_map_sg(drm, attach, sg, rk_obj);  in rockchip_gem_prime_import_sg_table()
502 		ret = rockchip_gem_dma_map_sg(drm, attach, sg, rk_obj);  in rockchip_gem_prime_import_sg_table()
509 	return &rk_obj->base;  in rockchip_gem_prime_import_sg_table()
512 	rockchip_gem_release_object(rk_obj);  in rockchip_gem_prime_import_sg_table()
518 	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);  in rockchip_gem_prime_vmap()  local
520 	if (rk_obj->pages) {  in rockchip_gem_prime_vmap()
521 		void *vaddr = vmap(rk_obj->pages, rk_obj->num_pages, VM_MAP,  in rockchip_gem_prime_vmap()
529 	if (rk_obj->dma_attrs & DMA_ATTR_NO_KERNEL_MAPPING)  in rockchip_gem_prime_vmap()
531 	iosys_map_set_vaddr(map, rk_obj->kvaddr);  in rockchip_gem_prime_vmap()
539 	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);  in rockchip_gem_prime_vunmap()  local
541 	if (rk_obj->pages) {  in rockchip_gem_prime_vunmap()