151 static bool omap_gem_is_contiguous(struct omap_gem_object *omap_obj)  in omap_gem_is_contiguous()  argument
153 	if (omap_obj->flags & OMAP_BO_MEM_DMA_API)  in omap_gem_is_contiguous()
156 	if ((omap_obj->flags & OMAP_BO_MEM_DMABUF) && omap_obj->sgt->nents == 1)  in omap_gem_is_contiguous()
169 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_evict_entry()  local
175 	const int m = DIV_ROUND_UP(omap_obj->width << fmt, PAGE_SIZE);  in omap_gem_evict_entry()
196 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_evict()  local
199 	if (omap_obj->flags & OMAP_BO_TILED) {  in omap_gem_evict()
200 		enum tiler_fmt fmt = gem2fmt(omap_obj->flags);  in omap_gem_evict()
224 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_attach_pages()  local
230 	lockdep_assert_held(&omap_obj->lock);  in omap_gem_attach_pages()
236 	if (!(omap_obj->flags & OMAP_BO_MEM_SHMEM) || omap_obj->pages)  in omap_gem_attach_pages()
248 	if (omap_obj->flags & (OMAP_BO_WC|OMAP_BO_UNCACHED)) {  in omap_gem_attach_pages()
280 	omap_obj->dma_addrs = addrs;  in omap_gem_attach_pages()
281 	omap_obj->pages = pages;  in omap_gem_attach_pages()
296 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_detach_pages()  local
300 	lockdep_assert_held(&omap_obj->lock);  in omap_gem_detach_pages()
303 		if (omap_obj->dma_addrs[i])  in omap_gem_detach_pages()
304 			dma_unmap_page(obj->dev->dev, omap_obj->dma_addrs[i],  in omap_gem_detach_pages()
308 	kfree(omap_obj->dma_addrs);  in omap_gem_detach_pages()
309 	omap_obj->dma_addrs = NULL;  in omap_gem_detach_pages()
311 	drm_gem_put_pages(obj, omap_obj->pages, true, false);  in omap_gem_detach_pages()
312 	omap_obj->pages = NULL;  in omap_gem_detach_pages()
324 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_mmap_size()  local
327 	if (omap_obj->flags & OMAP_BO_TILED) {  in omap_gem_mmap_size()
334 		size = tiler_vsize(gem2fmt(omap_obj->flags),  in omap_gem_mmap_size()
335 				omap_obj->width, omap_obj->height);  in omap_gem_mmap_size()
349 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_fault_1d()  local
356 	if (omap_obj->pages) {  in omap_gem_fault_1d()
358 		pfn = page_to_pfn(omap_obj->pages[pgoff]);  in omap_gem_fault_1d()
360 		BUG_ON(!omap_gem_is_contiguous(omap_obj));  in omap_gem_fault_1d()
361 		pfn = (omap_obj->dma_addr >> PAGE_SHIFT) + pgoff;  in omap_gem_fault_1d()
375 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_fault_2d()  local
378 	enum tiler_fmt fmt = gem2fmt(omap_obj->flags);  in omap_gem_fault_2d()
400 	const int m = DIV_ROUND_UP(omap_obj->width << fmt, PAGE_SIZE);  in omap_gem_fault_2d()
412 	slots = omap_obj->width >> priv->usergart[fmt].slot_shift;  in omap_gem_fault_2d()
445 	memcpy(pages, &omap_obj->pages[base_pgoff],  in omap_gem_fault_2d()
494 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_fault()  local
501 	mutex_lock(&omap_obj->lock);  in omap_gem_fault()
516 	if (omap_obj->flags & OMAP_BO_TILED)  in omap_gem_fault()
523 	mutex_unlock(&omap_obj->lock);  in omap_gem_fault()
544 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_mmap_obj()  local
549 	if (omap_obj->flags & OMAP_BO_WC) {  in omap_gem_mmap_obj()
551 	} else if (omap_obj->flags & OMAP_BO_UNCACHED) {  in omap_gem_mmap_obj()
646 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_roll()  local
655 	omap_obj->roll = roll;  in omap_gem_roll()
657 	mutex_lock(&omap_obj->lock);  in omap_gem_roll()
660 	if (omap_obj->block) {  in omap_gem_roll()
665 		ret = tiler_pin(omap_obj->block, omap_obj->pages, npages,  in omap_gem_roll()
672 	mutex_unlock(&omap_obj->lock);  in omap_gem_roll()
693 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_is_cached_coherent()  local
695 	return !((omap_obj->flags & OMAP_BO_MEM_SHMEM) &&  in omap_gem_is_cached_coherent()
696 		((omap_obj->flags & OMAP_BO_CACHE_MASK) == OMAP_BO_CACHED));  in omap_gem_is_cached_coherent()
705 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_cpu_sync_page()  local
710 	if (omap_obj->dma_addrs[pgoff]) {  in omap_gem_cpu_sync_page()
711 		dma_unmap_page(dev->dev, omap_obj->dma_addrs[pgoff],  in omap_gem_cpu_sync_page()
713 		omap_obj->dma_addrs[pgoff] = 0;  in omap_gem_cpu_sync_page()
722 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_dma_sync_buffer()  local
724 	struct page **pages = omap_obj->pages;  in omap_gem_dma_sync_buffer()
731 		if (!omap_obj->dma_addrs[i]) {  in omap_gem_dma_sync_buffer()
743 			omap_obj->dma_addrs[i] = addr;  in omap_gem_dma_sync_buffer()
770 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_pin()  local
773 	mutex_lock(&omap_obj->lock);  in omap_gem_pin()
775 	if (!omap_gem_is_contiguous(omap_obj) && priv->has_dmm) {  in omap_gem_pin()
776 		if (omap_obj->dma_addr_cnt == 0) {  in omap_gem_pin()
778 			enum tiler_fmt fmt = gem2fmt(omap_obj->flags);  in omap_gem_pin()
781 			BUG_ON(omap_obj->block);  in omap_gem_pin()
787 			if (omap_obj->flags & OMAP_BO_TILED) {  in omap_gem_pin()
789 						omap_obj->width,  in omap_gem_pin()
790 						omap_obj->height, 0);  in omap_gem_pin()
803 			ret = tiler_pin(block, omap_obj->pages, npages,  in omap_gem_pin()
804 					omap_obj->roll, true);  in omap_gem_pin()
812 			omap_obj->dma_addr = tiler_ssptr(block);  in omap_gem_pin()
813 			omap_obj->block = block;  in omap_gem_pin()
815 			DBG("got dma address: %pad", &omap_obj->dma_addr);  in omap_gem_pin()
818 		omap_obj->dma_addr_cnt++;  in omap_gem_pin()
820 		*dma_addr = omap_obj->dma_addr;  in omap_gem_pin()
821 	} else if (omap_gem_is_contiguous(omap_obj)) {  in omap_gem_pin()
822 		*dma_addr = omap_obj->dma_addr;  in omap_gem_pin()
829 	mutex_unlock(&omap_obj->lock);  in omap_gem_pin()
844 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_unpin()  local
847 	mutex_lock(&omap_obj->lock);  in omap_gem_unpin()
849 	if (omap_obj->dma_addr_cnt > 0) {  in omap_gem_unpin()
850 		omap_obj->dma_addr_cnt--;  in omap_gem_unpin()
851 		if (omap_obj->dma_addr_cnt == 0) {  in omap_gem_unpin()
852 			ret = tiler_unpin(omap_obj->block);  in omap_gem_unpin()
857 			ret = tiler_release(omap_obj->block);  in omap_gem_unpin()
862 			omap_obj->dma_addr = 0;  in omap_gem_unpin()
863 			omap_obj->block = NULL;  in omap_gem_unpin()
867 	mutex_unlock(&omap_obj->lock);  in omap_gem_unpin()
877 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_rotated_dma_addr()  local
880 	mutex_lock(&omap_obj->lock);  in omap_gem_rotated_dma_addr()
882 	if ((omap_obj->dma_addr_cnt > 0) && omap_obj->block &&  in omap_gem_rotated_dma_addr()
883 			(omap_obj->flags & OMAP_BO_TILED)) {  in omap_gem_rotated_dma_addr()
884 		*dma_addr = tiler_tsptr(omap_obj->block, orient, x, y);  in omap_gem_rotated_dma_addr()
888 	mutex_unlock(&omap_obj->lock);  in omap_gem_rotated_dma_addr()
896 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_tiled_stride()  local
898 	if (omap_obj->flags & OMAP_BO_TILED)  in omap_gem_tiled_stride()
899 		ret = tiler_stride(gem2fmt(omap_obj->flags), orient);  in omap_gem_tiled_stride()
916 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_get_pages()  local
919 	mutex_lock(&omap_obj->lock);  in omap_gem_get_pages()
927 	if (!omap_obj->pages) {  in omap_gem_get_pages()
932 	*pages = omap_obj->pages;  in omap_gem_get_pages()
935 	mutex_unlock(&omap_obj->lock);  in omap_gem_get_pages()
957 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_vaddr()  local
961 	mutex_lock(&omap_obj->lock);  in omap_gem_vaddr()
963 	if (!omap_obj->vaddr) {  in omap_gem_vaddr()
970 		omap_obj->vaddr = vmap(omap_obj->pages, obj->size >> PAGE_SHIFT,  in omap_gem_vaddr()
974 	vaddr = omap_obj->vaddr;  in omap_gem_vaddr()
977 	mutex_unlock(&omap_obj->lock);  in omap_gem_vaddr()
991 	struct omap_gem_object *omap_obj;  in omap_gem_resume()  local
995 	list_for_each_entry(omap_obj, &priv->obj_list, mm_list) {  in omap_gem_resume()
996 		if (omap_obj->block) {  in omap_gem_resume()
997 			struct drm_gem_object *obj = &omap_obj->base;  in omap_gem_resume()
1000 			WARN_ON(!omap_obj->pages);  /* this can't happen */  in omap_gem_resume()
1001 			ret = tiler_pin(omap_obj->block,  in omap_gem_resume()
1002 					omap_obj->pages, npages,  in omap_gem_resume()
1003 					omap_obj->roll, true);  in omap_gem_resume()
1024 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_describe()  local
1029 	mutex_lock(&omap_obj->lock);  in omap_gem_describe()
1032 			omap_obj->flags, obj->name, kref_read(&obj->refcount),  in omap_gem_describe()
1033 			off, &omap_obj->dma_addr, omap_obj->dma_addr_cnt,  in omap_gem_describe()
1034 			omap_obj->vaddr, omap_obj->roll);  in omap_gem_describe()
1036 	if (omap_obj->flags & OMAP_BO_TILED) {  in omap_gem_describe()
1037 		seq_printf(m, " %dx%d", omap_obj->width, omap_obj->height);  in omap_gem_describe()
1038 		if (omap_obj->block) {  in omap_gem_describe()
1039 			struct tcm_area *area = &omap_obj->block->area;  in omap_gem_describe()
1048 	mutex_unlock(&omap_obj->lock);  in omap_gem_describe()
1055 	struct omap_gem_object *omap_obj;  in omap_gem_describe_objects()  local
1059 	list_for_each_entry(omap_obj, list, mm_list) {  in omap_gem_describe_objects()
1060 		struct drm_gem_object *obj = &omap_obj->base;  in omap_gem_describe_objects()
1079 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_free_object()  local
1084 	list_del(&omap_obj->mm_list);  in omap_gem_free_object()
1093 	mutex_lock(&omap_obj->lock);  in omap_gem_free_object()
1096 	WARN_ON(omap_obj->dma_addr_cnt > 0);  in omap_gem_free_object()
1098 	if (omap_obj->pages) {  in omap_gem_free_object()
1099 		if (omap_obj->flags & OMAP_BO_MEM_DMABUF)  in omap_gem_free_object()
1100 			kfree(omap_obj->pages);  in omap_gem_free_object()
1105 	if (omap_obj->flags & OMAP_BO_MEM_DMA_API) {  in omap_gem_free_object()
1106 		dma_free_wc(dev->dev, obj->size, omap_obj->vaddr,  in omap_gem_free_object()
1107 			    omap_obj->dma_addr);  in omap_gem_free_object()
1108 	} else if (omap_obj->vaddr) {  in omap_gem_free_object()
1109 		vunmap(omap_obj->vaddr);  in omap_gem_free_object()
1111 		drm_prime_gem_destroy(obj, omap_obj->sgt);  in omap_gem_free_object()
1114 	mutex_unlock(&omap_obj->lock);  in omap_gem_free_object()
1118 	mutex_destroy(&omap_obj->lock);  in omap_gem_free_object()
1120 	kfree(omap_obj);  in omap_gem_free_object()
1128 	struct omap_gem_object *omap_obj;  in omap_gem_new()  local
1169 	omap_obj = kzalloc(sizeof(*omap_obj), GFP_KERNEL);  in omap_gem_new()
1170 	if (!omap_obj)  in omap_gem_new()
1173 	obj = &omap_obj->base;  in omap_gem_new()
1174 	omap_obj->flags = flags;  in omap_gem_new()
1175 	mutex_init(&omap_obj->lock);  in omap_gem_new()
1188 		omap_obj->width = gsize.tiled.width;  in omap_gem_new()
1189 		omap_obj->height = gsize.tiled.height;  in omap_gem_new()
1208 		omap_obj->vaddr = dma_alloc_wc(dev->dev, size,  in omap_gem_new()
1209 					       &omap_obj->dma_addr,  in omap_gem_new()
1211 		if (!omap_obj->vaddr)  in omap_gem_new()
1216 	list_add(&omap_obj->mm_list, &priv->obj_list);  in omap_gem_new()
1224 	kfree(omap_obj);  in omap_gem_new()
1232 	struct omap_gem_object *omap_obj;  in omap_gem_new_dmabuf()  local
1245 	omap_obj = to_omap_bo(obj);  in omap_gem_new_dmabuf()
1247 	mutex_lock(&omap_obj->lock);  in omap_gem_new_dmabuf()
1249 	omap_obj->sgt = sgt;  in omap_gem_new_dmabuf()
1252 		omap_obj->dma_addr = sg_dma_address(sgt->sgl);  in omap_gem_new_dmabuf()
1268 		omap_obj->pages = pages;  in omap_gem_new_dmabuf()
1284 	mutex_unlock(&omap_obj->lock);  in omap_gem_new_dmabuf()