160 static bool omap_gem_is_contiguous(struct omap_gem_object *omap_obj)  in omap_gem_is_contiguous()  argument
162 	if (omap_obj->flags & OMAP_BO_MEM_DMA_API)  in omap_gem_is_contiguous()
165 	if ((omap_obj->flags & OMAP_BO_MEM_DMABUF) && omap_obj->sgt->nents == 1)  in omap_gem_is_contiguous()
178 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_evict_entry()  local
184 	const int m = DIV_ROUND_UP(omap_obj->width << fmt, PAGE_SIZE);  in omap_gem_evict_entry()
205 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_evict()  local
208 	if (omap_obj->flags & OMAP_BO_TILED) {  in omap_gem_evict()
209 		enum tiler_fmt fmt = gem2fmt(omap_obj->flags);  in omap_gem_evict()
233 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_attach_pages()  local
239 	lockdep_assert_held(&omap_obj->lock);  in omap_gem_attach_pages()
245 	if (!(omap_obj->flags & OMAP_BO_MEM_SHMEM) || omap_obj->pages)  in omap_gem_attach_pages()
257 	if (omap_obj->flags & (OMAP_BO_WC|OMAP_BO_UNCACHED)) {  in omap_gem_attach_pages()
289 	omap_obj->dma_addrs = addrs;  in omap_gem_attach_pages()
290 	omap_obj->pages = pages;  in omap_gem_attach_pages()
305 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_detach_pages()  local
309 	lockdep_assert_held(&omap_obj->lock);  in omap_gem_detach_pages()
312 		if (omap_obj->dma_addrs[i])  in omap_gem_detach_pages()
313 			dma_unmap_page(obj->dev->dev, omap_obj->dma_addrs[i],  in omap_gem_detach_pages()
317 	kfree(omap_obj->dma_addrs);  in omap_gem_detach_pages()
318 	omap_obj->dma_addrs = NULL;  in omap_gem_detach_pages()
320 	drm_gem_put_pages(obj, omap_obj->pages, true, false);  in omap_gem_detach_pages()
321 	omap_obj->pages = NULL;  in omap_gem_detach_pages()
333 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_mmap_size()  local
336 	if (omap_obj->flags & OMAP_BO_TILED) {  in omap_gem_mmap_size()
343 		size = tiler_vsize(gem2fmt(omap_obj->flags),  in omap_gem_mmap_size()
344 				omap_obj->width, omap_obj->height);  in omap_gem_mmap_size()
358 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_fault_1d()  local
365 	if (omap_obj->pages) {  in omap_gem_fault_1d()
367 		pfn = page_to_pfn(omap_obj->pages[pgoff]);  in omap_gem_fault_1d()
369 		BUG_ON(!omap_gem_is_contiguous(omap_obj));  in omap_gem_fault_1d()
370 		pfn = (omap_obj->dma_addr >> PAGE_SHIFT) + pgoff;  in omap_gem_fault_1d()
384 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_fault_2d()  local
387 	enum tiler_fmt fmt = gem2fmt(omap_obj->flags);  in omap_gem_fault_2d()
409 	const int m = DIV_ROUND_UP(omap_obj->width << fmt, PAGE_SIZE);  in omap_gem_fault_2d()
421 	slots = omap_obj->width >> priv->usergart[fmt].slot_shift;  in omap_gem_fault_2d()
454 	memcpy(pages, &omap_obj->pages[base_pgoff],  in omap_gem_fault_2d()
503 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_fault()  local
510 	mutex_lock(&omap_obj->lock);  in omap_gem_fault()
525 	if (omap_obj->flags & OMAP_BO_TILED)  in omap_gem_fault()
532 	mutex_unlock(&omap_obj->lock);  in omap_gem_fault()
553 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_mmap_obj()  local
558 	if (omap_obj->flags & OMAP_BO_WC) {  in omap_gem_mmap_obj()
560 	} else if (omap_obj->flags & OMAP_BO_UNCACHED) {  in omap_gem_mmap_obj()
655 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_roll()  local
664 	omap_obj->roll = roll;  in omap_gem_roll()
666 	mutex_lock(&omap_obj->lock);  in omap_gem_roll()
669 	if (omap_obj->block) {  in omap_gem_roll()
674 		ret = tiler_pin(omap_obj->block, omap_obj->pages, npages,  in omap_gem_roll()
681 	mutex_unlock(&omap_obj->lock);  in omap_gem_roll()
702 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_is_cached_coherent()  local
704 	return !((omap_obj->flags & OMAP_BO_MEM_SHMEM) &&  in omap_gem_is_cached_coherent()
705 		((omap_obj->flags & OMAP_BO_CACHE_MASK) == OMAP_BO_CACHED));  in omap_gem_is_cached_coherent()
714 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_cpu_sync_page()  local
719 	if (omap_obj->dma_addrs[pgoff]) {  in omap_gem_cpu_sync_page()
720 		dma_unmap_page(dev->dev, omap_obj->dma_addrs[pgoff],  in omap_gem_cpu_sync_page()
722 		omap_obj->dma_addrs[pgoff] = 0;  in omap_gem_cpu_sync_page()
731 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_dma_sync_buffer()  local
733 	struct page **pages = omap_obj->pages;  in omap_gem_dma_sync_buffer()
740 		if (!omap_obj->dma_addrs[i]) {  in omap_gem_dma_sync_buffer()
752 			omap_obj->dma_addrs[i] = addr;  in omap_gem_dma_sync_buffer()
779 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_pin()  local
782 	mutex_lock(&omap_obj->lock);  in omap_gem_pin()
784 	if (!omap_gem_is_contiguous(omap_obj) && priv->has_dmm) {  in omap_gem_pin()
785 		if (omap_obj->dma_addr_cnt == 0) {  in omap_gem_pin()
787 			enum tiler_fmt fmt = gem2fmt(omap_obj->flags);  in omap_gem_pin()
790 			BUG_ON(omap_obj->block);  in omap_gem_pin()
796 			if (omap_obj->flags & OMAP_BO_TILED) {  in omap_gem_pin()
798 						omap_obj->width,  in omap_gem_pin()
799 						omap_obj->height, 0);  in omap_gem_pin()
812 			ret = tiler_pin(block, omap_obj->pages, npages,  in omap_gem_pin()
813 					omap_obj->roll, true);  in omap_gem_pin()
821 			omap_obj->dma_addr = tiler_ssptr(block);  in omap_gem_pin()
822 			omap_obj->block = block;  in omap_gem_pin()
824 			DBG("got dma address: %pad", &omap_obj->dma_addr);  in omap_gem_pin()
827 		omap_obj->dma_addr_cnt++;  in omap_gem_pin()
829 		*dma_addr = omap_obj->dma_addr;  in omap_gem_pin()
830 	} else if (omap_gem_is_contiguous(omap_obj)) {  in omap_gem_pin()
831 		*dma_addr = omap_obj->dma_addr;  in omap_gem_pin()
838 	mutex_unlock(&omap_obj->lock);  in omap_gem_pin()
853 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_unpin()  local
856 	mutex_lock(&omap_obj->lock);  in omap_gem_unpin()
858 	if (omap_obj->dma_addr_cnt > 0) {  in omap_gem_unpin()
859 		omap_obj->dma_addr_cnt--;  in omap_gem_unpin()
860 		if (omap_obj->dma_addr_cnt == 0) {  in omap_gem_unpin()
861 			ret = tiler_unpin(omap_obj->block);  in omap_gem_unpin()
866 			ret = tiler_release(omap_obj->block);  in omap_gem_unpin()
871 			omap_obj->dma_addr = 0;  in omap_gem_unpin()
872 			omap_obj->block = NULL;  in omap_gem_unpin()
876 	mutex_unlock(&omap_obj->lock);  in omap_gem_unpin()
886 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_rotated_dma_addr()  local
889 	mutex_lock(&omap_obj->lock);  in omap_gem_rotated_dma_addr()
891 	if ((omap_obj->dma_addr_cnt > 0) && omap_obj->block &&  in omap_gem_rotated_dma_addr()
892 			(omap_obj->flags & OMAP_BO_TILED)) {  in omap_gem_rotated_dma_addr()
893 		*dma_addr = tiler_tsptr(omap_obj->block, orient, x, y);  in omap_gem_rotated_dma_addr()
897 	mutex_unlock(&omap_obj->lock);  in omap_gem_rotated_dma_addr()
905 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_tiled_stride()  local
907 	if (omap_obj->flags & OMAP_BO_TILED)  in omap_gem_tiled_stride()
908 		ret = tiler_stride(gem2fmt(omap_obj->flags), orient);  in omap_gem_tiled_stride()
925 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_get_pages()  local
928 	mutex_lock(&omap_obj->lock);  in omap_gem_get_pages()
936 	if (!omap_obj->pages) {  in omap_gem_get_pages()
941 	*pages = omap_obj->pages;  in omap_gem_get_pages()
944 	mutex_unlock(&omap_obj->lock);  in omap_gem_get_pages()
966 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_vaddr()  local
970 	mutex_lock(&omap_obj->lock);  in omap_gem_vaddr()
972 	if (!omap_obj->vaddr) {  in omap_gem_vaddr()
979 		omap_obj->vaddr = vmap(omap_obj->pages, obj->size >> PAGE_SHIFT,  in omap_gem_vaddr()
983 	vaddr = omap_obj->vaddr;  in omap_gem_vaddr()
986 	mutex_unlock(&omap_obj->lock);  in omap_gem_vaddr()
1000 	struct omap_gem_object *omap_obj;  in omap_gem_resume()  local
1004 	list_for_each_entry(omap_obj, &priv->obj_list, mm_list) {  in omap_gem_resume()
1005 		if (omap_obj->block) {  in omap_gem_resume()
1006 			struct drm_gem_object *obj = &omap_obj->base;  in omap_gem_resume()
1009 			WARN_ON(!omap_obj->pages);  /* this can't happen */  in omap_gem_resume()
1010 			ret = tiler_pin(omap_obj->block,  in omap_gem_resume()
1011 					omap_obj->pages, npages,  in omap_gem_resume()
1012 					omap_obj->roll, true);  in omap_gem_resume()
1033 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_describe()  local
1038 	mutex_lock(&omap_obj->lock);  in omap_gem_describe()
1041 			omap_obj->flags, obj->name, kref_read(&obj->refcount),  in omap_gem_describe()
1042 			off, &omap_obj->dma_addr, omap_obj->dma_addr_cnt,  in omap_gem_describe()
1043 			omap_obj->vaddr, omap_obj->roll);  in omap_gem_describe()
1045 	if (omap_obj->flags & OMAP_BO_TILED) {  in omap_gem_describe()
1046 		seq_printf(m, " %dx%d", omap_obj->width, omap_obj->height);  in omap_gem_describe()
1047 		if (omap_obj->block) {  in omap_gem_describe()
1048 			struct tcm_area *area = &omap_obj->block->area;  in omap_gem_describe()
1057 	mutex_unlock(&omap_obj->lock);  in omap_gem_describe()
1064 	struct omap_gem_object *omap_obj;  in omap_gem_describe_objects()  local
1068 	list_for_each_entry(omap_obj, list, mm_list) {  in omap_gem_describe_objects()
1069 		struct drm_gem_object *obj = &omap_obj->base;  in omap_gem_describe_objects()
1088 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_free_object()  local
1093 	list_del(&omap_obj->mm_list);  in omap_gem_free_object()
1102 	mutex_lock(&omap_obj->lock);  in omap_gem_free_object()
1105 	WARN_ON(omap_obj->dma_addr_cnt > 0);  in omap_gem_free_object()
1107 	if (omap_obj->pages) {  in omap_gem_free_object()
1108 		if (omap_obj->flags & OMAP_BO_MEM_DMABUF)  in omap_gem_free_object()
1109 			kfree(omap_obj->pages);  in omap_gem_free_object()
1114 	if (omap_obj->flags & OMAP_BO_MEM_DMA_API) {  in omap_gem_free_object()
1115 		dma_free_wc(dev->dev, obj->size, omap_obj->vaddr,  in omap_gem_free_object()
1116 			    omap_obj->dma_addr);  in omap_gem_free_object()
1117 	} else if (omap_obj->vaddr) {  in omap_gem_free_object()
1118 		vunmap(omap_obj->vaddr);  in omap_gem_free_object()
1120 		drm_prime_gem_destroy(obj, omap_obj->sgt);  in omap_gem_free_object()
1123 	mutex_unlock(&omap_obj->lock);  in omap_gem_free_object()
1127 	mutex_destroy(&omap_obj->lock);  in omap_gem_free_object()
1129 	kfree(omap_obj);  in omap_gem_free_object()
1137 	struct omap_gem_object *omap_obj;  in omap_gem_new()  local
1178 	omap_obj = kzalloc(sizeof(*omap_obj), GFP_KERNEL);  in omap_gem_new()
1179 	if (!omap_obj)  in omap_gem_new()
1182 	obj = &omap_obj->base;  in omap_gem_new()
1183 	omap_obj->flags = flags;  in omap_gem_new()
1184 	mutex_init(&omap_obj->lock);  in omap_gem_new()
1197 		omap_obj->width = gsize.tiled.width;  in omap_gem_new()
1198 		omap_obj->height = gsize.tiled.height;  in omap_gem_new()
1217 		omap_obj->vaddr = dma_alloc_wc(dev->dev, size,  in omap_gem_new()
1218 					       &omap_obj->dma_addr,  in omap_gem_new()
1220 		if (!omap_obj->vaddr)  in omap_gem_new()
1225 	list_add(&omap_obj->mm_list, &priv->obj_list);  in omap_gem_new()
1233 	kfree(omap_obj);  in omap_gem_new()
1241 	struct omap_gem_object *omap_obj;  in omap_gem_new_dmabuf()  local
1254 	omap_obj = to_omap_bo(obj);  in omap_gem_new_dmabuf()
1256 	mutex_lock(&omap_obj->lock);  in omap_gem_new_dmabuf()
1258 	omap_obj->sgt = sgt;  in omap_gem_new_dmabuf()
1261 		omap_obj->dma_addr = sg_dma_address(sgt->sgl);  in omap_gem_new_dmabuf()
1277 		omap_obj->pages = pages;  in omap_gem_new_dmabuf()
1293 	mutex_unlock(&omap_obj->lock);  in omap_gem_new_dmabuf()