151 static bool omap_gem_is_contiguous(struct omap_gem_object *omap_obj)  in omap_gem_is_contiguous()  argument
153 	if (omap_obj->flags & OMAP_BO_MEM_DMA_API)  in omap_gem_is_contiguous()
156 	if ((omap_obj->flags & OMAP_BO_MEM_DMABUF) && omap_obj->sgt->nents == 1)  in omap_gem_is_contiguous()
169 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_evict_entry()  local
175 	const int m = DIV_ROUND_UP(omap_obj->width << fmt, PAGE_SIZE);  in omap_gem_evict_entry()
196 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_evict()  local
199 	if (omap_obj->flags & OMAP_BO_TILED_MASK) {  in omap_gem_evict()
200 		enum tiler_fmt fmt = gem2fmt(omap_obj->flags);  in omap_gem_evict()
224 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_attach_pages()  local
230 	lockdep_assert_held(&omap_obj->lock);  in omap_gem_attach_pages()
236 	if (!(omap_obj->flags & OMAP_BO_MEM_SHMEM) || omap_obj->pages)  in omap_gem_attach_pages()
248 	if (omap_obj->flags & (OMAP_BO_WC|OMAP_BO_UNCACHED)) {  in omap_gem_attach_pages()
280 	omap_obj->dma_addrs = addrs;  in omap_gem_attach_pages()
281 	omap_obj->pages = pages;  in omap_gem_attach_pages()
296 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_detach_pages()  local
300 	lockdep_assert_held(&omap_obj->lock);  in omap_gem_detach_pages()
303 		if (omap_obj->dma_addrs[i])  in omap_gem_detach_pages()
304 			dma_unmap_page(obj->dev->dev, omap_obj->dma_addrs[i],  in omap_gem_detach_pages()
308 	kfree(omap_obj->dma_addrs);  in omap_gem_detach_pages()
309 	omap_obj->dma_addrs = NULL;  in omap_gem_detach_pages()
311 	drm_gem_put_pages(obj, omap_obj->pages, true, false);  in omap_gem_detach_pages()
312 	omap_obj->pages = NULL;  in omap_gem_detach_pages()
324 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_mmap_size()  local
327 	if (omap_obj->flags & OMAP_BO_TILED_MASK) {  in omap_gem_mmap_size()
334 		size = tiler_vsize(gem2fmt(omap_obj->flags),  in omap_gem_mmap_size()
335 				omap_obj->width, omap_obj->height);  in omap_gem_mmap_size()
349 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_fault_1d()  local
356 	if (omap_obj->pages) {  in omap_gem_fault_1d()
358 		pfn = page_to_pfn(omap_obj->pages[pgoff]);  in omap_gem_fault_1d()
360 		BUG_ON(!omap_gem_is_contiguous(omap_obj));  in omap_gem_fault_1d()
361 		pfn = (omap_obj->dma_addr >> PAGE_SHIFT) + pgoff;  in omap_gem_fault_1d()
375 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_fault_2d()  local
378 	enum tiler_fmt fmt = gem2fmt(omap_obj->flags);  in omap_gem_fault_2d()
400 	const int m = DIV_ROUND_UP(omap_obj->width << fmt, PAGE_SIZE);  in omap_gem_fault_2d()
412 	slots = omap_obj->width >> priv->usergart[fmt].slot_shift;  in omap_gem_fault_2d()
445 	memcpy(pages, &omap_obj->pages[base_pgoff],  in omap_gem_fault_2d()
494 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_fault()  local
501 	mutex_lock(&omap_obj->lock);  in omap_gem_fault()
516 	if (omap_obj->flags & OMAP_BO_TILED_MASK)  in omap_gem_fault()
523 	mutex_unlock(&omap_obj->lock);  in omap_gem_fault()
544 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_mmap_obj()  local
549 	if (omap_obj->flags & OMAP_BO_WC) {  in omap_gem_mmap_obj()
551 	} else if (omap_obj->flags & OMAP_BO_UNCACHED) {  in omap_gem_mmap_obj()
646 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_roll()  local
655 	omap_obj->roll = roll;  in omap_gem_roll()
657 	mutex_lock(&omap_obj->lock);  in omap_gem_roll()
660 	if (omap_obj->block) {  in omap_gem_roll()
665 		ret = tiler_pin(omap_obj->block, omap_obj->pages, npages,  in omap_gem_roll()
672 	mutex_unlock(&omap_obj->lock);  in omap_gem_roll()
693 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_is_cached_coherent()  local
695 	return !((omap_obj->flags & OMAP_BO_MEM_SHMEM) &&  in omap_gem_is_cached_coherent()
696 		((omap_obj->flags & OMAP_BO_CACHE_MASK) == OMAP_BO_CACHED));  in omap_gem_is_cached_coherent()
705 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_cpu_sync_page()  local
710 	if (omap_obj->dma_addrs[pgoff]) {  in omap_gem_cpu_sync_page()
711 		dma_unmap_page(dev->dev, omap_obj->dma_addrs[pgoff],  in omap_gem_cpu_sync_page()
713 		omap_obj->dma_addrs[pgoff] = 0;  in omap_gem_cpu_sync_page()
722 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_dma_sync_buffer()  local
724 	struct page **pages = omap_obj->pages;  in omap_gem_dma_sync_buffer()
731 		if (!omap_obj->dma_addrs[i]) {  in omap_gem_dma_sync_buffer()
743 			omap_obj->dma_addrs[i] = addr;  in omap_gem_dma_sync_buffer()
770 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_pin()  local
773 	mutex_lock(&omap_obj->lock);  in omap_gem_pin()
775 	if (!omap_gem_is_contiguous(omap_obj) && priv->has_dmm) {  in omap_gem_pin()
776 		if (refcount_read(&omap_obj->dma_addr_cnt) == 0) {  in omap_gem_pin()
778 			enum tiler_fmt fmt = gem2fmt(omap_obj->flags);  in omap_gem_pin()
781 			BUG_ON(omap_obj->block);  in omap_gem_pin()
783 			refcount_set(&omap_obj->dma_addr_cnt, 1);  in omap_gem_pin()
789 			if (omap_obj->flags & OMAP_BO_TILED_MASK) {  in omap_gem_pin()
791 						omap_obj->width,  in omap_gem_pin()
792 						omap_obj->height, 0);  in omap_gem_pin()
805 			ret = tiler_pin(block, omap_obj->pages, npages,  in omap_gem_pin()
806 					omap_obj->roll, true);  in omap_gem_pin()
814 			omap_obj->dma_addr = tiler_ssptr(block);  in omap_gem_pin()
815 			omap_obj->block = block;  in omap_gem_pin()
817 			DBG("got dma address: %pad", &omap_obj->dma_addr);  in omap_gem_pin()
819 			refcount_inc(&omap_obj->dma_addr_cnt);  in omap_gem_pin()
823 			*dma_addr = omap_obj->dma_addr;  in omap_gem_pin()
824 	} else if (omap_gem_is_contiguous(omap_obj)) {  in omap_gem_pin()
826 			*dma_addr = omap_obj->dma_addr;  in omap_gem_pin()
833 	mutex_unlock(&omap_obj->lock);  in omap_gem_pin()
847 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_unpin_locked()  local
850 	if (omap_gem_is_contiguous(omap_obj) || !priv->has_dmm)  in omap_gem_unpin_locked()
853 	if (refcount_dec_and_test(&omap_obj->dma_addr_cnt)) {  in omap_gem_unpin_locked()
854 		ret = tiler_unpin(omap_obj->block);  in omap_gem_unpin_locked()
859 		ret = tiler_release(omap_obj->block);  in omap_gem_unpin_locked()
864 		omap_obj->dma_addr = 0;  in omap_gem_unpin_locked()
865 		omap_obj->block = NULL;  in omap_gem_unpin_locked()
879 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_unpin()  local
881 	mutex_lock(&omap_obj->lock);  in omap_gem_unpin()
883 	mutex_unlock(&omap_obj->lock);  in omap_gem_unpin()
893 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_rotated_dma_addr()  local
896 	mutex_lock(&omap_obj->lock);  in omap_gem_rotated_dma_addr()
898 	if ((refcount_read(&omap_obj->dma_addr_cnt) > 0) && omap_obj->block &&  in omap_gem_rotated_dma_addr()
899 			(omap_obj->flags & OMAP_BO_TILED_MASK)) {  in omap_gem_rotated_dma_addr()
900 		*dma_addr = tiler_tsptr(omap_obj->block, orient, x, y);  in omap_gem_rotated_dma_addr()
904 	mutex_unlock(&omap_obj->lock);  in omap_gem_rotated_dma_addr()
912 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_tiled_stride()  local
914 	if (omap_obj->flags & OMAP_BO_TILED_MASK)  in omap_gem_tiled_stride()
915 		ret = tiler_stride(gem2fmt(omap_obj->flags), orient);  in omap_gem_tiled_stride()
932 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_get_pages()  local
935 	mutex_lock(&omap_obj->lock);  in omap_gem_get_pages()
943 	if (!omap_obj->pages) {  in omap_gem_get_pages()
948 	*pages = omap_obj->pages;  in omap_gem_get_pages()
951 	mutex_unlock(&omap_obj->lock);  in omap_gem_get_pages()
973 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_vaddr()  local
977 	mutex_lock(&omap_obj->lock);  in omap_gem_vaddr()
979 	if (!omap_obj->vaddr) {  in omap_gem_vaddr()
986 		omap_obj->vaddr = vmap(omap_obj->pages, obj->size >> PAGE_SHIFT,  in omap_gem_vaddr()
990 	vaddr = omap_obj->vaddr;  in omap_gem_vaddr()
993 	mutex_unlock(&omap_obj->lock);  in omap_gem_vaddr()
1007 	struct omap_gem_object *omap_obj;  in omap_gem_resume()  local
1011 	list_for_each_entry(omap_obj, &priv->obj_list, mm_list) {  in omap_gem_resume()
1012 		if (omap_obj->block) {  in omap_gem_resume()
1013 			struct drm_gem_object *obj = &omap_obj->base;  in omap_gem_resume()
1016 			WARN_ON(!omap_obj->pages);  /* this can't happen */  in omap_gem_resume()
1017 			ret = tiler_pin(omap_obj->block,  in omap_gem_resume()
1018 					omap_obj->pages, npages,  in omap_gem_resume()
1019 					omap_obj->roll, true);  in omap_gem_resume()
1040 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_describe()  local
1045 	mutex_lock(&omap_obj->lock);  in omap_gem_describe()
1048 			omap_obj->flags, obj->name, kref_read(&obj->refcount),  in omap_gem_describe()
1049 			off, &omap_obj->dma_addr,  in omap_gem_describe()
1050 			refcount_read(&omap_obj->dma_addr_cnt),  in omap_gem_describe()
1051 			omap_obj->vaddr, omap_obj->roll);  in omap_gem_describe()
1053 	if (omap_obj->flags & OMAP_BO_TILED_MASK) {  in omap_gem_describe()
1054 		seq_printf(m, " %dx%d", omap_obj->width, omap_obj->height);  in omap_gem_describe()
1055 		if (omap_obj->block) {  in omap_gem_describe()
1056 			struct tcm_area *area = &omap_obj->block->area;  in omap_gem_describe()
1065 	mutex_unlock(&omap_obj->lock);  in omap_gem_describe()
1072 	struct omap_gem_object *omap_obj;  in omap_gem_describe_objects()  local
1076 	list_for_each_entry(omap_obj, list, mm_list) {  in omap_gem_describe_objects()
1077 		struct drm_gem_object *obj = &omap_obj->base;  in omap_gem_describe_objects()
1096 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_free_object()  local
1101 	list_del(&omap_obj->mm_list);  in omap_gem_free_object()
1110 	mutex_lock(&omap_obj->lock);  in omap_gem_free_object()
1113 	WARN_ON(refcount_read(&omap_obj->dma_addr_cnt) > 0);  in omap_gem_free_object()
1115 	if (omap_obj->pages) {  in omap_gem_free_object()
1116 		if (omap_obj->flags & OMAP_BO_MEM_DMABUF)  in omap_gem_free_object()
1117 			kfree(omap_obj->pages);  in omap_gem_free_object()
1122 	if (omap_obj->flags & OMAP_BO_MEM_DMA_API) {  in omap_gem_free_object()
1123 		dma_free_wc(dev->dev, obj->size, omap_obj->vaddr,  in omap_gem_free_object()
1124 			    omap_obj->dma_addr);  in omap_gem_free_object()
1125 	} else if (omap_obj->vaddr) {  in omap_gem_free_object()
1126 		vunmap(omap_obj->vaddr);  in omap_gem_free_object()
1128 		drm_prime_gem_destroy(obj, omap_obj->sgt);  in omap_gem_free_object()
1131 	mutex_unlock(&omap_obj->lock);  in omap_gem_free_object()
1135 	mutex_destroy(&omap_obj->lock);  in omap_gem_free_object()
1137 	kfree(omap_obj);  in omap_gem_free_object()
1177 	struct omap_gem_object *omap_obj;  in omap_gem_new()  local
1214 	omap_obj = kzalloc(sizeof(*omap_obj), GFP_KERNEL);  in omap_gem_new()
1215 	if (!omap_obj)  in omap_gem_new()
1218 	obj = &omap_obj->base;  in omap_gem_new()
1219 	omap_obj->flags = flags;  in omap_gem_new()
1220 	mutex_init(&omap_obj->lock);  in omap_gem_new()
1233 		omap_obj->width = gsize.tiled.width;  in omap_gem_new()
1234 		omap_obj->height = gsize.tiled.height;  in omap_gem_new()
1253 		omap_obj->vaddr = dma_alloc_wc(dev->dev, size,  in omap_gem_new()
1254 					       &omap_obj->dma_addr,  in omap_gem_new()
1256 		if (!omap_obj->vaddr)  in omap_gem_new()
1261 	list_add(&omap_obj->mm_list, &priv->obj_list);  in omap_gem_new()
1269 	kfree(omap_obj);  in omap_gem_new()
1277 	struct omap_gem_object *omap_obj;  in omap_gem_new_dmabuf()  local
1290 	omap_obj = to_omap_bo(obj);  in omap_gem_new_dmabuf()
1292 	mutex_lock(&omap_obj->lock);  in omap_gem_new_dmabuf()
1294 	omap_obj->sgt = sgt;  in omap_gem_new_dmabuf()
1297 		omap_obj->dma_addr = sg_dma_address(sgt->sgl);  in omap_gem_new_dmabuf()
1312 		omap_obj->pages = pages;  in omap_gem_new_dmabuf()
1323 	mutex_unlock(&omap_obj->lock);  in omap_gem_new_dmabuf()