151 static bool omap_gem_is_contiguous(struct omap_gem_object *omap_obj)  in omap_gem_is_contiguous()  argument
153 	if (omap_obj->flags & OMAP_BO_MEM_DMA_API)  in omap_gem_is_contiguous()
156 	if ((omap_obj->flags & OMAP_BO_MEM_DMABUF) && omap_obj->sgt->nents == 1)  in omap_gem_is_contiguous()
169 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_evict_entry()  local
175 	const int m = DIV_ROUND_UP(omap_obj->width << fmt, PAGE_SIZE);  in omap_gem_evict_entry()
196 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_evict()  local
199 	if (omap_obj->flags & OMAP_BO_TILED_MASK) {  in omap_gem_evict()
200 		enum tiler_fmt fmt = gem2fmt(omap_obj->flags);  in omap_gem_evict()
224 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_attach_pages()  local
230 	lockdep_assert_held(&omap_obj->lock);  in omap_gem_attach_pages()
236 	if (!(omap_obj->flags & OMAP_BO_MEM_SHMEM) || omap_obj->pages)  in omap_gem_attach_pages()
248 	if (omap_obj->flags & (OMAP_BO_WC|OMAP_BO_UNCACHED)) {  in omap_gem_attach_pages()
280 	omap_obj->dma_addrs = addrs;  in omap_gem_attach_pages()
281 	omap_obj->pages = pages;  in omap_gem_attach_pages()
296 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_detach_pages()  local
300 	lockdep_assert_held(&omap_obj->lock);  in omap_gem_detach_pages()
303 		if (omap_obj->dma_addrs[i])  in omap_gem_detach_pages()
304 			dma_unmap_page(obj->dev->dev, omap_obj->dma_addrs[i],  in omap_gem_detach_pages()
308 	kfree(omap_obj->dma_addrs);  in omap_gem_detach_pages()
309 	omap_obj->dma_addrs = NULL;  in omap_gem_detach_pages()
311 	drm_gem_put_pages(obj, omap_obj->pages, true, false);  in omap_gem_detach_pages()
312 	omap_obj->pages = NULL;  in omap_gem_detach_pages()
324 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_mmap_size()  local
327 	if (omap_obj->flags & OMAP_BO_TILED_MASK) {  in omap_gem_mmap_size()
334 		size = tiler_vsize(gem2fmt(omap_obj->flags),  in omap_gem_mmap_size()
335 				omap_obj->width, omap_obj->height);  in omap_gem_mmap_size()
349 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_fault_1d()  local
356 	if (omap_obj->pages) {  in omap_gem_fault_1d()
358 		pfn = page_to_pfn(omap_obj->pages[pgoff]);  in omap_gem_fault_1d()
360 		BUG_ON(!omap_gem_is_contiguous(omap_obj));  in omap_gem_fault_1d()
361 		pfn = (omap_obj->dma_addr >> PAGE_SHIFT) + pgoff;  in omap_gem_fault_1d()
375 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_fault_2d()  local
378 	enum tiler_fmt fmt = gem2fmt(omap_obj->flags);  in omap_gem_fault_2d()
400 	const int m = DIV_ROUND_UP(omap_obj->width << fmt, PAGE_SIZE);  in omap_gem_fault_2d()
412 	slots = omap_obj->width >> priv->usergart[fmt].slot_shift;  in omap_gem_fault_2d()
445 	memcpy(pages, &omap_obj->pages[base_pgoff],  in omap_gem_fault_2d()
494 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_fault()  local
501 	mutex_lock(&omap_obj->lock);  in omap_gem_fault()
516 	if (omap_obj->flags & OMAP_BO_TILED_MASK)  in omap_gem_fault()
523 	mutex_unlock(&omap_obj->lock);  in omap_gem_fault()
544 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_mmap_obj()  local
549 	if (omap_obj->flags & OMAP_BO_WC) {  in omap_gem_mmap_obj()
551 	} else if (omap_obj->flags & OMAP_BO_UNCACHED) {  in omap_gem_mmap_obj()
646 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_roll()  local
655 	omap_obj->roll = roll;  in omap_gem_roll()
657 	mutex_lock(&omap_obj->lock);  in omap_gem_roll()
660 	if (omap_obj->block) {  in omap_gem_roll()
665 		ret = tiler_pin(omap_obj->block, omap_obj->pages, npages,  in omap_gem_roll()
672 	mutex_unlock(&omap_obj->lock);  in omap_gem_roll()
693 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_is_cached_coherent()  local
695 	return !((omap_obj->flags & OMAP_BO_MEM_SHMEM) &&  in omap_gem_is_cached_coherent()
696 		((omap_obj->flags & OMAP_BO_CACHE_MASK) == OMAP_BO_CACHED));  in omap_gem_is_cached_coherent()
705 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_cpu_sync_page()  local
710 	if (omap_obj->dma_addrs[pgoff]) {  in omap_gem_cpu_sync_page()
711 		dma_unmap_page(dev->dev, omap_obj->dma_addrs[pgoff],  in omap_gem_cpu_sync_page()
713 		omap_obj->dma_addrs[pgoff] = 0;  in omap_gem_cpu_sync_page()
722 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_dma_sync_buffer()  local
724 	struct page **pages = omap_obj->pages;  in omap_gem_dma_sync_buffer()
731 		if (!omap_obj->dma_addrs[i]) {  in omap_gem_dma_sync_buffer()
743 			omap_obj->dma_addrs[i] = addr;  in omap_gem_dma_sync_buffer()
755 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_pin_tiler()  local
757 	enum tiler_fmt fmt = gem2fmt(omap_obj->flags);  in omap_gem_pin_tiler()
761 	BUG_ON(omap_obj->block);  in omap_gem_pin_tiler()
763 	if (omap_obj->flags & OMAP_BO_TILED_MASK) {  in omap_gem_pin_tiler()
764 		block = tiler_reserve_2d(fmt, omap_obj->width, omap_obj->height,  in omap_gem_pin_tiler()
777 	ret = tiler_pin(block, omap_obj->pages, npages, omap_obj->roll, true);  in omap_gem_pin_tiler()
784 	omap_obj->dma_addr = tiler_ssptr(block);  in omap_gem_pin_tiler()
785 	omap_obj->block = block;  in omap_gem_pin_tiler()
787 	DBG("got dma address: %pad", &omap_obj->dma_addr);  in omap_gem_pin_tiler()
810 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_pin()  local
813 	mutex_lock(&omap_obj->lock);  in omap_gem_pin()
815 	if (!omap_gem_is_contiguous(omap_obj)) {  in omap_gem_pin()
816 		if (refcount_read(&omap_obj->pin_cnt) == 0) {  in omap_gem_pin()
818 			refcount_set(&omap_obj->pin_cnt, 1);  in omap_gem_pin()
824 			if (omap_obj->flags & OMAP_BO_SCANOUT) {  in omap_gem_pin()
832 			refcount_inc(&omap_obj->pin_cnt);  in omap_gem_pin()
837 		*dma_addr = omap_obj->dma_addr;  in omap_gem_pin()
840 	mutex_unlock(&omap_obj->lock);  in omap_gem_pin()
854 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_unpin_locked()  local
857 	if (omap_gem_is_contiguous(omap_obj))  in omap_gem_unpin_locked()
860 	if (refcount_dec_and_test(&omap_obj->pin_cnt)) {  in omap_gem_unpin_locked()
861 		if (omap_obj->sgt) {  in omap_gem_unpin_locked()
862 			sg_free_table(omap_obj->sgt);  in omap_gem_unpin_locked()
863 			kfree(omap_obj->sgt);  in omap_gem_unpin_locked()
864 			omap_obj->sgt = NULL;  in omap_gem_unpin_locked()
866 		if (!(omap_obj->flags & OMAP_BO_SCANOUT))  in omap_gem_unpin_locked()
869 			ret = tiler_unpin(omap_obj->block);  in omap_gem_unpin_locked()
874 			ret = tiler_release(omap_obj->block);  in omap_gem_unpin_locked()
879 			omap_obj->dma_addr = 0;  in omap_gem_unpin_locked()
880 			omap_obj->block = NULL;  in omap_gem_unpin_locked()
895 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_unpin()  local
897 	mutex_lock(&omap_obj->lock);  in omap_gem_unpin()
899 	mutex_unlock(&omap_obj->lock);  in omap_gem_unpin()
909 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_rotated_dma_addr()  local
912 	mutex_lock(&omap_obj->lock);  in omap_gem_rotated_dma_addr()
914 	if ((refcount_read(&omap_obj->pin_cnt) > 0) && omap_obj->block &&  in omap_gem_rotated_dma_addr()
915 			(omap_obj->flags & OMAP_BO_TILED_MASK)) {  in omap_gem_rotated_dma_addr()
916 		*dma_addr = tiler_tsptr(omap_obj->block, orient, x, y);  in omap_gem_rotated_dma_addr()
920 	mutex_unlock(&omap_obj->lock);  in omap_gem_rotated_dma_addr()
928 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_tiled_stride()  local
930 	if (omap_obj->flags & OMAP_BO_TILED_MASK)  in omap_gem_tiled_stride()
931 		ret = tiler_stride(gem2fmt(omap_obj->flags), orient);  in omap_gem_tiled_stride()
948 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_get_pages()  local
951 	mutex_lock(&omap_obj->lock);  in omap_gem_get_pages()
959 	if (!omap_obj->pages) {  in omap_gem_get_pages()
964 	*pages = omap_obj->pages;  in omap_gem_get_pages()
967 	mutex_unlock(&omap_obj->lock);  in omap_gem_get_pages()
985 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_get_sg()  local
996 	mutex_lock(&omap_obj->lock);  in omap_gem_get_sg()
998 	sgt = omap_obj->sgt;  in omap_gem_get_sg()
1009 		if (omap_obj->flags & OMAP_BO_TILED_MASK) {  in omap_gem_get_sg()
1010 			enum tiler_fmt fmt = gem2fmt(omap_obj->flags);  in omap_gem_get_sg()
1012 			len = omap_obj->width << (int)fmt;  in omap_gem_get_sg()
1013 			count = omap_obj->height;  in omap_gem_get_sg()
1042 			sg_set_page(sg, omap_obj->pages[i], PAGE_SIZE, 0);  in omap_gem_get_sg()
1043 			sg_dma_address(sg) = omap_obj->dma_addrs[i];  in omap_gem_get_sg()
1048 	omap_obj->sgt = sgt;  in omap_gem_get_sg()
1050 	mutex_unlock(&omap_obj->lock);  in omap_gem_get_sg()
1056 	mutex_unlock(&omap_obj->lock);  in omap_gem_get_sg()
1063 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_put_sg()  local
1065 	if (WARN_ON(omap_obj->sgt != sgt))  in omap_gem_put_sg()
1078 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_vaddr()  local
1082 	mutex_lock(&omap_obj->lock);  in omap_gem_vaddr()
1084 	if (!omap_obj->vaddr) {  in omap_gem_vaddr()
1091 		omap_obj->vaddr = vmap(omap_obj->pages, obj->size >> PAGE_SHIFT,  in omap_gem_vaddr()
1095 	vaddr = omap_obj->vaddr;  in omap_gem_vaddr()
1098 	mutex_unlock(&omap_obj->lock);  in omap_gem_vaddr()
1112 	struct omap_gem_object *omap_obj;  in omap_gem_resume()  local
1116 	list_for_each_entry(omap_obj, &priv->obj_list, mm_list) {  in omap_gem_resume()
1117 		if (omap_obj->block) {  in omap_gem_resume()
1118 			struct drm_gem_object *obj = &omap_obj->base;  in omap_gem_resume()
1121 			WARN_ON(!omap_obj->pages);  /* this can't happen */  in omap_gem_resume()
1122 			ret = tiler_pin(omap_obj->block,  in omap_gem_resume()
1123 					omap_obj->pages, npages,  in omap_gem_resume()
1124 					omap_obj->roll, true);  in omap_gem_resume()
1145 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_describe()  local
1150 	mutex_lock(&omap_obj->lock);  in omap_gem_describe()
1153 			omap_obj->flags, obj->name, kref_read(&obj->refcount),  in omap_gem_describe()
1154 			off, &omap_obj->dma_addr,  in omap_gem_describe()
1155 			refcount_read(&omap_obj->pin_cnt),  in omap_gem_describe()
1156 			omap_obj->vaddr, omap_obj->roll);  in omap_gem_describe()
1158 	if (omap_obj->flags & OMAP_BO_TILED_MASK) {  in omap_gem_describe()
1159 		seq_printf(m, " %dx%d", omap_obj->width, omap_obj->height);  in omap_gem_describe()
1160 		if (omap_obj->block) {  in omap_gem_describe()
1161 			struct tcm_area *area = &omap_obj->block->area;  in omap_gem_describe()
1170 	mutex_unlock(&omap_obj->lock);  in omap_gem_describe()
1177 	struct omap_gem_object *omap_obj;  in omap_gem_describe_objects()  local
1181 	list_for_each_entry(omap_obj, list, mm_list) {  in omap_gem_describe_objects()
1182 		struct drm_gem_object *obj = &omap_obj->base;  in omap_gem_describe_objects()
1201 	struct omap_gem_object *omap_obj = to_omap_bo(obj);  in omap_gem_free_object()  local
1206 	list_del(&omap_obj->mm_list);  in omap_gem_free_object()
1215 	mutex_lock(&omap_obj->lock);  in omap_gem_free_object()
1218 	WARN_ON(refcount_read(&omap_obj->pin_cnt) > 0);  in omap_gem_free_object()
1220 	if (omap_obj->pages) {  in omap_gem_free_object()
1221 		if (omap_obj->flags & OMAP_BO_MEM_DMABUF)  in omap_gem_free_object()
1222 			kfree(omap_obj->pages);  in omap_gem_free_object()
1227 	if (omap_obj->flags & OMAP_BO_MEM_DMA_API) {  in omap_gem_free_object()
1228 		dma_free_wc(dev->dev, obj->size, omap_obj->vaddr,  in omap_gem_free_object()
1229 			    omap_obj->dma_addr);  in omap_gem_free_object()
1230 	} else if (omap_obj->vaddr) {  in omap_gem_free_object()
1231 		vunmap(omap_obj->vaddr);  in omap_gem_free_object()
1233 		drm_prime_gem_destroy(obj, omap_obj->sgt);  in omap_gem_free_object()
1236 	mutex_unlock(&omap_obj->lock);  in omap_gem_free_object()
1240 	mutex_destroy(&omap_obj->lock);  in omap_gem_free_object()
1242 	kfree(omap_obj);  in omap_gem_free_object()
1294 	struct omap_gem_object *omap_obj;  in omap_gem_new()  local
1331 	omap_obj = kzalloc(sizeof(*omap_obj), GFP_KERNEL);  in omap_gem_new()
1332 	if (!omap_obj)  in omap_gem_new()
1335 	obj = &omap_obj->base;  in omap_gem_new()
1336 	omap_obj->flags = flags;  in omap_gem_new()
1337 	mutex_init(&omap_obj->lock);  in omap_gem_new()
1350 		omap_obj->width = gsize.tiled.width;  in omap_gem_new()
1351 		omap_obj->height = gsize.tiled.height;  in omap_gem_new()
1372 		omap_obj->vaddr = dma_alloc_wc(dev->dev, size,  in omap_gem_new()
1373 					       &omap_obj->dma_addr,  in omap_gem_new()
1375 		if (!omap_obj->vaddr)  in omap_gem_new()
1380 	list_add(&omap_obj->mm_list, &priv->obj_list);  in omap_gem_new()
1388 	kfree(omap_obj);  in omap_gem_new()
1396 	struct omap_gem_object *omap_obj;  in omap_gem_new_dmabuf()  local
1409 	omap_obj = to_omap_bo(obj);  in omap_gem_new_dmabuf()
1411 	mutex_lock(&omap_obj->lock);  in omap_gem_new_dmabuf()
1413 	omap_obj->sgt = sgt;  in omap_gem_new_dmabuf()
1416 		omap_obj->dma_addr = sg_dma_address(sgt->sgl);  in omap_gem_new_dmabuf()
1431 		omap_obj->pages = pages;  in omap_gem_new_dmabuf()
1441 	mutex_unlock(&omap_obj->lock);  in omap_gem_new_dmabuf()