23  * This library provides helpers for GEM objects backed by shmem buffers
41 	struct drm_gem_shmem_object *shmem;  in __drm_gem_shmem_create()  local
50 		obj = kzalloc(sizeof(*shmem), GFP_KERNEL);  in __drm_gem_shmem_create()
54 	shmem = to_drm_gem_shmem_obj(obj);  in __drm_gem_shmem_create()
61 		shmem->map_wc = false; /* dma-buf mappings use always writecombine */  in __drm_gem_shmem_create()
72 	mutex_init(&shmem->pages_lock);  in __drm_gem_shmem_create()
73 	mutex_init(&shmem->vmap_lock);  in __drm_gem_shmem_create()
74 	INIT_LIST_HEAD(&shmem->madv_list);  in __drm_gem_shmem_create()
88 	return shmem;  in __drm_gem_shmem_create()
102  * This function creates a shmem GEM object.
115  * drm_gem_shmem_free_object - Free resources associated with a shmem GEM object
124 	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);  in drm_gem_shmem_free_object()  local
126 	WARN_ON(shmem->vmap_use_count);  in drm_gem_shmem_free_object()
129 		drm_prime_gem_destroy(obj, shmem->sgt);  in drm_gem_shmem_free_object()
131 		if (shmem->sgt) {  in drm_gem_shmem_free_object()
132 			dma_unmap_sgtable(obj->dev->dev, shmem->sgt,  in drm_gem_shmem_free_object()
134 			sg_free_table(shmem->sgt);  in drm_gem_shmem_free_object()
135 			kfree(shmem->sgt);  in drm_gem_shmem_free_object()
137 		if (shmem->pages)  in drm_gem_shmem_free_object()
138 			drm_gem_shmem_put_pages(shmem);  in drm_gem_shmem_free_object()
141 	WARN_ON(shmem->pages_use_count);  in drm_gem_shmem_free_object()
144 	mutex_destroy(&shmem->pages_lock);  in drm_gem_shmem_free_object()
145 	mutex_destroy(&shmem->vmap_lock);  in drm_gem_shmem_free_object()
146 	kfree(shmem);  in drm_gem_shmem_free_object()
150 static int drm_gem_shmem_get_pages_locked(struct drm_gem_shmem_object *shmem)  in drm_gem_shmem_get_pages_locked()  argument
152 	struct drm_gem_object *obj = &shmem->base;  in drm_gem_shmem_get_pages_locked()
155 	if (shmem->pages_use_count++ > 0)  in drm_gem_shmem_get_pages_locked()
161 		shmem->pages_use_count = 0;  in drm_gem_shmem_get_pages_locked()
165 	shmem->pages = pages;  in drm_gem_shmem_get_pages_locked()
171  * drm_gem_shmem_get_pages - Allocate backing pages for a shmem GEM object
172  * @shmem: shmem GEM object
174  * This function makes sure that backing pages exists for the shmem GEM object
180 int drm_gem_shmem_get_pages(struct drm_gem_shmem_object *shmem)  in drm_gem_shmem_get_pages()  argument
184 	WARN_ON(shmem->base.import_attach);  in drm_gem_shmem_get_pages()
186 	ret = mutex_lock_interruptible(&shmem->pages_lock);  in drm_gem_shmem_get_pages()
189 	ret = drm_gem_shmem_get_pages_locked(shmem);  in drm_gem_shmem_get_pages()
190 	mutex_unlock(&shmem->pages_lock);  in drm_gem_shmem_get_pages()
196 static void drm_gem_shmem_put_pages_locked(struct drm_gem_shmem_object *shmem)  in drm_gem_shmem_put_pages_locked()  argument
198 	struct drm_gem_object *obj = &shmem->base;  in drm_gem_shmem_put_pages_locked()
200 	if (WARN_ON_ONCE(!shmem->pages_use_count))  in drm_gem_shmem_put_pages_locked()
203 	if (--shmem->pages_use_count > 0)  in drm_gem_shmem_put_pages_locked()
206 	drm_gem_put_pages(obj, shmem->pages,  in drm_gem_shmem_put_pages_locked()
207 			  shmem->pages_mark_dirty_on_put,  in drm_gem_shmem_put_pages_locked()
208 			  shmem->pages_mark_accessed_on_put);  in drm_gem_shmem_put_pages_locked()
209 	shmem->pages = NULL;  in drm_gem_shmem_put_pages_locked()
213  * drm_gem_shmem_put_pages - Decrease use count on the backing pages for a shmem GEM object
214  * @shmem: shmem GEM object
218 void drm_gem_shmem_put_pages(struct drm_gem_shmem_object *shmem)  in drm_gem_shmem_put_pages()  argument
220 	mutex_lock(&shmem->pages_lock);  in drm_gem_shmem_put_pages()
221 	drm_gem_shmem_put_pages_locked(shmem);  in drm_gem_shmem_put_pages()
222 	mutex_unlock(&shmem->pages_lock);  in drm_gem_shmem_put_pages()
227  * drm_gem_shmem_pin - Pin backing pages for a shmem GEM object
239 	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);  in drm_gem_shmem_pin()  local
241 	WARN_ON(shmem->base.import_attach);  in drm_gem_shmem_pin()
243 	return drm_gem_shmem_get_pages(shmem);  in drm_gem_shmem_pin()
248  * drm_gem_shmem_unpin - Unpin backing pages for a shmem GEM object
256 	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);  in drm_gem_shmem_unpin()  local
258 	WARN_ON(shmem->base.import_attach);  in drm_gem_shmem_unpin()
260 	drm_gem_shmem_put_pages(shmem);  in drm_gem_shmem_unpin()
264 static int drm_gem_shmem_vmap_locked(struct drm_gem_shmem_object *shmem, struct dma_buf_map *map)  in drm_gem_shmem_vmap_locked()  argument
266 	struct drm_gem_object *obj = &shmem->base;  in drm_gem_shmem_vmap_locked()
269 	if (shmem->vmap_use_count++ > 0) {  in drm_gem_shmem_vmap_locked()
270 		dma_buf_map_set_vaddr(map, shmem->vaddr);  in drm_gem_shmem_vmap_locked()
281 			shmem->vaddr = map->vaddr;  in drm_gem_shmem_vmap_locked()
286 		ret = drm_gem_shmem_get_pages(shmem);  in drm_gem_shmem_vmap_locked()
290 		if (shmem->map_wc)  in drm_gem_shmem_vmap_locked()
292 		shmem->vaddr = vmap(shmem->pages, obj->size >> PAGE_SHIFT,  in drm_gem_shmem_vmap_locked()
294 		if (!shmem->vaddr)  in drm_gem_shmem_vmap_locked()
297 			dma_buf_map_set_vaddr(map, shmem->vaddr);  in drm_gem_shmem_vmap_locked()
309 		drm_gem_shmem_put_pages(shmem);  in drm_gem_shmem_vmap_locked()
311 	shmem->vmap_use_count = 0;  in drm_gem_shmem_vmap_locked()
317  * drm_gem_shmem_vmap - Create a virtual mapping for a shmem GEM object
318  * @shmem: shmem GEM object
319  * @map: Returns the kernel virtual address of the SHMEM GEM object's backing
323  * exists for the buffer backing the shmem GEM object.
336 	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);  in drm_gem_shmem_vmap()  local
339 	ret = mutex_lock_interruptible(&shmem->vmap_lock);  in drm_gem_shmem_vmap()
342 	ret = drm_gem_shmem_vmap_locked(shmem, map);  in drm_gem_shmem_vmap()
343 	mutex_unlock(&shmem->vmap_lock);  in drm_gem_shmem_vmap()
349 static void drm_gem_shmem_vunmap_locked(struct drm_gem_shmem_object *shmem,  in drm_gem_shmem_vunmap_locked()  argument
352 	struct drm_gem_object *obj = &shmem->base;  in drm_gem_shmem_vunmap_locked()
354 	if (WARN_ON_ONCE(!shmem->vmap_use_count))  in drm_gem_shmem_vunmap_locked()
357 	if (--shmem->vmap_use_count > 0)  in drm_gem_shmem_vunmap_locked()
363 		vunmap(shmem->vaddr);  in drm_gem_shmem_vunmap_locked()
364 		drm_gem_shmem_put_pages(shmem);  in drm_gem_shmem_vunmap_locked()
367 	shmem->vaddr = NULL;  in drm_gem_shmem_vunmap_locked()
371  * drm_gem_shmem_vunmap - Unmap a virtual mapping for a shmem GEM object
372  * @shmem: shmem GEM object
373  * @map: Kernel virtual address where the SHMEM GEM object was mapped
385 	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);  in drm_gem_shmem_vunmap()  local
387 	mutex_lock(&shmem->vmap_lock);  in drm_gem_shmem_vunmap()
388 	drm_gem_shmem_vunmap_locked(shmem, map);  in drm_gem_shmem_vunmap()
389 	mutex_unlock(&shmem->vmap_lock);  in drm_gem_shmem_vunmap()
398 	struct drm_gem_shmem_object *shmem;  in drm_gem_shmem_create_with_handle()  local
401 	shmem = drm_gem_shmem_create(dev, size);  in drm_gem_shmem_create_with_handle()
402 	if (IS_ERR(shmem))  in drm_gem_shmem_create_with_handle()
403 		return shmem;  in drm_gem_shmem_create_with_handle()
409 	ret = drm_gem_handle_create(file_priv, &shmem->base, handle);  in drm_gem_shmem_create_with_handle()
411 	drm_gem_object_put(&shmem->base);  in drm_gem_shmem_create_with_handle()
415 	return shmem;  in drm_gem_shmem_create_with_handle()
424 	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);  in drm_gem_shmem_madvise()  local
426 	mutex_lock(&shmem->pages_lock);  in drm_gem_shmem_madvise()
428 	if (shmem->madv >= 0)  in drm_gem_shmem_madvise()
429 		shmem->madv = madv;  in drm_gem_shmem_madvise()
431 	madv = shmem->madv;  in drm_gem_shmem_madvise()
433 	mutex_unlock(&shmem->pages_lock);  in drm_gem_shmem_madvise()
442 	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);  in drm_gem_shmem_purge_locked()  local
444 	WARN_ON(!drm_gem_shmem_is_purgeable(shmem));  in drm_gem_shmem_purge_locked()
446 	dma_unmap_sgtable(obj->dev->dev, shmem->sgt, DMA_BIDIRECTIONAL, 0);  in drm_gem_shmem_purge_locked()
447 	sg_free_table(shmem->sgt);  in drm_gem_shmem_purge_locked()
448 	kfree(shmem->sgt);  in drm_gem_shmem_purge_locked()
449 	shmem->sgt = NULL;  in drm_gem_shmem_purge_locked()
451 	drm_gem_shmem_put_pages_locked(shmem);  in drm_gem_shmem_purge_locked()
453 	shmem->madv = -1;  in drm_gem_shmem_purge_locked()
472 	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);  in drm_gem_shmem_purge()  local
474 	if (!mutex_trylock(&shmem->pages_lock))  in drm_gem_shmem_purge()
477 	mutex_unlock(&shmem->pages_lock);  in drm_gem_shmem_purge()
484  * drm_gem_shmem_dumb_create - Create a dumb shmem buffer object
504 	struct drm_gem_shmem_object *shmem;  in drm_gem_shmem_dumb_create()  local
517 	shmem = drm_gem_shmem_create_with_handle(file, dev, args->size, &args->handle);  in drm_gem_shmem_dumb_create()
519 	return PTR_ERR_OR_ZERO(shmem);  in drm_gem_shmem_dumb_create()
527 	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);  in drm_gem_shmem_fault()  local
536 	mutex_lock(&shmem->pages_lock);  in drm_gem_shmem_fault()
539 	    WARN_ON_ONCE(!shmem->pages) ||  in drm_gem_shmem_fault()
540 	    shmem->madv < 0) {  in drm_gem_shmem_fault()
543 		page = shmem->pages[page_offset];  in drm_gem_shmem_fault()
548 	mutex_unlock(&shmem->pages_lock);  in drm_gem_shmem_fault()
556 	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);  in drm_gem_shmem_vm_open()  local
559 	WARN_ON(shmem->base.import_attach);  in drm_gem_shmem_vm_open()
561 	ret = drm_gem_shmem_get_pages(shmem);  in drm_gem_shmem_vm_open()
570 	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);  in drm_gem_shmem_vm_close()  local
572 	drm_gem_shmem_put_pages(shmem);  in drm_gem_shmem_vm_close()
583  * drm_gem_shmem_mmap - Memory-map a shmem GEM object
588  * operation for shmem objects. Drivers which employ the shmem helpers should
596 	struct drm_gem_shmem_object *shmem;  in drm_gem_shmem_mmap()  local
607 	shmem = to_drm_gem_shmem_obj(obj);  in drm_gem_shmem_mmap()
609 	ret = drm_gem_shmem_get_pages(shmem);  in drm_gem_shmem_mmap()
617 	if (shmem->map_wc)  in drm_gem_shmem_mmap()
636 	const struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);  in drm_gem_shmem_print_info()  local
638 	drm_printf_indent(p, indent, "pages_use_count=%u\n", shmem->pages_use_count);  in drm_gem_shmem_print_info()
639 	drm_printf_indent(p, indent, "vmap_use_count=%u\n", shmem->vmap_use_count);  in drm_gem_shmem_print_info()
640 	drm_printf_indent(p, indent, "vaddr=%p\n", shmem->vaddr);  in drm_gem_shmem_print_info()
646  *                              pages for a shmem GEM object
662 	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);  in drm_gem_shmem_get_sg_table()  local
664 	WARN_ON(shmem->base.import_attach);  in drm_gem_shmem_get_sg_table()
666 	return drm_prime_pages_to_sg(obj->dev, shmem->pages, obj->size >> PAGE_SHIFT);  in drm_gem_shmem_get_sg_table()
672  *				 scatter/gather table for a shmem GEM object.
689 	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);  in drm_gem_shmem_get_pages_sgt()  local
692 	if (shmem->sgt)  in drm_gem_shmem_get_pages_sgt()
693 		return shmem->sgt;  in drm_gem_shmem_get_pages_sgt()
697 	ret = drm_gem_shmem_get_pages(shmem);  in drm_gem_shmem_get_pages_sgt()
701 	sgt = drm_gem_shmem_get_sg_table(&shmem->base);  in drm_gem_shmem_get_pages_sgt()
711 	shmem->sgt = sgt;  in drm_gem_shmem_get_pages_sgt()
719 	drm_gem_shmem_put_pages(shmem);  in drm_gem_shmem_get_pages_sgt()
725  * drm_gem_shmem_prime_import_sg_table - Produce a shmem GEM object from
732  * another driver. Drivers that use the shmem helpers should set this as their
745 	struct drm_gem_shmem_object *shmem;  in drm_gem_shmem_prime_import_sg_table()  local
747 	shmem = __drm_gem_shmem_create(dev, size, true);  in drm_gem_shmem_prime_import_sg_table()
748 	if (IS_ERR(shmem))  in drm_gem_shmem_prime_import_sg_table()
749 		return ERR_CAST(shmem);  in drm_gem_shmem_prime_import_sg_table()
751 	shmem->sgt = sgt;  in drm_gem_shmem_prime_import_sg_table()
755 	return &shmem->base;  in drm_gem_shmem_prime_import_sg_table()