1 // SPDX-License-Identifier: GPL-2.0
6 #include <linux/dma-buf.h>
61 	if (dev->driver->gem_create_object) {  in __drm_gem_shmem_create()
62 		obj = dev->driver->gem_create_object(dev, size);  in __drm_gem_shmem_create()
69 			return ERR_PTR(-ENOMEM);  in __drm_gem_shmem_create()
70 		obj = &shmem->base;  in __drm_gem_shmem_create()
73 	if (!obj->funcs)  in __drm_gem_shmem_create()
74 		obj->funcs = &drm_gem_shmem_funcs;  in __drm_gem_shmem_create()
78 		shmem->map_wc = false; /* dma-buf mappings use always writecombine */  in __drm_gem_shmem_create()
89 	mutex_init(&shmem->pages_lock);  in __drm_gem_shmem_create()
90 	mutex_init(&shmem->vmap_lock);  in __drm_gem_shmem_create()
91 	INIT_LIST_HEAD(&shmem->madv_list);  in __drm_gem_shmem_create()
101 		mapping_set_gfp_mask(obj->filp->f_mapping, GFP_HIGHUSER |  in __drm_gem_shmem_create()
115  * drm_gem_shmem_create - Allocate an object with the given size
122  * A struct drm_gem_shmem_object * on success or an ERR_PTR()-encoded negative
132  * drm_gem_shmem_free - Free resources associated with a shmem GEM object
140 	struct drm_gem_object *obj = &shmem->base;  in drm_gem_shmem_free()
142 	WARN_ON(shmem->vmap_use_count);  in drm_gem_shmem_free()
144 	if (obj->import_attach) {  in drm_gem_shmem_free()
145 		drm_prime_gem_destroy(obj, shmem->sgt);  in drm_gem_shmem_free()
147 		if (shmem->sgt) {  in drm_gem_shmem_free()
148 			dma_unmap_sgtable(obj->dev->dev, shmem->sgt,  in drm_gem_shmem_free()
150 			sg_free_table(shmem->sgt);  in drm_gem_shmem_free()
151 			kfree(shmem->sgt);  in drm_gem_shmem_free()
153 		if (shmem->pages)  in drm_gem_shmem_free()
157 	WARN_ON(shmem->pages_use_count);  in drm_gem_shmem_free()
160 	mutex_destroy(&shmem->pages_lock);  in drm_gem_shmem_free()
161 	mutex_destroy(&shmem->vmap_lock);  in drm_gem_shmem_free()
168 	struct drm_gem_object *obj = &shmem->base;  in drm_gem_shmem_get_pages_locked()
171 	if (shmem->pages_use_count++ > 0)  in drm_gem_shmem_get_pages_locked()
177 		shmem->pages_use_count = 0;  in drm_gem_shmem_get_pages_locked()
187 	if (shmem->map_wc)  in drm_gem_shmem_get_pages_locked()
188 		set_pages_array_wc(pages, obj->size >> PAGE_SHIFT);  in drm_gem_shmem_get_pages_locked()
191 	shmem->pages = pages;  in drm_gem_shmem_get_pages_locked()
197  * drm_gem_shmem_get_pages - Allocate backing pages for a shmem GEM object
210 	WARN_ON(shmem->base.import_attach);  in drm_gem_shmem_get_pages()
212 	ret = mutex_lock_interruptible(&shmem->pages_lock);  in drm_gem_shmem_get_pages()
216 	mutex_unlock(&shmem->pages_lock);  in drm_gem_shmem_get_pages()
224 	struct drm_gem_object *obj = &shmem->base;  in drm_gem_shmem_put_pages_locked()
226 	if (WARN_ON_ONCE(!shmem->pages_use_count))  in drm_gem_shmem_put_pages_locked()
229 	if (--shmem->pages_use_count > 0)  in drm_gem_shmem_put_pages_locked()
233 	if (shmem->map_wc)  in drm_gem_shmem_put_pages_locked()
234 		set_pages_array_wb(shmem->pages, obj->size >> PAGE_SHIFT);  in drm_gem_shmem_put_pages_locked()
237 	drm_gem_put_pages(obj, shmem->pages,  in drm_gem_shmem_put_pages_locked()
238 			  shmem->pages_mark_dirty_on_put,  in drm_gem_shmem_put_pages_locked()
239 			  shmem->pages_mark_accessed_on_put);  in drm_gem_shmem_put_pages_locked()
240 	shmem->pages = NULL;  in drm_gem_shmem_put_pages_locked()
244  * drm_gem_shmem_put_pages - Decrease use count on the backing pages for a shmem GEM object
251 	mutex_lock(&shmem->pages_lock);  in drm_gem_shmem_put_pages()
253 	mutex_unlock(&shmem->pages_lock);  in drm_gem_shmem_put_pages()
258  * drm_gem_shmem_pin - Pin backing pages for a shmem GEM object
269 	WARN_ON(shmem->base.import_attach);  in drm_gem_shmem_pin()
276  * drm_gem_shmem_unpin - Unpin backing pages for a shmem GEM object
284 	WARN_ON(shmem->base.import_attach);  in drm_gem_shmem_unpin()
293 	struct drm_gem_object *obj = &shmem->base;  in drm_gem_shmem_vmap_locked()
296 	if (shmem->vmap_use_count++ > 0) {  in drm_gem_shmem_vmap_locked()
297 		iosys_map_set_vaddr(map, shmem->vaddr);  in drm_gem_shmem_vmap_locked()
301 	if (obj->import_attach) {  in drm_gem_shmem_vmap_locked()
302 		ret = dma_buf_vmap(obj->import_attach->dmabuf, map);  in drm_gem_shmem_vmap_locked()
304 			if (WARN_ON(map->is_iomem)) {  in drm_gem_shmem_vmap_locked()
305 				dma_buf_vunmap(obj->import_attach->dmabuf, map);  in drm_gem_shmem_vmap_locked()
306 				ret = -EIO;  in drm_gem_shmem_vmap_locked()
309 			shmem->vaddr = map->vaddr;  in drm_gem_shmem_vmap_locked()
318 		if (shmem->map_wc)  in drm_gem_shmem_vmap_locked()
320 		shmem->vaddr = vmap(shmem->pages, obj->size >> PAGE_SHIFT,  in drm_gem_shmem_vmap_locked()
322 		if (!shmem->vaddr)  in drm_gem_shmem_vmap_locked()
323 			ret = -ENOMEM;  in drm_gem_shmem_vmap_locked()
325 			iosys_map_set_vaddr(map, shmem->vaddr);  in drm_gem_shmem_vmap_locked()
336 	if (!obj->import_attach)  in drm_gem_shmem_vmap_locked()
339 	shmem->vmap_use_count = 0;  in drm_gem_shmem_vmap_locked()
345  * drm_gem_shmem_vmap - Create a virtual mapping for a shmem GEM object
352  * between dma-buf imported and natively allocated objects.
364 	ret = mutex_lock_interruptible(&shmem->vmap_lock);  in drm_gem_shmem_vmap()
368 	mutex_unlock(&shmem->vmap_lock);  in drm_gem_shmem_vmap()
377 	struct drm_gem_object *obj = &shmem->base;  in drm_gem_shmem_vunmap_locked()
379 	if (WARN_ON_ONCE(!shmem->vmap_use_count))  in drm_gem_shmem_vunmap_locked()
382 	if (--shmem->vmap_use_count > 0)  in drm_gem_shmem_vunmap_locked()
385 	if (obj->import_attach) {  in drm_gem_shmem_vunmap_locked()
386 		dma_buf_vunmap(obj->import_attach->dmabuf, map);  in drm_gem_shmem_vunmap_locked()
388 		vunmap(shmem->vaddr);  in drm_gem_shmem_vunmap_locked()
392 	shmem->vaddr = NULL;  in drm_gem_shmem_vunmap_locked()
396  * drm_gem_shmem_vunmap - Unmap a virtual mapping for a shmem GEM object
404  * This function hides the differences between dma-buf imported and natively
410 	mutex_lock(&shmem->vmap_lock);  in drm_gem_shmem_vunmap()
412 	mutex_unlock(&shmem->vmap_lock);  in drm_gem_shmem_vunmap()
432 	ret = drm_gem_handle_create(file_priv, &shmem->base, handle);  in drm_gem_shmem_create_with_handle()
433 	/* drop reference from allocate - handle holds it now. */  in drm_gem_shmem_create_with_handle()
434 	drm_gem_object_put(&shmem->base);  in drm_gem_shmem_create_with_handle()
442  * false or -errno.
446 	mutex_lock(&shmem->pages_lock);  in drm_gem_shmem_madvise()
448 	if (shmem->madv >= 0)  in drm_gem_shmem_madvise()
449 		shmem->madv = madv;  in drm_gem_shmem_madvise()
451 	madv = shmem->madv;  in drm_gem_shmem_madvise()
453 	mutex_unlock(&shmem->pages_lock);  in drm_gem_shmem_madvise()
461 	struct drm_gem_object *obj = &shmem->base;  in drm_gem_shmem_purge_locked()
462 	struct drm_device *dev = obj->dev;  in drm_gem_shmem_purge_locked()
466 	dma_unmap_sgtable(dev->dev, shmem->sgt, DMA_BIDIRECTIONAL, 0);  in drm_gem_shmem_purge_locked()
467 	sg_free_table(shmem->sgt);  in drm_gem_shmem_purge_locked()
468 	kfree(shmem->sgt);  in drm_gem_shmem_purge_locked()
469 	shmem->sgt = NULL;  in drm_gem_shmem_purge_locked()
473 	shmem->madv = -1;  in drm_gem_shmem_purge_locked()
475 	drm_vma_node_unmap(&obj->vma_node, dev->anon_inode->i_mapping);  in drm_gem_shmem_purge_locked()
483 	shmem_truncate_range(file_inode(obj->filp), 0, (loff_t)-1);  in drm_gem_shmem_purge_locked()
485 	invalidate_mapping_pages(file_inode(obj->filp)->i_mapping, 0, (loff_t)-1);  in drm_gem_shmem_purge_locked()
491 	if (!mutex_trylock(&shmem->pages_lock))  in drm_gem_shmem_purge()
494 	mutex_unlock(&shmem->pages_lock);  in drm_gem_shmem_purge()
501  * drm_gem_shmem_dumb_create - Create a dumb shmem buffer object
520 	u32 min_pitch = DIV_ROUND_UP(args->width * args->bpp, 8);  in drm_gem_shmem_dumb_create()
523 	if (!args->pitch || !args->size) {  in drm_gem_shmem_dumb_create()
524 		args->pitch = min_pitch;  in drm_gem_shmem_dumb_create()
525 		args->size = PAGE_ALIGN(args->pitch * args->height);  in drm_gem_shmem_dumb_create()
528 		if (args->pitch < min_pitch)  in drm_gem_shmem_dumb_create()
529 			args->pitch = min_pitch;  in drm_gem_shmem_dumb_create()
530 		if (args->size < args->pitch * args->height)  in drm_gem_shmem_dumb_create()
531 			args->size = PAGE_ALIGN(args->pitch * args->height);  in drm_gem_shmem_dumb_create()
534 	shmem = drm_gem_shmem_create_with_handle(file, dev, args->size, &args->handle);  in drm_gem_shmem_dumb_create()
542 	struct vm_area_struct *vma = vmf->vma;  in drm_gem_shmem_fault()
543 	struct drm_gem_object *obj = vma->vm_private_data;  in drm_gem_shmem_fault()
545 	loff_t num_pages = obj->size >> PAGE_SHIFT;  in drm_gem_shmem_fault()
550 	/* We don't use vmf->pgoff since that has the fake offset */  in drm_gem_shmem_fault()
551 	page_offset = (vmf->address - vma->vm_start) >> PAGE_SHIFT;  in drm_gem_shmem_fault()
553 	mutex_lock(&shmem->pages_lock);  in drm_gem_shmem_fault()
556 	    WARN_ON_ONCE(!shmem->pages) ||  in drm_gem_shmem_fault()
557 	    shmem->madv < 0) {  in drm_gem_shmem_fault()
560 		page = shmem->pages[page_offset];  in drm_gem_shmem_fault()
562 		ret = vmf_insert_pfn(vma, vmf->address, page_to_pfn(page));  in drm_gem_shmem_fault()
565 	mutex_unlock(&shmem->pages_lock);  in drm_gem_shmem_fault()
572 	struct drm_gem_object *obj = vma->vm_private_data;  in drm_gem_shmem_vm_open()
575 	WARN_ON(shmem->base.import_attach);  in drm_gem_shmem_vm_open()
577 	mutex_lock(&shmem->pages_lock);  in drm_gem_shmem_vm_open()
584 	if (!WARN_ON_ONCE(!shmem->pages_use_count))  in drm_gem_shmem_vm_open()
585 		shmem->pages_use_count++;  in drm_gem_shmem_vm_open()
587 	mutex_unlock(&shmem->pages_lock);  in drm_gem_shmem_vm_open()
594 	struct drm_gem_object *obj = vma->vm_private_data;  in drm_gem_shmem_vm_close()
609  * drm_gem_shmem_mmap - Memory-map a shmem GEM object
621 	struct drm_gem_object *obj = &shmem->base;  in drm_gem_shmem_mmap()
624 	if (obj->import_attach) {  in drm_gem_shmem_mmap()
627 		vma->vm_private_data = NULL;  in drm_gem_shmem_mmap()
629 		return dma_buf_mmap(obj->dma_buf, vma, 0);  in drm_gem_shmem_mmap()
636 	vma->vm_flags |= VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP;  in drm_gem_shmem_mmap()
637 	vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);  in drm_gem_shmem_mmap()
638 	if (shmem->map_wc)  in drm_gem_shmem_mmap()
639 		vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);  in drm_gem_shmem_mmap()
646  * drm_gem_shmem_print_info() - Print &drm_gem_shmem_object info for debugfs
654 	drm_printf_indent(p, indent, "pages_use_count=%u\n", shmem->pages_use_count);  in drm_gem_shmem_print_info()
655 	drm_printf_indent(p, indent, "vmap_use_count=%u\n", shmem->vmap_use_count);  in drm_gem_shmem_print_info()
656 	drm_printf_indent(p, indent, "vaddr=%p\n", shmem->vaddr);  in drm_gem_shmem_print_info()
661  * drm_gem_shmem_get_sg_table - Provide a scatter/gather table of pinned
665  * This function exports a scatter/gather table suitable for PRIME usage by
668  * Drivers who need to acquire an scatter/gather table for objects need to call
672  * A pointer to the scatter/gather table of pinned pages or error pointer on failure.
676 	struct drm_gem_object *obj = &shmem->base;  in drm_gem_shmem_get_sg_table()
678 	WARN_ON(shmem->base.import_attach);  in drm_gem_shmem_get_sg_table()
680 	return drm_prime_pages_to_sg(obj->dev, shmem->pages, obj->size >> PAGE_SHIFT);  in drm_gem_shmem_get_sg_table()
685  * drm_gem_shmem_get_pages_sgt - Pin pages, dma map them, and return a
686  *				 scatter/gather table for a shmem GEM object.
689  * This function returns a scatter/gather table suitable for driver usage. If
690  * the sg table doesn't exist, the pages are pinned, dma-mapped, and a sg
694  * and difference between dma-buf imported and natively allocated objects.
698  * A pointer to the scatter/gather table of pinned pages or errno on failure.
702 	struct drm_gem_object *obj = &shmem->base;  in drm_gem_shmem_get_pages_sgt()
706 	if (shmem->sgt)  in drm_gem_shmem_get_pages_sgt()
707 		return shmem->sgt;  in drm_gem_shmem_get_pages_sgt()
709 	WARN_ON(obj->import_attach);  in drm_gem_shmem_get_pages_sgt()
721 	ret = dma_map_sgtable(obj->dev->dev, sgt, DMA_BIDIRECTIONAL, 0);  in drm_gem_shmem_get_pages_sgt()
725 	shmem->sgt = sgt;  in drm_gem_shmem_get_pages_sgt()
739  * drm_gem_shmem_prime_import_sg_table - Produce a shmem GEM object from
740  *                 another driver's scatter/gather table of pinned pages
742  * @attach: DMA-BUF attachment
743  * @sgt: Scatter/gather table of pinned pages
745  * This function imports a scatter/gather table exported via DMA-BUF by
750  * A pointer to a newly created GEM object or an ERR_PTR-encoded negative
758 	size_t size = PAGE_ALIGN(attach->dmabuf->size);  in drm_gem_shmem_prime_import_sg_table()
765 	shmem->sgt = sgt;  in drm_gem_shmem_prime_import_sg_table()
769 	return &shmem->base;  in drm_gem_shmem_prime_import_sg_table()
773 MODULE_DESCRIPTION("DRM SHMEM memory-management helpers");