1 /* SPDX-License-Identifier: GPL-2.0 OR MIT */
4  * Copyright (c) 2009-2022 VMware, Inc., Palo Alto, CA., USA
21  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
29  * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
39  * Base- and reference object implementation for the various
41  * and release on file close.
70  * file release.
92  * This is the per-device data structure needed for ttm object management.
106  * @hash: Hash entry for the per-file object reference hash.
108  * @head: List entry for the per-file list of ref-objects.
117  * that allows lookup with a pointer to the referenced object as a key. In
138 	kref_get(&tfile->refcount);  in ttm_object_file_ref()
143 				  uint64_t key,  in ttm_tfile_find_ref_rcu()  argument
148 	hash_for_each_possible_rcu(tfile->ref_hash, hash, head, key) {  in ttm_tfile_find_ref_rcu()
149 		if (hash->key == key) {  in ttm_tfile_find_ref_rcu()
154 	return -EINVAL;  in ttm_tfile_find_ref_rcu()
158 			      uint64_t key,  in ttm_tfile_find_ref()  argument
163 	hash_for_each_possible(tfile->ref_hash, hash, head, key) {  in ttm_tfile_find_ref()
164 		if (hash->key == key) {  in ttm_tfile_find_ref()
169 	return -EINVAL;  in ttm_tfile_find_ref()
186 	kref_put(&tfile->refcount, ttm_object_file_destroy);  in ttm_object_file_unref()
196 	struct ttm_object_device *tdev = tfile->tdev;  in ttm_base_object_init()
199 	base->shareable = shareable;  in ttm_base_object_init()
200 	base->tfile = ttm_object_file_ref(tfile);  in ttm_base_object_init()
201 	base->refcount_release = refcount_release;  in ttm_base_object_init()
202 	base->object_type = object_type;  in ttm_base_object_init()
203 	kref_init(&base->refcount);  in ttm_base_object_init()
205 	spin_lock(&tdev->object_lock);  in ttm_base_object_init()
206 	ret = idr_alloc(&tdev->idr, base, 1, 0, GFP_NOWAIT);  in ttm_base_object_init()
207 	spin_unlock(&tdev->object_lock);  in ttm_base_object_init()
212 	base->handle = ret;  in ttm_base_object_init()
221 	spin_lock(&tdev->object_lock);  in ttm_base_object_init()
222 	idr_remove(&tdev->idr, base->handle);  in ttm_base_object_init()
223 	spin_unlock(&tdev->object_lock);  in ttm_base_object_init()
231 	struct ttm_object_device *tdev = base->tfile->tdev;  in ttm_release_base()
233 	spin_lock(&tdev->object_lock);  in ttm_release_base()
234 	idr_remove(&tdev->idr, base->handle);  in ttm_release_base()
235 	spin_unlock(&tdev->object_lock);  in ttm_release_base()
243 	ttm_object_file_unref(&base->tfile);  in ttm_release_base()
244 	if (base->refcount_release)  in ttm_release_base()
245 		base->refcount_release(&base);  in ttm_release_base()
254 	kref_put(&base->refcount, ttm_release_base);  in ttm_base_object_unref()
258 					       uint64_t key)  in ttm_base_object_lookup()  argument
264 	spin_lock(&tfile->lock);  in ttm_base_object_lookup()
265 	ret = ttm_tfile_find_ref(tfile, key, &hash);  in ttm_base_object_lookup()
268 		base = hlist_entry(hash, struct ttm_ref_object, hash)->obj;  in ttm_base_object_lookup()
269 		if (!kref_get_unless_zero(&base->refcount))  in ttm_base_object_lookup()
272 	spin_unlock(&tfile->lock);  in ttm_base_object_lookup()
279 ttm_base_object_lookup_for_ref(struct ttm_object_device *tdev, uint64_t key)  in ttm_base_object_lookup_for_ref()  argument
284 	base = idr_find(&tdev->idr, key);  in ttm_base_object_lookup_for_ref()
286 	if (base && !kref_get_unless_zero(&base->refcount))  in ttm_base_object_lookup_for_ref()
300 	int ret = -EINVAL;  in ttm_ref_object_add()
302 	if (base->tfile != tfile && !base->shareable)  in ttm_ref_object_add()
303 		return -EPERM;  in ttm_ref_object_add()
308 	while (ret == -EINVAL) {  in ttm_ref_object_add()
310 		ret = ttm_tfile_find_ref_rcu(tfile, base->handle, &hash);  in ttm_ref_object_add()
314 			if (kref_get_unless_zero(&ref->kref)) {  in ttm_ref_object_add()
322 			return -EPERM;  in ttm_ref_object_add()
326 			return -ENOMEM;  in ttm_ref_object_add()
329 		ref->hash.key = base->handle;  in ttm_ref_object_add()
330 		ref->obj = base;  in ttm_ref_object_add()
331 		ref->tfile = tfile;  in ttm_ref_object_add()
332 		kref_init(&ref->kref);  in ttm_ref_object_add()
334 		spin_lock(&tfile->lock);  in ttm_ref_object_add()
335 		hash_add_rcu(tfile->ref_hash, &ref->hash.head, ref->hash.key);  in ttm_ref_object_add()
338 		list_add_tail(&ref->head, &tfile->ref_list);  in ttm_ref_object_add()
339 		kref_get(&base->refcount);  in ttm_ref_object_add()
340 		spin_unlock(&tfile->lock);  in ttm_ref_object_add()
348 static void __releases(tfile->lock) __acquires(tfile->lock)
353 	struct ttm_object_file *tfile = ref->tfile;  in ttm_ref_object_release()
355 	hash_del_rcu(&ref->hash.head);  in ttm_ref_object_release()
356 	list_del(&ref->head);  in ttm_ref_object_release()
357 	spin_unlock(&tfile->lock);  in ttm_ref_object_release()
359 	ttm_base_object_unref(&ref->obj);  in ttm_ref_object_release()
361 	spin_lock(&tfile->lock);  in ttm_ref_object_release()
365 			      unsigned long key)  in ttm_ref_object_base_unref()  argument
371 	spin_lock(&tfile->lock);  in ttm_ref_object_base_unref()
372 	ret = ttm_tfile_find_ref(tfile, key, &hash);  in ttm_ref_object_base_unref()
374 		spin_unlock(&tfile->lock);  in ttm_ref_object_base_unref()
375 		return -EINVAL;  in ttm_ref_object_base_unref()
378 	kref_put(&ref->kref, ttm_ref_object_release);  in ttm_ref_object_base_unref()
379 	spin_unlock(&tfile->lock);  in ttm_ref_object_base_unref()
390 	spin_lock(&tfile->lock);  in ttm_object_file_release()
393 	 * Since we release the lock within the loop, we have to  in ttm_object_file_release()
397 	while (!list_empty(&tfile->ref_list)) {  in ttm_object_file_release()
398 		list = tfile->ref_list.next;  in ttm_object_file_release()
400 		ttm_ref_object_release(&ref->kref);  in ttm_object_file_release()
403 	spin_unlock(&tfile->lock);  in ttm_object_file_release()
415 	spin_lock_init(&tfile->lock);  in ttm_object_file_init()
416 	tfile->tdev = tdev;  in ttm_object_file_init()
417 	kref_init(&tfile->refcount);  in ttm_object_file_init()
418 	INIT_LIST_HEAD(&tfile->ref_list);  in ttm_object_file_init()
420 	hash_init(tfile->ref_hash);  in ttm_object_file_init()
433 	spin_lock_init(&tdev->object_lock);  in ttm_object_device_init()
434 	atomic_set(&tdev->object_count, 0);  in ttm_object_device_init()
444 	idr_init_base(&tdev->idr, VMWGFX_NUM_MOB + 1);  in ttm_object_device_init()
445 	tdev->ops = *ops;  in ttm_object_device_init()
446 	tdev->dmabuf_release = tdev->ops.release;  in ttm_object_device_init()
447 	tdev->ops.release = ttm_prime_dmabuf_release;  in ttm_object_device_init()
457 	WARN_ON_ONCE(!idr_is_empty(&tdev->idr));  in ttm_object_device_release()
458 	idr_destroy(&tdev->idr);  in ttm_object_device_release()
464  * get_dma_buf_unless_doomed - get a dma_buf reference if possible.
466  * @dmabuf: Non-refcounted pointer to a struct dma-buf.
469  * the file, but synchronizes with its release method to make sure it has
478 	return atomic_long_inc_not_zero(&dmabuf->file->f_count) != 0L;  in get_dma_buf_unless_doomed()
482  * ttm_prime_refcount_release - refcount release method for a prime object.
498 	BUG_ON(prime->dma_buf != NULL);  in ttm_prime_refcount_release()
499 	mutex_destroy(&prime->mutex);  in ttm_prime_refcount_release()
500 	if (prime->refcount_release)  in ttm_prime_refcount_release()
501 		prime->refcount_release(&base);  in ttm_prime_refcount_release()
505  * ttm_prime_dmabuf_release - Release method for the dma-bufs we export
509  * This function first calls the dma_buf release method the driver
517 		(struct ttm_prime_object *) dma_buf->priv;  in ttm_prime_dmabuf_release()
518 	struct ttm_base_object *base = &prime->base;  in ttm_prime_dmabuf_release()
519 	struct ttm_object_device *tdev = base->tfile->tdev;  in ttm_prime_dmabuf_release()
521 	if (tdev->dmabuf_release)  in ttm_prime_dmabuf_release()
522 		tdev->dmabuf_release(dma_buf);  in ttm_prime_dmabuf_release()
523 	mutex_lock(&prime->mutex);  in ttm_prime_dmabuf_release()
524 	if (prime->dma_buf == dma_buf)  in ttm_prime_dmabuf_release()
525 		prime->dma_buf = NULL;  in ttm_prime_dmabuf_release()
526 	mutex_unlock(&prime->mutex);  in ttm_prime_dmabuf_release()
531  * ttm_prime_fd_to_handle - Get a base object handle from a prime fd
538  * a dma-buf. Note that we don't handle imports yet, because we simply
544 	struct ttm_object_device *tdev = tfile->tdev;  in ttm_prime_fd_to_handle()
554 	if (dma_buf->ops != &tdev->ops)  in ttm_prime_fd_to_handle()
555 		return -ENOSYS;  in ttm_prime_fd_to_handle()
557 	prime = (struct ttm_prime_object *) dma_buf->priv;  in ttm_prime_fd_to_handle()
558 	base = &prime->base;  in ttm_prime_fd_to_handle()
559 	*handle = base->handle;  in ttm_prime_fd_to_handle()
568  * ttm_prime_handle_to_fd - Return a dma_buf fd from a ttm prime object
572  * @flags: flags for dma-buf creation. We just pass them on.
580 	struct ttm_object_device *tdev = tfile->tdev;  in ttm_prime_handle_to_fd()
588 		     base->object_type != ttm_prime_type)) {  in ttm_prime_handle_to_fd()
589 		ret = -ENOENT;  in ttm_prime_handle_to_fd()
594 	if (unlikely(!base->shareable)) {  in ttm_prime_handle_to_fd()
595 		ret = -EPERM;  in ttm_prime_handle_to_fd()
599 	ret = mutex_lock_interruptible(&prime->mutex);  in ttm_prime_handle_to_fd()
601 		ret = -ERESTARTSYS;  in ttm_prime_handle_to_fd()
605 	dma_buf = prime->dma_buf;  in ttm_prime_handle_to_fd()
608 		exp_info.ops = &tdev->ops;  in ttm_prime_handle_to_fd()
609 		exp_info.size = prime->size;  in ttm_prime_handle_to_fd()
620 			mutex_unlock(&prime->mutex);  in ttm_prime_handle_to_fd()
628 		prime->dma_buf = dma_buf;  in ttm_prime_handle_to_fd()
630 	mutex_unlock(&prime->mutex);  in ttm_prime_handle_to_fd()
646  * ttm_prime_object_init - Initialize a ttm_prime_object
663 	mutex_init(&prime->mutex);  in ttm_prime_object_init()
664 	prime->size = PAGE_ALIGN(size);  in ttm_prime_object_init()
665 	prime->real_type = type;  in ttm_prime_object_init()
666 	prime->dma_buf = NULL;  in ttm_prime_object_init()
667 	prime->refcount_release = refcount_release;  in ttm_prime_object_init()
668 	return ttm_base_object_init(tfile, &prime->base, shareable,  in ttm_prime_object_init()