1 // SPDX-License-Identifier: GPL-2.0-only
3  * Fence mechanism for dma-buf and to allow for asynchronous dma access
16 #include <linux/dma-fence.h>
50  * Since the purposes of fences is to facilitate cross-device and
51  * cross-application synchronization, there's multiple ways to use one:
53  * - Individual fences can be exposed as a &sync_file, accessed as a file
58  * - Some subsystems also have their own explicit fencing primitives, like
62  * - Then there's also implicit fencing, where the synchronization points are
69  * DOC: fence cross-driver contract
77  *   further command submission and force complete all in-flight fences, e.g.
110  * track asynchronous compute work using &dma_fence. No driver outside of
125  * dma_fence_get_stub - return a signaled fence
152  * dma_fence_allocate_private_stub - return a private, signaled fence
162 		return ERR_PTR(-ENOMEM);  in dma_fence_allocate_private_stub()
179  * dma_fence_context_alloc - allocate an array of fence contexts
199  * * It is a cross-driver contract, and therefore all drivers must follow the
201  *   and anything else significant for in-kernel interfaces. But it is also
202  *   impossible to test all drivers in a single machine, hence brute-force N vs.
218  *   this pulls in there is extremely thin wiggle-room for driver-specific
224  * Furthermore lockdep does not handle cross-release dependencies, which means
256  *   dma_fence_signal() is called. Un-annotated code can contain deadlock issues,
277  *   lockdep with the relevant hierarchy at boot-up. This means even just
288  * dma_fence_begin_signalling - begin a critical DMA fence signalling section
311 	/* ... and non-recursive readlock */  in dma_fence_begin_signalling()
319  * dma_fence_end_signalling - end a critical DMA fence signalling section
349  * dma_fence_signal_timestamp_locked - signal completion of a fence
372 	lockdep_assert_held(fence->lock);  in dma_fence_signal_timestamp_locked()
375 				      &fence->flags)))  in dma_fence_signal_timestamp_locked()
376 		return -EINVAL;  in dma_fence_signal_timestamp_locked()
379 	list_replace(&fence->cb_list, &cb_list);  in dma_fence_signal_timestamp_locked()
381 	fence->timestamp = timestamp;  in dma_fence_signal_timestamp_locked()
382 	set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags);  in dma_fence_signal_timestamp_locked()
386 		INIT_LIST_HEAD(&cur->node);  in dma_fence_signal_timestamp_locked()
387 		cur->func(fence, cur);  in dma_fence_signal_timestamp_locked()
395  * dma_fence_signal_timestamp - signal completion of a fence
415 		return -EINVAL;  in dma_fence_signal_timestamp()
417 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_signal_timestamp()
419 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_signal_timestamp()
426  * dma_fence_signal_locked - signal completion of a fence
448  * dma_fence_signal - signal completion of a fence
467 		return -EINVAL;  in dma_fence_signal()
471 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_signal()
473 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_signal()
482  * dma_fence_wait_timeout - sleep until the fence gets signaled
488  * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or the
493  * directly or indirectly (buf-mgr between reservation and committing)
505 		return -EINVAL;  in dma_fence_wait_timeout()
514 	if (fence->ops->wait)  in dma_fence_wait_timeout()
515 		ret = fence->ops->wait(fence, intr, timeout);  in dma_fence_wait_timeout()
524  * dma_fence_release - default relese function for fences
537 	if (WARN(!list_empty(&fence->cb_list) &&  in dma_fence_release()
538 		 !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags),  in dma_fence_release()
540 		 fence->ops->get_driver_name(fence),  in dma_fence_release()
541 		 fence->ops->get_timeline_name(fence),  in dma_fence_release()
542 		 fence->context, fence->seqno)) {  in dma_fence_release()
552 		spin_lock_irqsave(fence->lock, flags);  in dma_fence_release()
553 		fence->error = -EDEADLK;  in dma_fence_release()
555 		spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_release()
558 	if (fence->ops->release)  in dma_fence_release()
559 		fence->ops->release(fence);  in dma_fence_release()
566  * dma_fence_free - default release function for &dma_fence.
582 	lockdep_assert_held(fence->lock);  in __dma_fence_enable_signaling()
585 				   &fence->flags);  in __dma_fence_enable_signaling()
587 	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))  in __dma_fence_enable_signaling()
590 	if (!was_set && fence->ops->enable_signaling) {  in __dma_fence_enable_signaling()
593 		if (!fence->ops->enable_signaling(fence)) {  in __dma_fence_enable_signaling()
603  * dma_fence_enable_sw_signaling - enable signaling on fence
614 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_enable_sw_signaling()
616 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_enable_sw_signaling()
621  * dma_fence_add_callback - add a callback to be called when the fence
624  * @cb: the callback to register
630  * @cb will be initialized by dma_fence_add_callback(), no initialization
634  * If fence is already signaled, this function will return -ENOENT (and
639  * Returns 0 in case of success, -ENOENT if the fence is already signaled
640  * and -EINVAL in case of error.
642 int dma_fence_add_callback(struct dma_fence *fence, struct dma_fence_cb *cb,  in dma_fence_add_callback()  argument
649 		return -EINVAL;  in dma_fence_add_callback()
651 	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {  in dma_fence_add_callback()
652 		INIT_LIST_HEAD(&cb->node);  in dma_fence_add_callback()
653 		return -ENOENT;  in dma_fence_add_callback()
656 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_add_callback()
659 		cb->func = func;  in dma_fence_add_callback()
660 		list_add_tail(&cb->node, &fence->cb_list);  in dma_fence_add_callback()
662 		INIT_LIST_HEAD(&cb->node);  in dma_fence_add_callback()
663 		ret = -ENOENT;  in dma_fence_add_callback()
666 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_add_callback()
673  * dma_fence_get_status - returns the status upon completion
689 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_get_status()
691 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_get_status()
698  * dma_fence_remove_callback - remove a callback from the signaling list
700  * @cb: the callback to remove
712  * Behaviour is undefined if @cb has not been added to @fence using
716 dma_fence_remove_callback(struct dma_fence *fence, struct dma_fence_cb *cb)  in dma_fence_remove_callback()  argument
721 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_remove_callback()
723 	ret = !list_empty(&cb->node);  in dma_fence_remove_callback()
725 		list_del_init(&cb->node);  in dma_fence_remove_callback()
727 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_remove_callback()
739 dma_fence_default_wait_cb(struct dma_fence *fence, struct dma_fence_cb *cb)  in dma_fence_default_wait_cb()  argument
742 		container_of(cb, struct default_wait_cb, base);  in dma_fence_default_wait_cb()
744 	wake_up_state(wait->task, TASK_NORMAL);  in dma_fence_default_wait_cb()
748  * dma_fence_default_wait - default sleep until the fence gets signaled
754  * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or the
762 	struct default_wait_cb cb;  in dma_fence_default_wait()  local
766 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_default_wait()
768 	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))  in dma_fence_default_wait()
772 		ret = -ERESTARTSYS;  in dma_fence_default_wait()
781 	cb.base.func = dma_fence_default_wait_cb;  in dma_fence_default_wait()
782 	cb.task = current;  in dma_fence_default_wait()
783 	list_add(&cb.base.node, &fence->cb_list);  in dma_fence_default_wait()
785 	while (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) && ret > 0) {  in dma_fence_default_wait()
790 		spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_default_wait()
794 		spin_lock_irqsave(fence->lock, flags);  in dma_fence_default_wait()
796 			ret = -ERESTARTSYS;  in dma_fence_default_wait()
799 	if (!list_empty(&cb.base.node))  in dma_fence_default_wait()
800 		list_del(&cb.base.node);  in dma_fence_default_wait()
804 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_default_wait()
817 		if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {  in dma_fence_test_signaled_any()
827  * dma_fence_wait_any_timeout - sleep until any fence gets signaled
836  * Returns -EINVAL on custom fence wait implementation, -ERESTARTSYS if
850 	struct default_wait_cb *cb;  in dma_fence_wait_any_timeout()  local
855 		return -EINVAL;  in dma_fence_wait_any_timeout()
868 	cb = kcalloc(count, sizeof(struct default_wait_cb), GFP_KERNEL);  in dma_fence_wait_any_timeout()
869 	if (cb == NULL) {  in dma_fence_wait_any_timeout()
870 		ret = -ENOMEM;  in dma_fence_wait_any_timeout()
877 		cb[i].task = current;  in dma_fence_wait_any_timeout()
878 		if (dma_fence_add_callback(fence, &cb[i].base,  in dma_fence_wait_any_timeout()
899 			ret = -ERESTARTSYS;  in dma_fence_wait_any_timeout()
905 	while (i-- > 0)  in dma_fence_wait_any_timeout()
906 		dma_fence_remove_callback(fences[i], &cb[i].base);  in dma_fence_wait_any_timeout()
909 	kfree(cb);  in dma_fence_wait_any_timeout()
916  * dma_fence_describe - Dump fence describtion into seq_file
925 		   fence->ops->get_driver_name(fence),  in dma_fence_describe()
926 		   fence->ops->get_timeline_name(fence), fence->seqno,  in dma_fence_describe()
932  * dma_fence_init - Initialize a custom fence.
951 	BUG_ON(!ops || !ops->get_driver_name || !ops->get_timeline_name);  in dma_fence_init()
953 	kref_init(&fence->refcount);  in dma_fence_init()
954 	fence->ops = ops;  in dma_fence_init()
955 	INIT_LIST_HEAD(&fence->cb_list);  in dma_fence_init()
956 	fence->lock = lock;  in dma_fence_init()
957 	fence->context = context;  in dma_fence_init()
958 	fence->seqno = seqno;  in dma_fence_init()
959 	fence->flags = 0UL;  in dma_fence_init()
960 	fence->error = 0;  in dma_fence_init()