102 int dma_fence_signal_locked(struct dma_fence *fence)  in dma_fence_signal_locked()  argument
107 	lockdep_assert_held(fence->lock);  in dma_fence_signal_locked()
109 	if (WARN_ON(!fence))  in dma_fence_signal_locked()
112 	if (test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {  in dma_fence_signal_locked()
120 		fence->timestamp = ktime_get();  in dma_fence_signal_locked()
121 		set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags);  in dma_fence_signal_locked()
122 		trace_dma_fence_signaled(fence);  in dma_fence_signal_locked()
125 	list_for_each_entry_safe(cur, tmp, &fence->cb_list, node) {  in dma_fence_signal_locked()
127 		cur->func(fence, cur);  in dma_fence_signal_locked()
146 int dma_fence_signal(struct dma_fence *fence)  in dma_fence_signal()  argument
150 	if (!fence)  in dma_fence_signal()
153 	if (test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))  in dma_fence_signal()
156 	fence->timestamp = ktime_get();  in dma_fence_signal()
157 	set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags);  in dma_fence_signal()
158 	trace_dma_fence_signaled(fence);  in dma_fence_signal()
160 	if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &fence->flags)) {  in dma_fence_signal()
163 		spin_lock_irqsave(fence->lock, flags);  in dma_fence_signal()
164 		list_for_each_entry_safe(cur, tmp, &fence->cb_list, node) {  in dma_fence_signal()
166 			cur->func(fence, cur);  in dma_fence_signal()
168 		spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_signal()
193 dma_fence_wait_timeout(struct dma_fence *fence, bool intr, signed long timeout)  in dma_fence_wait_timeout()  argument
200 	trace_dma_fence_wait_start(fence);  in dma_fence_wait_timeout()
201 	if (fence->ops->wait)  in dma_fence_wait_timeout()
202 		ret = fence->ops->wait(fence, intr, timeout);  in dma_fence_wait_timeout()
204 		ret = dma_fence_default_wait(fence, intr, timeout);  in dma_fence_wait_timeout()
205 	trace_dma_fence_wait_end(fence);  in dma_fence_wait_timeout()
219 	struct dma_fence *fence =  in dma_fence_release()  local
222 	trace_dma_fence_destroy(fence);  in dma_fence_release()
225 	WARN_ON(!list_empty(&fence->cb_list));  in dma_fence_release()
227 	if (fence->ops->release)  in dma_fence_release()
228 		fence->ops->release(fence);  in dma_fence_release()
230 		dma_fence_free(fence);  in dma_fence_release()
241 void dma_fence_free(struct dma_fence *fence)  in dma_fence_free()  argument
243 	kfree_rcu(fence, rcu);  in dma_fence_free()
255 void dma_fence_enable_sw_signaling(struct dma_fence *fence)  in dma_fence_enable_sw_signaling()  argument
260 			      &fence->flags) &&  in dma_fence_enable_sw_signaling()
261 	    !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) &&  in dma_fence_enable_sw_signaling()
262 	    fence->ops->enable_signaling) {  in dma_fence_enable_sw_signaling()
263 		trace_dma_fence_enable_signal(fence);  in dma_fence_enable_sw_signaling()
265 		spin_lock_irqsave(fence->lock, flags);  in dma_fence_enable_sw_signaling()
267 		if (!fence->ops->enable_signaling(fence))  in dma_fence_enable_sw_signaling()
268 			dma_fence_signal_locked(fence);  in dma_fence_enable_sw_signaling()
270 		spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_enable_sw_signaling()
300 int dma_fence_add_callback(struct dma_fence *fence, struct dma_fence_cb *cb,  in dma_fence_add_callback()  argument
307 	if (WARN_ON(!fence || !func))  in dma_fence_add_callback()
310 	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {  in dma_fence_add_callback()
315 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_add_callback()
318 				   &fence->flags);  in dma_fence_add_callback()
320 	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))  in dma_fence_add_callback()
322 	else if (!was_set && fence->ops->enable_signaling) {  in dma_fence_add_callback()
323 		trace_dma_fence_enable_signal(fence);  in dma_fence_add_callback()
325 		if (!fence->ops->enable_signaling(fence)) {  in dma_fence_add_callback()
326 			dma_fence_signal_locked(fence);  in dma_fence_add_callback()
333 		list_add_tail(&cb->node, &fence->cb_list);  in dma_fence_add_callback()
336 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_add_callback()
354 int dma_fence_get_status(struct dma_fence *fence)  in dma_fence_get_status()  argument
359 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_get_status()
360 	status = dma_fence_get_status_locked(fence);  in dma_fence_get_status()
361 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_get_status()
386 dma_fence_remove_callback(struct dma_fence *fence, struct dma_fence_cb *cb)  in dma_fence_remove_callback()  argument
391 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_remove_callback()
397 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_remove_callback()
409 dma_fence_default_wait_cb(struct dma_fence *fence, struct dma_fence_cb *cb)  in dma_fence_default_wait_cb()  argument
430 dma_fence_default_wait(struct dma_fence *fence, bool intr, signed long timeout)  in dma_fence_default_wait()  argument
437 	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))  in dma_fence_default_wait()
440 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_default_wait()
448 				   &fence->flags);  in dma_fence_default_wait()
450 	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))  in dma_fence_default_wait()
453 	if (!was_set && fence->ops->enable_signaling) {  in dma_fence_default_wait()
454 		trace_dma_fence_enable_signal(fence);  in dma_fence_default_wait()
456 		if (!fence->ops->enable_signaling(fence)) {  in dma_fence_default_wait()
457 			dma_fence_signal_locked(fence);  in dma_fence_default_wait()
469 	list_add(&cb.base.node, &fence->cb_list);  in dma_fence_default_wait()
471 	while (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) && ret > 0) {  in dma_fence_default_wait()
476 		spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_default_wait()
480 		spin_lock_irqsave(fence->lock, flags);  in dma_fence_default_wait()
490 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_default_wait()
502 		struct dma_fence *fence = fences[i];  in dma_fence_test_signaled_any()  local
503 		if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {  in dma_fence_test_signaled_any()
561 		struct dma_fence *fence = fences[i];  in dma_fence_wait_any_timeout()  local
564 		if (dma_fence_add_callback(fence, &cb[i].base,  in dma_fence_wait_any_timeout()
617 dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops,  in dma_fence_init()  argument
623 	kref_init(&fence->refcount);  in dma_fence_init()
624 	fence->ops = ops;  in dma_fence_init()
625 	INIT_LIST_HEAD(&fence->cb_list);  in dma_fence_init()
626 	fence->lock = lock;  in dma_fence_init()
627 	fence->context = context;  in dma_fence_init()
628 	fence->seqno = seqno;  in dma_fence_init()
629 	fence->flags = 0UL;  in dma_fence_init()
630 	fence->error = 0;  in dma_fence_init()
632 	trace_dma_fence_init(fence);  in dma_fence_init()