39 static inline void debug_fence_init(struct i915_sw_fence *fence)  in debug_fence_init()  argument
41 	debug_object_init(fence, &i915_sw_fence_debug_descr);  in debug_fence_init()
44 static inline void debug_fence_init_onstack(struct i915_sw_fence *fence)  in debug_fence_init_onstack()  argument
46 	debug_object_init_on_stack(fence, &i915_sw_fence_debug_descr);  in debug_fence_init_onstack()
49 static inline void debug_fence_activate(struct i915_sw_fence *fence)  in debug_fence_activate()  argument
51 	debug_object_activate(fence, &i915_sw_fence_debug_descr);  in debug_fence_activate()
54 static inline void debug_fence_set_state(struct i915_sw_fence *fence,  in debug_fence_set_state()  argument
57 	debug_object_active_state(fence, &i915_sw_fence_debug_descr, old, new);  in debug_fence_set_state()
60 static inline void debug_fence_deactivate(struct i915_sw_fence *fence)  in debug_fence_deactivate()  argument
62 	debug_object_deactivate(fence, &i915_sw_fence_debug_descr);  in debug_fence_deactivate()
65 static inline void debug_fence_destroy(struct i915_sw_fence *fence)  in debug_fence_destroy()  argument
67 	debug_object_destroy(fence, &i915_sw_fence_debug_descr);  in debug_fence_destroy()
70 static inline void debug_fence_free(struct i915_sw_fence *fence)  in debug_fence_free()  argument
72 	debug_object_free(fence, &i915_sw_fence_debug_descr);  in debug_fence_free()
76 static inline void debug_fence_assert(struct i915_sw_fence *fence)  in debug_fence_assert()  argument
78 	debug_object_assert_init(fence, &i915_sw_fence_debug_descr);  in debug_fence_assert()
83 static inline void debug_fence_init(struct i915_sw_fence *fence)  in debug_fence_init()  argument
87 static inline void debug_fence_init_onstack(struct i915_sw_fence *fence)  in debug_fence_init_onstack()  argument
91 static inline void debug_fence_activate(struct i915_sw_fence *fence)  in debug_fence_activate()  argument
95 static inline void debug_fence_set_state(struct i915_sw_fence *fence,  in debug_fence_set_state()  argument
100 static inline void debug_fence_deactivate(struct i915_sw_fence *fence)  in debug_fence_deactivate()  argument
104 static inline void debug_fence_destroy(struct i915_sw_fence *fence)  in debug_fence_destroy()  argument
108 static inline void debug_fence_free(struct i915_sw_fence *fence)  in debug_fence_free()  argument
112 static inline void debug_fence_assert(struct i915_sw_fence *fence)  in debug_fence_assert()  argument
118 static int __i915_sw_fence_notify(struct i915_sw_fence *fence,  in __i915_sw_fence_notify()  argument
123 	fn = (i915_sw_fence_notify_t)(fence->flags & I915_SW_FENCE_MASK);  in __i915_sw_fence_notify()
124 	return fn(fence, state);  in __i915_sw_fence_notify()
128 void i915_sw_fence_fini(struct i915_sw_fence *fence)  in i915_sw_fence_fini()  argument
130 	debug_fence_free(fence);  in i915_sw_fence_fini()
134 static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence,  in __i915_sw_fence_wake_up_all()  argument
137 	wait_queue_head_t *x = &fence->wait;  in __i915_sw_fence_wake_up_all()
141 	debug_fence_deactivate(fence);  in __i915_sw_fence_wake_up_all()
142 	atomic_set_release(&fence->pending, -1); /* 0 -> -1 [done] */  in __i915_sw_fence_wake_up_all()
174 	debug_fence_assert(fence);  in __i915_sw_fence_wake_up_all()
177 static void __i915_sw_fence_complete(struct i915_sw_fence *fence,  in __i915_sw_fence_complete()  argument
180 	debug_fence_assert(fence);  in __i915_sw_fence_complete()
182 	if (!atomic_dec_and_test(&fence->pending))  in __i915_sw_fence_complete()
185 	debug_fence_set_state(fence, DEBUG_FENCE_IDLE, DEBUG_FENCE_NOTIFY);  in __i915_sw_fence_complete()
187 	if (__i915_sw_fence_notify(fence, FENCE_COMPLETE) != NOTIFY_DONE)  in __i915_sw_fence_complete()
190 	debug_fence_set_state(fence, DEBUG_FENCE_NOTIFY, DEBUG_FENCE_IDLE);  in __i915_sw_fence_complete()
192 	__i915_sw_fence_wake_up_all(fence, continuation);  in __i915_sw_fence_complete()
194 	debug_fence_destroy(fence);  in __i915_sw_fence_complete()
195 	__i915_sw_fence_notify(fence, FENCE_FREE);  in __i915_sw_fence_complete()
198 static void i915_sw_fence_complete(struct i915_sw_fence *fence)  in i915_sw_fence_complete()  argument
200 	debug_fence_assert(fence);  in i915_sw_fence_complete()
202 	if (WARN_ON(i915_sw_fence_done(fence)))  in i915_sw_fence_complete()
205 	__i915_sw_fence_complete(fence, NULL);  in i915_sw_fence_complete()
208 static void i915_sw_fence_await(struct i915_sw_fence *fence)  in i915_sw_fence_await()  argument
210 	debug_fence_assert(fence);  in i915_sw_fence_await()
211 	WARN_ON(atomic_inc_return(&fence->pending) <= 1);  in i915_sw_fence_await()
214 void __i915_sw_fence_init(struct i915_sw_fence *fence,  in __i915_sw_fence_init()  argument
221 	debug_fence_init(fence);  in __i915_sw_fence_init()
223 	__init_waitqueue_head(&fence->wait, name, key);  in __i915_sw_fence_init()
224 	atomic_set(&fence->pending, 1);  in __i915_sw_fence_init()
225 	fence->flags = (unsigned long)fn;  in __i915_sw_fence_init()
228 void i915_sw_fence_commit(struct i915_sw_fence *fence)  in i915_sw_fence_commit()  argument
230 	debug_fence_activate(fence);  in i915_sw_fence_commit()
231 	i915_sw_fence_complete(fence);  in i915_sw_fence_commit()
244 static bool __i915_sw_fence_check_if_after(struct i915_sw_fence *fence,  in __i915_sw_fence_check_if_after()  argument
249 	if (__test_and_set_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))  in __i915_sw_fence_check_if_after()
252 	if (fence == signaler)  in __i915_sw_fence_check_if_after()
255 	list_for_each_entry(wq, &fence->wait.head, entry) {  in __i915_sw_fence_check_if_after()
266 static void __i915_sw_fence_clear_checked_bit(struct i915_sw_fence *fence)  in __i915_sw_fence_clear_checked_bit()  argument
270 	if (!__test_and_clear_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))  in __i915_sw_fence_clear_checked_bit()
273 	list_for_each_entry(wq, &fence->wait.head, entry) {  in __i915_sw_fence_clear_checked_bit()
281 static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence,  in i915_sw_fence_check_if_after()  argument
291 	err = __i915_sw_fence_check_if_after(fence, signaler);  in i915_sw_fence_check_if_after()
292 	__i915_sw_fence_clear_checked_bit(fence);  in i915_sw_fence_check_if_after()
298 static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,  in __i915_sw_fence_await_sw_fence()  argument
305 	debug_fence_assert(fence);  in __i915_sw_fence_await_sw_fence()
314 	if (unlikely(i915_sw_fence_check_if_after(fence, signaler)))  in __i915_sw_fence_await_sw_fence()
334 	wq->private = fence;  in __i915_sw_fence_await_sw_fence()
336 	i915_sw_fence_await(fence);  in __i915_sw_fence_await_sw_fence()
351 int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,  in i915_sw_fence_await_sw_fence()  argument
355 	return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0);  in i915_sw_fence_await_sw_fence()
358 int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence,  in i915_sw_fence_await_sw_fence_gfp()  argument
362 	return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp);  in i915_sw_fence_await_sw_fence_gfp()
367 	struct i915_sw_fence *fence;  member
383 	i915_sw_fence_complete(cb->fence);  in dma_i915_sw_fence_wake()
390 	struct i915_sw_fence *fence;  in timer_i915_sw_fence_wake()  local
392 	fence = xchg(&cb->base.fence, NULL);  in timer_i915_sw_fence_wake()
393 	if (!fence)  in timer_i915_sw_fence_wake()
401 	i915_sw_fence_complete(fence);  in timer_i915_sw_fence_wake()
409 	struct i915_sw_fence *fence;  in dma_i915_sw_fence_wake_timer()  local
411 	fence = xchg(&cb->base.fence, NULL);  in dma_i915_sw_fence_wake_timer()
412 	if (fence)  in dma_i915_sw_fence_wake_timer()
413 		i915_sw_fence_complete(fence);  in dma_i915_sw_fence_wake_timer()
429 int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,  in i915_sw_fence_await_dma_fence()  argument
438 	debug_fence_assert(fence);  in i915_sw_fence_await_dma_fence()
455 	cb->fence = fence;  in i915_sw_fence_await_dma_fence()
456 	i915_sw_fence_await(fence);  in i915_sw_fence_await_dma_fence()
485 int i915_sw_fence_await_reservation(struct i915_sw_fence *fence,  in i915_sw_fence_await_reservation()  argument
495 	debug_fence_assert(fence);  in i915_sw_fence_await_reservation()
511 			pending = i915_sw_fence_await_dma_fence(fence,  in i915_sw_fence_await_reservation()
531 		pending = i915_sw_fence_await_dma_fence(fence,  in i915_sw_fence_await_reservation()