1 // SPDX-License-Identifier: GPL-2.0-only
5  * Copyright (c) 2010-2013, NVIDIA Corporation.
41 	u32 thresh = waiter->thresh;  in add_waiter_to_queue()
44 		if ((s32)(pos->thresh - thresh) <= 0) {  in add_waiter_to_queue()
45 			list_add(&waiter->list, &pos->list);  in add_waiter_to_queue()
49 	list_add(&waiter->list, queue);  in add_waiter_to_queue()
54  * run through a waiter queue for a single sync point ID
64 		if ((s32)(waiter->thresh - sync) > 0)  in remove_completed_waiters()
67 		dest = completed + waiter->action;  in remove_completed_waiters()
70 		if (waiter->action == HOST1X_INTR_ACTION_SUBMIT_COMPLETE &&  in remove_completed_waiters()
72 			prev = list_entry(dest->prev,  in remove_completed_waiters()
74 			if (prev->data == waiter->data) {  in remove_completed_waiters()
75 				prev->count++;  in remove_completed_waiters()
80 		/* PENDING->REMOVED or CANCELLED->HANDLED */  in remove_completed_waiters()
81 		if (atomic_inc_return(&waiter->state) == WLS_HANDLED || !dest) {  in remove_completed_waiters()
82 			list_del(&waiter->list);  in remove_completed_waiters()
83 			kref_put(&waiter->refcount, waiter_release);  in remove_completed_waiters()
85 			list_move_tail(&waiter->list, dest);  in remove_completed_waiters()
89 static void reset_threshold_interrupt(struct host1x *host,  in reset_threshold_interrupt()  argument
91 				      unsigned int id)  in reset_threshold_interrupt()  argument
94 		list_first_entry(head, struct host1x_waitlist, list)->thresh;  in reset_threshold_interrupt()
96 	host1x_hw_intr_set_syncpt_threshold(host, id, thresh);  in reset_threshold_interrupt()
97 	host1x_hw_intr_enable_syncpt_intr(host, id);  in reset_threshold_interrupt()
102 	struct host1x_channel *channel = waiter->data;  in action_submit_complete()
104 	host1x_cdma_update(&channel->cdma);  in action_submit_complete()
107 	trace_host1x_channel_submit_complete(dev_name(channel->dev),  in action_submit_complete()
108 					     waiter->count, waiter->thresh);  in action_submit_complete()
113 	wait_queue_head_t *wq = waiter->data;  in action_wakeup()
120 	wait_queue_head_t *wq = waiter->data;  in action_wakeup_interruptible()
127 	struct host1x_syncpt_fence *f = waiter->data;  in action_signal_fence()
151 			list_del(&waiter->list);  in run_handlers()
153 			WARN_ON(atomic_xchg(&waiter->state, WLS_HANDLED) !=  in run_handlers()
155 			kref_put(&waiter->refcount, waiter_release);  in run_handlers()
163 static int process_wait_list(struct host1x *host,  in process_wait_list()  argument
174 	spin_lock(&syncpt->intr.lock);  in process_wait_list()
176 	remove_completed_waiters(&syncpt->intr.wait_head, threshold,  in process_wait_list()
179 	empty = list_empty(&syncpt->intr.wait_head);  in process_wait_list()
181 		host1x_hw_intr_disable_syncpt_intr(host, syncpt->id);  in process_wait_list()
183 		reset_threshold_interrupt(host, &syncpt->intr.wait_head,  in process_wait_list()
184 					  syncpt->id);  in process_wait_list()
186 	spin_unlock(&syncpt->intr.lock);  in process_wait_list()
204 	unsigned int id = syncpt->id;  in syncpt_thresh_work()  local
205 	struct host1x *host = syncpt->host;  in syncpt_thresh_work()  local
207 	(void)process_wait_list(host, syncpt,  in syncpt_thresh_work()
208 				host1x_syncpt_load(host->syncpt + id));  in syncpt_thresh_work()
211 int host1x_intr_add_action(struct host1x *host, struct host1x_syncpt *syncpt,  in host1x_intr_add_action()  argument
220 		return -EINVAL;  in host1x_intr_add_action()
224 	INIT_LIST_HEAD(&waiter->list);  in host1x_intr_add_action()
225 	kref_init(&waiter->refcount);  in host1x_intr_add_action()
227 		kref_get(&waiter->refcount);  in host1x_intr_add_action()
228 	waiter->thresh = thresh;  in host1x_intr_add_action()
229 	waiter->action = action;  in host1x_intr_add_action()
230 	atomic_set(&waiter->state, WLS_PENDING);  in host1x_intr_add_action()
231 	waiter->data = data;  in host1x_intr_add_action()
232 	waiter->count = 1;  in host1x_intr_add_action()
234 	spin_lock(&syncpt->intr.lock);  in host1x_intr_add_action()
236 	queue_was_empty = list_empty(&syncpt->intr.wait_head);  in host1x_intr_add_action()
238 	if (add_waiter_to_queue(waiter, &syncpt->intr.wait_head)) {  in host1x_intr_add_action()
239 		/* added at head of list - new threshold value */  in host1x_intr_add_action()
240 		host1x_hw_intr_set_syncpt_threshold(host, syncpt->id, thresh);  in host1x_intr_add_action()
242 		/* added as first waiter - enable interrupt */  in host1x_intr_add_action()
244 			host1x_hw_intr_enable_syncpt_intr(host, syncpt->id);  in host1x_intr_add_action()
250 	spin_unlock(&syncpt->intr.lock);  in host1x_intr_add_action()
255 void host1x_intr_put_ref(struct host1x *host, unsigned int id, void *ref,  in host1x_intr_put_ref()  argument
261 	atomic_cmpxchg(&waiter->state, WLS_PENDING, WLS_CANCELLED);  in host1x_intr_put_ref()
263 	syncpt = host->syncpt + id;  in host1x_intr_put_ref()
265 	spin_lock(&syncpt->intr.lock);  in host1x_intr_put_ref()
266 	if (atomic_cmpxchg(&waiter->state, WLS_CANCELLED, WLS_HANDLED) ==  in host1x_intr_put_ref()
268 		list_del(&waiter->list);  in host1x_intr_put_ref()
269 		kref_put(&waiter->refcount, waiter_release);  in host1x_intr_put_ref()
271 	spin_unlock(&syncpt->intr.lock);  in host1x_intr_put_ref()
275 		while (atomic_read(&waiter->state) != WLS_HANDLED)  in host1x_intr_put_ref()
279 	kref_put(&waiter->refcount, waiter_release);  in host1x_intr_put_ref()
282 int host1x_intr_init(struct host1x *host, unsigned int irq_sync)  in host1x_intr_init()  argument
284 	unsigned int id;  in host1x_intr_init()  local
285 	u32 nb_pts = host1x_syncpt_nb_pts(host);  in host1x_intr_init()
287 	mutex_init(&host->intr_mutex);  in host1x_intr_init()
288 	host->intr_syncpt_irq = irq_sync;  in host1x_intr_init()
290 	for (id = 0; id < nb_pts; ++id) {  in host1x_intr_init()
291 		struct host1x_syncpt *syncpt = host->syncpt + id;  in host1x_intr_init()
293 		spin_lock_init(&syncpt->intr.lock);  in host1x_intr_init()
294 		INIT_LIST_HEAD(&syncpt->intr.wait_head);  in host1x_intr_init()
295 		snprintf(syncpt->intr.thresh_irq_name,  in host1x_intr_init()
296 			 sizeof(syncpt->intr.thresh_irq_name),  in host1x_intr_init()
297 			 "host1x_sp_%02u", id);  in host1x_intr_init()
300 	host1x_intr_start(host);  in host1x_intr_init()
305 void host1x_intr_deinit(struct host1x *host)  in host1x_intr_deinit()  argument
307 	host1x_intr_stop(host);  in host1x_intr_deinit()
310 void host1x_intr_start(struct host1x *host)  in host1x_intr_start()  argument
312 	u32 hz = clk_get_rate(host->clk);  in host1x_intr_start()
315 	mutex_lock(&host->intr_mutex);  in host1x_intr_start()
316 	err = host1x_hw_intr_init_host_sync(host, DIV_ROUND_UP(hz, 1000000),  in host1x_intr_start()
319 		mutex_unlock(&host->intr_mutex);  in host1x_intr_start()
322 	mutex_unlock(&host->intr_mutex);  in host1x_intr_start()
325 void host1x_intr_stop(struct host1x *host)  in host1x_intr_stop()  argument
327 	unsigned int id;  in host1x_intr_stop()  local
328 	struct host1x_syncpt *syncpt = host->syncpt;  in host1x_intr_stop()
329 	u32 nb_pts = host1x_syncpt_nb_pts(host);  in host1x_intr_stop()
331 	mutex_lock(&host->intr_mutex);  in host1x_intr_stop()
333 	host1x_hw_intr_disable_all_syncpt_intrs(host);  in host1x_intr_stop()
335 	for (id = 0; id < nb_pts; ++id) {  in host1x_intr_stop()
339 			&syncpt[id].intr.wait_head, list) {  in host1x_intr_stop()
340 			if (atomic_cmpxchg(&waiter->state,  in host1x_intr_stop()
342 				list_del(&waiter->list);  in host1x_intr_stop()
343 				kref_put(&waiter->refcount, waiter_release);  in host1x_intr_stop()
347 		if (!list_empty(&syncpt[id].intr.wait_head)) {  in host1x_intr_stop()
349 			mutex_unlock(&host->intr_mutex);  in host1x_intr_stop()
350 			pr_warn("%s cannot stop syncpt intr id=%u\n",  in host1x_intr_stop()
351 				__func__, id);  in host1x_intr_stop()
356 	host1x_hw_intr_free_syncpt_irq(host);  in host1x_intr_stop()
358 	mutex_unlock(&host->intr_mutex);  in host1x_intr_stop()