91  * queue->lock and bh_disabled in order to avoid races within
95 __rpc_disable_timer(struct rpc_wait_queue *queue, struct rpc_task *task)  in __rpc_disable_timer()  argument
101 	if (list_empty(&queue->timer_list.list))  in __rpc_disable_timer()
102 		cancel_delayed_work(&queue->timer_list.dwork);  in __rpc_disable_timer()
106 rpc_set_queue_timer(struct rpc_wait_queue *queue, unsigned long expires)  in rpc_set_queue_timer()  argument
109 	queue->timer_list.expires = expires;  in rpc_set_queue_timer()
114 	mod_delayed_work(rpciod_workqueue, &queue->timer_list.dwork, expires);  in rpc_set_queue_timer()
121 __rpc_add_timer(struct rpc_wait_queue *queue, struct rpc_task *task,  in __rpc_add_timer()  argument
125 	if (list_empty(&queue->timer_list.list) || time_before(timeout, queue->timer_list.expires))  in __rpc_add_timer()
126 		rpc_set_queue_timer(queue, timeout);  in __rpc_add_timer()
127 	list_add(&task->u.tk_wait.timer_list, &queue->timer_list.list);  in __rpc_add_timer()
130 static void rpc_set_waitqueue_priority(struct rpc_wait_queue *queue, int priority)  in rpc_set_waitqueue_priority()  argument
132 	if (queue->priority != priority) {  in rpc_set_waitqueue_priority()
133 		queue->priority = priority;  in rpc_set_waitqueue_priority()
134 		queue->nr = 1U << priority;  in rpc_set_waitqueue_priority()
138 static void rpc_reset_waitqueue_priority(struct rpc_wait_queue *queue)  in rpc_reset_waitqueue_priority()  argument
140 	rpc_set_waitqueue_priority(queue, queue->maxpriority);  in rpc_reset_waitqueue_priority()
144  * Add a request to a queue list
155 			/* Cache the queue head in task->u.tk_wait.list */  in __rpc_list_enqueue_task()
166  * Remove request from a queue list
182 		/* Assume __rpc_list_enqueue_task() cached the queue head */  in __rpc_list_dequeue_task()
191  * Add new request to a priority queue.
193 static void __rpc_add_wait_queue_priority(struct rpc_wait_queue *queue,  in __rpc_add_wait_queue_priority()  argument
197 	if (unlikely(queue_priority > queue->maxpriority))  in __rpc_add_wait_queue_priority()
198 		queue_priority = queue->maxpriority;  in __rpc_add_wait_queue_priority()
199 	__rpc_list_enqueue_task(&queue->tasks[queue_priority], task);  in __rpc_add_wait_queue_priority()
203  * Add new request to wait queue.
205 static void __rpc_add_wait_queue(struct rpc_wait_queue *queue,  in __rpc_add_wait_queue()  argument
210 	if (RPC_IS_PRIORITY(queue))  in __rpc_add_wait_queue()
211 		__rpc_add_wait_queue_priority(queue, task, queue_priority);  in __rpc_add_wait_queue()
213 		list_add_tail(&task->u.tk_wait.list, &queue->tasks[0]);  in __rpc_add_wait_queue()
214 	task->tk_waitqueue = queue;  in __rpc_add_wait_queue()
215 	queue->qlen++;  in __rpc_add_wait_queue()
222  * Remove request from a priority queue.
230  * Remove request from queue.
233 static void __rpc_remove_wait_queue(struct rpc_wait_queue *queue, struct rpc_task *task)  in __rpc_remove_wait_queue()  argument
235 	__rpc_disable_timer(queue, task);  in __rpc_remove_wait_queue()
236 	if (RPC_IS_PRIORITY(queue))  in __rpc_remove_wait_queue()
240 	queue->qlen--;  in __rpc_remove_wait_queue()
243 static void __rpc_init_priority_wait_queue(struct rpc_wait_queue *queue, const char *qname, unsigne…  in __rpc_init_priority_wait_queue()  argument
247 	spin_lock_init(&queue->lock);  in __rpc_init_priority_wait_queue()
248 	for (i = 0; i < ARRAY_SIZE(queue->tasks); i++)  in __rpc_init_priority_wait_queue()
249 		INIT_LIST_HEAD(&queue->tasks[i]);  in __rpc_init_priority_wait_queue()
250 	queue->maxpriority = nr_queues - 1;  in __rpc_init_priority_wait_queue()
251 	rpc_reset_waitqueue_priority(queue);  in __rpc_init_priority_wait_queue()
252 	queue->qlen = 0;  in __rpc_init_priority_wait_queue()
253 	queue->timer_list.expires = 0;  in __rpc_init_priority_wait_queue()
254 	INIT_DELAYED_WORK(&queue->timer_list.dwork, __rpc_queue_timer_fn);  in __rpc_init_priority_wait_queue()
255 	INIT_LIST_HEAD(&queue->timer_list.list);  in __rpc_init_priority_wait_queue()
256 	rpc_assign_waitqueue_name(queue, qname);  in __rpc_init_priority_wait_queue()
259 void rpc_init_priority_wait_queue(struct rpc_wait_queue *queue, const char *qname)  in rpc_init_priority_wait_queue()  argument
261 	__rpc_init_priority_wait_queue(queue, qname, RPC_NR_PRIORITY);  in rpc_init_priority_wait_queue()
265 void rpc_init_wait_queue(struct rpc_wait_queue *queue, const char *qname)  in rpc_init_wait_queue()  argument
267 	__rpc_init_priority_wait_queue(queue, qname, 1);  in rpc_init_wait_queue()
271 void rpc_destroy_wait_queue(struct rpc_wait_queue *queue)  in rpc_destroy_wait_queue()  argument
273 	cancel_delayed_work_sync(&queue->timer_list.dwork);  in rpc_destroy_wait_queue()
354  * rpc_wait_queue, this must be called with the queue spinlock held to protect
355  * the wait queue operation.
377  * Prepare for sleeping on a wait queue.
380  * as it's on a wait queue.
439 	 * Protect the queue operations.  in rpc_sleep_on_timeout()
457 	 * Protect the queue operations.  in rpc_sleep_on()
473 	 * Protect the queue operations.  in rpc_sleep_on_priority_timeout()
490 	 * Protect the queue operations.  in rpc_sleep_on_priority()
501  * @queue: wait queue
504  * Caller must hold queue->lock, and have cleared the task queued flag.
507 		struct rpc_wait_queue *queue,  in __rpc_do_wake_up_task_on_wq()  argument
516 	trace_rpc_task_wakeup(task, queue);  in __rpc_do_wake_up_task_on_wq()
518 	__rpc_remove_wait_queue(queue, task);  in __rpc_do_wake_up_task_on_wq()
524  * Wake up a queued task while the queue lock is being held
528 		struct rpc_wait_queue *queue, struct rpc_task *task,  in rpc_wake_up_task_on_wq_queue_action_locked()  argument
533 		if (task->tk_waitqueue == queue) {  in rpc_wake_up_task_on_wq_queue_action_locked()
535 				__rpc_do_wake_up_task_on_wq(wq, queue, task);  in rpc_wake_up_task_on_wq_queue_action_locked()
544  * Wake up a queued task while the queue lock is being held
546 static void rpc_wake_up_task_queue_locked(struct rpc_wait_queue *queue,  in rpc_wake_up_task_queue_locked()  argument
549 	rpc_wake_up_task_on_wq_queue_action_locked(rpciod_workqueue, queue,  in rpc_wake_up_task_queue_locked()
554  * Wake up a task on a specific queue
556 void rpc_wake_up_queued_task(struct rpc_wait_queue *queue, struct rpc_task *task)  in rpc_wake_up_queued_task()  argument
560 	spin_lock(&queue->lock);  in rpc_wake_up_queued_task()
561 	rpc_wake_up_task_queue_locked(queue, task);  in rpc_wake_up_queued_task()
562 	spin_unlock(&queue->lock);  in rpc_wake_up_queued_task()
573 rpc_wake_up_task_queue_set_status_locked(struct rpc_wait_queue *queue,  in rpc_wake_up_task_queue_set_status_locked()  argument
576 	rpc_wake_up_task_on_wq_queue_action_locked(rpciod_workqueue, queue,  in rpc_wake_up_task_queue_set_status_locked()
582  * @queue: pointer to rpc_wait_queue
586  * If @task is queued on @queue, then it is woken up, and @task->tk_status is
590 rpc_wake_up_queued_task_set_status(struct rpc_wait_queue *queue,  in rpc_wake_up_queued_task_set_status()  argument
595 	spin_lock(&queue->lock);  in rpc_wake_up_queued_task_set_status()
596 	rpc_wake_up_task_queue_set_status_locked(queue, task, status);  in rpc_wake_up_queued_task_set_status()
597 	spin_unlock(&queue->lock);  in rpc_wake_up_queued_task_set_status()
601  * Wake up the next task on a priority queue.
603 static struct rpc_task *__rpc_find_next_queued_priority(struct rpc_wait_queue *queue)  in __rpc_find_next_queued_priority()  argument
609 	 * Service the privileged queue.  in __rpc_find_next_queued_priority()
611 	q = &queue->tasks[RPC_NR_PRIORITY - 1];  in __rpc_find_next_queued_priority()
612 	if (queue->maxpriority > RPC_PRIORITY_PRIVILEGED && !list_empty(q)) {  in __rpc_find_next_queued_priority()
620 	q = &queue->tasks[queue->priority];  in __rpc_find_next_queued_priority()
621 	if (!list_empty(q) && queue->nr) {  in __rpc_find_next_queued_priority()
622 		queue->nr--;  in __rpc_find_next_queued_priority()
628 	 * Service the next queue.  in __rpc_find_next_queued_priority()
631 		if (q == &queue->tasks[0])  in __rpc_find_next_queued_priority()
632 			q = &queue->tasks[queue->maxpriority];  in __rpc_find_next_queued_priority()
639 	} while (q != &queue->tasks[queue->priority]);  in __rpc_find_next_queued_priority()
641 	rpc_reset_waitqueue_priority(queue);  in __rpc_find_next_queued_priority()
645 	rpc_set_waitqueue_priority(queue, (unsigned int)(q - &queue->tasks[0]));  in __rpc_find_next_queued_priority()
650 static struct rpc_task *__rpc_find_next_queued(struct rpc_wait_queue *queue)  in __rpc_find_next_queued()  argument
652 	if (RPC_IS_PRIORITY(queue))  in __rpc_find_next_queued()
653 		return __rpc_find_next_queued_priority(queue);  in __rpc_find_next_queued()
654 	if (!list_empty(&queue->tasks[0]))  in __rpc_find_next_queued()
655 		return list_first_entry(&queue->tasks[0], struct rpc_task, u.tk_wait.list);  in __rpc_find_next_queued()
660  * Wake up the first task on the wait queue.
663 		struct rpc_wait_queue *queue,  in rpc_wake_up_first_on_wq()  argument
668 	spin_lock(&queue->lock);  in rpc_wake_up_first_on_wq()
669 	task = __rpc_find_next_queued(queue);  in rpc_wake_up_first_on_wq()
671 		task = rpc_wake_up_task_on_wq_queue_action_locked(wq, queue,  in rpc_wake_up_first_on_wq()
673 	spin_unlock(&queue->lock);  in rpc_wake_up_first_on_wq()
679  * Wake up the first task on the wait queue.
681 struct rpc_task *rpc_wake_up_first(struct rpc_wait_queue *queue,  in rpc_wake_up_first()  argument
684 	return rpc_wake_up_first_on_wq(rpciod_workqueue, queue, func, data);  in rpc_wake_up_first()
694  * Wake up the next task on the wait queue.
696 struct rpc_task *rpc_wake_up_next(struct rpc_wait_queue *queue)  in rpc_wake_up_next()  argument
698 	return rpc_wake_up_first(queue, rpc_wake_up_next_func, NULL);  in rpc_wake_up_next()
704  * @queue: rpc_wait_queue on which the tasks are sleeping
707 static void rpc_wake_up_locked(struct rpc_wait_queue *queue)  in rpc_wake_up_locked()  argument
712 		task = __rpc_find_next_queued(queue);  in rpc_wake_up_locked()
715 		rpc_wake_up_task_queue_locked(queue, task);  in rpc_wake_up_locked()
721  * @queue: rpc_wait_queue on which the tasks are sleeping
723  * Grabs queue->lock
725 void rpc_wake_up(struct rpc_wait_queue *queue)  in rpc_wake_up()  argument
727 	spin_lock(&queue->lock);  in rpc_wake_up()
728 	rpc_wake_up_locked(queue);  in rpc_wake_up()
729 	spin_unlock(&queue->lock);  in rpc_wake_up()
735  * @queue: rpc_wait_queue on which the tasks are sleeping
738 static void rpc_wake_up_status_locked(struct rpc_wait_queue *queue, int status)  in rpc_wake_up_status_locked()  argument
743 		task = __rpc_find_next_queued(queue);  in rpc_wake_up_status_locked()
746 		rpc_wake_up_task_queue_set_status_locked(queue, task, status);  in rpc_wake_up_status_locked()
752  * @queue: rpc_wait_queue on which the tasks are sleeping
755  * Grabs queue->lock
757 void rpc_wake_up_status(struct rpc_wait_queue *queue, int status)  in rpc_wake_up_status()  argument
759 	spin_lock(&queue->lock);  in rpc_wake_up_status()
760 	rpc_wake_up_status_locked(queue, status);  in rpc_wake_up_status()
761 	spin_unlock(&queue->lock);  in rpc_wake_up_status()
767 	struct rpc_wait_queue *queue = container_of(work,  in __rpc_queue_timer_fn()  local
773 	spin_lock(&queue->lock);  in __rpc_queue_timer_fn()
775 	list_for_each_entry_safe(task, n, &queue->timer_list.list, u.tk_wait.timer_list) {  in __rpc_queue_timer_fn()
780 			rpc_wake_up_task_queue_locked(queue, task);  in __rpc_queue_timer_fn()
786 	if (!list_empty(&queue->timer_list.list))  in __rpc_queue_timer_fn()
787 		rpc_set_queue_timer(queue, expires);  in __rpc_queue_timer_fn()
788 	spin_unlock(&queue->lock);  in __rpc_queue_timer_fn()
858 	struct rpc_wait_queue *queue;  in rpc_signal_task()  local
868 	queue = READ_ONCE(task->tk_waitqueue);  in rpc_signal_task()
869 	if (queue)  in rpc_signal_task()
870 		rpc_wake_up_queued_task(queue, task);  in rpc_signal_task()
875 	struct rpc_wait_queue *queue;  in rpc_task_try_cancel()  local
879 	queue = READ_ONCE(task->tk_waitqueue);  in rpc_task_try_cancel()
880 	if (queue)  in rpc_task_try_cancel()
881 		rpc_wake_up_queued_task(queue, task);  in rpc_task_try_cancel()
912 	struct rpc_wait_queue *queue;  in __rpc_execute()  local
960 		 * The queue->lock protects against races with  in __rpc_execute()
968 		queue = task->tk_waitqueue;  in __rpc_execute()
969 		spin_lock(&queue->lock);  in __rpc_execute()
971 			spin_unlock(&queue->lock);  in __rpc_execute()
976 			rpc_wake_up_task_queue_locked(queue, task);  in __rpc_execute()
977 			spin_unlock(&queue->lock);  in __rpc_execute()
981 		spin_unlock(&queue->lock);  in __rpc_execute()
994 			 * clean up after sleeping on some queue, we don't  in __rpc_execute()