66 __rpc_disable_timer(struct rpc_wait_queue *queue, struct rpc_task *task)  in __rpc_disable_timer()  argument
73 	if (list_empty(&queue->timer_list.list))  in __rpc_disable_timer()
74 		del_timer(&queue->timer_list.timer);  in __rpc_disable_timer()
78 rpc_set_queue_timer(struct rpc_wait_queue *queue, unsigned long expires)  in rpc_set_queue_timer()  argument
80 	queue->timer_list.expires = expires;  in rpc_set_queue_timer()
81 	mod_timer(&queue->timer_list.timer, expires);  in rpc_set_queue_timer()
88 __rpc_add_timer(struct rpc_wait_queue *queue, struct rpc_task *task)  in __rpc_add_timer()  argument
97 …if (list_empty(&queue->timer_list.list) || time_before(task->u.tk_wait.expires, queue->timer_list.…  in __rpc_add_timer()
98 		rpc_set_queue_timer(queue, task->u.tk_wait.expires);  in __rpc_add_timer()
99 	list_add(&task->u.tk_wait.timer_list, &queue->timer_list.list);  in __rpc_add_timer()
102 static void rpc_rotate_queue_owner(struct rpc_wait_queue *queue)  in rpc_rotate_queue_owner()  argument
104 	struct list_head *q = &queue->tasks[queue->priority];  in rpc_rotate_queue_owner()
109 		if (task->tk_owner == queue->owner)  in rpc_rotate_queue_owner()
114 static void rpc_set_waitqueue_priority(struct rpc_wait_queue *queue, int priority)  in rpc_set_waitqueue_priority()  argument
116 	if (queue->priority != priority) {  in rpc_set_waitqueue_priority()
118 		rpc_rotate_queue_owner(queue);  in rpc_set_waitqueue_priority()
119 		queue->priority = priority;  in rpc_set_waitqueue_priority()
123 static void rpc_set_waitqueue_owner(struct rpc_wait_queue *queue, pid_t pid)  in rpc_set_waitqueue_owner()  argument
125 	queue->owner = pid;  in rpc_set_waitqueue_owner()
126 	queue->nr = RPC_BATCH_COUNT;  in rpc_set_waitqueue_owner()
129 static void rpc_reset_waitqueue_priority(struct rpc_wait_queue *queue)  in rpc_reset_waitqueue_priority()  argument
131 	rpc_set_waitqueue_priority(queue, queue->maxpriority);  in rpc_reset_waitqueue_priority()
132 	rpc_set_waitqueue_owner(queue, 0);  in rpc_reset_waitqueue_priority()
138 static void __rpc_add_wait_queue_priority(struct rpc_wait_queue *queue,  in __rpc_add_wait_queue_priority()  argument
146 	if (unlikely(queue_priority > queue->maxpriority))  in __rpc_add_wait_queue_priority()
147 		queue_priority = queue->maxpriority;  in __rpc_add_wait_queue_priority()
148 	if (queue_priority > queue->priority)  in __rpc_add_wait_queue_priority()
149 		rpc_set_waitqueue_priority(queue, queue_priority);  in __rpc_add_wait_queue_priority()
150 	q = &queue->tasks[queue_priority];  in __rpc_add_wait_queue_priority()
168 static void __rpc_add_wait_queue(struct rpc_wait_queue *queue,  in __rpc_add_wait_queue()  argument
176 	if (RPC_IS_PRIORITY(queue))  in __rpc_add_wait_queue()
177 		__rpc_add_wait_queue_priority(queue, task, queue_priority);  in __rpc_add_wait_queue()
179 		list_add(&task->u.tk_wait.list, &queue->tasks[0]);  in __rpc_add_wait_queue()
181 		list_add_tail(&task->u.tk_wait.list, &queue->tasks[0]);  in __rpc_add_wait_queue()
182 	task->tk_waitqueue = queue;  in __rpc_add_wait_queue()
183 	queue->qlen++;  in __rpc_add_wait_queue()
189 			task->tk_pid, queue, rpc_qname(queue));  in __rpc_add_wait_queue()
210 static void __rpc_remove_wait_queue(struct rpc_wait_queue *queue, struct rpc_task *task)  in __rpc_remove_wait_queue()  argument
212 	__rpc_disable_timer(queue, task);  in __rpc_remove_wait_queue()
213 	if (RPC_IS_PRIORITY(queue))  in __rpc_remove_wait_queue()
216 	queue->qlen--;  in __rpc_remove_wait_queue()
218 			task->tk_pid, queue, rpc_qname(queue));  in __rpc_remove_wait_queue()
221 static void __rpc_init_priority_wait_queue(struct rpc_wait_queue *queue, const char *qname, unsigne…  in __rpc_init_priority_wait_queue()  argument
225 	spin_lock_init(&queue->lock);  in __rpc_init_priority_wait_queue()
226 	for (i = 0; i < ARRAY_SIZE(queue->tasks); i++)  in __rpc_init_priority_wait_queue()
227 		INIT_LIST_HEAD(&queue->tasks[i]);  in __rpc_init_priority_wait_queue()
228 	queue->maxpriority = nr_queues - 1;  in __rpc_init_priority_wait_queue()
229 	rpc_reset_waitqueue_priority(queue);  in __rpc_init_priority_wait_queue()
230 	queue->qlen = 0;  in __rpc_init_priority_wait_queue()
231 	timer_setup(&queue->timer_list.timer, __rpc_queue_timer_fn, 0);  in __rpc_init_priority_wait_queue()
232 	INIT_LIST_HEAD(&queue->timer_list.list);  in __rpc_init_priority_wait_queue()
233 	rpc_assign_waitqueue_name(queue, qname);  in __rpc_init_priority_wait_queue()
236 void rpc_init_priority_wait_queue(struct rpc_wait_queue *queue, const char *qname)  in rpc_init_priority_wait_queue()  argument
238 	__rpc_init_priority_wait_queue(queue, qname, RPC_NR_PRIORITY);  in rpc_init_priority_wait_queue()
242 void rpc_init_wait_queue(struct rpc_wait_queue *queue, const char *qname)  in rpc_init_wait_queue()  argument
244 	__rpc_init_priority_wait_queue(queue, qname, 1);  in rpc_init_wait_queue()
248 void rpc_destroy_wait_queue(struct rpc_wait_queue *queue)  in rpc_destroy_wait_queue()  argument
250 	del_timer_sync(&queue->timer_list.timer);  in rpc_destroy_wait_queue()
419 		struct rpc_wait_queue *queue,  in __rpc_do_wake_up_task_on_wq()  argument
431 	trace_rpc_task_wakeup(task, queue);  in __rpc_do_wake_up_task_on_wq()
433 	__rpc_remove_wait_queue(queue, task);  in __rpc_do_wake_up_task_on_wq()
444 		struct rpc_wait_queue *queue, struct rpc_task *task)  in rpc_wake_up_task_on_wq_queue_locked()  argument
448 		if (task->tk_waitqueue == queue)  in rpc_wake_up_task_on_wq_queue_locked()
449 			__rpc_do_wake_up_task_on_wq(wq, queue, task);  in rpc_wake_up_task_on_wq_queue_locked()
456 static void rpc_wake_up_task_queue_locked(struct rpc_wait_queue *queue, struct rpc_task *task)  in rpc_wake_up_task_queue_locked()  argument
458 	rpc_wake_up_task_on_wq_queue_locked(rpciod_workqueue, queue, task);  in rpc_wake_up_task_queue_locked()
465 		struct rpc_wait_queue *queue,  in rpc_wake_up_queued_task_on_wq()  argument
468 	spin_lock_bh(&queue->lock);  in rpc_wake_up_queued_task_on_wq()
469 	rpc_wake_up_task_on_wq_queue_locked(wq, queue, task);  in rpc_wake_up_queued_task_on_wq()
470 	spin_unlock_bh(&queue->lock);  in rpc_wake_up_queued_task_on_wq()
476 void rpc_wake_up_queued_task(struct rpc_wait_queue *queue, struct rpc_task *task)  in rpc_wake_up_queued_task()  argument
478 	spin_lock_bh(&queue->lock);  in rpc_wake_up_queued_task()
479 	rpc_wake_up_task_queue_locked(queue, task);  in rpc_wake_up_queued_task()
480 	spin_unlock_bh(&queue->lock);  in rpc_wake_up_queued_task()
487 static struct rpc_task *__rpc_find_next_queued_priority(struct rpc_wait_queue *queue)  in __rpc_find_next_queued_priority()  argument
495 	q = &queue->tasks[queue->priority];  in __rpc_find_next_queued_priority()
498 		if (queue->owner == task->tk_owner) {  in __rpc_find_next_queued_priority()
499 			if (--queue->nr)  in __rpc_find_next_queued_priority()
513 		if (q == &queue->tasks[0])  in __rpc_find_next_queued_priority()
514 			q = &queue->tasks[queue->maxpriority];  in __rpc_find_next_queued_priority()
521 	} while (q != &queue->tasks[queue->priority]);  in __rpc_find_next_queued_priority()
523 	rpc_reset_waitqueue_priority(queue);  in __rpc_find_next_queued_priority()
527 	rpc_set_waitqueue_priority(queue, (unsigned int)(q - &queue->tasks[0]));  in __rpc_find_next_queued_priority()
529 	rpc_set_waitqueue_owner(queue, task->tk_owner);  in __rpc_find_next_queued_priority()
534 static struct rpc_task *__rpc_find_next_queued(struct rpc_wait_queue *queue)  in __rpc_find_next_queued()  argument
536 	if (RPC_IS_PRIORITY(queue))  in __rpc_find_next_queued()
537 		return __rpc_find_next_queued_priority(queue);  in __rpc_find_next_queued()
538 	if (!list_empty(&queue->tasks[0]))  in __rpc_find_next_queued()
539 		return list_first_entry(&queue->tasks[0], struct rpc_task, u.tk_wait.list);  in __rpc_find_next_queued()
547 		struct rpc_wait_queue *queue,  in rpc_wake_up_first_on_wq()  argument
553 			queue, rpc_qname(queue));  in rpc_wake_up_first_on_wq()
554 	spin_lock_bh(&queue->lock);  in rpc_wake_up_first_on_wq()
555 	task = __rpc_find_next_queued(queue);  in rpc_wake_up_first_on_wq()
558 			rpc_wake_up_task_on_wq_queue_locked(wq, queue, task);  in rpc_wake_up_first_on_wq()
562 	spin_unlock_bh(&queue->lock);  in rpc_wake_up_first_on_wq()
570 struct rpc_task *rpc_wake_up_first(struct rpc_wait_queue *queue,  in rpc_wake_up_first()  argument
573 	return rpc_wake_up_first_on_wq(rpciod_workqueue, queue, func, data);  in rpc_wake_up_first()
585 struct rpc_task *rpc_wake_up_next(struct rpc_wait_queue *queue)  in rpc_wake_up_next()  argument
587 	return rpc_wake_up_first(queue, rpc_wake_up_next_func, NULL);  in rpc_wake_up_next()
597 void rpc_wake_up(struct rpc_wait_queue *queue)  in rpc_wake_up()  argument
601 	spin_lock_bh(&queue->lock);  in rpc_wake_up()
602 	head = &queue->tasks[queue->maxpriority];  in rpc_wake_up()
609 			rpc_wake_up_task_queue_locked(queue, task);  in rpc_wake_up()
611 		if (head == &queue->tasks[0])  in rpc_wake_up()
615 	spin_unlock_bh(&queue->lock);  in rpc_wake_up()
626 void rpc_wake_up_status(struct rpc_wait_queue *queue, int status)  in rpc_wake_up_status()  argument
630 	spin_lock_bh(&queue->lock);  in rpc_wake_up_status()
631 	head = &queue->tasks[queue->maxpriority];  in rpc_wake_up_status()
639 			rpc_wake_up_task_queue_locked(queue, task);  in rpc_wake_up_status()
641 		if (head == &queue->tasks[0])  in rpc_wake_up_status()
645 	spin_unlock_bh(&queue->lock);  in rpc_wake_up_status()
651 	struct rpc_wait_queue *queue = from_timer(queue, t, timer_list.timer);  in __rpc_queue_timer_fn()  local
655 	spin_lock(&queue->lock);  in __rpc_queue_timer_fn()
657 	list_for_each_entry_safe(task, n, &queue->timer_list.list, u.tk_wait.timer_list) {  in __rpc_queue_timer_fn()
662 			rpc_wake_up_task_queue_locked(queue, task);  in __rpc_queue_timer_fn()
668 	if (!list_empty(&queue->timer_list.list))  in __rpc_queue_timer_fn()
669 		rpc_set_queue_timer(queue, expires);  in __rpc_queue_timer_fn()
670 	spin_unlock(&queue->lock);  in __rpc_queue_timer_fn()
755 	struct rpc_wait_queue *queue;  in __rpc_execute()  local
800 		queue = task->tk_waitqueue;  in __rpc_execute()
801 		spin_lock_bh(&queue->lock);  in __rpc_execute()
803 			spin_unlock_bh(&queue->lock);  in __rpc_execute()
807 		spin_unlock_bh(&queue->lock);  in __rpc_execute()