46 static void			 rpc_release_task(struct rpc_task *task);
66 __rpc_disable_timer(struct rpc_wait_queue *queue, struct rpc_task *task)  in __rpc_disable_timer()  argument
68 	if (task->tk_timeout == 0)  in __rpc_disable_timer()
70 	dprintk("RPC: %5u disabling timer\n", task->tk_pid);  in __rpc_disable_timer()
71 	task->tk_timeout = 0;  in __rpc_disable_timer()
72 	list_del(&task->u.tk_wait.timer_list);  in __rpc_disable_timer()
88 __rpc_add_timer(struct rpc_wait_queue *queue, struct rpc_task *task)  in __rpc_add_timer()  argument
90 	if (!task->tk_timeout)  in __rpc_add_timer()
94 		task->tk_pid, jiffies_to_msecs(task->tk_timeout));  in __rpc_add_timer()
96 	task->u.tk_wait.expires = jiffies + task->tk_timeout;  in __rpc_add_timer()
97 …if (list_empty(&queue->timer_list.list) || time_before(task->u.tk_wait.expires, queue->timer_list.…  in __rpc_add_timer()
98 		rpc_set_queue_timer(queue, task->u.tk_wait.expires);  in __rpc_add_timer()
99 	list_add(&task->u.tk_wait.timer_list, &queue->timer_list.list);  in __rpc_add_timer()
105 	struct rpc_task *task;  in rpc_rotate_queue_owner()  local
108 		task = list_first_entry(q, struct rpc_task, u.tk_wait.list);  in rpc_rotate_queue_owner()
109 		if (task->tk_owner == queue->owner)  in rpc_rotate_queue_owner()
110 			list_move_tail(&task->u.tk_wait.list, q);  in rpc_rotate_queue_owner()
139 		struct rpc_task *task,  in __rpc_add_wait_queue_priority()  argument
145 	INIT_LIST_HEAD(&task->u.tk_wait.links);  in __rpc_add_wait_queue_priority()
152 		if (t->tk_owner == task->tk_owner) {  in __rpc_add_wait_queue_priority()
153 			list_add_tail(&task->u.tk_wait.list, &t->u.tk_wait.links);  in __rpc_add_wait_queue_priority()
157 	list_add_tail(&task->u.tk_wait.list, q);  in __rpc_add_wait_queue_priority()
169 		struct rpc_task *task,  in __rpc_add_wait_queue()  argument
172 	WARN_ON_ONCE(RPC_IS_QUEUED(task));  in __rpc_add_wait_queue()
173 	if (RPC_IS_QUEUED(task))  in __rpc_add_wait_queue()
177 		__rpc_add_wait_queue_priority(queue, task, queue_priority);  in __rpc_add_wait_queue()
178 	else if (RPC_IS_SWAPPER(task))  in __rpc_add_wait_queue()
179 		list_add(&task->u.tk_wait.list, &queue->tasks[0]);  in __rpc_add_wait_queue()
181 		list_add_tail(&task->u.tk_wait.list, &queue->tasks[0]);  in __rpc_add_wait_queue()
182 	task->tk_waitqueue = queue;  in __rpc_add_wait_queue()
186 	rpc_set_queued(task);  in __rpc_add_wait_queue()
189 			task->tk_pid, queue, rpc_qname(queue));  in __rpc_add_wait_queue()
195 static void __rpc_remove_wait_queue_priority(struct rpc_task *task)  in __rpc_remove_wait_queue_priority()  argument
199 	if (!list_empty(&task->u.tk_wait.links)) {  in __rpc_remove_wait_queue_priority()
200 		t = list_entry(task->u.tk_wait.links.next, struct rpc_task, u.tk_wait.list);  in __rpc_remove_wait_queue_priority()
201 		list_move(&t->u.tk_wait.list, &task->u.tk_wait.list);  in __rpc_remove_wait_queue_priority()
202 		list_splice_init(&task->u.tk_wait.links, &t->u.tk_wait.links);  in __rpc_remove_wait_queue_priority()
210 static void __rpc_remove_wait_queue(struct rpc_wait_queue *queue, struct rpc_task *task)  in __rpc_remove_wait_queue()  argument
212 	__rpc_disable_timer(queue, task);  in __rpc_remove_wait_queue()
214 		__rpc_remove_wait_queue_priority(task);  in __rpc_remove_wait_queue()
215 	list_del(&task->u.tk_wait.list);  in __rpc_remove_wait_queue()
218 			task->tk_pid, queue, rpc_qname(queue));  in __rpc_remove_wait_queue()
263 static void rpc_task_set_debuginfo(struct rpc_task *task)  in rpc_task_set_debuginfo()  argument
267 	task->tk_pid = atomic_inc_return(&rpc_pid);  in rpc_task_set_debuginfo()
270 static inline void rpc_task_set_debuginfo(struct rpc_task *task)  in rpc_task_set_debuginfo()  argument
275 static void rpc_set_active(struct rpc_task *task)  in rpc_set_active()  argument
277 	rpc_task_set_debuginfo(task);  in rpc_set_active()
278 	set_bit(RPC_TASK_ACTIVE, &task->tk_runstate);  in rpc_set_active()
279 	trace_rpc_task_begin(task, NULL);  in rpc_set_active()
286 static int rpc_complete_task(struct rpc_task *task)  in rpc_complete_task()  argument
288 	void *m = &task->tk_runstate;  in rpc_complete_task()
294 	trace_rpc_task_complete(task, NULL);  in rpc_complete_task()
297 	clear_bit(RPC_TASK_ACTIVE, &task->tk_runstate);  in rpc_complete_task()
298 	ret = atomic_dec_and_test(&task->tk_count);  in rpc_complete_task()
312 int __rpc_wait_for_completion_task(struct rpc_task *task, wait_bit_action_f *action)  in __rpc_wait_for_completion_task()  argument
316 	return out_of_line_wait_on_bit(&task->tk_runstate, RPC_TASK_ACTIVE,  in __rpc_wait_for_completion_task()
333 		struct rpc_task *task)  in rpc_make_runnable()  argument
335 	bool need_wakeup = !rpc_test_and_set_running(task);  in rpc_make_runnable()
337 	rpc_clear_queued(task);  in rpc_make_runnable()
340 	if (RPC_IS_ASYNC(task)) {  in rpc_make_runnable()
341 		INIT_WORK(&task->u.tk_work, rpc_async_schedule);  in rpc_make_runnable()
342 		queue_work(wq, &task->u.tk_work);  in rpc_make_runnable()
344 		wake_up_bit(&task->tk_runstate, RPC_TASK_QUEUED);  in rpc_make_runnable()
354 		struct rpc_task *task,  in __rpc_sleep_on_priority()  argument
359 			task->tk_pid, rpc_qname(q), jiffies);  in __rpc_sleep_on_priority()
361 	trace_rpc_task_sleep(task, q);  in __rpc_sleep_on_priority()
363 	__rpc_add_wait_queue(q, task, queue_priority);  in __rpc_sleep_on_priority()
365 	WARN_ON_ONCE(task->tk_callback != NULL);  in __rpc_sleep_on_priority()
366 	task->tk_callback = action;  in __rpc_sleep_on_priority()
367 	__rpc_add_timer(q, task);  in __rpc_sleep_on_priority()
370 void rpc_sleep_on(struct rpc_wait_queue *q, struct rpc_task *task,  in rpc_sleep_on()  argument
374 	WARN_ON_ONCE(!RPC_IS_ACTIVATED(task));  in rpc_sleep_on()
375 	if (!RPC_IS_ACTIVATED(task)) {  in rpc_sleep_on()
376 		task->tk_status = -EIO;  in rpc_sleep_on()
377 		rpc_put_task_async(task);  in rpc_sleep_on()
385 	__rpc_sleep_on_priority(q, task, action, task->tk_priority);  in rpc_sleep_on()
390 void rpc_sleep_on_priority(struct rpc_wait_queue *q, struct rpc_task *task,  in rpc_sleep_on_priority()  argument
394 	WARN_ON_ONCE(!RPC_IS_ACTIVATED(task));  in rpc_sleep_on_priority()
395 	if (!RPC_IS_ACTIVATED(task)) {  in rpc_sleep_on_priority()
396 		task->tk_status = -EIO;  in rpc_sleep_on_priority()
397 		rpc_put_task_async(task);  in rpc_sleep_on_priority()
405 	__rpc_sleep_on_priority(q, task, action, priority - RPC_PRIORITY_LOW);  in rpc_sleep_on_priority()
420 		struct rpc_task *task)  in __rpc_do_wake_up_task_on_wq()  argument
423 			task->tk_pid, jiffies);  in __rpc_do_wake_up_task_on_wq()
426 	if (!RPC_IS_ACTIVATED(task)) {  in __rpc_do_wake_up_task_on_wq()
427 		printk(KERN_ERR "RPC: Inactive task (%p) being woken up!\n", task);  in __rpc_do_wake_up_task_on_wq()
431 	trace_rpc_task_wakeup(task, queue);  in __rpc_do_wake_up_task_on_wq()
433 	__rpc_remove_wait_queue(queue, task);  in __rpc_do_wake_up_task_on_wq()
435 	rpc_make_runnable(wq, task);  in __rpc_do_wake_up_task_on_wq()
444 		struct rpc_wait_queue *queue, struct rpc_task *task)  in rpc_wake_up_task_on_wq_queue_locked()  argument
446 	if (RPC_IS_QUEUED(task)) {  in rpc_wake_up_task_on_wq_queue_locked()
448 		if (task->tk_waitqueue == queue)  in rpc_wake_up_task_on_wq_queue_locked()
449 			__rpc_do_wake_up_task_on_wq(wq, queue, task);  in rpc_wake_up_task_on_wq_queue_locked()
456 static void rpc_wake_up_task_queue_locked(struct rpc_wait_queue *queue, struct rpc_task *task)  in rpc_wake_up_task_queue_locked()  argument
458 	rpc_wake_up_task_on_wq_queue_locked(rpciod_workqueue, queue, task);  in rpc_wake_up_task_queue_locked()
466 		struct rpc_task *task)  in rpc_wake_up_queued_task_on_wq()  argument
469 	rpc_wake_up_task_on_wq_queue_locked(wq, queue, task);  in rpc_wake_up_queued_task_on_wq()
476 void rpc_wake_up_queued_task(struct rpc_wait_queue *queue, struct rpc_task *task)  in rpc_wake_up_queued_task()  argument
479 	rpc_wake_up_task_queue_locked(queue, task);  in rpc_wake_up_queued_task()
490 	struct rpc_task *task;  in __rpc_find_next_queued_priority()  local
497 		task = list_entry(q->next, struct rpc_task, u.tk_wait.list);  in __rpc_find_next_queued_priority()
498 		if (queue->owner == task->tk_owner) {  in __rpc_find_next_queued_priority()
501 			list_move_tail(&task->u.tk_wait.list, q);  in __rpc_find_next_queued_priority()
518 			task = list_entry(q->next, struct rpc_task, u.tk_wait.list);  in __rpc_find_next_queued_priority()
529 	rpc_set_waitqueue_owner(queue, task->tk_owner);  in __rpc_find_next_queued_priority()
531 	return task;  in __rpc_find_next_queued_priority()
550 	struct rpc_task	*task = NULL;  in rpc_wake_up_first_on_wq()  local
555 	task = __rpc_find_next_queued(queue);  in rpc_wake_up_first_on_wq()
556 	if (task != NULL) {  in rpc_wake_up_first_on_wq()
557 		if (func(task, data))  in rpc_wake_up_first_on_wq()
558 			rpc_wake_up_task_on_wq_queue_locked(wq, queue, task);  in rpc_wake_up_first_on_wq()
560 			task = NULL;  in rpc_wake_up_first_on_wq()
564 	return task;  in rpc_wake_up_first_on_wq()
577 static bool rpc_wake_up_next_func(struct rpc_task *task, void *data)  in rpc_wake_up_next_func()  argument
605 			struct rpc_task *task;  in rpc_wake_up()  local
606 			task = list_first_entry(head,  in rpc_wake_up()
609 			rpc_wake_up_task_queue_locked(queue, task);  in rpc_wake_up()
634 			struct rpc_task *task;  in rpc_wake_up_status()  local
635 			task = list_first_entry(head,  in rpc_wake_up_status()
638 			task->tk_status = status;  in rpc_wake_up_status()
639 			rpc_wake_up_task_queue_locked(queue, task);  in rpc_wake_up_status()
652 	struct rpc_task *task, *n;  in __rpc_queue_timer_fn()  local
657 	list_for_each_entry_safe(task, n, &queue->timer_list.list, u.tk_wait.timer_list) {  in __rpc_queue_timer_fn()
658 		timeo = task->u.tk_wait.expires;  in __rpc_queue_timer_fn()
660 			dprintk("RPC: %5u timeout\n", task->tk_pid);  in __rpc_queue_timer_fn()
661 			task->tk_status = -ETIMEDOUT;  in __rpc_queue_timer_fn()
662 			rpc_wake_up_task_queue_locked(queue, task);  in __rpc_queue_timer_fn()
673 static void __rpc_atrun(struct rpc_task *task)  in __rpc_atrun()  argument
675 	if (task->tk_status == -ETIMEDOUT)  in __rpc_atrun()
676 		task->tk_status = 0;  in __rpc_atrun()
682 void rpc_delay(struct rpc_task *task, unsigned long delay)  in rpc_delay()  argument
684 	task->tk_timeout = delay;  in rpc_delay()
685 	rpc_sleep_on(&delay_queue, task, __rpc_atrun);  in rpc_delay()
692 void rpc_prepare_task(struct rpc_task *task)  in rpc_prepare_task()  argument
694 	task->tk_ops->rpc_call_prepare(task, task->tk_calldata);  in rpc_prepare_task()
698 rpc_init_task_statistics(struct rpc_task *task)  in rpc_init_task_statistics()  argument
701 	task->tk_garb_retry = 2;  in rpc_init_task_statistics()
702 	task->tk_cred_retry = 2;  in rpc_init_task_statistics()
703 	task->tk_rebind_retry = 2;  in rpc_init_task_statistics()
706 	task->tk_start = ktime_get();  in rpc_init_task_statistics()
710 rpc_reset_task_statistics(struct rpc_task *task)  in rpc_reset_task_statistics()  argument
712 	task->tk_timeouts = 0;  in rpc_reset_task_statistics()
713 	task->tk_flags &= ~(RPC_CALL_MAJORSEEN|RPC_TASK_KILLED|RPC_TASK_SENT);  in rpc_reset_task_statistics()
715 	rpc_init_task_statistics(task);  in rpc_reset_task_statistics()
721 void rpc_exit_task(struct rpc_task *task)  in rpc_exit_task()  argument
723 	task->tk_action = NULL;  in rpc_exit_task()
724 	if (task->tk_ops->rpc_call_done != NULL) {  in rpc_exit_task()
725 		task->tk_ops->rpc_call_done(task, task->tk_calldata);  in rpc_exit_task()
726 		if (task->tk_action != NULL) {  in rpc_exit_task()
727 			WARN_ON(RPC_ASSASSINATED(task));  in rpc_exit_task()
729 			xprt_release(task);  in rpc_exit_task()
730 			rpc_reset_task_statistics(task);  in rpc_exit_task()
735 void rpc_exit(struct rpc_task *task, int status)  in rpc_exit()  argument
737 	task->tk_status = status;  in rpc_exit()
738 	task->tk_action = rpc_exit_task;  in rpc_exit()
739 	if (RPC_IS_QUEUED(task))  in rpc_exit()
740 		rpc_wake_up_queued_task(task->tk_waitqueue, task);  in rpc_exit()
753 static void __rpc_execute(struct rpc_task *task)  in __rpc_execute()  argument
756 	int task_is_async = RPC_IS_ASYNC(task);  in __rpc_execute()
760 			task->tk_pid, task->tk_flags);  in __rpc_execute()
762 	WARN_ON_ONCE(RPC_IS_QUEUED(task));  in __rpc_execute()
763 	if (RPC_IS_QUEUED(task))  in __rpc_execute()
776 		do_action = task->tk_action;  in __rpc_execute()
777 		if (task->tk_callback) {  in __rpc_execute()
778 			do_action = task->tk_callback;  in __rpc_execute()
779 			task->tk_callback = NULL;  in __rpc_execute()
783 		trace_rpc_task_run_action(task, do_action);  in __rpc_execute()
784 		do_action(task);  in __rpc_execute()
789 		if (!RPC_IS_QUEUED(task))  in __rpc_execute()
800 		queue = task->tk_waitqueue;  in __rpc_execute()
802 		if (!RPC_IS_QUEUED(task)) {  in __rpc_execute()
806 		rpc_clear_running(task);  in __rpc_execute()
812 		dprintk("RPC: %5u sync task going to sleep\n", task->tk_pid);  in __rpc_execute()
813 		status = out_of_line_wait_on_bit(&task->tk_runstate,  in __rpc_execute()
823 			dprintk("RPC: %5u got signal\n", task->tk_pid);  in __rpc_execute()
824 			task->tk_flags |= RPC_TASK_KILLED;  in __rpc_execute()
825 			rpc_exit(task, -ERESTARTSYS);  in __rpc_execute()
827 		dprintk("RPC: %5u sync task resuming\n", task->tk_pid);  in __rpc_execute()
830 	dprintk("RPC: %5u return %d, status %d\n", task->tk_pid, status,  in __rpc_execute()
831 			task->tk_status);  in __rpc_execute()
833 	rpc_release_task(task);  in __rpc_execute()
845 void rpc_execute(struct rpc_task *task)  in rpc_execute()  argument
847 	bool is_async = RPC_IS_ASYNC(task);  in rpc_execute()
849 	rpc_set_active(task);  in rpc_execute()
850 	rpc_make_runnable(rpciod_workqueue, task);  in rpc_execute()
852 		__rpc_execute(task);  in rpc_execute()
880 int rpc_malloc(struct rpc_task *task)  in rpc_malloc()  argument
882 	struct rpc_rqst *rqst = task->tk_rqstp;  in rpc_malloc()
887 	if (RPC_IS_SWAPPER(task))  in rpc_malloc()
901 			task->tk_pid, size, buf);  in rpc_malloc()
913 void rpc_free(struct rpc_task *task)  in rpc_free()  argument
915 	void *buffer = task->tk_rqstp->rq_buffer;  in rpc_free()
935 static void rpc_init_task(struct rpc_task *task, const struct rpc_task_setup *task_setup_data)  in rpc_init_task()  argument
937 	memset(task, 0, sizeof(*task));  in rpc_init_task()
938 	atomic_set(&task->tk_count, 1);  in rpc_init_task()
939 	task->tk_flags  = task_setup_data->flags;  in rpc_init_task()
940 	task->tk_ops = task_setup_data->callback_ops;  in rpc_init_task()
941 	task->tk_calldata = task_setup_data->callback_data;  in rpc_init_task()
942 	INIT_LIST_HEAD(&task->tk_task);  in rpc_init_task()
944 	task->tk_priority = task_setup_data->priority - RPC_PRIORITY_LOW;  in rpc_init_task()
945 	task->tk_owner = current->tgid;  in rpc_init_task()
948 	task->tk_workqueue = task_setup_data->workqueue;  in rpc_init_task()
950 	task->tk_xprt = xprt_get(task_setup_data->rpc_xprt);  in rpc_init_task()
952 	if (task->tk_ops->rpc_call_prepare != NULL)  in rpc_init_task()
953 		task->tk_action = rpc_prepare_task;  in rpc_init_task()
955 	rpc_init_task_statistics(task);  in rpc_init_task()
972 	struct rpc_task	*task = setup_data->task;  in rpc_new_task()  local
975 	if (task == NULL) {  in rpc_new_task()
976 		task = rpc_alloc_task();  in rpc_new_task()
980 	rpc_init_task(task, setup_data);  in rpc_new_task()
981 	task->tk_flags |= flags;  in rpc_new_task()
982 	dprintk("RPC:       allocated task %p\n", task);  in rpc_new_task()
983 	return task;  in rpc_new_task()
1005 static void rpc_free_task(struct rpc_task *task)  in rpc_free_task()  argument
1007 	unsigned short tk_flags = task->tk_flags;  in rpc_free_task()
1009 	rpc_release_calldata(task->tk_ops, task->tk_calldata);  in rpc_free_task()
1012 		dprintk("RPC: %5u freeing task\n", task->tk_pid);  in rpc_free_task()
1013 		mempool_free(task, rpc_task_mempool);  in rpc_free_task()
1022 static void rpc_release_resources_task(struct rpc_task *task)  in rpc_release_resources_task()  argument
1024 	xprt_release(task);  in rpc_release_resources_task()
1025 	if (task->tk_msg.rpc_cred) {  in rpc_release_resources_task()
1026 		put_rpccred(task->tk_msg.rpc_cred);  in rpc_release_resources_task()
1027 		task->tk_msg.rpc_cred = NULL;  in rpc_release_resources_task()
1029 	rpc_task_release_client(task);  in rpc_release_resources_task()
1032 static void rpc_final_put_task(struct rpc_task *task,  in rpc_final_put_task()  argument
1036 		INIT_WORK(&task->u.tk_work, rpc_async_release);  in rpc_final_put_task()
1037 		queue_work(q, &task->u.tk_work);  in rpc_final_put_task()
1039 		rpc_free_task(task);  in rpc_final_put_task()
1042 static void rpc_do_put_task(struct rpc_task *task, struct workqueue_struct *q)  in rpc_do_put_task()  argument
1044 	if (atomic_dec_and_test(&task->tk_count)) {  in rpc_do_put_task()
1045 		rpc_release_resources_task(task);  in rpc_do_put_task()
1046 		rpc_final_put_task(task, q);  in rpc_do_put_task()
1050 void rpc_put_task(struct rpc_task *task)  in rpc_put_task()  argument
1052 	rpc_do_put_task(task, NULL);  in rpc_put_task()
1056 void rpc_put_task_async(struct rpc_task *task)  in rpc_put_task_async()  argument
1058 	rpc_do_put_task(task, task->tk_workqueue);  in rpc_put_task_async()
1062 static void rpc_release_task(struct rpc_task *task)  in rpc_release_task()  argument
1064 	dprintk("RPC: %5u release task\n", task->tk_pid);  in rpc_release_task()
1066 	WARN_ON_ONCE(RPC_IS_QUEUED(task));  in rpc_release_task()
1068 	rpc_release_resources_task(task);  in rpc_release_task()
1075 	if (atomic_read(&task->tk_count) != 1 + !RPC_IS_ASYNC(task)) {  in rpc_release_task()
1077 		if (!rpc_complete_task(task))  in rpc_release_task()
1080 		if (!atomic_dec_and_test(&task->tk_count))  in rpc_release_task()
1083 	rpc_final_put_task(task, task->tk_workqueue);  in rpc_release_task()