Lines Matching refs:task
49 static void rpc_release_task(struct rpc_task *task);
65 rpc_task_timeout(const struct rpc_task *task) in rpc_task_timeout() argument
67 unsigned long timeout = READ_ONCE(task->tk_timeout); in rpc_task_timeout()
84 __rpc_disable_timer(struct rpc_wait_queue *queue, struct rpc_task *task) in __rpc_disable_timer() argument
86 if (list_empty(&task->u.tk_wait.timer_list)) in __rpc_disable_timer()
88 dprintk("RPC: %5u disabling timer\n", task->tk_pid); in __rpc_disable_timer()
89 task->tk_timeout = 0; in __rpc_disable_timer()
90 list_del(&task->u.tk_wait.timer_list); in __rpc_disable_timer()
111 __rpc_add_timer(struct rpc_wait_queue *queue, struct rpc_task *task, in __rpc_add_timer() argument
115 task->tk_pid, jiffies_to_msecs(timeout - jiffies)); in __rpc_add_timer()
117 task->tk_timeout = timeout; in __rpc_add_timer()
120 list_add(&task->u.tk_wait.timer_list, &queue->timer_list.list); in __rpc_add_timer()
140 __rpc_list_enqueue_task(struct list_head *q, struct rpc_task *task) in __rpc_list_enqueue_task() argument
145 if (t->tk_owner == task->tk_owner) { in __rpc_list_enqueue_task()
146 list_add_tail(&task->u.tk_wait.links, in __rpc_list_enqueue_task()
149 task->u.tk_wait.list.next = q; in __rpc_list_enqueue_task()
150 task->u.tk_wait.list.prev = NULL; in __rpc_list_enqueue_task()
154 INIT_LIST_HEAD(&task->u.tk_wait.links); in __rpc_list_enqueue_task()
155 list_add_tail(&task->u.tk_wait.list, q); in __rpc_list_enqueue_task()
162 __rpc_list_dequeue_task(struct rpc_task *task) in __rpc_list_dequeue_task() argument
167 if (task->u.tk_wait.list.prev == NULL) { in __rpc_list_dequeue_task()
168 list_del(&task->u.tk_wait.links); in __rpc_list_dequeue_task()
171 if (!list_empty(&task->u.tk_wait.links)) { in __rpc_list_dequeue_task()
172 t = list_first_entry(&task->u.tk_wait.links, in __rpc_list_dequeue_task()
178 list_del(&task->u.tk_wait.links); in __rpc_list_dequeue_task()
180 list_del(&task->u.tk_wait.list); in __rpc_list_dequeue_task()
187 struct rpc_task *task, in __rpc_add_wait_queue_priority() argument
192 __rpc_list_enqueue_task(&queue->tasks[queue_priority], task); in __rpc_add_wait_queue_priority()
204 struct rpc_task *task, in __rpc_add_wait_queue() argument
207 WARN_ON_ONCE(RPC_IS_QUEUED(task)); in __rpc_add_wait_queue()
208 if (RPC_IS_QUEUED(task)) in __rpc_add_wait_queue()
211 INIT_LIST_HEAD(&task->u.tk_wait.timer_list); in __rpc_add_wait_queue()
213 __rpc_add_wait_queue_priority(queue, task, queue_priority); in __rpc_add_wait_queue()
214 else if (RPC_IS_SWAPPER(task)) in __rpc_add_wait_queue()
215 list_add(&task->u.tk_wait.list, &queue->tasks[0]); in __rpc_add_wait_queue()
217 list_add_tail(&task->u.tk_wait.list, &queue->tasks[0]); in __rpc_add_wait_queue()
218 task->tk_waitqueue = queue; in __rpc_add_wait_queue()
222 rpc_set_queued(task); in __rpc_add_wait_queue()
225 task->tk_pid, queue, rpc_qname(queue)); in __rpc_add_wait_queue()
231 static void __rpc_remove_wait_queue_priority(struct rpc_task *task) in __rpc_remove_wait_queue_priority() argument
233 __rpc_list_dequeue_task(task); in __rpc_remove_wait_queue_priority()
240 static void __rpc_remove_wait_queue(struct rpc_wait_queue *queue, struct rpc_task *task) in __rpc_remove_wait_queue() argument
242 __rpc_disable_timer(queue, task); in __rpc_remove_wait_queue()
244 __rpc_remove_wait_queue_priority(task); in __rpc_remove_wait_queue()
246 list_del(&task->u.tk_wait.list); in __rpc_remove_wait_queue()
249 task->tk_pid, queue, rpc_qname(queue)); in __rpc_remove_wait_queue()
295 static void rpc_task_set_debuginfo(struct rpc_task *task) in rpc_task_set_debuginfo() argument
299 task->tk_pid = atomic_inc_return(&rpc_pid); in rpc_task_set_debuginfo()
302 static inline void rpc_task_set_debuginfo(struct rpc_task *task) in rpc_task_set_debuginfo() argument
307 static void rpc_set_active(struct rpc_task *task) in rpc_set_active() argument
309 rpc_task_set_debuginfo(task); in rpc_set_active()
310 set_bit(RPC_TASK_ACTIVE, &task->tk_runstate); in rpc_set_active()
311 trace_rpc_task_begin(task, NULL); in rpc_set_active()
318 static int rpc_complete_task(struct rpc_task *task) in rpc_complete_task() argument
320 void *m = &task->tk_runstate; in rpc_complete_task()
326 trace_rpc_task_complete(task, NULL); in rpc_complete_task()
329 clear_bit(RPC_TASK_ACTIVE, &task->tk_runstate); in rpc_complete_task()
330 ret = atomic_dec_and_test(&task->tk_count); in rpc_complete_task()
344 int __rpc_wait_for_completion_task(struct rpc_task *task, wait_bit_action_f *action) in __rpc_wait_for_completion_task() argument
348 return out_of_line_wait_on_bit(&task->tk_runstate, RPC_TASK_ACTIVE, in __rpc_wait_for_completion_task()
365 struct rpc_task *task) in rpc_make_runnable() argument
367 bool need_wakeup = !rpc_test_and_set_running(task); in rpc_make_runnable()
369 rpc_clear_queued(task); in rpc_make_runnable()
372 if (RPC_IS_ASYNC(task)) { in rpc_make_runnable()
373 INIT_WORK(&task->u.tk_work, rpc_async_schedule); in rpc_make_runnable()
374 queue_work(wq, &task->u.tk_work); in rpc_make_runnable()
376 wake_up_bit(&task->tk_runstate, RPC_TASK_QUEUED); in rpc_make_runnable()
386 struct rpc_task *task, in __rpc_sleep_on_priority() argument
390 task->tk_pid, rpc_qname(q), jiffies); in __rpc_sleep_on_priority()
392 trace_rpc_task_sleep(task, q); in __rpc_sleep_on_priority()
394 __rpc_add_wait_queue(q, task, queue_priority); in __rpc_sleep_on_priority()
399 struct rpc_task *task, unsigned long timeout, in __rpc_sleep_on_priority_timeout() argument
403 __rpc_sleep_on_priority(q, task, queue_priority); in __rpc_sleep_on_priority_timeout()
404 __rpc_add_timer(q, task, timeout); in __rpc_sleep_on_priority_timeout()
406 task->tk_status = -ETIMEDOUT; in __rpc_sleep_on_priority_timeout()
409 static void rpc_set_tk_callback(struct rpc_task *task, rpc_action action) in rpc_set_tk_callback() argument
411 if (action && !WARN_ON_ONCE(task->tk_callback != NULL)) in rpc_set_tk_callback()
412 task->tk_callback = action; in rpc_set_tk_callback()
415 static bool rpc_sleep_check_activated(struct rpc_task *task) in rpc_sleep_check_activated() argument
418 if (WARN_ON_ONCE(!RPC_IS_ACTIVATED(task))) { in rpc_sleep_check_activated()
419 task->tk_status = -EIO; in rpc_sleep_check_activated()
420 rpc_put_task_async(task); in rpc_sleep_check_activated()
426 void rpc_sleep_on_timeout(struct rpc_wait_queue *q, struct rpc_task *task, in rpc_sleep_on_timeout() argument
429 if (!rpc_sleep_check_activated(task)) in rpc_sleep_on_timeout()
432 rpc_set_tk_callback(task, action); in rpc_sleep_on_timeout()
438 __rpc_sleep_on_priority_timeout(q, task, timeout, task->tk_priority); in rpc_sleep_on_timeout()
443 void rpc_sleep_on(struct rpc_wait_queue *q, struct rpc_task *task, in rpc_sleep_on() argument
446 if (!rpc_sleep_check_activated(task)) in rpc_sleep_on()
449 rpc_set_tk_callback(task, action); in rpc_sleep_on()
451 WARN_ON_ONCE(task->tk_timeout != 0); in rpc_sleep_on()
456 __rpc_sleep_on_priority(q, task, task->tk_priority); in rpc_sleep_on()
462 struct rpc_task *task, unsigned long timeout, int priority) in rpc_sleep_on_priority_timeout() argument
464 if (!rpc_sleep_check_activated(task)) in rpc_sleep_on_priority_timeout()
472 __rpc_sleep_on_priority_timeout(q, task, timeout, priority); in rpc_sleep_on_priority_timeout()
477 void rpc_sleep_on_priority(struct rpc_wait_queue *q, struct rpc_task *task, in rpc_sleep_on_priority() argument
480 if (!rpc_sleep_check_activated(task)) in rpc_sleep_on_priority()
483 WARN_ON_ONCE(task->tk_timeout != 0); in rpc_sleep_on_priority()
489 __rpc_sleep_on_priority(q, task, priority); in rpc_sleep_on_priority()
504 struct rpc_task *task) in __rpc_do_wake_up_task_on_wq() argument
507 task->tk_pid, jiffies); in __rpc_do_wake_up_task_on_wq()
510 if (!RPC_IS_ACTIVATED(task)) { in __rpc_do_wake_up_task_on_wq()
511 printk(KERN_ERR "RPC: Inactive task (%p) being woken up!\n", task); in __rpc_do_wake_up_task_on_wq()
515 trace_rpc_task_wakeup(task, queue); in __rpc_do_wake_up_task_on_wq()
517 __rpc_remove_wait_queue(queue, task); in __rpc_do_wake_up_task_on_wq()
519 rpc_make_runnable(wq, task); in __rpc_do_wake_up_task_on_wq()
529 struct rpc_wait_queue *queue, struct rpc_task *task, in rpc_wake_up_task_on_wq_queue_action_locked() argument
532 if (RPC_IS_QUEUED(task)) { in rpc_wake_up_task_on_wq_queue_action_locked()
534 if (task->tk_waitqueue == queue) { in rpc_wake_up_task_on_wq_queue_action_locked()
535 if (action == NULL || action(task, data)) { in rpc_wake_up_task_on_wq_queue_action_locked()
536 __rpc_do_wake_up_task_on_wq(wq, queue, task); in rpc_wake_up_task_on_wq_queue_action_locked()
537 return task; in rpc_wake_up_task_on_wq_queue_action_locked()
548 struct rpc_task *task) in rpc_wake_up_task_queue_locked() argument
551 task, NULL, NULL); in rpc_wake_up_task_queue_locked()
557 void rpc_wake_up_queued_task(struct rpc_wait_queue *queue, struct rpc_task *task) in rpc_wake_up_queued_task() argument
559 if (!RPC_IS_QUEUED(task)) in rpc_wake_up_queued_task()
562 rpc_wake_up_task_queue_locked(queue, task); in rpc_wake_up_queued_task()
567 static bool rpc_task_action_set_status(struct rpc_task *task, void *status) in rpc_task_action_set_status() argument
569 task->tk_status = *(int *)status; in rpc_task_action_set_status()
575 struct rpc_task *task, int status) in rpc_wake_up_task_queue_set_status_locked() argument
578 task, rpc_task_action_set_status, &status); in rpc_wake_up_task_queue_set_status_locked()
592 struct rpc_task *task, int status) in rpc_wake_up_queued_task_set_status() argument
594 if (!RPC_IS_QUEUED(task)) in rpc_wake_up_queued_task_set_status()
597 rpc_wake_up_task_queue_set_status_locked(queue, task, status); in rpc_wake_up_queued_task_set_status()
607 struct rpc_task *task; in __rpc_find_next_queued_priority() local
614 task = list_first_entry(q, struct rpc_task, u.tk_wait.list); in __rpc_find_next_queued_priority()
627 task = list_first_entry(q, struct rpc_task, u.tk_wait.list); in __rpc_find_next_queued_priority()
638 return task; in __rpc_find_next_queued_priority()
657 struct rpc_task *task = NULL; in rpc_wake_up_first_on_wq() local
662 task = __rpc_find_next_queued(queue); in rpc_wake_up_first_on_wq()
663 if (task != NULL) in rpc_wake_up_first_on_wq()
664 task = rpc_wake_up_task_on_wq_queue_action_locked(wq, queue, in rpc_wake_up_first_on_wq()
665 task, func, data); in rpc_wake_up_first_on_wq()
668 return task; in rpc_wake_up_first_on_wq()
681 static bool rpc_wake_up_next_func(struct rpc_task *task, void *data) in rpc_wake_up_next_func() argument
709 struct rpc_task *task; in rpc_wake_up() local
710 task = list_first_entry(head, in rpc_wake_up()
713 rpc_wake_up_task_queue_locked(queue, task); in rpc_wake_up()
738 struct rpc_task *task; in rpc_wake_up_status() local
739 task = list_first_entry(head, in rpc_wake_up_status()
742 task->tk_status = status; in rpc_wake_up_status()
743 rpc_wake_up_task_queue_locked(queue, task); in rpc_wake_up_status()
758 struct rpc_task *task, *n; in __rpc_queue_timer_fn() local
763 list_for_each_entry_safe(task, n, &queue->timer_list.list, u.tk_wait.timer_list) { in __rpc_queue_timer_fn()
764 timeo = task->tk_timeout; in __rpc_queue_timer_fn()
766 dprintk("RPC: %5u timeout\n", task->tk_pid); in __rpc_queue_timer_fn()
767 task->tk_status = -ETIMEDOUT; in __rpc_queue_timer_fn()
768 rpc_wake_up_task_queue_locked(queue, task); in __rpc_queue_timer_fn()
779 static void __rpc_atrun(struct rpc_task *task) in __rpc_atrun() argument
781 if (task->tk_status == -ETIMEDOUT) in __rpc_atrun()
782 task->tk_status = 0; in __rpc_atrun()
788 void rpc_delay(struct rpc_task *task, unsigned long delay) in rpc_delay() argument
790 rpc_sleep_on_timeout(&delay_queue, task, __rpc_atrun, jiffies + delay); in rpc_delay()
797 void rpc_prepare_task(struct rpc_task *task) in rpc_prepare_task() argument
799 task->tk_ops->rpc_call_prepare(task, task->tk_calldata); in rpc_prepare_task()
803 rpc_init_task_statistics(struct rpc_task *task) in rpc_init_task_statistics() argument
806 task->tk_garb_retry = 2; in rpc_init_task_statistics()
807 task->tk_cred_retry = 2; in rpc_init_task_statistics()
808 task->tk_rebind_retry = 2; in rpc_init_task_statistics()
811 task->tk_start = ktime_get(); in rpc_init_task_statistics()
815 rpc_reset_task_statistics(struct rpc_task *task) in rpc_reset_task_statistics() argument
817 task->tk_timeouts = 0; in rpc_reset_task_statistics()
818 task->tk_flags &= ~(RPC_CALL_MAJORSEEN|RPC_TASK_SENT); in rpc_reset_task_statistics()
819 rpc_init_task_statistics(task); in rpc_reset_task_statistics()
825 void rpc_exit_task(struct rpc_task *task) in rpc_exit_task() argument
827 task->tk_action = NULL; in rpc_exit_task()
828 if (task->tk_ops->rpc_count_stats) in rpc_exit_task()
829 task->tk_ops->rpc_count_stats(task, task->tk_calldata); in rpc_exit_task()
830 else if (task->tk_client) in rpc_exit_task()
831 rpc_count_iostats(task, task->tk_client->cl_metrics); in rpc_exit_task()
832 if (task->tk_ops->rpc_call_done != NULL) { in rpc_exit_task()
833 task->tk_ops->rpc_call_done(task, task->tk_calldata); in rpc_exit_task()
834 if (task->tk_action != NULL) { in rpc_exit_task()
836 xprt_release(task); in rpc_exit_task()
837 rpc_reset_task_statistics(task); in rpc_exit_task()
842 void rpc_signal_task(struct rpc_task *task) in rpc_signal_task() argument
846 if (!RPC_IS_ACTIVATED(task)) in rpc_signal_task()
848 set_bit(RPC_TASK_SIGNALLED, &task->tk_runstate); in rpc_signal_task()
850 queue = READ_ONCE(task->tk_waitqueue); in rpc_signal_task()
852 rpc_wake_up_queued_task_set_status(queue, task, -ERESTARTSYS); in rpc_signal_task()
855 void rpc_exit(struct rpc_task *task, int status) in rpc_exit() argument
857 task->tk_status = status; in rpc_exit()
858 task->tk_action = rpc_exit_task; in rpc_exit()
859 rpc_wake_up_queued_task(task->tk_waitqueue, task); in rpc_exit()
872 static void __rpc_execute(struct rpc_task *task) in __rpc_execute() argument
875 int task_is_async = RPC_IS_ASYNC(task); in __rpc_execute()
879 task->tk_pid, task->tk_flags); in __rpc_execute()
881 WARN_ON_ONCE(RPC_IS_QUEUED(task)); in __rpc_execute()
882 if (RPC_IS_QUEUED(task)) in __rpc_execute()
895 do_action = task->tk_action; in __rpc_execute()
896 if (task->tk_callback) { in __rpc_execute()
897 do_action = task->tk_callback; in __rpc_execute()
898 task->tk_callback = NULL; in __rpc_execute()
902 trace_rpc_task_run_action(task, do_action); in __rpc_execute()
903 do_action(task); in __rpc_execute()
908 if (!RPC_IS_QUEUED(task)) in __rpc_execute()
914 if (RPC_SIGNALLED(task)) { in __rpc_execute()
915 task->tk_rpc_status = -ERESTARTSYS; in __rpc_execute()
916 rpc_exit(task, -ERESTARTSYS); in __rpc_execute()
928 queue = task->tk_waitqueue; in __rpc_execute()
930 if (!RPC_IS_QUEUED(task)) { in __rpc_execute()
934 rpc_clear_running(task); in __rpc_execute()
940 dprintk("RPC: %5u sync task going to sleep\n", task->tk_pid); in __rpc_execute()
941 status = out_of_line_wait_on_bit(&task->tk_runstate, in __rpc_execute()
951 dprintk("RPC: %5u got signal\n", task->tk_pid); in __rpc_execute()
952 set_bit(RPC_TASK_SIGNALLED, &task->tk_runstate); in __rpc_execute()
953 task->tk_rpc_status = -ERESTARTSYS; in __rpc_execute()
954 rpc_exit(task, -ERESTARTSYS); in __rpc_execute()
956 dprintk("RPC: %5u sync task resuming\n", task->tk_pid); in __rpc_execute()
959 dprintk("RPC: %5u return %d, status %d\n", task->tk_pid, status, in __rpc_execute()
960 task->tk_status); in __rpc_execute()
962 rpc_release_task(task); in __rpc_execute()
974 void rpc_execute(struct rpc_task *task) in rpc_execute() argument
976 bool is_async = RPC_IS_ASYNC(task); in rpc_execute()
978 rpc_set_active(task); in rpc_execute()
979 rpc_make_runnable(rpciod_workqueue, task); in rpc_execute()
981 __rpc_execute(task); in rpc_execute()
1009 int rpc_malloc(struct rpc_task *task) in rpc_malloc() argument
1011 struct rpc_rqst *rqst = task->tk_rqstp; in rpc_malloc()
1016 if (RPC_IS_SWAPPER(task)) in rpc_malloc()
1030 task->tk_pid, size, buf); in rpc_malloc()
1042 void rpc_free(struct rpc_task *task) in rpc_free() argument
1044 void *buffer = task->tk_rqstp->rq_buffer; in rpc_free()
1064 static void rpc_init_task(struct rpc_task *task, const struct rpc_task_setup *task_setup_data) in rpc_init_task() argument
1066 memset(task, 0, sizeof(*task)); in rpc_init_task()
1067 atomic_set(&task->tk_count, 1); in rpc_init_task()
1068 task->tk_flags = task_setup_data->flags; in rpc_init_task()
1069 task->tk_ops = task_setup_data->callback_ops; in rpc_init_task()
1070 task->tk_calldata = task_setup_data->callback_data; in rpc_init_task()
1071 INIT_LIST_HEAD(&task->tk_task); in rpc_init_task()
1073 task->tk_priority = task_setup_data->priority - RPC_PRIORITY_LOW; in rpc_init_task()
1074 task->tk_owner = current->tgid; in rpc_init_task()
1077 task->tk_workqueue = task_setup_data->workqueue; in rpc_init_task()
1079 task->tk_xprt = rpc_task_get_xprt(task_setup_data->rpc_client, in rpc_init_task()
1082 task->tk_op_cred = get_rpccred(task_setup_data->rpc_op_cred); in rpc_init_task()
1084 if (task->tk_ops->rpc_call_prepare != NULL) in rpc_init_task()
1085 task->tk_action = rpc_prepare_task; in rpc_init_task()
1087 rpc_init_task_statistics(task); in rpc_init_task()
1104 struct rpc_task *task = setup_data->task; in rpc_new_task() local
1107 if (task == NULL) { in rpc_new_task()
1108 task = rpc_alloc_task(); in rpc_new_task()
1112 rpc_init_task(task, setup_data); in rpc_new_task()
1113 task->tk_flags |= flags; in rpc_new_task()
1114 dprintk("RPC: allocated task %p\n", task); in rpc_new_task()
1115 return task; in rpc_new_task()
1137 static void rpc_free_task(struct rpc_task *task) in rpc_free_task() argument
1139 unsigned short tk_flags = task->tk_flags; in rpc_free_task()
1141 put_rpccred(task->tk_op_cred); in rpc_free_task()
1142 rpc_release_calldata(task->tk_ops, task->tk_calldata); in rpc_free_task()
1145 dprintk("RPC: %5u freeing task\n", task->tk_pid); in rpc_free_task()
1146 mempool_free(task, rpc_task_mempool); in rpc_free_task()
1158 static void rpc_release_resources_task(struct rpc_task *task) in rpc_release_resources_task() argument
1160 xprt_release(task); in rpc_release_resources_task()
1161 if (task->tk_msg.rpc_cred) { in rpc_release_resources_task()
1162 put_cred(task->tk_msg.rpc_cred); in rpc_release_resources_task()
1163 task->tk_msg.rpc_cred = NULL; in rpc_release_resources_task()
1165 rpc_task_release_client(task); in rpc_release_resources_task()
1168 static void rpc_final_put_task(struct rpc_task *task, in rpc_final_put_task() argument
1172 INIT_WORK(&task->u.tk_work, rpc_async_release); in rpc_final_put_task()
1173 queue_work(q, &task->u.tk_work); in rpc_final_put_task()
1175 rpc_free_task(task); in rpc_final_put_task()
1178 static void rpc_do_put_task(struct rpc_task *task, struct workqueue_struct *q) in rpc_do_put_task() argument
1180 if (atomic_dec_and_test(&task->tk_count)) { in rpc_do_put_task()
1181 rpc_release_resources_task(task); in rpc_do_put_task()
1182 rpc_final_put_task(task, q); in rpc_do_put_task()
1186 void rpc_put_task(struct rpc_task *task) in rpc_put_task() argument
1188 rpc_do_put_task(task, NULL); in rpc_put_task()
1192 void rpc_put_task_async(struct rpc_task *task) in rpc_put_task_async() argument
1194 rpc_do_put_task(task, task->tk_workqueue); in rpc_put_task_async()
1198 static void rpc_release_task(struct rpc_task *task) in rpc_release_task() argument
1200 dprintk("RPC: %5u release task\n", task->tk_pid); in rpc_release_task()
1202 WARN_ON_ONCE(RPC_IS_QUEUED(task)); in rpc_release_task()
1204 rpc_release_resources_task(task); in rpc_release_task()
1211 if (atomic_read(&task->tk_count) != 1 + !RPC_IS_ASYNC(task)) { in rpc_release_task()
1213 if (!rpc_complete_task(task)) in rpc_release_task()
1216 if (!atomic_dec_and_test(&task->tk_count)) in rpc_release_task()
1219 rpc_final_put_task(task, task->tk_workqueue); in rpc_release_task()