23  * struct rcu_tasks - Definition for a Tasks-RCU-like mechanism.
84 /* Track exiting tasks in order to allow them to be waited for. */
97 /* RCU tasks grace-period state for debugging. */
151 // Enqueue a callback for the specified flavor of Tasks RCU.
170 // Wait for a grace period for the specified flavor of Tasks RCU.
181 /* RCU-tasks kthread that detects grace periods and invokes callbacks. */
196 	 * one RCU-tasks grace period and then invokes the callbacks.  in rcu_tasks_kthread()
244 /* Spawn RCU-tasks grace-period kthread. */
258  * Print any non-default Tasks RCU settings.
267 	pr_info("\tTrampoline variant of Tasks RCU enabled.\n");  in rcu_tasks_bootup_oddness()
270 	pr_info("\tRude variant of Tasks RCU enabled.\n");  in rcu_tasks_bootup_oddness()
273 	pr_info("\tTracing variant of Tasks RCU enabled.\n");  in rcu_tasks_bootup_oddness()
280 /* Dump out rcutorture-relevant state common to all RCU-tasks flavors. */
301 // Shared code between task-list-scanning variants of Tasks RCU.
303 /* Wait for one RCU-tasks grace period. */
315 	 * There were callbacks, so we need to wait for an RCU-tasks  in rcu_tasks_wait_gp()
316 	 * grace period.  Start off by scanning the task list for tasks  in rcu_tasks_wait_gp()
317 	 * that are not already voluntarily blocked.  Mark these tasks  in rcu_tasks_wait_gp()
331 	 * tasks, removing any that are no longer holdouts.  When the list  in rcu_tasks_wait_gp()
396 //	Invoke synchronize_srcu() to ensure that all tasks that were
398 //	synchronize with this RCU Tasks grace period) have completed
406 //	t->on_rq and t->nvcsw transitions are seen by all CPUs and tasks
407 //	to have happened before the end of this RCU Tasks grace period.
455 	 * Wait for tasks that are in the process of exiting.  This  in rcu_tasks_postscan()
456 	 * does only part of the job, ensuring that all tasks that were  in rcu_tasks_postscan()
464 /* See if tasks are still holding out, complain if so. */
484 		pr_err("INFO: rcu_tasks detected stalls on tasks:\n");  in check_holdout_task()
496 /* Scan the holdout lists for tasks no longer holding out. */
508 /* Finish off the Tasks-RCU grace period. */
514 	 * reordering on other CPUs could cause their RCU-tasks read-side  in rcu_tasks_postgp()
524 	 * In addition, this synchronize_rcu() waits for exiting tasks  in rcu_tasks_postgp()
532 DEFINE_RCU_TASKS(rcu_tasks, rcu_tasks_wait_gp, call_rcu_tasks, "RCU Tasks");
546  * this primitive is intended to determine that all tasks have passed
559  * synchronize_rcu_tasks - wait until an rcu-tasks grace period has elapsed.
561  * Control will return to the caller some time after a full rcu-tasks
563  * executing rcu-tasks read-side critical sections have elapsed.  These
644 // "Rude" variant of Tasks RCU, inspired by Steve Rostedt's trick of
661 // Wait for one rude RCU-tasks grace period.
670 		 "RCU Tasks Rude");
684  * that all tasks have passed through a safe state, not so much for
697  * synchronize_rcu_tasks_rude - wait for a rude rcu-tasks grace period
699  * Control will return to the caller some time after a rude rcu-tasks
701  * executing rcu-tasks read-side critical sections have elapsed.  These
751 // Tracing variant of Tasks RCU.  This variant is designed to be used
766 // It is necessary to scan the full tasklist, much as for Tasks RCU.  There
768 // Tasks RCU.  If needed, these downsides can be at least partially remedied.
771 // flavors, rcu_preempt and rcu_sched.  The fact that RCU Tasks Trace
789 //	to invoke trc_read_check_handler() in the context of running tasks
794 //	state as above (but only for idle tasks), unblock CPU-hotplug
796 //	tasks that are in the process of exiting.
806 // The exit_tasks_rcu_finish_trace() synchronizes with exiting tasks.
825 static DECLARE_WAIT_QUEUE_HEAD(trc_wait);	// List of holdout tasks.
838 		 "RCU Tasks Trace");
948 		if (!ofl && // Check for "running" idle tasks on offline CPUs.  in trc_inspect_reader()
1037 /* Initialize for a new RCU-tasks-trace grace period. */
1071  * pick up the idle tasks.
1083 	// Wait for late-stage exiting tasks to finish exiting.  in rcu_tasks_trace_postscan()
1086 	// Any tasks that exit after this point will set ->trc_reader_checked.  in rcu_tasks_trace_postscan()
1089 /* Show the state of a task stalling the current RCU tasks trace GP. */
1095 		pr_err("INFO: rcu_tasks_trace detected stalls on tasks:\n");  in show_stalled_task_trace()
1111 /* List stalled IPIs for RCU tasks trace. */
1218  * that all tasks have passed through a safe state, not so much for
1231  * synchronize_rcu_tasks_trace - wait for a trace rcu-tasks grace period
1233  * Control will return to the caller some time after a trace rcu-tasks
1235  * rcu-tasks read-side critical sections have elapsed.  These read-side
1249 …_held(&rcu_trace_lock_map), "Illegal synchronize_rcu_tasks_trace() in RCU Tasks Trace read-side cr…  in synchronize_rcu_tasks_trace()
1353 	pr_info("Running RCU-tasks wait API self tests\n");  in rcu_tasks_initiate_self_tests()