23  * Definition for a Tasks-RCU-like mechanism.
84 /* Track exiting tasks in order to allow them to be waited for. */
97 /* RCU tasks grace-period state for debugging. */
151 // Enqueue a callback for the specified flavor of Tasks RCU.
170 // Wait for a grace period for the specified flavor of Tasks RCU.
181 /* RCU-tasks kthread that detects grace periods and invokes callbacks. */
196 	 * one RCU-tasks grace period and then invokes the callbacks.  in rcu_tasks_kthread()
244 /* Spawn RCU-tasks grace-period kthread, e.g., at core_initcall() time. */
258  * Print any non-default Tasks RCU settings.
267 	pr_info("\tTrampoline variant of Tasks RCU enabled.\n");  in rcu_tasks_bootup_oddness()
270 	pr_info("\tRude variant of Tasks RCU enabled.\n");  in rcu_tasks_bootup_oddness()
273 	pr_info("\tTracing variant of Tasks RCU enabled.\n");  in rcu_tasks_bootup_oddness()
280 /* Dump out rcutorture-relevant state common to all RCU-tasks flavors. */
301 // Shared code between task-list-scanning variants of Tasks RCU.
303 /* Wait for one RCU-tasks grace period. */
315 	 * There were callbacks, so we need to wait for an RCU-tasks  in rcu_tasks_wait_gp()
316 	 * grace period.  Start off by scanning the task list for tasks  in rcu_tasks_wait_gp()
317 	 * that are not already voluntarily blocked.  Mark these tasks  in rcu_tasks_wait_gp()
331 	 * tasks, removing any that are no longer holdouts.  When the list  in rcu_tasks_wait_gp()
420 	 * Wait for tasks that are in the process of exiting.  This  in rcu_tasks_postscan()
421 	 * does only part of the job, ensuring that all tasks that were  in rcu_tasks_postscan()
429 /* See if tasks are still holding out, complain if so. */
449 		pr_err("INFO: rcu_tasks detected stalls on tasks:\n");  in check_holdout_task()
461 /* Scan the holdout lists for tasks no longer holding out. */
473 /* Finish off the Tasks-RCU grace period. */
479 	 * reordering on other CPUs could cause their RCU-tasks read-side  in rcu_tasks_postgp()
489 	 * In addition, this synchronize_rcu() waits for exiting tasks  in rcu_tasks_postgp()
497 DEFINE_RCU_TASKS(rcu_tasks, rcu_tasks_wait_gp, call_rcu_tasks, "RCU Tasks");
511  * this primitive is intended to determine that all tasks have passed
524  * synchronize_rcu_tasks - wait until an rcu-tasks grace period has elapsed.
526  * Control will return to the caller some time after a full rcu-tasks
528  * executing rcu-tasks read-side critical sections have elapsed.  These
610 // "Rude" variant of Tasks RCU, inspired by Steve Rostedt's trick of
622 // Wait for one rude RCU-tasks grace period.
631 		 "RCU Tasks Rude");
645  * that all tasks have passed through a safe state, not so much for
658  * synchronize_rcu_tasks_rude - wait for a rude rcu-tasks grace period
660  * Control will return to the caller some time after a rude rcu-tasks
662  * executing rcu-tasks read-side critical sections have elapsed.  These
715 // Tracing variant of Tasks RCU.  This variant is designed to be used
730 // It is necessary to scan the full tasklist, much as for Tasks RCU.  There
732 // Tasks RCU.  If needed, these downsides can be at least partially remedied.
735 // flavors, rcu_preempt and rcu_sched.  The fact that RCU Tasks Trace
750 static DECLARE_WAIT_QUEUE_HEAD(trc_wait);	// List of holdout tasks.
763 		 "RCU Tasks Trace");
873 		if (!ofl && // Check for "running" idle tasks on offline CPUs.  in trc_inspect_reader()
957 /* Initialize for a new RCU-tasks-trace grace period. */
986  * pick up the idle tasks.
998 	// Wait for late-stage exiting tasks to finish exiting.  in rcu_tasks_trace_postscan()
1001 	// Any tasks that exit after this point will set ->trc_reader_checked.  in rcu_tasks_trace_postscan()
1004 /* Show the state of a task stalling the current RCU tasks trace GP. */
1010 		pr_err("INFO: rcu_tasks_trace detected stalls on tasks:\n");  in show_stalled_task_trace()
1026 /* List stalled IPIs for RCU tasks trace. */
1133  * that all tasks have passed through a safe state, not so much for
1146  * synchronize_rcu_tasks_trace - wait for a trace rcu-tasks grace period
1148  * Control will return to the caller some time after a trace rcu-tasks
1150  * rcu-tasks read-side critical sections have elapsed.  These read-side
1164 …_held(&rcu_trace_lock_map), "Illegal synchronize_rcu_tasks_trace() in RCU Tasks Trace read-side cr…  in synchronize_rcu_tasks_trace()