Lines Matching +full:cpu +full:- +full:read
1 /* SPDX-License-Identifier: GPL-2.0+ */
3 * Task-based RCU implementations.
24 * struct rcu_tasks_percpu - Per-CPU component of definition for a Tasks-RCU-like mechanism.
26 * @lock: Lock protecting per-CPU callback list.
28 * @rtp_n_lock_retries: Rough lock-contention statistic.
33 * @cpu: CPU number corresponding to this entry.
45 int cpu; member
50 * struct rcu_tasks - Definition for a Tasks-RCU-like mechanism.
53 * @tasks_gp_mutex: Mutex protecting grace period, needed during mid-boot dead zone.
54 * @kthread_ptr: This flavor's grace-period/callback-invocation kthread.
55 * @gp_func: This flavor's grace-period-wait function.
57 * @gp_sleep: Per-grace-period sleep to prevent CPU-bound looping.
60 * @gp_start: Most recent grace-period start in jiffies.
63 * @n_ipis_fails: Number of IPI-send failures.
64 * @pregp_func: This flavor's pre-grace-period function (optional).
65 * @pertask_func: This flavor's per-task scan function (optional).
66 * @postscan_func: This flavor's post-task scan function (optional).
67 * @holdouts_func: This flavor's holdout-list scan function (optional).
68 * @postgp_func: This flavor's post-grace-period function (optional).
69 * @call_func: This flavor's call_rcu()-equivalent function.
71 * @percpu_enqueue_shift: Shift down CPU ID this much when enqueuing callbacks.
72 * @percpu_enqueue_lim: Number of per-CPU callback queues in use for enqueuing.
73 * @percpu_dequeue_lim: Number of per-CPU callback queues in use for dequeuing.
74 * @percpu_dequeue_gpseq: RCU grace-period number to propagate enqueue limit to dequeuers.
135 .barrier_q_seq = (0UL - 50UL) << RCU_SEQ_CTR_SHIFT, \
158 static int rcu_task_enqueue_lim __read_mostly = -1;
167 /* RCU tasks grace-period state for debugging. */
203 /* Record grace-period phase and time. */
206 rtp->gp_state = newstate; in set_tasks_gp_state()
207 rtp->gp_jiffies = jiffies; in set_tasks_gp_state()
214 int i = data_race(rtp->gp_state); // Let KCSAN detect update races in tasks_gp_state_getname()
223 // Initialize per-CPU callback lists for the specified flavor of
227 int cpu; in cblist_init_generic() local
232 raw_spin_lock_irqsave(&rtp->cbs_gbl_lock, flags); in cblist_init_generic()
245 if (((nr_cpu_ids - 1) >> shift) >= lim) in cblist_init_generic()
247 WRITE_ONCE(rtp->percpu_enqueue_shift, shift); in cblist_init_generic()
248 WRITE_ONCE(rtp->percpu_dequeue_lim, lim); in cblist_init_generic()
249 smp_store_release(&rtp->percpu_enqueue_lim, lim); in cblist_init_generic()
250 for_each_possible_cpu(cpu) { in cblist_init_generic()
251 struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu); in cblist_init_generic()
254 if (cpu) in cblist_init_generic()
257 if (rcu_segcblist_empty(&rtpcp->cblist)) in cblist_init_generic()
258 rcu_segcblist_init(&rtpcp->cblist); in cblist_init_generic()
259 INIT_WORK(&rtpcp->rtp_work, rcu_tasks_invoke_cbs_wq); in cblist_init_generic()
260 rtpcp->cpu = cpu; in cblist_init_generic()
261 rtpcp->rtpp = rtp; in cblist_init_generic()
262 if (!rtpcp->rtp_blkd_tasks.next) in cblist_init_generic()
263 INIT_LIST_HEAD(&rtpcp->rtp_blkd_tasks); in cblist_init_generic()
266 raw_spin_unlock_irqrestore(&rtp->cbs_gbl_lock, flags); in cblist_init_generic()
267 …ng shift to %d and lim to %d.\n", __func__, data_race(rtp->percpu_enqueue_shift), data_race(rtp->p… in cblist_init_generic()
270 // IRQ-work handler that does deferred wakeup for call_rcu_tasks_generic().
276 rtp = rtpcp->rtpp; in call_rcu_tasks_iw_wakeup()
277 rcuwait_wake_up(&rtp->cbs_wait); in call_rcu_tasks_iw_wakeup()
292 rhp->next = NULL; in call_rcu_tasks_generic()
293 rhp->func = func; in call_rcu_tasks_generic()
296 ideal_cpu = smp_processor_id() >> READ_ONCE(rtp->percpu_enqueue_shift); in call_rcu_tasks_generic()
297 chosen_cpu = cpumask_next(ideal_cpu - 1, cpu_possible_mask); in call_rcu_tasks_generic()
298 rtpcp = per_cpu_ptr(rtp->rtpcpu, chosen_cpu); in call_rcu_tasks_generic()
302 if (rtpcp->rtp_jiffies != j) { in call_rcu_tasks_generic()
303 rtpcp->rtp_jiffies = j; in call_rcu_tasks_generic()
304 rtpcp->rtp_n_lock_retries = 0; in call_rcu_tasks_generic()
306 if (rcu_task_cb_adjust && ++rtpcp->rtp_n_lock_retries > rcu_task_contend_lim && in call_rcu_tasks_generic()
307 READ_ONCE(rtp->percpu_enqueue_lim) != nr_cpu_ids) in call_rcu_tasks_generic()
310 if (!rcu_segcblist_is_enabled(&rtpcp->cblist)) { in call_rcu_tasks_generic()
315 needwake = rcu_segcblist_empty(&rtpcp->cblist); in call_rcu_tasks_generic()
316 rcu_segcblist_enqueue(&rtpcp->cblist, rhp); in call_rcu_tasks_generic()
319 raw_spin_lock_irqsave(&rtp->cbs_gbl_lock, flags); in call_rcu_tasks_generic()
320 if (rtp->percpu_enqueue_lim != nr_cpu_ids) { in call_rcu_tasks_generic()
321 WRITE_ONCE(rtp->percpu_enqueue_shift, 0); in call_rcu_tasks_generic()
322 WRITE_ONCE(rtp->percpu_dequeue_lim, nr_cpu_ids); in call_rcu_tasks_generic()
323 smp_store_release(&rtp->percpu_enqueue_lim, nr_cpu_ids); in call_rcu_tasks_generic()
324 pr_info("Switching %s to per-CPU callback queuing.\n", rtp->name); in call_rcu_tasks_generic()
326 raw_spin_unlock_irqrestore(&rtp->cbs_gbl_lock, flags); in call_rcu_tasks_generic()
330 if (needwake && READ_ONCE(rtp->kthread_ptr)) in call_rcu_tasks_generic()
331 irq_work_queue(&rtpcp->rtp_irq_work); in call_rcu_tasks_generic()
341 rtp = rtpcp->rtpp; in rcu_barrier_tasks_generic_cb()
342 if (atomic_dec_and_test(&rtp->barrier_q_count)) in rcu_barrier_tasks_generic_cb()
343 complete(&rtp->barrier_q_completion); in rcu_barrier_tasks_generic_cb()
346 // Wait for all in-flight callbacks for the specified RCU Tasks flavor.
350 int cpu; in rcu_barrier_tasks_generic() local
353 unsigned long s = rcu_seq_snap(&rtp->barrier_q_seq); in rcu_barrier_tasks_generic()
355 mutex_lock(&rtp->barrier_q_mutex); in rcu_barrier_tasks_generic()
356 if (rcu_seq_done(&rtp->barrier_q_seq, s)) { in rcu_barrier_tasks_generic()
358 mutex_unlock(&rtp->barrier_q_mutex); in rcu_barrier_tasks_generic()
361 rcu_seq_start(&rtp->barrier_q_seq); in rcu_barrier_tasks_generic()
362 init_completion(&rtp->barrier_q_completion); in rcu_barrier_tasks_generic()
363 atomic_set(&rtp->barrier_q_count, 2); in rcu_barrier_tasks_generic()
364 for_each_possible_cpu(cpu) { in rcu_barrier_tasks_generic()
365 if (cpu >= smp_load_acquire(&rtp->percpu_dequeue_lim)) in rcu_barrier_tasks_generic()
367 rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu); in rcu_barrier_tasks_generic()
368 rtpcp->barrier_q_head.func = rcu_barrier_tasks_generic_cb; in rcu_barrier_tasks_generic()
370 if (rcu_segcblist_entrain(&rtpcp->cblist, &rtpcp->barrier_q_head)) in rcu_barrier_tasks_generic()
371 atomic_inc(&rtp->barrier_q_count); in rcu_barrier_tasks_generic()
374 if (atomic_sub_and_test(2, &rtp->barrier_q_count)) in rcu_barrier_tasks_generic()
375 complete(&rtp->barrier_q_completion); in rcu_barrier_tasks_generic()
376 wait_for_completion(&rtp->barrier_q_completion); in rcu_barrier_tasks_generic()
377 rcu_seq_end(&rtp->barrier_q_seq); in rcu_barrier_tasks_generic()
378 mutex_unlock(&rtp->barrier_q_mutex); in rcu_barrier_tasks_generic()
385 int cpu; in rcu_tasks_need_gpcb() local
392 for (cpu = 0; cpu < smp_load_acquire(&rtp->percpu_dequeue_lim); cpu++) { in rcu_tasks_need_gpcb()
393 struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu); in rcu_tasks_need_gpcb()
396 if (!rcu_segcblist_n_cbs(&rtpcp->cblist)) in rcu_tasks_need_gpcb()
400 n = rcu_segcblist_n_cbs(&rtpcp->cblist); in rcu_tasks_need_gpcb()
403 if (cpu > 0) in rcu_tasks_need_gpcb()
406 rcu_segcblist_advance(&rtpcp->cblist, rcu_seq_current(&rtp->tasks_gp_seq)); in rcu_tasks_need_gpcb()
407 (void)rcu_segcblist_accelerate(&rtpcp->cblist, rcu_seq_snap(&rtp->tasks_gp_seq)); in rcu_tasks_need_gpcb()
408 if (rcu_segcblist_pend_cbs(&rtpcp->cblist)) in rcu_tasks_need_gpcb()
410 if (!rcu_segcblist_empty(&rtpcp->cblist)) in rcu_tasks_need_gpcb()
417 // rcu_task_collapse_lim callbacks on CPU 0 and none on any other in rcu_tasks_need_gpcb()
418 // CPU, limit enqueueing to CPU 0. (2) After an RCU grace period, in rcu_tasks_need_gpcb()
420 // to CPU 0. Note the matching RCU read-side critical section in in rcu_tasks_need_gpcb()
423 raw_spin_lock_irqsave(&rtp->cbs_gbl_lock, flags); in rcu_tasks_need_gpcb()
424 if (rtp->percpu_enqueue_lim > 1) { in rcu_tasks_need_gpcb()
425 WRITE_ONCE(rtp->percpu_enqueue_shift, order_base_2(nr_cpu_ids)); in rcu_tasks_need_gpcb()
426 smp_store_release(&rtp->percpu_enqueue_lim, 1); in rcu_tasks_need_gpcb()
427 rtp->percpu_dequeue_gpseq = get_state_synchronize_rcu(); in rcu_tasks_need_gpcb()
428 pr_info("Starting switch %s to CPU-0 callback queuing.\n", rtp->name); in rcu_tasks_need_gpcb()
430 raw_spin_unlock_irqrestore(&rtp->cbs_gbl_lock, flags); in rcu_tasks_need_gpcb()
433 poll_state_synchronize_rcu(rtp->percpu_dequeue_gpseq)) { in rcu_tasks_need_gpcb()
434 raw_spin_lock_irqsave(&rtp->cbs_gbl_lock, flags); in rcu_tasks_need_gpcb()
435 if (rtp->percpu_enqueue_lim < rtp->percpu_dequeue_lim) { in rcu_tasks_need_gpcb()
436 WRITE_ONCE(rtp->percpu_dequeue_lim, 1); in rcu_tasks_need_gpcb()
437 pr_info("Completing switch %s to CPU-0 callback queuing.\n", rtp->name); in rcu_tasks_need_gpcb()
439 for (cpu = rtp->percpu_dequeue_lim; cpu < nr_cpu_ids; cpu++) { in rcu_tasks_need_gpcb()
440 struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu); in rcu_tasks_need_gpcb()
442 WARN_ON_ONCE(rcu_segcblist_n_cbs(&rtpcp->cblist)); in rcu_tasks_need_gpcb()
444 raw_spin_unlock_irqrestore(&rtp->cbs_gbl_lock, flags); in rcu_tasks_need_gpcb()
453 int cpu; in rcu_tasks_invoke_cbs() local
461 cpu = rtpcp->cpu; in rcu_tasks_invoke_cbs()
462 cpunext = cpu * 2 + 1; in rcu_tasks_invoke_cbs()
463 if (cpunext < smp_load_acquire(&rtp->percpu_dequeue_lim)) { in rcu_tasks_invoke_cbs()
464 rtpcp_next = per_cpu_ptr(rtp->rtpcpu, cpunext); in rcu_tasks_invoke_cbs()
465 queue_work_on(cpunext, system_wq, &rtpcp_next->rtp_work); in rcu_tasks_invoke_cbs()
467 if (cpunext < smp_load_acquire(&rtp->percpu_dequeue_lim)) { in rcu_tasks_invoke_cbs()
468 rtpcp_next = per_cpu_ptr(rtp->rtpcpu, cpunext); in rcu_tasks_invoke_cbs()
469 queue_work_on(cpunext, system_wq, &rtpcp_next->rtp_work); in rcu_tasks_invoke_cbs()
473 if (rcu_segcblist_empty(&rtpcp->cblist) || !cpu_possible(cpu)) in rcu_tasks_invoke_cbs()
476 rcu_segcblist_advance(&rtpcp->cblist, rcu_seq_current(&rtp->tasks_gp_seq)); in rcu_tasks_invoke_cbs()
477 rcu_segcblist_extract_done_cbs(&rtpcp->cblist, &rcl); in rcu_tasks_invoke_cbs()
482 rhp->func(rhp); in rcu_tasks_invoke_cbs()
487 rcu_segcblist_add_len(&rtpcp->cblist, -len); in rcu_tasks_invoke_cbs()
488 (void)rcu_segcblist_accelerate(&rtpcp->cblist, rcu_seq_snap(&rtp->tasks_gp_seq)); in rcu_tasks_invoke_cbs()
498 rtp = rtpcp->rtpp; in rcu_tasks_invoke_cbs_wq()
507 mutex_lock(&rtp->tasks_gp_mutex); in rcu_tasks_one_gp()
514 rcuwait_wait_event(&rtp->cbs_wait, in rcu_tasks_one_gp()
522 rtp->gp_start = jiffies; in rcu_tasks_one_gp()
523 rcu_seq_start(&rtp->tasks_gp_seq); in rcu_tasks_one_gp()
524 rtp->gp_func(rtp); in rcu_tasks_one_gp()
525 rcu_seq_end(&rtp->tasks_gp_seq); in rcu_tasks_one_gp()
530 rcu_tasks_invoke_cbs(rtp, per_cpu_ptr(rtp->rtpcpu, 0)); in rcu_tasks_one_gp()
531 mutex_unlock(&rtp->tasks_gp_mutex); in rcu_tasks_one_gp()
534 // RCU-tasks kthread that detects grace periods and invokes callbacks.
541 WRITE_ONCE(rtp->kthread_ptr, current); // Let GPs start! in rcu_tasks_kthread()
546 * one RCU-tasks grace period and then invokes the callbacks. in rcu_tasks_kthread()
547 * This loop is terminated by the system going down. ;-) in rcu_tasks_kthread()
555 schedule_timeout_idle(rtp->gp_sleep); in rcu_tasks_kthread()
566 // If the grace-period kthread is running, use it. in synchronize_rcu_tasks_generic()
567 if (READ_ONCE(rtp->kthread_ptr)) { in synchronize_rcu_tasks_generic()
568 wait_rcu_gp(rtp->call_func); in synchronize_rcu_tasks_generic()
574 /* Spawn RCU-tasks grace-period kthread. */
579 t = kthread_run(rcu_tasks_kthread, rtp, "%s_kthread", rtp->kname); in rcu_spawn_tasks_kthread_generic()
580 …NCE(IS_ERR(t), "%s: Could not start %s grace-period kthread, OOM is now expected behavior\n", __fu… in rcu_spawn_tasks_kthread_generic()
588 * Print any non-default Tasks RCU settings.
596 …pr_info("\tTasks-RCU CPU stall warnings timeout set to %d (rcu_task_stall_timeout).\n", rcu_task_s… in rcu_tasks_bootup_oddness()
599 …pr_info("\tTasks-RCU CPU stall info multiplier clamped to %d (rcu_task_stall_info_mult).\n", rtsim… in rcu_tasks_bootup_oddness()
617 /* Dump out rcutorture-relevant state common to all RCU-tasks flavors. */
620 int cpu; in show_rcu_tasks_generic_gp_kthread() local
623 for_each_possible_cpu(cpu) { in show_rcu_tasks_generic_gp_kthread()
624 struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu); in show_rcu_tasks_generic_gp_kthread()
626 if (!data_race(rcu_segcblist_empty(&rtpcp->cblist))) { in show_rcu_tasks_generic_gp_kthread()
632 rtp->kname, in show_rcu_tasks_generic_gp_kthread()
633 tasks_gp_state_getname(rtp), data_race(rtp->gp_state), in show_rcu_tasks_generic_gp_kthread()
634 jiffies - data_race(rtp->gp_jiffies), in show_rcu_tasks_generic_gp_kthread()
635 data_race(rcu_seq_current(&rtp->tasks_gp_seq)), in show_rcu_tasks_generic_gp_kthread()
636 data_race(rtp->n_ipis_fails), data_race(rtp->n_ipis), in show_rcu_tasks_generic_gp_kthread()
637 ".k"[!!data_race(rtp->kthread_ptr)], in show_rcu_tasks_generic_gp_kthread()
649 // Shared code between task-list-scanning variants of Tasks RCU.
651 /* Wait for one RCU-tasks grace period. */
665 rtp->pregp_func(&holdouts); in rcu_tasks_wait_gp()
668 * There were callbacks, so we need to wait for an RCU-tasks in rcu_tasks_wait_gp()
674 if (rtp->pertask_func) { in rcu_tasks_wait_gp()
677 rtp->pertask_func(t, &holdouts); in rcu_tasks_wait_gp()
682 rtp->postscan_func(&holdouts); in rcu_tasks_wait_gp()
694 fract = rtp->init_fract; in rcu_tasks_wait_gp()
724 rtp->holdouts_func(&holdouts, needreport, &firstreport); in rcu_tasks_wait_gp()
726 // Print pre-stall informational messages if needed. in rcu_tasks_wait_gp()
732 __func__, rtp->kname, rtp->tasks_gp_seq, j - rtp->gp_start); in rcu_tasks_wait_gp()
737 rtp->postgp_func(rtp); in rcu_tasks_wait_gp()
747 // switch, cond_resched_tasks_rcu_qs(), user-space execution, and idle.
749 // read-side primitives similar to rcu_read_lock() and rcu_read_unlock()
753 // rates from multiple CPUs. If this is required, per-CPU callback lists
762 // Invokes synchronize_rcu() in order to wait for all in-flight
763 // t->on_rq and t->nvcsw transitions to complete. This works because
765 // rcu_tasks_pertask(), invoked on every non-idle task:
766 // For every runnable non-idle task other than the current one, use
781 // t->on_rq and t->nvcsw transitions are seen by all CPUs and tasks
788 // read-side critical sections waited for by rcu_tasks_postscan().
790 // Pre-grace-period update-side code is ordered before the grace
791 // via the raw_spin_lock.*rcu_node(). Pre-grace-period read-side code
796 /* Pre-grace-period preparation. */
800 * Wait for all pre-existing t->on_rq and t->nvcsw transitions in rcu_tasks_pregp_step()
803 * synchronize_rcu(), a read-side critical section that started in rcu_tasks_pregp_step()
808 * memory barrier on the first store to t->rcu_tasks_holdout, in rcu_tasks_pregp_step()
815 /* Per-task initial processing. */
818 if (t != current && READ_ONCE(t->on_rq) && !is_idle_task(t)) { in rcu_tasks_pertask()
820 t->rcu_tasks_nvcsw = READ_ONCE(t->nvcsw); in rcu_tasks_pertask()
821 WRITE_ONCE(t->rcu_tasks_holdout, true); in rcu_tasks_pertask()
822 list_add(&t->rcu_tasks_holdout_list, hop); in rcu_tasks_pertask()
843 int cpu; in check_holdout_task() local
845 if (!READ_ONCE(t->rcu_tasks_holdout) || in check_holdout_task()
846 t->rcu_tasks_nvcsw != READ_ONCE(t->nvcsw) || in check_holdout_task()
847 !READ_ONCE(t->on_rq) || in check_holdout_task()
849 !is_idle_task(t) && t->rcu_tasks_idle_cpu >= 0)) { in check_holdout_task()
850 WRITE_ONCE(t->rcu_tasks_holdout, false); in check_holdout_task()
851 list_del_init(&t->rcu_tasks_holdout_list); in check_holdout_task()
862 cpu = task_cpu(t); in check_holdout_task()
865 "N."[cpu < 0 || !tick_nohz_full_cpu(cpu)], in check_holdout_task()
866 t->rcu_tasks_nvcsw, t->nvcsw, t->rcu_tasks_holdout, in check_holdout_task()
867 t->rcu_tasks_idle_cpu, cpu); in check_holdout_task()
883 /* Finish off the Tasks-RCU grace period. */
887 * Because ->on_rq and ->nvcsw are not guaranteed to have a full in rcu_tasks_postgp()
889 * reordering on other CPUs could cause their RCU-tasks read-side in rcu_tasks_postgp()
891 * However, because these ->nvcsw updates are carried out with in rcu_tasks_postgp()
895 * This synchronize_rcu() also confines all ->rcu_tasks_holdout in rcu_tasks_postgp()
897 * memory barriers for ->rcu_tasks_holdout accesses. in rcu_tasks_postgp()
910 * call_rcu_tasks() - Queue an RCU for invocation task-based grace period
916 * read-side critical sections have completed. call_rcu_tasks() assumes
917 * that the read-side critical sections end at a voluntary context
919 * or transition to usermode execution. As such, there are no read-side
922 * through a safe state, not so much for data-structure synchronization.
934 * synchronize_rcu_tasks - wait until an rcu-tasks grace period has elapsed.
936 * Control will return to the caller some time after a full rcu-tasks
938 * executing rcu-tasks read-side critical sections have elapsed. These
939 * read-side critical sections are delimited by calls to schedule(),
958 * rcu_barrier_tasks - Wait for in-flight call_rcu_tasks() callbacks.
995 current->rcu_tasks_idx = __srcu_read_lock(&tasks_rcu_exit_srcu); in exit_tasks_rcu_start()
1005 __srcu_read_unlock(&tasks_rcu_exit_srcu, t->rcu_tasks_idx); in exit_tasks_rcu_finish()
1029 // Ordering is provided by the scheduler's context-switch code.
1036 // Wait for one rude RCU-tasks grace period.
1040 return; // Fastpath for only one CPU. in rcu_tasks_rude_wait_gp()
1042 rtp->n_ipis += cpumask_weight(cpu_online_mask); in rcu_tasks_rude_wait_gp()
1051 * call_rcu_tasks_rude() - Queue a callback rude task-based grace period
1057 * read-side critical sections have completed. call_rcu_tasks_rude()
1058 * assumes that the read-side critical sections end at context switch,
1060 * usermode execution is schedulable). As such, there are no read-side
1063 * through a safe state, not so much for data-structure synchronization.
1075 * synchronize_rcu_tasks_rude - wait for a rude rcu-tasks grace period
1077 * Control will return to the caller some time after a rude rcu-tasks
1079 * executing rcu-tasks read-side critical sections have elapsed. These
1080 * read-side critical sections are delimited by calls to schedule(),
1099 * rcu_barrier_tasks_rude - Wait for in-flight call_rcu_tasks_rude() callbacks.
1133 // 1. Has explicit read-side markers to allow finite grace periods
1134 // in the face of in-kernel loops for PREEMPT=n builds.
1137 // CPU-hotplug code paths, similar to the capabilities of SRCU.
1139 // 3. Avoids expensive read-side instructions, having overhead similar
1142 // There are of course downsides. For example, the grace-period code
1158 // Disables CPU hotplug, adds all currently executing tasks to the
1160 // or were preempted within their current RCU Tasks Trace read-side
1162 // Finally, this function re-enables CPU hotplug.
1163 // The ->pertask_func() pointer is NULL, so there is no per-task processing.
1165 // Invokes synchronize_rcu() to wait for late-stage exiting tasks
1177 // Pre-grace-period update-side code is ordered before the grace period
1178 // via the ->cbs_lock and barriers in rcu_tasks_kthread(). Pre-grace-period
1179 // read-side code is ordered before the grace period by atomic operations
1181 // context-switch ordering (for locked-down non-running readers).
1193 // Record outstanding IPIs to each CPU. No point in sending two...
1207 /* Load from ->trc_reader_special.b.need_qs with proper ordering. */
1210 smp_mb(); // Enforce full grace-period ordering. in rcu_ld_need_qs()
1211 return smp_load_acquire(&t->trc_reader_special.b.need_qs); in rcu_ld_need_qs()
1214 /* Store to ->trc_reader_special.b.need_qs with proper ordering. */
1217 smp_store_release(&t->trc_reader_special.b.need_qs, v); in rcu_st_need_qs()
1218 smp_mb(); // Enforce full grace-period ordering. in rcu_st_need_qs()
1222 * Do a cmpxchg() on ->trc_reader_special.b.need_qs, allowing for
1223 * the four-byte operand-size restriction of some platforms.
1229 union rcu_special trs_old = READ_ONCE(t->trc_reader_special); in rcu_trc_cmpxchg_need_qs()
1235 ret.s = cmpxchg(&t->trc_reader_special.s, trs_old.s, trs_new.s); in rcu_trc_cmpxchg_need_qs()
1241 * If we are the last reader, signal the grace-period kthread.
1242 * Also remove from the per-CPU list of blocked tasks.
1250 // Open-coded full-word version of rcu_ld_need_qs(). in rcu_read_unlock_trace_special()
1251 smp_mb(); // Enforce full grace-period ordering. in rcu_read_unlock_trace_special()
1252 trs = smp_load_acquire(&t->trc_reader_special); in rcu_read_unlock_trace_special()
1254 if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB) && t->trc_reader_special.b.need_mb) in rcu_read_unlock_trace_special()
1255 smp_mb(); // Pairs with update-side barriers. in rcu_read_unlock_trace_special()
1256 // Update .need_qs before ->trc_reader_nesting for irq/NMI handlers. in rcu_read_unlock_trace_special()
1264 rtpcp = per_cpu_ptr(rcu_tasks_trace.rtpcpu, t->trc_blkd_cpu); in rcu_read_unlock_trace_special()
1266 list_del_init(&t->trc_blkd_node); in rcu_read_unlock_trace_special()
1267 WRITE_ONCE(t->trc_reader_special.b.blocked, false); in rcu_read_unlock_trace_special()
1270 WRITE_ONCE(t->trc_reader_nesting, 0); in rcu_read_unlock_trace_special()
1274 /* Add a newly blocked reader task to its CPU's list. */
1283 t->trc_blkd_cpu = smp_processor_id(); in rcu_tasks_trace_qs_blkd()
1284 if (!rtpcp->rtp_blkd_tasks.next) in rcu_tasks_trace_qs_blkd()
1285 INIT_LIST_HEAD(&rtpcp->rtp_blkd_tasks); in rcu_tasks_trace_qs_blkd()
1286 list_add(&t->trc_blkd_node, &rtpcp->rtp_blkd_tasks); in rcu_tasks_trace_qs_blkd()
1287 WRITE_ONCE(t->trc_reader_special.b.blocked, true); in rcu_tasks_trace_qs_blkd()
1295 if (list_empty(&t->trc_holdout_list)) { in trc_add_holdout()
1297 list_add(&t->trc_holdout_list, bhp); in trc_add_holdout()
1305 if (!list_empty(&t->trc_holdout_list)) { in trc_del_holdout()
1306 list_del_init(&t->trc_holdout_list); in trc_del_holdout()
1308 n_trc_holdouts--; in trc_del_holdout()
1319 // If the task is no longer running on this CPU, leave. in trc_read_check_handler()
1323 // If the task is not in a read-side critical section, and in trc_read_check_handler()
1324 // if this is the last reader, awaken the grace-period kthread. in trc_read_check_handler()
1325 nesting = READ_ONCE(t->trc_reader_nesting); in trc_read_check_handler()
1334 // Get here if the task is in a read-side critical section. in trc_read_check_handler()
1335 // Set its state so that it will update state for the grace-period in trc_read_check_handler()
1340 // Allow future IPIs to be sent on CPU and for task. in trc_read_check_handler()
1344 smp_store_release(&texp->trc_ipi_to_cpu, -1); // ^^^ in trc_read_check_handler()
1347 /* Callback function for scheduler to check locked-down task. */
1351 int cpu = task_cpu(t); in trc_inspect_reader() local
1353 bool ofl = cpu_is_offline(cpu); in trc_inspect_reader()
1358 return -EINVAL; in trc_inspect_reader()
1365 if (!rcu_dynticks_zero_in_eqs(cpu, &t->trc_reader_nesting)) in trc_inspect_reader()
1366 return -EINVAL; // No quiescent state, do it the hard way. in trc_inspect_reader()
1370 // The task is not running, so C-language access is safe. in trc_inspect_reader()
1371 nesting = t->trc_reader_nesting; in trc_inspect_reader()
1377 // If not exiting a read-side critical section, mark as checked in trc_inspect_reader()
1378 // so that the grace-period kthread will remove it from the in trc_inspect_reader()
1385 return -EINVAL; // Reader transitioning, try again later. in trc_inspect_reader()
1387 // The task is in a read-side critical section, so set up its in trc_inspect_reader()
1399 int cpu; in trc_wait_for_one_reader() local
1402 if (smp_load_acquire(&t->trc_ipi_to_cpu) != -1) // Order IPI in trc_wait_for_one_reader()
1408 WARN_ON_ONCE(READ_ONCE(t->trc_reader_nesting)); in trc_wait_for_one_reader()
1421 // an RCU read-side critical section. Otherwise, the invocation of in trc_wait_for_one_reader()
1431 cpu = task_cpu(t); in trc_wait_for_one_reader()
1434 if (per_cpu(trc_ipi_to_cpu, cpu) || t->trc_ipi_to_cpu >= 0) in trc_wait_for_one_reader()
1437 per_cpu(trc_ipi_to_cpu, cpu) = true; in trc_wait_for_one_reader()
1438 t->trc_ipi_to_cpu = cpu; in trc_wait_for_one_reader()
1440 if (smp_call_function_single(cpu, trc_read_check_handler, t, 0)) { in trc_wait_for_one_reader()
1442 // failure than the target CPU being offline. in trc_wait_for_one_reader()
1443 WARN_ONCE(1, "%s(): smp_call_function_single() failed for CPU: %d\n", in trc_wait_for_one_reader()
1444 __func__, cpu); in trc_wait_for_one_reader()
1446 per_cpu(trc_ipi_to_cpu, cpu) = false; in trc_wait_for_one_reader()
1447 t->trc_ipi_to_cpu = -1; in trc_wait_for_one_reader()
1453 * Initialize for first-round processing for the specified task.
1458 // During early boot when there is only the one boot CPU, there in rcu_tasks_trace_pertask_prep()
1459 // is no idle task for the other CPUs. Also, the grace-period in rcu_tasks_trace_pertask_prep()
1462 if (unlikely(t == NULL) || (t == current && notself) || !list_empty(&t->trc_holdout_list)) in rcu_tasks_trace_pertask_prep()
1466 t->trc_ipi_to_cpu = -1; in rcu_tasks_trace_pertask_prep()
1470 /* Do first-round processing for the specified task. */
1477 /* Initialize for a new RCU-tasks-trace grace period. */
1481 int cpu; in rcu_tasks_trace_pregp_step() local
1487 for_each_possible_cpu(cpu) in rcu_tasks_trace_pregp_step()
1488 WARN_ON_ONCE(per_cpu(trc_ipi_to_cpu, cpu)); in rcu_tasks_trace_pregp_step()
1490 // Disable CPU hotplug across the CPU scan for the benefit of in rcu_tasks_trace_pregp_step()
1492 // in CPU-hotplug code paths. in rcu_tasks_trace_pregp_step()
1497 for_each_online_cpu(cpu) { in rcu_tasks_trace_pregp_step()
1499 t = cpu_curr_snapshot(cpu); in rcu_tasks_trace_pregp_step()
1508 // current RCU tasks trace read-side critical section. in rcu_tasks_trace_pregp_step()
1509 for_each_possible_cpu(cpu) { in rcu_tasks_trace_pregp_step()
1510 rtpcp = per_cpu_ptr(rcu_tasks_trace.rtpcpu, cpu); in rcu_tasks_trace_pregp_step()
1512 list_splice_init(&rtpcp->rtp_blkd_tasks, &blkd_tasks); in rcu_tasks_trace_pregp_step()
1516 list_del_init(&t->trc_blkd_node); in rcu_tasks_trace_pregp_step()
1517 list_add(&t->trc_blkd_node, &rtpcp->rtp_blkd_tasks); in rcu_tasks_trace_pregp_step()
1527 // Re-enable CPU hotplug now that the holdout list is populated. in rcu_tasks_trace_pregp_step()
1536 // Wait for late-stage exiting tasks to finish exiting. in rcu_tasks_trace_postscan()
1540 // TRC_NEED_QS_CHECKED in ->trc_reader_special.b.need_qs. in rcu_tasks_trace_postscan()
1556 trc_rdrp->nesting = READ_ONCE(t->trc_reader_nesting); in trc_check_slow_task()
1557 trc_rdrp->ipi_to_cpu = READ_ONCE(t->trc_ipi_to_cpu); in trc_check_slow_task()
1558 trc_rdrp->needqs = rcu_ld_need_qs(t); in trc_check_slow_task()
1565 int cpu; in show_stalled_task_trace() local
1573 cpu = task_cpu(t); in show_stalled_task_trace()
1576 t->pid, in show_stalled_task_trace()
1577 ".I"[t->trc_ipi_to_cpu >= 0], in show_stalled_task_trace()
1580 pr_alert("P%d: %c%c%c%c nesting: %d%c%c cpu: %d%s\n", in show_stalled_task_trace()
1581 t->pid, in show_stalled_task_trace()
1584 ".N"[cpu >= 0 && tick_nohz_full_cpu(cpu)], in show_stalled_task_trace()
1585 ".B"[!!data_race(t->trc_reader_special.b.blocked)], in show_stalled_task_trace()
1589 cpu, cpu_online(cpu) ? "" : "(offline)"); in show_stalled_task_trace()
1596 int cpu; in show_stalled_ipi_trace() local
1598 for_each_possible_cpu(cpu) in show_stalled_ipi_trace()
1599 if (per_cpu(trc_ipi_to_cpu, cpu)) in show_stalled_ipi_trace()
1600 pr_alert("\tIPI outstanding to CPU %d\n", cpu); in show_stalled_ipi_trace()
1609 // Disable CPU hotplug across the holdout list scan for IPIs. in check_all_holdout_tasks_trace()
1614 if (READ_ONCE(t->trc_ipi_to_cpu) == -1 && in check_all_holdout_tasks_trace()
1619 if (smp_load_acquire(&t->trc_ipi_to_cpu) == -1 && in check_all_holdout_tasks_trace()
1627 // Re-enable CPU hotplug now that the holdout list scan has completed. in check_all_holdout_tasks_trace()
1644 int cpu; in rcu_tasks_trace_postgp() local
1647 // if a CPU has gone offline or transitioned to userspace in the in rcu_tasks_trace_postgp()
1651 for_each_online_cpu(cpu) in rcu_tasks_trace_postgp()
1652 if (WARN_ON_ONCE(smp_load_acquire(per_cpu_ptr(&trc_ipi_to_cpu, cpu)))) in rcu_tasks_trace_postgp()
1653 smp_call_function_single(cpu, rcu_tasks_trace_empty_fn, NULL, 1); in rcu_tasks_trace_postgp()
1662 union rcu_special trs = READ_ONCE(t->trc_reader_special); in exit_tasks_rcu_finish_trace()
1665 WARN_ON_ONCE(READ_ONCE(t->trc_reader_nesting)); in exit_tasks_rcu_finish_trace()
1669 WRITE_ONCE(t->trc_reader_nesting, 0); in exit_tasks_rcu_finish_trace()
1673 * call_rcu_tasks_trace() - Queue a callback trace task-based grace period
1677 * The callback function will be invoked some time after a trace rcu-tasks
1679 * trace rcu-tasks read-side critical sections have completed. These
1680 * read-side critical sections are delimited by calls to rcu_read_lock_trace()
1693 * synchronize_rcu_tasks_trace - wait for a trace rcu-tasks grace period
1695 * Control will return to the caller some time after a trace rcu-tasks
1697 * trace rcu-tasks read-side critical sections have elapsed. These read-side
1711 …race_lock_map), "Illegal synchronize_rcu_tasks_trace() in RCU Tasks Trace read-side critical secti… in synchronize_rcu_tasks_trace()
1717 * rcu_barrier_tasks_trace - Wait for in-flight call_rcu_tasks_trace() callbacks.
1809 pr_info("Callback from %s invoked.\n", rttd->name); in test_rcu_tasks_callback()
1811 rttd->notrun = false; in test_rcu_tasks_callback()
1818 pr_info("Running RCU-tasks wait API self tests\n"); in rcu_tasks_initiate_self_tests()
1839 * Return: 0 - test passed
1840 * 1 - test failed, but have not timed out yet
1841 * -1 - test failed and timed out
1854 pr_err("%s has failed boot-time tests.\n", tests[i].name); in rcu_tasks_verify_self_tests()
1855 ret = -1; in rcu_tasks_verify_self_tests()
1908 // Run the self-tests. in rcu_init_tasks_generic()