Lines Matching refs:rsp
162 rcu_report_qs_rnp(unsigned long mask, struct rcu_state *rsp,
168 static void invoke_rcu_callbacks(struct rcu_state *rsp, struct rcu_data *rdp);
169 static void rcu_report_exp_rdp(struct rcu_state *rsp,
220 static int rcu_gp_in_progress(struct rcu_state *rsp) in rcu_gp_in_progress() argument
222 return rcu_seq_state(rcu_seq_current(&rsp->gp_seq)); in rcu_gp_in_progress()
549 static void force_qs_rnp(struct rcu_state *rsp, int (*f)(struct rcu_data *rsp));
550 static void force_quiescent_state(struct rcu_state *rsp);
637 struct rcu_state *rsp; in show_rcu_gp_kthreads() local
639 for_each_rcu_flavor(rsp) { in show_rcu_gp_kthreads()
641 rsp->name, rsp->gp_state, rsp->gp_kthread->state); in show_rcu_gp_kthreads()
642 rcu_for_each_node_breadth_first(rsp, rnp) { in show_rcu_gp_kthreads()
643 if (ULONG_CMP_GE(rsp->gp_seq, rnp->gp_seq_needed)) in show_rcu_gp_kthreads()
651 rdp = per_cpu_ptr(rsp->rda, cpu); in show_rcu_gp_kthreads()
653 ULONG_CMP_GE(rsp->gp_seq, in show_rcu_gp_kthreads()
671 struct rcu_state *rsp = NULL; in rcutorture_get_gp_data() local
675 rsp = rcu_state_p; in rcutorture_get_gp_data()
678 rsp = &rcu_bh_state; in rcutorture_get_gp_data()
681 rsp = &rcu_sched_state; in rcutorture_get_gp_data()
686 if (rsp == NULL) in rcutorture_get_gp_data()
688 *flags = READ_ONCE(rsp->gp_flags); in rcutorture_get_gp_data()
689 *gp_seq = rcu_seq_current(&rsp->gp_seq); in rcutorture_get_gp_data()
696 static struct rcu_node *rcu_get_root(struct rcu_state *rsp) in rcu_get_root() argument
698 return &rsp->node[0]; in rcu_get_root()
711 struct rcu_state *rsp; in rcu_eqs_enter() local
727 for_each_rcu_flavor(rsp) { in rcu_eqs_enter()
728 rdp = this_cpu_ptr(rsp->rda); in rcu_eqs_enter()
1072 struct rcu_state *rsp; in rcu_lockdep_current_cpu_online() local
1077 for_each_rcu_flavor(rsp) { in rcu_lockdep_current_cpu_online()
1078 rdp = this_cpu_ptr(rsp->rda); in rcu_lockdep_current_cpu_online()
1131 trace_rcu_fqs(rdp->rsp->name, rdp->gp_seq, rdp->cpu, TPS("dti")); in dyntick_save_progress_counter()
1181 trace_rcu_fqs(rdp->rsp->name, rdp->gp_seq, rdp->cpu, TPS("dti")); in rcu_implicit_dynticks_qs()
1195 if (time_after(jiffies, rdp->rsp->gp_start + jtsq) && in rcu_implicit_dynticks_qs()
1198 trace_rcu_fqs(rdp->rsp->name, rdp->gp_seq, rdp->cpu, TPS("rqc")); in rcu_implicit_dynticks_qs()
1201 } else if (time_after(jiffies, rdp->rsp->gp_start + jtsq)) { in rcu_implicit_dynticks_qs()
1208 time_after(jiffies, rdp->rsp->gp_start + HZ)) { in rcu_implicit_dynticks_qs()
1246 (time_after(jiffies, rdp->rsp->gp_start + jtsq) || in rcu_implicit_dynticks_qs()
1247 time_after(jiffies, rdp->rsp->jiffies_resched))) { in rcu_implicit_dynticks_qs()
1251 rdp->rsp->jiffies_resched += jtsq; /* Re-enable beating. */ in rcu_implicit_dynticks_qs()
1260 if (jiffies - rdp->rsp->gp_start > rcu_jiffies_till_stall_check() / 2) { in rcu_implicit_dynticks_qs()
1275 static void record_gp_stall_check_time(struct rcu_state *rsp) in record_gp_stall_check_time() argument
1280 rsp->gp_start = j; in record_gp_stall_check_time()
1283 smp_store_release(&rsp->jiffies_stall, j + j1); /* ^^^ */ in record_gp_stall_check_time()
1284 rsp->jiffies_resched = j + j1 / 2; in record_gp_stall_check_time()
1285 rsp->n_force_qs_gpstart = READ_ONCE(rsp->n_force_qs); in record_gp_stall_check_time()
1301 static void rcu_check_gp_kthread_starvation(struct rcu_state *rsp) in rcu_check_gp_kthread_starvation() argument
1307 gpa = READ_ONCE(rsp->gp_activity); in rcu_check_gp_kthread_starvation()
1310 rsp->name, j - gpa, in rcu_check_gp_kthread_starvation()
1311 (long)rcu_seq_current(&rsp->gp_seq), in rcu_check_gp_kthread_starvation()
1312 rsp->gp_flags, in rcu_check_gp_kthread_starvation()
1313 gp_state_getname(rsp->gp_state), rsp->gp_state, in rcu_check_gp_kthread_starvation()
1314 rsp->gp_kthread ? rsp->gp_kthread->state : ~0, in rcu_check_gp_kthread_starvation()
1315 rsp->gp_kthread ? task_cpu(rsp->gp_kthread) : -1); in rcu_check_gp_kthread_starvation()
1316 if (rsp->gp_kthread) { in rcu_check_gp_kthread_starvation()
1318 sched_show_task(rsp->gp_kthread); in rcu_check_gp_kthread_starvation()
1319 wake_up_process(rsp->gp_kthread); in rcu_check_gp_kthread_starvation()
1330 static void rcu_dump_cpu_stacks(struct rcu_state *rsp) in rcu_dump_cpu_stacks() argument
1336 rcu_for_each_leaf_node(rsp, rnp) { in rcu_dump_cpu_stacks()
1350 static void rcu_stall_kick_kthreads(struct rcu_state *rsp) in rcu_stall_kick_kthreads() argument
1356 j = READ_ONCE(rsp->jiffies_kick_kthreads); in rcu_stall_kick_kthreads()
1357 if (time_after(jiffies, j) && rsp->gp_kthread && in rcu_stall_kick_kthreads()
1358 (rcu_gp_in_progress(rsp) || READ_ONCE(rsp->gp_flags))) { in rcu_stall_kick_kthreads()
1359 WARN_ONCE(1, "Kicking %s grace-period kthread\n", rsp->name); in rcu_stall_kick_kthreads()
1361 wake_up_process(rsp->gp_kthread); in rcu_stall_kick_kthreads()
1362 WRITE_ONCE(rsp->jiffies_kick_kthreads, j + HZ); in rcu_stall_kick_kthreads()
1372 static void print_other_cpu_stall(struct rcu_state *rsp, unsigned long gp_seq) in print_other_cpu_stall() argument
1379 struct rcu_node *rnp = rcu_get_root(rsp); in print_other_cpu_stall()
1383 rcu_stall_kick_kthreads(rsp); in print_other_cpu_stall()
1392 pr_err("INFO: %s detected stalls on CPUs/tasks:", rsp->name); in print_other_cpu_stall()
1394 rcu_for_each_leaf_node(rsp, rnp) { in print_other_cpu_stall()
1400 print_cpu_stall_info(rsp, cpu); in print_other_cpu_stall()
1409 totqlen += rcu_segcblist_n_cbs(&per_cpu_ptr(rsp->rda, in print_other_cpu_stall()
1412 smp_processor_id(), (long)(jiffies - rsp->gp_start), in print_other_cpu_stall()
1413 (long)rcu_seq_current(&rsp->gp_seq), totqlen); in print_other_cpu_stall()
1415 rcu_dump_cpu_stacks(rsp); in print_other_cpu_stall()
1418 rcu_print_detail_task_stall(rsp); in print_other_cpu_stall()
1420 if (rcu_seq_current(&rsp->gp_seq) != gp_seq) { in print_other_cpu_stall()
1424 gpa = READ_ONCE(rsp->gp_activity); in print_other_cpu_stall()
1426 rsp->name, j - gpa, j, gpa, in print_other_cpu_stall()
1428 rcu_get_root(rsp)->qsmask); in print_other_cpu_stall()
1434 if (ULONG_CMP_GE(jiffies, READ_ONCE(rsp->jiffies_stall))) in print_other_cpu_stall()
1435 WRITE_ONCE(rsp->jiffies_stall, in print_other_cpu_stall()
1438 rcu_check_gp_kthread_starvation(rsp); in print_other_cpu_stall()
1442 force_quiescent_state(rsp); /* Kick them all. */ in print_other_cpu_stall()
1445 static void print_cpu_stall(struct rcu_state *rsp) in print_cpu_stall() argument
1449 struct rcu_data *rdp = this_cpu_ptr(rsp->rda); in print_cpu_stall()
1450 struct rcu_node *rnp = rcu_get_root(rsp); in print_cpu_stall()
1454 rcu_stall_kick_kthreads(rsp); in print_cpu_stall()
1463 pr_err("INFO: %s self-detected stall on CPU", rsp->name); in print_cpu_stall()
1466 print_cpu_stall_info(rsp, smp_processor_id()); in print_cpu_stall()
1470 totqlen += rcu_segcblist_n_cbs(&per_cpu_ptr(rsp->rda, in print_cpu_stall()
1473 jiffies - rsp->gp_start, in print_cpu_stall()
1474 (long)rcu_seq_current(&rsp->gp_seq), totqlen); in print_cpu_stall()
1476 rcu_check_gp_kthread_starvation(rsp); in print_cpu_stall()
1478 rcu_dump_cpu_stacks(rsp); in print_cpu_stall()
1482 if (ULONG_CMP_GE(jiffies, READ_ONCE(rsp->jiffies_stall))) in print_cpu_stall()
1483 WRITE_ONCE(rsp->jiffies_stall, in print_cpu_stall()
1499 static void check_cpu_stall(struct rcu_state *rsp, struct rcu_data *rdp) in check_cpu_stall() argument
1510 !rcu_gp_in_progress(rsp)) in check_cpu_stall()
1512 rcu_stall_kick_kthreads(rsp); in check_cpu_stall()
1532 gs1 = READ_ONCE(rsp->gp_seq); in check_cpu_stall()
1534 js = READ_ONCE(rsp->jiffies_stall); in check_cpu_stall()
1536 gps = READ_ONCE(rsp->gp_start); in check_cpu_stall()
1538 gs2 = READ_ONCE(rsp->gp_seq); in check_cpu_stall()
1545 if (rcu_gp_in_progress(rsp) && in check_cpu_stall()
1547 cmpxchg(&rsp->jiffies_stall, js, jn) == js) { in check_cpu_stall()
1550 print_cpu_stall(rsp); in check_cpu_stall()
1552 } else if (rcu_gp_in_progress(rsp) && in check_cpu_stall()
1554 cmpxchg(&rsp->jiffies_stall, js, jn) == js) { in check_cpu_stall()
1557 print_other_cpu_stall(rsp, gs2); in check_cpu_stall()
1572 struct rcu_state *rsp; in rcu_cpu_stall_reset() local
1574 for_each_rcu_flavor(rsp) in rcu_cpu_stall_reset()
1575 WRITE_ONCE(rsp->jiffies_stall, jiffies + ULONG_MAX / 2); in rcu_cpu_stall_reset()
1582 trace_rcu_future_grace_period(rdp->rsp->name, rnp->gp_seq, gp_seq_req, in trace_rcu_this_gp()
1606 struct rcu_state *rsp = rdp->rsp; in rcu_start_this_gp() local
1650 if (rcu_gp_in_progress(rsp)) { in rcu_start_this_gp()
1655 WRITE_ONCE(rsp->gp_flags, rsp->gp_flags | RCU_GP_FLAG_INIT); in rcu_start_this_gp()
1656 rsp->gp_req_activity = jiffies; in rcu_start_this_gp()
1657 if (!rsp->gp_kthread) { in rcu_start_this_gp()
1661 trace_rcu_grace_period(rsp->name, READ_ONCE(rsp->gp_seq), TPS("newreq")); in rcu_start_this_gp()
1678 static bool rcu_future_gp_cleanup(struct rcu_state *rsp, struct rcu_node *rnp) in rcu_future_gp_cleanup() argument
1681 struct rcu_data *rdp = this_cpu_ptr(rsp->rda); in rcu_future_gp_cleanup()
1698 static void rcu_gp_kthread_wake(struct rcu_state *rsp) in rcu_gp_kthread_wake() argument
1700 if (current == rsp->gp_kthread || in rcu_gp_kthread_wake()
1701 !READ_ONCE(rsp->gp_flags) || in rcu_gp_kthread_wake()
1702 !rsp->gp_kthread) in rcu_gp_kthread_wake()
1704 swake_up_one(&rsp->gp_wq); in rcu_gp_kthread_wake()
1719 static bool rcu_accelerate_cbs(struct rcu_state *rsp, struct rcu_node *rnp, in rcu_accelerate_cbs() argument
1741 gp_seq_req = rcu_seq_snap(&rsp->gp_seq); in rcu_accelerate_cbs()
1747 trace_rcu_grace_period(rsp->name, rdp->gp_seq, TPS("AccWaitCB")); in rcu_accelerate_cbs()
1749 trace_rcu_grace_period(rsp->name, rdp->gp_seq, TPS("AccReadyCB")); in rcu_accelerate_cbs()
1760 static void rcu_accelerate_cbs_unlocked(struct rcu_state *rsp, in rcu_accelerate_cbs_unlocked() argument
1768 c = rcu_seq_snap(&rsp->gp_seq); in rcu_accelerate_cbs_unlocked()
1775 needwake = rcu_accelerate_cbs(rsp, rnp, rdp); in rcu_accelerate_cbs_unlocked()
1778 rcu_gp_kthread_wake(rsp); in rcu_accelerate_cbs_unlocked()
1791 static bool rcu_advance_cbs(struct rcu_state *rsp, struct rcu_node *rnp, in rcu_advance_cbs() argument
1807 return rcu_accelerate_cbs(rsp, rnp, rdp); in rcu_advance_cbs()
1816 static bool __note_gp_changes(struct rcu_state *rsp, struct rcu_node *rnp, in __note_gp_changes() argument
1830 ret = rcu_advance_cbs(rsp, rnp, rdp); /* Advance callbacks. */ in __note_gp_changes()
1831 trace_rcu_grace_period(rsp->name, rdp->gp_seq, TPS("cpuend")); in __note_gp_changes()
1833 ret = rcu_accelerate_cbs(rsp, rnp, rdp); /* Recent callbacks. */ in __note_gp_changes()
1844 trace_rcu_grace_period(rsp->name, rnp->gp_seq, TPS("cpustart")); in __note_gp_changes()
1859 static void note_gp_changes(struct rcu_state *rsp, struct rcu_data *rdp) in note_gp_changes() argument
1873 needwake = __note_gp_changes(rsp, rnp, rdp); in note_gp_changes()
1876 rcu_gp_kthread_wake(rsp); in note_gp_changes()
1879 static void rcu_gp_slow(struct rcu_state *rsp, int delay) in rcu_gp_slow() argument
1882 !(rcu_seq_ctr(rsp->gp_seq) % in rcu_gp_slow()
1890 static bool rcu_gp_init(struct rcu_state *rsp) in rcu_gp_init() argument
1896 struct rcu_node *rnp = rcu_get_root(rsp); in rcu_gp_init()
1898 WRITE_ONCE(rsp->gp_activity, jiffies); in rcu_gp_init()
1900 if (!READ_ONCE(rsp->gp_flags)) { in rcu_gp_init()
1905 WRITE_ONCE(rsp->gp_flags, 0); /* Clear all flags: New grace period. */ in rcu_gp_init()
1907 if (WARN_ON_ONCE(rcu_gp_in_progress(rsp))) { in rcu_gp_init()
1917 record_gp_stall_check_time(rsp); in rcu_gp_init()
1919 rcu_seq_start(&rsp->gp_seq); in rcu_gp_init()
1920 trace_rcu_grace_period(rsp->name, rsp->gp_seq, TPS("start")); in rcu_gp_init()
1929 rsp->gp_state = RCU_GP_ONOFF; in rcu_gp_init()
1930 rcu_for_each_leaf_node(rsp, rnp) { in rcu_gp_init()
1931 spin_lock(&rsp->ofl_lock); in rcu_gp_init()
1937 spin_unlock(&rsp->ofl_lock); in rcu_gp_init()
1973 spin_unlock(&rsp->ofl_lock); in rcu_gp_init()
1975 rcu_gp_slow(rsp, gp_preinit_delay); /* Races with CPU hotplug. */ in rcu_gp_init()
1989 rsp->gp_state = RCU_GP_INIT; in rcu_gp_init()
1990 rcu_for_each_node_breadth_first(rsp, rnp) { in rcu_gp_init()
1991 rcu_gp_slow(rsp, gp_init_delay); in rcu_gp_init()
1993 rdp = this_cpu_ptr(rsp->rda); in rcu_gp_init()
1994 rcu_preempt_check_blocked_tasks(rsp, rnp); in rcu_gp_init()
1996 WRITE_ONCE(rnp->gp_seq, rsp->gp_seq); in rcu_gp_init()
1998 (void)__note_gp_changes(rsp, rnp, rdp); in rcu_gp_init()
2000 trace_rcu_grace_period_init(rsp->name, rnp->gp_seq, in rcu_gp_init()
2007 rcu_report_qs_rnp(mask, rsp, rnp, rnp->gp_seq, flags); in rcu_gp_init()
2011 WRITE_ONCE(rsp->gp_activity, jiffies); in rcu_gp_init()
2021 static bool rcu_gp_fqs_check_wake(struct rcu_state *rsp, int *gfp) in rcu_gp_fqs_check_wake() argument
2023 struct rcu_node *rnp = rcu_get_root(rsp); in rcu_gp_fqs_check_wake()
2026 *gfp = READ_ONCE(rsp->gp_flags); in rcu_gp_fqs_check_wake()
2040 static void rcu_gp_fqs(struct rcu_state *rsp, bool first_time) in rcu_gp_fqs() argument
2042 struct rcu_node *rnp = rcu_get_root(rsp); in rcu_gp_fqs()
2044 WRITE_ONCE(rsp->gp_activity, jiffies); in rcu_gp_fqs()
2045 rsp->n_force_qs++; in rcu_gp_fqs()
2048 force_qs_rnp(rsp, dyntick_save_progress_counter); in rcu_gp_fqs()
2051 force_qs_rnp(rsp, rcu_implicit_dynticks_qs); in rcu_gp_fqs()
2054 if (READ_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) { in rcu_gp_fqs()
2056 WRITE_ONCE(rsp->gp_flags, in rcu_gp_fqs()
2057 READ_ONCE(rsp->gp_flags) & ~RCU_GP_FLAG_FQS); in rcu_gp_fqs()
2065 static void rcu_gp_cleanup(struct rcu_state *rsp) in rcu_gp_cleanup() argument
2071 struct rcu_node *rnp = rcu_get_root(rsp); in rcu_gp_cleanup()
2074 WRITE_ONCE(rsp->gp_activity, jiffies); in rcu_gp_cleanup()
2076 gp_duration = jiffies - rsp->gp_start; in rcu_gp_cleanup()
2077 if (gp_duration > rsp->gp_max) in rcu_gp_cleanup()
2078 rsp->gp_max = gp_duration; in rcu_gp_cleanup()
2099 new_gp_seq = rsp->gp_seq; in rcu_gp_cleanup()
2101 rcu_for_each_node_breadth_first(rsp, rnp) { in rcu_gp_cleanup()
2104 dump_blkd_tasks(rsp, rnp, 10); in rcu_gp_cleanup()
2107 rdp = this_cpu_ptr(rsp->rda); in rcu_gp_cleanup()
2109 needgp = __note_gp_changes(rsp, rnp, rdp) || needgp; in rcu_gp_cleanup()
2111 needgp = rcu_future_gp_cleanup(rsp, rnp) || needgp; in rcu_gp_cleanup()
2116 WRITE_ONCE(rsp->gp_activity, jiffies); in rcu_gp_cleanup()
2117 rcu_gp_slow(rsp, gp_cleanup_delay); in rcu_gp_cleanup()
2119 rnp = rcu_get_root(rsp); in rcu_gp_cleanup()
2123 rcu_seq_end(&rsp->gp_seq); in rcu_gp_cleanup()
2124 trace_rcu_grace_period(rsp->name, rsp->gp_seq, TPS("end")); in rcu_gp_cleanup()
2125 rsp->gp_state = RCU_GP_IDLE; in rcu_gp_cleanup()
2127 rdp = this_cpu_ptr(rsp->rda); in rcu_gp_cleanup()
2134 if (!rcu_accelerate_cbs(rsp, rnp, rdp) && needgp) { in rcu_gp_cleanup()
2135 WRITE_ONCE(rsp->gp_flags, RCU_GP_FLAG_INIT); in rcu_gp_cleanup()
2136 rsp->gp_req_activity = jiffies; in rcu_gp_cleanup()
2137 trace_rcu_grace_period(rsp->name, READ_ONCE(rsp->gp_seq), in rcu_gp_cleanup()
2140 WRITE_ONCE(rsp->gp_flags, rsp->gp_flags & RCU_GP_FLAG_INIT); in rcu_gp_cleanup()
2154 struct rcu_state *rsp = arg; in rcu_gp_kthread() local
2155 struct rcu_node *rnp = rcu_get_root(rsp); in rcu_gp_kthread()
2162 trace_rcu_grace_period(rsp->name, in rcu_gp_kthread()
2163 READ_ONCE(rsp->gp_seq), in rcu_gp_kthread()
2165 rsp->gp_state = RCU_GP_WAIT_GPS; in rcu_gp_kthread()
2166 swait_event_idle_exclusive(rsp->gp_wq, READ_ONCE(rsp->gp_flags) & in rcu_gp_kthread()
2168 rsp->gp_state = RCU_GP_DONE_GPS; in rcu_gp_kthread()
2170 if (rcu_gp_init(rsp)) in rcu_gp_kthread()
2173 WRITE_ONCE(rsp->gp_activity, jiffies); in rcu_gp_kthread()
2175 trace_rcu_grace_period(rsp->name, in rcu_gp_kthread()
2176 READ_ONCE(rsp->gp_seq), in rcu_gp_kthread()
2186 rsp->jiffies_force_qs = jiffies + j; in rcu_gp_kthread()
2187 WRITE_ONCE(rsp->jiffies_kick_kthreads, in rcu_gp_kthread()
2190 trace_rcu_grace_period(rsp->name, in rcu_gp_kthread()
2191 READ_ONCE(rsp->gp_seq), in rcu_gp_kthread()
2193 rsp->gp_state = RCU_GP_WAIT_FQS; in rcu_gp_kthread()
2194 ret = swait_event_idle_timeout_exclusive(rsp->gp_wq, in rcu_gp_kthread()
2195 rcu_gp_fqs_check_wake(rsp, &gf), j); in rcu_gp_kthread()
2196 rsp->gp_state = RCU_GP_DOING_FQS; in rcu_gp_kthread()
2203 if (ULONG_CMP_GE(jiffies, rsp->jiffies_force_qs) || in rcu_gp_kthread()
2205 trace_rcu_grace_period(rsp->name, in rcu_gp_kthread()
2206 READ_ONCE(rsp->gp_seq), in rcu_gp_kthread()
2208 rcu_gp_fqs(rsp, first_gp_fqs); in rcu_gp_kthread()
2210 trace_rcu_grace_period(rsp->name, in rcu_gp_kthread()
2211 READ_ONCE(rsp->gp_seq), in rcu_gp_kthread()
2214 WRITE_ONCE(rsp->gp_activity, jiffies); in rcu_gp_kthread()
2220 WRITE_ONCE(rsp->gp_activity, jiffies); in rcu_gp_kthread()
2222 trace_rcu_grace_period(rsp->name, in rcu_gp_kthread()
2223 READ_ONCE(rsp->gp_seq), in rcu_gp_kthread()
2227 if (time_after(jiffies, rsp->jiffies_force_qs)) in rcu_gp_kthread()
2230 j = rsp->jiffies_force_qs - j; in rcu_gp_kthread()
2235 rsp->gp_state = RCU_GP_CLEANUP; in rcu_gp_kthread()
2236 rcu_gp_cleanup(rsp); in rcu_gp_kthread()
2237 rsp->gp_state = RCU_GP_CLEANED; in rcu_gp_kthread()
2250 static void rcu_report_qs_rsp(struct rcu_state *rsp, unsigned long flags) in rcu_report_qs_rsp() argument
2251 __releases(rcu_get_root(rsp)->lock) in rcu_report_qs_rsp()
2253 raw_lockdep_assert_held_rcu_node(rcu_get_root(rsp)); in rcu_report_qs_rsp()
2254 WARN_ON_ONCE(!rcu_gp_in_progress(rsp)); in rcu_report_qs_rsp()
2255 WRITE_ONCE(rsp->gp_flags, READ_ONCE(rsp->gp_flags) | RCU_GP_FLAG_FQS); in rcu_report_qs_rsp()
2256 raw_spin_unlock_irqrestore_rcu_node(rcu_get_root(rsp), flags); in rcu_report_qs_rsp()
2257 rcu_gp_kthread_wake(rsp); in rcu_report_qs_rsp()
2275 rcu_report_qs_rnp(unsigned long mask, struct rcu_state *rsp, in rcu_report_qs_rnp() argument
2299 trace_rcu_quiescent_state_report(rsp->name, rnp->gp_seq, in rcu_report_qs_rnp()
2329 rcu_report_qs_rsp(rsp, flags); /* releases rnp->lock. */ in rcu_report_qs_rnp()
2340 rcu_report_unblock_qs_rnp(struct rcu_state *rsp, in rcu_report_unblock_qs_rnp() argument
2350 WARN_ON_ONCE(rsp != rcu_state_p) || in rcu_report_unblock_qs_rnp()
2364 rcu_report_qs_rsp(rsp, flags); in rcu_report_unblock_qs_rnp()
2373 rcu_report_qs_rnp(mask, rsp, rnp_p, gps, flags); in rcu_report_unblock_qs_rnp()
2381 rcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct rcu_data *rdp) in rcu_report_qs_rdp() argument
2414 needwake = rcu_accelerate_cbs(rsp, rnp, rdp); in rcu_report_qs_rdp()
2416 rcu_report_qs_rnp(mask, rsp, rnp, rnp->gp_seq, flags); in rcu_report_qs_rdp()
2419 rcu_gp_kthread_wake(rsp); in rcu_report_qs_rdp()
2430 rcu_check_quiescent_state(struct rcu_state *rsp, struct rcu_data *rdp) in rcu_check_quiescent_state() argument
2433 note_gp_changes(rsp, rdp); in rcu_check_quiescent_state()
2453 rcu_report_qs_rdp(rdp->cpu, rsp, rdp); in rcu_check_quiescent_state()
2459 static void rcu_cleanup_dying_cpu(struct rcu_state *rsp) in rcu_cleanup_dying_cpu() argument
2462 RCU_TRACE(struct rcu_data *rdp = this_cpu_ptr(rsp->rda);) in rcu_cleanup_dying_cpu()
2469 trace_rcu_grace_period(rsp->name, rnp->gp_seq, in rcu_cleanup_dying_cpu()
2524 static void rcu_cleanup_dead_cpu(int cpu, struct rcu_state *rsp) in rcu_cleanup_dead_cpu() argument
2526 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu); in rcu_cleanup_dead_cpu()
2540 static void rcu_do_batch(struct rcu_state *rsp, struct rcu_data *rdp) in rcu_do_batch() argument
2549 trace_rcu_batch_start(rsp->name, in rcu_do_batch()
2552 trace_rcu_batch_end(rsp->name, 0, in rcu_do_batch()
2567 trace_rcu_batch_start(rsp->name, rcu_segcblist_n_lazy_cbs(&rdp->cblist), in rcu_do_batch()
2576 if (__rcu_reclaim(rsp->name, rhp)) in rcu_do_batch()
2590 trace_rcu_batch_end(rsp->name, count, !!rcl.head, need_resched(), in rcu_do_batch()
2606 rdp->n_force_qs_snap = rsp->n_force_qs; in rcu_do_batch()
2678 static void force_qs_rnp(struct rcu_state *rsp, int (*f)(struct rcu_data *rsp)) in force_qs_rnp() argument
2685 rcu_for_each_leaf_node(rsp, rnp) { in force_qs_rnp()
2691 rsp != rcu_state_p || in force_qs_rnp()
2708 if (f(per_cpu_ptr(rsp->rda, cpu))) in force_qs_rnp()
2714 rcu_report_qs_rnp(mask, rsp, rnp, rnp->gp_seq, flags); in force_qs_rnp()
2726 static void force_quiescent_state(struct rcu_state *rsp) in force_quiescent_state() argument
2734 rnp = __this_cpu_read(rsp->rda->mynode); in force_quiescent_state()
2736 ret = (READ_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) || in force_quiescent_state()
2749 if (READ_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) { in force_quiescent_state()
2753 WRITE_ONCE(rsp->gp_flags, READ_ONCE(rsp->gp_flags) | RCU_GP_FLAG_FQS); in force_quiescent_state()
2755 rcu_gp_kthread_wake(rsp); in force_quiescent_state()
2763 rcu_check_gp_start_stall(struct rcu_state *rsp, struct rcu_node *rnp, in rcu_check_gp_start_stall() argument
2769 struct rcu_node *rnp_root = rcu_get_root(rsp); in rcu_check_gp_start_stall()
2772 if (!IS_ENABLED(CONFIG_PROVE_RCU) || rcu_gp_in_progress(rsp) || in rcu_check_gp_start_stall()
2776 if (time_before(j, READ_ONCE(rsp->gp_req_activity) + gpssdelay) || in rcu_check_gp_start_stall()
2777 time_before(j, READ_ONCE(rsp->gp_activity) + gpssdelay) || in rcu_check_gp_start_stall()
2783 if (rcu_gp_in_progress(rsp) || in rcu_check_gp_start_stall()
2785 time_before(j, READ_ONCE(rsp->gp_req_activity) + gpssdelay) || in rcu_check_gp_start_stall()
2786 time_before(j, READ_ONCE(rsp->gp_activity) + gpssdelay) || in rcu_check_gp_start_stall()
2796 if (rcu_gp_in_progress(rsp) || in rcu_check_gp_start_stall()
2798 time_before(j, rsp->gp_req_activity + gpssdelay) || in rcu_check_gp_start_stall()
2799 time_before(j, rsp->gp_activity + gpssdelay) || in rcu_check_gp_start_stall()
2806 __func__, (long)READ_ONCE(rsp->gp_seq), in rcu_check_gp_start_stall()
2808 j - rsp->gp_req_activity, j - rsp->gp_activity, in rcu_check_gp_start_stall()
2809 rsp->gp_flags, rsp->gp_state, rsp->name, in rcu_check_gp_start_stall()
2810 rsp->gp_kthread ? rsp->gp_kthread->state : 0x1ffffL); in rcu_check_gp_start_stall()
2823 __rcu_process_callbacks(struct rcu_state *rsp) in __rcu_process_callbacks() argument
2826 struct rcu_data *rdp = raw_cpu_ptr(rsp->rda); in __rcu_process_callbacks()
2832 rcu_check_quiescent_state(rsp, rdp); in __rcu_process_callbacks()
2835 if (!rcu_gp_in_progress(rsp) && in __rcu_process_callbacks()
2839 rcu_accelerate_cbs_unlocked(rsp, rnp, rdp); in __rcu_process_callbacks()
2843 rcu_check_gp_start_stall(rsp, rnp, rdp); in __rcu_process_callbacks()
2847 invoke_rcu_callbacks(rsp, rdp); in __rcu_process_callbacks()
2858 struct rcu_state *rsp; in rcu_process_callbacks() local
2863 for_each_rcu_flavor(rsp) in rcu_process_callbacks()
2864 __rcu_process_callbacks(rsp); in rcu_process_callbacks()
2875 static void invoke_rcu_callbacks(struct rcu_state *rsp, struct rcu_data *rdp) in invoke_rcu_callbacks() argument
2879 if (likely(!rsp->boost)) { in invoke_rcu_callbacks()
2880 rcu_do_batch(rsp, rdp); in invoke_rcu_callbacks()
2895 static void __call_rcu_core(struct rcu_state *rsp, struct rcu_data *rdp, in __call_rcu_core() argument
2920 note_gp_changes(rsp, rdp); in __call_rcu_core()
2923 if (!rcu_gp_in_progress(rsp)) { in __call_rcu_core()
2924 rcu_accelerate_cbs_unlocked(rsp, rdp->mynode, rdp); in __call_rcu_core()
2928 if (rsp->n_force_qs == rdp->n_force_qs_snap && in __call_rcu_core()
2930 force_quiescent_state(rsp); in __call_rcu_core()
2931 rdp->n_force_qs_snap = rsp->n_force_qs; in __call_rcu_core()
2952 struct rcu_state *rsp, int cpu, bool lazy) in __call_rcu() argument
2974 rdp = this_cpu_ptr(rsp->rda); in __call_rcu()
2981 rdp = per_cpu_ptr(rsp->rda, cpu); in __call_rcu()
3004 trace_rcu_kfree_callback(rsp->name, head, (unsigned long)func, in __call_rcu()
3008 trace_rcu_callback(rsp->name, head, in __call_rcu()
3013 __call_rcu_core(rsp, rdp, head, flags); in __call_rcu()
3271 static int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp) in __rcu_pending() argument
3276 check_cpu_stall(rsp, rdp); in __rcu_pending()
3279 if (rcu_nohz_full_cpu(rsp)) in __rcu_pending()
3291 if (!rcu_gp_in_progress(rsp) && in __rcu_pending()
3316 struct rcu_state *rsp; in rcu_pending() local
3318 for_each_rcu_flavor(rsp) in rcu_pending()
3319 if (__rcu_pending(rsp, this_cpu_ptr(rsp->rda))) in rcu_pending()
3334 struct rcu_state *rsp; in rcu_cpu_has_callbacks() local
3336 for_each_rcu_flavor(rsp) { in rcu_cpu_has_callbacks()
3337 rdp = this_cpu_ptr(rsp->rda); in rcu_cpu_has_callbacks()
3355 static void _rcu_barrier_trace(struct rcu_state *rsp, const char *s, in _rcu_barrier_trace() argument
3358 trace_rcu_barrier(rsp->name, s, cpu, in _rcu_barrier_trace()
3359 atomic_read(&rsp->barrier_cpu_count), done); in _rcu_barrier_trace()
3369 struct rcu_state *rsp = rdp->rsp; in rcu_barrier_callback() local
3371 if (atomic_dec_and_test(&rsp->barrier_cpu_count)) { in rcu_barrier_callback()
3372 _rcu_barrier_trace(rsp, TPS("LastCB"), -1, in rcu_barrier_callback()
3373 rsp->barrier_sequence); in rcu_barrier_callback()
3374 complete(&rsp->barrier_completion); in rcu_barrier_callback()
3376 _rcu_barrier_trace(rsp, TPS("CB"), -1, rsp->barrier_sequence); in rcu_barrier_callback()
3385 struct rcu_state *rsp = type; in rcu_barrier_func() local
3386 struct rcu_data *rdp = raw_cpu_ptr(rsp->rda); in rcu_barrier_func()
3388 _rcu_barrier_trace(rsp, TPS("IRQ"), -1, rsp->barrier_sequence); in rcu_barrier_func()
3392 atomic_inc(&rsp->barrier_cpu_count); in rcu_barrier_func()
3395 _rcu_barrier_trace(rsp, TPS("IRQNQ"), -1, in rcu_barrier_func()
3396 rsp->barrier_sequence); in rcu_barrier_func()
3404 static void _rcu_barrier(struct rcu_state *rsp) in _rcu_barrier() argument
3408 unsigned long s = rcu_seq_snap(&rsp->barrier_sequence); in _rcu_barrier()
3410 _rcu_barrier_trace(rsp, TPS("Begin"), -1, s); in _rcu_barrier()
3413 mutex_lock(&rsp->barrier_mutex); in _rcu_barrier()
3416 if (rcu_seq_done(&rsp->barrier_sequence, s)) { in _rcu_barrier()
3417 _rcu_barrier_trace(rsp, TPS("EarlyExit"), -1, in _rcu_barrier()
3418 rsp->barrier_sequence); in _rcu_barrier()
3420 mutex_unlock(&rsp->barrier_mutex); in _rcu_barrier()
3425 rcu_seq_start(&rsp->barrier_sequence); in _rcu_barrier()
3426 _rcu_barrier_trace(rsp, TPS("Inc1"), -1, rsp->barrier_sequence); in _rcu_barrier()
3434 init_completion(&rsp->barrier_completion); in _rcu_barrier()
3435 atomic_set(&rsp->barrier_cpu_count, 1); in _rcu_barrier()
3446 rdp = per_cpu_ptr(rsp->rda, cpu); in _rcu_barrier()
3448 if (!rcu_nocb_cpu_needs_barrier(rsp, cpu)) { in _rcu_barrier()
3449 _rcu_barrier_trace(rsp, TPS("OfflineNoCB"), cpu, in _rcu_barrier()
3450 rsp->barrier_sequence); in _rcu_barrier()
3452 _rcu_barrier_trace(rsp, TPS("OnlineNoCB"), cpu, in _rcu_barrier()
3453 rsp->barrier_sequence); in _rcu_barrier()
3455 atomic_inc(&rsp->barrier_cpu_count); in _rcu_barrier()
3457 rcu_barrier_callback, rsp, cpu, 0); in _rcu_barrier()
3460 _rcu_barrier_trace(rsp, TPS("OnlineQ"), cpu, in _rcu_barrier()
3461 rsp->barrier_sequence); in _rcu_barrier()
3462 smp_call_function_single(cpu, rcu_barrier_func, rsp, 1); in _rcu_barrier()
3464 _rcu_barrier_trace(rsp, TPS("OnlineNQ"), cpu, in _rcu_barrier()
3465 rsp->barrier_sequence); in _rcu_barrier()
3474 if (atomic_dec_and_test(&rsp->barrier_cpu_count)) in _rcu_barrier()
3475 complete(&rsp->barrier_completion); in _rcu_barrier()
3478 wait_for_completion(&rsp->barrier_completion); in _rcu_barrier()
3481 _rcu_barrier_trace(rsp, TPS("Inc2"), -1, rsp->barrier_sequence); in _rcu_barrier()
3482 rcu_seq_end(&rsp->barrier_sequence); in _rcu_barrier()
3485 mutex_unlock(&rsp->barrier_mutex); in _rcu_barrier()
3538 rcu_boot_init_percpu_data(int cpu, struct rcu_state *rsp) in rcu_boot_init_percpu_data() argument
3540 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu); in rcu_boot_init_percpu_data()
3547 rdp->rcu_ofl_gp_seq = rsp->gp_seq; in rcu_boot_init_percpu_data()
3549 rdp->rcu_onl_gp_seq = rsp->gp_seq; in rcu_boot_init_percpu_data()
3552 rdp->rsp = rsp; in rcu_boot_init_percpu_data()
3563 rcu_init_percpu_data(int cpu, struct rcu_state *rsp) in rcu_init_percpu_data() argument
3566 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu); in rcu_init_percpu_data()
3567 struct rcu_node *rnp = rcu_get_root(rsp); in rcu_init_percpu_data()
3572 rdp->n_force_qs_snap = rsp->n_force_qs; in rcu_init_percpu_data()
3596 trace_rcu_grace_period(rsp->name, rdp->gp_seq, TPS("cpuonl")); in rcu_init_percpu_data()
3606 struct rcu_state *rsp; in rcutree_prepare_cpu() local
3608 for_each_rcu_flavor(rsp) in rcutree_prepare_cpu()
3609 rcu_init_percpu_data(cpu, rsp); in rcutree_prepare_cpu()
3636 struct rcu_state *rsp; in rcutree_online_cpu() local
3638 for_each_rcu_flavor(rsp) { in rcutree_online_cpu()
3639 rdp = per_cpu_ptr(rsp->rda, cpu); in rcutree_online_cpu()
3663 struct rcu_state *rsp; in rcutree_offline_cpu() local
3665 for_each_rcu_flavor(rsp) { in rcutree_offline_cpu()
3666 rdp = per_cpu_ptr(rsp->rda, cpu); in rcutree_offline_cpu()
3684 struct rcu_state *rsp; in rcutree_dying_cpu() local
3686 for_each_rcu_flavor(rsp) in rcutree_dying_cpu()
3687 rcu_cleanup_dying_cpu(rsp); in rcutree_dying_cpu()
3696 struct rcu_state *rsp; in rcutree_dead_cpu() local
3698 for_each_rcu_flavor(rsp) { in rcutree_dead_cpu()
3699 rcu_cleanup_dead_cpu(cpu, rsp); in rcutree_dead_cpu()
3700 do_nocb_deferred_wakeup(per_cpu_ptr(rsp->rda, cpu)); in rcutree_dead_cpu()
3726 struct rcu_state *rsp; in rcu_cpu_starting() local
3733 for_each_rcu_flavor(rsp) { in rcu_cpu_starting()
3734 rdp = per_cpu_ptr(rsp->rda, cpu); in rcu_cpu_starting()
3744 smp_store_release(&rsp->ncpus, rsp->ncpus + nbits); /* ^^^ */ in rcu_cpu_starting()
3746 rdp->rcu_onl_gp_seq = READ_ONCE(rsp->gp_seq); in rcu_cpu_starting()
3747 rdp->rcu_onl_gp_flags = READ_ONCE(rsp->gp_flags); in rcu_cpu_starting()
3750 rcu_report_qs_rnp(mask, rsp, rnp, rnp->gp_seq, flags); in rcu_cpu_starting()
3764 static void rcu_cleanup_dying_idle_cpu(int cpu, struct rcu_state *rsp) in rcu_cleanup_dying_idle_cpu() argument
3768 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu); in rcu_cleanup_dying_idle_cpu()
3773 spin_lock(&rsp->ofl_lock); in rcu_cleanup_dying_idle_cpu()
3775 rdp->rcu_ofl_gp_seq = READ_ONCE(rsp->gp_seq); in rcu_cleanup_dying_idle_cpu()
3776 rdp->rcu_ofl_gp_flags = READ_ONCE(rsp->gp_flags); in rcu_cleanup_dying_idle_cpu()
3779 rcu_report_qs_rnp(mask, rsp, rnp, rnp->gp_seq, flags); in rcu_cleanup_dying_idle_cpu()
3784 spin_unlock(&rsp->ofl_lock); in rcu_cleanup_dying_idle_cpu()
3797 struct rcu_state *rsp; in rcu_report_dead() local
3804 for_each_rcu_flavor(rsp) in rcu_report_dead()
3805 rcu_cleanup_dying_idle_cpu(cpu, rsp); in rcu_report_dead()
3811 static void rcu_migrate_callbacks(int cpu, struct rcu_state *rsp) in rcu_migrate_callbacks() argument
3815 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu); in rcu_migrate_callbacks()
3816 struct rcu_node *rnp_root = rcu_get_root(rdp->rsp); in rcu_migrate_callbacks()
3823 my_rdp = this_cpu_ptr(rsp->rda); in rcu_migrate_callbacks()
3830 needwake = rcu_advance_cbs(rsp, rnp_root, rdp) || in rcu_migrate_callbacks()
3831 rcu_advance_cbs(rsp, rnp_root, my_rdp); in rcu_migrate_callbacks()
3837 rcu_gp_kthread_wake(rsp); in rcu_migrate_callbacks()
3852 struct rcu_state *rsp; in rcutree_migrate_callbacks() local
3854 for_each_rcu_flavor(rsp) in rcutree_migrate_callbacks()
3855 rcu_migrate_callbacks(cpu, rsp); in rcutree_migrate_callbacks()
3891 struct rcu_state *rsp; in rcu_spawn_gp_kthread() local
3911 for_each_rcu_flavor(rsp) { in rcu_spawn_gp_kthread()
3912 t = kthread_create(rcu_gp_kthread, rsp, "%s", rsp->name); in rcu_spawn_gp_kthread()
3914 rnp = rcu_get_root(rsp); in rcu_spawn_gp_kthread()
3916 rsp->gp_kthread = t; in rcu_spawn_gp_kthread()
3952 static void __init rcu_init_one(struct rcu_state *rsp) in rcu_init_one() argument
3974 rsp->level[i] = rsp->level[i - 1] + num_rcu_lvl[i - 1]; in rcu_init_one()
3981 rnp = rsp->level[i]; in rcu_init_one()
3989 rnp->gp_seq = rsp->gp_seq; in rcu_init_one()
3990 rnp->gp_seq_needed = rsp->gp_seq; in rcu_init_one()
3991 rnp->completedqs = rsp->gp_seq; in rcu_init_one()
4005 rnp->parent = rsp->level[i - 1] + in rcu_init_one()
4019 init_swait_queue_head(&rsp->gp_wq); in rcu_init_one()
4020 init_swait_queue_head(&rsp->expedited_wq); in rcu_init_one()
4021 rnp = rcu_first_leaf_node(rsp); in rcu_init_one()
4025 per_cpu_ptr(rsp->rda, i)->mynode = rnp; in rcu_init_one()
4026 rcu_boot_init_percpu_data(i, rsp); in rcu_init_one()
4028 list_add(&rsp->flavors, &rcu_struct_flavors); in rcu_init_one()
4114 static void __init rcu_dump_rcu_node_tree(struct rcu_state *rsp) in rcu_dump_rcu_node_tree() argument
4121 rcu_for_each_node_breadth_first(rsp, rnp) { in rcu_dump_rcu_node_tree()