Lines Matching refs:rdp
151 static void rcu_report_exp_rdp(struct rcu_data *rdp);
153 static void check_cb_ovld_locked(struct rcu_data *rdp, struct rcu_node *rnp);
154 static bool rcu_rdp_is_offloaded(struct rcu_data *rdp);
233 static bool rcu_rdp_cpu_online(struct rcu_data *rdp) in rcu_rdp_cpu_online() argument
235 return !!(rdp->grpmask & rcu_rnp_online_cpus(rdp->mynode)); in rcu_rdp_cpu_online()
254 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_get_n_cbs_cpu() local
256 if (rcu_segcblist_is_enabled(&rdp->cblist)) in rcu_get_n_cbs_cpu()
257 return rcu_segcblist_n_cbs(&rdp->cblist); in rcu_get_n_cbs_cpu()
315 static bool rcu_dynticks_in_eqs_since(struct rcu_data *rdp, int snap) in rcu_dynticks_in_eqs_since() argument
317 return snap != rcu_dynticks_snap(rdp->cpu); in rcu_dynticks_in_eqs_since()
500 static void force_qs_rnp(int (*f)(struct rcu_data *rdp));
573 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_irq_work_resched() local
582 if (do_nocb_deferred_wakeup(rdp) && need_resched()) { in rcu_irq_work_resched()
636 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in __rcu_irq_enter_check_tick() local
645 if (!tick_nohz_full_cpu(rdp->cpu) || in __rcu_irq_enter_check_tick()
646 !READ_ONCE(rdp->rcu_urgent_qs) || in __rcu_irq_enter_check_tick()
647 READ_ONCE(rdp->rcu_forced_tick)) { in __rcu_irq_enter_check_tick()
659 raw_spin_lock_rcu_node(rdp->mynode); in __rcu_irq_enter_check_tick()
660 if (rdp->rcu_urgent_qs && !rdp->rcu_forced_tick) { in __rcu_irq_enter_check_tick()
663 WRITE_ONCE(rdp->rcu_forced_tick, true); in __rcu_irq_enter_check_tick()
664 tick_dep_set_cpu(rdp->cpu, TICK_DEP_BIT_RCU); in __rcu_irq_enter_check_tick()
666 raw_spin_unlock_rcu_node(rdp->mynode); in __rcu_irq_enter_check_tick()
692 static void rcu_disable_urgency_upon_qs(struct rcu_data *rdp) in rcu_disable_urgency_upon_qs() argument
694 raw_lockdep_assert_held_rcu_node(rdp->mynode); in rcu_disable_urgency_upon_qs()
695 WRITE_ONCE(rdp->rcu_urgent_qs, false); in rcu_disable_urgency_upon_qs()
696 WRITE_ONCE(rdp->rcu_need_heavy_qs, false); in rcu_disable_urgency_upon_qs()
697 if (tick_nohz_full_cpu(rdp->cpu) && rdp->rcu_forced_tick) { in rcu_disable_urgency_upon_qs()
698 tick_dep_clear_cpu(rdp->cpu, TICK_DEP_BIT_RCU); in rcu_disable_urgency_upon_qs()
699 WRITE_ONCE(rdp->rcu_forced_tick, false); in rcu_disable_urgency_upon_qs()
760 struct rcu_data *rdp; in rcu_lockdep_current_cpu_online() local
766 rdp = this_cpu_ptr(&rcu_data); in rcu_lockdep_current_cpu_online()
774 if (rcu_rdp_cpu_online(rdp) || arch_spin_is_locked(&rcu_state.ofl_lock)) in rcu_lockdep_current_cpu_online()
790 static void rcu_gpnum_ovf(struct rcu_node *rnp, struct rcu_data *rdp) in rcu_gpnum_ovf() argument
793 if (ULONG_CMP_LT(rcu_seq_current(&rdp->gp_seq) + ULONG_MAX / 4, in rcu_gpnum_ovf()
795 WRITE_ONCE(rdp->gpwrap, true); in rcu_gpnum_ovf()
796 if (ULONG_CMP_LT(rdp->rcu_iw_gp_seq + ULONG_MAX / 4, rnp->gp_seq)) in rcu_gpnum_ovf()
797 rdp->rcu_iw_gp_seq = rnp->gp_seq + ULONG_MAX / 4; in rcu_gpnum_ovf()
805 static int dyntick_save_progress_counter(struct rcu_data *rdp) in dyntick_save_progress_counter() argument
807 rdp->dynticks_snap = rcu_dynticks_snap(rdp->cpu); in dyntick_save_progress_counter()
808 if (rcu_dynticks_in_eqs(rdp->dynticks_snap)) { in dyntick_save_progress_counter()
809 trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti")); in dyntick_save_progress_counter()
810 rcu_gpnum_ovf(rdp->mynode, rdp); in dyntick_save_progress_counter()
822 static int rcu_implicit_dynticks_qs(struct rcu_data *rdp) in rcu_implicit_dynticks_qs() argument
825 struct rcu_node *rnp = rdp->mynode; in rcu_implicit_dynticks_qs()
835 if (rcu_dynticks_in_eqs_since(rdp, rdp->dynticks_snap)) { in rcu_implicit_dynticks_qs()
836 trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti")); in rcu_implicit_dynticks_qs()
837 rcu_gpnum_ovf(rnp, rdp); in rcu_implicit_dynticks_qs()
859 if (WARN_ON_ONCE(!rcu_rdp_cpu_online(rdp))) { in rcu_implicit_dynticks_qs()
869 __func__, rdp->cpu, ".o"[rcu_rdp_cpu_online(rdp)], in rcu_implicit_dynticks_qs()
870 (long)rdp->rcu_onl_gp_seq, rdp->rcu_onl_gp_flags, in rcu_implicit_dynticks_qs()
871 (long)rdp->rcu_ofl_gp_seq, rdp->rcu_ofl_gp_flags); in rcu_implicit_dynticks_qs()
887 if (!READ_ONCE(rdp->rcu_need_heavy_qs) && in rcu_implicit_dynticks_qs()
891 WRITE_ONCE(rdp->rcu_need_heavy_qs, true); in rcu_implicit_dynticks_qs()
893 smp_store_release(&rdp->rcu_urgent_qs, true); in rcu_implicit_dynticks_qs()
895 WRITE_ONCE(rdp->rcu_urgent_qs, true); in rcu_implicit_dynticks_qs()
906 if (tick_nohz_full_cpu(rdp->cpu) && in rcu_implicit_dynticks_qs()
907 (time_after(jiffies, READ_ONCE(rdp->last_fqs_resched) + jtsq * 3) || in rcu_implicit_dynticks_qs()
909 WRITE_ONCE(rdp->rcu_urgent_qs, true); in rcu_implicit_dynticks_qs()
910 resched_cpu(rdp->cpu); in rcu_implicit_dynticks_qs()
911 WRITE_ONCE(rdp->last_fqs_resched, jiffies); in rcu_implicit_dynticks_qs()
923 READ_ONCE(rdp->last_fqs_resched) + jtsq)) { in rcu_implicit_dynticks_qs()
924 resched_cpu(rdp->cpu); in rcu_implicit_dynticks_qs()
925 WRITE_ONCE(rdp->last_fqs_resched, jiffies); in rcu_implicit_dynticks_qs()
928 !rdp->rcu_iw_pending && rdp->rcu_iw_gp_seq != rnp->gp_seq && in rcu_implicit_dynticks_qs()
929 (rnp->ffmask & rdp->grpmask)) { in rcu_implicit_dynticks_qs()
930 rdp->rcu_iw_pending = true; in rcu_implicit_dynticks_qs()
931 rdp->rcu_iw_gp_seq = rnp->gp_seq; in rcu_implicit_dynticks_qs()
932 irq_work_queue_on(&rdp->rcu_iw, rdp->cpu); in rcu_implicit_dynticks_qs()
940 static void trace_rcu_this_gp(struct rcu_node *rnp, struct rcu_data *rdp, in trace_rcu_this_gp() argument
964 static bool rcu_start_this_gp(struct rcu_node *rnp_start, struct rcu_data *rdp, in rcu_start_this_gp() argument
980 trace_rcu_this_gp(rnp_start, rdp, gp_seq_req, TPS("Startleaf")); in rcu_start_this_gp()
988 trace_rcu_this_gp(rnp, rdp, gp_seq_req, in rcu_start_this_gp()
1000 trace_rcu_this_gp(rnp_start, rdp, gp_seq_req, in rcu_start_this_gp()
1012 trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("Startedleafroot")); in rcu_start_this_gp()
1015 trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("Startedroot")); in rcu_start_this_gp()
1019 trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("NoGPkthread")); in rcu_start_this_gp()
1028 WRITE_ONCE(rdp->gp_seq_needed, rnp->gp_seq_needed); in rcu_start_this_gp()
1042 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_future_gp_cleanup() local
1047 trace_rcu_this_gp(rnp, rdp, rnp->gp_seq, in rcu_future_gp_cleanup()
1091 static bool rcu_accelerate_cbs(struct rcu_node *rnp, struct rcu_data *rdp) in rcu_accelerate_cbs() argument
1096 rcu_lockdep_assert_cblist_protected(rdp); in rcu_accelerate_cbs()
1100 if (!rcu_segcblist_pend_cbs(&rdp->cblist)) in rcu_accelerate_cbs()
1103 trace_rcu_segcb_stats(&rdp->cblist, TPS("SegCbPreAcc")); in rcu_accelerate_cbs()
1116 if (rcu_segcblist_accelerate(&rdp->cblist, gp_seq_req)) in rcu_accelerate_cbs()
1117 ret = rcu_start_this_gp(rnp, rdp, gp_seq_req); in rcu_accelerate_cbs()
1120 if (rcu_segcblist_restempty(&rdp->cblist, RCU_WAIT_TAIL)) in rcu_accelerate_cbs()
1125 trace_rcu_segcb_stats(&rdp->cblist, TPS("SegCbPostAcc")); in rcu_accelerate_cbs()
1138 struct rcu_data *rdp) in rcu_accelerate_cbs_unlocked() argument
1143 rcu_lockdep_assert_cblist_protected(rdp); in rcu_accelerate_cbs_unlocked()
1145 if (!READ_ONCE(rdp->gpwrap) && ULONG_CMP_GE(rdp->gp_seq_needed, c)) { in rcu_accelerate_cbs_unlocked()
1147 (void)rcu_segcblist_accelerate(&rdp->cblist, c); in rcu_accelerate_cbs_unlocked()
1151 needwake = rcu_accelerate_cbs(rnp, rdp); in rcu_accelerate_cbs_unlocked()
1167 static bool rcu_advance_cbs(struct rcu_node *rnp, struct rcu_data *rdp) in rcu_advance_cbs() argument
1169 rcu_lockdep_assert_cblist_protected(rdp); in rcu_advance_cbs()
1173 if (!rcu_segcblist_pend_cbs(&rdp->cblist)) in rcu_advance_cbs()
1180 rcu_segcblist_advance(&rdp->cblist, rnp->gp_seq); in rcu_advance_cbs()
1183 return rcu_accelerate_cbs(rnp, rdp); in rcu_advance_cbs()
1191 struct rcu_data *rdp) in rcu_advance_cbs_nowake() argument
1193 rcu_lockdep_assert_cblist_protected(rdp); in rcu_advance_cbs_nowake()
1198 WARN_ON_ONCE(rcu_advance_cbs(rnp, rdp)); in rcu_advance_cbs_nowake()
1221 static bool __note_gp_changes(struct rcu_node *rnp, struct rcu_data *rdp) in __note_gp_changes() argument
1225 const bool offloaded = rcu_rdp_is_offloaded(rdp); in __note_gp_changes()
1229 if (rdp->gp_seq == rnp->gp_seq) in __note_gp_changes()
1233 if (rcu_seq_completed_gp(rdp->gp_seq, rnp->gp_seq) || in __note_gp_changes()
1234 unlikely(READ_ONCE(rdp->gpwrap))) { in __note_gp_changes()
1236 ret = rcu_advance_cbs(rnp, rdp); /* Advance CBs. */ in __note_gp_changes()
1237 rdp->core_needs_qs = false; in __note_gp_changes()
1238 trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuend")); in __note_gp_changes()
1241 ret = rcu_accelerate_cbs(rnp, rdp); /* Recent CBs. */ in __note_gp_changes()
1242 if (rdp->core_needs_qs) in __note_gp_changes()
1243 rdp->core_needs_qs = !!(rnp->qsmask & rdp->grpmask); in __note_gp_changes()
1247 if (rcu_seq_new_gp(rdp->gp_seq, rnp->gp_seq) || in __note_gp_changes()
1248 unlikely(READ_ONCE(rdp->gpwrap))) { in __note_gp_changes()
1255 need_qs = !!(rnp->qsmask & rdp->grpmask); in __note_gp_changes()
1256 rdp->cpu_no_qs.b.norm = need_qs; in __note_gp_changes()
1257 rdp->core_needs_qs = need_qs; in __note_gp_changes()
1258 zero_cpu_stall_ticks(rdp); in __note_gp_changes()
1260 rdp->gp_seq = rnp->gp_seq; /* Remember new grace-period state. */ in __note_gp_changes()
1261 if (ULONG_CMP_LT(rdp->gp_seq_needed, rnp->gp_seq_needed) || rdp->gpwrap) in __note_gp_changes()
1262 WRITE_ONCE(rdp->gp_seq_needed, rnp->gp_seq_needed); in __note_gp_changes()
1263 if (IS_ENABLED(CONFIG_PROVE_RCU) && READ_ONCE(rdp->gpwrap)) in __note_gp_changes()
1264 WRITE_ONCE(rdp->last_sched_clock, jiffies); in __note_gp_changes()
1265 WRITE_ONCE(rdp->gpwrap, false); in __note_gp_changes()
1266 rcu_gpnum_ovf(rnp, rdp); in __note_gp_changes()
1270 static void note_gp_changes(struct rcu_data *rdp) in note_gp_changes() argument
1277 rnp = rdp->mynode; in note_gp_changes()
1278 if ((rdp->gp_seq == rcu_seq_current(&rnp->gp_seq) && in note_gp_changes()
1279 !unlikely(READ_ONCE(rdp->gpwrap))) || /* w/out lock. */ in note_gp_changes()
1284 needwake = __note_gp_changes(rnp, rdp); in note_gp_changes()
1442 struct rcu_data *rdp; in rcu_gp_init() local
1549 rdp = this_cpu_ptr(&rcu_data); in rcu_gp_init()
1553 if (rnp == rdp->mynode) in rcu_gp_init()
1554 (void)__note_gp_changes(rnp, rdp); in rcu_gp_init()
1722 struct rcu_data *rdp; in rcu_gp_cleanup() local
1763 rdp = this_cpu_ptr(&rcu_data); in rcu_gp_cleanup()
1764 if (rnp == rdp->mynode) in rcu_gp_cleanup()
1765 needgp = __note_gp_changes(rnp, rdp) || needgp; in rcu_gp_cleanup()
1771 rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_gp_cleanup()
1772 check_cb_ovld_locked(rdp, rnp); in rcu_gp_cleanup()
1790 rdp = this_cpu_ptr(&rcu_data); in rcu_gp_cleanup()
1792 trace_rcu_this_gp(rnp, rdp, rnp->gp_seq_needed, in rcu_gp_cleanup()
1797 offloaded = rcu_rdp_is_offloaded(rdp); in rcu_gp_cleanup()
1798 if ((offloaded || !rcu_accelerate_cbs(rnp, rdp)) && needgp) { in rcu_gp_cleanup()
2006 rcu_report_qs_rdp(struct rcu_data *rdp) in rcu_report_qs_rdp() argument
2014 WARN_ON_ONCE(rdp->cpu != smp_processor_id()); in rcu_report_qs_rdp()
2015 rnp = rdp->mynode; in rcu_report_qs_rdp()
2017 if (rdp->cpu_no_qs.b.norm || rdp->gp_seq != rnp->gp_seq || in rcu_report_qs_rdp()
2018 rdp->gpwrap) { in rcu_report_qs_rdp()
2026 rdp->cpu_no_qs.b.norm = true; /* need qs for new gp. */ in rcu_report_qs_rdp()
2030 mask = rdp->grpmask; in rcu_report_qs_rdp()
2031 rdp->core_needs_qs = false; in rcu_report_qs_rdp()
2041 if (!rcu_rdp_is_offloaded(rdp)) { in rcu_report_qs_rdp()
2042 needwake = rcu_accelerate_cbs(rnp, rdp); in rcu_report_qs_rdp()
2043 } else if (!rcu_segcblist_completely_offloaded(&rdp->cblist)) { in rcu_report_qs_rdp()
2051 rcu_disable_urgency_upon_qs(rdp); in rcu_report_qs_rdp()
2058 rcu_nocb_lock_irqsave(rdp, flags); in rcu_report_qs_rdp()
2059 rcu_accelerate_cbs_unlocked(rnp, rdp); in rcu_report_qs_rdp()
2060 rcu_nocb_unlock_irqrestore(rdp, flags); in rcu_report_qs_rdp()
2072 rcu_check_quiescent_state(struct rcu_data *rdp) in rcu_check_quiescent_state() argument
2075 note_gp_changes(rdp); in rcu_check_quiescent_state()
2081 if (!rdp->core_needs_qs) in rcu_check_quiescent_state()
2088 if (rdp->cpu_no_qs.b.norm) in rcu_check_quiescent_state()
2095 rcu_report_qs_rdp(rdp); in rcu_check_quiescent_state()
2105 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcutree_dying_cpu() local
2106 struct rcu_node *rnp = rdp->mynode; in rcutree_dying_cpu()
2111 blkd = !!(rnp->qsmask & rdp->grpmask); in rcutree_dying_cpu()
2170 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcutree_dead_cpu() local
2171 struct rcu_node *rnp = rdp->mynode; /* Outgoing CPU's rdp & rnp. */ in rcutree_dead_cpu()
2188 static void rcu_do_batch(struct rcu_data *rdp) in rcu_do_batch() argument
2199 if (!rcu_segcblist_ready_cbs(&rdp->cblist)) { in rcu_do_batch()
2201 rcu_segcblist_n_cbs(&rdp->cblist), 0); in rcu_do_batch()
2203 !rcu_segcblist_empty(&rdp->cblist), in rcu_do_batch()
2205 rcu_is_callbacks_kthread(rdp)); in rcu_do_batch()
2214 rcu_nocb_lock_irqsave(rdp, flags); in rcu_do_batch()
2216 pending = rcu_segcblist_n_cbs(&rdp->cblist); in rcu_do_batch()
2219 bl = max(rdp->blimit, pending >> div); in rcu_do_batch()
2227 rcu_segcblist_n_cbs(&rdp->cblist), bl); in rcu_do_batch()
2228 rcu_segcblist_extract_done_cbs(&rdp->cblist, &rcl); in rcu_do_batch()
2229 if (rcu_rdp_is_offloaded(rdp)) in rcu_do_batch()
2230 rdp->qlen_last_fqs_check = rcu_segcblist_n_cbs(&rdp->cblist); in rcu_do_batch()
2232 trace_rcu_segcb_stats(&rdp->cblist, TPS("SegCbDequeued")); in rcu_do_batch()
2233 rcu_nocb_unlock_irqrestore(rdp, flags); in rcu_do_batch()
2280 rcu_nocb_lock_irqsave(rdp, flags); in rcu_do_batch()
2281 rdp->n_cbs_invoked += count; in rcu_do_batch()
2283 is_idle_task(current), rcu_is_callbacks_kthread(rdp)); in rcu_do_batch()
2286 rcu_segcblist_insert_done_cbs(&rdp->cblist, &rcl); in rcu_do_batch()
2287 rcu_segcblist_add_len(&rdp->cblist, -count); in rcu_do_batch()
2290 count = rcu_segcblist_n_cbs(&rdp->cblist); in rcu_do_batch()
2291 if (rdp->blimit >= DEFAULT_MAX_RCU_BLIMIT && count <= qlowmark) in rcu_do_batch()
2292 rdp->blimit = blimit; in rcu_do_batch()
2295 if (count == 0 && rdp->qlen_last_fqs_check != 0) { in rcu_do_batch()
2296 rdp->qlen_last_fqs_check = 0; in rcu_do_batch()
2297 rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs); in rcu_do_batch()
2298 } else if (count < rdp->qlen_last_fqs_check - qhimark) in rcu_do_batch()
2299 rdp->qlen_last_fqs_check = count; in rcu_do_batch()
2305 empty = rcu_segcblist_empty(&rdp->cblist); in rcu_do_batch()
2309 WARN_ON_ONCE(count == 0 && rcu_segcblist_n_segment_cbs(&rdp->cblist) != 0); in rcu_do_batch()
2310 WARN_ON_ONCE(!empty && rcu_segcblist_n_segment_cbs(&rdp->cblist) == 0); in rcu_do_batch()
2312 rcu_nocb_unlock_irqrestore(rdp, flags); in rcu_do_batch()
2363 static void force_qs_rnp(int (*f)(struct rcu_data *rdp)) in force_qs_rnp() argument
2368 struct rcu_data *rdp; in force_qs_rnp() local
2393 rdp = per_cpu_ptr(&rcu_data, cpu); in force_qs_rnp()
2394 if (f(rdp)) { in force_qs_rnp()
2395 mask |= rdp->grpmask; in force_qs_rnp()
2396 rcu_disable_urgency_upon_qs(rdp); in force_qs_rnp()
2459 struct rcu_data *rdp = raw_cpu_ptr(&rcu_data); in rcu_core() local
2460 struct rcu_node *rnp = rdp->mynode; in rcu_core()
2478 const bool do_batch = !rcu_segcblist_completely_offloaded(&rdp->cblist); in rcu_core()
2483 WARN_ON_ONCE(!rdp->beenonline); in rcu_core()
2494 rcu_check_quiescent_state(rdp); in rcu_core()
2498 rcu_segcblist_is_enabled(&rdp->cblist) && do_batch) { in rcu_core()
2499 rcu_nocb_lock_irqsave(rdp, flags); in rcu_core()
2500 if (!rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL)) in rcu_core()
2501 rcu_accelerate_cbs_unlocked(rnp, rdp); in rcu_core()
2502 rcu_nocb_unlock_irqrestore(rdp, flags); in rcu_core()
2505 rcu_check_gp_start_stall(rnp, rdp, rcu_jiffies_till_stall_check()); in rcu_core()
2508 if (do_batch && rcu_segcblist_ready_cbs(&rdp->cblist) && in rcu_core()
2510 rcu_do_batch(rdp); in rcu_core()
2512 if (rcu_segcblist_ready_cbs(&rdp->cblist)) in rcu_core()
2517 do_nocb_deferred_wakeup(rdp); in rcu_core()
2522 queue_work_on(rdp->cpu, rcu_gp_wq, &rdp->strict_work); in rcu_core()
2642 static void __call_rcu_core(struct rcu_data *rdp, struct rcu_head *head, in __call_rcu_core() argument
2663 if (unlikely(rcu_segcblist_n_cbs(&rdp->cblist) > in __call_rcu_core()
2664 rdp->qlen_last_fqs_check + qhimark)) { in __call_rcu_core()
2667 note_gp_changes(rdp); in __call_rcu_core()
2671 rcu_accelerate_cbs_unlocked(rdp->mynode, rdp); in __call_rcu_core()
2674 rdp->blimit = DEFAULT_MAX_RCU_BLIMIT; in __call_rcu_core()
2675 if (READ_ONCE(rcu_state.n_force_qs) == rdp->n_force_qs_snap && in __call_rcu_core()
2676 rcu_segcblist_first_pend_cb(&rdp->cblist) != head) in __call_rcu_core()
2678 rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs); in __call_rcu_core()
2679 rdp->qlen_last_fqs_check = rcu_segcblist_n_cbs(&rdp->cblist); in __call_rcu_core()
2697 static void check_cb_ovld_locked(struct rcu_data *rdp, struct rcu_node *rnp) in check_cb_ovld_locked() argument
2702 if (rcu_segcblist_n_cbs(&rdp->cblist) >= qovld_calc) in check_cb_ovld_locked()
2703 WRITE_ONCE(rnp->cbovldmask, rnp->cbovldmask | rdp->grpmask); in check_cb_ovld_locked()
2705 WRITE_ONCE(rnp->cbovldmask, rnp->cbovldmask & ~rdp->grpmask); in check_cb_ovld_locked()
2720 static void check_cb_ovld(struct rcu_data *rdp) in check_cb_ovld() argument
2722 struct rcu_node *const rnp = rdp->mynode; in check_cb_ovld()
2725 ((rcu_segcblist_n_cbs(&rdp->cblist) >= qovld_calc) == in check_cb_ovld()
2726 !!(READ_ONCE(rnp->cbovldmask) & rdp->grpmask))) in check_cb_ovld()
2729 check_cb_ovld_locked(rdp, rnp); in check_cb_ovld()
2777 struct rcu_data *rdp; in call_rcu() local
2800 rdp = this_cpu_ptr(&rcu_data); in call_rcu()
2803 if (unlikely(!rcu_segcblist_is_enabled(&rdp->cblist))) { in call_rcu()
2809 if (rcu_segcblist_empty(&rdp->cblist)) in call_rcu()
2810 rcu_segcblist_init(&rdp->cblist); in call_rcu()
2813 check_cb_ovld(rdp); in call_rcu()
2814 if (rcu_nocb_try_bypass(rdp, head, &was_alldone, flags)) in call_rcu()
2817 rcu_segcblist_enqueue(&rdp->cblist, head); in call_rcu()
2821 rcu_segcblist_n_cbs(&rdp->cblist)); in call_rcu()
2824 rcu_segcblist_n_cbs(&rdp->cblist)); in call_rcu()
2826 trace_rcu_segcb_stats(&rdp->cblist, TPS("SegCBQueued")); in call_rcu()
2829 if (unlikely(rcu_rdp_is_offloaded(rdp))) { in call_rcu()
2830 __call_rcu_nocb_wake(rdp, was_alldone, flags); /* unlocks */ in call_rcu()
2832 __call_rcu_core(rdp, head, flags); in call_rcu()
3609 struct rcu_data *rdp; in start_poll_synchronize_rcu_common() local
3614 rdp = this_cpu_ptr(&rcu_data); in start_poll_synchronize_rcu_common()
3615 rnp = rdp->mynode; in start_poll_synchronize_rcu_common()
3623 needwake = rcu_start_this_gp(rnp, rdp, rcu_seq_snap(&rcu_state.gp_seq)); in start_poll_synchronize_rcu_common()
3819 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_pending() local
3820 struct rcu_node *rnp = rdp->mynode; in rcu_pending()
3825 check_cpu_stall(rdp); in rcu_pending()
3828 if (rcu_nocb_need_deferred_wakeup(rdp, RCU_NOCB_WAKE)) in rcu_pending()
3837 if (rdp->core_needs_qs && !rdp->cpu_no_qs.b.norm && gp_in_progress) in rcu_pending()
3841 if (!rcu_rdp_is_offloaded(rdp) && in rcu_pending()
3842 rcu_segcblist_ready_cbs(&rdp->cblist)) in rcu_pending()
3846 if (!gp_in_progress && rcu_segcblist_is_enabled(&rdp->cblist) && in rcu_pending()
3847 !rcu_rdp_is_offloaded(rdp) && in rcu_pending()
3848 !rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL)) in rcu_pending()
3852 if (rcu_seq_current(&rnp->gp_seq) != rdp->gp_seq || in rcu_pending()
3853 unlikely(READ_ONCE(rdp->gpwrap))) /* outside lock */ in rcu_pending()
3895 static void rcu_barrier_entrain(struct rcu_data *rdp) in rcu_barrier_entrain() argument
3898 unsigned long lseq = READ_ONCE(rdp->barrier_seq_snap); in rcu_barrier_entrain()
3904 rdp->barrier_head.func = rcu_barrier_callback; in rcu_barrier_entrain()
3905 debug_rcu_head_queue(&rdp->barrier_head); in rcu_barrier_entrain()
3906 rcu_nocb_lock(rdp); in rcu_barrier_entrain()
3907 WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, jiffies)); in rcu_barrier_entrain()
3908 if (rcu_segcblist_entrain(&rdp->cblist, &rdp->barrier_head)) { in rcu_barrier_entrain()
3911 debug_rcu_head_unqueue(&rdp->barrier_head); in rcu_barrier_entrain()
3914 rcu_nocb_unlock(rdp); in rcu_barrier_entrain()
3915 smp_store_release(&rdp->barrier_seq_snap, gseq); in rcu_barrier_entrain()
3924 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_barrier_handler() local
3927 WARN_ON_ONCE(cpu != rdp->cpu); in rcu_barrier_handler()
3930 rcu_barrier_entrain(rdp); in rcu_barrier_handler()
3947 struct rcu_data *rdp; in rcu_barrier() local
3986 rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_barrier()
3988 if (smp_load_acquire(&rdp->barrier_seq_snap) == gseq) in rcu_barrier()
3991 if (!rcu_segcblist_n_cbs(&rdp->cblist)) { in rcu_barrier()
3992 WRITE_ONCE(rdp->barrier_seq_snap, gseq); in rcu_barrier()
3997 if (!rcu_rdp_cpu_online(rdp)) { in rcu_barrier()
3998 rcu_barrier_entrain(rdp); in rcu_barrier()
3999 WARN_ON_ONCE(READ_ONCE(rdp->barrier_seq_snap) != gseq); in rcu_barrier()
4009 WARN_ON_ONCE(READ_ONCE(rdp->barrier_seq_snap) != gseq); in rcu_barrier()
4028 rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_barrier()
4030 WRITE_ONCE(rdp->barrier_seq_snap, gseq); in rcu_barrier()
4073 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_boot_init_percpu_data() local
4076 rdp->grpmask = leaf_node_cpu_bit(rdp->mynode, cpu); in rcu_boot_init_percpu_data()
4077 INIT_WORK(&rdp->strict_work, strict_work_handler); in rcu_boot_init_percpu_data()
4080 rdp->barrier_seq_snap = rcu_state.barrier_sequence; in rcu_boot_init_percpu_data()
4081 rdp->rcu_ofl_gp_seq = rcu_state.gp_seq; in rcu_boot_init_percpu_data()
4082 rdp->rcu_ofl_gp_flags = RCU_GP_CLEANED; in rcu_boot_init_percpu_data()
4083 rdp->rcu_onl_gp_seq = rcu_state.gp_seq; in rcu_boot_init_percpu_data()
4084 rdp->rcu_onl_gp_flags = RCU_GP_CLEANED; in rcu_boot_init_percpu_data()
4085 rdp->last_sched_clock = jiffies; in rcu_boot_init_percpu_data()
4086 rdp->cpu = cpu; in rcu_boot_init_percpu_data()
4087 rcu_boot_init_nocb_percpu_data(rdp); in rcu_boot_init_percpu_data()
4104 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcutree_prepare_cpu() local
4109 rdp->qlen_last_fqs_check = 0; in rcutree_prepare_cpu()
4110 rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs); in rcutree_prepare_cpu()
4111 rdp->blimit = blimit; in rcutree_prepare_cpu()
4119 if (!rcu_segcblist_is_enabled(&rdp->cblist)) in rcutree_prepare_cpu()
4120 rcu_segcblist_init(&rdp->cblist); /* Re-enable callbacks. */ in rcutree_prepare_cpu()
4127 rnp = rdp->mynode; in rcutree_prepare_cpu()
4129 rdp->beenonline = true; /* We have now been online. */ in rcutree_prepare_cpu()
4130 rdp->gp_seq = READ_ONCE(rnp->gp_seq); in rcutree_prepare_cpu()
4131 rdp->gp_seq_needed = rdp->gp_seq; in rcutree_prepare_cpu()
4132 rdp->cpu_no_qs.b.norm = true; in rcutree_prepare_cpu()
4133 rdp->core_needs_qs = false; in rcutree_prepare_cpu()
4134 rdp->rcu_iw_pending = false; in rcutree_prepare_cpu()
4135 rdp->rcu_iw = IRQ_WORK_INIT_HARD(rcu_iw_handler); in rcutree_prepare_cpu()
4136 rdp->rcu_iw_gp_seq = rdp->gp_seq - 1; in rcutree_prepare_cpu()
4137 trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuonl")); in rcutree_prepare_cpu()
4151 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcutree_affinity_setting() local
4153 rcu_boost_kthread_setaffinity(rdp->mynode, outgoing); in rcutree_affinity_setting()
4163 struct rcu_data *rdp; in rcutree_online_cpu() local
4166 rdp = per_cpu_ptr(&rcu_data, cpu); in rcutree_online_cpu()
4167 rnp = rdp->mynode; in rcutree_online_cpu()
4169 rnp->ffmask |= rdp->grpmask; in rcutree_online_cpu()
4188 struct rcu_data *rdp; in rcutree_offline_cpu() local
4191 rdp = per_cpu_ptr(&rcu_data, cpu); in rcutree_offline_cpu()
4192 rnp = rdp->mynode; in rcutree_offline_cpu()
4194 rnp->ffmask &= ~rdp->grpmask; in rcutree_offline_cpu()
4219 struct rcu_data *rdp; in rcu_cpu_starting() local
4223 rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_cpu_starting()
4224 if (rdp->cpu_started) in rcu_cpu_starting()
4226 rdp->cpu_started = true; in rcu_cpu_starting()
4228 rnp = rdp->mynode; in rcu_cpu_starting()
4229 mask = rdp->grpmask; in rcu_cpu_starting()
4242 rcu_gpnum_ovf(rnp, rdp); /* Offline-induced counter wrap? */ in rcu_cpu_starting()
4243 rdp->rcu_onl_gp_seq = READ_ONCE(rcu_state.gp_seq); in rcu_cpu_starting()
4244 rdp->rcu_onl_gp_flags = READ_ONCE(rcu_state.gp_flags); in rcu_cpu_starting()
4252 rcu_disable_urgency_upon_qs(rdp); in rcu_cpu_starting()
4275 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_report_dead() local
4276 struct rcu_node *rnp = rdp->mynode; /* Outgoing CPU's rdp & rnp. */ in rcu_report_dead()
4279 do_nocb_deferred_wakeup(rdp); in rcu_report_dead()
4282 rcu_report_exp_rdp(rdp); in rcu_report_dead()
4286 mask = rdp->grpmask; in rcu_report_dead()
4290 rdp->rcu_ofl_gp_seq = READ_ONCE(rcu_state.gp_seq); in rcu_report_dead()
4291 rdp->rcu_ofl_gp_flags = READ_ONCE(rcu_state.gp_flags); in rcu_report_dead()
4294 rcu_disable_urgency_upon_qs(rdp); in rcu_report_dead()
4303 rdp->cpu_started = false; in rcu_report_dead()
4317 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcutree_migrate_callbacks() local
4320 if (rcu_rdp_is_offloaded(rdp) || in rcutree_migrate_callbacks()
4321 rcu_segcblist_empty(&rdp->cblist)) in rcutree_migrate_callbacks()
4325 WARN_ON_ONCE(rcu_rdp_cpu_online(rdp)); in rcutree_migrate_callbacks()
4326 rcu_barrier_entrain(rdp); in rcutree_migrate_callbacks()
4333 needwake = rcu_advance_cbs(my_rnp, rdp) || in rcutree_migrate_callbacks()
4335 rcu_segcblist_merge(&my_rdp->cblist, &rdp->cblist); in rcutree_migrate_callbacks()
4338 rcu_segcblist_disable(&rdp->cblist); in rcutree_migrate_callbacks()
4351 WARN_ONCE(rcu_segcblist_n_cbs(&rdp->cblist) != 0 || in rcutree_migrate_callbacks()
4352 !rcu_segcblist_empty(&rdp->cblist), in rcutree_migrate_callbacks()
4354 cpu, rcu_segcblist_n_cbs(&rdp->cblist), in rcutree_migrate_callbacks()
4355 rcu_segcblist_first_cb(&rdp->cblist)); in rcutree_migrate_callbacks()
4435 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_spawn_gp_kthread() local
4460 rcu_spawn_one_boost_kthread(rdp->mynode); in rcu_spawn_gp_kthread()