Lines Matching refs:rdp
177 static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp) in rcu_preempt_ctxt_queue() argument
182 (rnp->qsmask & rdp->grpmask ? RCU_GP_BLKD : 0) + in rcu_preempt_ctxt_queue()
183 (rnp->expmask & rdp->grpmask ? RCU_EXP_BLKD : 0); in rcu_preempt_ctxt_queue()
187 WARN_ON_ONCE(rdp->mynode != rnp); in rcu_preempt_ctxt_queue()
191 rdp->grpmask); in rcu_preempt_ctxt_queue()
276 !(rnp->qsmask & rdp->grpmask)); in rcu_preempt_ctxt_queue()
278 !(rnp->expmask & rdp->grpmask)); in rcu_preempt_ctxt_queue()
290 rcu_report_exp_rdp(rdp->rsp, rdp, true); in rcu_preempt_ctxt_queue()
338 struct rcu_data *rdp; in rcu_preempt_note_context_switch() local
347 rdp = this_cpu_ptr(rcu_state_p->rda); in rcu_preempt_note_context_switch()
348 rnp = rdp->mynode; in rcu_preempt_note_context_switch()
358 WARN_ON_ONCE((rdp->grpmask & rcu_rnp_online_cpus(rnp)) == 0); in rcu_preempt_note_context_switch()
360 trace_rcu_preempt_task(rdp->rsp->name, in rcu_preempt_note_context_switch()
362 (rnp->qsmask & rdp->grpmask) in rcu_preempt_note_context_switch()
365 rcu_preempt_ctxt_queue(rnp, rdp); in rcu_preempt_note_context_switch()
479 struct rcu_data *rdp; in rcu_read_unlock_special() local
514 rdp = this_cpu_ptr(rcu_state_p->rda); in rcu_read_unlock_special()
515 rcu_report_exp_rdp(rcu_state_p, rdp, true); in rcu_read_unlock_special()
875 struct rcu_data *rdp; in dump_blkd_tasks() local
896 rdp = per_cpu_ptr(rsp->rda, cpu); in dump_blkd_tasks()
897 onl = !!(rdp->grpmask & rcu_rnp_online_cpus(rnp)); in dump_blkd_tasks()
900 (long)rdp->rcu_onl_gp_seq, rdp->rcu_onl_gp_flags, in dump_blkd_tasks()
901 (long)rdp->rcu_ofl_gp_seq, rdp->rcu_ofl_gp_flags); in dump_blkd_tasks()
1362 struct rcu_data *rdp = per_cpu_ptr(rcu_state_p->rda, cpu); in rcu_prepare_kthreads() local
1363 struct rcu_node *rnp = rdp->mynode; in rcu_prepare_kthreads()
1488 struct rcu_data *rdp; in rcu_try_advance_all_cbs() local
1499 rdp = this_cpu_ptr(rsp->rda); in rcu_try_advance_all_cbs()
1500 rnp = rdp->mynode; in rcu_try_advance_all_cbs()
1507 if ((rcu_seq_completed_gp(rdp->gp_seq, in rcu_try_advance_all_cbs()
1509 unlikely(READ_ONCE(rdp->gpwrap))) && in rcu_try_advance_all_cbs()
1510 rcu_segcblist_pend_cbs(&rdp->cblist)) in rcu_try_advance_all_cbs()
1511 note_gp_changes(rsp, rdp); in rcu_try_advance_all_cbs()
1513 if (rcu_segcblist_ready_cbs(&rdp->cblist)) in rcu_try_advance_all_cbs()
1575 struct rcu_data *rdp; in rcu_prepare_for_idle() local
1617 rdp = this_cpu_ptr(rsp->rda); in rcu_prepare_for_idle()
1618 if (!rcu_segcblist_pend_cbs(&rdp->cblist)) in rcu_prepare_for_idle()
1620 rnp = rdp->mynode; in rcu_prepare_for_idle()
1622 needwake = rcu_accelerate_cbs(rsp, rnp, rdp); in rcu_prepare_for_idle()
1682 struct rcu_data *rdp; in rcu_oom_notify_cpu() local
1685 rdp = raw_cpu_ptr(rsp->rda); in rcu_oom_notify_cpu()
1686 if (rcu_segcblist_n_lazy_cbs(&rdp->cblist)) { in rcu_oom_notify_cpu()
1688 rsp->call(&rdp->oom_head, rcu_oom_callback); in rcu_oom_notify_cpu()
1784 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu); in print_cpu_stall_info() local
1785 struct rcu_dynticks *rdtp = rdp->dynticks; in print_cpu_stall_info()
1795 ticks_value = rcu_seq_ctr(rsp->gp_seq - rdp->gp_seq); in print_cpu_stall_info()
1800 ticks_value = rdp->ticks_this_gp; in print_cpu_stall_info()
1803 delta = rcu_seq_ctr(rdp->mynode->gp_seq - rdp->rcu_iw_gp_seq); in print_cpu_stall_info()
1807 "o."[!!(rdp->grpmask & rdp->mynode->qsmaskinit)], in print_cpu_stall_info()
1808 "N."[!!(rdp->grpmask & rdp->mynode->qsmaskinitnext)], in print_cpu_stall_info()
1810 rdp->rcu_iw_pending ? (int)min(delta, 9UL) + '0' : in print_cpu_stall_info()
1815 rdp->softirq_snap, kstat_softirqs_cpu(RCU_SOFTIRQ, cpu), in print_cpu_stall_info()
1827 static void zero_cpu_stall_ticks(struct rcu_data *rdp) in zero_cpu_stall_ticks() argument
1829 rdp->ticks_this_gp = 0; in zero_cpu_stall_ticks()
1830 rdp->softirq_snap = kstat_softirqs_cpu(RCU_SOFTIRQ, smp_processor_id()); in zero_cpu_stall_ticks()
1913 static void __wake_nocb_leader(struct rcu_data *rdp, bool force, in __wake_nocb_leader() argument
1915 __releases(rdp->nocb_lock) in __wake_nocb_leader()
1917 struct rcu_data *rdp_leader = rdp->nocb_leader; in __wake_nocb_leader()
1919 lockdep_assert_held(&rdp->nocb_lock); in __wake_nocb_leader()
1921 raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags); in __wake_nocb_leader()
1927 del_timer(&rdp->nocb_timer); in __wake_nocb_leader()
1928 raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags); in __wake_nocb_leader()
1932 raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags); in __wake_nocb_leader()
1940 static void wake_nocb_leader(struct rcu_data *rdp, bool force) in wake_nocb_leader() argument
1944 raw_spin_lock_irqsave(&rdp->nocb_lock, flags); in wake_nocb_leader()
1945 __wake_nocb_leader(rdp, force, flags); in wake_nocb_leader()
1952 static void wake_nocb_leader_defer(struct rcu_data *rdp, int waketype, in wake_nocb_leader_defer() argument
1957 raw_spin_lock_irqsave(&rdp->nocb_lock, flags); in wake_nocb_leader_defer()
1958 if (rdp->nocb_defer_wakeup == RCU_NOCB_WAKE_NOT) in wake_nocb_leader_defer()
1959 mod_timer(&rdp->nocb_timer, jiffies + 1); in wake_nocb_leader_defer()
1960 WRITE_ONCE(rdp->nocb_defer_wakeup, waketype); in wake_nocb_leader_defer()
1961 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, reason); in wake_nocb_leader_defer()
1962 raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags); in wake_nocb_leader_defer()
1971 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu); in rcu_nocb_cpu_needs_barrier() local
1989 ret = atomic_long_read(&rdp->nocb_q_count); in rcu_nocb_cpu_needs_barrier()
1992 rhp = READ_ONCE(rdp->nocb_head); in rcu_nocb_cpu_needs_barrier()
1994 rhp = READ_ONCE(rdp->nocb_gp_head); in rcu_nocb_cpu_needs_barrier()
1996 rhp = READ_ONCE(rdp->nocb_follower_head); in rcu_nocb_cpu_needs_barrier()
1999 if (!READ_ONCE(rdp->nocb_kthread) && rhp && in rcu_nocb_cpu_needs_barrier()
2019 static void __call_rcu_nocb_enqueue(struct rcu_data *rdp, in __call_rcu_nocb_enqueue() argument
2030 atomic_long_add(rhcount, &rdp->nocb_q_count); in __call_rcu_nocb_enqueue()
2032 old_rhpp = xchg(&rdp->nocb_tail, rhtp); in __call_rcu_nocb_enqueue()
2034 atomic_long_add(rhcount_lazy, &rdp->nocb_q_count_lazy); in __call_rcu_nocb_enqueue()
2038 t = READ_ONCE(rdp->nocb_kthread); in __call_rcu_nocb_enqueue()
2040 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, in __call_rcu_nocb_enqueue()
2044 len = atomic_long_read(&rdp->nocb_q_count); in __call_rcu_nocb_enqueue()
2045 if (old_rhpp == &rdp->nocb_head) { in __call_rcu_nocb_enqueue()
2048 wake_nocb_leader(rdp, false); in __call_rcu_nocb_enqueue()
2049 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, in __call_rcu_nocb_enqueue()
2052 wake_nocb_leader_defer(rdp, RCU_NOCB_WAKE, in __call_rcu_nocb_enqueue()
2055 rdp->qlen_last_fqs_check = 0; in __call_rcu_nocb_enqueue()
2056 } else if (len > rdp->qlen_last_fqs_check + qhimark) { in __call_rcu_nocb_enqueue()
2059 wake_nocb_leader(rdp, true); in __call_rcu_nocb_enqueue()
2060 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, in __call_rcu_nocb_enqueue()
2063 wake_nocb_leader_defer(rdp, RCU_NOCB_WAKE_FORCE, in __call_rcu_nocb_enqueue()
2066 rdp->qlen_last_fqs_check = LONG_MAX / 2; in __call_rcu_nocb_enqueue()
2068 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("WakeNot")); in __call_rcu_nocb_enqueue()
2082 static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp, in __call_rcu_nocb() argument
2086 if (!rcu_is_nocb_cpu(rdp->cpu)) in __call_rcu_nocb()
2088 __call_rcu_nocb_enqueue(rdp, rhp, &rhp->next, 1, lazy, flags); in __call_rcu_nocb()
2090 trace_rcu_kfree_callback(rdp->rsp->name, rhp, in __call_rcu_nocb()
2092 -atomic_long_read(&rdp->nocb_q_count_lazy), in __call_rcu_nocb()
2093 -atomic_long_read(&rdp->nocb_q_count)); in __call_rcu_nocb()
2095 trace_rcu_callback(rdp->rsp->name, rhp, in __call_rcu_nocb()
2096 -atomic_long_read(&rdp->nocb_q_count_lazy), in __call_rcu_nocb()
2097 -atomic_long_read(&rdp->nocb_q_count)); in __call_rcu_nocb()
2117 struct rcu_data *rdp, in rcu_nocb_adopt_orphan_cbs() argument
2123 __call_rcu_nocb_enqueue(my_rdp, rcu_segcblist_head(&rdp->cblist), in rcu_nocb_adopt_orphan_cbs()
2124 rcu_segcblist_tail(&rdp->cblist), in rcu_nocb_adopt_orphan_cbs()
2125 rcu_segcblist_n_cbs(&rdp->cblist), in rcu_nocb_adopt_orphan_cbs()
2126 rcu_segcblist_n_lazy_cbs(&rdp->cblist), flags); in rcu_nocb_adopt_orphan_cbs()
2127 rcu_segcblist_init(&rdp->cblist); in rcu_nocb_adopt_orphan_cbs()
2128 rcu_segcblist_disable(&rdp->cblist); in rcu_nocb_adopt_orphan_cbs()
2136 static void rcu_nocb_wait_gp(struct rcu_data *rdp) in rcu_nocb_wait_gp() argument
2142 struct rcu_node *rnp = rdp->mynode; in rcu_nocb_wait_gp()
2145 c = rcu_seq_snap(&rdp->rsp->gp_seq); in rcu_nocb_wait_gp()
2146 if (!rdp->gpwrap && ULONG_CMP_GE(rdp->gp_seq_needed, c)) { in rcu_nocb_wait_gp()
2150 needwake = rcu_start_this_gp(rnp, rdp, c); in rcu_nocb_wait_gp()
2153 rcu_gp_kthread_wake(rdp->rsp); in rcu_nocb_wait_gp()
2160 trace_rcu_this_gp(rnp, rdp, c, TPS("StartWait")); in rcu_nocb_wait_gp()
2168 trace_rcu_this_gp(rnp, rdp, c, TPS("ResumeWait")); in rcu_nocb_wait_gp()
2170 trace_rcu_this_gp(rnp, rdp, c, TPS("EndWait")); in rcu_nocb_wait_gp()
2183 struct rcu_data *rdp; in nocb_leader_wait() local
2210 for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower) { in nocb_leader_wait()
2211 rdp->nocb_gp_head = READ_ONCE(rdp->nocb_head); in nocb_leader_wait()
2212 if (!rdp->nocb_gp_head) in nocb_leader_wait()
2216 WRITE_ONCE(rdp->nocb_head, NULL); in nocb_leader_wait()
2217 rdp->nocb_gp_tail = xchg(&rdp->nocb_tail, &rdp->nocb_head); in nocb_leader_wait()
2237 for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower) { in nocb_leader_wait()
2239 READ_ONCE(rdp->nocb_head) && in nocb_leader_wait()
2245 if (!rdp->nocb_gp_head) in nocb_leader_wait()
2249 raw_spin_lock_irqsave(&rdp->nocb_lock, flags); in nocb_leader_wait()
2250 tail = rdp->nocb_follower_tail; in nocb_leader_wait()
2251 rdp->nocb_follower_tail = rdp->nocb_gp_tail; in nocb_leader_wait()
2252 *tail = rdp->nocb_gp_head; in nocb_leader_wait()
2253 raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags); in nocb_leader_wait()
2254 if (rdp != my_rdp && tail == &rdp->nocb_follower_head) { in nocb_leader_wait()
2256 swake_up_one(&rdp->nocb_wq); in nocb_leader_wait()
2269 static void nocb_follower_wait(struct rcu_data *rdp) in nocb_follower_wait() argument
2272 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("FollowerSleep")); in nocb_follower_wait()
2273 swait_event_interruptible_exclusive(rdp->nocb_wq, in nocb_follower_wait()
2274 READ_ONCE(rdp->nocb_follower_head)); in nocb_follower_wait()
2275 if (smp_load_acquire(&rdp->nocb_follower_head)) { in nocb_follower_wait()
2280 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("WokeEmpty")); in nocb_follower_wait()
2297 struct rcu_data *rdp = arg; in rcu_nocb_kthread() local
2302 if (rdp->nocb_leader == rdp) in rcu_nocb_kthread()
2303 nocb_leader_wait(rdp); in rcu_nocb_kthread()
2305 nocb_follower_wait(rdp); in rcu_nocb_kthread()
2308 raw_spin_lock_irqsave(&rdp->nocb_lock, flags); in rcu_nocb_kthread()
2309 list = rdp->nocb_follower_head; in rcu_nocb_kthread()
2310 rdp->nocb_follower_head = NULL; in rcu_nocb_kthread()
2311 tail = rdp->nocb_follower_tail; in rcu_nocb_kthread()
2312 rdp->nocb_follower_tail = &rdp->nocb_follower_head; in rcu_nocb_kthread()
2313 raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags); in rcu_nocb_kthread()
2315 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("WokeNonEmpty")); in rcu_nocb_kthread()
2318 trace_rcu_batch_start(rdp->rsp->name, in rcu_nocb_kthread()
2319 atomic_long_read(&rdp->nocb_q_count_lazy), in rcu_nocb_kthread()
2320 atomic_long_read(&rdp->nocb_q_count), -1); in rcu_nocb_kthread()
2326 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, in rcu_nocb_kthread()
2329 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, in rcu_nocb_kthread()
2335 if (__rcu_reclaim(rdp->rsp->name, list)) in rcu_nocb_kthread()
2342 trace_rcu_batch_end(rdp->rsp->name, c, !!list, 0, 0, 1); in rcu_nocb_kthread()
2344 atomic_long_add(-c, &rdp->nocb_q_count); in rcu_nocb_kthread()
2345 atomic_long_add(-cl, &rdp->nocb_q_count_lazy); in rcu_nocb_kthread()
2351 static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp) in rcu_nocb_need_deferred_wakeup() argument
2353 return READ_ONCE(rdp->nocb_defer_wakeup); in rcu_nocb_need_deferred_wakeup()
2357 static void do_nocb_deferred_wakeup_common(struct rcu_data *rdp) in do_nocb_deferred_wakeup_common() argument
2362 raw_spin_lock_irqsave(&rdp->nocb_lock, flags); in do_nocb_deferred_wakeup_common()
2363 if (!rcu_nocb_need_deferred_wakeup(rdp)) { in do_nocb_deferred_wakeup_common()
2364 raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags); in do_nocb_deferred_wakeup_common()
2367 ndw = READ_ONCE(rdp->nocb_defer_wakeup); in do_nocb_deferred_wakeup_common()
2368 WRITE_ONCE(rdp->nocb_defer_wakeup, RCU_NOCB_WAKE_NOT); in do_nocb_deferred_wakeup_common()
2369 __wake_nocb_leader(rdp, ndw == RCU_NOCB_WAKE_FORCE, flags); in do_nocb_deferred_wakeup_common()
2370 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("DeferredWake")); in do_nocb_deferred_wakeup_common()
2376 struct rcu_data *rdp = from_timer(rdp, t, nocb_timer); in do_nocb_deferred_wakeup_timer() local
2378 do_nocb_deferred_wakeup_common(rdp); in do_nocb_deferred_wakeup_timer()
2386 static void do_nocb_deferred_wakeup(struct rcu_data *rdp) in do_nocb_deferred_wakeup() argument
2388 if (rcu_nocb_need_deferred_wakeup(rdp)) in do_nocb_deferred_wakeup()
2389 do_nocb_deferred_wakeup_common(rdp); in do_nocb_deferred_wakeup()
2438 static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp) in rcu_boot_init_nocb_percpu_data() argument
2440 rdp->nocb_tail = &rdp->nocb_head; in rcu_boot_init_nocb_percpu_data()
2441 init_swait_queue_head(&rdp->nocb_wq); in rcu_boot_init_nocb_percpu_data()
2442 rdp->nocb_follower_tail = &rdp->nocb_follower_head; in rcu_boot_init_nocb_percpu_data()
2443 raw_spin_lock_init(&rdp->nocb_lock); in rcu_boot_init_nocb_percpu_data()
2444 timer_setup(&rdp->nocb_timer, do_nocb_deferred_wakeup_timer, 0); in rcu_boot_init_nocb_percpu_data()
2455 struct rcu_data *rdp; in rcu_spawn_one_nocb_kthread() local
2472 rdp = rdp_old_leader; in rcu_spawn_one_nocb_kthread()
2474 rdp->nocb_leader = rdp_spawn; in rcu_spawn_one_nocb_kthread()
2475 if (rdp_last && rdp != rdp_spawn) in rcu_spawn_one_nocb_kthread()
2476 rdp_last->nocb_next_follower = rdp; in rcu_spawn_one_nocb_kthread()
2477 if (rdp == rdp_spawn) { in rcu_spawn_one_nocb_kthread()
2478 rdp = rdp->nocb_next_follower; in rcu_spawn_one_nocb_kthread()
2480 rdp_last = rdp; in rcu_spawn_one_nocb_kthread()
2481 rdp = rdp->nocb_next_follower; in rcu_spawn_one_nocb_kthread()
2484 } while (rdp); in rcu_spawn_one_nocb_kthread()
2534 struct rcu_data *rdp; in rcu_organize_nocb_kthreads() local
2551 rdp = per_cpu_ptr(rsp->rda, cpu); in rcu_organize_nocb_kthreads()
2552 if (rdp->cpu >= nl) { in rcu_organize_nocb_kthreads()
2554 nl = DIV_ROUND_UP(rdp->cpu + 1, ls) * ls; in rcu_organize_nocb_kthreads()
2555 rdp->nocb_leader = rdp; in rcu_organize_nocb_kthreads()
2556 rdp_leader = rdp; in rcu_organize_nocb_kthreads()
2559 rdp->nocb_leader = rdp_leader; in rcu_organize_nocb_kthreads()
2560 rdp_prev->nocb_next_follower = rdp; in rcu_organize_nocb_kthreads()
2562 rdp_prev = rdp; in rcu_organize_nocb_kthreads()
2567 static bool init_nocb_callback_list(struct rcu_data *rdp) in init_nocb_callback_list() argument
2569 if (!rcu_is_nocb_cpu(rdp->cpu)) in init_nocb_callback_list()
2573 if (!rcu_segcblist_empty(&rdp->cblist)) { in init_nocb_callback_list()
2574 rdp->nocb_head = rcu_segcblist_head(&rdp->cblist); in init_nocb_callback_list()
2575 rdp->nocb_tail = rcu_segcblist_tail(&rdp->cblist); in init_nocb_callback_list()
2576 atomic_long_set(&rdp->nocb_q_count, in init_nocb_callback_list()
2577 rcu_segcblist_n_cbs(&rdp->cblist)); in init_nocb_callback_list()
2578 atomic_long_set(&rdp->nocb_q_count_lazy, in init_nocb_callback_list()
2579 rcu_segcblist_n_lazy_cbs(&rdp->cblist)); in init_nocb_callback_list()
2580 rcu_segcblist_init(&rdp->cblist); in init_nocb_callback_list()
2582 rcu_segcblist_disable(&rdp->cblist); in init_nocb_callback_list()
2607 static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp, in __call_rcu_nocb() argument
2614 struct rcu_data *rdp, in rcu_nocb_adopt_orphan_cbs() argument
2620 static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp) in rcu_boot_init_nocb_percpu_data() argument
2624 static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp) in rcu_nocb_need_deferred_wakeup() argument
2629 static void do_nocb_deferred_wakeup(struct rcu_data *rdp) in do_nocb_deferred_wakeup() argument
2641 static bool init_nocb_callback_list(struct rcu_data *rdp) in init_nocb_callback_list() argument