Lines Matching refs:rcu_dynticks
268 static DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = {
280 struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks); in rcu_dynticks_eqs_enter()
303 struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks); in rcu_dynticks_eqs_exit()
334 struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks); in rcu_dynticks_eqs_online()
348 struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks); in rcu_dynticks_curr_cpu_in_eqs()
357 int rcu_dynticks_snap(struct rcu_dynticks *rdtp) in rcu_dynticks_snap()
378 static bool rcu_dynticks_in_eqs_since(struct rcu_dynticks *rdtp, int snap) in rcu_dynticks_in_eqs_since()
394 struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu); in rcu_eqs_special_set()
418 struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks); in rcu_momentary_dyntick_idle()
421 raw_cpu_write(rcu_dynticks.rcu_need_heavy_qs, false); in rcu_momentary_dyntick_idle()
439 if (!smp_load_acquire(this_cpu_ptr(&rcu_dynticks.rcu_urgent_qs))) in rcu_note_context_switch()
441 this_cpu_write(rcu_dynticks.rcu_urgent_qs, false); in rcu_note_context_switch()
442 if (unlikely(raw_cpu_read(rcu_dynticks.rcu_need_heavy_qs))) in rcu_note_context_switch()
444 this_cpu_inc(rcu_dynticks.rcu_qs_ctr); in rcu_note_context_switch()
470 if (!raw_cpu_read(rcu_dynticks.rcu_urgent_qs)) in rcu_all_qs()
474 if (!smp_load_acquire(this_cpu_ptr(&rcu_dynticks.rcu_urgent_qs))) { in rcu_all_qs()
478 this_cpu_write(rcu_dynticks.rcu_urgent_qs, false); in rcu_all_qs()
480 if (unlikely(raw_cpu_read(rcu_dynticks.rcu_need_heavy_qs))) { in rcu_all_qs()
487 this_cpu_inc(rcu_dynticks.rcu_qs_ctr); in rcu_all_qs()
713 struct rcu_dynticks *rdtp; in rcu_eqs_enter()
715 rdtp = this_cpu_ptr(&rcu_dynticks); in rcu_eqs_enter()
786 struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks); in rcu_nmi_exit()
834 struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks); in rcu_irq_exit()
869 struct rcu_dynticks *rdtp; in rcu_eqs_exit()
873 rdtp = this_cpu_ptr(&rcu_dynticks); in rcu_eqs_exit()
937 struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks); in rcu_nmi_enter()
987 struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks); in rcu_irq_enter()
1046 smp_store_release(per_cpu_ptr(&rcu_dynticks.rcu_urgent_qs, cpu), true); in rcu_request_urgent_qs_task()
1101 return __this_cpu_read(rcu_dynticks.dynticks_nesting) <= 0 && in rcu_is_cpu_rrupt_from_idle()
1102 __this_cpu_read(rcu_dynticks.dynticks_nmi_nesting) <= 1; in rcu_is_cpu_rrupt_from_idle()
1194 ruqp = per_cpu_ptr(&rcu_dynticks.rcu_urgent_qs, rdp->cpu); in rcu_implicit_dynticks_qs()
1196 READ_ONCE(rdp->rcu_qs_ctr_snap) != per_cpu(rcu_dynticks.rcu_qs_ctr, rdp->cpu) && in rcu_implicit_dynticks_qs()
1244 rnhqp = &per_cpu(rcu_dynticks.rcu_need_heavy_qs, rdp->cpu); in rcu_implicit_dynticks_qs()
1847 rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_dynticks.rcu_qs_ctr); in __note_gp_changes()
2400 rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_dynticks.rcu_qs_ctr); in rcu_report_qs_rdp()
3544 rdp->dynticks = &per_cpu(rcu_dynticks, cpu); in rcu_boot_init_percpu_data()
3592 rdp->rcu_qs_ctr_snap = per_cpu(rcu_dynticks.rcu_qs_ctr, cpu); in rcu_init_percpu_data()