Lines Matching refs:gp_seq
101 .gp_seq = (0UL - 300UL) << RCU_SEQ_CTR_SHIFT, \
222 return rcu_seq_state(rcu_seq_current(&rsp->gp_seq)); in rcu_gp_in_progress()
237 __this_cpu_read(rcu_sched_data.gp_seq), in rcu_sched_qs()
252 __this_cpu_read(rcu_bh_data.gp_seq), in rcu_bh_qs()
558 return READ_ONCE(rcu_state_p->gp_seq); in rcu_get_gp_seq()
567 return READ_ONCE(rcu_sched_state.gp_seq); in rcu_sched_get_gp_seq()
576 return READ_ONCE(rcu_bh_state.gp_seq); in rcu_bh_get_gp_seq()
643 if (ULONG_CMP_GE(rsp->gp_seq, rnp->gp_seq_needed)) in show_rcu_gp_kthreads()
646 rnp->grplo, rnp->grphi, rnp->gp_seq, in show_rcu_gp_kthreads()
653 ULONG_CMP_GE(rsp->gp_seq, in show_rcu_gp_kthreads()
669 unsigned long *gp_seq) in rcutorture_get_gp_data() argument
689 *gp_seq = rcu_seq_current(&rsp->gp_seq); in rcutorture_get_gp_data()
1115 if (ULONG_CMP_LT(rcu_seq_current(&rdp->gp_seq) + ULONG_MAX / 4, in rcu_gpnum_ovf()
1116 rnp->gp_seq)) in rcu_gpnum_ovf()
1118 if (ULONG_CMP_LT(rdp->rcu_iw_gp_seq + ULONG_MAX / 4, rnp->gp_seq)) in rcu_gpnum_ovf()
1119 rdp->rcu_iw_gp_seq = rnp->gp_seq + ULONG_MAX / 4; in rcu_gpnum_ovf()
1131 trace_rcu_fqs(rdp->rsp->name, rdp->gp_seq, rdp->cpu, TPS("dti")); in dyntick_save_progress_counter()
1153 rdp->rcu_iw_gp_seq = rnp->gp_seq; in rcu_iw_handler()
1181 trace_rcu_fqs(rdp->rsp->name, rdp->gp_seq, rdp->cpu, TPS("dti")); in rcu_implicit_dynticks_qs()
1197 rcu_seq_current(&rdp->gp_seq) == rnp->gp_seq && !rdp->gpwrap) { in rcu_implicit_dynticks_qs()
1198 trace_rcu_fqs(rdp->rsp->name, rdp->gp_seq, rdp->cpu, TPS("rqc")); in rcu_implicit_dynticks_qs()
1215 (long)rnp->gp_seq, (long)rnp->completedqs); in rcu_implicit_dynticks_qs()
1263 !rdp->rcu_iw_pending && rdp->rcu_iw_gp_seq != rnp->gp_seq && in rcu_implicit_dynticks_qs()
1267 rdp->rcu_iw_gp_seq = rnp->gp_seq; in rcu_implicit_dynticks_qs()
1311 (long)rcu_seq_current(&rsp->gp_seq), in rcu_check_gp_kthread_starvation()
1372 static void print_other_cpu_stall(struct rcu_state *rsp, unsigned long gp_seq) in print_other_cpu_stall() argument
1413 (long)rcu_seq_current(&rsp->gp_seq), totqlen); in print_other_cpu_stall()
1420 if (rcu_seq_current(&rsp->gp_seq) != gp_seq) { in print_other_cpu_stall()
1474 (long)rcu_seq_current(&rsp->gp_seq), totqlen); in print_cpu_stall()
1532 gs1 = READ_ONCE(rsp->gp_seq); in check_cpu_stall()
1538 gs2 = READ_ONCE(rsp->gp_seq); in check_cpu_stall()
1582 trace_rcu_future_grace_period(rdp->rsp->name, rnp->gp_seq, gp_seq_req, in trace_rcu_this_gp()
1624 rcu_seq_started(&rnp->gp_seq, gp_seq_req) || in rcu_start_this_gp()
1626 rcu_seq_state(rcu_seq_current(&rnp->gp_seq)))) { in rcu_start_this_gp()
1632 if (rcu_seq_state(rcu_seq_current(&rnp->gp_seq))) { in rcu_start_this_gp()
1661 trace_rcu_grace_period(rsp->name, READ_ONCE(rsp->gp_seq), TPS("newreq")); in rcu_start_this_gp()
1683 needmore = ULONG_CMP_LT(rnp->gp_seq, rnp->gp_seq_needed); in rcu_future_gp_cleanup()
1685 rnp->gp_seq_needed = rnp->gp_seq; /* Avoid counter wrap. */ in rcu_future_gp_cleanup()
1686 trace_rcu_this_gp(rnp, rdp, rnp->gp_seq, in rcu_future_gp_cleanup()
1741 gp_seq_req = rcu_seq_snap(&rsp->gp_seq); in rcu_accelerate_cbs()
1747 trace_rcu_grace_period(rsp->name, rdp->gp_seq, TPS("AccWaitCB")); in rcu_accelerate_cbs()
1749 trace_rcu_grace_period(rsp->name, rdp->gp_seq, TPS("AccReadyCB")); in rcu_accelerate_cbs()
1768 c = rcu_seq_snap(&rsp->gp_seq); in rcu_accelerate_cbs_unlocked()
1804 rcu_segcblist_advance(&rdp->cblist, rnp->gp_seq); in rcu_advance_cbs()
1824 if (rdp->gp_seq == rnp->gp_seq) in __note_gp_changes()
1828 if (rcu_seq_completed_gp(rdp->gp_seq, rnp->gp_seq) || in __note_gp_changes()
1831 trace_rcu_grace_period(rsp->name, rdp->gp_seq, TPS("cpuend")); in __note_gp_changes()
1837 if (rcu_seq_new_gp(rdp->gp_seq, rnp->gp_seq) || in __note_gp_changes()
1844 trace_rcu_grace_period(rsp->name, rnp->gp_seq, TPS("cpustart")); in __note_gp_changes()
1851 rdp->gp_seq = rnp->gp_seq; /* Remember new grace-period state. */ in __note_gp_changes()
1867 if ((rdp->gp_seq == rcu_seq_current(&rnp->gp_seq) && in note_gp_changes()
1882 !(rcu_seq_ctr(rsp->gp_seq) % in rcu_gp_slow()
1919 rcu_seq_start(&rsp->gp_seq); in rcu_gp_init()
1920 trace_rcu_grace_period(rsp->name, rsp->gp_seq, TPS("start")); in rcu_gp_init()
1996 WRITE_ONCE(rnp->gp_seq, rsp->gp_seq); in rcu_gp_init()
2000 trace_rcu_grace_period_init(rsp->name, rnp->gp_seq, in rcu_gp_init()
2007 rcu_report_qs_rnp(mask, rsp, rnp, rnp->gp_seq, flags); in rcu_gp_init()
2099 new_gp_seq = rsp->gp_seq; in rcu_gp_cleanup()
2106 WRITE_ONCE(rnp->gp_seq, new_gp_seq); in rcu_gp_cleanup()
2123 rcu_seq_end(&rsp->gp_seq); in rcu_gp_cleanup()
2124 trace_rcu_grace_period(rsp->name, rsp->gp_seq, TPS("end")); in rcu_gp_cleanup()
2128 if (!needgp && ULONG_CMP_LT(rnp->gp_seq, rnp->gp_seq_needed)) { in rcu_gp_cleanup()
2137 trace_rcu_grace_period(rsp->name, READ_ONCE(rsp->gp_seq), in rcu_gp_cleanup()
2163 READ_ONCE(rsp->gp_seq), in rcu_gp_kthread()
2176 READ_ONCE(rsp->gp_seq), in rcu_gp_kthread()
2191 READ_ONCE(rsp->gp_seq), in rcu_gp_kthread()
2206 READ_ONCE(rsp->gp_seq), in rcu_gp_kthread()
2211 READ_ONCE(rsp->gp_seq), in rcu_gp_kthread()
2223 READ_ONCE(rsp->gp_seq), in rcu_gp_kthread()
2286 if ((!(rnp->qsmask & mask) && mask) || rnp->gp_seq != gps) { in rcu_report_qs_rnp()
2299 trace_rcu_quiescent_state_report(rsp->name, rnp->gp_seq, in rcu_report_qs_rnp()
2309 rnp->completedqs = rnp->gp_seq; in rcu_report_qs_rnp()
2357 rnp->completedqs = rnp->gp_seq; in rcu_report_unblock_qs_rnp()
2369 gps = rnp->gp_seq; in rcu_report_unblock_qs_rnp()
2390 if (rdp->cpu_no_qs.b.norm || rdp->gp_seq != rnp->gp_seq || in rcu_report_qs_rdp()
2416 rcu_report_qs_rnp(mask, rsp, rnp, rnp->gp_seq, flags); in rcu_report_qs_rdp()
2469 trace_rcu_grace_period(rsp->name, rnp->gp_seq, in rcu_cleanup_dying_cpu()
2714 rcu_report_qs_rnp(mask, rsp, rnp, rnp->gp_seq, flags); in force_qs_rnp()
2773 ULONG_CMP_GE(rnp_root->gp_seq, rnp_root->gp_seq_needed)) in rcu_check_gp_start_stall()
2784 ULONG_CMP_GE(rnp_root->gp_seq, rnp_root->gp_seq_needed) || in rcu_check_gp_start_stall()
2797 ULONG_CMP_GE(rnp_root->gp_seq, rnp_root->gp_seq_needed) || in rcu_check_gp_start_stall()
2806 __func__, (long)READ_ONCE(rsp->gp_seq), in rcu_check_gp_start_stall()
3196 return rcu_seq_snap(&rcu_state_p->gp_seq); in get_state_synchronize_rcu()
3216 if (!rcu_seq_done(&rcu_state_p->gp_seq, oldstate)) in cond_synchronize_rcu()
3237 return rcu_seq_snap(&rcu_sched_state.gp_seq); in get_state_synchronize_sched()
3257 if (!rcu_seq_done(&rcu_sched_state.gp_seq, oldstate)) in cond_synchronize_sched()
3297 if (rcu_seq_current(&rnp->gp_seq) != rdp->gp_seq || in __rcu_pending()
3547 rdp->rcu_ofl_gp_seq = rsp->gp_seq; in rcu_boot_init_percpu_data()
3549 rdp->rcu_onl_gp_seq = rsp->gp_seq; in rcu_boot_init_percpu_data()
3589 rdp->gp_seq = rnp->gp_seq; in rcu_init_percpu_data()
3590 rdp->gp_seq_needed = rnp->gp_seq; in rcu_init_percpu_data()
3595 rdp->rcu_iw_gp_seq = rnp->gp_seq - 1; in rcu_init_percpu_data()
3596 trace_rcu_grace_period(rsp->name, rdp->gp_seq, TPS("cpuonl")); in rcu_init_percpu_data()
3746 rdp->rcu_onl_gp_seq = READ_ONCE(rsp->gp_seq); in rcu_cpu_starting()
3750 rcu_report_qs_rnp(mask, rsp, rnp, rnp->gp_seq, flags); in rcu_cpu_starting()
3775 rdp->rcu_ofl_gp_seq = READ_ONCE(rsp->gp_seq); in rcu_cleanup_dying_idle_cpu()
3779 rcu_report_qs_rnp(mask, rsp, rnp, rnp->gp_seq, flags); in rcu_cleanup_dying_idle_cpu()
3989 rnp->gp_seq = rsp->gp_seq; in rcu_init_one()
3990 rnp->gp_seq_needed = rsp->gp_seq; in rcu_init_one()
3991 rnp->completedqs = rsp->gp_seq; in rcu_init_one()