87 struct rcu_state rcu_state = {  variable
88 	.level = { &rcu_state.node[0] },
91 	.barrier_mutex = __MUTEX_INITIALIZER(rcu_state.barrier_mutex),
94 	.exp_mutex = __MUTEX_INITIALIZER(rcu_state.exp_mutex),
95 	.exp_wake_mutex = __MUTEX_INITIALIZER(rcu_state.exp_wake_mutex),
96 	.ofl_lock = __RAW_SPIN_LOCK_UNLOCKED(rcu_state.ofl_lock),
203 	return rcu_seq_state(rcu_seq_current(&rcu_state.gp_seq));  in rcu_gp_in_progress()
506 	return READ_ONCE(rcu_state.gp_seq);  in rcu_get_gp_seq()
518 	return rcu_state.expedited_sequence;  in rcu_exp_batches_completed()
527 	return &rcu_state.node[0];  in rcu_get_root()
548 		*flags = READ_ONCE(rcu_state.gp_flags);  in rcutorture_get_gp_data()
549 		*gp_seq = rcu_seq_current(&rcu_state.gp_seq);  in rcutorture_get_gp_data()
986 		trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti"));  in dyntick_save_progress_counter()
1015 		trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti"));  in rcu_implicit_dynticks_qs()
1022 	    time_after(jiffies, rcu_state.gp_start + HZ)) {  in rcu_implicit_dynticks_qs()
1056 	    (time_after(jiffies, rcu_state.gp_start + jtsq * 2) ||  in rcu_implicit_dynticks_qs()
1057 	     time_after(jiffies, rcu_state.jiffies_resched))) {  in rcu_implicit_dynticks_qs()
1061 	} else if (time_after(jiffies, rcu_state.gp_start + jtsq)) {  in rcu_implicit_dynticks_qs()
1087 	if (time_after(jiffies, rcu_state.jiffies_resched)) {  in rcu_implicit_dynticks_qs()
1110 	trace_rcu_future_grace_period(rcu_state.name, rnp->gp_seq, gp_seq_req,  in trace_rcu_this_gp()
1182 	WRITE_ONCE(rcu_state.gp_flags, rcu_state.gp_flags | RCU_GP_FLAG_INIT);  in rcu_start_this_gp()
1183 	rcu_state.gp_req_activity = jiffies;  in rcu_start_this_gp()
1184 	if (!rcu_state.gp_kthread) {  in rcu_start_this_gp()
1188 	trace_rcu_grace_period(rcu_state.name, READ_ONCE(rcu_state.gp_seq), TPS("newreq"));  in rcu_start_this_gp()
1234 	if ((current == rcu_state.gp_kthread &&  in rcu_gp_kthread_wake()
1236 	    !READ_ONCE(rcu_state.gp_flags) ||  in rcu_gp_kthread_wake()
1237 	    !rcu_state.gp_kthread)  in rcu_gp_kthread_wake()
1239 	WRITE_ONCE(rcu_state.gp_wake_time, jiffies);  in rcu_gp_kthread_wake()
1240 	WRITE_ONCE(rcu_state.gp_wake_seq, READ_ONCE(rcu_state.gp_seq));  in rcu_gp_kthread_wake()
1241 	swake_up_one(&rcu_state.gp_wq);  in rcu_gp_kthread_wake()
1278 	gp_seq_req = rcu_seq_snap(&rcu_state.gp_seq);  in rcu_accelerate_cbs()
1284 		trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("AccWaitCB"));  in rcu_accelerate_cbs()
1286 		trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("AccReadyCB"));  in rcu_accelerate_cbs()
1304 	c = rcu_seq_snap(&rcu_state.gp_seq);  in rcu_accelerate_cbs_unlocked()
1384 		trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuend"));  in __note_gp_changes()
1398 		trace_rcu_grace_period(rcu_state.name, rnp->gp_seq, TPS("cpustart"));  in __note_gp_changes()
1435 	    !(rcu_seq_ctr(rcu_state.gp_seq) %  in rcu_gp_slow()
1451 	WRITE_ONCE(rcu_state.gp_activity, jiffies);  in rcu_gp_init()
1453 	if (!READ_ONCE(rcu_state.gp_flags)) {  in rcu_gp_init()
1458 	WRITE_ONCE(rcu_state.gp_flags, 0); /* Clear all flags: New GP. */  in rcu_gp_init()
1472 	rcu_seq_start(&rcu_state.gp_seq);  in rcu_gp_init()
1473 	trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("start"));  in rcu_gp_init()
1482 	rcu_state.gp_state = RCU_GP_ONOFF;  in rcu_gp_init()
1484 		raw_spin_lock(&rcu_state.ofl_lock);  in rcu_gp_init()
1490 			raw_spin_unlock(&rcu_state.ofl_lock);  in rcu_gp_init()
1526 		raw_spin_unlock(&rcu_state.ofl_lock);  in rcu_gp_init()
1542 	rcu_state.gp_state = RCU_GP_INIT;  in rcu_gp_init()
1549 		WRITE_ONCE(rnp->gp_seq, rcu_state.gp_seq);  in rcu_gp_init()
1553 		trace_rcu_grace_period_init(rcu_state.name, rnp->gp_seq,  in rcu_gp_init()
1564 		WRITE_ONCE(rcu_state.gp_activity, jiffies);  in rcu_gp_init()
1579 	*gfp = READ_ONCE(rcu_state.gp_flags);  in rcu_gp_fqs_check_wake()
1597 	WRITE_ONCE(rcu_state.gp_activity, jiffies);  in rcu_gp_fqs()
1598 	rcu_state.n_force_qs++;  in rcu_gp_fqs()
1607 	if (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) {  in rcu_gp_fqs()
1609 		WRITE_ONCE(rcu_state.gp_flags,  in rcu_gp_fqs()
1610 			   READ_ONCE(rcu_state.gp_flags) & ~RCU_GP_FLAG_FQS);  in rcu_gp_fqs()
1631 			rcu_state.jiffies_force_qs = jiffies + j;  in rcu_gp_fqs_loop()
1632 			WRITE_ONCE(rcu_state.jiffies_kick_kthreads,  in rcu_gp_fqs_loop()
1635 		trace_rcu_grace_period(rcu_state.name,  in rcu_gp_fqs_loop()
1636 				       READ_ONCE(rcu_state.gp_seq),  in rcu_gp_fqs_loop()
1638 		rcu_state.gp_state = RCU_GP_WAIT_FQS;  in rcu_gp_fqs_loop()
1640 				rcu_state.gp_wq, rcu_gp_fqs_check_wake(&gf), j);  in rcu_gp_fqs_loop()
1641 		rcu_state.gp_state = RCU_GP_DOING_FQS;  in rcu_gp_fqs_loop()
1648 		if (ULONG_CMP_GE(jiffies, rcu_state.jiffies_force_qs) ||  in rcu_gp_fqs_loop()
1650 			trace_rcu_grace_period(rcu_state.name,  in rcu_gp_fqs_loop()
1651 					       READ_ONCE(rcu_state.gp_seq),  in rcu_gp_fqs_loop()
1655 			trace_rcu_grace_period(rcu_state.name,  in rcu_gp_fqs_loop()
1656 					       READ_ONCE(rcu_state.gp_seq),  in rcu_gp_fqs_loop()
1659 			WRITE_ONCE(rcu_state.gp_activity, jiffies);  in rcu_gp_fqs_loop()
1665 			WRITE_ONCE(rcu_state.gp_activity, jiffies);  in rcu_gp_fqs_loop()
1667 			trace_rcu_grace_period(rcu_state.name,  in rcu_gp_fqs_loop()
1668 					       READ_ONCE(rcu_state.gp_seq),  in rcu_gp_fqs_loop()
1672 			if (time_after(jiffies, rcu_state.jiffies_force_qs))  in rcu_gp_fqs_loop()
1675 				j = rcu_state.jiffies_force_qs - j;  in rcu_gp_fqs_loop()
1693 	WRITE_ONCE(rcu_state.gp_activity, jiffies);  in rcu_gp_cleanup()
1695 	rcu_state.gp_end = jiffies;  in rcu_gp_cleanup()
1696 	gp_duration = rcu_state.gp_end - rcu_state.gp_start;  in rcu_gp_cleanup()
1697 	if (gp_duration > rcu_state.gp_max)  in rcu_gp_cleanup()
1698 		rcu_state.gp_max = gp_duration;  in rcu_gp_cleanup()
1719 	new_gp_seq = rcu_state.gp_seq;  in rcu_gp_cleanup()
1736 		WRITE_ONCE(rcu_state.gp_activity, jiffies);  in rcu_gp_cleanup()
1743 	trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("end"));  in rcu_gp_cleanup()
1744 	rcu_seq_end(&rcu_state.gp_seq);  in rcu_gp_cleanup()
1745 	rcu_state.gp_state = RCU_GP_IDLE;  in rcu_gp_cleanup()
1757 		WRITE_ONCE(rcu_state.gp_flags, RCU_GP_FLAG_INIT);  in rcu_gp_cleanup()
1758 		rcu_state.gp_req_activity = jiffies;  in rcu_gp_cleanup()
1759 		trace_rcu_grace_period(rcu_state.name,  in rcu_gp_cleanup()
1760 				       READ_ONCE(rcu_state.gp_seq),  in rcu_gp_cleanup()
1763 		WRITE_ONCE(rcu_state.gp_flags,  in rcu_gp_cleanup()
1764 			   rcu_state.gp_flags & RCU_GP_FLAG_INIT);  in rcu_gp_cleanup()
1779 			trace_rcu_grace_period(rcu_state.name,  in rcu_gp_kthread()
1780 					       READ_ONCE(rcu_state.gp_seq),  in rcu_gp_kthread()
1782 			rcu_state.gp_state = RCU_GP_WAIT_GPS;  in rcu_gp_kthread()
1783 			swait_event_idle_exclusive(rcu_state.gp_wq,  in rcu_gp_kthread()
1784 					 READ_ONCE(rcu_state.gp_flags) &  in rcu_gp_kthread()
1786 			rcu_state.gp_state = RCU_GP_DONE_GPS;  in rcu_gp_kthread()
1791 			WRITE_ONCE(rcu_state.gp_activity, jiffies);  in rcu_gp_kthread()
1793 			trace_rcu_grace_period(rcu_state.name,  in rcu_gp_kthread()
1794 					       READ_ONCE(rcu_state.gp_seq),  in rcu_gp_kthread()
1802 		rcu_state.gp_state = RCU_GP_CLEANUP;  in rcu_gp_kthread()
1804 		rcu_state.gp_state = RCU_GP_CLEANED;  in rcu_gp_kthread()
1822 	WRITE_ONCE(rcu_state.gp_flags,  in rcu_report_qs_rsp()
1823 		   READ_ONCE(rcu_state.gp_flags) | RCU_GP_FLAG_FQS);  in rcu_report_qs_rsp()
1866 		trace_rcu_quiescent_state_report(rcu_state.name, rnp->gp_seq,  in rcu_report_qs_rnp()
2036 	trace_rcu_grace_period(rcu_state.name, rnp->gp_seq,  in rcutree_dying_cpu()
2123 		trace_rcu_batch_start(rcu_state.name,  in rcu_do_batch()
2126 		trace_rcu_batch_end(rcu_state.name, 0,  in rcu_do_batch()
2145 	trace_rcu_batch_start(rcu_state.name,  in rcu_do_batch()
2157 		if (__rcu_reclaim(rcu_state.name, rhp))  in rcu_do_batch()
2187 	trace_rcu_batch_end(rcu_state.name, count, !!rcl.head, need_resched(),  in rcu_do_batch()
2203 		rdp->n_force_qs_snap = rcu_state.n_force_qs;  in rcu_do_batch()
2314 		ret = (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) ||  in rcu_force_quiescent_state()
2327 	if (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) {  in rcu_force_quiescent_state()
2331 	WRITE_ONCE(rcu_state.gp_flags,  in rcu_force_quiescent_state()
2332 		   READ_ONCE(rcu_state.gp_flags) | RCU_GP_FLAG_FQS);  in rcu_force_quiescent_state()
2532 			if (rcu_state.n_force_qs == rdp->n_force_qs_snap &&  in __call_rcu_core()
2535 			rdp->n_force_qs_snap = rcu_state.n_force_qs;  in __call_rcu_core()
2595 		trace_rcu_kfree_callback(rcu_state.name, head,  in __call_rcu()
2600 		trace_rcu_callback(rcu_state.name, head,  in __call_rcu()
2755 	return rcu_seq_snap(&rcu_state.gp_seq);  in get_state_synchronize_rcu()
2775 	if (!rcu_seq_done(&rcu_state.gp_seq, oldstate))  in cond_synchronize_rcu()
2836 	trace_rcu_barrier(rcu_state.name, s, cpu,  in rcu_barrier_trace()
2837 			  atomic_read(&rcu_state.barrier_cpu_count), done);  in rcu_barrier_trace()
2846 	if (atomic_dec_and_test(&rcu_state.barrier_cpu_count)) {  in rcu_barrier_callback()
2848 				   rcu_state.barrier_sequence);  in rcu_barrier_callback()
2849 		complete(&rcu_state.barrier_completion);  in rcu_barrier_callback()
2851 		rcu_barrier_trace(TPS("CB"), -1, rcu_state.barrier_sequence);  in rcu_barrier_callback()
2862 	rcu_barrier_trace(TPS("IRQ"), -1, rcu_state.barrier_sequence);  in rcu_barrier_func()
2868 		atomic_inc(&rcu_state.barrier_cpu_count);  in rcu_barrier_func()
2872 				   rcu_state.barrier_sequence);  in rcu_barrier_func()
2889 	unsigned long s = rcu_seq_snap(&rcu_state.barrier_sequence);  in rcu_barrier()
2894 	mutex_lock(&rcu_state.barrier_mutex);  in rcu_barrier()
2897 	if (rcu_seq_done(&rcu_state.barrier_sequence, s)) {  in rcu_barrier()
2899 				   rcu_state.barrier_sequence);  in rcu_barrier()
2901 		mutex_unlock(&rcu_state.barrier_mutex);  in rcu_barrier()
2906 	rcu_seq_start(&rcu_state.barrier_sequence);  in rcu_barrier()
2907 	rcu_barrier_trace(TPS("Inc1"), -1, rcu_state.barrier_sequence);  in rcu_barrier()
2915 	init_completion(&rcu_state.barrier_completion);  in rcu_barrier()
2916 	atomic_set(&rcu_state.barrier_cpu_count, 1);  in rcu_barrier()
2931 					   rcu_state.barrier_sequence);  in rcu_barrier()
2935 					   rcu_state.barrier_sequence);  in rcu_barrier()
2944 	if (atomic_dec_and_test(&rcu_state.barrier_cpu_count))  in rcu_barrier()
2945 		complete(&rcu_state.barrier_completion);  in rcu_barrier()
2948 	wait_for_completion(&rcu_state.barrier_completion);  in rcu_barrier()
2951 	rcu_barrier_trace(TPS("Inc2"), -1, rcu_state.barrier_sequence);  in rcu_barrier()
2952 	rcu_seq_end(&rcu_state.barrier_sequence);  in rcu_barrier()
2955 	mutex_unlock(&rcu_state.barrier_mutex);  in rcu_barrier()
2999 	rdp->rcu_ofl_gp_seq = rcu_state.gp_seq;  in rcu_boot_init_percpu_data()
3001 	rdp->rcu_onl_gp_seq = rcu_state.gp_seq;  in rcu_boot_init_percpu_data()
3026 	rdp->n_force_qs_snap = rcu_state.n_force_qs;  in rcutree_prepare_cpu()
3049 	trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuonl"));  in rcutree_prepare_cpu()
3146 	smp_store_release(&rcu_state.ncpus, rcu_state.ncpus + nbits); /* ^^^ */  in rcu_cpu_starting()
3148 	rdp->rcu_onl_gp_seq = READ_ONCE(rcu_state.gp_seq);  in rcu_cpu_starting()
3149 	rdp->rcu_onl_gp_flags = READ_ONCE(rcu_state.gp_flags);  in rcu_cpu_starting()
3183 	raw_spin_lock(&rcu_state.ofl_lock);  in rcu_report_dead()
3185 	rdp->rcu_ofl_gp_seq = READ_ONCE(rcu_state.gp_seq);  in rcu_report_dead()
3186 	rdp->rcu_ofl_gp_flags = READ_ONCE(rcu_state.gp_flags);  in rcu_report_dead()
3194 	raw_spin_unlock(&rcu_state.ofl_lock);  in rcu_report_dead()
3297 	t = kthread_create(rcu_gp_kthread, NULL, "%s", rcu_state.name);  in rcu_spawn_gp_kthread()
3306 	rcu_state.gp_kthread = t;  in rcu_spawn_gp_kthread()
3359 		rcu_state.level[i] =  in rcu_init_one()
3360 			rcu_state.level[i - 1] + num_rcu_lvl[i - 1];  in rcu_init_one()
3367 		rnp = rcu_state.level[i];  in rcu_init_one()
3375 			rnp->gp_seq = rcu_state.gp_seq;  in rcu_init_one()
3376 			rnp->gp_seq_needed = rcu_state.gp_seq;  in rcu_init_one()
3377 			rnp->completedqs = rcu_state.gp_seq;  in rcu_init_one()
3391 				rnp->parent = rcu_state.level[i - 1] +  in rcu_init_one()
3405 	init_swait_queue_head(&rcu_state.gp_wq);  in rcu_init_one()
3406 	init_swait_queue_head(&rcu_state.expedited_wq);  in rcu_init_one()