87 static struct rcu_state rcu_state = {  variable
88 	.level = { &rcu_state.node[0] },
91 	.barrier_mutex = __MUTEX_INITIALIZER(rcu_state.barrier_mutex),
94 	.exp_mutex = __MUTEX_INITIALIZER(rcu_state.exp_mutex),
95 	.exp_wake_mutex = __MUTEX_INITIALIZER(rcu_state.exp_wake_mutex),
96 	.ofl_lock = __RAW_SPIN_LOCK_UNLOCKED(rcu_state.ofl_lock),
219 	return rcu_seq_state(rcu_seq_current(&rcu_state.gp_seq));  in rcu_gp_in_progress()
571 	return READ_ONCE(rcu_state.gp_seq);  in rcu_get_gp_seq()
583 	return rcu_state.expedited_sequence;  in rcu_exp_batches_completed()
592 	return &rcu_state.node[0];  in rcu_get_root()
603 		*flags = READ_ONCE(rcu_state.gp_flags);  in rcutorture_get_gp_data()
604 		*gp_seq = rcu_seq_current(&rcu_state.gp_seq);  in rcutorture_get_gp_data()
1187 		trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti"));  in dyntick_save_progress_counter()
1216 		trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti"));  in rcu_implicit_dynticks_qs()
1272 	    (time_after(jiffies, rcu_state.gp_start + jtsq * 2) ||  in rcu_implicit_dynticks_qs()
1273 	     time_after(jiffies, rcu_state.jiffies_resched) ||  in rcu_implicit_dynticks_qs()
1274 	     rcu_state.cbovld)) {  in rcu_implicit_dynticks_qs()
1278 	} else if (time_after(jiffies, rcu_state.gp_start + jtsq)) {  in rcu_implicit_dynticks_qs()
1292 	     rcu_state.cbovld)) {  in rcu_implicit_dynticks_qs()
1305 	if (time_after(jiffies, rcu_state.jiffies_resched)) {  in rcu_implicit_dynticks_qs()
1329 	trace_rcu_future_grace_period(rcu_state.name, READ_ONCE(rnp->gp_seq),  in trace_rcu_this_gp()
1402 	WRITE_ONCE(rcu_state.gp_flags, rcu_state.gp_flags | RCU_GP_FLAG_INIT);  in rcu_start_this_gp()
1403 	WRITE_ONCE(rcu_state.gp_req_activity, jiffies);  in rcu_start_this_gp()
1404 	if (!READ_ONCE(rcu_state.gp_kthread)) {  in rcu_start_this_gp()
1408 	trace_rcu_grace_period(rcu_state.name, data_race(rcu_state.gp_seq), TPS("newreq"));  in rcu_start_this_gp()
1455 	struct task_struct *t = READ_ONCE(rcu_state.gp_kthread);  in rcu_gp_kthread_wake()
1458 	    !READ_ONCE(rcu_state.gp_flags) || !t)  in rcu_gp_kthread_wake()
1460 	WRITE_ONCE(rcu_state.gp_wake_time, jiffies);  in rcu_gp_kthread_wake()
1461 	WRITE_ONCE(rcu_state.gp_wake_seq, READ_ONCE(rcu_state.gp_seq));  in rcu_gp_kthread_wake()
1462 	swake_up_one(&rcu_state.gp_wq);  in rcu_gp_kthread_wake()
1499 	gp_seq_req = rcu_seq_snap(&rcu_state.gp_seq);  in rcu_accelerate_cbs()
1505 		trace_rcu_grace_period(rcu_state.name, gp_seq_req, TPS("AccWaitCB"));  in rcu_accelerate_cbs()
1507 		trace_rcu_grace_period(rcu_state.name, gp_seq_req, TPS("AccReadyCB"));  in rcu_accelerate_cbs()
1526 	c = rcu_seq_snap(&rcu_state.gp_seq);  in rcu_accelerate_cbs_unlocked()
1620 		trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuend"));  in __note_gp_changes()
1636 		trace_rcu_grace_period(rcu_state.name, rnp->gp_seq, TPS("cpustart"));  in __note_gp_changes()
1674 	    !(rcu_seq_ctr(rcu_state.gp_seq) %  in rcu_gp_slow()
1724 	WRITE_ONCE(rcu_state.gp_activity, jiffies);  in rcu_gp_init()
1726 	if (!READ_ONCE(rcu_state.gp_flags)) {  in rcu_gp_init()
1731 	WRITE_ONCE(rcu_state.gp_flags, 0); /* Clear all flags: New GP. */  in rcu_gp_init()
1745 	rcu_seq_start(&rcu_state.gp_seq);  in rcu_gp_init()
1746 	ASSERT_EXCLUSIVE_WRITER(rcu_state.gp_seq);  in rcu_gp_init()
1747 	trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("start"));  in rcu_gp_init()
1759 	rcu_state.gp_state = RCU_GP_ONOFF;  in rcu_gp_init()
1761 		raw_spin_lock(&rcu_state.ofl_lock);  in rcu_gp_init()
1767 			raw_spin_unlock(&rcu_state.ofl_lock);  in rcu_gp_init()
1803 		raw_spin_unlock(&rcu_state.ofl_lock);  in rcu_gp_init()
1819 	rcu_state.gp_state = RCU_GP_INIT;  in rcu_gp_init()
1826 		WRITE_ONCE(rnp->gp_seq, rcu_state.gp_seq);  in rcu_gp_init()
1830 		trace_rcu_grace_period_init(rcu_state.name, rnp->gp_seq,  in rcu_gp_init()
1841 		WRITE_ONCE(rcu_state.gp_activity, jiffies);  in rcu_gp_init()
1864 	*gfp = READ_ONCE(rcu_state.gp_flags);  in rcu_gp_fqs_check_wake()
1882 	WRITE_ONCE(rcu_state.gp_activity, jiffies);  in rcu_gp_fqs()
1883 	rcu_state.n_force_qs++;  in rcu_gp_fqs()
1892 	if (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) {  in rcu_gp_fqs()
1894 		WRITE_ONCE(rcu_state.gp_flags,  in rcu_gp_fqs()
1895 			   READ_ONCE(rcu_state.gp_flags) & ~RCU_GP_FLAG_FQS);  in rcu_gp_fqs()
1913 	if (rcu_state.cbovld)  in rcu_gp_fqs_loop()
1918 			rcu_state.jiffies_force_qs = jiffies + j;  in rcu_gp_fqs_loop()
1919 			WRITE_ONCE(rcu_state.jiffies_kick_kthreads,  in rcu_gp_fqs_loop()
1922 		trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,  in rcu_gp_fqs_loop()
1924 		rcu_state.gp_state = RCU_GP_WAIT_FQS;  in rcu_gp_fqs_loop()
1926 				rcu_state.gp_wq, rcu_gp_fqs_check_wake(&gf), j);  in rcu_gp_fqs_loop()
1928 		rcu_state.gp_state = RCU_GP_DOING_FQS;  in rcu_gp_fqs_loop()
1935 		if (!time_after(rcu_state.jiffies_force_qs, jiffies) ||  in rcu_gp_fqs_loop()
1937 			trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,  in rcu_gp_fqs_loop()
1943 				gf = rcu_state.cbovld ? RCU_GP_FLAG_OVLD : 0;  in rcu_gp_fqs_loop()
1945 			trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,  in rcu_gp_fqs_loop()
1948 			WRITE_ONCE(rcu_state.gp_activity, jiffies);  in rcu_gp_fqs_loop()
1954 			WRITE_ONCE(rcu_state.gp_activity, jiffies);  in rcu_gp_fqs_loop()
1956 			trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,  in rcu_gp_fqs_loop()
1960 			if (time_after(jiffies, rcu_state.jiffies_force_qs))  in rcu_gp_fqs_loop()
1963 				j = rcu_state.jiffies_force_qs - j;  in rcu_gp_fqs_loop()
1983 	WRITE_ONCE(rcu_state.gp_activity, jiffies);  in rcu_gp_cleanup()
1985 	rcu_state.gp_end = jiffies;  in rcu_gp_cleanup()
1986 	gp_duration = rcu_state.gp_end - rcu_state.gp_start;  in rcu_gp_cleanup()
1987 	if (gp_duration > rcu_state.gp_max)  in rcu_gp_cleanup()
1988 		rcu_state.gp_max = gp_duration;  in rcu_gp_cleanup()
2009 	new_gp_seq = rcu_state.gp_seq;  in rcu_gp_cleanup()
2032 		WRITE_ONCE(rcu_state.gp_activity, jiffies);  in rcu_gp_cleanup()
2039 	trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("end"));  in rcu_gp_cleanup()
2040 	rcu_seq_end(&rcu_state.gp_seq);  in rcu_gp_cleanup()
2041 	ASSERT_EXCLUSIVE_WRITER(rcu_state.gp_seq);  in rcu_gp_cleanup()
2042 	rcu_state.gp_state = RCU_GP_IDLE;  in rcu_gp_cleanup()
2054 		WRITE_ONCE(rcu_state.gp_flags, RCU_GP_FLAG_INIT);  in rcu_gp_cleanup()
2055 		WRITE_ONCE(rcu_state.gp_req_activity, jiffies);  in rcu_gp_cleanup()
2056 		trace_rcu_grace_period(rcu_state.name,  in rcu_gp_cleanup()
2057 				       rcu_state.gp_seq,  in rcu_gp_cleanup()
2060 		WRITE_ONCE(rcu_state.gp_flags,  in rcu_gp_cleanup()
2061 			   rcu_state.gp_flags & RCU_GP_FLAG_INIT);  in rcu_gp_cleanup()
2080 			trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,  in rcu_gp_kthread()
2082 			rcu_state.gp_state = RCU_GP_WAIT_GPS;  in rcu_gp_kthread()
2083 			swait_event_idle_exclusive(rcu_state.gp_wq,  in rcu_gp_kthread()
2084 					 READ_ONCE(rcu_state.gp_flags) &  in rcu_gp_kthread()
2087 			rcu_state.gp_state = RCU_GP_DONE_GPS;  in rcu_gp_kthread()
2092 			WRITE_ONCE(rcu_state.gp_activity, jiffies);  in rcu_gp_kthread()
2094 			trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,  in rcu_gp_kthread()
2102 		rcu_state.gp_state = RCU_GP_CLEANUP;  in rcu_gp_kthread()
2104 		rcu_state.gp_state = RCU_GP_CLEANED;  in rcu_gp_kthread()
2122 	WRITE_ONCE(rcu_state.gp_flags,  in rcu_report_qs_rsp()
2123 		   READ_ONCE(rcu_state.gp_flags) | RCU_GP_FLAG_FQS);  in rcu_report_qs_rsp()
2166 		trace_rcu_quiescent_state_report(rcu_state.name, rnp->gp_seq,  in rcu_report_qs_rnp()
2338 	trace_rcu_grace_period(rcu_state.name, READ_ONCE(rnp->gp_seq),  in rcutree_dying_cpu()
2429 		trace_rcu_batch_start(rcu_state.name,  in rcu_do_batch()
2431 		trace_rcu_batch_end(rcu_state.name, 0,  in rcu_do_batch()
2456 	trace_rcu_batch_start(rcu_state.name,  in rcu_do_batch()
2472 		trace_rcu_invoke_callback(rcu_state.name, rhp);  in rcu_do_batch()
2509 	trace_rcu_batch_end(rcu_state.name, count, !!rcl.head, need_resched(),  in rcu_do_batch()
2525 		rdp->n_force_qs_snap = rcu_state.n_force_qs;  in rcu_do_batch()
2588 	rcu_state.cbovld = rcu_state.cbovldnext;  in force_qs_rnp()
2589 	rcu_state.cbovldnext = false;  in force_qs_rnp()
2594 		rcu_state.cbovldnext |= !!rnp->cbovldmask;  in force_qs_rnp()
2640 		ret = (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) ||  in rcu_force_quiescent_state()
2653 	if (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) {  in rcu_force_quiescent_state()
2657 	WRITE_ONCE(rcu_state.gp_flags,  in rcu_force_quiescent_state()
2658 		   READ_ONCE(rcu_state.gp_flags) | RCU_GP_FLAG_FQS);  in rcu_force_quiescent_state()
2870 			if (rcu_state.n_force_qs == rdp->n_force_qs_snap &&  in __call_rcu_core()
2873 			rdp->n_force_qs_snap = rcu_state.n_force_qs;  in __call_rcu_core()
2973 		trace_rcu_kvfree_callback(rcu_state.name, head,  in __call_rcu()
2977 		trace_rcu_callback(rcu_state.name, head,  in __call_rcu()
3204 					rcu_state.name, bkvhead[i]->nr_records,  in kfree_rcu_work()
3212 						rcu_state.name,  in kfree_rcu_work()
3244 		trace_rcu_invoke_kvfree_callback(rcu_state.name, head, offset);  in kfree_rcu_work()
3653 	return rcu_seq_snap(&rcu_state.gp_seq);  in get_state_synchronize_rcu()
3673 	if (!rcu_seq_done(&rcu_state.gp_seq, oldstate))  in cond_synchronize_rcu()
3735 	trace_rcu_barrier(rcu_state.name, s, cpu,  in rcu_barrier_trace()
3736 			  atomic_read(&rcu_state.barrier_cpu_count), done);  in rcu_barrier_trace()
3751 	unsigned long __maybe_unused s = rcu_state.barrier_sequence;  in rcu_barrier_callback()
3753 	if (atomic_dec_and_test(&rcu_state.barrier_cpu_count)) {  in rcu_barrier_callback()
3755 		complete(&rcu_state.barrier_completion);  in rcu_barrier_callback()
3769 	rcu_barrier_trace(TPS("IRQ"), -1, rcu_state.barrier_sequence);  in rcu_barrier_func()
3775 		atomic_inc(&rcu_state.barrier_cpu_count);  in rcu_barrier_func()
3779 				  rcu_state.barrier_sequence);  in rcu_barrier_func()
3796 	unsigned long s = rcu_seq_snap(&rcu_state.barrier_sequence);  in rcu_barrier()
3801 	mutex_lock(&rcu_state.barrier_mutex);  in rcu_barrier()
3804 	if (rcu_seq_done(&rcu_state.barrier_sequence, s)) {  in rcu_barrier()
3806 				  rcu_state.barrier_sequence);  in rcu_barrier()
3808 		mutex_unlock(&rcu_state.barrier_mutex);  in rcu_barrier()
3813 	rcu_seq_start(&rcu_state.barrier_sequence);  in rcu_barrier()
3814 	rcu_barrier_trace(TPS("Inc1"), -1, rcu_state.barrier_sequence);  in rcu_barrier()
3823 	init_completion(&rcu_state.barrier_completion);  in rcu_barrier()
3824 	atomic_set(&rcu_state.barrier_cpu_count, 2);  in rcu_barrier()
3839 					  rcu_state.barrier_sequence);  in rcu_barrier()
3844 					  rcu_state.barrier_sequence);  in rcu_barrier()
3850 					  rcu_state.barrier_sequence);  in rcu_barrier()
3853 					  rcu_state.barrier_sequence);  in rcu_barrier()
3862 	if (atomic_sub_and_test(2, &rcu_state.barrier_cpu_count))  in rcu_barrier()
3863 		complete(&rcu_state.barrier_completion);  in rcu_barrier()
3866 	wait_for_completion(&rcu_state.barrier_completion);  in rcu_barrier()
3869 	rcu_barrier_trace(TPS("Inc2"), -1, rcu_state.barrier_sequence);  in rcu_barrier()
3870 	rcu_seq_end(&rcu_state.barrier_sequence);  in rcu_barrier()
3873 	mutex_unlock(&rcu_state.barrier_mutex);  in rcu_barrier()
3918 	rdp->rcu_ofl_gp_seq = rcu_state.gp_seq;  in rcu_boot_init_percpu_data()
3920 	rdp->rcu_onl_gp_seq = rcu_state.gp_seq;  in rcu_boot_init_percpu_data()
3945 	rdp->n_force_qs_snap = rcu_state.n_force_qs;  in rcutree_prepare_cpu()
3968 	trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuonl"));  in rcutree_prepare_cpu()
4065 	smp_store_release(&rcu_state.ncpus, rcu_state.ncpus + newcpu); /* ^^^ */  in rcu_cpu_starting()
4066 	ASSERT_EXCLUSIVE_WRITER(rcu_state.ncpus);  in rcu_cpu_starting()
4068 	rdp->rcu_onl_gp_seq = READ_ONCE(rcu_state.gp_seq);  in rcu_cpu_starting()
4069 	rdp->rcu_onl_gp_flags = READ_ONCE(rcu_state.gp_flags);  in rcu_cpu_starting()
4103 	raw_spin_lock(&rcu_state.ofl_lock);  in rcu_report_dead()
4105 	rdp->rcu_ofl_gp_seq = READ_ONCE(rcu_state.gp_seq);  in rcu_report_dead()
4106 	rdp->rcu_ofl_gp_flags = READ_ONCE(rcu_state.gp_flags);  in rcu_report_dead()
4114 	raw_spin_unlock(&rcu_state.ofl_lock);  in rcu_report_dead()
4218 	t = kthread_create(rcu_gp_kthread, NULL, "%s", rcu_state.name);  in rcu_spawn_gp_kthread()
4227 	WRITE_ONCE(rcu_state.gp_activity, jiffies);  in rcu_spawn_gp_kthread()
4228 	WRITE_ONCE(rcu_state.gp_req_activity, jiffies);  in rcu_spawn_gp_kthread()
4230 	smp_store_release(&rcu_state.gp_kthread, t);  /* ^^^ */  in rcu_spawn_gp_kthread()
4283 		rcu_state.level[i] =  in rcu_init_one()
4284 			rcu_state.level[i - 1] + num_rcu_lvl[i - 1];  in rcu_init_one()
4291 		rnp = rcu_state.level[i];  in rcu_init_one()
4299 			rnp->gp_seq = rcu_state.gp_seq;  in rcu_init_one()
4300 			rnp->gp_seq_needed = rcu_state.gp_seq;  in rcu_init_one()
4301 			rnp->completedqs = rcu_state.gp_seq;  in rcu_init_one()
4315 				rnp->parent = rcu_state.level[i - 1] +  in rcu_init_one()
4329 	init_swait_queue_head(&rcu_state.gp_wq);  in rcu_init_one()
4330 	init_swait_queue_head(&rcu_state.expedited_wq);  in rcu_init_one()