28 static void rcu_exp_gp_seq_start(struct rcu_state *rsp)  in rcu_exp_gp_seq_start()  argument
30 	rcu_seq_start(&rsp->expedited_sequence);  in rcu_exp_gp_seq_start()
37 static __maybe_unused unsigned long rcu_exp_gp_seq_endval(struct rcu_state *rsp)  in rcu_exp_gp_seq_endval()  argument
39 	return rcu_seq_endval(&rsp->expedited_sequence);  in rcu_exp_gp_seq_endval()
45 static void rcu_exp_gp_seq_end(struct rcu_state *rsp)  in rcu_exp_gp_seq_end()  argument
47 	rcu_seq_end(&rsp->expedited_sequence);  in rcu_exp_gp_seq_end()
54 static unsigned long rcu_exp_gp_seq_snap(struct rcu_state *rsp)  in rcu_exp_gp_seq_snap()  argument
59 	s = rcu_seq_snap(&rsp->expedited_sequence);  in rcu_exp_gp_seq_snap()
60 	trace_rcu_exp_grace_period(rsp->name, s, TPS("snap"));  in rcu_exp_gp_seq_snap()
69 static bool rcu_exp_gp_seq_done(struct rcu_state *rsp, unsigned long s)  in rcu_exp_gp_seq_done()  argument
71 	return rcu_seq_done(&rsp->expedited_sequence, s);  in rcu_exp_gp_seq_done()
81 static void sync_exp_reset_tree_hotplug(struct rcu_state *rsp)  in sync_exp_reset_tree_hotplug()  argument
87 	int ncpus = smp_load_acquire(&rsp->ncpus); /* Order against locking. */  in sync_exp_reset_tree_hotplug()
92 	if (likely(ncpus == rsp->ncpus_snap))  in sync_exp_reset_tree_hotplug()
94 	rsp->ncpus_snap = ncpus;  in sync_exp_reset_tree_hotplug()
100 	rcu_for_each_leaf_node(rsp, rnp) {  in sync_exp_reset_tree_hotplug()
138 static void __maybe_unused sync_exp_reset_tree(struct rcu_state *rsp)  in sync_exp_reset_tree()  argument
143 	sync_exp_reset_tree_hotplug(rsp);  in sync_exp_reset_tree()
144 	rcu_for_each_node_breadth_first(rsp, rnp) {  in sync_exp_reset_tree()
197 static void __rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp,  in __rcu_report_exp_rnp()  argument
215 				swake_up_one(&rsp->expedited_wq);  in __rcu_report_exp_rnp()
232 static void __maybe_unused rcu_report_exp_rnp(struct rcu_state *rsp,  in rcu_report_exp_rnp()  argument
238 	__rcu_report_exp_rnp(rsp, rnp, wake, flags);  in rcu_report_exp_rnp()
245 static void rcu_report_exp_cpu_mult(struct rcu_state *rsp, struct rcu_node *rnp,  in rcu_report_exp_cpu_mult()  argument
256 	__rcu_report_exp_rnp(rsp, rnp, wake, flags); /* Releases rnp->lock. */  in rcu_report_exp_cpu_mult()
262 static void rcu_report_exp_rdp(struct rcu_state *rsp, struct rcu_data *rdp,  in rcu_report_exp_rdp()  argument
265 	rcu_report_exp_cpu_mult(rsp, rdp->mynode, rdp->grpmask, wake);  in rcu_report_exp_rdp()
269 static bool sync_exp_work_done(struct rcu_state *rsp, unsigned long s)  in sync_exp_work_done()  argument
271 	if (rcu_exp_gp_seq_done(rsp, s)) {  in sync_exp_work_done()
272 		trace_rcu_exp_grace_period(rsp->name, s, TPS("done"));  in sync_exp_work_done()
287 static bool exp_funnel_lock(struct rcu_state *rsp, unsigned long s)  in exp_funnel_lock()  argument
289 	struct rcu_data *rdp = per_cpu_ptr(rsp->rda, raw_smp_processor_id());  in exp_funnel_lock()
291 	struct rcu_node *rnp_root = rcu_get_root(rsp);  in exp_funnel_lock()
297 	    mutex_trylock(&rsp->exp_mutex))  in exp_funnel_lock()
308 		if (sync_exp_work_done(rsp, s))  in exp_funnel_lock()
317 			trace_rcu_exp_funnel_lock(rsp->name, rnp->level,  in exp_funnel_lock()
321 				   sync_exp_work_done(rsp, s));  in exp_funnel_lock()
326 		trace_rcu_exp_funnel_lock(rsp->name, rnp->level, rnp->grplo,  in exp_funnel_lock()
329 	mutex_lock(&rsp->exp_mutex);  in exp_funnel_lock()
331 	if (sync_exp_work_done(rsp, s)) {  in exp_funnel_lock()
332 		mutex_unlock(&rsp->exp_mutex);  in exp_funnel_lock()
335 	rcu_exp_gp_seq_start(rsp);  in exp_funnel_lock()
336 	trace_rcu_exp_grace_period(rsp->name, s, TPS("start"));  in exp_funnel_lock()
345 	struct rcu_state *rsp = data;  in sync_sched_exp_handler()  local
347 	rdp = this_cpu_ptr(rsp->rda);  in sync_sched_exp_handler()
369 	struct rcu_state *rsp = &rcu_sched_state;  in sync_sched_exp_online_cleanup()  local
371 	rdp = per_cpu_ptr(rsp->rda, cpu);  in sync_sched_exp_online_cleanup()
375 	ret = smp_call_function_single(cpu, sync_sched_exp_handler, rsp, 0);  in sync_sched_exp_online_cleanup()
394 	struct rcu_state *rsp = rewp->rew_rsp;  in sync_rcu_exp_select_node_cpus()  local
403 		struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);  in sync_rcu_exp_select_node_cpus()
432 		struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);  in sync_rcu_exp_select_node_cpus()
442 		ret = smp_call_function_single(cpu, func, rsp, 0);  in sync_rcu_exp_select_node_cpus()
453 			trace_rcu_exp_grace_period(rsp->name, rcu_exp_gp_seq_endval(rsp), TPS("selectofl"));  in sync_rcu_exp_select_node_cpus()
465 		rcu_report_exp_cpu_mult(rsp, rnp, mask_ofl_test, false);  in sync_rcu_exp_select_node_cpus()
472 static void sync_rcu_exp_select_cpus(struct rcu_state *rsp,  in sync_rcu_exp_select_cpus()  argument
478 	trace_rcu_exp_grace_period(rsp->name, rcu_exp_gp_seq_endval(rsp), TPS("reset"));  in sync_rcu_exp_select_cpus()
479 	sync_exp_reset_tree(rsp);  in sync_rcu_exp_select_cpus()
480 	trace_rcu_exp_grace_period(rsp->name, rcu_exp_gp_seq_endval(rsp), TPS("select"));  in sync_rcu_exp_select_cpus()
483 	rcu_for_each_leaf_node(rsp, rnp) {  in sync_rcu_exp_select_cpus()
488 		rnp->rew.rew_rsp = rsp;  in sync_rcu_exp_select_cpus()
491 		    rcu_is_last_leaf_node(rsp, rnp)) {  in sync_rcu_exp_select_cpus()
508 	rcu_for_each_leaf_node(rsp, rnp)  in sync_rcu_exp_select_cpus()
513 static void synchronize_sched_expedited_wait(struct rcu_state *rsp)  in synchronize_sched_expedited_wait()  argument
521 	struct rcu_node *rnp_root = rcu_get_root(rsp);  in synchronize_sched_expedited_wait()
524 	trace_rcu_exp_grace_period(rsp->name, rcu_exp_gp_seq_endval(rsp), TPS("startwait"));  in synchronize_sched_expedited_wait()
530 				rsp->expedited_wq,  in synchronize_sched_expedited_wait()
540 		       rsp->name);  in synchronize_sched_expedited_wait()
542 		rcu_for_each_leaf_node(rsp, rnp) {  in synchronize_sched_expedited_wait()
551 				rdp = per_cpu_ptr(rsp->rda, cpu);  in synchronize_sched_expedited_wait()
559 			jiffies - jiffies_start, rsp->expedited_sequence,  in synchronize_sched_expedited_wait()
563 			rcu_for_each_node_breadth_first(rsp, rnp) {  in synchronize_sched_expedited_wait()
575 		rcu_for_each_leaf_node(rsp, rnp) {  in synchronize_sched_expedited_wait()
593 static void rcu_exp_wait_wake(struct rcu_state *rsp, unsigned long s)  in rcu_exp_wait_wake()  argument
597 	synchronize_sched_expedited_wait(rsp);  in rcu_exp_wait_wake()
598 	rcu_exp_gp_seq_end(rsp);  in rcu_exp_wait_wake()
599 	trace_rcu_exp_grace_period(rsp->name, s, TPS("end"));  in rcu_exp_wait_wake()
605 	mutex_lock(&rsp->exp_wake_mutex);  in rcu_exp_wait_wake()
607 	rcu_for_each_node_breadth_first(rsp, rnp) {  in rcu_exp_wait_wake()
616 		wake_up_all(&rnp->exp_wq[rcu_seq_ctr(rsp->expedited_sequence) & 0x3]);  in rcu_exp_wait_wake()
618 	trace_rcu_exp_grace_period(rsp->name, s, TPS("endwake"));  in rcu_exp_wait_wake()
619 	mutex_unlock(&rsp->exp_wake_mutex);  in rcu_exp_wait_wake()
626 static void rcu_exp_sel_wait_wake(struct rcu_state *rsp,  in rcu_exp_sel_wait_wake()  argument
630 	sync_rcu_exp_select_cpus(rsp, func);  in rcu_exp_sel_wait_wake()
633 	rcu_exp_wait_wake(rsp, s);  in rcu_exp_sel_wait_wake()
651 static void _synchronize_rcu_expedited(struct rcu_state *rsp,  in _synchronize_rcu_expedited()  argument
661 		wait_rcu_gp(rsp->call);  in _synchronize_rcu_expedited()
666 	s = rcu_exp_gp_seq_snap(rsp);  in _synchronize_rcu_expedited()
667 	if (exp_funnel_lock(rsp, s))  in _synchronize_rcu_expedited()
673 		rcu_exp_sel_wait_wake(rsp, func, s);  in _synchronize_rcu_expedited()
677 		rew.rew_rsp = rsp;  in _synchronize_rcu_expedited()
684 	rdp = per_cpu_ptr(rsp->rda, raw_smp_processor_id());  in _synchronize_rcu_expedited()
685 	rnp = rcu_get_root(rsp);  in _synchronize_rcu_expedited()
687 		   sync_exp_work_done(rsp, s));  in _synchronize_rcu_expedited()
691 	mutex_unlock(&rsp->exp_mutex);  in _synchronize_rcu_expedited()
712 	struct rcu_state *rsp = &rcu_sched_state;  in synchronize_sched_expedited()  local
723 	_synchronize_rcu_expedited(rsp, sync_sched_exp_handler);  in synchronize_sched_expedited()
739 	struct rcu_state *rsp = info;  in sync_rcu_exp_handler()  local
762 	rdp = this_cpu_ptr(rsp->rda);  in sync_rcu_exp_handler()
763 	rcu_report_exp_rdp(rsp, rdp, true);  in sync_rcu_exp_handler()
786 	struct rcu_state *rsp = rcu_state_p;  in synchronize_rcu_expedited()  local
795 	_synchronize_rcu_expedited(rsp, sync_rcu_exp_handler);  in synchronize_rcu_expedited()