130 static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp,
290 		rcu_report_exp_rdp(rdp->rsp, rdp, true);  in rcu_preempt_ctxt_queue()
360 		trace_rcu_preempt_task(rdp->rsp->name,  in rcu_preempt_note_context_switch()
636 static void rcu_print_detail_task_stall(struct rcu_state *rsp)  in rcu_print_detail_task_stall()  argument
638 	struct rcu_node *rnp = rcu_get_root(rsp);  in rcu_print_detail_task_stall()
641 	rcu_for_each_leaf_node(rsp, rnp)  in rcu_print_detail_task_stall()
710 rcu_preempt_check_blocked_tasks(struct rcu_state *rsp, struct rcu_node *rnp)  in rcu_preempt_check_blocked_tasks()  argument
716 		dump_blkd_tasks(rsp, rnp, 10);  in rcu_preempt_check_blocked_tasks()
737 	struct rcu_state *rsp = &rcu_preempt_state;  in rcu_preempt_check_callbacks()  local
748 	    time_after(jiffies, rsp->gp_start + HZ))  in rcu_preempt_check_callbacks()
869 dump_blkd_tasks(struct rcu_state *rsp, struct rcu_node *rnp, int ncheck)  in dump_blkd_tasks()  argument
896 		rdp = per_cpu_ptr(rsp->rda, cpu);  in dump_blkd_tasks()
947 static void rcu_print_detail_task_stall(struct rcu_state *rsp)  in rcu_print_detail_task_stall()  argument
976 rcu_preempt_check_blocked_tasks(struct rcu_state *rsp, struct rcu_node *rnp)  in rcu_preempt_check_blocked_tasks()  argument
1018 dump_blkd_tasks(struct rcu_state *rsp, struct rcu_node *rnp, int ncheck)  in dump_blkd_tasks()  argument
1215 static int rcu_spawn_one_boost_kthread(struct rcu_state *rsp,  in rcu_spawn_one_boost_kthread()  argument
1218 	int rnp_index = rnp - &rsp->node[0];  in rcu_spawn_one_boost_kthread()
1223 	if (rcu_state_p != rsp)  in rcu_spawn_one_boost_kthread()
1229 	rsp->boost = 1;  in rcu_spawn_one_boost_kthread()
1491 	struct rcu_state *rsp;  in rcu_try_advance_all_cbs()  local
1498 	for_each_rcu_flavor(rsp) {  in rcu_try_advance_all_cbs()
1499 		rdp = this_cpu_ptr(rsp->rda);  in rcu_try_advance_all_cbs()
1511 			note_gp_changes(rsp, rdp);  in rcu_try_advance_all_cbs()
1578 	struct rcu_state *rsp;  in rcu_prepare_for_idle()  local
1616 	for_each_rcu_flavor(rsp) {  in rcu_prepare_for_idle()
1617 		rdp = this_cpu_ptr(rsp->rda);  in rcu_prepare_for_idle()
1622 		needwake = rcu_accelerate_cbs(rsp, rnp, rdp);  in rcu_prepare_for_idle()
1625 			rcu_gp_kthread_wake(rsp);  in rcu_prepare_for_idle()
1681 	struct rcu_state *rsp;  in rcu_oom_notify_cpu()  local
1684 	for_each_rcu_flavor(rsp) {  in rcu_oom_notify_cpu()
1685 		rdp = raw_cpu_ptr(rsp->rda);  in rcu_oom_notify_cpu()
1688 			rsp->call(&rdp->oom_head, rcu_oom_callback);  in rcu_oom_notify_cpu()
1780 static void print_cpu_stall_info(struct rcu_state *rsp, int cpu)  in print_cpu_stall_info()  argument
1784 	struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);  in print_cpu_stall_info()
1795 	ticks_value = rcu_seq_ctr(rsp->gp_seq - rdp->gp_seq);  in print_cpu_stall_info()
1816 	       READ_ONCE(rsp->n_force_qs) - rsp->n_force_qs_gpstart,  in print_cpu_stall_info()
1836 	struct rcu_state *rsp;  in increment_cpu_stall_ticks()  local
1838 	for_each_rcu_flavor(rsp)  in increment_cpu_stall_ticks()
1839 		raw_cpu_inc(rsp->rda->ticks_this_gp);  in increment_cpu_stall_ticks()
1961 	trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, reason);  in wake_nocb_leader_defer()
1969 static bool rcu_nocb_cpu_needs_barrier(struct rcu_state *rsp, int cpu)  in rcu_nocb_cpu_needs_barrier()  argument
1971 	struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);  in rcu_nocb_cpu_needs_barrier()
2040 		trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,  in __call_rcu_nocb_enqueue()
2049 			trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,  in __call_rcu_nocb_enqueue()
2060 			trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,  in __call_rcu_nocb_enqueue()
2068 		trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("WakeNot"));  in __call_rcu_nocb_enqueue()
2090 		trace_rcu_kfree_callback(rdp->rsp->name, rhp,  in __call_rcu_nocb()
2095 		trace_rcu_callback(rdp->rsp->name, rhp,  in __call_rcu_nocb()
2145 	c = rcu_seq_snap(&rdp->rsp->gp_seq);  in rcu_nocb_wait_gp()
2153 			rcu_gp_kthread_wake(rdp->rsp);  in rcu_nocb_wait_gp()
2190 		trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu, TPS("Sleep"));  in nocb_leader_wait()
2200 		trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu, TPS("Poll"));  in nocb_leader_wait()
2227 			trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu,  in nocb_leader_wait()
2272 		trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("FollowerSleep"));  in nocb_follower_wait()
2280 		trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("WokeEmpty"));  in nocb_follower_wait()
2315 		trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("WokeNonEmpty"));  in rcu_nocb_kthread()
2318 		trace_rcu_batch_start(rdp->rsp->name,  in rcu_nocb_kthread()
2326 				trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,  in rcu_nocb_kthread()
2329 				trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu,  in rcu_nocb_kthread()
2335 			if (__rcu_reclaim(rdp->rsp->name, list))  in rcu_nocb_kthread()
2342 		trace_rcu_batch_end(rdp->rsp->name, c, !!list, 0, 0, 1);  in rcu_nocb_kthread()
2370 	trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, TPS("DeferredWake"));  in do_nocb_deferred_wakeup_common()
2396 	struct rcu_state *rsp;  in rcu_init_nohz()  local
2430 	for_each_rcu_flavor(rsp) {  in rcu_init_nohz()
2432 			init_nocb_callback_list(per_cpu_ptr(rsp->rda, cpu));  in rcu_init_nohz()
2433 		rcu_organize_nocb_kthreads(rsp);  in rcu_init_nohz()
2453 static void rcu_spawn_one_nocb_kthread(struct rcu_state *rsp, int cpu)  in rcu_spawn_one_nocb_kthread()  argument
2458 	struct rcu_data *rdp_spawn = per_cpu_ptr(rsp->rda, cpu);  in rcu_spawn_one_nocb_kthread()
2490 			"rcuo%c/%d", rsp->abbr, cpu);  in rcu_spawn_one_nocb_kthread()
2501 	struct rcu_state *rsp;  in rcu_spawn_all_nocb_kthreads()  local
2504 		for_each_rcu_flavor(rsp)  in rcu_spawn_all_nocb_kthreads()
2505 			rcu_spawn_one_nocb_kthread(rsp, cpu);  in rcu_spawn_all_nocb_kthreads()
2529 static void __init rcu_organize_nocb_kthreads(struct rcu_state *rsp)  in rcu_organize_nocb_kthreads()  argument
2551 		rdp = per_cpu_ptr(rsp->rda, cpu);  in rcu_organize_nocb_kthreads()
2588 static bool rcu_nocb_cpu_needs_barrier(struct rcu_state *rsp, int cpu)  in rcu_nocb_cpu_needs_barrier()  argument
2657 static bool rcu_nohz_full_cpu(struct rcu_state *rsp)  in rcu_nohz_full_cpu()  argument
2661 	    (!rcu_gp_in_progress(rsp) ||  in rcu_nohz_full_cpu()
2662 	     ULONG_CMP_LT(jiffies, READ_ONCE(rsp->gp_start) + HZ)))  in rcu_nohz_full_cpu()