220 static bool __wake_nocb_gp(struct rcu_data *rdp_gp,  in __wake_nocb_gp()  argument
223 	__releases(rdp_gp->nocb_gp_lock)  in __wake_nocb_gp()
227 	if (!READ_ONCE(rdp_gp->nocb_gp_kthread)) {  in __wake_nocb_gp()
228 		raw_spin_unlock_irqrestore(&rdp_gp->nocb_gp_lock, flags);  in __wake_nocb_gp()
234 	if (rdp_gp->nocb_defer_wakeup > RCU_NOCB_WAKE_NOT) {  in __wake_nocb_gp()
235 		WRITE_ONCE(rdp_gp->nocb_defer_wakeup, RCU_NOCB_WAKE_NOT);  in __wake_nocb_gp()
236 		del_timer(&rdp_gp->nocb_timer);  in __wake_nocb_gp()
239 	if (force || READ_ONCE(rdp_gp->nocb_gp_sleep)) {  in __wake_nocb_gp()
240 		WRITE_ONCE(rdp_gp->nocb_gp_sleep, false);  in __wake_nocb_gp()
243 	raw_spin_unlock_irqrestore(&rdp_gp->nocb_gp_lock, flags);  in __wake_nocb_gp()
246 		wake_up_process(rdp_gp->nocb_gp_kthread);  in __wake_nocb_gp()
258 	struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;  in wake_nocb_gp()  local
260 	raw_spin_lock_irqsave(&rdp_gp->nocb_gp_lock, flags);  in wake_nocb_gp()
261 	return __wake_nocb_gp(rdp_gp, rdp, force, flags);  in wake_nocb_gp()
272 	struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;  in wake_nocb_gp_defer()  local
274 	raw_spin_lock_irqsave(&rdp_gp->nocb_gp_lock, flags);  in wake_nocb_gp_defer()
281 		mod_timer(&rdp_gp->nocb_timer, jiffies + 2);  in wake_nocb_gp_defer()
282 		WRITE_ONCE(rdp_gp->nocb_defer_wakeup, waketype);  in wake_nocb_gp_defer()
284 		if (rdp_gp->nocb_defer_wakeup < RCU_NOCB_WAKE)  in wake_nocb_gp_defer()
285 			mod_timer(&rdp_gp->nocb_timer, jiffies + 1);  in wake_nocb_gp_defer()
286 		if (rdp_gp->nocb_defer_wakeup < waketype)  in wake_nocb_gp_defer()
287 			WRITE_ONCE(rdp_gp->nocb_defer_wakeup, waketype);  in wake_nocb_gp_defer()
290 	raw_spin_unlock_irqrestore(&rdp_gp->nocb_gp_lock, flags);  in wake_nocb_gp_defer()
881 static bool do_nocb_deferred_wakeup_common(struct rcu_data *rdp_gp,  in do_nocb_deferred_wakeup_common()  argument
884 	__releases(rdp_gp->nocb_gp_lock)  in do_nocb_deferred_wakeup_common()
889 	if (!rcu_nocb_need_deferred_wakeup(rdp_gp, level)) {  in do_nocb_deferred_wakeup_common()
890 		raw_spin_unlock_irqrestore(&rdp_gp->nocb_gp_lock, flags);  in do_nocb_deferred_wakeup_common()
894 	ndw = rdp_gp->nocb_defer_wakeup;  in do_nocb_deferred_wakeup_common()
895 	ret = __wake_nocb_gp(rdp_gp, rdp, ndw == RCU_NOCB_WAKE_FORCE, flags);  in do_nocb_deferred_wakeup_common()
923 	struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;  in do_nocb_deferred_wakeup()  local
925 	if (!rdp_gp || !rcu_nocb_need_deferred_wakeup(rdp_gp, RCU_NOCB_WAKE))  in do_nocb_deferred_wakeup()
928 	raw_spin_lock_irqsave(&rdp_gp->nocb_gp_lock, flags);  in do_nocb_deferred_wakeup()
929 	return do_nocb_deferred_wakeup_common(rdp_gp, rdp, RCU_NOCB_WAKE, flags);  in do_nocb_deferred_wakeup()
943 	struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;  in rdp_offload_toggle()  local
958 	raw_spin_lock_irqsave(&rdp_gp->nocb_gp_lock, flags);  in rdp_offload_toggle()
959 	if (rdp_gp->nocb_gp_sleep) {  in rdp_offload_toggle()
960 		rdp_gp->nocb_gp_sleep = false;  in rdp_offload_toggle()
963 	raw_spin_unlock_irqrestore(&rdp_gp->nocb_gp_lock, flags);  in rdp_offload_toggle()
966 		wake_up_process(rdp_gp->nocb_gp_kthread);  in rdp_offload_toggle()
1184 	struct rcu_data *rdp_gp;  in rcu_spawn_one_nocb_kthread()  local
1195 	rdp_gp = rdp->nocb_gp_rdp;  in rcu_spawn_one_nocb_kthread()
1196 	if (!rdp_gp->nocb_gp_kthread) {  in rcu_spawn_one_nocb_kthread()
1197 		t = kthread_run(rcu_nocb_gp_kthread, rdp_gp,  in rcu_spawn_one_nocb_kthread()
1198 				"rcuog/%d", rdp_gp->cpu);  in rcu_spawn_one_nocb_kthread()
1201 		WRITE_ONCE(rdp_gp->nocb_gp_kthread, t);  in rcu_spawn_one_nocb_kthread()
1210 	WRITE_ONCE(rdp->nocb_gp_kthread, rdp_gp->nocb_gp_kthread);  in rcu_spawn_one_nocb_kthread()
1253 	struct rcu_data *rdp_gp = NULL;  /* Suppress misguided gcc warn. */  in rcu_organize_nocb_kthreads()  local
1275 			rdp_gp = rdp;  in rcu_organize_nocb_kthreads()
1288 			rdp->nocb_gp_rdp = rdp_gp;  in rcu_organize_nocb_kthreads()