19 static inline int rcu_lockdep_is_held_nocb(struct rcu_data *rdp)  in rcu_lockdep_is_held_nocb()  argument
21 	return lockdep_is_held(&rdp->nocb_lock);  in rcu_lockdep_is_held_nocb()
24 static inline bool rcu_current_is_nocb_kthread(struct rcu_data *rdp)  in rcu_current_is_nocb_kthread()  argument
27 	if (!rdp->nocb_cb_kthread || !rdp->nocb_gp_kthread)  in rcu_current_is_nocb_kthread()
30 	if (current == rdp->nocb_cb_kthread || current == rdp->nocb_gp_kthread)  in rcu_current_is_nocb_kthread()
97 static void rcu_nocb_bypass_lock(struct rcu_data *rdp)  in rcu_nocb_bypass_lock()  argument
98 	__acquires(&rdp->nocb_bypass_lock)  in rcu_nocb_bypass_lock()
101 	if (raw_spin_trylock(&rdp->nocb_bypass_lock))  in rcu_nocb_bypass_lock()
103 	atomic_inc(&rdp->nocb_lock_contended);  in rcu_nocb_bypass_lock()
104 	WARN_ON_ONCE(smp_processor_id() != rdp->cpu);  in rcu_nocb_bypass_lock()
106 	raw_spin_lock(&rdp->nocb_bypass_lock);  in rcu_nocb_bypass_lock()
108 	atomic_dec(&rdp->nocb_lock_contended);  in rcu_nocb_bypass_lock()
121 static void rcu_nocb_wait_contended(struct rcu_data *rdp)  in rcu_nocb_wait_contended()  argument
123 	WARN_ON_ONCE(smp_processor_id() != rdp->cpu);  in rcu_nocb_wait_contended()
124 	while (WARN_ON_ONCE(atomic_read(&rdp->nocb_lock_contended)))  in rcu_nocb_wait_contended()
132 static bool rcu_nocb_bypass_trylock(struct rcu_data *rdp)  in rcu_nocb_bypass_trylock()  argument
135 	return raw_spin_trylock(&rdp->nocb_bypass_lock);  in rcu_nocb_bypass_trylock()
141 static void rcu_nocb_bypass_unlock(struct rcu_data *rdp)  in rcu_nocb_bypass_unlock()  argument
142 	__releases(&rdp->nocb_bypass_lock)  in rcu_nocb_bypass_unlock()
145 	raw_spin_unlock(&rdp->nocb_bypass_lock);  in rcu_nocb_bypass_unlock()
152 static void rcu_nocb_lock(struct rcu_data *rdp)  in rcu_nocb_lock()  argument
155 	if (!rcu_rdp_is_offloaded(rdp))  in rcu_nocb_lock()
157 	raw_spin_lock(&rdp->nocb_lock);  in rcu_nocb_lock()
164 static void rcu_nocb_unlock(struct rcu_data *rdp)  in rcu_nocb_unlock()  argument
166 	if (rcu_rdp_is_offloaded(rdp)) {  in rcu_nocb_unlock()
168 		raw_spin_unlock(&rdp->nocb_lock);  in rcu_nocb_unlock()
176 static void rcu_nocb_unlock_irqrestore(struct rcu_data *rdp,  in rcu_nocb_unlock_irqrestore()  argument
179 	if (rcu_rdp_is_offloaded(rdp)) {  in rcu_nocb_unlock_irqrestore()
181 		raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags);  in rcu_nocb_unlock_irqrestore()
188 static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp)  in rcu_lockdep_assert_cblist_protected()  argument
191 	if (rcu_rdp_is_offloaded(rdp))  in rcu_lockdep_assert_cblist_protected()
192 		lockdep_assert_held(&rdp->nocb_lock);  in rcu_lockdep_assert_cblist_protected()
216 			   struct rcu_data *rdp,  in __wake_nocb_gp()
224 		trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,  in __wake_nocb_gp()
240 		trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("DoWake"));  in __wake_nocb_gp()
250 static bool wake_nocb_gp(struct rcu_data *rdp, bool force)  in wake_nocb_gp()  argument
253 	struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;  in wake_nocb_gp()
256 	return __wake_nocb_gp(rdp_gp, rdp, force, flags);  in wake_nocb_gp()
263 static void wake_nocb_gp_defer(struct rcu_data *rdp, int waketype,  in wake_nocb_gp_defer()  argument
267 	struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;  in wake_nocb_gp_defer()
287 	trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, reason);  in wake_nocb_gp_defer()
298 static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp,  in rcu_nocb_do_flush_bypass()  argument
303 	WARN_ON_ONCE(!rcu_rdp_is_offloaded(rdp));  in rcu_nocb_do_flush_bypass()
304 	rcu_lockdep_assert_cblist_protected(rdp);  in rcu_nocb_do_flush_bypass()
305 	lockdep_assert_held(&rdp->nocb_bypass_lock);  in rcu_nocb_do_flush_bypass()
306 	if (rhp && !rcu_cblist_n_cbs(&rdp->nocb_bypass)) {  in rcu_nocb_do_flush_bypass()
307 		raw_spin_unlock(&rdp->nocb_bypass_lock);  in rcu_nocb_do_flush_bypass()
312 		rcu_segcblist_inc_len(&rdp->cblist); /* Must precede enqueue. */  in rcu_nocb_do_flush_bypass()
313 	rcu_cblist_flush_enqueue(&rcl, &rdp->nocb_bypass, rhp);  in rcu_nocb_do_flush_bypass()
314 	rcu_segcblist_insert_pend_cbs(&rdp->cblist, &rcl);  in rcu_nocb_do_flush_bypass()
315 	WRITE_ONCE(rdp->nocb_bypass_first, j);  in rcu_nocb_do_flush_bypass()
316 	rcu_nocb_bypass_unlock(rdp);  in rcu_nocb_do_flush_bypass()
328 static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp,  in rcu_nocb_flush_bypass()  argument
331 	if (!rcu_rdp_is_offloaded(rdp))  in rcu_nocb_flush_bypass()
333 	rcu_lockdep_assert_cblist_protected(rdp);  in rcu_nocb_flush_bypass()
334 	rcu_nocb_bypass_lock(rdp);  in rcu_nocb_flush_bypass()
335 	return rcu_nocb_do_flush_bypass(rdp, rhp, j);  in rcu_nocb_flush_bypass()
342 static void rcu_nocb_try_flush_bypass(struct rcu_data *rdp, unsigned long j)  in rcu_nocb_try_flush_bypass()  argument
344 	rcu_lockdep_assert_cblist_protected(rdp);  in rcu_nocb_try_flush_bypass()
345 	if (!rcu_rdp_is_offloaded(rdp) ||  in rcu_nocb_try_flush_bypass()
346 	    !rcu_nocb_bypass_trylock(rdp))  in rcu_nocb_try_flush_bypass()
348 	WARN_ON_ONCE(!rcu_nocb_do_flush_bypass(rdp, NULL, j));  in rcu_nocb_try_flush_bypass()
369 static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp,  in rcu_nocb_try_bypass()  argument
375 	long ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass);  in rcu_nocb_try_bypass()
381 	if (!rcu_rdp_is_offloaded(rdp)) {  in rcu_nocb_try_bypass()
382 		*was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist);  in rcu_nocb_try_bypass()
388 	if (!rcu_segcblist_completely_offloaded(&rdp->cblist)) {  in rcu_nocb_try_bypass()
389 		rcu_nocb_lock(rdp);  in rcu_nocb_try_bypass()
390 		*was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist);  in rcu_nocb_try_bypass()
396 		rcu_nocb_lock(rdp);  in rcu_nocb_try_bypass()
397 		WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass));  in rcu_nocb_try_bypass()
398 		*was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist);  in rcu_nocb_try_bypass()
404 	if (j == rdp->nocb_nobypass_last) {  in rcu_nocb_try_bypass()
405 		c = rdp->nocb_nobypass_count + 1;  in rcu_nocb_try_bypass()
407 		WRITE_ONCE(rdp->nocb_nobypass_last, j);  in rcu_nocb_try_bypass()
408 		c = rdp->nocb_nobypass_count - nocb_nobypass_lim_per_jiffy;  in rcu_nocb_try_bypass()
409 		if (ULONG_CMP_LT(rdp->nocb_nobypass_count,  in rcu_nocb_try_bypass()
415 	WRITE_ONCE(rdp->nocb_nobypass_count, c);  in rcu_nocb_try_bypass()
420 	if (rdp->nocb_nobypass_count < nocb_nobypass_lim_per_jiffy) {  in rcu_nocb_try_bypass()
421 		rcu_nocb_lock(rdp);  in rcu_nocb_try_bypass()
422 		*was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist);  in rcu_nocb_try_bypass()
424 			trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,  in rcu_nocb_try_bypass()
426 		WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, j));  in rcu_nocb_try_bypass()
427 		WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass));  in rcu_nocb_try_bypass()
433 	if ((ncbs && j != READ_ONCE(rdp->nocb_bypass_first)) ||  in rcu_nocb_try_bypass()
435 		rcu_nocb_lock(rdp);  in rcu_nocb_try_bypass()
436 		if (!rcu_nocb_flush_bypass(rdp, rhp, j)) {  in rcu_nocb_try_bypass()
437 			*was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist);  in rcu_nocb_try_bypass()
439 				trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,  in rcu_nocb_try_bypass()
441 			WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass));  in rcu_nocb_try_bypass()
444 		if (j != rdp->nocb_gp_adv_time &&  in rcu_nocb_try_bypass()
445 		    rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) &&  in rcu_nocb_try_bypass()
446 		    rcu_seq_done(&rdp->mynode->gp_seq, cur_gp_seq)) {  in rcu_nocb_try_bypass()
447 			rcu_advance_cbs_nowake(rdp->mynode, rdp);  in rcu_nocb_try_bypass()
448 			rdp->nocb_gp_adv_time = j;  in rcu_nocb_try_bypass()
450 		rcu_nocb_unlock_irqrestore(rdp, flags);  in rcu_nocb_try_bypass()
455 	rcu_nocb_wait_contended(rdp);  in rcu_nocb_try_bypass()
456 	rcu_nocb_bypass_lock(rdp);  in rcu_nocb_try_bypass()
457 	ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass);  in rcu_nocb_try_bypass()
458 	rcu_segcblist_inc_len(&rdp->cblist); /* Must precede enqueue. */  in rcu_nocb_try_bypass()
459 	rcu_cblist_enqueue(&rdp->nocb_bypass, rhp);  in rcu_nocb_try_bypass()
461 		WRITE_ONCE(rdp->nocb_bypass_first, j);  in rcu_nocb_try_bypass()
462 		trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("FirstBQ"));  in rcu_nocb_try_bypass()
464 	rcu_nocb_bypass_unlock(rdp);  in rcu_nocb_try_bypass()
470 		rcu_nocb_lock(rdp); // Rare during call_rcu() flood.  in rcu_nocb_try_bypass()
471 		if (!rcu_segcblist_pend_cbs(&rdp->cblist)) {  in rcu_nocb_try_bypass()
472 			trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,  in rcu_nocb_try_bypass()
474 			__call_rcu_nocb_wake(rdp, true, flags);  in rcu_nocb_try_bypass()
476 			trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,  in rcu_nocb_try_bypass()
478 			rcu_nocb_unlock_irqrestore(rdp, flags);  in rcu_nocb_try_bypass()
490 static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone,  in __call_rcu_nocb_wake()  argument
492 				 __releases(rdp->nocb_lock)  in __call_rcu_nocb_wake()
500 	t = READ_ONCE(rdp->nocb_gp_kthread);  in __call_rcu_nocb_wake()
502 		rcu_nocb_unlock_irqrestore(rdp, flags);  in __call_rcu_nocb_wake()
503 		trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,  in __call_rcu_nocb_wake()
508 	len = rcu_segcblist_n_cbs(&rdp->cblist);  in __call_rcu_nocb_wake()
510 		rdp->qlen_last_fqs_check = len;  in __call_rcu_nocb_wake()
513 			rcu_nocb_unlock_irqrestore(rdp, flags);  in __call_rcu_nocb_wake()
514 			wake_nocb_gp(rdp, false);  in __call_rcu_nocb_wake()
515 			trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,  in __call_rcu_nocb_wake()
518 			rcu_nocb_unlock_irqrestore(rdp, flags);  in __call_rcu_nocb_wake()
519 			wake_nocb_gp_defer(rdp, RCU_NOCB_WAKE,  in __call_rcu_nocb_wake()
522 	} else if (len > rdp->qlen_last_fqs_check + qhimark) {  in __call_rcu_nocb_wake()
524 		rdp->qlen_last_fqs_check = len;  in __call_rcu_nocb_wake()
526 		if (j != rdp->nocb_gp_adv_time &&  in __call_rcu_nocb_wake()
527 		    rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) &&  in __call_rcu_nocb_wake()
528 		    rcu_seq_done(&rdp->mynode->gp_seq, cur_gp_seq)) {  in __call_rcu_nocb_wake()
529 			rcu_advance_cbs_nowake(rdp->mynode, rdp);  in __call_rcu_nocb_wake()
530 			rdp->nocb_gp_adv_time = j;  in __call_rcu_nocb_wake()
533 		if ((rdp->nocb_cb_sleep ||  in __call_rcu_nocb_wake()
534 		     !rcu_segcblist_ready_cbs(&rdp->cblist)) &&  in __call_rcu_nocb_wake()
535 		    !timer_pending(&rdp->nocb_timer)) {  in __call_rcu_nocb_wake()
536 			rcu_nocb_unlock_irqrestore(rdp, flags);  in __call_rcu_nocb_wake()
537 			wake_nocb_gp_defer(rdp, RCU_NOCB_WAKE_FORCE,  in __call_rcu_nocb_wake()
540 			rcu_nocb_unlock_irqrestore(rdp, flags);  in __call_rcu_nocb_wake()
541 			trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WakeNot"));  in __call_rcu_nocb_wake()
544 		rcu_nocb_unlock_irqrestore(rdp, flags);  in __call_rcu_nocb_wake()
545 		trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WakeNot"));  in __call_rcu_nocb_wake()
549 static int nocb_gp_toggle_rdp(struct rcu_data *rdp,  in nocb_gp_toggle_rdp()  argument
552 	struct rcu_segcblist *cblist = &rdp->cblist;  in nocb_gp_toggle_rdp()
556 	rcu_nocb_lock_irqsave(rdp, flags);  in nocb_gp_toggle_rdp()
582 	rcu_nocb_unlock_irqrestore(rdp, flags);  in nocb_gp_toggle_rdp()
611 	struct rcu_data *rdp, *rdp_toggling = NULL;  in nocb_gp_wait()  local
636 	list_for_each_entry(rdp, &my_rdp->nocb_head_rdp, nocb_entry_rdp) {  in nocb_gp_wait()
637 		trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("Check"));  in nocb_gp_wait()
638 		rcu_nocb_lock_irqsave(rdp, flags);  in nocb_gp_wait()
639 		lockdep_assert_held(&rdp->nocb_lock);  in nocb_gp_wait()
640 		bypass_ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass);  in nocb_gp_wait()
642 		    (time_after(j, READ_ONCE(rdp->nocb_bypass_first) + 1) ||  in nocb_gp_wait()
645 			(void)rcu_nocb_try_flush_bypass(rdp, j);  in nocb_gp_wait()
646 			bypass_ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass);  in nocb_gp_wait()
647 		} else if (!bypass_ncbs && rcu_segcblist_empty(&rdp->cblist)) {  in nocb_gp_wait()
648 			rcu_nocb_unlock_irqrestore(rdp, flags);  in nocb_gp_wait()
652 			trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,  in nocb_gp_wait()
656 		rnp = rdp->mynode;  in nocb_gp_wait()
660 		if (!rcu_segcblist_restempty(&rdp->cblist,  in nocb_gp_wait()
662 		    (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) &&  in nocb_gp_wait()
665 			needwake_gp = rcu_advance_cbs(rnp, rdp);  in nocb_gp_wait()
666 			wasempty = rcu_segcblist_restempty(&rdp->cblist,  in nocb_gp_wait()
672 			     !rcu_segcblist_restempty(&rdp->cblist,  in nocb_gp_wait()
674 		if (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq)) {  in nocb_gp_wait()
679 			trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,  in nocb_gp_wait()
682 		if (rcu_segcblist_ready_cbs(&rdp->cblist)) {  in nocb_gp_wait()
683 			needwake = rdp->nocb_cb_sleep;  in nocb_gp_wait()
684 			WRITE_ONCE(rdp->nocb_cb_sleep, false);  in nocb_gp_wait()
689 		rcu_nocb_unlock_irqrestore(rdp, flags);  in nocb_gp_wait()
691 			swake_up_one(&rdp->nocb_cb_wq);  in nocb_gp_wait()
791 	struct rcu_data *rdp = arg;  in rcu_nocb_gp_kthread()  local
794 		WRITE_ONCE(rdp->nocb_gp_loops, rdp->nocb_gp_loops + 1);  in rcu_nocb_gp_kthread()
795 		nocb_gp_wait(rdp);  in rcu_nocb_gp_kthread()
801 static inline bool nocb_cb_can_run(struct rcu_data *rdp)  in nocb_cb_can_run()  argument
805 	return rcu_segcblist_test_flags(&rdp->cblist, flags);  in nocb_cb_can_run()
808 static inline bool nocb_cb_wait_cond(struct rcu_data *rdp)  in nocb_cb_wait_cond()  argument
810 	return nocb_cb_can_run(rdp) && !READ_ONCE(rdp->nocb_cb_sleep);  in nocb_cb_wait_cond()
817 static void nocb_cb_wait(struct rcu_data *rdp)  in nocb_cb_wait()  argument
819 	struct rcu_segcblist *cblist = &rdp->cblist;  in nocb_cb_wait()
825 	struct rcu_node *rnp = rdp->mynode;  in nocb_cb_wait()
828 		swait_event_interruptible_exclusive(rdp->nocb_cb_wq,  in nocb_cb_wait()
829 						    nocb_cb_wait_cond(rdp));  in nocb_cb_wait()
832 		if (smp_load_acquire(&rdp->nocb_cb_sleep)) { // ^^^  in nocb_cb_wait()
834 			trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WokeEmpty"));  in nocb_cb_wait()
836 	} while (!nocb_cb_can_run(rdp));  in nocb_cb_wait()
849 	rcu_do_batch(rdp);  in nocb_cb_wait()
852 	rcu_nocb_lock_irqsave(rdp, flags);  in nocb_cb_wait()
856 		needwake_gp = rcu_advance_cbs(rdp->mynode, rdp);  in nocb_cb_wait()
880 	WRITE_ONCE(rdp->nocb_cb_sleep, can_sleep);  in nocb_cb_wait()
882 	if (rdp->nocb_cb_sleep)  in nocb_cb_wait()
883 		trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("CBSleep"));  in nocb_cb_wait()
885 	rcu_nocb_unlock_irqrestore(rdp, flags);  in nocb_cb_wait()
890 		swake_up_one(&rdp->nocb_state_wq);  in nocb_cb_wait()
899 	struct rcu_data *rdp = arg;  in rcu_nocb_cb_kthread()  local
904 		nocb_cb_wait(rdp);  in rcu_nocb_cb_kthread()
911 static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp, int level)  in rcu_nocb_need_deferred_wakeup()  argument
913 	return READ_ONCE(rdp->nocb_defer_wakeup) >= level;  in rcu_nocb_need_deferred_wakeup()
918 					   struct rcu_data *rdp, int level,  in do_nocb_deferred_wakeup_common()
931 	ret = __wake_nocb_gp(rdp_gp, rdp, ndw == RCU_NOCB_WAKE_FORCE, flags);  in do_nocb_deferred_wakeup_common()
932 	trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("DeferredWake"));  in do_nocb_deferred_wakeup_common()
941 	struct rcu_data *rdp = from_timer(rdp, t, nocb_timer);  in do_nocb_deferred_wakeup_timer()  local
943 	WARN_ON_ONCE(rdp->nocb_gp_rdp != rdp);  in do_nocb_deferred_wakeup_timer()
944 	trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("Timer"));  in do_nocb_deferred_wakeup_timer()
946 	raw_spin_lock_irqsave(&rdp->nocb_gp_lock, flags);  in do_nocb_deferred_wakeup_timer()
948 	do_nocb_deferred_wakeup_common(rdp, rdp, RCU_NOCB_WAKE_BYPASS, flags);  in do_nocb_deferred_wakeup_timer()
956 static bool do_nocb_deferred_wakeup(struct rcu_data *rdp)  in do_nocb_deferred_wakeup()  argument
959 	struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;  in do_nocb_deferred_wakeup()
965 	return do_nocb_deferred_wakeup_common(rdp_gp, rdp, RCU_NOCB_WAKE, flags);  in do_nocb_deferred_wakeup()
974 static int rdp_offload_toggle(struct rcu_data *rdp,  in rdp_offload_toggle()  argument
976 	__releases(rdp->nocb_lock)  in rdp_offload_toggle()
978 	struct rcu_segcblist *cblist = &rdp->cblist;  in rdp_offload_toggle()
979 	struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;  in rdp_offload_toggle()
984 	if (rdp->nocb_cb_sleep)  in rdp_offload_toggle()
985 		rdp->nocb_cb_sleep = false;  in rdp_offload_toggle()
986 	rcu_nocb_unlock_irqrestore(rdp, flags);  in rdp_offload_toggle()
992 	swake_up_one(&rdp->nocb_cb_wq);  in rdp_offload_toggle()
996 	WRITE_ONCE(rdp_gp->nocb_toggling_rdp, rdp);  in rdp_offload_toggle()
1008 	struct rcu_data *rdp = arg;  in rcu_nocb_rdp_deoffload()  local
1009 	struct rcu_segcblist *cblist = &rdp->cblist;  in rcu_nocb_rdp_deoffload()
1012 	struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;  in rcu_nocb_rdp_deoffload()
1019 	WARN_ON_ONCE((rdp->cpu != raw_smp_processor_id()) && cpu_online(rdp->cpu));  in rcu_nocb_rdp_deoffload()
1021 	pr_info("De-offloading %d\n", rdp->cpu);  in rcu_nocb_rdp_deoffload()
1023 	rcu_nocb_lock_irqsave(rdp, flags);  in rcu_nocb_rdp_deoffload()
1033 	WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, jiffies));  in rcu_nocb_rdp_deoffload()
1043 	wake_gp = rdp_offload_toggle(rdp, false, flags);  in rcu_nocb_rdp_deoffload()
1054 		if (!rdp->nocb_cb_kthread) {  in rcu_nocb_rdp_deoffload()
1055 			rcu_nocb_lock_irqsave(rdp, flags);  in rcu_nocb_rdp_deoffload()
1056 			rcu_segcblist_clear_flags(&rdp->cblist, SEGCBLIST_KTHREAD_CB);  in rcu_nocb_rdp_deoffload()
1057 			rcu_nocb_unlock_irqrestore(rdp, flags);  in rcu_nocb_rdp_deoffload()
1060 		swait_event_exclusive(rdp->nocb_state_wq,  in rcu_nocb_rdp_deoffload()
1069 		rcu_nocb_lock_irqsave(rdp, flags);  in rcu_nocb_rdp_deoffload()
1070 		rcu_segcblist_clear_flags(&rdp->cblist,  in rcu_nocb_rdp_deoffload()
1072 		rcu_nocb_unlock_irqrestore(rdp, flags);  in rcu_nocb_rdp_deoffload()
1074 		list_del(&rdp->nocb_entry_rdp);  in rcu_nocb_rdp_deoffload()
1082 	rcu_nocb_lock_irqsave(rdp, flags);  in rcu_nocb_rdp_deoffload()
1092 	raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags);  in rcu_nocb_rdp_deoffload()
1095 	WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass));  in rcu_nocb_rdp_deoffload()
1103 	struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);  in rcu_nocb_cpu_deoffload()  local
1108 	if (rcu_rdp_is_offloaded(rdp)) {  in rcu_nocb_cpu_deoffload()
1110 			ret = work_on_cpu(cpu, rcu_nocb_rdp_deoffload, rdp);  in rcu_nocb_cpu_deoffload()
1114 			pr_info("NOCB: Cannot CB-deoffload offline CPU %d\n", rdp->cpu);  in rcu_nocb_cpu_deoffload()
1127 	struct rcu_data *rdp = arg;  in rcu_nocb_rdp_offload()  local
1128 	struct rcu_segcblist *cblist = &rdp->cblist;  in rcu_nocb_rdp_offload()
1131 	struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;  in rcu_nocb_rdp_offload()
1133 	WARN_ON_ONCE(rdp->cpu != raw_smp_processor_id());  in rcu_nocb_rdp_offload()
1138 	if (!rdp->nocb_gp_rdp)  in rcu_nocb_rdp_offload()
1144 	pr_info("Offloading %d\n", rdp->cpu);  in rcu_nocb_rdp_offload()
1150 	raw_spin_lock_irqsave(&rdp->nocb_lock, flags);  in rcu_nocb_rdp_offload()
1168 	wake_gp = rdp_offload_toggle(rdp, true, flags);  in rcu_nocb_rdp_offload()
1171 	swait_event_exclusive(rdp->nocb_state_wq,  in rcu_nocb_rdp_offload()
1179 	rcu_nocb_lock_irqsave(rdp, flags);  in rcu_nocb_rdp_offload()
1181 	rcu_nocb_unlock_irqrestore(rdp, flags);  in rcu_nocb_rdp_offload()
1188 	struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);  in rcu_nocb_cpu_offload()  local
1193 	if (!rcu_rdp_is_offloaded(rdp)) {  in rcu_nocb_cpu_offload()
1195 			ret = work_on_cpu(cpu, rcu_nocb_rdp_offload, rdp);  in rcu_nocb_cpu_offload()
1199 			pr_info("NOCB: Cannot CB-offload offline CPU %d\n", rdp->cpu);  in rcu_nocb_cpu_offload()
1215 	struct rcu_data *rdp;  in rcu_init_nohz()  local
1266 		rdp = per_cpu_ptr(&rcu_data, cpu);  in rcu_init_nohz()
1267 		if (rcu_segcblist_empty(&rdp->cblist))  in rcu_init_nohz()
1268 			rcu_segcblist_init(&rdp->cblist);  in rcu_init_nohz()
1269 		rcu_segcblist_offload(&rdp->cblist, true);  in rcu_init_nohz()
1270 		rcu_segcblist_set_flags(&rdp->cblist, SEGCBLIST_KTHREAD_CB | SEGCBLIST_KTHREAD_GP);  in rcu_init_nohz()
1271 		rcu_segcblist_clear_flags(&rdp->cblist, SEGCBLIST_RCU_CORE);  in rcu_init_nohz()
1277 static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)  in rcu_boot_init_nocb_percpu_data()  argument
1279 	init_swait_queue_head(&rdp->nocb_cb_wq);  in rcu_boot_init_nocb_percpu_data()
1280 	init_swait_queue_head(&rdp->nocb_gp_wq);  in rcu_boot_init_nocb_percpu_data()
1281 	init_swait_queue_head(&rdp->nocb_state_wq);  in rcu_boot_init_nocb_percpu_data()
1282 	raw_spin_lock_init(&rdp->nocb_lock);  in rcu_boot_init_nocb_percpu_data()
1283 	raw_spin_lock_init(&rdp->nocb_bypass_lock);  in rcu_boot_init_nocb_percpu_data()
1284 	raw_spin_lock_init(&rdp->nocb_gp_lock);  in rcu_boot_init_nocb_percpu_data()
1285 	timer_setup(&rdp->nocb_timer, do_nocb_deferred_wakeup_timer, 0);  in rcu_boot_init_nocb_percpu_data()
1286 	rcu_cblist_init(&rdp->nocb_bypass);  in rcu_boot_init_nocb_percpu_data()
1287 	mutex_init(&rdp->nocb_gp_kthread_mutex);  in rcu_boot_init_nocb_percpu_data()
1297 	struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);  in rcu_spawn_cpu_nocb_kthread()  local
1306 	if (rdp->nocb_cb_kthread)  in rcu_spawn_cpu_nocb_kthread()
1311 	rdp_gp = rdp->nocb_gp_rdp;  in rcu_spawn_cpu_nocb_kthread()
1327 	t = kthread_run(rcu_nocb_cb_kthread, rdp,  in rcu_spawn_cpu_nocb_kthread()
1335 	WRITE_ONCE(rdp->nocb_cb_kthread, t);  in rcu_spawn_cpu_nocb_kthread()
1336 	WRITE_ONCE(rdp->nocb_gp_kthread, rdp_gp->nocb_gp_kthread);  in rcu_spawn_cpu_nocb_kthread()
1340 	if (rcu_rdp_is_offloaded(rdp)) {  in rcu_spawn_cpu_nocb_kthread()
1341 		rcu_nocb_rdp_deoffload(rdp);  in rcu_spawn_cpu_nocb_kthread()
1362 	struct rcu_data *rdp;  in rcu_organize_nocb_kthreads()  local
1378 		rdp = per_cpu_ptr(&rcu_data, cpu);  in rcu_organize_nocb_kthreads()
1379 		if (rdp->cpu >= nl) {  in rcu_organize_nocb_kthreads()
1382 			nl = DIV_ROUND_UP(rdp->cpu + 1, ls) * ls;  in rcu_organize_nocb_kthreads()
1383 			rdp_gp = rdp;  in rcu_organize_nocb_kthreads()
1384 			INIT_LIST_HEAD(&rdp->nocb_head_rdp);  in rcu_organize_nocb_kthreads()
1400 		rdp->nocb_gp_rdp = rdp_gp;  in rcu_organize_nocb_kthreads()
1402 			list_add_tail(&rdp->nocb_entry_rdp, &rdp_gp->nocb_head_rdp);  in rcu_organize_nocb_kthreads()
1436 static void show_rcu_nocb_gp_state(struct rcu_data *rdp)  in show_rcu_nocb_gp_state()  argument
1438 	struct rcu_node *rnp = rdp->mynode;  in show_rcu_nocb_gp_state()
1441 		rdp->cpu,  in show_rcu_nocb_gp_state()
1442 		"kK"[!!rdp->nocb_gp_kthread],  in show_rcu_nocb_gp_state()
1443 		"lL"[raw_spin_is_locked(&rdp->nocb_gp_lock)],  in show_rcu_nocb_gp_state()
1444 		"dD"[!!rdp->nocb_defer_wakeup],  in show_rcu_nocb_gp_state()
1445 		"tT"[timer_pending(&rdp->nocb_timer)],  in show_rcu_nocb_gp_state()
1446 		"sS"[!!rdp->nocb_gp_sleep],  in show_rcu_nocb_gp_state()
1447 		".W"[swait_active(&rdp->nocb_gp_wq)],  in show_rcu_nocb_gp_state()
1450 		".B"[!!rdp->nocb_gp_bypass],  in show_rcu_nocb_gp_state()
1451 		".G"[!!rdp->nocb_gp_gp],  in show_rcu_nocb_gp_state()
1452 		(long)rdp->nocb_gp_seq,  in show_rcu_nocb_gp_state()
1453 		rnp->grplo, rnp->grphi, READ_ONCE(rdp->nocb_gp_loops),  in show_rcu_nocb_gp_state()
1454 		rdp->nocb_gp_kthread ? task_state_to_char(rdp->nocb_gp_kthread) : '.',  in show_rcu_nocb_gp_state()
1455 		rdp->nocb_gp_kthread ? (int)task_cpu(rdp->nocb_gp_kthread) : -1,  in show_rcu_nocb_gp_state()
1456 		show_rcu_should_be_on_cpu(rdp->nocb_gp_kthread));  in show_rcu_nocb_gp_state()
1460 static void show_rcu_nocb_state(struct rcu_data *rdp)  in show_rcu_nocb_state()  argument
1465 	struct rcu_segcblist *rsclp = &rdp->cblist;  in show_rcu_nocb_state()
1469 	if (rdp->nocb_gp_rdp == rdp)  in show_rcu_nocb_state()
1470 		show_rcu_nocb_gp_state(rdp);  in show_rcu_nocb_state()
1472 	nocb_next_rdp = list_next_or_null_rcu(&rdp->nocb_gp_rdp->nocb_head_rdp,  in show_rcu_nocb_state()
1473 					      &rdp->nocb_entry_rdp,  in show_rcu_nocb_state()
1474 					      typeof(*rdp),  in show_rcu_nocb_state()
1480 		rdp->cpu, rdp->nocb_gp_rdp->cpu,  in show_rcu_nocb_state()
1482 		"kK"[!!rdp->nocb_cb_kthread],  in show_rcu_nocb_state()
1483 		"bB"[raw_spin_is_locked(&rdp->nocb_bypass_lock)],  in show_rcu_nocb_state()
1484 		"cC"[!!atomic_read(&rdp->nocb_lock_contended)],  in show_rcu_nocb_state()
1485 		"lL"[raw_spin_is_locked(&rdp->nocb_lock)],  in show_rcu_nocb_state()
1486 		"sS"[!!rdp->nocb_cb_sleep],  in show_rcu_nocb_state()
1487 		".W"[swait_active(&rdp->nocb_cb_wq)],  in show_rcu_nocb_state()
1488 		jiffies - rdp->nocb_bypass_first,  in show_rcu_nocb_state()
1489 		jiffies - rdp->nocb_nobypass_last,  in show_rcu_nocb_state()
1490 		rdp->nocb_nobypass_count,  in show_rcu_nocb_state()
1497 		".B"[!!rcu_cblist_n_cbs(&rdp->nocb_bypass)],  in show_rcu_nocb_state()
1498 		rcu_segcblist_n_cbs(&rdp->cblist),  in show_rcu_nocb_state()
1499 		rdp->nocb_cb_kthread ? task_state_to_char(rdp->nocb_cb_kthread) : '.',  in show_rcu_nocb_state()
1500 		rdp->nocb_cb_kthread ? (int)task_cpu(rdp->nocb_cb_kthread) : -1,  in show_rcu_nocb_state()
1501 		show_rcu_should_be_on_cpu(rdp->nocb_cb_kthread));  in show_rcu_nocb_state()
1504 	if (rdp->nocb_gp_rdp == rdp)  in show_rcu_nocb_state()
1507 	waslocked = raw_spin_is_locked(&rdp->nocb_gp_lock);  in show_rcu_nocb_state()
1508 	wassleep = swait_active(&rdp->nocb_gp_wq);  in show_rcu_nocb_state()
1509 	if (!rdp->nocb_gp_sleep && !waslocked && !wassleep)  in show_rcu_nocb_state()
1514 		"dD"[!!rdp->nocb_defer_wakeup],  in show_rcu_nocb_state()
1515 		"sS"[!!rdp->nocb_gp_sleep],  in show_rcu_nocb_state()
1521 static inline int rcu_lockdep_is_held_nocb(struct rcu_data *rdp)  in rcu_lockdep_is_held_nocb()  argument
1526 static inline bool rcu_current_is_nocb_kthread(struct rcu_data *rdp)  in rcu_current_is_nocb_kthread()  argument
1532 static void rcu_nocb_lock(struct rcu_data *rdp)  in rcu_nocb_lock()  argument
1537 static void rcu_nocb_unlock(struct rcu_data *rdp)  in rcu_nocb_unlock()  argument
1542 static void rcu_nocb_unlock_irqrestore(struct rcu_data *rdp,  in rcu_nocb_unlock_irqrestore()  argument
1549 static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp)  in rcu_lockdep_assert_cblist_protected()  argument
1567 static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp,  in rcu_nocb_flush_bypass()  argument
1573 static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp,  in rcu_nocb_try_bypass()  argument
1579 static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_empty,  in __call_rcu_nocb_wake()  argument
1585 static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)  in rcu_boot_init_nocb_percpu_data()  argument
1589 static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp, int level)  in rcu_nocb_need_deferred_wakeup()  argument
1594 static bool do_nocb_deferred_wakeup(struct rcu_data *rdp)  in do_nocb_deferred_wakeup()  argument
1603 static void show_rcu_nocb_state(struct rcu_data *rdp)  in show_rcu_nocb_state()  argument