19 static inline int rcu_lockdep_is_held_nocb(struct rcu_data *rdp)  in rcu_lockdep_is_held_nocb()  argument
21 	return lockdep_is_held(&rdp->nocb_lock);  in rcu_lockdep_is_held_nocb()
24 static inline bool rcu_current_is_nocb_kthread(struct rcu_data *rdp)  in rcu_current_is_nocb_kthread()  argument
27 	if (!rdp->nocb_cb_kthread || !rdp->nocb_gp_kthread)  in rcu_current_is_nocb_kthread()
30 	if (current == rdp->nocb_cb_kthread || current == rdp->nocb_gp_kthread)  in rcu_current_is_nocb_kthread()
97 static void rcu_nocb_bypass_lock(struct rcu_data *rdp)  in rcu_nocb_bypass_lock()  argument
98 	__acquires(&rdp->nocb_bypass_lock)  in rcu_nocb_bypass_lock()
101 	if (raw_spin_trylock(&rdp->nocb_bypass_lock))  in rcu_nocb_bypass_lock()
103 	atomic_inc(&rdp->nocb_lock_contended);  in rcu_nocb_bypass_lock()
104 	WARN_ON_ONCE(smp_processor_id() != rdp->cpu);  in rcu_nocb_bypass_lock()
106 	raw_spin_lock(&rdp->nocb_bypass_lock);  in rcu_nocb_bypass_lock()
108 	atomic_dec(&rdp->nocb_lock_contended);  in rcu_nocb_bypass_lock()
121 static void rcu_nocb_wait_contended(struct rcu_data *rdp)  in rcu_nocb_wait_contended()  argument
123 	WARN_ON_ONCE(smp_processor_id() != rdp->cpu);  in rcu_nocb_wait_contended()
124 	while (WARN_ON_ONCE(atomic_read(&rdp->nocb_lock_contended)))  in rcu_nocb_wait_contended()
132 static bool rcu_nocb_bypass_trylock(struct rcu_data *rdp)  in rcu_nocb_bypass_trylock()  argument
135 	return raw_spin_trylock(&rdp->nocb_bypass_lock);  in rcu_nocb_bypass_trylock()
141 static void rcu_nocb_bypass_unlock(struct rcu_data *rdp)  in rcu_nocb_bypass_unlock()  argument
142 	__releases(&rdp->nocb_bypass_lock)  in rcu_nocb_bypass_unlock()
145 	raw_spin_unlock(&rdp->nocb_bypass_lock);  in rcu_nocb_bypass_unlock()
152 static void rcu_nocb_lock(struct rcu_data *rdp)  in rcu_nocb_lock()  argument
155 	if (!rcu_rdp_is_offloaded(rdp))  in rcu_nocb_lock()
157 	raw_spin_lock(&rdp->nocb_lock);  in rcu_nocb_lock()
164 static void rcu_nocb_unlock(struct rcu_data *rdp)  in rcu_nocb_unlock()  argument
166 	if (rcu_rdp_is_offloaded(rdp)) {  in rcu_nocb_unlock()
168 		raw_spin_unlock(&rdp->nocb_lock);  in rcu_nocb_unlock()
176 static void rcu_nocb_unlock_irqrestore(struct rcu_data *rdp,  in rcu_nocb_unlock_irqrestore()  argument
179 	if (rcu_rdp_is_offloaded(rdp)) {  in rcu_nocb_unlock_irqrestore()
181 		raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags);  in rcu_nocb_unlock_irqrestore()
188 static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp)  in rcu_lockdep_assert_cblist_protected()  argument
191 	if (rcu_rdp_is_offloaded(rdp))  in rcu_lockdep_assert_cblist_protected()
192 		lockdep_assert_held(&rdp->nocb_lock);  in rcu_lockdep_assert_cblist_protected()
216 			   struct rcu_data *rdp,  in __wake_nocb_gp()
224 		trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,  in __wake_nocb_gp()
240 		trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("DoWake"));  in __wake_nocb_gp()
250 static bool wake_nocb_gp(struct rcu_data *rdp, bool force)  in wake_nocb_gp()  argument
253 	struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;  in wake_nocb_gp()
256 	return __wake_nocb_gp(rdp_gp, rdp, force, flags);  in wake_nocb_gp()
288 static void wake_nocb_gp_defer(struct rcu_data *rdp, int waketype,  in wake_nocb_gp_defer()  argument
292 	struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;  in wake_nocb_gp_defer()
301 	    rdp->nocb_defer_wakeup == RCU_NOCB_WAKE_NOT) {  in wake_nocb_gp_defer()
316 	trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, reason);  in wake_nocb_gp_defer()
330 static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp_in,  in rcu_nocb_do_flush_bypass()  argument
336 	WARN_ON_ONCE(!rcu_rdp_is_offloaded(rdp));  in rcu_nocb_do_flush_bypass()
337 	rcu_lockdep_assert_cblist_protected(rdp);  in rcu_nocb_do_flush_bypass()
338 	lockdep_assert_held(&rdp->nocb_bypass_lock);  in rcu_nocb_do_flush_bypass()
339 	if (rhp && !rcu_cblist_n_cbs(&rdp->nocb_bypass)) {  in rcu_nocb_do_flush_bypass()
340 		raw_spin_unlock(&rdp->nocb_bypass_lock);  in rcu_nocb_do_flush_bypass()
345 		rcu_segcblist_inc_len(&rdp->cblist); /* Must precede enqueue. */  in rcu_nocb_do_flush_bypass()
354 		rcu_cblist_enqueue(&rdp->nocb_bypass, rhp);  in rcu_nocb_do_flush_bypass()
357 	rcu_cblist_flush_enqueue(&rcl, &rdp->nocb_bypass, rhp);  in rcu_nocb_do_flush_bypass()
358 	WRITE_ONCE(rdp->lazy_len, 0);  in rcu_nocb_do_flush_bypass()
360 	rcu_segcblist_insert_pend_cbs(&rdp->cblist, &rcl);  in rcu_nocb_do_flush_bypass()
361 	WRITE_ONCE(rdp->nocb_bypass_first, j);  in rcu_nocb_do_flush_bypass()
362 	rcu_nocb_bypass_unlock(rdp);  in rcu_nocb_do_flush_bypass()
374 static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp,  in rcu_nocb_flush_bypass()  argument
377 	if (!rcu_rdp_is_offloaded(rdp))  in rcu_nocb_flush_bypass()
379 	rcu_lockdep_assert_cblist_protected(rdp);  in rcu_nocb_flush_bypass()
380 	rcu_nocb_bypass_lock(rdp);  in rcu_nocb_flush_bypass()
381 	return rcu_nocb_do_flush_bypass(rdp, rhp, j, lazy);  in rcu_nocb_flush_bypass()
388 static void rcu_nocb_try_flush_bypass(struct rcu_data *rdp, unsigned long j)  in rcu_nocb_try_flush_bypass()  argument
390 	rcu_lockdep_assert_cblist_protected(rdp);  in rcu_nocb_try_flush_bypass()
391 	if (!rcu_rdp_is_offloaded(rdp) ||  in rcu_nocb_try_flush_bypass()
392 	    !rcu_nocb_bypass_trylock(rdp))  in rcu_nocb_try_flush_bypass()
394 	WARN_ON_ONCE(!rcu_nocb_do_flush_bypass(rdp, NULL, j, false));  in rcu_nocb_try_flush_bypass()
415 static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp,  in rcu_nocb_try_bypass()  argument
422 	long ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass);  in rcu_nocb_try_bypass()
423 	bool bypass_is_lazy = (ncbs == READ_ONCE(rdp->lazy_len));  in rcu_nocb_try_bypass()
429 	if (!rcu_rdp_is_offloaded(rdp)) {  in rcu_nocb_try_bypass()
430 		*was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist);  in rcu_nocb_try_bypass()
436 	if (!rcu_segcblist_completely_offloaded(&rdp->cblist)) {  in rcu_nocb_try_bypass()
437 		rcu_nocb_lock(rdp);  in rcu_nocb_try_bypass()
438 		*was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist);  in rcu_nocb_try_bypass()
444 		rcu_nocb_lock(rdp);  in rcu_nocb_try_bypass()
445 		WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass));  in rcu_nocb_try_bypass()
446 		*was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist);  in rcu_nocb_try_bypass()
452 	if (j == rdp->nocb_nobypass_last) {  in rcu_nocb_try_bypass()
453 		c = rdp->nocb_nobypass_count + 1;  in rcu_nocb_try_bypass()
455 		WRITE_ONCE(rdp->nocb_nobypass_last, j);  in rcu_nocb_try_bypass()
456 		c = rdp->nocb_nobypass_count - nocb_nobypass_lim_per_jiffy;  in rcu_nocb_try_bypass()
457 		if (ULONG_CMP_LT(rdp->nocb_nobypass_count,  in rcu_nocb_try_bypass()
463 	WRITE_ONCE(rdp->nocb_nobypass_count, c);  in rcu_nocb_try_bypass()
469 	if (rdp->nocb_nobypass_count < nocb_nobypass_lim_per_jiffy && !lazy) {  in rcu_nocb_try_bypass()
470 		rcu_nocb_lock(rdp);  in rcu_nocb_try_bypass()
471 		*was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist);  in rcu_nocb_try_bypass()
473 			trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,  in rcu_nocb_try_bypass()
476 		WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, j, false));  in rcu_nocb_try_bypass()
477 		WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass));  in rcu_nocb_try_bypass()
483 	if ((ncbs && !bypass_is_lazy && j != READ_ONCE(rdp->nocb_bypass_first)) ||  in rcu_nocb_try_bypass()
485 	     (time_after(j, READ_ONCE(rdp->nocb_bypass_first) + jiffies_till_flush))) ||  in rcu_nocb_try_bypass()
487 		rcu_nocb_lock(rdp);  in rcu_nocb_try_bypass()
488 		*was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist);  in rcu_nocb_try_bypass()
490 		if (!rcu_nocb_flush_bypass(rdp, rhp, j, lazy)) {  in rcu_nocb_try_bypass()
492 				trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,  in rcu_nocb_try_bypass()
494 			WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass));  in rcu_nocb_try_bypass()
497 		if (j != rdp->nocb_gp_adv_time &&  in rcu_nocb_try_bypass()
498 		    rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) &&  in rcu_nocb_try_bypass()
499 		    rcu_seq_done(&rdp->mynode->gp_seq, cur_gp_seq)) {  in rcu_nocb_try_bypass()
500 			rcu_advance_cbs_nowake(rdp->mynode, rdp);  in rcu_nocb_try_bypass()
501 			rdp->nocb_gp_adv_time = j;  in rcu_nocb_try_bypass()
507 		__call_rcu_nocb_wake(rdp, *was_alldone, flags);  in rcu_nocb_try_bypass()
513 	rcu_nocb_wait_contended(rdp);  in rcu_nocb_try_bypass()
514 	rcu_nocb_bypass_lock(rdp);  in rcu_nocb_try_bypass()
515 	ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass);  in rcu_nocb_try_bypass()
516 	rcu_segcblist_inc_len(&rdp->cblist); /* Must precede enqueue. */  in rcu_nocb_try_bypass()
517 	rcu_cblist_enqueue(&rdp->nocb_bypass, rhp);  in rcu_nocb_try_bypass()
520 		WRITE_ONCE(rdp->lazy_len, rdp->lazy_len + 1);  in rcu_nocb_try_bypass()
523 		WRITE_ONCE(rdp->nocb_bypass_first, j);  in rcu_nocb_try_bypass()
524 		trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("FirstBQ"));  in rcu_nocb_try_bypass()
526 	rcu_nocb_bypass_unlock(rdp);  in rcu_nocb_try_bypass()
539 		rcu_nocb_lock(rdp); // Rare during call_rcu() flood.  in rcu_nocb_try_bypass()
540 		if (!rcu_segcblist_pend_cbs(&rdp->cblist)) {  in rcu_nocb_try_bypass()
541 			trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,  in rcu_nocb_try_bypass()
543 			__call_rcu_nocb_wake(rdp, true, flags);  in rcu_nocb_try_bypass()
545 			trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,  in rcu_nocb_try_bypass()
547 			rcu_nocb_unlock_irqrestore(rdp, flags);  in rcu_nocb_try_bypass()
559 static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone,  in __call_rcu_nocb_wake()  argument
561 				 __releases(rdp->nocb_lock)  in __call_rcu_nocb_wake()
571 	t = READ_ONCE(rdp->nocb_gp_kthread);  in __call_rcu_nocb_wake()
573 		rcu_nocb_unlock_irqrestore(rdp, flags);  in __call_rcu_nocb_wake()
574 		trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,  in __call_rcu_nocb_wake()
579 	len = rcu_segcblist_n_cbs(&rdp->cblist);  in __call_rcu_nocb_wake()
580 	bypass_len = rcu_cblist_n_cbs(&rdp->nocb_bypass);  in __call_rcu_nocb_wake()
581 	lazy_len = READ_ONCE(rdp->lazy_len);  in __call_rcu_nocb_wake()
583 		rdp->qlen_last_fqs_check = len;  in __call_rcu_nocb_wake()
586 			rcu_nocb_unlock_irqrestore(rdp, flags);  in __call_rcu_nocb_wake()
587 			wake_nocb_gp_defer(rdp, RCU_NOCB_WAKE_LAZY,  in __call_rcu_nocb_wake()
591 			rcu_nocb_unlock_irqrestore(rdp, flags);  in __call_rcu_nocb_wake()
592 			wake_nocb_gp(rdp, false);  in __call_rcu_nocb_wake()
593 			trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,  in __call_rcu_nocb_wake()
596 			rcu_nocb_unlock_irqrestore(rdp, flags);  in __call_rcu_nocb_wake()
597 			wake_nocb_gp_defer(rdp, RCU_NOCB_WAKE,  in __call_rcu_nocb_wake()
600 	} else if (len > rdp->qlen_last_fqs_check + qhimark) {  in __call_rcu_nocb_wake()
602 		rdp->qlen_last_fqs_check = len;  in __call_rcu_nocb_wake()
604 		if (j != rdp->nocb_gp_adv_time &&  in __call_rcu_nocb_wake()
605 		    rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) &&  in __call_rcu_nocb_wake()
606 		    rcu_seq_done(&rdp->mynode->gp_seq, cur_gp_seq)) {  in __call_rcu_nocb_wake()
607 			rcu_advance_cbs_nowake(rdp->mynode, rdp);  in __call_rcu_nocb_wake()
608 			rdp->nocb_gp_adv_time = j;  in __call_rcu_nocb_wake()
611 		if ((rdp->nocb_cb_sleep ||  in __call_rcu_nocb_wake()
612 		     !rcu_segcblist_ready_cbs(&rdp->cblist)) &&  in __call_rcu_nocb_wake()
613 		    !timer_pending(&rdp->nocb_timer)) {  in __call_rcu_nocb_wake()
614 			rcu_nocb_unlock_irqrestore(rdp, flags);  in __call_rcu_nocb_wake()
615 			wake_nocb_gp_defer(rdp, RCU_NOCB_WAKE_FORCE,  in __call_rcu_nocb_wake()
618 			rcu_nocb_unlock_irqrestore(rdp, flags);  in __call_rcu_nocb_wake()
619 			trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WakeNot"));  in __call_rcu_nocb_wake()
622 		rcu_nocb_unlock_irqrestore(rdp, flags);  in __call_rcu_nocb_wake()
623 		trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WakeNot"));  in __call_rcu_nocb_wake()
627 static int nocb_gp_toggle_rdp(struct rcu_data *rdp,  in nocb_gp_toggle_rdp()  argument
630 	struct rcu_segcblist *cblist = &rdp->cblist;  in nocb_gp_toggle_rdp()
634 	rcu_nocb_lock_irqsave(rdp, flags);  in nocb_gp_toggle_rdp()
660 	rcu_nocb_unlock_irqrestore(rdp, flags);  in nocb_gp_toggle_rdp()
689 	struct rcu_data *rdp, *rdp_toggling = NULL;  in nocb_gp_wait()  local
714 	list_for_each_entry(rdp, &my_rdp->nocb_head_rdp, nocb_entry_rdp) {  in nocb_gp_wait()
719 		trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("Check"));  in nocb_gp_wait()
720 		rcu_nocb_lock_irqsave(rdp, flags);  in nocb_gp_wait()
721 		lockdep_assert_held(&rdp->nocb_lock);  in nocb_gp_wait()
722 		bypass_ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass);  in nocb_gp_wait()
723 		lazy_ncbs = READ_ONCE(rdp->lazy_len);  in nocb_gp_wait()
726 		    (time_after(j, READ_ONCE(rdp->nocb_bypass_first) + jiffies_till_flush) ||  in nocb_gp_wait()
730 		    (time_after(j, READ_ONCE(rdp->nocb_bypass_first) + 1) ||  in nocb_gp_wait()
733 		} else if (!bypass_ncbs && rcu_segcblist_empty(&rdp->cblist)) {  in nocb_gp_wait()
734 			rcu_nocb_unlock_irqrestore(rdp, flags);  in nocb_gp_wait()
740 			(void)rcu_nocb_try_flush_bypass(rdp, j);  in nocb_gp_wait()
741 			bypass_ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass);  in nocb_gp_wait()
742 			lazy_ncbs = READ_ONCE(rdp->lazy_len);  in nocb_gp_wait()
746 			trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,  in nocb_gp_wait()
753 		rnp = rdp->mynode;  in nocb_gp_wait()
757 		if (!rcu_segcblist_restempty(&rdp->cblist,  in nocb_gp_wait()
759 		    (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) &&  in nocb_gp_wait()
762 			needwake_gp = rcu_advance_cbs(rnp, rdp);  in nocb_gp_wait()
763 			wasempty = rcu_segcblist_restempty(&rdp->cblist,  in nocb_gp_wait()
769 			     !rcu_segcblist_restempty(&rdp->cblist,  in nocb_gp_wait()
771 		if (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq)) {  in nocb_gp_wait()
776 			trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,  in nocb_gp_wait()
779 		if (rcu_segcblist_ready_cbs(&rdp->cblist)) {  in nocb_gp_wait()
780 			needwake = rdp->nocb_cb_sleep;  in nocb_gp_wait()
781 			WRITE_ONCE(rdp->nocb_cb_sleep, false);  in nocb_gp_wait()
786 		rcu_nocb_unlock_irqrestore(rdp, flags);  in nocb_gp_wait()
788 			swake_up_one(&rdp->nocb_cb_wq);  in nocb_gp_wait()
896 	struct rcu_data *rdp = arg;  in rcu_nocb_gp_kthread()  local
899 		WRITE_ONCE(rdp->nocb_gp_loops, rdp->nocb_gp_loops + 1);  in rcu_nocb_gp_kthread()
900 		nocb_gp_wait(rdp);  in rcu_nocb_gp_kthread()
906 static inline bool nocb_cb_can_run(struct rcu_data *rdp)  in nocb_cb_can_run()  argument
910 	return rcu_segcblist_test_flags(&rdp->cblist, flags);  in nocb_cb_can_run()
913 static inline bool nocb_cb_wait_cond(struct rcu_data *rdp)  in nocb_cb_wait_cond()  argument
915 	return nocb_cb_can_run(rdp) && !READ_ONCE(rdp->nocb_cb_sleep);  in nocb_cb_wait_cond()
922 static void nocb_cb_wait(struct rcu_data *rdp)  in nocb_cb_wait()  argument
924 	struct rcu_segcblist *cblist = &rdp->cblist;  in nocb_cb_wait()
930 	struct rcu_node *rnp = rdp->mynode;  in nocb_cb_wait()
933 		swait_event_interruptible_exclusive(rdp->nocb_cb_wq,  in nocb_cb_wait()
934 						    nocb_cb_wait_cond(rdp));  in nocb_cb_wait()
937 		if (smp_load_acquire(&rdp->nocb_cb_sleep)) { // ^^^  in nocb_cb_wait()
939 			trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WokeEmpty"));  in nocb_cb_wait()
941 	} while (!nocb_cb_can_run(rdp));  in nocb_cb_wait()
954 	rcu_do_batch(rdp);  in nocb_cb_wait()
957 	rcu_nocb_lock_irqsave(rdp, flags);  in nocb_cb_wait()
961 		needwake_gp = rcu_advance_cbs(rdp->mynode, rdp);  in nocb_cb_wait()
985 	WRITE_ONCE(rdp->nocb_cb_sleep, can_sleep);  in nocb_cb_wait()
987 	if (rdp->nocb_cb_sleep)  in nocb_cb_wait()
988 		trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("CBSleep"));  in nocb_cb_wait()
990 	rcu_nocb_unlock_irqrestore(rdp, flags);  in nocb_cb_wait()
995 		swake_up_one(&rdp->nocb_state_wq);  in nocb_cb_wait()
1004 	struct rcu_data *rdp = arg;  in rcu_nocb_cb_kthread()  local
1009 		nocb_cb_wait(rdp);  in rcu_nocb_cb_kthread()
1016 static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp, int level)  in rcu_nocb_need_deferred_wakeup()  argument
1018 	return READ_ONCE(rdp->nocb_defer_wakeup) >= level;  in rcu_nocb_need_deferred_wakeup()
1023 					   struct rcu_data *rdp, int level,  in do_nocb_deferred_wakeup_common()
1036 	ret = __wake_nocb_gp(rdp_gp, rdp, ndw == RCU_NOCB_WAKE_FORCE, flags);  in do_nocb_deferred_wakeup_common()
1037 	trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("DeferredWake"));  in do_nocb_deferred_wakeup_common()
1046 	struct rcu_data *rdp = from_timer(rdp, t, nocb_timer);  in do_nocb_deferred_wakeup_timer()  local
1048 	WARN_ON_ONCE(rdp->nocb_gp_rdp != rdp);  in do_nocb_deferred_wakeup_timer()
1049 	trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("Timer"));  in do_nocb_deferred_wakeup_timer()
1051 	raw_spin_lock_irqsave(&rdp->nocb_gp_lock, flags);  in do_nocb_deferred_wakeup_timer()
1053 	do_nocb_deferred_wakeup_common(rdp, rdp, RCU_NOCB_WAKE_BYPASS, flags);  in do_nocb_deferred_wakeup_timer()
1061 static bool do_nocb_deferred_wakeup(struct rcu_data *rdp)  in do_nocb_deferred_wakeup()  argument
1064 	struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;  in do_nocb_deferred_wakeup()
1070 	return do_nocb_deferred_wakeup_common(rdp_gp, rdp, RCU_NOCB_WAKE, flags);  in do_nocb_deferred_wakeup()
1079 static int rdp_offload_toggle(struct rcu_data *rdp,  in rdp_offload_toggle()  argument
1081 	__releases(rdp->nocb_lock)  in rdp_offload_toggle()
1083 	struct rcu_segcblist *cblist = &rdp->cblist;  in rdp_offload_toggle()
1084 	struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;  in rdp_offload_toggle()
1089 	if (rdp->nocb_cb_sleep)  in rdp_offload_toggle()
1090 		rdp->nocb_cb_sleep = false;  in rdp_offload_toggle()
1091 	rcu_nocb_unlock_irqrestore(rdp, flags);  in rdp_offload_toggle()
1097 	swake_up_one(&rdp->nocb_cb_wq);  in rdp_offload_toggle()
1101 	WRITE_ONCE(rdp_gp->nocb_toggling_rdp, rdp);  in rdp_offload_toggle()
1113 	struct rcu_data *rdp = arg;  in rcu_nocb_rdp_deoffload()  local
1114 	struct rcu_segcblist *cblist = &rdp->cblist;  in rcu_nocb_rdp_deoffload()
1117 	struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;  in rcu_nocb_rdp_deoffload()
1124 	WARN_ON_ONCE((rdp->cpu != raw_smp_processor_id()) && cpu_online(rdp->cpu));  in rcu_nocb_rdp_deoffload()
1126 	pr_info("De-offloading %d\n", rdp->cpu);  in rcu_nocb_rdp_deoffload()
1128 	rcu_nocb_lock_irqsave(rdp, flags);  in rcu_nocb_rdp_deoffload()
1138 	WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, jiffies, false));  in rcu_nocb_rdp_deoffload()
1148 	wake_gp = rdp_offload_toggle(rdp, false, flags);  in rcu_nocb_rdp_deoffload()
1159 		if (!rdp->nocb_cb_kthread) {  in rcu_nocb_rdp_deoffload()
1160 			rcu_nocb_lock_irqsave(rdp, flags);  in rcu_nocb_rdp_deoffload()
1161 			rcu_segcblist_clear_flags(&rdp->cblist, SEGCBLIST_KTHREAD_CB);  in rcu_nocb_rdp_deoffload()
1162 			rcu_nocb_unlock_irqrestore(rdp, flags);  in rcu_nocb_rdp_deoffload()
1165 		swait_event_exclusive(rdp->nocb_state_wq,  in rcu_nocb_rdp_deoffload()
1174 		rcu_nocb_lock_irqsave(rdp, flags);  in rcu_nocb_rdp_deoffload()
1175 		rcu_segcblist_clear_flags(&rdp->cblist,  in rcu_nocb_rdp_deoffload()
1177 		rcu_nocb_unlock_irqrestore(rdp, flags);  in rcu_nocb_rdp_deoffload()
1179 		list_del(&rdp->nocb_entry_rdp);  in rcu_nocb_rdp_deoffload()
1187 	rcu_nocb_lock_irqsave(rdp, flags);  in rcu_nocb_rdp_deoffload()
1197 	raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags);  in rcu_nocb_rdp_deoffload()
1200 	WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass));  in rcu_nocb_rdp_deoffload()
1208 	struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);  in rcu_nocb_cpu_deoffload()  local
1213 	if (rcu_rdp_is_offloaded(rdp)) {  in rcu_nocb_cpu_deoffload()
1215 			ret = work_on_cpu(cpu, rcu_nocb_rdp_deoffload, rdp);  in rcu_nocb_cpu_deoffload()
1219 			pr_info("NOCB: Cannot CB-deoffload offline CPU %d\n", rdp->cpu);  in rcu_nocb_cpu_deoffload()
1232 	struct rcu_data *rdp = arg;  in rcu_nocb_rdp_offload()  local
1233 	struct rcu_segcblist *cblist = &rdp->cblist;  in rcu_nocb_rdp_offload()
1236 	struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;  in rcu_nocb_rdp_offload()
1238 	WARN_ON_ONCE(rdp->cpu != raw_smp_processor_id());  in rcu_nocb_rdp_offload()
1243 	if (!rdp->nocb_gp_rdp)  in rcu_nocb_rdp_offload()
1249 	pr_info("Offloading %d\n", rdp->cpu);  in rcu_nocb_rdp_offload()
1255 	raw_spin_lock_irqsave(&rdp->nocb_lock, flags);  in rcu_nocb_rdp_offload()
1273 	wake_gp = rdp_offload_toggle(rdp, true, flags);  in rcu_nocb_rdp_offload()
1276 	swait_event_exclusive(rdp->nocb_state_wq,  in rcu_nocb_rdp_offload()
1284 	rcu_nocb_lock_irqsave(rdp, flags);  in rcu_nocb_rdp_offload()
1286 	rcu_nocb_unlock_irqrestore(rdp, flags);  in rcu_nocb_rdp_offload()
1293 	struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);  in rcu_nocb_cpu_offload()  local
1298 	if (!rcu_rdp_is_offloaded(rdp)) {  in rcu_nocb_cpu_offload()
1300 			ret = work_on_cpu(cpu, rcu_nocb_rdp_offload, rdp);  in rcu_nocb_cpu_offload()
1304 			pr_info("NOCB: Cannot CB-offload offline CPU %d\n", rdp->cpu);  in rcu_nocb_cpu_offload()
1331 		struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);  in lazy_rcu_shrink_count()  local
1333 		count +=  READ_ONCE(rdp->lazy_len);  in lazy_rcu_shrink_count()
1366 		struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);  in lazy_rcu_shrink_scan()  local
1369 		if (WARN_ON_ONCE(!rcu_rdp_is_offloaded(rdp)))  in lazy_rcu_shrink_scan()
1372 		if (!READ_ONCE(rdp->lazy_len))  in lazy_rcu_shrink_scan()
1375 		rcu_nocb_lock_irqsave(rdp, flags);  in lazy_rcu_shrink_scan()
1381 		_count = READ_ONCE(rdp->lazy_len);  in lazy_rcu_shrink_scan()
1383 			rcu_nocb_unlock_irqrestore(rdp, flags);  in lazy_rcu_shrink_scan()
1386 		WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, jiffies, false));  in lazy_rcu_shrink_scan()
1387 		rcu_nocb_unlock_irqrestore(rdp, flags);  in lazy_rcu_shrink_scan()
1388 		wake_nocb_gp(rdp, false);  in lazy_rcu_shrink_scan()
1411 	struct rcu_data *rdp;  in rcu_init_nohz()  local
1457 		rdp = per_cpu_ptr(&rcu_data, cpu);  in rcu_init_nohz()
1458 		if (rcu_segcblist_empty(&rdp->cblist))  in rcu_init_nohz()
1459 			rcu_segcblist_init(&rdp->cblist);  in rcu_init_nohz()
1460 		rcu_segcblist_offload(&rdp->cblist, true);  in rcu_init_nohz()
1461 		rcu_segcblist_set_flags(&rdp->cblist, SEGCBLIST_KTHREAD_CB | SEGCBLIST_KTHREAD_GP);  in rcu_init_nohz()
1462 		rcu_segcblist_clear_flags(&rdp->cblist, SEGCBLIST_RCU_CORE);  in rcu_init_nohz()
1468 static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)  in rcu_boot_init_nocb_percpu_data()  argument
1470 	init_swait_queue_head(&rdp->nocb_cb_wq);  in rcu_boot_init_nocb_percpu_data()
1471 	init_swait_queue_head(&rdp->nocb_gp_wq);  in rcu_boot_init_nocb_percpu_data()
1472 	init_swait_queue_head(&rdp->nocb_state_wq);  in rcu_boot_init_nocb_percpu_data()
1473 	raw_spin_lock_init(&rdp->nocb_lock);  in rcu_boot_init_nocb_percpu_data()
1474 	raw_spin_lock_init(&rdp->nocb_bypass_lock);  in rcu_boot_init_nocb_percpu_data()
1475 	raw_spin_lock_init(&rdp->nocb_gp_lock);  in rcu_boot_init_nocb_percpu_data()
1476 	timer_setup(&rdp->nocb_timer, do_nocb_deferred_wakeup_timer, 0);  in rcu_boot_init_nocb_percpu_data()
1477 	rcu_cblist_init(&rdp->nocb_bypass);  in rcu_boot_init_nocb_percpu_data()
1478 	WRITE_ONCE(rdp->lazy_len, 0);  in rcu_boot_init_nocb_percpu_data()
1479 	mutex_init(&rdp->nocb_gp_kthread_mutex);  in rcu_boot_init_nocb_percpu_data()
1489 	struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);  in rcu_spawn_cpu_nocb_kthread()  local
1498 	if (rdp->nocb_cb_kthread)  in rcu_spawn_cpu_nocb_kthread()
1503 	rdp_gp = rdp->nocb_gp_rdp;  in rcu_spawn_cpu_nocb_kthread()
1519 	t = kthread_run(rcu_nocb_cb_kthread, rdp,  in rcu_spawn_cpu_nocb_kthread()
1527 	WRITE_ONCE(rdp->nocb_cb_kthread, t);  in rcu_spawn_cpu_nocb_kthread()
1528 	WRITE_ONCE(rdp->nocb_gp_kthread, rdp_gp->nocb_gp_kthread);  in rcu_spawn_cpu_nocb_kthread()
1532 	if (rcu_rdp_is_offloaded(rdp)) {  in rcu_spawn_cpu_nocb_kthread()
1533 		rcu_nocb_rdp_deoffload(rdp);  in rcu_spawn_cpu_nocb_kthread()
1554 	struct rcu_data *rdp;  in rcu_organize_nocb_kthreads()  local
1570 		rdp = per_cpu_ptr(&rcu_data, cpu);  in rcu_organize_nocb_kthreads()
1571 		if (rdp->cpu >= nl) {  in rcu_organize_nocb_kthreads()
1574 			nl = DIV_ROUND_UP(rdp->cpu + 1, ls) * ls;  in rcu_organize_nocb_kthreads()
1575 			rdp_gp = rdp;  in rcu_organize_nocb_kthreads()
1576 			INIT_LIST_HEAD(&rdp->nocb_head_rdp);  in rcu_organize_nocb_kthreads()
1592 		rdp->nocb_gp_rdp = rdp_gp;  in rcu_organize_nocb_kthreads()
1594 			list_add_tail(&rdp->nocb_entry_rdp, &rdp_gp->nocb_head_rdp);  in rcu_organize_nocb_kthreads()
1628 static void show_rcu_nocb_gp_state(struct rcu_data *rdp)  in show_rcu_nocb_gp_state()  argument
1630 	struct rcu_node *rnp = rdp->mynode;  in show_rcu_nocb_gp_state()
1633 		rdp->cpu,  in show_rcu_nocb_gp_state()
1634 		"kK"[!!rdp->nocb_gp_kthread],  in show_rcu_nocb_gp_state()
1635 		"lL"[raw_spin_is_locked(&rdp->nocb_gp_lock)],  in show_rcu_nocb_gp_state()
1636 		"dD"[!!rdp->nocb_defer_wakeup],  in show_rcu_nocb_gp_state()
1637 		"tT"[timer_pending(&rdp->nocb_timer)],  in show_rcu_nocb_gp_state()
1638 		"sS"[!!rdp->nocb_gp_sleep],  in show_rcu_nocb_gp_state()
1639 		".W"[swait_active(&rdp->nocb_gp_wq)],  in show_rcu_nocb_gp_state()
1642 		".B"[!!rdp->nocb_gp_bypass],  in show_rcu_nocb_gp_state()
1643 		".G"[!!rdp->nocb_gp_gp],  in show_rcu_nocb_gp_state()
1644 		(long)rdp->nocb_gp_seq,  in show_rcu_nocb_gp_state()
1645 		rnp->grplo, rnp->grphi, READ_ONCE(rdp->nocb_gp_loops),  in show_rcu_nocb_gp_state()
1646 		rdp->nocb_gp_kthread ? task_state_to_char(rdp->nocb_gp_kthread) : '.',  in show_rcu_nocb_gp_state()
1647 		rdp->nocb_gp_kthread ? (int)task_cpu(rdp->nocb_gp_kthread) : -1,  in show_rcu_nocb_gp_state()
1648 		show_rcu_should_be_on_cpu(rdp->nocb_gp_kthread));  in show_rcu_nocb_gp_state()
1652 static void show_rcu_nocb_state(struct rcu_data *rdp)  in show_rcu_nocb_state()  argument
1657 	struct rcu_segcblist *rsclp = &rdp->cblist;  in show_rcu_nocb_state()
1661 	if (rdp->nocb_gp_rdp == rdp)  in show_rcu_nocb_state()
1662 		show_rcu_nocb_gp_state(rdp);  in show_rcu_nocb_state()
1664 	nocb_next_rdp = list_next_or_null_rcu(&rdp->nocb_gp_rdp->nocb_head_rdp,  in show_rcu_nocb_state()
1665 					      &rdp->nocb_entry_rdp,  in show_rcu_nocb_state()
1666 					      typeof(*rdp),  in show_rcu_nocb_state()
1672 		rdp->cpu, rdp->nocb_gp_rdp->cpu,  in show_rcu_nocb_state()
1674 		"kK"[!!rdp->nocb_cb_kthread],  in show_rcu_nocb_state()
1675 		"bB"[raw_spin_is_locked(&rdp->nocb_bypass_lock)],  in show_rcu_nocb_state()
1676 		"cC"[!!atomic_read(&rdp->nocb_lock_contended)],  in show_rcu_nocb_state()
1677 		"lL"[raw_spin_is_locked(&rdp->nocb_lock)],  in show_rcu_nocb_state()
1678 		"sS"[!!rdp->nocb_cb_sleep],  in show_rcu_nocb_state()
1679 		".W"[swait_active(&rdp->nocb_cb_wq)],  in show_rcu_nocb_state()
1680 		jiffies - rdp->nocb_bypass_first,  in show_rcu_nocb_state()
1681 		jiffies - rdp->nocb_nobypass_last,  in show_rcu_nocb_state()
1682 		rdp->nocb_nobypass_count,  in show_rcu_nocb_state()
1689 		".B"[!!rcu_cblist_n_cbs(&rdp->nocb_bypass)],  in show_rcu_nocb_state()
1690 		rcu_segcblist_n_cbs(&rdp->cblist),  in show_rcu_nocb_state()
1691 		rdp->nocb_cb_kthread ? task_state_to_char(rdp->nocb_cb_kthread) : '.',  in show_rcu_nocb_state()
1692 		rdp->nocb_cb_kthread ? (int)task_cpu(rdp->nocb_cb_kthread) : -1,  in show_rcu_nocb_state()
1693 		show_rcu_should_be_on_cpu(rdp->nocb_cb_kthread));  in show_rcu_nocb_state()
1696 	if (rdp->nocb_gp_rdp == rdp)  in show_rcu_nocb_state()
1699 	waslocked = raw_spin_is_locked(&rdp->nocb_gp_lock);  in show_rcu_nocb_state()
1700 	wassleep = swait_active(&rdp->nocb_gp_wq);  in show_rcu_nocb_state()
1701 	if (!rdp->nocb_gp_sleep && !waslocked && !wassleep)  in show_rcu_nocb_state()
1706 		"dD"[!!rdp->nocb_defer_wakeup],  in show_rcu_nocb_state()
1707 		"sS"[!!rdp->nocb_gp_sleep],  in show_rcu_nocb_state()
1713 static inline int rcu_lockdep_is_held_nocb(struct rcu_data *rdp)  in rcu_lockdep_is_held_nocb()  argument
1718 static inline bool rcu_current_is_nocb_kthread(struct rcu_data *rdp)  in rcu_current_is_nocb_kthread()  argument
1724 static void rcu_nocb_lock(struct rcu_data *rdp)  in rcu_nocb_lock()  argument
1729 static void rcu_nocb_unlock(struct rcu_data *rdp)  in rcu_nocb_unlock()  argument
1734 static void rcu_nocb_unlock_irqrestore(struct rcu_data *rdp,  in rcu_nocb_unlock_irqrestore()  argument
1741 static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp)  in rcu_lockdep_assert_cblist_protected()  argument
1759 static bool wake_nocb_gp(struct rcu_data *rdp, bool force)  in wake_nocb_gp()  argument
1764 static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp,  in rcu_nocb_flush_bypass()  argument
1770 static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp,  in rcu_nocb_try_bypass()  argument
1776 static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_empty,  in __call_rcu_nocb_wake()  argument
1782 static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)  in rcu_boot_init_nocb_percpu_data()  argument
1786 static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp, int level)  in rcu_nocb_need_deferred_wakeup()  argument
1791 static bool do_nocb_deferred_wakeup(struct rcu_data *rdp)  in do_nocb_deferred_wakeup()  argument
1800 static void show_rcu_nocb_state(struct rcu_data *rdp)  in show_rcu_nocb_state()  argument