13 static int rcu_print_task_exp_stall(struct rcu_node *rnp);
78 	struct rcu_node *rnp;  in sync_exp_reset_tree_hotplug()  local
90 	rcu_for_each_leaf_node(rnp) {  in sync_exp_reset_tree_hotplug()
91 		raw_spin_lock_irqsave_rcu_node(rnp, flags);  in sync_exp_reset_tree_hotplug()
92 		if (rnp->expmaskinit == rnp->expmaskinitnext) {  in sync_exp_reset_tree_hotplug()
93 			raw_spin_unlock_irqrestore_rcu_node(rnp, flags);  in sync_exp_reset_tree_hotplug()
98 		oldmask = rnp->expmaskinit;  in sync_exp_reset_tree_hotplug()
99 		rnp->expmaskinit = rnp->expmaskinitnext;  in sync_exp_reset_tree_hotplug()
100 		raw_spin_unlock_irqrestore_rcu_node(rnp, flags);  in sync_exp_reset_tree_hotplug()
107 		mask = rnp->grpmask;  in sync_exp_reset_tree_hotplug()
108 		rnp_up = rnp->parent;  in sync_exp_reset_tree_hotplug()
131 	struct rcu_node *rnp;  in sync_exp_reset_tree()  local
134 	rcu_for_each_node_breadth_first(rnp) {  in sync_exp_reset_tree()
135 		raw_spin_lock_irqsave_rcu_node(rnp, flags);  in sync_exp_reset_tree()
136 		WARN_ON_ONCE(rnp->expmask);  in sync_exp_reset_tree()
137 		rnp->expmask = rnp->expmaskinit;  in sync_exp_reset_tree()
138 		raw_spin_unlock_irqrestore_rcu_node(rnp, flags);  in sync_exp_reset_tree()
151 static bool sync_rcu_preempt_exp_done(struct rcu_node *rnp)  in sync_rcu_preempt_exp_done()  argument
153 	raw_lockdep_assert_held_rcu_node(rnp);  in sync_rcu_preempt_exp_done()
155 	return rnp->exp_tasks == NULL &&  in sync_rcu_preempt_exp_done()
156 	       READ_ONCE(rnp->expmask) == 0;  in sync_rcu_preempt_exp_done()
164 static bool sync_rcu_preempt_exp_done_unlocked(struct rcu_node *rnp)  in sync_rcu_preempt_exp_done_unlocked()  argument
169 	raw_spin_lock_irqsave_rcu_node(rnp, flags);  in sync_rcu_preempt_exp_done_unlocked()
170 	ret = sync_rcu_preempt_exp_done(rnp);  in sync_rcu_preempt_exp_done_unlocked()
171 	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);  in sync_rcu_preempt_exp_done_unlocked()
187 static void __rcu_report_exp_rnp(struct rcu_node *rnp,  in __rcu_report_exp_rnp()  argument
189 	__releases(rnp->lock)  in __rcu_report_exp_rnp()
194 		if (!sync_rcu_preempt_exp_done(rnp)) {  in __rcu_report_exp_rnp()
195 			if (!rnp->expmask)  in __rcu_report_exp_rnp()
196 				rcu_initiate_boost(rnp, flags);  in __rcu_report_exp_rnp()
198 				raw_spin_unlock_irqrestore_rcu_node(rnp, flags);  in __rcu_report_exp_rnp()
201 		if (rnp->parent == NULL) {  in __rcu_report_exp_rnp()
202 			raw_spin_unlock_irqrestore_rcu_node(rnp, flags);  in __rcu_report_exp_rnp()
209 		mask = rnp->grpmask;  in __rcu_report_exp_rnp()
210 		raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled */  in __rcu_report_exp_rnp()
211 		rnp = rnp->parent;  in __rcu_report_exp_rnp()
212 		raw_spin_lock_rcu_node(rnp); /* irqs already disabled */  in __rcu_report_exp_rnp()
213 		WARN_ON_ONCE(!(rnp->expmask & mask));  in __rcu_report_exp_rnp()
214 		rnp->expmask &= ~mask;  in __rcu_report_exp_rnp()
222 static void __maybe_unused rcu_report_exp_rnp(struct rcu_node *rnp, bool wake)  in rcu_report_exp_rnp()  argument
226 	raw_spin_lock_irqsave_rcu_node(rnp, flags);  in rcu_report_exp_rnp()
227 	__rcu_report_exp_rnp(rnp, wake, flags);  in rcu_report_exp_rnp()
234 static void rcu_report_exp_cpu_mult(struct rcu_node *rnp,  in rcu_report_exp_cpu_mult()  argument
239 	raw_spin_lock_irqsave_rcu_node(rnp, flags);  in rcu_report_exp_cpu_mult()
240 	if (!(rnp->expmask & mask)) {  in rcu_report_exp_cpu_mult()
241 		raw_spin_unlock_irqrestore_rcu_node(rnp, flags);  in rcu_report_exp_cpu_mult()
244 	rnp->expmask &= ~mask;  in rcu_report_exp_cpu_mult()
245 	__rcu_report_exp_rnp(rnp, wake, flags); /* Releases rnp->lock. */  in rcu_report_exp_cpu_mult()
278 	struct rcu_node *rnp = rdp->mynode;  in exp_funnel_lock()  local
282 	if (ULONG_CMP_LT(READ_ONCE(rnp->exp_seq_rq), s) &&  in exp_funnel_lock()
283 	    (rnp == rnp_root ||  in exp_funnel_lock()
295 	for (; rnp != NULL; rnp = rnp->parent) {  in exp_funnel_lock()
300 		spin_lock(&rnp->exp_lock);  in exp_funnel_lock()
301 		if (ULONG_CMP_GE(rnp->exp_seq_rq, s)) {  in exp_funnel_lock()
304 			spin_unlock(&rnp->exp_lock);  in exp_funnel_lock()
305 			trace_rcu_exp_funnel_lock(rcu_state.name, rnp->level,  in exp_funnel_lock()
306 						  rnp->grplo, rnp->grphi,  in exp_funnel_lock()
308 			wait_event(rnp->exp_wq[rcu_seq_ctr(s) & 0x3],  in exp_funnel_lock()
312 		rnp->exp_seq_rq = s; /* Followers can wait on us. */  in exp_funnel_lock()
313 		spin_unlock(&rnp->exp_lock);  in exp_funnel_lock()
314 		trace_rcu_exp_funnel_lock(rcu_state.name, rnp->level,  in exp_funnel_lock()
315 					  rnp->grplo, rnp->grphi, TPS("nxtlvl"));  in exp_funnel_lock()
341 	struct rcu_node *rnp = container_of(rewp, struct rcu_node, rew);  in sync_rcu_exp_select_node_cpus()  local
343 	raw_spin_lock_irqsave_rcu_node(rnp, flags);  in sync_rcu_exp_select_node_cpus()
347 	for_each_leaf_node_cpu_mask(rnp, cpu, rnp->expmask) {  in sync_rcu_exp_select_node_cpus()
348 		unsigned long mask = leaf_node_cpu_bit(rnp, cpu);  in sync_rcu_exp_select_node_cpus()
353 		    !(rnp->qsmaskinitnext & mask)) {  in sync_rcu_exp_select_node_cpus()
363 	mask_ofl_ipi = rnp->expmask & ~mask_ofl_test;  in sync_rcu_exp_select_node_cpus()
370 	if (rcu_preempt_has_tasks(rnp))  in sync_rcu_exp_select_node_cpus()
371 		rnp->exp_tasks = rnp->blkd_tasks.next;  in sync_rcu_exp_select_node_cpus()
372 	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);  in sync_rcu_exp_select_node_cpus()
375 	for_each_leaf_node_cpu_mask(rnp, cpu, rnp->expmask) {  in sync_rcu_exp_select_node_cpus()
376 		unsigned long mask = leaf_node_cpu_bit(rnp, cpu);  in sync_rcu_exp_select_node_cpus()
397 		raw_spin_lock_irqsave_rcu_node(rnp, flags);  in sync_rcu_exp_select_node_cpus()
398 		if ((rnp->qsmaskinitnext & mask) &&  in sync_rcu_exp_select_node_cpus()
399 		    (rnp->expmask & mask)) {  in sync_rcu_exp_select_node_cpus()
401 			raw_spin_unlock_irqrestore_rcu_node(rnp, flags);  in sync_rcu_exp_select_node_cpus()
407 		if (!(rnp->expmask & mask))  in sync_rcu_exp_select_node_cpus()
409 		raw_spin_unlock_irqrestore_rcu_node(rnp, flags);  in sync_rcu_exp_select_node_cpus()
414 		rcu_report_exp_cpu_mult(rnp, mask_ofl_test, false);  in sync_rcu_exp_select_node_cpus()
424 	struct rcu_node *rnp;  in sync_rcu_exp_select_cpus()  local
431 	rcu_for_each_leaf_node(rnp) {  in sync_rcu_exp_select_cpus()
432 		rnp->exp_need_flush = false;  in sync_rcu_exp_select_cpus()
433 		if (!READ_ONCE(rnp->expmask))  in sync_rcu_exp_select_cpus()
437 		    rcu_is_last_leaf_node(rnp)) {  in sync_rcu_exp_select_cpus()
439 			sync_rcu_exp_select_node_cpus(&rnp->rew.rew_work);  in sync_rcu_exp_select_cpus()
442 		INIT_WORK(&rnp->rew.rew_work, sync_rcu_exp_select_node_cpus);  in sync_rcu_exp_select_cpus()
443 		cpu = find_next_bit(&rnp->ffmask, BITS_PER_LONG, -1);  in sync_rcu_exp_select_cpus()
445 		if (unlikely(cpu > rnp->grphi - rnp->grplo))  in sync_rcu_exp_select_cpus()
448 			cpu += rnp->grplo;  in sync_rcu_exp_select_cpus()
449 		queue_work_on(cpu, rcu_par_gp_wq, &rnp->rew.rew_work);  in sync_rcu_exp_select_cpus()
450 		rnp->exp_need_flush = true;  in sync_rcu_exp_select_cpus()
454 	rcu_for_each_leaf_node(rnp)  in sync_rcu_exp_select_cpus()
455 		if (rnp->exp_need_flush)  in sync_rcu_exp_select_cpus()
456 			flush_work(&rnp->rew.rew_work);  in sync_rcu_exp_select_cpus()
466 	struct rcu_node *rnp;  in synchronize_sched_expedited_wait()  local
488 		rcu_for_each_leaf_node(rnp) {  in synchronize_sched_expedited_wait()
489 			ndetected += rcu_print_task_exp_stall(rnp);  in synchronize_sched_expedited_wait()
490 			for_each_leaf_node_possible_cpu(rnp, cpu) {  in synchronize_sched_expedited_wait()
493 				mask = leaf_node_cpu_bit(rnp, cpu);  in synchronize_sched_expedited_wait()
494 				if (!(rnp->expmask & mask))  in synchronize_sched_expedited_wait()
500 					"o."[!!(rdp->grpmask & rnp->expmaskinit)],  in synchronize_sched_expedited_wait()
501 					"N."[!!(rdp->grpmask & rnp->expmaskinitnext)]);  in synchronize_sched_expedited_wait()
509 			rcu_for_each_node_breadth_first(rnp) {  in synchronize_sched_expedited_wait()
510 				if (rnp == rnp_root)  in synchronize_sched_expedited_wait()
512 				if (sync_rcu_preempt_exp_done_unlocked(rnp))  in synchronize_sched_expedited_wait()
515 					rnp->level, rnp->grplo, rnp->grphi,  in synchronize_sched_expedited_wait()
516 					rnp->expmask,  in synchronize_sched_expedited_wait()
517 					".T"[!!rnp->exp_tasks]);  in synchronize_sched_expedited_wait()
521 		rcu_for_each_leaf_node(rnp) {  in synchronize_sched_expedited_wait()
522 			for_each_leaf_node_possible_cpu(rnp, cpu) {  in synchronize_sched_expedited_wait()
523 				mask = leaf_node_cpu_bit(rnp, cpu);  in synchronize_sched_expedited_wait()
524 				if (!(rnp->expmask & mask))  in synchronize_sched_expedited_wait()
541 	struct rcu_node *rnp;  in rcu_exp_wait_wake()  local
553 	rcu_for_each_node_breadth_first(rnp) {  in rcu_exp_wait_wake()
554 		if (ULONG_CMP_LT(READ_ONCE(rnp->exp_seq_rq), s)) {  in rcu_exp_wait_wake()
555 			spin_lock(&rnp->exp_lock);  in rcu_exp_wait_wake()
557 			if (ULONG_CMP_LT(rnp->exp_seq_rq, s))  in rcu_exp_wait_wake()
558 				rnp->exp_seq_rq = s;  in rcu_exp_wait_wake()
559 			spin_unlock(&rnp->exp_lock);  in rcu_exp_wait_wake()
562 		wake_up_all(&rnp->exp_wq[rcu_seq_ctr(rcu_state.expedited_sequence) & 0x3]);  in rcu_exp_wait_wake()
605 	struct rcu_node *rnp = rdp->mynode;  in rcu_exp_handler()  local
638 		raw_spin_lock_irqsave_rcu_node(rnp, flags);  in rcu_exp_handler()
639 		if (rnp->expmask & rdp->grpmask) {  in rcu_exp_handler()
643 		raw_spin_unlock_irqrestore_rcu_node(rnp, flags);  in rcu_exp_handler()
683 static int rcu_print_task_exp_stall(struct rcu_node *rnp)  in rcu_print_task_exp_stall()  argument
688 	if (!rnp->exp_tasks)  in rcu_print_task_exp_stall()
690 	t = list_entry(rnp->exp_tasks->prev,  in rcu_print_task_exp_stall()
692 	list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) {  in rcu_print_task_exp_stall()
715 	struct rcu_node *rnp;  in rcu_exp_handler()  local
718 	rnp = rdp->mynode;  in rcu_exp_handler()
719 	if (!(READ_ONCE(rnp->expmask) & rdp->grpmask) ||  in rcu_exp_handler()
736 	struct rcu_node *rnp;  in sync_sched_exp_online_cleanup()  local
739 	rnp = rdp->mynode;  in sync_sched_exp_online_cleanup()
742 	if (!(READ_ONCE(rnp->expmask) & rdp->grpmask) ||  in sync_sched_exp_online_cleanup()
766 static int rcu_print_task_exp_stall(struct rcu_node *rnp)  in rcu_print_task_exp_stall()  argument
797 	struct rcu_node *rnp;  in synchronize_rcu_expedited()  local
832 	rnp = rcu_get_root();  in synchronize_rcu_expedited()
833 	wait_event(rnp->exp_wq[rcu_seq_ctr(s) & 0x3],  in synchronize_rcu_expedited()