84 static void rcu_report_exp_rnp(struct rcu_node *rnp, bool wake);
130 static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp)  in rcu_preempt_ctxt_queue()  argument
131 	__releases(rnp->lock) /* But leaves rrupts disabled. */  in rcu_preempt_ctxt_queue()
133 	int blkd_state = (rnp->gp_tasks ? RCU_GP_TASKS : 0) +  in rcu_preempt_ctxt_queue()
134 			 (rnp->exp_tasks ? RCU_EXP_TASKS : 0) +  in rcu_preempt_ctxt_queue()
135 			 (rnp->qsmask & rdp->grpmask ? RCU_GP_BLKD : 0) +  in rcu_preempt_ctxt_queue()
136 			 (rnp->expmask & rdp->grpmask ? RCU_EXP_BLKD : 0);  in rcu_preempt_ctxt_queue()
139 	raw_lockdep_assert_held_rcu_node(rnp);  in rcu_preempt_ctxt_queue()
140 	WARN_ON_ONCE(rdp->mynode != rnp);  in rcu_preempt_ctxt_queue()
141 	WARN_ON_ONCE(!rcu_is_leaf_node(rnp));  in rcu_preempt_ctxt_queue()
143 	WARN_ON_ONCE(rnp->qsmaskinitnext & ~rnp->qsmaskinit & rnp->qsmask &  in rcu_preempt_ctxt_queue()
164 		list_add(&t->rcu_node_entry, &rnp->blkd_tasks);  in rcu_preempt_ctxt_queue()
182 		list_add_tail(&t->rcu_node_entry, &rnp->blkd_tasks);  in rcu_preempt_ctxt_queue()
195 		list_add(&t->rcu_node_entry, rnp->exp_tasks);  in rcu_preempt_ctxt_queue()
206 		list_add(&t->rcu_node_entry, rnp->gp_tasks);  in rcu_preempt_ctxt_queue()
222 	if (!rnp->gp_tasks && (blkd_state & RCU_GP_BLKD)) {  in rcu_preempt_ctxt_queue()
223 		rnp->gp_tasks = &t->rcu_node_entry;  in rcu_preempt_ctxt_queue()
224 		WARN_ON_ONCE(rnp->completedqs == rnp->gp_seq);  in rcu_preempt_ctxt_queue()
226 	if (!rnp->exp_tasks && (blkd_state & RCU_EXP_BLKD))  in rcu_preempt_ctxt_queue()
227 		rnp->exp_tasks = &t->rcu_node_entry;  in rcu_preempt_ctxt_queue()
229 		     !(rnp->qsmask & rdp->grpmask));  in rcu_preempt_ctxt_queue()
231 		     !(rnp->expmask & rdp->grpmask));  in rcu_preempt_ctxt_queue()
232 	raw_spin_unlock_rcu_node(rnp); /* interrupts remain disabled. */  in rcu_preempt_ctxt_queue()
289 	struct rcu_node *rnp;  in rcu_note_context_switch()  local
298 		rnp = rdp->mynode;  in rcu_note_context_switch()
299 		raw_spin_lock_rcu_node(rnp);  in rcu_note_context_switch()
301 		t->rcu_blocked_node = rnp;  in rcu_note_context_switch()
308 		WARN_ON_ONCE((rdp->grpmask & rcu_rnp_online_cpus(rnp)) == 0);  in rcu_note_context_switch()
312 				       (rnp->qsmask & rdp->grpmask)  in rcu_note_context_switch()
313 				       ? rnp->gp_seq  in rcu_note_context_switch()
314 				       : rcu_seq_snap(&rnp->gp_seq));  in rcu_note_context_switch()
315 		rcu_preempt_ctxt_queue(rnp, rdp);  in rcu_note_context_switch()
341 static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp)  in rcu_preempt_blocked_readers_cgp()  argument
343 	return rnp->gp_tasks != NULL;  in rcu_preempt_blocked_readers_cgp()
400 					     struct rcu_node *rnp)  in rcu_next_node_entry()  argument
405 	if (np == &rnp->blkd_tasks)  in rcu_next_node_entry()
414 static bool rcu_preempt_has_tasks(struct rcu_node *rnp)  in rcu_preempt_has_tasks()  argument
416 	return !list_empty(&rnp->blkd_tasks);  in rcu_preempt_has_tasks()
433 	struct rcu_node *rnp;  in rcu_preempt_deferred_qs_irqrestore()  local
481 		rnp = t->rcu_blocked_node;  in rcu_preempt_deferred_qs_irqrestore()
482 		raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */  in rcu_preempt_deferred_qs_irqrestore()
483 		WARN_ON_ONCE(rnp != t->rcu_blocked_node);  in rcu_preempt_deferred_qs_irqrestore()
484 		WARN_ON_ONCE(!rcu_is_leaf_node(rnp));  in rcu_preempt_deferred_qs_irqrestore()
485 		empty_norm = !rcu_preempt_blocked_readers_cgp(rnp);  in rcu_preempt_deferred_qs_irqrestore()
486 		WARN_ON_ONCE(rnp->completedqs == rnp->gp_seq &&  in rcu_preempt_deferred_qs_irqrestore()
487 			     (!empty_norm || rnp->qsmask));  in rcu_preempt_deferred_qs_irqrestore()
488 		empty_exp = sync_rcu_preempt_exp_done(rnp);  in rcu_preempt_deferred_qs_irqrestore()
490 		np = rcu_next_node_entry(t, rnp);  in rcu_preempt_deferred_qs_irqrestore()
494 						rnp->gp_seq, t->pid);  in rcu_preempt_deferred_qs_irqrestore()
495 		if (&t->rcu_node_entry == rnp->gp_tasks)  in rcu_preempt_deferred_qs_irqrestore()
496 			rnp->gp_tasks = np;  in rcu_preempt_deferred_qs_irqrestore()
497 		if (&t->rcu_node_entry == rnp->exp_tasks)  in rcu_preempt_deferred_qs_irqrestore()
498 			rnp->exp_tasks = np;  in rcu_preempt_deferred_qs_irqrestore()
501 			drop_boost_mutex = rt_mutex_owner(&rnp->boost_mtx) == t;  in rcu_preempt_deferred_qs_irqrestore()
502 			if (&t->rcu_node_entry == rnp->boost_tasks)  in rcu_preempt_deferred_qs_irqrestore()
503 				rnp->boost_tasks = np;  in rcu_preempt_deferred_qs_irqrestore()
512 		empty_exp_now = sync_rcu_preempt_exp_done(rnp);  in rcu_preempt_deferred_qs_irqrestore()
513 		if (!empty_norm && !rcu_preempt_blocked_readers_cgp(rnp)) {  in rcu_preempt_deferred_qs_irqrestore()
515 							 rnp->gp_seq,  in rcu_preempt_deferred_qs_irqrestore()
516 							 0, rnp->qsmask,  in rcu_preempt_deferred_qs_irqrestore()
517 							 rnp->level,  in rcu_preempt_deferred_qs_irqrestore()
518 							 rnp->grplo,  in rcu_preempt_deferred_qs_irqrestore()
519 							 rnp->grphi,  in rcu_preempt_deferred_qs_irqrestore()
520 							 !!rnp->gp_tasks);  in rcu_preempt_deferred_qs_irqrestore()
521 			rcu_report_unblock_qs_rnp(rnp, flags);  in rcu_preempt_deferred_qs_irqrestore()
523 			raw_spin_unlock_irqrestore_rcu_node(rnp, flags);  in rcu_preempt_deferred_qs_irqrestore()
528 			rt_mutex_futex_unlock(&rnp->boost_mtx);  in rcu_preempt_deferred_qs_irqrestore()
535 			rcu_report_exp_rnp(rnp, true);  in rcu_preempt_deferred_qs_irqrestore()
611 		struct rcu_node *rnp = rdp->mynode;  in rcu_read_unlock_special()  local
615 		      (rdp->grpmask & rnp->expmask) ||  in rcu_read_unlock_special()
657 static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp)  in rcu_preempt_check_blocked_tasks()  argument
662 	if (WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp)))  in rcu_preempt_check_blocked_tasks()
663 		dump_blkd_tasks(rnp, 10);  in rcu_preempt_check_blocked_tasks()
664 	if (rcu_preempt_has_tasks(rnp) &&  in rcu_preempt_check_blocked_tasks()
665 	    (rnp->qsmaskinit || rnp->wait_blkd_tasks)) {  in rcu_preempt_check_blocked_tasks()
666 		rnp->gp_tasks = rnp->blkd_tasks.next;  in rcu_preempt_check_blocked_tasks()
667 		t = container_of(rnp->gp_tasks, struct task_struct,  in rcu_preempt_check_blocked_tasks()
670 						rnp->gp_seq, t->pid);  in rcu_preempt_check_blocked_tasks()
672 	WARN_ON_ONCE(rnp->qsmask);  in rcu_preempt_check_blocked_tasks()
743 dump_blkd_tasks(struct rcu_node *rnp, int ncheck)  in dump_blkd_tasks()  argument
752 	raw_lockdep_assert_held_rcu_node(rnp);  in dump_blkd_tasks()
754 		__func__, rnp->grplo, rnp->grphi, rnp->level,  in dump_blkd_tasks()
755 		(long)rnp->gp_seq, (long)rnp->completedqs);  in dump_blkd_tasks()
756 	for (rnp1 = rnp; rnp1; rnp1 = rnp1->parent)  in dump_blkd_tasks()
760 		__func__, rnp->gp_tasks, rnp->boost_tasks, rnp->exp_tasks);  in dump_blkd_tasks()
763 	list_for_each(lhp, &rnp->blkd_tasks) {  in dump_blkd_tasks()
769 	for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++) {  in dump_blkd_tasks()
771 		onl = !!(rdp->grpmask & rcu_rnp_online_cpus(rnp));  in dump_blkd_tasks()
864 static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp)  in rcu_preempt_blocked_readers_cgp()  argument
872 static bool rcu_preempt_has_tasks(struct rcu_node *rnp)  in rcu_preempt_has_tasks()  argument
892 static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp)  in rcu_preempt_check_blocked_tasks()  argument
894 	WARN_ON_ONCE(rnp->qsmask);  in rcu_preempt_check_blocked_tasks()
933 dump_blkd_tasks(struct rcu_node *rnp, int ncheck)  in dump_blkd_tasks()  argument
935 	WARN_ON_ONCE(!list_empty(&rnp->blkd_tasks));  in dump_blkd_tasks()
963 static int rcu_boost(struct rcu_node *rnp)  in rcu_boost()  argument
969 	if (READ_ONCE(rnp->exp_tasks) == NULL &&  in rcu_boost()
970 	    READ_ONCE(rnp->boost_tasks) == NULL)  in rcu_boost()
973 	raw_spin_lock_irqsave_rcu_node(rnp, flags);  in rcu_boost()
979 	if (rnp->exp_tasks == NULL && rnp->boost_tasks == NULL) {  in rcu_boost()
980 		raw_spin_unlock_irqrestore_rcu_node(rnp, flags);  in rcu_boost()
990 	if (rnp->exp_tasks != NULL)  in rcu_boost()
991 		tb = rnp->exp_tasks;  in rcu_boost()
993 		tb = rnp->boost_tasks;  in rcu_boost()
1012 	rt_mutex_init_proxy_locked(&rnp->boost_mtx, t);  in rcu_boost()
1013 	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);  in rcu_boost()
1015 	rt_mutex_lock(&rnp->boost_mtx);  in rcu_boost()
1016 	rt_mutex_unlock(&rnp->boost_mtx);  /* Then keep lockdep happy. */  in rcu_boost()
1018 	return READ_ONCE(rnp->exp_tasks) != NULL ||  in rcu_boost()
1019 	       READ_ONCE(rnp->boost_tasks) != NULL;  in rcu_boost()
1027 	struct rcu_node *rnp = (struct rcu_node *)arg;  in rcu_boost_kthread()  local
1033 		rnp->boost_kthread_status = RCU_KTHREAD_WAITING;  in rcu_boost_kthread()
1035 		rcu_wait(rnp->boost_tasks || rnp->exp_tasks);  in rcu_boost_kthread()
1037 		rnp->boost_kthread_status = RCU_KTHREAD_RUNNING;  in rcu_boost_kthread()
1038 		more2boost = rcu_boost(rnp);  in rcu_boost_kthread()
1044 			rnp->boost_kthread_status = RCU_KTHREAD_YIELDING;  in rcu_boost_kthread()
1066 static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags)  in rcu_initiate_boost()  argument
1067 	__releases(rnp->lock)  in rcu_initiate_boost()
1069 	raw_lockdep_assert_held_rcu_node(rnp);  in rcu_initiate_boost()
1070 	if (!rcu_preempt_blocked_readers_cgp(rnp) && rnp->exp_tasks == NULL) {  in rcu_initiate_boost()
1071 		raw_spin_unlock_irqrestore_rcu_node(rnp, flags);  in rcu_initiate_boost()
1074 	if (rnp->exp_tasks != NULL ||  in rcu_initiate_boost()
1075 	    (rnp->gp_tasks != NULL &&  in rcu_initiate_boost()
1076 	     rnp->boost_tasks == NULL &&  in rcu_initiate_boost()
1077 	     rnp->qsmask == 0 &&  in rcu_initiate_boost()
1078 	     ULONG_CMP_GE(jiffies, rnp->boost_time))) {  in rcu_initiate_boost()
1079 		if (rnp->exp_tasks == NULL)  in rcu_initiate_boost()
1080 			rnp->boost_tasks = rnp->gp_tasks;  in rcu_initiate_boost()
1081 		raw_spin_unlock_irqrestore_rcu_node(rnp, flags);  in rcu_initiate_boost()
1082 		rcu_wake_cond(rnp->boost_kthread_task,  in rcu_initiate_boost()
1083 			      rnp->boost_kthread_status);  in rcu_initiate_boost()
1085 		raw_spin_unlock_irqrestore_rcu_node(rnp, flags);  in rcu_initiate_boost()
1103 static void rcu_preempt_boost_start_gp(struct rcu_node *rnp)  in rcu_preempt_boost_start_gp()  argument
1105 	rnp->boost_time = jiffies + RCU_BOOST_DELAY_JIFFIES;  in rcu_preempt_boost_start_gp()
1113 static void rcu_spawn_one_boost_kthread(struct rcu_node *rnp)  in rcu_spawn_one_boost_kthread()  argument
1115 	int rnp_index = rnp - rcu_get_root();  in rcu_spawn_one_boost_kthread()
1123 	if (!rcu_scheduler_fully_active || rcu_rnp_online_cpus(rnp) == 0)  in rcu_spawn_one_boost_kthread()
1128 	if (rnp->boost_kthread_task != NULL)  in rcu_spawn_one_boost_kthread()
1131 	t = kthread_create(rcu_boost_kthread, (void *)rnp,  in rcu_spawn_one_boost_kthread()
1136 	raw_spin_lock_irqsave_rcu_node(rnp, flags);  in rcu_spawn_one_boost_kthread()
1137 	rnp->boost_kthread_task = t;  in rcu_spawn_one_boost_kthread()
1138 	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);  in rcu_spawn_one_boost_kthread()
1153 static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu)  in rcu_boost_kthread_setaffinity()  argument
1155 	struct task_struct *t = rnp->boost_kthread_task;  in rcu_boost_kthread_setaffinity()
1156 	unsigned long mask = rcu_rnp_online_cpus(rnp);  in rcu_boost_kthread_setaffinity()
1164 	for_each_leaf_node_possible_cpu(rnp, cpu)  in rcu_boost_kthread_setaffinity()
1165 		if ((mask & leaf_node_cpu_bit(rnp, cpu)) &&  in rcu_boost_kthread_setaffinity()
1179 	struct rcu_node *rnp;  in rcu_spawn_boost_kthreads()  local
1181 	rcu_for_each_leaf_node(rnp)  in rcu_spawn_boost_kthreads()
1182 		rcu_spawn_one_boost_kthread(rnp);  in rcu_spawn_boost_kthreads()
1188 	struct rcu_node *rnp = rdp->mynode;  in rcu_prepare_kthreads()  local
1192 		rcu_spawn_one_boost_kthread(rnp);  in rcu_prepare_kthreads()
1197 static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags)  in rcu_initiate_boost()  argument
1198 	__releases(rnp->lock)  in rcu_initiate_boost()
1200 	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);  in rcu_initiate_boost()
1208 static void rcu_preempt_boost_start_gp(struct rcu_node *rnp)  in rcu_preempt_boost_start_gp()  argument
1212 static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu)  in rcu_boost_kthread_setaffinity()  argument
1302 	struct rcu_node *rnp;  in rcu_try_advance_all_cbs()  local
1309 	rnp = rdp->mynode;  in rcu_try_advance_all_cbs()
1317 				  rcu_seq_current(&rnp->gp_seq)) ||  in rcu_try_advance_all_cbs()
1383 	struct rcu_node *rnp;  in rcu_prepare_for_idle()  local
1420 		rnp = rdp->mynode;  in rcu_prepare_for_idle()
1421 		raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */  in rcu_prepare_for_idle()
1422 		needwake = rcu_accelerate_cbs(rnp, rdp);  in rcu_prepare_for_idle()
1423 		raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */  in rcu_prepare_for_idle()
1618 static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp)  in rcu_nocb_gp_get()  argument
1620 	return &rnp->nocb_gp_wq[rcu_seq_ctr(rnp->gp_seq) & 0x1];  in rcu_nocb_gp_get()
1623 static void rcu_init_one_nocb(struct rcu_node *rnp)  in rcu_init_one_nocb()  argument
1625 	init_swait_queue_head(&rnp->nocb_gp_wq[0]);  in rcu_init_one_nocb()
1626 	init_swait_queue_head(&rnp->nocb_gp_wq[1]);  in rcu_init_one_nocb()
1955 	struct rcu_node *rnp;  in nocb_gp_wait()  local
1982 		rnp = rdp->mynode;  in nocb_gp_wait()
1993 		     rcu_seq_done(&rnp->gp_seq, cur_gp_seq))) {  in nocb_gp_wait()
1994 			raw_spin_lock_rcu_node(rnp); /* irqs disabled. */  in nocb_gp_wait()
1995 			needwake_gp = rcu_advance_cbs(rnp, rdp);  in nocb_gp_wait()
1996 			raw_spin_unlock_rcu_node(rnp); /* irqs disabled. */  in nocb_gp_wait()
2047 		rnp = my_rdp->mynode;  in nocb_gp_wait()
2048 		trace_rcu_this_gp(rnp, my_rdp, wait_gp_seq, TPS("StartWait"));  in nocb_gp_wait()
2050 			rnp->nocb_gp_wq[rcu_seq_ctr(wait_gp_seq) & 0x1],  in nocb_gp_wait()
2051 			rcu_seq_done(&rnp->gp_seq, wait_gp_seq) ||  in nocb_gp_wait()
2053 		trace_rcu_this_gp(rnp, my_rdp, wait_gp_seq, TPS("EndWait"));  in nocb_gp_wait()
2095 	struct rcu_node *rnp = rdp->mynode;  in nocb_cb_wait()  local
2106 	    rcu_seq_done(&rnp->gp_seq, cur_gp_seq) &&  in nocb_cb_wait()
2107 	    raw_spin_trylock_rcu_node(rnp)) { /* irqs already disabled. */  in nocb_cb_wait()
2109 		raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */  in nocb_cb_wait()
2380 	struct rcu_node *rnp = rdp->mynode;  in show_rcu_nocb_gp_state()  local
2391 		".W"[swait_active(&rnp->nocb_gp_wq[0])],  in show_rcu_nocb_gp_state()
2392 		".W"[swait_active(&rnp->nocb_gp_wq[1])],  in show_rcu_nocb_gp_state()
2396 		rnp->grplo, rnp->grphi, READ_ONCE(rdp->nocb_gp_loops));  in show_rcu_nocb_gp_state()
2476 static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp)  in rcu_nocb_gp_get()  argument
2481 static void rcu_init_one_nocb(struct rcu_node *rnp)  in rcu_init_one_nocb()  argument