16 static bool rcu_rdp_is_offloaded(struct rcu_data *rdp)  in rcu_rdp_is_offloaded()  argument
30 		  rcu_lockdep_is_held_nocb(rdp) ||  in rcu_rdp_is_offloaded()
31 		  (rdp == this_cpu_ptr(&rcu_data) &&  in rcu_rdp_is_offloaded()
33 		  rcu_current_is_nocb_kthread(rdp)),  in rcu_rdp_is_offloaded()
37 	return rcu_segcblist_is_offloaded(&rdp->cblist);  in rcu_rdp_is_offloaded()
153 static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp)  in rcu_preempt_ctxt_queue()  argument
158 			 (rnp->qsmask & rdp->grpmask ? RCU_GP_BLKD : 0) +  in rcu_preempt_ctxt_queue()
159 			 (rnp->expmask & rdp->grpmask ? RCU_EXP_BLKD : 0);  in rcu_preempt_ctxt_queue()
163 	WARN_ON_ONCE(rdp->mynode != rnp);  in rcu_preempt_ctxt_queue()
167 		     rdp->grpmask);  in rcu_preempt_ctxt_queue()
252 		     !(rnp->qsmask & rdp->grpmask));  in rcu_preempt_ctxt_queue()
254 		     !(rnp->expmask & rdp->grpmask));  in rcu_preempt_ctxt_queue()
263 	if (blkd_state & RCU_EXP_BLKD && rdp->exp_deferred_qs)  in rcu_preempt_ctxt_queue()
264 		rcu_report_exp_rdp(rdp);  in rcu_preempt_ctxt_queue()
266 		WARN_ON_ONCE(rdp->exp_deferred_qs);  in rcu_preempt_ctxt_queue()
311 	struct rcu_data *rdp = this_cpu_ptr(&rcu_data);  in rcu_note_context_switch()  local
321 		rnp = rdp->mynode;  in rcu_note_context_switch()
331 		WARN_ON_ONCE((rdp->grpmask & rcu_rnp_online_cpus(rnp)) == 0);  in rcu_note_context_switch()
335 				       (rnp->qsmask & rdp->grpmask)  in rcu_note_context_switch()
338 		rcu_preempt_ctxt_queue(rnp, rdp);  in rcu_note_context_switch()
353 	if (rdp->exp_deferred_qs)  in rcu_note_context_switch()
354 		rcu_report_exp_rdp(rdp);  in rcu_note_context_switch()
469 	struct rcu_data *rdp;  in rcu_preempt_deferred_qs_irqrestore()  local
479 	rdp = this_cpu_ptr(&rcu_data);  in rcu_preempt_deferred_qs_irqrestore()
480 	if (!special.s && !rdp->exp_deferred_qs) {  in rcu_preempt_deferred_qs_irqrestore()
487 			rcu_report_qs_rdp(rdp);  in rcu_preempt_deferred_qs_irqrestore()
500 	if (rdp->exp_deferred_qs)  in rcu_preempt_deferred_qs_irqrestore()
501 		rcu_report_exp_rdp(rdp);  in rcu_preempt_deferred_qs_irqrestore()
610 	struct rcu_data *rdp;  in rcu_preempt_deferred_qs_handler()  local
612 	rdp = container_of(iwp, struct rcu_data, defer_qs_iw);  in rcu_preempt_deferred_qs_handler()
613 	rdp->defer_qs_iw_pending = false;  in rcu_preempt_deferred_qs_handler()
636 		struct rcu_data *rdp = this_cpu_ptr(&rcu_data);  in rcu_read_unlock_special()  local
637 		struct rcu_node *rnp = rdp->mynode;  in rcu_read_unlock_special()
640 			   (rdp->grpmask & READ_ONCE(rnp->expmask)) ||  in rcu_read_unlock_special()
658 			    expboost && !rdp->defer_qs_iw_pending && cpu_online(rdp->cpu)) {  in rcu_read_unlock_special()
661 				init_irq_work(&rdp->defer_qs_iw, rcu_preempt_deferred_qs_handler);  in rcu_read_unlock_special()
662 				rdp->defer_qs_iw_pending = true;  in rcu_read_unlock_special()
663 				irq_work_queue_on(&rdp->defer_qs_iw, rdp->cpu);  in rcu_read_unlock_special()
775 	struct rcu_data *rdp;  in dump_blkd_tasks()  local
797 		rdp = per_cpu_ptr(&rcu_data, cpu);  in dump_blkd_tasks()
798 		onl = !!(rdp->grpmask & rcu_rnp_online_cpus(rnp));  in dump_blkd_tasks()
801 			(long)rdp->rcu_onl_gp_seq, rdp->rcu_onl_gp_flags,  in dump_blkd_tasks()
802 			(long)rdp->rcu_ofl_gp_seq, rdp->rcu_ofl_gp_flags);  in dump_blkd_tasks()
815 	struct rcu_data *rdp;  in rcu_read_unlock_strict()  local
820 	rdp = this_cpu_ptr(&rcu_data);  in rcu_read_unlock_strict()
821 	rcu_report_qs_rdp(rdp);  in rcu_read_unlock_strict()
1325 	struct rcu_data *rdp = this_cpu_ptr(&rcu_data);  in rcu_try_advance_all_cbs()  local
1329 	if (jiffies == rdp->last_advance_all)  in rcu_try_advance_all_cbs()
1331 	rdp->last_advance_all = jiffies;  in rcu_try_advance_all_cbs()
1333 	rnp = rdp->mynode;  in rcu_try_advance_all_cbs()
1340 	if ((rcu_seq_completed_gp(rdp->gp_seq,  in rcu_try_advance_all_cbs()
1342 	     unlikely(READ_ONCE(rdp->gpwrap))) &&  in rcu_try_advance_all_cbs()
1343 	    rcu_segcblist_pend_cbs(&rdp->cblist))  in rcu_try_advance_all_cbs()
1344 		note_gp_changes(rdp);  in rcu_try_advance_all_cbs()
1346 	if (rcu_segcblist_ready_cbs(&rdp->cblist))  in rcu_try_advance_all_cbs()
1360 	struct rcu_data *rdp = this_cpu_ptr(&rcu_data);  in rcu_needs_cpu()  local
1366 	if (rcu_segcblist_empty(&rdp->cblist) ||  in rcu_needs_cpu()
1367 	    rcu_rdp_is_offloaded(rdp)) {  in rcu_needs_cpu()
1378 	rdp->last_accelerate = jiffies;  in rcu_needs_cpu()
1398 	struct rcu_data *rdp = this_cpu_ptr(&rcu_data);  in rcu_prepare_for_idle()  local
1403 	if (rcu_rdp_is_offloaded(rdp))  in rcu_prepare_for_idle()
1408 	if (tne != rdp->tick_nohz_enabled_snap) {  in rcu_prepare_for_idle()
1409 		if (!rcu_segcblist_empty(&rdp->cblist))  in rcu_prepare_for_idle()
1411 		rdp->tick_nohz_enabled_snap = tne;  in rcu_prepare_for_idle()
1421 	if (rdp->last_accelerate == jiffies)  in rcu_prepare_for_idle()
1423 	rdp->last_accelerate = jiffies;  in rcu_prepare_for_idle()
1424 	if (rcu_segcblist_pend_cbs(&rdp->cblist)) {  in rcu_prepare_for_idle()
1425 		rnp = rdp->mynode;  in rcu_prepare_for_idle()
1427 		needwake = rcu_accelerate_cbs(rnp, rdp);  in rcu_prepare_for_idle()
1441 	struct rcu_data *rdp = this_cpu_ptr(&rcu_data);  in rcu_cleanup_after_idle()  local
1444 	if (rcu_rdp_is_offloaded(rdp))  in rcu_cleanup_after_idle()