16 static bool rcu_rdp_is_offloaded(struct rcu_data *rdp)  in rcu_rdp_is_offloaded()  argument
30 		  rcu_lockdep_is_held_nocb(rdp) ||  in rcu_rdp_is_offloaded()
31 		  (rdp == this_cpu_ptr(&rcu_data) &&  in rcu_rdp_is_offloaded()
33 		  rcu_current_is_nocb_kthread(rdp)),  in rcu_rdp_is_offloaded()
37 	return rcu_segcblist_is_offloaded(&rdp->cblist);  in rcu_rdp_is_offloaded()
151 static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp)  in rcu_preempt_ctxt_queue()  argument
156 			 (rnp->qsmask & rdp->grpmask ? RCU_GP_BLKD : 0) +  in rcu_preempt_ctxt_queue()
157 			 (rnp->expmask & rdp->grpmask ? RCU_EXP_BLKD : 0);  in rcu_preempt_ctxt_queue()
161 	WARN_ON_ONCE(rdp->mynode != rnp);  in rcu_preempt_ctxt_queue()
165 		     rdp->grpmask);  in rcu_preempt_ctxt_queue()
250 		     !(rnp->qsmask & rdp->grpmask));  in rcu_preempt_ctxt_queue()
252 		     !(rnp->expmask & rdp->grpmask));  in rcu_preempt_ctxt_queue()
263 	if (blkd_state & RCU_EXP_BLKD && rdp->cpu_no_qs.b.exp)  in rcu_preempt_ctxt_queue()
264 		rcu_report_exp_rdp(rdp);  in rcu_preempt_ctxt_queue()
266 		WARN_ON_ONCE(rdp->cpu_no_qs.b.exp);  in rcu_preempt_ctxt_queue()
315 	struct rcu_data *rdp = this_cpu_ptr(&rcu_data);  in rcu_note_context_switch()  local
325 		rnp = rdp->mynode;  in rcu_note_context_switch()
335 		WARN_ON_ONCE(!rcu_rdp_cpu_online(rdp));  in rcu_note_context_switch()
339 				       (rnp->qsmask & rdp->grpmask)  in rcu_note_context_switch()
342 		rcu_preempt_ctxt_queue(rnp, rdp);  in rcu_note_context_switch()
357 	if (rdp->cpu_no_qs.b.exp)  in rcu_note_context_switch()
358 		rcu_report_exp_rdp(rdp);  in rcu_note_context_switch()
473 	struct rcu_data *rdp;  in rcu_preempt_deferred_qs_irqrestore()  local
483 	rdp = this_cpu_ptr(&rcu_data);  in rcu_preempt_deferred_qs_irqrestore()
484 	if (!special.s && !rdp->cpu_no_qs.b.exp) {  in rcu_preempt_deferred_qs_irqrestore()
491 			rdp->cpu_no_qs.b.norm = false;  in rcu_preempt_deferred_qs_irqrestore()
492 			rcu_report_qs_rdp(rdp);  in rcu_preempt_deferred_qs_irqrestore()
505 	if (rdp->cpu_no_qs.b.exp)  in rcu_preempt_deferred_qs_irqrestore()
506 		rcu_report_exp_rdp(rdp);  in rcu_preempt_deferred_qs_irqrestore()
615 	struct rcu_data *rdp;  in rcu_preempt_deferred_qs_handler()  local
617 	rdp = container_of(iwp, struct rcu_data, defer_qs_iw);  in rcu_preempt_deferred_qs_handler()
618 	rdp->defer_qs_iw_pending = false;  in rcu_preempt_deferred_qs_handler()
641 		struct rcu_data *rdp = this_cpu_ptr(&rcu_data);  in rcu_read_unlock_special()  local
642 		struct rcu_node *rnp = rdp->mynode;  in rcu_read_unlock_special()
645 			   (rdp->grpmask & READ_ONCE(rnp->expmask)) ||  in rcu_read_unlock_special()
647 			   ((rdp->grpmask & READ_ONCE(rnp->qsmask)) || t->rcu_blocked_node)) ||  in rcu_read_unlock_special()
664 			    expboost && !rdp->defer_qs_iw_pending && cpu_online(rdp->cpu)) {  in rcu_read_unlock_special()
669 					rdp->defer_qs_iw = IRQ_WORK_INIT_HARD(  in rcu_read_unlock_special()
672 					init_irq_work(&rdp->defer_qs_iw,  in rcu_read_unlock_special()
674 				rdp->defer_qs_iw_pending = true;  in rcu_read_unlock_special()
675 				irq_work_queue_on(&rdp->defer_qs_iw, rdp->cpu);  in rcu_read_unlock_special()
783 	struct rcu_data *rdp;  in dump_blkd_tasks()  local
805 		rdp = per_cpu_ptr(&rcu_data, cpu);  in dump_blkd_tasks()
807 			cpu, ".o"[rcu_rdp_cpu_online(rdp)],  in dump_blkd_tasks()
808 			(long)rdp->rcu_onl_gp_seq, rdp->rcu_onl_gp_flags,  in dump_blkd_tasks()
809 			(long)rdp->rcu_ofl_gp_seq, rdp->rcu_ofl_gp_flags);  in dump_blkd_tasks()
822 	struct rcu_data *rdp;  in rcu_read_unlock_strict()  local
826 	rdp = this_cpu_ptr(&rcu_data);  in rcu_read_unlock_strict()
827 	rdp->cpu_no_qs.b.norm = false;  in rcu_read_unlock_strict()
828 	rcu_report_qs_rdp(rdp);  in rcu_read_unlock_strict()
944 	struct rcu_data *rdp = this_cpu_ptr(&rcu_data);  in rcu_preempt_deferred_qs()  local
946 	if (READ_ONCE(rdp->cpu_no_qs.b.exp))  in rcu_preempt_deferred_qs()
947 		rcu_report_exp_rdp(rdp);  in rcu_preempt_deferred_qs()
1007 	struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);  in rcu_cpu_kthread_setup()  local
1015 	WRITE_ONCE(rdp->rcuc_activity, jiffies);  in rcu_cpu_kthread_setup()
1018 static bool rcu_is_callbacks_nocb_kthread(struct rcu_data *rdp)  in rcu_is_callbacks_nocb_kthread()  argument
1021 	return rdp->nocb_cb_kthread == current;  in rcu_is_callbacks_nocb_kthread()
1031 static bool rcu_is_callbacks_kthread(struct rcu_data *rdp)  in rcu_is_callbacks_kthread()  argument
1033 	return rdp->rcu_cpu_kthread_task == current ||  in rcu_is_callbacks_kthread()
1034 			rcu_is_callbacks_nocb_kthread(rdp);  in rcu_is_callbacks_kthread()