251 		struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu);  in cblist_init_generic()  local
253 		WARN_ON_ONCE(!rtpcp);  in cblist_init_generic()
255 			raw_spin_lock_init(&ACCESS_PRIVATE(rtpcp, lock));  in cblist_init_generic()
256 		raw_spin_lock_rcu_node(rtpcp); // irqs already disabled.  in cblist_init_generic()
257 		if (rcu_segcblist_empty(&rtpcp->cblist))  in cblist_init_generic()
258 			rcu_segcblist_init(&rtpcp->cblist);  in cblist_init_generic()
259 		INIT_WORK(&rtpcp->rtp_work, rcu_tasks_invoke_cbs_wq);  in cblist_init_generic()
260 		rtpcp->cpu = cpu;  in cblist_init_generic()
261 		rtpcp->rtpp = rtp;  in cblist_init_generic()
262 		if (!rtpcp->rtp_blkd_tasks.next)  in cblist_init_generic()
263 			INIT_LIST_HEAD(&rtpcp->rtp_blkd_tasks);  in cblist_init_generic()
264 		raw_spin_unlock_rcu_node(rtpcp); // irqs remain disabled.  in cblist_init_generic()
274 	struct rcu_tasks_percpu *rtpcp = container_of(iwp, struct rcu_tasks_percpu, rtp_irq_work);  in call_rcu_tasks_iw_wakeup()  local
276 	rtp = rtpcp->rtpp;  in call_rcu_tasks_iw_wakeup()
290 	struct rcu_tasks_percpu *rtpcp;  in call_rcu_tasks_generic()  local
298 	rtpcp = per_cpu_ptr(rtp->rtpcpu, chosen_cpu);  in call_rcu_tasks_generic()
299 	if (!raw_spin_trylock_rcu_node(rtpcp)) { // irqs already disabled.  in call_rcu_tasks_generic()
300 		raw_spin_lock_rcu_node(rtpcp); // irqs already disabled.  in call_rcu_tasks_generic()
302 		if (rtpcp->rtp_jiffies != j) {  in call_rcu_tasks_generic()
303 			rtpcp->rtp_jiffies = j;  in call_rcu_tasks_generic()
304 			rtpcp->rtp_n_lock_retries = 0;  in call_rcu_tasks_generic()
306 		if (rcu_task_cb_adjust && ++rtpcp->rtp_n_lock_retries > rcu_task_contend_lim &&  in call_rcu_tasks_generic()
310 	if (!rcu_segcblist_is_enabled(&rtpcp->cblist)) {  in call_rcu_tasks_generic()
311 		raw_spin_unlock_rcu_node(rtpcp); // irqs remain disabled.  in call_rcu_tasks_generic()
313 		raw_spin_lock_rcu_node(rtpcp); // irqs already disabled.  in call_rcu_tasks_generic()
315 	needwake = rcu_segcblist_empty(&rtpcp->cblist);  in call_rcu_tasks_generic()
316 	rcu_segcblist_enqueue(&rtpcp->cblist, rhp);  in call_rcu_tasks_generic()
317 	raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);  in call_rcu_tasks_generic()
331 		irq_work_queue(&rtpcp->rtp_irq_work);  in call_rcu_tasks_generic()
338 	struct rcu_tasks_percpu *rtpcp;  in rcu_barrier_tasks_generic_cb()  local
340 	rtpcp = container_of(rhp, struct rcu_tasks_percpu, barrier_q_head);  in rcu_barrier_tasks_generic_cb()
341 	rtp = rtpcp->rtpp;  in rcu_barrier_tasks_generic_cb()
352 	struct rcu_tasks_percpu *rtpcp;  in rcu_barrier_tasks_generic()  local
367 		rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu);  in rcu_barrier_tasks_generic()
368 		rtpcp->barrier_q_head.func = rcu_barrier_tasks_generic_cb;  in rcu_barrier_tasks_generic()
369 		raw_spin_lock_irqsave_rcu_node(rtpcp, flags);  in rcu_barrier_tasks_generic()
370 		if (rcu_segcblist_entrain(&rtpcp->cblist, &rtpcp->barrier_q_head))  in rcu_barrier_tasks_generic()
372 		raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);  in rcu_barrier_tasks_generic()
393 		struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu);  in rcu_tasks_need_gpcb()  local
396 		if (!rcu_segcblist_n_cbs(&rtpcp->cblist))  in rcu_tasks_need_gpcb()
398 		raw_spin_lock_irqsave_rcu_node(rtpcp, flags);  in rcu_tasks_need_gpcb()
400 		n = rcu_segcblist_n_cbs(&rtpcp->cblist);  in rcu_tasks_need_gpcb()
406 		rcu_segcblist_advance(&rtpcp->cblist, rcu_seq_current(&rtp->tasks_gp_seq));  in rcu_tasks_need_gpcb()
407 		(void)rcu_segcblist_accelerate(&rtpcp->cblist, rcu_seq_snap(&rtp->tasks_gp_seq));  in rcu_tasks_need_gpcb()
408 		if (rcu_segcblist_pend_cbs(&rtpcp->cblist))  in rcu_tasks_need_gpcb()
410 		if (!rcu_segcblist_empty(&rtpcp->cblist))  in rcu_tasks_need_gpcb()
412 		raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);  in rcu_tasks_need_gpcb()
440 			struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu);  in rcu_tasks_need_gpcb()  local
442 			WARN_ON_ONCE(rcu_segcblist_n_cbs(&rtpcp->cblist));  in rcu_tasks_need_gpcb()
451 static void rcu_tasks_invoke_cbs(struct rcu_tasks *rtp, struct rcu_tasks_percpu *rtpcp)  in rcu_tasks_invoke_cbs()  argument
461 	cpu = rtpcp->cpu;  in rcu_tasks_invoke_cbs()
473 	if (rcu_segcblist_empty(&rtpcp->cblist) || !cpu_possible(cpu))  in rcu_tasks_invoke_cbs()
475 	raw_spin_lock_irqsave_rcu_node(rtpcp, flags);  in rcu_tasks_invoke_cbs()
476 	rcu_segcblist_advance(&rtpcp->cblist, rcu_seq_current(&rtp->tasks_gp_seq));  in rcu_tasks_invoke_cbs()
477 	rcu_segcblist_extract_done_cbs(&rtpcp->cblist, &rcl);  in rcu_tasks_invoke_cbs()
478 	raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);  in rcu_tasks_invoke_cbs()
486 	raw_spin_lock_irqsave_rcu_node(rtpcp, flags);  in rcu_tasks_invoke_cbs()
487 	rcu_segcblist_add_len(&rtpcp->cblist, -len);  in rcu_tasks_invoke_cbs()
488 	(void)rcu_segcblist_accelerate(&rtpcp->cblist, rcu_seq_snap(&rtp->tasks_gp_seq));  in rcu_tasks_invoke_cbs()
489 	raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);  in rcu_tasks_invoke_cbs()
496 	struct rcu_tasks_percpu *rtpcp = container_of(wp, struct rcu_tasks_percpu, rtp_work);  in rcu_tasks_invoke_cbs_wq()  local
498 	rtp = rtpcp->rtpp;  in rcu_tasks_invoke_cbs_wq()
499 	rcu_tasks_invoke_cbs(rtp, rtpcp);  in rcu_tasks_invoke_cbs_wq()
624 		struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu);  in show_rcu_tasks_generic_gp_kthread()  local
626 		if (!data_race(rcu_segcblist_empty(&rtpcp->cblist))) {  in show_rcu_tasks_generic_gp_kthread()
1247 	struct rcu_tasks_percpu *rtpcp;  in rcu_read_unlock_trace_special()  local
1264 		rtpcp = per_cpu_ptr(rcu_tasks_trace.rtpcpu, t->trc_blkd_cpu);  in rcu_read_unlock_trace_special()
1265 		raw_spin_lock_irqsave_rcu_node(rtpcp, flags);  in rcu_read_unlock_trace_special()
1268 		raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);  in rcu_read_unlock_trace_special()
1278 	struct rcu_tasks_percpu *rtpcp;  in rcu_tasks_trace_qs_blkd()  local
1281 	rtpcp = this_cpu_ptr(rcu_tasks_trace.rtpcpu);  in rcu_tasks_trace_qs_blkd()
1282 	raw_spin_lock_rcu_node(rtpcp); // irqs already disabled  in rcu_tasks_trace_qs_blkd()
1284 	if (!rtpcp->rtp_blkd_tasks.next)  in rcu_tasks_trace_qs_blkd()
1285 		INIT_LIST_HEAD(&rtpcp->rtp_blkd_tasks);  in rcu_tasks_trace_qs_blkd()
1286 	list_add(&t->trc_blkd_node, &rtpcp->rtp_blkd_tasks);  in rcu_tasks_trace_qs_blkd()
1288 	raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);  in rcu_tasks_trace_qs_blkd()
1483 	struct rcu_tasks_percpu *rtpcp;  in rcu_tasks_trace_pregp_step()  local
1510 		rtpcp = per_cpu_ptr(rcu_tasks_trace.rtpcpu, cpu);  in rcu_tasks_trace_pregp_step()
1511 		raw_spin_lock_irqsave_rcu_node(rtpcp, flags);  in rcu_tasks_trace_pregp_step()
1512 		list_splice_init(&rtpcp->rtp_blkd_tasks, &blkd_tasks);  in rcu_tasks_trace_pregp_step()
1517 			list_add(&t->trc_blkd_node, &rtpcp->rtp_blkd_tasks);  in rcu_tasks_trace_pregp_step()
1518 			raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);  in rcu_tasks_trace_pregp_step()
1521 			raw_spin_lock_irqsave_rcu_node(rtpcp, flags);  in rcu_tasks_trace_pregp_step()
1523 		raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);  in rcu_tasks_trace_pregp_step()