264 		struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu);  in cblist_init_generic()  local
266 		WARN_ON_ONCE(!rtpcp);  in cblist_init_generic()
268 			raw_spin_lock_init(&ACCESS_PRIVATE(rtpcp, lock));  in cblist_init_generic()
270 		if (rcu_segcblist_empty(&rtpcp->cblist))  in cblist_init_generic()
271 			rcu_segcblist_init(&rtpcp->cblist);  in cblist_init_generic()
273 		INIT_WORK(&rtpcp->rtp_work, rcu_tasks_invoke_cbs_wq);  in cblist_init_generic()
274 		rtpcp->cpu = cpu;  in cblist_init_generic()
275 		rtpcp->rtpp = rtp;  in cblist_init_generic()
276 		if (!rtpcp->rtp_blkd_tasks.next)  in cblist_init_generic()
277 			INIT_LIST_HEAD(&rtpcp->rtp_blkd_tasks);  in cblist_init_generic()
296 	struct rcu_tasks_percpu *rtpcp = from_timer(rtpcp, tlp, lazy_timer);  in call_rcu_tasks_generic_timer()  local
298 	rtp = rtpcp->rtpp;  in call_rcu_tasks_generic_timer()
299 	raw_spin_lock_irqsave_rcu_node(rtpcp, flags);  in call_rcu_tasks_generic_timer()
300 	if (!rcu_segcblist_empty(&rtpcp->cblist) && rtp->lazy_jiffies) {  in call_rcu_tasks_generic_timer()
301 		if (!rtpcp->urgent_gp)  in call_rcu_tasks_generic_timer()
302 			rtpcp->urgent_gp = 1;  in call_rcu_tasks_generic_timer()
304 		mod_timer(&rtpcp->lazy_timer, rcu_tasks_lazy_time(rtp));  in call_rcu_tasks_generic_timer()
306 	raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);  in call_rcu_tasks_generic_timer()
315 	struct rcu_tasks_percpu *rtpcp = container_of(iwp, struct rcu_tasks_percpu, rtp_irq_work);  in call_rcu_tasks_iw_wakeup()  local
317 	rtp = rtpcp->rtpp;  in call_rcu_tasks_iw_wakeup()
332 	struct rcu_tasks_percpu *rtpcp;  in call_rcu_tasks_generic()  local
340 	rtpcp = per_cpu_ptr(rtp->rtpcpu, chosen_cpu);  in call_rcu_tasks_generic()
341 	if (!raw_spin_trylock_rcu_node(rtpcp)) { // irqs already disabled.  in call_rcu_tasks_generic()
342 		raw_spin_lock_rcu_node(rtpcp); // irqs already disabled.  in call_rcu_tasks_generic()
344 		if (rtpcp->rtp_jiffies != j) {  in call_rcu_tasks_generic()
345 			rtpcp->rtp_jiffies = j;  in call_rcu_tasks_generic()
346 			rtpcp->rtp_n_lock_retries = 0;  in call_rcu_tasks_generic()
348 		if (rcu_task_cb_adjust && ++rtpcp->rtp_n_lock_retries > rcu_task_contend_lim &&  in call_rcu_tasks_generic()
353 	if (WARN_ON_ONCE(!rcu_segcblist_is_enabled(&rtpcp->cblist)))  in call_rcu_tasks_generic()
354 		rcu_segcblist_init(&rtpcp->cblist);  in call_rcu_tasks_generic()
356 		   (rcu_segcblist_n_cbs(&rtpcp->cblist) == rcu_task_lazy_lim);  in call_rcu_tasks_generic()
357 	if (havekthread && !needwake && !timer_pending(&rtpcp->lazy_timer)) {  in call_rcu_tasks_generic()
359 			mod_timer(&rtpcp->lazy_timer, rcu_tasks_lazy_time(rtp));  in call_rcu_tasks_generic()
361 			needwake = rcu_segcblist_empty(&rtpcp->cblist);  in call_rcu_tasks_generic()
364 		rtpcp->urgent_gp = 3;  in call_rcu_tasks_generic()
365 	rcu_segcblist_enqueue(&rtpcp->cblist, rhp);  in call_rcu_tasks_generic()
366 	raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);  in call_rcu_tasks_generic()
380 		irq_work_queue(&rtpcp->rtp_irq_work);  in call_rcu_tasks_generic()
387 	struct rcu_tasks_percpu *rtpcp;  in rcu_barrier_tasks_generic_cb()  local
389 	rtpcp = container_of(rhp, struct rcu_tasks_percpu, barrier_q_head);  in rcu_barrier_tasks_generic_cb()
390 	rtp = rtpcp->rtpp;  in rcu_barrier_tasks_generic_cb()
401 	struct rcu_tasks_percpu *rtpcp;  in rcu_barrier_tasks_generic()  local
416 		rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu);  in rcu_barrier_tasks_generic()
417 		rtpcp->barrier_q_head.func = rcu_barrier_tasks_generic_cb;  in rcu_barrier_tasks_generic()
418 		raw_spin_lock_irqsave_rcu_node(rtpcp, flags);  in rcu_barrier_tasks_generic()
419 		if (rcu_segcblist_entrain(&rtpcp->cblist, &rtpcp->barrier_q_head))  in rcu_barrier_tasks_generic()
421 		raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);  in rcu_barrier_tasks_generic()
443 		struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu);  in rcu_tasks_need_gpcb()  local
446 		if (!rcu_segcblist_n_cbs(&rtpcp->cblist))  in rcu_tasks_need_gpcb()
448 		raw_spin_lock_irqsave_rcu_node(rtpcp, flags);  in rcu_tasks_need_gpcb()
450 		n = rcu_segcblist_n_cbs(&rtpcp->cblist);  in rcu_tasks_need_gpcb()
456 		rcu_segcblist_advance(&rtpcp->cblist, rcu_seq_current(&rtp->tasks_gp_seq));  in rcu_tasks_need_gpcb()
457 		(void)rcu_segcblist_accelerate(&rtpcp->cblist, rcu_seq_snap(&rtp->tasks_gp_seq));  in rcu_tasks_need_gpcb()
458 		if (rtpcp->urgent_gp > 0 && rcu_segcblist_pend_cbs(&rtpcp->cblist)) {  in rcu_tasks_need_gpcb()
460 				rtpcp->urgent_gp--;  in rcu_tasks_need_gpcb()
462 		} else if (rcu_segcblist_empty(&rtpcp->cblist)) {  in rcu_tasks_need_gpcb()
463 			rtpcp->urgent_gp = 0;  in rcu_tasks_need_gpcb()
465 		if (rcu_segcblist_ready_cbs(&rtpcp->cblist))  in rcu_tasks_need_gpcb()
467 		raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);  in rcu_tasks_need_gpcb()
496 				struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu);  in rcu_tasks_need_gpcb()  local
498 				WARN_ON_ONCE(rcu_segcblist_n_cbs(&rtpcp->cblist));  in rcu_tasks_need_gpcb()
508 static void rcu_tasks_invoke_cbs(struct rcu_tasks *rtp, struct rcu_tasks_percpu *rtpcp)  in rcu_tasks_invoke_cbs()  argument
519 	cpu = rtpcp->cpu;  in rcu_tasks_invoke_cbs()
533 	if (rcu_segcblist_empty(&rtpcp->cblist) || !cpu_possible(cpu))  in rcu_tasks_invoke_cbs()
535 	raw_spin_lock_irqsave_rcu_node(rtpcp, flags);  in rcu_tasks_invoke_cbs()
536 	rcu_segcblist_advance(&rtpcp->cblist, rcu_seq_current(&rtp->tasks_gp_seq));  in rcu_tasks_invoke_cbs()
537 	rcu_segcblist_extract_done_cbs(&rtpcp->cblist, &rcl);  in rcu_tasks_invoke_cbs()
538 	raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);  in rcu_tasks_invoke_cbs()
546 	raw_spin_lock_irqsave_rcu_node(rtpcp, flags);  in rcu_tasks_invoke_cbs()
547 	rcu_segcblist_add_len(&rtpcp->cblist, -len);  in rcu_tasks_invoke_cbs()
548 	(void)rcu_segcblist_accelerate(&rtpcp->cblist, rcu_seq_snap(&rtp->tasks_gp_seq));  in rcu_tasks_invoke_cbs()
549 	raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);  in rcu_tasks_invoke_cbs()
556 	struct rcu_tasks_percpu *rtpcp = container_of(wp, struct rcu_tasks_percpu, rtp_work);  in rcu_tasks_invoke_cbs_wq()  local
558 	rtp = rtpcp->rtpp;  in rcu_tasks_invoke_cbs_wq()
559 	rcu_tasks_invoke_cbs(rtp, rtpcp);  in rcu_tasks_invoke_cbs_wq()
603 		struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu);  in rcu_tasks_kthread()  local
605 		timer_setup(&rtpcp->lazy_timer, call_rcu_tasks_generic_timer, 0);  in rcu_tasks_kthread()
606 		rtpcp->urgent_gp = 1;  in rcu_tasks_kthread()
697 		struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu);  in show_rcu_tasks_generic_gp_kthread()  local
699 		if (!data_race(rcu_segcblist_empty(&rtpcp->cblist)))  in show_rcu_tasks_generic_gp_kthread()
701 		if (data_race(rtpcp->urgent_gp))  in show_rcu_tasks_generic_gp_kthread()
703 		if (!data_race(rcu_segcblist_empty(&rtpcp->cblist)) && data_race(rtpcp->urgent_gp))  in show_rcu_tasks_generic_gp_kthread()
1400 	struct rcu_tasks_percpu *rtpcp;  in rcu_read_unlock_trace_special()  local
1417 		rtpcp = per_cpu_ptr(rcu_tasks_trace.rtpcpu, t->trc_blkd_cpu);  in rcu_read_unlock_trace_special()
1418 		raw_spin_lock_irqsave_rcu_node(rtpcp, flags);  in rcu_read_unlock_trace_special()
1421 		raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);  in rcu_read_unlock_trace_special()
1431 	struct rcu_tasks_percpu *rtpcp;  in rcu_tasks_trace_qs_blkd()  local
1434 	rtpcp = this_cpu_ptr(rcu_tasks_trace.rtpcpu);  in rcu_tasks_trace_qs_blkd()
1435 	raw_spin_lock_rcu_node(rtpcp); // irqs already disabled  in rcu_tasks_trace_qs_blkd()
1437 	if (!rtpcp->rtp_blkd_tasks.next)  in rcu_tasks_trace_qs_blkd()
1438 		INIT_LIST_HEAD(&rtpcp->rtp_blkd_tasks);  in rcu_tasks_trace_qs_blkd()
1439 	list_add(&t->trc_blkd_node, &rtpcp->rtp_blkd_tasks);  in rcu_tasks_trace_qs_blkd()
1441 	raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);  in rcu_tasks_trace_qs_blkd()
1636 	struct rcu_tasks_percpu *rtpcp;  in rcu_tasks_trace_pregp_step()  local
1663 		rtpcp = per_cpu_ptr(rcu_tasks_trace.rtpcpu, cpu);  in rcu_tasks_trace_pregp_step()
1664 		raw_spin_lock_irqsave_rcu_node(rtpcp, flags);  in rcu_tasks_trace_pregp_step()
1665 		list_splice_init(&rtpcp->rtp_blkd_tasks, &blkd_tasks);  in rcu_tasks_trace_pregp_step()
1670 			list_add(&t->trc_blkd_node, &rtpcp->rtp_blkd_tasks);  in rcu_tasks_trace_pregp_step()
1671 			raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);  in rcu_tasks_trace_pregp_step()
1674 			raw_spin_lock_irqsave_rcu_node(rtpcp, flags);  in rcu_tasks_trace_pregp_step()
1676 		raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);  in rcu_tasks_trace_pregp_step()