89 	struct srcu_data *sdp;  in init_srcu_struct_nodes()  local
129 	WARN_ON_ONCE(ARRAY_SIZE(sdp->srcu_lock_count) !=  in init_srcu_struct_nodes()
130 		     ARRAY_SIZE(sdp->srcu_unlock_count));  in init_srcu_struct_nodes()
134 		sdp = per_cpu_ptr(ssp->sda, cpu);  in init_srcu_struct_nodes()
135 		spin_lock_init(&ACCESS_PRIVATE(sdp, lock));  in init_srcu_struct_nodes()
136 		rcu_segcblist_init(&sdp->srcu_cblist);  in init_srcu_struct_nodes()
137 		sdp->srcu_cblist_invoking = false;  in init_srcu_struct_nodes()
138 		sdp->srcu_gp_seq_needed = ssp->srcu_gp_seq;  in init_srcu_struct_nodes()
139 		sdp->srcu_gp_seq_needed_exp = ssp->srcu_gp_seq;  in init_srcu_struct_nodes()
140 		sdp->mynode = &snp_first[cpu / levelspread[level]];  in init_srcu_struct_nodes()
141 		for (snp = sdp->mynode; snp != NULL; snp = snp->srcu_parent) {  in init_srcu_struct_nodes()
146 		sdp->cpu = cpu;  in init_srcu_struct_nodes()
147 		INIT_WORK(&sdp->work, srcu_invoke_callbacks);  in init_srcu_struct_nodes()
148 		timer_setup(&sdp->delay_work, srcu_delay_timer, 0);  in init_srcu_struct_nodes()
149 		sdp->ssp = ssp;  in init_srcu_struct_nodes()
150 		sdp->grpmask = 1 << (cpu - sdp->mynode->grplo);  in init_srcu_struct_nodes()
155 		for (i = 0; i < ARRAY_SIZE(sdp->srcu_lock_count); i++) {  in init_srcu_struct_nodes()
156 			sdp->srcu_lock_count[i] = 0;  in init_srcu_struct_nodes()
157 			sdp->srcu_unlock_count[i] = 0;  in init_srcu_struct_nodes()
380 		struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu);  in cleanup_srcu_struct()  local
382 		del_timer_sync(&sdp->delay_work);  in cleanup_srcu_struct()
383 		flush_work(&sdp->work);  in cleanup_srcu_struct()
384 		if (WARN_ON(rcu_segcblist_n_cbs(&sdp->srcu_cblist)))  in cleanup_srcu_struct()
440 	struct srcu_data *sdp = this_cpu_ptr(ssp->sda);  in srcu_gp_start()  local
445 	spin_lock_rcu_node(sdp);  /* Interrupts already disabled. */  in srcu_gp_start()
446 	rcu_segcblist_advance(&sdp->srcu_cblist,  in srcu_gp_start()
448 	(void)rcu_segcblist_accelerate(&sdp->srcu_cblist,  in srcu_gp_start()
450 	spin_unlock_rcu_node(sdp);  /* Interrupts remain disabled. */  in srcu_gp_start()
460 	struct srcu_data *sdp = container_of(t, struct srcu_data, delay_work);  in srcu_delay_timer()  local
462 	queue_work_on(sdp->cpu, rcu_gp_wq, &sdp->work);  in srcu_delay_timer()
465 static void srcu_queue_delayed_work_on(struct srcu_data *sdp,  in srcu_queue_delayed_work_on()  argument
469 		queue_work_on(sdp->cpu, rcu_gp_wq, &sdp->work);  in srcu_queue_delayed_work_on()
473 	timer_reduce(&sdp->delay_work, jiffies + delay);  in srcu_queue_delayed_work_on()
480 static void srcu_schedule_cbs_sdp(struct srcu_data *sdp, unsigned long delay)  in srcu_schedule_cbs_sdp()  argument
482 	srcu_queue_delayed_work_on(sdp, delay);  in srcu_schedule_cbs_sdp()
522 	struct srcu_data *sdp;  in srcu_gp_end()  local
563 				sdp = per_cpu_ptr(ssp->sda, cpu);  in srcu_gp_end()
564 				spin_lock_irqsave_rcu_node(sdp, flags);  in srcu_gp_end()
566 						 sdp->srcu_gp_seq_needed + 100))  in srcu_gp_end()
567 					sdp->srcu_gp_seq_needed = gpseq;  in srcu_gp_end()
569 						 sdp->srcu_gp_seq_needed_exp + 100))  in srcu_gp_end()
570 					sdp->srcu_gp_seq_needed_exp = gpseq;  in srcu_gp_end()
571 				spin_unlock_irqrestore_rcu_node(sdp, flags);  in srcu_gp_end()
631 static void srcu_funnel_gp_start(struct srcu_struct *ssp, struct srcu_data *sdp,  in srcu_funnel_gp_start()  argument
635 	int idx = rcu_seq_ctr(s) % ARRAY_SIZE(sdp->mynode->srcu_have_cbs);  in srcu_funnel_gp_start()
636 	struct srcu_node *snp = sdp->mynode;  in srcu_funnel_gp_start()
641 		if (rcu_seq_done(&ssp->srcu_gp_seq, s) && snp != sdp->mynode)  in srcu_funnel_gp_start()
646 			if (snp == sdp->mynode && snp_seq == s)  in srcu_funnel_gp_start()
647 				snp->srcu_data_have_cbs[idx] |= sdp->grpmask;  in srcu_funnel_gp_start()
649 			if (snp == sdp->mynode && snp_seq != s) {  in srcu_funnel_gp_start()
650 				srcu_schedule_cbs_sdp(sdp, do_norm  in srcu_funnel_gp_start()
660 		if (snp == sdp->mynode)  in srcu_funnel_gp_start()
661 			snp->srcu_data_have_cbs[idx] |= sdp->grpmask;  in srcu_funnel_gp_start()
763 	struct srcu_data *sdp;  in srcu_might_be_idle()  local
769 	sdp = raw_cpu_ptr(ssp->sda);  in srcu_might_be_idle()
770 	spin_lock_irqsave_rcu_node(sdp, flags);  in srcu_might_be_idle()
771 	if (rcu_segcblist_pend_cbs(&sdp->srcu_cblist)) {  in srcu_might_be_idle()
772 		spin_unlock_irqrestore_rcu_node(sdp, flags);  in srcu_might_be_idle()
775 	spin_unlock_irqrestore_rcu_node(sdp, flags);  in srcu_might_be_idle()
844 	struct srcu_data *sdp;  in __call_srcu()  local
855 	sdp = raw_cpu_ptr(ssp->sda);  in __call_srcu()
856 	spin_lock_irqsave_rcu_node(sdp, flags);  in __call_srcu()
857 	rcu_segcblist_enqueue(&sdp->srcu_cblist, rhp);  in __call_srcu()
858 	rcu_segcblist_advance(&sdp->srcu_cblist,  in __call_srcu()
861 	(void)rcu_segcblist_accelerate(&sdp->srcu_cblist, s);  in __call_srcu()
862 	if (ULONG_CMP_LT(sdp->srcu_gp_seq_needed, s)) {  in __call_srcu()
863 		sdp->srcu_gp_seq_needed = s;  in __call_srcu()
866 	if (!do_norm && ULONG_CMP_LT(sdp->srcu_gp_seq_needed_exp, s)) {  in __call_srcu()
867 		sdp->srcu_gp_seq_needed_exp = s;  in __call_srcu()
870 	spin_unlock_irqrestore_rcu_node(sdp, flags);  in __call_srcu()
872 		srcu_funnel_gp_start(ssp, sdp, s, do_norm);  in __call_srcu()
874 		srcu_funnel_exp_start(ssp, sdp->mynode, s);  in __call_srcu()
1009 	struct srcu_data *sdp;  in srcu_barrier_cb()  local
1012 	sdp = container_of(rhp, struct srcu_data, srcu_barrier_head);  in srcu_barrier_cb()
1013 	ssp = sdp->ssp;  in srcu_barrier_cb()
1025 	struct srcu_data *sdp;  in srcu_barrier()  local
1050 		sdp = per_cpu_ptr(ssp->sda, cpu);  in srcu_barrier()
1051 		spin_lock_irq_rcu_node(sdp);  in srcu_barrier()
1053 		sdp->srcu_barrier_head.func = srcu_barrier_cb;  in srcu_barrier()
1054 		debug_rcu_head_queue(&sdp->srcu_barrier_head);  in srcu_barrier()
1055 		if (!rcu_segcblist_entrain(&sdp->srcu_cblist,  in srcu_barrier()
1056 					   &sdp->srcu_barrier_head)) {  in srcu_barrier()
1057 			debug_rcu_head_unqueue(&sdp->srcu_barrier_head);  in srcu_barrier()
1060 		spin_unlock_irq_rcu_node(sdp);  in srcu_barrier()
1164 	struct srcu_data *sdp;  in srcu_invoke_callbacks()  local
1167 	sdp = container_of(work, struct srcu_data, work);  in srcu_invoke_callbacks()
1169 	ssp = sdp->ssp;  in srcu_invoke_callbacks()
1171 	spin_lock_irq_rcu_node(sdp);  in srcu_invoke_callbacks()
1172 	rcu_segcblist_advance(&sdp->srcu_cblist,  in srcu_invoke_callbacks()
1174 	if (sdp->srcu_cblist_invoking ||  in srcu_invoke_callbacks()
1175 	    !rcu_segcblist_ready_cbs(&sdp->srcu_cblist)) {  in srcu_invoke_callbacks()
1176 		spin_unlock_irq_rcu_node(sdp);  in srcu_invoke_callbacks()
1181 	sdp->srcu_cblist_invoking = true;  in srcu_invoke_callbacks()
1182 	rcu_segcblist_extract_done_cbs(&sdp->srcu_cblist, &ready_cbs);  in srcu_invoke_callbacks()
1183 	spin_unlock_irq_rcu_node(sdp);  in srcu_invoke_callbacks()
1196 	spin_lock_irq_rcu_node(sdp);  in srcu_invoke_callbacks()
1197 	rcu_segcblist_insert_count(&sdp->srcu_cblist, &ready_cbs);  in srcu_invoke_callbacks()
1198 	(void)rcu_segcblist_accelerate(&sdp->srcu_cblist,  in srcu_invoke_callbacks()
1200 	sdp->srcu_cblist_invoking = false;  in srcu_invoke_callbacks()
1201 	more = rcu_segcblist_ready_cbs(&sdp->srcu_cblist);  in srcu_invoke_callbacks()
1202 	spin_unlock_irq_rcu_node(sdp);  in srcu_invoke_callbacks()
1204 		srcu_schedule_cbs_sdp(sdp, 0);  in srcu_invoke_callbacks()
1268 		struct srcu_data *sdp;  in srcu_torture_stats_print()  local
1270 		sdp = per_cpu_ptr(ssp->sda, cpu);  in srcu_torture_stats_print()
1271 		u0 = data_race(sdp->srcu_unlock_count[!idx]);  in srcu_torture_stats_print()
1272 		u1 = data_race(sdp->srcu_unlock_count[idx]);  in srcu_torture_stats_print()
1280 		l0 = data_race(sdp->srcu_lock_count[!idx]);  in srcu_torture_stats_print()
1281 		l1 = data_race(sdp->srcu_lock_count[idx]);  in srcu_torture_stats_print()
1287 			"C."[rcu_segcblist_empty(&sdp->srcu_cblist)]);  in srcu_torture_stats_print()