97 	struct srcu_data *sdp;  in init_srcu_struct_nodes()  local
137 	WARN_ON_ONCE(ARRAY_SIZE(sdp->srcu_lock_count) !=  in init_srcu_struct_nodes()
138 		     ARRAY_SIZE(sdp->srcu_unlock_count));  in init_srcu_struct_nodes()
142 		sdp = per_cpu_ptr(sp->sda, cpu);  in init_srcu_struct_nodes()
143 		spin_lock_init(&ACCESS_PRIVATE(sdp, lock));  in init_srcu_struct_nodes()
144 		rcu_segcblist_init(&sdp->srcu_cblist);  in init_srcu_struct_nodes()
145 		sdp->srcu_cblist_invoking = false;  in init_srcu_struct_nodes()
146 		sdp->srcu_gp_seq_needed = sp->srcu_gp_seq;  in init_srcu_struct_nodes()
147 		sdp->srcu_gp_seq_needed_exp = sp->srcu_gp_seq;  in init_srcu_struct_nodes()
148 		sdp->mynode = &snp_first[cpu / levelspread[level]];  in init_srcu_struct_nodes()
149 		for (snp = sdp->mynode; snp != NULL; snp = snp->srcu_parent) {  in init_srcu_struct_nodes()
154 		sdp->cpu = cpu;  in init_srcu_struct_nodes()
155 		INIT_DELAYED_WORK(&sdp->work, srcu_invoke_callbacks);  in init_srcu_struct_nodes()
156 		sdp->sp = sp;  in init_srcu_struct_nodes()
157 		sdp->grpmask = 1 << (cpu - sdp->mynode->grplo);  in init_srcu_struct_nodes()
162 		for (i = 0; i < ARRAY_SIZE(sdp->srcu_lock_count); i++) {  in init_srcu_struct_nodes()
163 			sdp->srcu_lock_count[i] = 0;  in init_srcu_struct_nodes()
164 			sdp->srcu_unlock_count[i] = 0;  in init_srcu_struct_nodes()
446 	struct srcu_data *sdp = this_cpu_ptr(sp->sda);  in srcu_gp_start()  local
451 	rcu_segcblist_advance(&sdp->srcu_cblist,  in srcu_gp_start()
453 	(void)rcu_segcblist_accelerate(&sdp->srcu_cblist,  in srcu_gp_start()
500 static void srcu_schedule_cbs_sdp(struct srcu_data *sdp, unsigned long delay)  in srcu_schedule_cbs_sdp()  argument
502 	srcu_queue_delayed_work_on(sdp->cpu, rcu_gp_wq, &sdp->work, delay);  in srcu_schedule_cbs_sdp()
542 	struct srcu_data *sdp;  in srcu_gp_end()  local
583 				sdp = per_cpu_ptr(sp->sda, cpu);  in srcu_gp_end()
584 				spin_lock_irqsave_rcu_node(sdp, flags);  in srcu_gp_end()
586 						 sdp->srcu_gp_seq_needed + 100))  in srcu_gp_end()
587 					sdp->srcu_gp_seq_needed = gpseq;  in srcu_gp_end()
589 						 sdp->srcu_gp_seq_needed_exp + 100))  in srcu_gp_end()
590 					sdp->srcu_gp_seq_needed_exp = gpseq;  in srcu_gp_end()
591 				spin_unlock_irqrestore_rcu_node(sdp, flags);  in srcu_gp_end()
651 static void srcu_funnel_gp_start(struct srcu_struct *sp, struct srcu_data *sdp,  in srcu_funnel_gp_start()  argument
655 	int idx = rcu_seq_ctr(s) % ARRAY_SIZE(sdp->mynode->srcu_have_cbs);  in srcu_funnel_gp_start()
656 	struct srcu_node *snp = sdp->mynode;  in srcu_funnel_gp_start()
661 		if (rcu_seq_done(&sp->srcu_gp_seq, s) && snp != sdp->mynode)  in srcu_funnel_gp_start()
666 			if (snp == sdp->mynode && snp_seq == s)  in srcu_funnel_gp_start()
667 				snp->srcu_data_have_cbs[idx] |= sdp->grpmask;  in srcu_funnel_gp_start()
669 			if (snp == sdp->mynode && snp_seq != s) {  in srcu_funnel_gp_start()
670 				srcu_schedule_cbs_sdp(sdp, do_norm  in srcu_funnel_gp_start()
680 		if (snp == sdp->mynode)  in srcu_funnel_gp_start()
681 			snp->srcu_data_have_cbs[idx] |= sdp->grpmask;  in srcu_funnel_gp_start()
779 	struct srcu_data *sdp;  in srcu_might_be_idle()  local
784 	sdp = this_cpu_ptr(sp->sda);  in srcu_might_be_idle()
785 	if (rcu_segcblist_pend_cbs(&sdp->srcu_cblist)) {  in srcu_might_be_idle()
857 	struct srcu_data *sdp;  in __call_srcu()  local
868 	sdp = this_cpu_ptr(sp->sda);  in __call_srcu()
869 	spin_lock_rcu_node(sdp);  in __call_srcu()
870 	rcu_segcblist_enqueue(&sdp->srcu_cblist, rhp, false);  in __call_srcu()
871 	rcu_segcblist_advance(&sdp->srcu_cblist,  in __call_srcu()
874 	(void)rcu_segcblist_accelerate(&sdp->srcu_cblist, s);  in __call_srcu()
875 	if (ULONG_CMP_LT(sdp->srcu_gp_seq_needed, s)) {  in __call_srcu()
876 		sdp->srcu_gp_seq_needed = s;  in __call_srcu()
879 	if (!do_norm && ULONG_CMP_LT(sdp->srcu_gp_seq_needed_exp, s)) {  in __call_srcu()
880 		sdp->srcu_gp_seq_needed_exp = s;  in __call_srcu()
883 	spin_unlock_irqrestore_rcu_node(sdp, flags);  in __call_srcu()
885 		srcu_funnel_gp_start(sp, sdp, s, do_norm);  in __call_srcu()
887 		srcu_funnel_exp_start(sp, sdp->mynode, s);  in __call_srcu()
1021 	struct srcu_data *sdp;  in srcu_barrier_cb()  local
1024 	sdp = container_of(rhp, struct srcu_data, srcu_barrier_head);  in srcu_barrier_cb()
1025 	sp = sdp->sp;  in srcu_barrier_cb()
1037 	struct srcu_data *sdp;  in srcu_barrier()  local
1062 		sdp = per_cpu_ptr(sp->sda, cpu);  in srcu_barrier()
1063 		spin_lock_irq_rcu_node(sdp);  in srcu_barrier()
1065 		sdp->srcu_barrier_head.func = srcu_barrier_cb;  in srcu_barrier()
1066 		debug_rcu_head_queue(&sdp->srcu_barrier_head);  in srcu_barrier()
1067 		if (!rcu_segcblist_entrain(&sdp->srcu_cblist,  in srcu_barrier()
1068 					   &sdp->srcu_barrier_head, 0)) {  in srcu_barrier()
1069 			debug_rcu_head_unqueue(&sdp->srcu_barrier_head);  in srcu_barrier()
1072 		spin_unlock_irq_rcu_node(sdp);  in srcu_barrier()
1174 	struct srcu_data *sdp;  in srcu_invoke_callbacks()  local
1177 	sdp = container_of(work, struct srcu_data, work.work);  in srcu_invoke_callbacks()
1178 	sp = sdp->sp;  in srcu_invoke_callbacks()
1180 	spin_lock_irq_rcu_node(sdp);  in srcu_invoke_callbacks()
1181 	rcu_segcblist_advance(&sdp->srcu_cblist,  in srcu_invoke_callbacks()
1183 	if (sdp->srcu_cblist_invoking ||  in srcu_invoke_callbacks()
1184 	    !rcu_segcblist_ready_cbs(&sdp->srcu_cblist)) {  in srcu_invoke_callbacks()
1185 		spin_unlock_irq_rcu_node(sdp);  in srcu_invoke_callbacks()
1190 	sdp->srcu_cblist_invoking = true;  in srcu_invoke_callbacks()
1191 	rcu_segcblist_extract_done_cbs(&sdp->srcu_cblist, &ready_cbs);  in srcu_invoke_callbacks()
1192 	spin_unlock_irq_rcu_node(sdp);  in srcu_invoke_callbacks()
1205 	spin_lock_irq_rcu_node(sdp);  in srcu_invoke_callbacks()
1206 	rcu_segcblist_insert_count(&sdp->srcu_cblist, &ready_cbs);  in srcu_invoke_callbacks()
1207 	(void)rcu_segcblist_accelerate(&sdp->srcu_cblist,  in srcu_invoke_callbacks()
1209 	sdp->srcu_cblist_invoking = false;  in srcu_invoke_callbacks()
1210 	more = rcu_segcblist_ready_cbs(&sdp->srcu_cblist);  in srcu_invoke_callbacks()
1211 	spin_unlock_irq_rcu_node(sdp);  in srcu_invoke_callbacks()
1213 		srcu_schedule_cbs_sdp(sdp, 0);  in srcu_invoke_callbacks()
1277 		struct srcu_data *sdp;  in srcu_torture_stats_print()  local
1279 		sdp = per_cpu_ptr(sp->sda, cpu);  in srcu_torture_stats_print()
1280 		u0 = sdp->srcu_unlock_count[!idx];  in srcu_torture_stats_print()
1281 		u1 = sdp->srcu_unlock_count[idx];  in srcu_torture_stats_print()
1289 		l0 = sdp->srcu_lock_count[!idx];  in srcu_torture_stats_print()
1290 		l1 = sdp->srcu_lock_count[idx];  in srcu_torture_stats_print()
1295 			cpu, c0, c1, rcu_segcblist_head(&sdp->srcu_cblist));  in srcu_torture_stats_print()