171 	struct srcu_node *snp;  in init_srcu_struct_nodes()  local
187 	srcu_for_each_node_breadth_first(ssp, snp) {  in init_srcu_struct_nodes()
188 		spin_lock_init(&ACCESS_PRIVATE(snp, lock));  in init_srcu_struct_nodes()
189 		WARN_ON_ONCE(ARRAY_SIZE(snp->srcu_have_cbs) !=  in init_srcu_struct_nodes()
190 			     ARRAY_SIZE(snp->srcu_data_have_cbs));  in init_srcu_struct_nodes()
191 		for (i = 0; i < ARRAY_SIZE(snp->srcu_have_cbs); i++) {  in init_srcu_struct_nodes()
192 			snp->srcu_have_cbs[i] = SRCU_SNP_INIT_SEQ;  in init_srcu_struct_nodes()
193 			snp->srcu_data_have_cbs[i] = 0;  in init_srcu_struct_nodes()
195 		snp->srcu_gp_seq_needed_exp = SRCU_SNP_INIT_SEQ;  in init_srcu_struct_nodes()
196 		snp->grplo = -1;  in init_srcu_struct_nodes()
197 		snp->grphi = -1;  in init_srcu_struct_nodes()
198 		if (snp == &ssp->srcu_sup->node[0]) {  in init_srcu_struct_nodes()
200 			snp->srcu_parent = NULL;  in init_srcu_struct_nodes()
205 		if (snp == ssp->srcu_sup->level[level + 1])  in init_srcu_struct_nodes()
207 		snp->srcu_parent = ssp->srcu_sup->level[level - 1] +  in init_srcu_struct_nodes()
208 				   (snp - ssp->srcu_sup->level[level]) /  in init_srcu_struct_nodes()
221 		for (snp = sdp->mynode; snp != NULL; snp = snp->srcu_parent) {  in init_srcu_struct_nodes()
222 			if (snp->grplo < 0)  in init_srcu_struct_nodes()
223 				snp->grplo = cpu;  in init_srcu_struct_nodes()
224 			snp->grphi = cpu;  in init_srcu_struct_nodes()
830 static void srcu_schedule_cbs_snp(struct srcu_struct *ssp, struct srcu_node *snp,  in srcu_schedule_cbs_snp()  argument
835 	for (cpu = snp->grplo; cpu <= snp->grphi; cpu++) {  in srcu_schedule_cbs_snp()
836 		if (!(mask & (1 << (cpu - snp->grplo))))  in srcu_schedule_cbs_snp()
863 	struct srcu_node *snp;  in srcu_gp_end()  local
892 		idx = rcu_seq_ctr(gpseq) % ARRAY_SIZE(snp->srcu_have_cbs);  in srcu_gp_end()
893 		srcu_for_each_node_breadth_first(ssp, snp) {  in srcu_gp_end()
894 			spin_lock_irq_rcu_node(snp);  in srcu_gp_end()
896 			last_lvl = snp >= sup->level[rcu_num_lvls - 1];  in srcu_gp_end()
898 				cbs = ss_state < SRCU_SIZE_BIG || snp->srcu_have_cbs[idx] == gpseq;  in srcu_gp_end()
899 			snp->srcu_have_cbs[idx] = gpseq;  in srcu_gp_end()
900 			rcu_seq_set_state(&snp->srcu_have_cbs[idx], 1);  in srcu_gp_end()
901 			sgsne = snp->srcu_gp_seq_needed_exp;  in srcu_gp_end()
903 				WRITE_ONCE(snp->srcu_gp_seq_needed_exp, gpseq);  in srcu_gp_end()
907 				mask = snp->srcu_data_have_cbs[idx];  in srcu_gp_end()
908 			snp->srcu_data_have_cbs[idx] = 0;  in srcu_gp_end()
909 			spin_unlock_irq_rcu_node(snp);  in srcu_gp_end()
911 				srcu_schedule_cbs_snp(ssp, snp, mask, cbdelay);  in srcu_gp_end()
958 static void srcu_funnel_exp_start(struct srcu_struct *ssp, struct srcu_node *snp,  in srcu_funnel_exp_start()  argument
964 	if (snp)  in srcu_funnel_exp_start()
965 		for (; snp != NULL; snp = snp->srcu_parent) {  in srcu_funnel_exp_start()
966 			sgsne = READ_ONCE(snp->srcu_gp_seq_needed_exp);  in srcu_funnel_exp_start()
970 			spin_lock_irqsave_rcu_node(snp, flags);  in srcu_funnel_exp_start()
971 			sgsne = snp->srcu_gp_seq_needed_exp;  in srcu_funnel_exp_start()
973 				spin_unlock_irqrestore_rcu_node(snp, flags);  in srcu_funnel_exp_start()
976 			WRITE_ONCE(snp->srcu_gp_seq_needed_exp, s);  in srcu_funnel_exp_start()
977 			spin_unlock_irqrestore_rcu_node(snp, flags);  in srcu_funnel_exp_start()
1004 	struct srcu_node *snp;  in srcu_funnel_gp_start()  local
1017 		for (snp = snp_leaf; snp != NULL; snp = snp->srcu_parent) {  in srcu_funnel_gp_start()
1018 			if (WARN_ON_ONCE(rcu_seq_done(&sup->srcu_gp_seq, s)) && snp != snp_leaf)  in srcu_funnel_gp_start()
1020 			spin_lock_irqsave_rcu_node(snp, flags);  in srcu_funnel_gp_start()
1021 			snp_seq = snp->srcu_have_cbs[idx];  in srcu_funnel_gp_start()
1023 				if (snp == snp_leaf && snp_seq == s)  in srcu_funnel_gp_start()
1024 					snp->srcu_data_have_cbs[idx] |= sdp->grpmask;  in srcu_funnel_gp_start()
1025 				spin_unlock_irqrestore_rcu_node(snp, flags);  in srcu_funnel_gp_start()
1026 				if (snp == snp_leaf && snp_seq != s) {  in srcu_funnel_gp_start()
1031 					srcu_funnel_exp_start(ssp, snp, s);  in srcu_funnel_gp_start()
1034 			snp->srcu_have_cbs[idx] = s;  in srcu_funnel_gp_start()
1035 			if (snp == snp_leaf)  in srcu_funnel_gp_start()
1036 				snp->srcu_data_have_cbs[idx] |= sdp->grpmask;  in srcu_funnel_gp_start()
1037 			sgsne = snp->srcu_gp_seq_needed_exp;  in srcu_funnel_gp_start()
1039 				WRITE_ONCE(snp->srcu_gp_seq_needed_exp, s);  in srcu_funnel_gp_start()
1040 			spin_unlock_irqrestore_rcu_node(snp, flags);  in srcu_funnel_gp_start()