98 	struct srcu_node *snp;  in init_srcu_struct_nodes()  local
108 	rcu_for_each_node_breadth_first(sp, snp) {  in init_srcu_struct_nodes()
109 		spin_lock_init(&ACCESS_PRIVATE(snp, lock));  in init_srcu_struct_nodes()
110 		WARN_ON_ONCE(ARRAY_SIZE(snp->srcu_have_cbs) !=  in init_srcu_struct_nodes()
111 			     ARRAY_SIZE(snp->srcu_data_have_cbs));  in init_srcu_struct_nodes()
112 		for (i = 0; i < ARRAY_SIZE(snp->srcu_have_cbs); i++) {  in init_srcu_struct_nodes()
113 			snp->srcu_have_cbs[i] = 0;  in init_srcu_struct_nodes()
114 			snp->srcu_data_have_cbs[i] = 0;  in init_srcu_struct_nodes()
116 		snp->srcu_gp_seq_needed_exp = 0;  in init_srcu_struct_nodes()
117 		snp->grplo = -1;  in init_srcu_struct_nodes()
118 		snp->grphi = -1;  in init_srcu_struct_nodes()
119 		if (snp == &sp->node[0]) {  in init_srcu_struct_nodes()
121 			snp->srcu_parent = NULL;  in init_srcu_struct_nodes()
126 		if (snp == sp->level[level + 1])  in init_srcu_struct_nodes()
128 		snp->srcu_parent = sp->level[level - 1] +  in init_srcu_struct_nodes()
129 				   (snp - sp->level[level]) /  in init_srcu_struct_nodes()
149 		for (snp = sdp->mynode; snp != NULL; snp = snp->srcu_parent) {  in init_srcu_struct_nodes()
150 			if (snp->grplo < 0)  in init_srcu_struct_nodes()
151 				snp->grplo = cpu;  in init_srcu_struct_nodes()
152 			snp->grphi = cpu;  in init_srcu_struct_nodes()
511 static void srcu_schedule_cbs_snp(struct srcu_struct *sp, struct srcu_node *snp,  in srcu_schedule_cbs_snp()  argument
516 	for (cpu = snp->grplo; cpu <= snp->grphi; cpu++) {  in srcu_schedule_cbs_snp()
517 		if (!(mask & (1 << (cpu - snp->grplo))))  in srcu_schedule_cbs_snp()
543 	struct srcu_node *snp;  in srcu_gp_end()  local
563 	idx = rcu_seq_ctr(gpseq) % ARRAY_SIZE(snp->srcu_have_cbs);  in srcu_gp_end()
564 	rcu_for_each_node_breadth_first(sp, snp) {  in srcu_gp_end()
565 		spin_lock_irq_rcu_node(snp);  in srcu_gp_end()
567 		last_lvl = snp >= sp->level[rcu_num_lvls - 1];  in srcu_gp_end()
569 			cbs = snp->srcu_have_cbs[idx] == gpseq;  in srcu_gp_end()
570 		snp->srcu_have_cbs[idx] = gpseq;  in srcu_gp_end()
571 		rcu_seq_set_state(&snp->srcu_have_cbs[idx], 1);  in srcu_gp_end()
572 		if (ULONG_CMP_LT(snp->srcu_gp_seq_needed_exp, gpseq))  in srcu_gp_end()
573 			snp->srcu_gp_seq_needed_exp = gpseq;  in srcu_gp_end()
574 		mask = snp->srcu_data_have_cbs[idx];  in srcu_gp_end()
575 		snp->srcu_data_have_cbs[idx] = 0;  in srcu_gp_end()
576 		spin_unlock_irq_rcu_node(snp);  in srcu_gp_end()
578 			srcu_schedule_cbs_snp(sp, snp, mask, cbdelay);  in srcu_gp_end()
582 			for (cpu = snp->grplo; cpu <= snp->grphi; cpu++) {  in srcu_gp_end()
618 static void srcu_funnel_exp_start(struct srcu_struct *sp, struct srcu_node *snp,  in srcu_funnel_exp_start()  argument
623 	for (; snp != NULL; snp = snp->srcu_parent) {  in srcu_funnel_exp_start()
625 		    ULONG_CMP_GE(READ_ONCE(snp->srcu_gp_seq_needed_exp), s))  in srcu_funnel_exp_start()
627 		spin_lock_irqsave_rcu_node(snp, flags);  in srcu_funnel_exp_start()
628 		if (ULONG_CMP_GE(snp->srcu_gp_seq_needed_exp, s)) {  in srcu_funnel_exp_start()
629 			spin_unlock_irqrestore_rcu_node(snp, flags);  in srcu_funnel_exp_start()
632 		WRITE_ONCE(snp->srcu_gp_seq_needed_exp, s);  in srcu_funnel_exp_start()
633 		spin_unlock_irqrestore_rcu_node(snp, flags);  in srcu_funnel_exp_start()
656 	struct srcu_node *snp = sdp->mynode;  in srcu_funnel_gp_start()  local
660 	for (; snp != NULL; snp = snp->srcu_parent) {  in srcu_funnel_gp_start()
661 		if (rcu_seq_done(&sp->srcu_gp_seq, s) && snp != sdp->mynode)  in srcu_funnel_gp_start()
663 		spin_lock_irqsave_rcu_node(snp, flags);  in srcu_funnel_gp_start()
664 		if (ULONG_CMP_GE(snp->srcu_have_cbs[idx], s)) {  in srcu_funnel_gp_start()
665 			snp_seq = snp->srcu_have_cbs[idx];  in srcu_funnel_gp_start()
666 			if (snp == sdp->mynode && snp_seq == s)  in srcu_funnel_gp_start()
667 				snp->srcu_data_have_cbs[idx] |= sdp->grpmask;  in srcu_funnel_gp_start()
668 			spin_unlock_irqrestore_rcu_node(snp, flags);  in srcu_funnel_gp_start()
669 			if (snp == sdp->mynode && snp_seq != s) {  in srcu_funnel_gp_start()
676 				srcu_funnel_exp_start(sp, snp, s);  in srcu_funnel_gp_start()
679 		snp->srcu_have_cbs[idx] = s;  in srcu_funnel_gp_start()
680 		if (snp == sdp->mynode)  in srcu_funnel_gp_start()
681 			snp->srcu_data_have_cbs[idx] |= sdp->grpmask;  in srcu_funnel_gp_start()
682 		if (!do_norm && ULONG_CMP_LT(snp->srcu_gp_seq_needed_exp, s))  in srcu_funnel_gp_start()
683 			snp->srcu_gp_seq_needed_exp = s;  in srcu_funnel_gp_start()
684 		spin_unlock_irqrestore_rcu_node(snp, flags);  in srcu_funnel_gp_start()