90 	struct srcu_node *snp;  in init_srcu_struct_nodes()  local
103 	srcu_for_each_node_breadth_first(ssp, snp) {  in init_srcu_struct_nodes()
104 		spin_lock_init(&ACCESS_PRIVATE(snp, lock));  in init_srcu_struct_nodes()
105 		WARN_ON_ONCE(ARRAY_SIZE(snp->srcu_have_cbs) !=  in init_srcu_struct_nodes()
106 			     ARRAY_SIZE(snp->srcu_data_have_cbs));  in init_srcu_struct_nodes()
107 		for (i = 0; i < ARRAY_SIZE(snp->srcu_have_cbs); i++) {  in init_srcu_struct_nodes()
108 			snp->srcu_have_cbs[i] = 0;  in init_srcu_struct_nodes()
109 			snp->srcu_data_have_cbs[i] = 0;  in init_srcu_struct_nodes()
111 		snp->srcu_gp_seq_needed_exp = 0;  in init_srcu_struct_nodes()
112 		snp->grplo = -1;  in init_srcu_struct_nodes()
113 		snp->grphi = -1;  in init_srcu_struct_nodes()
114 		if (snp == &ssp->node[0]) {  in init_srcu_struct_nodes()
116 			snp->srcu_parent = NULL;  in init_srcu_struct_nodes()
121 		if (snp == ssp->level[level + 1])  in init_srcu_struct_nodes()
123 		snp->srcu_parent = ssp->level[level - 1] +  in init_srcu_struct_nodes()
124 				   (snp - ssp->level[level]) /  in init_srcu_struct_nodes()
144 		for (snp = sdp->mynode; snp != NULL; snp = snp->srcu_parent) {  in init_srcu_struct_nodes()
145 			if (snp->grplo < 0)  in init_srcu_struct_nodes()
146 				snp->grplo = cpu;  in init_srcu_struct_nodes()
147 			snp->grphi = cpu;  in init_srcu_struct_nodes()
488 static void srcu_schedule_cbs_snp(struct srcu_struct *ssp, struct srcu_node *snp,  in srcu_schedule_cbs_snp()  argument
493 	for (cpu = snp->grplo; cpu <= snp->grphi; cpu++) {  in srcu_schedule_cbs_snp()
494 		if (!(mask & (1 << (cpu - snp->grplo))))  in srcu_schedule_cbs_snp()
520 	struct srcu_node *snp;  in srcu_gp_end()  local
540 	idx = rcu_seq_ctr(gpseq) % ARRAY_SIZE(snp->srcu_have_cbs);  in srcu_gp_end()
541 	srcu_for_each_node_breadth_first(ssp, snp) {  in srcu_gp_end()
542 		spin_lock_irq_rcu_node(snp);  in srcu_gp_end()
544 		last_lvl = snp >= ssp->level[rcu_num_lvls - 1];  in srcu_gp_end()
546 			cbs = snp->srcu_have_cbs[idx] == gpseq;  in srcu_gp_end()
547 		snp->srcu_have_cbs[idx] = gpseq;  in srcu_gp_end()
548 		rcu_seq_set_state(&snp->srcu_have_cbs[idx], 1);  in srcu_gp_end()
549 		if (ULONG_CMP_LT(snp->srcu_gp_seq_needed_exp, gpseq))  in srcu_gp_end()
550 			WRITE_ONCE(snp->srcu_gp_seq_needed_exp, gpseq);  in srcu_gp_end()
551 		mask = snp->srcu_data_have_cbs[idx];  in srcu_gp_end()
552 		snp->srcu_data_have_cbs[idx] = 0;  in srcu_gp_end()
553 		spin_unlock_irq_rcu_node(snp);  in srcu_gp_end()
555 			srcu_schedule_cbs_snp(ssp, snp, mask, cbdelay);  in srcu_gp_end()
559 			for (cpu = snp->grplo; cpu <= snp->grphi; cpu++) {  in srcu_gp_end()
595 static void srcu_funnel_exp_start(struct srcu_struct *ssp, struct srcu_node *snp,  in srcu_funnel_exp_start()  argument
600 	for (; snp != NULL; snp = snp->srcu_parent) {  in srcu_funnel_exp_start()
602 		    ULONG_CMP_GE(READ_ONCE(snp->srcu_gp_seq_needed_exp), s))  in srcu_funnel_exp_start()
604 		spin_lock_irqsave_rcu_node(snp, flags);  in srcu_funnel_exp_start()
605 		if (ULONG_CMP_GE(snp->srcu_gp_seq_needed_exp, s)) {  in srcu_funnel_exp_start()
606 			spin_unlock_irqrestore_rcu_node(snp, flags);  in srcu_funnel_exp_start()
609 		WRITE_ONCE(snp->srcu_gp_seq_needed_exp, s);  in srcu_funnel_exp_start()
610 		spin_unlock_irqrestore_rcu_node(snp, flags);  in srcu_funnel_exp_start()
633 	struct srcu_node *snp = sdp->mynode;  in srcu_funnel_gp_start()  local
637 	for (; snp != NULL; snp = snp->srcu_parent) {  in srcu_funnel_gp_start()
638 		if (rcu_seq_done(&ssp->srcu_gp_seq, s) && snp != sdp->mynode)  in srcu_funnel_gp_start()
640 		spin_lock_irqsave_rcu_node(snp, flags);  in srcu_funnel_gp_start()
641 		if (ULONG_CMP_GE(snp->srcu_have_cbs[idx], s)) {  in srcu_funnel_gp_start()
642 			snp_seq = snp->srcu_have_cbs[idx];  in srcu_funnel_gp_start()
643 			if (snp == sdp->mynode && snp_seq == s)  in srcu_funnel_gp_start()
644 				snp->srcu_data_have_cbs[idx] |= sdp->grpmask;  in srcu_funnel_gp_start()
645 			spin_unlock_irqrestore_rcu_node(snp, flags);  in srcu_funnel_gp_start()
646 			if (snp == sdp->mynode && snp_seq != s) {  in srcu_funnel_gp_start()
653 				srcu_funnel_exp_start(ssp, snp, s);  in srcu_funnel_gp_start()
656 		snp->srcu_have_cbs[idx] = s;  in srcu_funnel_gp_start()
657 		if (snp == sdp->mynode)  in srcu_funnel_gp_start()
658 			snp->srcu_data_have_cbs[idx] |= sdp->grpmask;  in srcu_funnel_gp_start()
659 		if (!do_norm && ULONG_CMP_LT(snp->srcu_gp_seq_needed_exp, s))  in srcu_funnel_gp_start()
660 			WRITE_ONCE(snp->srcu_gp_seq_needed_exp, s);  in srcu_funnel_gp_start()
661 		spin_unlock_irqrestore_rcu_node(snp, flags);  in srcu_funnel_gp_start()