90 	struct srcu_node *snp;  in init_srcu_struct_nodes()  local
100 	srcu_for_each_node_breadth_first(ssp, snp) {  in init_srcu_struct_nodes()
101 		spin_lock_init(&ACCESS_PRIVATE(snp, lock));  in init_srcu_struct_nodes()
102 		WARN_ON_ONCE(ARRAY_SIZE(snp->srcu_have_cbs) !=  in init_srcu_struct_nodes()
103 			     ARRAY_SIZE(snp->srcu_data_have_cbs));  in init_srcu_struct_nodes()
104 		for (i = 0; i < ARRAY_SIZE(snp->srcu_have_cbs); i++) {  in init_srcu_struct_nodes()
105 			snp->srcu_have_cbs[i] = 0;  in init_srcu_struct_nodes()
106 			snp->srcu_data_have_cbs[i] = 0;  in init_srcu_struct_nodes()
108 		snp->srcu_gp_seq_needed_exp = 0;  in init_srcu_struct_nodes()
109 		snp->grplo = -1;  in init_srcu_struct_nodes()
110 		snp->grphi = -1;  in init_srcu_struct_nodes()
111 		if (snp == &ssp->node[0]) {  in init_srcu_struct_nodes()
113 			snp->srcu_parent = NULL;  in init_srcu_struct_nodes()
118 		if (snp == ssp->level[level + 1])  in init_srcu_struct_nodes()
120 		snp->srcu_parent = ssp->level[level - 1] +  in init_srcu_struct_nodes()
121 				   (snp - ssp->level[level]) /  in init_srcu_struct_nodes()
141 		for (snp = sdp->mynode; snp != NULL; snp = snp->srcu_parent) {  in init_srcu_struct_nodes()
142 			if (snp->grplo < 0)  in init_srcu_struct_nodes()
143 				snp->grplo = cpu;  in init_srcu_struct_nodes()
144 			snp->grphi = cpu;  in init_srcu_struct_nodes()
491 static void srcu_schedule_cbs_snp(struct srcu_struct *ssp, struct srcu_node *snp,  in srcu_schedule_cbs_snp()  argument
496 	for (cpu = snp->grplo; cpu <= snp->grphi; cpu++) {  in srcu_schedule_cbs_snp()
497 		if (!(mask & (1 << (cpu - snp->grplo))))  in srcu_schedule_cbs_snp()
523 	struct srcu_node *snp;  in srcu_gp_end()  local
543 	idx = rcu_seq_ctr(gpseq) % ARRAY_SIZE(snp->srcu_have_cbs);  in srcu_gp_end()
544 	srcu_for_each_node_breadth_first(ssp, snp) {  in srcu_gp_end()
545 		spin_lock_irq_rcu_node(snp);  in srcu_gp_end()
547 		last_lvl = snp >= ssp->level[rcu_num_lvls - 1];  in srcu_gp_end()
549 			cbs = snp->srcu_have_cbs[idx] == gpseq;  in srcu_gp_end()
550 		snp->srcu_have_cbs[idx] = gpseq;  in srcu_gp_end()
551 		rcu_seq_set_state(&snp->srcu_have_cbs[idx], 1);  in srcu_gp_end()
552 		if (ULONG_CMP_LT(snp->srcu_gp_seq_needed_exp, gpseq))  in srcu_gp_end()
553 			snp->srcu_gp_seq_needed_exp = gpseq;  in srcu_gp_end()
554 		mask = snp->srcu_data_have_cbs[idx];  in srcu_gp_end()
555 		snp->srcu_data_have_cbs[idx] = 0;  in srcu_gp_end()
556 		spin_unlock_irq_rcu_node(snp);  in srcu_gp_end()
558 			srcu_schedule_cbs_snp(ssp, snp, mask, cbdelay);  in srcu_gp_end()
562 			for (cpu = snp->grplo; cpu <= snp->grphi; cpu++) {  in srcu_gp_end()
598 static void srcu_funnel_exp_start(struct srcu_struct *ssp, struct srcu_node *snp,  in srcu_funnel_exp_start()  argument
603 	for (; snp != NULL; snp = snp->srcu_parent) {  in srcu_funnel_exp_start()
605 		    ULONG_CMP_GE(READ_ONCE(snp->srcu_gp_seq_needed_exp), s))  in srcu_funnel_exp_start()
607 		spin_lock_irqsave_rcu_node(snp, flags);  in srcu_funnel_exp_start()
608 		if (ULONG_CMP_GE(snp->srcu_gp_seq_needed_exp, s)) {  in srcu_funnel_exp_start()
609 			spin_unlock_irqrestore_rcu_node(snp, flags);  in srcu_funnel_exp_start()
612 		WRITE_ONCE(snp->srcu_gp_seq_needed_exp, s);  in srcu_funnel_exp_start()
613 		spin_unlock_irqrestore_rcu_node(snp, flags);  in srcu_funnel_exp_start()
636 	struct srcu_node *snp = sdp->mynode;  in srcu_funnel_gp_start()  local
640 	for (; snp != NULL; snp = snp->srcu_parent) {  in srcu_funnel_gp_start()
641 		if (rcu_seq_done(&ssp->srcu_gp_seq, s) && snp != sdp->mynode)  in srcu_funnel_gp_start()
643 		spin_lock_irqsave_rcu_node(snp, flags);  in srcu_funnel_gp_start()
644 		if (ULONG_CMP_GE(snp->srcu_have_cbs[idx], s)) {  in srcu_funnel_gp_start()
645 			snp_seq = snp->srcu_have_cbs[idx];  in srcu_funnel_gp_start()
646 			if (snp == sdp->mynode && snp_seq == s)  in srcu_funnel_gp_start()
647 				snp->srcu_data_have_cbs[idx] |= sdp->grpmask;  in srcu_funnel_gp_start()
648 			spin_unlock_irqrestore_rcu_node(snp, flags);  in srcu_funnel_gp_start()
649 			if (snp == sdp->mynode && snp_seq != s) {  in srcu_funnel_gp_start()
656 				srcu_funnel_exp_start(ssp, snp, s);  in srcu_funnel_gp_start()
659 		snp->srcu_have_cbs[idx] = s;  in srcu_funnel_gp_start()
660 		if (snp == sdp->mynode)  in srcu_funnel_gp_start()
661 			snp->srcu_data_have_cbs[idx] |= sdp->grpmask;  in srcu_funnel_gp_start()
662 		if (!do_norm && ULONG_CMP_LT(snp->srcu_gp_seq_needed_exp, s))  in srcu_funnel_gp_start()
663 			snp->srcu_gp_seq_needed_exp = s;  in srcu_funnel_gp_start()
664 		spin_unlock_irqrestore_rcu_node(snp, flags);  in srcu_funnel_gp_start()