619 	struct srcu_usage *sup = ssp->srcu_sup;  in srcu_get_delay()  local
621 	if (ULONG_CMP_LT(READ_ONCE(sup->srcu_gp_seq), READ_ONCE(sup->srcu_gp_seq_needed_exp)))  in srcu_get_delay()
623 	if (rcu_seq_state(READ_ONCE(sup->srcu_gp_seq))) {  in srcu_get_delay()
625 		gpstart = READ_ONCE(sup->srcu_gp_start);  in srcu_get_delay()
629 			WRITE_ONCE(sup->srcu_n_exp_nodelay, READ_ONCE(sup->srcu_n_exp_nodelay) + 1);  in srcu_get_delay()
630 			if (READ_ONCE(sup->srcu_n_exp_nodelay) > srcu_max_nodelay_phase)  in srcu_get_delay()
647 	struct srcu_usage *sup = ssp->srcu_sup;  in cleanup_srcu_struct()  local
653 	flush_delayed_work(&sup->work);  in cleanup_srcu_struct()
662 	if (WARN_ON(rcu_seq_state(READ_ONCE(sup->srcu_gp_seq)) != SRCU_STATE_IDLE) ||  in cleanup_srcu_struct()
663 	    WARN_ON(rcu_seq_current(&sup->srcu_gp_seq) != sup->srcu_gp_seq_needed) ||  in cleanup_srcu_struct()
666 			__func__, ssp, rcu_seq_state(READ_ONCE(sup->srcu_gp_seq)),  in cleanup_srcu_struct()
667 			rcu_seq_current(&sup->srcu_gp_seq), sup->srcu_gp_seq_needed);  in cleanup_srcu_struct()
670 	kfree(sup->node);  in cleanup_srcu_struct()
671 	sup->node = NULL;  in cleanup_srcu_struct()
672 	sup->srcu_size_state = SRCU_SIZE_SMALL;  in cleanup_srcu_struct()
673 	if (!sup->sda_is_static) {  in cleanup_srcu_struct()
676 		kfree(sup);  in cleanup_srcu_struct()
865 	struct srcu_usage *sup = ssp->srcu_sup;  in srcu_gp_end()  local
868 	mutex_lock(&sup->srcu_cb_mutex);  in srcu_gp_end()
871 	spin_lock_irq_rcu_node(sup);  in srcu_gp_end()
872 	idx = rcu_seq_state(sup->srcu_gp_seq);  in srcu_gp_end()
874 	if (ULONG_CMP_LT(READ_ONCE(sup->srcu_gp_seq), READ_ONCE(sup->srcu_gp_seq_needed_exp)))  in srcu_gp_end()
877 	WRITE_ONCE(sup->srcu_last_gp_end, ktime_get_mono_fast_ns());  in srcu_gp_end()
878 	rcu_seq_end(&sup->srcu_gp_seq);  in srcu_gp_end()
879 	gpseq = rcu_seq_current(&sup->srcu_gp_seq);  in srcu_gp_end()
880 	if (ULONG_CMP_LT(sup->srcu_gp_seq_needed_exp, gpseq))  in srcu_gp_end()
881 		WRITE_ONCE(sup->srcu_gp_seq_needed_exp, gpseq);  in srcu_gp_end()
882 	spin_unlock_irq_rcu_node(sup);  in srcu_gp_end()
883 	mutex_unlock(&sup->srcu_gp_mutex);  in srcu_gp_end()
887 	ss_state = smp_load_acquire(&sup->srcu_size_state);  in srcu_gp_end()
896 			last_lvl = snp >= sup->level[rcu_num_lvls - 1];  in srcu_gp_end()
928 	mutex_unlock(&sup->srcu_cb_mutex);  in srcu_gp_end()
931 	spin_lock_irq_rcu_node(sup);  in srcu_gp_end()
932 	gpseq = rcu_seq_current(&sup->srcu_gp_seq);  in srcu_gp_end()
934 	    ULONG_CMP_LT(gpseq, sup->srcu_gp_seq_needed)) {  in srcu_gp_end()
936 		spin_unlock_irq_rcu_node(sup);  in srcu_gp_end()
939 		spin_unlock_irq_rcu_node(sup);  in srcu_gp_end()
947 			smp_store_release(&sup->srcu_size_state, ss_state + 1);  in srcu_gp_end()
1007 	struct srcu_usage *sup = ssp->srcu_sup;  in srcu_funnel_gp_start()  local
1010 	if (smp_load_acquire(&sup->srcu_size_state) < SRCU_SIZE_WAIT_BARRIER)  in srcu_funnel_gp_start()
1018 			if (WARN_ON_ONCE(rcu_seq_done(&sup->srcu_gp_seq, s)) && snp != snp_leaf)  in srcu_funnel_gp_start()
1045 	if (ULONG_CMP_LT(sup->srcu_gp_seq_needed, s)) {  in srcu_funnel_gp_start()
1050 		smp_store_release(&sup->srcu_gp_seq_needed, s); /*^^^*/  in srcu_funnel_gp_start()
1052 	if (!do_norm && ULONG_CMP_LT(sup->srcu_gp_seq_needed_exp, s))  in srcu_funnel_gp_start()
1053 		WRITE_ONCE(sup->srcu_gp_seq_needed_exp, s);  in srcu_funnel_gp_start()
1056 	if (!WARN_ON_ONCE(rcu_seq_done(&sup->srcu_gp_seq, s)) &&  in srcu_funnel_gp_start()
1057 	    rcu_seq_state(sup->srcu_gp_seq) == SRCU_STATE_IDLE) {  in srcu_funnel_gp_start()
1058 		WARN_ON_ONCE(ULONG_CMP_GE(sup->srcu_gp_seq, sup->srcu_gp_seq_needed));  in srcu_funnel_gp_start()
1067 			queue_delayed_work(rcu_gp_wq, &sup->work,  in srcu_funnel_gp_start()
1069 		else if (list_empty(&sup->work.work.entry))  in srcu_funnel_gp_start()
1070 			list_add(&sup->work.work.entry, &srcu_boot_list);  in srcu_funnel_gp_start()
1072 	spin_unlock_irqrestore_rcu_node(sup, flags);  in srcu_funnel_gp_start()
1764 	struct srcu_usage *sup;  in process_srcu()  local
1766 	sup = container_of(work, struct srcu_usage, work.work);  in process_srcu()
1767 	ssp = sup->srcu_ssp;  in process_srcu()
1772 		WRITE_ONCE(sup->reschedule_count, 0);  in process_srcu()
1775 		if (READ_ONCE(sup->reschedule_jiffies) == j) {  in process_srcu()
1776 			WRITE_ONCE(sup->reschedule_count, READ_ONCE(sup->reschedule_count) + 1);  in process_srcu()
1777 			if (READ_ONCE(sup->reschedule_count) > srcu_max_nodelay)  in process_srcu()
1780 			WRITE_ONCE(sup->reschedule_count, 1);  in process_srcu()
1781 			WRITE_ONCE(sup->reschedule_jiffies, j);  in process_srcu()
1880 	struct srcu_usage *sup;  in srcu_init()  local
1900 		sup = list_first_entry(&srcu_boot_list, struct srcu_usage,  in srcu_init()
1902 		list_del_init(&sup->work.work.entry);  in srcu_init()
1904 		    sup->srcu_size_state == SRCU_SIZE_SMALL)  in srcu_init()
1905 			sup->srcu_size_state = SRCU_SIZE_ALLOC;  in srcu_init()
1906 		queue_work(rcu_gp_wq, &sup->work.work);  in srcu_init()