76 static void srcu_reschedule(struct srcu_struct *ssp, unsigned long delay);
122 static void init_srcu_struct_data(struct srcu_struct *ssp)  in init_srcu_struct_data()  argument
134 		sdp = per_cpu_ptr(ssp->sda, cpu);  in init_srcu_struct_data()
138 		sdp->srcu_gp_seq_needed = ssp->srcu_sup->srcu_gp_seq;  in init_srcu_struct_data()
139 		sdp->srcu_gp_seq_needed_exp = ssp->srcu_sup->srcu_gp_seq;  in init_srcu_struct_data()
144 		sdp->ssp = ssp;  in init_srcu_struct_data()
164 static bool init_srcu_struct_nodes(struct srcu_struct *ssp, gfp_t gfp_flags)  in init_srcu_struct_nodes()  argument
176 	ssp->srcu_sup->node = kcalloc(rcu_num_nodes, sizeof(*ssp->srcu_sup->node), gfp_flags);  in init_srcu_struct_nodes()
177 	if (!ssp->srcu_sup->node)  in init_srcu_struct_nodes()
181 	ssp->srcu_sup->level[0] = &ssp->srcu_sup->node[0];  in init_srcu_struct_nodes()
183 		ssp->srcu_sup->level[i] = ssp->srcu_sup->level[i - 1] + num_rcu_lvl[i - 1];  in init_srcu_struct_nodes()
187 	srcu_for_each_node_breadth_first(ssp, snp) {  in init_srcu_struct_nodes()
198 		if (snp == &ssp->srcu_sup->node[0]) {  in init_srcu_struct_nodes()
205 		if (snp == ssp->srcu_sup->level[level + 1])  in init_srcu_struct_nodes()
207 		snp->srcu_parent = ssp->srcu_sup->level[level - 1] +  in init_srcu_struct_nodes()
208 				   (snp - ssp->srcu_sup->level[level]) /  in init_srcu_struct_nodes()
217 	snp_first = ssp->srcu_sup->level[level];  in init_srcu_struct_nodes()
219 		sdp = per_cpu_ptr(ssp->sda, cpu);  in init_srcu_struct_nodes()
228 	smp_store_release(&ssp->srcu_sup->srcu_size_state, SRCU_SIZE_WAIT_BARRIER);  in init_srcu_struct_nodes()
237 static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static)  in init_srcu_struct_fields()  argument
240 		ssp->srcu_sup = kzalloc(sizeof(*ssp->srcu_sup), GFP_KERNEL);  in init_srcu_struct_fields()
241 	if (!ssp->srcu_sup)  in init_srcu_struct_fields()
244 		spin_lock_init(&ACCESS_PRIVATE(ssp->srcu_sup, lock));  in init_srcu_struct_fields()
245 	ssp->srcu_sup->srcu_size_state = SRCU_SIZE_SMALL;  in init_srcu_struct_fields()
246 	ssp->srcu_sup->node = NULL;  in init_srcu_struct_fields()
247 	mutex_init(&ssp->srcu_sup->srcu_cb_mutex);  in init_srcu_struct_fields()
248 	mutex_init(&ssp->srcu_sup->srcu_gp_mutex);  in init_srcu_struct_fields()
249 	ssp->srcu_idx = 0;  in init_srcu_struct_fields()
250 	ssp->srcu_sup->srcu_gp_seq = 0;  in init_srcu_struct_fields()
251 	ssp->srcu_sup->srcu_barrier_seq = 0;  in init_srcu_struct_fields()
252 	mutex_init(&ssp->srcu_sup->srcu_barrier_mutex);  in init_srcu_struct_fields()
253 	atomic_set(&ssp->srcu_sup->srcu_barrier_cpu_cnt, 0);  in init_srcu_struct_fields()
254 	INIT_DELAYED_WORK(&ssp->srcu_sup->work, process_srcu);  in init_srcu_struct_fields()
255 	ssp->srcu_sup->sda_is_static = is_static;  in init_srcu_struct_fields()
257 		ssp->sda = alloc_percpu(struct srcu_data);  in init_srcu_struct_fields()
258 	if (!ssp->sda) {  in init_srcu_struct_fields()
260 			kfree(ssp->srcu_sup);  in init_srcu_struct_fields()
263 	init_srcu_struct_data(ssp);  in init_srcu_struct_fields()
264 	ssp->srcu_sup->srcu_gp_seq_needed_exp = 0;  in init_srcu_struct_fields()
265 	ssp->srcu_sup->srcu_last_gp_end = ktime_get_mono_fast_ns();  in init_srcu_struct_fields()
266 	if (READ_ONCE(ssp->srcu_sup->srcu_size_state) == SRCU_SIZE_SMALL && SRCU_SIZING_IS_INIT()) {  in init_srcu_struct_fields()
267 		if (!init_srcu_struct_nodes(ssp, GFP_ATOMIC)) {  in init_srcu_struct_fields()
268 			if (!ssp->srcu_sup->sda_is_static) {  in init_srcu_struct_fields()
269 				free_percpu(ssp->sda);  in init_srcu_struct_fields()
270 				ssp->sda = NULL;  in init_srcu_struct_fields()
271 				kfree(ssp->srcu_sup);  in init_srcu_struct_fields()
275 			WRITE_ONCE(ssp->srcu_sup->srcu_size_state, SRCU_SIZE_BIG);  in init_srcu_struct_fields()
278 	ssp->srcu_sup->srcu_ssp = ssp;  in init_srcu_struct_fields()
279 	smp_store_release(&ssp->srcu_sup->srcu_gp_seq_needed, 0); /* Init done. */  in init_srcu_struct_fields()
285 int __init_srcu_struct(struct srcu_struct *ssp, const char *name,  in __init_srcu_struct()  argument
289 	debug_check_no_locks_freed((void *)ssp, sizeof(*ssp));  in __init_srcu_struct()
290 	lockdep_init_map(&ssp->dep_map, name, key, 0);  in __init_srcu_struct()
291 	return init_srcu_struct_fields(ssp, false);  in __init_srcu_struct()
305 int init_srcu_struct(struct srcu_struct *ssp)  in init_srcu_struct()  argument
307 	return init_srcu_struct_fields(ssp, false);  in init_srcu_struct()
316 static void __srcu_transition_to_big(struct srcu_struct *ssp)  in __srcu_transition_to_big()  argument
318 	lockdep_assert_held(&ACCESS_PRIVATE(ssp->srcu_sup, lock));  in __srcu_transition_to_big()
319 	smp_store_release(&ssp->srcu_sup->srcu_size_state, SRCU_SIZE_ALLOC);  in __srcu_transition_to_big()
325 static void srcu_transition_to_big(struct srcu_struct *ssp)  in srcu_transition_to_big()  argument
330 	if (smp_load_acquire(&ssp->srcu_sup->srcu_size_state) != SRCU_SIZE_SMALL)  in srcu_transition_to_big()
332 	spin_lock_irqsave_rcu_node(ssp->srcu_sup, flags);  in srcu_transition_to_big()
333 	if (smp_load_acquire(&ssp->srcu_sup->srcu_size_state) != SRCU_SIZE_SMALL) {  in srcu_transition_to_big()
334 		spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);  in srcu_transition_to_big()
337 	__srcu_transition_to_big(ssp);  in srcu_transition_to_big()
338 	spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);  in srcu_transition_to_big()
345 static void spin_lock_irqsave_check_contention(struct srcu_struct *ssp)  in spin_lock_irqsave_check_contention()  argument
349 	if (!SRCU_SIZING_IS_CONTEND() || ssp->srcu_sup->srcu_size_state)  in spin_lock_irqsave_check_contention()
352 	if (ssp->srcu_sup->srcu_size_jiffies != j) {  in spin_lock_irqsave_check_contention()
353 		ssp->srcu_sup->srcu_size_jiffies = j;  in spin_lock_irqsave_check_contention()
354 		ssp->srcu_sup->srcu_n_lock_retries = 0;  in spin_lock_irqsave_check_contention()
356 	if (++ssp->srcu_sup->srcu_n_lock_retries <= small_contention_lim)  in spin_lock_irqsave_check_contention()
358 	__srcu_transition_to_big(ssp);  in spin_lock_irqsave_check_contention()
369 	struct srcu_struct *ssp = sdp->ssp;  in spin_lock_irqsave_sdp_contention()  local
373 	spin_lock_irqsave_rcu_node(ssp->srcu_sup, *flags);  in spin_lock_irqsave_sdp_contention()
374 	spin_lock_irqsave_check_contention(ssp);  in spin_lock_irqsave_sdp_contention()
375 	spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, *flags);  in spin_lock_irqsave_sdp_contention()
385 static void spin_lock_irqsave_ssp_contention(struct srcu_struct *ssp, unsigned long *flags)  in spin_lock_irqsave_ssp_contention()  argument
387 	if (spin_trylock_irqsave_rcu_node(ssp->srcu_sup, *flags))  in spin_lock_irqsave_ssp_contention()
389 	spin_lock_irqsave_rcu_node(ssp->srcu_sup, *flags);  in spin_lock_irqsave_ssp_contention()
390 	spin_lock_irqsave_check_contention(ssp);  in spin_lock_irqsave_ssp_contention()
401 static void check_init_srcu_struct(struct srcu_struct *ssp)  in check_init_srcu_struct()  argument
406 	if (!rcu_seq_state(smp_load_acquire(&ssp->srcu_sup->srcu_gp_seq_needed))) /*^^^*/  in check_init_srcu_struct()
408 	spin_lock_irqsave_rcu_node(ssp->srcu_sup, flags);  in check_init_srcu_struct()
409 	if (!rcu_seq_state(ssp->srcu_sup->srcu_gp_seq_needed)) {  in check_init_srcu_struct()
410 		spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);  in check_init_srcu_struct()
413 	init_srcu_struct_fields(ssp, true);  in check_init_srcu_struct()
414 	spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);  in check_init_srcu_struct()
421 static unsigned long srcu_readers_lock_idx(struct srcu_struct *ssp, int idx)  in srcu_readers_lock_idx()  argument
427 		struct srcu_data *cpuc = per_cpu_ptr(ssp->sda, cpu);  in srcu_readers_lock_idx()
438 static unsigned long srcu_readers_unlock_idx(struct srcu_struct *ssp, int idx)  in srcu_readers_unlock_idx()  argument
445 		struct srcu_data *cpuc = per_cpu_ptr(ssp->sda, cpu);  in srcu_readers_unlock_idx()
452 		  "Mixed NMI-safe readers for srcu_struct at %ps.\n", ssp);  in srcu_readers_unlock_idx()
460 static bool srcu_readers_active_idx_check(struct srcu_struct *ssp, int idx)  in srcu_readers_active_idx_check()  argument
464 	unlocks = srcu_readers_unlock_idx(ssp, idx);  in srcu_readers_active_idx_check()
535 	return srcu_readers_lock_idx(ssp, idx) == unlocks;  in srcu_readers_active_idx_check()
547 static bool srcu_readers_active(struct srcu_struct *ssp)  in srcu_readers_active()  argument
553 		struct srcu_data *cpuc = per_cpu_ptr(ssp->sda, cpu);  in srcu_readers_active()
614 static unsigned long srcu_get_delay(struct srcu_struct *ssp)  in srcu_get_delay()  argument
619 	struct srcu_usage *sup = ssp->srcu_sup;  in srcu_get_delay()
644 void cleanup_srcu_struct(struct srcu_struct *ssp)  in cleanup_srcu_struct()  argument
647 	struct srcu_usage *sup = ssp->srcu_sup;  in cleanup_srcu_struct()
649 	if (WARN_ON(!srcu_get_delay(ssp)))  in cleanup_srcu_struct()
651 	if (WARN_ON(srcu_readers_active(ssp)))  in cleanup_srcu_struct()
655 		struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu);  in cleanup_srcu_struct()
664 	    WARN_ON(srcu_readers_active(ssp))) {  in cleanup_srcu_struct()
666 			__func__, ssp, rcu_seq_state(READ_ONCE(sup->srcu_gp_seq)),  in cleanup_srcu_struct()
674 		free_percpu(ssp->sda);  in cleanup_srcu_struct()
675 		ssp->sda = NULL;  in cleanup_srcu_struct()
677 		ssp->srcu_sup = NULL;  in cleanup_srcu_struct()
686 void srcu_check_nmi_safety(struct srcu_struct *ssp, bool nmi_safe)  in srcu_check_nmi_safety()  argument
694 	sdp = raw_cpu_ptr(ssp->sda);  in srcu_check_nmi_safety()
710 int __srcu_read_lock(struct srcu_struct *ssp)  in __srcu_read_lock()  argument
714 	idx = READ_ONCE(ssp->srcu_idx) & 0x1;  in __srcu_read_lock()
715 	this_cpu_inc(ssp->sda->srcu_lock_count[idx].counter);  in __srcu_read_lock()
726 void __srcu_read_unlock(struct srcu_struct *ssp, int idx)  in __srcu_read_unlock()  argument
729 	this_cpu_inc(ssp->sda->srcu_unlock_count[idx].counter);  in __srcu_read_unlock()
740 int __srcu_read_lock_nmisafe(struct srcu_struct *ssp)  in __srcu_read_lock_nmisafe()  argument
743 	struct srcu_data *sdp = raw_cpu_ptr(ssp->sda);  in __srcu_read_lock_nmisafe()
745 	idx = READ_ONCE(ssp->srcu_idx) & 0x1;  in __srcu_read_lock_nmisafe()
757 void __srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx)  in __srcu_read_unlock_nmisafe()  argument
759 	struct srcu_data *sdp = raw_cpu_ptr(ssp->sda);  in __srcu_read_unlock_nmisafe()
771 static void srcu_gp_start(struct srcu_struct *ssp)  in srcu_gp_start()  argument
776 	if (smp_load_acquire(&ssp->srcu_sup->srcu_size_state) < SRCU_SIZE_WAIT_BARRIER)  in srcu_gp_start()
777 		sdp = per_cpu_ptr(ssp->sda, get_boot_cpu_id());  in srcu_gp_start()
779 		sdp = this_cpu_ptr(ssp->sda);  in srcu_gp_start()
780 	lockdep_assert_held(&ACCESS_PRIVATE(ssp->srcu_sup, lock));  in srcu_gp_start()
781 	WARN_ON_ONCE(ULONG_CMP_GE(ssp->srcu_sup->srcu_gp_seq, ssp->srcu_sup->srcu_gp_seq_needed));  in srcu_gp_start()
784 			      rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq));  in srcu_gp_start()
786 				       rcu_seq_snap(&ssp->srcu_sup->srcu_gp_seq));  in srcu_gp_start()
788 	WRITE_ONCE(ssp->srcu_sup->srcu_gp_start, jiffies);  in srcu_gp_start()
789 	WRITE_ONCE(ssp->srcu_sup->srcu_n_exp_nodelay, 0);  in srcu_gp_start()
791 	rcu_seq_start(&ssp->srcu_sup->srcu_gp_seq);  in srcu_gp_start()
792 	state = rcu_seq_state(ssp->srcu_sup->srcu_gp_seq);  in srcu_gp_start()
830 static void srcu_schedule_cbs_snp(struct srcu_struct *ssp, struct srcu_node *snp,  in srcu_schedule_cbs_snp()  argument
838 		srcu_schedule_cbs_sdp(per_cpu_ptr(ssp->sda, cpu), delay);  in srcu_schedule_cbs_snp()
851 static void srcu_gp_end(struct srcu_struct *ssp)  in srcu_gp_end()  argument
865 	struct srcu_usage *sup = ssp->srcu_sup;  in srcu_gp_end()
889 		srcu_schedule_cbs_sdp(per_cpu_ptr(ssp->sda, get_boot_cpu_id()),  in srcu_gp_end()
893 		srcu_for_each_node_breadth_first(ssp, snp) {  in srcu_gp_end()
911 				srcu_schedule_cbs_snp(ssp, snp, mask, cbdelay);  in srcu_gp_end()
918 			sdp = per_cpu_ptr(ssp->sda, cpu);  in srcu_gp_end()
935 		srcu_gp_start(ssp);  in srcu_gp_end()
937 		srcu_reschedule(ssp, 0);  in srcu_gp_end()
945 			init_srcu_struct_nodes(ssp, GFP_KERNEL);  in srcu_gp_end()
958 static void srcu_funnel_exp_start(struct srcu_struct *ssp, struct srcu_node *snp,  in srcu_funnel_exp_start()  argument
967 			if (WARN_ON_ONCE(rcu_seq_done(&ssp->srcu_sup->srcu_gp_seq, s)) ||  in srcu_funnel_exp_start()
979 	spin_lock_irqsave_ssp_contention(ssp, &flags);  in srcu_funnel_exp_start()
980 	if (ULONG_CMP_LT(ssp->srcu_sup->srcu_gp_seq_needed_exp, s))  in srcu_funnel_exp_start()
981 		WRITE_ONCE(ssp->srcu_sup->srcu_gp_seq_needed_exp, s);  in srcu_funnel_exp_start()
982 	spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);  in srcu_funnel_exp_start()
998 static void srcu_funnel_gp_start(struct srcu_struct *ssp, struct srcu_data *sdp,  in srcu_funnel_gp_start()  argument
1007 	struct srcu_usage *sup = ssp->srcu_sup;  in srcu_funnel_gp_start()
1031 					srcu_funnel_exp_start(ssp, snp, s);  in srcu_funnel_gp_start()
1044 	spin_lock_irqsave_ssp_contention(ssp, &flags);  in srcu_funnel_gp_start()
1059 		srcu_gp_start(ssp);  in srcu_funnel_gp_start()
1068 					   !!srcu_get_delay(ssp));  in srcu_funnel_gp_start()
1080 static bool try_check_zero(struct srcu_struct *ssp, int idx, int trycount)  in try_check_zero()  argument
1084 	curdelay = !srcu_get_delay(ssp);  in try_check_zero()
1087 		if (srcu_readers_active_idx_check(ssp, idx))  in try_check_zero()
1100 static void srcu_flip(struct srcu_struct *ssp)  in srcu_flip()  argument
1132 	WRITE_ONCE(ssp->srcu_idx, ssp->srcu_idx + 1); // Flip the counter.  in srcu_flip()
1166 static bool srcu_might_be_idle(struct srcu_struct *ssp)  in srcu_might_be_idle()  argument
1174 	check_init_srcu_struct(ssp);  in srcu_might_be_idle()
1176 	sdp = raw_cpu_ptr(ssp->sda);  in srcu_might_be_idle()
1192 	tlast = READ_ONCE(ssp->srcu_sup->srcu_last_gp_end);  in srcu_might_be_idle()
1198 	curseq = rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq);  in srcu_might_be_idle()
1200 	if (ULONG_CMP_LT(curseq, READ_ONCE(ssp->srcu_sup->srcu_gp_seq_needed)))  in srcu_might_be_idle()
1203 	if (curseq != rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq))  in srcu_might_be_idle()
1218 static unsigned long srcu_gp_start_if_needed(struct srcu_struct *ssp,  in srcu_gp_start_if_needed()  argument
1230 	check_init_srcu_struct(ssp);  in srcu_gp_start_if_needed()
1236 	idx = __srcu_read_lock_nmisafe(ssp);  in srcu_gp_start_if_needed()
1237 	ss_state = smp_load_acquire(&ssp->srcu_sup->srcu_size_state);  in srcu_gp_start_if_needed()
1239 		sdp = per_cpu_ptr(ssp->sda, get_boot_cpu_id());  in srcu_gp_start_if_needed()
1241 		sdp = raw_cpu_ptr(ssp->sda);  in srcu_gp_start_if_needed()
1246 			      rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq));  in srcu_gp_start_if_needed()
1247 	s = rcu_seq_snap(&ssp->srcu_sup->srcu_gp_seq);  in srcu_gp_start_if_needed()
1266 		srcu_funnel_gp_start(ssp, sdp, s, do_norm);  in srcu_gp_start_if_needed()
1268 		srcu_funnel_exp_start(ssp, sdp_mynode, s);  in srcu_gp_start_if_needed()
1269 	__srcu_read_unlock_nmisafe(ssp, idx);  in srcu_gp_start_if_needed()
1301 static void __call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp,  in __call_srcu()  argument
1311 	(void)srcu_gp_start_if_needed(ssp, rhp, do_norm);  in __call_srcu()
1331 void call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp,  in call_srcu()  argument
1334 	__call_srcu(ssp, rhp, func, true);  in call_srcu()
1341 static void __synchronize_srcu(struct srcu_struct *ssp, bool do_norm)  in __synchronize_srcu()  argument
1345 	srcu_lock_sync(&ssp->dep_map);  in __synchronize_srcu()
1347 	RCU_LOCKDEP_WARN(lockdep_is_held(ssp) ||  in __synchronize_srcu()
1356 	check_init_srcu_struct(ssp);  in __synchronize_srcu()
1359 	__call_srcu(ssp, &rcu.head, wakeme_after_rcu, do_norm);  in __synchronize_srcu()
1383 void synchronize_srcu_expedited(struct srcu_struct *ssp)  in synchronize_srcu_expedited()  argument
1385 	__synchronize_srcu(ssp, rcu_gp_is_normal());  in synchronize_srcu_expedited()
1436 void synchronize_srcu(struct srcu_struct *ssp)  in synchronize_srcu()  argument
1438 	if (srcu_might_be_idle(ssp) || rcu_gp_is_expedited())  in synchronize_srcu()
1439 		synchronize_srcu_expedited(ssp);  in synchronize_srcu()
1441 		__synchronize_srcu(ssp, true);  in synchronize_srcu()
1455 unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp)  in get_state_synchronize_srcu()  argument
1460 	return rcu_seq_snap(&ssp->srcu_sup->srcu_gp_seq);  in get_state_synchronize_srcu()
1474 unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp)  in start_poll_synchronize_srcu()  argument
1476 	return srcu_gp_start_if_needed(ssp, NULL, true);  in start_poll_synchronize_srcu()
1505 bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie)  in poll_state_synchronize_srcu()  argument
1507 	if (!rcu_seq_done(&ssp->srcu_sup->srcu_gp_seq, cookie))  in poll_state_synchronize_srcu()
1522 	struct srcu_struct *ssp;  in srcu_barrier_cb()  local
1525 	ssp = sdp->ssp;  in srcu_barrier_cb()
1526 	if (atomic_dec_and_test(&ssp->srcu_sup->srcu_barrier_cpu_cnt))  in srcu_barrier_cb()
1527 		complete(&ssp->srcu_sup->srcu_barrier_completion);  in srcu_barrier_cb()
1538 static void srcu_barrier_one_cpu(struct srcu_struct *ssp, struct srcu_data *sdp)  in srcu_barrier_one_cpu()  argument
1541 	atomic_inc(&ssp->srcu_sup->srcu_barrier_cpu_cnt);  in srcu_barrier_one_cpu()
1547 		atomic_dec(&ssp->srcu_sup->srcu_barrier_cpu_cnt);  in srcu_barrier_one_cpu()
1556 void srcu_barrier(struct srcu_struct *ssp)  in srcu_barrier()  argument
1560 	unsigned long s = rcu_seq_snap(&ssp->srcu_sup->srcu_barrier_seq);  in srcu_barrier()
1562 	check_init_srcu_struct(ssp);  in srcu_barrier()
1563 	mutex_lock(&ssp->srcu_sup->srcu_barrier_mutex);  in srcu_barrier()
1564 	if (rcu_seq_done(&ssp->srcu_sup->srcu_barrier_seq, s)) {  in srcu_barrier()
1566 		mutex_unlock(&ssp->srcu_sup->srcu_barrier_mutex);  in srcu_barrier()
1569 	rcu_seq_start(&ssp->srcu_sup->srcu_barrier_seq);  in srcu_barrier()
1570 	init_completion(&ssp->srcu_sup->srcu_barrier_completion);  in srcu_barrier()
1573 	atomic_set(&ssp->srcu_sup->srcu_barrier_cpu_cnt, 1);  in srcu_barrier()
1575 	idx = __srcu_read_lock_nmisafe(ssp);  in srcu_barrier()
1576 	if (smp_load_acquire(&ssp->srcu_sup->srcu_size_state) < SRCU_SIZE_WAIT_BARRIER)  in srcu_barrier()
1577 		srcu_barrier_one_cpu(ssp, per_cpu_ptr(ssp->sda,	get_boot_cpu_id()));  in srcu_barrier()
1580 			srcu_barrier_one_cpu(ssp, per_cpu_ptr(ssp->sda, cpu));  in srcu_barrier()
1581 	__srcu_read_unlock_nmisafe(ssp, idx);  in srcu_barrier()
1584 	if (atomic_dec_and_test(&ssp->srcu_sup->srcu_barrier_cpu_cnt))  in srcu_barrier()
1585 		complete(&ssp->srcu_sup->srcu_barrier_completion);  in srcu_barrier()
1586 	wait_for_completion(&ssp->srcu_sup->srcu_barrier_completion);  in srcu_barrier()
1588 	rcu_seq_end(&ssp->srcu_sup->srcu_barrier_seq);  in srcu_barrier()
1589 	mutex_unlock(&ssp->srcu_sup->srcu_barrier_mutex);  in srcu_barrier()
1600 unsigned long srcu_batches_completed(struct srcu_struct *ssp)  in srcu_batches_completed()  argument
1602 	return READ_ONCE(ssp->srcu_idx);  in srcu_batches_completed()
1611 static void srcu_advance_state(struct srcu_struct *ssp)  in srcu_advance_state()  argument
1615 	mutex_lock(&ssp->srcu_sup->srcu_gp_mutex);  in srcu_advance_state()
1627 	idx = rcu_seq_state(smp_load_acquire(&ssp->srcu_sup->srcu_gp_seq)); /* ^^^ */  in srcu_advance_state()
1629 		spin_lock_irq_rcu_node(ssp->srcu_sup);  in srcu_advance_state()
1630 		if (ULONG_CMP_GE(ssp->srcu_sup->srcu_gp_seq, ssp->srcu_sup->srcu_gp_seq_needed)) {  in srcu_advance_state()
1631 			WARN_ON_ONCE(rcu_seq_state(ssp->srcu_sup->srcu_gp_seq));  in srcu_advance_state()
1632 			spin_unlock_irq_rcu_node(ssp->srcu_sup);  in srcu_advance_state()
1633 			mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex);  in srcu_advance_state()
1636 		idx = rcu_seq_state(READ_ONCE(ssp->srcu_sup->srcu_gp_seq));  in srcu_advance_state()
1638 			srcu_gp_start(ssp);  in srcu_advance_state()
1639 		spin_unlock_irq_rcu_node(ssp->srcu_sup);  in srcu_advance_state()
1641 			mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex);  in srcu_advance_state()
1646 	if (rcu_seq_state(READ_ONCE(ssp->srcu_sup->srcu_gp_seq)) == SRCU_STATE_SCAN1) {  in srcu_advance_state()
1647 		idx = 1 ^ (ssp->srcu_idx & 1);  in srcu_advance_state()
1648 		if (!try_check_zero(ssp, idx, 1)) {  in srcu_advance_state()
1649 			mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex);  in srcu_advance_state()
1652 		srcu_flip(ssp);  in srcu_advance_state()
1653 		spin_lock_irq_rcu_node(ssp->srcu_sup);  in srcu_advance_state()
1654 		rcu_seq_set_state(&ssp->srcu_sup->srcu_gp_seq, SRCU_STATE_SCAN2);  in srcu_advance_state()
1655 		ssp->srcu_sup->srcu_n_exp_nodelay = 0;  in srcu_advance_state()
1656 		spin_unlock_irq_rcu_node(ssp->srcu_sup);  in srcu_advance_state()
1659 	if (rcu_seq_state(READ_ONCE(ssp->srcu_sup->srcu_gp_seq)) == SRCU_STATE_SCAN2) {  in srcu_advance_state()
1665 		idx = 1 ^ (ssp->srcu_idx & 1);  in srcu_advance_state()
1666 		if (!try_check_zero(ssp, idx, 2)) {  in srcu_advance_state()
1667 			mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex);  in srcu_advance_state()
1670 		ssp->srcu_sup->srcu_n_exp_nodelay = 0;  in srcu_advance_state()
1671 		srcu_gp_end(ssp);  /* Releases ->srcu_gp_mutex. */  in srcu_advance_state()
1688 	struct srcu_struct *ssp;  in srcu_invoke_callbacks()  local
1692 	ssp = sdp->ssp;  in srcu_invoke_callbacks()
1696 			      rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq));  in srcu_invoke_callbacks()
1724 				       rcu_seq_snap(&ssp->srcu_sup->srcu_gp_seq));  in srcu_invoke_callbacks()
1736 static void srcu_reschedule(struct srcu_struct *ssp, unsigned long delay)  in srcu_reschedule()  argument
1740 	spin_lock_irq_rcu_node(ssp->srcu_sup);  in srcu_reschedule()
1741 	if (ULONG_CMP_GE(ssp->srcu_sup->srcu_gp_seq, ssp->srcu_sup->srcu_gp_seq_needed)) {  in srcu_reschedule()
1742 		if (!WARN_ON_ONCE(rcu_seq_state(ssp->srcu_sup->srcu_gp_seq))) {  in srcu_reschedule()
1746 	} else if (!rcu_seq_state(ssp->srcu_sup->srcu_gp_seq)) {  in srcu_reschedule()
1748 		srcu_gp_start(ssp);  in srcu_reschedule()
1750 	spin_unlock_irq_rcu_node(ssp->srcu_sup);  in srcu_reschedule()
1753 		queue_delayed_work(rcu_gp_wq, &ssp->srcu_sup->work, delay);  in srcu_reschedule()
1763 	struct srcu_struct *ssp;  in process_srcu()  local
1767 	ssp = sup->srcu_ssp;  in process_srcu()
1769 	srcu_advance_state(ssp);  in process_srcu()
1770 	curdelay = srcu_get_delay(ssp);  in process_srcu()
1784 	srcu_reschedule(ssp, curdelay);  in process_srcu()
1788 			     struct srcu_struct *ssp, int *flags,  in srcutorture_get_gp_data()  argument
1794 	*gp_seq = rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq);  in srcutorture_get_gp_data()
1811 void srcu_torture_stats_print(struct srcu_struct *ssp, char *tt, char *tf)  in srcu_torture_stats_print()  argument
1816 	int ss_state = READ_ONCE(ssp->srcu_sup->srcu_size_state);  in srcu_torture_stats_print()
1819 	idx = ssp->srcu_idx & 0x1;  in srcu_torture_stats_print()
1823 		 tt, tf, rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq), ss_state,  in srcu_torture_stats_print()
1825 	if (!ssp->sda) {  in srcu_torture_stats_print()
1836 			sdp = per_cpu_ptr(ssp->sda, cpu);  in srcu_torture_stats_print()
1860 		srcu_transition_to_big(ssp);  in srcu_torture_stats_print()
1916 	struct srcu_struct *ssp;  in srcu_module_coming()  local
1920 		ssp = *(sspp++);  in srcu_module_coming()
1921 		ssp->sda = alloc_percpu(struct srcu_data);  in srcu_module_coming()
1922 		if (WARN_ON_ONCE(!ssp->sda))  in srcu_module_coming()
1932 	struct srcu_struct *ssp;  in srcu_module_going()  local
1936 		ssp = *(sspp++);  in srcu_module_going()
1937 		if (!rcu_seq_state(smp_load_acquire(&ssp->srcu_sup->srcu_gp_seq_needed)) &&  in srcu_module_going()
1938 		    !WARN_ON_ONCE(!ssp->srcu_sup->sda_is_static))  in srcu_module_going()
1939 			cleanup_srcu_struct(ssp);  in srcu_module_going()
1940 		if (!WARN_ON(srcu_readers_active(ssp)))  in srcu_module_going()
1941 			free_percpu(ssp->sda);  in srcu_module_going()