46 static void srcu_reschedule(struct srcu_struct *ssp, unsigned long delay);
83 static void init_srcu_struct_nodes(struct srcu_struct *ssp)  in init_srcu_struct_nodes()  argument
97 	ssp->level[0] = &ssp->node[0];  in init_srcu_struct_nodes()
99 		ssp->level[i] = ssp->level[i - 1] + num_rcu_lvl[i - 1];  in init_srcu_struct_nodes()
103 	srcu_for_each_node_breadth_first(ssp, snp) {  in init_srcu_struct_nodes()
114 		if (snp == &ssp->node[0]) {  in init_srcu_struct_nodes()
121 		if (snp == ssp->level[level + 1])  in init_srcu_struct_nodes()
123 		snp->srcu_parent = ssp->level[level - 1] +  in init_srcu_struct_nodes()
124 				   (snp - ssp->level[level]) /  in init_srcu_struct_nodes()
135 	snp_first = ssp->level[level];  in init_srcu_struct_nodes()
137 		sdp = per_cpu_ptr(ssp->sda, cpu);  in init_srcu_struct_nodes()
141 		sdp->srcu_gp_seq_needed = ssp->srcu_gp_seq;  in init_srcu_struct_nodes()
142 		sdp->srcu_gp_seq_needed_exp = ssp->srcu_gp_seq;  in init_srcu_struct_nodes()
152 		sdp->ssp = ssp;  in init_srcu_struct_nodes()
163 static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static)  in init_srcu_struct_fields()  argument
165 	mutex_init(&ssp->srcu_cb_mutex);  in init_srcu_struct_fields()
166 	mutex_init(&ssp->srcu_gp_mutex);  in init_srcu_struct_fields()
167 	ssp->srcu_idx = 0;  in init_srcu_struct_fields()
168 	ssp->srcu_gp_seq = 0;  in init_srcu_struct_fields()
169 	ssp->srcu_barrier_seq = 0;  in init_srcu_struct_fields()
170 	mutex_init(&ssp->srcu_barrier_mutex);  in init_srcu_struct_fields()
171 	atomic_set(&ssp->srcu_barrier_cpu_cnt, 0);  in init_srcu_struct_fields()
172 	INIT_DELAYED_WORK(&ssp->work, process_srcu);  in init_srcu_struct_fields()
174 		ssp->sda = alloc_percpu(struct srcu_data);  in init_srcu_struct_fields()
175 	if (!ssp->sda)  in init_srcu_struct_fields()
177 	init_srcu_struct_nodes(ssp);  in init_srcu_struct_fields()
178 	ssp->srcu_gp_seq_needed_exp = 0;  in init_srcu_struct_fields()
179 	ssp->srcu_last_gp_end = ktime_get_mono_fast_ns();  in init_srcu_struct_fields()
180 	smp_store_release(&ssp->srcu_gp_seq_needed, 0); /* Init done. */  in init_srcu_struct_fields()
186 int __init_srcu_struct(struct srcu_struct *ssp, const char *name,  in __init_srcu_struct()  argument
190 	debug_check_no_locks_freed((void *)ssp, sizeof(*ssp));  in __init_srcu_struct()
191 	lockdep_init_map(&ssp->dep_map, name, key, 0);  in __init_srcu_struct()
192 	spin_lock_init(&ACCESS_PRIVATE(ssp, lock));  in __init_srcu_struct()
193 	return init_srcu_struct_fields(ssp, false);  in __init_srcu_struct()
201  * @ssp: structure to initialize.
207 int init_srcu_struct(struct srcu_struct *ssp)  in init_srcu_struct()  argument
209 	spin_lock_init(&ACCESS_PRIVATE(ssp, lock));  in init_srcu_struct()
210 	return init_srcu_struct_fields(ssp, false);  in init_srcu_struct()
220  * to each update-side SRCU primitive.  Use ssp->lock, which -is-
224 static void check_init_srcu_struct(struct srcu_struct *ssp)  in check_init_srcu_struct()  argument
229 	if (!rcu_seq_state(smp_load_acquire(&ssp->srcu_gp_seq_needed))) /*^^^*/  in check_init_srcu_struct()
231 	spin_lock_irqsave_rcu_node(ssp, flags);  in check_init_srcu_struct()
232 	if (!rcu_seq_state(ssp->srcu_gp_seq_needed)) {  in check_init_srcu_struct()
233 		spin_unlock_irqrestore_rcu_node(ssp, flags);  in check_init_srcu_struct()
236 	init_srcu_struct_fields(ssp, true);  in check_init_srcu_struct()
237 	spin_unlock_irqrestore_rcu_node(ssp, flags);  in check_init_srcu_struct()
244 static unsigned long srcu_readers_lock_idx(struct srcu_struct *ssp, int idx)  in srcu_readers_lock_idx()  argument
250 		struct srcu_data *cpuc = per_cpu_ptr(ssp->sda, cpu);  in srcu_readers_lock_idx()
261 static unsigned long srcu_readers_unlock_idx(struct srcu_struct *ssp, int idx)  in srcu_readers_unlock_idx()  argument
267 		struct srcu_data *cpuc = per_cpu_ptr(ssp->sda, cpu);  in srcu_readers_unlock_idx()
278 static bool srcu_readers_active_idx_check(struct srcu_struct *ssp, int idx)  in srcu_readers_active_idx_check()  argument
282 	unlocks = srcu_readers_unlock_idx(ssp, idx);  in srcu_readers_active_idx_check()
318 	return srcu_readers_lock_idx(ssp, idx) == unlocks;  in srcu_readers_active_idx_check()
324  * @ssp: which srcu_struct to count active readers (holding srcu_read_lock).
330 static bool srcu_readers_active(struct srcu_struct *ssp)  in srcu_readers_active()  argument
336 		struct srcu_data *cpuc = per_cpu_ptr(ssp->sda, cpu);  in srcu_readers_active()
352 static unsigned long srcu_get_delay(struct srcu_struct *ssp)  in srcu_get_delay()  argument
354 	if (ULONG_CMP_LT(READ_ONCE(ssp->srcu_gp_seq),  in srcu_get_delay()
355 			 READ_ONCE(ssp->srcu_gp_seq_needed_exp)))  in srcu_get_delay()
362  * @ssp: structure to clean up.
367 void cleanup_srcu_struct(struct srcu_struct *ssp)  in cleanup_srcu_struct()  argument
371 	if (WARN_ON(!srcu_get_delay(ssp)))  in cleanup_srcu_struct()
373 	if (WARN_ON(srcu_readers_active(ssp)))  in cleanup_srcu_struct()
375 	flush_delayed_work(&ssp->work);  in cleanup_srcu_struct()
377 		struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu);  in cleanup_srcu_struct()
384 	if (WARN_ON(rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq)) != SRCU_STATE_IDLE) ||  in cleanup_srcu_struct()
385 	    WARN_ON(srcu_readers_active(ssp))) {  in cleanup_srcu_struct()
387 			__func__, ssp, rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq)));  in cleanup_srcu_struct()
390 	free_percpu(ssp->sda);  in cleanup_srcu_struct()
391 	ssp->sda = NULL;  in cleanup_srcu_struct()
400 int __srcu_read_lock(struct srcu_struct *ssp)  in __srcu_read_lock()  argument
404 	idx = READ_ONCE(ssp->srcu_idx) & 0x1;  in __srcu_read_lock()
405 	this_cpu_inc(ssp->sda->srcu_lock_count[idx]);  in __srcu_read_lock()
416 void __srcu_read_unlock(struct srcu_struct *ssp, int idx)  in __srcu_read_unlock()  argument
419 	this_cpu_inc(ssp->sda->srcu_unlock_count[idx]);  in __srcu_read_unlock()
435 static void srcu_gp_start(struct srcu_struct *ssp)  in srcu_gp_start()  argument
437 	struct srcu_data *sdp = this_cpu_ptr(ssp->sda);  in srcu_gp_start()
440 	lockdep_assert_held(&ACCESS_PRIVATE(ssp, lock));  in srcu_gp_start()
441 	WARN_ON_ONCE(ULONG_CMP_GE(ssp->srcu_gp_seq, ssp->srcu_gp_seq_needed));  in srcu_gp_start()
444 			      rcu_seq_current(&ssp->srcu_gp_seq));  in srcu_gp_start()
446 				       rcu_seq_snap(&ssp->srcu_gp_seq));  in srcu_gp_start()
449 	rcu_seq_start(&ssp->srcu_gp_seq);  in srcu_gp_start()
450 	state = rcu_seq_state(ssp->srcu_gp_seq);  in srcu_gp_start()
488 static void srcu_schedule_cbs_snp(struct srcu_struct *ssp, struct srcu_node *snp,  in srcu_schedule_cbs_snp()  argument
496 		srcu_schedule_cbs_sdp(per_cpu_ptr(ssp->sda, cpu), delay);  in srcu_schedule_cbs_snp()
509 static void srcu_gp_end(struct srcu_struct *ssp)  in srcu_gp_end()  argument
523 	mutex_lock(&ssp->srcu_cb_mutex);  in srcu_gp_end()
526 	spin_lock_irq_rcu_node(ssp);  in srcu_gp_end()
527 	idx = rcu_seq_state(ssp->srcu_gp_seq);  in srcu_gp_end()
529 	cbdelay = srcu_get_delay(ssp);  in srcu_gp_end()
530 	WRITE_ONCE(ssp->srcu_last_gp_end, ktime_get_mono_fast_ns());  in srcu_gp_end()
531 	rcu_seq_end(&ssp->srcu_gp_seq);  in srcu_gp_end()
532 	gpseq = rcu_seq_current(&ssp->srcu_gp_seq);  in srcu_gp_end()
533 	if (ULONG_CMP_LT(ssp->srcu_gp_seq_needed_exp, gpseq))  in srcu_gp_end()
534 		WRITE_ONCE(ssp->srcu_gp_seq_needed_exp, gpseq);  in srcu_gp_end()
535 	spin_unlock_irq_rcu_node(ssp);  in srcu_gp_end()
536 	mutex_unlock(&ssp->srcu_gp_mutex);  in srcu_gp_end()
541 	srcu_for_each_node_breadth_first(ssp, snp) {  in srcu_gp_end()
544 		last_lvl = snp >= ssp->level[rcu_num_lvls - 1];  in srcu_gp_end()
555 			srcu_schedule_cbs_snp(ssp, snp, mask, cbdelay);  in srcu_gp_end()
560 				sdp = per_cpu_ptr(ssp->sda, cpu);  in srcu_gp_end()
573 	mutex_unlock(&ssp->srcu_cb_mutex);  in srcu_gp_end()
576 	spin_lock_irq_rcu_node(ssp);  in srcu_gp_end()
577 	gpseq = rcu_seq_current(&ssp->srcu_gp_seq);  in srcu_gp_end()
579 	    ULONG_CMP_LT(gpseq, ssp->srcu_gp_seq_needed)) {  in srcu_gp_end()
580 		srcu_gp_start(ssp);  in srcu_gp_end()
581 		spin_unlock_irq_rcu_node(ssp);  in srcu_gp_end()
582 		srcu_reschedule(ssp, 0);  in srcu_gp_end()
584 		spin_unlock_irq_rcu_node(ssp);  in srcu_gp_end()
595 static void srcu_funnel_exp_start(struct srcu_struct *ssp, struct srcu_node *snp,  in srcu_funnel_exp_start()  argument
601 		if (rcu_seq_done(&ssp->srcu_gp_seq, s) ||  in srcu_funnel_exp_start()
612 	spin_lock_irqsave_rcu_node(ssp, flags);  in srcu_funnel_exp_start()
613 	if (ULONG_CMP_LT(ssp->srcu_gp_seq_needed_exp, s))  in srcu_funnel_exp_start()
614 		WRITE_ONCE(ssp->srcu_gp_seq_needed_exp, s);  in srcu_funnel_exp_start()
615 	spin_unlock_irqrestore_rcu_node(ssp, flags);  in srcu_funnel_exp_start()
628 static void srcu_funnel_gp_start(struct srcu_struct *ssp, struct srcu_data *sdp,  in srcu_funnel_gp_start()  argument
638 		if (rcu_seq_done(&ssp->srcu_gp_seq, s) && snp != sdp->mynode)  in srcu_funnel_gp_start()
653 				srcu_funnel_exp_start(ssp, snp, s);  in srcu_funnel_gp_start()
665 	spin_lock_irqsave_rcu_node(ssp, flags);  in srcu_funnel_gp_start()
666 	if (ULONG_CMP_LT(ssp->srcu_gp_seq_needed, s)) {  in srcu_funnel_gp_start()
671 		smp_store_release(&ssp->srcu_gp_seq_needed, s); /*^^^*/  in srcu_funnel_gp_start()
673 	if (!do_norm && ULONG_CMP_LT(ssp->srcu_gp_seq_needed_exp, s))  in srcu_funnel_gp_start()
674 		WRITE_ONCE(ssp->srcu_gp_seq_needed_exp, s);  in srcu_funnel_gp_start()
677 	if (!rcu_seq_done(&ssp->srcu_gp_seq, s) &&  in srcu_funnel_gp_start()
678 	    rcu_seq_state(ssp->srcu_gp_seq) == SRCU_STATE_IDLE) {  in srcu_funnel_gp_start()
679 		WARN_ON_ONCE(ULONG_CMP_GE(ssp->srcu_gp_seq, ssp->srcu_gp_seq_needed));  in srcu_funnel_gp_start()
680 		srcu_gp_start(ssp);  in srcu_funnel_gp_start()
682 			queue_delayed_work(rcu_gp_wq, &ssp->work,  in srcu_funnel_gp_start()
683 					   srcu_get_delay(ssp));  in srcu_funnel_gp_start()
684 		else if (list_empty(&ssp->work.work.entry))  in srcu_funnel_gp_start()
685 			list_add(&ssp->work.work.entry, &srcu_boot_list);  in srcu_funnel_gp_start()
687 	spin_unlock_irqrestore_rcu_node(ssp, flags);  in srcu_funnel_gp_start()
695 static bool try_check_zero(struct srcu_struct *ssp, int idx, int trycount)  in try_check_zero()  argument
698 		if (srcu_readers_active_idx_check(ssp, idx))  in try_check_zero()
700 		if (--trycount + !srcu_get_delay(ssp) <= 0)  in try_check_zero()
711 static void srcu_flip(struct srcu_struct *ssp)  in srcu_flip()  argument
723 	WRITE_ONCE(ssp->srcu_idx, ssp->srcu_idx + 1);  in srcu_flip()
756 static bool srcu_might_be_idle(struct srcu_struct *ssp)  in srcu_might_be_idle()  argument
764 	check_init_srcu_struct(ssp);  in srcu_might_be_idle()
766 	sdp = raw_cpu_ptr(ssp->sda);  in srcu_might_be_idle()
782 	tlast = READ_ONCE(ssp->srcu_last_gp_end);  in srcu_might_be_idle()
788 	curseq = rcu_seq_current(&ssp->srcu_gp_seq);  in srcu_might_be_idle()
790 	if (ULONG_CMP_LT(curseq, READ_ONCE(ssp->srcu_gp_seq_needed)))  in srcu_might_be_idle()
793 	if (curseq != rcu_seq_current(&ssp->srcu_gp_seq))  in srcu_might_be_idle()
808 static unsigned long srcu_gp_start_if_needed(struct srcu_struct *ssp,  in srcu_gp_start_if_needed()  argument
818 	check_init_srcu_struct(ssp);  in srcu_gp_start_if_needed()
819 	idx = srcu_read_lock(ssp);  in srcu_gp_start_if_needed()
820 	sdp = raw_cpu_ptr(ssp->sda);  in srcu_gp_start_if_needed()
825 			      rcu_seq_current(&ssp->srcu_gp_seq));  in srcu_gp_start_if_needed()
826 	s = rcu_seq_snap(&ssp->srcu_gp_seq);  in srcu_gp_start_if_needed()
838 		srcu_funnel_gp_start(ssp, sdp, s, do_norm);  in srcu_gp_start_if_needed()
840 		srcu_funnel_exp_start(ssp, sdp->mynode, s);  in srcu_gp_start_if_needed()
841 	srcu_read_unlock(ssp, idx);  in srcu_gp_start_if_needed()
873 static void __call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp,  in __call_srcu()  argument
883 	(void)srcu_gp_start_if_needed(ssp, rhp, do_norm);  in __call_srcu()
888  * @ssp: srcu_struct in queue the callback
903 void call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp,  in call_srcu()  argument
906 	__call_srcu(ssp, rhp, func, true);  in call_srcu()
913 static void __synchronize_srcu(struct srcu_struct *ssp, bool do_norm)  in __synchronize_srcu()  argument
917 	RCU_LOCKDEP_WARN(lockdep_is_held(ssp) ||  in __synchronize_srcu()
926 	check_init_srcu_struct(ssp);  in __synchronize_srcu()
929 	__call_srcu(ssp, &rcu.head, wakeme_after_rcu, do_norm);  in __synchronize_srcu()
945  * @ssp: srcu_struct with which to synchronize.
953 void synchronize_srcu_expedited(struct srcu_struct *ssp)  in synchronize_srcu_expedited()  argument
955 	__synchronize_srcu(ssp, rcu_gp_is_normal());  in synchronize_srcu_expedited()
961  * @ssp: srcu_struct with which to synchronize.
1006 void synchronize_srcu(struct srcu_struct *ssp)  in synchronize_srcu()  argument
1008 	if (srcu_might_be_idle(ssp) || rcu_gp_is_expedited())  in synchronize_srcu()
1009 		synchronize_srcu_expedited(ssp);  in synchronize_srcu()
1011 		__synchronize_srcu(ssp, true);  in synchronize_srcu()
1017  * @ssp: srcu_struct to provide cookie for.
1025 unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp)  in get_state_synchronize_srcu()  argument
1030 	return rcu_seq_snap(&ssp->srcu_gp_seq);  in get_state_synchronize_srcu()
1036  * @ssp: srcu_struct to provide cookie for.
1044 unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp)  in start_poll_synchronize_srcu()  argument
1046 	return srcu_gp_start_if_needed(ssp, NULL, true);  in start_poll_synchronize_srcu()
1052  * @ssp: srcu_struct to provide cookie for.
1075 bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie)  in poll_state_synchronize_srcu()  argument
1077 	if (!rcu_seq_done(&ssp->srcu_gp_seq, cookie))  in poll_state_synchronize_srcu()
1092 	struct srcu_struct *ssp;  in srcu_barrier_cb()  local
1095 	ssp = sdp->ssp;  in srcu_barrier_cb()
1096 	if (atomic_dec_and_test(&ssp->srcu_barrier_cpu_cnt))  in srcu_barrier_cb()
1097 		complete(&ssp->srcu_barrier_completion);  in srcu_barrier_cb()
1102  * @ssp: srcu_struct on which to wait for in-flight callbacks.
1104 void srcu_barrier(struct srcu_struct *ssp)  in srcu_barrier()  argument
1108 	unsigned long s = rcu_seq_snap(&ssp->srcu_barrier_seq);  in srcu_barrier()
1110 	check_init_srcu_struct(ssp);  in srcu_barrier()
1111 	mutex_lock(&ssp->srcu_barrier_mutex);  in srcu_barrier()
1112 	if (rcu_seq_done(&ssp->srcu_barrier_seq, s)) {  in srcu_barrier()
1114 		mutex_unlock(&ssp->srcu_barrier_mutex);  in srcu_barrier()
1117 	rcu_seq_start(&ssp->srcu_barrier_seq);  in srcu_barrier()
1118 	init_completion(&ssp->srcu_barrier_completion);  in srcu_barrier()
1121 	atomic_set(&ssp->srcu_barrier_cpu_cnt, 1);  in srcu_barrier()
1132 		sdp = per_cpu_ptr(ssp->sda, cpu);  in srcu_barrier()
1134 		atomic_inc(&ssp->srcu_barrier_cpu_cnt);  in srcu_barrier()
1140 			atomic_dec(&ssp->srcu_barrier_cpu_cnt);  in srcu_barrier()
1146 	if (atomic_dec_and_test(&ssp->srcu_barrier_cpu_cnt))  in srcu_barrier()
1147 		complete(&ssp->srcu_barrier_completion);  in srcu_barrier()
1148 	wait_for_completion(&ssp->srcu_barrier_completion);  in srcu_barrier()
1150 	rcu_seq_end(&ssp->srcu_barrier_seq);  in srcu_barrier()
1151 	mutex_unlock(&ssp->srcu_barrier_mutex);  in srcu_barrier()
1157  * @ssp: srcu_struct on which to report batch completion.
1162 unsigned long srcu_batches_completed(struct srcu_struct *ssp)  in srcu_batches_completed()  argument
1164 	return READ_ONCE(ssp->srcu_idx);  in srcu_batches_completed()
1173 static void srcu_advance_state(struct srcu_struct *ssp)  in srcu_advance_state()  argument
1177 	mutex_lock(&ssp->srcu_gp_mutex);  in srcu_advance_state()
1189 	idx = rcu_seq_state(smp_load_acquire(&ssp->srcu_gp_seq)); /* ^^^ */  in srcu_advance_state()
1191 		spin_lock_irq_rcu_node(ssp);  in srcu_advance_state()
1192 		if (ULONG_CMP_GE(ssp->srcu_gp_seq, ssp->srcu_gp_seq_needed)) {  in srcu_advance_state()
1193 			WARN_ON_ONCE(rcu_seq_state(ssp->srcu_gp_seq));  in srcu_advance_state()
1194 			spin_unlock_irq_rcu_node(ssp);  in srcu_advance_state()
1195 			mutex_unlock(&ssp->srcu_gp_mutex);  in srcu_advance_state()
1198 		idx = rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq));  in srcu_advance_state()
1200 			srcu_gp_start(ssp);  in srcu_advance_state()
1201 		spin_unlock_irq_rcu_node(ssp);  in srcu_advance_state()
1203 			mutex_unlock(&ssp->srcu_gp_mutex);  in srcu_advance_state()
1208 	if (rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq)) == SRCU_STATE_SCAN1) {  in srcu_advance_state()
1209 		idx = 1 ^ (ssp->srcu_idx & 1);  in srcu_advance_state()
1210 		if (!try_check_zero(ssp, idx, 1)) {  in srcu_advance_state()
1211 			mutex_unlock(&ssp->srcu_gp_mutex);  in srcu_advance_state()
1214 		srcu_flip(ssp);  in srcu_advance_state()
1215 		spin_lock_irq_rcu_node(ssp);  in srcu_advance_state()
1216 		rcu_seq_set_state(&ssp->srcu_gp_seq, SRCU_STATE_SCAN2);  in srcu_advance_state()
1217 		spin_unlock_irq_rcu_node(ssp);  in srcu_advance_state()
1220 	if (rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq)) == SRCU_STATE_SCAN2) {  in srcu_advance_state()
1226 		idx = 1 ^ (ssp->srcu_idx & 1);  in srcu_advance_state()
1227 		if (!try_check_zero(ssp, idx, 2)) {  in srcu_advance_state()
1228 			mutex_unlock(&ssp->srcu_gp_mutex);  in srcu_advance_state()
1231 		srcu_gp_end(ssp);  /* Releases ->srcu_gp_mutex. */  in srcu_advance_state()
1248 	struct srcu_struct *ssp;  in srcu_invoke_callbacks()  local
1252 	ssp = sdp->ssp;  in srcu_invoke_callbacks()
1256 			      rcu_seq_current(&ssp->srcu_gp_seq));  in srcu_invoke_callbacks()
1284 				       rcu_seq_snap(&ssp->srcu_gp_seq));  in srcu_invoke_callbacks()
1296 static void srcu_reschedule(struct srcu_struct *ssp, unsigned long delay)  in srcu_reschedule()  argument
1300 	spin_lock_irq_rcu_node(ssp);  in srcu_reschedule()
1301 	if (ULONG_CMP_GE(ssp->srcu_gp_seq, ssp->srcu_gp_seq_needed)) {  in srcu_reschedule()
1302 		if (!WARN_ON_ONCE(rcu_seq_state(ssp->srcu_gp_seq))) {  in srcu_reschedule()
1306 	} else if (!rcu_seq_state(ssp->srcu_gp_seq)) {  in srcu_reschedule()
1308 		srcu_gp_start(ssp);  in srcu_reschedule()
1310 	spin_unlock_irq_rcu_node(ssp);  in srcu_reschedule()
1313 		queue_delayed_work(rcu_gp_wq, &ssp->work, delay);  in srcu_reschedule()
1321 	struct srcu_struct *ssp;  in process_srcu()  local
1323 	ssp = container_of(work, struct srcu_struct, work.work);  in process_srcu()
1325 	srcu_advance_state(ssp);  in process_srcu()
1326 	srcu_reschedule(ssp, srcu_get_delay(ssp));  in process_srcu()
1330 			     struct srcu_struct *ssp, int *flags,  in srcutorture_get_gp_data()  argument
1336 	*gp_seq = rcu_seq_current(&ssp->srcu_gp_seq);  in srcutorture_get_gp_data()
1340 void srcu_torture_stats_print(struct srcu_struct *ssp, char *tt, char *tf)  in srcu_torture_stats_print()  argument
1346 	idx = ssp->srcu_idx & 0x1;  in srcu_torture_stats_print()
1348 		 tt, tf, rcu_seq_current(&ssp->srcu_gp_seq), idx);  in srcu_torture_stats_print()
1355 		sdp = per_cpu_ptr(ssp->sda, cpu);  in srcu_torture_stats_print()
1391 	struct srcu_struct *ssp;  in srcu_init()  local
1400 		ssp = list_first_entry(&srcu_boot_list, struct srcu_struct,  in srcu_init()
1402 		list_del_init(&ssp->work.work.entry);  in srcu_init()
1403 		queue_work(rcu_gp_wq, &ssp->work.work);  in srcu_init()