Lines Matching full:ssp
76 static void srcu_reschedule(struct srcu_struct *ssp, unsigned long delay);
122 static void init_srcu_struct_data(struct srcu_struct *ssp) in init_srcu_struct_data() argument
134 sdp = per_cpu_ptr(ssp->sda, cpu); in init_srcu_struct_data()
138 sdp->srcu_gp_seq_needed = ssp->srcu_gp_seq; in init_srcu_struct_data()
139 sdp->srcu_gp_seq_needed_exp = ssp->srcu_gp_seq; in init_srcu_struct_data()
144 sdp->ssp = ssp; in init_srcu_struct_data()
164 static bool init_srcu_struct_nodes(struct srcu_struct *ssp, gfp_t gfp_flags) in init_srcu_struct_nodes() argument
176 ssp->node = kcalloc(rcu_num_nodes, sizeof(*ssp->node), gfp_flags); in init_srcu_struct_nodes()
177 if (!ssp->node) in init_srcu_struct_nodes()
181 ssp->level[0] = &ssp->node[0]; in init_srcu_struct_nodes()
183 ssp->level[i] = ssp->level[i - 1] + num_rcu_lvl[i - 1]; in init_srcu_struct_nodes()
187 srcu_for_each_node_breadth_first(ssp, snp) { in init_srcu_struct_nodes()
198 if (snp == &ssp->node[0]) { in init_srcu_struct_nodes()
205 if (snp == ssp->level[level + 1]) in init_srcu_struct_nodes()
207 snp->srcu_parent = ssp->level[level - 1] + in init_srcu_struct_nodes()
208 (snp - ssp->level[level]) / in init_srcu_struct_nodes()
217 snp_first = ssp->level[level]; in init_srcu_struct_nodes()
219 sdp = per_cpu_ptr(ssp->sda, cpu); in init_srcu_struct_nodes()
228 smp_store_release(&ssp->srcu_size_state, SRCU_SIZE_WAIT_BARRIER); in init_srcu_struct_nodes()
237 static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static) in init_srcu_struct_fields() argument
239 ssp->srcu_size_state = SRCU_SIZE_SMALL; in init_srcu_struct_fields()
240 ssp->node = NULL; in init_srcu_struct_fields()
241 mutex_init(&ssp->srcu_cb_mutex); in init_srcu_struct_fields()
242 mutex_init(&ssp->srcu_gp_mutex); in init_srcu_struct_fields()
243 ssp->srcu_idx = 0; in init_srcu_struct_fields()
244 ssp->srcu_gp_seq = 0; in init_srcu_struct_fields()
245 ssp->srcu_barrier_seq = 0; in init_srcu_struct_fields()
246 mutex_init(&ssp->srcu_barrier_mutex); in init_srcu_struct_fields()
247 atomic_set(&ssp->srcu_barrier_cpu_cnt, 0); in init_srcu_struct_fields()
248 INIT_DELAYED_WORK(&ssp->work, process_srcu); in init_srcu_struct_fields()
249 ssp->sda_is_static = is_static; in init_srcu_struct_fields()
251 ssp->sda = alloc_percpu(struct srcu_data); in init_srcu_struct_fields()
252 if (!ssp->sda) in init_srcu_struct_fields()
254 init_srcu_struct_data(ssp); in init_srcu_struct_fields()
255 ssp->srcu_gp_seq_needed_exp = 0; in init_srcu_struct_fields()
256 ssp->srcu_last_gp_end = ktime_get_mono_fast_ns(); in init_srcu_struct_fields()
257 if (READ_ONCE(ssp->srcu_size_state) == SRCU_SIZE_SMALL && SRCU_SIZING_IS_INIT()) { in init_srcu_struct_fields()
258 if (!init_srcu_struct_nodes(ssp, GFP_ATOMIC)) { in init_srcu_struct_fields()
259 if (!ssp->sda_is_static) { in init_srcu_struct_fields()
260 free_percpu(ssp->sda); in init_srcu_struct_fields()
261 ssp->sda = NULL; in init_srcu_struct_fields()
265 WRITE_ONCE(ssp->srcu_size_state, SRCU_SIZE_BIG); in init_srcu_struct_fields()
268 smp_store_release(&ssp->srcu_gp_seq_needed, 0); /* Init done. */ in init_srcu_struct_fields()
274 int __init_srcu_struct(struct srcu_struct *ssp, const char *name, in __init_srcu_struct() argument
278 debug_check_no_locks_freed((void *)ssp, sizeof(*ssp)); in __init_srcu_struct()
279 lockdep_init_map(&ssp->dep_map, name, key, 0); in __init_srcu_struct()
280 spin_lock_init(&ACCESS_PRIVATE(ssp, lock)); in __init_srcu_struct()
281 return init_srcu_struct_fields(ssp, false); in __init_srcu_struct()
289 * @ssp: structure to initialize.
295 int init_srcu_struct(struct srcu_struct *ssp) in init_srcu_struct() argument
297 spin_lock_init(&ACCESS_PRIVATE(ssp, lock)); in init_srcu_struct()
298 return init_srcu_struct_fields(ssp, false); in init_srcu_struct()
307 static void __srcu_transition_to_big(struct srcu_struct *ssp) in __srcu_transition_to_big() argument
309 lockdep_assert_held(&ACCESS_PRIVATE(ssp, lock)); in __srcu_transition_to_big()
310 smp_store_release(&ssp->srcu_size_state, SRCU_SIZE_ALLOC); in __srcu_transition_to_big()
316 static void srcu_transition_to_big(struct srcu_struct *ssp) in srcu_transition_to_big() argument
321 if (smp_load_acquire(&ssp->srcu_size_state) != SRCU_SIZE_SMALL) in srcu_transition_to_big()
323 spin_lock_irqsave_rcu_node(ssp, flags); in srcu_transition_to_big()
324 if (smp_load_acquire(&ssp->srcu_size_state) != SRCU_SIZE_SMALL) { in srcu_transition_to_big()
325 spin_unlock_irqrestore_rcu_node(ssp, flags); in srcu_transition_to_big()
328 __srcu_transition_to_big(ssp); in srcu_transition_to_big()
329 spin_unlock_irqrestore_rcu_node(ssp, flags); in srcu_transition_to_big()
336 static void spin_lock_irqsave_check_contention(struct srcu_struct *ssp) in spin_lock_irqsave_check_contention() argument
340 if (!SRCU_SIZING_IS_CONTEND() || ssp->srcu_size_state) in spin_lock_irqsave_check_contention()
343 if (ssp->srcu_size_jiffies != j) { in spin_lock_irqsave_check_contention()
344 ssp->srcu_size_jiffies = j; in spin_lock_irqsave_check_contention()
345 ssp->srcu_n_lock_retries = 0; in spin_lock_irqsave_check_contention()
347 if (++ssp->srcu_n_lock_retries <= small_contention_lim) in spin_lock_irqsave_check_contention()
349 __srcu_transition_to_big(ssp); in spin_lock_irqsave_check_contention()
360 struct srcu_struct *ssp = sdp->ssp; in spin_lock_irqsave_sdp_contention() local
364 spin_lock_irqsave_rcu_node(ssp, *flags); in spin_lock_irqsave_sdp_contention()
365 spin_lock_irqsave_check_contention(ssp); in spin_lock_irqsave_sdp_contention()
366 spin_unlock_irqrestore_rcu_node(ssp, *flags); in spin_lock_irqsave_sdp_contention()
376 static void spin_lock_irqsave_ssp_contention(struct srcu_struct *ssp, unsigned long *flags) in spin_lock_irqsave_ssp_contention() argument
378 if (spin_trylock_irqsave_rcu_node(ssp, *flags)) in spin_lock_irqsave_ssp_contention()
380 spin_lock_irqsave_rcu_node(ssp, *flags); in spin_lock_irqsave_ssp_contention()
381 spin_lock_irqsave_check_contention(ssp); in spin_lock_irqsave_ssp_contention()
388 * to each update-side SRCU primitive. Use ssp->lock, which -is-
392 static void check_init_srcu_struct(struct srcu_struct *ssp) in check_init_srcu_struct() argument
397 if (!rcu_seq_state(smp_load_acquire(&ssp->srcu_gp_seq_needed))) /*^^^*/ in check_init_srcu_struct()
399 spin_lock_irqsave_rcu_node(ssp, flags); in check_init_srcu_struct()
400 if (!rcu_seq_state(ssp->srcu_gp_seq_needed)) { in check_init_srcu_struct()
401 spin_unlock_irqrestore_rcu_node(ssp, flags); in check_init_srcu_struct()
404 init_srcu_struct_fields(ssp, true); in check_init_srcu_struct()
405 spin_unlock_irqrestore_rcu_node(ssp, flags); in check_init_srcu_struct()
412 static unsigned long srcu_readers_lock_idx(struct srcu_struct *ssp, int idx) in srcu_readers_lock_idx() argument
418 struct srcu_data *cpuc = per_cpu_ptr(ssp->sda, cpu); in srcu_readers_lock_idx()
429 static unsigned long srcu_readers_unlock_idx(struct srcu_struct *ssp, int idx) in srcu_readers_unlock_idx() argument
435 struct srcu_data *cpuc = per_cpu_ptr(ssp->sda, cpu); in srcu_readers_unlock_idx()
446 static bool srcu_readers_active_idx_check(struct srcu_struct *ssp, int idx) in srcu_readers_active_idx_check() argument
450 unlocks = srcu_readers_unlock_idx(ssp, idx); in srcu_readers_active_idx_check()
486 return srcu_readers_lock_idx(ssp, idx) == unlocks; in srcu_readers_active_idx_check()
492 * @ssp: which srcu_struct to count active readers (holding srcu_read_lock).
498 static bool srcu_readers_active(struct srcu_struct *ssp) in srcu_readers_active() argument
504 struct srcu_data *cpuc = per_cpu_ptr(ssp->sda, cpu); in srcu_readers_active()
565 static unsigned long srcu_get_delay(struct srcu_struct *ssp) in srcu_get_delay() argument
571 if (ULONG_CMP_LT(READ_ONCE(ssp->srcu_gp_seq), READ_ONCE(ssp->srcu_gp_seq_needed_exp))) in srcu_get_delay()
573 if (rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq))) { in srcu_get_delay()
575 gpstart = READ_ONCE(ssp->srcu_gp_start); in srcu_get_delay()
579 WRITE_ONCE(ssp->srcu_n_exp_nodelay, READ_ONCE(ssp->srcu_n_exp_nodelay) + 1); in srcu_get_delay()
580 if (READ_ONCE(ssp->srcu_n_exp_nodelay) > srcu_max_nodelay_phase) in srcu_get_delay()
589 * @ssp: structure to clean up.
594 void cleanup_srcu_struct(struct srcu_struct *ssp) in cleanup_srcu_struct() argument
598 if (WARN_ON(!srcu_get_delay(ssp))) in cleanup_srcu_struct()
600 if (WARN_ON(srcu_readers_active(ssp))) in cleanup_srcu_struct()
602 flush_delayed_work(&ssp->work); in cleanup_srcu_struct()
604 struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu); in cleanup_srcu_struct()
611 if (WARN_ON(rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq)) != SRCU_STATE_IDLE) || in cleanup_srcu_struct()
612 WARN_ON(rcu_seq_current(&ssp->srcu_gp_seq) != ssp->srcu_gp_seq_needed) || in cleanup_srcu_struct()
613 WARN_ON(srcu_readers_active(ssp))) { in cleanup_srcu_struct()
615 __func__, ssp, rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq)), in cleanup_srcu_struct()
616 rcu_seq_current(&ssp->srcu_gp_seq), ssp->srcu_gp_seq_needed); in cleanup_srcu_struct()
619 if (!ssp->sda_is_static) { in cleanup_srcu_struct()
620 free_percpu(ssp->sda); in cleanup_srcu_struct()
621 ssp->sda = NULL; in cleanup_srcu_struct()
623 kfree(ssp->node); in cleanup_srcu_struct()
624 ssp->node = NULL; in cleanup_srcu_struct()
625 ssp->srcu_size_state = SRCU_SIZE_SMALL; in cleanup_srcu_struct()
634 int __srcu_read_lock(struct srcu_struct *ssp) in __srcu_read_lock() argument
638 idx = READ_ONCE(ssp->srcu_idx) & 0x1; in __srcu_read_lock()
639 this_cpu_inc(ssp->sda->srcu_lock_count[idx]); in __srcu_read_lock()
650 void __srcu_read_unlock(struct srcu_struct *ssp, int idx) in __srcu_read_unlock() argument
653 this_cpu_inc(ssp->sda->srcu_unlock_count[idx]); in __srcu_read_unlock()
660 static void srcu_gp_start(struct srcu_struct *ssp) in srcu_gp_start() argument
665 if (smp_load_acquire(&ssp->srcu_size_state) < SRCU_SIZE_WAIT_BARRIER) in srcu_gp_start()
666 sdp = per_cpu_ptr(ssp->sda, 0); in srcu_gp_start()
668 sdp = this_cpu_ptr(ssp->sda); in srcu_gp_start()
669 lockdep_assert_held(&ACCESS_PRIVATE(ssp, lock)); in srcu_gp_start()
670 WARN_ON_ONCE(ULONG_CMP_GE(ssp->srcu_gp_seq, ssp->srcu_gp_seq_needed)); in srcu_gp_start()
673 rcu_seq_current(&ssp->srcu_gp_seq)); in srcu_gp_start()
675 rcu_seq_snap(&ssp->srcu_gp_seq)); in srcu_gp_start()
677 WRITE_ONCE(ssp->srcu_gp_start, jiffies); in srcu_gp_start()
678 WRITE_ONCE(ssp->srcu_n_exp_nodelay, 0); in srcu_gp_start()
680 rcu_seq_start(&ssp->srcu_gp_seq); in srcu_gp_start()
681 state = rcu_seq_state(ssp->srcu_gp_seq); in srcu_gp_start()
719 static void srcu_schedule_cbs_snp(struct srcu_struct *ssp, struct srcu_node *snp, in srcu_schedule_cbs_snp() argument
727 srcu_schedule_cbs_sdp(per_cpu_ptr(ssp->sda, cpu), delay); in srcu_schedule_cbs_snp()
740 static void srcu_gp_end(struct srcu_struct *ssp) in srcu_gp_end() argument
756 mutex_lock(&ssp->srcu_cb_mutex); in srcu_gp_end()
759 spin_lock_irq_rcu_node(ssp); in srcu_gp_end()
760 idx = rcu_seq_state(ssp->srcu_gp_seq); in srcu_gp_end()
762 if (ULONG_CMP_LT(READ_ONCE(ssp->srcu_gp_seq), READ_ONCE(ssp->srcu_gp_seq_needed_exp))) in srcu_gp_end()
765 WRITE_ONCE(ssp->srcu_last_gp_end, ktime_get_mono_fast_ns()); in srcu_gp_end()
766 rcu_seq_end(&ssp->srcu_gp_seq); in srcu_gp_end()
767 gpseq = rcu_seq_current(&ssp->srcu_gp_seq); in srcu_gp_end()
768 if (ULONG_CMP_LT(ssp->srcu_gp_seq_needed_exp, gpseq)) in srcu_gp_end()
769 WRITE_ONCE(ssp->srcu_gp_seq_needed_exp, gpseq); in srcu_gp_end()
770 spin_unlock_irq_rcu_node(ssp); in srcu_gp_end()
771 mutex_unlock(&ssp->srcu_gp_mutex); in srcu_gp_end()
775 ss_state = smp_load_acquire(&ssp->srcu_size_state); in srcu_gp_end()
777 srcu_schedule_cbs_sdp(per_cpu_ptr(ssp->sda, 0), cbdelay); in srcu_gp_end()
780 srcu_for_each_node_breadth_first(ssp, snp) { in srcu_gp_end()
783 last_lvl = snp >= ssp->level[rcu_num_lvls - 1]; in srcu_gp_end()
798 srcu_schedule_cbs_snp(ssp, snp, mask, cbdelay); in srcu_gp_end()
805 sdp = per_cpu_ptr(ssp->sda, cpu); in srcu_gp_end()
815 mutex_unlock(&ssp->srcu_cb_mutex); in srcu_gp_end()
818 spin_lock_irq_rcu_node(ssp); in srcu_gp_end()
819 gpseq = rcu_seq_current(&ssp->srcu_gp_seq); in srcu_gp_end()
821 ULONG_CMP_LT(gpseq, ssp->srcu_gp_seq_needed)) { in srcu_gp_end()
822 srcu_gp_start(ssp); in srcu_gp_end()
823 spin_unlock_irq_rcu_node(ssp); in srcu_gp_end()
824 srcu_reschedule(ssp, 0); in srcu_gp_end()
826 spin_unlock_irq_rcu_node(ssp); in srcu_gp_end()
832 init_srcu_struct_nodes(ssp, GFP_KERNEL); in srcu_gp_end()
834 smp_store_release(&ssp->srcu_size_state, ss_state + 1); in srcu_gp_end()
845 static void srcu_funnel_exp_start(struct srcu_struct *ssp, struct srcu_node *snp, in srcu_funnel_exp_start() argument
854 if (rcu_seq_done(&ssp->srcu_gp_seq, s) || in srcu_funnel_exp_start()
866 spin_lock_irqsave_ssp_contention(ssp, &flags); in srcu_funnel_exp_start()
867 if (ULONG_CMP_LT(ssp->srcu_gp_seq_needed_exp, s)) in srcu_funnel_exp_start()
868 WRITE_ONCE(ssp->srcu_gp_seq_needed_exp, s); in srcu_funnel_exp_start()
869 spin_unlock_irqrestore_rcu_node(ssp, flags); in srcu_funnel_exp_start()
882 static void srcu_funnel_gp_start(struct srcu_struct *ssp, struct srcu_data *sdp, in srcu_funnel_gp_start() argument
893 if (smp_load_acquire(&ssp->srcu_size_state) < SRCU_SIZE_WAIT_BARRIER) in srcu_funnel_gp_start()
901 if (rcu_seq_done(&ssp->srcu_gp_seq, s) && snp != snp_leaf) in srcu_funnel_gp_start()
914 srcu_funnel_exp_start(ssp, snp, s); in srcu_funnel_gp_start()
927 spin_lock_irqsave_ssp_contention(ssp, &flags); in srcu_funnel_gp_start()
928 if (ULONG_CMP_LT(ssp->srcu_gp_seq_needed, s)) { in srcu_funnel_gp_start()
933 smp_store_release(&ssp->srcu_gp_seq_needed, s); /*^^^*/ in srcu_funnel_gp_start()
935 if (!do_norm && ULONG_CMP_LT(ssp->srcu_gp_seq_needed_exp, s)) in srcu_funnel_gp_start()
936 WRITE_ONCE(ssp->srcu_gp_seq_needed_exp, s); in srcu_funnel_gp_start()
939 if (!rcu_seq_done(&ssp->srcu_gp_seq, s) && in srcu_funnel_gp_start()
940 rcu_seq_state(ssp->srcu_gp_seq) == SRCU_STATE_IDLE) { in srcu_funnel_gp_start()
941 WARN_ON_ONCE(ULONG_CMP_GE(ssp->srcu_gp_seq, ssp->srcu_gp_seq_needed)); in srcu_funnel_gp_start()
942 srcu_gp_start(ssp); in srcu_funnel_gp_start()
950 queue_delayed_work(rcu_gp_wq, &ssp->work, in srcu_funnel_gp_start()
951 !!srcu_get_delay(ssp)); in srcu_funnel_gp_start()
952 else if (list_empty(&ssp->work.work.entry)) in srcu_funnel_gp_start()
953 list_add(&ssp->work.work.entry, &srcu_boot_list); in srcu_funnel_gp_start()
955 spin_unlock_irqrestore_rcu_node(ssp, flags); in srcu_funnel_gp_start()
963 static bool try_check_zero(struct srcu_struct *ssp, int idx, int trycount) in try_check_zero() argument
967 curdelay = !srcu_get_delay(ssp); in try_check_zero()
970 if (srcu_readers_active_idx_check(ssp, idx)) in try_check_zero()
983 static void srcu_flip(struct srcu_struct *ssp) in srcu_flip() argument
995 WRITE_ONCE(ssp->srcu_idx, ssp->srcu_idx + 1); in srcu_flip()
1028 static bool srcu_might_be_idle(struct srcu_struct *ssp) in srcu_might_be_idle() argument
1036 check_init_srcu_struct(ssp); in srcu_might_be_idle()
1038 sdp = raw_cpu_ptr(ssp->sda); in srcu_might_be_idle()
1054 tlast = READ_ONCE(ssp->srcu_last_gp_end); in srcu_might_be_idle()
1060 curseq = rcu_seq_current(&ssp->srcu_gp_seq); in srcu_might_be_idle()
1062 if (ULONG_CMP_LT(curseq, READ_ONCE(ssp->srcu_gp_seq_needed))) in srcu_might_be_idle()
1065 if (curseq != rcu_seq_current(&ssp->srcu_gp_seq)) in srcu_might_be_idle()
1080 static unsigned long srcu_gp_start_if_needed(struct srcu_struct *ssp, in srcu_gp_start_if_needed() argument
1092 check_init_srcu_struct(ssp); in srcu_gp_start_if_needed()
1093 idx = srcu_read_lock(ssp); in srcu_gp_start_if_needed()
1094 ss_state = smp_load_acquire(&ssp->srcu_size_state); in srcu_gp_start_if_needed()
1096 sdp = per_cpu_ptr(ssp->sda, 0); in srcu_gp_start_if_needed()
1098 sdp = raw_cpu_ptr(ssp->sda); in srcu_gp_start_if_needed()
1103 rcu_seq_current(&ssp->srcu_gp_seq)); in srcu_gp_start_if_needed()
1104 s = rcu_seq_snap(&ssp->srcu_gp_seq); in srcu_gp_start_if_needed()
1123 srcu_funnel_gp_start(ssp, sdp, s, do_norm); in srcu_gp_start_if_needed()
1125 srcu_funnel_exp_start(ssp, sdp_mynode, s); in srcu_gp_start_if_needed()
1126 srcu_read_unlock(ssp, idx); in srcu_gp_start_if_needed()
1158 static void __call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, in __call_srcu() argument
1168 (void)srcu_gp_start_if_needed(ssp, rhp, do_norm); in __call_srcu()
1173 * @ssp: srcu_struct in queue the callback
1188 void call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, in call_srcu() argument
1191 __call_srcu(ssp, rhp, func, true); in call_srcu()
1198 static void __synchronize_srcu(struct srcu_struct *ssp, bool do_norm) in __synchronize_srcu() argument
1202 RCU_LOCKDEP_WARN(lockdep_is_held(ssp) || in __synchronize_srcu()
1211 check_init_srcu_struct(ssp); in __synchronize_srcu()
1214 __call_srcu(ssp, &rcu.head, wakeme_after_rcu, do_norm); in __synchronize_srcu()
1230 * @ssp: srcu_struct with which to synchronize.
1238 void synchronize_srcu_expedited(struct srcu_struct *ssp) in synchronize_srcu_expedited() argument
1240 __synchronize_srcu(ssp, rcu_gp_is_normal()); in synchronize_srcu_expedited()
1246 * @ssp: srcu_struct with which to synchronize.
1291 void synchronize_srcu(struct srcu_struct *ssp) in synchronize_srcu() argument
1293 if (srcu_might_be_idle(ssp) || rcu_gp_is_expedited()) in synchronize_srcu()
1294 synchronize_srcu_expedited(ssp); in synchronize_srcu()
1296 __synchronize_srcu(ssp, true); in synchronize_srcu()
1302 * @ssp: srcu_struct to provide cookie for.
1310 unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp) in get_state_synchronize_srcu() argument
1315 return rcu_seq_snap(&ssp->srcu_gp_seq); in get_state_synchronize_srcu()
1321 * @ssp: srcu_struct to provide cookie for.
1329 unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp) in start_poll_synchronize_srcu() argument
1331 return srcu_gp_start_if_needed(ssp, NULL, true); in start_poll_synchronize_srcu()
1337 * @ssp: srcu_struct to provide cookie for.
1360 bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie) in poll_state_synchronize_srcu() argument
1362 if (!rcu_seq_done(&ssp->srcu_gp_seq, cookie)) in poll_state_synchronize_srcu()
1377 struct srcu_struct *ssp; in srcu_barrier_cb() local
1380 ssp = sdp->ssp; in srcu_barrier_cb()
1381 if (atomic_dec_and_test(&ssp->srcu_barrier_cpu_cnt)) in srcu_barrier_cb()
1382 complete(&ssp->srcu_barrier_completion); in srcu_barrier_cb()
1393 static void srcu_barrier_one_cpu(struct srcu_struct *ssp, struct srcu_data *sdp) in srcu_barrier_one_cpu() argument
1396 atomic_inc(&ssp->srcu_barrier_cpu_cnt); in srcu_barrier_one_cpu()
1402 atomic_dec(&ssp->srcu_barrier_cpu_cnt); in srcu_barrier_one_cpu()
1409 * @ssp: srcu_struct on which to wait for in-flight callbacks.
1411 void srcu_barrier(struct srcu_struct *ssp) in srcu_barrier() argument
1415 unsigned long s = rcu_seq_snap(&ssp->srcu_barrier_seq); in srcu_barrier()
1417 check_init_srcu_struct(ssp); in srcu_barrier()
1418 mutex_lock(&ssp->srcu_barrier_mutex); in srcu_barrier()
1419 if (rcu_seq_done(&ssp->srcu_barrier_seq, s)) { in srcu_barrier()
1421 mutex_unlock(&ssp->srcu_barrier_mutex); in srcu_barrier()
1424 rcu_seq_start(&ssp->srcu_barrier_seq); in srcu_barrier()
1425 init_completion(&ssp->srcu_barrier_completion); in srcu_barrier()
1428 atomic_set(&ssp->srcu_barrier_cpu_cnt, 1); in srcu_barrier()
1430 idx = srcu_read_lock(ssp); in srcu_barrier()
1431 if (smp_load_acquire(&ssp->srcu_size_state) < SRCU_SIZE_WAIT_BARRIER) in srcu_barrier()
1432 srcu_barrier_one_cpu(ssp, per_cpu_ptr(ssp->sda, 0)); in srcu_barrier()
1435 srcu_barrier_one_cpu(ssp, per_cpu_ptr(ssp->sda, cpu)); in srcu_barrier()
1436 srcu_read_unlock(ssp, idx); in srcu_barrier()
1439 if (atomic_dec_and_test(&ssp->srcu_barrier_cpu_cnt)) in srcu_barrier()
1440 complete(&ssp->srcu_barrier_completion); in srcu_barrier()
1441 wait_for_completion(&ssp->srcu_barrier_completion); in srcu_barrier()
1443 rcu_seq_end(&ssp->srcu_barrier_seq); in srcu_barrier()
1444 mutex_unlock(&ssp->srcu_barrier_mutex); in srcu_barrier()
1450 * @ssp: srcu_struct on which to report batch completion.
1455 unsigned long srcu_batches_completed(struct srcu_struct *ssp) in srcu_batches_completed() argument
1457 return READ_ONCE(ssp->srcu_idx); in srcu_batches_completed()
1466 static void srcu_advance_state(struct srcu_struct *ssp) in srcu_advance_state() argument
1470 mutex_lock(&ssp->srcu_gp_mutex); in srcu_advance_state()
1482 idx = rcu_seq_state(smp_load_acquire(&ssp->srcu_gp_seq)); /* ^^^ */ in srcu_advance_state()
1484 spin_lock_irq_rcu_node(ssp); in srcu_advance_state()
1485 if (ULONG_CMP_GE(ssp->srcu_gp_seq, ssp->srcu_gp_seq_needed)) { in srcu_advance_state()
1486 WARN_ON_ONCE(rcu_seq_state(ssp->srcu_gp_seq)); in srcu_advance_state()
1487 spin_unlock_irq_rcu_node(ssp); in srcu_advance_state()
1488 mutex_unlock(&ssp->srcu_gp_mutex); in srcu_advance_state()
1491 idx = rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq)); in srcu_advance_state()
1493 srcu_gp_start(ssp); in srcu_advance_state()
1494 spin_unlock_irq_rcu_node(ssp); in srcu_advance_state()
1496 mutex_unlock(&ssp->srcu_gp_mutex); in srcu_advance_state()
1501 if (rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq)) == SRCU_STATE_SCAN1) { in srcu_advance_state()
1502 idx = 1 ^ (ssp->srcu_idx & 1); in srcu_advance_state()
1503 if (!try_check_zero(ssp, idx, 1)) { in srcu_advance_state()
1504 mutex_unlock(&ssp->srcu_gp_mutex); in srcu_advance_state()
1507 srcu_flip(ssp); in srcu_advance_state()
1508 spin_lock_irq_rcu_node(ssp); in srcu_advance_state()
1509 rcu_seq_set_state(&ssp->srcu_gp_seq, SRCU_STATE_SCAN2); in srcu_advance_state()
1510 ssp->srcu_n_exp_nodelay = 0; in srcu_advance_state()
1511 spin_unlock_irq_rcu_node(ssp); in srcu_advance_state()
1514 if (rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq)) == SRCU_STATE_SCAN2) { in srcu_advance_state()
1520 idx = 1 ^ (ssp->srcu_idx & 1); in srcu_advance_state()
1521 if (!try_check_zero(ssp, idx, 2)) { in srcu_advance_state()
1522 mutex_unlock(&ssp->srcu_gp_mutex); in srcu_advance_state()
1525 ssp->srcu_n_exp_nodelay = 0; in srcu_advance_state()
1526 srcu_gp_end(ssp); /* Releases ->srcu_gp_mutex. */ in srcu_advance_state()
1543 struct srcu_struct *ssp; in srcu_invoke_callbacks() local
1547 ssp = sdp->ssp; in srcu_invoke_callbacks()
1551 rcu_seq_current(&ssp->srcu_gp_seq)); in srcu_invoke_callbacks()
1579 rcu_seq_snap(&ssp->srcu_gp_seq)); in srcu_invoke_callbacks()
1591 static void srcu_reschedule(struct srcu_struct *ssp, unsigned long delay) in srcu_reschedule() argument
1595 spin_lock_irq_rcu_node(ssp); in srcu_reschedule()
1596 if (ULONG_CMP_GE(ssp->srcu_gp_seq, ssp->srcu_gp_seq_needed)) { in srcu_reschedule()
1597 if (!WARN_ON_ONCE(rcu_seq_state(ssp->srcu_gp_seq))) { in srcu_reschedule()
1601 } else if (!rcu_seq_state(ssp->srcu_gp_seq)) { in srcu_reschedule()
1603 srcu_gp_start(ssp); in srcu_reschedule()
1605 spin_unlock_irq_rcu_node(ssp); in srcu_reschedule()
1608 queue_delayed_work(rcu_gp_wq, &ssp->work, delay); in srcu_reschedule()
1618 struct srcu_struct *ssp; in process_srcu() local
1620 ssp = container_of(work, struct srcu_struct, work.work); in process_srcu()
1622 srcu_advance_state(ssp); in process_srcu()
1623 curdelay = srcu_get_delay(ssp); in process_srcu()
1625 WRITE_ONCE(ssp->reschedule_count, 0); in process_srcu()
1628 if (READ_ONCE(ssp->reschedule_jiffies) == j) { in process_srcu()
1629 WRITE_ONCE(ssp->reschedule_count, READ_ONCE(ssp->reschedule_count) + 1); in process_srcu()
1630 if (READ_ONCE(ssp->reschedule_count) > srcu_max_nodelay) in process_srcu()
1633 WRITE_ONCE(ssp->reschedule_count, 1); in process_srcu()
1634 WRITE_ONCE(ssp->reschedule_jiffies, j); in process_srcu()
1637 srcu_reschedule(ssp, curdelay); in process_srcu()
1641 struct srcu_struct *ssp, int *flags, in srcutorture_get_gp_data() argument
1647 *gp_seq = rcu_seq_current(&ssp->srcu_gp_seq); in srcutorture_get_gp_data()
1664 void srcu_torture_stats_print(struct srcu_struct *ssp, char *tt, char *tf) in srcu_torture_stats_print() argument
1669 int ss_state = READ_ONCE(ssp->srcu_size_state); in srcu_torture_stats_print()
1672 idx = ssp->srcu_idx & 0x1; in srcu_torture_stats_print()
1676 tt, tf, rcu_seq_current(&ssp->srcu_gp_seq), ss_state, in srcu_torture_stats_print()
1678 if (!ssp->sda) { in srcu_torture_stats_print()
1689 sdp = per_cpu_ptr(ssp->sda, cpu); in srcu_torture_stats_print()
1713 srcu_transition_to_big(ssp); in srcu_torture_stats_print()
1733 struct srcu_struct *ssp; in srcu_init() local
1753 ssp = list_first_entry(&srcu_boot_list, struct srcu_struct, in srcu_init()
1755 list_del_init(&ssp->work.work.entry); in srcu_init()
1756 if (SRCU_SIZING_IS(SRCU_SIZING_INIT) && ssp->srcu_size_state == SRCU_SIZE_SMALL) in srcu_init()
1757 ssp->srcu_size_state = SRCU_SIZE_ALLOC; in srcu_init()
1758 queue_work(rcu_gp_wq, &ssp->work.work); in srcu_init()