Lines Matching refs:rdp
134 static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp) in rcu_preempt_ctxt_queue() argument
139 (rnp->qsmask & rdp->grpmask ? RCU_GP_BLKD : 0) + in rcu_preempt_ctxt_queue()
140 (rnp->expmask & rdp->grpmask ? RCU_EXP_BLKD : 0); in rcu_preempt_ctxt_queue()
144 WARN_ON_ONCE(rdp->mynode != rnp); in rcu_preempt_ctxt_queue()
148 rdp->grpmask); in rcu_preempt_ctxt_queue()
233 !(rnp->qsmask & rdp->grpmask)); in rcu_preempt_ctxt_queue()
235 !(rnp->expmask & rdp->grpmask)); in rcu_preempt_ctxt_queue()
244 if (blkd_state & RCU_EXP_BLKD && rdp->exp_deferred_qs) in rcu_preempt_ctxt_queue()
245 rcu_report_exp_rdp(rdp); in rcu_preempt_ctxt_queue()
247 WARN_ON_ONCE(rdp->exp_deferred_qs); in rcu_preempt_ctxt_queue()
292 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_note_context_switch() local
302 rnp = rdp->mynode; in rcu_note_context_switch()
312 WARN_ON_ONCE((rdp->grpmask & rcu_rnp_online_cpus(rnp)) == 0); in rcu_note_context_switch()
316 (rnp->qsmask & rdp->grpmask) in rcu_note_context_switch()
319 rcu_preempt_ctxt_queue(rnp, rdp); in rcu_note_context_switch()
334 if (rdp->exp_deferred_qs) in rcu_note_context_switch()
335 rcu_report_exp_rdp(rdp); in rcu_note_context_switch()
446 struct rcu_data *rdp; in rcu_preempt_deferred_qs_irqrestore() local
456 rdp = this_cpu_ptr(&rcu_data); in rcu_preempt_deferred_qs_irqrestore()
457 if (!special.s && !rdp->exp_deferred_qs) { in rcu_preempt_deferred_qs_irqrestore()
464 rcu_report_qs_rdp(rdp); in rcu_preempt_deferred_qs_irqrestore()
477 if (rdp->exp_deferred_qs) in rcu_preempt_deferred_qs_irqrestore()
478 rcu_report_exp_rdp(rdp); in rcu_preempt_deferred_qs_irqrestore()
587 struct rcu_data *rdp; in rcu_preempt_deferred_qs_handler() local
589 rdp = container_of(iwp, struct rcu_data, defer_qs_iw); in rcu_preempt_deferred_qs_handler()
590 rdp->defer_qs_iw_pending = false; in rcu_preempt_deferred_qs_handler()
613 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_read_unlock_special() local
614 struct rcu_node *rnp = rdp->mynode; in rcu_read_unlock_special()
618 (rdp->grpmask & READ_ONCE(rnp->expmask)); in rcu_read_unlock_special()
631 !rdp->defer_qs_iw_pending && exp) { in rcu_read_unlock_special()
634 init_irq_work(&rdp->defer_qs_iw, in rcu_read_unlock_special()
636 rdp->defer_qs_iw_pending = true; in rcu_read_unlock_special()
637 irq_work_queue_on(&rdp->defer_qs_iw, rdp->cpu); in rcu_read_unlock_special()
748 struct rcu_data *rdp; in dump_blkd_tasks() local
770 rdp = per_cpu_ptr(&rcu_data, cpu); in dump_blkd_tasks()
771 onl = !!(rdp->grpmask & rcu_rnp_online_cpus(rnp)); in dump_blkd_tasks()
774 (long)rdp->rcu_onl_gp_seq, rdp->rcu_onl_gp_flags, in dump_blkd_tasks()
775 (long)rdp->rcu_ofl_gp_seq, rdp->rcu_ofl_gp_flags); in dump_blkd_tasks()
788 struct rcu_data *rdp; in rcu_read_unlock_strict() local
793 rdp = this_cpu_ptr(&rcu_data); in rcu_read_unlock_strict()
794 rcu_report_qs_rdp(rdp); in rcu_read_unlock_strict()
1205 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_prepare_kthreads() local
1206 struct rcu_node *rnp = rdp->mynode; in rcu_prepare_kthreads()
1312 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_try_advance_all_cbs() local
1316 if (jiffies == rdp->last_advance_all) in rcu_try_advance_all_cbs()
1318 rdp->last_advance_all = jiffies; in rcu_try_advance_all_cbs()
1320 rnp = rdp->mynode; in rcu_try_advance_all_cbs()
1327 if ((rcu_seq_completed_gp(rdp->gp_seq, in rcu_try_advance_all_cbs()
1329 unlikely(READ_ONCE(rdp->gpwrap))) && in rcu_try_advance_all_cbs()
1330 rcu_segcblist_pend_cbs(&rdp->cblist)) in rcu_try_advance_all_cbs()
1331 note_gp_changes(rdp); in rcu_try_advance_all_cbs()
1333 if (rcu_segcblist_ready_cbs(&rdp->cblist)) in rcu_try_advance_all_cbs()
1347 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_needs_cpu() local
1353 if (rcu_segcblist_empty(&rdp->cblist) || in rcu_needs_cpu()
1365 rdp->last_accelerate = jiffies; in rcu_needs_cpu()
1385 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_prepare_for_idle() local
1390 if (rcu_segcblist_is_offloaded(&rdp->cblist)) in rcu_prepare_for_idle()
1395 if (tne != rdp->tick_nohz_enabled_snap) { in rcu_prepare_for_idle()
1396 if (!rcu_segcblist_empty(&rdp->cblist)) in rcu_prepare_for_idle()
1398 rdp->tick_nohz_enabled_snap = tne; in rcu_prepare_for_idle()
1408 if (rdp->last_accelerate == jiffies) in rcu_prepare_for_idle()
1410 rdp->last_accelerate = jiffies; in rcu_prepare_for_idle()
1411 if (rcu_segcblist_pend_cbs(&rdp->cblist)) { in rcu_prepare_for_idle()
1412 rnp = rdp->mynode; in rcu_prepare_for_idle()
1414 needwake = rcu_accelerate_cbs(rnp, rdp); in rcu_prepare_for_idle()
1428 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_cleanup_after_idle() local
1431 if (rcu_segcblist_is_offloaded(&rdp->cblist)) in rcu_cleanup_after_idle()
1504 static void rcu_nocb_bypass_lock(struct rcu_data *rdp) in rcu_nocb_bypass_lock() argument
1505 __acquires(&rdp->nocb_bypass_lock) in rcu_nocb_bypass_lock()
1508 if (raw_spin_trylock(&rdp->nocb_bypass_lock)) in rcu_nocb_bypass_lock()
1510 atomic_inc(&rdp->nocb_lock_contended); in rcu_nocb_bypass_lock()
1511 WARN_ON_ONCE(smp_processor_id() != rdp->cpu); in rcu_nocb_bypass_lock()
1513 raw_spin_lock(&rdp->nocb_bypass_lock); in rcu_nocb_bypass_lock()
1515 atomic_dec(&rdp->nocb_lock_contended); in rcu_nocb_bypass_lock()
1528 static void rcu_nocb_wait_contended(struct rcu_data *rdp) in rcu_nocb_wait_contended() argument
1530 WARN_ON_ONCE(smp_processor_id() != rdp->cpu); in rcu_nocb_wait_contended()
1531 while (WARN_ON_ONCE(atomic_read(&rdp->nocb_lock_contended))) in rcu_nocb_wait_contended()
1539 static bool rcu_nocb_bypass_trylock(struct rcu_data *rdp) in rcu_nocb_bypass_trylock() argument
1542 return raw_spin_trylock(&rdp->nocb_bypass_lock); in rcu_nocb_bypass_trylock()
1548 static void rcu_nocb_bypass_unlock(struct rcu_data *rdp) in rcu_nocb_bypass_unlock() argument
1549 __releases(&rdp->nocb_bypass_lock) in rcu_nocb_bypass_unlock()
1552 raw_spin_unlock(&rdp->nocb_bypass_lock); in rcu_nocb_bypass_unlock()
1559 static void rcu_nocb_lock(struct rcu_data *rdp) in rcu_nocb_lock() argument
1562 if (!rcu_segcblist_is_offloaded(&rdp->cblist)) in rcu_nocb_lock()
1564 raw_spin_lock(&rdp->nocb_lock); in rcu_nocb_lock()
1571 static void rcu_nocb_unlock(struct rcu_data *rdp) in rcu_nocb_unlock() argument
1573 if (rcu_segcblist_is_offloaded(&rdp->cblist)) { in rcu_nocb_unlock()
1575 raw_spin_unlock(&rdp->nocb_lock); in rcu_nocb_unlock()
1583 static void rcu_nocb_unlock_irqrestore(struct rcu_data *rdp, in rcu_nocb_unlock_irqrestore() argument
1586 if (rcu_segcblist_is_offloaded(&rdp->cblist)) { in rcu_nocb_unlock_irqrestore()
1588 raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags); in rcu_nocb_unlock_irqrestore()
1595 static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp) in rcu_lockdep_assert_cblist_protected() argument
1598 if (rcu_segcblist_is_offloaded(&rdp->cblist)) in rcu_lockdep_assert_cblist_protected()
1599 lockdep_assert_held(&rdp->nocb_lock); in rcu_lockdep_assert_cblist_protected()
1634 static void wake_nocb_gp(struct rcu_data *rdp, bool force, in wake_nocb_gp() argument
1636 __releases(rdp->nocb_lock) in wake_nocb_gp()
1639 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp; in wake_nocb_gp()
1641 lockdep_assert_held(&rdp->nocb_lock); in wake_nocb_gp()
1643 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in wake_nocb_gp()
1645 rcu_nocb_unlock_irqrestore(rdp, flags); in wake_nocb_gp()
1648 del_timer(&rdp->nocb_timer); in wake_nocb_gp()
1649 rcu_nocb_unlock_irqrestore(rdp, flags); in wake_nocb_gp()
1654 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("DoWake")); in wake_nocb_gp()
1665 static void wake_nocb_gp_defer(struct rcu_data *rdp, int waketype, in wake_nocb_gp_defer() argument
1668 if (rdp->nocb_defer_wakeup == RCU_NOCB_WAKE_NOT) in wake_nocb_gp_defer()
1669 mod_timer(&rdp->nocb_timer, jiffies + 1); in wake_nocb_gp_defer()
1670 if (rdp->nocb_defer_wakeup < waketype) in wake_nocb_gp_defer()
1671 WRITE_ONCE(rdp->nocb_defer_wakeup, waketype); in wake_nocb_gp_defer()
1672 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, reason); in wake_nocb_gp_defer()
1683 static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, in rcu_nocb_do_flush_bypass() argument
1688 WARN_ON_ONCE(!rcu_segcblist_is_offloaded(&rdp->cblist)); in rcu_nocb_do_flush_bypass()
1689 rcu_lockdep_assert_cblist_protected(rdp); in rcu_nocb_do_flush_bypass()
1690 lockdep_assert_held(&rdp->nocb_bypass_lock); in rcu_nocb_do_flush_bypass()
1691 if (rhp && !rcu_cblist_n_cbs(&rdp->nocb_bypass)) { in rcu_nocb_do_flush_bypass()
1692 raw_spin_unlock(&rdp->nocb_bypass_lock); in rcu_nocb_do_flush_bypass()
1697 rcu_segcblist_inc_len(&rdp->cblist); /* Must precede enqueue. */ in rcu_nocb_do_flush_bypass()
1698 rcu_cblist_flush_enqueue(&rcl, &rdp->nocb_bypass, rhp); in rcu_nocb_do_flush_bypass()
1699 rcu_segcblist_insert_pend_cbs(&rdp->cblist, &rcl); in rcu_nocb_do_flush_bypass()
1700 WRITE_ONCE(rdp->nocb_bypass_first, j); in rcu_nocb_do_flush_bypass()
1701 rcu_nocb_bypass_unlock(rdp); in rcu_nocb_do_flush_bypass()
1713 static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, in rcu_nocb_flush_bypass() argument
1716 if (!rcu_segcblist_is_offloaded(&rdp->cblist)) in rcu_nocb_flush_bypass()
1718 rcu_lockdep_assert_cblist_protected(rdp); in rcu_nocb_flush_bypass()
1719 rcu_nocb_bypass_lock(rdp); in rcu_nocb_flush_bypass()
1720 return rcu_nocb_do_flush_bypass(rdp, rhp, j); in rcu_nocb_flush_bypass()
1727 static void rcu_nocb_try_flush_bypass(struct rcu_data *rdp, unsigned long j) in rcu_nocb_try_flush_bypass() argument
1729 rcu_lockdep_assert_cblist_protected(rdp); in rcu_nocb_try_flush_bypass()
1730 if (!rcu_segcblist_is_offloaded(&rdp->cblist) || in rcu_nocb_try_flush_bypass()
1731 !rcu_nocb_bypass_trylock(rdp)) in rcu_nocb_try_flush_bypass()
1733 WARN_ON_ONCE(!rcu_nocb_do_flush_bypass(rdp, NULL, j)); in rcu_nocb_try_flush_bypass()
1754 static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp, in rcu_nocb_try_bypass() argument
1760 long ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass); in rcu_nocb_try_bypass()
1762 if (!rcu_segcblist_is_offloaded(&rdp->cblist)) { in rcu_nocb_try_bypass()
1763 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist); in rcu_nocb_try_bypass()
1770 rcu_nocb_lock(rdp); in rcu_nocb_try_bypass()
1771 WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass)); in rcu_nocb_try_bypass()
1772 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist); in rcu_nocb_try_bypass()
1778 if (j == rdp->nocb_nobypass_last) { in rcu_nocb_try_bypass()
1779 c = rdp->nocb_nobypass_count + 1; in rcu_nocb_try_bypass()
1781 WRITE_ONCE(rdp->nocb_nobypass_last, j); in rcu_nocb_try_bypass()
1782 c = rdp->nocb_nobypass_count - nocb_nobypass_lim_per_jiffy; in rcu_nocb_try_bypass()
1783 if (ULONG_CMP_LT(rdp->nocb_nobypass_count, in rcu_nocb_try_bypass()
1789 WRITE_ONCE(rdp->nocb_nobypass_count, c); in rcu_nocb_try_bypass()
1794 if (rdp->nocb_nobypass_count < nocb_nobypass_lim_per_jiffy) { in rcu_nocb_try_bypass()
1795 rcu_nocb_lock(rdp); in rcu_nocb_try_bypass()
1796 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist); in rcu_nocb_try_bypass()
1798 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in rcu_nocb_try_bypass()
1800 WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, j)); in rcu_nocb_try_bypass()
1801 WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass)); in rcu_nocb_try_bypass()
1807 if ((ncbs && j != READ_ONCE(rdp->nocb_bypass_first)) || in rcu_nocb_try_bypass()
1809 rcu_nocb_lock(rdp); in rcu_nocb_try_bypass()
1810 if (!rcu_nocb_flush_bypass(rdp, rhp, j)) { in rcu_nocb_try_bypass()
1811 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist); in rcu_nocb_try_bypass()
1813 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in rcu_nocb_try_bypass()
1815 WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass)); in rcu_nocb_try_bypass()
1818 if (j != rdp->nocb_gp_adv_time && in rcu_nocb_try_bypass()
1819 rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) && in rcu_nocb_try_bypass()
1820 rcu_seq_done(&rdp->mynode->gp_seq, cur_gp_seq)) { in rcu_nocb_try_bypass()
1821 rcu_advance_cbs_nowake(rdp->mynode, rdp); in rcu_nocb_try_bypass()
1822 rdp->nocb_gp_adv_time = j; in rcu_nocb_try_bypass()
1824 rcu_nocb_unlock_irqrestore(rdp, flags); in rcu_nocb_try_bypass()
1829 rcu_nocb_wait_contended(rdp); in rcu_nocb_try_bypass()
1830 rcu_nocb_bypass_lock(rdp); in rcu_nocb_try_bypass()
1831 ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass); in rcu_nocb_try_bypass()
1832 rcu_segcblist_inc_len(&rdp->cblist); /* Must precede enqueue. */ in rcu_nocb_try_bypass()
1833 rcu_cblist_enqueue(&rdp->nocb_bypass, rhp); in rcu_nocb_try_bypass()
1835 WRITE_ONCE(rdp->nocb_bypass_first, j); in rcu_nocb_try_bypass()
1836 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("FirstBQ")); in rcu_nocb_try_bypass()
1838 rcu_nocb_bypass_unlock(rdp); in rcu_nocb_try_bypass()
1844 rcu_nocb_lock(rdp); // Rare during call_rcu() flood. in rcu_nocb_try_bypass()
1845 if (!rcu_segcblist_pend_cbs(&rdp->cblist)) { in rcu_nocb_try_bypass()
1846 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in rcu_nocb_try_bypass()
1848 __call_rcu_nocb_wake(rdp, true, flags); in rcu_nocb_try_bypass()
1850 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in rcu_nocb_try_bypass()
1852 rcu_nocb_unlock_irqrestore(rdp, flags); in rcu_nocb_try_bypass()
1864 static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone, in __call_rcu_nocb_wake() argument
1866 __releases(rdp->nocb_lock) in __call_rcu_nocb_wake()
1874 t = READ_ONCE(rdp->nocb_gp_kthread); in __call_rcu_nocb_wake()
1876 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in __call_rcu_nocb_wake()
1878 rcu_nocb_unlock_irqrestore(rdp, flags); in __call_rcu_nocb_wake()
1882 len = rcu_segcblist_n_cbs(&rdp->cblist); in __call_rcu_nocb_wake()
1884 rdp->qlen_last_fqs_check = len; in __call_rcu_nocb_wake()
1887 wake_nocb_gp(rdp, false, flags); in __call_rcu_nocb_wake()
1888 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in __call_rcu_nocb_wake()
1891 wake_nocb_gp_defer(rdp, RCU_NOCB_WAKE, in __call_rcu_nocb_wake()
1893 rcu_nocb_unlock_irqrestore(rdp, flags); in __call_rcu_nocb_wake()
1895 } else if (len > rdp->qlen_last_fqs_check + qhimark) { in __call_rcu_nocb_wake()
1897 rdp->qlen_last_fqs_check = len; in __call_rcu_nocb_wake()
1899 if (j != rdp->nocb_gp_adv_time && in __call_rcu_nocb_wake()
1900 rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) && in __call_rcu_nocb_wake()
1901 rcu_seq_done(&rdp->mynode->gp_seq, cur_gp_seq)) { in __call_rcu_nocb_wake()
1902 rcu_advance_cbs_nowake(rdp->mynode, rdp); in __call_rcu_nocb_wake()
1903 rdp->nocb_gp_adv_time = j; in __call_rcu_nocb_wake()
1906 if ((rdp->nocb_cb_sleep || in __call_rcu_nocb_wake()
1907 !rcu_segcblist_ready_cbs(&rdp->cblist)) && in __call_rcu_nocb_wake()
1908 !timer_pending(&rdp->nocb_bypass_timer)) in __call_rcu_nocb_wake()
1909 wake_nocb_gp_defer(rdp, RCU_NOCB_WAKE_FORCE, in __call_rcu_nocb_wake()
1911 rcu_nocb_unlock_irqrestore(rdp, flags); in __call_rcu_nocb_wake()
1913 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WakeNot")); in __call_rcu_nocb_wake()
1914 rcu_nocb_unlock_irqrestore(rdp, flags); in __call_rcu_nocb_wake()
1923 struct rcu_data *rdp = from_timer(rdp, t, nocb_bypass_timer); in do_nocb_bypass_wakeup_timer() local
1925 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("Timer")); in do_nocb_bypass_wakeup_timer()
1926 rcu_nocb_lock_irqsave(rdp, flags); in do_nocb_bypass_wakeup_timer()
1928 __call_rcu_nocb_wake(rdp, true, flags); in do_nocb_bypass_wakeup_timer()
1947 struct rcu_data *rdp; in nocb_gp_wait() local
1958 for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_cb_rdp) { in nocb_gp_wait()
1959 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("Check")); in nocb_gp_wait()
1960 rcu_nocb_lock_irqsave(rdp, flags); in nocb_gp_wait()
1961 bypass_ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass); in nocb_gp_wait()
1963 (time_after(j, READ_ONCE(rdp->nocb_bypass_first) + 1) || in nocb_gp_wait()
1966 (void)rcu_nocb_try_flush_bypass(rdp, j); in nocb_gp_wait()
1967 bypass_ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass); in nocb_gp_wait()
1968 } else if (!bypass_ncbs && rcu_segcblist_empty(&rdp->cblist)) { in nocb_gp_wait()
1969 rcu_nocb_unlock_irqrestore(rdp, flags); in nocb_gp_wait()
1973 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in nocb_gp_wait()
1977 rnp = rdp->mynode; in nocb_gp_wait()
1985 if (!rcu_segcblist_restempty(&rdp->cblist, in nocb_gp_wait()
1987 (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) && in nocb_gp_wait()
1990 needwake_gp = rcu_advance_cbs(rnp, rdp); in nocb_gp_wait()
1991 wasempty = rcu_segcblist_restempty(&rdp->cblist, in nocb_gp_wait()
1997 !rcu_segcblist_restempty(&rdp->cblist, in nocb_gp_wait()
1999 if (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq)) { in nocb_gp_wait()
2004 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in nocb_gp_wait()
2007 if (rcu_segcblist_ready_cbs(&rdp->cblist)) { in nocb_gp_wait()
2008 needwake = rdp->nocb_cb_sleep; in nocb_gp_wait()
2009 WRITE_ONCE(rdp->nocb_cb_sleep, false); in nocb_gp_wait()
2014 rcu_nocb_unlock_irqrestore(rdp, flags); in nocb_gp_wait()
2016 swake_up_one(&rdp->nocb_cb_wq); in nocb_gp_wait()
2074 struct rcu_data *rdp = arg; in rcu_nocb_gp_kthread() local
2077 WRITE_ONCE(rdp->nocb_gp_loops, rdp->nocb_gp_loops + 1); in rcu_nocb_gp_kthread()
2078 nocb_gp_wait(rdp); in rcu_nocb_gp_kthread()
2088 static void nocb_cb_wait(struct rcu_data *rdp) in nocb_cb_wait() argument
2093 struct rcu_node *rnp = rdp->mynode; in nocb_cb_wait()
2099 rcu_do_batch(rdp); in nocb_cb_wait()
2102 rcu_nocb_lock_irqsave(rdp, flags); in nocb_cb_wait()
2103 if (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) && in nocb_cb_wait()
2106 needwake_gp = rcu_advance_cbs(rdp->mynode, rdp); in nocb_cb_wait()
2109 if (rcu_segcblist_ready_cbs(&rdp->cblist)) { in nocb_cb_wait()
2110 rcu_nocb_unlock_irqrestore(rdp, flags); in nocb_cb_wait()
2116 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("CBSleep")); in nocb_cb_wait()
2117 WRITE_ONCE(rdp->nocb_cb_sleep, true); in nocb_cb_wait()
2118 rcu_nocb_unlock_irqrestore(rdp, flags); in nocb_cb_wait()
2121 swait_event_interruptible_exclusive(rdp->nocb_cb_wq, in nocb_cb_wait()
2122 !READ_ONCE(rdp->nocb_cb_sleep)); in nocb_cb_wait()
2123 if (!smp_load_acquire(&rdp->nocb_cb_sleep)) { /* VVV */ in nocb_cb_wait()
2128 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WokeEmpty")); in nocb_cb_wait()
2137 struct rcu_data *rdp = arg; in rcu_nocb_cb_kthread() local
2142 nocb_cb_wait(rdp); in rcu_nocb_cb_kthread()
2149 static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp) in rcu_nocb_need_deferred_wakeup() argument
2151 return READ_ONCE(rdp->nocb_defer_wakeup); in rcu_nocb_need_deferred_wakeup()
2155 static void do_nocb_deferred_wakeup_common(struct rcu_data *rdp) in do_nocb_deferred_wakeup_common() argument
2160 rcu_nocb_lock_irqsave(rdp, flags); in do_nocb_deferred_wakeup_common()
2161 if (!rcu_nocb_need_deferred_wakeup(rdp)) { in do_nocb_deferred_wakeup_common()
2162 rcu_nocb_unlock_irqrestore(rdp, flags); in do_nocb_deferred_wakeup_common()
2165 ndw = READ_ONCE(rdp->nocb_defer_wakeup); in do_nocb_deferred_wakeup_common()
2166 WRITE_ONCE(rdp->nocb_defer_wakeup, RCU_NOCB_WAKE_NOT); in do_nocb_deferred_wakeup_common()
2167 wake_nocb_gp(rdp, ndw == RCU_NOCB_WAKE_FORCE, flags); in do_nocb_deferred_wakeup_common()
2168 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("DeferredWake")); in do_nocb_deferred_wakeup_common()
2174 struct rcu_data *rdp = from_timer(rdp, t, nocb_timer); in do_nocb_deferred_wakeup_timer() local
2176 do_nocb_deferred_wakeup_common(rdp); in do_nocb_deferred_wakeup_timer()
2184 static void do_nocb_deferred_wakeup(struct rcu_data *rdp) in do_nocb_deferred_wakeup() argument
2186 if (rcu_nocb_need_deferred_wakeup(rdp)) in do_nocb_deferred_wakeup()
2187 do_nocb_deferred_wakeup_common(rdp); in do_nocb_deferred_wakeup()
2194 struct rcu_data *rdp; in rcu_init_nohz() local
2229 rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_init_nohz()
2230 if (rcu_segcblist_empty(&rdp->cblist)) in rcu_init_nohz()
2231 rcu_segcblist_init(&rdp->cblist); in rcu_init_nohz()
2232 rcu_segcblist_offload(&rdp->cblist); in rcu_init_nohz()
2238 static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp) in rcu_boot_init_nocb_percpu_data() argument
2240 init_swait_queue_head(&rdp->nocb_cb_wq); in rcu_boot_init_nocb_percpu_data()
2241 init_swait_queue_head(&rdp->nocb_gp_wq); in rcu_boot_init_nocb_percpu_data()
2242 raw_spin_lock_init(&rdp->nocb_lock); in rcu_boot_init_nocb_percpu_data()
2243 raw_spin_lock_init(&rdp->nocb_bypass_lock); in rcu_boot_init_nocb_percpu_data()
2244 raw_spin_lock_init(&rdp->nocb_gp_lock); in rcu_boot_init_nocb_percpu_data()
2245 timer_setup(&rdp->nocb_timer, do_nocb_deferred_wakeup_timer, 0); in rcu_boot_init_nocb_percpu_data()
2246 timer_setup(&rdp->nocb_bypass_timer, do_nocb_bypass_wakeup_timer, 0); in rcu_boot_init_nocb_percpu_data()
2247 rcu_cblist_init(&rdp->nocb_bypass); in rcu_boot_init_nocb_percpu_data()
2257 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_spawn_one_nocb_kthread() local
2265 if (!rcu_is_nocb_cpu(cpu) || rdp->nocb_cb_kthread) in rcu_spawn_one_nocb_kthread()
2269 rdp_gp = rdp->nocb_gp_rdp; in rcu_spawn_one_nocb_kthread()
2279 t = kthread_run(rcu_nocb_cb_kthread, rdp, in rcu_spawn_one_nocb_kthread()
2283 WRITE_ONCE(rdp->nocb_cb_kthread, t); in rcu_spawn_one_nocb_kthread()
2284 WRITE_ONCE(rdp->nocb_gp_kthread, rdp_gp->nocb_gp_kthread); in rcu_spawn_one_nocb_kthread()
2326 struct rcu_data *rdp; in rcu_organize_nocb_kthreads() local
2343 rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_organize_nocb_kthreads()
2344 if (rdp->cpu >= nl) { in rcu_organize_nocb_kthreads()
2347 nl = DIV_ROUND_UP(rdp->cpu + 1, ls) * ls; in rcu_organize_nocb_kthreads()
2348 rdp->nocb_gp_rdp = rdp; in rcu_organize_nocb_kthreads()
2349 rdp_gp = rdp; in rcu_organize_nocb_kthreads()
2362 rdp->nocb_gp_rdp = rdp_gp; in rcu_organize_nocb_kthreads()
2363 rdp_prev->nocb_next_cb_rdp = rdp; in rcu_organize_nocb_kthreads()
2367 rdp_prev = rdp; in rcu_organize_nocb_kthreads()
2388 static void show_rcu_nocb_gp_state(struct rcu_data *rdp) in show_rcu_nocb_gp_state() argument
2390 struct rcu_node *rnp = rdp->mynode; in show_rcu_nocb_gp_state()
2393 rdp->cpu, in show_rcu_nocb_gp_state()
2394 "kK"[!!rdp->nocb_gp_kthread], in show_rcu_nocb_gp_state()
2395 "lL"[raw_spin_is_locked(&rdp->nocb_gp_lock)], in show_rcu_nocb_gp_state()
2396 "dD"[!!rdp->nocb_defer_wakeup], in show_rcu_nocb_gp_state()
2397 "tT"[timer_pending(&rdp->nocb_timer)], in show_rcu_nocb_gp_state()
2398 "bB"[timer_pending(&rdp->nocb_bypass_timer)], in show_rcu_nocb_gp_state()
2399 "sS"[!!rdp->nocb_gp_sleep], in show_rcu_nocb_gp_state()
2400 ".W"[swait_active(&rdp->nocb_gp_wq)], in show_rcu_nocb_gp_state()
2403 ".B"[!!rdp->nocb_gp_bypass], in show_rcu_nocb_gp_state()
2404 ".G"[!!rdp->nocb_gp_gp], in show_rcu_nocb_gp_state()
2405 (long)rdp->nocb_gp_seq, in show_rcu_nocb_gp_state()
2406 rnp->grplo, rnp->grphi, READ_ONCE(rdp->nocb_gp_loops)); in show_rcu_nocb_gp_state()
2410 static void show_rcu_nocb_state(struct rcu_data *rdp) in show_rcu_nocb_state() argument
2412 struct rcu_segcblist *rsclp = &rdp->cblist; in show_rcu_nocb_state()
2417 if (rdp->nocb_gp_rdp == rdp) in show_rcu_nocb_state()
2418 show_rcu_nocb_gp_state(rdp); in show_rcu_nocb_state()
2421 rdp->cpu, rdp->nocb_gp_rdp->cpu, in show_rcu_nocb_state()
2422 "kK"[!!rdp->nocb_cb_kthread], in show_rcu_nocb_state()
2423 "bB"[raw_spin_is_locked(&rdp->nocb_bypass_lock)], in show_rcu_nocb_state()
2424 "cC"[!!atomic_read(&rdp->nocb_lock_contended)], in show_rcu_nocb_state()
2425 "lL"[raw_spin_is_locked(&rdp->nocb_lock)], in show_rcu_nocb_state()
2426 "sS"[!!rdp->nocb_cb_sleep], in show_rcu_nocb_state()
2427 ".W"[swait_active(&rdp->nocb_cb_wq)], in show_rcu_nocb_state()
2428 jiffies - rdp->nocb_bypass_first, in show_rcu_nocb_state()
2429 jiffies - rdp->nocb_nobypass_last, in show_rcu_nocb_state()
2430 rdp->nocb_nobypass_count, in show_rcu_nocb_state()
2435 ".B"[!!rcu_cblist_n_cbs(&rdp->nocb_bypass)], in show_rcu_nocb_state()
2436 rcu_segcblist_n_cbs(&rdp->cblist)); in show_rcu_nocb_state()
2439 if (rdp->nocb_gp_rdp == rdp) in show_rcu_nocb_state()
2442 waslocked = raw_spin_is_locked(&rdp->nocb_gp_lock); in show_rcu_nocb_state()
2443 wastimer = timer_pending(&rdp->nocb_bypass_timer); in show_rcu_nocb_state()
2444 wassleep = swait_active(&rdp->nocb_gp_wq); in show_rcu_nocb_state()
2445 if (!rdp->nocb_gp_sleep && !waslocked && !wastimer && !wassleep) in show_rcu_nocb_state()
2450 "dD"[!!rdp->nocb_defer_wakeup], in show_rcu_nocb_state()
2452 "sS"[!!rdp->nocb_gp_sleep], in show_rcu_nocb_state()
2459 static void rcu_nocb_lock(struct rcu_data *rdp) in rcu_nocb_lock() argument
2464 static void rcu_nocb_unlock(struct rcu_data *rdp) in rcu_nocb_unlock() argument
2469 static void rcu_nocb_unlock_irqrestore(struct rcu_data *rdp, in rcu_nocb_unlock_irqrestore() argument
2476 static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp) in rcu_lockdep_assert_cblist_protected() argument
2494 static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, in rcu_nocb_flush_bypass() argument
2500 static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp, in rcu_nocb_try_bypass() argument
2506 static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_empty, in __call_rcu_nocb_wake() argument
2512 static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp) in rcu_boot_init_nocb_percpu_data() argument
2516 static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp) in rcu_nocb_need_deferred_wakeup() argument
2521 static void do_nocb_deferred_wakeup(struct rcu_data *rdp) in do_nocb_deferred_wakeup() argument
2533 static void show_rcu_nocb_state(struct rcu_data *rdp) in show_rcu_nocb_state() argument