Lines Matching refs:rdp
130 static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp) in rcu_preempt_ctxt_queue() argument
135 (rnp->qsmask & rdp->grpmask ? RCU_GP_BLKD : 0) + in rcu_preempt_ctxt_queue()
136 (rnp->expmask & rdp->grpmask ? RCU_EXP_BLKD : 0); in rcu_preempt_ctxt_queue()
140 WARN_ON_ONCE(rdp->mynode != rnp); in rcu_preempt_ctxt_queue()
144 rdp->grpmask); in rcu_preempt_ctxt_queue()
229 !(rnp->qsmask & rdp->grpmask)); in rcu_preempt_ctxt_queue()
231 !(rnp->expmask & rdp->grpmask)); in rcu_preempt_ctxt_queue()
240 if (blkd_state & RCU_EXP_BLKD && rdp->exp_deferred_qs) in rcu_preempt_ctxt_queue()
241 rcu_report_exp_rdp(rdp); in rcu_preempt_ctxt_queue()
243 WARN_ON_ONCE(rdp->exp_deferred_qs); in rcu_preempt_ctxt_queue()
288 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_note_context_switch() local
298 rnp = rdp->mynode; in rcu_note_context_switch()
308 WARN_ON_ONCE((rdp->grpmask & rcu_rnp_online_cpus(rnp)) == 0); in rcu_note_context_switch()
312 (rnp->qsmask & rdp->grpmask) in rcu_note_context_switch()
315 rcu_preempt_ctxt_queue(rnp, rdp); in rcu_note_context_switch()
330 if (rdp->exp_deferred_qs) in rcu_note_context_switch()
331 rcu_report_exp_rdp(rdp); in rcu_note_context_switch()
432 struct rcu_data *rdp; in rcu_preempt_deferred_qs_irqrestore() local
442 rdp = this_cpu_ptr(&rcu_data); in rcu_preempt_deferred_qs_irqrestore()
443 if (!special.s && !rdp->exp_deferred_qs) { in rcu_preempt_deferred_qs_irqrestore()
451 if (!t->rcu_read_unlock_special.s && !rdp->exp_deferred_qs) { in rcu_preempt_deferred_qs_irqrestore()
463 if (rdp->exp_deferred_qs) { in rcu_preempt_deferred_qs_irqrestore()
464 rcu_report_exp_rdp(rdp); in rcu_preempt_deferred_qs_irqrestore()
584 struct rcu_data *rdp; in rcu_preempt_deferred_qs_handler() local
586 rdp = container_of(iwp, struct rcu_data, defer_qs_iw); in rcu_preempt_deferred_qs_handler()
587 rdp->defer_qs_iw_pending = false; in rcu_preempt_deferred_qs_handler()
610 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_read_unlock_special() local
611 struct rcu_node *rnp = rdp->mynode; in rcu_read_unlock_special()
615 (rdp->grpmask & rnp->expmask) || in rcu_read_unlock_special()
616 tick_nohz_full_cpu(rdp->cpu); in rcu_read_unlock_special()
630 !rdp->defer_qs_iw_pending && exp) { in rcu_read_unlock_special()
633 init_irq_work(&rdp->defer_qs_iw, in rcu_read_unlock_special()
635 rdp->defer_qs_iw_pending = true; in rcu_read_unlock_special()
636 irq_work_queue_on(&rdp->defer_qs_iw, rdp->cpu); in rcu_read_unlock_special()
749 struct rcu_data *rdp; in dump_blkd_tasks() local
770 rdp = per_cpu_ptr(&rcu_data, cpu); in dump_blkd_tasks()
771 onl = !!(rdp->grpmask & rcu_rnp_online_cpus(rnp)); in dump_blkd_tasks()
774 (long)rdp->rcu_onl_gp_seq, rdp->rcu_onl_gp_flags, in dump_blkd_tasks()
775 (long)rdp->rcu_ofl_gp_seq, rdp->rcu_ofl_gp_flags); in dump_blkd_tasks()
1187 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_prepare_kthreads() local
1188 struct rcu_node *rnp = rdp->mynode; in rcu_prepare_kthreads()
1301 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_try_advance_all_cbs() local
1305 if (jiffies == rdp->last_advance_all) in rcu_try_advance_all_cbs()
1307 rdp->last_advance_all = jiffies; in rcu_try_advance_all_cbs()
1309 rnp = rdp->mynode; in rcu_try_advance_all_cbs()
1316 if ((rcu_seq_completed_gp(rdp->gp_seq, in rcu_try_advance_all_cbs()
1318 unlikely(READ_ONCE(rdp->gpwrap))) && in rcu_try_advance_all_cbs()
1319 rcu_segcblist_pend_cbs(&rdp->cblist)) in rcu_try_advance_all_cbs()
1320 note_gp_changes(rdp); in rcu_try_advance_all_cbs()
1322 if (rcu_segcblist_ready_cbs(&rdp->cblist)) in rcu_try_advance_all_cbs()
1337 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_needs_cpu() local
1343 if (rcu_segcblist_empty(&rdp->cblist) || in rcu_needs_cpu()
1355 rdp->last_accelerate = jiffies; in rcu_needs_cpu()
1358 rdp->all_lazy = !rcu_segcblist_n_nonlazy_cbs(&rdp->cblist); in rcu_needs_cpu()
1359 if (rdp->all_lazy) { in rcu_needs_cpu()
1382 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_prepare_for_idle() local
1387 if (rcu_segcblist_is_offloaded(&rdp->cblist)) in rcu_prepare_for_idle()
1392 if (tne != rdp->tick_nohz_enabled_snap) { in rcu_prepare_for_idle()
1393 if (!rcu_segcblist_empty(&rdp->cblist)) in rcu_prepare_for_idle()
1395 rdp->tick_nohz_enabled_snap = tne; in rcu_prepare_for_idle()
1406 if (rdp->all_lazy && rcu_segcblist_n_nonlazy_cbs(&rdp->cblist)) { in rcu_prepare_for_idle()
1407 rdp->all_lazy = false; in rcu_prepare_for_idle()
1416 if (rdp->last_accelerate == jiffies) in rcu_prepare_for_idle()
1418 rdp->last_accelerate = jiffies; in rcu_prepare_for_idle()
1419 if (rcu_segcblist_pend_cbs(&rdp->cblist)) { in rcu_prepare_for_idle()
1420 rnp = rdp->mynode; in rcu_prepare_for_idle()
1422 needwake = rcu_accelerate_cbs(rnp, rdp); in rcu_prepare_for_idle()
1436 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_cleanup_after_idle() local
1439 if (rcu_segcblist_is_offloaded(&rdp->cblist)) in rcu_cleanup_after_idle()
1512 static void rcu_nocb_bypass_lock(struct rcu_data *rdp) in rcu_nocb_bypass_lock() argument
1515 if (raw_spin_trylock(&rdp->nocb_bypass_lock)) in rcu_nocb_bypass_lock()
1517 atomic_inc(&rdp->nocb_lock_contended); in rcu_nocb_bypass_lock()
1518 WARN_ON_ONCE(smp_processor_id() != rdp->cpu); in rcu_nocb_bypass_lock()
1520 raw_spin_lock(&rdp->nocb_bypass_lock); in rcu_nocb_bypass_lock()
1522 atomic_dec(&rdp->nocb_lock_contended); in rcu_nocb_bypass_lock()
1535 static void rcu_nocb_wait_contended(struct rcu_data *rdp) in rcu_nocb_wait_contended() argument
1537 WARN_ON_ONCE(smp_processor_id() != rdp->cpu); in rcu_nocb_wait_contended()
1538 while (WARN_ON_ONCE(atomic_read(&rdp->nocb_lock_contended))) in rcu_nocb_wait_contended()
1546 static bool rcu_nocb_bypass_trylock(struct rcu_data *rdp) in rcu_nocb_bypass_trylock() argument
1549 return raw_spin_trylock(&rdp->nocb_bypass_lock); in rcu_nocb_bypass_trylock()
1555 static void rcu_nocb_bypass_unlock(struct rcu_data *rdp) in rcu_nocb_bypass_unlock() argument
1558 raw_spin_unlock(&rdp->nocb_bypass_lock); in rcu_nocb_bypass_unlock()
1565 static void rcu_nocb_lock(struct rcu_data *rdp) in rcu_nocb_lock() argument
1568 if (!rcu_segcblist_is_offloaded(&rdp->cblist)) in rcu_nocb_lock()
1570 raw_spin_lock(&rdp->nocb_lock); in rcu_nocb_lock()
1577 static void rcu_nocb_unlock(struct rcu_data *rdp) in rcu_nocb_unlock() argument
1579 if (rcu_segcblist_is_offloaded(&rdp->cblist)) { in rcu_nocb_unlock()
1581 raw_spin_unlock(&rdp->nocb_lock); in rcu_nocb_unlock()
1589 static void rcu_nocb_unlock_irqrestore(struct rcu_data *rdp, in rcu_nocb_unlock_irqrestore() argument
1592 if (rcu_segcblist_is_offloaded(&rdp->cblist)) { in rcu_nocb_unlock_irqrestore()
1594 raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags); in rcu_nocb_unlock_irqrestore()
1601 static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp) in rcu_lockdep_assert_cblist_protected() argument
1604 if (rcu_segcblist_is_offloaded(&rdp->cblist) && in rcu_lockdep_assert_cblist_protected()
1605 cpu_online(rdp->cpu)) in rcu_lockdep_assert_cblist_protected()
1606 lockdep_assert_held(&rdp->nocb_lock); in rcu_lockdep_assert_cblist_protected()
1641 static void wake_nocb_gp(struct rcu_data *rdp, bool force, in wake_nocb_gp() argument
1643 __releases(rdp->nocb_lock) in wake_nocb_gp()
1646 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp; in wake_nocb_gp()
1648 lockdep_assert_held(&rdp->nocb_lock); in wake_nocb_gp()
1650 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in wake_nocb_gp()
1652 rcu_nocb_unlock_irqrestore(rdp, flags); in wake_nocb_gp()
1655 del_timer(&rdp->nocb_timer); in wake_nocb_gp()
1656 rcu_nocb_unlock_irqrestore(rdp, flags); in wake_nocb_gp()
1661 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("DoWake")); in wake_nocb_gp()
1672 static void wake_nocb_gp_defer(struct rcu_data *rdp, int waketype, in wake_nocb_gp_defer() argument
1675 if (rdp->nocb_defer_wakeup == RCU_NOCB_WAKE_NOT) in wake_nocb_gp_defer()
1676 mod_timer(&rdp->nocb_timer, jiffies + 1); in wake_nocb_gp_defer()
1677 if (rdp->nocb_defer_wakeup < waketype) in wake_nocb_gp_defer()
1678 WRITE_ONCE(rdp->nocb_defer_wakeup, waketype); in wake_nocb_gp_defer()
1679 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, reason); in wake_nocb_gp_defer()
1690 static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, in rcu_nocb_do_flush_bypass() argument
1695 WARN_ON_ONCE(!rcu_segcblist_is_offloaded(&rdp->cblist)); in rcu_nocb_do_flush_bypass()
1696 rcu_lockdep_assert_cblist_protected(rdp); in rcu_nocb_do_flush_bypass()
1697 lockdep_assert_held(&rdp->nocb_bypass_lock); in rcu_nocb_do_flush_bypass()
1698 if (rhp && !rcu_cblist_n_cbs(&rdp->nocb_bypass)) { in rcu_nocb_do_flush_bypass()
1699 raw_spin_unlock(&rdp->nocb_bypass_lock); in rcu_nocb_do_flush_bypass()
1704 rcu_segcblist_inc_len(&rdp->cblist); /* Must precede enqueue. */ in rcu_nocb_do_flush_bypass()
1705 rcu_cblist_flush_enqueue(&rcl, &rdp->nocb_bypass, rhp); in rcu_nocb_do_flush_bypass()
1706 rcu_segcblist_insert_pend_cbs(&rdp->cblist, &rcl); in rcu_nocb_do_flush_bypass()
1707 WRITE_ONCE(rdp->nocb_bypass_first, j); in rcu_nocb_do_flush_bypass()
1708 rcu_nocb_bypass_unlock(rdp); in rcu_nocb_do_flush_bypass()
1720 static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, in rcu_nocb_flush_bypass() argument
1723 if (!rcu_segcblist_is_offloaded(&rdp->cblist)) in rcu_nocb_flush_bypass()
1725 rcu_lockdep_assert_cblist_protected(rdp); in rcu_nocb_flush_bypass()
1726 rcu_nocb_bypass_lock(rdp); in rcu_nocb_flush_bypass()
1727 return rcu_nocb_do_flush_bypass(rdp, rhp, j); in rcu_nocb_flush_bypass()
1734 static void rcu_nocb_try_flush_bypass(struct rcu_data *rdp, unsigned long j) in rcu_nocb_try_flush_bypass() argument
1736 rcu_lockdep_assert_cblist_protected(rdp); in rcu_nocb_try_flush_bypass()
1737 if (!rcu_segcblist_is_offloaded(&rdp->cblist) || in rcu_nocb_try_flush_bypass()
1738 !rcu_nocb_bypass_trylock(rdp)) in rcu_nocb_try_flush_bypass()
1740 WARN_ON_ONCE(!rcu_nocb_do_flush_bypass(rdp, NULL, j)); in rcu_nocb_try_flush_bypass()
1761 static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp, in rcu_nocb_try_bypass() argument
1767 long ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass); in rcu_nocb_try_bypass()
1769 if (!rcu_segcblist_is_offloaded(&rdp->cblist)) { in rcu_nocb_try_bypass()
1770 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist); in rcu_nocb_try_bypass()
1777 rcu_nocb_lock(rdp); in rcu_nocb_try_bypass()
1778 WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass)); in rcu_nocb_try_bypass()
1779 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist); in rcu_nocb_try_bypass()
1785 if (j == rdp->nocb_nobypass_last) { in rcu_nocb_try_bypass()
1786 c = rdp->nocb_nobypass_count + 1; in rcu_nocb_try_bypass()
1788 WRITE_ONCE(rdp->nocb_nobypass_last, j); in rcu_nocb_try_bypass()
1789 c = rdp->nocb_nobypass_count - nocb_nobypass_lim_per_jiffy; in rcu_nocb_try_bypass()
1790 if (ULONG_CMP_LT(rdp->nocb_nobypass_count, in rcu_nocb_try_bypass()
1796 WRITE_ONCE(rdp->nocb_nobypass_count, c); in rcu_nocb_try_bypass()
1801 if (rdp->nocb_nobypass_count < nocb_nobypass_lim_per_jiffy) { in rcu_nocb_try_bypass()
1802 rcu_nocb_lock(rdp); in rcu_nocb_try_bypass()
1803 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist); in rcu_nocb_try_bypass()
1805 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in rcu_nocb_try_bypass()
1807 WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, j)); in rcu_nocb_try_bypass()
1808 WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass)); in rcu_nocb_try_bypass()
1814 if ((ncbs && j != READ_ONCE(rdp->nocb_bypass_first)) || in rcu_nocb_try_bypass()
1816 rcu_nocb_lock(rdp); in rcu_nocb_try_bypass()
1817 if (!rcu_nocb_flush_bypass(rdp, rhp, j)) { in rcu_nocb_try_bypass()
1818 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist); in rcu_nocb_try_bypass()
1820 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in rcu_nocb_try_bypass()
1822 WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass)); in rcu_nocb_try_bypass()
1825 if (j != rdp->nocb_gp_adv_time && in rcu_nocb_try_bypass()
1826 rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) && in rcu_nocb_try_bypass()
1827 rcu_seq_done(&rdp->mynode->gp_seq, cur_gp_seq)) { in rcu_nocb_try_bypass()
1828 rcu_advance_cbs_nowake(rdp->mynode, rdp); in rcu_nocb_try_bypass()
1829 rdp->nocb_gp_adv_time = j; in rcu_nocb_try_bypass()
1831 rcu_nocb_unlock_irqrestore(rdp, flags); in rcu_nocb_try_bypass()
1836 rcu_nocb_wait_contended(rdp); in rcu_nocb_try_bypass()
1837 rcu_nocb_bypass_lock(rdp); in rcu_nocb_try_bypass()
1838 ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass); in rcu_nocb_try_bypass()
1839 rcu_segcblist_inc_len(&rdp->cblist); /* Must precede enqueue. */ in rcu_nocb_try_bypass()
1840 rcu_cblist_enqueue(&rdp->nocb_bypass, rhp); in rcu_nocb_try_bypass()
1842 WRITE_ONCE(rdp->nocb_bypass_first, j); in rcu_nocb_try_bypass()
1843 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("FirstBQ")); in rcu_nocb_try_bypass()
1845 rcu_nocb_bypass_unlock(rdp); in rcu_nocb_try_bypass()
1851 rcu_nocb_lock(rdp); // Rare during call_rcu() flood. in rcu_nocb_try_bypass()
1852 if (!rcu_segcblist_pend_cbs(&rdp->cblist)) { in rcu_nocb_try_bypass()
1853 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in rcu_nocb_try_bypass()
1855 __call_rcu_nocb_wake(rdp, true, flags); in rcu_nocb_try_bypass()
1857 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in rcu_nocb_try_bypass()
1859 rcu_nocb_unlock_irqrestore(rdp, flags); in rcu_nocb_try_bypass()
1871 static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone, in __call_rcu_nocb_wake() argument
1873 __releases(rdp->nocb_lock) in __call_rcu_nocb_wake()
1881 t = READ_ONCE(rdp->nocb_gp_kthread); in __call_rcu_nocb_wake()
1883 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in __call_rcu_nocb_wake()
1885 rcu_nocb_unlock_irqrestore(rdp, flags); in __call_rcu_nocb_wake()
1889 len = rcu_segcblist_n_cbs(&rdp->cblist); in __call_rcu_nocb_wake()
1891 rdp->qlen_last_fqs_check = len; in __call_rcu_nocb_wake()
1894 wake_nocb_gp(rdp, false, flags); in __call_rcu_nocb_wake()
1895 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in __call_rcu_nocb_wake()
1898 wake_nocb_gp_defer(rdp, RCU_NOCB_WAKE, in __call_rcu_nocb_wake()
1900 rcu_nocb_unlock_irqrestore(rdp, flags); in __call_rcu_nocb_wake()
1902 } else if (len > rdp->qlen_last_fqs_check + qhimark) { in __call_rcu_nocb_wake()
1904 rdp->qlen_last_fqs_check = len; in __call_rcu_nocb_wake()
1906 if (j != rdp->nocb_gp_adv_time && in __call_rcu_nocb_wake()
1907 rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) && in __call_rcu_nocb_wake()
1908 rcu_seq_done(&rdp->mynode->gp_seq, cur_gp_seq)) { in __call_rcu_nocb_wake()
1909 rcu_advance_cbs_nowake(rdp->mynode, rdp); in __call_rcu_nocb_wake()
1910 rdp->nocb_gp_adv_time = j; in __call_rcu_nocb_wake()
1913 if ((rdp->nocb_cb_sleep || in __call_rcu_nocb_wake()
1914 !rcu_segcblist_ready_cbs(&rdp->cblist)) && in __call_rcu_nocb_wake()
1915 !timer_pending(&rdp->nocb_bypass_timer)) in __call_rcu_nocb_wake()
1916 wake_nocb_gp_defer(rdp, RCU_NOCB_WAKE_FORCE, in __call_rcu_nocb_wake()
1918 rcu_nocb_unlock_irqrestore(rdp, flags); in __call_rcu_nocb_wake()
1920 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WakeNot")); in __call_rcu_nocb_wake()
1921 rcu_nocb_unlock_irqrestore(rdp, flags); in __call_rcu_nocb_wake()
1930 struct rcu_data *rdp = from_timer(rdp, t, nocb_bypass_timer); in do_nocb_bypass_wakeup_timer() local
1932 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("Timer")); in do_nocb_bypass_wakeup_timer()
1933 rcu_nocb_lock_irqsave(rdp, flags); in do_nocb_bypass_wakeup_timer()
1935 __call_rcu_nocb_wake(rdp, true, flags); in do_nocb_bypass_wakeup_timer()
1954 struct rcu_data *rdp; in nocb_gp_wait() local
1963 for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_cb_rdp) { in nocb_gp_wait()
1964 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("Check")); in nocb_gp_wait()
1965 rcu_nocb_lock_irqsave(rdp, flags); in nocb_gp_wait()
1966 bypass_ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass); in nocb_gp_wait()
1968 (time_after(j, READ_ONCE(rdp->nocb_bypass_first) + 1) || in nocb_gp_wait()
1971 (void)rcu_nocb_try_flush_bypass(rdp, j); in nocb_gp_wait()
1972 bypass_ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass); in nocb_gp_wait()
1973 } else if (!bypass_ncbs && rcu_segcblist_empty(&rdp->cblist)) { in nocb_gp_wait()
1974 rcu_nocb_unlock_irqrestore(rdp, flags); in nocb_gp_wait()
1978 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in nocb_gp_wait()
1982 rnp = rdp->mynode; in nocb_gp_wait()
1990 if (!rcu_segcblist_restempty(&rdp->cblist, in nocb_gp_wait()
1992 (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) && in nocb_gp_wait()
1995 needwake_gp = rcu_advance_cbs(rnp, rdp); in nocb_gp_wait()
1999 WARN_ON_ONCE(!rcu_segcblist_restempty(&rdp->cblist, in nocb_gp_wait()
2001 if (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq)) { in nocb_gp_wait()
2006 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in nocb_gp_wait()
2009 if (rcu_segcblist_ready_cbs(&rdp->cblist)) { in nocb_gp_wait()
2010 needwake = rdp->nocb_cb_sleep; in nocb_gp_wait()
2011 WRITE_ONCE(rdp->nocb_cb_sleep, false); in nocb_gp_wait()
2016 rcu_nocb_unlock_irqrestore(rdp, flags); in nocb_gp_wait()
2018 swake_up_one(&rdp->nocb_cb_wq); in nocb_gp_wait()
2076 struct rcu_data *rdp = arg; in rcu_nocb_gp_kthread() local
2079 WRITE_ONCE(rdp->nocb_gp_loops, rdp->nocb_gp_loops + 1); in rcu_nocb_gp_kthread()
2080 nocb_gp_wait(rdp); in rcu_nocb_gp_kthread()
2090 static void nocb_cb_wait(struct rcu_data *rdp) in nocb_cb_wait() argument
2095 struct rcu_node *rnp = rdp->mynode; in nocb_cb_wait()
2101 rcu_do_batch(rdp); in nocb_cb_wait()
2104 rcu_nocb_lock_irqsave(rdp, flags); in nocb_cb_wait()
2105 if (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) && in nocb_cb_wait()
2108 needwake_gp = rcu_advance_cbs(rdp->mynode, rdp); in nocb_cb_wait()
2111 if (rcu_segcblist_ready_cbs(&rdp->cblist)) { in nocb_cb_wait()
2112 rcu_nocb_unlock_irqrestore(rdp, flags); in nocb_cb_wait()
2118 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("CBSleep")); in nocb_cb_wait()
2119 WRITE_ONCE(rdp->nocb_cb_sleep, true); in nocb_cb_wait()
2120 rcu_nocb_unlock_irqrestore(rdp, flags); in nocb_cb_wait()
2123 swait_event_interruptible_exclusive(rdp->nocb_cb_wq, in nocb_cb_wait()
2124 !READ_ONCE(rdp->nocb_cb_sleep)); in nocb_cb_wait()
2125 if (!smp_load_acquire(&rdp->nocb_cb_sleep)) { /* VVV */ in nocb_cb_wait()
2130 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WokeEmpty")); in nocb_cb_wait()
2139 struct rcu_data *rdp = arg; in rcu_nocb_cb_kthread() local
2144 nocb_cb_wait(rdp); in rcu_nocb_cb_kthread()
2151 static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp) in rcu_nocb_need_deferred_wakeup() argument
2153 return READ_ONCE(rdp->nocb_defer_wakeup); in rcu_nocb_need_deferred_wakeup()
2157 static void do_nocb_deferred_wakeup_common(struct rcu_data *rdp) in do_nocb_deferred_wakeup_common() argument
2162 rcu_nocb_lock_irqsave(rdp, flags); in do_nocb_deferred_wakeup_common()
2163 if (!rcu_nocb_need_deferred_wakeup(rdp)) { in do_nocb_deferred_wakeup_common()
2164 rcu_nocb_unlock_irqrestore(rdp, flags); in do_nocb_deferred_wakeup_common()
2167 ndw = READ_ONCE(rdp->nocb_defer_wakeup); in do_nocb_deferred_wakeup_common()
2168 WRITE_ONCE(rdp->nocb_defer_wakeup, RCU_NOCB_WAKE_NOT); in do_nocb_deferred_wakeup_common()
2169 wake_nocb_gp(rdp, ndw == RCU_NOCB_WAKE_FORCE, flags); in do_nocb_deferred_wakeup_common()
2170 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("DeferredWake")); in do_nocb_deferred_wakeup_common()
2176 struct rcu_data *rdp = from_timer(rdp, t, nocb_timer); in do_nocb_deferred_wakeup_timer() local
2178 do_nocb_deferred_wakeup_common(rdp); in do_nocb_deferred_wakeup_timer()
2186 static void do_nocb_deferred_wakeup(struct rcu_data *rdp) in do_nocb_deferred_wakeup() argument
2188 if (rcu_nocb_need_deferred_wakeup(rdp)) in do_nocb_deferred_wakeup()
2189 do_nocb_deferred_wakeup_common(rdp); in do_nocb_deferred_wakeup()
2196 struct rcu_data *rdp; in rcu_init_nohz() local
2231 rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_init_nohz()
2232 if (rcu_segcblist_empty(&rdp->cblist)) in rcu_init_nohz()
2233 rcu_segcblist_init(&rdp->cblist); in rcu_init_nohz()
2234 rcu_segcblist_offload(&rdp->cblist); in rcu_init_nohz()
2240 static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp) in rcu_boot_init_nocb_percpu_data() argument
2242 init_swait_queue_head(&rdp->nocb_cb_wq); in rcu_boot_init_nocb_percpu_data()
2243 init_swait_queue_head(&rdp->nocb_gp_wq); in rcu_boot_init_nocb_percpu_data()
2244 raw_spin_lock_init(&rdp->nocb_lock); in rcu_boot_init_nocb_percpu_data()
2245 raw_spin_lock_init(&rdp->nocb_bypass_lock); in rcu_boot_init_nocb_percpu_data()
2246 raw_spin_lock_init(&rdp->nocb_gp_lock); in rcu_boot_init_nocb_percpu_data()
2247 timer_setup(&rdp->nocb_timer, do_nocb_deferred_wakeup_timer, 0); in rcu_boot_init_nocb_percpu_data()
2248 timer_setup(&rdp->nocb_bypass_timer, do_nocb_bypass_wakeup_timer, 0); in rcu_boot_init_nocb_percpu_data()
2249 rcu_cblist_init(&rdp->nocb_bypass); in rcu_boot_init_nocb_percpu_data()
2259 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_spawn_one_nocb_kthread() local
2267 if (!rcu_is_nocb_cpu(cpu) || rdp->nocb_cb_kthread) in rcu_spawn_one_nocb_kthread()
2271 rdp_gp = rdp->nocb_gp_rdp; in rcu_spawn_one_nocb_kthread()
2281 t = kthread_run(rcu_nocb_cb_kthread, rdp, in rcu_spawn_one_nocb_kthread()
2285 WRITE_ONCE(rdp->nocb_cb_kthread, t); in rcu_spawn_one_nocb_kthread()
2286 WRITE_ONCE(rdp->nocb_gp_kthread, rdp_gp->nocb_gp_kthread); in rcu_spawn_one_nocb_kthread()
2326 struct rcu_data *rdp; in rcu_organize_nocb_kthreads() local
2343 rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_organize_nocb_kthreads()
2344 if (rdp->cpu >= nl) { in rcu_organize_nocb_kthreads()
2346 nl = DIV_ROUND_UP(rdp->cpu + 1, ls) * ls; in rcu_organize_nocb_kthreads()
2347 rdp->nocb_gp_rdp = rdp; in rcu_organize_nocb_kthreads()
2348 rdp_gp = rdp; in rcu_organize_nocb_kthreads()
2355 rdp->nocb_gp_rdp = rdp_gp; in rcu_organize_nocb_kthreads()
2356 rdp_prev->nocb_next_cb_rdp = rdp; in rcu_organize_nocb_kthreads()
2359 rdp_prev = rdp; in rcu_organize_nocb_kthreads()
2378 static void show_rcu_nocb_gp_state(struct rcu_data *rdp) in show_rcu_nocb_gp_state() argument
2380 struct rcu_node *rnp = rdp->mynode; in show_rcu_nocb_gp_state()
2383 rdp->cpu, in show_rcu_nocb_gp_state()
2384 "kK"[!!rdp->nocb_gp_kthread], in show_rcu_nocb_gp_state()
2385 "lL"[raw_spin_is_locked(&rdp->nocb_gp_lock)], in show_rcu_nocb_gp_state()
2386 "dD"[!!rdp->nocb_defer_wakeup], in show_rcu_nocb_gp_state()
2387 "tT"[timer_pending(&rdp->nocb_timer)], in show_rcu_nocb_gp_state()
2388 "bB"[timer_pending(&rdp->nocb_bypass_timer)], in show_rcu_nocb_gp_state()
2389 "sS"[!!rdp->nocb_gp_sleep], in show_rcu_nocb_gp_state()
2390 ".W"[swait_active(&rdp->nocb_gp_wq)], in show_rcu_nocb_gp_state()
2393 ".B"[!!rdp->nocb_gp_bypass], in show_rcu_nocb_gp_state()
2394 ".G"[!!rdp->nocb_gp_gp], in show_rcu_nocb_gp_state()
2395 (long)rdp->nocb_gp_seq, in show_rcu_nocb_gp_state()
2396 rnp->grplo, rnp->grphi, READ_ONCE(rdp->nocb_gp_loops)); in show_rcu_nocb_gp_state()
2400 static void show_rcu_nocb_state(struct rcu_data *rdp) in show_rcu_nocb_state() argument
2402 struct rcu_segcblist *rsclp = &rdp->cblist; in show_rcu_nocb_state()
2407 if (rdp->nocb_gp_rdp == rdp) in show_rcu_nocb_state()
2408 show_rcu_nocb_gp_state(rdp); in show_rcu_nocb_state()
2411 rdp->cpu, rdp->nocb_gp_rdp->cpu, in show_rcu_nocb_state()
2412 "kK"[!!rdp->nocb_cb_kthread], in show_rcu_nocb_state()
2413 "bB"[raw_spin_is_locked(&rdp->nocb_bypass_lock)], in show_rcu_nocb_state()
2414 "cC"[!!atomic_read(&rdp->nocb_lock_contended)], in show_rcu_nocb_state()
2415 "lL"[raw_spin_is_locked(&rdp->nocb_lock)], in show_rcu_nocb_state()
2416 "sS"[!!rdp->nocb_cb_sleep], in show_rcu_nocb_state()
2417 ".W"[swait_active(&rdp->nocb_cb_wq)], in show_rcu_nocb_state()
2418 jiffies - rdp->nocb_bypass_first, in show_rcu_nocb_state()
2419 jiffies - rdp->nocb_nobypass_last, in show_rcu_nocb_state()
2420 rdp->nocb_nobypass_count, in show_rcu_nocb_state()
2425 ".B"[!!rcu_cblist_n_cbs(&rdp->nocb_bypass)], in show_rcu_nocb_state()
2426 rcu_segcblist_n_cbs(&rdp->cblist)); in show_rcu_nocb_state()
2429 if (rdp->nocb_gp_rdp == rdp) in show_rcu_nocb_state()
2432 waslocked = raw_spin_is_locked(&rdp->nocb_gp_lock); in show_rcu_nocb_state()
2433 wastimer = timer_pending(&rdp->nocb_timer); in show_rcu_nocb_state()
2434 wassleep = swait_active(&rdp->nocb_gp_wq); in show_rcu_nocb_state()
2435 if (!rdp->nocb_defer_wakeup && !rdp->nocb_gp_sleep && in show_rcu_nocb_state()
2441 "dD"[!!rdp->nocb_defer_wakeup], in show_rcu_nocb_state()
2443 "sS"[!!rdp->nocb_gp_sleep], in show_rcu_nocb_state()
2450 static void rcu_nocb_lock(struct rcu_data *rdp) in rcu_nocb_lock() argument
2455 static void rcu_nocb_unlock(struct rcu_data *rdp) in rcu_nocb_unlock() argument
2460 static void rcu_nocb_unlock_irqrestore(struct rcu_data *rdp, in rcu_nocb_unlock_irqrestore() argument
2467 static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp) in rcu_lockdep_assert_cblist_protected() argument
2485 static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, in rcu_nocb_flush_bypass() argument
2491 static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp, in rcu_nocb_try_bypass() argument
2497 static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_empty, in __call_rcu_nocb_wake() argument
2503 static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp) in rcu_boot_init_nocb_percpu_data() argument
2507 static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp) in rcu_nocb_need_deferred_wakeup() argument
2512 static void do_nocb_deferred_wakeup(struct rcu_data *rdp) in do_nocb_deferred_wakeup() argument
2524 static void show_rcu_nocb_state(struct rcu_data *rdp) in show_rcu_nocb_state() argument