Lines Matching refs:cblist
230 if (rcu_segcblist_is_enabled(&rdp->cblist)) in rcu_get_n_cbs_cpu()
231 return rcu_segcblist_n_cbs(&rdp->cblist); in rcu_get_n_cbs_cpu()
1486 if (!rcu_segcblist_pend_cbs(&rdp->cblist)) in rcu_accelerate_cbs()
1500 if (rcu_segcblist_accelerate(&rdp->cblist, gp_seq_req)) in rcu_accelerate_cbs()
1504 if (rcu_segcblist_restempty(&rdp->cblist, RCU_WAIT_TAIL)) in rcu_accelerate_cbs()
1529 (void)rcu_segcblist_accelerate(&rdp->cblist, c); in rcu_accelerate_cbs_unlocked()
1555 if (!rcu_segcblist_pend_cbs(&rdp->cblist)) in rcu_advance_cbs()
1562 rcu_segcblist_advance(&rdp->cblist, rnp->gp_seq); in rcu_advance_cbs()
1607 rcu_segcblist_is_offloaded(&rdp->cblist); in __note_gp_changes()
2052 rcu_segcblist_is_offloaded(&rdp->cblist); in rcu_gp_cleanup()
2252 rcu_segcblist_is_offloaded(&rdp->cblist); in rcu_report_qs_rdp()
2421 rcu_segcblist_is_offloaded(&rdp->cblist); in rcu_do_batch()
2428 if (!rcu_segcblist_ready_cbs(&rdp->cblist)) { in rcu_do_batch()
2430 rcu_segcblist_n_cbs(&rdp->cblist), 0); in rcu_do_batch()
2432 !rcu_segcblist_empty(&rdp->cblist), in rcu_do_batch()
2446 pending = rcu_segcblist_n_cbs(&rdp->cblist); in rcu_do_batch()
2457 rcu_segcblist_n_cbs(&rdp->cblist), bl); in rcu_do_batch()
2458 rcu_segcblist_extract_done_cbs(&rdp->cblist, &rcl); in rcu_do_batch()
2460 rdp->qlen_last_fqs_check = rcu_segcblist_n_cbs(&rdp->cblist); in rcu_do_batch()
2513 rcu_segcblist_insert_done_cbs(&rdp->cblist, &rcl); in rcu_do_batch()
2515 rcu_segcblist_insert_count(&rdp->cblist, &rcl); in rcu_do_batch()
2518 count = rcu_segcblist_n_cbs(&rdp->cblist); in rcu_do_batch()
2533 WARN_ON_ONCE(count == 0 && !rcu_segcblist_empty(&rdp->cblist)); in rcu_do_batch()
2535 count != 0 && rcu_segcblist_empty(&rdp->cblist)); in rcu_do_batch()
2540 if (!offloaded && rcu_segcblist_ready_cbs(&rdp->cblist)) in rcu_do_batch()
2679 rcu_segcblist_is_offloaded(&rdp->cblist); in rcu_core()
2699 rcu_segcblist_is_enabled(&rdp->cblist) && !offloaded) { in rcu_core()
2701 if (!rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL)) in rcu_core()
2709 if (!offloaded && rcu_segcblist_ready_cbs(&rdp->cblist) && in rcu_core()
2858 if (unlikely(rcu_segcblist_n_cbs(&rdp->cblist) > in __call_rcu_core()
2871 rcu_segcblist_first_pend_cb(&rdp->cblist) != head) in __call_rcu_core()
2874 rdp->qlen_last_fqs_check = rcu_segcblist_n_cbs(&rdp->cblist); in __call_rcu_core()
2897 if (rcu_segcblist_n_cbs(&rdp->cblist) >= qovld_calc) in check_cb_ovld_locked()
2920 ((rcu_segcblist_n_cbs(&rdp->cblist) >= qovld_calc) == in check_cb_ovld()
2957 if (unlikely(!rcu_segcblist_is_enabled(&rdp->cblist))) { in __call_rcu()
2963 if (rcu_segcblist_empty(&rdp->cblist)) in __call_rcu()
2964 rcu_segcblist_init(&rdp->cblist); in __call_rcu()
2971 rcu_segcblist_enqueue(&rdp->cblist, head); in __call_rcu()
2975 rcu_segcblist_n_cbs(&rdp->cblist)); in __call_rcu()
2978 rcu_segcblist_n_cbs(&rdp->cblist)); in __call_rcu()
2982 unlikely(rcu_segcblist_is_offloaded(&rdp->cblist))) { in __call_rcu()
3710 if (rcu_segcblist_ready_cbs(&rdp->cblist)) in rcu_pending()
3714 if (!gp_in_progress && rcu_segcblist_is_enabled(&rdp->cblist) && in rcu_pending()
3716 !rcu_segcblist_is_offloaded(&rdp->cblist)) && in rcu_pending()
3717 !rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL)) in rcu_pending()
3774 if (rcu_segcblist_entrain(&rdp->cblist, &rdp->barrier_head)) { in rcu_barrier_func()
3835 !rcu_segcblist_is_offloaded(&rdp->cblist)) in rcu_barrier()
3837 if (rcu_segcblist_n_cbs(&rdp->cblist) && cpu_online(cpu)) { in rcu_barrier()
3841 } else if (rcu_segcblist_n_cbs(&rdp->cblist) && in rcu_barrier()
3947 if (rcu_segcblist_empty(&rdp->cblist) && /* No early-boot CBs? */ in rcutree_prepare_cpu()
3948 !rcu_segcblist_is_offloaded(&rdp->cblist)) in rcutree_prepare_cpu()
3949 rcu_segcblist_init(&rdp->cblist); /* Re-enable callbacks. */ in rcutree_prepare_cpu()
4133 if (rcu_segcblist_is_offloaded(&rdp->cblist) || in rcutree_migrate_callbacks()
4134 rcu_segcblist_empty(&rdp->cblist)) in rcutree_migrate_callbacks()
4146 rcu_segcblist_merge(&my_rdp->cblist, &rdp->cblist); in rcutree_migrate_callbacks()
4148 rcu_segcblist_disable(&rdp->cblist); in rcutree_migrate_callbacks()
4149 WARN_ON_ONCE(rcu_segcblist_empty(&my_rdp->cblist) != in rcutree_migrate_callbacks()
4150 !rcu_segcblist_n_cbs(&my_rdp->cblist)); in rcutree_migrate_callbacks()
4151 if (rcu_segcblist_is_offloaded(&my_rdp->cblist)) { in rcutree_migrate_callbacks()
4161 WARN_ONCE(rcu_segcblist_n_cbs(&rdp->cblist) != 0 || in rcutree_migrate_callbacks()
4162 !rcu_segcblist_empty(&rdp->cblist), in rcutree_migrate_callbacks()
4164 cpu, rcu_segcblist_n_cbs(&rdp->cblist), in rcutree_migrate_callbacks()
4165 rcu_segcblist_first_cb(&rdp->cblist)); in rcutree_migrate_callbacks()