Lines Matching refs:cblist

214 	if (rcu_segcblist_is_enabled(&rdp->cblist))  in rcu_get_n_cbs_cpu()
215 return rcu_segcblist_n_cbs(&rdp->cblist); in rcu_get_n_cbs_cpu()
1265 if (!rcu_segcblist_pend_cbs(&rdp->cblist)) in rcu_accelerate_cbs()
1279 if (rcu_segcblist_accelerate(&rdp->cblist, gp_seq_req)) in rcu_accelerate_cbs()
1283 if (rcu_segcblist_restempty(&rdp->cblist, RCU_WAIT_TAIL)) in rcu_accelerate_cbs()
1307 (void)rcu_segcblist_accelerate(&rdp->cblist, c); in rcu_accelerate_cbs_unlocked()
1333 if (!rcu_segcblist_pend_cbs(&rdp->cblist)) in rcu_advance_cbs()
1340 rcu_segcblist_advance(&rdp->cblist, rnp->gp_seq); in rcu_advance_cbs()
1372 rcu_segcblist_is_offloaded(&rdp->cblist); in __note_gp_changes()
1755 rcu_segcblist_is_offloaded(&rdp->cblist); in rcu_gp_cleanup()
1952 rcu_segcblist_is_offloaded(&rdp->cblist); in rcu_report_qs_rdp()
2115 rcu_segcblist_is_offloaded(&rdp->cblist); in rcu_do_batch()
2122 if (!rcu_segcblist_ready_cbs(&rdp->cblist)) { in rcu_do_batch()
2124 rcu_segcblist_n_lazy_cbs(&rdp->cblist), in rcu_do_batch()
2125 rcu_segcblist_n_cbs(&rdp->cblist), 0); in rcu_do_batch()
2127 !rcu_segcblist_empty(&rdp->cblist), in rcu_do_batch()
2141 pending = rcu_segcblist_n_cbs(&rdp->cblist); in rcu_do_batch()
2146 rcu_segcblist_n_lazy_cbs(&rdp->cblist), in rcu_do_batch()
2147 rcu_segcblist_n_cbs(&rdp->cblist), bl); in rcu_do_batch()
2148 rcu_segcblist_extract_done_cbs(&rdp->cblist, &rcl); in rcu_do_batch()
2150 rdp->qlen_last_fqs_check = rcu_segcblist_n_cbs(&rdp->cblist); in rcu_do_batch()
2191 rcu_segcblist_insert_done_cbs(&rdp->cblist, &rcl); in rcu_do_batch()
2193 rcu_segcblist_insert_count(&rdp->cblist, &rcl); in rcu_do_batch()
2196 count = rcu_segcblist_n_cbs(&rdp->cblist); in rcu_do_batch()
2211 WARN_ON_ONCE(count == 0 && !rcu_segcblist_empty(&rdp->cblist)); in rcu_do_batch()
2213 count != 0 && rcu_segcblist_empty(&rdp->cblist)); in rcu_do_batch()
2218 if (!offloaded && rcu_segcblist_ready_cbs(&rdp->cblist)) in rcu_do_batch()
2345 rcu_segcblist_is_offloaded(&rdp->cblist); in rcu_core()
2365 rcu_segcblist_is_enabled(&rdp->cblist) && !offloaded) { in rcu_core()
2367 if (!rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL)) in rcu_core()
2375 if (!offloaded && rcu_segcblist_ready_cbs(&rdp->cblist) && in rcu_core()
2520 if (unlikely(rcu_segcblist_n_cbs(&rdp->cblist) > in __call_rcu_core()
2533 rcu_segcblist_first_pend_cb(&rdp->cblist) != head) in __call_rcu_core()
2536 rdp->qlen_last_fqs_check = rcu_segcblist_n_cbs(&rdp->cblist); in __call_rcu_core()
2581 if (unlikely(!rcu_segcblist_is_enabled(&rdp->cblist))) { in __call_rcu()
2587 if (rcu_segcblist_empty(&rdp->cblist)) in __call_rcu()
2588 rcu_segcblist_init(&rdp->cblist); in __call_rcu()
2593 rcu_segcblist_enqueue(&rdp->cblist, head, lazy); in __call_rcu()
2597 rcu_segcblist_n_lazy_cbs(&rdp->cblist), in __call_rcu()
2598 rcu_segcblist_n_cbs(&rdp->cblist)); in __call_rcu()
2601 rcu_segcblist_n_lazy_cbs(&rdp->cblist), in __call_rcu()
2602 rcu_segcblist_n_cbs(&rdp->cblist)); in __call_rcu()
2606 unlikely(rcu_segcblist_is_offloaded(&rdp->cblist))) { in __call_rcu()
2810 if (rcu_segcblist_ready_cbs(&rdp->cblist)) in rcu_pending()
2815 rcu_segcblist_is_enabled(&rdp->cblist) && in rcu_pending()
2817 !rcu_segcblist_is_offloaded(&rdp->cblist)) && in rcu_pending()
2818 !rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL)) in rcu_pending()
2867 if (rcu_segcblist_entrain(&rdp->cblist, &rdp->barrier_head, 0)) { in rcu_barrier_func()
2927 !rcu_segcblist_is_offloaded(&rdp->cblist)) in rcu_barrier()
2929 if (rcu_segcblist_n_cbs(&rdp->cblist)) { in rcu_barrier()
3028 if (rcu_segcblist_empty(&rdp->cblist) && /* No early-boot CBs? */ in rcutree_prepare_cpu()
3029 !rcu_segcblist_is_offloaded(&rdp->cblist)) in rcutree_prepare_cpu()
3030 rcu_segcblist_init(&rdp->cblist); /* Re-enable callbacks. */ in rcutree_prepare_cpu()
3212 if (rcu_segcblist_is_offloaded(&rdp->cblist) || in rcutree_migrate_callbacks()
3213 rcu_segcblist_empty(&rdp->cblist)) in rcutree_migrate_callbacks()
3225 rcu_segcblist_merge(&my_rdp->cblist, &rdp->cblist); in rcutree_migrate_callbacks()
3227 rcu_segcblist_disable(&rdp->cblist); in rcutree_migrate_callbacks()
3228 WARN_ON_ONCE(rcu_segcblist_empty(&my_rdp->cblist) != in rcutree_migrate_callbacks()
3229 !rcu_segcblist_n_cbs(&my_rdp->cblist)); in rcutree_migrate_callbacks()
3230 if (rcu_segcblist_is_offloaded(&my_rdp->cblist)) { in rcutree_migrate_callbacks()
3240 WARN_ONCE(rcu_segcblist_n_cbs(&rdp->cblist) != 0 || in rcutree_migrate_callbacks()
3241 !rcu_segcblist_empty(&rdp->cblist), in rcutree_migrate_callbacks()
3243 cpu, rcu_segcblist_n_cbs(&rdp->cblist), in rcutree_migrate_callbacks()
3244 rcu_segcblist_first_cb(&rdp->cblist)); in rcutree_migrate_callbacks()