Lines Matching refs:cblist
82 .cblist.flags = SEGCBLIST_SOFTIRQ_ONLY,
242 if (rcu_segcblist_is_enabled(&rdp->cblist)) in rcu_get_n_cbs_cpu()
243 return rcu_segcblist_n_cbs(&rdp->cblist); in rcu_get_n_cbs_cpu()
1503 if (!rcu_segcblist_pend_cbs(&rdp->cblist)) in rcu_accelerate_cbs()
1506 trace_rcu_segcb_stats(&rdp->cblist, TPS("SegCbPreAcc")); in rcu_accelerate_cbs()
1519 if (rcu_segcblist_accelerate(&rdp->cblist, gp_seq_req)) in rcu_accelerate_cbs()
1523 if (rcu_segcblist_restempty(&rdp->cblist, RCU_WAIT_TAIL)) in rcu_accelerate_cbs()
1528 trace_rcu_segcb_stats(&rdp->cblist, TPS("SegCbPostAcc")); in rcu_accelerate_cbs()
1550 (void)rcu_segcblist_accelerate(&rdp->cblist, c); in rcu_accelerate_cbs_unlocked()
1576 if (!rcu_segcblist_pend_cbs(&rdp->cblist)) in rcu_advance_cbs()
1583 rcu_segcblist_advance(&rdp->cblist, rnp->gp_seq); in rcu_advance_cbs()
2456 if (!rcu_segcblist_ready_cbs(&rdp->cblist)) { in rcu_do_batch()
2458 rcu_segcblist_n_cbs(&rdp->cblist), 0); in rcu_do_batch()
2460 !rcu_segcblist_empty(&rdp->cblist), in rcu_do_batch()
2474 pending = rcu_segcblist_n_cbs(&rdp->cblist); in rcu_do_batch()
2485 rcu_segcblist_n_cbs(&rdp->cblist), bl); in rcu_do_batch()
2486 rcu_segcblist_extract_done_cbs(&rdp->cblist, &rcl); in rcu_do_batch()
2488 rdp->qlen_last_fqs_check = rcu_segcblist_n_cbs(&rdp->cblist); in rcu_do_batch()
2490 trace_rcu_segcb_stats(&rdp->cblist, TPS("SegCbDequeued")); in rcu_do_batch()
2542 rcu_segcblist_insert_done_cbs(&rdp->cblist, &rcl); in rcu_do_batch()
2543 rcu_segcblist_add_len(&rdp->cblist, -count); in rcu_do_batch()
2546 count = rcu_segcblist_n_cbs(&rdp->cblist); in rcu_do_batch()
2561 empty = rcu_segcblist_empty(&rdp->cblist); in rcu_do_batch()
2565 WARN_ON_ONCE(count == 0 && rcu_segcblist_n_segment_cbs(&rdp->cblist) != 0); in rcu_do_batch()
2566 WARN_ON_ONCE(!empty && rcu_segcblist_n_segment_cbs(&rdp->cblist) == 0); in rcu_do_batch()
2571 if (!offloaded && rcu_segcblist_ready_cbs(&rdp->cblist)) in rcu_do_batch()
2711 const bool do_batch = !rcu_segcblist_completely_offloaded(&rdp->cblist); in rcu_core()
2731 rcu_segcblist_is_enabled(&rdp->cblist) && do_batch) { in rcu_core()
2733 if (!rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL)) in rcu_core()
2741 if (do_batch && rcu_segcblist_ready_cbs(&rdp->cblist) && in rcu_core()
2889 if (unlikely(rcu_segcblist_n_cbs(&rdp->cblist) > in __call_rcu_core()
2902 rcu_segcblist_first_pend_cb(&rdp->cblist) != head) in __call_rcu_core()
2905 rdp->qlen_last_fqs_check = rcu_segcblist_n_cbs(&rdp->cblist); in __call_rcu_core()
2928 if (rcu_segcblist_n_cbs(&rdp->cblist) >= qovld_calc) in check_cb_ovld_locked()
2951 ((rcu_segcblist_n_cbs(&rdp->cblist) >= qovld_calc) == in check_cb_ovld()
2991 if (unlikely(!rcu_segcblist_is_enabled(&rdp->cblist))) { in __call_rcu()
2997 if (rcu_segcblist_empty(&rdp->cblist)) in __call_rcu()
2998 rcu_segcblist_init(&rdp->cblist); in __call_rcu()
3005 rcu_segcblist_enqueue(&rdp->cblist, head); in __call_rcu()
3009 rcu_segcblist_n_cbs(&rdp->cblist)); in __call_rcu()
3012 rcu_segcblist_n_cbs(&rdp->cblist)); in __call_rcu()
3014 trace_rcu_segcb_stats(&rdp->cblist, TPS("SegCBQueued")); in __call_rcu()
3897 rcu_segcblist_ready_cbs(&rdp->cblist)) in rcu_pending()
3901 if (!gp_in_progress && rcu_segcblist_is_enabled(&rdp->cblist) && in rcu_pending()
3903 !rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL)) in rcu_pending()
3960 if (rcu_segcblist_entrain(&rdp->cblist, &rdp->barrier_head)) { in rcu_barrier_func()
4023 if (rcu_segcblist_n_cbs(&rdp->cblist) && cpu_online(cpu)) { in rcu_barrier()
4027 } else if (rcu_segcblist_n_cbs(&rdp->cblist) && in rcu_barrier()
4141 if (!rcu_segcblist_is_enabled(&rdp->cblist)) in rcutree_prepare_cpu()
4142 rcu_segcblist_init(&rdp->cblist); /* Re-enable callbacks. */ in rcutree_prepare_cpu()
4343 rcu_segcblist_empty(&rdp->cblist)) in rcutree_migrate_callbacks()
4355 rcu_segcblist_merge(&my_rdp->cblist, &rdp->cblist); in rcutree_migrate_callbacks()
4357 rcu_segcblist_disable(&rdp->cblist); in rcutree_migrate_callbacks()
4358 WARN_ON_ONCE(rcu_segcblist_empty(&my_rdp->cblist) != in rcutree_migrate_callbacks()
4359 !rcu_segcblist_n_cbs(&my_rdp->cblist)); in rcutree_migrate_callbacks()
4370 WARN_ONCE(rcu_segcblist_n_cbs(&rdp->cblist) != 0 || in rcutree_migrate_callbacks()
4371 !rcu_segcblist_empty(&rdp->cblist), in rcutree_migrate_callbacks()
4373 cpu, rcu_segcblist_n_cbs(&rdp->cblist), in rcutree_migrate_callbacks()
4374 rcu_segcblist_first_cb(&rdp->cblist)); in rcutree_migrate_callbacks()