Lines Matching refs:RCU_DONE_TAIL
185 &rsclp->head != READ_ONCE(rsclp->tails[RCU_DONE_TAIL]); in rcu_segcblist_ready_cbs()
195 !rcu_segcblist_restempty(rsclp, RCU_DONE_TAIL); in rcu_segcblist_pend_cbs()
219 return *rsclp->tails[RCU_DONE_TAIL]; in rcu_segcblist_first_pend_cb()
274 for (i = RCU_NEXT_TAIL; i > RCU_DONE_TAIL; i--) in rcu_segcblist_entrain()
311 WRITE_ONCE(rsclp->head, *rsclp->tails[RCU_DONE_TAIL]); in rcu_segcblist_extract_done_cbs()
312 WRITE_ONCE(*rsclp->tails[RCU_DONE_TAIL], NULL); in rcu_segcblist_extract_done_cbs()
313 rclp->tail = rsclp->tails[RCU_DONE_TAIL]; in rcu_segcblist_extract_done_cbs()
314 for (i = RCU_CBLIST_NSEGS - 1; i >= RCU_DONE_TAIL; i--) in rcu_segcblist_extract_done_cbs()
315 if (rsclp->tails[i] == rsclp->tails[RCU_DONE_TAIL]) in rcu_segcblist_extract_done_cbs()
333 *rclp->tail = *rsclp->tails[RCU_DONE_TAIL]; in rcu_segcblist_extract_pend_cbs()
335 WRITE_ONCE(*rsclp->tails[RCU_DONE_TAIL], NULL); in rcu_segcblist_extract_pend_cbs()
336 for (i = RCU_DONE_TAIL + 1; i < RCU_CBLIST_NSEGS; i++) in rcu_segcblist_extract_pend_cbs()
337 WRITE_ONCE(rsclp->tails[i], rsclp->tails[RCU_DONE_TAIL]); in rcu_segcblist_extract_pend_cbs()
364 for (i = RCU_DONE_TAIL; i < RCU_CBLIST_NSEGS; i++) in rcu_segcblist_insert_done_cbs()
395 if (rcu_segcblist_restempty(rsclp, RCU_DONE_TAIL)) in rcu_segcblist_advance()
405 WRITE_ONCE(rsclp->tails[RCU_DONE_TAIL], rsclp->tails[i]); in rcu_segcblist_advance()
414 WRITE_ONCE(rsclp->tails[j], rsclp->tails[RCU_DONE_TAIL]); in rcu_segcblist_advance()
450 if (rcu_segcblist_restempty(rsclp, RCU_DONE_TAIL)) in rcu_segcblist_accelerate()
461 for (i = RCU_NEXT_READY_TAIL; i > RCU_DONE_TAIL; i--) in rcu_segcblist_accelerate()