1 /* SPDX-License-Identifier: GPL-2.0+ */
2 /*
3 * RCU segmented callback lists, internal-to-rcu header file
4 *
5 * Copyright IBM Corporation, 2017
6 *
7 * Authors: Paul E. McKenney <paulmck@linux.ibm.com>
8 */
9
10 #include <linux/rcu_segcblist.h>
11
12 /* Return number of callbacks in the specified callback list. */
rcu_cblist_n_cbs(struct rcu_cblist * rclp)13 static inline long rcu_cblist_n_cbs(struct rcu_cblist *rclp)
14 {
15 return READ_ONCE(rclp->len);
16 }
17
18 void rcu_cblist_init(struct rcu_cblist *rclp);
19 void rcu_cblist_enqueue(struct rcu_cblist *rclp, struct rcu_head *rhp);
20 void rcu_cblist_flush_enqueue(struct rcu_cblist *drclp,
21 struct rcu_cblist *srclp,
22 struct rcu_head *rhp);
23 struct rcu_head *rcu_cblist_dequeue(struct rcu_cblist *rclp);
24
25 /*
26 * Is the specified rcu_segcblist structure empty?
27 *
28 * But careful! The fact that the ->head field is NULL does not
29 * necessarily imply that there are no callbacks associated with
30 * this structure. When callbacks are being invoked, they are
31 * removed as a group. If callback invocation must be preempted,
32 * the remaining callbacks will be added back to the list. Either
33 * way, the counts are updated later.
34 *
35 * So it is often the case that rcu_segcblist_n_cbs() should be used
36 * instead.
37 */
rcu_segcblist_empty(struct rcu_segcblist * rsclp)38 static inline bool rcu_segcblist_empty(struct rcu_segcblist *rsclp)
39 {
40 return !READ_ONCE(rsclp->head);
41 }
42
43 /* Return number of callbacks in segmented callback list. */
rcu_segcblist_n_cbs(struct rcu_segcblist * rsclp)44 static inline long rcu_segcblist_n_cbs(struct rcu_segcblist *rsclp)
45 {
46 #ifdef CONFIG_RCU_NOCB_CPU
47 return atomic_long_read(&rsclp->len);
48 #else
49 return READ_ONCE(rsclp->len);
50 #endif
51 }
52
53 /*
54 * Is the specified rcu_segcblist enabled, for example, not corresponding
55 * to an offline CPU?
56 */
rcu_segcblist_is_enabled(struct rcu_segcblist * rsclp)57 static inline bool rcu_segcblist_is_enabled(struct rcu_segcblist *rsclp)
58 {
59 return rsclp->enabled;
60 }
61
62 /* Is the specified rcu_segcblist offloaded? */
rcu_segcblist_is_offloaded(struct rcu_segcblist * rsclp)63 static inline bool rcu_segcblist_is_offloaded(struct rcu_segcblist *rsclp)
64 {
65 return rsclp->offloaded;
66 }
67
68 /*
69 * Are all segments following the specified segment of the specified
70 * rcu_segcblist structure empty of callbacks? (The specified
71 * segment might well contain callbacks.)
72 */
rcu_segcblist_restempty(struct rcu_segcblist * rsclp,int seg)73 static inline bool rcu_segcblist_restempty(struct rcu_segcblist *rsclp, int seg)
74 {
75 return !READ_ONCE(*READ_ONCE(rsclp->tails[seg]));
76 }
77
78 void rcu_segcblist_inc_len(struct rcu_segcblist *rsclp);
79 void rcu_segcblist_init(struct rcu_segcblist *rsclp);
80 void rcu_segcblist_disable(struct rcu_segcblist *rsclp);
81 void rcu_segcblist_offload(struct rcu_segcblist *rsclp);
82 bool rcu_segcblist_ready_cbs(struct rcu_segcblist *rsclp);
83 bool rcu_segcblist_pend_cbs(struct rcu_segcblist *rsclp);
84 struct rcu_head *rcu_segcblist_first_cb(struct rcu_segcblist *rsclp);
85 struct rcu_head *rcu_segcblist_first_pend_cb(struct rcu_segcblist *rsclp);
86 bool rcu_segcblist_nextgp(struct rcu_segcblist *rsclp, unsigned long *lp);
87 void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp,
88 struct rcu_head *rhp);
89 bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp,
90 struct rcu_head *rhp);
91 void rcu_segcblist_extract_count(struct rcu_segcblist *rsclp,
92 struct rcu_cblist *rclp);
93 void rcu_segcblist_extract_done_cbs(struct rcu_segcblist *rsclp,
94 struct rcu_cblist *rclp);
95 void rcu_segcblist_extract_pend_cbs(struct rcu_segcblist *rsclp,
96 struct rcu_cblist *rclp);
97 void rcu_segcblist_insert_count(struct rcu_segcblist *rsclp,
98 struct rcu_cblist *rclp);
99 void rcu_segcblist_insert_done_cbs(struct rcu_segcblist *rsclp,
100 struct rcu_cblist *rclp);
101 void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp,
102 struct rcu_cblist *rclp);
103 void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq);
104 bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq);
105 void rcu_segcblist_merge(struct rcu_segcblist *dst_rsclp,
106 struct rcu_segcblist *src_rsclp);
107
