21  * puts the ref back in single atomic_t mode, collecting the per cpu refs and
22  * issuing the appropriate barriers, and then marks the ref as shutting down so
23  * that percpu_ref_put() will check for the ref hitting 0.  After it returns,
24  * it's safe to drop the initial ref.
35  * and it's then safe to drop the initial ref with percpu_ref_put().
74 	 * Start w/ ref == 1 in atomic mode.  Can be switched to percpu
76 	 * with this flag, the ref will stay in atomic mode until
83 	 * Start dead w/ ref == 0 in atomic mode.  Must be revived with
98 	 * The low bit of the pointer indicates whether the ref is in percpu
109 int __must_check percpu_ref_init(struct percpu_ref *ref,
112 void percpu_ref_exit(struct percpu_ref *ref);
113 void percpu_ref_switch_to_atomic(struct percpu_ref *ref,
115 void percpu_ref_switch_to_atomic_sync(struct percpu_ref *ref);
116 void percpu_ref_switch_to_percpu(struct percpu_ref *ref);
117 void percpu_ref_kill_and_confirm(struct percpu_ref *ref,
119 void percpu_ref_resurrect(struct percpu_ref *ref);
120 void percpu_ref_reinit(struct percpu_ref *ref);
123  * percpu_ref_kill - drop the initial ref
124  * @ref: percpu_ref to kill
126  * Must be used to drop the initial ref on a percpu refcount; must be called
129  * Switches @ref into atomic mode before gathering up the percpu counters
130  * and dropping the initial ref.
134 static inline void percpu_ref_kill(struct percpu_ref *ref)  in percpu_ref_kill()  argument
136 	percpu_ref_kill_and_confirm(ref, NULL);  in percpu_ref_kill()
143  * branches as it can't assume that @ref->percpu_count is not NULL.
145 static inline bool __ref_is_percpu(struct percpu_ref *ref,  in __ref_is_percpu()  argument
151 	 * The value of @ref->percpu_count_ptr is tested for  in __ref_is_percpu()
161 	percpu_ptr = READ_ONCE(ref->percpu_count_ptr);  in __ref_is_percpu()
178  * @ref: percpu_ref to get
183  * This function is safe to call as long as @ref is between init and exit.
185 static inline void percpu_ref_get_many(struct percpu_ref *ref, unsigned long nr)  in percpu_ref_get_many()  argument
191 	if (__ref_is_percpu(ref, &percpu_count))  in percpu_ref_get_many()
194 		atomic_long_add(nr, &ref->count);  in percpu_ref_get_many()
201  * @ref: percpu_ref to get
205  * This function is safe to call as long as @ref is between init and exit.
207 static inline void percpu_ref_get(struct percpu_ref *ref)  in percpu_ref_get()  argument
209 	percpu_ref_get_many(ref, 1);  in percpu_ref_get()
214  * @ref: percpu_ref to try-get
219  * This function is safe to call as long as @ref is between init and exit.
221 static inline bool percpu_ref_tryget(struct percpu_ref *ref)  in percpu_ref_tryget()  argument
228 	if (__ref_is_percpu(ref, &percpu_count)) {  in percpu_ref_tryget()
232 		ret = atomic_long_inc_not_zero(&ref->count);  in percpu_ref_tryget()
242  * @ref: percpu_ref to try-get
253  * This function is safe to call as long as @ref is between init and exit.
255 static inline bool percpu_ref_tryget_live(struct percpu_ref *ref)  in percpu_ref_tryget_live()  argument
262 	if (__ref_is_percpu(ref, &percpu_count)) {  in percpu_ref_tryget_live()
265 	} else if (!(ref->percpu_count_ptr & __PERCPU_REF_DEAD)) {  in percpu_ref_tryget_live()
266 		ret = atomic_long_inc_not_zero(&ref->count);  in percpu_ref_tryget_live()
276  * @ref: percpu_ref to put
282  * This function is safe to call as long as @ref is between init and exit.
284 static inline void percpu_ref_put_many(struct percpu_ref *ref, unsigned long nr)  in percpu_ref_put_many()  argument
290 	if (__ref_is_percpu(ref, &percpu_count))  in percpu_ref_put_many()
292 	else if (unlikely(atomic_long_sub_and_test(nr, &ref->count)))  in percpu_ref_put_many()
293 		ref->release(ref);  in percpu_ref_put_many()
300  * @ref: percpu_ref to put
305  * This function is safe to call as long as @ref is between init and exit.
307 static inline void percpu_ref_put(struct percpu_ref *ref)  in percpu_ref_put()  argument
309 	percpu_ref_put_many(ref, 1);  in percpu_ref_put()
314  * @ref: percpu_ref to test
316  * Returns %true if @ref is dying or dead.
318  * This function is safe to call as long as @ref is between init and exit
321 static inline bool percpu_ref_is_dying(struct percpu_ref *ref)  in percpu_ref_is_dying()  argument
323 	return ref->percpu_count_ptr & __PERCPU_REF_DEAD;  in percpu_ref_is_dying()
328  * @ref: percpu_ref to test
330  * Returns %true if @ref reached zero.
332  * This function is safe to call as long as @ref is between init and exit.
334 static inline bool percpu_ref_is_zero(struct percpu_ref *ref)  in percpu_ref_is_zero()  argument
338 	if (__ref_is_percpu(ref, &percpu_count))  in percpu_ref_is_zero()
340 	return !atomic_long_read(&ref->count);  in percpu_ref_is_zero()