8  * RCU-protected list version
10 #include <linux/list.h>
15  * @list: list to be initialized
18  * cleanup tasks, when readers have no access to the list being initialized.
19  * However, if the list being initialized is visible to readers, you
22 static inline void INIT_LIST_HEAD_RCU(struct list_head *list)  in INIT_LIST_HEAD_RCU()  argument
24 	WRITE_ONCE(list->next, list);  in INIT_LIST_HEAD_RCU()
25 	WRITE_ONCE(list->prev, list);  in INIT_LIST_HEAD_RCU()
32 #define list_next_rcu(list)	(*((struct list_head __rcu **)(&(list)->next)))  argument
35  * list_tail_rcu - returns the prev pointer of the head of the list
36  * @head: the head of the list
38  * Note: This should only be used with the list header, and even then
40  * list header.
45  * Check during list traversal that we are within an RCU reader
55 			 "RCU-list traversed in non-reader section!");	\
61 		"RCU-list traversed without holding the required lock!");\
73  * This is only for internal list manipulation where we know
89  * list_add_rcu - add a new entry to rcu-protected list
91  * @head: list head to add it after
98  * with another list-mutation primitive, such as list_add_rcu()
99  * or list_del_rcu(), running on this same list.
101  * the _rcu list-traversal primitives, such as
110  * list_add_tail_rcu - add a new entry to rcu-protected list
112  * @head: list head to add it before
119  * with another list-mutation primitive, such as list_add_tail_rcu()
120  * or list_del_rcu(), running on this same list.
122  * the _rcu list-traversal primitives, such as
132  * list_del_rcu - deletes entry from list without re-initialization
133  * @entry: the element to delete from the list.
140  * pointers that may still be used for walking the list.
144  * with another list-mutation primitive, such as list_del_rcu()
145  * or list_add_rcu(), running on this same list.
147  * the _rcu list-traversal primitives, such as
162  * hlist_del_init_rcu - deletes entry from hash list with re-initialization
163  * @n: the element to delete from the hash list.
167  * must know if the list entry is still hashed or already unhashed.
170  * that may still be used for walking the hash list and we can only
176  * list-mutation primitive, such as hlist_add_head_rcu() or
177  * hlist_del_rcu(), running on this same list.  However, it is
178  * perfectly legal to run concurrently with the _rcu list-traversal
208  * __list_splice_init_rcu - join an RCU-protected list into an existing list.
209  * @list:	the RCU-protected list to splice
210  * @prev:	points to the last element of the existing list
211  * @next:	points to the first element of the existing list
214  * The list pointed to by @prev and @next can be RCU-read traversed
220  * any other updates to the existing list.  In principle, it is possible to
221  * modify the list as soon as sync() begins execution. If this sort of thing
226 static inline void __list_splice_init_rcu(struct list_head *list,  in __list_splice_init_rcu()  argument
231 	struct list_head *first = list->next;  in __list_splice_init_rcu()
232 	struct list_head *last = list->prev;  in __list_splice_init_rcu()
235 	 * "first" and "last" tracking list, so initialize it.  RCU readers  in __list_splice_init_rcu()
236 	 * have access to this list, so we must use INIT_LIST_HEAD_RCU()  in __list_splice_init_rcu()
240 	INIT_LIST_HEAD_RCU(list);  in __list_splice_init_rcu()
243 	 * At this point, the list body still points to the source list.  in __list_splice_init_rcu()
244 	 * Wait for any readers to finish using the list before splicing  in __list_splice_init_rcu()
245 	 * the list body into the new list.  Any new readers will see  in __list_splice_init_rcu()
246 	 * an empty list.  in __list_splice_init_rcu()
254 	 * Readers are finished with the source list, so perform splice.  in __list_splice_init_rcu()
255 	 * The order is important if the new list is global and accessible  in __list_splice_init_rcu()
268  * list_splice_init_rcu - splice an RCU-protected list into an existing list,
270  * @list:	the RCU-protected list to splice
271  * @head:	the place in the existing list to splice the first list into
274 static inline void list_splice_init_rcu(struct list_head *list,  in list_splice_init_rcu()  argument
278 	if (!list_empty(list))  in list_splice_init_rcu()
279 		__list_splice_init_rcu(list, head, head->next, sync);  in list_splice_init_rcu()
283  * list_splice_tail_init_rcu - splice an RCU-protected list into an existing
284  *                             list, designed for queues.
285  * @list:	the RCU-protected list to splice
286  * @head:	the place in the existing list to splice the first list into
289 static inline void list_splice_tail_init_rcu(struct list_head *list,  in list_splice_tail_init_rcu()  argument
293 	if (!list_empty(list))  in list_splice_tail_init_rcu()
294 		__list_splice_init_rcu(list, head->prev, head, sync);  in list_splice_tail_init_rcu()
303  * This primitive may safely run concurrently with the _rcu list-mutation
319  * The list might be non-empty when list_empty_rcu() checks it, but it
329  * list head.  However, it neither dereferences this pointer nor provides
340  * list_first_or_null_rcu - get the first element from a list
341  * @ptr:        the list head to take the element from.
345  * Note that if the list is empty, it returns NULL.
347  * This primitive may safely run concurrently with the _rcu list-mutation
358  * list_next_or_null_rcu - get the first element from a list
359  * @head:	the head for the list.
360  * @ptr:        the list head to take the next element from.
364  * Note that if the ptr is at the end of the list, NULL is returned.
366  * This primitive may safely run concurrently with the _rcu list-mutation
379  * list_for_each_entry_rcu	-	iterate over rcu list of given type
381  * @head:	the head for your list.
385  * This list-traversal primitive may safely run concurrently with
386  * the _rcu list-mutation primitives such as list_add_rcu()
396  * list_for_each_entry_srcu	-	iterate over rcu list of given type
398  * @head:	the head for your list.
400  * @cond:	lockdep expression for the lock required to traverse the list.
402  * This list-traversal primitive may safely run concurrently with
403  * the _rcu list-mutation primitives such as list_add_rcu()
421  * list-mutation primitives such as list_add_rcu(), but requires some
424  * cannot be invoked.  Another example is when items are added to the list,
431  * list_for_each_entry_lockless - iterate over rcu list of given type
433  * @head:	the head for your list.
437  * list-mutation primitives such as list_add_rcu(), but requires some
440  * cannot be invoked.  Another example is when items are added to the list,
449  * list_for_each_entry_continue_rcu - continue iteration over list of given type
451  * @head:	the head for your list.
454  * Continue to iterate over list of given type, continuing after
455  * the current position which must have been in the list when the RCU read
458  * previous walk of the list in the same RCU read-side critical section, or
460  * to keep the node alive *and* in the list.
472  * list_for_each_entry_from_rcu - iterate over a list from current point
474  * @head:	the head for your list.
477  * Iterate over the tail of a list starting from a given position,
478  * which must have been in the list when the RCU read lock was taken.
480  * previous walk of the list in the same RCU read-side critical section, or
482  * to keep the node alive *and* in the list.
493  * hlist_del_rcu - deletes entry from hash list without re-initialization
494  * @n: the element to delete from the hash list.
501  * pointers that may still be used for walking the hash list.
505  * with another list-mutation primitive, such as hlist_add_head_rcu()
506  * or hlist_del_rcu(), running on this same list.
508  * the _rcu list-traversal primitives, such as
567  * @n: the element to add to the hash list.
568  * @h: the list to add to.
576  * with another list-mutation primitive, such as hlist_add_head_rcu()
577  * or hlist_del_rcu(), running on this same list.
579  * the _rcu list-traversal primitives, such as
582  * list-traversal primitive must be guarded by rcu_read_lock().
598  * @n: the element to add to the hash list.
599  * @h: the list to add to.
607  * with another list-mutation primitive, such as hlist_add_head_rcu()
608  * or hlist_del_rcu(), running on this same list.
610  * the _rcu list-traversal primitives, such as
613  * list-traversal primitive must be guarded by rcu_read_lock().
635  * @n: the new element to add to the hash list.
644  * with another list-mutation primitive, such as hlist_add_head_rcu()
645  * or hlist_del_rcu(), running on this same list.
647  * the _rcu list-traversal primitives, such as
662  * @n: the new element to add to the hash list.
671  * with another list-mutation primitive, such as hlist_add_head_rcu()
672  * or hlist_del_rcu(), running on this same list.
674  * the _rcu list-traversal primitives, such as
694  * hlist_for_each_entry_rcu - iterate over rcu list of given type
696  * @head:	the head for your list.
700  * This list-traversal primitive may safely run concurrently with
701  * the _rcu list-mutation primitives such as hlist_add_head_rcu()
713  * hlist_for_each_entry_srcu - iterate over rcu list of given type
715  * @head:	the head for your list.
717  * @cond:	lockdep expression for the lock required to traverse the list.
719  * This list-traversal primitive may safely run concurrently with
720  * the _rcu list-mutation primitives such as hlist_add_head_rcu()
734  * hlist_for_each_entry_rcu_notrace - iterate over rcu list of given type (for tracing)
736  * @head:	the head for your list.
739  * This list-traversal primitive may safely run concurrently with
740  * the _rcu list-mutation primitives such as hlist_add_head_rcu()
754  * hlist_for_each_entry_rcu_bh - iterate over rcu list of given type
756  * @head:	the head for your list.
759  * This list-traversal primitive may safely run concurrently with
760  * the _rcu list-mutation primitives such as hlist_add_head_rcu()