1 /* SPDX-License-Identifier: GPL-2.0 */
8  * RCU-protected list version
14  * INIT_LIST_HEAD_RCU - Initialize a list_head visible to RCU readers
24 	WRITE_ONCE(list->next, list);  in INIT_LIST_HEAD_RCU()
25 	WRITE_ONCE(list->prev, list);  in INIT_LIST_HEAD_RCU()
29  * return the ->next pointer of a list_head in an rcu safe
32 #define list_next_rcu(list)	(*((struct list_head __rcu **)(&(list)->next)))
35  * list_tail_rcu - returns the prev pointer of the head of the list
42 #define list_tail_rcu(head)	(*((struct list_head __rcu **)(&(head)->prev)))
55 			 "RCU-list traversed in non-reader section!");	\
61 		"RCU-list traversed without holding the required lock!");\
82 	new->next = next;  in __list_add_rcu()
83 	new->prev = prev;  in __list_add_rcu()
85 	next->prev = new;  in __list_add_rcu()
89  * list_add_rcu - add a new entry to rcu-protected list
98  * with another list-mutation primitive, such as list_add_rcu()
101  * the _rcu list-traversal primitives, such as
106 	__list_add_rcu(new, head, head->next);  in list_add_rcu()
110  * list_add_tail_rcu - add a new entry to rcu-protected list
119  * with another list-mutation primitive, such as list_add_tail_rcu()
122  * the _rcu list-traversal primitives, such as
128 	__list_add_rcu(new, head->prev, head);  in list_add_tail_rcu()
132  * list_del_rcu - deletes entry from list without re-initialization
144  * with another list-mutation primitive, such as list_del_rcu()
147  * the _rcu list-traversal primitives, such as
158 	entry->prev = LIST_POISON2;  in list_del_rcu()
162  * hlist_del_init_rcu - deletes entry from hash list with re-initialization
176  * list-mutation primitive, such as hlist_add_head_rcu() or
178  * perfectly legal to run concurrently with the _rcu list-traversal
185 		WRITE_ONCE(n->pprev, NULL);  in hlist_del_init_rcu()
190  * list_replace_rcu - replace old entry by new one
200 	new->next = old->next;  in list_replace_rcu()
201 	new->prev = old->prev;  in list_replace_rcu()
202 	rcu_assign_pointer(list_next_rcu(new->prev), new);  in list_replace_rcu()
203 	new->next->prev = new;  in list_replace_rcu()
204 	old->prev = LIST_POISON2;  in list_replace_rcu()
208  * __list_splice_init_rcu - join an RCU-protected list into an existing list.
209  * @list:	the RCU-protected list to splice
212  * @sync:	synchronize_rcu, synchronize_rcu_expedited, ...
214  * The list pointed to by @prev and @next can be RCU-read traversed
221  * modify the list as soon as sync() begins execution. If this sort of thing
223  * created.  But only if -really- needed -- there is no shortage of RCU API
229 					  void (*sync)(void))  in __list_splice_init_rcu()
231 	struct list_head *first = list->next;  in __list_splice_init_rcu()
232 	struct list_head *last = list->prev;  in __list_splice_init_rcu()
249 	sync();  in __list_splice_init_rcu()
261 	last->next = next;  in __list_splice_init_rcu()
263 	first->prev = prev;  in __list_splice_init_rcu()
264 	next->prev = last;  in __list_splice_init_rcu()
268  * list_splice_init_rcu - splice an RCU-protected list into an existing list,
270  * @list:	the RCU-protected list to splice
272  * @sync:	synchronize_rcu, synchronize_rcu_expedited, ...
276 					void (*sync)(void))  in list_splice_init_rcu()
279 		__list_splice_init_rcu(list, head, head->next, sync);  in list_splice_init_rcu()
283  * list_splice_tail_init_rcu - splice an RCU-protected list into an existing
285  * @list:	the RCU-protected list to splice
287  * @sync:	synchronize_rcu, synchronize_rcu_expedited, ...
291 					     void (*sync)(void))  in list_splice_tail_init_rcu()
294 		__list_splice_init_rcu(list, head->prev, head, sync);  in list_splice_tail_init_rcu()
298  * list_entry_rcu - get the struct for this entry
300  * @type:       the type of the struct this is embedded in.
303  * This primitive may safely run concurrently with the _rcu list-mutation
319  * The list might be non-empty when list_empty_rcu() checks it, but it
321  * the ->next pointer, which would result in a SEGV.
323  * When not using RCU, it is OK for list_first_entry() to re-read that
328  * RCU-protected ->next pointer and then compares it to the address of the
340  * list_first_or_null_rcu - get the first element from a list
342  * @type:       the type of the struct this is embedded in.
347  * This primitive may safely run concurrently with the _rcu list-mutation
353 	struct list_head *__next = READ_ONCE(__ptr->next); \
358  * list_next_or_null_rcu - get the first element from a list
361  * @type:       the type of the struct this is embedded in.
366  * This primitive may safely run concurrently with the _rcu list-mutation
373 	struct list_head *__next = READ_ONCE(__ptr->next); \
379  * list_for_each_entry_rcu	-	iterate over rcu list of given type
383  * @cond:	optional lockdep expression if called from non-RCU protection.
385  * This list-traversal primitive may safely run concurrently with
386  * the _rcu list-mutation primitives such as list_add_rcu()
391 	     pos = list_entry_rcu((head)->next, typeof(*pos), member);	\
392 		&pos->member != (head);					\
393 		pos = list_entry_rcu(pos->member.next, typeof(*pos), member))
396  * list_for_each_entry_srcu	-	iterate over rcu list of given type
402  * This list-traversal primitive may safely run concurrently with
403  * the _rcu list-mutation primitives such as list_add_rcu()
410 	     pos = list_entry_rcu((head)->next, typeof(*pos), member);	\
411 		&pos->member != (head);					\
412 		pos = list_entry_rcu(pos->member.next, typeof(*pos), member))
415  * list_entry_lockless - get the struct for this entry
417  * @type:       the type of the struct this is embedded in.
421  * list-mutation primitives such as list_add_rcu(), but requires some
422  * implicit RCU read-side guarding.  One example is running within a special
423  * exception-time environment where preemption is disabled and where lockdep
431  * list_for_each_entry_lockless - iterate over rcu list of given type
437  * list-mutation primitives such as list_add_rcu(), but requires some
438  * implicit RCU read-side guarding.  One example is running within a special
439  * exception-time environment where preemption is disabled and where lockdep
444 	for (pos = list_entry_lockless((head)->next, typeof(*pos), member); \
445 	     &pos->member != (head); \
446 	     pos = list_entry_lockless(pos->member.next, typeof(*pos), member))
449  * list_for_each_entry_continue_rcu - continue iteration over list of given type
458  * previous walk of the list in the same RCU read-side critical section, or
459  * that you held some sort of non-RCU reference (such as a reference count)
467 	for (pos = list_entry_rcu(pos->member.next, typeof(*pos), member); \
468 	     &pos->member != (head);	\
469 	     pos = list_entry_rcu(pos->member.next, typeof(*pos), member))
472  * list_for_each_entry_from_rcu - iterate over a list from current point
480  * previous walk of the list in the same RCU read-side critical section, or
481  * that you held some sort of non-RCU reference (such as a reference count)
489 	for (; &(pos)->member != (head);					\
490 		pos = list_entry_rcu(pos->member.next, typeof(*(pos)), member))
493  * hlist_del_rcu - deletes entry from hash list without re-initialization
505  * with another list-mutation primitive, such as hlist_add_head_rcu()
508  * the _rcu list-traversal primitives, such as
514 	WRITE_ONCE(n->pprev, LIST_POISON2);  in hlist_del_rcu()
518  * hlist_replace_rcu - replace old entry by new one
527 	struct hlist_node *next = old->next;  in hlist_replace_rcu()
529 	new->next = next;  in hlist_replace_rcu()
530 	WRITE_ONCE(new->pprev, old->pprev);  in hlist_replace_rcu()
531 	rcu_assign_pointer(*(struct hlist_node __rcu **)new->pprev, new);  in hlist_replace_rcu()
533 		WRITE_ONCE(new->next->pprev, &new->next);  in hlist_replace_rcu()
534 	WRITE_ONCE(old->pprev, LIST_POISON2);  in hlist_replace_rcu()
538  * hlists_swap_heads_rcu - swap the lists the hlist heads point to
549 	struct hlist_node *node1 = left->first;  in hlists_swap_heads_rcu()
550 	struct hlist_node *node2 = right->first;  in hlists_swap_heads_rcu()
552 	rcu_assign_pointer(left->first, node2);  in hlists_swap_heads_rcu()
553 	rcu_assign_pointer(right->first, node1);  in hlists_swap_heads_rcu()
554 	WRITE_ONCE(node2->pprev, &left->first);  in hlists_swap_heads_rcu()
555 	WRITE_ONCE(node1->pprev, &right->first);  in hlists_swap_heads_rcu()
561 #define hlist_first_rcu(head)	(*((struct hlist_node __rcu **)(&(head)->first)))
562 #define hlist_next_rcu(node)	(*((struct hlist_node __rcu **)(&(node)->next)))
563 #define hlist_pprev_rcu(node)	(*((struct hlist_node __rcu **)((node)->pprev)))
576  * with another list-mutation primitive, such as hlist_add_head_rcu()
579  * the _rcu list-traversal primitives, such as
580  * hlist_for_each_entry_rcu(), used to prevent memory-consistency
582  * list-traversal primitive must be guarded by rcu_read_lock().
587 	struct hlist_node *first = h->first;  in hlist_add_head_rcu()
589 	n->next = first;  in hlist_add_head_rcu()
590 	WRITE_ONCE(n->pprev, &h->first);  in hlist_add_head_rcu()
593 		WRITE_ONCE(first->pprev, &n->next);  in hlist_add_head_rcu()
607  * with another list-mutation primitive, such as hlist_add_head_rcu()
610  * the _rcu list-traversal primitives, such as
611  * hlist_for_each_entry_rcu(), used to prevent memory-consistency
613  * list-traversal primitive must be guarded by rcu_read_lock().
621 	for (i = h->first; i; i = i->next)  in hlist_add_tail_rcu()
625 		n->next = last->next;  in hlist_add_tail_rcu()
626 		WRITE_ONCE(n->pprev, &last->next);  in hlist_add_tail_rcu()
644  * with another list-mutation primitive, such as hlist_add_head_rcu()
647  * the _rcu list-traversal primitives, such as
648  * hlist_for_each_entry_rcu(), used to prevent memory-consistency
654 	WRITE_ONCE(n->pprev, next->pprev);  in hlist_add_before_rcu()
655 	n->next = next;  in hlist_add_before_rcu()
657 	WRITE_ONCE(next->pprev, &n->next);  in hlist_add_before_rcu()
671  * with another list-mutation primitive, such as hlist_add_head_rcu()
674  * the _rcu list-traversal primitives, such as
675  * hlist_for_each_entry_rcu(), used to prevent memory-consistency
681 	n->next = prev->next;  in hlist_add_behind_rcu()
682 	WRITE_ONCE(n->pprev, &prev->next);  in hlist_add_behind_rcu()
684 	if (n->next)  in hlist_add_behind_rcu()
685 		WRITE_ONCE(n->next->pprev, &n->next);  in hlist_add_behind_rcu()
694  * hlist_for_each_entry_rcu - iterate over rcu list of given type
698  * @cond:	optional lockdep expression if called from non-RCU protection.
700  * This list-traversal primitive may safely run concurrently with
701  * the _rcu list-mutation primitives such as hlist_add_head_rcu()
710 			&(pos)->member)), typeof(*(pos)), member))
713  * hlist_for_each_entry_srcu - iterate over rcu list of given type
719  * This list-traversal primitive may safely run concurrently with
720  * the _rcu list-mutation primitives such as hlist_add_head_rcu()
731 			&(pos)->member)), typeof(*(pos)), member))
734  * hlist_for_each_entry_rcu_notrace - iterate over rcu list of given type (for tracing)
739  * This list-traversal primitive may safely run concurrently with
740  * the _rcu list-mutation primitives such as hlist_add_head_rcu()
751 			&(pos)->member)), typeof(*(pos)), member))
754  * hlist_for_each_entry_rcu_bh - iterate over rcu list of given type
759  * This list-traversal primitive may safely run concurrently with
760  * the _rcu list-mutation primitives such as hlist_add_head_rcu()
768 			&(pos)->member)), typeof(*(pos)), member))
771  * hlist_for_each_entry_continue_rcu - iterate over a hlist continuing after current point
777 			&(pos)->member)), typeof(*(pos)), member);	\
780 			&(pos)->member)), typeof(*(pos)), member))
783  * hlist_for_each_entry_continue_rcu_bh - iterate over a hlist continuing after current point
789 			&(pos)->member)), typeof(*(pos)), member);	\
792 			&(pos)->member)), typeof(*(pos)), member))
795  * hlist_for_each_entry_from_rcu - iterate over a hlist continuing from current point
802 			&(pos)->member)), typeof(*(pos)), member))