1 /* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */
10  * A simple CAS-based lock-free free list. Not the fastest thing in the world
15  * Adapted from: https://moodycamel.com/blog/2014/solving-the-aba-problem-for-lock-free-free-lists
33 	 * Since the refcount is zero, and nobody can increase it once it's  in __freelist_add()
34 	 * zero (except us, and we run only one copy of this method per node at  in __freelist_add()
37 	 * back above zero, then other threads could increase it (happens under  in __freelist_add()
38 	 * heavy contention, when the refcount goes to zero in between a load  in __freelist_add()
40 	 * something non-zero, then the refcount increment is done by the other  in __freelist_add()
41 	 * thread) -- so if the CAS to add the node to the actual list fails,  in __freelist_add()
43 	 * who puts the refcount back to zero (which could be us, hence the  in __freelist_add()
46 	struct freelist_node *head = READ_ONCE(list->head);  in __freelist_add()
49 		WRITE_ONCE(node->next, head);  in __freelist_add()
50 		atomic_set_release(&node->refs, 1);  in __freelist_add()
52 		if (!try_cmpxchg_release(&list->head, &head, node)) {  in __freelist_add()
55 			 * the refcount goes back to zero.  in __freelist_add()
57 			if (atomic_fetch_add_release(REFS_ON_FREELIST - 1, &node->refs) == 1)  in __freelist_add()
67 	 * We know that the should-be-on-freelist bit is 0 at this point, so  in freelist_add()
70 	if (!atomic_fetch_add_release(REFS_ON_FREELIST, &node->refs)) {  in freelist_add()
81 	struct freelist_node *prev, *next, *head = smp_load_acquire(&list->head);  in freelist_try_get()
86 		refs = atomic_read(&head->refs);  in freelist_try_get()
88 		    !atomic_try_cmpxchg_acquire(&head->refs, &refs, refs+1)) {  in freelist_try_get()
89 			head = smp_load_acquire(&list->head);  in freelist_try_get()
95 		 * zero), which means we can read the next and not worry about  in freelist_try_get()
98 		next = READ_ONCE(head->next);  in freelist_try_get()
99 		if (try_cmpxchg_acquire(&list->head, &head, next)) {  in freelist_try_get()
102 			 * which means should-be-on-freelist must be false no  in freelist_try_get()
106 			WARN_ON_ONCE(atomic_read(&head->refs) & REFS_ON_FREELIST);  in freelist_try_get()
112 			atomic_fetch_add(-2, &head->refs);  in freelist_try_get()
121 		refs = atomic_fetch_add(-1, &prev->refs);  in freelist_try_get()