19 	      struct bpf_local_storage_elem *selem)  in select_bucket()  argument
21 	return &smap->buckets[hash_ptr(selem, smap->bucket_log)];  in select_bucket()
51 static bool selem_linked_to_storage(const struct bpf_local_storage_elem *selem)  in selem_linked_to_storage()  argument
53 	return !hlist_unhashed(&selem->snode);  in selem_linked_to_storage()
56 static bool selem_linked_to_map(const struct bpf_local_storage_elem *selem)  in selem_linked_to_map()  argument
58 	return !hlist_unhashed(&selem->map_node);  in selem_linked_to_map()
65 	struct bpf_local_storage_elem *selem;  in bpf_selem_alloc()  local
70 	selem = kzalloc(smap->elem_size, GFP_ATOMIC | __GFP_NOWARN);  in bpf_selem_alloc()
71 	if (selem) {  in bpf_selem_alloc()
73 			memcpy(SDATA(selem)->data, value, smap->map.value_size);  in bpf_selem_alloc()
74 		return selem;  in bpf_selem_alloc()
88 				     struct bpf_local_storage_elem *selem,  in bpf_selem_unlink_storage_nolock()  argument
95 	smap = rcu_dereference(SDATA(selem)->smap);  in bpf_selem_unlink_storage_nolock()
105 	free_local_storage = hlist_is_singular_node(&selem->snode,  in bpf_selem_unlink_storage_nolock()
128 	hlist_del_init_rcu(&selem->snode);  in bpf_selem_unlink_storage_nolock()
130 	    SDATA(selem))  in bpf_selem_unlink_storage_nolock()
133 	kfree_rcu(selem, rcu);  in bpf_selem_unlink_storage_nolock()
138 static void __bpf_selem_unlink_storage(struct bpf_local_storage_elem *selem)  in __bpf_selem_unlink_storage()  argument
143 	if (unlikely(!selem_linked_to_storage(selem)))  in __bpf_selem_unlink_storage()
147 	local_storage = rcu_dereference(selem->local_storage);  in __bpf_selem_unlink_storage()
149 	if (likely(selem_linked_to_storage(selem)))  in __bpf_selem_unlink_storage()
151 			local_storage, selem, true);  in __bpf_selem_unlink_storage()
159 				   struct bpf_local_storage_elem *selem)  in bpf_selem_link_storage_nolock()  argument
161 	RCU_INIT_POINTER(selem->local_storage, local_storage);  in bpf_selem_link_storage_nolock()
162 	hlist_add_head_rcu(&selem->snode, &local_storage->list);  in bpf_selem_link_storage_nolock()
165 void bpf_selem_unlink_map(struct bpf_local_storage_elem *selem)  in bpf_selem_unlink_map()  argument
170 	if (unlikely(!selem_linked_to_map(selem)))  in bpf_selem_unlink_map()
174 	smap = rcu_dereference(SDATA(selem)->smap);  in bpf_selem_unlink_map()
175 	b = select_bucket(smap, selem);  in bpf_selem_unlink_map()
177 	if (likely(selem_linked_to_map(selem)))  in bpf_selem_unlink_map()
178 		hlist_del_init_rcu(&selem->map_node);  in bpf_selem_unlink_map()
183 			struct bpf_local_storage_elem *selem)  in bpf_selem_link_map()  argument
185 	struct bpf_local_storage_map_bucket *b = select_bucket(smap, selem);  in bpf_selem_link_map()
188 	RCU_INIT_POINTER(SDATA(selem)->smap, smap);  in bpf_selem_link_map()
189 	hlist_add_head_rcu(&selem->map_node, &b->list);  in bpf_selem_link_map()
193 void bpf_selem_unlink(struct bpf_local_storage_elem *selem)  in bpf_selem_unlink()  argument
199 	bpf_selem_unlink_map(selem);  in bpf_selem_unlink()
200 	__bpf_selem_unlink_storage(selem);  in bpf_selem_unlink()
209 	struct bpf_local_storage_elem *selem;  in bpf_local_storage_lookup()  local
217 	hlist_for_each_entry_rcu(selem, &local_storage->list, snode)  in bpf_local_storage_lookup()
218 		if (rcu_access_pointer(SDATA(selem)->smap) == smap)  in bpf_local_storage_lookup()
221 	if (!selem)  in bpf_local_storage_lookup()
224 	sdata = SDATA(selem);  in bpf_local_storage_lookup()
232 		if (selem_linked_to_storage(selem))  in bpf_local_storage_lookup()
326 	struct bpf_local_storage_elem *selem;  in bpf_local_storage_update()  local
344 		selem = bpf_selem_alloc(smap, owner, value, true);  in bpf_local_storage_update()
345 		if (!selem)  in bpf_local_storage_update()
348 		err = bpf_local_storage_alloc(owner, smap, selem);  in bpf_local_storage_update()
350 			kfree(selem);  in bpf_local_storage_update()
355 		return SDATA(selem);  in bpf_local_storage_update()
396 		selem = SELEM(old_sdata);  in bpf_local_storage_update()
409 	selem = bpf_selem_alloc(smap, owner, value, !old_sdata);  in bpf_local_storage_update()
410 	if (!selem) {  in bpf_local_storage_update()
416 	bpf_selem_link_map(smap, selem);  in bpf_local_storage_update()
419 	bpf_selem_link_storage_nolock(local_storage, selem);  in bpf_local_storage_update()
430 	return SDATA(selem);  in bpf_local_storage_update()
471 	struct bpf_local_storage_elem *selem;  in bpf_local_storage_map_free()  local
495 		while ((selem = hlist_entry_safe(  in bpf_local_storage_map_free()
498 			bpf_selem_unlink(selem);  in bpf_local_storage_map_free()