22 	      struct bpf_local_storage_elem *selem)  in select_bucket()  argument
24 	return &smap->buckets[hash_ptr(selem, smap->bucket_log)];  in select_bucket()
54 static bool selem_linked_to_storage(const struct bpf_local_storage_elem *selem)  in selem_linked_to_storage()  argument
56 	return !hlist_unhashed(&selem->snode);  in selem_linked_to_storage()
59 static bool selem_linked_to_map(const struct bpf_local_storage_elem *selem)  in selem_linked_to_map()  argument
61 	return !hlist_unhashed(&selem->map_node);  in selem_linked_to_map()
68 	struct bpf_local_storage_elem *selem;  in bpf_selem_alloc()  local
73 	selem = bpf_map_kzalloc(&smap->map, smap->elem_size,  in bpf_selem_alloc()
75 	if (selem) {  in bpf_selem_alloc()
77 			copy_map_value(&smap->map, SDATA(selem)->data, value);  in bpf_selem_alloc()
78 		return selem;  in bpf_selem_alloc()
97 	struct bpf_local_storage_elem *selem;  in bpf_selem_free_rcu()  local
99 	selem = container_of(rcu, struct bpf_local_storage_elem, rcu);  in bpf_selem_free_rcu()
100 	kfree_rcu(selem, rcu);  in bpf_selem_free_rcu()
108 				     struct bpf_local_storage_elem *selem,  in bpf_selem_unlink_storage_nolock()  argument
115 	smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held());  in bpf_selem_unlink_storage_nolock()
125 	free_local_storage = hlist_is_singular_node(&selem->snode,  in bpf_selem_unlink_storage_nolock()
148 	hlist_del_init_rcu(&selem->snode);  in bpf_selem_unlink_storage_nolock()
150 	    SDATA(selem))  in bpf_selem_unlink_storage_nolock()
154 		call_rcu_tasks_trace(&selem->rcu, bpf_selem_free_rcu);  in bpf_selem_unlink_storage_nolock()
156 		kfree_rcu(selem, rcu);  in bpf_selem_unlink_storage_nolock()
161 static void __bpf_selem_unlink_storage(struct bpf_local_storage_elem *selem,  in __bpf_selem_unlink_storage()  argument
168 	if (unlikely(!selem_linked_to_storage(selem)))  in __bpf_selem_unlink_storage()
172 	local_storage = rcu_dereference_check(selem->local_storage,  in __bpf_selem_unlink_storage()
175 	if (likely(selem_linked_to_storage(selem)))  in __bpf_selem_unlink_storage()
177 			local_storage, selem, true, use_trace_rcu);  in __bpf_selem_unlink_storage()
190 				   struct bpf_local_storage_elem *selem)  in bpf_selem_link_storage_nolock()  argument
192 	RCU_INIT_POINTER(selem->local_storage, local_storage);  in bpf_selem_link_storage_nolock()
193 	hlist_add_head_rcu(&selem->snode, &local_storage->list);  in bpf_selem_link_storage_nolock()
196 void bpf_selem_unlink_map(struct bpf_local_storage_elem *selem)  in bpf_selem_unlink_map()  argument
202 	if (unlikely(!selem_linked_to_map(selem)))  in bpf_selem_unlink_map()
206 	smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held());  in bpf_selem_unlink_map()
207 	b = select_bucket(smap, selem);  in bpf_selem_unlink_map()
209 	if (likely(selem_linked_to_map(selem)))  in bpf_selem_unlink_map()
210 		hlist_del_init_rcu(&selem->map_node);  in bpf_selem_unlink_map()
215 			struct bpf_local_storage_elem *selem)  in bpf_selem_link_map()  argument
217 	struct bpf_local_storage_map_bucket *b = select_bucket(smap, selem);  in bpf_selem_link_map()
221 	RCU_INIT_POINTER(SDATA(selem)->smap, smap);  in bpf_selem_link_map()
222 	hlist_add_head_rcu(&selem->map_node, &b->list);  in bpf_selem_link_map()
226 void bpf_selem_unlink(struct bpf_local_storage_elem *selem, bool use_trace_rcu)  in bpf_selem_unlink()  argument
232 	bpf_selem_unlink_map(selem);  in bpf_selem_unlink()
233 	__bpf_selem_unlink_storage(selem, use_trace_rcu);  in bpf_selem_unlink()
242 	struct bpf_local_storage_elem *selem;  in bpf_local_storage_lookup()  local
251 	hlist_for_each_entry_rcu(selem, &local_storage->list, snode,  in bpf_local_storage_lookup()
253 		if (rcu_access_pointer(SDATA(selem)->smap) == smap)  in bpf_local_storage_lookup()
256 	if (!selem)  in bpf_local_storage_lookup()
259 	sdata = SDATA(selem);  in bpf_local_storage_lookup()
269 		if (selem_linked_to_storage(selem))  in bpf_local_storage_lookup()
366 	struct bpf_local_storage_elem *selem = NULL;  in bpf_local_storage_update()  local
389 		selem = bpf_selem_alloc(smap, owner, value, true, gfp_flags);  in bpf_local_storage_update()
390 		if (!selem)  in bpf_local_storage_update()
393 		err = bpf_local_storage_alloc(owner, smap, selem, gfp_flags);  in bpf_local_storage_update()
395 			kfree(selem);  in bpf_local_storage_update()
400 		return SDATA(selem);  in bpf_local_storage_update()
421 		selem = bpf_selem_alloc(smap, owner, value, true, gfp_flags);  in bpf_local_storage_update()
422 		if (!selem)  in bpf_local_storage_update()
447 		selem = SELEM(old_sdata);  in bpf_local_storage_update()
461 		selem = bpf_selem_alloc(smap, owner, value, !old_sdata, gfp_flags);  in bpf_local_storage_update()
462 		if (!selem) {  in bpf_local_storage_update()
469 	bpf_selem_link_map(smap, selem);  in bpf_local_storage_update()
472 	bpf_selem_link_storage_nolock(local_storage, selem);  in bpf_local_storage_update()
483 	return SDATA(selem);  in bpf_local_storage_update()
487 	if (selem) {  in bpf_local_storage_update()
489 		kfree(selem);  in bpf_local_storage_update()
529 	struct bpf_local_storage_elem *selem;  in bpf_local_storage_map_free()  local
553 		while ((selem = hlist_entry_safe(  in bpf_local_storage_map_free()
560 			bpf_selem_unlink(selem, false);  in bpf_local_storage_map_free()