22 	      struct bpf_local_storage_elem *selem)  in select_bucket()  argument
24 	return &smap->buckets[hash_ptr(selem, smap->bucket_log)];  in select_bucket()
54 static bool selem_linked_to_storage_lockless(const struct bpf_local_storage_elem *selem)  in selem_linked_to_storage_lockless()  argument
56 	return !hlist_unhashed_lockless(&selem->snode);  in selem_linked_to_storage_lockless()
59 static bool selem_linked_to_storage(const struct bpf_local_storage_elem *selem)  in selem_linked_to_storage()  argument
61 	return !hlist_unhashed(&selem->snode);  in selem_linked_to_storage()
64 static bool selem_linked_to_map_lockless(const struct bpf_local_storage_elem *selem)  in selem_linked_to_map_lockless()  argument
66 	return !hlist_unhashed_lockless(&selem->map_node);  in selem_linked_to_map_lockless()
69 static bool selem_linked_to_map(const struct bpf_local_storage_elem *selem)  in selem_linked_to_map()  argument
71 	return !hlist_unhashed(&selem->map_node);  in selem_linked_to_map()
78 	struct bpf_local_storage_elem *selem;  in bpf_selem_alloc()  local
85 		selem = bpf_mem_cache_alloc_flags(&smap->selem_ma, gfp_flags);  in bpf_selem_alloc()
87 		if (selem)  in bpf_selem_alloc()
95 			memset(SDATA(selem)->data, 0, smap->map.value_size);  in bpf_selem_alloc()
97 		selem = bpf_map_kzalloc(&smap->map, smap->elem_size,  in bpf_selem_alloc()
101 	if (selem) {  in bpf_selem_alloc()
103 			copy_map_value(&smap->map, SDATA(selem)->data, value);  in bpf_selem_alloc()
105 		return selem;  in bpf_selem_alloc()
190 	struct bpf_local_storage_elem *selem;  in __bpf_selem_free_trace_rcu()  local
192 	selem = container_of(rcu, struct bpf_local_storage_elem, rcu);  in __bpf_selem_free_trace_rcu()
194 		kfree(selem);  in __bpf_selem_free_trace_rcu()
196 		kfree_rcu(selem, rcu);  in __bpf_selem_free_trace_rcu()
200 static void __bpf_selem_free(struct bpf_local_storage_elem *selem,  in __bpf_selem_free()  argument
204 		kfree_rcu(selem, rcu);  in __bpf_selem_free()
206 		call_rcu_tasks_trace(&selem->rcu, __bpf_selem_free_trace_rcu);  in __bpf_selem_free()
211 	struct bpf_local_storage_elem *selem;  in bpf_selem_free_rcu()  local
213 	selem = container_of(rcu, struct bpf_local_storage_elem, rcu);  in bpf_selem_free_rcu()
214 	bpf_mem_cache_raw_free(selem);  in bpf_selem_free_rcu()
225 void bpf_selem_free(struct bpf_local_storage_elem *selem,  in bpf_selem_free()  argument
229 	bpf_obj_free_fields(smap->map.record, SDATA(selem)->data);  in bpf_selem_free()
232 		__bpf_selem_free(selem, reuse_now);  in bpf_selem_free()
237 		call_rcu_tasks_trace(&selem->rcu, bpf_selem_free_trace_rcu);  in bpf_selem_free()
244 		bpf_mem_cache_free(&smap->selem_ma, selem);  in bpf_selem_free()
254 					    struct bpf_local_storage_elem *selem,  in bpf_selem_unlink_storage_nolock()  argument
261 	smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held());  in bpf_selem_unlink_storage_nolock()
271 	free_local_storage = hlist_is_singular_node(&selem->snode,  in bpf_selem_unlink_storage_nolock()
294 	hlist_del_init_rcu(&selem->snode);  in bpf_selem_unlink_storage_nolock()
296 	    SDATA(selem))  in bpf_selem_unlink_storage_nolock()
299 	bpf_selem_free(selem, smap, reuse_now);  in bpf_selem_unlink_storage_nolock()
309 				 struct bpf_local_storage_elem *selem)  in check_storage_bpf_ma()  argument
327 	if (!selem) {  in check_storage_bpf_ma()
335 		selem = hlist_entry(n, struct bpf_local_storage_elem, snode);  in check_storage_bpf_ma()
337 	selem_smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held());  in check_storage_bpf_ma()
342 static void bpf_selem_unlink_storage(struct bpf_local_storage_elem *selem,  in bpf_selem_unlink_storage()  argument
350 	if (unlikely(!selem_linked_to_storage_lockless(selem)))  in bpf_selem_unlink_storage()
354 	local_storage = rcu_dereference_check(selem->local_storage,  in bpf_selem_unlink_storage()
358 	bpf_ma = check_storage_bpf_ma(local_storage, storage_smap, selem);  in bpf_selem_unlink_storage()
361 	if (likely(selem_linked_to_storage(selem)))  in bpf_selem_unlink_storage()
363 			local_storage, selem, true, reuse_now);  in bpf_selem_unlink_storage()
371 				   struct bpf_local_storage_elem *selem)  in bpf_selem_link_storage_nolock()  argument
373 	RCU_INIT_POINTER(selem->local_storage, local_storage);  in bpf_selem_link_storage_nolock()
374 	hlist_add_head_rcu(&selem->snode, &local_storage->list);  in bpf_selem_link_storage_nolock()
377 static void bpf_selem_unlink_map(struct bpf_local_storage_elem *selem)  in bpf_selem_unlink_map()  argument
383 	if (unlikely(!selem_linked_to_map_lockless(selem)))  in bpf_selem_unlink_map()
387 	smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held());  in bpf_selem_unlink_map()
388 	b = select_bucket(smap, selem);  in bpf_selem_unlink_map()
390 	if (likely(selem_linked_to_map(selem)))  in bpf_selem_unlink_map()
391 		hlist_del_init_rcu(&selem->map_node);  in bpf_selem_unlink_map()
396 			struct bpf_local_storage_elem *selem)  in bpf_selem_link_map()  argument
398 	struct bpf_local_storage_map_bucket *b = select_bucket(smap, selem);  in bpf_selem_link_map()
402 	RCU_INIT_POINTER(SDATA(selem)->smap, smap);  in bpf_selem_link_map()
403 	hlist_add_head_rcu(&selem->map_node, &b->list);  in bpf_selem_link_map()
407 void bpf_selem_unlink(struct bpf_local_storage_elem *selem, bool reuse_now)  in bpf_selem_unlink()  argument
413 	bpf_selem_unlink_map(selem);  in bpf_selem_unlink()
414 	bpf_selem_unlink_storage(selem, reuse_now);  in bpf_selem_unlink()
424 	struct bpf_local_storage_elem *selem;  in bpf_local_storage_lookup()  local
433 	hlist_for_each_entry_rcu(selem, &local_storage->list, snode,  in bpf_local_storage_lookup()
435 		if (rcu_access_pointer(SDATA(selem)->smap) == smap)  in bpf_local_storage_lookup()
438 	if (!selem)  in bpf_local_storage_lookup()
441 	sdata = SDATA(selem);  in bpf_local_storage_lookup()
451 		if (selem_linked_to_storage(selem))  in bpf_local_storage_lookup()
556 	struct bpf_local_storage_elem *alloc_selem, *selem = NULL;  in bpf_local_storage_update()  local
579 		selem = bpf_selem_alloc(smap, owner, value, true, gfp_flags);  in bpf_local_storage_update()
580 		if (!selem)  in bpf_local_storage_update()
583 		err = bpf_local_storage_alloc(owner, smap, selem, gfp_flags);  in bpf_local_storage_update()
585 			bpf_selem_free(selem, smap, true);  in bpf_local_storage_update()
590 		return SDATA(selem);  in bpf_local_storage_update()
613 	alloc_selem = selem = bpf_selem_alloc(smap, owner, value, true, gfp_flags);  in bpf_local_storage_update()
638 		selem = SELEM(old_sdata);  in bpf_local_storage_update()
644 	bpf_selem_link_map(smap, selem);  in bpf_local_storage_update()
647 	bpf_selem_link_storage_nolock(local_storage, selem);  in bpf_local_storage_update()
662 	return err ? ERR_PTR(err) : SDATA(selem);  in bpf_local_storage_update()
733 	struct bpf_local_storage_elem *selem;  in bpf_local_storage_destroy()  local
751 	hlist_for_each_entry_safe(selem, n, &local_storage->list, snode) {  in bpf_local_storage_destroy()
755 		bpf_selem_unlink_map(selem);  in bpf_local_storage_destroy()
763 			local_storage, selem, true, true);  in bpf_local_storage_destroy()
853 	struct bpf_local_storage_elem *selem;  in bpf_local_storage_map_free()  local
880 		while ((selem = hlist_entry_safe(  in bpf_local_storage_map_free()
887 			bpf_selem_unlink(selem, true);  in bpf_local_storage_map_free()