87 bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage,  in bpf_selem_unlink_storage_nolock()  argument
96 	owner = local_storage->owner;  in bpf_selem_unlink_storage_nolock()
106 						    &local_storage->list);  in bpf_selem_unlink_storage_nolock()
109 		local_storage->owner = NULL;  in bpf_selem_unlink_storage_nolock()
129 	if (rcu_access_pointer(local_storage->cache[smap->cache_idx]) ==  in bpf_selem_unlink_storage_nolock()
131 		RCU_INIT_POINTER(local_storage->cache[smap->cache_idx], NULL);  in bpf_selem_unlink_storage_nolock()
140 	struct bpf_local_storage *local_storage;  in __bpf_selem_unlink_storage()  local
147 	local_storage = rcu_dereference(selem->local_storage);  in __bpf_selem_unlink_storage()
148 	raw_spin_lock_bh(&local_storage->lock);  in __bpf_selem_unlink_storage()
151 			local_storage, selem, true);  in __bpf_selem_unlink_storage()
152 	raw_spin_unlock_bh(&local_storage->lock);  in __bpf_selem_unlink_storage()
155 		kfree_rcu(local_storage, rcu);  in __bpf_selem_unlink_storage()
158 void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage,  in bpf_selem_link_storage_nolock()  argument
161 	RCU_INIT_POINTER(selem->local_storage, local_storage);  in bpf_selem_link_storage_nolock()
162 	hlist_add_head_rcu(&selem->snode, &local_storage->list);  in bpf_selem_link_storage_nolock()
204 bpf_local_storage_lookup(struct bpf_local_storage *local_storage,  in bpf_local_storage_lookup()  argument
212 	sdata = rcu_dereference(local_storage->cache[smap->cache_idx]);  in bpf_local_storage_lookup()
217 	hlist_for_each_entry_rcu(selem, &local_storage->list, snode)  in bpf_local_storage_lookup()
231 		raw_spin_lock_bh(&local_storage->lock);  in bpf_local_storage_lookup()
233 			rcu_assign_pointer(local_storage->cache[smap->cache_idx],  in bpf_local_storage_lookup()
235 		raw_spin_unlock_bh(&local_storage->lock);  in bpf_local_storage_lookup()
327 	struct bpf_local_storage *local_storage;  in bpf_local_storage_update()  local
337 	local_storage = rcu_dereference(*owner_storage(smap, owner));  in bpf_local_storage_update()
338 	if (!local_storage || hlist_empty(&local_storage->list)) {  in bpf_local_storage_update()
364 			bpf_local_storage_lookup(local_storage, smap, false);  in bpf_local_storage_update()
375 	raw_spin_lock_bh(&local_storage->lock);  in bpf_local_storage_update()
378 	if (unlikely(hlist_empty(&local_storage->list))) {  in bpf_local_storage_update()
388 	old_sdata = bpf_local_storage_lookup(local_storage, smap, false);  in bpf_local_storage_update()
419 	bpf_selem_link_storage_nolock(local_storage, selem);  in bpf_local_storage_update()
424 		bpf_selem_unlink_storage_nolock(local_storage, SELEM(old_sdata),  in bpf_local_storage_update()
429 	raw_spin_unlock_bh(&local_storage->lock);  in bpf_local_storage_update()
433 	raw_spin_unlock_bh(&local_storage->lock);  in bpf_local_storage_update()