88 bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage,  in bpf_selem_unlink_storage_nolock()  argument
97 	owner = local_storage->owner;  in bpf_selem_unlink_storage_nolock()
107 						    &local_storage->list);  in bpf_selem_unlink_storage_nolock()
110 		local_storage->owner = NULL;  in bpf_selem_unlink_storage_nolock()
130 	if (rcu_access_pointer(local_storage->cache[smap->cache_idx]) ==  in bpf_selem_unlink_storage_nolock()
132 		RCU_INIT_POINTER(local_storage->cache[smap->cache_idx], NULL);  in bpf_selem_unlink_storage_nolock()
141 	struct bpf_local_storage *local_storage;  in __bpf_selem_unlink_storage()  local
149 	local_storage = rcu_dereference(selem->local_storage);  in __bpf_selem_unlink_storage()
150 	raw_spin_lock_irqsave(&local_storage->lock, flags);  in __bpf_selem_unlink_storage()
153 			local_storage, selem, true);  in __bpf_selem_unlink_storage()
154 	raw_spin_unlock_irqrestore(&local_storage->lock, flags);  in __bpf_selem_unlink_storage()
157 		kfree_rcu(local_storage, rcu);  in __bpf_selem_unlink_storage()
160 void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage,  in bpf_selem_link_storage_nolock()  argument
163 	RCU_INIT_POINTER(selem->local_storage, local_storage);  in bpf_selem_link_storage_nolock()
164 	hlist_add_head_rcu(&selem->snode, &local_storage->list);  in bpf_selem_link_storage_nolock()
208 bpf_local_storage_lookup(struct bpf_local_storage *local_storage,  in bpf_local_storage_lookup()  argument
216 	sdata = rcu_dereference(local_storage->cache[smap->cache_idx]);  in bpf_local_storage_lookup()
221 	hlist_for_each_entry_rcu(selem, &local_storage->list, snode)  in bpf_local_storage_lookup()
237 		raw_spin_lock_irqsave(&local_storage->lock, flags);  in bpf_local_storage_lookup()
239 			rcu_assign_pointer(local_storage->cache[smap->cache_idx],  in bpf_local_storage_lookup()
241 		raw_spin_unlock_irqrestore(&local_storage->lock, flags);  in bpf_local_storage_lookup()
334 	struct bpf_local_storage *local_storage;  in bpf_local_storage_update()  local
345 	local_storage = rcu_dereference(*owner_storage(smap, owner));  in bpf_local_storage_update()
346 	if (!local_storage || hlist_empty(&local_storage->list)) {  in bpf_local_storage_update()
372 			bpf_local_storage_lookup(local_storage, smap, false);  in bpf_local_storage_update()
383 	raw_spin_lock_irqsave(&local_storage->lock, flags);  in bpf_local_storage_update()
386 	if (unlikely(hlist_empty(&local_storage->list))) {  in bpf_local_storage_update()
396 	old_sdata = bpf_local_storage_lookup(local_storage, smap, false);  in bpf_local_storage_update()
427 	bpf_selem_link_storage_nolock(local_storage, selem);  in bpf_local_storage_update()
432 		bpf_selem_unlink_storage_nolock(local_storage, SELEM(old_sdata),  in bpf_local_storage_update()
437 	raw_spin_unlock_irqrestore(&local_storage->lock, flags);  in bpf_local_storage_update()
441 	raw_spin_unlock_irqrestore(&local_storage->lock, flags);  in bpf_local_storage_update()