18 select_bucket(struct bpf_local_storage_map *smap,  in select_bucket()  argument
21 	return &smap->buckets[hash_ptr(selem, smap->bucket_log)];  in select_bucket()
24 static int mem_charge(struct bpf_local_storage_map *smap, void *owner, u32 size)  in mem_charge()  argument
26 	struct bpf_map *map = &smap->map;  in mem_charge()
31 	return map->ops->map_local_storage_charge(smap, owner, size);  in mem_charge()
34 static void mem_uncharge(struct bpf_local_storage_map *smap, void *owner,  in mem_uncharge()  argument
37 	struct bpf_map *map = &smap->map;  in mem_uncharge()
40 		map->ops->map_local_storage_uncharge(smap, owner, size);  in mem_uncharge()
44 owner_storage(struct bpf_local_storage_map *smap, void *owner)  in owner_storage()  argument
46 	struct bpf_map *map = &smap->map;  in owner_storage()
62 bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner,  in bpf_selem_alloc()  argument
67 	if (charge_mem && mem_charge(smap, owner, smap->elem_size))  in bpf_selem_alloc()
70 	selem = kzalloc(smap->elem_size, GFP_ATOMIC | __GFP_NOWARN);  in bpf_selem_alloc()
73 			memcpy(SDATA(selem)->data, value, smap->map.value_size);  in bpf_selem_alloc()
78 		mem_uncharge(smap, owner, smap->elem_size);  in bpf_selem_alloc()
91 	struct bpf_local_storage_map *smap;  in bpf_selem_unlink_storage_nolock()  local
95 	smap = rcu_dereference(SDATA(selem)->smap);  in bpf_selem_unlink_storage_nolock()
103 		mem_uncharge(smap, owner, smap->elem_size);  in bpf_selem_unlink_storage_nolock()
108 		mem_uncharge(smap, owner, sizeof(struct bpf_local_storage));  in bpf_selem_unlink_storage_nolock()
112 		RCU_INIT_POINTER(*owner_storage(smap, owner), NULL);  in bpf_selem_unlink_storage_nolock()
129 	if (rcu_access_pointer(local_storage->cache[smap->cache_idx]) ==  in bpf_selem_unlink_storage_nolock()
131 		RCU_INIT_POINTER(local_storage->cache[smap->cache_idx], NULL);  in bpf_selem_unlink_storage_nolock()
167 	struct bpf_local_storage_map *smap;  in bpf_selem_unlink_map()  local
174 	smap = rcu_dereference(SDATA(selem)->smap);  in bpf_selem_unlink_map()
175 	b = select_bucket(smap, selem);  in bpf_selem_unlink_map()
182 void bpf_selem_link_map(struct bpf_local_storage_map *smap,  in bpf_selem_link_map()  argument
185 	struct bpf_local_storage_map_bucket *b = select_bucket(smap, selem);  in bpf_selem_link_map()
188 	RCU_INIT_POINTER(SDATA(selem)->smap, smap);  in bpf_selem_link_map()
205 			 struct bpf_local_storage_map *smap,  in bpf_local_storage_lookup()  argument
212 	sdata = rcu_dereference(local_storage->cache[smap->cache_idx]);  in bpf_local_storage_lookup()
213 	if (sdata && rcu_access_pointer(sdata->smap) == smap)  in bpf_local_storage_lookup()
218 		if (rcu_access_pointer(SDATA(selem)->smap) == smap)  in bpf_local_storage_lookup()
233 			rcu_assign_pointer(local_storage->cache[smap->cache_idx],  in bpf_local_storage_lookup()
256 			    struct bpf_local_storage_map *smap,  in bpf_local_storage_alloc()  argument
263 	err = mem_charge(smap, owner, sizeof(*storage));  in bpf_local_storage_alloc()
278 	bpf_selem_link_map(smap, first_selem);  in bpf_local_storage_alloc()
281 		(struct bpf_local_storage **)owner_storage(smap, owner);  in bpf_local_storage_alloc()
312 	mem_uncharge(smap, owner, sizeof(*storage));  in bpf_local_storage_alloc()
322 bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,  in bpf_local_storage_update()  argument
334 		     !map_value_has_spin_lock(&smap->map)))  in bpf_local_storage_update()
337 	local_storage = rcu_dereference(*owner_storage(smap, owner));  in bpf_local_storage_update()
344 		selem = bpf_selem_alloc(smap, owner, value, true);  in bpf_local_storage_update()
348 		err = bpf_local_storage_alloc(owner, smap, selem);  in bpf_local_storage_update()
351 			mem_uncharge(smap, owner, smap->elem_size);  in bpf_local_storage_update()
364 			bpf_local_storage_lookup(local_storage, smap, false);  in bpf_local_storage_update()
369 			copy_map_value_locked(&smap->map, old_sdata->data,  in bpf_local_storage_update()
388 	old_sdata = bpf_local_storage_lookup(local_storage, smap, false);  in bpf_local_storage_update()
394 		copy_map_value_locked(&smap->map, old_sdata->data, value,  in bpf_local_storage_update()
409 	selem = bpf_selem_alloc(smap, owner, value, !old_sdata);  in bpf_local_storage_update()
416 	bpf_selem_link_map(smap, selem);  in bpf_local_storage_update()
469 void bpf_local_storage_map_free(struct bpf_local_storage_map *smap)  in bpf_local_storage_map_free()  argument
490 	for (i = 0; i < (1U << smap->bucket_log); i++) {  in bpf_local_storage_map_free()
491 		b = &smap->buckets[i];  in bpf_local_storage_map_free()
518 	kvfree(smap->buckets);  in bpf_local_storage_map_free()
519 	kfree(smap);  in bpf_local_storage_map_free()
543 	struct bpf_local_storage_map *smap;  in bpf_local_storage_map_alloc()  local
549 	smap = kzalloc(sizeof(*smap), GFP_USER | __GFP_NOWARN);  in bpf_local_storage_map_alloc()
550 	if (!smap)  in bpf_local_storage_map_alloc()
552 	bpf_map_init_from_attr(&smap->map, attr);  in bpf_local_storage_map_alloc()
557 	smap->bucket_log = ilog2(nbuckets);  in bpf_local_storage_map_alloc()
558 	cost = sizeof(*smap->buckets) * nbuckets + sizeof(*smap);  in bpf_local_storage_map_alloc()
560 	ret = bpf_map_charge_init(&smap->map.memory, cost);  in bpf_local_storage_map_alloc()
562 		kfree(smap);  in bpf_local_storage_map_alloc()
566 	smap->buckets = kvcalloc(sizeof(*smap->buckets), nbuckets,  in bpf_local_storage_map_alloc()
568 	if (!smap->buckets) {  in bpf_local_storage_map_alloc()
569 		bpf_map_charge_finish(&smap->map.memory);  in bpf_local_storage_map_alloc()
570 		kfree(smap);  in bpf_local_storage_map_alloc()
575 		INIT_HLIST_HEAD(&smap->buckets[i].list);  in bpf_local_storage_map_alloc()
576 		raw_spin_lock_init(&smap->buckets[i].lock);  in bpf_local_storage_map_alloc()
579 	smap->elem_size =  in bpf_local_storage_map_alloc()
582 	return smap;  in bpf_local_storage_map_alloc()