21 select_bucket(struct bpf_local_storage_map *smap,  in select_bucket()  argument
24 	return &smap->buckets[hash_ptr(selem, smap->bucket_log)];  in select_bucket()
27 static int mem_charge(struct bpf_local_storage_map *smap, void *owner, u32 size)  in mem_charge()  argument
29 	struct bpf_map *map = &smap->map;  in mem_charge()
34 	return map->ops->map_local_storage_charge(smap, owner, size);  in mem_charge()
37 static void mem_uncharge(struct bpf_local_storage_map *smap, void *owner,  in mem_uncharge()  argument
40 	struct bpf_map *map = &smap->map;  in mem_uncharge()
43 		map->ops->map_local_storage_uncharge(smap, owner, size);  in mem_uncharge()
47 owner_storage(struct bpf_local_storage_map *smap, void *owner)  in owner_storage()  argument
49 	struct bpf_map *map = &smap->map;  in owner_storage()
65 bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner,  in bpf_selem_alloc()  argument
70 	if (charge_mem && mem_charge(smap, owner, smap->elem_size))  in bpf_selem_alloc()
73 	selem = bpf_map_kzalloc(&smap->map, smap->elem_size,  in bpf_selem_alloc()
77 			copy_map_value(&smap->map, SDATA(selem)->data, value);  in bpf_selem_alloc()
82 		mem_uncharge(smap, owner, smap->elem_size);  in bpf_selem_alloc()
111 	struct bpf_local_storage_map *smap;  in bpf_selem_unlink_storage_nolock()  local
115 	smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held());  in bpf_selem_unlink_storage_nolock()
123 		mem_uncharge(smap, owner, smap->elem_size);  in bpf_selem_unlink_storage_nolock()
128 		mem_uncharge(smap, owner, sizeof(struct bpf_local_storage));  in bpf_selem_unlink_storage_nolock()
132 		RCU_INIT_POINTER(*owner_storage(smap, owner), NULL);  in bpf_selem_unlink_storage_nolock()
149 	if (rcu_access_pointer(local_storage->cache[smap->cache_idx]) ==  in bpf_selem_unlink_storage_nolock()
151 		RCU_INIT_POINTER(local_storage->cache[smap->cache_idx], NULL);  in bpf_selem_unlink_storage_nolock()
198 	struct bpf_local_storage_map *smap;  in bpf_selem_unlink_map()  local
206 	smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held());  in bpf_selem_unlink_map()
207 	b = select_bucket(smap, selem);  in bpf_selem_unlink_map()
214 void bpf_selem_link_map(struct bpf_local_storage_map *smap,  in bpf_selem_link_map()  argument
217 	struct bpf_local_storage_map_bucket *b = select_bucket(smap, selem);  in bpf_selem_link_map()
221 	RCU_INIT_POINTER(SDATA(selem)->smap, smap);  in bpf_selem_link_map()
238 			 struct bpf_local_storage_map *smap,  in bpf_local_storage_lookup()  argument
245 	sdata = rcu_dereference_check(local_storage->cache[smap->cache_idx],  in bpf_local_storage_lookup()
247 	if (sdata && rcu_access_pointer(sdata->smap) == smap)  in bpf_local_storage_lookup()
253 		if (rcu_access_pointer(SDATA(selem)->smap) == smap)  in bpf_local_storage_lookup()
270 			rcu_assign_pointer(local_storage->cache[smap->cache_idx],  in bpf_local_storage_lookup()
293 			    struct bpf_local_storage_map *smap,  in bpf_local_storage_alloc()  argument
301 	err = mem_charge(smap, owner, sizeof(*storage));  in bpf_local_storage_alloc()
305 	storage = bpf_map_kzalloc(&smap->map, sizeof(*storage),  in bpf_local_storage_alloc()
317 	bpf_selem_link_map(smap, first_selem);  in bpf_local_storage_alloc()
320 		(struct bpf_local_storage **)owner_storage(smap, owner);  in bpf_local_storage_alloc()
352 	mem_uncharge(smap, owner, sizeof(*storage));  in bpf_local_storage_alloc()
362 bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,  in bpf_local_storage_update()  argument
375 		     !map_value_has_spin_lock(&smap->map)))  in bpf_local_storage_update()
381 	local_storage = rcu_dereference_check(*owner_storage(smap, owner),  in bpf_local_storage_update()
389 		selem = bpf_selem_alloc(smap, owner, value, true, gfp_flags);  in bpf_local_storage_update()
393 		err = bpf_local_storage_alloc(owner, smap, selem, gfp_flags);  in bpf_local_storage_update()
396 			mem_uncharge(smap, owner, smap->elem_size);  in bpf_local_storage_update()
409 			bpf_local_storage_lookup(local_storage, smap, false);  in bpf_local_storage_update()
414 			copy_map_value_locked(&smap->map, old_sdata->data,  in bpf_local_storage_update()
421 		selem = bpf_selem_alloc(smap, owner, value, true, gfp_flags);  in bpf_local_storage_update()
439 	old_sdata = bpf_local_storage_lookup(local_storage, smap, false);  in bpf_local_storage_update()
445 		copy_map_value_locked(&smap->map, old_sdata->data, value,  in bpf_local_storage_update()
461 		selem = bpf_selem_alloc(smap, owner, value, !old_sdata, gfp_flags);  in bpf_local_storage_update()
469 	bpf_selem_link_map(smap, selem);  in bpf_local_storage_update()
488 		mem_uncharge(smap, owner, smap->elem_size);  in bpf_local_storage_update()
526 void bpf_local_storage_map_free(struct bpf_local_storage_map *smap,  in bpf_local_storage_map_free()  argument
548 	for (i = 0; i < (1U << smap->bucket_log); i++) {  in bpf_local_storage_map_free()
549 		b = &smap->buckets[i];  in bpf_local_storage_map_free()
584 	kvfree(smap->buckets);  in bpf_local_storage_map_free()
585 	bpf_map_area_free(smap);  in bpf_local_storage_map_free()
609 	struct bpf_local_storage_map *smap;  in bpf_local_storage_map_alloc()  local
613 	smap = bpf_map_area_alloc(sizeof(*smap), NUMA_NO_NODE);  in bpf_local_storage_map_alloc()
614 	if (!smap)  in bpf_local_storage_map_alloc()
616 	bpf_map_init_from_attr(&smap->map, attr);  in bpf_local_storage_map_alloc()
621 	smap->bucket_log = ilog2(nbuckets);  in bpf_local_storage_map_alloc()
623 	smap->buckets = kvcalloc(sizeof(*smap->buckets), nbuckets,  in bpf_local_storage_map_alloc()
625 	if (!smap->buckets) {  in bpf_local_storage_map_alloc()
626 		bpf_map_area_free(smap);  in bpf_local_storage_map_alloc()
631 		INIT_HLIST_HEAD(&smap->buckets[i].list);  in bpf_local_storage_map_alloc()
632 		raw_spin_lock_init(&smap->buckets[i].lock);  in bpf_local_storage_map_alloc()
635 	smap->elem_size =  in bpf_local_storage_map_alloc()
638 	return smap;  in bpf_local_storage_map_alloc()