20 	struct bpf_map map;  member
28 static struct bpf_cgroup_storage_map *map_to_storage(struct bpf_map *map)  in map_to_storage()  argument
30 	return container_of(map, struct bpf_cgroup_storage_map, map);  in map_to_storage()
49 	struct bpf_cgroup_storage_map *map, struct bpf_cgroup_storage_key *key,  in cgroup_storage_lookup()  argument
52 	struct rb_root *root = &map->root;  in cgroup_storage_lookup()
56 		spin_lock_bh(&map->lock);  in cgroup_storage_lookup()
73 				spin_unlock_bh(&map->lock);  in cgroup_storage_lookup()
79 		spin_unlock_bh(&map->lock);  in cgroup_storage_lookup()
84 static int cgroup_storage_insert(struct bpf_cgroup_storage_map *map,  in cgroup_storage_insert()  argument
87 	struct rb_root *root = &map->root;  in cgroup_storage_insert()
116 	struct bpf_cgroup_storage_map *map = map_to_storage(_map);  in cgroup_storage_lookup_elem()  local
120 	storage = cgroup_storage_lookup(map, key, false);  in cgroup_storage_lookup_elem()
127 static int cgroup_storage_update_elem(struct bpf_map *map, void *_key,  in cgroup_storage_update_elem()  argument
141 		     !map_value_has_spin_lock(map)))  in cgroup_storage_update_elem()
144 	storage = cgroup_storage_lookup((struct bpf_cgroup_storage_map *)map,  in cgroup_storage_update_elem()
150 		copy_map_value_locked(map, storage->buf->data, value, false);  in cgroup_storage_update_elem()
155 			   map->value_size,  in cgroup_storage_update_elem()
157 			   map->numa_node);  in cgroup_storage_update_elem()
161 	memcpy(&new->data[0], value, map->value_size);  in cgroup_storage_update_elem()
162 	check_and_init_map_lock(map, new->data);  in cgroup_storage_update_elem()
173 	struct bpf_cgroup_storage_map *map = map_to_storage(_map);  in bpf_percpu_cgroup_storage_copy()  local
180 	storage = cgroup_storage_lookup(map, key, false);  in bpf_percpu_cgroup_storage_copy()
203 	struct bpf_cgroup_storage_map *map = map_to_storage(_map);  in bpf_percpu_cgroup_storage_update()  local
213 	storage = cgroup_storage_lookup(map, key, false);  in bpf_percpu_cgroup_storage_update()
238 	struct bpf_cgroup_storage_map *map = map_to_storage(_map);  in cgroup_storage_get_next_key()  local
243 	spin_lock_bh(&map->lock);  in cgroup_storage_get_next_key()
245 	if (list_empty(&map->list))  in cgroup_storage_get_next_key()
249 		storage = cgroup_storage_lookup(map, key, true);  in cgroup_storage_get_next_key()
257 		storage = list_first_entry(&map->list,  in cgroup_storage_get_next_key()
261 	spin_unlock_bh(&map->lock);  in cgroup_storage_get_next_key()
267 	spin_unlock_bh(&map->lock);  in cgroup_storage_get_next_key()
274 	struct bpf_cgroup_storage_map *map;  in cgroup_storage_map_alloc()  local
299 	map = kmalloc_node(sizeof(struct bpf_cgroup_storage_map),  in cgroup_storage_map_alloc()
301 	if (!map) {  in cgroup_storage_map_alloc()
306 	bpf_map_charge_move(&map->map.memory, &mem);  in cgroup_storage_map_alloc()
309 	bpf_map_init_from_attr(&map->map, attr);  in cgroup_storage_map_alloc()
311 	spin_lock_init(&map->lock);  in cgroup_storage_map_alloc()
312 	map->root = RB_ROOT;  in cgroup_storage_map_alloc()
313 	INIT_LIST_HEAD(&map->list);  in cgroup_storage_map_alloc()
315 	return &map->map;  in cgroup_storage_map_alloc()
320 	struct bpf_cgroup_storage_map *map = map_to_storage(_map);  in cgroup_storage_map_free()  local
322 	WARN_ON(!RB_EMPTY_ROOT(&map->root));  in cgroup_storage_map_free()
323 	WARN_ON(!list_empty(&map->list));  in cgroup_storage_map_free()
325 	kfree(map);  in cgroup_storage_map_free()
328 static int cgroup_storage_delete_elem(struct bpf_map *map, void *key)  in cgroup_storage_delete_elem()  argument
333 static int cgroup_storage_check_btf(const struct bpf_map *map,  in cgroup_storage_check_btf()  argument
376 static void cgroup_storage_seq_show_elem(struct bpf_map *map, void *_key,  in cgroup_storage_seq_show_elem()  argument
379 	enum bpf_cgroup_storage_type stype = cgroup_storage_type(map);  in cgroup_storage_seq_show_elem()
385 	storage = cgroup_storage_lookup(map_to_storage(map), key, false);  in cgroup_storage_seq_show_elem()
391 	btf_type_seq_show(map->btf, map->btf_key_type_id, key, m);  in cgroup_storage_seq_show_elem()
392 	stype = cgroup_storage_type(map);  in cgroup_storage_seq_show_elem()
395 		btf_type_seq_show(map->btf, map->btf_value_type_id,  in cgroup_storage_seq_show_elem()
402 			btf_type_seq_show(map->btf, map->btf_value_type_id,  in cgroup_storage_seq_show_elem()
426 	struct bpf_cgroup_storage_map *map = map_to_storage(_map);  in bpf_cgroup_storage_assign()  local
429 	spin_lock_bh(&map->lock);  in bpf_cgroup_storage_assign()
431 	if (map->prog && map->prog != prog)  in bpf_cgroup_storage_assign()
437 	map->prog = prog;  in bpf_cgroup_storage_assign()
441 	spin_unlock_bh(&map->lock);  in bpf_cgroup_storage_assign()
449 	struct bpf_cgroup_storage_map *map = map_to_storage(_map);  in bpf_cgroup_storage_release()  local
451 	spin_lock_bh(&map->lock);  in bpf_cgroup_storage_release()
452 	if (map->prog == prog) {  in bpf_cgroup_storage_release()
454 		map->prog = NULL;  in bpf_cgroup_storage_release()
457 	spin_unlock_bh(&map->lock);  in bpf_cgroup_storage_release()
460 static size_t bpf_cgroup_storage_calculate_size(struct bpf_map *map, u32 *pages)  in bpf_cgroup_storage_calculate_size()  argument
464 	if (cgroup_storage_type(map) == BPF_CGROUP_STORAGE_SHARED) {  in bpf_cgroup_storage_calculate_size()
465 		size = sizeof(struct bpf_storage_buffer) + map->value_size;  in bpf_cgroup_storage_calculate_size()
469 		size = map->value_size;  in bpf_cgroup_storage_calculate_size()
481 	struct bpf_map *map;  in bpf_cgroup_storage_alloc()  local
486 	map = prog->aux->cgroup_storage[stype];  in bpf_cgroup_storage_alloc()
487 	if (!map)  in bpf_cgroup_storage_alloc()
490 	size = bpf_cgroup_storage_calculate_size(map, &pages);  in bpf_cgroup_storage_alloc()
492 	if (bpf_map_charge_memlock(map, pages))  in bpf_cgroup_storage_alloc()
496 			       __GFP_ZERO | GFP_USER, map->numa_node);  in bpf_cgroup_storage_alloc()
503 		storage->buf = kmalloc_node(size, flags, map->numa_node);  in bpf_cgroup_storage_alloc()
506 		check_and_init_map_lock(map, storage->buf->data);  in bpf_cgroup_storage_alloc()
513 	storage->map = (struct bpf_cgroup_storage_map *)map;  in bpf_cgroup_storage_alloc()
518 	bpf_map_uncharge_memlock(map, pages);  in bpf_cgroup_storage_alloc()
544 	struct bpf_map *map;  in bpf_cgroup_storage_free()  local
550 	map = &storage->map->map;  in bpf_cgroup_storage_free()
552 	bpf_cgroup_storage_calculate_size(map, &pages);  in bpf_cgroup_storage_free()
553 	bpf_map_uncharge_memlock(map, pages);  in bpf_cgroup_storage_free()
555 	stype = cgroup_storage_type(map);  in bpf_cgroup_storage_free()
566 	struct bpf_cgroup_storage_map *map;  in bpf_cgroup_storage_link()  local
574 	map = storage->map;  in bpf_cgroup_storage_link()
576 	spin_lock_bh(&map->lock);  in bpf_cgroup_storage_link()
577 	WARN_ON(cgroup_storage_insert(map, storage));  in bpf_cgroup_storage_link()
578 	list_add(&storage->list, &map->list);  in bpf_cgroup_storage_link()
579 	spin_unlock_bh(&map->lock);  in bpf_cgroup_storage_link()
584 	struct bpf_cgroup_storage_map *map;  in bpf_cgroup_storage_unlink()  local
590 	map = storage->map;  in bpf_cgroup_storage_unlink()
592 	spin_lock_bh(&map->lock);  in bpf_cgroup_storage_unlink()
593 	root = &map->root;  in bpf_cgroup_storage_unlink()
597 	spin_unlock_bh(&map->lock);  in bpf_cgroup_storage_unlink()