21 	struct bpf_map map;  member
28 static struct bpf_cgroup_storage_map *map_to_storage(struct bpf_map *map)  in map_to_storage()  argument
30 	return container_of(map, struct bpf_cgroup_storage_map, map);  in map_to_storage()
33 static bool attach_type_isolated(const struct bpf_map *map)  in attach_type_isolated()  argument
35 	return map->key_size == sizeof(struct bpf_cgroup_storage_key);  in attach_type_isolated()
38 static int bpf_cgroup_storage_key_cmp(const struct bpf_cgroup_storage_map *map,  in bpf_cgroup_storage_key_cmp()  argument
41 	if (attach_type_isolated(&map->map)) {  in bpf_cgroup_storage_key_cmp()
66 cgroup_storage_lookup(struct bpf_cgroup_storage_map *map,  in cgroup_storage_lookup()  argument
69 	struct rb_root *root = &map->root;  in cgroup_storage_lookup()
73 		spin_lock_bh(&map->lock);  in cgroup_storage_lookup()
81 		switch (bpf_cgroup_storage_key_cmp(map, key, &storage->key)) {  in cgroup_storage_lookup()
90 				spin_unlock_bh(&map->lock);  in cgroup_storage_lookup()
96 		spin_unlock_bh(&map->lock);  in cgroup_storage_lookup()
101 static int cgroup_storage_insert(struct bpf_cgroup_storage_map *map,  in cgroup_storage_insert()  argument
104 	struct rb_root *root = &map->root;  in cgroup_storage_insert()
113 		switch (bpf_cgroup_storage_key_cmp(map, &storage->key, &this->key)) {  in cgroup_storage_insert()
133 	struct bpf_cgroup_storage_map *map = map_to_storage(_map);  in cgroup_storage_lookup_elem()  local
136 	storage = cgroup_storage_lookup(map, key, false);  in cgroup_storage_lookup_elem()
143 static int cgroup_storage_update_elem(struct bpf_map *map, void *key,  in cgroup_storage_update_elem()  argument
153 		     !map_value_has_spin_lock(map)))  in cgroup_storage_update_elem()
156 	storage = cgroup_storage_lookup((struct bpf_cgroup_storage_map *)map,  in cgroup_storage_update_elem()
162 		copy_map_value_locked(map, storage->buf->data, value, false);  in cgroup_storage_update_elem()
166 	new = bpf_map_kmalloc_node(map, sizeof(struct bpf_storage_buffer) +  in cgroup_storage_update_elem()
167 				   map->value_size,  in cgroup_storage_update_elem()
169 				   map->numa_node);  in cgroup_storage_update_elem()
173 	memcpy(&new->data[0], value, map->value_size);  in cgroup_storage_update_elem()
174 	check_and_init_map_value(map, new->data);  in cgroup_storage_update_elem()
185 	struct bpf_cgroup_storage_map *map = map_to_storage(_map);  in bpf_percpu_cgroup_storage_copy()  local
191 	storage = cgroup_storage_lookup(map, key, false);  in bpf_percpu_cgroup_storage_copy()
214 	struct bpf_cgroup_storage_map *map = map_to_storage(_map);  in bpf_percpu_cgroup_storage_update()  local
223 	storage = cgroup_storage_lookup(map, key, false);  in bpf_percpu_cgroup_storage_update()
248 	struct bpf_cgroup_storage_map *map = map_to_storage(_map);  in cgroup_storage_get_next_key()  local
251 	spin_lock_bh(&map->lock);  in cgroup_storage_get_next_key()
253 	if (list_empty(&map->list))  in cgroup_storage_get_next_key()
257 		storage = cgroup_storage_lookup(map, key, true);  in cgroup_storage_get_next_key()
265 		storage = list_first_entry(&map->list,  in cgroup_storage_get_next_key()
269 	spin_unlock_bh(&map->lock);  in cgroup_storage_get_next_key()
271 	if (attach_type_isolated(&map->map)) {  in cgroup_storage_get_next_key()
281 	spin_unlock_bh(&map->lock);  in cgroup_storage_get_next_key()
289 	struct bpf_cgroup_storage_map *map;  in cgroup_storage_map_alloc()  local
316 	map = kmalloc_node(sizeof(struct bpf_cgroup_storage_map),  in cgroup_storage_map_alloc()
318 	if (!map)  in cgroup_storage_map_alloc()
321 	/* copy mandatory map attributes */  in cgroup_storage_map_alloc()
322 	bpf_map_init_from_attr(&map->map, attr);  in cgroup_storage_map_alloc()
324 	spin_lock_init(&map->lock);  in cgroup_storage_map_alloc()
325 	map->root = RB_ROOT;  in cgroup_storage_map_alloc()
326 	INIT_LIST_HEAD(&map->list);  in cgroup_storage_map_alloc()
328 	return &map->map;  in cgroup_storage_map_alloc()
333 	struct bpf_cgroup_storage_map *map = map_to_storage(_map);  in cgroup_storage_map_free()  local
334 	struct list_head *storages = &map->list;  in cgroup_storage_map_free()
346 	WARN_ON(!RB_EMPTY_ROOT(&map->root));  in cgroup_storage_map_free()
347 	WARN_ON(!list_empty(&map->list));  in cgroup_storage_map_free()
349 	kfree(map);  in cgroup_storage_map_free()
352 static int cgroup_storage_delete_elem(struct bpf_map *map, void *key)  in cgroup_storage_delete_elem()  argument
357 static int cgroup_storage_check_btf(const struct bpf_map *map,  in cgroup_storage_check_btf()  argument
362 	if (attach_type_isolated(map)) {  in cgroup_storage_check_btf()
415 static void cgroup_storage_seq_show_elem(struct bpf_map *map, void *key,  in cgroup_storage_seq_show_elem()  argument
423 	storage = cgroup_storage_lookup(map_to_storage(map), key, false);  in cgroup_storage_seq_show_elem()
429 	btf_type_seq_show(map->btf, map->btf_key_type_id, key, m);  in cgroup_storage_seq_show_elem()
430 	stype = cgroup_storage_type(map);  in cgroup_storage_seq_show_elem()
433 		btf_type_seq_show(map->btf, map->btf_value_type_id,  in cgroup_storage_seq_show_elem()
440 			btf_type_seq_show(map->btf, map->btf_value_type_id,  in cgroup_storage_seq_show_elem()
476 static size_t bpf_cgroup_storage_calculate_size(struct bpf_map *map, u32 *pages)  in bpf_cgroup_storage_calculate_size()  argument
480 	if (cgroup_storage_type(map) == BPF_CGROUP_STORAGE_SHARED) {  in bpf_cgroup_storage_calculate_size()
481 		size = sizeof(struct bpf_storage_buffer) + map->value_size;  in bpf_cgroup_storage_calculate_size()
485 		size = map->value_size;  in bpf_cgroup_storage_calculate_size()
498 	struct bpf_map *map;  in bpf_cgroup_storage_alloc()  local
502 	map = prog->aux->cgroup_storage[stype];  in bpf_cgroup_storage_alloc()
503 	if (!map)  in bpf_cgroup_storage_alloc()
506 	size = bpf_cgroup_storage_calculate_size(map, &pages);  in bpf_cgroup_storage_alloc()
508 	storage = bpf_map_kmalloc_node(map, sizeof(struct bpf_cgroup_storage),  in bpf_cgroup_storage_alloc()
509 				       gfp, map->numa_node);  in bpf_cgroup_storage_alloc()
514 		storage->buf = bpf_map_kmalloc_node(map, size, gfp,  in bpf_cgroup_storage_alloc()
515 						    map->numa_node);  in bpf_cgroup_storage_alloc()
518 		check_and_init_map_value(map, storage->buf->data);  in bpf_cgroup_storage_alloc()
520 		storage->percpu_buf = bpf_map_alloc_percpu(map, size, 8, gfp);  in bpf_cgroup_storage_alloc()
525 	storage->map = (struct bpf_cgroup_storage_map *)map;  in bpf_cgroup_storage_alloc()
555 	struct bpf_map *map;  in bpf_cgroup_storage_free()  local
560 	map = &storage->map->map;  in bpf_cgroup_storage_free()
561 	stype = cgroup_storage_type(map);  in bpf_cgroup_storage_free()
572 	struct bpf_cgroup_storage_map *map;  in bpf_cgroup_storage_link()  local
580 	map = storage->map;  in bpf_cgroup_storage_link()
582 	spin_lock_bh(&map->lock);  in bpf_cgroup_storage_link()
583 	WARN_ON(cgroup_storage_insert(map, storage));  in bpf_cgroup_storage_link()
584 	list_add(&storage->list_map, &map->list);  in bpf_cgroup_storage_link()
586 	spin_unlock_bh(&map->lock);  in bpf_cgroup_storage_link()
591 	struct bpf_cgroup_storage_map *map;  in bpf_cgroup_storage_unlink()  local
597 	map = storage->map;  in bpf_cgroup_storage_unlink()
599 	spin_lock_bh(&map->lock);  in bpf_cgroup_storage_unlink()
600 	root = &map->root;  in bpf_cgroup_storage_unlink()
605 	spin_unlock_bh(&map->lock);  in bpf_cgroup_storage_unlink()