78 		struct bpf_cgroup_storage *storage;  in cgroup_storage_lookup()  local
80 		storage = container_of(node, struct bpf_cgroup_storage, node);  in cgroup_storage_lookup()
82 		switch (bpf_cgroup_storage_key_cmp(map, key, &storage->key)) {  in cgroup_storage_lookup()
92 			return storage;  in cgroup_storage_lookup()
103 				 struct bpf_cgroup_storage *storage)  in cgroup_storage_insert()  argument
114 		switch (bpf_cgroup_storage_key_cmp(map, &storage->key, &this->key)) {  in cgroup_storage_insert()
126 	rb_link_node(&storage->node, parent, new);  in cgroup_storage_insert()
127 	rb_insert_color(&storage->node, root);  in cgroup_storage_insert()
135 	struct bpf_cgroup_storage *storage;  in cgroup_storage_lookup_elem()  local
137 	storage = cgroup_storage_lookup(map, key, false);  in cgroup_storage_lookup_elem()
138 	if (!storage)  in cgroup_storage_lookup_elem()
141 	return &READ_ONCE(storage->buf)->data[0];  in cgroup_storage_lookup_elem()
147 	struct bpf_cgroup_storage *storage;  in cgroup_storage_update_elem()  local
157 	storage = cgroup_storage_lookup((struct bpf_cgroup_storage_map *)map,  in cgroup_storage_update_elem()
159 	if (!storage)  in cgroup_storage_update_elem()
163 		copy_map_value_locked(map, storage->buf->data, value, false);  in cgroup_storage_update_elem()
177 	new = xchg(&storage->buf, new);  in cgroup_storage_update_elem()
187 	struct bpf_cgroup_storage *storage;  in bpf_percpu_cgroup_storage_copy()  local
192 	storage = cgroup_storage_lookup(map, key, false);  in bpf_percpu_cgroup_storage_copy()
193 	if (!storage) {  in bpf_percpu_cgroup_storage_copy()
205 				per_cpu_ptr(storage->percpu_buf, cpu), size);  in bpf_percpu_cgroup_storage_copy()
216 	struct bpf_cgroup_storage *storage;  in bpf_percpu_cgroup_storage_update()  local
224 	storage = cgroup_storage_lookup(map, key, false);  in bpf_percpu_cgroup_storage_update()
225 	if (!storage) {  in bpf_percpu_cgroup_storage_update()
238 		bpf_long_memcpy(per_cpu_ptr(storage->percpu_buf, cpu),  in bpf_percpu_cgroup_storage_update()
250 	struct bpf_cgroup_storage *storage;  in cgroup_storage_get_next_key()  local
258 		storage = cgroup_storage_lookup(map, key, true);  in cgroup_storage_get_next_key()
259 		if (!storage)  in cgroup_storage_get_next_key()
262 		storage = list_next_entry(storage, list_map);  in cgroup_storage_get_next_key()
263 		if (!storage)  in cgroup_storage_get_next_key()
266 		storage = list_first_entry(&map->list,  in cgroup_storage_get_next_key()
274 		*next = storage->key;  in cgroup_storage_get_next_key()
277 		*next = storage->key.cgroup_inode_id;  in cgroup_storage_get_next_key()
338 	struct bpf_cgroup_storage *storage, *stmp;  in cgroup_storage_map_free()  local
342 	list_for_each_entry_safe(storage, stmp, storages, list_map) {  in cgroup_storage_map_free()
343 		bpf_cgroup_storage_unlink(storage);  in cgroup_storage_map_free()
344 		bpf_cgroup_storage_free(storage);  in cgroup_storage_map_free()
422 	struct bpf_cgroup_storage *storage;  in cgroup_storage_seq_show_elem()  local
426 	storage = cgroup_storage_lookup(map_to_storage(map), key, false);  in cgroup_storage_seq_show_elem()
427 	if (!storage) {  in cgroup_storage_seq_show_elem()
437 				  &READ_ONCE(storage->buf)->data[0], m);  in cgroup_storage_seq_show_elem()
444 					  per_cpu_ptr(storage->percpu_buf, cpu),  in cgroup_storage_seq_show_elem()
499 	struct bpf_cgroup_storage *storage;  in bpf_cgroup_storage_alloc()  local
514 	storage = kmalloc_node(sizeof(struct bpf_cgroup_storage),  in bpf_cgroup_storage_alloc()
516 	if (!storage)  in bpf_cgroup_storage_alloc()
522 		storage->buf = kmalloc_node(size, flags, map->numa_node);  in bpf_cgroup_storage_alloc()
523 		if (!storage->buf)  in bpf_cgroup_storage_alloc()
525 		check_and_init_map_lock(map, storage->buf->data);  in bpf_cgroup_storage_alloc()
527 		storage->percpu_buf = __alloc_percpu_gfp(size, 8, flags);  in bpf_cgroup_storage_alloc()
528 		if (!storage->percpu_buf)  in bpf_cgroup_storage_alloc()
532 	storage->map = (struct bpf_cgroup_storage_map *)map;  in bpf_cgroup_storage_alloc()
534 	return storage;  in bpf_cgroup_storage_alloc()
538 	kfree(storage);  in bpf_cgroup_storage_alloc()
544 	struct bpf_cgroup_storage *storage =  in free_shared_cgroup_storage_rcu()  local
547 	kfree(storage->buf);  in free_shared_cgroup_storage_rcu()
548 	kfree(storage);  in free_shared_cgroup_storage_rcu()
553 	struct bpf_cgroup_storage *storage =  in free_percpu_cgroup_storage_rcu()  local
556 	free_percpu(storage->percpu_buf);  in free_percpu_cgroup_storage_rcu()
557 	kfree(storage);  in free_percpu_cgroup_storage_rcu()
560 void bpf_cgroup_storage_free(struct bpf_cgroup_storage *storage)  in bpf_cgroup_storage_free()  argument
566 	if (!storage)  in bpf_cgroup_storage_free()
569 	map = &storage->map->map;  in bpf_cgroup_storage_free()
576 		call_rcu(&storage->rcu, free_shared_cgroup_storage_rcu);  in bpf_cgroup_storage_free()
578 		call_rcu(&storage->rcu, free_percpu_cgroup_storage_rcu);  in bpf_cgroup_storage_free()
581 void bpf_cgroup_storage_link(struct bpf_cgroup_storage *storage,  in bpf_cgroup_storage_link()  argument
587 	if (!storage)  in bpf_cgroup_storage_link()
590 	storage->key.attach_type = type;  in bpf_cgroup_storage_link()
591 	storage->key.cgroup_inode_id = cgroup_id(cgroup);  in bpf_cgroup_storage_link()
593 	map = storage->map;  in bpf_cgroup_storage_link()
596 	WARN_ON(cgroup_storage_insert(map, storage));  in bpf_cgroup_storage_link()
597 	list_add(&storage->list_map, &map->list);  in bpf_cgroup_storage_link()
598 	list_add(&storage->list_cg, &cgroup->bpf.storages);  in bpf_cgroup_storage_link()
602 void bpf_cgroup_storage_unlink(struct bpf_cgroup_storage *storage)  in bpf_cgroup_storage_unlink()  argument
607 	if (!storage)  in bpf_cgroup_storage_unlink()
610 	map = storage->map;  in bpf_cgroup_storage_unlink()
614 	rb_erase(&storage->node, root);  in bpf_cgroup_storage_unlink()
616 	list_del(&storage->list_map);  in bpf_cgroup_storage_unlink()
617 	list_del(&storage->list_cg);  in bpf_cgroup_storage_unlink()