Lines Matching full:storage

78 		struct bpf_cgroup_storage *storage;  in cgroup_storage_lookup()  local
80 storage = container_of(node, struct bpf_cgroup_storage, node); in cgroup_storage_lookup()
82 switch (bpf_cgroup_storage_key_cmp(map, key, &storage->key)) { in cgroup_storage_lookup()
92 return storage; in cgroup_storage_lookup()
103 struct bpf_cgroup_storage *storage) in cgroup_storage_insert() argument
114 switch (bpf_cgroup_storage_key_cmp(map, &storage->key, &this->key)) { in cgroup_storage_insert()
126 rb_link_node(&storage->node, parent, new); in cgroup_storage_insert()
127 rb_insert_color(&storage->node, root); in cgroup_storage_insert()
135 struct bpf_cgroup_storage *storage; in cgroup_storage_lookup_elem() local
137 storage = cgroup_storage_lookup(map, key, false); in cgroup_storage_lookup_elem()
138 if (!storage) in cgroup_storage_lookup_elem()
141 return &READ_ONCE(storage->buf)->data[0]; in cgroup_storage_lookup_elem()
147 struct bpf_cgroup_storage *storage; in cgroup_storage_update_elem() local
157 storage = cgroup_storage_lookup((struct bpf_cgroup_storage_map *)map, in cgroup_storage_update_elem()
159 if (!storage) in cgroup_storage_update_elem()
163 copy_map_value_locked(map, storage->buf->data, value, false); in cgroup_storage_update_elem()
177 new = xchg(&storage->buf, new); in cgroup_storage_update_elem()
187 struct bpf_cgroup_storage *storage; in bpf_percpu_cgroup_storage_copy() local
192 storage = cgroup_storage_lookup(map, key, false); in bpf_percpu_cgroup_storage_copy()
193 if (!storage) { in bpf_percpu_cgroup_storage_copy()
205 per_cpu_ptr(storage->percpu_buf, cpu), size); in bpf_percpu_cgroup_storage_copy()
216 struct bpf_cgroup_storage *storage; in bpf_percpu_cgroup_storage_update() local
224 storage = cgroup_storage_lookup(map, key, false); in bpf_percpu_cgroup_storage_update()
225 if (!storage) { in bpf_percpu_cgroup_storage_update()
238 bpf_long_memcpy(per_cpu_ptr(storage->percpu_buf, cpu), in bpf_percpu_cgroup_storage_update()
250 struct bpf_cgroup_storage *storage; in cgroup_storage_get_next_key() local
258 storage = cgroup_storage_lookup(map, key, true); in cgroup_storage_get_next_key()
259 if (!storage) in cgroup_storage_get_next_key()
262 storage = list_next_entry(storage, list_map); in cgroup_storage_get_next_key()
263 if (!storage) in cgroup_storage_get_next_key()
266 storage = list_first_entry(&map->list, in cgroup_storage_get_next_key()
274 *next = storage->key; in cgroup_storage_get_next_key()
277 *next = storage->key.cgroup_inode_id; in cgroup_storage_get_next_key()
338 struct bpf_cgroup_storage *storage, *stmp; in cgroup_storage_map_free() local
342 list_for_each_entry_safe(storage, stmp, storages, list_map) { in cgroup_storage_map_free()
343 bpf_cgroup_storage_unlink(storage); in cgroup_storage_map_free()
344 bpf_cgroup_storage_free(storage); in cgroup_storage_map_free()
422 struct bpf_cgroup_storage *storage; in cgroup_storage_seq_show_elem() local
426 storage = cgroup_storage_lookup(map_to_storage(map), key, false); in cgroup_storage_seq_show_elem()
427 if (!storage) { in cgroup_storage_seq_show_elem()
437 &READ_ONCE(storage->buf)->data[0], m); in cgroup_storage_seq_show_elem()
444 per_cpu_ptr(storage->percpu_buf, cpu), in cgroup_storage_seq_show_elem()
499 struct bpf_cgroup_storage *storage; in bpf_cgroup_storage_alloc() local
514 storage = kmalloc_node(sizeof(struct bpf_cgroup_storage), in bpf_cgroup_storage_alloc()
516 if (!storage) in bpf_cgroup_storage_alloc()
522 storage->buf = kmalloc_node(size, flags, map->numa_node); in bpf_cgroup_storage_alloc()
523 if (!storage->buf) in bpf_cgroup_storage_alloc()
525 check_and_init_map_lock(map, storage->buf->data); in bpf_cgroup_storage_alloc()
527 storage->percpu_buf = __alloc_percpu_gfp(size, 8, flags); in bpf_cgroup_storage_alloc()
528 if (!storage->percpu_buf) in bpf_cgroup_storage_alloc()
532 storage->map = (struct bpf_cgroup_storage_map *)map; in bpf_cgroup_storage_alloc()
534 return storage; in bpf_cgroup_storage_alloc()
538 kfree(storage); in bpf_cgroup_storage_alloc()
544 struct bpf_cgroup_storage *storage = in free_shared_cgroup_storage_rcu() local
547 kfree(storage->buf); in free_shared_cgroup_storage_rcu()
548 kfree(storage); in free_shared_cgroup_storage_rcu()
553 struct bpf_cgroup_storage *storage = in free_percpu_cgroup_storage_rcu() local
556 free_percpu(storage->percpu_buf); in free_percpu_cgroup_storage_rcu()
557 kfree(storage); in free_percpu_cgroup_storage_rcu()
560 void bpf_cgroup_storage_free(struct bpf_cgroup_storage *storage) in bpf_cgroup_storage_free() argument
566 if (!storage) in bpf_cgroup_storage_free()
569 map = &storage->map->map; in bpf_cgroup_storage_free()
576 call_rcu(&storage->rcu, free_shared_cgroup_storage_rcu); in bpf_cgroup_storage_free()
578 call_rcu(&storage->rcu, free_percpu_cgroup_storage_rcu); in bpf_cgroup_storage_free()
581 void bpf_cgroup_storage_link(struct bpf_cgroup_storage *storage, in bpf_cgroup_storage_link() argument
587 if (!storage) in bpf_cgroup_storage_link()
590 storage->key.attach_type = type; in bpf_cgroup_storage_link()
591 storage->key.cgroup_inode_id = cgroup_id(cgroup); in bpf_cgroup_storage_link()
593 map = storage->map; in bpf_cgroup_storage_link()
596 WARN_ON(cgroup_storage_insert(map, storage)); in bpf_cgroup_storage_link()
597 list_add(&storage->list_map, &map->list); in bpf_cgroup_storage_link()
598 list_add(&storage->list_cg, &cgroup->bpf.storages); in bpf_cgroup_storage_link()
602 void bpf_cgroup_storage_unlink(struct bpf_cgroup_storage *storage) in bpf_cgroup_storage_unlink() argument
607 if (!storage) in bpf_cgroup_storage_unlink()
610 map = storage->map; in bpf_cgroup_storage_unlink()
614 rb_erase(&storage->node, root); in bpf_cgroup_storage_unlink()
616 list_del(&storage->list_map); in bpf_cgroup_storage_unlink()
617 list_del(&storage->list_cg); in bpf_cgroup_storage_unlink()