1 //SPDX-License-Identifier: GPL-2.0
2 #include <linux/bpf-cgroup.h>
3 #include <linux/bpf.h>
4 #include <linux/bug.h>
5 #include <linux/filter.h>
6 #include <linux/mm.h>
7 #include <linux/rbtree.h>
8 #include <linux/slab.h>
9
10 DEFINE_PER_CPU(void*, bpf_cgroup_storage);
11
12 #ifdef CONFIG_CGROUP_BPF
13
14 #define LOCAL_STORAGE_CREATE_FLAG_MASK \
15 (BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY)
16
17 struct bpf_cgroup_storage_map {
18 struct bpf_map map;
19
20 spinlock_t lock;
21 struct bpf_prog *prog;
22 struct rb_root root;
23 struct list_head list;
24 };
25
map_to_storage(struct bpf_map * map)26 static struct bpf_cgroup_storage_map *map_to_storage(struct bpf_map *map)
27 {
28 return container_of(map, struct bpf_cgroup_storage_map, map);
29 }
30
bpf_cgroup_storage_key_cmp(const struct bpf_cgroup_storage_key * key1,const struct bpf_cgroup_storage_key * key2)31 static int bpf_cgroup_storage_key_cmp(
32 const struct bpf_cgroup_storage_key *key1,
33 const struct bpf_cgroup_storage_key *key2)
34 {
35 if (key1->cgroup_inode_id < key2->cgroup_inode_id)
36 return -1;
37 else if (key1->cgroup_inode_id > key2->cgroup_inode_id)
38 return 1;
39 else if (key1->attach_type < key2->attach_type)
40 return -1;
41 else if (key1->attach_type > key2->attach_type)
42 return 1;
43 return 0;
44 }
45
cgroup_storage_lookup(struct bpf_cgroup_storage_map * map,struct bpf_cgroup_storage_key * key,bool locked)46 static struct bpf_cgroup_storage *cgroup_storage_lookup(
47 struct bpf_cgroup_storage_map *map, struct bpf_cgroup_storage_key *key,
48 bool locked)
49 {
50 struct rb_root *root = &map->root;
51 struct rb_node *node;
52
53 if (!locked)
54 spin_lock_bh(&map->lock);
55
56 node = root->rb_node;
57 while (node) {
58 struct bpf_cgroup_storage *storage;
59
60 storage = container_of(node, struct bpf_cgroup_storage, node);
61
62 switch (bpf_cgroup_storage_key_cmp(key, &storage->key)) {
63 case -1:
64 node = node->rb_left;
65 break;
66 case 1:
67 node = node->rb_right;
68 break;
69 default:
70 if (!locked)
71 spin_unlock_bh(&map->lock);
72 return storage;
73 }
74 }
75
76 if (!locked)
77 spin_unlock_bh(&map->lock);
78
79 return NULL;
80 }
81
cgroup_storage_insert(struct bpf_cgroup_storage_map * map,struct bpf_cgroup_storage * storage)82 static int cgroup_storage_insert(struct bpf_cgroup_storage_map *map,
83 struct bpf_cgroup_storage *storage)
84 {
85 struct rb_root *root = &map->root;
86 struct rb_node **new = &(root->rb_node), *parent = NULL;
87
88 while (*new) {
89 struct bpf_cgroup_storage *this;
90
91 this = container_of(*new, struct bpf_cgroup_storage, node);
92
93 parent = *new;
94 switch (bpf_cgroup_storage_key_cmp(&storage->key, &this->key)) {
95 case -1:
96 new = &((*new)->rb_left);
97 break;
98 case 1:
99 new = &((*new)->rb_right);
100 break;
101 default:
102 return -EEXIST;
103 }
104 }
105
106 rb_link_node(&storage->node, parent, new);
107 rb_insert_color(&storage->node, root);
108
109 return 0;
110 }
111
cgroup_storage_lookup_elem(struct bpf_map * _map,void * _key)112 static void *cgroup_storage_lookup_elem(struct bpf_map *_map, void *_key)
113 {
114 struct bpf_cgroup_storage_map *map = map_to_storage(_map);
115 struct bpf_cgroup_storage_key *key = _key;
116 struct bpf_cgroup_storage *storage;
117
118 storage = cgroup_storage_lookup(map, key, false);
119 if (!storage)
120 return NULL;
121
122 return &READ_ONCE(storage->buf)->data[0];
123 }
124
cgroup_storage_update_elem(struct bpf_map * map,void * _key,void * value,u64 flags)125 static int cgroup_storage_update_elem(struct bpf_map *map, void *_key,
126 void *value, u64 flags)
127 {
128 struct bpf_cgroup_storage_key *key = _key;
129 struct bpf_cgroup_storage *storage;
130 struct bpf_storage_buffer *new;
131
132 if (flags != BPF_ANY && flags != BPF_EXIST)
133 return -EINVAL;
134
135 storage = cgroup_storage_lookup((struct bpf_cgroup_storage_map *)map,
136 key, false);
137 if (!storage)
138 return -ENOENT;
139
140 new = kmalloc_node(sizeof(struct bpf_storage_buffer) +
141 map->value_size, __GFP_ZERO | GFP_USER,
142 map->numa_node);
143 if (!new)
144 return -ENOMEM;
145
146 memcpy(&new->data[0], value, map->value_size);
147
148 new = xchg(&storage->buf, new);
149 kfree_rcu(new, rcu);
150
151 return 0;
152 }
153
cgroup_storage_get_next_key(struct bpf_map * _map,void * _key,void * _next_key)154 static int cgroup_storage_get_next_key(struct bpf_map *_map, void *_key,
155 void *_next_key)
156 {
157 struct bpf_cgroup_storage_map *map = map_to_storage(_map);
158 struct bpf_cgroup_storage_key *key = _key;
159 struct bpf_cgroup_storage_key *next = _next_key;
160 struct bpf_cgroup_storage *storage;
161
162 spin_lock_bh(&map->lock);
163
164 if (list_empty(&map->list))
165 goto enoent;
166
167 if (key) {
168 storage = cgroup_storage_lookup(map, key, true);
169 if (!storage)
170 goto enoent;
171
172 storage = list_next_entry(storage, list);
173 if (!storage)
174 goto enoent;
175 } else {
176 storage = list_first_entry(&map->list,
177 struct bpf_cgroup_storage, list);
178 }
179
180 spin_unlock_bh(&map->lock);
181 next->attach_type = storage->key.attach_type;
182 next->cgroup_inode_id = storage->key.cgroup_inode_id;
183 return 0;
184
185 enoent:
186 spin_unlock_bh(&map->lock);
187 return -ENOENT;
188 }
189
cgroup_storage_map_alloc(union bpf_attr * attr)190 static struct bpf_map *cgroup_storage_map_alloc(union bpf_attr *attr)
191 {
192 int numa_node = bpf_map_attr_numa_node(attr);
193 struct bpf_cgroup_storage_map *map;
194
195 if (attr->key_size != sizeof(struct bpf_cgroup_storage_key))
196 return ERR_PTR(-EINVAL);
197
198 if (attr->value_size == 0)
199 return ERR_PTR(-EINVAL);
200
201 if (attr->value_size > PAGE_SIZE)
202 return ERR_PTR(-E2BIG);
203
204 if (attr->map_flags & ~LOCAL_STORAGE_CREATE_FLAG_MASK)
205 /* reserved bits should not be used */
206 return ERR_PTR(-EINVAL);
207
208 if (attr->max_entries)
209 /* max_entries is not used and enforced to be 0 */
210 return ERR_PTR(-EINVAL);
211
212 map = kmalloc_node(sizeof(struct bpf_cgroup_storage_map),
213 __GFP_ZERO | GFP_USER, numa_node);
214 if (!map)
215 return ERR_PTR(-ENOMEM);
216
217 map->map.pages = round_up(sizeof(struct bpf_cgroup_storage_map),
218 PAGE_SIZE) >> PAGE_SHIFT;
219
220 /* copy mandatory map attributes */
221 bpf_map_init_from_attr(&map->map, attr);
222
223 spin_lock_init(&map->lock);
224 map->root = RB_ROOT;
225 INIT_LIST_HEAD(&map->list);
226
227 return &map->map;
228 }
229
cgroup_storage_map_free(struct bpf_map * _map)230 static void cgroup_storage_map_free(struct bpf_map *_map)
231 {
232 struct bpf_cgroup_storage_map *map = map_to_storage(_map);
233
234 WARN_ON(!RB_EMPTY_ROOT(&map->root));
235 WARN_ON(!list_empty(&map->list));
236
237 kfree(map);
238 }
239
cgroup_storage_delete_elem(struct bpf_map * map,void * key)240 static int cgroup_storage_delete_elem(struct bpf_map *map, void *key)
241 {
242 return -EINVAL;
243 }
244
245 const struct bpf_map_ops cgroup_storage_map_ops = {
246 .map_alloc = cgroup_storage_map_alloc,
247 .map_free = cgroup_storage_map_free,
248 .map_get_next_key = cgroup_storage_get_next_key,
249 .map_lookup_elem = cgroup_storage_lookup_elem,
250 .map_update_elem = cgroup_storage_update_elem,
251 .map_delete_elem = cgroup_storage_delete_elem,
252 .map_check_btf = map_check_no_btf,
253 };
254
bpf_cgroup_storage_assign(struct bpf_prog * prog,struct bpf_map * _map)255 int bpf_cgroup_storage_assign(struct bpf_prog *prog, struct bpf_map *_map)
256 {
257 struct bpf_cgroup_storage_map *map = map_to_storage(_map);
258 int ret = -EBUSY;
259
260 spin_lock_bh(&map->lock);
261
262 if (map->prog && map->prog != prog)
263 goto unlock;
264 if (prog->aux->cgroup_storage && prog->aux->cgroup_storage != _map)
265 goto unlock;
266
267 map->prog = prog;
268 prog->aux->cgroup_storage = _map;
269 ret = 0;
270 unlock:
271 spin_unlock_bh(&map->lock);
272
273 return ret;
274 }
275
bpf_cgroup_storage_release(struct bpf_prog * prog,struct bpf_map * _map)276 void bpf_cgroup_storage_release(struct bpf_prog *prog, struct bpf_map *_map)
277 {
278 struct bpf_cgroup_storage_map *map = map_to_storage(_map);
279
280 spin_lock_bh(&map->lock);
281 if (map->prog == prog) {
282 WARN_ON(prog->aux->cgroup_storage != _map);
283 map->prog = NULL;
284 prog->aux->cgroup_storage = NULL;
285 }
286 spin_unlock_bh(&map->lock);
287 }
288
bpf_cgroup_storage_alloc(struct bpf_prog * prog)289 struct bpf_cgroup_storage *bpf_cgroup_storage_alloc(struct bpf_prog *prog)
290 {
291 struct bpf_cgroup_storage *storage;
292 struct bpf_map *map;
293 u32 pages;
294
295 map = prog->aux->cgroup_storage;
296 if (!map)
297 return NULL;
298
299 pages = round_up(sizeof(struct bpf_cgroup_storage) +
300 sizeof(struct bpf_storage_buffer) +
301 map->value_size, PAGE_SIZE) >> PAGE_SHIFT;
302 if (bpf_map_charge_memlock(map, pages))
303 return ERR_PTR(-EPERM);
304
305 storage = kmalloc_node(sizeof(struct bpf_cgroup_storage),
306 __GFP_ZERO | GFP_USER, map->numa_node);
307 if (!storage) {
308 bpf_map_uncharge_memlock(map, pages);
309 return ERR_PTR(-ENOMEM);
310 }
311
312 storage->buf = kmalloc_node(sizeof(struct bpf_storage_buffer) +
313 map->value_size, __GFP_ZERO | GFP_USER,
314 map->numa_node);
315 if (!storage->buf) {
316 bpf_map_uncharge_memlock(map, pages);
317 kfree(storage);
318 return ERR_PTR(-ENOMEM);
319 }
320
321 storage->map = (struct bpf_cgroup_storage_map *)map;
322
323 return storage;
324 }
325
bpf_cgroup_storage_free(struct bpf_cgroup_storage * storage)326 void bpf_cgroup_storage_free(struct bpf_cgroup_storage *storage)
327 {
328 u32 pages;
329 struct bpf_map *map;
330
331 if (!storage)
332 return;
333
334 map = &storage->map->map;
335 pages = round_up(sizeof(struct bpf_cgroup_storage) +
336 sizeof(struct bpf_storage_buffer) +
337 map->value_size, PAGE_SIZE) >> PAGE_SHIFT;
338 bpf_map_uncharge_memlock(map, pages);
339
340 kfree_rcu(storage->buf, rcu);
341 kfree_rcu(storage, rcu);
342 }
343
bpf_cgroup_storage_link(struct bpf_cgroup_storage * storage,struct cgroup * cgroup,enum bpf_attach_type type)344 void bpf_cgroup_storage_link(struct bpf_cgroup_storage *storage,
345 struct cgroup *cgroup,
346 enum bpf_attach_type type)
347 {
348 struct bpf_cgroup_storage_map *map;
349
350 if (!storage)
351 return;
352
353 storage->key.attach_type = type;
354 storage->key.cgroup_inode_id = cgroup->kn->id.id;
355
356 map = storage->map;
357
358 spin_lock_bh(&map->lock);
359 WARN_ON(cgroup_storage_insert(map, storage));
360 list_add(&storage->list, &map->list);
361 spin_unlock_bh(&map->lock);
362 }
363
bpf_cgroup_storage_unlink(struct bpf_cgroup_storage * storage)364 void bpf_cgroup_storage_unlink(struct bpf_cgroup_storage *storage)
365 {
366 struct bpf_cgroup_storage_map *map;
367 struct rb_root *root;
368
369 if (!storage)
370 return;
371
372 map = storage->map;
373
374 spin_lock_bh(&map->lock);
375 root = &map->root;
376 rb_erase(&storage->node, root);
377
378 list_del(&storage->list);
379 spin_unlock_bh(&map->lock);
380 }
381
382 #endif
383
