Lines Matching +full:cpu +full:- +full:map
1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
26 for (i = 0; i < array->map.max_entries; i++) { in bpf_array_free_percpu()
27 free_percpu(array->pptrs[i]); in bpf_array_free_percpu()
37 for (i = 0; i < array->map.max_entries; i++) { in bpf_array_alloc_percpu()
38 ptr = bpf_map_alloc_percpu(&array->map, array->elem_size, 8, in bpf_array_alloc_percpu()
42 return -ENOMEM; in bpf_array_alloc_percpu()
44 array->pptrs[i] = ptr; in bpf_array_alloc_percpu()
54 bool percpu = attr->map_type == BPF_MAP_TYPE_PERCPU_ARRAY; in array_map_alloc_check()
58 if (attr->max_entries == 0 || attr->key_size != 4 || in array_map_alloc_check()
59 attr->value_size == 0 || in array_map_alloc_check()
60 attr->map_flags & ~ARRAY_CREATE_FLAG_MASK || in array_map_alloc_check()
61 !bpf_map_flags_access_ok(attr->map_flags) || in array_map_alloc_check()
63 return -EINVAL; in array_map_alloc_check()
65 if (attr->map_type != BPF_MAP_TYPE_ARRAY && in array_map_alloc_check()
66 attr->map_flags & (BPF_F_MMAPABLE | BPF_F_INNER_MAP)) in array_map_alloc_check()
67 return -EINVAL; in array_map_alloc_check()
69 if (attr->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY && in array_map_alloc_check()
70 attr->map_flags & BPF_F_PRESERVE_ELEMS) in array_map_alloc_check()
71 return -EINVAL; in array_map_alloc_check()
73 /* avoid overflow on round_up(map->value_size) */ in array_map_alloc_check()
74 if (attr->value_size > INT_MAX) in array_map_alloc_check()
75 return -E2BIG; in array_map_alloc_check()
82 bool percpu = attr->map_type == BPF_MAP_TYPE_PERCPU_ARRAY; in array_map_alloc()
89 elem_size = round_up(attr->value_size, 8); in array_map_alloc()
91 max_entries = attr->max_entries; in array_map_alloc()
97 mask64 = fls_long(max_entries - 1); in array_map_alloc()
99 mask64 -= 1; in array_map_alloc()
104 * since cpu will speculate within index_mask limits in array_map_alloc()
108 if (max_entries < attr->max_entries) in array_map_alloc()
109 return ERR_PTR(-E2BIG); in array_map_alloc()
116 /* rely on vmalloc() to return page-aligned memory and in array_map_alloc()
117 * ensure array->value is exactly page-aligned in array_map_alloc()
119 if (attr->map_flags & BPF_F_MMAPABLE) { in array_map_alloc()
127 /* allocate all map elements and zero-initialize them */ in array_map_alloc()
128 if (attr->map_flags & BPF_F_MMAPABLE) { in array_map_alloc()
134 return ERR_PTR(-ENOMEM); in array_map_alloc()
136 - offsetof(struct bpf_array, value); in array_map_alloc()
141 return ERR_PTR(-ENOMEM); in array_map_alloc()
142 array->index_mask = index_mask; in array_map_alloc()
143 array->map.bypass_spec_v1 = bypass_spec_v1; in array_map_alloc()
145 /* copy mandatory map attributes */ in array_map_alloc()
146 bpf_map_init_from_attr(&array->map, attr); in array_map_alloc()
147 array->elem_size = elem_size; in array_map_alloc()
151 return ERR_PTR(-ENOMEM); in array_map_alloc()
154 return &array->map; in array_map_alloc()
159 return array->value + (u64)array->elem_size * index; in array_map_elem_ptr()
163 static void *array_map_lookup_elem(struct bpf_map *map, void *key) in array_map_lookup_elem() argument
165 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_lookup_elem()
168 if (unlikely(index >= array->map.max_entries)) in array_map_lookup_elem()
171 return array->value + (u64)array->elem_size * (index & array->index_mask); in array_map_lookup_elem()
174 static int array_map_direct_value_addr(const struct bpf_map *map, u64 *imm, in array_map_direct_value_addr() argument
177 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_direct_value_addr()
179 if (map->max_entries != 1) in array_map_direct_value_addr()
180 return -ENOTSUPP; in array_map_direct_value_addr()
181 if (off >= map->value_size) in array_map_direct_value_addr()
182 return -EINVAL; in array_map_direct_value_addr()
184 *imm = (unsigned long)array->value; in array_map_direct_value_addr()
188 static int array_map_direct_value_meta(const struct bpf_map *map, u64 imm, in array_map_direct_value_meta() argument
191 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_direct_value_meta()
192 u64 base = (unsigned long)array->value; in array_map_direct_value_meta()
193 u64 range = array->elem_size; in array_map_direct_value_meta()
195 if (map->max_entries != 1) in array_map_direct_value_meta()
196 return -ENOTSUPP; in array_map_direct_value_meta()
198 return -ENOENT; in array_map_direct_value_meta()
200 *off = imm - base; in array_map_direct_value_meta()
205 static int array_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf) in array_map_gen_lookup() argument
207 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_gen_lookup()
209 u32 elem_size = array->elem_size; in array_map_gen_lookup()
214 if (map->map_flags & BPF_F_INNER_MAP) in array_map_gen_lookup()
215 return -EOPNOTSUPP; in array_map_gen_lookup()
219 if (!map->bypass_spec_v1) { in array_map_gen_lookup()
220 *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 4); in array_map_gen_lookup()
221 *insn++ = BPF_ALU32_IMM(BPF_AND, ret, array->index_mask); in array_map_gen_lookup()
223 *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 3); in array_map_gen_lookup()
234 return insn - insn_buf; in array_map_gen_lookup()
238 static void *percpu_array_map_lookup_elem(struct bpf_map *map, void *key) in percpu_array_map_lookup_elem() argument
240 struct bpf_array *array = container_of(map, struct bpf_array, map); in percpu_array_map_lookup_elem()
243 if (unlikely(index >= array->map.max_entries)) in percpu_array_map_lookup_elem()
246 return this_cpu_ptr(array->pptrs[index & array->index_mask]); in percpu_array_map_lookup_elem()
249 static void *percpu_array_map_lookup_percpu_elem(struct bpf_map *map, void *key, u32 cpu) in percpu_array_map_lookup_percpu_elem() argument
251 struct bpf_array *array = container_of(map, struct bpf_array, map); in percpu_array_map_lookup_percpu_elem()
254 if (cpu >= nr_cpu_ids) in percpu_array_map_lookup_percpu_elem()
257 if (unlikely(index >= array->map.max_entries)) in percpu_array_map_lookup_percpu_elem()
260 return per_cpu_ptr(array->pptrs[index & array->index_mask], cpu); in percpu_array_map_lookup_percpu_elem()
263 int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value) in bpf_percpu_array_copy() argument
265 struct bpf_array *array = container_of(map, struct bpf_array, map); in bpf_percpu_array_copy()
268 int cpu, off = 0; in bpf_percpu_array_copy() local
271 if (unlikely(index >= array->map.max_entries)) in bpf_percpu_array_copy()
272 return -ENOENT; in bpf_percpu_array_copy()
274 /* per_cpu areas are zero-filled and bpf programs can only in bpf_percpu_array_copy()
278 size = array->elem_size; in bpf_percpu_array_copy()
280 pptr = array->pptrs[index & array->index_mask]; in bpf_percpu_array_copy()
281 for_each_possible_cpu(cpu) { in bpf_percpu_array_copy()
282 copy_map_value_long(map, value + off, per_cpu_ptr(pptr, cpu)); in bpf_percpu_array_copy()
283 check_and_init_map_value(map, value + off); in bpf_percpu_array_copy()
291 static int array_map_get_next_key(struct bpf_map *map, void *key, void *next_key) in array_map_get_next_key() argument
293 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_get_next_key()
297 if (index >= array->map.max_entries) { in array_map_get_next_key()
302 if (index == array->map.max_entries - 1) in array_map_get_next_key()
303 return -ENOENT; in array_map_get_next_key()
311 if (map_value_has_timer(&arr->map)) in check_and_free_fields()
312 bpf_timer_cancel_and_free(val + arr->map.timer_off); in check_and_free_fields()
313 if (map_value_has_kptrs(&arr->map)) in check_and_free_fields()
314 bpf_map_free_kptrs(&arr->map, val); in check_and_free_fields()
318 static int array_map_update_elem(struct bpf_map *map, void *key, void *value, in array_map_update_elem() argument
321 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_update_elem()
327 return -EINVAL; in array_map_update_elem()
329 if (unlikely(index >= array->map.max_entries)) in array_map_update_elem()
330 /* all elements were pre-allocated, cannot insert a new one */ in array_map_update_elem()
331 return -E2BIG; in array_map_update_elem()
335 return -EEXIST; in array_map_update_elem()
338 !map_value_has_spin_lock(map))) in array_map_update_elem()
339 return -EINVAL; in array_map_update_elem()
341 if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { in array_map_update_elem()
342 val = this_cpu_ptr(array->pptrs[index & array->index_mask]); in array_map_update_elem()
343 copy_map_value(map, val, value); in array_map_update_elem()
346 val = array->value + in array_map_update_elem()
347 (u64)array->elem_size * (index & array->index_mask); in array_map_update_elem()
349 copy_map_value_locked(map, val, value, false); in array_map_update_elem()
351 copy_map_value(map, val, value); in array_map_update_elem()
357 int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, in bpf_percpu_array_update() argument
360 struct bpf_array *array = container_of(map, struct bpf_array, map); in bpf_percpu_array_update()
363 int cpu, off = 0; in bpf_percpu_array_update() local
368 return -EINVAL; in bpf_percpu_array_update()
370 if (unlikely(index >= array->map.max_entries)) in bpf_percpu_array_update()
371 /* all elements were pre-allocated, cannot insert a new one */ in bpf_percpu_array_update()
372 return -E2BIG; in bpf_percpu_array_update()
376 return -EEXIST; in bpf_percpu_array_update()
379 * will be copied into per-cpu area. bpf programs can only access in bpf_percpu_array_update()
381 * returned or zeros which were zero-filled by percpu_alloc, in bpf_percpu_array_update()
384 size = array->elem_size; in bpf_percpu_array_update()
386 pptr = array->pptrs[index & array->index_mask]; in bpf_percpu_array_update()
387 for_each_possible_cpu(cpu) { in bpf_percpu_array_update()
388 copy_map_value_long(map, per_cpu_ptr(pptr, cpu), value + off); in bpf_percpu_array_update()
389 check_and_free_fields(array, per_cpu_ptr(pptr, cpu)); in bpf_percpu_array_update()
397 static int array_map_delete_elem(struct bpf_map *map, void *key) in array_map_delete_elem() argument
399 return -EINVAL; in array_map_delete_elem()
407 static void array_map_free_timers(struct bpf_map *map) in array_map_free_timers() argument
409 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_free_timers()
413 if (!map_value_has_timer(map)) in array_map_free_timers()
416 for (i = 0; i < array->map.max_entries; i++) in array_map_free_timers()
417 bpf_timer_cancel_and_free(array_map_elem_ptr(array, i) + map->timer_off); in array_map_free_timers()
420 /* Called when map->refcnt goes to zero, either from workqueue or from syscall */
421 static void array_map_free(struct bpf_map *map) in array_map_free() argument
423 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_free()
426 if (map_value_has_kptrs(map)) { in array_map_free()
427 if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { in array_map_free()
428 for (i = 0; i < array->map.max_entries; i++) { in array_map_free()
429 void __percpu *pptr = array->pptrs[i & array->index_mask]; in array_map_free()
430 int cpu; in array_map_free() local
432 for_each_possible_cpu(cpu) { in array_map_free()
433 bpf_map_free_kptrs(map, per_cpu_ptr(pptr, cpu)); in array_map_free()
438 for (i = 0; i < array->map.max_entries; i++) in array_map_free()
439 bpf_map_free_kptrs(map, array_map_elem_ptr(array, i)); in array_map_free()
441 bpf_map_free_kptr_off_tab(map); in array_map_free()
444 if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) in array_map_free()
447 if (array->map.map_flags & BPF_F_MMAPABLE) in array_map_free()
453 static void array_map_seq_show_elem(struct bpf_map *map, void *key, in array_map_seq_show_elem() argument
460 value = array_map_lookup_elem(map, key); in array_map_seq_show_elem()
466 if (map->btf_key_type_id) in array_map_seq_show_elem()
468 btf_type_seq_show(map->btf, map->btf_value_type_id, value, m); in array_map_seq_show_elem()
474 static void percpu_array_map_seq_show_elem(struct bpf_map *map, void *key, in percpu_array_map_seq_show_elem() argument
477 struct bpf_array *array = container_of(map, struct bpf_array, map); in percpu_array_map_seq_show_elem()
480 int cpu; in percpu_array_map_seq_show_elem() local
485 pptr = array->pptrs[index & array->index_mask]; in percpu_array_map_seq_show_elem()
486 for_each_possible_cpu(cpu) { in percpu_array_map_seq_show_elem()
487 seq_printf(m, "\tcpu%d: ", cpu); in percpu_array_map_seq_show_elem()
488 btf_type_seq_show(map->btf, map->btf_value_type_id, in percpu_array_map_seq_show_elem()
489 per_cpu_ptr(pptr, cpu), m); in percpu_array_map_seq_show_elem()
497 static int array_map_check_btf(const struct bpf_map *map, in array_map_check_btf() argument
504 /* One exception for keyless BTF: .bss/.data/.rodata map */ in array_map_check_btf()
506 if (map->map_type != BPF_MAP_TYPE_ARRAY || in array_map_check_btf()
507 map->max_entries != 1) in array_map_check_btf()
508 return -EINVAL; in array_map_check_btf()
510 if (BTF_INFO_KIND(value_type->info) != BTF_KIND_DATASEC) in array_map_check_btf()
511 return -EINVAL; in array_map_check_btf()
516 if (BTF_INFO_KIND(key_type->info) != BTF_KIND_INT) in array_map_check_btf()
517 return -EINVAL; in array_map_check_btf()
524 return -EINVAL; in array_map_check_btf()
529 static int array_map_mmap(struct bpf_map *map, struct vm_area_struct *vma) in array_map_mmap() argument
531 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_mmap()
534 if (!(map->map_flags & BPF_F_MMAPABLE)) in array_map_mmap()
535 return -EINVAL; in array_map_mmap()
537 if (vma->vm_pgoff * PAGE_SIZE + (vma->vm_end - vma->vm_start) > in array_map_mmap()
538 PAGE_ALIGN((u64)array->map.max_entries * array->elem_size)) in array_map_mmap()
539 return -EINVAL; in array_map_mmap()
542 vma->vm_pgoff + pgoff); in array_map_mmap()
550 return meta0->map_flags & BPF_F_INNER_MAP ? true : in array_map_meta_equal()
551 meta0->max_entries == meta1->max_entries; in array_map_meta_equal()
555 struct bpf_map *map; member
562 struct bpf_iter_seq_array_map_info *info = seq->private; in bpf_array_map_seq_start()
563 struct bpf_map *map = info->map; in bpf_array_map_seq_start() local
567 if (info->index >= map->max_entries) in bpf_array_map_seq_start()
572 array = container_of(map, struct bpf_array, map); in bpf_array_map_seq_start()
573 index = info->index & array->index_mask; in bpf_array_map_seq_start()
574 if (info->percpu_value_buf) in bpf_array_map_seq_start()
575 return array->pptrs[index]; in bpf_array_map_seq_start()
581 struct bpf_iter_seq_array_map_info *info = seq->private; in bpf_array_map_seq_next()
582 struct bpf_map *map = info->map; in bpf_array_map_seq_next() local
587 ++info->index; in bpf_array_map_seq_next()
588 if (info->index >= map->max_entries) in bpf_array_map_seq_next()
591 array = container_of(map, struct bpf_array, map); in bpf_array_map_seq_next()
592 index = info->index & array->index_mask; in bpf_array_map_seq_next()
593 if (info->percpu_value_buf) in bpf_array_map_seq_next()
594 return array->pptrs[index]; in bpf_array_map_seq_next()
600 struct bpf_iter_seq_array_map_info *info = seq->private; in __bpf_array_map_seq_show()
602 struct bpf_map *map = info->map; in __bpf_array_map_seq_show() local
603 struct bpf_array *array = container_of(map, struct bpf_array, map); in __bpf_array_map_seq_show()
606 int off = 0, cpu = 0; in __bpf_array_map_seq_show() local
616 ctx.map = info->map; in __bpf_array_map_seq_show()
618 ctx.key = &info->index; in __bpf_array_map_seq_show()
620 if (!info->percpu_value_buf) { in __bpf_array_map_seq_show()
624 size = array->elem_size; in __bpf_array_map_seq_show()
625 for_each_possible_cpu(cpu) { in __bpf_array_map_seq_show()
626 copy_map_value_long(map, info->percpu_value_buf + off, in __bpf_array_map_seq_show()
627 per_cpu_ptr(pptr, cpu)); in __bpf_array_map_seq_show()
628 check_and_init_map_value(map, info->percpu_value_buf + off); in __bpf_array_map_seq_show()
631 ctx.value = info->percpu_value_buf; in __bpf_array_map_seq_show()
653 struct bpf_map *map = aux->map; in bpf_iter_init_array_map() local
654 struct bpf_array *array = container_of(map, struct bpf_array, map); in bpf_iter_init_array_map()
658 if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { in bpf_iter_init_array_map()
659 buf_size = array->elem_size * num_possible_cpus(); in bpf_iter_init_array_map()
662 return -ENOMEM; in bpf_iter_init_array_map()
664 seq_info->percpu_value_buf = value_buf; in bpf_iter_init_array_map()
667 /* bpf_iter_attach_map() acquires a map uref, and the uref may be in bpf_iter_init_array_map()
668 * released before or in the middle of iterating map elements, so in bpf_iter_init_array_map()
669 * acquire an extra map uref for iterator. in bpf_iter_init_array_map()
671 bpf_map_inc_with_uref(map); in bpf_iter_init_array_map()
672 seq_info->map = map; in bpf_iter_init_array_map()
680 bpf_map_put_with_uref(seq_info->map); in bpf_iter_fini_array_map()
681 kfree(seq_info->percpu_value_buf); in bpf_iter_fini_array_map()
698 static int bpf_for_each_array_elem(struct bpf_map *map, bpf_callback_t callback_fn, in bpf_for_each_array_elem() argument
708 return -EINVAL; in bpf_for_each_array_elem()
710 is_percpu = map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY; in bpf_for_each_array_elem()
711 array = container_of(map, struct bpf_array, map); in bpf_for_each_array_elem()
714 for (i = 0; i < map->max_entries; i++) { in bpf_for_each_array_elem()
716 val = this_cpu_ptr(array->pptrs[i]); in bpf_for_each_array_elem()
721 ret = callback_fn((u64)(long)map, (u64)(long)&key, in bpf_for_each_array_elem()
723 /* return value: 0 - continue, 1 - stop and return */ in bpf_for_each_array_elem()
780 /* only file descriptors can be stored in this type of map */ in fd_array_map_alloc_check()
781 if (attr->value_size != sizeof(u32)) in fd_array_map_alloc_check()
782 return -EINVAL; in fd_array_map_alloc_check()
783 /* Program read-only/write-only not supported for special maps yet. */ in fd_array_map_alloc_check()
784 if (attr->map_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG)) in fd_array_map_alloc_check()
785 return -EINVAL; in fd_array_map_alloc_check()
789 static void fd_array_map_free(struct bpf_map *map) in fd_array_map_free() argument
791 struct bpf_array *array = container_of(map, struct bpf_array, map); in fd_array_map_free()
795 for (i = 0; i < array->map.max_entries; i++) in fd_array_map_free()
796 BUG_ON(array->ptrs[i] != NULL); in fd_array_map_free()
801 static void *fd_array_map_lookup_elem(struct bpf_map *map, void *key) in fd_array_map_lookup_elem() argument
803 return ERR_PTR(-EOPNOTSUPP); in fd_array_map_lookup_elem()
807 int bpf_fd_array_map_lookup_elem(struct bpf_map *map, void *key, u32 *value) in bpf_fd_array_map_lookup_elem() argument
812 if (!map->ops->map_fd_sys_lookup_elem) in bpf_fd_array_map_lookup_elem()
813 return -ENOTSUPP; in bpf_fd_array_map_lookup_elem()
816 elem = array_map_lookup_elem(map, key); in bpf_fd_array_map_lookup_elem()
818 *value = map->ops->map_fd_sys_lookup_elem(ptr); in bpf_fd_array_map_lookup_elem()
820 ret = -ENOENT; in bpf_fd_array_map_lookup_elem()
827 int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file, in bpf_fd_array_map_update_elem() argument
830 struct bpf_array *array = container_of(map, struct bpf_array, map); in bpf_fd_array_map_update_elem()
835 return -EINVAL; in bpf_fd_array_map_update_elem()
837 if (index >= array->map.max_entries) in bpf_fd_array_map_update_elem()
838 return -E2BIG; in bpf_fd_array_map_update_elem()
841 new_ptr = map->ops->map_fd_get_ptr(map, map_file, ufd); in bpf_fd_array_map_update_elem()
845 if (map->ops->map_poke_run) { in bpf_fd_array_map_update_elem()
846 mutex_lock(&array->aux->poke_mutex); in bpf_fd_array_map_update_elem()
847 old_ptr = xchg(array->ptrs + index, new_ptr); in bpf_fd_array_map_update_elem()
848 map->ops->map_poke_run(map, index, old_ptr, new_ptr); in bpf_fd_array_map_update_elem()
849 mutex_unlock(&array->aux->poke_mutex); in bpf_fd_array_map_update_elem()
851 old_ptr = xchg(array->ptrs + index, new_ptr); in bpf_fd_array_map_update_elem()
855 map->ops->map_fd_put_ptr(old_ptr); in bpf_fd_array_map_update_elem()
859 static int fd_array_map_delete_elem(struct bpf_map *map, void *key) in fd_array_map_delete_elem() argument
861 struct bpf_array *array = container_of(map, struct bpf_array, map); in fd_array_map_delete_elem()
865 if (index >= array->map.max_entries) in fd_array_map_delete_elem()
866 return -E2BIG; in fd_array_map_delete_elem()
868 if (map->ops->map_poke_run) { in fd_array_map_delete_elem()
869 mutex_lock(&array->aux->poke_mutex); in fd_array_map_delete_elem()
870 old_ptr = xchg(array->ptrs + index, NULL); in fd_array_map_delete_elem()
871 map->ops->map_poke_run(map, index, old_ptr, NULL); in fd_array_map_delete_elem()
872 mutex_unlock(&array->aux->poke_mutex); in fd_array_map_delete_elem()
874 old_ptr = xchg(array->ptrs + index, NULL); in fd_array_map_delete_elem()
878 map->ops->map_fd_put_ptr(old_ptr); in fd_array_map_delete_elem()
881 return -ENOENT; in fd_array_map_delete_elem()
885 static void *prog_fd_array_get_ptr(struct bpf_map *map, in prog_fd_array_get_ptr() argument
893 if (!bpf_prog_map_compatible(map, prog)) { in prog_fd_array_get_ptr()
895 return ERR_PTR(-EINVAL); in prog_fd_array_get_ptr()
908 return ((struct bpf_prog *)ptr)->aux->id; in prog_fd_array_sys_lookup_elem()
911 /* decrement refcnt of all bpf_progs that are stored in this map */
912 static void bpf_fd_array_map_clear(struct bpf_map *map) in bpf_fd_array_map_clear() argument
914 struct bpf_array *array = container_of(map, struct bpf_array, map); in bpf_fd_array_map_clear()
917 for (i = 0; i < array->map.max_entries; i++) in bpf_fd_array_map_clear()
918 fd_array_map_delete_elem(map, &i); in bpf_fd_array_map_clear()
921 static void prog_array_map_seq_show_elem(struct bpf_map *map, void *key, in prog_array_map_seq_show_elem() argument
929 elem = array_map_lookup_elem(map, key); in prog_array_map_seq_show_elem()
935 btf_type_seq_show(map->btf, map->btf_value_type_id, in prog_array_map_seq_show_elem()
949 static int prog_array_map_poke_track(struct bpf_map *map, in prog_array_map_poke_track() argument
956 aux = container_of(map, struct bpf_array, map)->aux; in prog_array_map_poke_track()
957 mutex_lock(&aux->poke_mutex); in prog_array_map_poke_track()
958 list_for_each_entry(elem, &aux->poke_progs, list) { in prog_array_map_poke_track()
959 if (elem->aux == prog_aux) in prog_array_map_poke_track()
965 ret = -ENOMEM; in prog_array_map_poke_track()
969 INIT_LIST_HEAD(&elem->list); in prog_array_map_poke_track()
974 elem->aux = prog_aux; in prog_array_map_poke_track()
976 list_add_tail(&elem->list, &aux->poke_progs); in prog_array_map_poke_track()
978 mutex_unlock(&aux->poke_mutex); in prog_array_map_poke_track()
982 static void prog_array_map_poke_untrack(struct bpf_map *map, in prog_array_map_poke_untrack() argument
988 aux = container_of(map, struct bpf_array, map)->aux; in prog_array_map_poke_untrack()
989 mutex_lock(&aux->poke_mutex); in prog_array_map_poke_untrack()
990 list_for_each_entry_safe(elem, tmp, &aux->poke_progs, list) { in prog_array_map_poke_untrack()
991 if (elem->aux == prog_aux) { in prog_array_map_poke_untrack()
992 list_del_init(&elem->list); in prog_array_map_poke_untrack()
997 mutex_unlock(&aux->poke_mutex); in prog_array_map_poke_untrack()
1000 static void prog_array_map_poke_run(struct bpf_map *map, u32 key, in prog_array_map_poke_run() argument
1008 aux = container_of(map, struct bpf_array, map)->aux; in prog_array_map_poke_run()
1009 WARN_ON_ONCE(!mutex_is_locked(&aux->poke_mutex)); in prog_array_map_poke_run()
1011 list_for_each_entry(elem, &aux->poke_progs, list) { in prog_array_map_poke_run()
1015 for (i = 0; i < elem->aux->size_poke_tab; i++) { in prog_array_map_poke_run()
1016 poke = &elem->aux->poke_tab[i]; in prog_array_map_poke_run()
1021 * not aux->prog since it might not be stable yet and in prog_array_map_poke_run()
1025 * entry. We skip these as poke->tailcall_target_stable in prog_array_map_poke_run()
1028 * poke->tailcall_target_stable are successively in prog_array_map_poke_run()
1031 * non-activated poke entries. in prog_array_map_poke_run()
1036 * bails out with -EINVAL. As these are unreachable since in prog_array_map_poke_run()
1048 if (!READ_ONCE(poke->tailcall_target_stable)) in prog_array_map_poke_run()
1050 if (poke->reason != BPF_POKE_REASON_TAIL_CALL) in prog_array_map_poke_run()
1052 if (poke->tail_call.map != map || in prog_array_map_poke_run()
1053 poke->tail_call.key != key) in prog_array_map_poke_run()
1056 old_bypass_addr = old ? NULL : poke->bypass_addr; in prog_array_map_poke_run()
1057 old_addr = old ? (u8 *)old->bpf_func + poke->adj_off : NULL; in prog_array_map_poke_run()
1058 new_addr = new ? (u8 *)new->bpf_func + poke->adj_off : NULL; in prog_array_map_poke_run()
1061 ret = bpf_arch_text_poke(poke->tailcall_target, in prog_array_map_poke_run()
1064 BUG_ON(ret < 0 && ret != -EINVAL); in prog_array_map_poke_run()
1066 ret = bpf_arch_text_poke(poke->tailcall_bypass, in prog_array_map_poke_run()
1068 poke->bypass_addr, in prog_array_map_poke_run()
1070 BUG_ON(ret < 0 && ret != -EINVAL); in prog_array_map_poke_run()
1073 ret = bpf_arch_text_poke(poke->tailcall_bypass, in prog_array_map_poke_run()
1076 poke->bypass_addr); in prog_array_map_poke_run()
1077 BUG_ON(ret < 0 && ret != -EINVAL); in prog_array_map_poke_run()
1084 ret = bpf_arch_text_poke(poke->tailcall_target, in prog_array_map_poke_run()
1087 BUG_ON(ret < 0 && ret != -EINVAL); in prog_array_map_poke_run()
1095 struct bpf_map *map = container_of(work, struct bpf_array_aux, in prog_array_map_clear_deferred() local
1096 work)->map; in prog_array_map_clear_deferred()
1097 bpf_fd_array_map_clear(map); in prog_array_map_clear_deferred()
1098 bpf_map_put(map); in prog_array_map_clear_deferred()
1101 static void prog_array_map_clear(struct bpf_map *map) in prog_array_map_clear() argument
1103 struct bpf_array_aux *aux = container_of(map, struct bpf_array, in prog_array_map_clear()
1104 map)->aux; in prog_array_map_clear()
1105 bpf_map_inc(map); in prog_array_map_clear()
1106 schedule_work(&aux->work); in prog_array_map_clear()
1112 struct bpf_map *map; in prog_array_map_alloc() local
1116 return ERR_PTR(-ENOMEM); in prog_array_map_alloc()
1118 INIT_WORK(&aux->work, prog_array_map_clear_deferred); in prog_array_map_alloc()
1119 INIT_LIST_HEAD(&aux->poke_progs); in prog_array_map_alloc()
1120 mutex_init(&aux->poke_mutex); in prog_array_map_alloc()
1122 map = array_map_alloc(attr); in prog_array_map_alloc()
1123 if (IS_ERR(map)) { in prog_array_map_alloc()
1125 return map; in prog_array_map_alloc()
1128 container_of(map, struct bpf_array, map)->aux = aux; in prog_array_map_alloc()
1129 aux->map = map; in prog_array_map_alloc()
1131 return map; in prog_array_map_alloc()
1134 static void prog_array_map_free(struct bpf_map *map) in prog_array_map_free() argument
1139 aux = container_of(map, struct bpf_array, map)->aux; in prog_array_map_free()
1140 list_for_each_entry_safe(elem, tmp, &aux->poke_progs, list) { in prog_array_map_free()
1141 list_del_init(&elem->list); in prog_array_map_free()
1145 fd_array_map_free(map); in prog_array_map_free()
1148 /* prog_array->aux->{type,jited} is a runtime binding.
1178 ee->event = perf_file->private_data; in bpf_event_entry_gen()
1179 ee->perf_file = perf_file; in bpf_event_entry_gen()
1180 ee->map_file = map_file; in bpf_event_entry_gen()
1191 fput(ee->perf_file); in __bpf_event_entry_free()
1197 call_rcu(&ee->rcu, __bpf_event_entry_free); in bpf_event_entry_free_rcu()
1200 static void *perf_event_fd_array_get_ptr(struct bpf_map *map, in perf_event_fd_array_get_ptr() argument
1212 ee = ERR_PTR(-EOPNOTSUPP); in perf_event_fd_array_get_ptr()
1213 event = perf_file->private_data; in perf_event_fd_array_get_ptr()
1214 if (perf_event_read_local(event, &value, NULL, NULL) == -EOPNOTSUPP) in perf_event_fd_array_get_ptr()
1220 ee = ERR_PTR(-ENOMEM); in perf_event_fd_array_get_ptr()
1231 static void perf_event_fd_array_release(struct bpf_map *map, in perf_event_fd_array_release() argument
1234 struct bpf_array *array = container_of(map, struct bpf_array, map); in perf_event_fd_array_release()
1238 if (map->map_flags & BPF_F_PRESERVE_ELEMS) in perf_event_fd_array_release()
1242 for (i = 0; i < array->map.max_entries; i++) { in perf_event_fd_array_release()
1243 ee = READ_ONCE(array->ptrs[i]); in perf_event_fd_array_release()
1244 if (ee && ee->map_file == map_file) in perf_event_fd_array_release()
1245 fd_array_map_delete_elem(map, &i); in perf_event_fd_array_release()
1250 static void perf_event_fd_array_map_free(struct bpf_map *map) in perf_event_fd_array_map_free() argument
1252 if (map->map_flags & BPF_F_PRESERVE_ELEMS) in perf_event_fd_array_map_free()
1253 bpf_fd_array_map_clear(map); in perf_event_fd_array_map_free()
1254 fd_array_map_free(map); in perf_event_fd_array_map_free()
1273 static void *cgroup_fd_array_get_ptr(struct bpf_map *map, in cgroup_fd_array_get_ptr() argument
1286 static void cgroup_fd_array_free(struct bpf_map *map) in cgroup_fd_array_free() argument
1288 bpf_fd_array_map_clear(map); in cgroup_fd_array_free()
1289 fd_array_map_free(map); in cgroup_fd_array_free()
1309 struct bpf_map *map, *inner_map_meta; in array_of_map_alloc() local
1311 inner_map_meta = bpf_map_meta_alloc(attr->inner_map_fd); in array_of_map_alloc()
1315 map = array_map_alloc(attr); in array_of_map_alloc()
1316 if (IS_ERR(map)) { in array_of_map_alloc()
1318 return map; in array_of_map_alloc()
1321 map->inner_map_meta = inner_map_meta; in array_of_map_alloc()
1323 return map; in array_of_map_alloc()
1326 static void array_of_map_free(struct bpf_map *map) in array_of_map_free() argument
1328 /* map->inner_map_meta is only accessed by syscall which in array_of_map_free()
1331 bpf_map_meta_free(map->inner_map_meta); in array_of_map_free()
1332 bpf_fd_array_map_clear(map); in array_of_map_free()
1333 fd_array_map_free(map); in array_of_map_free()
1336 static void *array_of_map_lookup_elem(struct bpf_map *map, void *key) in array_of_map_lookup_elem() argument
1338 struct bpf_map **inner_map = array_map_lookup_elem(map, key); in array_of_map_lookup_elem()
1346 static int array_of_map_gen_lookup(struct bpf_map *map, in array_of_map_gen_lookup() argument
1349 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_of_map_gen_lookup()
1350 u32 elem_size = array->elem_size; in array_of_map_gen_lookup()
1358 if (!map->bypass_spec_v1) { in array_of_map_gen_lookup()
1359 *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 6); in array_of_map_gen_lookup()
1360 *insn++ = BPF_ALU32_IMM(BPF_AND, ret, array->index_mask); in array_of_map_gen_lookup()
1362 *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 5); in array_of_map_gen_lookup()
1374 return insn - insn_buf; in array_of_map_gen_lookup()