Lines Matching +full:cpu +full:- +full:map
1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
26 for (i = 0; i < array->map.max_entries; i++) { in bpf_array_free_percpu()
27 free_percpu(array->pptrs[i]); in bpf_array_free_percpu()
37 for (i = 0; i < array->map.max_entries; i++) { in bpf_array_alloc_percpu()
38 ptr = bpf_map_alloc_percpu(&array->map, array->elem_size, 8, in bpf_array_alloc_percpu()
42 return -ENOMEM; in bpf_array_alloc_percpu()
44 array->pptrs[i] = ptr; in bpf_array_alloc_percpu()
54 bool percpu = attr->map_type == BPF_MAP_TYPE_PERCPU_ARRAY; in array_map_alloc_check()
58 if (attr->max_entries == 0 || attr->key_size != 4 || in array_map_alloc_check()
59 attr->value_size == 0 || in array_map_alloc_check()
60 attr->map_flags & ~ARRAY_CREATE_FLAG_MASK || in array_map_alloc_check()
61 !bpf_map_flags_access_ok(attr->map_flags) || in array_map_alloc_check()
63 return -EINVAL; in array_map_alloc_check()
65 if (attr->map_type != BPF_MAP_TYPE_ARRAY && in array_map_alloc_check()
66 attr->map_flags & (BPF_F_MMAPABLE | BPF_F_INNER_MAP)) in array_map_alloc_check()
67 return -EINVAL; in array_map_alloc_check()
69 if (attr->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY && in array_map_alloc_check()
70 attr->map_flags & BPF_F_PRESERVE_ELEMS) in array_map_alloc_check()
71 return -EINVAL; in array_map_alloc_check()
73 /* avoid overflow on round_up(map->value_size) */ in array_map_alloc_check()
74 if (attr->value_size > INT_MAX) in array_map_alloc_check()
75 return -E2BIG; in array_map_alloc_check()
82 bool percpu = attr->map_type == BPF_MAP_TYPE_PERCPU_ARRAY; in array_map_alloc()
89 elem_size = round_up(attr->value_size, 8); in array_map_alloc()
91 max_entries = attr->max_entries; in array_map_alloc()
97 mask64 = fls_long(max_entries - 1); in array_map_alloc()
99 mask64 -= 1; in array_map_alloc()
104 * since cpu will speculate within index_mask limits in array_map_alloc()
108 if (max_entries < attr->max_entries) in array_map_alloc()
109 return ERR_PTR(-E2BIG); in array_map_alloc()
116 /* rely on vmalloc() to return page-aligned memory and in array_map_alloc()
117 * ensure array->value is exactly page-aligned in array_map_alloc()
119 if (attr->map_flags & BPF_F_MMAPABLE) { in array_map_alloc()
127 /* allocate all map elements and zero-initialize them */ in array_map_alloc()
128 if (attr->map_flags & BPF_F_MMAPABLE) { in array_map_alloc()
134 return ERR_PTR(-ENOMEM); in array_map_alloc()
136 - offsetof(struct bpf_array, value); in array_map_alloc()
141 return ERR_PTR(-ENOMEM); in array_map_alloc()
142 array->index_mask = index_mask; in array_map_alloc()
143 array->map.bypass_spec_v1 = bypass_spec_v1; in array_map_alloc()
145 /* copy mandatory map attributes */ in array_map_alloc()
146 bpf_map_init_from_attr(&array->map, attr); in array_map_alloc()
147 array->elem_size = elem_size; in array_map_alloc()
151 return ERR_PTR(-ENOMEM); in array_map_alloc()
154 return &array->map; in array_map_alloc()
159 return array->value + (u64)array->elem_size * index; in array_map_elem_ptr()
163 static void *array_map_lookup_elem(struct bpf_map *map, void *key) in array_map_lookup_elem() argument
165 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_lookup_elem()
168 if (unlikely(index >= array->map.max_entries)) in array_map_lookup_elem()
171 return array->value + (u64)array->elem_size * (index & array->index_mask); in array_map_lookup_elem()
174 static int array_map_direct_value_addr(const struct bpf_map *map, u64 *imm, in array_map_direct_value_addr() argument
177 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_direct_value_addr()
179 if (map->max_entries != 1) in array_map_direct_value_addr()
180 return -ENOTSUPP; in array_map_direct_value_addr()
181 if (off >= map->value_size) in array_map_direct_value_addr()
182 return -EINVAL; in array_map_direct_value_addr()
184 *imm = (unsigned long)array->value; in array_map_direct_value_addr()
188 static int array_map_direct_value_meta(const struct bpf_map *map, u64 imm, in array_map_direct_value_meta() argument
191 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_direct_value_meta()
192 u64 base = (unsigned long)array->value; in array_map_direct_value_meta()
193 u64 range = array->elem_size; in array_map_direct_value_meta()
195 if (map->max_entries != 1) in array_map_direct_value_meta()
196 return -ENOTSUPP; in array_map_direct_value_meta()
198 return -ENOENT; in array_map_direct_value_meta()
200 *off = imm - base; in array_map_direct_value_meta()
205 static int array_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf) in array_map_gen_lookup() argument
207 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_gen_lookup()
209 u32 elem_size = array->elem_size; in array_map_gen_lookup()
214 if (map->map_flags & BPF_F_INNER_MAP) in array_map_gen_lookup()
215 return -EOPNOTSUPP; in array_map_gen_lookup()
219 if (!map->bypass_spec_v1) { in array_map_gen_lookup()
220 *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 4); in array_map_gen_lookup()
221 *insn++ = BPF_ALU32_IMM(BPF_AND, ret, array->index_mask); in array_map_gen_lookup()
223 *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 3); in array_map_gen_lookup()
234 return insn - insn_buf; in array_map_gen_lookup()
238 static void *percpu_array_map_lookup_elem(struct bpf_map *map, void *key) in percpu_array_map_lookup_elem() argument
240 struct bpf_array *array = container_of(map, struct bpf_array, map); in percpu_array_map_lookup_elem()
243 if (unlikely(index >= array->map.max_entries)) in percpu_array_map_lookup_elem()
246 return this_cpu_ptr(array->pptrs[index & array->index_mask]); in percpu_array_map_lookup_elem()
249 static void *percpu_array_map_lookup_percpu_elem(struct bpf_map *map, void *key, u32 cpu) in percpu_array_map_lookup_percpu_elem() argument
251 struct bpf_array *array = container_of(map, struct bpf_array, map); in percpu_array_map_lookup_percpu_elem()
254 if (cpu >= nr_cpu_ids) in percpu_array_map_lookup_percpu_elem()
257 if (unlikely(index >= array->map.max_entries)) in percpu_array_map_lookup_percpu_elem()
260 return per_cpu_ptr(array->pptrs[index & array->index_mask], cpu); in percpu_array_map_lookup_percpu_elem()
263 int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value) in bpf_percpu_array_copy() argument
265 struct bpf_array *array = container_of(map, struct bpf_array, map); in bpf_percpu_array_copy()
268 int cpu, off = 0; in bpf_percpu_array_copy() local
271 if (unlikely(index >= array->map.max_entries)) in bpf_percpu_array_copy()
272 return -ENOENT; in bpf_percpu_array_copy()
274 /* per_cpu areas are zero-filled and bpf programs can only in bpf_percpu_array_copy()
278 size = array->elem_size; in bpf_percpu_array_copy()
280 pptr = array->pptrs[index & array->index_mask]; in bpf_percpu_array_copy()
281 for_each_possible_cpu(cpu) { in bpf_percpu_array_copy()
282 copy_map_value_long(map, value + off, per_cpu_ptr(pptr, cpu)); in bpf_percpu_array_copy()
283 check_and_init_map_value(map, value + off); in bpf_percpu_array_copy()
291 static int array_map_get_next_key(struct bpf_map *map, void *key, void *next_key) in array_map_get_next_key() argument
293 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_get_next_key()
297 if (index >= array->map.max_entries) { in array_map_get_next_key()
302 if (index == array->map.max_entries - 1) in array_map_get_next_key()
303 return -ENOENT; in array_map_get_next_key()
310 static long array_map_update_elem(struct bpf_map *map, void *key, void *value, in array_map_update_elem() argument
313 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_update_elem()
319 return -EINVAL; in array_map_update_elem()
321 if (unlikely(index >= array->map.max_entries)) in array_map_update_elem()
322 /* all elements were pre-allocated, cannot insert a new one */ in array_map_update_elem()
323 return -E2BIG; in array_map_update_elem()
327 return -EEXIST; in array_map_update_elem()
330 !btf_record_has_field(map->record, BPF_SPIN_LOCK))) in array_map_update_elem()
331 return -EINVAL; in array_map_update_elem()
333 if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { in array_map_update_elem()
334 val = this_cpu_ptr(array->pptrs[index & array->index_mask]); in array_map_update_elem()
335 copy_map_value(map, val, value); in array_map_update_elem()
336 bpf_obj_free_fields(array->map.record, val); in array_map_update_elem()
338 val = array->value + in array_map_update_elem()
339 (u64)array->elem_size * (index & array->index_mask); in array_map_update_elem()
341 copy_map_value_locked(map, val, value, false); in array_map_update_elem()
343 copy_map_value(map, val, value); in array_map_update_elem()
344 bpf_obj_free_fields(array->map.record, val); in array_map_update_elem()
349 int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, in bpf_percpu_array_update() argument
352 struct bpf_array *array = container_of(map, struct bpf_array, map); in bpf_percpu_array_update()
355 int cpu, off = 0; in bpf_percpu_array_update() local
360 return -EINVAL; in bpf_percpu_array_update()
362 if (unlikely(index >= array->map.max_entries)) in bpf_percpu_array_update()
363 /* all elements were pre-allocated, cannot insert a new one */ in bpf_percpu_array_update()
364 return -E2BIG; in bpf_percpu_array_update()
368 return -EEXIST; in bpf_percpu_array_update()
371 * will be copied into per-cpu area. bpf programs can only access in bpf_percpu_array_update()
373 * returned or zeros which were zero-filled by percpu_alloc, in bpf_percpu_array_update()
376 size = array->elem_size; in bpf_percpu_array_update()
378 pptr = array->pptrs[index & array->index_mask]; in bpf_percpu_array_update()
379 for_each_possible_cpu(cpu) { in bpf_percpu_array_update()
380 copy_map_value_long(map, per_cpu_ptr(pptr, cpu), value + off); in bpf_percpu_array_update()
381 bpf_obj_free_fields(array->map.record, per_cpu_ptr(pptr, cpu)); in bpf_percpu_array_update()
389 static long array_map_delete_elem(struct bpf_map *map, void *key) in array_map_delete_elem() argument
391 return -EINVAL; in array_map_delete_elem()
399 static void array_map_free_timers(struct bpf_map *map) in array_map_free_timers() argument
401 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_free_timers()
405 if (!btf_record_has_field(map->record, BPF_TIMER)) in array_map_free_timers()
408 for (i = 0; i < array->map.max_entries; i++) in array_map_free_timers()
409 bpf_obj_free_timer(map->record, array_map_elem_ptr(array, i)); in array_map_free_timers()
412 /* Called when map->refcnt goes to zero, either from workqueue or from syscall */
413 static void array_map_free(struct bpf_map *map) in array_map_free() argument
415 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_free()
418 if (!IS_ERR_OR_NULL(map->record)) { in array_map_free()
419 if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { in array_map_free()
420 for (i = 0; i < array->map.max_entries; i++) { in array_map_free()
421 void __percpu *pptr = array->pptrs[i & array->index_mask]; in array_map_free()
422 int cpu; in array_map_free() local
424 for_each_possible_cpu(cpu) { in array_map_free()
425 bpf_obj_free_fields(map->record, per_cpu_ptr(pptr, cpu)); in array_map_free()
430 for (i = 0; i < array->map.max_entries; i++) in array_map_free()
431 bpf_obj_free_fields(map->record, array_map_elem_ptr(array, i)); in array_map_free()
435 if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) in array_map_free()
438 if (array->map.map_flags & BPF_F_MMAPABLE) in array_map_free()
444 static void array_map_seq_show_elem(struct bpf_map *map, void *key, in array_map_seq_show_elem() argument
451 value = array_map_lookup_elem(map, key); in array_map_seq_show_elem()
457 if (map->btf_key_type_id) in array_map_seq_show_elem()
459 btf_type_seq_show(map->btf, map->btf_value_type_id, value, m); in array_map_seq_show_elem()
465 static void percpu_array_map_seq_show_elem(struct bpf_map *map, void *key, in percpu_array_map_seq_show_elem() argument
468 struct bpf_array *array = container_of(map, struct bpf_array, map); in percpu_array_map_seq_show_elem()
471 int cpu; in percpu_array_map_seq_show_elem() local
476 pptr = array->pptrs[index & array->index_mask]; in percpu_array_map_seq_show_elem()
477 for_each_possible_cpu(cpu) { in percpu_array_map_seq_show_elem()
478 seq_printf(m, "\tcpu%d: ", cpu); in percpu_array_map_seq_show_elem()
479 btf_type_seq_show(map->btf, map->btf_value_type_id, in percpu_array_map_seq_show_elem()
480 per_cpu_ptr(pptr, cpu), m); in percpu_array_map_seq_show_elem()
488 static int array_map_check_btf(const struct bpf_map *map, in array_map_check_btf() argument
495 /* One exception for keyless BTF: .bss/.data/.rodata map */ in array_map_check_btf()
497 if (map->map_type != BPF_MAP_TYPE_ARRAY || in array_map_check_btf()
498 map->max_entries != 1) in array_map_check_btf()
499 return -EINVAL; in array_map_check_btf()
501 if (BTF_INFO_KIND(value_type->info) != BTF_KIND_DATASEC) in array_map_check_btf()
502 return -EINVAL; in array_map_check_btf()
507 if (BTF_INFO_KIND(key_type->info) != BTF_KIND_INT) in array_map_check_btf()
508 return -EINVAL; in array_map_check_btf()
515 return -EINVAL; in array_map_check_btf()
520 static int array_map_mmap(struct bpf_map *map, struct vm_area_struct *vma) in array_map_mmap() argument
522 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_mmap()
525 if (!(map->map_flags & BPF_F_MMAPABLE)) in array_map_mmap()
526 return -EINVAL; in array_map_mmap()
528 if (vma->vm_pgoff * PAGE_SIZE + (vma->vm_end - vma->vm_start) > in array_map_mmap()
529 PAGE_ALIGN((u64)array->map.max_entries * array->elem_size)) in array_map_mmap()
530 return -EINVAL; in array_map_mmap()
533 vma->vm_pgoff + pgoff); in array_map_mmap()
541 return meta0->map_flags & BPF_F_INNER_MAP ? true : in array_map_meta_equal()
542 meta0->max_entries == meta1->max_entries; in array_map_meta_equal()
546 struct bpf_map *map; member
553 struct bpf_iter_seq_array_map_info *info = seq->private; in bpf_array_map_seq_start()
554 struct bpf_map *map = info->map; in bpf_array_map_seq_start() local
558 if (info->index >= map->max_entries) in bpf_array_map_seq_start()
563 array = container_of(map, struct bpf_array, map); in bpf_array_map_seq_start()
564 index = info->index & array->index_mask; in bpf_array_map_seq_start()
565 if (info->percpu_value_buf) in bpf_array_map_seq_start()
566 return array->pptrs[index]; in bpf_array_map_seq_start()
572 struct bpf_iter_seq_array_map_info *info = seq->private; in bpf_array_map_seq_next()
573 struct bpf_map *map = info->map; in bpf_array_map_seq_next() local
578 ++info->index; in bpf_array_map_seq_next()
579 if (info->index >= map->max_entries) in bpf_array_map_seq_next()
582 array = container_of(map, struct bpf_array, map); in bpf_array_map_seq_next()
583 index = info->index & array->index_mask; in bpf_array_map_seq_next()
584 if (info->percpu_value_buf) in bpf_array_map_seq_next()
585 return array->pptrs[index]; in bpf_array_map_seq_next()
591 struct bpf_iter_seq_array_map_info *info = seq->private; in __bpf_array_map_seq_show()
593 struct bpf_map *map = info->map; in __bpf_array_map_seq_show() local
594 struct bpf_array *array = container_of(map, struct bpf_array, map); in __bpf_array_map_seq_show()
597 int off = 0, cpu = 0; in __bpf_array_map_seq_show() local
607 ctx.map = info->map; in __bpf_array_map_seq_show()
609 ctx.key = &info->index; in __bpf_array_map_seq_show()
611 if (!info->percpu_value_buf) { in __bpf_array_map_seq_show()
615 size = array->elem_size; in __bpf_array_map_seq_show()
616 for_each_possible_cpu(cpu) { in __bpf_array_map_seq_show()
617 copy_map_value_long(map, info->percpu_value_buf + off, in __bpf_array_map_seq_show()
618 per_cpu_ptr(pptr, cpu)); in __bpf_array_map_seq_show()
619 check_and_init_map_value(map, info->percpu_value_buf + off); in __bpf_array_map_seq_show()
622 ctx.value = info->percpu_value_buf; in __bpf_array_map_seq_show()
644 struct bpf_map *map = aux->map; in bpf_iter_init_array_map() local
645 struct bpf_array *array = container_of(map, struct bpf_array, map); in bpf_iter_init_array_map()
649 if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { in bpf_iter_init_array_map()
650 buf_size = array->elem_size * num_possible_cpus(); in bpf_iter_init_array_map()
653 return -ENOMEM; in bpf_iter_init_array_map()
655 seq_info->percpu_value_buf = value_buf; in bpf_iter_init_array_map()
658 /* bpf_iter_attach_map() acquires a map uref, and the uref may be in bpf_iter_init_array_map()
659 * released before or in the middle of iterating map elements, so in bpf_iter_init_array_map()
660 * acquire an extra map uref for iterator. in bpf_iter_init_array_map()
662 bpf_map_inc_with_uref(map); in bpf_iter_init_array_map()
663 seq_info->map = map; in bpf_iter_init_array_map()
671 bpf_map_put_with_uref(seq_info->map); in bpf_iter_fini_array_map()
672 kfree(seq_info->percpu_value_buf); in bpf_iter_fini_array_map()
689 static long bpf_for_each_array_elem(struct bpf_map *map, bpf_callback_t callback_fn, in bpf_for_each_array_elem() argument
699 return -EINVAL; in bpf_for_each_array_elem()
701 is_percpu = map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY; in bpf_for_each_array_elem()
702 array = container_of(map, struct bpf_array, map); in bpf_for_each_array_elem()
705 for (i = 0; i < map->max_entries; i++) { in bpf_for_each_array_elem()
707 val = this_cpu_ptr(array->pptrs[i]); in bpf_for_each_array_elem()
712 ret = callback_fn((u64)(long)map, (u64)(long)&key, in bpf_for_each_array_elem()
714 /* return value: 0 - continue, 1 - stop and return */ in bpf_for_each_array_elem()
724 static u64 array_map_mem_usage(const struct bpf_map *map) in array_map_mem_usage() argument
726 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_mem_usage()
727 bool percpu = map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY; in array_map_mem_usage()
728 u32 elem_size = array->elem_size; in array_map_mem_usage()
729 u64 entries = map->max_entries; in array_map_mem_usage()
736 if (map->map_flags & BPF_F_MMAPABLE) { in array_map_mem_usage()
795 /* only file descriptors can be stored in this type of map */ in fd_array_map_alloc_check()
796 if (attr->value_size != sizeof(u32)) in fd_array_map_alloc_check()
797 return -EINVAL; in fd_array_map_alloc_check()
798 /* Program read-only/write-only not supported for special maps yet. */ in fd_array_map_alloc_check()
799 if (attr->map_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG)) in fd_array_map_alloc_check()
800 return -EINVAL; in fd_array_map_alloc_check()
804 static void fd_array_map_free(struct bpf_map *map) in fd_array_map_free() argument
806 struct bpf_array *array = container_of(map, struct bpf_array, map); in fd_array_map_free()
810 for (i = 0; i < array->map.max_entries; i++) in fd_array_map_free()
811 BUG_ON(array->ptrs[i] != NULL); in fd_array_map_free()
816 static void *fd_array_map_lookup_elem(struct bpf_map *map, void *key) in fd_array_map_lookup_elem() argument
818 return ERR_PTR(-EOPNOTSUPP); in fd_array_map_lookup_elem()
822 int bpf_fd_array_map_lookup_elem(struct bpf_map *map, void *key, u32 *value) in bpf_fd_array_map_lookup_elem() argument
827 if (!map->ops->map_fd_sys_lookup_elem) in bpf_fd_array_map_lookup_elem()
828 return -ENOTSUPP; in bpf_fd_array_map_lookup_elem()
831 elem = array_map_lookup_elem(map, key); in bpf_fd_array_map_lookup_elem()
833 *value = map->ops->map_fd_sys_lookup_elem(ptr); in bpf_fd_array_map_lookup_elem()
835 ret = -ENOENT; in bpf_fd_array_map_lookup_elem()
842 int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file, in bpf_fd_array_map_update_elem() argument
845 struct bpf_array *array = container_of(map, struct bpf_array, map); in bpf_fd_array_map_update_elem()
850 return -EINVAL; in bpf_fd_array_map_update_elem()
852 if (index >= array->map.max_entries) in bpf_fd_array_map_update_elem()
853 return -E2BIG; in bpf_fd_array_map_update_elem()
856 new_ptr = map->ops->map_fd_get_ptr(map, map_file, ufd); in bpf_fd_array_map_update_elem()
860 if (map->ops->map_poke_run) { in bpf_fd_array_map_update_elem()
861 mutex_lock(&array->aux->poke_mutex); in bpf_fd_array_map_update_elem()
862 old_ptr = xchg(array->ptrs + index, new_ptr); in bpf_fd_array_map_update_elem()
863 map->ops->map_poke_run(map, index, old_ptr, new_ptr); in bpf_fd_array_map_update_elem()
864 mutex_unlock(&array->aux->poke_mutex); in bpf_fd_array_map_update_elem()
866 old_ptr = xchg(array->ptrs + index, new_ptr); in bpf_fd_array_map_update_elem()
870 map->ops->map_fd_put_ptr(old_ptr); in bpf_fd_array_map_update_elem()
874 static long fd_array_map_delete_elem(struct bpf_map *map, void *key) in fd_array_map_delete_elem() argument
876 struct bpf_array *array = container_of(map, struct bpf_array, map); in fd_array_map_delete_elem()
880 if (index >= array->map.max_entries) in fd_array_map_delete_elem()
881 return -E2BIG; in fd_array_map_delete_elem()
883 if (map->ops->map_poke_run) { in fd_array_map_delete_elem()
884 mutex_lock(&array->aux->poke_mutex); in fd_array_map_delete_elem()
885 old_ptr = xchg(array->ptrs + index, NULL); in fd_array_map_delete_elem()
886 map->ops->map_poke_run(map, index, old_ptr, NULL); in fd_array_map_delete_elem()
887 mutex_unlock(&array->aux->poke_mutex); in fd_array_map_delete_elem()
889 old_ptr = xchg(array->ptrs + index, NULL); in fd_array_map_delete_elem()
893 map->ops->map_fd_put_ptr(old_ptr); in fd_array_map_delete_elem()
896 return -ENOENT; in fd_array_map_delete_elem()
900 static void *prog_fd_array_get_ptr(struct bpf_map *map, in prog_fd_array_get_ptr() argument
908 if (!bpf_prog_map_compatible(map, prog)) { in prog_fd_array_get_ptr()
910 return ERR_PTR(-EINVAL); in prog_fd_array_get_ptr()
923 return ((struct bpf_prog *)ptr)->aux->id; in prog_fd_array_sys_lookup_elem()
926 /* decrement refcnt of all bpf_progs that are stored in this map */
927 static void bpf_fd_array_map_clear(struct bpf_map *map) in bpf_fd_array_map_clear() argument
929 struct bpf_array *array = container_of(map, struct bpf_array, map); in bpf_fd_array_map_clear()
932 for (i = 0; i < array->map.max_entries; i++) in bpf_fd_array_map_clear()
933 fd_array_map_delete_elem(map, &i); in bpf_fd_array_map_clear()
936 static void prog_array_map_seq_show_elem(struct bpf_map *map, void *key, in prog_array_map_seq_show_elem() argument
944 elem = array_map_lookup_elem(map, key); in prog_array_map_seq_show_elem()
950 btf_type_seq_show(map->btf, map->btf_value_type_id, in prog_array_map_seq_show_elem()
964 static int prog_array_map_poke_track(struct bpf_map *map, in prog_array_map_poke_track() argument
971 aux = container_of(map, struct bpf_array, map)->aux; in prog_array_map_poke_track()
972 mutex_lock(&aux->poke_mutex); in prog_array_map_poke_track()
973 list_for_each_entry(elem, &aux->poke_progs, list) { in prog_array_map_poke_track()
974 if (elem->aux == prog_aux) in prog_array_map_poke_track()
980 ret = -ENOMEM; in prog_array_map_poke_track()
984 INIT_LIST_HEAD(&elem->list); in prog_array_map_poke_track()
989 elem->aux = prog_aux; in prog_array_map_poke_track()
991 list_add_tail(&elem->list, &aux->poke_progs); in prog_array_map_poke_track()
993 mutex_unlock(&aux->poke_mutex); in prog_array_map_poke_track()
997 static void prog_array_map_poke_untrack(struct bpf_map *map, in prog_array_map_poke_untrack() argument
1003 aux = container_of(map, struct bpf_array, map)->aux; in prog_array_map_poke_untrack()
1004 mutex_lock(&aux->poke_mutex); in prog_array_map_poke_untrack()
1005 list_for_each_entry_safe(elem, tmp, &aux->poke_progs, list) { in prog_array_map_poke_untrack()
1006 if (elem->aux == prog_aux) { in prog_array_map_poke_untrack()
1007 list_del_init(&elem->list); in prog_array_map_poke_untrack()
1012 mutex_unlock(&aux->poke_mutex); in prog_array_map_poke_untrack()
1015 static void prog_array_map_poke_run(struct bpf_map *map, u32 key, in prog_array_map_poke_run() argument
1023 aux = container_of(map, struct bpf_array, map)->aux; in prog_array_map_poke_run()
1024 WARN_ON_ONCE(!mutex_is_locked(&aux->poke_mutex)); in prog_array_map_poke_run()
1026 list_for_each_entry(elem, &aux->poke_progs, list) { in prog_array_map_poke_run()
1030 for (i = 0; i < elem->aux->size_poke_tab; i++) { in prog_array_map_poke_run()
1031 poke = &elem->aux->poke_tab[i]; in prog_array_map_poke_run()
1036 * not aux->prog since it might not be stable yet and in prog_array_map_poke_run()
1040 * entry. We skip these as poke->tailcall_target_stable in prog_array_map_poke_run()
1043 * poke->tailcall_target_stable are successively in prog_array_map_poke_run()
1046 * non-activated poke entries. in prog_array_map_poke_run()
1051 * bails out with -EINVAL. As these are unreachable since in prog_array_map_poke_run()
1063 if (!READ_ONCE(poke->tailcall_target_stable)) in prog_array_map_poke_run()
1065 if (poke->reason != BPF_POKE_REASON_TAIL_CALL) in prog_array_map_poke_run()
1067 if (poke->tail_call.map != map || in prog_array_map_poke_run()
1068 poke->tail_call.key != key) in prog_array_map_poke_run()
1071 old_bypass_addr = old ? NULL : poke->bypass_addr; in prog_array_map_poke_run()
1072 old_addr = old ? (u8 *)old->bpf_func + poke->adj_off : NULL; in prog_array_map_poke_run()
1073 new_addr = new ? (u8 *)new->bpf_func + poke->adj_off : NULL; in prog_array_map_poke_run()
1076 ret = bpf_arch_text_poke(poke->tailcall_target, in prog_array_map_poke_run()
1079 BUG_ON(ret < 0 && ret != -EINVAL); in prog_array_map_poke_run()
1081 ret = bpf_arch_text_poke(poke->tailcall_bypass, in prog_array_map_poke_run()
1083 poke->bypass_addr, in prog_array_map_poke_run()
1085 BUG_ON(ret < 0 && ret != -EINVAL); in prog_array_map_poke_run()
1088 ret = bpf_arch_text_poke(poke->tailcall_bypass, in prog_array_map_poke_run()
1091 poke->bypass_addr); in prog_array_map_poke_run()
1092 BUG_ON(ret < 0 && ret != -EINVAL); in prog_array_map_poke_run()
1099 ret = bpf_arch_text_poke(poke->tailcall_target, in prog_array_map_poke_run()
1102 BUG_ON(ret < 0 && ret != -EINVAL); in prog_array_map_poke_run()
1110 struct bpf_map *map = container_of(work, struct bpf_array_aux, in prog_array_map_clear_deferred() local
1111 work)->map; in prog_array_map_clear_deferred()
1112 bpf_fd_array_map_clear(map); in prog_array_map_clear_deferred()
1113 bpf_map_put(map); in prog_array_map_clear_deferred()
1116 static void prog_array_map_clear(struct bpf_map *map) in prog_array_map_clear() argument
1118 struct bpf_array_aux *aux = container_of(map, struct bpf_array, in prog_array_map_clear()
1119 map)->aux; in prog_array_map_clear()
1120 bpf_map_inc(map); in prog_array_map_clear()
1121 schedule_work(&aux->work); in prog_array_map_clear()
1127 struct bpf_map *map; in prog_array_map_alloc() local
1131 return ERR_PTR(-ENOMEM); in prog_array_map_alloc()
1133 INIT_WORK(&aux->work, prog_array_map_clear_deferred); in prog_array_map_alloc()
1134 INIT_LIST_HEAD(&aux->poke_progs); in prog_array_map_alloc()
1135 mutex_init(&aux->poke_mutex); in prog_array_map_alloc()
1137 map = array_map_alloc(attr); in prog_array_map_alloc()
1138 if (IS_ERR(map)) { in prog_array_map_alloc()
1140 return map; in prog_array_map_alloc()
1143 container_of(map, struct bpf_array, map)->aux = aux; in prog_array_map_alloc()
1144 aux->map = map; in prog_array_map_alloc()
1146 return map; in prog_array_map_alloc()
1149 static void prog_array_map_free(struct bpf_map *map) in prog_array_map_free() argument
1154 aux = container_of(map, struct bpf_array, map)->aux; in prog_array_map_free()
1155 list_for_each_entry_safe(elem, tmp, &aux->poke_progs, list) { in prog_array_map_free()
1156 list_del_init(&elem->list); in prog_array_map_free()
1160 fd_array_map_free(map); in prog_array_map_free()
1163 /* prog_array->aux->{type,jited} is a runtime binding.
1194 ee->event = perf_file->private_data; in bpf_event_entry_gen()
1195 ee->perf_file = perf_file; in bpf_event_entry_gen()
1196 ee->map_file = map_file; in bpf_event_entry_gen()
1207 fput(ee->perf_file); in __bpf_event_entry_free()
1213 call_rcu(&ee->rcu, __bpf_event_entry_free); in bpf_event_entry_free_rcu()
1216 static void *perf_event_fd_array_get_ptr(struct bpf_map *map, in perf_event_fd_array_get_ptr() argument
1228 ee = ERR_PTR(-EOPNOTSUPP); in perf_event_fd_array_get_ptr()
1229 event = perf_file->private_data; in perf_event_fd_array_get_ptr()
1230 if (perf_event_read_local(event, &value, NULL, NULL) == -EOPNOTSUPP) in perf_event_fd_array_get_ptr()
1236 ee = ERR_PTR(-ENOMEM); in perf_event_fd_array_get_ptr()
1247 static void perf_event_fd_array_release(struct bpf_map *map, in perf_event_fd_array_release() argument
1250 struct bpf_array *array = container_of(map, struct bpf_array, map); in perf_event_fd_array_release()
1254 if (map->map_flags & BPF_F_PRESERVE_ELEMS) in perf_event_fd_array_release()
1258 for (i = 0; i < array->map.max_entries; i++) { in perf_event_fd_array_release()
1259 ee = READ_ONCE(array->ptrs[i]); in perf_event_fd_array_release()
1260 if (ee && ee->map_file == map_file) in perf_event_fd_array_release()
1261 fd_array_map_delete_elem(map, &i); in perf_event_fd_array_release()
1266 static void perf_event_fd_array_map_free(struct bpf_map *map) in perf_event_fd_array_map_free() argument
1268 if (map->map_flags & BPF_F_PRESERVE_ELEMS) in perf_event_fd_array_map_free()
1269 bpf_fd_array_map_clear(map); in perf_event_fd_array_map_free()
1270 fd_array_map_free(map); in perf_event_fd_array_map_free()
1290 static void *cgroup_fd_array_get_ptr(struct bpf_map *map, in cgroup_fd_array_get_ptr() argument
1303 static void cgroup_fd_array_free(struct bpf_map *map) in cgroup_fd_array_free() argument
1305 bpf_fd_array_map_clear(map); in cgroup_fd_array_free()
1306 fd_array_map_free(map); in cgroup_fd_array_free()
1327 struct bpf_map *map, *inner_map_meta; in array_of_map_alloc() local
1329 inner_map_meta = bpf_map_meta_alloc(attr->inner_map_fd); in array_of_map_alloc()
1333 map = array_map_alloc(attr); in array_of_map_alloc()
1334 if (IS_ERR(map)) { in array_of_map_alloc()
1336 return map; in array_of_map_alloc()
1339 map->inner_map_meta = inner_map_meta; in array_of_map_alloc()
1341 return map; in array_of_map_alloc()
1344 static void array_of_map_free(struct bpf_map *map) in array_of_map_free() argument
1346 /* map->inner_map_meta is only accessed by syscall which in array_of_map_free()
1349 bpf_map_meta_free(map->inner_map_meta); in array_of_map_free()
1350 bpf_fd_array_map_clear(map); in array_of_map_free()
1351 fd_array_map_free(map); in array_of_map_free()
1354 static void *array_of_map_lookup_elem(struct bpf_map *map, void *key) in array_of_map_lookup_elem() argument
1356 struct bpf_map **inner_map = array_map_lookup_elem(map, key); in array_of_map_lookup_elem()
1364 static int array_of_map_gen_lookup(struct bpf_map *map, in array_of_map_gen_lookup() argument
1367 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_of_map_gen_lookup()
1368 u32 elem_size = array->elem_size; in array_of_map_gen_lookup()
1376 if (!map->bypass_spec_v1) { in array_of_map_gen_lookup()
1377 *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 6); in array_of_map_gen_lookup()
1378 *insn++ = BPF_ALU32_IMM(BPF_AND, ret, array->index_mask); in array_of_map_gen_lookup()
1380 *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 5); in array_of_map_gen_lookup()
1392 return insn - insn_buf; in array_of_map_gen_lookup()