Lines Matching full:array
22 static void bpf_array_free_percpu(struct bpf_array *array) in bpf_array_free_percpu() argument
26 for (i = 0; i < array->map.max_entries; i++) { in bpf_array_free_percpu()
27 free_percpu(array->pptrs[i]); in bpf_array_free_percpu()
32 static int bpf_array_alloc_percpu(struct bpf_array *array) in bpf_array_alloc_percpu() argument
37 for (i = 0; i < array->map.max_entries; i++) { in bpf_array_alloc_percpu()
38 ptr = bpf_map_alloc_percpu(&array->map, array->elem_size, 8, in bpf_array_alloc_percpu()
41 bpf_array_free_percpu(array); in bpf_array_alloc_percpu()
44 array->pptrs[i] = ptr; in bpf_array_alloc_percpu()
87 struct bpf_array *array; in array_map_alloc() local
103 /* round up array size to nearest power of 2, in array_map_alloc()
112 array_size = sizeof(*array); in array_map_alloc()
117 * ensure array->value is exactly page-aligned in array_map_alloc()
135 array = data + PAGE_ALIGN(sizeof(struct bpf_array)) in array_map_alloc()
138 array = bpf_map_area_alloc(array_size, numa_node); in array_map_alloc()
140 if (!array) in array_map_alloc()
142 array->index_mask = index_mask; in array_map_alloc()
143 array->map.bypass_spec_v1 = bypass_spec_v1; in array_map_alloc()
146 bpf_map_init_from_attr(&array->map, attr); in array_map_alloc()
147 array->elem_size = elem_size; in array_map_alloc()
149 if (percpu && bpf_array_alloc_percpu(array)) { in array_map_alloc()
150 bpf_map_area_free(array); in array_map_alloc()
154 return &array->map; in array_map_alloc()
157 static void *array_map_elem_ptr(struct bpf_array* array, u32 index) in array_map_elem_ptr() argument
159 return array->value + (u64)array->elem_size * index; in array_map_elem_ptr()
165 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_lookup_elem() local
168 if (unlikely(index >= array->map.max_entries)) in array_map_lookup_elem()
171 return array->value + (u64)array->elem_size * (index & array->index_mask); in array_map_lookup_elem()
177 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_direct_value_addr() local
184 *imm = (unsigned long)array->value; in array_map_direct_value_addr()
191 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_direct_value_meta() local
192 u64 base = (unsigned long)array->value; in array_map_direct_value_meta()
193 u64 range = array->elem_size; in array_map_direct_value_meta()
207 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_gen_lookup() local
209 u32 elem_size = array->elem_size; in array_map_gen_lookup()
221 *insn++ = BPF_ALU32_IMM(BPF_AND, ret, array->index_mask); in array_map_gen_lookup()
240 struct bpf_array *array = container_of(map, struct bpf_array, map); in percpu_array_map_lookup_elem() local
243 if (unlikely(index >= array->map.max_entries)) in percpu_array_map_lookup_elem()
246 return this_cpu_ptr(array->pptrs[index & array->index_mask]); in percpu_array_map_lookup_elem()
251 struct bpf_array *array = container_of(map, struct bpf_array, map); in percpu_array_map_lookup_percpu_elem() local
257 if (unlikely(index >= array->map.max_entries)) in percpu_array_map_lookup_percpu_elem()
260 return per_cpu_ptr(array->pptrs[index & array->index_mask], cpu); in percpu_array_map_lookup_percpu_elem()
265 struct bpf_array *array = container_of(map, struct bpf_array, map); in bpf_percpu_array_copy() local
271 if (unlikely(index >= array->map.max_entries)) in bpf_percpu_array_copy()
278 size = array->elem_size; in bpf_percpu_array_copy()
280 pptr = array->pptrs[index & array->index_mask]; in bpf_percpu_array_copy()
293 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_get_next_key() local
297 if (index >= array->map.max_entries) { in array_map_get_next_key()
302 if (index == array->map.max_entries - 1) in array_map_get_next_key()
321 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_update_elem() local
329 if (unlikely(index >= array->map.max_entries)) in array_map_update_elem()
341 if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { in array_map_update_elem()
342 val = this_cpu_ptr(array->pptrs[index & array->index_mask]); in array_map_update_elem()
344 check_and_free_fields(array, val); in array_map_update_elem()
346 val = array->value + in array_map_update_elem()
347 (u64)array->elem_size * (index & array->index_mask); in array_map_update_elem()
352 check_and_free_fields(array, val); in array_map_update_elem()
360 struct bpf_array *array = container_of(map, struct bpf_array, map); in bpf_percpu_array_update() local
370 if (unlikely(index >= array->map.max_entries)) in bpf_percpu_array_update()
384 size = array->elem_size; in bpf_percpu_array_update()
386 pptr = array->pptrs[index & array->index_mask]; in bpf_percpu_array_update()
389 check_and_free_fields(array, per_cpu_ptr(pptr, cpu)); in bpf_percpu_array_update()
402 static void *array_map_vmalloc_addr(struct bpf_array *array) in array_map_vmalloc_addr() argument
404 return (void *)round_down((unsigned long)array, PAGE_SIZE); in array_map_vmalloc_addr()
409 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_free_timers() local
416 for (i = 0; i < array->map.max_entries; i++) in array_map_free_timers()
417 bpf_timer_cancel_and_free(array_map_elem_ptr(array, i) + map->timer_off); in array_map_free_timers()
423 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_free() local
427 if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { in array_map_free()
428 for (i = 0; i < array->map.max_entries; i++) { in array_map_free()
429 void __percpu *pptr = array->pptrs[i & array->index_mask]; in array_map_free()
438 for (i = 0; i < array->map.max_entries; i++) in array_map_free()
439 bpf_map_free_kptrs(map, array_map_elem_ptr(array, i)); in array_map_free()
444 if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) in array_map_free()
445 bpf_array_free_percpu(array); in array_map_free()
447 if (array->map.map_flags & BPF_F_MMAPABLE) in array_map_free()
448 bpf_map_area_free(array_map_vmalloc_addr(array)); in array_map_free()
450 bpf_map_area_free(array); in array_map_free()
477 struct bpf_array *array = container_of(map, struct bpf_array, map); in percpu_array_map_seq_show_elem() local
485 pptr = array->pptrs[index & array->index_mask]; in percpu_array_map_seq_show_elem()
520 /* bpf array can only take a u32 key. This check makes sure in array_map_check_btf()
531 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_mmap() local
532 pgoff_t pgoff = PAGE_ALIGN(sizeof(*array)) >> PAGE_SHIFT; in array_map_mmap()
538 PAGE_ALIGN((u64)array->map.max_entries * array->elem_size)) in array_map_mmap()
541 return remap_vmalloc_range(vma, array_map_vmalloc_addr(array), in array_map_mmap()
564 struct bpf_array *array; in bpf_array_map_seq_start() local
572 array = container_of(map, struct bpf_array, map); in bpf_array_map_seq_start()
573 index = info->index & array->index_mask; in bpf_array_map_seq_start()
575 return array->pptrs[index]; in bpf_array_map_seq_start()
576 return array_map_elem_ptr(array, index); in bpf_array_map_seq_start()
583 struct bpf_array *array; in bpf_array_map_seq_next() local
591 array = container_of(map, struct bpf_array, map); in bpf_array_map_seq_next()
592 index = info->index & array->index_mask; in bpf_array_map_seq_next()
594 return array->pptrs[index]; in bpf_array_map_seq_next()
595 return array_map_elem_ptr(array, index); in bpf_array_map_seq_next()
603 struct bpf_array *array = container_of(map, struct bpf_array, map); in __bpf_array_map_seq_show() local
624 size = array->elem_size; in __bpf_array_map_seq_show()
654 struct bpf_array *array = container_of(map, struct bpf_array, map); in bpf_iter_init_array_map() local
659 buf_size = array->elem_size * num_possible_cpus(); in bpf_iter_init_array_map()
702 struct bpf_array *array; in bpf_for_each_array_elem() local
711 array = container_of(map, struct bpf_array, map); in bpf_for_each_array_elem()
716 val = this_cpu_ptr(array->pptrs[i]); in bpf_for_each_array_elem()
718 val = array_map_elem_ptr(array, i); in bpf_for_each_array_elem()
791 struct bpf_array *array = container_of(map, struct bpf_array, map); in fd_array_map_free() local
795 for (i = 0; i < array->map.max_entries; i++) in fd_array_map_free()
796 BUG_ON(array->ptrs[i] != NULL); in fd_array_map_free()
798 bpf_map_area_free(array); in fd_array_map_free()
830 struct bpf_array *array = container_of(map, struct bpf_array, map); in bpf_fd_array_map_update_elem() local
837 if (index >= array->map.max_entries) in bpf_fd_array_map_update_elem()
846 mutex_lock(&array->aux->poke_mutex); in bpf_fd_array_map_update_elem()
847 old_ptr = xchg(array->ptrs + index, new_ptr); in bpf_fd_array_map_update_elem()
849 mutex_unlock(&array->aux->poke_mutex); in bpf_fd_array_map_update_elem()
851 old_ptr = xchg(array->ptrs + index, new_ptr); in bpf_fd_array_map_update_elem()
861 struct bpf_array *array = container_of(map, struct bpf_array, map); in fd_array_map_delete_elem() local
865 if (index >= array->map.max_entries) in fd_array_map_delete_elem()
869 mutex_lock(&array->aux->poke_mutex); in fd_array_map_delete_elem()
870 old_ptr = xchg(array->ptrs + index, NULL); in fd_array_map_delete_elem()
872 mutex_unlock(&array->aux->poke_mutex); in fd_array_map_delete_elem()
874 old_ptr = xchg(array->ptrs + index, NULL); in fd_array_map_delete_elem()
914 struct bpf_array *array = container_of(map, struct bpf_array, map); in bpf_fd_array_map_clear() local
917 for (i = 0; i < array->map.max_entries; i++) in bpf_fd_array_map_clear()
1234 struct bpf_array *array = container_of(map, struct bpf_array, map); in perf_event_fd_array_release() local
1242 for (i = 0; i < array->map.max_entries; i++) { in perf_event_fd_array_release()
1243 ee = READ_ONCE(array->ptrs[i]); in perf_event_fd_array_release()
1349 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_of_map_gen_lookup() local
1350 u32 elem_size = array->elem_size; in array_of_map_gen_lookup()
1360 *insn++ = BPF_ALU32_IMM(BPF_AND, ret, array->index_mask); in array_of_map_gen_lookup()