26 	for (i = 0; i < array->map.max_entries; i++) {  in bpf_array_free_percpu()
37 	for (i = 0; i < array->map.max_entries; i++) {  in bpf_array_alloc_percpu()
38 		ptr = bpf_map_alloc_percpu(&array->map, array->elem_size, 8,  in bpf_array_alloc_percpu()
73 	/* avoid overflow on round_up(map->value_size) */  in array_map_alloc_check()
127 	/* allocate all map elements and zero-initialize them */  in array_map_alloc()
143 	array->map.bypass_spec_v1 = bypass_spec_v1;  in array_map_alloc()
145 	/* copy mandatory map attributes */  in array_map_alloc()
146 	bpf_map_init_from_attr(&array->map, attr);  in array_map_alloc()
154 	return &array->map;  in array_map_alloc()
163 static void *array_map_lookup_elem(struct bpf_map *map, void *key)  in array_map_lookup_elem()  argument
165 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_lookup_elem()
168 	if (unlikely(index >= array->map.max_entries))  in array_map_lookup_elem()
174 static int array_map_direct_value_addr(const struct bpf_map *map, u64 *imm,  in array_map_direct_value_addr()  argument
177 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_direct_value_addr()
179 	if (map->max_entries != 1)  in array_map_direct_value_addr()
181 	if (off >= map->value_size)  in array_map_direct_value_addr()
188 static int array_map_direct_value_meta(const struct bpf_map *map, u64 imm,  in array_map_direct_value_meta()  argument
191 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_direct_value_meta()
195 	if (map->max_entries != 1)  in array_map_direct_value_meta()
205 static int array_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf)  in array_map_gen_lookup()  argument
207 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_gen_lookup()
214 	if (map->map_flags & BPF_F_INNER_MAP)  in array_map_gen_lookup()
219 	if (!map->bypass_spec_v1) {  in array_map_gen_lookup()
220 		*insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 4);  in array_map_gen_lookup()
223 		*insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 3);  in array_map_gen_lookup()
238 static void *percpu_array_map_lookup_elem(struct bpf_map *map, void *key)  in percpu_array_map_lookup_elem()  argument
240 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in percpu_array_map_lookup_elem()
243 	if (unlikely(index >= array->map.max_entries))  in percpu_array_map_lookup_elem()
249 static void *percpu_array_map_lookup_percpu_elem(struct bpf_map *map, void *key, u32 cpu)  in percpu_array_map_lookup_percpu_elem()  argument
251 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in percpu_array_map_lookup_percpu_elem()
257 	if (unlikely(index >= array->map.max_entries))  in percpu_array_map_lookup_percpu_elem()
263 int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value)  in bpf_percpu_array_copy()  argument
265 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in bpf_percpu_array_copy()
271 	if (unlikely(index >= array->map.max_entries))  in bpf_percpu_array_copy()
282 		copy_map_value_long(map, value + off, per_cpu_ptr(pptr, cpu));  in bpf_percpu_array_copy()
283 		check_and_init_map_value(map, value + off);  in bpf_percpu_array_copy()
291 static int array_map_get_next_key(struct bpf_map *map, void *key, void *next_key)  in array_map_get_next_key()  argument
293 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_get_next_key()
297 	if (index >= array->map.max_entries) {  in array_map_get_next_key()
302 	if (index == array->map.max_entries - 1)  in array_map_get_next_key()
311 	if (map_value_has_timer(&arr->map))  in check_and_free_fields()
312 		bpf_timer_cancel_and_free(val + arr->map.timer_off);  in check_and_free_fields()
313 	if (map_value_has_kptrs(&arr->map))  in check_and_free_fields()
314 		bpf_map_free_kptrs(&arr->map, val);  in check_and_free_fields()
318 static int array_map_update_elem(struct bpf_map *map, void *key, void *value,  in array_map_update_elem()  argument
321 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_update_elem()
329 	if (unlikely(index >= array->map.max_entries))  in array_map_update_elem()
338 		     !map_value_has_spin_lock(map)))  in array_map_update_elem()
341 	if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {  in array_map_update_elem()
343 		copy_map_value(map, val, value);  in array_map_update_elem()
349 			copy_map_value_locked(map, val, value, false);  in array_map_update_elem()
351 			copy_map_value(map, val, value);  in array_map_update_elem()
357 int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,  in bpf_percpu_array_update()  argument
360 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in bpf_percpu_array_update()
370 	if (unlikely(index >= array->map.max_entries))  in bpf_percpu_array_update()
388 		copy_map_value_long(map, per_cpu_ptr(pptr, cpu), value + off);  in bpf_percpu_array_update()
397 static int array_map_delete_elem(struct bpf_map *map, void *key)  in array_map_delete_elem()  argument
407 static void array_map_free_timers(struct bpf_map *map)  in array_map_free_timers()  argument
409 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_free_timers()
413 	if (!map_value_has_timer(map))  in array_map_free_timers()
416 	for (i = 0; i < array->map.max_entries; i++)  in array_map_free_timers()
417 		bpf_timer_cancel_and_free(array_map_elem_ptr(array, i) + map->timer_off);  in array_map_free_timers()
420 /* Called when map->refcnt goes to zero, either from workqueue or from syscall */
421 static void array_map_free(struct bpf_map *map)  in array_map_free()  argument
423 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_free()
426 	if (map_value_has_kptrs(map)) {  in array_map_free()
427 		if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {  in array_map_free()
428 			for (i = 0; i < array->map.max_entries; i++) {  in array_map_free()
433 					bpf_map_free_kptrs(map, per_cpu_ptr(pptr, cpu));  in array_map_free()
438 			for (i = 0; i < array->map.max_entries; i++)  in array_map_free()
439 				bpf_map_free_kptrs(map, array_map_elem_ptr(array, i));  in array_map_free()
441 		bpf_map_free_kptr_off_tab(map);  in array_map_free()
444 	if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY)  in array_map_free()
447 	if (array->map.map_flags & BPF_F_MMAPABLE)  in array_map_free()
453 static void array_map_seq_show_elem(struct bpf_map *map, void *key,  in array_map_seq_show_elem()  argument
460 	value = array_map_lookup_elem(map, key);  in array_map_seq_show_elem()
466 	if (map->btf_key_type_id)  in array_map_seq_show_elem()
468 	btf_type_seq_show(map->btf, map->btf_value_type_id, value, m);  in array_map_seq_show_elem()
474 static void percpu_array_map_seq_show_elem(struct bpf_map *map, void *key,  in percpu_array_map_seq_show_elem()  argument
477 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in percpu_array_map_seq_show_elem()
488 		btf_type_seq_show(map->btf, map->btf_value_type_id,  in percpu_array_map_seq_show_elem()
497 static int array_map_check_btf(const struct bpf_map *map,  in array_map_check_btf()  argument
504 	/* One exception for keyless BTF: .bss/.data/.rodata map */  in array_map_check_btf()
506 		if (map->map_type != BPF_MAP_TYPE_ARRAY ||  in array_map_check_btf()
507 		    map->max_entries != 1)  in array_map_check_btf()
529 static int array_map_mmap(struct bpf_map *map, struct vm_area_struct *vma)  in array_map_mmap()  argument
531 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_mmap()
534 	if (!(map->map_flags & BPF_F_MMAPABLE))  in array_map_mmap()
538 	    PAGE_ALIGN((u64)array->map.max_entries * array->elem_size))  in array_map_mmap()
555 	struct bpf_map *map;  member
563 	struct bpf_map *map = info->map;  in bpf_array_map_seq_start()  local
567 	if (info->index >= map->max_entries)  in bpf_array_map_seq_start()
572 	array = container_of(map, struct bpf_array, map);  in bpf_array_map_seq_start()
582 	struct bpf_map *map = info->map;  in bpf_array_map_seq_next()  local
588 	if (info->index >= map->max_entries)  in bpf_array_map_seq_next()
591 	array = container_of(map, struct bpf_array, map);  in bpf_array_map_seq_next()
602 	struct bpf_map *map = info->map;  in __bpf_array_map_seq_show()  local
603 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in __bpf_array_map_seq_show()
616 	ctx.map = info->map;  in __bpf_array_map_seq_show()
626 				copy_map_value_long(map, info->percpu_value_buf + off,  in __bpf_array_map_seq_show()
628 				check_and_init_map_value(map, info->percpu_value_buf + off);  in __bpf_array_map_seq_show()
653 	struct bpf_map *map = aux->map;  in bpf_iter_init_array_map()  local
654 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in bpf_iter_init_array_map()
658 	if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {  in bpf_iter_init_array_map()
667 	/* bpf_iter_attach_map() acquires a map uref, and the uref may be  in bpf_iter_init_array_map()
668 	 * released before or in the middle of iterating map elements, so  in bpf_iter_init_array_map()
669 	 * acquire an extra map uref for iterator.  in bpf_iter_init_array_map()
671 	bpf_map_inc_with_uref(map);  in bpf_iter_init_array_map()
672 	seq_info->map = map;  in bpf_iter_init_array_map()
680 	bpf_map_put_with_uref(seq_info->map);  in bpf_iter_fini_array_map()
698 static int bpf_for_each_array_elem(struct bpf_map *map, bpf_callback_t callback_fn,  in bpf_for_each_array_elem()  argument
710 	is_percpu = map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY;  in bpf_for_each_array_elem()
711 	array = container_of(map, struct bpf_array, map);  in bpf_for_each_array_elem()
714 	for (i = 0; i < map->max_entries; i++) {  in bpf_for_each_array_elem()
721 		ret = callback_fn((u64)(long)map, (u64)(long)&key,  in bpf_for_each_array_elem()
780 	/* only file descriptors can be stored in this type of map */  in fd_array_map_alloc_check()
789 static void fd_array_map_free(struct bpf_map *map)  in fd_array_map_free()  argument
791 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in fd_array_map_free()
795 	for (i = 0; i < array->map.max_entries; i++)  in fd_array_map_free()
801 static void *fd_array_map_lookup_elem(struct bpf_map *map, void *key)  in fd_array_map_lookup_elem()  argument
807 int bpf_fd_array_map_lookup_elem(struct bpf_map *map, void *key, u32 *value)  in bpf_fd_array_map_lookup_elem()  argument
812 	if (!map->ops->map_fd_sys_lookup_elem)  in bpf_fd_array_map_lookup_elem()
816 	elem = array_map_lookup_elem(map, key);  in bpf_fd_array_map_lookup_elem()
818 		*value = map->ops->map_fd_sys_lookup_elem(ptr);  in bpf_fd_array_map_lookup_elem()
827 int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file,  in bpf_fd_array_map_update_elem()  argument
830 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in bpf_fd_array_map_update_elem()
837 	if (index >= array->map.max_entries)  in bpf_fd_array_map_update_elem()
841 	new_ptr = map->ops->map_fd_get_ptr(map, map_file, ufd);  in bpf_fd_array_map_update_elem()
845 	if (map->ops->map_poke_run) {  in bpf_fd_array_map_update_elem()
848 		map->ops->map_poke_run(map, index, old_ptr, new_ptr);  in bpf_fd_array_map_update_elem()
855 		map->ops->map_fd_put_ptr(old_ptr);  in bpf_fd_array_map_update_elem()
859 static int fd_array_map_delete_elem(struct bpf_map *map, void *key)  in fd_array_map_delete_elem()  argument
861 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in fd_array_map_delete_elem()
865 	if (index >= array->map.max_entries)  in fd_array_map_delete_elem()
868 	if (map->ops->map_poke_run) {  in fd_array_map_delete_elem()
871 		map->ops->map_poke_run(map, index, old_ptr, NULL);  in fd_array_map_delete_elem()
878 		map->ops->map_fd_put_ptr(old_ptr);  in fd_array_map_delete_elem()
885 static void *prog_fd_array_get_ptr(struct bpf_map *map,  in prog_fd_array_get_ptr()  argument
893 	if (!bpf_prog_map_compatible(map, prog)) {  in prog_fd_array_get_ptr()
911 /* decrement refcnt of all bpf_progs that are stored in this map */
912 static void bpf_fd_array_map_clear(struct bpf_map *map)  in bpf_fd_array_map_clear()  argument
914 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in bpf_fd_array_map_clear()
917 	for (i = 0; i < array->map.max_entries; i++)  in bpf_fd_array_map_clear()
918 		fd_array_map_delete_elem(map, &i);  in bpf_fd_array_map_clear()
921 static void prog_array_map_seq_show_elem(struct bpf_map *map, void *key,  in prog_array_map_seq_show_elem()  argument
929 	elem = array_map_lookup_elem(map, key);  in prog_array_map_seq_show_elem()
935 			btf_type_seq_show(map->btf, map->btf_value_type_id,  in prog_array_map_seq_show_elem()
949 static int prog_array_map_poke_track(struct bpf_map *map,  in prog_array_map_poke_track()  argument
956 	aux = container_of(map, struct bpf_array, map)->aux;  in prog_array_map_poke_track()
982 static void prog_array_map_poke_untrack(struct bpf_map *map,  in prog_array_map_poke_untrack()  argument
988 	aux = container_of(map, struct bpf_array, map)->aux;  in prog_array_map_poke_untrack()
1000 static void prog_array_map_poke_run(struct bpf_map *map, u32 key,  in prog_array_map_poke_run()  argument
1008 	aux = container_of(map, struct bpf_array, map)->aux;  in prog_array_map_poke_run()
1052 			if (poke->tail_call.map != map ||  in prog_array_map_poke_run()
1095 	struct bpf_map *map = container_of(work, struct bpf_array_aux,  in prog_array_map_clear_deferred()  local
1096 					   work)->map;  in prog_array_map_clear_deferred()
1097 	bpf_fd_array_map_clear(map);  in prog_array_map_clear_deferred()
1098 	bpf_map_put(map);  in prog_array_map_clear_deferred()
1101 static void prog_array_map_clear(struct bpf_map *map)  in prog_array_map_clear()  argument
1103 	struct bpf_array_aux *aux = container_of(map, struct bpf_array,  in prog_array_map_clear()
1104 						 map)->aux;  in prog_array_map_clear()
1105 	bpf_map_inc(map);  in prog_array_map_clear()
1112 	struct bpf_map *map;  in prog_array_map_alloc()  local
1122 	map = array_map_alloc(attr);  in prog_array_map_alloc()
1123 	if (IS_ERR(map)) {  in prog_array_map_alloc()
1125 		return map;  in prog_array_map_alloc()
1128 	container_of(map, struct bpf_array, map)->aux = aux;  in prog_array_map_alloc()
1129 	aux->map = map;  in prog_array_map_alloc()
1131 	return map;  in prog_array_map_alloc()
1134 static void prog_array_map_free(struct bpf_map *map)  in prog_array_map_free()  argument
1139 	aux = container_of(map, struct bpf_array, map)->aux;  in prog_array_map_free()
1145 	fd_array_map_free(map);  in prog_array_map_free()
1200 static void *perf_event_fd_array_get_ptr(struct bpf_map *map,  in perf_event_fd_array_get_ptr()  argument
1231 static void perf_event_fd_array_release(struct bpf_map *map,  in perf_event_fd_array_release()  argument
1234 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in perf_event_fd_array_release()
1238 	if (map->map_flags & BPF_F_PRESERVE_ELEMS)  in perf_event_fd_array_release()
1242 	for (i = 0; i < array->map.max_entries; i++) {  in perf_event_fd_array_release()
1245 			fd_array_map_delete_elem(map, &i);  in perf_event_fd_array_release()
1250 static void perf_event_fd_array_map_free(struct bpf_map *map)  in perf_event_fd_array_map_free()  argument
1252 	if (map->map_flags & BPF_F_PRESERVE_ELEMS)  in perf_event_fd_array_map_free()
1253 		bpf_fd_array_map_clear(map);  in perf_event_fd_array_map_free()
1254 	fd_array_map_free(map);  in perf_event_fd_array_map_free()
1273 static void *cgroup_fd_array_get_ptr(struct bpf_map *map,  in cgroup_fd_array_get_ptr()  argument
1286 static void cgroup_fd_array_free(struct bpf_map *map)  in cgroup_fd_array_free()  argument
1288 	bpf_fd_array_map_clear(map);  in cgroup_fd_array_free()
1289 	fd_array_map_free(map);  in cgroup_fd_array_free()
1309 	struct bpf_map *map, *inner_map_meta;  in array_of_map_alloc()  local
1315 	map = array_map_alloc(attr);  in array_of_map_alloc()
1316 	if (IS_ERR(map)) {  in array_of_map_alloc()
1318 		return map;  in array_of_map_alloc()
1321 	map->inner_map_meta = inner_map_meta;  in array_of_map_alloc()
1323 	return map;  in array_of_map_alloc()
1326 static void array_of_map_free(struct bpf_map *map)  in array_of_map_free()  argument
1328 	/* map->inner_map_meta is only accessed by syscall which  in array_of_map_free()
1331 	bpf_map_meta_free(map->inner_map_meta);  in array_of_map_free()
1332 	bpf_fd_array_map_clear(map);  in array_of_map_free()
1333 	fd_array_map_free(map);  in array_of_map_free()
1336 static void *array_of_map_lookup_elem(struct bpf_map *map, void *key)  in array_of_map_lookup_elem()  argument
1338 	struct bpf_map **inner_map = array_map_lookup_elem(map, key);  in array_of_map_lookup_elem()
1346 static int array_of_map_gen_lookup(struct bpf_map *map,  in array_of_map_gen_lookup()  argument
1349 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_of_map_gen_lookup()
1358 	if (!map->bypass_spec_v1) {  in array_of_map_gen_lookup()
1359 		*insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 6);  in array_of_map_gen_lookup()
1362 		*insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 5);  in array_of_map_gen_lookup()