27 static void bpf_array_free_percpu(struct bpf_array *array)  in bpf_array_free_percpu()  argument
31 	for (i = 0; i < array->map.max_entries; i++) {  in bpf_array_free_percpu()
32 		free_percpu(array->pptrs[i]);  in bpf_array_free_percpu()
37 static int bpf_array_alloc_percpu(struct bpf_array *array)  in bpf_array_alloc_percpu()  argument
42 	for (i = 0; i < array->map.max_entries; i++) {  in bpf_array_alloc_percpu()
43 		ptr = __alloc_percpu_gfp(array->elem_size, 8,  in bpf_array_alloc_percpu()
46 			bpf_array_free_percpu(array);  in bpf_array_alloc_percpu()
49 		array->pptrs[i] = ptr;  in bpf_array_alloc_percpu()
85 	struct bpf_array *array;  in array_map_alloc()  local
110 	array_size = sizeof(*array);  in array_map_alloc()
132 	array = bpf_map_area_alloc(array_size, numa_node);  in array_map_alloc()
133 	if (!array)  in array_map_alloc()
135 	array->index_mask = index_mask;  in array_map_alloc()
136 	array->map.unpriv_array = unpriv;  in array_map_alloc()
139 	bpf_map_init_from_attr(&array->map, attr);  in array_map_alloc()
140 	array->map.pages = cost;  in array_map_alloc()
141 	array->elem_size = elem_size;  in array_map_alloc()
143 	if (percpu && bpf_array_alloc_percpu(array)) {  in array_map_alloc()
144 		bpf_map_area_free(array);  in array_map_alloc()
148 	return &array->map;  in array_map_alloc()
154 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_lookup_elem()  local
157 	if (unlikely(index >= array->map.max_entries))  in array_map_lookup_elem()
160 	return array->value + array->elem_size * (index & array->index_mask);  in array_map_lookup_elem()
166 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_gen_lookup()  local
177 		*insn++ = BPF_ALU32_IMM(BPF_AND, ret, array->index_mask);  in array_map_gen_lookup()
196 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in percpu_array_map_lookup_elem()  local
199 	if (unlikely(index >= array->map.max_entries))  in percpu_array_map_lookup_elem()
202 	return this_cpu_ptr(array->pptrs[index & array->index_mask]);  in percpu_array_map_lookup_elem()
207 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in bpf_percpu_array_copy()  local
213 	if (unlikely(index >= array->map.max_entries))  in bpf_percpu_array_copy()
222 	pptr = array->pptrs[index & array->index_mask];  in bpf_percpu_array_copy()
234 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_get_next_key()  local
238 	if (index >= array->map.max_entries) {  in array_map_get_next_key()
243 	if (index == array->map.max_entries - 1)  in array_map_get_next_key()
254 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_update_elem()  local
261 	if (unlikely(index >= array->map.max_entries))  in array_map_update_elem()
269 	if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY)  in array_map_update_elem()
270 		memcpy(this_cpu_ptr(array->pptrs[index & array->index_mask]),  in array_map_update_elem()
273 		memcpy(array->value +  in array_map_update_elem()
274 		       array->elem_size * (index & array->index_mask),  in array_map_update_elem()
282 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in bpf_percpu_array_update()  local
292 	if (unlikely(index >= array->map.max_entries))  in bpf_percpu_array_update()
308 	pptr = array->pptrs[index & array->index_mask];  in bpf_percpu_array_update()
326 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_map_free()  local
335 	if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY)  in array_map_free()
336 		bpf_array_free_percpu(array);  in array_map_free()
338 	bpf_map_area_free(array);  in array_map_free()
414 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in fd_array_map_free()  local
420 	for (i = 0; i < array->map.max_entries; i++)  in fd_array_map_free()
421 		BUG_ON(array->ptrs[i] != NULL);  in fd_array_map_free()
423 	bpf_map_area_free(array);  in fd_array_map_free()
455 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in bpf_fd_array_map_update_elem()  local
462 	if (index >= array->map.max_entries)  in bpf_fd_array_map_update_elem()
470 	old_ptr = xchg(array->ptrs + index, new_ptr);  in bpf_fd_array_map_update_elem()
479 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in fd_array_map_delete_elem()  local
483 	if (index >= array->map.max_entries)  in fd_array_map_delete_elem()
486 	old_ptr = xchg(array->ptrs + index, NULL);  in fd_array_map_delete_elem()
498 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in prog_fd_array_get_ptr()  local
504 	if (!bpf_prog_array_compatible(array, prog)) {  in prog_fd_array_get_ptr()
525 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in bpf_fd_array_map_clear()  local
528 	for (i = 0; i < array->map.max_entries; i++)  in bpf_fd_array_map_clear()
609 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in perf_event_fd_array_release()  local
614 	for (i = 0; i < array->map.max_entries; i++) {  in perf_event_fd_array_release()
615 		ee = READ_ONCE(array->ptrs[i]);  in perf_event_fd_array_release()
710 	struct bpf_array *array = container_of(map, struct bpf_array, map);  in array_of_map_gen_lookup()  local
721 		*insn++ = BPF_ALU32_IMM(BPF_AND, ret, array->index_mask);  in array_of_map_gen_lookup()