64 static int prealloc_elems_and_freelist(struct bpf_stack_map *smap)  in prealloc_elems_and_freelist()  argument
66 	u32 elem_size = sizeof(struct stack_map_bucket) + smap->map.value_size;  in prealloc_elems_and_freelist()
69 	smap->elems = bpf_map_area_alloc(elem_size * smap->map.max_entries,  in prealloc_elems_and_freelist()
70 					 smap->map.numa_node);  in prealloc_elems_and_freelist()
71 	if (!smap->elems)  in prealloc_elems_and_freelist()
74 	err = pcpu_freelist_init(&smap->freelist);  in prealloc_elems_and_freelist()
78 	pcpu_freelist_populate(&smap->freelist, smap->elems, elem_size,  in prealloc_elems_and_freelist()
79 			       smap->map.max_entries);  in prealloc_elems_and_freelist()
83 	bpf_map_area_free(smap->elems);  in prealloc_elems_and_freelist()
91 	struct bpf_stack_map *smap;  in stack_map_alloc()  local
118 	cost = n_buckets * sizeof(struct stack_map_bucket *) + sizeof(*smap);  in stack_map_alloc()
122 	smap = bpf_map_area_alloc(cost, bpf_map_attr_numa_node(attr));  in stack_map_alloc()
123 	if (!smap)  in stack_map_alloc()
131 	bpf_map_init_from_attr(&smap->map, attr);  in stack_map_alloc()
132 	smap->map.value_size = value_size;  in stack_map_alloc()
133 	smap->n_buckets = n_buckets;  in stack_map_alloc()
134 	smap->map.pages = round_up(cost, PAGE_SIZE) >> PAGE_SHIFT;  in stack_map_alloc()
136 	err = bpf_map_precharge_memlock(smap->map.pages);  in stack_map_alloc()
144 	err = prealloc_elems_and_freelist(smap);  in stack_map_alloc()
148 	return &smap->map;  in stack_map_alloc()
153 	bpf_map_area_free(smap);  in stack_map_alloc()
341 	struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);  in BPF_CALL_3()  local
378 	id = hash & (smap->n_buckets - 1);  in BPF_CALL_3()
379 	bucket = READ_ONCE(smap->buckets[id]);  in BPF_CALL_3()
389 			pcpu_freelist_pop(&smap->freelist);  in BPF_CALL_3()
399 			pcpu_freelist_push(&smap->freelist, &new_bucket->fnode);  in BPF_CALL_3()
403 			pcpu_freelist_push(&smap->freelist, &new_bucket->fnode);  in BPF_CALL_3()
414 			pcpu_freelist_pop(&smap->freelist);  in BPF_CALL_3()
423 	old_bucket = xchg(&smap->buckets[id], new_bucket);  in BPF_CALL_3()
425 		pcpu_freelist_push(&smap->freelist, &old_bucket->fnode);  in BPF_CALL_3()
514 	struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);  in bpf_stackmap_copy()  local
518 	if (unlikely(id >= smap->n_buckets))  in bpf_stackmap_copy()
521 	bucket = xchg(&smap->buckets[id], NULL);  in bpf_stackmap_copy()
529 	old_bucket = xchg(&smap->buckets[id], bucket);  in bpf_stackmap_copy()
531 		pcpu_freelist_push(&smap->freelist, &old_bucket->fnode);  in bpf_stackmap_copy()
538 	struct bpf_stack_map *smap = container_of(map,  in stack_map_get_next_key()  local
548 		if (id >= smap->n_buckets || !smap->buckets[id])  in stack_map_get_next_key()
554 	while (id < smap->n_buckets && !smap->buckets[id])  in stack_map_get_next_key()
557 	if (id >= smap->n_buckets)  in stack_map_get_next_key()
573 	struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);  in stack_map_delete_elem()  local
577 	if (unlikely(id >= smap->n_buckets))  in stack_map_delete_elem()
580 	old_bucket = xchg(&smap->buckets[id], NULL);  in stack_map_delete_elem()
582 		pcpu_freelist_push(&smap->freelist, &old_bucket->fnode);  in stack_map_delete_elem()
592 	struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);  in stack_map_free()  local
597 	bpf_map_area_free(smap->elems);  in stack_map_free()
598 	pcpu_freelist_destroy(&smap->freelist);  in stack_map_free()
599 	bpf_map_area_free(smap);  in stack_map_free()