65 static int prealloc_elems_and_freelist(struct bpf_stack_map *smap)  in prealloc_elems_and_freelist()  argument
67 	u32 elem_size = sizeof(struct stack_map_bucket) + smap->map.value_size;  in prealloc_elems_and_freelist()
70 	smap->elems = bpf_map_area_alloc(elem_size * smap->map.max_entries,  in prealloc_elems_and_freelist()
71 					 smap->map.numa_node);  in prealloc_elems_and_freelist()
72 	if (!smap->elems)  in prealloc_elems_and_freelist()
75 	err = pcpu_freelist_init(&smap->freelist);  in prealloc_elems_and_freelist()
79 	pcpu_freelist_populate(&smap->freelist, smap->elems, elem_size,  in prealloc_elems_and_freelist()
80 			       smap->map.max_entries);  in prealloc_elems_and_freelist()
84 	bpf_map_area_free(smap->elems);  in prealloc_elems_and_freelist()
92 	struct bpf_stack_map *smap;  in stack_map_alloc()  local
120 	cost = n_buckets * sizeof(struct stack_map_bucket *) + sizeof(*smap);  in stack_map_alloc()
126 	smap = bpf_map_area_alloc(cost, bpf_map_attr_numa_node(attr));  in stack_map_alloc()
127 	if (!smap) {  in stack_map_alloc()
132 	bpf_map_init_from_attr(&smap->map, attr);  in stack_map_alloc()
133 	smap->map.value_size = value_size;  in stack_map_alloc()
134 	smap->n_buckets = n_buckets;  in stack_map_alloc()
140 	err = prealloc_elems_and_freelist(smap);  in stack_map_alloc()
144 	bpf_map_charge_move(&smap->map.memory, &mem);  in stack_map_alloc()
146 	return &smap->map;  in stack_map_alloc()
152 	bpf_map_area_free(smap);  in stack_map_alloc()
398 	struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);  in __bpf_get_stackid()  local
422 	id = hash & (smap->n_buckets - 1);  in __bpf_get_stackid()
423 	bucket = READ_ONCE(smap->buckets[id]);  in __bpf_get_stackid()
433 			pcpu_freelist_pop(&smap->freelist);  in __bpf_get_stackid()
443 			pcpu_freelist_push(&smap->freelist, &new_bucket->fnode);  in __bpf_get_stackid()
447 			pcpu_freelist_push(&smap->freelist, &new_bucket->fnode);  in __bpf_get_stackid()
458 			pcpu_freelist_pop(&smap->freelist);  in __bpf_get_stackid()
467 	old_bucket = xchg(&smap->buckets[id], new_bucket);  in __bpf_get_stackid()
469 		pcpu_freelist_push(&smap->freelist, &old_bucket->fnode);  in __bpf_get_stackid()
753 	struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);  in bpf_stackmap_copy()  local
757 	if (unlikely(id >= smap->n_buckets))  in bpf_stackmap_copy()
760 	bucket = xchg(&smap->buckets[id], NULL);  in bpf_stackmap_copy()
768 	old_bucket = xchg(&smap->buckets[id], bucket);  in bpf_stackmap_copy()
770 		pcpu_freelist_push(&smap->freelist, &old_bucket->fnode);  in bpf_stackmap_copy()
777 	struct bpf_stack_map *smap = container_of(map,  in stack_map_get_next_key()  local
787 		if (id >= smap->n_buckets || !smap->buckets[id])  in stack_map_get_next_key()
793 	while (id < smap->n_buckets && !smap->buckets[id])  in stack_map_get_next_key()
796 	if (id >= smap->n_buckets)  in stack_map_get_next_key()
812 	struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);  in stack_map_delete_elem()  local
816 	if (unlikely(id >= smap->n_buckets))  in stack_map_delete_elem()
819 	old_bucket = xchg(&smap->buckets[id], NULL);  in stack_map_delete_elem()
821 		pcpu_freelist_push(&smap->freelist, &old_bucket->fnode);  in stack_map_delete_elem()
831 	struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);  in stack_map_free()  local
833 	bpf_map_area_free(smap->elems);  in stack_map_free()
834 	pcpu_freelist_destroy(&smap->freelist);  in stack_map_free()
835 	bpf_map_area_free(smap);  in stack_map_free()