45 static int prealloc_elems_and_freelist(struct bpf_stack_map *smap)  in prealloc_elems_and_freelist()  argument
48 			(u64)smap->map.value_size;  in prealloc_elems_and_freelist()
51 	smap->elems = bpf_map_area_alloc(elem_size * smap->map.max_entries,  in prealloc_elems_and_freelist()
52 					 smap->map.numa_node);  in prealloc_elems_and_freelist()
53 	if (!smap->elems)  in prealloc_elems_and_freelist()
56 	err = pcpu_freelist_init(&smap->freelist);  in prealloc_elems_and_freelist()
60 	pcpu_freelist_populate(&smap->freelist, smap->elems, elem_size,  in prealloc_elems_and_freelist()
61 			       smap->map.max_entries);  in prealloc_elems_and_freelist()
65 	bpf_map_area_free(smap->elems);  in prealloc_elems_and_freelist()
73 	struct bpf_stack_map *smap;  in stack_map_alloc()  local
99 	cost = n_buckets * sizeof(struct stack_map_bucket *) + sizeof(*smap);  in stack_map_alloc()
100 	smap = bpf_map_area_alloc(cost, bpf_map_attr_numa_node(attr));  in stack_map_alloc()
101 	if (!smap)  in stack_map_alloc()
104 	bpf_map_init_from_attr(&smap->map, attr);  in stack_map_alloc()
105 	smap->n_buckets = n_buckets;  in stack_map_alloc()
111 	err = prealloc_elems_and_freelist(smap);  in stack_map_alloc()
115 	return &smap->map;  in stack_map_alloc()
120 	bpf_map_area_free(smap);  in stack_map_alloc()
213 	struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);  in __bpf_get_stackid()  local
229 	id = hash & (smap->n_buckets - 1);  in __bpf_get_stackid()
230 	bucket = READ_ONCE(smap->buckets[id]);  in __bpf_get_stackid()
240 			pcpu_freelist_pop(&smap->freelist);  in __bpf_get_stackid()
250 			pcpu_freelist_push(&smap->freelist, &new_bucket->fnode);  in __bpf_get_stackid()
254 			pcpu_freelist_push(&smap->freelist, &new_bucket->fnode);  in __bpf_get_stackid()
265 			pcpu_freelist_pop(&smap->freelist);  in __bpf_get_stackid()
274 	old_bucket = xchg(&smap->buckets[id], new_bucket);  in __bpf_get_stackid()
276 		pcpu_freelist_push(&smap->freelist, &old_bucket->fnode);  in __bpf_get_stackid()
568 	struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);  in bpf_stackmap_copy()  local
572 	if (unlikely(id >= smap->n_buckets))  in bpf_stackmap_copy()
575 	bucket = xchg(&smap->buckets[id], NULL);  in bpf_stackmap_copy()
583 	old_bucket = xchg(&smap->buckets[id], bucket);  in bpf_stackmap_copy()
585 		pcpu_freelist_push(&smap->freelist, &old_bucket->fnode);  in bpf_stackmap_copy()
592 	struct bpf_stack_map *smap = container_of(map,  in stack_map_get_next_key()  local
602 		if (id >= smap->n_buckets || !smap->buckets[id])  in stack_map_get_next_key()
608 	while (id < smap->n_buckets && !smap->buckets[id])  in stack_map_get_next_key()
611 	if (id >= smap->n_buckets)  in stack_map_get_next_key()
627 	struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);  in stack_map_delete_elem()  local
631 	if (unlikely(id >= smap->n_buckets))  in stack_map_delete_elem()
634 	old_bucket = xchg(&smap->buckets[id], NULL);  in stack_map_delete_elem()
636 		pcpu_freelist_push(&smap->freelist, &old_bucket->fnode);  in stack_map_delete_elem()
646 	struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);  in stack_map_free()  local
648 	bpf_map_area_free(smap->elems);  in stack_map_free()
649 	pcpu_freelist_destroy(&smap->freelist);  in stack_map_free()
650 	bpf_map_area_free(smap);  in stack_map_free()
656 	struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);  in stack_map_mem_usage()  local
658 	u64 n_buckets = smap->n_buckets;  in stack_map_mem_usage()
660 	u64 usage = sizeof(*smap);  in stack_map_mem_usage()