61 static int prealloc_elems_and_freelist(struct bpf_stack_map *smap)  in prealloc_elems_and_freelist()  argument
63 	u32 elem_size = sizeof(struct stack_map_bucket) + smap->map.value_size;  in prealloc_elems_and_freelist()
66 	smap->elems = bpf_map_area_alloc(elem_size * smap->map.max_entries,  in prealloc_elems_and_freelist()
67 					 smap->map.numa_node);  in prealloc_elems_and_freelist()
68 	if (!smap->elems)  in prealloc_elems_and_freelist()
71 	err = pcpu_freelist_init(&smap->freelist);  in prealloc_elems_and_freelist()
75 	pcpu_freelist_populate(&smap->freelist, smap->elems, elem_size,  in prealloc_elems_and_freelist()
76 			       smap->map.max_entries);  in prealloc_elems_and_freelist()
80 	bpf_map_area_free(smap->elems);  in prealloc_elems_and_freelist()
88 	struct bpf_stack_map *smap;  in stack_map_alloc()  local
116 	cost = n_buckets * sizeof(struct stack_map_bucket *) + sizeof(*smap);  in stack_map_alloc()
122 	smap = bpf_map_area_alloc(cost, bpf_map_attr_numa_node(attr));  in stack_map_alloc()
123 	if (!smap) {  in stack_map_alloc()
128 	bpf_map_init_from_attr(&smap->map, attr);  in stack_map_alloc()
129 	smap->map.value_size = value_size;  in stack_map_alloc()
130 	smap->n_buckets = n_buckets;  in stack_map_alloc()
136 	err = prealloc_elems_and_freelist(smap);  in stack_map_alloc()
140 	bpf_map_charge_move(&smap->map.memory, &mem);  in stack_map_alloc()
142 	return &smap->map;  in stack_map_alloc()
148 	bpf_map_area_free(smap);  in stack_map_alloc()
348 	struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);  in BPF_CALL_3()  local
385 	id = hash & (smap->n_buckets - 1);  in BPF_CALL_3()
386 	bucket = READ_ONCE(smap->buckets[id]);  in BPF_CALL_3()
396 			pcpu_freelist_pop(&smap->freelist);  in BPF_CALL_3()
406 			pcpu_freelist_push(&smap->freelist, &new_bucket->fnode);  in BPF_CALL_3()
410 			pcpu_freelist_push(&smap->freelist, &new_bucket->fnode);  in BPF_CALL_3()
421 			pcpu_freelist_pop(&smap->freelist);  in BPF_CALL_3()
430 	old_bucket = xchg(&smap->buckets[id], new_bucket);  in BPF_CALL_3()
432 		pcpu_freelist_push(&smap->freelist, &old_bucket->fnode);  in BPF_CALL_3()
521 	struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);  in bpf_stackmap_copy()  local
525 	if (unlikely(id >= smap->n_buckets))  in bpf_stackmap_copy()
528 	bucket = xchg(&smap->buckets[id], NULL);  in bpf_stackmap_copy()
536 	old_bucket = xchg(&smap->buckets[id], bucket);  in bpf_stackmap_copy()
538 		pcpu_freelist_push(&smap->freelist, &old_bucket->fnode);  in bpf_stackmap_copy()
545 	struct bpf_stack_map *smap = container_of(map,  in stack_map_get_next_key()  local
555 		if (id >= smap->n_buckets || !smap->buckets[id])  in stack_map_get_next_key()
561 	while (id < smap->n_buckets && !smap->buckets[id])  in stack_map_get_next_key()
564 	if (id >= smap->n_buckets)  in stack_map_get_next_key()
580 	struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);  in stack_map_delete_elem()  local
584 	if (unlikely(id >= smap->n_buckets))  in stack_map_delete_elem()
587 	old_bucket = xchg(&smap->buckets[id], NULL);  in stack_map_delete_elem()
589 		pcpu_freelist_push(&smap->freelist, &old_bucket->fnode);  in stack_map_delete_elem()
599 	struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);  in stack_map_free()  local
604 	bpf_map_area_free(smap->elems);  in stack_map_free()
605 	pcpu_freelist_destroy(&smap->freelist);  in stack_map_free()
606 	bpf_map_area_free(smap);  in stack_map_free()