281 static inline struct bucket *__select_bucket(struct bpf_htab *htab, u32 hash)  in __select_bucket()  argument
283 	return &htab->buckets[hash & (htab->n_buckets - 1)];  in __select_bucket()
286 static inline struct hlist_head *select_bucket(struct bpf_htab *htab, u32 hash)  in select_bucket()  argument
288 	return &__select_bucket(htab, hash)->head;  in select_bucket()
291 static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l)  in free_htab_elem()  argument
293 	atomic_dec(&htab->count);  in free_htab_elem()
344 			struct bpf_htab *htab = container_of(e->map, struct bpf_htab, map);  in bpf_tcp_remove()  local
349 			b = __select_bucket(htab, link->hash);  in bpf_tcp_remove()
354 					    htab->map.key_size);  in bpf_tcp_remove()
361 				free_htab_elem(htab, link);  in bpf_tcp_remove()
2054 		struct bpf_htab *htab = container_of(map, struct bpf_htab, map);  in sock_map_prog()  local
2056 		progs = &htab->progs;  in sock_map_prog()
2153 		struct bpf_htab *htab = container_of(map, struct bpf_htab, map);  in sock_map_release()  local
2155 		progs = &htab->progs;  in sock_map_release()
2172 	struct bpf_htab *htab;  in sock_hash_alloc()  local
2196 	htab = kzalloc(sizeof(*htab), GFP_USER);  in sock_hash_alloc()
2197 	if (!htab)  in sock_hash_alloc()
2200 	bpf_map_init_from_attr(&htab->map, attr);  in sock_hash_alloc()
2202 	htab->n_buckets = roundup_pow_of_two(htab->map.max_entries);  in sock_hash_alloc()
2203 	htab->elem_size = sizeof(struct htab_elem) +  in sock_hash_alloc()
2204 			  round_up(htab->map.key_size, 8);  in sock_hash_alloc()
2206 	if (htab->n_buckets == 0 ||  in sock_hash_alloc()
2207 	    htab->n_buckets > U32_MAX / sizeof(struct bucket))  in sock_hash_alloc()
2210 	cost = (u64) htab->n_buckets * sizeof(struct bucket) +  in sock_hash_alloc()
2211 	       (u64) htab->elem_size * htab->map.max_entries;  in sock_hash_alloc()
2216 	htab->map.pages = round_up(cost, PAGE_SIZE) >> PAGE_SHIFT;  in sock_hash_alloc()
2217 	err = bpf_map_precharge_memlock(htab->map.pages);  in sock_hash_alloc()
2222 	htab->buckets = bpf_map_area_alloc(  in sock_hash_alloc()
2223 				htab->n_buckets * sizeof(struct bucket),  in sock_hash_alloc()
2224 				htab->map.numa_node);  in sock_hash_alloc()
2225 	if (!htab->buckets)  in sock_hash_alloc()
2228 	for (i = 0; i < htab->n_buckets; i++) {  in sock_hash_alloc()
2229 		INIT_HLIST_HEAD(&htab->buckets[i].head);  in sock_hash_alloc()
2230 		raw_spin_lock_init(&htab->buckets[i].lock);  in sock_hash_alloc()
2233 	return &htab->map;  in sock_hash_alloc()
2235 	kfree(htab);  in sock_hash_alloc()
2241 	struct bpf_htab *htab;  in __bpf_htab_free()  local
2243 	htab = container_of(rcu, struct bpf_htab, rcu);  in __bpf_htab_free()
2244 	bpf_map_area_free(htab->buckets);  in __bpf_htab_free()
2245 	kfree(htab);  in __bpf_htab_free()
2250 	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);  in sock_hash_free()  local
2263 	for (i = 0; i < htab->n_buckets; i++) {  in sock_hash_free()
2264 		struct bucket *b = __select_bucket(htab, i);  in sock_hash_free()
2286 			free_htab_elem(htab, l);  in sock_hash_free()
2291 	call_rcu(&htab->rcu, __bpf_htab_free);  in sock_hash_free()
2294 static struct htab_elem *alloc_sock_hash_elem(struct bpf_htab *htab,  in alloc_sock_hash_elem()  argument
2301 	if (atomic_inc_return(&htab->count) > htab->map.max_entries) {  in alloc_sock_hash_elem()
2303 			atomic_dec(&htab->count);  in alloc_sock_hash_elem()
2307 	l_new = kmalloc_node(htab->elem_size, GFP_ATOMIC | __GFP_NOWARN,  in alloc_sock_hash_elem()
2308 			     htab->map.numa_node);  in alloc_sock_hash_elem()
2310 		atomic_dec(&htab->count);  in alloc_sock_hash_elem()
2328 	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);  in sock_hash_get_next_key()  local
2340 	h = select_bucket(htab, hash);  in sock_hash_get_next_key()
2354 	i = hash & (htab->n_buckets - 1);  in sock_hash_get_next_key()
2359 	for (; i < htab->n_buckets; i++) {  in sock_hash_get_next_key()
2360 		h = select_bucket(htab, i);  in sock_hash_get_next_key()
2381 	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);  in sock_hash_ctx_update_elem()  local
2382 	struct bpf_sock_progs *progs = &htab->progs;  in sock_hash_ctx_update_elem()
2408 	b = __select_bucket(htab, hash);  in sock_hash_ctx_update_elem()
2430 	l_new = alloc_sock_hash_elem(htab, key, key_size, hash, sock, l_old);  in sock_hash_ctx_update_elem()
2452 		free_htab_elem(htab, l_old);  in sock_hash_ctx_update_elem()
2505 	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);  in sock_hash_delete_elem()  local
2514 	b = __select_bucket(htab, hash);  in sock_hash_delete_elem()
2534 		free_htab_elem(htab, l);  in sock_hash_delete_elem()
2543 	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);  in __sock_hash_lookup_elem()  local
2552 	b = __select_bucket(htab, hash);  in __sock_hash_lookup_elem()