537 static struct bpf_htab_bucket *sock_hash_select_bucket(struct bpf_htab *htab,  in sock_hash_select_bucket()  argument
540 	return &htab->buckets[hash & (htab->buckets_num - 1)];  in sock_hash_select_bucket()
560 	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);  in __sock_hash_lookup_elem()  local
568 	bucket = sock_hash_select_bucket(htab, hash);  in __sock_hash_lookup_elem()
574 static void sock_hash_free_elem(struct bpf_htab *htab,  in sock_hash_free_elem()  argument
577 	atomic_dec(&htab->count);  in sock_hash_free_elem()
584 	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);  in sock_hash_delete_from_link()  local
589 	bucket = sock_hash_select_bucket(htab, elem->hash);  in sock_hash_delete_from_link()
601 		sock_hash_free_elem(htab, elem);  in sock_hash_delete_from_link()
608 	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);  in sock_hash_delete_elem()  local
615 	bucket = sock_hash_select_bucket(htab, hash);  in sock_hash_delete_elem()
622 		sock_hash_free_elem(htab, elem);  in sock_hash_delete_elem()
629 static struct bpf_htab_elem *sock_hash_alloc_elem(struct bpf_htab *htab,  in sock_hash_alloc_elem()  argument
636 	if (atomic_inc_return(&htab->count) > htab->map.max_entries) {  in sock_hash_alloc_elem()
638 			atomic_dec(&htab->count);  in sock_hash_alloc_elem()
643 	new = kmalloc_node(htab->elem_size, GFP_ATOMIC | __GFP_NOWARN,  in sock_hash_alloc_elem()
644 			   htab->map.numa_node);  in sock_hash_alloc_elem()
646 		atomic_dec(&htab->count);  in sock_hash_alloc_elem()
658 	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);  in sock_hash_update_common()  local
677 	ret = sock_map_link(map, &htab->progs, sk);  in sock_hash_update_common()
685 	bucket = sock_hash_select_bucket(htab, hash);  in sock_hash_update_common()
697 	elem_new = sock_hash_alloc_elem(htab, key, key_size, hash, sk, elem);  in sock_hash_update_common()
711 		sock_hash_free_elem(htab, elem);  in sock_hash_update_common()
756 	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);  in sock_hash_get_next_key()  local
765 	head = &sock_hash_select_bucket(htab, hash)->head;  in sock_hash_get_next_key()
777 	i = hash & (htab->buckets_num - 1);  in sock_hash_get_next_key()
780 	for (; i < htab->buckets_num; i++) {  in sock_hash_get_next_key()
781 		head = &sock_hash_select_bucket(htab, i)->head;  in sock_hash_get_next_key()
795 	struct bpf_htab *htab;  in sock_hash_alloc()  local
809 	htab = kzalloc(sizeof(*htab), GFP_USER);  in sock_hash_alloc()
810 	if (!htab)  in sock_hash_alloc()
813 	bpf_map_init_from_attr(&htab->map, attr);  in sock_hash_alloc()
815 	htab->buckets_num = roundup_pow_of_two(htab->map.max_entries);  in sock_hash_alloc()
816 	htab->elem_size = sizeof(struct bpf_htab_elem) +  in sock_hash_alloc()
817 			  round_up(htab->map.key_size, 8);  in sock_hash_alloc()
818 	if (htab->buckets_num == 0 ||  in sock_hash_alloc()
819 	    htab->buckets_num > U32_MAX / sizeof(struct bpf_htab_bucket)) {  in sock_hash_alloc()
824 	cost = (u64) htab->buckets_num * sizeof(struct bpf_htab_bucket) +  in sock_hash_alloc()
825 	       (u64) htab->elem_size * htab->map.max_entries;  in sock_hash_alloc()
831 	htab->buckets = bpf_map_area_alloc(htab->buckets_num *  in sock_hash_alloc()
833 					   htab->map.numa_node);  in sock_hash_alloc()
834 	if (!htab->buckets) {  in sock_hash_alloc()
839 	for (i = 0; i < htab->buckets_num; i++) {  in sock_hash_alloc()
840 		INIT_HLIST_HEAD(&htab->buckets[i].head);  in sock_hash_alloc()
841 		raw_spin_lock_init(&htab->buckets[i].lock);  in sock_hash_alloc()
844 	return &htab->map;  in sock_hash_alloc()
846 	kfree(htab);  in sock_hash_alloc()
852 	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);  in sock_hash_free()  local
860 	for (i = 0; i < htab->buckets_num; i++) {  in sock_hash_free()
861 		bucket = sock_hash_select_bucket(htab, i);  in sock_hash_free()
871 	bpf_map_area_free(htab->buckets);  in sock_hash_free()
872 	kfree(htab);  in sock_hash_free()