66 	struct bpf_dtab *dtab;  member
103 static inline struct hlist_head *dev_map_index_hash(struct bpf_dtab *dtab,  in dev_map_index_hash()  argument
106 	return &dtab->dev_index_head[idx & (dtab->n_buckets - 1)];  in dev_map_index_hash()
109 static int dev_map_init_map(struct bpf_dtab *dtab, union bpf_attr *attr)  in dev_map_init_map()  argument
131 	bpf_map_init_from_attr(&dtab->map, attr);  in dev_map_init_map()
134 		dtab->n_buckets = roundup_pow_of_two(dtab->map.max_entries);  in dev_map_init_map()
136 		if (!dtab->n_buckets) /* Overflow check */  in dev_map_init_map()
138 		cost += (u64) sizeof(struct hlist_head) * dtab->n_buckets;  in dev_map_init_map()
140 		cost += (u64) dtab->map.max_entries * sizeof(struct bpf_dtab_netdev *);  in dev_map_init_map()
144 	err = bpf_map_charge_init(&dtab->map.memory, cost);  in dev_map_init_map()
149 		dtab->dev_index_head = dev_map_create_hash(dtab->n_buckets,  in dev_map_init_map()
150 							   dtab->map.numa_node);  in dev_map_init_map()
151 		if (!dtab->dev_index_head)  in dev_map_init_map()
154 		spin_lock_init(&dtab->index_lock);  in dev_map_init_map()
156 		dtab->netdev_map = bpf_map_area_alloc(dtab->map.max_entries *  in dev_map_init_map()
158 						      dtab->map.numa_node);  in dev_map_init_map()
159 		if (!dtab->netdev_map)  in dev_map_init_map()
166 	bpf_map_charge_finish(&dtab->map.memory);  in dev_map_init_map()
172 	struct bpf_dtab *dtab;  in dev_map_alloc()  local
178 	dtab = kzalloc(sizeof(*dtab), GFP_USER);  in dev_map_alloc()
179 	if (!dtab)  in dev_map_alloc()
182 	err = dev_map_init_map(dtab, attr);  in dev_map_alloc()
184 		kfree(dtab);  in dev_map_alloc()
189 	list_add_tail_rcu(&dtab->list, &dev_map_list);  in dev_map_alloc()
192 	return &dtab->map;  in dev_map_alloc()
197 	struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);  in dev_map_free()  local
211 	list_del_rcu(&dtab->list);  in dev_map_free()
220 	if (dtab->map.map_type == BPF_MAP_TYPE_DEVMAP_HASH) {  in dev_map_free()
221 		for (i = 0; i < dtab->n_buckets; i++) {  in dev_map_free()
226 			head = dev_map_index_hash(dtab, i);  in dev_map_free()
237 		bpf_map_area_free(dtab->dev_index_head);  in dev_map_free()
239 		for (i = 0; i < dtab->map.max_entries; i++) {  in dev_map_free()
242 			dev = dtab->netdev_map[i];  in dev_map_free()
252 		bpf_map_area_free(dtab->netdev_map);  in dev_map_free()
255 	kfree(dtab);  in dev_map_free()
260 	struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);  in dev_map_get_next_key()  local
264 	if (index >= dtab->map.max_entries) {  in dev_map_get_next_key()
269 	if (index == dtab->map.max_entries - 1)  in dev_map_get_next_key()
277 	struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);  in __dev_map_hash_lookup_elem()  local
278 	struct hlist_head *head = dev_map_index_hash(dtab, key);  in __dev_map_hash_lookup_elem()
282 				 lockdep_is_held(&dtab->index_lock))  in __dev_map_hash_lookup_elem()
292 	struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);  in dev_map_hash_get_next_key()  local
315 	i = idx & (dtab->n_buckets - 1);  in dev_map_hash_get_next_key()
319 	for (; i < dtab->n_buckets; i++) {  in dev_map_hash_get_next_key()
320 		head = dev_map_index_hash(dtab, i);  in dev_map_hash_get_next_key()
411 	struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);  in __dev_map_lookup_elem()  local
417 	obj = READ_ONCE(dtab->netdev_map[key]);  in __dev_map_lookup_elem()
555 	struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);  in dev_map_delete_elem()  local
569 	old_dev = xchg(&dtab->netdev_map[k], NULL);  in dev_map_delete_elem()
577 	struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);  in dev_map_hash_delete_elem()  local
583 	spin_lock_irqsave(&dtab->index_lock, flags);  in dev_map_hash_delete_elem()
587 		dtab->items--;  in dev_map_hash_delete_elem()
592 	spin_unlock_irqrestore(&dtab->index_lock, flags);  in dev_map_hash_delete_elem()
598 						    struct bpf_dtab *dtab,  in __dev_map_alloc_node()  argument
606 			   dtab->map.numa_node);  in __dev_map_alloc_node()
624 	dev->dtab = dtab;  in __dev_map_alloc_node()
647 	struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);  in __dev_map_update_elem()  local
654 	if (unlikely(i >= dtab->map.max_entries))  in __dev_map_update_elem()
668 		dev = __dev_map_alloc_node(net, dtab, &val, i);  in __dev_map_update_elem()
677 	old_dev = xchg(&dtab->netdev_map[i], dev);  in __dev_map_update_elem()
694 	struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);  in __dev_map_hash_update_elem()  local
707 	spin_lock_irqsave(&dtab->index_lock, flags);  in __dev_map_hash_update_elem()
713 	dev = __dev_map_alloc_node(net, dtab, &val, idx);  in __dev_map_hash_update_elem()
722 		if (dtab->items >= dtab->map.max_entries) {  in __dev_map_hash_update_elem()
723 			spin_unlock_irqrestore(&dtab->index_lock, flags);  in __dev_map_hash_update_elem()
727 		dtab->items++;  in __dev_map_hash_update_elem()
731 			   dev_map_index_hash(dtab, idx));  in __dev_map_hash_update_elem()
732 	spin_unlock_irqrestore(&dtab->index_lock, flags);  in __dev_map_hash_update_elem()
740 	spin_unlock_irqrestore(&dtab->index_lock, flags);  in __dev_map_hash_update_elem()
779 static void dev_map_hash_remove_netdev(struct bpf_dtab *dtab,  in dev_map_hash_remove_netdev()  argument
785 	spin_lock_irqsave(&dtab->index_lock, flags);  in dev_map_hash_remove_netdev()
786 	for (i = 0; i < dtab->n_buckets; i++) {  in dev_map_hash_remove_netdev()
791 		head = dev_map_index_hash(dtab, i);  in dev_map_hash_remove_netdev()
797 			dtab->items--;  in dev_map_hash_remove_netdev()
802 	spin_unlock_irqrestore(&dtab->index_lock, flags);  in dev_map_hash_remove_netdev()
809 	struct bpf_dtab *dtab;  in dev_map_notification()  local
834 		list_for_each_entry_rcu(dtab, &dev_map_list, list) {  in dev_map_notification()
835 			if (dtab->map.map_type == BPF_MAP_TYPE_DEVMAP_HASH) {  in dev_map_notification()
836 				dev_map_hash_remove_netdev(dtab, netdev);  in dev_map_notification()
840 			for (i = 0; i < dtab->map.max_entries; i++) {  in dev_map_notification()
843 				dev = READ_ONCE(dtab->netdev_map[i]);  in dev_map_notification()
846 				odev = cmpxchg(&dtab->netdev_map[i], dev, NULL);  in dev_map_notification()