67 	struct bpf_dtab *dtab;  member
90 	struct bpf_dtab *dtab;  in dev_map_alloc()  local
102 	dtab = kzalloc(sizeof(*dtab), GFP_USER);  in dev_map_alloc()
103 	if (!dtab)  in dev_map_alloc()
106 	bpf_map_init_from_attr(&dtab->map, attr);  in dev_map_alloc()
109 	cost = (u64) dtab->map.max_entries * sizeof(struct bpf_dtab_netdev *);  in dev_map_alloc()
114 	dtab->map.pages = round_up(cost, PAGE_SIZE) >> PAGE_SHIFT;  in dev_map_alloc()
117 	err = bpf_map_precharge_memlock(dtab->map.pages);  in dev_map_alloc()
124 	dtab->flush_needed = __alloc_percpu_gfp(dev_map_bitmap_size(attr),  in dev_map_alloc()
127 	if (!dtab->flush_needed)  in dev_map_alloc()
130 	dtab->netdev_map = bpf_map_area_alloc(dtab->map.max_entries *  in dev_map_alloc()
132 					      dtab->map.numa_node);  in dev_map_alloc()
133 	if (!dtab->netdev_map)  in dev_map_alloc()
137 	list_add_tail_rcu(&dtab->list, &dev_map_list);  in dev_map_alloc()
140 	return &dtab->map;  in dev_map_alloc()
142 	free_percpu(dtab->flush_needed);  in dev_map_alloc()
143 	kfree(dtab);  in dev_map_alloc()
149 	struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);  in dev_map_free()  local
161 	list_del_rcu(&dtab->list);  in dev_map_free()
173 		unsigned long *bitmap = per_cpu_ptr(dtab->flush_needed, cpu);  in dev_map_free()
175 		while (!bitmap_empty(bitmap, dtab->map.max_entries))  in dev_map_free()
179 	for (i = 0; i < dtab->map.max_entries; i++) {  in dev_map_free()
182 		dev = dtab->netdev_map[i];  in dev_map_free()
190 	free_percpu(dtab->flush_needed);  in dev_map_free()
191 	bpf_map_area_free(dtab->netdev_map);  in dev_map_free()
192 	kfree(dtab);  in dev_map_free()
197 	struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);  in dev_map_get_next_key()  local
201 	if (index >= dtab->map.max_entries) {  in dev_map_get_next_key()
206 	if (index == dtab->map.max_entries - 1)  in dev_map_get_next_key()
214 	struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);  in __dev_map_insert_ctx()  local
215 	unsigned long *bitmap = this_cpu_ptr(dtab->flush_needed);  in __dev_map_insert_ctx()
247 	trace_xdp_devmap_xmit(&obj->dtab->map, obj->bit,  in bq_xmit_all()
277 	struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);  in __dev_map_flush()  local
278 	unsigned long *bitmap = this_cpu_ptr(dtab->flush_needed);  in __dev_map_flush()
282 		struct bpf_dtab_netdev *dev = READ_ONCE(dtab->netdev_map[bit]);  in __dev_map_flush()
304 	struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);  in __dev_map_lookup_elem()  local
310 	obj = READ_ONCE(dtab->netdev_map[key]);  in __dev_map_lookup_elem()
389 			bitmap = per_cpu_ptr(dev->dtab->flush_needed, cpu);  in dev_map_flush_old()
411 	struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);  in dev_map_delete_elem()  local
426 	old_dev = xchg(&dtab->netdev_map[k], NULL);  in dev_map_delete_elem()
435 	struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map);  in dev_map_update_elem()  local
444 	if (unlikely(i >= dtab->map.max_entries))  in dev_map_update_elem()
471 		dev->dtab = dtab;  in dev_map_update_elem()
478 	old_dev = xchg(&dtab->netdev_map[i], dev);  in dev_map_update_elem()
499 	struct bpf_dtab *dtab;  in dev_map_notification()  local
510 		list_for_each_entry_rcu(dtab, &dev_map_list, list) {  in dev_map_notification()
511 			for (i = 0; i < dtab->map.max_entries; i++) {  in dev_map_notification()
514 				dev = READ_ONCE(dtab->netdev_map[i]);  in dev_map_notification()
518 				odev = cmpxchg(&dtab->netdev_map[i], dev, NULL);  in dev_map_notification()