41 	struct htab_elem *__percpu *extra_elems;
49 struct htab_elem {  struct
87 static inline void htab_elem_set_ptr(struct htab_elem *l, u32 key_size,  in htab_elem_set_ptr()  argument
93 static inline void __percpu *htab_elem_get_ptr(struct htab_elem *l, u32 key_size)  in htab_elem_get_ptr()
98 static void *fd_htab_map_get_ptr(const struct bpf_map *map, struct htab_elem *l)  in fd_htab_map_get_ptr()
103 static struct htab_elem *get_htab_elem(struct bpf_htab *htab, int i)  in get_htab_elem()
105 	return (struct htab_elem *) (htab->elems + i * htab->elem_size);  in get_htab_elem()
127 static struct htab_elem *prealloc_lru_pop(struct bpf_htab *htab, void *key,  in prealloc_lru_pop()
131 	struct htab_elem *l;  in prealloc_lru_pop()
134 		l = container_of(node, struct htab_elem, lru_node);  in prealloc_lru_pop()
174 				   offsetof(struct htab_elem, hash) -  in prealloc_init()
175 				   offsetof(struct htab_elem, lru_node),  in prealloc_init()
186 				 offsetof(struct htab_elem, lru_node),  in prealloc_init()
190 				       htab->elems + offsetof(struct htab_elem, fnode),  in prealloc_init()
212 	struct htab_elem *__percpu *pptr, *l_new;  in alloc_extra_elems()
216 	pptr = __alloc_percpu_gfp(sizeof(struct htab_elem *), 8,  in alloc_extra_elems()
226 		l_new = container_of(l, struct htab_elem, fnode);  in alloc_extra_elems()
249 	BUILD_BUG_ON(offsetof(struct htab_elem, htab) !=  in htab_map_alloc_check()
250 		     offsetof(struct htab_elem, hash_node.pprev));  in htab_map_alloc_check()
251 	BUILD_BUG_ON(offsetof(struct htab_elem, fnode.next) !=  in htab_map_alloc_check()
252 		     offsetof(struct htab_elem, hash_node.pprev));  in htab_map_alloc_check()
287 	    MAX_BPF_STACK - sizeof(struct htab_elem))  in htab_map_alloc_check()
336 	htab->elem_size = sizeof(struct htab_elem) +  in htab_map_alloc()
424 static struct htab_elem *lookup_elem_raw(struct hlist_nulls_head *head, u32 hash,  in lookup_elem_raw()
428 	struct htab_elem *l;  in lookup_elem_raw()
441 static struct htab_elem *lookup_nulls_elem_raw(struct hlist_nulls_head *head,  in lookup_nulls_elem_raw()
446 	struct htab_elem *l;  in lookup_nulls_elem_raw()
468 	struct htab_elem *l;  in __htab_map_lookup_elem()
487 	struct htab_elem *l = __htab_map_lookup_elem(map, key);  in htab_map_lookup_elem()
516 				offsetof(struct htab_elem, key) +  in htab_map_gen_lookup()
523 	struct htab_elem *l = __htab_map_lookup_elem(map, key);  in htab_lru_map_lookup_elem()
545 			      offsetof(struct htab_elem, lru_node) +  in htab_lru_map_gen_lookup()
549 			     offsetof(struct htab_elem, lru_node) +  in htab_lru_map_gen_lookup()
553 				offsetof(struct htab_elem, key) +  in htab_lru_map_gen_lookup()
564 	struct htab_elem *l = NULL, *tgt_l;  in htab_lru_map_delete_node()
570 	tgt_l = container_of(node, struct htab_elem, lru_node);  in htab_lru_map_delete_node()
592 	struct htab_elem *l, *next_l;  in htab_map_get_next_key()
615 				  struct htab_elem, hash_node);  in htab_map_get_next_key()
634 					  struct htab_elem, hash_node);  in htab_map_get_next_key()
646 static void htab_elem_free(struct bpf_htab *htab, struct htab_elem *l)  in htab_elem_free()
655 	struct htab_elem *l = container_of(head, struct htab_elem, rcu);  in htab_elem_free_rcu()
669 static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l)  in free_htab_elem()
721 static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key,  in alloc_htab_elem()
724 					 struct htab_elem *old_elem)  in alloc_htab_elem()
728 	struct htab_elem *l_new, **pl_new;  in alloc_htab_elem()
745 			l_new = container_of(l, struct htab_elem, fnode);  in alloc_htab_elem()
796 static int check_flags(struct bpf_htab *htab, struct htab_elem *l_old,  in check_flags()
815 	struct htab_elem *l_new = NULL, *l_old;  in htab_map_update_elem()
871 	struct htab_elem *l_new, *l_old = NULL;  in htab_lru_map_update_elem()
936 	struct htab_elem *l_new = NULL, *l_old;  in __htab_percpu_map_update_elem()
989 	struct htab_elem *l_new = NULL, *l_old;  in __htab_lru_percpu_map_update_elem()
1068 	struct htab_elem *l;  in htab_map_delete_elem()
1100 	struct htab_elem *l;  in htab_lru_map_delete_elem()
1135 		struct htab_elem *l;  in delete_all_elements()
1218 	struct htab_elem *l = __htab_map_lookup_elem(map, key);  in htab_percpu_map_lookup_elem()
1228 	struct htab_elem *l = __htab_map_lookup_elem(map, key);  in htab_lru_percpu_map_lookup_elem()
1241 	struct htab_elem *l;  in bpf_percpu_hash_copy()
1320 	struct htab_elem *l;  in fd_htab_map_free()
1415 				offsetof(struct htab_elem, key) +  in htab_of_map_gen_lookup()