87 static inline void htab_elem_set_ptr(struct htab_elem *l, u32 key_size,  in htab_elem_set_ptr()  argument
90 	*(void __percpu **)(l->key + key_size) = pptr;  in htab_elem_set_ptr()
93 static inline void __percpu *htab_elem_get_ptr(struct htab_elem *l, u32 key_size)  in htab_elem_get_ptr()  argument
95 	return *(void __percpu **)(l->key + key_size);  in htab_elem_get_ptr()
100 	return *(void **)(l->key + roundup(map->key_size, 8));  in fd_htab_map_get_ptr()
119 					 htab->map.key_size);  in htab_free_elems()
135 		memcpy(l->key, key, htab->map.key_size);  in prealloc_lru_pop()
165 		htab_elem_set_ptr(get_htab_elem(htab, i), htab->map.key_size,  in prealloc_init()
276 	if (attr->max_entries == 0 || attr->key_size == 0 ||  in htab_map_alloc_check()
280 	if (attr->key_size > MAX_BPF_STACK)  in htab_map_alloc_check()
337 			  round_up(htab->map.key_size, 8);  in htab_map_alloc()
425 					 void *key, u32 key_size)  in lookup_elem_raw()  argument
431 		if (l->hash == hash && !memcmp(&l->key, key, key_size))  in lookup_elem_raw()
443 					       u32 key_size, u32 n_buckets)  in lookup_nulls_elem_raw()  argument
450 		if (l->hash == hash && !memcmp(&l->key, key, key_size))  in lookup_nulls_elem_raw()
469 	u32 hash, key_size;  in __htab_map_lookup_elem()  local
474 	key_size = map->key_size;  in __htab_map_lookup_elem()
476 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in __htab_map_lookup_elem()
480 	l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets);  in __htab_map_lookup_elem()
490 		return l->key + round_up(map->key_size, 8);  in htab_map_lookup_elem()
517 				round_up(map->key_size, 8));  in htab_map_gen_lookup()
527 		return l->key + round_up(map->key_size, 8);  in htab_lru_map_lookup_elem()
554 				round_up(map->key_size, 8));  in htab_lru_map_gen_lookup()
593 	u32 hash, key_size;  in htab_map_get_next_key()  local
598 	key_size = map->key_size;  in htab_map_get_next_key()
603 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in htab_map_get_next_key()
608 	l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets);  in htab_map_get_next_key()
619 		memcpy(next_key, next_l->key, key_size);  in htab_map_get_next_key()
637 			memcpy(next_key, next_l->key, key_size);  in htab_map_get_next_key()
649 		free_percpu(htab_elem_get_ptr(l, htab->map.key_size));  in htab_elem_free()
722 					 void *value, u32 key_size, u32 hash,  in alloc_htab_elem()  argument
766 	memcpy(l_new->key, key, key_size);  in alloc_htab_elem()
769 			pptr = htab_elem_get_ptr(l_new, key_size);  in alloc_htab_elem()
784 			htab_elem_set_ptr(l_new, key_size, pptr);  in alloc_htab_elem()
786 		memcpy(l_new->key + round_up(key_size, 8), value, size);  in alloc_htab_elem()
819 	u32 key_size, hash;  in htab_map_update_elem()  local
828 	key_size = map->key_size;  in htab_map_update_elem()
830 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in htab_map_update_elem()
838 	l_old = lookup_elem_raw(head, hash, key, key_size);  in htab_map_update_elem()
844 	l_new = alloc_htab_elem(htab, key, value, key_size, hash, false, false,  in htab_map_update_elem()
875 	u32 key_size, hash;  in htab_lru_map_update_elem()  local
884 	key_size = map->key_size;  in htab_lru_map_update_elem()
886 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in htab_lru_map_update_elem()
899 	memcpy(l_new->key + round_up(map->key_size, 8), value, map->value_size);  in htab_lru_map_update_elem()
904 	l_old = lookup_elem_raw(head, hash, key, key_size);  in htab_lru_map_update_elem()
940 	u32 key_size, hash;  in __htab_percpu_map_update_elem()  local
949 	key_size = map->key_size;  in __htab_percpu_map_update_elem()
951 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in __htab_percpu_map_update_elem()
959 	l_old = lookup_elem_raw(head, hash, key, key_size);  in __htab_percpu_map_update_elem()
967 		pcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size),  in __htab_percpu_map_update_elem()
970 		l_new = alloc_htab_elem(htab, key, value, key_size,  in __htab_percpu_map_update_elem()
993 	u32 key_size, hash;  in __htab_lru_percpu_map_update_elem()  local
1002 	key_size = map->key_size;  in __htab_lru_percpu_map_update_elem()
1004 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in __htab_lru_percpu_map_update_elem()
1023 	l_old = lookup_elem_raw(head, hash, key, key_size);  in __htab_lru_percpu_map_update_elem()
1033 		pcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size),  in __htab_lru_percpu_map_update_elem()
1036 		pcpu_copy_value(htab, htab_elem_get_ptr(l_new, key_size),  in __htab_lru_percpu_map_update_elem()
1070 	u32 hash, key_size;  in htab_map_delete_elem()  local
1075 	key_size = map->key_size;  in htab_map_delete_elem()
1077 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in htab_map_delete_elem()
1083 	l = lookup_elem_raw(head, hash, key, key_size);  in htab_map_delete_elem()
1102 	u32 hash, key_size;  in htab_lru_map_delete_elem()  local
1107 	key_size = map->key_size;  in htab_lru_map_delete_elem()
1109 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in htab_lru_map_delete_elem()
1115 	l = lookup_elem_raw(head, hash, key, key_size);  in htab_lru_map_delete_elem()
1221 		return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size));  in htab_percpu_map_lookup_elem()
1232 		return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size));  in htab_lru_percpu_map_lookup_elem()
1258 	pptr = htab_elem_get_ptr(l, map->key_size);  in bpf_percpu_hash_copy()
1416 				round_up(map->key_size, 8));  in htab_of_map_gen_lookup()