79 static inline void htab_elem_set_ptr(struct htab_elem *l, u32 key_size,  in htab_elem_set_ptr()  argument
82 	*(void __percpu **)(l->key + key_size) = pptr;  in htab_elem_set_ptr()
85 static inline void __percpu *htab_elem_get_ptr(struct htab_elem *l, u32 key_size)  in htab_elem_get_ptr()  argument
87 	return *(void __percpu **)(l->key + key_size);  in htab_elem_get_ptr()
92 	return *(void **)(l->key + roundup(map->key_size, 8));  in fd_htab_map_get_ptr()
111 					 htab->map.key_size);  in htab_free_elems()
127 		memcpy(l->key, key, htab->map.key_size);  in prealloc_lru_pop()
157 		htab_elem_set_ptr(get_htab_elem(htab, i), htab->map.key_size,  in prealloc_init()
273 	if (attr->max_entries == 0 || attr->key_size == 0 ||  in htab_map_alloc_check()
277 	if (attr->key_size > MAX_BPF_STACK)  in htab_map_alloc_check()
334 			  round_up(htab->map.key_size, 8);  in htab_map_alloc()
422 					 void *key, u32 key_size)  in lookup_elem_raw()  argument
428 		if (l->hash == hash && !memcmp(&l->key, key, key_size))  in lookup_elem_raw()
440 					       u32 key_size, u32 n_buckets)  in lookup_nulls_elem_raw()  argument
447 		if (l->hash == hash && !memcmp(&l->key, key, key_size))  in lookup_nulls_elem_raw()
466 	u32 hash, key_size;  in __htab_map_lookup_elem()  local
471 	key_size = map->key_size;  in __htab_map_lookup_elem()
473 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in __htab_map_lookup_elem()
477 	l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets);  in __htab_map_lookup_elem()
487 		return l->key + round_up(map->key_size, 8);  in htab_map_lookup_elem()
514 				round_up(map->key_size, 8));  in htab_map_gen_lookup()
526 		return l->key + round_up(map->key_size, 8);  in __htab_lru_map_lookup_elem()
563 				round_up(map->key_size, 8));  in htab_lru_map_gen_lookup()
602 	u32 hash, key_size;  in htab_map_get_next_key()  local
607 	key_size = map->key_size;  in htab_map_get_next_key()
612 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in htab_map_get_next_key()
617 	l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets);  in htab_map_get_next_key()
628 		memcpy(next_key, next_l->key, key_size);  in htab_map_get_next_key()
646 			memcpy(next_key, next_l->key, key_size);  in htab_map_get_next_key()
658 		free_percpu(htab_elem_get_ptr(l, htab->map.key_size));  in htab_elem_free()
722 					 void *value, u32 key_size, u32 hash,  in alloc_htab_elem()  argument
765 					l_new->key + round_up(key_size, 8));  in alloc_htab_elem()
768 	memcpy(l_new->key, key, key_size);  in alloc_htab_elem()
772 			pptr = htab_elem_get_ptr(l_new, key_size);  in alloc_htab_elem()
787 			htab_elem_set_ptr(l_new, key_size, pptr);  in alloc_htab_elem()
790 		memcpy(l_new->key + round_up(key_size, 8), value, size);  in alloc_htab_elem()
793 			       l_new->key + round_up(key_size, 8),  in alloc_htab_elem()
827 	u32 key_size, hash;  in htab_map_update_elem()  local
836 	key_size = map->key_size;  in htab_map_update_elem()
838 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in htab_map_update_elem()
847 		l_old = lookup_nulls_elem_raw(head, hash, key, key_size,  in htab_map_update_elem()
855 					      l_old->key + round_up(key_size, 8),  in htab_map_update_elem()
868 	l_old = lookup_elem_raw(head, hash, key, key_size);  in htab_map_update_elem()
882 				      l_old->key + round_up(key_size, 8),  in htab_map_update_elem()
888 	l_new = alloc_htab_elem(htab, key, value, key_size, hash, false, false,  in htab_map_update_elem()
919 	u32 key_size, hash;  in htab_lru_map_update_elem()  local
928 	key_size = map->key_size;  in htab_lru_map_update_elem()
930 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in htab_lru_map_update_elem()
943 	memcpy(l_new->key + round_up(map->key_size, 8), value, map->value_size);  in htab_lru_map_update_elem()
948 	l_old = lookup_elem_raw(head, hash, key, key_size);  in htab_lru_map_update_elem()
984 	u32 key_size, hash;  in __htab_percpu_map_update_elem()  local
993 	key_size = map->key_size;  in __htab_percpu_map_update_elem()
995 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in __htab_percpu_map_update_elem()
1003 	l_old = lookup_elem_raw(head, hash, key, key_size);  in __htab_percpu_map_update_elem()
1011 		pcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size),  in __htab_percpu_map_update_elem()
1014 		l_new = alloc_htab_elem(htab, key, value, key_size,  in __htab_percpu_map_update_elem()
1037 	u32 key_size, hash;  in __htab_lru_percpu_map_update_elem()  local
1046 	key_size = map->key_size;  in __htab_lru_percpu_map_update_elem()
1048 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in __htab_lru_percpu_map_update_elem()
1067 	l_old = lookup_elem_raw(head, hash, key, key_size);  in __htab_lru_percpu_map_update_elem()
1077 		pcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size),  in __htab_lru_percpu_map_update_elem()
1080 		pcpu_copy_value(htab, htab_elem_get_ptr(l_new, key_size),  in __htab_lru_percpu_map_update_elem()
1114 	u32 hash, key_size;  in htab_map_delete_elem()  local
1119 	key_size = map->key_size;  in htab_map_delete_elem()
1121 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in htab_map_delete_elem()
1127 	l = lookup_elem_raw(head, hash, key, key_size);  in htab_map_delete_elem()
1146 	u32 hash, key_size;  in htab_lru_map_delete_elem()  local
1151 	key_size = map->key_size;  in htab_lru_map_delete_elem()
1153 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in htab_lru_map_delete_elem()
1159 	l = lookup_elem_raw(head, hash, key, key_size);  in htab_lru_map_delete_elem()
1266 		return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size));  in htab_percpu_map_lookup_elem()
1277 		return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size));  in htab_lru_percpu_map_lookup_elem()
1303 	pptr = htab_elem_get_ptr(l, map->key_size);  in bpf_percpu_hash_copy()
1350 	pptr = htab_elem_get_ptr(l, map->key_size);  in htab_percpu_map_seq_show_elem()
1492 				round_up(map->key_size, 8));  in htab_of_map_gen_lookup()