184 static inline void htab_elem_set_ptr(struct htab_elem *l, u32 key_size,  in htab_elem_set_ptr()  argument
187 	*(void __percpu **)(l->key + key_size) = pptr;  in htab_elem_set_ptr()
190 static inline void __percpu *htab_elem_get_ptr(struct htab_elem *l, u32 key_size)  in htab_elem_get_ptr()  argument
192 	return *(void __percpu **)(l->key + key_size);  in htab_elem_get_ptr()
197 	return *(void **)(l->key + roundup(map->key_size, 8));  in fd_htab_map_get_ptr()
216 					 htab->map.key_size);  in htab_free_elems()
243 		memcpy(l->key, key, htab->map.key_size);  in prealloc_lru_pop()
273 		htab_elem_set_ptr(get_htab_elem(htab, i), htab->map.key_size,  in prealloc_init()
389 	if (attr->max_entries == 0 || attr->key_size == 0 ||  in htab_map_alloc_check()
393 	if (attr->key_size > MAX_BPF_STACK)  in htab_map_alloc_check()
450 			  round_up(htab->map.key_size, 8);  in htab_map_alloc()
535 					 void *key, u32 key_size)  in lookup_elem_raw()  argument
541 		if (l->hash == hash && !memcmp(&l->key, key, key_size))  in lookup_elem_raw()
553 					       u32 key_size, u32 n_buckets)  in lookup_nulls_elem_raw()  argument
560 		if (l->hash == hash && !memcmp(&l->key, key, key_size))  in lookup_nulls_elem_raw()
579 	u32 hash, key_size;  in __htab_map_lookup_elem()  local
583 	key_size = map->key_size;  in __htab_map_lookup_elem()
585 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in __htab_map_lookup_elem()
589 	l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets);  in __htab_map_lookup_elem()
599 		return l->key + round_up(map->key_size, 8);  in htab_map_lookup_elem()
626 				round_up(map->key_size, 8));  in htab_map_gen_lookup()
638 		return l->key + round_up(map->key_size, 8);  in __htab_lru_map_lookup_elem()
675 				round_up(map->key_size, 8));  in htab_lru_map_gen_lookup()
714 	u32 hash, key_size;  in htab_map_get_next_key()  local
719 	key_size = map->key_size;  in htab_map_get_next_key()
724 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in htab_map_get_next_key()
729 	l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets);  in htab_map_get_next_key()
740 		memcpy(next_key, next_l->key, key_size);  in htab_map_get_next_key()
758 			memcpy(next_key, next_l->key, key_size);  in htab_map_get_next_key()
770 		free_percpu(htab_elem_get_ptr(l, htab->map.key_size));  in htab_elem_free()
857 					 void *value, u32 key_size, u32 hash,  in alloc_htab_elem()  argument
901 					l_new->key + round_up(key_size, 8));  in alloc_htab_elem()
904 	memcpy(l_new->key, key, key_size);  in alloc_htab_elem()
908 			pptr = htab_elem_get_ptr(l_new, key_size);  in alloc_htab_elem()
923 			htab_elem_set_ptr(l_new, key_size, pptr);  in alloc_htab_elem()
926 		memcpy(l_new->key + round_up(key_size, 8), value, size);  in alloc_htab_elem()
929 			       l_new->key + round_up(key_size, 8),  in alloc_htab_elem()
963 	u32 key_size, hash;  in htab_map_update_elem()  local
972 	key_size = map->key_size;  in htab_map_update_elem()
974 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in htab_map_update_elem()
983 		l_old = lookup_nulls_elem_raw(head, hash, key, key_size,  in htab_map_update_elem()
991 					      l_old->key + round_up(key_size, 8),  in htab_map_update_elem()
1003 	l_old = lookup_elem_raw(head, hash, key, key_size);  in htab_map_update_elem()
1017 				      l_old->key + round_up(key_size, 8),  in htab_map_update_elem()
1023 	l_new = alloc_htab_elem(htab, key, value, key_size, hash, false, false,  in htab_map_update_elem()
1054 	u32 key_size, hash;  in htab_lru_map_update_elem()  local
1063 	key_size = map->key_size;  in htab_lru_map_update_elem()
1065 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in htab_lru_map_update_elem()
1078 	memcpy(l_new->key + round_up(map->key_size, 8), value, map->value_size);  in htab_lru_map_update_elem()
1082 	l_old = lookup_elem_raw(head, hash, key, key_size);  in htab_lru_map_update_elem()
1118 	u32 key_size, hash;  in __htab_percpu_map_update_elem()  local
1127 	key_size = map->key_size;  in __htab_percpu_map_update_elem()
1129 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in __htab_percpu_map_update_elem()
1136 	l_old = lookup_elem_raw(head, hash, key, key_size);  in __htab_percpu_map_update_elem()
1144 		pcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size),  in __htab_percpu_map_update_elem()
1147 		l_new = alloc_htab_elem(htab, key, value, key_size,  in __htab_percpu_map_update_elem()
1170 	u32 key_size, hash;  in __htab_lru_percpu_map_update_elem()  local
1179 	key_size = map->key_size;  in __htab_lru_percpu_map_update_elem()
1181 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in __htab_lru_percpu_map_update_elem()
1199 	l_old = lookup_elem_raw(head, hash, key, key_size);  in __htab_lru_percpu_map_update_elem()
1209 		pcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size),  in __htab_lru_percpu_map_update_elem()
1212 		pcpu_init_value(htab, htab_elem_get_ptr(l_new, key_size),  in __htab_lru_percpu_map_update_elem()
1246 	u32 hash, key_size;  in htab_map_delete_elem()  local
1251 	key_size = map->key_size;  in htab_map_delete_elem()
1253 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in htab_map_delete_elem()
1259 	l = lookup_elem_raw(head, hash, key, key_size);  in htab_map_delete_elem()
1278 	u32 hash, key_size;  in htab_lru_map_delete_elem()  local
1283 	key_size = map->key_size;  in htab_lru_map_delete_elem()
1285 	hash = htab_map_hash(key, key_size, htab->hashrnd);  in htab_lru_map_delete_elem()
1291 	l = lookup_elem_raw(head, hash, key, key_size);  in htab_lru_map_delete_elem()
1373 	u32 bucket_cnt, total, key_size, value_size, roundup_key_size;  in __htab_map_lookup_and_delete_batch()  local
1412 	key_size = htab->map.key_size;  in __htab_map_lookup_and_delete_batch()
1413 	roundup_key_size = round_up(htab->map.key_size, 8);  in __htab_map_lookup_and_delete_batch()
1428 	keys = kvmalloc(key_size * bucket_size, GFP_USER | __GFP_NOWARN);  in __htab_map_lookup_and_delete_batch()
1486 		memcpy(dst_key, l->key, key_size);  in __htab_map_lookup_and_delete_batch()
1492 			pptr = htab_elem_get_ptr(l, map->key_size);  in __htab_map_lookup_and_delete_batch()
1522 		dst_key += key_size;  in __htab_map_lookup_and_delete_batch()
1546 	if (bucket_cnt && (copy_to_user(ukeys + total * key_size, keys,  in __htab_map_lookup_and_delete_batch()
1547 	    key_size * bucket_cnt) ||  in __htab_map_lookup_and_delete_batch()
1751 			roundup_key_size = round_up(map->key_size, 8);  in __bpf_hash_map_seq_show()
1757 				pptr = htab_elem_get_ptr(elem, map->key_size);  in __bpf_hash_map_seq_show()
1873 		return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size));  in htab_percpu_map_lookup_elem()
1884 		return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size));  in htab_lru_percpu_map_lookup_elem()
1910 	pptr = htab_elem_get_ptr(l, map->key_size);  in bpf_percpu_hash_copy()
1957 	pptr = htab_elem_get_ptr(l, map->key_size);  in htab_percpu_map_seq_show_elem()
2111 				round_up(map->key_size, 8));  in htab_of_map_gen_lookup()