Lines Matching +full:cpu +full:- +full:map
1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
46 * from sys_bpf(). BPF recursion is prevented by incrementing the per CPU
50 * by pinning the task to the current CPU and incrementing the recursion
51 * protection across the map operation.
72 * it is only safe to use raw spinlock for preallocated hash map on a RT kernel,
74 * after hash map was fully converted to use bpf_mem_alloc, there will be
75 * non-synchronous memory allocation for non-preallocated hash map, so it is
84 #define HASHTAB_MAP_LOCK_MASK (HASHTAB_MAP_LOCK_COUNT - 1)
87 struct bpf_map map; member
97 /* number of elements in non-preallocated hashtable are kept
123 /* pointer to per-cpu pointer */
133 return !(htab->map.map_flags & BPF_F_NO_PREALLOC); in htab_is_prealloc()
140 for (i = 0; i < htab->n_buckets; i++) { in htab_init_buckets()
141 INIT_HLIST_NULLS_HEAD(&htab->buckets[i].head, i); in htab_init_buckets()
142 raw_spin_lock_init(&htab->buckets[i].raw_lock); in htab_init_buckets()
143 lockdep_set_class(&htab->buckets[i].raw_lock, in htab_init_buckets()
144 &htab->lockdep_key); in htab_init_buckets()
155 hash = hash & min_t(u32, HASHTAB_MAP_LOCK_MASK, htab->n_buckets - 1); in htab_lock_bucket()
158 if (unlikely(__this_cpu_inc_return(*(htab->map_locked[hash])) != 1)) { in htab_lock_bucket()
159 __this_cpu_dec(*(htab->map_locked[hash])); in htab_lock_bucket()
161 return -EBUSY; in htab_lock_bucket()
164 raw_spin_lock_irqsave(&b->raw_lock, flags); in htab_lock_bucket()
174 hash = hash & min_t(u32, HASHTAB_MAP_LOCK_MASK, htab->n_buckets - 1); in htab_unlock_bucket()
175 raw_spin_unlock_irqrestore(&b->raw_lock, flags); in htab_unlock_bucket()
176 __this_cpu_dec(*(htab->map_locked[hash])); in htab_unlock_bucket()
184 return htab->map.map_type == BPF_MAP_TYPE_LRU_HASH || in htab_is_lru()
185 htab->map.map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH; in htab_is_lru()
190 return htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH || in htab_is_percpu()
191 htab->map.map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH; in htab_is_percpu()
197 *(void __percpu **)(l->key + key_size) = pptr; in htab_elem_set_ptr()
202 return *(void __percpu **)(l->key + key_size); in htab_elem_get_ptr()
205 static void *fd_htab_map_get_ptr(const struct bpf_map *map, struct htab_elem *l) in fd_htab_map_get_ptr() argument
207 return *(void **)(l->key + roundup(map->key_size, 8)); in fd_htab_map_get_ptr()
212 return (struct htab_elem *) (htab->elems + i * (u64)htab->elem_size); in get_htab_elem()
222 u32 num_entries = htab->map.max_entries; in htab_free_prealloced_timers()
225 if (!btf_record_has_field(htab->map.record, BPF_TIMER)) in htab_free_prealloced_timers()
234 bpf_obj_free_timer(htab->map.record, elem->key + round_up(htab->map.key_size, 8)); in htab_free_prealloced_timers()
241 u32 num_entries = htab->map.max_entries; in htab_free_prealloced_fields()
244 if (IS_ERR_OR_NULL(htab->map.record)) in htab_free_prealloced_fields()
253 void __percpu *pptr = htab_elem_get_ptr(elem, htab->map.key_size); in htab_free_prealloced_fields()
254 int cpu; in htab_free_prealloced_fields() local
256 for_each_possible_cpu(cpu) { in htab_free_prealloced_fields()
257 bpf_obj_free_fields(htab->map.record, per_cpu_ptr(pptr, cpu)); in htab_free_prealloced_fields()
261 bpf_obj_free_fields(htab->map.record, elem->key + round_up(htab->map.key_size, 8)); in htab_free_prealloced_fields()
275 for (i = 0; i < htab->map.max_entries; i++) { in htab_free_elems()
279 htab->map.key_size); in htab_free_elems()
284 bpf_map_area_free(htab->elems); in htab_free_elems()
289 * order is always lru_lock -> bucket_lock and this only happens in
301 struct bpf_lru_node *node = bpf_lru_pop_free(&htab->lru, hash); in prealloc_lru_pop()
305 bpf_map_inc_elem_count(&htab->map); in prealloc_lru_pop()
307 memcpy(l->key, key, htab->map.key_size); in prealloc_lru_pop()
316 u32 num_entries = htab->map.max_entries; in prealloc_init()
317 int err = -ENOMEM, i; in prealloc_init()
322 htab->elems = bpf_map_area_alloc((u64)htab->elem_size * num_entries, in prealloc_init()
323 htab->map.numa_node); in prealloc_init()
324 if (!htab->elems) in prealloc_init()
325 return -ENOMEM; in prealloc_init()
331 u32 size = round_up(htab->map.value_size, 8); in prealloc_init()
334 pptr = bpf_map_alloc_percpu(&htab->map, size, 8, in prealloc_init()
338 htab_elem_set_ptr(get_htab_elem(htab, i), htab->map.key_size, in prealloc_init()
345 err = bpf_lru_init(&htab->lru, in prealloc_init()
346 htab->map.map_flags & BPF_F_NO_COMMON_LRU, in prealloc_init()
347 offsetof(struct htab_elem, hash) - in prealloc_init()
352 err = pcpu_freelist_init(&htab->freelist); in prealloc_init()
358 bpf_lru_populate(&htab->lru, htab->elems, in prealloc_init()
360 htab->elem_size, num_entries); in prealloc_init()
362 pcpu_freelist_populate(&htab->freelist, in prealloc_init()
363 htab->elems + offsetof(struct htab_elem, fnode), in prealloc_init()
364 htab->elem_size, num_entries); in prealloc_init()
378 bpf_lru_destroy(&htab->lru); in prealloc_destroy()
380 pcpu_freelist_destroy(&htab->freelist); in prealloc_destroy()
387 int cpu; in alloc_extra_elems() local
389 pptr = bpf_map_alloc_percpu(&htab->map, sizeof(struct htab_elem *), 8, in alloc_extra_elems()
392 return -ENOMEM; in alloc_extra_elems()
394 for_each_possible_cpu(cpu) { in alloc_extra_elems()
395 l = pcpu_freelist_pop(&htab->freelist); in alloc_extra_elems()
400 *per_cpu_ptr(pptr, cpu) = l_new; in alloc_extra_elems()
402 htab->extra_elems = pptr; in alloc_extra_elems()
409 bool percpu = (attr->map_type == BPF_MAP_TYPE_PERCPU_HASH || in htab_map_alloc_check()
410 attr->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH); in htab_map_alloc_check()
411 bool lru = (attr->map_type == BPF_MAP_TYPE_LRU_HASH || in htab_map_alloc_check()
412 attr->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH); in htab_map_alloc_check()
413 /* percpu_lru means each cpu has its own LRU list. in htab_map_alloc_check()
415 * the map's value itself is percpu. percpu_lru has in htab_map_alloc_check()
416 * nothing to do with the map's value. in htab_map_alloc_check()
418 bool percpu_lru = (attr->map_flags & BPF_F_NO_COMMON_LRU); in htab_map_alloc_check()
419 bool prealloc = !(attr->map_flags & BPF_F_NO_PREALLOC); in htab_map_alloc_check()
420 bool zero_seed = (attr->map_flags & BPF_F_ZERO_SEED); in htab_map_alloc_check()
428 return -EPERM; in htab_map_alloc_check()
430 if (attr->map_flags & ~HTAB_CREATE_FLAG_MASK || in htab_map_alloc_check()
431 !bpf_map_flags_access_ok(attr->map_flags)) in htab_map_alloc_check()
432 return -EINVAL; in htab_map_alloc_check()
435 return -EINVAL; in htab_map_alloc_check()
438 return -ENOTSUPP; in htab_map_alloc_check()
441 return -EINVAL; in htab_map_alloc_check()
444 * value_size == 0 may be allowed in the future to use map as a set in htab_map_alloc_check()
446 if (attr->max_entries == 0 || attr->key_size == 0 || in htab_map_alloc_check()
447 attr->value_size == 0) in htab_map_alloc_check()
448 return -EINVAL; in htab_map_alloc_check()
450 if ((u64)attr->key_size + attr->value_size >= KMALLOC_MAX_SIZE - in htab_map_alloc_check()
455 * kmalloc-able later in htab_map_update_elem() in htab_map_alloc_check()
457 return -E2BIG; in htab_map_alloc_check()
464 bool percpu = (attr->map_type == BPF_MAP_TYPE_PERCPU_HASH || in htab_map_alloc()
465 attr->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH); in htab_map_alloc()
466 bool lru = (attr->map_type == BPF_MAP_TYPE_LRU_HASH || in htab_map_alloc()
467 attr->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH); in htab_map_alloc()
468 /* percpu_lru means each cpu has its own LRU list. in htab_map_alloc()
470 * the map's value itself is percpu. percpu_lru has in htab_map_alloc()
471 * nothing to do with the map's value. in htab_map_alloc()
473 bool percpu_lru = (attr->map_flags & BPF_F_NO_COMMON_LRU); in htab_map_alloc()
474 bool prealloc = !(attr->map_flags & BPF_F_NO_PREALLOC); in htab_map_alloc()
480 return ERR_PTR(-ENOMEM); in htab_map_alloc()
482 lockdep_register_key(&htab->lockdep_key); in htab_map_alloc()
484 bpf_map_init_from_attr(&htab->map, attr); in htab_map_alloc()
487 /* ensure each CPU's lru list has >=1 elements. in htab_map_alloc()
491 htab->map.max_entries = roundup(attr->max_entries, in htab_map_alloc()
493 if (htab->map.max_entries < attr->max_entries) in htab_map_alloc()
494 htab->map.max_entries = rounddown(attr->max_entries, in htab_map_alloc()
499 htab->n_buckets = roundup_pow_of_two(htab->map.max_entries); in htab_map_alloc()
501 htab->elem_size = sizeof(struct htab_elem) + in htab_map_alloc()
502 round_up(htab->map.key_size, 8); in htab_map_alloc()
504 htab->elem_size += sizeof(void *); in htab_map_alloc()
506 htab->elem_size += round_up(htab->map.value_size, 8); in htab_map_alloc()
508 err = -E2BIG; in htab_map_alloc()
510 if (htab->n_buckets == 0 || in htab_map_alloc()
511 htab->n_buckets > U32_MAX / sizeof(struct bucket)) in htab_map_alloc()
514 err = bpf_map_init_elem_count(&htab->map); in htab_map_alloc()
518 err = -ENOMEM; in htab_map_alloc()
519 htab->buckets = bpf_map_area_alloc(htab->n_buckets * in htab_map_alloc()
521 htab->map.numa_node); in htab_map_alloc()
522 if (!htab->buckets) in htab_map_alloc()
526 htab->map_locked[i] = bpf_map_alloc_percpu(&htab->map, in htab_map_alloc()
530 if (!htab->map_locked[i]) in htab_map_alloc()
534 if (htab->map.map_flags & BPF_F_ZERO_SEED) in htab_map_alloc()
535 htab->hashrnd = 0; in htab_map_alloc()
537 htab->hashrnd = get_random_u32(); in htab_map_alloc()
543 * htab->max_entries - cur_number_of_elems to be more than batch * num_online_cpus() in htab_map_alloc()
545 * hash map size is 10k, which means that a system with 64 cpus will fill in htab_map_alloc()
547 * define our own batch count as 32 then 10k hash map can be filled up to 80%: in htab_map_alloc()
548 * 10k - 8k > 32 _batch_ * 64 _cpus_ in htab_map_alloc()
549 * and __percpu_counter_compare() will still be fast. At that point hash map in htab_map_alloc()
550 * collisions will dominate its performance anyway. Assume that hash map filled in htab_map_alloc()
555 if (attr->max_entries / 2 > num_online_cpus() * PERCPU_COUNTER_BATCH) in htab_map_alloc()
556 htab->use_percpu_counter = true; in htab_map_alloc()
558 if (htab->use_percpu_counter) { in htab_map_alloc()
559 err = percpu_counter_init(&htab->pcount, 0, GFP_KERNEL); in htab_map_alloc()
578 err = bpf_mem_alloc_init(&htab->ma, htab->elem_size, false); in htab_map_alloc()
582 err = bpf_mem_alloc_init(&htab->pcpu_ma, in htab_map_alloc()
583 round_up(htab->map.value_size, 8), true); in htab_map_alloc()
589 return &htab->map; in htab_map_alloc()
594 if (htab->use_percpu_counter) in htab_map_alloc()
595 percpu_counter_destroy(&htab->pcount); in htab_map_alloc()
597 free_percpu(htab->map_locked[i]); in htab_map_alloc()
598 bpf_map_area_free(htab->buckets); in htab_map_alloc()
599 bpf_mem_alloc_destroy(&htab->pcpu_ma); in htab_map_alloc()
600 bpf_mem_alloc_destroy(&htab->ma); in htab_map_alloc()
602 bpf_map_free_elem_count(&htab->map); in htab_map_alloc()
604 lockdep_unregister_key(&htab->lockdep_key); in htab_map_alloc()
618 return &htab->buckets[hash & (htab->n_buckets - 1)]; in __select_bucket()
623 return &__select_bucket(htab, hash)->head; in select_bucket()
634 if (l->hash == hash && !memcmp(&l->key, key, key_size)) in lookup_elem_raw()
653 if (l->hash == hash && !memcmp(&l->key, key, key_size)) in lookup_nulls_elem_raw()
656 if (unlikely(get_nulls_value(n) != (hash & (n_buckets - 1)))) in lookup_nulls_elem_raw()
667 static void *__htab_map_lookup_elem(struct bpf_map *map, void *key) in __htab_map_lookup_elem() argument
669 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in __htab_map_lookup_elem()
677 key_size = map->key_size; in __htab_map_lookup_elem()
679 hash = htab_map_hash(key, key_size, htab->hashrnd); in __htab_map_lookup_elem()
683 l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets); in __htab_map_lookup_elem()
688 static void *htab_map_lookup_elem(struct bpf_map *map, void *key) in htab_map_lookup_elem() argument
690 struct htab_elem *l = __htab_map_lookup_elem(map, key); in htab_map_lookup_elem()
693 return l->key + round_up(map->key_size, 8); in htab_map_lookup_elem()
702 * map->ops->map_lookup_elem
709 static int htab_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf) in htab_map_gen_lookup() argument
715 (void *(*)(struct bpf_map *map, void *key))NULL)); in htab_map_gen_lookup()
720 round_up(map->key_size, 8)); in htab_map_gen_lookup()
721 return insn - insn_buf; in htab_map_gen_lookup()
724 static __always_inline void *__htab_lru_map_lookup_elem(struct bpf_map *map, in __htab_lru_map_lookup_elem() argument
727 struct htab_elem *l = __htab_map_lookup_elem(map, key); in __htab_lru_map_lookup_elem()
731 bpf_lru_node_set_ref(&l->lru_node); in __htab_lru_map_lookup_elem()
732 return l->key + round_up(map->key_size, 8); in __htab_lru_map_lookup_elem()
738 static void *htab_lru_map_lookup_elem(struct bpf_map *map, void *key) in htab_lru_map_lookup_elem() argument
740 return __htab_lru_map_lookup_elem(map, key, true); in htab_lru_map_lookup_elem()
743 static void *htab_lru_map_lookup_elem_sys(struct bpf_map *map, void *key) in htab_lru_map_lookup_elem_sys() argument
745 return __htab_lru_map_lookup_elem(map, key, false); in htab_lru_map_lookup_elem_sys()
748 static int htab_lru_map_gen_lookup(struct bpf_map *map, in htab_lru_map_gen_lookup() argument
756 (void *(*)(struct bpf_map *map, void *key))NULL)); in htab_lru_map_gen_lookup()
769 round_up(map->key_size, 8)); in htab_lru_map_gen_lookup()
770 return insn - insn_buf; in htab_lru_map_gen_lookup()
777 void __percpu *pptr = htab_elem_get_ptr(elem, htab->map.key_size); in check_and_free_fields()
778 int cpu; in check_and_free_fields() local
780 for_each_possible_cpu(cpu) in check_and_free_fields()
781 bpf_obj_free_fields(htab->map.record, per_cpu_ptr(pptr, cpu)); in check_and_free_fields()
783 void *map_value = elem->key + round_up(htab->map.key_size, 8); in check_and_free_fields()
785 bpf_obj_free_fields(htab->map.record, map_value); in check_and_free_fields()
803 b = __select_bucket(htab, tgt_l->hash); in htab_lru_map_delete_node()
804 head = &b->head; in htab_lru_map_delete_node()
806 ret = htab_lock_bucket(htab, b, tgt_l->hash, &flags); in htab_lru_map_delete_node()
812 hlist_nulls_del_rcu(&l->hash_node); in htab_lru_map_delete_node()
814 bpf_map_dec_elem_count(&htab->map); in htab_lru_map_delete_node()
818 htab_unlock_bucket(htab, b, tgt_l->hash, flags); in htab_lru_map_delete_node()
824 static int htab_map_get_next_key(struct bpf_map *map, void *key, void *next_key) in htab_map_get_next_key() argument
826 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_get_next_key()
834 key_size = map->key_size; in htab_map_get_next_key()
839 hash = htab_map_hash(key, key_size, htab->hashrnd); in htab_map_get_next_key()
844 l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets); in htab_map_get_next_key()
850 next_l = hlist_nulls_entry_safe(rcu_dereference_raw(hlist_nulls_next_rcu(&l->hash_node)), in htab_map_get_next_key()
854 /* if next elem in this hash list is non-zero, just return it */ in htab_map_get_next_key()
855 memcpy(next_key, next_l->key, key_size); in htab_map_get_next_key()
860 i = hash & (htab->n_buckets - 1); in htab_map_get_next_key()
865 for (; i < htab->n_buckets; i++) { in htab_map_get_next_key()
873 memcpy(next_key, next_l->key, key_size); in htab_map_get_next_key()
879 return -ENOENT; in htab_map_get_next_key()
885 if (htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH) in htab_elem_free()
886 bpf_mem_cache_free(&htab->pcpu_ma, l->ptr_to_pptr); in htab_elem_free()
887 bpf_mem_cache_free(&htab->ma, l); in htab_elem_free()
892 struct bpf_map *map = &htab->map; in htab_put_fd_value() local
895 if (map->ops->map_fd_put_ptr) { in htab_put_fd_value()
896 ptr = fd_htab_map_get_ptr(map, l); in htab_put_fd_value()
897 map->ops->map_fd_put_ptr(ptr); in htab_put_fd_value()
903 if (htab->use_percpu_counter) in is_map_full()
904 return __percpu_counter_compare(&htab->pcount, htab->map.max_entries, in is_map_full()
906 return atomic_read(&htab->count) >= htab->map.max_entries; in is_map_full()
911 bpf_map_inc_elem_count(&htab->map); in inc_elem_count()
913 if (htab->use_percpu_counter) in inc_elem_count()
914 percpu_counter_add_batch(&htab->pcount, 1, PERCPU_COUNTER_BATCH); in inc_elem_count()
916 atomic_inc(&htab->count); in inc_elem_count()
921 bpf_map_dec_elem_count(&htab->map); in dec_elem_count()
923 if (htab->use_percpu_counter) in dec_elem_count()
924 percpu_counter_add_batch(&htab->pcount, -1, PERCPU_COUNTER_BATCH); in dec_elem_count()
926 atomic_dec(&htab->count); in dec_elem_count()
935 bpf_map_dec_elem_count(&htab->map); in free_htab_elem()
937 __pcpu_freelist_push(&htab->freelist, &l->fnode); in free_htab_elem()
949 copy_map_value(&htab->map, this_cpu_ptr(pptr), value); in pcpu_copy_value()
951 u32 size = round_up(htab->map.value_size, 8); in pcpu_copy_value()
952 int off = 0, cpu; in pcpu_copy_value() local
954 for_each_possible_cpu(cpu) { in pcpu_copy_value()
955 copy_map_value_long(&htab->map, per_cpu_ptr(pptr, cpu), value + off); in pcpu_copy_value()
964 /* When not setting the initial value on all cpus, zero-fill element in pcpu_init_value()
971 int cpu; in pcpu_init_value() local
973 for_each_possible_cpu(cpu) { in pcpu_init_value()
974 if (cpu == current_cpu) in pcpu_init_value()
975 copy_map_value_long(&htab->map, per_cpu_ptr(pptr, cpu), value); in pcpu_init_value()
977 zero_map_value(&htab->map, per_cpu_ptr(pptr, cpu)); in pcpu_init_value()
986 return htab->map.map_type == BPF_MAP_TYPE_HASH_OF_MAPS && in fd_htab_map_needs_adjust()
995 u32 size = htab->map.value_size; in alloc_htab_elem()
1003 * use per-cpu extra elems to avoid freelist_pop/push in alloc_htab_elem()
1005 pl_new = this_cpu_ptr(htab->extra_elems); in alloc_htab_elem()
1012 l = __pcpu_freelist_pop(&htab->freelist); in alloc_htab_elem()
1014 return ERR_PTR(-E2BIG); in alloc_htab_elem()
1016 bpf_map_inc_elem_count(&htab->map); in alloc_htab_elem()
1021 /* when map is full and update() is replacing in alloc_htab_elem()
1026 return ERR_PTR(-E2BIG); in alloc_htab_elem()
1028 l_new = bpf_mem_cache_alloc(&htab->ma); in alloc_htab_elem()
1030 l_new = ERR_PTR(-ENOMEM); in alloc_htab_elem()
1035 memcpy(l_new->key, key, key_size); in alloc_htab_elem()
1040 /* alloc_percpu zero-fills */ in alloc_htab_elem()
1041 pptr = bpf_mem_cache_alloc(&htab->pcpu_ma); in alloc_htab_elem()
1043 bpf_mem_cache_free(&htab->ma, l_new); in alloc_htab_elem()
1044 l_new = ERR_PTR(-ENOMEM); in alloc_htab_elem()
1047 l_new->ptr_to_pptr = pptr; in alloc_htab_elem()
1057 memcpy(l_new->key + round_up(key_size, 8), value, size); in alloc_htab_elem()
1059 copy_map_value(&htab->map, in alloc_htab_elem()
1060 l_new->key + round_up(key_size, 8), in alloc_htab_elem()
1064 l_new->hash = hash; in alloc_htab_elem()
1076 return -EEXIST; in check_flags()
1080 return -ENOENT; in check_flags()
1086 static long htab_map_update_elem(struct bpf_map *map, void *key, void *value, in htab_map_update_elem() argument
1089 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_update_elem()
1099 return -EINVAL; in htab_map_update_elem()
1104 key_size = map->key_size; in htab_map_update_elem()
1106 hash = htab_map_hash(key, key_size, htab->hashrnd); in htab_map_update_elem()
1109 head = &b->head; in htab_map_update_elem()
1112 if (unlikely(!btf_record_has_field(map->record, BPF_SPIN_LOCK))) in htab_map_update_elem()
1113 return -EINVAL; in htab_map_update_elem()
1116 htab->n_buckets); in htab_map_update_elem()
1122 copy_map_value_locked(map, in htab_map_update_elem()
1123 l_old->key + round_up(key_size, 8), in htab_map_update_elem()
1150 copy_map_value_locked(map, in htab_map_update_elem()
1151 l_old->key + round_up(key_size, 8), in htab_map_update_elem()
1160 /* all pre-allocated elements are in use or memory exhausted */ in htab_map_update_elem()
1168 hlist_nulls_add_head_rcu(&l_new->hash_node, head); in htab_map_update_elem()
1170 hlist_nulls_del_rcu(&l_old->hash_node); in htab_map_update_elem()
1185 bpf_map_dec_elem_count(&htab->map); in htab_lru_push_free()
1186 bpf_lru_push_free(&htab->lru, &elem->lru_node); in htab_lru_push_free()
1189 static long htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value, in htab_lru_map_update_elem() argument
1192 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_lru_map_update_elem()
1202 return -EINVAL; in htab_lru_map_update_elem()
1207 key_size = map->key_size; in htab_lru_map_update_elem()
1209 hash = htab_map_hash(key, key_size, htab->hashrnd); in htab_lru_map_update_elem()
1212 head = &b->head; in htab_lru_map_update_elem()
1221 return -ENOMEM; in htab_lru_map_update_elem()
1222 copy_map_value(&htab->map, in htab_lru_map_update_elem()
1223 l_new->key + round_up(map->key_size, 8), value); in htab_lru_map_update_elem()
1238 hlist_nulls_add_head_rcu(&l_new->hash_node, head); in htab_lru_map_update_elem()
1240 bpf_lru_node_set_ref(&l_new->lru_node); in htab_lru_map_update_elem()
1241 hlist_nulls_del_rcu(&l_old->hash_node); in htab_lru_map_update_elem()
1257 static long __htab_percpu_map_update_elem(struct bpf_map *map, void *key, in __htab_percpu_map_update_elem() argument
1261 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in __htab_percpu_map_update_elem()
1271 return -EINVAL; in __htab_percpu_map_update_elem()
1276 key_size = map->key_size; in __htab_percpu_map_update_elem()
1278 hash = htab_map_hash(key, key_size, htab->hashrnd); in __htab_percpu_map_update_elem()
1281 head = &b->head; in __htab_percpu_map_update_elem()
1294 /* per-cpu hash map can update value in-place */ in __htab_percpu_map_update_elem()
1304 hlist_nulls_add_head_rcu(&l_new->hash_node, head); in __htab_percpu_map_update_elem()
1312 static long __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key, in __htab_lru_percpu_map_update_elem() argument
1316 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in __htab_lru_percpu_map_update_elem()
1326 return -EINVAL; in __htab_lru_percpu_map_update_elem()
1331 key_size = map->key_size; in __htab_lru_percpu_map_update_elem()
1333 hash = htab_map_hash(key, key_size, htab->hashrnd); in __htab_lru_percpu_map_update_elem()
1336 head = &b->head; in __htab_lru_percpu_map_update_elem()
1346 return -ENOMEM; in __htab_lru_percpu_map_update_elem()
1360 bpf_lru_node_set_ref(&l_old->lru_node); in __htab_lru_percpu_map_update_elem()
1362 /* per-cpu hash map can update value in-place */ in __htab_lru_percpu_map_update_elem()
1368 hlist_nulls_add_head_rcu(&l_new->hash_node, head); in __htab_lru_percpu_map_update_elem()
1376 bpf_map_dec_elem_count(&htab->map); in __htab_lru_percpu_map_update_elem()
1377 bpf_lru_push_free(&htab->lru, &l_new->lru_node); in __htab_lru_percpu_map_update_elem()
1382 static long htab_percpu_map_update_elem(struct bpf_map *map, void *key, in htab_percpu_map_update_elem() argument
1385 return __htab_percpu_map_update_elem(map, key, value, map_flags, false); in htab_percpu_map_update_elem()
1388 static long htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key, in htab_lru_percpu_map_update_elem() argument
1391 return __htab_lru_percpu_map_update_elem(map, key, value, map_flags, in htab_lru_percpu_map_update_elem()
1396 static long htab_map_delete_elem(struct bpf_map *map, void *key) in htab_map_delete_elem() argument
1398 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_delete_elem()
1409 key_size = map->key_size; in htab_map_delete_elem()
1411 hash = htab_map_hash(key, key_size, htab->hashrnd); in htab_map_delete_elem()
1413 head = &b->head; in htab_map_delete_elem()
1422 hlist_nulls_del_rcu(&l->hash_node); in htab_map_delete_elem()
1425 ret = -ENOENT; in htab_map_delete_elem()
1432 static long htab_lru_map_delete_elem(struct bpf_map *map, void *key) in htab_lru_map_delete_elem() argument
1434 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_lru_map_delete_elem()
1445 key_size = map->key_size; in htab_lru_map_delete_elem()
1447 hash = htab_map_hash(key, key_size, htab->hashrnd); in htab_lru_map_delete_elem()
1449 head = &b->head; in htab_lru_map_delete_elem()
1458 hlist_nulls_del_rcu(&l->hash_node); in htab_lru_map_delete_elem()
1460 ret = -ENOENT; in htab_lru_map_delete_elem()
1476 for (i = 0; i < htab->n_buckets; i++) { in delete_all_elements()
1482 hlist_nulls_del_rcu(&l->hash_node); in delete_all_elements()
1494 for (i = 0; i < htab->n_buckets; i++) { in htab_free_malloced_timers()
1501 bpf_obj_free_timer(htab->map.record, l->key + round_up(htab->map.key_size, 8)); in htab_free_malloced_timers()
1508 static void htab_map_free_timers(struct bpf_map *map) in htab_map_free_timers() argument
1510 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_free_timers()
1513 if (!btf_record_has_field(htab->map.record, BPF_TIMER)) in htab_map_free_timers()
1521 /* Called when map->refcnt goes to zero, either from workqueue or from syscall */
1522 static void htab_map_free(struct bpf_map *map) in htab_map_free() argument
1524 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_free()
1529 * There is no need to synchronize_rcu() here to protect map elements. in htab_map_free()
1543 bpf_map_free_elem_count(map); in htab_map_free()
1544 free_percpu(htab->extra_elems); in htab_map_free()
1545 bpf_map_area_free(htab->buckets); in htab_map_free()
1546 bpf_mem_alloc_destroy(&htab->pcpu_ma); in htab_map_free()
1547 bpf_mem_alloc_destroy(&htab->ma); in htab_map_free()
1548 if (htab->use_percpu_counter) in htab_map_free()
1549 percpu_counter_destroy(&htab->pcount); in htab_map_free()
1551 free_percpu(htab->map_locked[i]); in htab_map_free()
1552 lockdep_unregister_key(&htab->lockdep_key); in htab_map_free()
1556 static void htab_map_seq_show_elem(struct bpf_map *map, void *key, in htab_map_seq_show_elem() argument
1563 value = htab_map_lookup_elem(map, key); in htab_map_seq_show_elem()
1569 btf_type_seq_show(map->btf, map->btf_key_type_id, key, m); in htab_map_seq_show_elem()
1571 btf_type_seq_show(map->btf, map->btf_value_type_id, value, m); in htab_map_seq_show_elem()
1577 static int __htab_map_lookup_and_delete_elem(struct bpf_map *map, void *key, in __htab_map_lookup_and_delete_elem() argument
1581 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in __htab_map_lookup_and_delete_elem()
1589 key_size = map->key_size; in __htab_map_lookup_and_delete_elem()
1591 hash = htab_map_hash(key, key_size, htab->hashrnd); in __htab_map_lookup_and_delete_elem()
1593 head = &b->head; in __htab_map_lookup_and_delete_elem()
1601 ret = -ENOENT; in __htab_map_lookup_and_delete_elem()
1604 u32 roundup_value_size = round_up(map->value_size, 8); in __htab_map_lookup_and_delete_elem()
1606 int off = 0, cpu; in __htab_map_lookup_and_delete_elem() local
1609 for_each_possible_cpu(cpu) { in __htab_map_lookup_and_delete_elem()
1610 copy_map_value_long(&htab->map, value + off, per_cpu_ptr(pptr, cpu)); in __htab_map_lookup_and_delete_elem()
1611 check_and_init_map_value(&htab->map, value + off); in __htab_map_lookup_and_delete_elem()
1615 u32 roundup_key_size = round_up(map->key_size, 8); in __htab_map_lookup_and_delete_elem()
1618 copy_map_value_locked(map, value, l->key + in __htab_map_lookup_and_delete_elem()
1622 copy_map_value(map, value, l->key + in __htab_map_lookup_and_delete_elem()
1625 check_and_init_map_value(map, value); in __htab_map_lookup_and_delete_elem()
1628 hlist_nulls_del_rcu(&l->hash_node); in __htab_map_lookup_and_delete_elem()
1641 static int htab_map_lookup_and_delete_elem(struct bpf_map *map, void *key, in htab_map_lookup_and_delete_elem() argument
1644 return __htab_map_lookup_and_delete_elem(map, key, value, false, false, in htab_map_lookup_and_delete_elem()
1648 static int htab_percpu_map_lookup_and_delete_elem(struct bpf_map *map, in htab_percpu_map_lookup_and_delete_elem() argument
1652 return __htab_map_lookup_and_delete_elem(map, key, value, false, true, in htab_percpu_map_lookup_and_delete_elem()
1656 static int htab_lru_map_lookup_and_delete_elem(struct bpf_map *map, void *key, in htab_lru_map_lookup_and_delete_elem() argument
1659 return __htab_map_lookup_and_delete_elem(map, key, value, true, false, in htab_lru_map_lookup_and_delete_elem()
1663 static int htab_lru_percpu_map_lookup_and_delete_elem(struct bpf_map *map, in htab_lru_percpu_map_lookup_and_delete_elem() argument
1667 return __htab_map_lookup_and_delete_elem(map, key, value, true, true, in htab_lru_percpu_map_lookup_and_delete_elem()
1672 __htab_map_lookup_and_delete_batch(struct bpf_map *map, in __htab_map_lookup_and_delete_batch() argument
1678 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in __htab_map_lookup_and_delete_batch()
1681 void __user *uvalues = u64_to_user_ptr(attr->batch.values); in __htab_map_lookup_and_delete_batch()
1682 void __user *ukeys = u64_to_user_ptr(attr->batch.keys); in __htab_map_lookup_and_delete_batch()
1683 void __user *ubatch = u64_to_user_ptr(attr->batch.in_batch); in __htab_map_lookup_and_delete_batch()
1695 elem_map_flags = attr->batch.elem_flags; in __htab_map_lookup_and_delete_batch()
1697 ((elem_map_flags & BPF_F_LOCK) && !btf_record_has_field(map->record, BPF_SPIN_LOCK))) in __htab_map_lookup_and_delete_batch()
1698 return -EINVAL; in __htab_map_lookup_and_delete_batch()
1700 map_flags = attr->batch.flags; in __htab_map_lookup_and_delete_batch()
1702 return -EINVAL; in __htab_map_lookup_and_delete_batch()
1704 max_count = attr->batch.count; in __htab_map_lookup_and_delete_batch()
1708 if (put_user(0, &uattr->batch.count)) in __htab_map_lookup_and_delete_batch()
1709 return -EFAULT; in __htab_map_lookup_and_delete_batch()
1713 return -EFAULT; in __htab_map_lookup_and_delete_batch()
1715 if (batch >= htab->n_buckets) in __htab_map_lookup_and_delete_batch()
1716 return -ENOENT; in __htab_map_lookup_and_delete_batch()
1718 key_size = htab->map.key_size; in __htab_map_lookup_and_delete_batch()
1719 roundup_key_size = round_up(htab->map.key_size, 8); in __htab_map_lookup_and_delete_batch()
1720 value_size = htab->map.value_size; in __htab_map_lookup_and_delete_batch()
1737 ret = -ENOMEM; in __htab_map_lookup_and_delete_batch()
1747 b = &htab->buckets[batch]; in __htab_map_lookup_and_delete_batch()
1748 head = &b->head; in __htab_map_lookup_and_delete_batch()
1768 if (bucket_cnt > (max_count - total)) { in __htab_map_lookup_and_delete_batch()
1770 ret = -ENOSPC; in __htab_map_lookup_and_delete_batch()
1798 memcpy(dst_key, l->key, key_size); in __htab_map_lookup_and_delete_batch()
1801 int off = 0, cpu; in __htab_map_lookup_and_delete_batch() local
1804 pptr = htab_elem_get_ptr(l, map->key_size); in __htab_map_lookup_and_delete_batch()
1805 for_each_possible_cpu(cpu) { in __htab_map_lookup_and_delete_batch()
1806 copy_map_value_long(&htab->map, dst_val + off, per_cpu_ptr(pptr, cpu)); in __htab_map_lookup_and_delete_batch()
1807 check_and_init_map_value(&htab->map, dst_val + off); in __htab_map_lookup_and_delete_batch()
1811 value = l->key + roundup_key_size; in __htab_map_lookup_and_delete_batch()
1812 if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) { in __htab_map_lookup_and_delete_batch()
1815 /* Actual value is the id of the inner map */ in __htab_map_lookup_and_delete_batch()
1816 map_id = map->ops->map_fd_sys_lookup_elem(*inner_map); in __htab_map_lookup_and_delete_batch()
1821 copy_map_value_locked(map, dst_val, value, in __htab_map_lookup_and_delete_batch()
1824 copy_map_value(map, dst_val, value); in __htab_map_lookup_and_delete_batch()
1826 check_and_init_map_value(map, dst_val); in __htab_map_lookup_and_delete_batch()
1829 hlist_nulls_del_rcu(&l->hash_node); in __htab_map_lookup_and_delete_batch()
1837 l->batch_flink = node_to_free; in __htab_map_lookup_and_delete_batch()
1852 node_to_free = node_to_free->batch_flink; in __htab_map_lookup_and_delete_batch()
1860 if (!bucket_cnt && (batch + 1 < htab->n_buckets)) { in __htab_map_lookup_and_delete_batch()
1871 ret = -EFAULT; in __htab_map_lookup_and_delete_batch()
1877 if (batch >= htab->n_buckets) { in __htab_map_lookup_and_delete_batch()
1878 ret = -ENOENT; in __htab_map_lookup_and_delete_batch()
1884 if (ret == -EFAULT) in __htab_map_lookup_and_delete_batch()
1888 ubatch = u64_to_user_ptr(attr->batch.out_batch); in __htab_map_lookup_and_delete_batch()
1890 put_user(total, &uattr->batch.count)) in __htab_map_lookup_and_delete_batch()
1891 ret = -EFAULT; in __htab_map_lookup_and_delete_batch()
1900 htab_percpu_map_lookup_batch(struct bpf_map *map, const union bpf_attr *attr, in htab_percpu_map_lookup_batch() argument
1903 return __htab_map_lookup_and_delete_batch(map, attr, uattr, false, in htab_percpu_map_lookup_batch()
1908 htab_percpu_map_lookup_and_delete_batch(struct bpf_map *map, in htab_percpu_map_lookup_and_delete_batch() argument
1912 return __htab_map_lookup_and_delete_batch(map, attr, uattr, true, in htab_percpu_map_lookup_and_delete_batch()
1917 htab_map_lookup_batch(struct bpf_map *map, const union bpf_attr *attr, in htab_map_lookup_batch() argument
1920 return __htab_map_lookup_and_delete_batch(map, attr, uattr, false, in htab_map_lookup_batch()
1925 htab_map_lookup_and_delete_batch(struct bpf_map *map, in htab_map_lookup_and_delete_batch() argument
1929 return __htab_map_lookup_and_delete_batch(map, attr, uattr, true, in htab_map_lookup_and_delete_batch()
1934 htab_lru_percpu_map_lookup_batch(struct bpf_map *map, in htab_lru_percpu_map_lookup_batch() argument
1938 return __htab_map_lookup_and_delete_batch(map, attr, uattr, false, in htab_lru_percpu_map_lookup_batch()
1943 htab_lru_percpu_map_lookup_and_delete_batch(struct bpf_map *map, in htab_lru_percpu_map_lookup_and_delete_batch() argument
1947 return __htab_map_lookup_and_delete_batch(map, attr, uattr, true, in htab_lru_percpu_map_lookup_and_delete_batch()
1952 htab_lru_map_lookup_batch(struct bpf_map *map, const union bpf_attr *attr, in htab_lru_map_lookup_batch() argument
1955 return __htab_map_lookup_and_delete_batch(map, attr, uattr, false, in htab_lru_map_lookup_batch()
1960 htab_lru_map_lookup_and_delete_batch(struct bpf_map *map, in htab_lru_map_lookup_and_delete_batch() argument
1964 return __htab_map_lookup_and_delete_batch(map, attr, uattr, true, in htab_lru_map_lookup_and_delete_batch()
1969 struct bpf_map *map; member
1971 void *percpu_value_buf; // non-zero means percpu hash
1980 const struct bpf_htab *htab = info->htab; in bpf_hash_map_seq_find_next()
1981 u32 skip_elems = info->skip_elems; in bpf_hash_map_seq_find_next()
1982 u32 bucket_id = info->bucket_id; in bpf_hash_map_seq_find_next()
1989 if (bucket_id >= htab->n_buckets) in bpf_hash_map_seq_find_next()
1997 n = rcu_dereference_raw(hlist_nulls_next_rcu(&prev_elem->hash_node)); in bpf_hash_map_seq_find_next()
2003 b = &htab->buckets[bucket_id++]; in bpf_hash_map_seq_find_next()
2008 for (i = bucket_id; i < htab->n_buckets; i++) { in bpf_hash_map_seq_find_next()
2009 b = &htab->buckets[i]; in bpf_hash_map_seq_find_next()
2013 head = &b->head; in bpf_hash_map_seq_find_next()
2016 info->bucket_id = i; in bpf_hash_map_seq_find_next()
2017 info->skip_elems = count; in bpf_hash_map_seq_find_next()
2027 info->bucket_id = i; in bpf_hash_map_seq_find_next()
2028 info->skip_elems = 0; in bpf_hash_map_seq_find_next()
2034 struct bpf_iter_seq_hash_map_info *info = seq->private; in bpf_hash_map_seq_start()
2048 struct bpf_iter_seq_hash_map_info *info = seq->private; in bpf_hash_map_seq_next()
2051 ++info->skip_elems; in bpf_hash_map_seq_next()
2057 struct bpf_iter_seq_hash_map_info *info = seq->private; in __bpf_hash_map_seq_show()
2060 struct bpf_map *map = info->map; in __bpf_hash_map_seq_show() local
2062 int ret = 0, off = 0, cpu; in __bpf_hash_map_seq_show() local
2070 ctx.map = info->map; in __bpf_hash_map_seq_show()
2072 roundup_key_size = round_up(map->key_size, 8); in __bpf_hash_map_seq_show()
2073 ctx.key = elem->key; in __bpf_hash_map_seq_show()
2074 if (!info->percpu_value_buf) { in __bpf_hash_map_seq_show()
2075 ctx.value = elem->key + roundup_key_size; in __bpf_hash_map_seq_show()
2077 roundup_value_size = round_up(map->value_size, 8); in __bpf_hash_map_seq_show()
2078 pptr = htab_elem_get_ptr(elem, map->key_size); in __bpf_hash_map_seq_show()
2079 for_each_possible_cpu(cpu) { in __bpf_hash_map_seq_show()
2080 copy_map_value_long(map, info->percpu_value_buf + off, in __bpf_hash_map_seq_show()
2081 per_cpu_ptr(pptr, cpu)); in __bpf_hash_map_seq_show()
2082 check_and_init_map_value(map, info->percpu_value_buf + off); in __bpf_hash_map_seq_show()
2085 ctx.value = info->percpu_value_buf; in __bpf_hash_map_seq_show()
2111 struct bpf_map *map = aux->map; in bpf_iter_init_hash_map() local
2115 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || in bpf_iter_init_hash_map()
2116 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { in bpf_iter_init_hash_map()
2117 buf_size = round_up(map->value_size, 8) * num_possible_cpus(); in bpf_iter_init_hash_map()
2120 return -ENOMEM; in bpf_iter_init_hash_map()
2122 seq_info->percpu_value_buf = value_buf; in bpf_iter_init_hash_map()
2125 bpf_map_inc_with_uref(map); in bpf_iter_init_hash_map()
2126 seq_info->map = map; in bpf_iter_init_hash_map()
2127 seq_info->htab = container_of(map, struct bpf_htab, map); in bpf_iter_init_hash_map()
2135 bpf_map_put_with_uref(seq_info->map); in bpf_iter_fini_hash_map()
2136 kfree(seq_info->percpu_value_buf); in bpf_iter_fini_hash_map()
2153 static long bpf_for_each_hash_elem(struct bpf_map *map, bpf_callback_t callback_fn, in bpf_for_each_hash_elem() argument
2156 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in bpf_for_each_hash_elem()
2169 return -EINVAL; in bpf_for_each_hash_elem()
2173 roundup_key_size = round_up(map->key_size, 8); in bpf_for_each_hash_elem()
2179 for (i = 0; i < htab->n_buckets; i++) { in bpf_for_each_hash_elem()
2180 b = &htab->buckets[i]; in bpf_for_each_hash_elem()
2182 head = &b->head; in bpf_for_each_hash_elem()
2184 key = elem->key; in bpf_for_each_hash_elem()
2186 /* current cpu value for percpu map */ in bpf_for_each_hash_elem()
2187 pptr = htab_elem_get_ptr(elem, map->key_size); in bpf_for_each_hash_elem()
2190 val = elem->key + roundup_key_size; in bpf_for_each_hash_elem()
2193 ret = callback_fn((u64)(long)map, (u64)(long)key, in bpf_for_each_hash_elem()
2195 /* return value: 0 - continue, 1 - stop and return */ in bpf_for_each_hash_elem()
2209 static u64 htab_map_mem_usage(const struct bpf_map *map) in htab_map_mem_usage() argument
2211 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_mem_usage()
2212 u32 value_size = round_up(htab->map.value_size, 8); in htab_map_mem_usage()
2219 usage += sizeof(struct bucket) * htab->n_buckets; in htab_map_mem_usage()
2222 num_entries = map->max_entries; in htab_map_mem_usage()
2226 usage += htab->elem_size * num_entries; in htab_map_mem_usage()
2235 num_entries = htab->use_percpu_counter ? in htab_map_mem_usage()
2236 percpu_counter_sum(&htab->pcount) : in htab_map_mem_usage()
2237 atomic_read(&htab->count); in htab_map_mem_usage()
2238 usage += (htab->elem_size + LLIST_NODE_SZ) * num_entries; in htab_map_mem_usage()
2292 static void *htab_percpu_map_lookup_elem(struct bpf_map *map, void *key) in htab_percpu_map_lookup_elem() argument
2294 struct htab_elem *l = __htab_map_lookup_elem(map, key); in htab_percpu_map_lookup_elem()
2297 return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size)); in htab_percpu_map_lookup_elem()
2302 static void *htab_percpu_map_lookup_percpu_elem(struct bpf_map *map, void *key, u32 cpu) in htab_percpu_map_lookup_percpu_elem() argument
2306 if (cpu >= nr_cpu_ids) in htab_percpu_map_lookup_percpu_elem()
2309 l = __htab_map_lookup_elem(map, key); in htab_percpu_map_lookup_percpu_elem()
2311 return per_cpu_ptr(htab_elem_get_ptr(l, map->key_size), cpu); in htab_percpu_map_lookup_percpu_elem()
2316 static void *htab_lru_percpu_map_lookup_elem(struct bpf_map *map, void *key) in htab_lru_percpu_map_lookup_elem() argument
2318 struct htab_elem *l = __htab_map_lookup_elem(map, key); in htab_lru_percpu_map_lookup_elem()
2321 bpf_lru_node_set_ref(&l->lru_node); in htab_lru_percpu_map_lookup_elem()
2322 return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size)); in htab_lru_percpu_map_lookup_elem()
2328 static void *htab_lru_percpu_map_lookup_percpu_elem(struct bpf_map *map, void *key, u32 cpu) in htab_lru_percpu_map_lookup_percpu_elem() argument
2332 if (cpu >= nr_cpu_ids) in htab_lru_percpu_map_lookup_percpu_elem()
2335 l = __htab_map_lookup_elem(map, key); in htab_lru_percpu_map_lookup_percpu_elem()
2337 bpf_lru_node_set_ref(&l->lru_node); in htab_lru_percpu_map_lookup_percpu_elem()
2338 return per_cpu_ptr(htab_elem_get_ptr(l, map->key_size), cpu); in htab_lru_percpu_map_lookup_percpu_elem()
2344 int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value) in bpf_percpu_hash_copy() argument
2348 int ret = -ENOENT; in bpf_percpu_hash_copy()
2349 int cpu, off = 0; in bpf_percpu_hash_copy() local
2352 /* per_cpu areas are zero-filled and bpf programs can only in bpf_percpu_hash_copy()
2356 size = round_up(map->value_size, 8); in bpf_percpu_hash_copy()
2358 l = __htab_map_lookup_elem(map, key); in bpf_percpu_hash_copy()
2361 /* We do not mark LRU map element here in order to not mess up in bpf_percpu_hash_copy()
2362 * eviction heuristics when user space does a map walk. in bpf_percpu_hash_copy()
2364 pptr = htab_elem_get_ptr(l, map->key_size); in bpf_percpu_hash_copy()
2365 for_each_possible_cpu(cpu) { in bpf_percpu_hash_copy()
2366 copy_map_value_long(map, value + off, per_cpu_ptr(pptr, cpu)); in bpf_percpu_hash_copy()
2367 check_and_init_map_value(map, value + off); in bpf_percpu_hash_copy()
2376 int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value, in bpf_percpu_hash_update() argument
2379 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in bpf_percpu_hash_update()
2384 ret = __htab_lru_percpu_map_update_elem(map, key, value, in bpf_percpu_hash_update()
2387 ret = __htab_percpu_map_update_elem(map, key, value, map_flags, in bpf_percpu_hash_update()
2394 static void htab_percpu_map_seq_show_elem(struct bpf_map *map, void *key, in htab_percpu_map_seq_show_elem() argument
2399 int cpu; in htab_percpu_map_seq_show_elem() local
2403 l = __htab_map_lookup_elem(map, key); in htab_percpu_map_seq_show_elem()
2409 btf_type_seq_show(map->btf, map->btf_key_type_id, key, m); in htab_percpu_map_seq_show_elem()
2411 pptr = htab_elem_get_ptr(l, map->key_size); in htab_percpu_map_seq_show_elem()
2412 for_each_possible_cpu(cpu) { in htab_percpu_map_seq_show_elem()
2413 seq_printf(m, "\tcpu%d: ", cpu); in htab_percpu_map_seq_show_elem()
2414 btf_type_seq_show(map->btf, map->btf_value_type_id, in htab_percpu_map_seq_show_elem()
2415 per_cpu_ptr(pptr, cpu), m); in htab_percpu_map_seq_show_elem()
2465 if (attr->value_size != sizeof(u32)) in fd_htab_map_alloc_check()
2466 return -EINVAL; in fd_htab_map_alloc_check()
2470 static void fd_htab_map_free(struct bpf_map *map) in fd_htab_map_free() argument
2472 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in fd_htab_map_free()
2478 for (i = 0; i < htab->n_buckets; i++) { in fd_htab_map_free()
2482 void *ptr = fd_htab_map_get_ptr(map, l); in fd_htab_map_free()
2484 map->ops->map_fd_put_ptr(ptr); in fd_htab_map_free()
2488 htab_map_free(map); in fd_htab_map_free()
2492 int bpf_fd_htab_map_lookup_elem(struct bpf_map *map, void *key, u32 *value) in bpf_fd_htab_map_lookup_elem() argument
2497 if (!map->ops->map_fd_sys_lookup_elem) in bpf_fd_htab_map_lookup_elem()
2498 return -ENOTSUPP; in bpf_fd_htab_map_lookup_elem()
2501 ptr = htab_map_lookup_elem(map, key); in bpf_fd_htab_map_lookup_elem()
2503 *value = map->ops->map_fd_sys_lookup_elem(READ_ONCE(*ptr)); in bpf_fd_htab_map_lookup_elem()
2505 ret = -ENOENT; in bpf_fd_htab_map_lookup_elem()
2512 int bpf_fd_htab_map_update_elem(struct bpf_map *map, struct file *map_file, in bpf_fd_htab_map_update_elem() argument
2519 ptr = map->ops->map_fd_get_ptr(map, map_file, ufd); in bpf_fd_htab_map_update_elem()
2523 ret = htab_map_update_elem(map, key, &ptr, map_flags); in bpf_fd_htab_map_update_elem()
2525 map->ops->map_fd_put_ptr(ptr); in bpf_fd_htab_map_update_elem()
2532 struct bpf_map *map, *inner_map_meta; in htab_of_map_alloc() local
2534 inner_map_meta = bpf_map_meta_alloc(attr->inner_map_fd); in htab_of_map_alloc()
2538 map = htab_map_alloc(attr); in htab_of_map_alloc()
2539 if (IS_ERR(map)) { in htab_of_map_alloc()
2541 return map; in htab_of_map_alloc()
2544 map->inner_map_meta = inner_map_meta; in htab_of_map_alloc()
2546 return map; in htab_of_map_alloc()
2549 static void *htab_of_map_lookup_elem(struct bpf_map *map, void *key) in htab_of_map_lookup_elem() argument
2551 struct bpf_map **inner_map = htab_map_lookup_elem(map, key); in htab_of_map_lookup_elem()
2559 static int htab_of_map_gen_lookup(struct bpf_map *map, in htab_of_map_gen_lookup() argument
2566 (void *(*)(struct bpf_map *map, void *key))NULL)); in htab_of_map_gen_lookup()
2571 round_up(map->key_size, 8)); in htab_of_map_gen_lookup()
2574 return insn - insn_buf; in htab_of_map_gen_lookup()
2577 static void htab_of_map_free(struct bpf_map *map) in htab_of_map_free() argument
2579 bpf_map_meta_free(map->inner_map_meta); in htab_of_map_free()
2580 fd_htab_map_free(map); in htab_of_map_free()