Lines Matching refs:ht
37 static u32 head_hashfn(struct rhashtable *ht, in head_hashfn() argument
41 return rht_head_hashfn(ht, tbl, he, ht->p); in head_hashfn()
47 int lockdep_rht_mutex_is_held(struct rhashtable *ht) in lockdep_rht_mutex_is_held() argument
49 return (debug_locks) ? lockdep_is_held(&ht->mutex) : 1; in lockdep_rht_mutex_is_held()
113 static union nested_table *nested_table_alloc(struct rhashtable *ht, in nested_table_alloc() argument
138 static struct bucket_table *nested_bucket_table_alloc(struct rhashtable *ht, in nested_bucket_table_alloc() argument
155 if (!nested_table_alloc(ht, (union nested_table __rcu **)tbl->buckets, in nested_bucket_table_alloc()
166 static struct bucket_table *bucket_table_alloc(struct rhashtable *ht, in bucket_table_alloc() argument
180 tbl = nested_bucket_table_alloc(ht, nbuckets, gfp); in bucket_table_alloc()
202 static struct bucket_table *rhashtable_last_table(struct rhashtable *ht, in rhashtable_last_table() argument
209 tbl = rht_dereference_rcu(tbl->future_tbl, ht); in rhashtable_last_table()
215 static int rhashtable_rehash_one(struct rhashtable *ht, in rhashtable_rehash_one() argument
219 struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht); in rhashtable_rehash_one()
220 struct bucket_table *new_tbl = rhashtable_last_table(ht, old_tbl); in rhashtable_rehash_one()
245 new_hash = head_hashfn(ht, new_tbl, entry); in rhashtable_rehash_one()
265 static int rhashtable_rehash_chain(struct rhashtable *ht, in rhashtable_rehash_chain() argument
268 struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht); in rhashtable_rehash_chain()
276 while (!(err = rhashtable_rehash_one(ht, bkt, old_hash))) in rhashtable_rehash_chain()
286 static int rhashtable_rehash_attach(struct rhashtable *ht, in rhashtable_rehash_attach() argument
303 static int rhashtable_rehash_table(struct rhashtable *ht) in rhashtable_rehash_table() argument
305 struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht); in rhashtable_rehash_table()
311 new_tbl = rht_dereference(old_tbl->future_tbl, ht); in rhashtable_rehash_table()
316 err = rhashtable_rehash_chain(ht, old_hash); in rhashtable_rehash_table()
323 rcu_assign_pointer(ht->tbl, new_tbl); in rhashtable_rehash_table()
325 spin_lock(&ht->lock); in rhashtable_rehash_table()
337 spin_unlock(&ht->lock); in rhashtable_rehash_table()
339 return rht_dereference(new_tbl->future_tbl, ht) ? -EAGAIN : 0; in rhashtable_rehash_table()
342 static int rhashtable_rehash_alloc(struct rhashtable *ht, in rhashtable_rehash_alloc() argument
349 ASSERT_RHT_MUTEX(ht); in rhashtable_rehash_alloc()
351 new_tbl = bucket_table_alloc(ht, size, GFP_KERNEL); in rhashtable_rehash_alloc()
355 err = rhashtable_rehash_attach(ht, old_tbl, new_tbl); in rhashtable_rehash_alloc()
378 static int rhashtable_shrink(struct rhashtable *ht) in rhashtable_shrink() argument
380 struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht); in rhashtable_shrink()
381 unsigned int nelems = atomic_read(&ht->nelems); in rhashtable_shrink()
386 if (size < ht->p.min_size) in rhashtable_shrink()
387 size = ht->p.min_size; in rhashtable_shrink()
392 if (rht_dereference(old_tbl->future_tbl, ht)) in rhashtable_shrink()
395 return rhashtable_rehash_alloc(ht, old_tbl, size); in rhashtable_shrink()
400 struct rhashtable *ht; in rht_deferred_worker() local
404 ht = container_of(work, struct rhashtable, run_work); in rht_deferred_worker()
405 mutex_lock(&ht->mutex); in rht_deferred_worker()
407 tbl = rht_dereference(ht->tbl, ht); in rht_deferred_worker()
408 tbl = rhashtable_last_table(ht, tbl); in rht_deferred_worker()
410 if (rht_grow_above_75(ht, tbl)) in rht_deferred_worker()
411 err = rhashtable_rehash_alloc(ht, tbl, tbl->size * 2); in rht_deferred_worker()
412 else if (ht->p.automatic_shrinking && rht_shrink_below_30(ht, tbl)) in rht_deferred_worker()
413 err = rhashtable_shrink(ht); in rht_deferred_worker()
415 err = rhashtable_rehash_alloc(ht, tbl, tbl->size); in rht_deferred_worker()
420 nerr = rhashtable_rehash_table(ht); in rht_deferred_worker()
424 mutex_unlock(&ht->mutex); in rht_deferred_worker()
427 schedule_work(&ht->run_work); in rht_deferred_worker()
430 static int rhashtable_insert_rehash(struct rhashtable *ht, in rhashtable_insert_rehash() argument
438 old_tbl = rht_dereference_rcu(ht->tbl, ht); in rhashtable_insert_rehash()
444 if (rht_grow_above_75(ht, tbl)) in rhashtable_insert_rehash()
452 new_tbl = bucket_table_alloc(ht, size, GFP_ATOMIC | __GFP_NOWARN); in rhashtable_insert_rehash()
456 err = rhashtable_rehash_attach(ht, tbl, new_tbl); in rhashtable_insert_rehash()
462 schedule_work(&ht->run_work); in rhashtable_insert_rehash()
473 schedule_work(&ht->run_work); in rhashtable_insert_rehash()
478 static void *rhashtable_lookup_one(struct rhashtable *ht, in rhashtable_lookup_one() argument
484 .ht = ht, in rhashtable_lookup_one()
498 (ht->p.obj_cmpfn ? in rhashtable_lookup_one()
499 ht->p.obj_cmpfn(&arg, rht_obj(ht, head)) : in rhashtable_lookup_one()
500 rhashtable_compare(&arg, rht_obj(ht, head)))) { in rhashtable_lookup_one()
505 if (!ht->rhlist) in rhashtable_lookup_one()
506 return rht_obj(ht, head); in rhashtable_lookup_one()
529 static struct bucket_table *rhashtable_insert_one(struct rhashtable *ht, in rhashtable_insert_one() argument
545 new_tbl = rht_dereference_rcu(tbl->future_tbl, ht); in rhashtable_insert_one()
552 if (unlikely(rht_grow_above_max(ht, tbl))) in rhashtable_insert_one()
555 if (unlikely(rht_grow_above_100(ht, tbl))) in rhashtable_insert_one()
561 if (ht->rhlist) { in rhashtable_insert_one()
573 atomic_inc(&ht->nelems); in rhashtable_insert_one()
574 if (rht_grow_above_75(ht, tbl)) in rhashtable_insert_one()
575 schedule_work(&ht->run_work); in rhashtable_insert_one()
580 static void *rhashtable_try_insert(struct rhashtable *ht, const void *key, in rhashtable_try_insert() argument
589 new_tbl = rcu_dereference(ht->tbl); in rhashtable_try_insert()
593 hash = rht_head_hashfn(ht, tbl, obj, ht->p); in rhashtable_try_insert()
598 bkt = rht_bucket_insert(ht, tbl, hash); in rhashtable_try_insert()
600 new_tbl = rht_dereference_rcu(tbl->future_tbl, ht); in rhashtable_try_insert()
604 data = rhashtable_lookup_one(ht, bkt, tbl, in rhashtable_try_insert()
606 new_tbl = rhashtable_insert_one(ht, bkt, tbl, in rhashtable_try_insert()
616 data = ERR_PTR(rhashtable_insert_rehash(ht, tbl) ?: in rhashtable_try_insert()
622 void *rhashtable_insert_slow(struct rhashtable *ht, const void *key, in rhashtable_insert_slow() argument
629 data = rhashtable_try_insert(ht, key, obj); in rhashtable_insert_slow()
658 void rhashtable_walk_enter(struct rhashtable *ht, struct rhashtable_iter *iter) in rhashtable_walk_enter() argument
660 iter->ht = ht; in rhashtable_walk_enter()
666 spin_lock(&ht->lock); in rhashtable_walk_enter()
668 rcu_dereference_protected(ht->tbl, lockdep_is_held(&ht->lock)); in rhashtable_walk_enter()
670 spin_unlock(&ht->lock); in rhashtable_walk_enter()
682 spin_lock(&iter->ht->lock); in rhashtable_walk_exit()
685 spin_unlock(&iter->ht->lock); in rhashtable_walk_exit()
710 struct rhashtable *ht = iter->ht; in rhashtable_walk_start_check() local
711 bool rhlist = ht->rhlist; in rhashtable_walk_start_check()
715 spin_lock(&ht->lock); in rhashtable_walk_start_check()
718 spin_unlock(&ht->lock); in rhashtable_walk_start_check()
723 iter->walker.tbl = rht_dereference_rcu(ht->tbl, ht); in rhashtable_walk_start_check()
784 struct rhashtable *ht = iter->ht; in __rhashtable_walk_find_next() local
786 bool rhlist = ht->rhlist; in __rhashtable_walk_find_next()
817 return rht_obj(ht, rhlist ? &list->rhead : p); in __rhashtable_walk_find_next()
828 iter->walker.tbl = rht_dereference_rcu(tbl->future_tbl, ht); in __rhashtable_walk_find_next()
855 struct rhashtable *ht = iter->ht; in rhashtable_walk_next() local
857 bool rhlist = ht->rhlist; in rhashtable_walk_next()
868 return rht_obj(ht, rhlist ? &list->rhead : p); in rhashtable_walk_next()
894 struct rhashtable *ht = iter->ht; in rhashtable_walk_peek() local
898 return rht_obj(ht, ht->rhlist ? &list->rhead : p); in rhashtable_walk_peek()
926 struct rhashtable *ht; in rhashtable_walk_stop() local
932 ht = iter->ht; in rhashtable_walk_stop()
934 spin_lock(&ht->lock); in rhashtable_walk_stop()
940 spin_unlock(&ht->lock); in rhashtable_walk_stop()
1008 int rhashtable_init(struct rhashtable *ht, in rhashtable_init() argument
1018 memset(ht, 0, sizeof(*ht)); in rhashtable_init()
1019 mutex_init(&ht->mutex); in rhashtable_init()
1020 spin_lock_init(&ht->lock); in rhashtable_init()
1021 memcpy(&ht->p, params, sizeof(*params)); in rhashtable_init()
1024 ht->p.min_size = roundup_pow_of_two(params->min_size); in rhashtable_init()
1027 ht->max_elems = 1u << 31; in rhashtable_init()
1030 ht->p.max_size = rounddown_pow_of_two(params->max_size); in rhashtable_init()
1031 if (ht->p.max_size < ht->max_elems / 2) in rhashtable_init()
1032 ht->max_elems = ht->p.max_size * 2; in rhashtable_init()
1035 ht->p.min_size = max_t(u16, ht->p.min_size, HASH_MIN_SIZE); in rhashtable_init()
1037 size = rounded_hashtable_size(&ht->p); in rhashtable_init()
1039 ht->key_len = ht->p.key_len; in rhashtable_init()
1041 ht->p.hashfn = jhash; in rhashtable_init()
1043 if (!(ht->key_len & (sizeof(u32) - 1))) { in rhashtable_init()
1044 ht->key_len /= sizeof(u32); in rhashtable_init()
1045 ht->p.hashfn = rhashtable_jhash2; in rhashtable_init()
1054 tbl = bucket_table_alloc(ht, size, GFP_KERNEL); in rhashtable_init()
1056 size = max_t(u16, ht->p.min_size, HASH_MIN_SIZE); in rhashtable_init()
1057 tbl = bucket_table_alloc(ht, size, GFP_KERNEL | __GFP_NOFAIL); in rhashtable_init()
1060 atomic_set(&ht->nelems, 0); in rhashtable_init()
1062 RCU_INIT_POINTER(ht->tbl, tbl); in rhashtable_init()
1064 INIT_WORK(&ht->run_work, rht_deferred_worker); in rhashtable_init()
1083 err = rhashtable_init(&hlt->ht, params); in rhltable_init()
1084 hlt->ht.rhlist = true; in rhltable_init()
1089 static void rhashtable_free_one(struct rhashtable *ht, struct rhash_head *obj, in rhashtable_free_one() argument
1095 if (!ht->rhlist) { in rhashtable_free_one()
1096 free_fn(rht_obj(ht, obj), arg); in rhashtable_free_one()
1103 list = rht_dereference(list->next, ht); in rhashtable_free_one()
1104 free_fn(rht_obj(ht, obj), arg); in rhashtable_free_one()
1123 void rhashtable_free_and_destroy(struct rhashtable *ht, in rhashtable_free_and_destroy() argument
1130 cancel_work_sync(&ht->run_work); in rhashtable_free_and_destroy()
1132 mutex_lock(&ht->mutex); in rhashtable_free_and_destroy()
1133 tbl = rht_dereference(ht->tbl, ht); in rhashtable_free_and_destroy()
1142 rht_dereference(pos->next, ht) : NULL; in rhashtable_free_and_destroy()
1146 rht_dereference(pos->next, ht) : NULL) in rhashtable_free_and_destroy()
1147 rhashtable_free_one(ht, pos, free_fn, arg); in rhashtable_free_and_destroy()
1151 next_tbl = rht_dereference(tbl->future_tbl, ht); in rhashtable_free_and_destroy()
1157 mutex_unlock(&ht->mutex); in rhashtable_free_and_destroy()
1161 void rhashtable_destroy(struct rhashtable *ht) in rhashtable_destroy() argument
1163 return rhashtable_free_and_destroy(ht, NULL, NULL); in rhashtable_destroy()
1207 struct rhash_lock_head **rht_bucket_nested_insert(struct rhashtable *ht, in rht_bucket_nested_insert() argument
1218 ntbl = nested_table_alloc(ht, &ntbl[index].table, in rht_bucket_nested_insert()
1225 ntbl = nested_table_alloc(ht, &ntbl[index].table, in rht_bucket_nested_insert()