86 static inline void *rht_obj(const struct rhashtable *ht,  in rht_obj()  argument
89 	return (char *)he - ht->p.head_offset;  in rht_obj()
98 static inline unsigned int rht_key_get_hash(struct rhashtable *ht,  in rht_key_get_hash()  argument
106 		hash = ht->p.hashfn(key, ht->key_len, hash_rnd);  in rht_key_get_hash()
117 		unsigned int key_len = ht->p.key_len;  in rht_key_get_hash()
129 	struct rhashtable *ht, const struct bucket_table *tbl,  in rht_key_hashfn()  argument
132 	unsigned int hash = rht_key_get_hash(ht, key, params, tbl->hash_rnd);  in rht_key_hashfn()
138 	struct rhashtable *ht, const struct bucket_table *tbl,  in rht_head_hashfn()  argument
141 	const char *ptr = rht_obj(ht, he);  in rht_head_hashfn()
145 							    ht->p.key_len,  in rht_head_hashfn()
147 	       rht_key_hashfn(ht, tbl, ptr + params.key_offset, params);  in rht_head_hashfn()
155 static inline bool rht_grow_above_75(const struct rhashtable *ht,  in rht_grow_above_75()  argument
159 	return atomic_read(&ht->nelems) > (tbl->size / 4 * 3) &&  in rht_grow_above_75()
160 	       (!ht->p.max_size || tbl->size < ht->p.max_size);  in rht_grow_above_75()
168 static inline bool rht_shrink_below_30(const struct rhashtable *ht,  in rht_shrink_below_30()  argument
172 	return atomic_read(&ht->nelems) < (tbl->size * 3 / 10) &&  in rht_shrink_below_30()
173 	       tbl->size > ht->p.min_size;  in rht_shrink_below_30()
181 static inline bool rht_grow_above_100(const struct rhashtable *ht,  in rht_grow_above_100()  argument
184 	return atomic_read(&ht->nelems) > tbl->size &&  in rht_grow_above_100()
185 		(!ht->p.max_size || tbl->size < ht->p.max_size);  in rht_grow_above_100()
193 static inline bool rht_grow_above_max(const struct rhashtable *ht,  in rht_grow_above_max()  argument
196 	return atomic_read(&ht->nelems) >= ht->max_elems;  in rht_grow_above_max()
219 int lockdep_rht_mutex_is_held(struct rhashtable *ht);
222 static inline int lockdep_rht_mutex_is_held(struct rhashtable *ht)  in lockdep_rht_mutex_is_held()  argument
234 void *rhashtable_insert_slow(struct rhashtable *ht, const void *key,
237 void rhashtable_walk_enter(struct rhashtable *ht,
251 void rhashtable_free_and_destroy(struct rhashtable *ht,
254 void rhashtable_destroy(struct rhashtable *ht);
258 struct rhash_head __rcu **rht_bucket_nested_insert(struct rhashtable *ht,
262 #define rht_dereference(p, ht) \  argument
263 	rcu_dereference_protected(p, lockdep_rht_mutex_is_held(ht))
265 #define rht_dereference_rcu(p, ht) \  argument
266 	rcu_dereference_check(p, lockdep_rht_mutex_is_held(ht))
292 	struct rhashtable *ht, struct bucket_table *tbl, unsigned int hash)  in rht_bucket_insert()  argument
294 	return unlikely(tbl->nest) ? rht_bucket_nested_insert(ht, tbl, hash) :  in rht_bucket_insert()
459 	struct rhashtable *ht = arg->ht;  in rhashtable_compare()  local
462 	return memcmp(ptr + ht->p.key_offset, arg->key, ht->p.key_len);  in rhashtable_compare()
467 	struct rhashtable *ht, const void *key,  in __rhashtable_lookup()  argument
471 		.ht = ht,  in __rhashtable_lookup()
478 	tbl = rht_dereference_rcu(ht->tbl, ht);  in __rhashtable_lookup()
480 	hash = rht_key_hashfn(ht, tbl, key, params);  in __rhashtable_lookup()
483 		    params.obj_cmpfn(&arg, rht_obj(ht, he)) :  in __rhashtable_lookup()
484 		    rhashtable_compare(&arg, rht_obj(ht, he)))  in __rhashtable_lookup()
492 	tbl = rht_dereference_rcu(tbl->future_tbl, ht);  in __rhashtable_lookup()
513 	struct rhashtable *ht, const void *key,  in rhashtable_lookup()  argument
516 	struct rhash_head *he = __rhashtable_lookup(ht, key, params);  in rhashtable_lookup()
518 	return he ? rht_obj(ht, he) : NULL;  in rhashtable_lookup()
536 	struct rhashtable *ht, const void *key,  in rhashtable_lookup_fast()  argument
542 	obj = rhashtable_lookup(ht, key, params);  in rhashtable_lookup_fast()
566 	struct rhash_head *he = __rhashtable_lookup(&hlt->ht, key, params);  in rhltable_lookup()
576 	struct rhashtable *ht, const void *key, struct rhash_head *obj,  in __rhashtable_insert_fast()  argument
580 		.ht = ht,  in __rhashtable_insert_fast()
593 	tbl = rht_dereference_rcu(ht->tbl, ht);  in __rhashtable_insert_fast()
594 	hash = rht_head_hashfn(ht, tbl, obj, params);  in __rhashtable_insert_fast()
602 		return rhashtable_insert_slow(ht, key, obj);  in __rhashtable_insert_fast()
606 	pprev = rht_bucket_insert(ht, tbl, hash);  in __rhashtable_insert_fast()
618 		     params.obj_cmpfn(&arg, rht_obj(ht, head)) :  in __rhashtable_insert_fast()
619 		     rhashtable_compare(&arg, rht_obj(ht, head)))) {  in __rhashtable_insert_fast()
624 		data = rht_obj(ht, head);  in __rhashtable_insert_fast()
645 	if (unlikely(rht_grow_above_max(ht, tbl)))  in __rhashtable_insert_fast()
648 	if (unlikely(rht_grow_above_100(ht, tbl)))  in __rhashtable_insert_fast()
663 	atomic_inc(&ht->nelems);  in __rhashtable_insert_fast()
664 	if (rht_grow_above_75(ht, tbl))  in __rhashtable_insert_fast()
665 		schedule_work(&ht->run_work);  in __rhashtable_insert_fast()
693 	struct rhashtable *ht, struct rhash_head *obj,  in rhashtable_insert_fast()  argument
698 	ret = __rhashtable_insert_fast(ht, NULL, obj, params, false);  in rhashtable_insert_fast()
725 	return PTR_ERR(__rhashtable_insert_fast(&hlt->ht, key, &list->rhead,  in rhltable_insert_key()
748 	const char *key = rht_obj(&hlt->ht, &list->rhead);  in rhltable_insert()
776 	struct rhashtable *ht, struct rhash_head *obj,  in rhashtable_lookup_insert_fast()  argument
779 	const char *key = rht_obj(ht, obj);  in rhashtable_lookup_insert_fast()
782 	BUG_ON(ht->p.obj_hashfn);  in rhashtable_lookup_insert_fast()
784 	ret = __rhashtable_insert_fast(ht, key + ht->p.key_offset, obj, params,  in rhashtable_lookup_insert_fast()
803 	struct rhashtable *ht, struct rhash_head *obj,  in rhashtable_lookup_get_insert_fast()  argument
806 	const char *key = rht_obj(ht, obj);  in rhashtable_lookup_get_insert_fast()
808 	BUG_ON(ht->p.obj_hashfn);  in rhashtable_lookup_get_insert_fast()
810 	return __rhashtable_insert_fast(ht, key + ht->p.key_offset, obj, params,  in rhashtable_lookup_get_insert_fast()
836 	struct rhashtable *ht, const void *key, struct rhash_head *obj,  in rhashtable_lookup_insert_key()  argument
841 	BUG_ON(!ht->p.obj_hashfn || !key);  in rhashtable_lookup_insert_key()
843 	ret = __rhashtable_insert_fast(ht, key, obj, params, false);  in rhashtable_lookup_insert_key()
862 	struct rhashtable *ht, const void *key, struct rhash_head *obj,  in rhashtable_lookup_get_insert_key()  argument
865 	BUG_ON(!ht->p.obj_hashfn || !key);  in rhashtable_lookup_get_insert_key()
867 	return __rhashtable_insert_fast(ht, key, obj, params, false);  in rhashtable_lookup_get_insert_key()
872 	struct rhashtable *ht, struct bucket_table *tbl,  in __rhashtable_remove_fast_one()  argument
882 	hash = rht_head_hashfn(ht, tbl, obj, params);  in __rhashtable_remove_fast_one()
935 		atomic_dec(&ht->nelems);  in __rhashtable_remove_fast_one()
936 		if (unlikely(ht->p.automatic_shrinking &&  in __rhashtable_remove_fast_one()
937 			     rht_shrink_below_30(ht, tbl)))  in __rhashtable_remove_fast_one()
938 			schedule_work(&ht->run_work);  in __rhashtable_remove_fast_one()
947 	struct rhashtable *ht, struct rhash_head *obj,  in __rhashtable_remove_fast()  argument
955 	tbl = rht_dereference_rcu(ht->tbl, ht);  in __rhashtable_remove_fast()
962 	while ((err = __rhashtable_remove_fast_one(ht, tbl, obj, params,  in __rhashtable_remove_fast()
964 	       (tbl = rht_dereference_rcu(tbl->future_tbl, ht)))  in __rhashtable_remove_fast()
988 	struct rhashtable *ht, struct rhash_head *obj,  in rhashtable_remove_fast()  argument
991 	return __rhashtable_remove_fast(ht, obj, params, false);  in rhashtable_remove_fast()
1013 	return __rhashtable_remove_fast(&hlt->ht, &list->rhead, params, true);  in rhltable_remove()
1018 	struct rhashtable *ht, struct bucket_table *tbl,  in __rhashtable_replace_fast()  argument
1031 	hash = rht_head_hashfn(ht, tbl, obj_old, params);  in __rhashtable_replace_fast()
1032 	if (hash != rht_head_hashfn(ht, tbl, obj_new, params))  in __rhashtable_replace_fast()
1072 	struct rhashtable *ht, struct rhash_head *obj_old,  in rhashtable_replace_fast()  argument
1081 	tbl = rht_dereference_rcu(ht->tbl, ht);  in rhashtable_replace_fast()
1088 	while ((err = __rhashtable_replace_fast(ht, tbl, obj_old,  in rhashtable_replace_fast()
1090 	       (tbl = rht_dereference_rcu(tbl->future_tbl, ht)))  in rhashtable_replace_fast()
1099 static inline int rhashtable_walk_init(struct rhashtable *ht,  in rhashtable_walk_init()  argument
1102 	rhashtable_walk_enter(ht, iter);  in rhashtable_walk_init()
1130 	return rhashtable_walk_enter(&hlt->ht, iter);  in rhltable_walk_enter()
1146 	return rhashtable_free_and_destroy(&hlt->ht, free_fn, arg);  in rhltable_free_and_destroy()