105 static inline const char *pool_name(struct rxe_pool *pool)  in pool_name()  argument
107 	return rxe_type_info[pool->type].name;  in pool_name()
110 static inline struct kmem_cache *pool_cache(struct rxe_pool *pool)  in pool_cache()  argument
112 	return rxe_type_info[pool->type].cache;  in pool_cache()
159 static int rxe_pool_init_index(struct rxe_pool *pool, u32 max, u32 min)  in rxe_pool_init_index()  argument
164 	if ((max - min + 1) < pool->max_elem) {  in rxe_pool_init_index()
170 	pool->max_index = max;  in rxe_pool_init_index()
171 	pool->min_index = min;  in rxe_pool_init_index()
174 	pool->table = kmalloc(size, GFP_KERNEL);  in rxe_pool_init_index()
175 	if (!pool->table) {  in rxe_pool_init_index()
180 	pool->table_size = size;  in rxe_pool_init_index()
181 	bitmap_zero(pool->table, max - min + 1);  in rxe_pool_init_index()
189 	struct rxe_pool		*pool,  in rxe_pool_init()  argument
196 	memset(pool, 0, sizeof(*pool));  in rxe_pool_init()
198 	pool->rxe		= rxe;  in rxe_pool_init()
199 	pool->type		= type;  in rxe_pool_init()
200 	pool->max_elem		= max_elem;  in rxe_pool_init()
201 	pool->elem_size		= ALIGN(size, RXE_POOL_ALIGN);  in rxe_pool_init()
202 	pool->flags		= rxe_type_info[type].flags;  in rxe_pool_init()
203 	pool->tree		= RB_ROOT;  in rxe_pool_init()
204 	pool->cleanup		= rxe_type_info[type].cleanup;  in rxe_pool_init()
206 	atomic_set(&pool->num_elem, 0);  in rxe_pool_init()
208 	kref_init(&pool->ref_cnt);  in rxe_pool_init()
210 	spin_lock_init(&pool->pool_lock);  in rxe_pool_init()
213 		err = rxe_pool_init_index(pool,  in rxe_pool_init()
221 		pool->key_offset = rxe_type_info[type].key_offset;  in rxe_pool_init()
222 		pool->key_size = rxe_type_info[type].key_size;  in rxe_pool_init()
225 	pool->state = rxe_pool_valid;  in rxe_pool_init()
233 	struct rxe_pool *pool = container_of(kref, struct rxe_pool, ref_cnt);  in rxe_pool_release()  local
235 	pool->state = rxe_pool_invalid;  in rxe_pool_release()
236 	kfree(pool->table);  in rxe_pool_release()
239 static void rxe_pool_put(struct rxe_pool *pool)  in rxe_pool_put()  argument
241 	kref_put(&pool->ref_cnt, rxe_pool_release);  in rxe_pool_put()
244 int rxe_pool_cleanup(struct rxe_pool *pool)  in rxe_pool_cleanup()  argument
248 	spin_lock_irqsave(&pool->pool_lock, flags);  in rxe_pool_cleanup()
249 	pool->state = rxe_pool_invalid;  in rxe_pool_cleanup()
250 	if (atomic_read(&pool->num_elem) > 0)  in rxe_pool_cleanup()
252 			pool_name(pool));  in rxe_pool_cleanup()
253 	spin_unlock_irqrestore(&pool->pool_lock, flags);  in rxe_pool_cleanup()
255 	rxe_pool_put(pool);  in rxe_pool_cleanup()
260 static u32 alloc_index(struct rxe_pool *pool)  in alloc_index()  argument
263 	u32 range = pool->max_index - pool->min_index + 1;  in alloc_index()
265 	index = find_next_zero_bit(pool->table, range, pool->last);  in alloc_index()
267 		index = find_first_zero_bit(pool->table, range);  in alloc_index()
270 	set_bit(index, pool->table);  in alloc_index()
271 	pool->last = index;  in alloc_index()
272 	return index + pool->min_index;  in alloc_index()
275 static void insert_index(struct rxe_pool *pool, struct rxe_pool_entry *new)  in insert_index()  argument
277 	struct rb_node **link = &pool->tree.rb_node;  in insert_index()
297 	rb_insert_color(&new->node, &pool->tree);  in insert_index()
302 static void insert_key(struct rxe_pool *pool, struct rxe_pool_entry *new)  in insert_key()  argument
304 	struct rb_node **link = &pool->tree.rb_node;  in insert_key()
313 		cmp = memcmp((u8 *)elem + pool->key_offset,  in insert_key()
314 			     (u8 *)new + pool->key_offset, pool->key_size);  in insert_key()
328 	rb_insert_color(&new->node, &pool->tree);  in insert_key()
336 	struct rxe_pool *pool = elem->pool;  in rxe_add_key()  local
339 	spin_lock_irqsave(&pool->pool_lock, flags);  in rxe_add_key()
340 	memcpy((u8 *)elem + pool->key_offset, key, pool->key_size);  in rxe_add_key()
341 	insert_key(pool, elem);  in rxe_add_key()
342 	spin_unlock_irqrestore(&pool->pool_lock, flags);  in rxe_add_key()
348 	struct rxe_pool *pool = elem->pool;  in rxe_drop_key()  local
351 	spin_lock_irqsave(&pool->pool_lock, flags);  in rxe_drop_key()
352 	rb_erase(&elem->node, &pool->tree);  in rxe_drop_key()
353 	spin_unlock_irqrestore(&pool->pool_lock, flags);  in rxe_drop_key()
359 	struct rxe_pool *pool = elem->pool;  in rxe_add_index()  local
362 	spin_lock_irqsave(&pool->pool_lock, flags);  in rxe_add_index()
363 	elem->index = alloc_index(pool);  in rxe_add_index()
364 	insert_index(pool, elem);  in rxe_add_index()
365 	spin_unlock_irqrestore(&pool->pool_lock, flags);  in rxe_add_index()
371 	struct rxe_pool *pool = elem->pool;  in rxe_drop_index()  local
374 	spin_lock_irqsave(&pool->pool_lock, flags);  in rxe_drop_index()
375 	clear_bit(elem->index - pool->min_index, pool->table);  in rxe_drop_index()
376 	rb_erase(&elem->node, &pool->tree);  in rxe_drop_index()
377 	spin_unlock_irqrestore(&pool->pool_lock, flags);  in rxe_drop_index()
380 void *rxe_alloc(struct rxe_pool *pool)  in rxe_alloc()  argument
385 	might_sleep_if(!(pool->flags & RXE_POOL_ATOMIC));  in rxe_alloc()
387 	spin_lock_irqsave(&pool->pool_lock, flags);  in rxe_alloc()
388 	if (pool->state != rxe_pool_valid) {  in rxe_alloc()
389 		spin_unlock_irqrestore(&pool->pool_lock, flags);  in rxe_alloc()
392 	kref_get(&pool->ref_cnt);  in rxe_alloc()
393 	spin_unlock_irqrestore(&pool->pool_lock, flags);  in rxe_alloc()
395 	kref_get(&pool->rxe->ref_cnt);  in rxe_alloc()
397 	if (atomic_inc_return(&pool->num_elem) > pool->max_elem)  in rxe_alloc()
400 	elem = kmem_cache_zalloc(pool_cache(pool),  in rxe_alloc()
401 				 (pool->flags & RXE_POOL_ATOMIC) ?  in rxe_alloc()
406 	elem->pool = pool;  in rxe_alloc()
412 	atomic_dec(&pool->num_elem);  in rxe_alloc()
413 	rxe_dev_put(pool->rxe);  in rxe_alloc()
414 	rxe_pool_put(pool);  in rxe_alloc()
422 	struct rxe_pool *pool = elem->pool;  in rxe_elem_release()  local
424 	if (pool->cleanup)  in rxe_elem_release()
425 		pool->cleanup(elem);  in rxe_elem_release()
427 	kmem_cache_free(pool_cache(pool), elem);  in rxe_elem_release()
428 	atomic_dec(&pool->num_elem);  in rxe_elem_release()
429 	rxe_dev_put(pool->rxe);  in rxe_elem_release()
430 	rxe_pool_put(pool);  in rxe_elem_release()
433 void *rxe_pool_get_index(struct rxe_pool *pool, u32 index)  in rxe_pool_get_index()  argument
439 	spin_lock_irqsave(&pool->pool_lock, flags);  in rxe_pool_get_index()
441 	if (pool->state != rxe_pool_valid)  in rxe_pool_get_index()
444 	node = pool->tree.rb_node;  in rxe_pool_get_index()
461 	spin_unlock_irqrestore(&pool->pool_lock, flags);  in rxe_pool_get_index()
465 void *rxe_pool_get_key(struct rxe_pool *pool, void *key)  in rxe_pool_get_key()  argument
472 	spin_lock_irqsave(&pool->pool_lock, flags);  in rxe_pool_get_key()
474 	if (pool->state != rxe_pool_valid)  in rxe_pool_get_key()
477 	node = pool->tree.rb_node;  in rxe_pool_get_key()
482 		cmp = memcmp((u8 *)elem + pool->key_offset,  in rxe_pool_get_key()
483 			     key, pool->key_size);  in rxe_pool_get_key()
497 	spin_unlock_irqrestore(&pool->pool_lock, flags);  in rxe_pool_get_key()