191 struct rds_ib_mr *rds_ib_reuse_mr(struct rds_ib_mr_pool *pool)  in rds_ib_reuse_mr()  argument
197 	spin_lock_irqsave(&pool->clean_lock, flags);  in rds_ib_reuse_mr()
198 	ret = llist_del_first(&pool->clean_list);  in rds_ib_reuse_mr()
199 	spin_unlock_irqrestore(&pool->clean_lock, flags);  in rds_ib_reuse_mr()
202 		if (pool->pool_type == RDS_IB_MR_8K_POOL)  in rds_ib_reuse_mr()
265 		struct rds_ib_mr_pool *pool = ibmr->pool;  in rds_ib_teardown_mr()  local
267 		atomic_sub(pinned, &pool->free_pinned);  in rds_ib_teardown_mr()
271 static inline unsigned int rds_ib_flush_goal(struct rds_ib_mr_pool *pool, int free_all)  in rds_ib_flush_goal()  argument
275 	item_count = atomic_read(&pool->item_count);  in rds_ib_flush_goal()
332 int rds_ib_flush_mr_pool(struct rds_ib_mr_pool *pool,  in rds_ib_flush_mr_pool()  argument
342 	if (pool->pool_type == RDS_IB_MR_8K_POOL)  in rds_ib_flush_mr_pool()
349 		while (!mutex_trylock(&pool->flush_lock)) {  in rds_ib_flush_mr_pool()
350 			ibmr = rds_ib_reuse_mr(pool);  in rds_ib_flush_mr_pool()
353 				finish_wait(&pool->flush_wait, &wait);  in rds_ib_flush_mr_pool()
357 			prepare_to_wait(&pool->flush_wait, &wait,  in rds_ib_flush_mr_pool()
359 			if (llist_empty(&pool->clean_list))  in rds_ib_flush_mr_pool()
362 			ibmr = rds_ib_reuse_mr(pool);  in rds_ib_flush_mr_pool()
365 				finish_wait(&pool->flush_wait, &wait);  in rds_ib_flush_mr_pool()
369 		finish_wait(&pool->flush_wait, &wait);  in rds_ib_flush_mr_pool()
371 		mutex_lock(&pool->flush_lock);  in rds_ib_flush_mr_pool()
374 		ibmr = rds_ib_reuse_mr(pool);  in rds_ib_flush_mr_pool()
384 	dirty_to_clean = llist_append_to_list(&pool->drop_list, &unmap_list);  in rds_ib_flush_mr_pool()
385 	dirty_to_clean += llist_append_to_list(&pool->free_list, &unmap_list);  in rds_ib_flush_mr_pool()
389 		spin_lock_irqsave(&pool->clean_lock, flags);  in rds_ib_flush_mr_pool()
390 		llist_append_to_list(&pool->clean_list, &unmap_list);  in rds_ib_flush_mr_pool()
391 		spin_unlock_irqrestore(&pool->clean_lock, flags);  in rds_ib_flush_mr_pool()
394 	free_goal = rds_ib_flush_goal(pool, free_all);  in rds_ib_flush_mr_pool()
399 	if (pool->use_fastreg)  in rds_ib_flush_mr_pool()
414 			spin_lock_irqsave(&pool->clean_lock, flags);  in rds_ib_flush_mr_pool()
416 					&pool->clean_list);  in rds_ib_flush_mr_pool()
417 			spin_unlock_irqrestore(&pool->clean_lock, flags);  in rds_ib_flush_mr_pool()
421 	atomic_sub(unpinned, &pool->free_pinned);  in rds_ib_flush_mr_pool()
422 	atomic_sub(dirty_to_clean, &pool->dirty_count);  in rds_ib_flush_mr_pool()
423 	atomic_sub(nfreed, &pool->item_count);  in rds_ib_flush_mr_pool()
426 	mutex_unlock(&pool->flush_lock);  in rds_ib_flush_mr_pool()
427 	if (waitqueue_active(&pool->flush_wait))  in rds_ib_flush_mr_pool()
428 		wake_up(&pool->flush_wait);  in rds_ib_flush_mr_pool()
433 struct rds_ib_mr *rds_ib_try_reuse_ibmr(struct rds_ib_mr_pool *pool)  in rds_ib_try_reuse_ibmr()  argument
439 		ibmr = rds_ib_reuse_mr(pool);  in rds_ib_try_reuse_ibmr()
443 		if (atomic_inc_return(&pool->item_count) <= pool->max_items)  in rds_ib_try_reuse_ibmr()
446 		atomic_dec(&pool->item_count);  in rds_ib_try_reuse_ibmr()
449 			if (pool->pool_type == RDS_IB_MR_8K_POOL)  in rds_ib_try_reuse_ibmr()
457 		if (pool->pool_type == RDS_IB_MR_8K_POOL)  in rds_ib_try_reuse_ibmr()
462 		rds_ib_flush_mr_pool(pool, 0, &ibmr);  in rds_ib_try_reuse_ibmr()
472 	struct rds_ib_mr_pool *pool = container_of(work, struct rds_ib_mr_pool, flush_worker.work);  in rds_ib_mr_pool_flush_worker()  local
474 	rds_ib_flush_mr_pool(pool, 0, NULL);  in rds_ib_mr_pool_flush_worker()
480 	struct rds_ib_mr_pool *pool = ibmr->pool;  in rds_ib_free_mr()  local
491 	atomic_add(ibmr->sg_len, &pool->free_pinned);  in rds_ib_free_mr()
492 	atomic_inc(&pool->dirty_count);  in rds_ib_free_mr()
495 	if (atomic_read(&pool->free_pinned) >= pool->max_free_pinned ||  in rds_ib_free_mr()
496 	    atomic_read(&pool->dirty_count) >= pool->max_items / 5)  in rds_ib_free_mr()
497 		queue_delayed_work(rds_ib_mr_wq, &pool->flush_worker, 10);  in rds_ib_free_mr()
501 			rds_ib_flush_mr_pool(pool, 0, NULL);  in rds_ib_free_mr()
507 					   &pool->flush_worker, 10);  in rds_ib_free_mr()
570 void rds_ib_destroy_mr_pool(struct rds_ib_mr_pool *pool)  in rds_ib_destroy_mr_pool()  argument
572 	cancel_delayed_work_sync(&pool->flush_worker);  in rds_ib_destroy_mr_pool()
573 	rds_ib_flush_mr_pool(pool, 1, NULL);  in rds_ib_destroy_mr_pool()
574 	WARN_ON(atomic_read(&pool->item_count));  in rds_ib_destroy_mr_pool()
575 	WARN_ON(atomic_read(&pool->free_pinned));  in rds_ib_destroy_mr_pool()
576 	kfree(pool);  in rds_ib_destroy_mr_pool()
582 	struct rds_ib_mr_pool *pool;  in rds_ib_create_mr_pool()  local
584 	pool = kzalloc(sizeof(*pool), GFP_KERNEL);  in rds_ib_create_mr_pool()
585 	if (!pool)  in rds_ib_create_mr_pool()
588 	pool->pool_type = pool_type;  in rds_ib_create_mr_pool()
589 	init_llist_head(&pool->free_list);  in rds_ib_create_mr_pool()
590 	init_llist_head(&pool->drop_list);  in rds_ib_create_mr_pool()
591 	init_llist_head(&pool->clean_list);  in rds_ib_create_mr_pool()
592 	spin_lock_init(&pool->clean_lock);  in rds_ib_create_mr_pool()
593 	mutex_init(&pool->flush_lock);  in rds_ib_create_mr_pool()
594 	init_waitqueue_head(&pool->flush_wait);  in rds_ib_create_mr_pool()
595 	INIT_DELAYED_WORK(&pool->flush_worker, rds_ib_mr_pool_flush_worker);  in rds_ib_create_mr_pool()
599 		pool->fmr_attr.max_pages = RDS_MR_1M_MSG_SIZE + 1;  in rds_ib_create_mr_pool()
600 		pool->max_items = rds_ibdev->max_1m_mrs;  in rds_ib_create_mr_pool()
603 		pool->fmr_attr.max_pages = RDS_MR_8K_MSG_SIZE + 1;  in rds_ib_create_mr_pool()
604 		pool->max_items = rds_ibdev->max_8k_mrs;  in rds_ib_create_mr_pool()
607 	pool->max_free_pinned = pool->max_items * pool->fmr_attr.max_pages / 4;  in rds_ib_create_mr_pool()
608 	pool->fmr_attr.max_maps = rds_ibdev->fmr_max_remaps;  in rds_ib_create_mr_pool()
609 	pool->fmr_attr.page_shift = PAGE_SHIFT;  in rds_ib_create_mr_pool()
610 	pool->max_items_soft = rds_ibdev->max_mrs * 3 / 4;  in rds_ib_create_mr_pool()
611 	pool->use_fastreg = rds_ibdev->use_fastreg;  in rds_ib_create_mr_pool()
613 	return pool;  in rds_ib_create_mr_pool()