151 	struct dma_pool *pool;  member
266 static int ttm_set_pages_caching(struct dma_pool *pool,  in ttm_set_pages_caching()  argument
271 	if (pool->type & IS_UC) {  in ttm_set_pages_caching()
275 			       pool->dev_name, cpages);  in ttm_set_pages_caching()
277 	if (pool->type & IS_WC) {  in ttm_set_pages_caching()
281 			       pool->dev_name, cpages);  in ttm_set_pages_caching()
286 static void __ttm_dma_free_page(struct dma_pool *pool, struct dma_page *d_page)  in __ttm_dma_free_page()  argument
290 	dma_free_coherent(pool->dev, pool->size, (void *)d_page->vaddr, dma);  in __ttm_dma_free_page()
295 static struct dma_page *__ttm_dma_alloc_page(struct dma_pool *pool)  in __ttm_dma_alloc_page()  argument
305 	if (pool->type & IS_HUGE)  in __ttm_dma_alloc_page()
308 	vaddr = dma_alloc_attrs(pool->dev, pool->size, &d_page->dma,  in __ttm_dma_alloc_page()
309 				pool->gfp_flags, attrs);  in __ttm_dma_alloc_page()
316 		if (pool->type & IS_HUGE)  in __ttm_dma_alloc_page()
340 static void ttm_pool_update_free_locked(struct dma_pool *pool,  in ttm_pool_update_free_locked()  argument
343 	pool->npages_free -= freed_pages;  in ttm_pool_update_free_locked()
344 	pool->nfrees += freed_pages;  in ttm_pool_update_free_locked()
349 static void ttm_dma_page_put(struct dma_pool *pool, struct dma_page *d_page)  in ttm_dma_page_put()  argument
355 	if (!(pool->type & IS_CACHED)) {  in ttm_dma_page_put()
356 		num_pages = pool->size / PAGE_SIZE;  in ttm_dma_page_put()
359 			       pool->dev_name, num_pages);  in ttm_dma_page_put()
363 	__ttm_dma_free_page(pool, d_page);  in ttm_dma_page_put()
366 static void ttm_dma_pages_put(struct dma_pool *pool, struct list_head *d_pages,  in ttm_dma_pages_put()  argument
371 	if (pool->type & IS_HUGE) {  in ttm_dma_pages_put()
373 			ttm_dma_page_put(pool, d_page);  in ttm_dma_pages_put()
379 	if (npages && !(pool->type & IS_CACHED) &&  in ttm_dma_pages_put()
382 		       pool->dev_name, npages);  in ttm_dma_pages_put()
386 		__ttm_dma_free_page(pool, d_page);  in ttm_dma_pages_put()
400 static unsigned ttm_dma_page_pool_free(struct dma_pool *pool, unsigned nr_free,  in ttm_dma_page_pool_free()  argument
416 			 pool->dev_name, pool->name, current->pid,  in ttm_dma_page_pool_free()
429 		       pool->dev_name);  in ttm_dma_page_pool_free()
434 	spin_lock_irqsave(&pool->lock, irq_flags);  in ttm_dma_page_pool_free()
437 	list_for_each_entry_safe_reverse(dma_p, tmp, &pool->free_list,  in ttm_dma_page_pool_free()
449 			ttm_pool_update_free_locked(pool, freed_pages);  in ttm_dma_page_pool_free()
454 			spin_unlock_irqrestore(&pool->lock, irq_flags);  in ttm_dma_page_pool_free()
456 			ttm_dma_pages_put(pool, &d_pages, pages_to_free,  in ttm_dma_page_pool_free()
486 		ttm_pool_update_free_locked(pool, freed_pages);  in ttm_dma_page_pool_free()
490 	spin_unlock_irqrestore(&pool->lock, irq_flags);  in ttm_dma_page_pool_free()
493 		ttm_dma_pages_put(pool, &d_pages, pages_to_free, freed_pages);  in ttm_dma_page_pool_free()
503 	struct dma_pool *pool;  in ttm_dma_free_pool()  local
512 		pool = p->pool;  in ttm_dma_free_pool()
513 		if (pool->type != type)  in ttm_dma_free_pool()
521 	list_for_each_entry_reverse(pool, &dev->dma_pools, pools) {  in ttm_dma_free_pool()
522 		if (pool->type != type)  in ttm_dma_free_pool()
526 		ttm_dma_page_pool_free(pool, FREE_ALL_PAGES, true);  in ttm_dma_free_pool()
527 		WARN_ON(((pool->npages_in_use + pool->npages_free) != 0));  in ttm_dma_free_pool()
532 		list_del(&pool->pools);  in ttm_dma_free_pool()
533 		kfree(pool);  in ttm_dma_free_pool()
545 	struct dma_pool *pool = *(struct dma_pool **)res;  in ttm_dma_pool_release()  local
547 	if (pool)  in ttm_dma_pool_release()
548 		ttm_dma_free_pool(dev, pool->type);  in ttm_dma_pool_release()
562 	struct dma_pool *pool = NULL, **ptr;  in ttm_dma_pool_init()  local
576 	pool = kmalloc_node(sizeof(struct dma_pool), GFP_KERNEL,  in ttm_dma_pool_init()
578 	if (!pool)  in ttm_dma_pool_init()
588 	sec_pool->pool =  pool;  in ttm_dma_pool_init()
590 	INIT_LIST_HEAD(&pool->free_list);  in ttm_dma_pool_init()
591 	INIT_LIST_HEAD(&pool->pools);  in ttm_dma_pool_init()
592 	spin_lock_init(&pool->lock);  in ttm_dma_pool_init()
593 	pool->dev = dev;  in ttm_dma_pool_init()
594 	pool->npages_free = pool->npages_in_use = 0;  in ttm_dma_pool_init()
595 	pool->nfrees = 0;  in ttm_dma_pool_init()
596 	pool->gfp_flags = flags;  in ttm_dma_pool_init()
599 		pool->size = HPAGE_PMD_SIZE;  in ttm_dma_pool_init()
604 		pool->size = PAGE_SIZE;  in ttm_dma_pool_init()
605 	pool->type = type;  in ttm_dma_pool_init()
606 	pool->nrefills = 0;  in ttm_dma_pool_init()
607 	p = pool->name;  in ttm_dma_pool_init()
610 			p += snprintf(p, sizeof(pool->name) - (p - pool->name),  in ttm_dma_pool_init()
617 	snprintf(pool->dev_name, sizeof(pool->dev_name), "%s %s",  in ttm_dma_pool_init()
624 	list_add(&pool->pools, &dev->dma_pools);  in ttm_dma_pool_init()
627 	*ptr = pool;  in ttm_dma_pool_init()
630 	return pool;  in ttm_dma_pool_init()
634 	kfree(pool);  in ttm_dma_pool_init()
641 	struct dma_pool *pool, *tmp;  in ttm_dma_find_pool()  local
657 	list_for_each_entry_safe(pool, tmp, &dev->dma_pools, pools)  in ttm_dma_find_pool()
658 		if (pool->type == type)  in ttm_dma_find_pool()
659 			return pool;  in ttm_dma_find_pool()
668 static void ttm_dma_handle_caching_state_failure(struct dma_pool *pool,  in ttm_dma_handle_caching_state_failure()  argument
686 		__ttm_dma_free_page(pool, d_page);  in ttm_dma_handle_caching_state_failure()
701 static int ttm_dma_pool_alloc_new_pages(struct dma_pool *pool,  in ttm_dma_pool_alloc_new_pages()  argument
719 		       pool->dev_name);  in ttm_dma_pool_alloc_new_pages()
725 			 pool->dev_name, pool->name, current->pid, count);  in ttm_dma_pool_alloc_new_pages()
728 		dma_p = __ttm_dma_alloc_page(pool);  in ttm_dma_pool_alloc_new_pages()
731 				 pool->dev_name, i);  in ttm_dma_pool_alloc_new_pages()
736 				r = ttm_set_pages_caching(pool, caching_array,  in ttm_dma_pool_alloc_new_pages()
740 						pool, d_pages, caching_array,  in ttm_dma_pool_alloc_new_pages()
757 		npages = pool->size / PAGE_SIZE;  in ttm_dma_pool_alloc_new_pages()
762 				r = ttm_set_pages_caching(pool, caching_array,  in ttm_dma_pool_alloc_new_pages()
766 					     pool, d_pages, caching_array,  in ttm_dma_pool_alloc_new_pages()
776 		r = ttm_set_pages_caching(pool, caching_array, cpages);  in ttm_dma_pool_alloc_new_pages()
778 			ttm_dma_handle_caching_state_failure(pool, d_pages,  in ttm_dma_pool_alloc_new_pages()
789 static int ttm_dma_page_pool_fill_locked(struct dma_pool *pool,  in ttm_dma_page_pool_fill_locked()  argument
793 	int r = pool->npages_free;  in ttm_dma_page_pool_fill_locked()
795 	if (count > pool->npages_free) {  in ttm_dma_page_pool_fill_locked()
800 		spin_unlock_irqrestore(&pool->lock, *irq_flags);  in ttm_dma_page_pool_fill_locked()
804 		r = ttm_dma_pool_alloc_new_pages(pool, &d_pages, count);  in ttm_dma_page_pool_fill_locked()
806 		spin_lock_irqsave(&pool->lock, *irq_flags);  in ttm_dma_page_pool_fill_locked()
809 			list_splice(&d_pages, &pool->free_list);  in ttm_dma_page_pool_fill_locked()
810 			++pool->nrefills;  in ttm_dma_page_pool_fill_locked()
811 			pool->npages_free += count;  in ttm_dma_page_pool_fill_locked()
818 				 pool->dev_name, pool->name, r);  in ttm_dma_page_pool_fill_locked()
823 			list_splice_tail(&d_pages, &pool->free_list);  in ttm_dma_page_pool_fill_locked()
824 			pool->npages_free += cpages;  in ttm_dma_page_pool_fill_locked()
836 static struct dma_page *ttm_dma_pool_get_pages(struct dma_pool *pool,  in ttm_dma_pool_get_pages()  argument
845 	spin_lock_irqsave(&pool->lock, irq_flags);  in ttm_dma_pool_get_pages()
846 	count = ttm_dma_page_pool_fill_locked(pool, &irq_flags);  in ttm_dma_pool_get_pages()
848 		d_page = list_first_entry(&pool->free_list, struct dma_page, page_list);  in ttm_dma_pool_get_pages()
852 		pool->npages_in_use += 1;  in ttm_dma_pool_get_pages()
853 		pool->npages_free -= 1;  in ttm_dma_pool_get_pages()
855 	spin_unlock_irqrestore(&pool->lock, irq_flags);  in ttm_dma_pool_get_pages()
894 	struct dma_pool *pool;  in ttm_dma_populate()  local
915 	pool = ttm_dma_find_pool(dev, type | IS_HUGE);  in ttm_dma_populate()
916 	if (!pool) {  in ttm_dma_populate()
919 		pool = ttm_dma_pool_init(dev, gfp_flags, type | IS_HUGE);  in ttm_dma_populate()
920 		if (IS_ERR_OR_NULL(pool))  in ttm_dma_populate()
927 		d_page = ttm_dma_pool_get_pages(pool, ttm_dma, i);  in ttm_dma_populate()
932 						pool->size, ctx);  in ttm_dma_populate()
952 	pool = ttm_dma_find_pool(dev, type);  in ttm_dma_populate()
953 	if (!pool) {  in ttm_dma_populate()
956 		pool = ttm_dma_pool_init(dev, gfp_flags, type);  in ttm_dma_populate()
957 		if (IS_ERR_OR_NULL(pool))  in ttm_dma_populate()
962 		d_page = ttm_dma_pool_get_pages(pool, ttm_dma, i);  in ttm_dma_populate()
969 						pool->size, ctx);  in ttm_dma_populate()
998 	struct dma_pool *pool;  in ttm_dma_unpopulate()  local
1008 	pool = ttm_dma_find_pool(dev, type | IS_HUGE);  in ttm_dma_unpopulate()
1009 	if (pool) {  in ttm_dma_unpopulate()
1019 							 pool->size);  in ttm_dma_unpopulate()
1022 			ttm_dma_page_put(pool, d_page);  in ttm_dma_unpopulate()
1025 		spin_lock_irqsave(&pool->lock, irq_flags);  in ttm_dma_unpopulate()
1026 		pool->npages_in_use -= count;  in ttm_dma_unpopulate()
1027 		pool->nfrees += count;  in ttm_dma_unpopulate()
1028 		spin_unlock_irqrestore(&pool->lock, irq_flags);  in ttm_dma_unpopulate()
1032 	pool = ttm_dma_find_pool(dev, type);  in ttm_dma_unpopulate()
1033 	if (!pool)  in ttm_dma_unpopulate()
1036 	is_cached = (ttm_dma_find_pool(pool->dev,  in ttm_dma_unpopulate()
1037 		     ttm_to_type(ttm->page_flags, tt_cached)) == pool);  in ttm_dma_unpopulate()
1048 						 pool->size);  in ttm_dma_unpopulate()
1053 			ttm_dma_page_put(pool, d_page);  in ttm_dma_unpopulate()
1056 	spin_lock_irqsave(&pool->lock, irq_flags);  in ttm_dma_unpopulate()
1057 	pool->npages_in_use -= count;  in ttm_dma_unpopulate()
1059 		pool->nfrees += count;  in ttm_dma_unpopulate()
1061 		pool->npages_free += count;  in ttm_dma_unpopulate()
1062 		list_splice(&ttm_dma->pages_list, &pool->free_list);  in ttm_dma_unpopulate()
1067 		if (pool->npages_free >= (_manager->options.max_size +  in ttm_dma_unpopulate()
1069 			npages = pool->npages_free - _manager->options.max_size;  in ttm_dma_unpopulate()
1071 	spin_unlock_irqrestore(&pool->lock, irq_flags);  in ttm_dma_unpopulate()
1081 		ttm_dma_page_pool_free(pool, npages, false);  in ttm_dma_unpopulate()
1124 		shrink_pages = ttm_dma_page_pool_free(p->pool, nr_free, true);  in ttm_dma_pool_shrink_scan()
1128 			 p->pool->dev_name, p->pool->name, current->pid,  in ttm_dma_pool_shrink_scan()
1145 		count += p->pool->npages_free;  in ttm_dma_pool_shrink_count()
1207 		dev_dbg(p->dev, "(%s:%d) Freeing.\n", p->pool->name,  in ttm_dma_page_alloc_fini()
1210 			ttm_dma_pool_match, p->pool));  in ttm_dma_page_alloc_fini()
1211 		ttm_dma_free_pool(p->dev, p->pool->type);  in ttm_dma_page_alloc_fini()
1220 	struct dma_pool *pool = NULL;  in ttm_dma_page_alloc_debugfs()  local
1232 		pool = p->pool;  in ttm_dma_page_alloc_debugfs()
1234 				pool->name, pool->nrefills,  in ttm_dma_page_alloc_debugfs()
1235 				pool->nfrees, pool->npages_in_use,  in ttm_dma_page_alloc_debugfs()
1236 				pool->npages_free,  in ttm_dma_page_alloc_debugfs()
1237 				pool->dev_name);  in ttm_dma_page_alloc_debugfs()