49 	struct ttm_mem_global *glob;  member
97 	spin_lock(&zone->glob->lock);  in ttm_mem_zone_show()
108 	spin_unlock(&zone->glob->lock);  in ttm_mem_zone_show()
114 static void ttm_check_swapping(struct ttm_mem_global *glob);
134 	spin_lock(&zone->glob->lock);  in ttm_mem_zone_store()
147 	spin_unlock(&zone->glob->lock);  in ttm_mem_zone_store()
149 	ttm_check_swapping(zone->glob);  in ttm_mem_zone_store()
183 	struct ttm_mem_global *glob =  in ttm_mem_global_show()  local
187 	spin_lock(&glob->lock);  in ttm_mem_global_show()
188 	val = glob->lower_mem_limit;  in ttm_mem_global_show()
189 	spin_unlock(&glob->lock);  in ttm_mem_global_show()
204 	struct ttm_mem_global *glob =  in ttm_mem_global_store()  local
215 	spin_lock(&glob->lock);  in ttm_mem_global_store()
216 	glob->lower_mem_limit = val64;  in ttm_mem_global_store()
217 	spin_unlock(&glob->lock);  in ttm_mem_global_store()
237 static bool ttm_zones_above_swap_target(struct ttm_mem_global *glob,  in ttm_zones_above_swap_target()  argument
244 	for (i = 0; i < glob->num_zones; ++i) {  in ttm_zones_above_swap_target()
245 		zone = glob->zones[i];  in ttm_zones_above_swap_target()
269 static void ttm_shrink(struct ttm_mem_global *glob, bool from_wq,  in ttm_shrink()  argument
274 	spin_lock(&glob->lock);  in ttm_shrink()
276 	while (ttm_zones_above_swap_target(glob, from_wq, extra)) {  in ttm_shrink()
277 		spin_unlock(&glob->lock);  in ttm_shrink()
279 		spin_lock(&glob->lock);  in ttm_shrink()
284 	spin_unlock(&glob->lock);  in ttm_shrink()
293 	struct ttm_mem_global *glob =  in ttm_shrink_work()  local
296 	ttm_shrink(glob, true, 0ULL, &ctx);  in ttm_shrink_work()
299 static int ttm_mem_init_kernel_zone(struct ttm_mem_global *glob,  in ttm_mem_init_kernel_zone()  argument
318 	zone->glob = glob;  in ttm_mem_init_kernel_zone()
319 	glob->zone_kernel = zone;  in ttm_mem_init_kernel_zone()
321 		&zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, zone->name);  in ttm_mem_init_kernel_zone()
326 	glob->zones[glob->num_zones++] = zone;  in ttm_mem_init_kernel_zone()
331 static int ttm_mem_init_highmem_zone(struct ttm_mem_global *glob,  in ttm_mem_init_highmem_zone()  argument
354 	zone->glob = glob;  in ttm_mem_init_highmem_zone()
355 	glob->zone_highmem = zone;  in ttm_mem_init_highmem_zone()
357 		&zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, "%s",  in ttm_mem_init_highmem_zone()
363 	glob->zones[glob->num_zones++] = zone;  in ttm_mem_init_highmem_zone()
367 static int ttm_mem_init_dma32_zone(struct ttm_mem_global *glob,  in ttm_mem_init_dma32_zone()  argument
402 	zone->glob = glob;  in ttm_mem_init_dma32_zone()
403 	glob->zone_dma32 = zone;  in ttm_mem_init_dma32_zone()
405 		&zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, zone->name);  in ttm_mem_init_dma32_zone()
410 	glob->zones[glob->num_zones++] = zone;  in ttm_mem_init_dma32_zone()
415 int ttm_mem_global_init(struct ttm_mem_global *glob)  in ttm_mem_global_init()  argument
422 	spin_lock_init(&glob->lock);  in ttm_mem_global_init()
423 	glob->swap_queue = create_singlethread_workqueue("ttm_swap");  in ttm_mem_global_init()
424 	INIT_WORK(&glob->work, ttm_shrink_work);  in ttm_mem_global_init()
426 		&glob->kobj, &ttm_mem_glob_kobj_type, ttm_get_kobj(), "memory_accounting");  in ttm_mem_global_init()
428 		kobject_put(&glob->kobj);  in ttm_mem_global_init()
435 	glob->lower_mem_limit = 0;  in ttm_mem_global_init()
437 	ret = ttm_mem_init_kernel_zone(glob, &si);  in ttm_mem_global_init()
441 	ret = ttm_mem_init_highmem_zone(glob, &si);  in ttm_mem_global_init()
445 	ret = ttm_mem_init_dma32_zone(glob, &si);  in ttm_mem_global_init()
449 	for (i = 0; i < glob->num_zones; ++i) {  in ttm_mem_global_init()
450 		zone = glob->zones[i];  in ttm_mem_global_init()
454 	ttm_page_alloc_init(glob, glob->zone_kernel->max_mem/(2*PAGE_SIZE));  in ttm_mem_global_init()
455 	ttm_dma_page_alloc_init(glob, glob->zone_kernel->max_mem/(2*PAGE_SIZE));  in ttm_mem_global_init()
458 	ttm_mem_global_release(glob);  in ttm_mem_global_init()
462 void ttm_mem_global_release(struct ttm_mem_global *glob)  in ttm_mem_global_release()  argument
471 	flush_workqueue(glob->swap_queue);  in ttm_mem_global_release()
472 	destroy_workqueue(glob->swap_queue);  in ttm_mem_global_release()
473 	glob->swap_queue = NULL;  in ttm_mem_global_release()
474 	for (i = 0; i < glob->num_zones; ++i) {  in ttm_mem_global_release()
475 		zone = glob->zones[i];  in ttm_mem_global_release()
479 	kobject_del(&glob->kobj);  in ttm_mem_global_release()
480 	kobject_put(&glob->kobj);  in ttm_mem_global_release()
481 	memset(glob, 0, sizeof(*glob));  in ttm_mem_global_release()
484 static void ttm_check_swapping(struct ttm_mem_global *glob)  in ttm_check_swapping()  argument
490 	spin_lock(&glob->lock);  in ttm_check_swapping()
491 	for (i = 0; i < glob->num_zones; ++i) {  in ttm_check_swapping()
492 		zone = glob->zones[i];  in ttm_check_swapping()
499 	spin_unlock(&glob->lock);  in ttm_check_swapping()
502 		(void)queue_work(glob->swap_queue, &glob->work);  in ttm_check_swapping()
506 static void ttm_mem_global_free_zone(struct ttm_mem_global *glob,  in ttm_mem_global_free_zone()  argument
513 	spin_lock(&glob->lock);  in ttm_mem_global_free_zone()
514 	for (i = 0; i < glob->num_zones; ++i) {  in ttm_mem_global_free_zone()
515 		zone = glob->zones[i];  in ttm_mem_global_free_zone()
520 	spin_unlock(&glob->lock);  in ttm_mem_global_free_zone()
523 void ttm_mem_global_free(struct ttm_mem_global *glob,  in ttm_mem_global_free()  argument
526 	return ttm_mem_global_free_zone(glob, glob->zone_kernel, amount);  in ttm_mem_global_free()
541 ttm_check_under_lowerlimit(struct ttm_mem_global *glob,  in ttm_check_under_lowerlimit()  argument
552 	if (available < glob->lower_mem_limit)  in ttm_check_under_lowerlimit()
558 static int ttm_mem_global_reserve(struct ttm_mem_global *glob,  in ttm_mem_global_reserve()  argument
567 	spin_lock(&glob->lock);  in ttm_mem_global_reserve()
568 	for (i = 0; i < glob->num_zones; ++i) {  in ttm_mem_global_reserve()
569 		zone = glob->zones[i];  in ttm_mem_global_reserve()
581 		for (i = 0; i < glob->num_zones; ++i) {  in ttm_mem_global_reserve()
582 			zone = glob->zones[i];  in ttm_mem_global_reserve()
591 	spin_unlock(&glob->lock);  in ttm_mem_global_reserve()
592 	ttm_check_swapping(glob);  in ttm_mem_global_reserve()
598 static int ttm_mem_global_alloc_zone(struct ttm_mem_global *glob,  in ttm_mem_global_alloc_zone()  argument
605 	while (unlikely(ttm_mem_global_reserve(glob,  in ttm_mem_global_alloc_zone()
613 		ttm_shrink(glob, false, memory + (memory >> 2) + 16, ctx);  in ttm_mem_global_alloc_zone()
619 int ttm_mem_global_alloc(struct ttm_mem_global *glob, uint64_t memory,  in ttm_mem_global_alloc()  argument
627 	return ttm_mem_global_alloc_zone(glob, glob->zone_kernel, memory, ctx);  in ttm_mem_global_alloc()
631 int ttm_mem_global_alloc_page(struct ttm_mem_global *glob,  in ttm_mem_global_alloc_page()  argument
643 	if (PageHighMem(page) && glob->zone_highmem != NULL)  in ttm_mem_global_alloc_page()
644 		zone = glob->zone_highmem;  in ttm_mem_global_alloc_page()
646 	if (glob->zone_dma32 && page_to_pfn(page) > 0x00100000UL)  in ttm_mem_global_alloc_page()
647 		zone = glob->zone_kernel;  in ttm_mem_global_alloc_page()
649 	return ttm_mem_global_alloc_zone(glob, zone, size, ctx);  in ttm_mem_global_alloc_page()
652 void ttm_mem_global_free_page(struct ttm_mem_global *glob, struct page *page,  in ttm_mem_global_free_page()  argument
658 	if (PageHighMem(page) && glob->zone_highmem != NULL)  in ttm_mem_global_free_page()
659 		zone = glob->zone_highmem;  in ttm_mem_global_free_page()
661 	if (glob->zone_dma32 && page_to_pfn(page) > 0x00100000UL)  in ttm_mem_global_free_page()
662 		zone = glob->zone_kernel;  in ttm_mem_global_free_page()
664 	ttm_mem_global_free_zone(glob, zone, size);  in ttm_mem_global_free_page()