52 	struct ttm_mem_global *glob;  member
100 	spin_lock(&zone->glob->lock);  in ttm_mem_zone_show()
111 	spin_unlock(&zone->glob->lock);  in ttm_mem_zone_show()
117 static void ttm_check_swapping(struct ttm_mem_global *glob);
137 	spin_lock(&zone->glob->lock);  in ttm_mem_zone_store()
150 	spin_unlock(&zone->glob->lock);  in ttm_mem_zone_store()
152 	ttm_check_swapping(zone->glob);  in ttm_mem_zone_store()
186 	struct ttm_mem_global *glob =  in ttm_mem_global_show()  local
190 	spin_lock(&glob->lock);  in ttm_mem_global_show()
191 	val = glob->lower_mem_limit;  in ttm_mem_global_show()
192 	spin_unlock(&glob->lock);  in ttm_mem_global_show()
207 	struct ttm_mem_global *glob =  in ttm_mem_global_store()  local
218 	spin_lock(&glob->lock);  in ttm_mem_global_store()
219 	glob->lower_mem_limit = val64;  in ttm_mem_global_store()
220 	spin_unlock(&glob->lock);  in ttm_mem_global_store()
240 static bool ttm_zones_above_swap_target(struct ttm_mem_global *glob,  in ttm_zones_above_swap_target()  argument
247 	for (i = 0; i < glob->num_zones; ++i) {  in ttm_zones_above_swap_target()
248 		zone = glob->zones[i];  in ttm_zones_above_swap_target()
272 static void ttm_shrink(struct ttm_mem_global *glob, bool from_wq,  in ttm_shrink()  argument
277 	spin_lock(&glob->lock);  in ttm_shrink()
279 	while (ttm_zones_above_swap_target(glob, from_wq, extra)) {  in ttm_shrink()
280 		spin_unlock(&glob->lock);  in ttm_shrink()
282 		spin_lock(&glob->lock);  in ttm_shrink()
287 	spin_unlock(&glob->lock);  in ttm_shrink()
296 	struct ttm_mem_global *glob =  in ttm_shrink_work()  local
299 	ttm_shrink(glob, true, 0ULL, &ctx);  in ttm_shrink_work()
302 static int ttm_mem_init_kernel_zone(struct ttm_mem_global *glob,  in ttm_mem_init_kernel_zone()  argument
321 	zone->glob = glob;  in ttm_mem_init_kernel_zone()
322 	glob->zone_kernel = zone;  in ttm_mem_init_kernel_zone()
324 		&zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, zone->name);  in ttm_mem_init_kernel_zone()
329 	glob->zones[glob->num_zones++] = zone;  in ttm_mem_init_kernel_zone()
334 static int ttm_mem_init_highmem_zone(struct ttm_mem_global *glob,  in ttm_mem_init_highmem_zone()  argument
357 	zone->glob = glob;  in ttm_mem_init_highmem_zone()
358 	glob->zone_highmem = zone;  in ttm_mem_init_highmem_zone()
360 		&zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, "%s",  in ttm_mem_init_highmem_zone()
366 	glob->zones[glob->num_zones++] = zone;  in ttm_mem_init_highmem_zone()
370 static int ttm_mem_init_dma32_zone(struct ttm_mem_global *glob,  in ttm_mem_init_dma32_zone()  argument
405 	zone->glob = glob;  in ttm_mem_init_dma32_zone()
406 	glob->zone_dma32 = zone;  in ttm_mem_init_dma32_zone()
408 		&zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, zone->name);  in ttm_mem_init_dma32_zone()
413 	glob->zones[glob->num_zones++] = zone;  in ttm_mem_init_dma32_zone()
418 int ttm_mem_global_init(struct ttm_mem_global *glob, struct device *dev)  in ttm_mem_global_init()  argument
425 	spin_lock_init(&glob->lock);  in ttm_mem_global_init()
426 	glob->swap_queue = create_singlethread_workqueue("ttm_swap");  in ttm_mem_global_init()
427 	INIT_WORK(&glob->work, ttm_shrink_work);  in ttm_mem_global_init()
429 	ret = kobject_init_and_add(&glob->kobj, &ttm_mem_glob_kobj_type,  in ttm_mem_global_init()
432 		kobject_put(&glob->kobj);  in ttm_mem_global_init()
438 	spin_lock(&glob->lock);  in ttm_mem_global_init()
440 	glob->lower_mem_limit = 0;  in ttm_mem_global_init()
441 	spin_unlock(&glob->lock);  in ttm_mem_global_init()
443 	ret = ttm_mem_init_kernel_zone(glob, &si);  in ttm_mem_global_init()
447 	ret = ttm_mem_init_highmem_zone(glob, &si);  in ttm_mem_global_init()
451 	ret = ttm_mem_init_dma32_zone(glob, &si);  in ttm_mem_global_init()
455 	for (i = 0; i < glob->num_zones; ++i) {  in ttm_mem_global_init()
456 		zone = glob->zones[i];  in ttm_mem_global_init()
462 	ttm_mem_global_release(glob);  in ttm_mem_global_init()
466 void ttm_mem_global_release(struct ttm_mem_global *glob)  in ttm_mem_global_release()  argument
471 	flush_workqueue(glob->swap_queue);  in ttm_mem_global_release()
472 	destroy_workqueue(glob->swap_queue);  in ttm_mem_global_release()
473 	glob->swap_queue = NULL;  in ttm_mem_global_release()
474 	for (i = 0; i < glob->num_zones; ++i) {  in ttm_mem_global_release()
475 		zone = glob->zones[i];  in ttm_mem_global_release()
479 	kobject_del(&glob->kobj);  in ttm_mem_global_release()
480 	kobject_put(&glob->kobj);  in ttm_mem_global_release()
481 	memset(glob, 0, sizeof(*glob));  in ttm_mem_global_release()
484 static void ttm_check_swapping(struct ttm_mem_global *glob)  in ttm_check_swapping()  argument
490 	spin_lock(&glob->lock);  in ttm_check_swapping()
491 	for (i = 0; i < glob->num_zones; ++i) {  in ttm_check_swapping()
492 		zone = glob->zones[i];  in ttm_check_swapping()
499 	spin_unlock(&glob->lock);  in ttm_check_swapping()
502 		(void)queue_work(glob->swap_queue, &glob->work);  in ttm_check_swapping()
506 static void ttm_mem_global_free_zone(struct ttm_mem_global *glob,  in ttm_mem_global_free_zone()  argument
513 	spin_lock(&glob->lock);  in ttm_mem_global_free_zone()
514 	for (i = 0; i < glob->num_zones; ++i) {  in ttm_mem_global_free_zone()
515 		zone = glob->zones[i];  in ttm_mem_global_free_zone()
520 	spin_unlock(&glob->lock);  in ttm_mem_global_free_zone()
523 void ttm_mem_global_free(struct ttm_mem_global *glob,  in ttm_mem_global_free()  argument
526 	return ttm_mem_global_free_zone(glob, glob->zone_kernel, amount);  in ttm_mem_global_free()
541 ttm_check_under_lowerlimit(struct ttm_mem_global *glob,  in ttm_check_under_lowerlimit()  argument
553 	if (available < glob->lower_mem_limit)  in ttm_check_under_lowerlimit()
559 static int ttm_mem_global_reserve(struct ttm_mem_global *glob,  in ttm_mem_global_reserve()  argument
568 	spin_lock(&glob->lock);  in ttm_mem_global_reserve()
569 	for (i = 0; i < glob->num_zones; ++i) {  in ttm_mem_global_reserve()
570 		zone = glob->zones[i];  in ttm_mem_global_reserve()
582 		for (i = 0; i < glob->num_zones; ++i) {  in ttm_mem_global_reserve()
583 			zone = glob->zones[i];  in ttm_mem_global_reserve()
592 	spin_unlock(&glob->lock);  in ttm_mem_global_reserve()
593 	ttm_check_swapping(glob);  in ttm_mem_global_reserve()
599 static int ttm_mem_global_alloc_zone(struct ttm_mem_global *glob,  in ttm_mem_global_alloc_zone()  argument
606 	while (unlikely(ttm_mem_global_reserve(glob,  in ttm_mem_global_alloc_zone()
614 		ttm_shrink(glob, false, memory + (memory >> 2) + 16, ctx);  in ttm_mem_global_alloc_zone()
620 int ttm_mem_global_alloc(struct ttm_mem_global *glob, uint64_t memory,  in ttm_mem_global_alloc()  argument
628 	return ttm_mem_global_alloc_zone(glob, glob->zone_kernel, memory, ctx);  in ttm_mem_global_alloc()
632 int ttm_mem_global_alloc_page(struct ttm_mem_global *glob,  in ttm_mem_global_alloc_page()  argument
644 	if (PageHighMem(page) && glob->zone_highmem != NULL)  in ttm_mem_global_alloc_page()
645 		zone = glob->zone_highmem;  in ttm_mem_global_alloc_page()
647 	if (glob->zone_dma32 && page_to_pfn(page) > 0x00100000UL)  in ttm_mem_global_alloc_page()
648 		zone = glob->zone_kernel;  in ttm_mem_global_alloc_page()
650 	return ttm_mem_global_alloc_zone(glob, zone, size, ctx);  in ttm_mem_global_alloc_page()
653 void ttm_mem_global_free_page(struct ttm_mem_global *glob, struct page *page,  in ttm_mem_global_free_page()  argument
659 	if (PageHighMem(page) && glob->zone_highmem != NULL)  in ttm_mem_global_free_page()
660 		zone = glob->zone_highmem;  in ttm_mem_global_free_page()
662 	if (glob->zone_dma32 && page_to_pfn(page) > 0x00100000UL)  in ttm_mem_global_free_page()
663 		zone = glob->zone_kernel;  in ttm_mem_global_free_page()
665 	ttm_mem_global_free_zone(glob, zone, size);  in ttm_mem_global_free_page()