201 #define SHRINKER_REGISTERING ((struct shrinker *)~0UL)
206 static int prealloc_memcg_shrinker(struct shrinker *shrinker)  in prealloc_memcg_shrinker()  argument
224 	shrinker->id = id;  in prealloc_memcg_shrinker()
231 static void unregister_memcg_shrinker(struct shrinker *shrinker)  in unregister_memcg_shrinker()  argument
233 	int id = shrinker->id;  in unregister_memcg_shrinker()
296 static int prealloc_memcg_shrinker(struct shrinker *shrinker)  in prealloc_memcg_shrinker()  argument
301 static void unregister_memcg_shrinker(struct shrinker *shrinker)  in unregister_memcg_shrinker()  argument
385 int prealloc_shrinker(struct shrinker *shrinker)  in prealloc_shrinker()  argument
387 	unsigned int size = sizeof(*shrinker->nr_deferred);  in prealloc_shrinker()
389 	if (shrinker->flags & SHRINKER_NUMA_AWARE)  in prealloc_shrinker()
392 	shrinker->nr_deferred = kzalloc(size, GFP_KERNEL);  in prealloc_shrinker()
393 	if (!shrinker->nr_deferred)  in prealloc_shrinker()
396 	if (shrinker->flags & SHRINKER_MEMCG_AWARE) {  in prealloc_shrinker()
397 		if (prealloc_memcg_shrinker(shrinker))  in prealloc_shrinker()
404 	kfree(shrinker->nr_deferred);  in prealloc_shrinker()
405 	shrinker->nr_deferred = NULL;  in prealloc_shrinker()
409 void free_prealloced_shrinker(struct shrinker *shrinker)  in free_prealloced_shrinker()  argument
411 	if (!shrinker->nr_deferred)  in free_prealloced_shrinker()
414 	if (shrinker->flags & SHRINKER_MEMCG_AWARE)  in free_prealloced_shrinker()
415 		unregister_memcg_shrinker(shrinker);  in free_prealloced_shrinker()
417 	kfree(shrinker->nr_deferred);  in free_prealloced_shrinker()
418 	shrinker->nr_deferred = NULL;  in free_prealloced_shrinker()
421 void register_shrinker_prepared(struct shrinker *shrinker)  in register_shrinker_prepared()  argument
424 	list_add_tail(&shrinker->list, &shrinker_list);  in register_shrinker_prepared()
426 	if (shrinker->flags & SHRINKER_MEMCG_AWARE)  in register_shrinker_prepared()
427 		idr_replace(&shrinker_idr, shrinker, shrinker->id);  in register_shrinker_prepared()
432 int register_shrinker(struct shrinker *shrinker)  in register_shrinker()  argument
434 	int err = prealloc_shrinker(shrinker);  in register_shrinker()
438 	register_shrinker_prepared(shrinker);  in register_shrinker()
446 void unregister_shrinker(struct shrinker *shrinker)  in unregister_shrinker()  argument
448 	if (!shrinker->nr_deferred)  in unregister_shrinker()
450 	if (shrinker->flags & SHRINKER_MEMCG_AWARE)  in unregister_shrinker()
451 		unregister_memcg_shrinker(shrinker);  in unregister_shrinker()
453 	list_del(&shrinker->list);  in unregister_shrinker()
455 	kfree(shrinker->nr_deferred);  in unregister_shrinker()
456 	shrinker->nr_deferred = NULL;  in unregister_shrinker()
463 				    struct shrinker *shrinker, int priority)  in do_shrink_slab()  argument
472 	long batch_size = shrinker->batch ? shrinker->batch  in do_shrink_slab()
476 	if (!(shrinker->flags & SHRINKER_NUMA_AWARE))  in do_shrink_slab()
479 	freeable = shrinker->count_objects(shrinker, shrinkctl);  in do_shrink_slab()
488 	nr = atomic_long_xchg(&shrinker->nr_deferred[nid], 0);  in do_shrink_slab()
491 	if (shrinker->seeks) {  in do_shrink_slab()
494 		do_div(delta, shrinker->seeks);  in do_shrink_slab()
507 		       shrinker->scan_objects, total_scan);  in do_shrink_slab()
536 	trace_mm_shrink_slab_start(shrinker, shrinkctl, nr,  in do_shrink_slab()
561 		ret = shrinker->scan_objects(shrinker, shrinkctl);  in do_shrink_slab()
584 						&shrinker->nr_deferred[nid]);  in do_shrink_slab()
586 		new_nr = atomic_long_read(&shrinker->nr_deferred[nid]);  in do_shrink_slab()
588 	trace_mm_shrink_slab_end(shrinker, nid, freed, nr, new_nr, total_scan);  in do_shrink_slab()
617 		struct shrinker *shrinker;  in shrink_slab_memcg()  local
619 		shrinker = idr_find(&shrinker_idr, i);  in shrink_slab_memcg()
620 		if (unlikely(!shrinker || shrinker == SHRINKER_REGISTERING)) {  in shrink_slab_memcg()
621 			if (!shrinker)  in shrink_slab_memcg()
628 		    !(shrinker->flags & SHRINKER_NONSLAB))  in shrink_slab_memcg()
631 		ret = do_shrink_slab(&sc, shrinker, priority);  in shrink_slab_memcg()
650 			ret = do_shrink_slab(&sc, shrinker, priority);  in shrink_slab_memcg()
700 	struct shrinker *shrinker;  in shrink_slab()  local
715 	list_for_each_entry(shrinker, &shrinker_list, list) {  in shrink_slab()
722 		ret = do_shrink_slab(&sc, shrinker, priority);  in shrink_slab()