201 #define SHRINKER_REGISTERING ((struct shrinker *)~0UL)
206 static int prealloc_memcg_shrinker(struct shrinker *shrinker)  in prealloc_memcg_shrinker()  argument
224 	shrinker->id = id;  in prealloc_memcg_shrinker()
231 static void unregister_memcg_shrinker(struct shrinker *shrinker)  in unregister_memcg_shrinker()  argument
233 	int id = shrinker->id;  in unregister_memcg_shrinker()
271 static int prealloc_memcg_shrinker(struct shrinker *shrinker)  in prealloc_memcg_shrinker()  argument
276 static void unregister_memcg_shrinker(struct shrinker *shrinker)  in unregister_memcg_shrinker()  argument
337 int prealloc_shrinker(struct shrinker *shrinker)  in prealloc_shrinker()  argument
339 	unsigned int size = sizeof(*shrinker->nr_deferred);  in prealloc_shrinker()
341 	if (shrinker->flags & SHRINKER_NUMA_AWARE)  in prealloc_shrinker()
344 	shrinker->nr_deferred = kzalloc(size, GFP_KERNEL);  in prealloc_shrinker()
345 	if (!shrinker->nr_deferred)  in prealloc_shrinker()
348 	if (shrinker->flags & SHRINKER_MEMCG_AWARE) {  in prealloc_shrinker()
349 		if (prealloc_memcg_shrinker(shrinker))  in prealloc_shrinker()
356 	kfree(shrinker->nr_deferred);  in prealloc_shrinker()
357 	shrinker->nr_deferred = NULL;  in prealloc_shrinker()
361 void free_prealloced_shrinker(struct shrinker *shrinker)  in free_prealloced_shrinker()  argument
363 	if (!shrinker->nr_deferred)  in free_prealloced_shrinker()
366 	if (shrinker->flags & SHRINKER_MEMCG_AWARE)  in free_prealloced_shrinker()
367 		unregister_memcg_shrinker(shrinker);  in free_prealloced_shrinker()
369 	kfree(shrinker->nr_deferred);  in free_prealloced_shrinker()
370 	shrinker->nr_deferred = NULL;  in free_prealloced_shrinker()
373 void register_shrinker_prepared(struct shrinker *shrinker)  in register_shrinker_prepared()  argument
376 	list_add_tail(&shrinker->list, &shrinker_list);  in register_shrinker_prepared()
378 	if (shrinker->flags & SHRINKER_MEMCG_AWARE)  in register_shrinker_prepared()
379 		idr_replace(&shrinker_idr, shrinker, shrinker->id);  in register_shrinker_prepared()
384 int register_shrinker(struct shrinker *shrinker)  in register_shrinker()  argument
386 	int err = prealloc_shrinker(shrinker);  in register_shrinker()
390 	register_shrinker_prepared(shrinker);  in register_shrinker()
398 void unregister_shrinker(struct shrinker *shrinker)  in unregister_shrinker()  argument
400 	if (!shrinker->nr_deferred)  in unregister_shrinker()
402 	if (shrinker->flags & SHRINKER_MEMCG_AWARE)  in unregister_shrinker()
403 		unregister_memcg_shrinker(shrinker);  in unregister_shrinker()
405 	list_del(&shrinker->list);  in unregister_shrinker()
407 	kfree(shrinker->nr_deferred);  in unregister_shrinker()
408 	shrinker->nr_deferred = NULL;  in unregister_shrinker()
415 				    struct shrinker *shrinker, int priority)  in do_shrink_slab()  argument
424 	long batch_size = shrinker->batch ? shrinker->batch  in do_shrink_slab()
428 	if (!(shrinker->flags & SHRINKER_NUMA_AWARE))  in do_shrink_slab()
431 	freeable = shrinker->count_objects(shrinker, shrinkctl);  in do_shrink_slab()
440 	nr = atomic_long_xchg(&shrinker->nr_deferred[nid], 0);  in do_shrink_slab()
443 	if (shrinker->seeks) {  in do_shrink_slab()
446 		do_div(delta, shrinker->seeks);  in do_shrink_slab()
459 		       shrinker->scan_objects, total_scan);  in do_shrink_slab()
488 	trace_mm_shrink_slab_start(shrinker, shrinkctl, nr,  in do_shrink_slab()
513 		ret = shrinker->scan_objects(shrinker, shrinkctl);  in do_shrink_slab()
536 						&shrinker->nr_deferred[nid]);  in do_shrink_slab()
538 		new_nr = atomic_long_read(&shrinker->nr_deferred[nid]);  in do_shrink_slab()
540 	trace_mm_shrink_slab_end(shrinker, nid, freed, nr, new_nr, total_scan);  in do_shrink_slab()
569 		struct shrinker *shrinker;  in shrink_slab_memcg()  local
571 		shrinker = idr_find(&shrinker_idr, i);  in shrink_slab_memcg()
572 		if (unlikely(!shrinker || shrinker == SHRINKER_REGISTERING)) {  in shrink_slab_memcg()
573 			if (!shrinker)  in shrink_slab_memcg()
580 		    !(shrinker->flags & SHRINKER_NONSLAB))  in shrink_slab_memcg()
583 		ret = do_shrink_slab(&sc, shrinker, priority);  in shrink_slab_memcg()
602 			ret = do_shrink_slab(&sc, shrinker, priority);  in shrink_slab_memcg()
652 	struct shrinker *shrinker;  in shrink_slab()  local
667 	list_for_each_entry(shrinker, &shrinker_list, list) {  in shrink_slab()
674 		ret = do_shrink_slab(&sc, shrinker, priority);  in shrink_slab()