185 #define SHRINKER_REGISTERING ((struct shrinker *)~0UL)
190 static int prealloc_memcg_shrinker(struct shrinker *shrinker)  in prealloc_memcg_shrinker()  argument
208 	shrinker->id = id;  in prealloc_memcg_shrinker()
215 static void unregister_memcg_shrinker(struct shrinker *shrinker)  in unregister_memcg_shrinker()  argument
217 	int id = shrinker->id;  in unregister_memcg_shrinker()
226 static int prealloc_memcg_shrinker(struct shrinker *shrinker)  in prealloc_memcg_shrinker()  argument
231 static void unregister_memcg_shrinker(struct shrinker *shrinker)  in unregister_memcg_shrinker()  argument
370 int prealloc_shrinker(struct shrinker *shrinker)  in prealloc_shrinker()  argument
372 	size_t size = sizeof(*shrinker->nr_deferred);  in prealloc_shrinker()
374 	if (shrinker->flags & SHRINKER_NUMA_AWARE)  in prealloc_shrinker()
377 	shrinker->nr_deferred = kzalloc(size, GFP_KERNEL);  in prealloc_shrinker()
378 	if (!shrinker->nr_deferred)  in prealloc_shrinker()
381 	if (shrinker->flags & SHRINKER_MEMCG_AWARE) {  in prealloc_shrinker()
382 		if (prealloc_memcg_shrinker(shrinker))  in prealloc_shrinker()
389 	kfree(shrinker->nr_deferred);  in prealloc_shrinker()
390 	shrinker->nr_deferred = NULL;  in prealloc_shrinker()
394 void free_prealloced_shrinker(struct shrinker *shrinker)  in free_prealloced_shrinker()  argument
396 	if (!shrinker->nr_deferred)  in free_prealloced_shrinker()
399 	if (shrinker->flags & SHRINKER_MEMCG_AWARE)  in free_prealloced_shrinker()
400 		unregister_memcg_shrinker(shrinker);  in free_prealloced_shrinker()
402 	kfree(shrinker->nr_deferred);  in free_prealloced_shrinker()
403 	shrinker->nr_deferred = NULL;  in free_prealloced_shrinker()
406 void register_shrinker_prepared(struct shrinker *shrinker)  in register_shrinker_prepared()  argument
409 	list_add_tail(&shrinker->list, &shrinker_list);  in register_shrinker_prepared()
411 	if (shrinker->flags & SHRINKER_MEMCG_AWARE)  in register_shrinker_prepared()
412 		idr_replace(&shrinker_idr, shrinker, shrinker->id);  in register_shrinker_prepared()
417 int register_shrinker(struct shrinker *shrinker)  in register_shrinker()  argument
419 	int err = prealloc_shrinker(shrinker);  in register_shrinker()
423 	register_shrinker_prepared(shrinker);  in register_shrinker()
431 void unregister_shrinker(struct shrinker *shrinker)  in unregister_shrinker()  argument
433 	if (!shrinker->nr_deferred)  in unregister_shrinker()
435 	if (shrinker->flags & SHRINKER_MEMCG_AWARE)  in unregister_shrinker()
436 		unregister_memcg_shrinker(shrinker);  in unregister_shrinker()
438 	list_del(&shrinker->list);  in unregister_shrinker()
440 	kfree(shrinker->nr_deferred);  in unregister_shrinker()
441 	shrinker->nr_deferred = NULL;  in unregister_shrinker()
448 				    struct shrinker *shrinker, int priority)  in do_shrink_slab()  argument
457 	long batch_size = shrinker->batch ? shrinker->batch  in do_shrink_slab()
461 	if (!(shrinker->flags & SHRINKER_NUMA_AWARE))  in do_shrink_slab()
464 	freeable = shrinker->count_objects(shrinker, shrinkctl);  in do_shrink_slab()
473 	nr = atomic_long_xchg(&shrinker->nr_deferred[nid], 0);  in do_shrink_slab()
478 	do_div(delta, shrinker->seeks);  in do_shrink_slab()
493 		       shrinker->scan_objects, total_scan);  in do_shrink_slab()
522 	trace_mm_shrink_slab_start(shrinker, shrinkctl, nr,  in do_shrink_slab()
547 		ret = shrinker->scan_objects(shrinker, shrinkctl);  in do_shrink_slab()
570 						&shrinker->nr_deferred[nid]);  in do_shrink_slab()
572 		new_nr = atomic_long_read(&shrinker->nr_deferred[nid]);  in do_shrink_slab()
574 	trace_mm_shrink_slab_end(shrinker, nid, freed, nr, new_nr, total_scan);  in do_shrink_slab()
603 		struct shrinker *shrinker;  in shrink_slab_memcg()  local
605 		shrinker = idr_find(&shrinker_idr, i);  in shrink_slab_memcg()
606 		if (unlikely(!shrinker || shrinker == SHRINKER_REGISTERING)) {  in shrink_slab_memcg()
607 			if (!shrinker)  in shrink_slab_memcg()
612 		ret = do_shrink_slab(&sc, shrinker, priority);  in shrink_slab_memcg()
631 			ret = do_shrink_slab(&sc, shrinker, priority);  in shrink_slab_memcg()
681 	struct shrinker *shrinker;  in shrink_slab()  local
689 	list_for_each_entry(shrinker, &shrinker_list, list) {  in shrink_slab()
696 		ret = do_shrink_slab(&sc, shrinker, priority);  in shrink_slab()