119 static inline int kmem_cache_debug(struct kmem_cache *s)  in kmem_cache_debug()
128 void *fixup_red_left(struct kmem_cache *s, void *p)  in fixup_red_left()
136 static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s)  in kmem_cache_has_cpu_partial()
217 static int sysfs_slab_add(struct kmem_cache *);
218 static int sysfs_slab_alias(struct kmem_cache *, const char *);
219 static void memcg_propagate_slab_attrs(struct kmem_cache *s);
220 static void sysfs_slab_remove(struct kmem_cache *s);
222 static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; }  in sysfs_slab_add()
223 static inline int sysfs_slab_alias(struct kmem_cache *s, const char *p)  in sysfs_slab_alias()
225 static inline void memcg_propagate_slab_attrs(struct kmem_cache *s) { }  in memcg_propagate_slab_attrs()
226 static inline void sysfs_slab_remove(struct kmem_cache *s) { }  in sysfs_slab_remove()
229 static inline void stat(const struct kmem_cache *s, enum stat_item si)  in stat()
249 static inline void *freelist_ptr(const struct kmem_cache *s, void *ptr,  in freelist_ptr()
271 static inline void *freelist_dereference(const struct kmem_cache *s,  in freelist_dereference()
278 static inline void *get_freepointer(struct kmem_cache *s, void *object)  in get_freepointer()
283 static void prefetch_freepointer(const struct kmem_cache *s, void *object)  in prefetch_freepointer()
288 static inline void *get_freepointer_safe(struct kmem_cache *s, void *object)  in get_freepointer_safe()
301 static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp)  in set_freepointer()
319 static inline unsigned int slab_index(void *p, struct kmem_cache *s, void *addr)  in slab_index()
365 static inline bool __cmpxchg_double_slab(struct kmem_cache *s, struct page *page,  in __cmpxchg_double_slab()
402 static inline bool cmpxchg_double_slab(struct kmem_cache *s, struct page *page,  in cmpxchg_double_slab()
450 static void get_map(struct kmem_cache *s, struct page *page, unsigned long *map)  in get_map()
459 static inline unsigned int size_from_object(struct kmem_cache *s)  in size_from_object()
467 static inline void *restore_red_left(struct kmem_cache *s, void *p)  in restore_red_left()
508 static inline int check_valid_pointer(struct kmem_cache *s,  in check_valid_pointer()
536 static struct track *get_track(struct kmem_cache *s, void *object,  in get_track()
549 static void set_track(struct kmem_cache *s, void *object,  in set_track()
574 static void init_tracking(struct kmem_cache *s, void *object)  in init_tracking()
602 static void print_tracking(struct kmem_cache *s, void *object)  in print_tracking()
619 static void slab_bug(struct kmem_cache *s, char *fmt, ...)  in slab_bug()
635 static void slab_fix(struct kmem_cache *s, char *fmt, ...)  in slab_fix()
647 static void print_trailer(struct kmem_cache *s, struct page *page, u8 *p)  in print_trailer()
689 void object_err(struct kmem_cache *s, struct page *page,  in object_err()
696 static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,  in slab_err()
710 static void init_object(struct kmem_cache *s, void *object, u8 val)  in init_object()
726 static void restore_bytes(struct kmem_cache *s, char *message, u8 data,  in restore_bytes()
733 static int check_bytes_and_report(struct kmem_cache *s, struct page *page,  in check_bytes_and_report()
797 static int check_pad_bytes(struct kmem_cache *s, struct page *page, u8 *p)  in check_pad_bytes()
819 static int slab_pad_check(struct kmem_cache *s, struct page *page)  in slab_pad_check()
854 static int check_object(struct kmem_cache *s, struct page *page,  in check_object()
910 static int check_slab(struct kmem_cache *s, struct page *page)  in check_slab()
941 static int on_freelist(struct kmem_cache *s, struct page *page, void *search)  in on_freelist()
990 static void trace(struct kmem_cache *s, struct page *page, void *object,  in trace()
1011 static void add_full(struct kmem_cache *s,  in add_full()
1021 static void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, struct page *page)  in remove_full()
1031 static inline unsigned long slabs_node(struct kmem_cache *s, int node)  in slabs_node()
1043 static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects)  in inc_slabs_node()
1058 static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects)  in dec_slabs_node()
1067 static void setup_object_debug(struct kmem_cache *s, struct page *page,  in setup_object_debug()
1078 void setup_page_debug(struct kmem_cache *s, struct page *page, void *addr)  in setup_page_debug()
1088 static inline int alloc_consistency_checks(struct kmem_cache *s,  in alloc_consistency_checks()
1105 static noinline int alloc_debug_processing(struct kmem_cache *s,  in alloc_debug_processing()
1135 static inline int free_consistency_checks(struct kmem_cache *s,  in free_consistency_checks()
1169 	struct kmem_cache *s, struct page *page,  in free_debug_processing()
1342 static inline void setup_object_debug(struct kmem_cache *s,  in setup_object_debug()
1345 void setup_page_debug(struct kmem_cache *s, struct page *page, void *addr) {}  in setup_page_debug()
1347 static inline int alloc_debug_processing(struct kmem_cache *s,  in alloc_debug_processing()
1351 	struct kmem_cache *s, struct page *page,  in free_debug_processing()
1355 static inline int slab_pad_check(struct kmem_cache *s, struct page *page)  in slab_pad_check()
1357 static inline int check_object(struct kmem_cache *s, struct page *page,  in check_object()
1359 static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n,  in add_full()
1361 static inline void remove_full(struct kmem_cache *s, struct kmem_cache_node *n,  in remove_full()
1373 static inline unsigned long slabs_node(struct kmem_cache *s, int node)  in slabs_node()
1377 static inline void inc_slabs_node(struct kmem_cache *s, int node,  in inc_slabs_node()
1379 static inline void dec_slabs_node(struct kmem_cache *s, int node,  in dec_slabs_node()
1402 static __always_inline bool slab_free_hook(struct kmem_cache *s, void *x)  in slab_free_hook()
1427 static inline bool slab_free_freelist_hook(struct kmem_cache *s,  in slab_free_freelist_hook()
1472 static void *setup_object(struct kmem_cache *s, struct page *page,  in setup_object()
1488 static inline struct page *alloc_slab_page(struct kmem_cache *s,  in alloc_slab_page()
1509 static int init_cache_random_seq(struct kmem_cache *s)  in init_cache_random_seq()
1538 	struct kmem_cache *s;  in init_freelist_randomization()
1549 static void *next_freelist_entry(struct kmem_cache *s, struct page *page,  in next_freelist_entry()
1571 static bool shuffle_freelist(struct kmem_cache *s, struct page *page)  in shuffle_freelist()
1605 static inline int init_cache_random_seq(struct kmem_cache *s)  in init_cache_random_seq()
1610 static inline bool shuffle_freelist(struct kmem_cache *s, struct page *page)  in shuffle_freelist()
1616 static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)  in allocate_slab()
1696 static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)  in new_slab()
1710 static void __free_slab(struct kmem_cache *s, struct page *page)  in __free_slab()
1741 static void free_slab(struct kmem_cache *s, struct page *page)  in free_slab()
1749 static void discard_slab(struct kmem_cache *s, struct page *page)  in discard_slab()
1789 static inline void *acquire_slab(struct kmem_cache *s,  in acquire_slab()
1829 static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);
1835 static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n,  in get_partial_node()
1884 static void *get_any_partial(struct kmem_cache *s, gfp_t flags,  in get_any_partial()
1948 static void *get_partial(struct kmem_cache *s, gfp_t flags, int node,  in get_partial()
2004 		const struct kmem_cache *s, unsigned long tid)  in note_cmpxchg_failure()
2027 static void init_kmem_cache_cpus(struct kmem_cache *s)  in init_kmem_cache_cpus()
2038 static void deactivate_slab(struct kmem_cache *s, struct page *page,  in deactivate_slab()
2183 static void unfreeze_partials(struct kmem_cache *s,  in unfreeze_partials()
2251 static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain)  in put_cpu_partial()
2303 static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)  in flush_slab()
2316 static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu)  in __flush_cpu_slab()
2328 	struct kmem_cache *s = d;  in flush_cpu_slab()
2335 	struct kmem_cache *s = info;  in has_cpu_slab()
2341 static void flush_all(struct kmem_cache *s)  in flush_all()
2352 	struct kmem_cache *s;  in slub_cpu_dead()
2407 slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid)  in slab_out_of_memory()
2443 static inline void *new_slab_objects(struct kmem_cache *s, gfp_t flags,  in new_slab_objects()
2496 static inline void *get_freelist(struct kmem_cache *s, struct page *page)  in get_freelist()
2539 static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,  in ___slab_alloc()
2632 static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,  in __slab_alloc()
2657 static __always_inline void maybe_wipe_obj_freeptr(struct kmem_cache *s,  in maybe_wipe_obj_freeptr()
2674 static __always_inline void *slab_alloc_node(struct kmem_cache *s,  in slab_alloc_node()
2763 static __always_inline void *slab_alloc(struct kmem_cache *s,  in slab_alloc()
2769 void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags)  in kmem_cache_alloc()
2781 void *kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size)  in kmem_cache_alloc_trace()
2792 void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node)  in kmem_cache_alloc_node()
2804 void *kmem_cache_alloc_node_trace(struct kmem_cache *s,  in kmem_cache_alloc_node_trace()
2828 static void __slab_free(struct kmem_cache *s, struct page *page,  in __slab_free()
2956 static __always_inline void do_slab_free(struct kmem_cache *s,  in do_slab_free()
2996 static __always_inline void slab_free(struct kmem_cache *s, struct page *page,  in slab_free()
3009 void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr)  in ___cache_free()
3015 void kmem_cache_free(struct kmem_cache *s, void *x)  in kmem_cache_free()
3030 	struct kmem_cache *s;
3046 int build_detached_freelist(struct kmem_cache *s, size_t size,  in build_detached_freelist()
3117 void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)  in kmem_cache_free_bulk()
3135 int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,  in kmem_cache_alloc_bulk()
3335 static inline int alloc_kmem_cache_cpus(struct kmem_cache *s)  in alloc_kmem_cache_cpus()
3355 static struct kmem_cache *kmem_cache_node;
3403 static void free_kmem_cache_nodes(struct kmem_cache *s)  in free_kmem_cache_nodes()
3414 void __kmem_cache_release(struct kmem_cache *s)  in __kmem_cache_release()
3421 static int init_kmem_cache_nodes(struct kmem_cache *s)  in init_kmem_cache_nodes()
3446 static void set_min_partial(struct kmem_cache *s, unsigned long min)  in set_min_partial()
3455 static void set_cpu_partial(struct kmem_cache *s)  in set_cpu_partial()
3492 static int calculate_sizes(struct kmem_cache *s, int forced_order)  in calculate_sizes()
3613 static int kmem_cache_open(struct kmem_cache *s, slab_flags_t flags)  in kmem_cache_open()
3671 static void list_slab_objects(struct kmem_cache *s, struct page *page,  in list_slab_objects()
3701 static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n)  in free_partial()
3723 bool __kmem_cache_empty(struct kmem_cache *s)  in __kmem_cache_empty()
3737 int __kmem_cache_shutdown(struct kmem_cache *s)  in __kmem_cache_shutdown()
3787 	struct kmem_cache *s;  in __kmalloc()
3828 	struct kmem_cache *s;  in __kmalloc_node()
3869 	struct kmem_cache *s;  in __check_heap_object()
3971 int __kmem_cache_shrink(struct kmem_cache *s)  in __kmem_cache_shrink()
4034 void __kmemcg_cache_deactivate_after_rcu(struct kmem_cache *s)  in __kmemcg_cache_deactivate_after_rcu()
4052 void __kmemcg_cache_deactivate(struct kmem_cache *s)  in __kmemcg_cache_deactivate()
4065 	struct kmem_cache *s;  in slab_mem_going_offline_callback()
4078 	struct kmem_cache *s;  in slab_mem_offline_callback()
4113 	struct kmem_cache *s;  in slab_mem_going_online_callback()
4192 static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache)  in bootstrap()
4195 	struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);  in bootstrap()
4198 	memcpy(s, static_cache, kmem_cache->object_size);  in bootstrap()
4225 	static __initdata struct kmem_cache boot_kmem_cache,  in kmem_cache_init()
4232 	kmem_cache = &boot_kmem_cache;  in kmem_cache_init()
4242 	create_boot_cache(kmem_cache, "kmem_cache",  in kmem_cache_init()
4243 			offsetof(struct kmem_cache, node) +  in kmem_cache_init()
4247 	kmem_cache = bootstrap(&boot_kmem_cache);  in kmem_cache_init()
4270 struct kmem_cache *
4274 	struct kmem_cache *s, *c;  in __kmem_cache_alias()
4301 int __kmem_cache_create(struct kmem_cache *s, slab_flags_t flags)  in __kmem_cache_create()
4323 	struct kmem_cache *s;  in __kmalloc_track_caller()
4346 	struct kmem_cache *s;  in __kmalloc_node_track_caller()
4386 static int validate_slab(struct kmem_cache *s, struct page *page,  in validate_slab()
4413 static void validate_slab_slab(struct kmem_cache *s, struct page *page,  in validate_slab_slab()
4421 static int validate_slab_node(struct kmem_cache *s,  in validate_slab_node()
4454 static long validate_slab_cache(struct kmem_cache *s)  in validate_slab_cache()
4520 static int add_location(struct loc_track *t, struct kmem_cache *s,  in add_location()
4596 static void process_slab(struct loc_track *t, struct kmem_cache *s,  in process_slab()
4611 static int list_locations(struct kmem_cache *s, char *buf,  in list_locations()
4787 static ssize_t show_slab_objects(struct kmem_cache *s,  in show_slab_objects()
4894 static int any_slab_objects(struct kmem_cache *s)  in any_slab_objects()
4908 #define to_slab(n) container_of(n, struct kmem_cache, kobj)
4912 	ssize_t (*show)(struct kmem_cache *s, char *buf);
4913 	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
4924 static ssize_t slab_size_show(struct kmem_cache *s, char *buf)  in slab_size_show()
4930 static ssize_t align_show(struct kmem_cache *s, char *buf)  in align_show()
4936 static ssize_t object_size_show(struct kmem_cache *s, char *buf)  in object_size_show()
4942 static ssize_t objs_per_slab_show(struct kmem_cache *s, char *buf)  in objs_per_slab_show()
4948 static ssize_t order_store(struct kmem_cache *s,  in order_store()
4965 static ssize_t order_show(struct kmem_cache *s, char *buf)  in order_show()
4971 static ssize_t min_partial_show(struct kmem_cache *s, char *buf)  in min_partial_show()
4976 static ssize_t min_partial_store(struct kmem_cache *s, const char *buf,  in min_partial_store()
4991 static ssize_t cpu_partial_show(struct kmem_cache *s, char *buf)  in cpu_partial_show()
4996 static ssize_t cpu_partial_store(struct kmem_cache *s, const char *buf,  in cpu_partial_store()
5014 static ssize_t ctor_show(struct kmem_cache *s, char *buf)  in ctor_show()
5022 static ssize_t aliases_show(struct kmem_cache *s, char *buf)  in aliases_show()
5028 static ssize_t partial_show(struct kmem_cache *s, char *buf)  in partial_show()
5034 static ssize_t cpu_slabs_show(struct kmem_cache *s, char *buf)  in cpu_slabs_show()
5040 static ssize_t objects_show(struct kmem_cache *s, char *buf)  in objects_show()
5046 static ssize_t objects_partial_show(struct kmem_cache *s, char *buf)  in objects_partial_show()
5052 static ssize_t slabs_cpu_partial_show(struct kmem_cache *s, char *buf)  in slabs_cpu_partial_show()
5087 static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf)  in reclaim_account_show()
5092 static ssize_t reclaim_account_store(struct kmem_cache *s,  in reclaim_account_store()
5102 static ssize_t hwcache_align_show(struct kmem_cache *s, char *buf)  in hwcache_align_show()
5109 static ssize_t cache_dma_show(struct kmem_cache *s, char *buf)  in cache_dma_show()
5116 static ssize_t usersize_show(struct kmem_cache *s, char *buf)  in usersize_show()
5122 static ssize_t destroy_by_rcu_show(struct kmem_cache *s, char *buf)  in destroy_by_rcu_show()
5129 static ssize_t slabs_show(struct kmem_cache *s, char *buf)  in slabs_show()
5135 static ssize_t total_objects_show(struct kmem_cache *s, char *buf)  in total_objects_show()
5141 static ssize_t sanity_checks_show(struct kmem_cache *s, char *buf)  in sanity_checks_show()
5146 static ssize_t sanity_checks_store(struct kmem_cache *s,  in sanity_checks_store()
5158 static ssize_t trace_show(struct kmem_cache *s, char *buf)  in trace_show()
5163 static ssize_t trace_store(struct kmem_cache *s, const char *buf,  in trace_store()
5183 static ssize_t red_zone_show(struct kmem_cache *s, char *buf)  in red_zone_show()
5188 static ssize_t red_zone_store(struct kmem_cache *s,  in red_zone_store()
5203 static ssize_t poison_show(struct kmem_cache *s, char *buf)  in poison_show()
5208 static ssize_t poison_store(struct kmem_cache *s,  in poison_store()
5223 static ssize_t store_user_show(struct kmem_cache *s, char *buf)  in store_user_show()
5228 static ssize_t store_user_store(struct kmem_cache *s,  in store_user_store()
5244 static ssize_t validate_show(struct kmem_cache *s, char *buf)  in validate_show()
5249 static ssize_t validate_store(struct kmem_cache *s,  in validate_store()
5263 static ssize_t alloc_calls_show(struct kmem_cache *s, char *buf)  in alloc_calls_show()
5271 static ssize_t free_calls_show(struct kmem_cache *s, char *buf)  in free_calls_show()
5281 static ssize_t failslab_show(struct kmem_cache *s, char *buf)  in failslab_show()
5286 static ssize_t failslab_store(struct kmem_cache *s, const char *buf,  in failslab_store()
5300 static ssize_t shrink_show(struct kmem_cache *s, char *buf)  in shrink_show()
5305 static ssize_t shrink_store(struct kmem_cache *s,  in shrink_store()
5317 static ssize_t remote_node_defrag_ratio_show(struct kmem_cache *s, char *buf)  in remote_node_defrag_ratio_show()
5322 static ssize_t remote_node_defrag_ratio_store(struct kmem_cache *s,  in remote_node_defrag_ratio_store()
5342 static int show_stat(struct kmem_cache *s, char *buf, enum stat_item si)  in show_stat()
5371 static void clear_stat(struct kmem_cache *s, enum stat_item si)  in clear_stat()
5380 static ssize_t text##_show(struct kmem_cache *s, char *buf)	\
5384 static ssize_t text##_store(struct kmem_cache *s,		\
5504 	struct kmem_cache *s;  in slab_attr_show()
5523 	struct kmem_cache *s;  in slab_attr_store()
5535 		struct kmem_cache *c;  in slab_attr_store()
5566 static void memcg_propagate_slab_attrs(struct kmem_cache *s)  in memcg_propagate_slab_attrs()
5571 	struct kmem_cache *root_cache;  in memcg_propagate_slab_attrs()
5654 static inline struct kset *cache_kset(struct kmem_cache *s)  in cache_kset()
5669 static char *create_unique_id(struct kmem_cache *s)  in create_unique_id()
5704 	struct kmem_cache *s =  in sysfs_slab_remove_workfn()
5705 		container_of(work, struct kmem_cache, kobj_remove_work);  in sysfs_slab_remove_workfn()
5724 static int sysfs_slab_add(struct kmem_cache *s)  in sysfs_slab_add()
5791 static void sysfs_slab_remove(struct kmem_cache *s)  in sysfs_slab_remove()
5804 void sysfs_slab_unlink(struct kmem_cache *s)  in sysfs_slab_unlink()
5810 void sysfs_slab_release(struct kmem_cache *s)  in sysfs_slab_release()
5821 	struct kmem_cache *s;
5828 static int sysfs_slab_alias(struct kmem_cache *s, const char *name)  in sysfs_slab_alias()
5853 	struct kmem_cache *s;  in slab_sysfs_init()
5897 void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo)  in get_slabinfo()
5919 void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s)  in slabinfo_show_stats()