1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef MM_SLAB_H
3 #define MM_SLAB_H
4 /*
5 * Internal slab definitions
6 */
7
8 #ifdef CONFIG_SLOB
9 /*
10 * Common fields provided in kmem_cache by all slab allocators
11 * This struct is either used directly by the allocator (SLOB)
12 * or the allocator must include definitions for all fields
13 * provided in kmem_cache_common in their definition of kmem_cache.
14 *
15 * Once we can do anonymous structs (C11 standard) we could put a
16 * anonymous struct definition in these allocators so that the
17 * separate allocations in the kmem_cache structure of SLAB and
18 * SLUB is no longer needed.
19 */
20 struct kmem_cache {
21 unsigned int object_size;/* The original size of the object */
22 unsigned int size; /* The aligned/padded/added on size */
23 unsigned int align; /* Alignment as calculated */
24 slab_flags_t flags; /* Active flags on the slab */
25 unsigned int useroffset;/* Usercopy region offset */
26 unsigned int usersize; /* Usercopy region size */
27 const char *name; /* Slab name for sysfs */
28 int refcount; /* Use counter */
29 void (*ctor)(void *); /* Called on object slot creation */
30 struct list_head list; /* List of all slab caches on the system */
31 };
32
33 #endif /* CONFIG_SLOB */
34
35 #ifdef CONFIG_SLAB
36 #include <linux/slab_def.h>
37 #endif
38
39 #ifdef CONFIG_SLUB
40 #include <linux/slub_def.h>
41 #endif
42
43 #include <linux/memcontrol.h>
44 #include <linux/fault-inject.h>
45 #include <linux/kasan.h>
46 #include <linux/kmemleak.h>
47 #include <linux/random.h>
48 #include <linux/sched/mm.h>
49
50 /*
51 * State of the slab allocator.
52 *
53 * This is used to describe the states of the allocator during bootup.
54 * Allocators use this to gradually bootstrap themselves. Most allocators
55 * have the problem that the structures used for managing slab caches are
56 * allocated from slab caches themselves.
57 */
58 enum slab_state {
59 DOWN, /* No slab functionality yet */
60 PARTIAL, /* SLUB: kmem_cache_node available */
61 PARTIAL_NODE, /* SLAB: kmalloc size for node struct available */
62 UP, /* Slab caches usable but not all extras yet */
63 FULL /* Everything is working */
64 };
65
66 extern enum slab_state slab_state;
67
68 /* The slab cache mutex protects the management structures during changes */
69 extern struct mutex slab_mutex;
70
71 /* The list of all slab caches on the system */
72 extern struct list_head slab_caches;
73
74 /* The slab cache that manages slab cache information */
75 extern struct kmem_cache *kmem_cache;
76
77 /* A table of kmalloc cache names and sizes */
78 extern const struct kmalloc_info_struct {
79 const char *name[NR_KMALLOC_TYPES];
80 unsigned int size;
81 } kmalloc_info[];
82
83 #ifndef CONFIG_SLOB
84 /* Kmalloc array related functions */
85 void setup_kmalloc_cache_index_table(void);
86 void create_kmalloc_caches(slab_flags_t);
87
88 /* Find the kmalloc slab corresponding for a certain size */
89 struct kmem_cache *kmalloc_slab(size_t, gfp_t);
90 #endif
91
92 gfp_t kmalloc_fix_flags(gfp_t flags);
93
94 /* Functions provided by the slab allocators */
95 int __kmem_cache_create(struct kmem_cache *, slab_flags_t flags);
96
97 struct kmem_cache *create_kmalloc_cache(const char *name, unsigned int size,
98 slab_flags_t flags, unsigned int useroffset,
99 unsigned int usersize);
100 extern void create_boot_cache(struct kmem_cache *, const char *name,
101 unsigned int size, slab_flags_t flags,
102 unsigned int useroffset, unsigned int usersize);
103
104 int slab_unmergeable(struct kmem_cache *s);
105 struct kmem_cache *find_mergeable(unsigned size, unsigned align,
106 slab_flags_t flags, const char *name, void (*ctor)(void *));
107 #ifndef CONFIG_SLOB
108 struct kmem_cache *
109 __kmem_cache_alias(const char *name, unsigned int size, unsigned int align,
110 slab_flags_t flags, void (*ctor)(void *));
111
112 slab_flags_t kmem_cache_flags(unsigned int object_size,
113 slab_flags_t flags, const char *name,
114 void (*ctor)(void *));
115 #else
116 static inline struct kmem_cache *
__kmem_cache_alias(const char * name,unsigned int size,unsigned int align,slab_flags_t flags,void (* ctor)(void *))117 __kmem_cache_alias(const char *name, unsigned int size, unsigned int align,
118 slab_flags_t flags, void (*ctor)(void *))
119 { return NULL; }
120
kmem_cache_flags(unsigned int object_size,slab_flags_t flags,const char * name,void (* ctor)(void *))121 static inline slab_flags_t kmem_cache_flags(unsigned int object_size,
122 slab_flags_t flags, const char *name,
123 void (*ctor)(void *))
124 {
125 return flags;
126 }
127 #endif
128
129
130 /* Legal flag mask for kmem_cache_create(), for various configurations */
131 #define SLAB_CORE_FLAGS (SLAB_HWCACHE_ALIGN | SLAB_CACHE_DMA | \
132 SLAB_CACHE_DMA32 | SLAB_PANIC | \
133 SLAB_TYPESAFE_BY_RCU | SLAB_DEBUG_OBJECTS )
134
135 #if defined(CONFIG_DEBUG_SLAB)
136 #define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER)
137 #elif defined(CONFIG_SLUB_DEBUG)
138 #define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
139 SLAB_TRACE | SLAB_CONSISTENCY_CHECKS)
140 #else
141 #define SLAB_DEBUG_FLAGS (0)
142 #endif
143
144 #if defined(CONFIG_SLAB)
145 #define SLAB_CACHE_FLAGS (SLAB_MEM_SPREAD | SLAB_NOLEAKTRACE | \
146 SLAB_RECLAIM_ACCOUNT | SLAB_TEMPORARY | \
147 SLAB_ACCOUNT)
148 #elif defined(CONFIG_SLUB)
149 #define SLAB_CACHE_FLAGS (SLAB_NOLEAKTRACE | SLAB_RECLAIM_ACCOUNT | \
150 SLAB_TEMPORARY | SLAB_ACCOUNT)
151 #else
152 #define SLAB_CACHE_FLAGS (0)
153 #endif
154
155 /* Common flags available with current configuration */
156 #define CACHE_CREATE_MASK (SLAB_CORE_FLAGS | SLAB_DEBUG_FLAGS | SLAB_CACHE_FLAGS)
157
158 /* Common flags permitted for kmem_cache_create */
159 #define SLAB_FLAGS_PERMITTED (SLAB_CORE_FLAGS | \
160 SLAB_RED_ZONE | \
161 SLAB_POISON | \
162 SLAB_STORE_USER | \
163 SLAB_TRACE | \
164 SLAB_CONSISTENCY_CHECKS | \
165 SLAB_MEM_SPREAD | \
166 SLAB_NOLEAKTRACE | \
167 SLAB_RECLAIM_ACCOUNT | \
168 SLAB_TEMPORARY | \
169 SLAB_ACCOUNT)
170
171 bool __kmem_cache_empty(struct kmem_cache *);
172 int __kmem_cache_shutdown(struct kmem_cache *);
173 void __kmem_cache_release(struct kmem_cache *);
174 int __kmem_cache_shrink(struct kmem_cache *);
175 void slab_kmem_cache_release(struct kmem_cache *);
176
177 struct seq_file;
178 struct file;
179
180 struct slabinfo {
181 unsigned long active_objs;
182 unsigned long num_objs;
183 unsigned long active_slabs;
184 unsigned long num_slabs;
185 unsigned long shared_avail;
186 unsigned int limit;
187 unsigned int batchcount;
188 unsigned int shared;
189 unsigned int objects_per_slab;
190 unsigned int cache_order;
191 };
192
193 void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo);
194 void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s);
195 ssize_t slabinfo_write(struct file *file, const char __user *buffer,
196 size_t count, loff_t *ppos);
197
198 /*
199 * Generic implementation of bulk operations
200 * These are useful for situations in which the allocator cannot
201 * perform optimizations. In that case segments of the object listed
202 * may be allocated or freed using these operations.
203 */
204 void __kmem_cache_free_bulk(struct kmem_cache *, size_t, void **);
205 int __kmem_cache_alloc_bulk(struct kmem_cache *, gfp_t, size_t, void **);
206
cache_vmstat_idx(struct kmem_cache * s)207 static inline int cache_vmstat_idx(struct kmem_cache *s)
208 {
209 return (s->flags & SLAB_RECLAIM_ACCOUNT) ?
210 NR_SLAB_RECLAIMABLE_B : NR_SLAB_UNRECLAIMABLE_B;
211 }
212
213 #ifdef CONFIG_SLUB_DEBUG
214 #ifdef CONFIG_SLUB_DEBUG_ON
215 DECLARE_STATIC_KEY_TRUE(slub_debug_enabled);
216 #else
217 DECLARE_STATIC_KEY_FALSE(slub_debug_enabled);
218 #endif
219 extern void print_tracking(struct kmem_cache *s, void *object);
220 #else
print_tracking(struct kmem_cache * s,void * object)221 static inline void print_tracking(struct kmem_cache *s, void *object)
222 {
223 }
224 #endif
225
226 /*
227 * Returns true if any of the specified slub_debug flags is enabled for the
228 * cache. Use only for flags parsed by setup_slub_debug() as it also enables
229 * the static key.
230 */
kmem_cache_debug_flags(struct kmem_cache * s,slab_flags_t flags)231 static inline bool kmem_cache_debug_flags(struct kmem_cache *s, slab_flags_t flags)
232 {
233 #ifdef CONFIG_SLUB_DEBUG
234 VM_WARN_ON_ONCE(!(flags & SLAB_DEBUG_FLAGS));
235 if (static_branch_unlikely(&slub_debug_enabled))
236 return s->flags & flags;
237 #endif
238 return false;
239 }
240
241 #ifdef CONFIG_MEMCG_KMEM
page_obj_cgroups(struct page * page)242 static inline struct obj_cgroup **page_obj_cgroups(struct page *page)
243 {
244 /*
245 * page->mem_cgroup and page->obj_cgroups are sharing the same
246 * space. To distinguish between them in case we don't know for sure
247 * that the page is a slab page (e.g. page_cgroup_ino()), let's
248 * always set the lowest bit of obj_cgroups.
249 */
250 return (struct obj_cgroup **)
251 ((unsigned long)page->obj_cgroups & ~0x1UL);
252 }
253
page_has_obj_cgroups(struct page * page)254 static inline bool page_has_obj_cgroups(struct page *page)
255 {
256 return ((unsigned long)page->obj_cgroups & 0x1UL);
257 }
258
259 int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s,
260 gfp_t gfp);
261
memcg_free_page_obj_cgroups(struct page * page)262 static inline void memcg_free_page_obj_cgroups(struct page *page)
263 {
264 kfree(page_obj_cgroups(page));
265 page->obj_cgroups = NULL;
266 }
267
obj_full_size(struct kmem_cache * s)268 static inline size_t obj_full_size(struct kmem_cache *s)
269 {
270 /*
271 * For each accounted object there is an extra space which is used
272 * to store obj_cgroup membership. Charge it too.
273 */
274 return s->size + sizeof(struct obj_cgroup *);
275 }
276
277 /*
278 * Returns false if the allocation should fail.
279 */
memcg_slab_pre_alloc_hook(struct kmem_cache * s,struct obj_cgroup ** objcgp,size_t objects,gfp_t flags)280 static inline bool memcg_slab_pre_alloc_hook(struct kmem_cache *s,
281 struct obj_cgroup **objcgp,
282 size_t objects, gfp_t flags)
283 {
284 struct obj_cgroup *objcg;
285
286 if (!memcg_kmem_enabled())
287 return true;
288
289 if (!(flags & __GFP_ACCOUNT) && !(s->flags & SLAB_ACCOUNT))
290 return true;
291
292 objcg = get_obj_cgroup_from_current();
293 if (!objcg)
294 return true;
295
296 if (obj_cgroup_charge(objcg, flags, objects * obj_full_size(s))) {
297 obj_cgroup_put(objcg);
298 return false;
299 }
300
301 *objcgp = objcg;
302 return true;
303 }
304
mod_objcg_state(struct obj_cgroup * objcg,struct pglist_data * pgdat,int idx,int nr)305 static inline void mod_objcg_state(struct obj_cgroup *objcg,
306 struct pglist_data *pgdat,
307 int idx, int nr)
308 {
309 struct mem_cgroup *memcg;
310 struct lruvec *lruvec;
311
312 rcu_read_lock();
313 memcg = obj_cgroup_memcg(objcg);
314 lruvec = mem_cgroup_lruvec(memcg, pgdat);
315 mod_memcg_lruvec_state(lruvec, idx, nr);
316 rcu_read_unlock();
317 }
318
memcg_slab_post_alloc_hook(struct kmem_cache * s,struct obj_cgroup * objcg,gfp_t flags,size_t size,void ** p)319 static inline void memcg_slab_post_alloc_hook(struct kmem_cache *s,
320 struct obj_cgroup *objcg,
321 gfp_t flags, size_t size,
322 void **p)
323 {
324 struct page *page;
325 unsigned long off;
326 size_t i;
327
328 if (!memcg_kmem_enabled() || !objcg)
329 return;
330
331 flags &= ~__GFP_ACCOUNT;
332 for (i = 0; i < size; i++) {
333 if (likely(p[i])) {
334 page = virt_to_head_page(p[i]);
335
336 if (!page_has_obj_cgroups(page) &&
337 memcg_alloc_page_obj_cgroups(page, s, flags)) {
338 obj_cgroup_uncharge(objcg, obj_full_size(s));
339 continue;
340 }
341
342 off = obj_to_index(s, page, p[i]);
343 obj_cgroup_get(objcg);
344 page_obj_cgroups(page)[off] = objcg;
345 mod_objcg_state(objcg, page_pgdat(page),
346 cache_vmstat_idx(s), obj_full_size(s));
347 } else {
348 obj_cgroup_uncharge(objcg, obj_full_size(s));
349 }
350 }
351 obj_cgroup_put(objcg);
352 }
353
memcg_slab_free_hook(struct kmem_cache * s_orig,void ** p,int objects)354 static inline void memcg_slab_free_hook(struct kmem_cache *s_orig,
355 void **p, int objects)
356 {
357 struct kmem_cache *s;
358 struct obj_cgroup *objcg;
359 struct page *page;
360 unsigned int off;
361 int i;
362
363 if (!memcg_kmem_enabled())
364 return;
365
366 for (i = 0; i < objects; i++) {
367 if (unlikely(!p[i]))
368 continue;
369
370 page = virt_to_head_page(p[i]);
371 if (!page_has_obj_cgroups(page))
372 continue;
373
374 if (!s_orig)
375 s = page->slab_cache;
376 else
377 s = s_orig;
378
379 off = obj_to_index(s, page, p[i]);
380 objcg = page_obj_cgroups(page)[off];
381 if (!objcg)
382 continue;
383
384 page_obj_cgroups(page)[off] = NULL;
385 obj_cgroup_uncharge(objcg, obj_full_size(s));
386 mod_objcg_state(objcg, page_pgdat(page), cache_vmstat_idx(s),
387 -obj_full_size(s));
388 obj_cgroup_put(objcg);
389 }
390 }
391
392 #else /* CONFIG_MEMCG_KMEM */
page_has_obj_cgroups(struct page * page)393 static inline bool page_has_obj_cgroups(struct page *page)
394 {
395 return false;
396 }
397
memcg_from_slab_obj(void * ptr)398 static inline struct mem_cgroup *memcg_from_slab_obj(void *ptr)
399 {
400 return NULL;
401 }
402
memcg_alloc_page_obj_cgroups(struct page * page,struct kmem_cache * s,gfp_t gfp)403 static inline int memcg_alloc_page_obj_cgroups(struct page *page,
404 struct kmem_cache *s, gfp_t gfp)
405 {
406 return 0;
407 }
408
memcg_free_page_obj_cgroups(struct page * page)409 static inline void memcg_free_page_obj_cgroups(struct page *page)
410 {
411 }
412
memcg_slab_pre_alloc_hook(struct kmem_cache * s,struct obj_cgroup ** objcgp,size_t objects,gfp_t flags)413 static inline bool memcg_slab_pre_alloc_hook(struct kmem_cache *s,
414 struct obj_cgroup **objcgp,
415 size_t objects, gfp_t flags)
416 {
417 return true;
418 }
419
memcg_slab_post_alloc_hook(struct kmem_cache * s,struct obj_cgroup * objcg,gfp_t flags,size_t size,void ** p)420 static inline void memcg_slab_post_alloc_hook(struct kmem_cache *s,
421 struct obj_cgroup *objcg,
422 gfp_t flags, size_t size,
423 void **p)
424 {
425 }
426
memcg_slab_free_hook(struct kmem_cache * s,void ** p,int objects)427 static inline void memcg_slab_free_hook(struct kmem_cache *s,
428 void **p, int objects)
429 {
430 }
431 #endif /* CONFIG_MEMCG_KMEM */
432
virt_to_cache(const void * obj)433 static inline struct kmem_cache *virt_to_cache(const void *obj)
434 {
435 struct page *page;
436
437 page = virt_to_head_page(obj);
438 if (WARN_ONCE(!PageSlab(page), "%s: Object is not a Slab page!\n",
439 __func__))
440 return NULL;
441 return page->slab_cache;
442 }
443
account_slab_page(struct page * page,int order,struct kmem_cache * s)444 static __always_inline void account_slab_page(struct page *page, int order,
445 struct kmem_cache *s)
446 {
447 mod_node_page_state(page_pgdat(page), cache_vmstat_idx(s),
448 PAGE_SIZE << order);
449 }
450
unaccount_slab_page(struct page * page,int order,struct kmem_cache * s)451 static __always_inline void unaccount_slab_page(struct page *page, int order,
452 struct kmem_cache *s)
453 {
454 if (memcg_kmem_enabled())
455 memcg_free_page_obj_cgroups(page);
456
457 mod_node_page_state(page_pgdat(page), cache_vmstat_idx(s),
458 -(PAGE_SIZE << order));
459 }
460
cache_from_obj(struct kmem_cache * s,void * x)461 static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x)
462 {
463 struct kmem_cache *cachep;
464
465 if (!IS_ENABLED(CONFIG_SLAB_FREELIST_HARDENED) &&
466 !kmem_cache_debug_flags(s, SLAB_CONSISTENCY_CHECKS))
467 return s;
468
469 cachep = virt_to_cache(x);
470 if (WARN(cachep && cachep != s,
471 "%s: Wrong slab cache. %s but object is from %s\n",
472 __func__, s->name, cachep->name))
473 print_tracking(cachep, x);
474 return cachep;
475 }
476
slab_ksize(const struct kmem_cache * s)477 static inline size_t slab_ksize(const struct kmem_cache *s)
478 {
479 #ifndef CONFIG_SLUB
480 return s->object_size;
481
482 #else /* CONFIG_SLUB */
483 # ifdef CONFIG_SLUB_DEBUG
484 /*
485 * Debugging requires use of the padding between object
486 * and whatever may come after it.
487 */
488 if (s->flags & (SLAB_RED_ZONE | SLAB_POISON))
489 return s->object_size;
490 # endif
491 if (s->flags & SLAB_KASAN)
492 return s->object_size;
493 /*
494 * If we have the need to store the freelist pointer
495 * back there or track user information then we can
496 * only use the space before that information.
497 */
498 if (s->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_STORE_USER))
499 return s->inuse;
500 /*
501 * Else we can use all the padding etc for the allocation
502 */
503 return s->size;
504 #endif
505 }
506
slab_pre_alloc_hook(struct kmem_cache * s,struct obj_cgroup ** objcgp,size_t size,gfp_t flags)507 static inline struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s,
508 struct obj_cgroup **objcgp,
509 size_t size, gfp_t flags)
510 {
511 flags &= gfp_allowed_mask;
512
513 fs_reclaim_acquire(flags);
514 fs_reclaim_release(flags);
515
516 might_sleep_if(gfpflags_allow_blocking(flags));
517
518 if (should_failslab(s, flags))
519 return NULL;
520
521 if (!memcg_slab_pre_alloc_hook(s, objcgp, size, flags))
522 return NULL;
523
524 return s;
525 }
526
slab_post_alloc_hook(struct kmem_cache * s,struct obj_cgroup * objcg,gfp_t flags,size_t size,void ** p)527 static inline void slab_post_alloc_hook(struct kmem_cache *s,
528 struct obj_cgroup *objcg,
529 gfp_t flags, size_t size, void **p)
530 {
531 size_t i;
532
533 flags &= gfp_allowed_mask;
534 for (i = 0; i < size; i++) {
535 p[i] = kasan_slab_alloc(s, p[i], flags);
536 /* As p[i] might get tagged, call kmemleak hook after KASAN. */
537 kmemleak_alloc_recursive(p[i], s->object_size, 1,
538 s->flags, flags);
539 }
540
541 memcg_slab_post_alloc_hook(s, objcg, flags, size, p);
542 }
543
544 #ifndef CONFIG_SLOB
545 /*
546 * The slab lists for all objects.
547 */
548 struct kmem_cache_node {
549 spinlock_t list_lock;
550
551 #ifdef CONFIG_SLAB
552 struct list_head slabs_partial; /* partial list first, better asm code */
553 struct list_head slabs_full;
554 struct list_head slabs_free;
555 unsigned long total_slabs; /* length of all slab lists */
556 unsigned long free_slabs; /* length of free slab list only */
557 unsigned long free_objects;
558 unsigned int free_limit;
559 unsigned int colour_next; /* Per-node cache coloring */
560 struct array_cache *shared; /* shared per node */
561 struct alien_cache **alien; /* on other nodes */
562 unsigned long next_reap; /* updated without locking */
563 int free_touched; /* updated without locking */
564 #endif
565
566 #ifdef CONFIG_SLUB
567 unsigned long nr_partial;
568 struct list_head partial;
569 #ifdef CONFIG_SLUB_DEBUG
570 atomic_long_t nr_slabs;
571 atomic_long_t total_objects;
572 struct list_head full;
573 #endif
574 #endif
575
576 };
577
get_node(struct kmem_cache * s,int node)578 static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node)
579 {
580 return s->node[node];
581 }
582
583 /*
584 * Iterator over all nodes. The body will be executed for each node that has
585 * a kmem_cache_node structure allocated (which is true for all online nodes)
586 */
587 #define for_each_kmem_cache_node(__s, __node, __n) \
588 for (__node = 0; __node < nr_node_ids; __node++) \
589 if ((__n = get_node(__s, __node)))
590
591 #endif
592
593 void *slab_start(struct seq_file *m, loff_t *pos);
594 void *slab_next(struct seq_file *m, void *p, loff_t *pos);
595 void slab_stop(struct seq_file *m, void *p);
596 int memcg_slab_show(struct seq_file *m, void *p);
597
598 #if defined(CONFIG_SLAB) || defined(CONFIG_SLUB_DEBUG)
599 void dump_unreclaimable_slab(void);
600 #else
dump_unreclaimable_slab(void)601 static inline void dump_unreclaimable_slab(void)
602 {
603 }
604 #endif
605
606 void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr);
607
608 #ifdef CONFIG_SLAB_FREELIST_RANDOM
609 int cache_random_seq_create(struct kmem_cache *cachep, unsigned int count,
610 gfp_t gfp);
611 void cache_random_seq_destroy(struct kmem_cache *cachep);
612 #else
cache_random_seq_create(struct kmem_cache * cachep,unsigned int count,gfp_t gfp)613 static inline int cache_random_seq_create(struct kmem_cache *cachep,
614 unsigned int count, gfp_t gfp)
615 {
616 return 0;
617 }
cache_random_seq_destroy(struct kmem_cache * cachep)618 static inline void cache_random_seq_destroy(struct kmem_cache *cachep) { }
619 #endif /* CONFIG_SLAB_FREELIST_RANDOM */
620
slab_want_init_on_alloc(gfp_t flags,struct kmem_cache * c)621 static inline bool slab_want_init_on_alloc(gfp_t flags, struct kmem_cache *c)
622 {
623 if (static_branch_unlikely(&init_on_alloc)) {
624 if (c->ctor)
625 return false;
626 if (c->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON))
627 return flags & __GFP_ZERO;
628 return true;
629 }
630 return flags & __GFP_ZERO;
631 }
632
slab_want_init_on_free(struct kmem_cache * c)633 static inline bool slab_want_init_on_free(struct kmem_cache *c)
634 {
635 if (static_branch_unlikely(&init_on_free))
636 return !(c->ctor ||
637 (c->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON)));
638 return false;
639 }
640
641 #endif /* MM_SLAB_H */
642
