Lines Matching refs:cachep
211 static void free_block(struct kmem_cache *cachep, void **objpp, int len,
213 static void slabs_destroy(struct kmem_cache *cachep, struct list_head *list);
214 static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp);
217 static inline void fixup_objfreelist_debug(struct kmem_cache *cachep,
219 static inline void fixup_slab_list(struct kmem_cache *cachep,
241 #define MAKE_LIST(cachep, listp, slab, nodeid) \ argument
244 list_splice(&get_node(cachep, nodeid)->slab, listp); \
247 #define MAKE_ALL_LISTS(cachep, ptr, nodeid) \ argument
249 MAKE_LIST((cachep), (&(ptr)->slabs_full), slabs_full, nodeid); \
250 MAKE_LIST((cachep), (&(ptr)->slabs_partial), slabs_partial, nodeid); \
251 MAKE_LIST((cachep), (&(ptr)->slabs_free), slabs_free, nodeid); \
327 static int obj_offset(struct kmem_cache *cachep) in obj_offset() argument
329 return cachep->obj_offset; in obj_offset()
332 static unsigned long long *dbg_redzone1(struct kmem_cache *cachep, void *objp) in dbg_redzone1() argument
334 BUG_ON(!(cachep->flags & SLAB_RED_ZONE)); in dbg_redzone1()
335 return (unsigned long long*) (objp + obj_offset(cachep) - in dbg_redzone1()
339 static unsigned long long *dbg_redzone2(struct kmem_cache *cachep, void *objp) in dbg_redzone2() argument
341 BUG_ON(!(cachep->flags & SLAB_RED_ZONE)); in dbg_redzone2()
342 if (cachep->flags & SLAB_STORE_USER) in dbg_redzone2()
343 return (unsigned long long *)(objp + cachep->size - in dbg_redzone2()
346 return (unsigned long long *) (objp + cachep->size - in dbg_redzone2()
350 static void **dbg_userword(struct kmem_cache *cachep, void *objp) in dbg_userword() argument
352 BUG_ON(!(cachep->flags & SLAB_STORE_USER)); in dbg_userword()
353 return (void **)(objp + cachep->size - BYTES_PER_WORD); in dbg_userword()
359 #define dbg_redzone1(cachep, objp) ({BUG(); (unsigned long long *)NULL;}) argument
360 #define dbg_redzone2(cachep, objp) ({BUG(); (unsigned long long *)NULL;}) argument
361 #define dbg_userword(cachep, objp) ({BUG(); (void **)NULL;}) argument
392 static inline struct array_cache *cpu_cache_get(struct kmem_cache *cachep) in cpu_cache_get() argument
394 return this_cpu_ptr(cachep->cpu_cache); in cpu_cache_get()
436 #define slab_error(cachep, msg) __slab_error(__func__, cachep, msg) argument
438 static void __slab_error(const char *function, struct kmem_cache *cachep, in __slab_error() argument
442 function, cachep->name, msg); in __slab_error()
551 static noinline void cache_free_pfmemalloc(struct kmem_cache *cachep, in cache_free_pfmemalloc() argument
559 n = get_node(cachep, page_node); in cache_free_pfmemalloc()
562 free_block(cachep, &objp, 1, page_node, &list); in cache_free_pfmemalloc()
565 slabs_destroy(cachep, &list); in cache_free_pfmemalloc()
603 #define drain_alien_cache(cachep, alien) do { } while (0) argument
604 #define reap_alien(cachep, n) do { } while (0) argument
616 static inline int cache_free_alien(struct kmem_cache *cachep, void *objp) in cache_free_alien() argument
621 static inline void *alternate_node_alloc(struct kmem_cache *cachep, in alternate_node_alloc() argument
627 static inline void *____cache_alloc_node(struct kmem_cache *cachep, in ____cache_alloc_node() argument
694 static void __drain_alien_cache(struct kmem_cache *cachep, in __drain_alien_cache() argument
698 struct kmem_cache_node *n = get_node(cachep, node); in __drain_alien_cache()
710 free_block(cachep, ac->entry, ac->avail, node, list); in __drain_alien_cache()
719 static void reap_alien(struct kmem_cache *cachep, struct kmem_cache_node *n) in reap_alien() argument
732 __drain_alien_cache(cachep, ac, node, &list); in reap_alien()
734 slabs_destroy(cachep, &list); in reap_alien()
740 static void drain_alien_cache(struct kmem_cache *cachep, in drain_alien_cache() argument
755 __drain_alien_cache(cachep, ac, i, &list); in drain_alien_cache()
757 slabs_destroy(cachep, &list); in drain_alien_cache()
762 static int __cache_free_alien(struct kmem_cache *cachep, void *objp, in __cache_free_alien() argument
770 n = get_node(cachep, node); in __cache_free_alien()
771 STATS_INC_NODEFREES(cachep); in __cache_free_alien()
777 STATS_INC_ACOVERFLOW(cachep); in __cache_free_alien()
778 __drain_alien_cache(cachep, ac, page_node, &list); in __cache_free_alien()
782 slabs_destroy(cachep, &list); in __cache_free_alien()
784 n = get_node(cachep, page_node); in __cache_free_alien()
786 free_block(cachep, &objp, 1, page_node, &list); in __cache_free_alien()
788 slabs_destroy(cachep, &list); in __cache_free_alien()
793 static inline int cache_free_alien(struct kmem_cache *cachep, void *objp) in cache_free_alien() argument
804 return __cache_free_alien(cachep, objp, node, page_node); in cache_free_alien()
817 static int init_cache_node(struct kmem_cache *cachep, int node, gfp_t gfp) in init_cache_node() argument
826 n = get_node(cachep, node); in init_cache_node()
829 n->free_limit = (1 + nr_cpus_node(node)) * cachep->batchcount + in init_cache_node()
830 cachep->num; in init_cache_node()
842 ((unsigned long)cachep) % REAPTIMEOUT_NODE; in init_cache_node()
845 (1 + nr_cpus_node(node)) * cachep->batchcount + cachep->num; in init_cache_node()
852 cachep->node[node] = n; in init_cache_node()
870 struct kmem_cache *cachep; in init_cache_node_node() local
872 list_for_each_entry(cachep, &slab_caches, list) { in init_cache_node_node()
873 ret = init_cache_node(cachep, node, GFP_KERNEL); in init_cache_node_node()
882 static int setup_kmem_cache_node(struct kmem_cache *cachep, in setup_kmem_cache_node() argument
893 new_alien = alloc_alien_cache(node, cachep->limit, gfp); in setup_kmem_cache_node()
898 if (cachep->shared) { in setup_kmem_cache_node()
900 cachep->shared * cachep->batchcount, 0xbaadf00d, gfp); in setup_kmem_cache_node()
905 ret = init_cache_node(cachep, node, gfp); in setup_kmem_cache_node()
909 n = get_node(cachep, node); in setup_kmem_cache_node()
912 free_block(cachep, n->shared->entry, in setup_kmem_cache_node()
929 slabs_destroy(cachep, &list); in setup_kmem_cache_node()
952 struct kmem_cache *cachep; in cpuup_canceled() local
957 list_for_each_entry(cachep, &slab_caches, list) { in cpuup_canceled()
963 n = get_node(cachep, node); in cpuup_canceled()
970 n->free_limit -= cachep->batchcount; in cpuup_canceled()
973 nc = per_cpu_ptr(cachep->cpu_cache, cpu); in cpuup_canceled()
974 free_block(cachep, nc->entry, nc->avail, node, &list); in cpuup_canceled()
984 free_block(cachep, shared->entry, in cpuup_canceled()
996 drain_alien_cache(cachep, alien); in cpuup_canceled()
1001 slabs_destroy(cachep, &list); in cpuup_canceled()
1008 list_for_each_entry(cachep, &slab_caches, list) { in cpuup_canceled()
1009 n = get_node(cachep, node); in cpuup_canceled()
1012 drain_freelist(cachep, n, INT_MAX); in cpuup_canceled()
1018 struct kmem_cache *cachep; in cpuup_prepare() local
1036 list_for_each_entry(cachep, &slab_caches, list) { in cpuup_prepare()
1037 err = setup_kmem_cache_node(cachep, node, GFP_KERNEL, false); in cpuup_prepare()
1107 struct kmem_cache *cachep; in drain_cache_node_node() local
1110 list_for_each_entry(cachep, &slab_caches, list) { in drain_cache_node_node()
1113 n = get_node(cachep, node); in drain_cache_node_node()
1117 drain_freelist(cachep, n, INT_MAX); in drain_cache_node_node()
1164 static void __init init_list(struct kmem_cache *cachep, struct kmem_cache_node *list, in init_list() argument
1178 MAKE_ALL_LISTS(cachep, ptr, nodeid); in init_list()
1179 cachep->node[nodeid] = ptr; in init_list()
1186 static void __init set_up_node(struct kmem_cache *cachep, int index) in set_up_node() argument
1191 cachep->node[node] = &init_kmem_cache_node[index + node]; in set_up_node()
1192 cachep->node[node]->next_reap = jiffies + in set_up_node()
1194 ((unsigned long)cachep) % REAPTIMEOUT_NODE; in set_up_node()
1285 struct kmem_cache *cachep; in kmem_cache_init_late() local
1289 list_for_each_entry(cachep, &slab_caches, list) in kmem_cache_init_late()
1290 if (enable_cpucache(cachep, GFP_NOWAIT)) in kmem_cache_init_late()
1327 slab_out_of_memory(struct kmem_cache *cachep, gfp_t gfpflags, int nodeid) in slab_out_of_memory() argument
1342 cachep->name, cachep->size, cachep->gfporder); in slab_out_of_memory()
1344 for_each_kmem_cache_node(cachep, node, n) { in slab_out_of_memory()
1355 (total_slabs * cachep->num) - free_objs, in slab_out_of_memory()
1356 total_slabs * cachep->num); in slab_out_of_memory()
1369 static struct page *kmem_getpages(struct kmem_cache *cachep, gfp_t flags, in kmem_getpages() argument
1374 flags |= cachep->allocflags; in kmem_getpages()
1376 page = __alloc_pages_node(nodeid, flags, cachep->gfporder); in kmem_getpages()
1378 slab_out_of_memory(cachep, flags, nodeid); in kmem_getpages()
1382 account_slab_page(page, cachep->gfporder, cachep); in kmem_getpages()
1394 static void kmem_freepages(struct kmem_cache *cachep, struct page *page) in kmem_freepages() argument
1396 int order = cachep->gfporder; in kmem_freepages()
1406 unaccount_slab_page(page, order, cachep); in kmem_freepages()
1412 struct kmem_cache *cachep; in kmem_rcu_free() local
1416 cachep = page->slab_cache; in kmem_rcu_free()
1418 kmem_freepages(cachep, page); in kmem_rcu_free()
1422 static bool is_debug_pagealloc_cache(struct kmem_cache *cachep) in is_debug_pagealloc_cache() argument
1424 if (debug_pagealloc_enabled_static() && OFF_SLAB(cachep) && in is_debug_pagealloc_cache()
1425 (cachep->size % PAGE_SIZE) == 0) in is_debug_pagealloc_cache()
1432 static void slab_kernel_map(struct kmem_cache *cachep, void *objp, int map) in slab_kernel_map() argument
1434 if (!is_debug_pagealloc_cache(cachep)) in slab_kernel_map()
1437 kernel_map_pages(virt_to_page(objp), cachep->size / PAGE_SIZE, map); in slab_kernel_map()
1441 static inline void slab_kernel_map(struct kmem_cache *cachep, void *objp, in slab_kernel_map() argument
1446 static void poison_obj(struct kmem_cache *cachep, void *addr, unsigned char val) in poison_obj() argument
1448 int size = cachep->object_size; in poison_obj()
1449 addr = &((char *)addr)[obj_offset(cachep)]; in poison_obj()
1487 static void print_objinfo(struct kmem_cache *cachep, void *objp, int lines) in print_objinfo() argument
1492 if (cachep->flags & SLAB_RED_ZONE) { in print_objinfo()
1494 *dbg_redzone1(cachep, objp), in print_objinfo()
1495 *dbg_redzone2(cachep, objp)); in print_objinfo()
1498 if (cachep->flags & SLAB_STORE_USER) in print_objinfo()
1499 pr_err("Last user: (%pSR)\n", *dbg_userword(cachep, objp)); in print_objinfo()
1500 realobj = (char *)objp + obj_offset(cachep); in print_objinfo()
1501 size = cachep->object_size; in print_objinfo()
1511 static void check_poison_obj(struct kmem_cache *cachep, void *objp) in check_poison_obj() argument
1517 if (is_debug_pagealloc_cache(cachep)) in check_poison_obj()
1520 realobj = (char *)objp + obj_offset(cachep); in check_poison_obj()
1521 size = cachep->object_size; in check_poison_obj()
1533 print_tainted(), cachep->name, in check_poison_obj()
1535 print_objinfo(cachep, objp, 0); in check_poison_obj()
1557 objnr = obj_to_index(cachep, page, objp); in check_poison_obj()
1559 objp = index_to_obj(cachep, page, objnr - 1); in check_poison_obj()
1560 realobj = (char *)objp + obj_offset(cachep); in check_poison_obj()
1562 print_objinfo(cachep, objp, 2); in check_poison_obj()
1564 if (objnr + 1 < cachep->num) { in check_poison_obj()
1565 objp = index_to_obj(cachep, page, objnr + 1); in check_poison_obj()
1566 realobj = (char *)objp + obj_offset(cachep); in check_poison_obj()
1568 print_objinfo(cachep, objp, 2); in check_poison_obj()
1575 static void slab_destroy_debugcheck(struct kmem_cache *cachep, in slab_destroy_debugcheck() argument
1580 if (OBJFREELIST_SLAB(cachep) && cachep->flags & SLAB_POISON) { in slab_destroy_debugcheck()
1581 poison_obj(cachep, page->freelist - obj_offset(cachep), in slab_destroy_debugcheck()
1585 for (i = 0; i < cachep->num; i++) { in slab_destroy_debugcheck()
1586 void *objp = index_to_obj(cachep, page, i); in slab_destroy_debugcheck()
1588 if (cachep->flags & SLAB_POISON) { in slab_destroy_debugcheck()
1589 check_poison_obj(cachep, objp); in slab_destroy_debugcheck()
1590 slab_kernel_map(cachep, objp, 1); in slab_destroy_debugcheck()
1592 if (cachep->flags & SLAB_RED_ZONE) { in slab_destroy_debugcheck()
1593 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE) in slab_destroy_debugcheck()
1594 slab_error(cachep, "start of a freed object was overwritten"); in slab_destroy_debugcheck()
1595 if (*dbg_redzone2(cachep, objp) != RED_INACTIVE) in slab_destroy_debugcheck()
1596 slab_error(cachep, "end of a freed object was overwritten"); in slab_destroy_debugcheck()
1601 static void slab_destroy_debugcheck(struct kmem_cache *cachep, in slab_destroy_debugcheck() argument
1616 static void slab_destroy(struct kmem_cache *cachep, struct page *page) in slab_destroy() argument
1621 slab_destroy_debugcheck(cachep, page); in slab_destroy()
1622 if (unlikely(cachep->flags & SLAB_TYPESAFE_BY_RCU)) in slab_destroy()
1625 kmem_freepages(cachep, page); in slab_destroy()
1631 if (OFF_SLAB(cachep)) in slab_destroy()
1632 kmem_cache_free(cachep->freelist_cache, freelist); in slab_destroy()
1639 static void slabs_destroy(struct kmem_cache *cachep, struct list_head *list) in slabs_destroy() argument
1645 slab_destroy(cachep, page); in slabs_destroy()
1663 static size_t calculate_slab_order(struct kmem_cache *cachep, in calculate_slab_order() argument
1698 if (freelist_cache->size > cachep->size / 2) in calculate_slab_order()
1703 cachep->num = num; in calculate_slab_order()
1704 cachep->gfporder = gfporder; in calculate_slab_order()
1732 struct kmem_cache *cachep, int entries, int batchcount) in alloc_kmem_cache_cpus() argument
1752 static int __ref setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp) in setup_cpu_cache() argument
1755 return enable_cpucache(cachep, gfp); in setup_cpu_cache()
1757 cachep->cpu_cache = alloc_kmem_cache_cpus(cachep, 1, 1); in setup_cpu_cache()
1758 if (!cachep->cpu_cache) in setup_cpu_cache()
1766 set_up_node(cachep, SIZE_NODE); in setup_cpu_cache()
1771 cachep->node[node] = kmalloc_node( in setup_cpu_cache()
1773 BUG_ON(!cachep->node[node]); in setup_cpu_cache()
1774 kmem_cache_node_init(cachep->node[node]); in setup_cpu_cache()
1778 cachep->node[numa_mem_id()]->next_reap = in setup_cpu_cache()
1780 ((unsigned long)cachep) % REAPTIMEOUT_NODE; in setup_cpu_cache()
1782 cpu_cache_get(cachep)->avail = 0; in setup_cpu_cache()
1783 cpu_cache_get(cachep)->limit = BOOT_CPUCACHE_ENTRIES; in setup_cpu_cache()
1784 cpu_cache_get(cachep)->batchcount = 1; in setup_cpu_cache()
1785 cpu_cache_get(cachep)->touched = 0; in setup_cpu_cache()
1786 cachep->batchcount = 1; in setup_cpu_cache()
1787 cachep->limit = BOOT_CPUCACHE_ENTRIES; in setup_cpu_cache()
1802 struct kmem_cache *cachep; in __kmem_cache_alias() local
1804 cachep = find_mergeable(size, align, flags, name, ctor); in __kmem_cache_alias()
1805 if (cachep) { in __kmem_cache_alias()
1806 cachep->refcount++; in __kmem_cache_alias()
1812 cachep->object_size = max_t(int, cachep->object_size, size); in __kmem_cache_alias()
1814 return cachep; in __kmem_cache_alias()
1817 static bool set_objfreelist_slab_cache(struct kmem_cache *cachep, in set_objfreelist_slab_cache() argument
1822 cachep->num = 0; in set_objfreelist_slab_cache()
1829 if (unlikely(slab_want_init_on_free(cachep))) in set_objfreelist_slab_cache()
1832 if (cachep->ctor || flags & SLAB_TYPESAFE_BY_RCU) in set_objfreelist_slab_cache()
1835 left = calculate_slab_order(cachep, size, in set_objfreelist_slab_cache()
1837 if (!cachep->num) in set_objfreelist_slab_cache()
1840 if (cachep->num * sizeof(freelist_idx_t) > cachep->object_size) in set_objfreelist_slab_cache()
1843 cachep->colour = left / cachep->colour_off; in set_objfreelist_slab_cache()
1848 static bool set_off_slab_cache(struct kmem_cache *cachep, in set_off_slab_cache() argument
1853 cachep->num = 0; in set_off_slab_cache()
1866 left = calculate_slab_order(cachep, size, flags | CFLGS_OFF_SLAB); in set_off_slab_cache()
1867 if (!cachep->num) in set_off_slab_cache()
1874 if (left >= cachep->num * sizeof(freelist_idx_t)) in set_off_slab_cache()
1877 cachep->colour = left / cachep->colour_off; in set_off_slab_cache()
1882 static bool set_on_slab_cache(struct kmem_cache *cachep, in set_on_slab_cache() argument
1887 cachep->num = 0; in set_on_slab_cache()
1889 left = calculate_slab_order(cachep, size, flags); in set_on_slab_cache()
1890 if (!cachep->num) in set_on_slab_cache()
1893 cachep->colour = left / cachep->colour_off; in set_on_slab_cache()
1921 int __kmem_cache_create(struct kmem_cache *cachep, slab_flags_t flags) in __kmem_cache_create() argument
1926 unsigned int size = cachep->size; in __kmem_cache_create()
1959 if (ralign < cachep->align) { in __kmem_cache_create()
1960 ralign = cachep->align; in __kmem_cache_create()
1968 cachep->align = ralign; in __kmem_cache_create()
1969 cachep->colour_off = cache_line_size(); in __kmem_cache_create()
1971 if (cachep->colour_off < cachep->align) in __kmem_cache_create()
1972 cachep->colour_off = cachep->align; in __kmem_cache_create()
1987 cachep->obj_offset += sizeof(unsigned long long); in __kmem_cache_create()
2002 kasan_cache_create(cachep, &size, &flags); in __kmem_cache_create()
2004 size = ALIGN(size, cachep->align); in __kmem_cache_create()
2010 size = ALIGN(SLAB_OBJ_MIN_SIZE, cachep->align); in __kmem_cache_create()
2021 size >= 256 && cachep->object_size > cache_line_size()) { in __kmem_cache_create()
2025 if (set_off_slab_cache(cachep, tmp_size, flags)) { in __kmem_cache_create()
2027 cachep->obj_offset += tmp_size - size; in __kmem_cache_create()
2035 if (set_objfreelist_slab_cache(cachep, size, flags)) { in __kmem_cache_create()
2040 if (set_off_slab_cache(cachep, size, flags)) { in __kmem_cache_create()
2045 if (set_on_slab_cache(cachep, size, flags)) in __kmem_cache_create()
2051 cachep->freelist_size = cachep->num * sizeof(freelist_idx_t); in __kmem_cache_create()
2052 cachep->flags = flags; in __kmem_cache_create()
2053 cachep->allocflags = __GFP_COMP; in __kmem_cache_create()
2055 cachep->allocflags |= GFP_DMA; in __kmem_cache_create()
2057 cachep->allocflags |= GFP_DMA32; in __kmem_cache_create()
2059 cachep->allocflags |= __GFP_RECLAIMABLE; in __kmem_cache_create()
2060 cachep->size = size; in __kmem_cache_create()
2061 cachep->reciprocal_buffer_size = reciprocal_value(size); in __kmem_cache_create()
2070 (cachep->flags & SLAB_POISON) && in __kmem_cache_create()
2071 is_debug_pagealloc_cache(cachep)) in __kmem_cache_create()
2072 cachep->flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER); in __kmem_cache_create()
2075 if (OFF_SLAB(cachep)) { in __kmem_cache_create()
2076 cachep->freelist_cache = in __kmem_cache_create()
2077 kmalloc_slab(cachep->freelist_size, 0u); in __kmem_cache_create()
2080 err = setup_cpu_cache(cachep, gfp); in __kmem_cache_create()
2082 __kmem_cache_release(cachep); in __kmem_cache_create()
2105 static void check_spinlock_acquired(struct kmem_cache *cachep) in check_spinlock_acquired() argument
2109 assert_spin_locked(&get_node(cachep, numa_mem_id())->list_lock); in check_spinlock_acquired()
2113 static void check_spinlock_acquired_node(struct kmem_cache *cachep, int node) in check_spinlock_acquired_node() argument
2117 assert_spin_locked(&get_node(cachep, node)->list_lock); in check_spinlock_acquired_node()
2129 static void drain_array_locked(struct kmem_cache *cachep, struct array_cache *ac, in drain_array_locked() argument
2141 free_block(cachep, ac->entry, tofree, node, list); in drain_array_locked()
2148 struct kmem_cache *cachep = arg; in do_drain() local
2155 ac = cpu_cache_get(cachep); in do_drain()
2156 n = get_node(cachep, node); in do_drain()
2158 free_block(cachep, ac->entry, ac->avail, node, &list); in do_drain()
2161 slabs_destroy(cachep, &list); in do_drain()
2164 static void drain_cpu_caches(struct kmem_cache *cachep) in drain_cpu_caches() argument
2170 on_each_cpu(do_drain, cachep, 1); in drain_cpu_caches()
2172 for_each_kmem_cache_node(cachep, node, n) in drain_cpu_caches()
2174 drain_alien_cache(cachep, n->alien); in drain_cpu_caches()
2176 for_each_kmem_cache_node(cachep, node, n) { in drain_cpu_caches()
2178 drain_array_locked(cachep, n->shared, node, true, &list); in drain_cpu_caches()
2181 slabs_destroy(cachep, &list); in drain_cpu_caches()
2237 int __kmem_cache_shrink(struct kmem_cache *cachep) in __kmem_cache_shrink() argument
2243 drain_cpu_caches(cachep); in __kmem_cache_shrink()
2246 for_each_kmem_cache_node(cachep, node, n) { in __kmem_cache_shrink()
2247 drain_freelist(cachep, n, INT_MAX); in __kmem_cache_shrink()
2255 int __kmem_cache_shutdown(struct kmem_cache *cachep) in __kmem_cache_shutdown() argument
2257 return __kmem_cache_shrink(cachep); in __kmem_cache_shutdown()
2260 void __kmem_cache_release(struct kmem_cache *cachep) in __kmem_cache_release() argument
2265 cache_random_seq_destroy(cachep); in __kmem_cache_release()
2267 free_percpu(cachep->cpu_cache); in __kmem_cache_release()
2270 for_each_kmem_cache_node(cachep, i, n) { in __kmem_cache_release()
2274 cachep->node[i] = NULL; in __kmem_cache_release()
2292 static void *alloc_slabmgmt(struct kmem_cache *cachep, in alloc_slabmgmt() argument
2302 if (OBJFREELIST_SLAB(cachep)) in alloc_slabmgmt()
2304 else if (OFF_SLAB(cachep)) { in alloc_slabmgmt()
2306 freelist = kmem_cache_alloc_node(cachep->freelist_cache, in alloc_slabmgmt()
2310 freelist = addr + (PAGE_SIZE << cachep->gfporder) - in alloc_slabmgmt()
2311 cachep->freelist_size; in alloc_slabmgmt()
2328 static void cache_init_objs_debug(struct kmem_cache *cachep, struct page *page) in cache_init_objs_debug() argument
2333 for (i = 0; i < cachep->num; i++) { in cache_init_objs_debug()
2334 void *objp = index_to_obj(cachep, page, i); in cache_init_objs_debug()
2336 if (cachep->flags & SLAB_STORE_USER) in cache_init_objs_debug()
2337 *dbg_userword(cachep, objp) = NULL; in cache_init_objs_debug()
2339 if (cachep->flags & SLAB_RED_ZONE) { in cache_init_objs_debug()
2340 *dbg_redzone1(cachep, objp) = RED_INACTIVE; in cache_init_objs_debug()
2341 *dbg_redzone2(cachep, objp) = RED_INACTIVE; in cache_init_objs_debug()
2348 if (cachep->ctor && !(cachep->flags & SLAB_POISON)) { in cache_init_objs_debug()
2349 kasan_unpoison_object_data(cachep, in cache_init_objs_debug()
2350 objp + obj_offset(cachep)); in cache_init_objs_debug()
2351 cachep->ctor(objp + obj_offset(cachep)); in cache_init_objs_debug()
2353 cachep, objp + obj_offset(cachep)); in cache_init_objs_debug()
2356 if (cachep->flags & SLAB_RED_ZONE) { in cache_init_objs_debug()
2357 if (*dbg_redzone2(cachep, objp) != RED_INACTIVE) in cache_init_objs_debug()
2358 slab_error(cachep, "constructor overwrote the end of an object"); in cache_init_objs_debug()
2359 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE) in cache_init_objs_debug()
2360 slab_error(cachep, "constructor overwrote the start of an object"); in cache_init_objs_debug()
2363 if (cachep->flags & SLAB_POISON) { in cache_init_objs_debug()
2364 poison_obj(cachep, objp, POISON_FREE); in cache_init_objs_debug()
2365 slab_kernel_map(cachep, objp, 0); in cache_init_objs_debug()
2387 struct kmem_cache *cachep, in freelist_state_initialize() argument
2397 if (!cachep->random_seq) { in freelist_state_initialize()
2401 state->list = cachep->random_seq; in freelist_state_initialize()
2428 static bool shuffle_freelist(struct kmem_cache *cachep, struct page *page) in shuffle_freelist() argument
2430 unsigned int objfreelist = 0, i, rand, count = cachep->num; in shuffle_freelist()
2437 precomputed = freelist_state_initialize(&state, cachep, count); in shuffle_freelist()
2440 if (OBJFREELIST_SLAB(cachep)) { in shuffle_freelist()
2445 page->freelist = index_to_obj(cachep, page, objfreelist) + in shuffle_freelist()
2446 obj_offset(cachep); in shuffle_freelist()
2469 if (OBJFREELIST_SLAB(cachep)) in shuffle_freelist()
2470 set_free_obj(page, cachep->num - 1, objfreelist); in shuffle_freelist()
2475 static inline bool shuffle_freelist(struct kmem_cache *cachep, in shuffle_freelist() argument
2482 static void cache_init_objs(struct kmem_cache *cachep, in cache_init_objs() argument
2489 cache_init_objs_debug(cachep, page); in cache_init_objs()
2492 shuffled = shuffle_freelist(cachep, page); in cache_init_objs()
2494 if (!shuffled && OBJFREELIST_SLAB(cachep)) { in cache_init_objs()
2495 page->freelist = index_to_obj(cachep, page, cachep->num - 1) + in cache_init_objs()
2496 obj_offset(cachep); in cache_init_objs()
2499 for (i = 0; i < cachep->num; i++) { in cache_init_objs()
2500 objp = index_to_obj(cachep, page, i); in cache_init_objs()
2501 objp = kasan_init_slab_obj(cachep, objp); in cache_init_objs()
2504 if (DEBUG == 0 && cachep->ctor) { in cache_init_objs()
2505 kasan_unpoison_object_data(cachep, objp); in cache_init_objs()
2506 cachep->ctor(objp); in cache_init_objs()
2507 kasan_poison_object_data(cachep, objp); in cache_init_objs()
2515 static void *slab_get_obj(struct kmem_cache *cachep, struct page *page) in slab_get_obj() argument
2519 objp = index_to_obj(cachep, page, get_free_obj(page, page->active)); in slab_get_obj()
2525 static void slab_put_obj(struct kmem_cache *cachep, in slab_put_obj() argument
2528 unsigned int objnr = obj_to_index(cachep, page, objp); in slab_put_obj()
2533 for (i = page->active; i < cachep->num; i++) { in slab_put_obj()
2536 cachep->name, objp); in slab_put_obj()
2543 page->freelist = objp + obj_offset(cachep); in slab_put_obj()
2564 static struct page *cache_grow_begin(struct kmem_cache *cachep, in cache_grow_begin() argument
2581 WARN_ON_ONCE(cachep->ctor && (flags & __GFP_ZERO)); in cache_grow_begin()
2592 page = kmem_getpages(cachep, local_flags, nodeid); in cache_grow_begin()
2597 n = get_node(cachep, page_node); in cache_grow_begin()
2601 if (n->colour_next >= cachep->colour) in cache_grow_begin()
2605 if (offset >= cachep->colour) in cache_grow_begin()
2608 offset *= cachep->colour_off; in cache_grow_begin()
2618 freelist = alloc_slabmgmt(cachep, page, offset, in cache_grow_begin()
2620 if (OFF_SLAB(cachep) && !freelist) in cache_grow_begin()
2623 slab_map_pages(cachep, page, freelist); in cache_grow_begin()
2625 cache_init_objs(cachep, page); in cache_grow_begin()
2633 kmem_freepages(cachep, page); in cache_grow_begin()
2640 static void cache_grow_end(struct kmem_cache *cachep, struct page *page) in cache_grow_end() argument
2651 n = get_node(cachep, page_to_nid(page)); in cache_grow_end()
2659 fixup_slab_list(cachep, n, page, &list); in cache_grow_end()
2661 STATS_INC_GROWN(cachep); in cache_grow_end()
2662 n->free_objects += cachep->num - page->active; in cache_grow_end()
2665 fixup_objfreelist_debug(cachep, &list); in cache_grow_end()
2706 static void *cache_free_debugcheck(struct kmem_cache *cachep, void *objp, in cache_free_debugcheck() argument
2712 BUG_ON(virt_to_cache(objp) != cachep); in cache_free_debugcheck()
2714 objp -= obj_offset(cachep); in cache_free_debugcheck()
2718 if (cachep->flags & SLAB_RED_ZONE) { in cache_free_debugcheck()
2719 verify_redzone_free(cachep, objp); in cache_free_debugcheck()
2720 *dbg_redzone1(cachep, objp) = RED_INACTIVE; in cache_free_debugcheck()
2721 *dbg_redzone2(cachep, objp) = RED_INACTIVE; in cache_free_debugcheck()
2723 if (cachep->flags & SLAB_STORE_USER) in cache_free_debugcheck()
2724 *dbg_userword(cachep, objp) = (void *)caller; in cache_free_debugcheck()
2726 objnr = obj_to_index(cachep, page, objp); in cache_free_debugcheck()
2728 BUG_ON(objnr >= cachep->num); in cache_free_debugcheck()
2729 BUG_ON(objp != index_to_obj(cachep, page, objnr)); in cache_free_debugcheck()
2731 if (cachep->flags & SLAB_POISON) { in cache_free_debugcheck()
2732 poison_obj(cachep, objp, POISON_FREE); in cache_free_debugcheck()
2733 slab_kernel_map(cachep, objp, 0); in cache_free_debugcheck()
2743 static inline void fixup_objfreelist_debug(struct kmem_cache *cachep, in fixup_objfreelist_debug() argument
2751 objp = next - obj_offset(cachep); in fixup_objfreelist_debug()
2753 poison_obj(cachep, objp, POISON_FREE); in fixup_objfreelist_debug()
2758 static inline void fixup_slab_list(struct kmem_cache *cachep, in fixup_slab_list() argument
2764 if (page->active == cachep->num) { in fixup_slab_list()
2766 if (OBJFREELIST_SLAB(cachep)) { in fixup_slab_list()
2769 if (cachep->flags & SLAB_POISON) { in fixup_slab_list()
2846 static noinline void *cache_alloc_pfmemalloc(struct kmem_cache *cachep, in cache_alloc_pfmemalloc() argument
2863 obj = slab_get_obj(cachep, page); in cache_alloc_pfmemalloc()
2866 fixup_slab_list(cachep, n, page, &list); in cache_alloc_pfmemalloc()
2869 fixup_objfreelist_debug(cachep, &list); in cache_alloc_pfmemalloc()
2878 static __always_inline int alloc_block(struct kmem_cache *cachep, in alloc_block() argument
2885 BUG_ON(page->active >= cachep->num); in alloc_block()
2887 while (page->active < cachep->num && batchcount--) { in alloc_block()
2888 STATS_INC_ALLOCED(cachep); in alloc_block()
2889 STATS_INC_ACTIVE(cachep); in alloc_block()
2890 STATS_SET_HIGH(cachep); in alloc_block()
2892 ac->entry[ac->avail++] = slab_get_obj(cachep, page); in alloc_block()
2898 static void *cache_alloc_refill(struct kmem_cache *cachep, gfp_t flags) in cache_alloc_refill() argument
2910 ac = cpu_cache_get(cachep); in cache_alloc_refill()
2920 n = get_node(cachep, node); in cache_alloc_refill()
2942 check_spinlock_acquired(cachep); in cache_alloc_refill()
2944 batchcount = alloc_block(cachep, ac, page, batchcount); in cache_alloc_refill()
2945 fixup_slab_list(cachep, n, page, &list); in cache_alloc_refill()
2952 fixup_objfreelist_debug(cachep, &list); in cache_alloc_refill()
2958 void *obj = cache_alloc_pfmemalloc(cachep, n, flags); in cache_alloc_refill()
2964 page = cache_grow_begin(cachep, gfp_exact_node(flags), node); in cache_alloc_refill()
2970 ac = cpu_cache_get(cachep); in cache_alloc_refill()
2972 alloc_block(cachep, ac, page, batchcount); in cache_alloc_refill()
2973 cache_grow_end(cachep, page); in cache_alloc_refill()
2983 static inline void cache_alloc_debugcheck_before(struct kmem_cache *cachep, in cache_alloc_debugcheck_before() argument
2990 static void *cache_alloc_debugcheck_after(struct kmem_cache *cachep, in cache_alloc_debugcheck_after() argument
2993 WARN_ON_ONCE(cachep->ctor && (flags & __GFP_ZERO)); in cache_alloc_debugcheck_after()
2996 if (cachep->flags & SLAB_POISON) { in cache_alloc_debugcheck_after()
2997 check_poison_obj(cachep, objp); in cache_alloc_debugcheck_after()
2998 slab_kernel_map(cachep, objp, 1); in cache_alloc_debugcheck_after()
2999 poison_obj(cachep, objp, POISON_INUSE); in cache_alloc_debugcheck_after()
3001 if (cachep->flags & SLAB_STORE_USER) in cache_alloc_debugcheck_after()
3002 *dbg_userword(cachep, objp) = (void *)caller; in cache_alloc_debugcheck_after()
3004 if (cachep->flags & SLAB_RED_ZONE) { in cache_alloc_debugcheck_after()
3005 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE || in cache_alloc_debugcheck_after()
3006 *dbg_redzone2(cachep, objp) != RED_INACTIVE) { in cache_alloc_debugcheck_after()
3007 slab_error(cachep, "double free, or memory outside object was overwritten"); in cache_alloc_debugcheck_after()
3009 objp, *dbg_redzone1(cachep, objp), in cache_alloc_debugcheck_after()
3010 *dbg_redzone2(cachep, objp)); in cache_alloc_debugcheck_after()
3012 *dbg_redzone1(cachep, objp) = RED_ACTIVE; in cache_alloc_debugcheck_after()
3013 *dbg_redzone2(cachep, objp) = RED_ACTIVE; in cache_alloc_debugcheck_after()
3016 objp += obj_offset(cachep); in cache_alloc_debugcheck_after()
3017 if (cachep->ctor && cachep->flags & SLAB_POISON) in cache_alloc_debugcheck_after()
3018 cachep->ctor(objp); in cache_alloc_debugcheck_after()
3030 static inline void *____cache_alloc(struct kmem_cache *cachep, gfp_t flags) in ____cache_alloc() argument
3037 ac = cpu_cache_get(cachep); in ____cache_alloc()
3042 STATS_INC_ALLOCHIT(cachep); in ____cache_alloc()
3046 STATS_INC_ALLOCMISS(cachep); in ____cache_alloc()
3047 objp = cache_alloc_refill(cachep, flags); in ____cache_alloc()
3052 ac = cpu_cache_get(cachep); in ____cache_alloc()
3072 static void *alternate_node_alloc(struct kmem_cache *cachep, gfp_t flags) in alternate_node_alloc() argument
3079 if (cpuset_do_slab_mem_spread() && (cachep->flags & SLAB_MEM_SPREAD)) in alternate_node_alloc()
3084 return ____cache_alloc_node(cachep, flags, nid_alloc); in alternate_node_alloc()
3163 static void *____cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, in ____cache_alloc_node() argument
3172 n = get_node(cachep, nodeid); in ____cache_alloc_node()
3181 check_spinlock_acquired_node(cachep, nodeid); in ____cache_alloc_node()
3183 STATS_INC_NODEALLOCS(cachep); in ____cache_alloc_node()
3184 STATS_INC_ACTIVE(cachep); in ____cache_alloc_node()
3185 STATS_SET_HIGH(cachep); in ____cache_alloc_node()
3187 BUG_ON(page->active == cachep->num); in ____cache_alloc_node()
3189 obj = slab_get_obj(cachep, page); in ____cache_alloc_node()
3192 fixup_slab_list(cachep, n, page, &list); in ____cache_alloc_node()
3195 fixup_objfreelist_debug(cachep, &list); in ____cache_alloc_node()
3200 page = cache_grow_begin(cachep, gfp_exact_node(flags), nodeid); in ____cache_alloc_node()
3203 obj = slab_get_obj(cachep, page); in ____cache_alloc_node()
3205 cache_grow_end(cachep, page); in ____cache_alloc_node()
3207 return obj ? obj : fallback_alloc(cachep, flags); in ____cache_alloc_node()
3211 slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid, in slab_alloc_node() argument
3220 cachep = slab_pre_alloc_hook(cachep, &objcg, 1, flags); in slab_alloc_node()
3221 if (unlikely(!cachep)) in slab_alloc_node()
3224 cache_alloc_debugcheck_before(cachep, flags); in slab_alloc_node()
3230 if (unlikely(!get_node(cachep, nodeid))) { in slab_alloc_node()
3232 ptr = fallback_alloc(cachep, flags); in slab_alloc_node()
3243 ptr = ____cache_alloc(cachep, flags); in slab_alloc_node()
3248 ptr = ____cache_alloc_node(cachep, flags, nodeid); in slab_alloc_node()
3251 ptr = cache_alloc_debugcheck_after(cachep, flags, ptr, caller); in slab_alloc_node()
3253 if (unlikely(slab_want_init_on_alloc(flags, cachep)) && ptr) in slab_alloc_node()
3254 memset(ptr, 0, cachep->object_size); in slab_alloc_node()
3256 slab_post_alloc_hook(cachep, objcg, flags, 1, &ptr); in slab_alloc_node()
3285 __do_cache_alloc(struct kmem_cache *cachep, gfp_t flags) in __do_cache_alloc() argument
3287 return ____cache_alloc(cachep, flags); in __do_cache_alloc()
3293 slab_alloc(struct kmem_cache *cachep, gfp_t flags, unsigned long caller) in slab_alloc() argument
3300 cachep = slab_pre_alloc_hook(cachep, &objcg, 1, flags); in slab_alloc()
3301 if (unlikely(!cachep)) in slab_alloc()
3304 cache_alloc_debugcheck_before(cachep, flags); in slab_alloc()
3306 objp = __do_cache_alloc(cachep, flags); in slab_alloc()
3308 objp = cache_alloc_debugcheck_after(cachep, flags, objp, caller); in slab_alloc()
3311 if (unlikely(slab_want_init_on_alloc(flags, cachep)) && objp) in slab_alloc()
3312 memset(objp, 0, cachep->object_size); in slab_alloc()
3314 slab_post_alloc_hook(cachep, objcg, flags, 1, &objp); in slab_alloc()
3322 static void free_block(struct kmem_cache *cachep, void **objpp, in free_block() argument
3326 struct kmem_cache_node *n = get_node(cachep, node); in free_block()
3339 check_spinlock_acquired_node(cachep, node); in free_block()
3340 slab_put_obj(cachep, page, objp); in free_block()
3341 STATS_DEC_ACTIVE(cachep); in free_block()
3357 n->free_objects -= cachep->num; in free_block()
3366 static void cache_flusharray(struct kmem_cache *cachep, struct array_cache *ac) in cache_flusharray() argument
3376 n = get_node(cachep, node); in cache_flusharray()
3391 free_block(cachep, ac->entry, batchcount, node, &list); in cache_flusharray()
3403 STATS_SET_FREEABLE(cachep, i); in cache_flusharray()
3409 slabs_destroy(cachep, &list); in cache_flusharray()
3416 static __always_inline void __cache_free(struct kmem_cache *cachep, void *objp, in __cache_free() argument
3420 if (kasan_slab_free(cachep, objp, _RET_IP_)) in __cache_free()
3424 if (!(cachep->flags & SLAB_TYPESAFE_BY_RCU)) in __cache_free()
3425 __kcsan_check_access(objp, cachep->object_size, in __cache_free()
3428 ___cache_free(cachep, objp, caller); in __cache_free()
3431 void ___cache_free(struct kmem_cache *cachep, void *objp, in ___cache_free() argument
3434 struct array_cache *ac = cpu_cache_get(cachep); in ___cache_free()
3437 if (unlikely(slab_want_init_on_free(cachep))) in ___cache_free()
3438 memset(objp, 0, cachep->object_size); in ___cache_free()
3439 kmemleak_free_recursive(objp, cachep->flags); in ___cache_free()
3440 objp = cache_free_debugcheck(cachep, objp, caller); in ___cache_free()
3441 memcg_slab_free_hook(cachep, &objp, 1); in ___cache_free()
3450 if (nr_online_nodes > 1 && cache_free_alien(cachep, objp)) in ___cache_free()
3454 STATS_INC_FREEHIT(cachep); in ___cache_free()
3456 STATS_INC_FREEMISS(cachep); in ___cache_free()
3457 cache_flusharray(cachep, ac); in ___cache_free()
3464 cache_free_pfmemalloc(cachep, page, objp); in ___cache_free()
3482 void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags) in kmem_cache_alloc() argument
3484 void *ret = slab_alloc(cachep, flags, _RET_IP_); in kmem_cache_alloc()
3487 cachep->object_size, cachep->size, flags); in kmem_cache_alloc()
3546 kmem_cache_alloc_trace(struct kmem_cache *cachep, gfp_t flags, size_t size) in kmem_cache_alloc_trace() argument
3550 ret = slab_alloc(cachep, flags, _RET_IP_); in kmem_cache_alloc_trace()
3552 ret = kasan_kmalloc(cachep, ret, size, flags); in kmem_cache_alloc_trace()
3554 size, cachep->size, flags); in kmem_cache_alloc_trace()
3574 void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid) in kmem_cache_alloc_node() argument
3576 void *ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_); in kmem_cache_alloc_node()
3579 cachep->object_size, cachep->size, in kmem_cache_alloc_node()
3587 void *kmem_cache_alloc_node_trace(struct kmem_cache *cachep, in kmem_cache_alloc_node_trace() argument
3594 ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_); in kmem_cache_alloc_node_trace()
3596 ret = kasan_kmalloc(cachep, ret, size, flags); in kmem_cache_alloc_node_trace()
3598 size, cachep->size, in kmem_cache_alloc_node_trace()
3608 struct kmem_cache *cachep; in __do_kmalloc_node() local
3613 cachep = kmalloc_slab(size, flags); in __do_kmalloc_node()
3614 if (unlikely(ZERO_OR_NULL_PTR(cachep))) in __do_kmalloc_node()
3615 return cachep; in __do_kmalloc_node()
3616 ret = kmem_cache_alloc_node_trace(cachep, flags, node, size); in __do_kmalloc_node()
3617 ret = kasan_kmalloc(cachep, ret, size, flags); in __do_kmalloc_node()
3647 struct kmem_cache *cachep; in __do_kmalloc() local
3652 cachep = kmalloc_slab(size, flags); in __do_kmalloc()
3653 if (unlikely(ZERO_OR_NULL_PTR(cachep))) in __do_kmalloc()
3654 return cachep; in __do_kmalloc()
3655 ret = slab_alloc(cachep, flags, caller); in __do_kmalloc()
3657 ret = kasan_kmalloc(cachep, ret, size, flags); in __do_kmalloc()
3659 size, cachep->size, flags); in __do_kmalloc()
3684 void kmem_cache_free(struct kmem_cache *cachep, void *objp) in kmem_cache_free() argument
3687 cachep = cache_from_obj(cachep, objp); in kmem_cache_free()
3688 if (!cachep) in kmem_cache_free()
3692 debug_check_no_locks_freed(objp, cachep->object_size); in kmem_cache_free()
3693 if (!(cachep->flags & SLAB_DEBUG_OBJECTS)) in kmem_cache_free()
3694 debug_check_no_obj_freed(objp, cachep->object_size); in kmem_cache_free()
3695 __cache_free(cachep, objp, _RET_IP_); in kmem_cache_free()
3766 static int setup_kmem_cache_nodes(struct kmem_cache *cachep, gfp_t gfp) in setup_kmem_cache_nodes() argument
3773 ret = setup_kmem_cache_node(cachep, node, gfp, true); in setup_kmem_cache_nodes()
3782 if (!cachep->list.next) { in setup_kmem_cache_nodes()
3786 n = get_node(cachep, node); in setup_kmem_cache_nodes()
3791 cachep->node[node] = NULL; in setup_kmem_cache_nodes()
3800 static int do_tune_cpucache(struct kmem_cache *cachep, int limit, in do_tune_cpucache() argument
3806 cpu_cache = alloc_kmem_cache_cpus(cachep, limit, batchcount); in do_tune_cpucache()
3810 prev = cachep->cpu_cache; in do_tune_cpucache()
3811 cachep->cpu_cache = cpu_cache; in do_tune_cpucache()
3820 cachep->batchcount = batchcount; in do_tune_cpucache()
3821 cachep->limit = limit; in do_tune_cpucache()
3822 cachep->shared = shared; in do_tune_cpucache()
3834 n = get_node(cachep, node); in do_tune_cpucache()
3836 free_block(cachep, ac->entry, ac->avail, node, &list); in do_tune_cpucache()
3838 slabs_destroy(cachep, &list); in do_tune_cpucache()
3843 return setup_kmem_cache_nodes(cachep, gfp); in do_tune_cpucache()
3847 static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp) in enable_cpucache() argument
3854 err = cache_random_seq_create(cachep, cachep->num, gfp); in enable_cpucache()
3869 if (cachep->size > 131072) in enable_cpucache()
3871 else if (cachep->size > PAGE_SIZE) in enable_cpucache()
3873 else if (cachep->size > 1024) in enable_cpucache()
3875 else if (cachep->size > 256) in enable_cpucache()
3890 if (cachep->size <= PAGE_SIZE && num_possible_cpus() > 1) in enable_cpucache()
3903 err = do_tune_cpucache(cachep, limit, batchcount, shared, gfp); in enable_cpucache()
3907 cachep->name, -err); in enable_cpucache()
3916 static void drain_array(struct kmem_cache *cachep, struct kmem_cache_node *n, in drain_array() argument
3933 drain_array_locked(cachep, ac, node, false, &list); in drain_array()
3936 slabs_destroy(cachep, &list); in drain_array()
4008 void get_slabinfo(struct kmem_cache *cachep, struct slabinfo *sinfo) in get_slabinfo() argument
4016 for_each_kmem_cache_node(cachep, node, n) { in get_slabinfo()
4029 num_objs = total_slabs * cachep->num; in get_slabinfo()
4038 sinfo->limit = cachep->limit; in get_slabinfo()
4039 sinfo->batchcount = cachep->batchcount; in get_slabinfo()
4040 sinfo->shared = cachep->shared; in get_slabinfo()
4041 sinfo->objects_per_slab = cachep->num; in get_slabinfo()
4042 sinfo->cache_order = cachep->gfporder; in get_slabinfo()
4045 void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *cachep) in slabinfo_show_stats() argument
4049 unsigned long high = cachep->high_mark; in slabinfo_show_stats()
4050 unsigned long allocs = cachep->num_allocations; in slabinfo_show_stats()
4051 unsigned long grown = cachep->grown; in slabinfo_show_stats()
4052 unsigned long reaped = cachep->reaped; in slabinfo_show_stats()
4053 unsigned long errors = cachep->errors; in slabinfo_show_stats()
4054 unsigned long max_freeable = cachep->max_freeable; in slabinfo_show_stats()
4055 unsigned long node_allocs = cachep->node_allocs; in slabinfo_show_stats()
4056 unsigned long node_frees = cachep->node_frees; in slabinfo_show_stats()
4057 unsigned long overflows = cachep->node_overflow; in slabinfo_show_stats()
4066 unsigned long allochit = atomic_read(&cachep->allochit); in slabinfo_show_stats()
4067 unsigned long allocmiss = atomic_read(&cachep->allocmiss); in slabinfo_show_stats()
4068 unsigned long freehit = atomic_read(&cachep->freehit); in slabinfo_show_stats()
4069 unsigned long freemiss = atomic_read(&cachep->freemiss); in slabinfo_show_stats()
4092 struct kmem_cache *cachep; in slabinfo_write() local
4111 list_for_each_entry(cachep, &slab_caches, list) { in slabinfo_write()
4112 if (!strcmp(cachep->name, kbuf)) { in slabinfo_write()
4117 res = do_tune_cpucache(cachep, limit, in slabinfo_write()
4142 struct kmem_cache *cachep; in __check_heap_object() local
4149 cachep = page->slab_cache; in __check_heap_object()
4150 objnr = obj_to_index(cachep, page, (void *)ptr); in __check_heap_object()
4151 BUG_ON(objnr >= cachep->num); in __check_heap_object()
4154 offset = ptr - index_to_obj(cachep, page, objnr) - obj_offset(cachep); in __check_heap_object()
4157 if (offset >= cachep->useroffset && in __check_heap_object()
4158 offset - cachep->useroffset <= cachep->usersize && in __check_heap_object()
4159 n <= cachep->useroffset - offset + cachep->usersize) in __check_heap_object()
4169 offset <= cachep->object_size && in __check_heap_object()
4170 n <= cachep->object_size - offset) { in __check_heap_object()
4171 usercopy_warn("SLAB object", cachep->name, to_user, offset, n); in __check_heap_object()
4175 usercopy_abort("SLAB object", cachep->name, to_user, offset, n); in __check_heap_object()