Lines Matching refs:cpu_slab
299 raw_cpu_inc(s->cpu_slab->stat[si]); in stat()
2275 unsigned long actual_tid = __this_cpu_read(s->cpu_slab->tid); in note_cmpxchg_failure()
2301 c = per_cpu_ptr(s->cpu_slab, cpu); in init_kmem_cache_cpus()
2515 local_lock_irqsave(&s->cpu_slab->lock, flags); in unfreeze_partials()
2516 partial_page = this_cpu_read(s->cpu_slab->partial); in unfreeze_partials()
2517 this_cpu_write(s->cpu_slab->partial, NULL); in unfreeze_partials()
2518 local_unlock_irqrestore(&s->cpu_slab->lock, flags); in unfreeze_partials()
2551 local_lock_irqsave(&s->cpu_slab->lock, flags); in put_cpu_partial()
2553 oldpage = this_cpu_read(s->cpu_slab->partial); in put_cpu_partial()
2577 this_cpu_write(s->cpu_slab->partial, page); in put_cpu_partial()
2579 local_unlock_irqrestore(&s->cpu_slab->lock, flags); in put_cpu_partial()
2601 local_lock_irqsave(&s->cpu_slab->lock, flags); in flush_slab()
2610 local_unlock_irqrestore(&s->cpu_slab->lock, flags); in flush_slab()
2620 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu); in __flush_cpu_slab()
2656 c = this_cpu_ptr(s->cpu_slab); in flush_cpu_slab()
2666 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu); in has_cpu_slab()
2839 lockdep_assert_held(this_cpu_ptr(&s->cpu_slab->lock)); in get_freelist()
2925 local_lock_irqsave(&s->cpu_slab->lock, flags); in ___slab_alloc()
2927 local_unlock_irqrestore(&s->cpu_slab->lock, flags); in ___slab_alloc()
2938 local_unlock_irqrestore(&s->cpu_slab->lock, flags); in ___slab_alloc()
2947 lockdep_assert_held(this_cpu_ptr(&s->cpu_slab->lock)); in ___slab_alloc()
2957 local_unlock_irqrestore(&s->cpu_slab->lock, flags); in ___slab_alloc()
2962 local_lock_irqsave(&s->cpu_slab->lock, flags); in ___slab_alloc()
2964 local_unlock_irqrestore(&s->cpu_slab->lock, flags); in ___slab_alloc()
2970 local_unlock_irqrestore(&s->cpu_slab->lock, flags); in ___slab_alloc()
2976 local_lock_irqsave(&s->cpu_slab->lock, flags); in ___slab_alloc()
2978 local_unlock_irqrestore(&s->cpu_slab->lock, flags); in ___slab_alloc()
2982 local_unlock_irqrestore(&s->cpu_slab->lock, flags); in ___slab_alloc()
2989 local_unlock_irqrestore(&s->cpu_slab->lock, flags); in ___slab_alloc()
3000 slub_put_cpu_ptr(s->cpu_slab); in ___slab_alloc()
3002 c = slub_get_cpu_ptr(s->cpu_slab); in ___slab_alloc()
3042 local_lock_irqsave(&s->cpu_slab->lock, flags); in ___slab_alloc()
3051 local_unlock_irqrestore(&s->cpu_slab->lock, flags); in ___slab_alloc()
3085 c = slub_get_cpu_ptr(s->cpu_slab); in __slab_alloc()
3090 slub_put_cpu_ptr(s->cpu_slab); in __slab_alloc()
3148 c = raw_cpu_ptr(s->cpu_slab); in slab_alloc_node()
3198 s->cpu_slab->freelist, s->cpu_slab->tid, in slab_alloc_node()
3433 c = raw_cpu_ptr(s->cpu_slab); in do_slab_free()
3446 s->cpu_slab->freelist, s->cpu_slab->tid, in do_slab_free()
3463 local_lock(&s->cpu_slab->lock); in do_slab_free()
3464 c = this_cpu_ptr(s->cpu_slab); in do_slab_free()
3466 local_unlock(&s->cpu_slab->lock); in do_slab_free()
3476 local_unlock(&s->cpu_slab->lock); in do_slab_free()
3655 c = slub_get_cpu_ptr(s->cpu_slab); in kmem_cache_alloc_bulk()
3656 local_lock_irq(&s->cpu_slab->lock); in kmem_cache_alloc_bulk()
3677 local_unlock_irq(&s->cpu_slab->lock); in kmem_cache_alloc_bulk()
3688 c = this_cpu_ptr(s->cpu_slab); in kmem_cache_alloc_bulk()
3691 local_lock_irq(&s->cpu_slab->lock); in kmem_cache_alloc_bulk()
3700 local_unlock_irq(&s->cpu_slab->lock); in kmem_cache_alloc_bulk()
3701 slub_put_cpu_ptr(s->cpu_slab); in kmem_cache_alloc_bulk()
3711 slub_put_cpu_ptr(s->cpu_slab); in kmem_cache_alloc_bulk()
3879 s->cpu_slab = __alloc_percpu(sizeof(struct kmem_cache_cpu), in alloc_kmem_cache_cpus()
3882 if (!s->cpu_slab) in alloc_kmem_cache_cpus()
3951 free_percpu(s->cpu_slab); in __kmem_cache_release()
5230 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, in show_slab_objects()
5462 page = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu)); in slabs_cpu_partial_show()
5476 page = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu)); in slabs_cpu_partial_show()
5648 unsigned x = per_cpu_ptr(s->cpu_slab, cpu)->stat[si]; in show_stat()
5674 per_cpu_ptr(s->cpu_slab, cpu)->stat[si] = 0; in clear_stat()