Lines Matching refs:cpuctx
158 static void perf_ctx_lock(struct perf_cpu_context *cpuctx, in perf_ctx_lock() argument
161 raw_spin_lock(&cpuctx->ctx.lock); in perf_ctx_lock()
166 static void perf_ctx_unlock(struct perf_cpu_context *cpuctx, in perf_ctx_unlock() argument
171 raw_spin_unlock(&cpuctx->ctx.lock); in perf_ctx_unlock()
222 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); in event_function() local
223 struct perf_event_context *task_ctx = cpuctx->task_ctx; in event_function()
228 perf_ctx_lock(cpuctx, task_ctx); in event_function()
253 WARN_ON_ONCE(&cpuctx->ctx != ctx); in event_function()
256 efs->func(event, cpuctx, ctx, efs->data); in event_function()
258 perf_ctx_unlock(cpuctx, task_ctx); in event_function()
319 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); in event_function_local() local
332 perf_ctx_lock(cpuctx, task_ctx); in event_function_local()
348 if (WARN_ON_ONCE(cpuctx->task_ctx != ctx)) in event_function_local()
352 WARN_ON_ONCE(&cpuctx->ctx != ctx); in event_function_local()
355 func(event, cpuctx, ctx, data); in event_function_local()
357 perf_ctx_unlock(cpuctx, task_ctx); in event_function_local()
711 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); in perf_cgroup_match() local
718 if (!cpuctx->cgrp) in perf_cgroup_match()
727 return cgroup_is_descendant(cpuctx->cgrp->css.cgroup, in perf_cgroup_match()
772 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx, bool final) in update_cgrp_time_from_cpuctx() argument
774 struct perf_cgroup *cgrp = cpuctx->cgrp; in update_cgrp_time_from_cpuctx()
812 perf_cgroup_set_timestamp(struct perf_cpu_context *cpuctx) in perf_cgroup_set_timestamp() argument
814 struct perf_event_context *ctx = &cpuctx->ctx; in perf_cgroup_set_timestamp()
815 struct perf_cgroup *cgrp = cpuctx->cgrp; in perf_cgroup_set_timestamp()
842 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); in perf_cgroup_switch() local
849 if (READ_ONCE(cpuctx->cgrp) == NULL) in perf_cgroup_switch()
852 WARN_ON_ONCE(cpuctx->ctx.nr_cgroups == 0); in perf_cgroup_switch()
855 if (READ_ONCE(cpuctx->cgrp) == cgrp) in perf_cgroup_switch()
858 perf_ctx_lock(cpuctx, cpuctx->task_ctx); in perf_cgroup_switch()
859 perf_ctx_disable(&cpuctx->ctx); in perf_cgroup_switch()
861 ctx_sched_out(&cpuctx->ctx, EVENT_ALL); in perf_cgroup_switch()
867 cpuctx->cgrp = cgrp; in perf_cgroup_switch()
873 ctx_sched_in(&cpuctx->ctx, EVENT_ALL); in perf_cgroup_switch()
875 perf_ctx_enable(&cpuctx->ctx); in perf_cgroup_switch()
876 perf_ctx_unlock(cpuctx, cpuctx->task_ctx); in perf_cgroup_switch()
882 struct perf_cpu_context *cpuctx; in perf_cgroup_ensure_storage() local
894 cpuctx = per_cpu_ptr(&perf_cpu_context, cpu); in perf_cgroup_ensure_storage()
895 if (heap_size <= cpuctx->heap_size) in perf_cgroup_ensure_storage()
905 raw_spin_lock_irq(&cpuctx->ctx.lock); in perf_cgroup_ensure_storage()
906 if (cpuctx->heap_size < heap_size) { in perf_cgroup_ensure_storage()
907 swap(cpuctx->heap, storage); in perf_cgroup_ensure_storage()
908 if (storage == cpuctx->heap_default) in perf_cgroup_ensure_storage()
910 cpuctx->heap_size = heap_size; in perf_cgroup_ensure_storage()
912 raw_spin_unlock_irq(&cpuctx->ctx.lock); in perf_cgroup_ensure_storage()
963 struct perf_cpu_context *cpuctx; in perf_cgroup_event_enable() local
972 cpuctx = container_of(ctx, struct perf_cpu_context, ctx); in perf_cgroup_event_enable()
977 cpuctx->cgrp = perf_cgroup_from_task(current, ctx); in perf_cgroup_event_enable()
983 struct perf_cpu_context *cpuctx; in perf_cgroup_event_disable() local
992 cpuctx = container_of(ctx, struct perf_cpu_context, ctx); in perf_cgroup_event_disable()
997 cpuctx->cgrp = NULL; in perf_cgroup_event_disable()
1020 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx, in update_cgrp_time_from_cpuctx() argument
1033 perf_cgroup_set_timestamp(struct perf_cpu_context *cpuctx) in perf_cgroup_set_timestamp() argument
2308 struct perf_cpu_context *cpuctx, in __perf_remove_from_context() argument
2317 update_cgrp_time_from_cpuctx(cpuctx, false); in __perf_remove_from_context()
2348 if (ctx == &cpuctx->ctx) in __perf_remove_from_context()
2349 update_cgrp_time_from_cpuctx(cpuctx, true); in __perf_remove_from_context()
2353 WARN_ON_ONCE(cpuctx->task_ctx != ctx); in __perf_remove_from_context()
2354 cpuctx->task_ctx = NULL; in __perf_remove_from_context()
2396 struct perf_cpu_context *cpuctx, in __perf_event_disable() argument
2625 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); in task_ctx_sched_out() local
2627 if (!cpuctx->task_ctx) in task_ctx_sched_out()
2630 if (WARN_ON_ONCE(ctx != cpuctx->task_ctx)) in task_ctx_sched_out()
2636 static void perf_event_sched_in(struct perf_cpu_context *cpuctx, in perf_event_sched_in() argument
2639 ctx_sched_in(&cpuctx->ctx, EVENT_PINNED); in perf_event_sched_in()
2642 ctx_sched_in(&cpuctx->ctx, EVENT_FLEXIBLE); in perf_event_sched_in()
2667 static void ctx_resched(struct perf_cpu_context *cpuctx, in ctx_resched() argument
2682 perf_ctx_disable(&cpuctx->ctx); in ctx_resched()
2696 ctx_sched_out(&cpuctx->ctx, event_type); in ctx_resched()
2698 ctx_sched_out(&cpuctx->ctx, EVENT_FLEXIBLE); in ctx_resched()
2700 perf_event_sched_in(cpuctx, task_ctx); in ctx_resched()
2702 perf_ctx_enable(&cpuctx->ctx); in ctx_resched()
2709 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); in perf_pmu_resched() local
2710 struct perf_event_context *task_ctx = cpuctx->task_ctx; in perf_pmu_resched()
2712 perf_ctx_lock(cpuctx, task_ctx); in perf_pmu_resched()
2713 ctx_resched(cpuctx, task_ctx, EVENT_ALL|EVENT_CPU); in perf_pmu_resched()
2714 perf_ctx_unlock(cpuctx, task_ctx); in perf_pmu_resched()
2727 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); in __perf_install_in_context() local
2728 struct perf_event_context *task_ctx = cpuctx->task_ctx; in __perf_install_in_context()
2732 raw_spin_lock(&cpuctx->ctx.lock); in __perf_install_in_context()
2751 WARN_ON_ONCE(reprogram && cpuctx->task_ctx && cpuctx->task_ctx != ctx); in __perf_install_in_context()
2771 ctx_resched(cpuctx, task_ctx, get_event_type(event)); in __perf_install_in_context()
2777 perf_ctx_unlock(cpuctx, task_ctx); in __perf_install_in_context()
2903 struct perf_cpu_context *cpuctx, in __perf_event_enable() argument
2937 task_ctx = cpuctx->task_ctx; in __perf_event_enable()
2941 ctx_resched(cpuctx, task_ctx, get_event_type(event)); in __perf_event_enable()
3246 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); in ctx_sched_out() local
3258 WARN_ON_ONCE(cpuctx->task_ctx); in ctx_sched_out()
3275 update_cgrp_time_from_cpuctx(cpuctx, ctx == &cpuctx->ctx); in ctx_sched_out()
3288 WARN_ON_ONCE(cpuctx->task_ctx != ctx); in ctx_sched_out()
3290 cpuctx->task_ctx = NULL; in ctx_sched_out()
3578 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); in __perf_pmu_sched_task() local
3587 perf_ctx_lock(cpuctx, cpuctx->task_ctx); in __perf_pmu_sched_task()
3593 perf_ctx_unlock(cpuctx, cpuctx->task_ctx); in __perf_pmu_sched_task()
3600 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); in perf_pmu_sched_task() local
3604 if (prev == next || cpuctx->task_ctx) in perf_pmu_sched_task()
3696 struct perf_cpu_context *cpuctx = NULL; in visit_groups_merge() local
3707 cpuctx = this_cpu_ptr(&perf_cpu_context); in visit_groups_merge()
3709 .data = cpuctx->heap, in visit_groups_merge()
3711 .size = cpuctx->heap_size, in visit_groups_merge()
3714 lockdep_assert_held(&cpuctx->ctx.lock); in visit_groups_merge()
3717 if (cpuctx->cgrp) in visit_groups_merge()
3718 css = &cpuctx->cgrp->css; in visit_groups_merge()
3869 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); in ctx_sched_in() local
3880 perf_cgroup_set_timestamp(cpuctx); in ctx_sched_in()
3891 cpuctx->task_ctx = ctx; in ctx_sched_in()
3893 WARN_ON_ONCE(cpuctx->task_ctx != ctx); in ctx_sched_in()
3912 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); in perf_event_context_sched_in() local
3920 if (cpuctx->task_ctx == ctx) { in perf_event_context_sched_in()
3921 perf_ctx_lock(cpuctx, ctx); in perf_event_context_sched_in()
3927 perf_ctx_unlock(cpuctx, ctx); in perf_event_context_sched_in()
3931 perf_ctx_lock(cpuctx, ctx); in perf_event_context_sched_in()
3949 perf_ctx_disable(&cpuctx->ctx); in perf_event_context_sched_in()
3950 ctx_sched_out(&cpuctx->ctx, EVENT_FLEXIBLE); in perf_event_context_sched_in()
3953 perf_event_sched_in(cpuctx, ctx); in perf_event_context_sched_in()
3955 perf_ctx_sched_task_cb(cpuctx->task_ctx, true); in perf_event_context_sched_in()
3958 perf_ctx_enable(&cpuctx->ctx); in perf_event_context_sched_in()
3963 perf_ctx_unlock(cpuctx, ctx); in perf_event_context_sched_in()
4236 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); in perf_rotate_context() local
4257 perf_ctx_lock(cpuctx, cpuctx->task_ctx); in perf_rotate_context()
4275 update_context_time(&cpuctx->ctx); in perf_rotate_context()
4277 rotate_ctx(&cpuctx->ctx, cpu_event); in perf_rotate_context()
4278 __pmu_ctx_sched_in(&cpuctx->ctx, pmu); in perf_rotate_context()
4288 perf_ctx_unlock(cpuctx, cpuctx->task_ctx); in perf_rotate_context()
4295 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); in perf_event_task_tick() local
4305 perf_adjust_freq_unthr_context(&cpuctx->ctx, !!throttled); in perf_event_task_tick()
4337 struct perf_cpu_context *cpuctx; in perf_event_enable_on_exec() local
4349 cpuctx = this_cpu_ptr(&perf_cpu_context); in perf_event_enable_on_exec()
4350 perf_ctx_lock(cpuctx, ctx); in perf_event_enable_on_exec()
4363 ctx_resched(cpuctx, ctx, event_type); in perf_event_enable_on_exec()
4367 perf_ctx_unlock(cpuctx, ctx); in perf_event_enable_on_exec()
4451 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); in __perf_event_read() local
4461 if (ctx->task && cpuctx->task_ctx != ctx) in __perf_event_read()
4737 struct perf_cpu_context *cpuctx; in find_get_context() local
4747 cpuctx = per_cpu_ptr(&perf_cpu_context, event->cpu); in find_get_context()
4748 ctx = &cpuctx->ctx; in find_get_context()
5729 struct perf_cpu_context *cpuctx, in __perf_event_period() argument
8062 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); in __perf_pmu_output_stop() local
8068 perf_iterate_ctx(&cpuctx->ctx, __perf_event_output_stop, &ro, false); in __perf_pmu_output_stop()
8069 if (cpuctx->task_ctx) in __perf_pmu_output_stop()
8070 perf_iterate_ctx(cpuctx->task_ctx, __perf_event_output_stop, in __perf_pmu_output_stop()
12528 struct perf_cpu_context *cpuctx = per_cpu_ptr(&perf_cpu_context, event->cpu); in SYSCALL_DEFINE5() local
12530 if (!cpuctx->online) { in SYSCALL_DEFINE5()
12817 struct perf_cpu_context *cpuctx = in perf_event_create_kernel_counter() local
12819 if (!cpuctx->online) { in perf_event_create_kernel_counter()
13551 struct perf_cpu_context *cpuctx; in perf_event_init_all_cpus() local
13565 cpuctx = per_cpu_ptr(&perf_cpu_context, cpu); in perf_event_init_all_cpus()
13566 __perf_event_init_context(&cpuctx->ctx); in perf_event_init_all_cpus()
13567 lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex); in perf_event_init_all_cpus()
13568 lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock); in perf_event_init_all_cpus()
13569 cpuctx->online = cpumask_test_cpu(cpu, perf_online_mask); in perf_event_init_all_cpus()
13570 cpuctx->heap_size = ARRAY_SIZE(cpuctx->heap_default); in perf_event_init_all_cpus()
13571 cpuctx->heap = cpuctx->heap_default; in perf_event_init_all_cpus()
13593 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); in __perf_event_exit_context() local
13600 __perf_remove_from_context(event, cpuctx, ctx, (void *)DETACH_GROUP); in __perf_event_exit_context()
13606 struct perf_cpu_context *cpuctx; in perf_event_exit_cpu_context() local
13611 cpuctx = per_cpu_ptr(&perf_cpu_context, cpu); in perf_event_exit_cpu_context()
13612 ctx = &cpuctx->ctx; in perf_event_exit_cpu_context()
13616 cpuctx->online = 0; in perf_event_exit_cpu_context()
13629 struct perf_cpu_context *cpuctx; in perf_event_init_cpu() local
13636 cpuctx = per_cpu_ptr(&perf_cpu_context, cpu); in perf_event_init_cpu()
13637 ctx = &cpuctx->ctx; in perf_event_init_cpu()
13640 cpuctx->online = 1; in perf_event_init_cpu()