152 static void perf_ctx_lock(struct perf_cpu_context *cpuctx,  in perf_ctx_lock()  argument
155 	raw_spin_lock(&cpuctx->ctx.lock);  in perf_ctx_lock()
160 static void perf_ctx_unlock(struct perf_cpu_context *cpuctx,  in perf_ctx_unlock()  argument
165 	raw_spin_unlock(&cpuctx->ctx.lock);  in perf_ctx_unlock()
208 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);  in event_function()  local
209 	struct perf_event_context *task_ctx = cpuctx->task_ctx;  in event_function()
214 	perf_ctx_lock(cpuctx, task_ctx);  in event_function()
239 		WARN_ON_ONCE(&cpuctx->ctx != ctx);  in event_function()
242 	efs->func(event, cpuctx, ctx, efs->data);  in event_function()
244 	perf_ctx_unlock(cpuctx, task_ctx);  in event_function()
305 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);  in event_function_local()  local
318 	perf_ctx_lock(cpuctx, task_ctx);  in event_function_local()
334 			if (WARN_ON_ONCE(cpuctx->task_ctx != ctx))  in event_function_local()
338 		WARN_ON_ONCE(&cpuctx->ctx != ctx);  in event_function_local()
341 	func(event, cpuctx, ctx, data);  in event_function_local()
343 	perf_ctx_unlock(cpuctx, task_ctx);  in event_function_local()
433 static bool perf_rotate_context(struct perf_cpu_context *cpuctx);
558 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
561 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
673 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);  in perf_cgroup_match()  local
680 	if (!cpuctx->cgrp)  in perf_cgroup_match()
689 	return cgroup_is_descendant(cpuctx->cgrp->css.cgroup,  in perf_cgroup_match()
725 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)  in update_cgrp_time_from_cpuctx()  argument
727 	struct perf_cgroup *cgrp = cpuctx->cgrp;  in update_cgrp_time_from_cpuctx()
795 	struct perf_cpu_context *cpuctx;  in perf_cgroup_switch()  local
806 	list_for_each_entry(cpuctx, list, cgrp_cpuctx_entry) {  in perf_cgroup_switch()
807 		WARN_ON_ONCE(cpuctx->ctx.nr_cgroups == 0);  in perf_cgroup_switch()
809 		perf_ctx_lock(cpuctx, cpuctx->task_ctx);  in perf_cgroup_switch()
810 		perf_pmu_disable(cpuctx->ctx.pmu);  in perf_cgroup_switch()
813 			cpu_ctx_sched_out(cpuctx, EVENT_ALL);  in perf_cgroup_switch()
818 			cpuctx->cgrp = NULL;  in perf_cgroup_switch()
822 			WARN_ON_ONCE(cpuctx->cgrp);  in perf_cgroup_switch()
830 			cpuctx->cgrp = perf_cgroup_from_task(task,  in perf_cgroup_switch()
831 							     &cpuctx->ctx);  in perf_cgroup_switch()
832 			cpu_ctx_sched_in(cpuctx, EVENT_ALL, task);  in perf_cgroup_switch()
834 		perf_pmu_enable(cpuctx->ctx.pmu);  in perf_cgroup_switch()
835 		perf_ctx_unlock(cpuctx, cpuctx->task_ctx);  in perf_cgroup_switch()
945 	struct perf_cpu_context *cpuctx;  in list_update_cgroup_event()  local
955 	cpuctx = __get_cpu_context(ctx);  in list_update_cgroup_event()
963 	if (add && !cpuctx->cgrp) {  in list_update_cgroup_event()
967 			cpuctx->cgrp = cgrp;  in list_update_cgroup_event()
977 		cpuctx->cgrp = NULL;  in list_update_cgroup_event()
979 	cpuctx_entry = &cpuctx->cgrp_cpuctx_entry;  in list_update_cgroup_event()
1006 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)  in update_cgrp_time_from_cpuctx()  argument
1066 	struct perf_cpu_context *cpuctx;  in perf_mux_hrtimer_handler()  local
1071 	cpuctx = container_of(hr, struct perf_cpu_context, hrtimer);  in perf_mux_hrtimer_handler()
1072 	rotations = perf_rotate_context(cpuctx);  in perf_mux_hrtimer_handler()
1074 	raw_spin_lock(&cpuctx->hrtimer_lock);  in perf_mux_hrtimer_handler()
1076 		hrtimer_forward_now(hr, cpuctx->hrtimer_interval);  in perf_mux_hrtimer_handler()
1078 		cpuctx->hrtimer_active = 0;  in perf_mux_hrtimer_handler()
1079 	raw_spin_unlock(&cpuctx->hrtimer_lock);  in perf_mux_hrtimer_handler()
1084 static void __perf_mux_hrtimer_init(struct perf_cpu_context *cpuctx, int cpu)  in __perf_mux_hrtimer_init()  argument
1086 	struct hrtimer *timer = &cpuctx->hrtimer;  in __perf_mux_hrtimer_init()
1087 	struct pmu *pmu = cpuctx->ctx.pmu;  in __perf_mux_hrtimer_init()
1102 	cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * interval);  in __perf_mux_hrtimer_init()
1104 	raw_spin_lock_init(&cpuctx->hrtimer_lock);  in __perf_mux_hrtimer_init()
1109 static int perf_mux_hrtimer_restart(struct perf_cpu_context *cpuctx)  in perf_mux_hrtimer_restart()  argument
1111 	struct hrtimer *timer = &cpuctx->hrtimer;  in perf_mux_hrtimer_restart()
1112 	struct pmu *pmu = cpuctx->ctx.pmu;  in perf_mux_hrtimer_restart()
1119 	raw_spin_lock_irqsave(&cpuctx->hrtimer_lock, flags);  in perf_mux_hrtimer_restart()
1120 	if (!cpuctx->hrtimer_active) {  in perf_mux_hrtimer_restart()
1121 		cpuctx->hrtimer_active = 1;  in perf_mux_hrtimer_restart()
1122 		hrtimer_forward_now(timer, cpuctx->hrtimer_interval);  in perf_mux_hrtimer_restart()
1125 	raw_spin_unlock_irqrestore(&cpuctx->hrtimer_lock, flags);  in perf_mux_hrtimer_restart()
1987 		  struct perf_cpu_context *cpuctx,  in event_sched_out()  argument
2017 		cpuctx->active_oncpu--;  in event_sched_out()
2022 	if (event->attr.exclusive || !cpuctx->active_oncpu)  in event_sched_out()
2023 		cpuctx->exclusive = 0;  in event_sched_out()
2030 		struct perf_cpu_context *cpuctx,  in group_sched_out()  argument
2040 	event_sched_out(group_event, cpuctx, ctx);  in group_sched_out()
2046 		event_sched_out(event, cpuctx, ctx);  in group_sched_out()
2051 		cpuctx->exclusive = 0;  in group_sched_out()
2064 			   struct perf_cpu_context *cpuctx,  in __perf_remove_from_context()  argument
2072 		update_cgrp_time_from_cpuctx(cpuctx);  in __perf_remove_from_context()
2075 	event_sched_out(event, cpuctx, ctx);  in __perf_remove_from_context()
2083 			WARN_ON_ONCE(cpuctx->task_ctx != ctx);  in __perf_remove_from_context()
2084 			cpuctx->task_ctx = NULL;  in __perf_remove_from_context()
2130 				 struct perf_cpu_context *cpuctx,  in __perf_event_disable()  argument
2143 		group_sched_out(event, cpuctx, ctx);  in __perf_event_disable()
2145 		event_sched_out(event, cpuctx, ctx);  in __perf_event_disable()
2244 		 struct perf_cpu_context *cpuctx,  in event_sched_in()  argument
2287 		cpuctx->active_oncpu++;  in event_sched_in()
2294 		cpuctx->exclusive = 1;  in event_sched_in()
2304 	       struct perf_cpu_context *cpuctx,  in group_sched_in()  argument
2315 	if (event_sched_in(group_event, cpuctx, ctx)) {  in group_sched_in()
2317 		perf_mux_hrtimer_restart(cpuctx);  in group_sched_in()
2325 		if (event_sched_in(event, cpuctx, ctx)) {  in group_sched_in()
2344 		event_sched_out(event, cpuctx, ctx);  in group_sched_in()
2346 	event_sched_out(group_event, cpuctx, ctx);  in group_sched_in()
2350 	perf_mux_hrtimer_restart(cpuctx);  in group_sched_in()
2359 			   struct perf_cpu_context *cpuctx,  in group_can_go_on()  argument
2371 	if (cpuctx->exclusive)  in group_can_go_on()
2377 	if (event->attr.exclusive && cpuctx->active_oncpu)  in group_can_go_on()
2394 			  struct perf_cpu_context *cpuctx,
2398 	     struct perf_cpu_context *cpuctx,
2402 static void task_ctx_sched_out(struct perf_cpu_context *cpuctx,  in task_ctx_sched_out()  argument
2406 	if (!cpuctx->task_ctx)  in task_ctx_sched_out()
2409 	if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))  in task_ctx_sched_out()
2412 	ctx_sched_out(ctx, cpuctx, event_type);  in task_ctx_sched_out()
2415 static void perf_event_sched_in(struct perf_cpu_context *cpuctx,  in perf_event_sched_in()  argument
2419 	cpu_ctx_sched_in(cpuctx, EVENT_PINNED, task);  in perf_event_sched_in()
2421 		ctx_sched_in(ctx, cpuctx, EVENT_PINNED, task);  in perf_event_sched_in()
2422 	cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE, task);  in perf_event_sched_in()
2424 		ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE, task);  in perf_event_sched_in()
2442 static void ctx_resched(struct perf_cpu_context *cpuctx,  in ctx_resched()  argument
2458 	perf_pmu_disable(cpuctx->ctx.pmu);  in ctx_resched()
2460 		task_ctx_sched_out(cpuctx, task_ctx, event_type);  in ctx_resched()
2470 		cpu_ctx_sched_out(cpuctx, ctx_event_type);  in ctx_resched()
2472 		cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);  in ctx_resched()
2474 	perf_event_sched_in(cpuctx, task_ctx, current);  in ctx_resched()
2475 	perf_pmu_enable(cpuctx->ctx.pmu);  in ctx_resched()
2488 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);  in __perf_install_in_context()  local
2489 	struct perf_event_context *task_ctx = cpuctx->task_ctx;  in __perf_install_in_context()
2493 	raw_spin_lock(&cpuctx->ctx.lock);  in __perf_install_in_context()
2512 		WARN_ON_ONCE(reprogram && cpuctx->task_ctx && cpuctx->task_ctx != ctx);  in __perf_install_in_context()
2530 		ctx_sched_out(ctx, cpuctx, EVENT_TIME);  in __perf_install_in_context()
2532 		ctx_resched(cpuctx, task_ctx, get_event_type(event));  in __perf_install_in_context()
2538 	perf_ctx_unlock(cpuctx, task_ctx);  in __perf_install_in_context()
2639 				struct perf_cpu_context *cpuctx,  in __perf_event_enable()  argument
2651 		ctx_sched_out(ctx, cpuctx, EVENT_TIME);  in __perf_event_enable()
2659 		ctx_sched_in(ctx, cpuctx, EVENT_TIME, current);  in __perf_event_enable()
2668 		ctx_sched_in(ctx, cpuctx, EVENT_TIME, current);  in __perf_event_enable()
2672 	task_ctx = cpuctx->task_ctx;  in __perf_event_enable()
2676 	ctx_resched(cpuctx, task_ctx, get_event_type(event));  in __perf_event_enable()
2893 			  struct perf_cpu_context *cpuctx,  in ctx_sched_out()  argument
2907 			WARN_ON_ONCE(cpuctx->task_ctx);  in ctx_sched_out()
2916 		WARN_ON_ONCE(cpuctx->task_ctx != ctx);  in ctx_sched_out()
2918 			cpuctx->task_ctx = NULL;  in ctx_sched_out()
2934 		update_cgrp_time_from_cpuctx(cpuctx);  in ctx_sched_out()
2945 			group_sched_out(event, cpuctx, ctx);  in ctx_sched_out()
2950 			group_sched_out(event, cpuctx, ctx);  in ctx_sched_out()
3063 	struct perf_cpu_context *cpuctx;  in perf_event_context_sched_out()  local
3069 	cpuctx = __get_cpu_context(ctx);  in perf_event_context_sched_out()
3070 	if (!cpuctx->task_ctx)  in perf_event_context_sched_out()
3125 		task_ctx_sched_out(cpuctx, ctx, EVENT_ALL);  in perf_event_context_sched_out()
3134 	struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);  in perf_sched_cb_dec()  local
3138 	if (!--cpuctx->sched_cb_usage)  in perf_sched_cb_dec()
3139 		list_del(&cpuctx->sched_cb_entry);  in perf_sched_cb_dec()
3145 	struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);  in perf_sched_cb_inc()  local
3147 	if (!cpuctx->sched_cb_usage++)  in perf_sched_cb_inc()
3148 		list_add(&cpuctx->sched_cb_entry, this_cpu_ptr(&sched_cb_list));  in perf_sched_cb_inc()
3165 	struct perf_cpu_context *cpuctx;  in perf_pmu_sched_task()  local
3171 	list_for_each_entry(cpuctx, this_cpu_ptr(&sched_cb_list), sched_cb_entry) {  in perf_pmu_sched_task()
3172 		pmu = cpuctx->ctx.pmu; /* software PMUs will not have sched_task */  in perf_pmu_sched_task()
3177 		perf_ctx_lock(cpuctx, cpuctx->task_ctx);  in perf_pmu_sched_task()
3180 		pmu->sched_task(cpuctx->task_ctx, sched_in);  in perf_pmu_sched_task()
3183 		perf_ctx_unlock(cpuctx, cpuctx->task_ctx);  in perf_pmu_sched_task()
3230 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,  in cpu_ctx_sched_out()  argument
3233 	ctx_sched_out(&cpuctx->ctx, cpuctx, event_type);  in cpu_ctx_sched_out()
3269 	struct perf_cpu_context *cpuctx;  member
3283 	if (group_can_go_on(event, sid->cpuctx, sid->can_add_hw)) {  in pinned_sched_in()
3284 		if (!group_sched_in(event, sid->cpuctx, sid->ctx))  in pinned_sched_in()
3308 	if (group_can_go_on(event, sid->cpuctx, sid->can_add_hw)) {  in flexible_sched_in()
3309 		if (!group_sched_in(event, sid->cpuctx, sid->ctx))  in flexible_sched_in()
3320 		    struct perf_cpu_context *cpuctx)  in ctx_pinned_sched_in()  argument
3324 		.cpuctx = cpuctx,  in ctx_pinned_sched_in()
3335 		      struct perf_cpu_context *cpuctx)  in ctx_flexible_sched_in()  argument
3339 		.cpuctx = cpuctx,  in ctx_flexible_sched_in()
3350 	     struct perf_cpu_context *cpuctx,  in ctx_sched_in()  argument
3365 			cpuctx->task_ctx = ctx;  in ctx_sched_in()
3367 			WARN_ON_ONCE(cpuctx->task_ctx != ctx);  in ctx_sched_in()
3384 		ctx_pinned_sched_in(ctx, cpuctx);  in ctx_sched_in()
3388 		ctx_flexible_sched_in(ctx, cpuctx);  in ctx_sched_in()
3391 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,  in cpu_ctx_sched_in()  argument
3395 	struct perf_event_context *ctx = &cpuctx->ctx;  in cpu_ctx_sched_in()
3397 	ctx_sched_in(ctx, cpuctx, event_type, task);  in cpu_ctx_sched_in()
3403 	struct perf_cpu_context *cpuctx;  in perf_event_context_sched_in()  local
3405 	cpuctx = __get_cpu_context(ctx);  in perf_event_context_sched_in()
3406 	if (cpuctx->task_ctx == ctx)  in perf_event_context_sched_in()
3409 	perf_ctx_lock(cpuctx, ctx);  in perf_event_context_sched_in()
3427 		cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);  in perf_event_context_sched_in()
3428 	perf_event_sched_in(cpuctx, ctx, task);  in perf_event_context_sched_in()
3432 	perf_ctx_unlock(cpuctx, ctx);  in perf_event_context_sched_in()
3677 static bool perf_rotate_context(struct perf_cpu_context *cpuctx)  in perf_rotate_context()  argument
3688 	if (cpuctx->ctx.nr_events) {  in perf_rotate_context()
3689 		if (cpuctx->ctx.nr_events != cpuctx->ctx.nr_active)  in perf_rotate_context()
3693 	ctx = cpuctx->task_ctx;  in perf_rotate_context()
3702 	perf_ctx_lock(cpuctx, cpuctx->task_ctx);  in perf_rotate_context()
3703 	perf_pmu_disable(cpuctx->ctx.pmu);  in perf_rotate_context()
3708 		cpu_event = ctx_first_active(&cpuctx->ctx);  in perf_rotate_context()
3715 		ctx_sched_out(ctx, cpuctx, EVENT_FLEXIBLE);  in perf_rotate_context()
3717 		cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);  in perf_rotate_context()
3722 		rotate_ctx(&cpuctx->ctx, cpu_event);  in perf_rotate_context()
3724 	perf_event_sched_in(cpuctx, ctx, current);  in perf_rotate_context()
3726 	perf_pmu_enable(cpuctx->ctx.pmu);  in perf_rotate_context()
3727 	perf_ctx_unlock(cpuctx, cpuctx->task_ctx);  in perf_rotate_context()
3771 	struct perf_cpu_context *cpuctx;  in perf_event_enable_on_exec()  local
3781 	cpuctx = __get_cpu_context(ctx);  in perf_event_enable_on_exec()
3782 	perf_ctx_lock(cpuctx, ctx);  in perf_event_enable_on_exec()
3783 	ctx_sched_out(ctx, cpuctx, EVENT_TIME);  in perf_event_enable_on_exec()
3794 		ctx_resched(cpuctx, ctx, event_type);  in perf_event_enable_on_exec()
3796 		ctx_sched_in(ctx, cpuctx, EVENT_TIME, current);  in perf_event_enable_on_exec()
3798 	perf_ctx_unlock(cpuctx, ctx);  in perf_event_enable_on_exec()
3838 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);  in __perf_event_read()  local
3848 	if (ctx->task && cpuctx->task_ctx != ctx)  in __perf_event_read()
4109 	struct perf_cpu_context *cpuctx;  in find_get_context()  local
4120 		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);  in find_get_context()
4121 		ctx = &cpuctx->ctx;  in find_get_context()
4930 				struct perf_cpu_context *cpuctx,  in __perf_event_period()  argument
6753 	struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);  in __perf_pmu_output_stop()  local
6759 	perf_iterate_ctx(&cpuctx->ctx, __perf_event_output_stop, &ro, false);  in __perf_pmu_output_stop()
6760 	if (cpuctx->task_ctx)  in __perf_pmu_output_stop()
6761 		perf_iterate_ctx(cpuctx->task_ctx, __perf_event_output_stop,  in __perf_pmu_output_stop()
9500 		struct perf_cpu_context *cpuctx;  in perf_event_mux_interval_ms_store()  local
9501 		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);  in perf_event_mux_interval_ms_store()
9502 		cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * timer);  in perf_event_mux_interval_ms_store()
9505 			(remote_function_f)perf_mux_hrtimer_restart, cpuctx);  in perf_event_mux_interval_ms_store()
9630 		struct perf_cpu_context *cpuctx;  in perf_pmu_register()  local
9632 		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);  in perf_pmu_register()
9633 		__perf_event_init_context(&cpuctx->ctx);  in perf_pmu_register()
9634 		lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex);  in perf_pmu_register()
9635 		lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock);  in perf_pmu_register()
9636 		cpuctx->ctx.pmu = pmu;  in perf_pmu_register()
9637 		cpuctx->online = cpumask_test_cpu(cpu, perf_online_mask);  in perf_pmu_register()
9639 		__perf_mux_hrtimer_init(cpuctx, cpu);  in perf_pmu_register()
10677 		struct perf_cpu_context *cpuctx =  in SYSCALL_DEFINE5()  local
10680 		if (!cpuctx->online) {  in SYSCALL_DEFINE5()
10868 		struct perf_cpu_context *cpuctx =  in perf_event_create_kernel_counter()  local
10870 		if (!cpuctx->online) {  in perf_event_create_kernel_counter()
11601 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);  in __perf_event_exit_context()  local
11605 	ctx_sched_out(ctx, cpuctx, EVENT_TIME);  in __perf_event_exit_context()
11607 		__perf_remove_from_context(event, cpuctx, ctx, (void *)DETACH_GROUP);  in __perf_event_exit_context()
11613 	struct perf_cpu_context *cpuctx;  in perf_event_exit_cpu_context()  local
11619 		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);  in perf_event_exit_cpu_context()
11620 		ctx = &cpuctx->ctx;  in perf_event_exit_cpu_context()
11624 		cpuctx->online = 0;  in perf_event_exit_cpu_context()
11638 	struct perf_cpu_context *cpuctx;  in perf_event_init_cpu()  local
11647 		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);  in perf_event_init_cpu()
11648 		ctx = &cpuctx->ctx;  in perf_event_init_cpu()
11651 		cpuctx->online = 1;  in perf_event_init_cpu()