151 static void perf_ctx_lock(struct perf_cpu_context *cpuctx,  in perf_ctx_lock()  argument
154 	raw_spin_lock(&cpuctx->ctx.lock);  in perf_ctx_lock()
159 static void perf_ctx_unlock(struct perf_cpu_context *cpuctx,  in perf_ctx_unlock()  argument
164 	raw_spin_unlock(&cpuctx->ctx.lock);  in perf_ctx_unlock()
207 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);  in event_function()  local
208 	struct perf_event_context *task_ctx = cpuctx->task_ctx;  in event_function()
213 	perf_ctx_lock(cpuctx, task_ctx);  in event_function()
238 		WARN_ON_ONCE(&cpuctx->ctx != ctx);  in event_function()
241 	efs->func(event, cpuctx, ctx, efs->data);  in event_function()
243 	perf_ctx_unlock(cpuctx, task_ctx);  in event_function()
304 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);  in event_function_local()  local
317 	perf_ctx_lock(cpuctx, task_ctx);  in event_function_local()
333 			if (WARN_ON_ONCE(cpuctx->task_ctx != ctx))  in event_function_local()
337 		WARN_ON_ONCE(&cpuctx->ctx != ctx);  in event_function_local()
340 	func(event, cpuctx, ctx, data);  in event_function_local()
342 	perf_ctx_unlock(cpuctx, task_ctx);  in event_function_local()
434 static bool perf_rotate_context(struct perf_cpu_context *cpuctx);
559 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
562 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
674 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);  in perf_cgroup_match()  local
681 	if (!cpuctx->cgrp)  in perf_cgroup_match()
690 	return cgroup_is_descendant(cpuctx->cgrp->css.cgroup,  in perf_cgroup_match()
726 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)  in update_cgrp_time_from_cpuctx()  argument
728 	struct perf_cgroup *cgrp = cpuctx->cgrp;  in update_cgrp_time_from_cpuctx()
796 	struct perf_cpu_context *cpuctx;  in perf_cgroup_switch()  local
807 	list_for_each_entry(cpuctx, list, cgrp_cpuctx_entry) {  in perf_cgroup_switch()
808 		WARN_ON_ONCE(cpuctx->ctx.nr_cgroups == 0);  in perf_cgroup_switch()
810 		perf_ctx_lock(cpuctx, cpuctx->task_ctx);  in perf_cgroup_switch()
811 		perf_pmu_disable(cpuctx->ctx.pmu);  in perf_cgroup_switch()
814 			cpu_ctx_sched_out(cpuctx, EVENT_ALL);  in perf_cgroup_switch()
819 			cpuctx->cgrp = NULL;  in perf_cgroup_switch()
823 			WARN_ON_ONCE(cpuctx->cgrp);  in perf_cgroup_switch()
831 			cpuctx->cgrp = perf_cgroup_from_task(task,  in perf_cgroup_switch()
832 							     &cpuctx->ctx);  in perf_cgroup_switch()
833 			cpu_ctx_sched_in(cpuctx, EVENT_ALL, task);  in perf_cgroup_switch()
835 		perf_pmu_enable(cpuctx->ctx.pmu);  in perf_cgroup_switch()
836 		perf_ctx_unlock(cpuctx, cpuctx->task_ctx);  in perf_cgroup_switch()
946 	struct perf_cpu_context *cpuctx;  in list_update_cgroup_event()  local
956 	cpuctx = __get_cpu_context(ctx);  in list_update_cgroup_event()
964 	if (add && !cpuctx->cgrp) {  in list_update_cgroup_event()
968 			cpuctx->cgrp = cgrp;  in list_update_cgroup_event()
978 		cpuctx->cgrp = NULL;  in list_update_cgroup_event()
980 	cpuctx_entry = &cpuctx->cgrp_cpuctx_entry;  in list_update_cgroup_event()
1007 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)  in update_cgrp_time_from_cpuctx()  argument
1067 	struct perf_cpu_context *cpuctx;  in perf_mux_hrtimer_handler()  local
1072 	cpuctx = container_of(hr, struct perf_cpu_context, hrtimer);  in perf_mux_hrtimer_handler()
1073 	rotations = perf_rotate_context(cpuctx);  in perf_mux_hrtimer_handler()
1075 	raw_spin_lock(&cpuctx->hrtimer_lock);  in perf_mux_hrtimer_handler()
1077 		hrtimer_forward_now(hr, cpuctx->hrtimer_interval);  in perf_mux_hrtimer_handler()
1079 		cpuctx->hrtimer_active = 0;  in perf_mux_hrtimer_handler()
1080 	raw_spin_unlock(&cpuctx->hrtimer_lock);  in perf_mux_hrtimer_handler()
1085 static void __perf_mux_hrtimer_init(struct perf_cpu_context *cpuctx, int cpu)  in __perf_mux_hrtimer_init()  argument
1087 	struct hrtimer *timer = &cpuctx->hrtimer;  in __perf_mux_hrtimer_init()
1088 	struct pmu *pmu = cpuctx->ctx.pmu;  in __perf_mux_hrtimer_init()
1103 	cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * interval);  in __perf_mux_hrtimer_init()
1105 	raw_spin_lock_init(&cpuctx->hrtimer_lock);  in __perf_mux_hrtimer_init()
1110 static int perf_mux_hrtimer_restart(struct perf_cpu_context *cpuctx)  in perf_mux_hrtimer_restart()  argument
1112 	struct hrtimer *timer = &cpuctx->hrtimer;  in perf_mux_hrtimer_restart()
1113 	struct pmu *pmu = cpuctx->ctx.pmu;  in perf_mux_hrtimer_restart()
1120 	raw_spin_lock_irqsave(&cpuctx->hrtimer_lock, flags);  in perf_mux_hrtimer_restart()
1121 	if (!cpuctx->hrtimer_active) {  in perf_mux_hrtimer_restart()
1122 		cpuctx->hrtimer_active = 1;  in perf_mux_hrtimer_restart()
1123 		hrtimer_forward_now(timer, cpuctx->hrtimer_interval);  in perf_mux_hrtimer_restart()
1126 	raw_spin_unlock_irqrestore(&cpuctx->hrtimer_lock, flags);  in perf_mux_hrtimer_restart()
1904 			    struct perf_cpu_context *cpuctx,
1910 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);  in perf_put_aux_event()  local
1939 		event_sched_out(iter, cpuctx, ctx);  in perf_put_aux_event()
2074 		  struct perf_cpu_context *cpuctx,  in event_sched_out()  argument
2104 		cpuctx->active_oncpu--;  in event_sched_out()
2109 	if (event->attr.exclusive || !cpuctx->active_oncpu)  in event_sched_out()
2110 		cpuctx->exclusive = 0;  in event_sched_out()
2117 		struct perf_cpu_context *cpuctx,  in group_sched_out()  argument
2127 	event_sched_out(group_event, cpuctx, ctx);  in group_sched_out()
2133 		event_sched_out(event, cpuctx, ctx);  in group_sched_out()
2138 		cpuctx->exclusive = 0;  in group_sched_out()
2151 			   struct perf_cpu_context *cpuctx,  in __perf_remove_from_context()  argument
2159 		update_cgrp_time_from_cpuctx(cpuctx);  in __perf_remove_from_context()
2162 	event_sched_out(event, cpuctx, ctx);  in __perf_remove_from_context()
2170 			WARN_ON_ONCE(cpuctx->task_ctx != ctx);  in __perf_remove_from_context()
2171 			cpuctx->task_ctx = NULL;  in __perf_remove_from_context()
2217 				 struct perf_cpu_context *cpuctx,  in __perf_event_disable()  argument
2230 		group_sched_out(event, cpuctx, ctx);  in __perf_event_disable()
2232 		event_sched_out(event, cpuctx, ctx);  in __perf_event_disable()
2332 		 struct perf_cpu_context *cpuctx,  in event_sched_in()  argument
2375 		cpuctx->active_oncpu++;  in event_sched_in()
2382 		cpuctx->exclusive = 1;  in event_sched_in()
2392 	       struct perf_cpu_context *cpuctx,  in group_sched_in()  argument
2403 	if (event_sched_in(group_event, cpuctx, ctx)) {  in group_sched_in()
2405 		perf_mux_hrtimer_restart(cpuctx);  in group_sched_in()
2413 		if (event_sched_in(event, cpuctx, ctx)) {  in group_sched_in()
2432 		event_sched_out(event, cpuctx, ctx);  in group_sched_in()
2434 	event_sched_out(group_event, cpuctx, ctx);  in group_sched_in()
2438 	perf_mux_hrtimer_restart(cpuctx);  in group_sched_in()
2447 			   struct perf_cpu_context *cpuctx,  in group_can_go_on()  argument
2459 	if (cpuctx->exclusive)  in group_can_go_on()
2465 	if (event->attr.exclusive && cpuctx->active_oncpu)  in group_can_go_on()
2482 			  struct perf_cpu_context *cpuctx,
2486 	     struct perf_cpu_context *cpuctx,
2490 static void task_ctx_sched_out(struct perf_cpu_context *cpuctx,  in task_ctx_sched_out()  argument
2494 	if (!cpuctx->task_ctx)  in task_ctx_sched_out()
2497 	if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))  in task_ctx_sched_out()
2500 	ctx_sched_out(ctx, cpuctx, event_type);  in task_ctx_sched_out()
2503 static void perf_event_sched_in(struct perf_cpu_context *cpuctx,  in perf_event_sched_in()  argument
2507 	cpu_ctx_sched_in(cpuctx, EVENT_PINNED, task);  in perf_event_sched_in()
2509 		ctx_sched_in(ctx, cpuctx, EVENT_PINNED, task);  in perf_event_sched_in()
2510 	cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE, task);  in perf_event_sched_in()
2512 		ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE, task);  in perf_event_sched_in()
2530 static void ctx_resched(struct perf_cpu_context *cpuctx,  in ctx_resched()  argument
2546 	perf_pmu_disable(cpuctx->ctx.pmu);  in ctx_resched()
2548 		task_ctx_sched_out(cpuctx, task_ctx, event_type);  in ctx_resched()
2558 		cpu_ctx_sched_out(cpuctx, ctx_event_type);  in ctx_resched()
2560 		cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);  in ctx_resched()
2562 	perf_event_sched_in(cpuctx, task_ctx, current);  in ctx_resched()
2563 	perf_pmu_enable(cpuctx->ctx.pmu);  in ctx_resched()
2568 	struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);  in perf_pmu_resched()  local
2569 	struct perf_event_context *task_ctx = cpuctx->task_ctx;  in perf_pmu_resched()
2571 	perf_ctx_lock(cpuctx, task_ctx);  in perf_pmu_resched()
2572 	ctx_resched(cpuctx, task_ctx, EVENT_ALL|EVENT_CPU);  in perf_pmu_resched()
2573 	perf_ctx_unlock(cpuctx, task_ctx);  in perf_pmu_resched()
2586 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);  in __perf_install_in_context()  local
2587 	struct perf_event_context *task_ctx = cpuctx->task_ctx;  in __perf_install_in_context()
2591 	raw_spin_lock(&cpuctx->ctx.lock);  in __perf_install_in_context()
2610 		WARN_ON_ONCE(reprogram && cpuctx->task_ctx && cpuctx->task_ctx != ctx);  in __perf_install_in_context()
2628 		ctx_sched_out(ctx, cpuctx, EVENT_TIME);  in __perf_install_in_context()
2630 		ctx_resched(cpuctx, task_ctx, get_event_type(event));  in __perf_install_in_context()
2636 	perf_ctx_unlock(cpuctx, task_ctx);  in __perf_install_in_context()
2742 				struct perf_cpu_context *cpuctx,  in __perf_event_enable()  argument
2754 		ctx_sched_out(ctx, cpuctx, EVENT_TIME);  in __perf_event_enable()
2762 		ctx_sched_in(ctx, cpuctx, EVENT_TIME, current);  in __perf_event_enable()
2771 		ctx_sched_in(ctx, cpuctx, EVENT_TIME, current);  in __perf_event_enable()
2775 	task_ctx = cpuctx->task_ctx;  in __perf_event_enable()
2779 	ctx_resched(cpuctx, task_ctx, get_event_type(event));  in __perf_event_enable()
2996 			  struct perf_cpu_context *cpuctx,  in ctx_sched_out()  argument
3010 			WARN_ON_ONCE(cpuctx->task_ctx);  in ctx_sched_out()
3019 		WARN_ON_ONCE(cpuctx->task_ctx != ctx);  in ctx_sched_out()
3021 			cpuctx->task_ctx = NULL;  in ctx_sched_out()
3037 		update_cgrp_time_from_cpuctx(cpuctx);  in ctx_sched_out()
3054 			group_sched_out(event, cpuctx, ctx);  in ctx_sched_out()
3059 			group_sched_out(event, cpuctx, ctx);  in ctx_sched_out()
3172 	struct perf_cpu_context *cpuctx;  in perf_event_context_sched_out()  local
3178 	cpuctx = __get_cpu_context(ctx);  in perf_event_context_sched_out()
3179 	if (!cpuctx->task_ctx)  in perf_event_context_sched_out()
3234 		task_ctx_sched_out(cpuctx, ctx, EVENT_ALL);  in perf_event_context_sched_out()
3243 	struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);  in perf_sched_cb_dec()  local
3247 	if (!--cpuctx->sched_cb_usage)  in perf_sched_cb_dec()
3248 		list_del(&cpuctx->sched_cb_entry);  in perf_sched_cb_dec()
3254 	struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);  in perf_sched_cb_inc()  local
3256 	if (!cpuctx->sched_cb_usage++)  in perf_sched_cb_inc()
3257 		list_add(&cpuctx->sched_cb_entry, this_cpu_ptr(&sched_cb_list));  in perf_sched_cb_inc()
3274 	struct perf_cpu_context *cpuctx;  in perf_pmu_sched_task()  local
3280 	list_for_each_entry(cpuctx, this_cpu_ptr(&sched_cb_list), sched_cb_entry) {  in perf_pmu_sched_task()
3281 		pmu = cpuctx->ctx.pmu; /* software PMUs will not have sched_task */  in perf_pmu_sched_task()
3286 		perf_ctx_lock(cpuctx, cpuctx->task_ctx);  in perf_pmu_sched_task()
3289 		pmu->sched_task(cpuctx->task_ctx, sched_in);  in perf_pmu_sched_task()
3292 		perf_ctx_unlock(cpuctx, cpuctx->task_ctx);  in perf_pmu_sched_task()
3339 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,  in cpu_ctx_sched_out()  argument
3342 	ctx_sched_out(&cpuctx->ctx, cpuctx, event_type);  in cpu_ctx_sched_out()
3378 	struct perf_cpu_context *cpuctx;  member
3392 	if (group_can_go_on(event, sid->cpuctx, sid->can_add_hw)) {  in pinned_sched_in()
3393 		if (!group_sched_in(event, sid->cpuctx, sid->ctx))  in pinned_sched_in()
3417 	if (group_can_go_on(event, sid->cpuctx, sid->can_add_hw)) {  in flexible_sched_in()
3418 		int ret = group_sched_in(event, sid->cpuctx, sid->ctx);  in flexible_sched_in()
3432 		    struct perf_cpu_context *cpuctx)  in ctx_pinned_sched_in()  argument
3436 		.cpuctx = cpuctx,  in ctx_pinned_sched_in()
3447 		      struct perf_cpu_context *cpuctx)  in ctx_flexible_sched_in()  argument
3451 		.cpuctx = cpuctx,  in ctx_flexible_sched_in()
3462 	     struct perf_cpu_context *cpuctx,  in ctx_sched_in()  argument
3477 			cpuctx->task_ctx = ctx;  in ctx_sched_in()
3479 			WARN_ON_ONCE(cpuctx->task_ctx != ctx);  in ctx_sched_in()
3496 		ctx_pinned_sched_in(ctx, cpuctx);  in ctx_sched_in()
3500 		ctx_flexible_sched_in(ctx, cpuctx);  in ctx_sched_in()
3503 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,  in cpu_ctx_sched_in()  argument
3507 	struct perf_event_context *ctx = &cpuctx->ctx;  in cpu_ctx_sched_in()
3509 	ctx_sched_in(ctx, cpuctx, event_type, task);  in cpu_ctx_sched_in()
3515 	struct perf_cpu_context *cpuctx;  in perf_event_context_sched_in()  local
3517 	cpuctx = __get_cpu_context(ctx);  in perf_event_context_sched_in()
3518 	if (cpuctx->task_ctx == ctx)  in perf_event_context_sched_in()
3521 	perf_ctx_lock(cpuctx, ctx);  in perf_event_context_sched_in()
3539 		cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);  in perf_event_context_sched_in()
3540 	perf_event_sched_in(cpuctx, ctx, task);  in perf_event_context_sched_in()
3544 	perf_ctx_unlock(cpuctx, ctx);  in perf_event_context_sched_in()
3801 static bool perf_rotate_context(struct perf_cpu_context *cpuctx)  in perf_rotate_context()  argument
3812 	cpu_rotate = cpuctx->ctx.rotate_necessary;  in perf_rotate_context()
3813 	task_ctx = cpuctx->task_ctx;  in perf_rotate_context()
3819 	perf_ctx_lock(cpuctx, cpuctx->task_ctx);  in perf_rotate_context()
3820 	perf_pmu_disable(cpuctx->ctx.pmu);  in perf_rotate_context()
3825 		cpu_event = ctx_event_to_rotate(&cpuctx->ctx);  in perf_rotate_context()
3832 		ctx_sched_out(task_ctx, cpuctx, EVENT_FLEXIBLE);  in perf_rotate_context()
3834 		cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);  in perf_rotate_context()
3839 		rotate_ctx(&cpuctx->ctx, cpu_event);  in perf_rotate_context()
3841 	perf_event_sched_in(cpuctx, task_ctx, current);  in perf_rotate_context()
3843 	perf_pmu_enable(cpuctx->ctx.pmu);  in perf_rotate_context()
3844 	perf_ctx_unlock(cpuctx, cpuctx->task_ctx);  in perf_rotate_context()
3888 	struct perf_cpu_context *cpuctx;  in perf_event_enable_on_exec()  local
3898 	cpuctx = __get_cpu_context(ctx);  in perf_event_enable_on_exec()
3899 	perf_ctx_lock(cpuctx, ctx);  in perf_event_enable_on_exec()
3900 	ctx_sched_out(ctx, cpuctx, EVENT_TIME);  in perf_event_enable_on_exec()
3911 		ctx_resched(cpuctx, ctx, event_type);  in perf_event_enable_on_exec()
3913 		ctx_sched_in(ctx, cpuctx, EVENT_TIME, current);  in perf_event_enable_on_exec()
3915 	perf_ctx_unlock(cpuctx, ctx);  in perf_event_enable_on_exec()
3955 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);  in __perf_event_read()  local
3965 	if (ctx->task && cpuctx->task_ctx != ctx)  in __perf_event_read()
4224 	struct perf_cpu_context *cpuctx;  in find_get_context()  local
4235 		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);  in find_get_context()
4236 		ctx = &cpuctx->ctx;  in find_get_context()
5068 				struct perf_cpu_context *cpuctx,  in __perf_event_period()  argument
6953 	struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);  in __perf_pmu_output_stop()  local
6959 	perf_iterate_ctx(&cpuctx->ctx, __perf_event_output_stop, &ro, false);  in __perf_pmu_output_stop()
6960 	if (cpuctx->task_ctx)  in __perf_pmu_output_stop()
6961 		perf_iterate_ctx(cpuctx->task_ctx, __perf_event_output_stop,  in __perf_pmu_output_stop()
9955 		struct perf_cpu_context *cpuctx;  in perf_event_mux_interval_ms_store()  local
9956 		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);  in perf_event_mux_interval_ms_store()
9957 		cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * timer);  in perf_event_mux_interval_ms_store()
9960 			(remote_function_f)perf_mux_hrtimer_restart, cpuctx);  in perf_event_mux_interval_ms_store()
10091 		struct perf_cpu_context *cpuctx;  in perf_pmu_register()  local
10093 		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);  in perf_pmu_register()
10094 		__perf_event_init_context(&cpuctx->ctx);  in perf_pmu_register()
10095 		lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex);  in perf_pmu_register()
10096 		lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock);  in perf_pmu_register()
10097 		cpuctx->ctx.pmu = pmu;  in perf_pmu_register()
10098 		cpuctx->online = cpumask_test_cpu(cpu, perf_online_mask);  in perf_pmu_register()
10100 		__perf_mux_hrtimer_init(cpuctx, cpu);  in perf_pmu_register()
11178 		struct perf_cpu_context *cpuctx =  in SYSCALL_DEFINE5()  local
11181 		if (!cpuctx->online) {  in SYSCALL_DEFINE5()
11374 		struct perf_cpu_context *cpuctx =  in perf_event_create_kernel_counter()  local
11376 		if (!cpuctx->online) {  in perf_event_create_kernel_counter()
12125 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);  in __perf_event_exit_context()  local
12129 	ctx_sched_out(ctx, cpuctx, EVENT_TIME);  in __perf_event_exit_context()
12131 		__perf_remove_from_context(event, cpuctx, ctx, (void *)DETACH_GROUP);  in __perf_event_exit_context()
12137 	struct perf_cpu_context *cpuctx;  in perf_event_exit_cpu_context()  local
12143 		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);  in perf_event_exit_cpu_context()
12144 		ctx = &cpuctx->ctx;  in perf_event_exit_cpu_context()
12148 		cpuctx->online = 0;  in perf_event_exit_cpu_context()
12162 	struct perf_cpu_context *cpuctx;  in perf_event_init_cpu()  local
12171 		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);  in perf_event_init_cpu()
12172 		ctx = &cpuctx->ctx;  in perf_event_init_cpu()
12175 		cpuctx->online = 1;  in perf_event_init_cpu()