107 static bool pmu_needs_timer(struct i915_pmu *pmu, bool gpu_active)  in pmu_needs_timer()  argument
109 	struct drm_i915_private *i915 = container_of(pmu, typeof(*i915), pmu);  in pmu_needs_timer()
117 	enable = pmu->enable;  in pmu_needs_timer()
170 	struct i915_pmu *pmu = &i915->pmu;  in get_rc6()  local
181 	spin_lock_irqsave(&pmu->lock, flags);  in get_rc6()
184 		pmu->sample[__I915_SAMPLE_RC6].cur = val;  in get_rc6()
193 		val = ktime_since_raw(pmu->sleep_last);  in get_rc6()
194 		val += pmu->sample[__I915_SAMPLE_RC6].cur;  in get_rc6()
197 	if (val < pmu->sample[__I915_SAMPLE_RC6_LAST_REPORTED].cur)  in get_rc6()
198 		val = pmu->sample[__I915_SAMPLE_RC6_LAST_REPORTED].cur;  in get_rc6()
200 		pmu->sample[__I915_SAMPLE_RC6_LAST_REPORTED].cur = val;  in get_rc6()
202 	spin_unlock_irqrestore(&pmu->lock, flags);  in get_rc6()
207 static void init_rc6(struct i915_pmu *pmu)  in init_rc6()  argument
209 	struct drm_i915_private *i915 = container_of(pmu, typeof(*i915), pmu);  in init_rc6()
213 		pmu->sample[__I915_SAMPLE_RC6].cur = __get_rc6(to_gt(i915));  in init_rc6()
214 		pmu->sample[__I915_SAMPLE_RC6_LAST_REPORTED].cur =  in init_rc6()
215 					pmu->sample[__I915_SAMPLE_RC6].cur;  in init_rc6()
216 		pmu->sleep_last = ktime_get_raw();  in init_rc6()
222 	struct i915_pmu *pmu = &i915->pmu;  in park_rc6()  local
224 	pmu->sample[__I915_SAMPLE_RC6].cur = __get_rc6(to_gt(i915));  in park_rc6()
225 	pmu->sleep_last = ktime_get_raw();  in park_rc6()
228 static void __i915_pmu_maybe_start_timer(struct i915_pmu *pmu)  in __i915_pmu_maybe_start_timer()  argument
230 	if (!pmu->timer_enabled && pmu_needs_timer(pmu, true)) {  in __i915_pmu_maybe_start_timer()
231 		pmu->timer_enabled = true;  in __i915_pmu_maybe_start_timer()
232 		pmu->timer_last = ktime_get();  in __i915_pmu_maybe_start_timer()
233 		hrtimer_start_range_ns(&pmu->timer,  in __i915_pmu_maybe_start_timer()
241 	struct i915_pmu *pmu = &i915->pmu;  in i915_pmu_gt_parked()  local
243 	if (!pmu->base.event_init)  in i915_pmu_gt_parked()
246 	spin_lock_irq(&pmu->lock);  in i915_pmu_gt_parked()
254 	pmu->timer_enabled = pmu_needs_timer(pmu, false);  in i915_pmu_gt_parked()
256 	spin_unlock_irq(&pmu->lock);  in i915_pmu_gt_parked()
261 	struct i915_pmu *pmu = &i915->pmu;  in i915_pmu_gt_unparked()  local
263 	if (!pmu->base.event_init)  in i915_pmu_gt_unparked()
266 	spin_lock_irq(&pmu->lock);  in i915_pmu_gt_unparked()
271 	__i915_pmu_maybe_start_timer(pmu);  in i915_pmu_gt_unparked()
273 	spin_unlock_irq(&pmu->lock);  in i915_pmu_gt_unparked()
294 	struct intel_engine_pmu *pmu = &engine->pmu;  in engine_sample()  local
303 		add_sample(&pmu->sample[I915_SAMPLE_WAIT], period_ns);  in engine_sample()
305 		add_sample(&pmu->sample[I915_SAMPLE_SEMA], period_ns);  in engine_sample()
324 		add_sample(&pmu->sample[I915_SAMPLE_BUSY], period_ns);  in engine_sample()
335 	if ((i915->pmu.enable & ENGINE_SAMPLE_MASK) == 0)  in engines_sample()
363 static bool frequency_sampling_enabled(struct i915_pmu *pmu)  in frequency_sampling_enabled()  argument
365 	return pmu->enable &  in frequency_sampling_enabled()
375 	struct i915_pmu *pmu = &i915->pmu;  in frequency_sample()  local
378 	if (!frequency_sampling_enabled(pmu))  in frequency_sample()
385 	if (pmu->enable & config_mask(I915_PMU_ACTUAL_FREQUENCY)) {  in frequency_sample()
403 		add_sample_mult(&pmu->sample[__I915_SAMPLE_FREQ_ACT],  in frequency_sample()
407 	if (pmu->enable & config_mask(I915_PMU_REQUESTED_FREQUENCY)) {  in frequency_sample()
408 		add_sample_mult(&pmu->sample[__I915_SAMPLE_FREQ_REQ],  in frequency_sample()
419 		container_of(hrtimer, struct drm_i915_private, pmu.timer);  in i915_sample()
420 	struct i915_pmu *pmu = &i915->pmu;  in i915_sample()  local
425 	if (!READ_ONCE(pmu->timer_enabled))  in i915_sample()
429 	period_ns = ktime_to_ns(ktime_sub(now, pmu->timer_last));  in i915_sample()
430 	pmu->timer_last = now;  in i915_sample()
449 		container_of(event->pmu, typeof(*i915), pmu.base);  in i915_pmu_event_destroy()
508 		container_of(event->pmu, typeof(*i915), pmu.base);  in engine_event_init()
522 		container_of(event->pmu, typeof(*i915), pmu.base);  in i915_pmu_event_init()
523 	struct i915_pmu *pmu = &i915->pmu;  in i915_pmu_event_init()  local
526 	if (pmu->closed)  in i915_pmu_event_init()
529 	if (event->attr.type != event->pmu->type)  in i915_pmu_event_init()
564 		container_of(event->pmu, typeof(*i915), pmu.base);  in __i915_pmu_event_read()
565 	struct i915_pmu *pmu = &i915->pmu;  in __i915_pmu_event_read()  local
585 			val = engine->pmu.sample[sample].cur;  in __i915_pmu_event_read()
591 			   div_u64(pmu->sample[__I915_SAMPLE_FREQ_ACT].cur,  in __i915_pmu_event_read()
596 			   div_u64(pmu->sample[__I915_SAMPLE_FREQ_REQ].cur,  in __i915_pmu_event_read()
600 			val = READ_ONCE(pmu->irq_count);  in __i915_pmu_event_read()
617 		container_of(event->pmu, typeof(*i915), pmu.base);  in i915_pmu_event_read()
619 	struct i915_pmu *pmu = &i915->pmu;  in i915_pmu_event_read()  local
622 	if (pmu->closed) {  in i915_pmu_event_read()
639 		container_of(event->pmu, typeof(*i915), pmu.base);  in i915_pmu_enable()
640 	struct i915_pmu *pmu = &i915->pmu;  in i915_pmu_enable()  local
648 	spin_lock_irqsave(&pmu->lock, flags);  in i915_pmu_enable()
654 	BUILD_BUG_ON(ARRAY_SIZE(pmu->enable_count) != I915_PMU_MASK_BITS);  in i915_pmu_enable()
655 	GEM_BUG_ON(bit >= ARRAY_SIZE(pmu->enable_count));  in i915_pmu_enable()
656 	GEM_BUG_ON(pmu->enable_count[bit] == ~0);  in i915_pmu_enable()
658 	pmu->enable |= BIT_ULL(bit);  in i915_pmu_enable()
659 	pmu->enable_count[bit]++;  in i915_pmu_enable()
664 	__i915_pmu_maybe_start_timer(pmu);  in i915_pmu_enable()
678 		BUILD_BUG_ON(ARRAY_SIZE(engine->pmu.enable_count) !=  in i915_pmu_enable()
680 		BUILD_BUG_ON(ARRAY_SIZE(engine->pmu.sample) !=  in i915_pmu_enable()
682 		GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.enable_count));  in i915_pmu_enable()
683 		GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.sample));  in i915_pmu_enable()
684 		GEM_BUG_ON(engine->pmu.enable_count[sample] == ~0);  in i915_pmu_enable()
686 		engine->pmu.enable |= BIT(sample);  in i915_pmu_enable()
687 		engine->pmu.enable_count[sample]++;  in i915_pmu_enable()
690 	spin_unlock_irqrestore(&pmu->lock, flags);  in i915_pmu_enable()
704 		container_of(event->pmu, typeof(*i915), pmu.base);  in i915_pmu_disable()
706 	struct i915_pmu *pmu = &i915->pmu;  in i915_pmu_disable()  local
712 	spin_lock_irqsave(&pmu->lock, flags);  in i915_pmu_disable()
722 		GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.enable_count));  in i915_pmu_disable()
723 		GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.sample));  in i915_pmu_disable()
724 		GEM_BUG_ON(engine->pmu.enable_count[sample] == 0);  in i915_pmu_disable()
730 		if (--engine->pmu.enable_count[sample] == 0)  in i915_pmu_disable()
731 			engine->pmu.enable &= ~BIT(sample);  in i915_pmu_disable()
734 	GEM_BUG_ON(bit >= ARRAY_SIZE(pmu->enable_count));  in i915_pmu_disable()
735 	GEM_BUG_ON(pmu->enable_count[bit] == 0);  in i915_pmu_disable()
740 	if (--pmu->enable_count[bit] == 0) {  in i915_pmu_disable()
741 		pmu->enable &= ~BIT_ULL(bit);  in i915_pmu_disable()
742 		pmu->timer_enabled &= pmu_needs_timer(pmu, true);  in i915_pmu_disable()
745 	spin_unlock_irqrestore(&pmu->lock, flags);  in i915_pmu_disable()
751 		container_of(event->pmu, typeof(*i915), pmu.base);  in i915_pmu_event_start()
752 	struct i915_pmu *pmu = &i915->pmu;  in i915_pmu_event_start()  local
754 	if (pmu->closed)  in i915_pmu_event_start()
772 		container_of(event->pmu, typeof(*i915), pmu.base);  in i915_pmu_event_add()
773 	struct i915_pmu *pmu = &i915->pmu;  in i915_pmu_event_add()  local
775 	if (pmu->closed)  in i915_pmu_event_add()
894 create_event_attributes(struct i915_pmu *pmu)  in create_event_attributes()  argument
896 	struct drm_i915_private *i915 = container_of(pmu, typeof(*i915), pmu);  in create_event_attributes()
1010 	pmu->i915_attr = i915_attr;  in create_event_attributes()
1011 	pmu->pmu_attr = pmu_attr;  in create_event_attributes()
1027 static void free_event_attributes(struct i915_pmu *pmu)  in free_event_attributes()  argument
1029 	struct attribute **attr_iter = pmu->events_attr_group.attrs;  in free_event_attributes()
1034 	kfree(pmu->events_attr_group.attrs);  in free_event_attributes()
1035 	kfree(pmu->i915_attr);  in free_event_attributes()
1036 	kfree(pmu->pmu_attr);  in free_event_attributes()
1038 	pmu->events_attr_group.attrs = NULL;  in free_event_attributes()
1039 	pmu->i915_attr = NULL;  in free_event_attributes()
1040 	pmu->pmu_attr = NULL;  in free_event_attributes()
1045 	struct i915_pmu *pmu = hlist_entry_safe(node, typeof(*pmu), cpuhp.node);  in i915_pmu_cpu_online()  local
1047 	GEM_BUG_ON(!pmu->base.event_init);  in i915_pmu_cpu_online()
1058 	struct i915_pmu *pmu = hlist_entry_safe(node, typeof(*pmu), cpuhp.node);  in i915_pmu_cpu_offline()  local
1061 	GEM_BUG_ON(!pmu->base.event_init);  in i915_pmu_cpu_offline()
1067 	if (pmu->closed)  in i915_pmu_cpu_offline()
1080 	if (target < nr_cpu_ids && target != pmu->cpuhp.cpu) {  in i915_pmu_cpu_offline()
1081 		perf_pmu_migrate_context(&pmu->base, cpu, target);  in i915_pmu_cpu_offline()
1082 		pmu->cpuhp.cpu = target;  in i915_pmu_cpu_offline()
1099 		pr_notice("Failed to setup cpuhp state for i915 PMU! (%d)\n",  in i915_pmu_init()
1113 static int i915_pmu_register_cpuhp_state(struct i915_pmu *pmu)  in i915_pmu_register_cpuhp_state()  argument
1118 	return cpuhp_state_add_instance(cpuhp_slot, &pmu->cpuhp.node);  in i915_pmu_register_cpuhp_state()
1121 static void i915_pmu_unregister_cpuhp_state(struct i915_pmu *pmu)  in i915_pmu_unregister_cpuhp_state()  argument
1123 	cpuhp_state_remove_instance(cpuhp_slot, &pmu->cpuhp.node);  in i915_pmu_unregister_cpuhp_state()
1139 	struct i915_pmu *pmu = &i915->pmu;  in i915_pmu_register()  local
1142 		&pmu->events_attr_group,  in i915_pmu_register()
1150 		drm_info(&i915->drm, "PMU not supported for this GPU.");  in i915_pmu_register()
1154 	spin_lock_init(&pmu->lock);  in i915_pmu_register()
1155 	hrtimer_init(&pmu->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);  in i915_pmu_register()
1156 	pmu->timer.function = i915_sample;  in i915_pmu_register()
1157 	pmu->cpuhp.cpu = -1;  in i915_pmu_register()
1158 	init_rc6(pmu);  in i915_pmu_register()
1161 		pmu->name = kasprintf(GFP_KERNEL,  in i915_pmu_register()
1164 		if (pmu->name) {  in i915_pmu_register()
1166 			strreplace((char *)pmu->name, ':', '_');  in i915_pmu_register()
1169 		pmu->name = "i915";  in i915_pmu_register()
1171 	if (!pmu->name)  in i915_pmu_register()
1174 	pmu->events_attr_group.name = "events";  in i915_pmu_register()
1175 	pmu->events_attr_group.attrs = create_event_attributes(pmu);  in i915_pmu_register()
1176 	if (!pmu->events_attr_group.attrs)  in i915_pmu_register()
1179 	pmu->base.attr_groups = kmemdup(attr_groups, sizeof(attr_groups),  in i915_pmu_register()
1181 	if (!pmu->base.attr_groups)  in i915_pmu_register()
1184 	pmu->base.module	= THIS_MODULE;  in i915_pmu_register()
1185 	pmu->base.task_ctx_nr	= perf_invalid_context;  in i915_pmu_register()
1186 	pmu->base.event_init	= i915_pmu_event_init;  in i915_pmu_register()
1187 	pmu->base.add		= i915_pmu_event_add;  in i915_pmu_register()
1188 	pmu->base.del		= i915_pmu_event_del;  in i915_pmu_register()
1189 	pmu->base.start		= i915_pmu_event_start;  in i915_pmu_register()
1190 	pmu->base.stop		= i915_pmu_event_stop;  in i915_pmu_register()
1191 	pmu->base.read		= i915_pmu_event_read;  in i915_pmu_register()
1192 	pmu->base.event_idx	= i915_pmu_event_event_idx;  in i915_pmu_register()
1194 	ret = perf_pmu_register(&pmu->base, pmu->name, -1);  in i915_pmu_register()
1198 	ret = i915_pmu_register_cpuhp_state(pmu);  in i915_pmu_register()
1205 	perf_pmu_unregister(&pmu->base);  in i915_pmu_register()
1207 	kfree(pmu->base.attr_groups);  in i915_pmu_register()
1209 	pmu->base.event_init = NULL;  in i915_pmu_register()
1210 	free_event_attributes(pmu);  in i915_pmu_register()
1213 		kfree(pmu->name);  in i915_pmu_register()
1215 	drm_notice(&i915->drm, "Failed to register PMU!\n");  in i915_pmu_register()
1220 	struct i915_pmu *pmu = &i915->pmu;  in i915_pmu_unregister()  local
1222 	if (!pmu->base.event_init)  in i915_pmu_unregister()
1226 	 * "Disconnect" the PMU callbacks - since all are atomic synchronize_rcu  in i915_pmu_unregister()
1230 	pmu->closed = true;  in i915_pmu_unregister()
1233 	hrtimer_cancel(&pmu->timer);  in i915_pmu_unregister()
1235 	i915_pmu_unregister_cpuhp_state(pmu);  in i915_pmu_unregister()
1237 	perf_pmu_unregister(&pmu->base);  in i915_pmu_unregister()
1238 	pmu->base.event_init = NULL;  in i915_pmu_unregister()
1239 	kfree(pmu->base.attr_groups);  in i915_pmu_unregister()
1241 		kfree(pmu->name);  in i915_pmu_unregister()
1242 	free_event_attributes(pmu);  in i915_pmu_unregister()