68 static void init_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm)  in init_intel_runtime_pm_wakeref()  argument
70 	spin_lock_init(&rpm->debug.lock);  in init_intel_runtime_pm_wakeref()
75 track_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm)  in track_intel_runtime_pm_wakeref()  argument
80 	if (rpm->no_wakeref_tracking)  in track_intel_runtime_pm_wakeref()
87 	spin_lock_irqsave(&rpm->debug.lock, flags);  in track_intel_runtime_pm_wakeref()
89 	if (!rpm->debug.count)  in track_intel_runtime_pm_wakeref()
90 		rpm->debug.last_acquire = stack;  in track_intel_runtime_pm_wakeref()
92 	stacks = krealloc(rpm->debug.owners,  in track_intel_runtime_pm_wakeref()
93 			  (rpm->debug.count + 1) * sizeof(*stacks),  in track_intel_runtime_pm_wakeref()
96 		stacks[rpm->debug.count++] = stack;  in track_intel_runtime_pm_wakeref()
97 		rpm->debug.owners = stacks;  in track_intel_runtime_pm_wakeref()
102 	spin_unlock_irqrestore(&rpm->debug.lock, flags);  in track_intel_runtime_pm_wakeref()
107 static void untrack_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm,  in untrack_intel_runtime_pm_wakeref()  argument
110 	struct drm_i915_private *i915 = container_of(rpm,  in untrack_intel_runtime_pm_wakeref()
119 	spin_lock_irqsave(&rpm->debug.lock, flags);  in untrack_intel_runtime_pm_wakeref()
120 	for (n = rpm->debug.count; n--; ) {  in untrack_intel_runtime_pm_wakeref()
121 		if (rpm->debug.owners[n] == stack) {  in untrack_intel_runtime_pm_wakeref()
122 			memmove(rpm->debug.owners + n,  in untrack_intel_runtime_pm_wakeref()
123 				rpm->debug.owners + n + 1,  in untrack_intel_runtime_pm_wakeref()
124 				(--rpm->debug.count - n) * sizeof(stack));  in untrack_intel_runtime_pm_wakeref()
129 	spin_unlock_irqrestore(&rpm->debug.lock, flags);  in untrack_intel_runtime_pm_wakeref()
133 		     rpm->debug.count, atomic_read(&rpm->wakeref_count))) {  in untrack_intel_runtime_pm_wakeref()
143 		stack = READ_ONCE(rpm->debug.last_release);  in untrack_intel_runtime_pm_wakeref()
228 __intel_wakeref_dec_and_check_tracking(struct intel_runtime_pm *rpm)  in __intel_wakeref_dec_and_check_tracking()  argument
233 	if (!atomic_dec_and_lock_irqsave(&rpm->wakeref_count,  in __intel_wakeref_dec_and_check_tracking()
234 					 &rpm->debug.lock,  in __intel_wakeref_dec_and_check_tracking()
238 	__untrack_all_wakerefs(&rpm->debug, &dbg);  in __intel_wakeref_dec_and_check_tracking()
239 	spin_unlock_irqrestore(&rpm->debug.lock, flags);  in __intel_wakeref_dec_and_check_tracking()
245 untrack_all_intel_runtime_pm_wakerefs(struct intel_runtime_pm *rpm)  in untrack_all_intel_runtime_pm_wakerefs()  argument
250 	spin_lock_irqsave(&rpm->debug.lock, flags);  in untrack_all_intel_runtime_pm_wakerefs()
251 	__untrack_all_wakerefs(&rpm->debug, &dbg);  in untrack_all_intel_runtime_pm_wakerefs()
252 	spin_unlock_irqrestore(&rpm->debug.lock, flags);  in untrack_all_intel_runtime_pm_wakerefs()
257 void print_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm,  in print_intel_runtime_pm_wakeref()  argument
266 		spin_lock_irq(&rpm->debug.lock);  in print_intel_runtime_pm_wakeref()
267 		dbg.count = rpm->debug.count;  in print_intel_runtime_pm_wakeref()
270 			       rpm->debug.owners,  in print_intel_runtime_pm_wakeref()
273 		dbg.last_acquire = rpm->debug.last_acquire;  in print_intel_runtime_pm_wakeref()
274 		dbg.last_release = rpm->debug.last_release;  in print_intel_runtime_pm_wakeref()
275 		spin_unlock_irq(&rpm->debug.lock);  in print_intel_runtime_pm_wakeref()
296 static void init_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm)  in init_intel_runtime_pm_wakeref()  argument
301 track_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm)  in track_intel_runtime_pm_wakeref()  argument
306 static void untrack_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm,  in untrack_intel_runtime_pm_wakeref()  argument
312 __intel_wakeref_dec_and_check_tracking(struct intel_runtime_pm *rpm)  in __intel_wakeref_dec_and_check_tracking()  argument
314 	atomic_dec(&rpm->wakeref_count);  in __intel_wakeref_dec_and_check_tracking()
318 untrack_all_intel_runtime_pm_wakerefs(struct intel_runtime_pm *rpm)  in untrack_all_intel_runtime_pm_wakerefs()  argument
325 intel_runtime_pm_acquire(struct intel_runtime_pm *rpm, bool wakelock)  in intel_runtime_pm_acquire()  argument
328 		atomic_add(1 + INTEL_RPM_WAKELOCK_BIAS, &rpm->wakeref_count);  in intel_runtime_pm_acquire()
329 		assert_rpm_wakelock_held(rpm);  in intel_runtime_pm_acquire()
331 		atomic_inc(&rpm->wakeref_count);  in intel_runtime_pm_acquire()
332 		assert_rpm_raw_wakeref_held(rpm);  in intel_runtime_pm_acquire()
337 intel_runtime_pm_release(struct intel_runtime_pm *rpm, int wakelock)  in intel_runtime_pm_release()  argument
340 		assert_rpm_wakelock_held(rpm);  in intel_runtime_pm_release()
341 		atomic_sub(INTEL_RPM_WAKELOCK_BIAS, &rpm->wakeref_count);  in intel_runtime_pm_release()
343 		assert_rpm_raw_wakeref_held(rpm);  in intel_runtime_pm_release()
346 	__intel_wakeref_dec_and_check_tracking(rpm);  in intel_runtime_pm_release()
349 static intel_wakeref_t __intel_runtime_pm_get(struct intel_runtime_pm *rpm,  in __intel_runtime_pm_get()  argument
352 	struct drm_i915_private *i915 = container_of(rpm,  in __intel_runtime_pm_get()
357 	ret = pm_runtime_get_sync(rpm->kdev);  in __intel_runtime_pm_get()
361 	intel_runtime_pm_acquire(rpm, wakelock);  in __intel_runtime_pm_get()
363 	return track_intel_runtime_pm_wakeref(rpm);  in __intel_runtime_pm_get()
368  * @rpm: the intel_runtime_pm structure
383 intel_wakeref_t intel_runtime_pm_get_raw(struct intel_runtime_pm *rpm)  in intel_runtime_pm_get_raw()  argument
385 	return __intel_runtime_pm_get(rpm, false);  in intel_runtime_pm_get_raw()
390  * @rpm: the intel_runtime_pm structure
400 intel_wakeref_t intel_runtime_pm_get(struct intel_runtime_pm *rpm)  in intel_runtime_pm_get()  argument
402 	return __intel_runtime_pm_get(rpm, true);  in intel_runtime_pm_get()
407  * @rpm: the intel_runtime_pm structure
427 static intel_wakeref_t __intel_runtime_pm_get_if_active(struct intel_runtime_pm *rpm,  in __intel_runtime_pm_get_if_active()  argument
432 		 * In cases runtime PM is disabled by the RPM core and we get  in __intel_runtime_pm_get_if_active()
437 		if (pm_runtime_get_if_active(rpm->kdev, ignore_usecount) <= 0)  in __intel_runtime_pm_get_if_active()
441 	intel_runtime_pm_acquire(rpm, true);  in __intel_runtime_pm_get_if_active()
443 	return track_intel_runtime_pm_wakeref(rpm);  in __intel_runtime_pm_get_if_active()
446 intel_wakeref_t intel_runtime_pm_get_if_in_use(struct intel_runtime_pm *rpm)  in intel_runtime_pm_get_if_in_use()  argument
448 	return __intel_runtime_pm_get_if_active(rpm, false);  in intel_runtime_pm_get_if_in_use()
451 intel_wakeref_t intel_runtime_pm_get_if_active(struct intel_runtime_pm *rpm)  in intel_runtime_pm_get_if_active()  argument
453 	return __intel_runtime_pm_get_if_active(rpm, true);  in intel_runtime_pm_get_if_active()
458  * @rpm: the intel_runtime_pm structure
475 intel_wakeref_t intel_runtime_pm_get_noresume(struct intel_runtime_pm *rpm)  in intel_runtime_pm_get_noresume()  argument
477 	assert_rpm_wakelock_held(rpm);  in intel_runtime_pm_get_noresume()
478 	pm_runtime_get_noresume(rpm->kdev);  in intel_runtime_pm_get_noresume()
480 	intel_runtime_pm_acquire(rpm, true);  in intel_runtime_pm_get_noresume()
482 	return track_intel_runtime_pm_wakeref(rpm);  in intel_runtime_pm_get_noresume()
485 static void __intel_runtime_pm_put(struct intel_runtime_pm *rpm,  in __intel_runtime_pm_put()  argument
489 	struct device *kdev = rpm->kdev;  in __intel_runtime_pm_put()
491 	untrack_intel_runtime_pm_wakeref(rpm, wref);  in __intel_runtime_pm_put()
493 	intel_runtime_pm_release(rpm, wakelock);  in __intel_runtime_pm_put()
501  * @rpm: the intel_runtime_pm structure
509 intel_runtime_pm_put_raw(struct intel_runtime_pm *rpm, intel_wakeref_t wref)  in intel_runtime_pm_put_raw()  argument
511 	__intel_runtime_pm_put(rpm, wref, false);  in intel_runtime_pm_put_raw()
516  * @rpm: the intel_runtime_pm structure
526 void intel_runtime_pm_put_unchecked(struct intel_runtime_pm *rpm)  in intel_runtime_pm_put_unchecked()  argument
528 	__intel_runtime_pm_put(rpm, -1, true);  in intel_runtime_pm_put_unchecked()
534  * @rpm: the intel_runtime_pm structure
541 void intel_runtime_pm_put(struct intel_runtime_pm *rpm, intel_wakeref_t wref)  in intel_runtime_pm_put()  argument
543 	__intel_runtime_pm_put(rpm, wref, true);  in intel_runtime_pm_put()
549  * @rpm: the intel_runtime_pm structure
557 void intel_runtime_pm_enable(struct intel_runtime_pm *rpm)  in intel_runtime_pm_enable()  argument
559 	struct drm_i915_private *i915 = container_of(rpm,  in intel_runtime_pm_enable()
562 	struct device *kdev = rpm->kdev;  in intel_runtime_pm_enable()
578 	 * Take a permanent reference to disable the RPM functionality and drop  in intel_runtime_pm_enable()
580 	 * so the driver's own RPM reference tracking asserts also work on  in intel_runtime_pm_enable()
581 	 * platforms without RPM support.  in intel_runtime_pm_enable()
583 	if (!rpm->available) {  in intel_runtime_pm_enable()
605 	 * The core calls the driver load handler with an RPM reference held.  in intel_runtime_pm_enable()
612 void intel_runtime_pm_disable(struct intel_runtime_pm *rpm)  in intel_runtime_pm_disable()  argument
614 	struct drm_i915_private *i915 = container_of(rpm,  in intel_runtime_pm_disable()
617 	struct device *kdev = rpm->kdev;  in intel_runtime_pm_disable()
619 	/* Transfer rpm ownership back to core */  in intel_runtime_pm_disable()
621 		 "Failed to pass rpm ownership back to core\n");  in intel_runtime_pm_disable()
625 	if (!rpm->available)  in intel_runtime_pm_disable()
629 void intel_runtime_pm_driver_release(struct intel_runtime_pm *rpm)  in intel_runtime_pm_driver_release()  argument
631 	struct drm_i915_private *i915 = container_of(rpm,  in intel_runtime_pm_driver_release()
634 	int count = atomic_read(&rpm->wakeref_count);  in intel_runtime_pm_driver_release()
641 	untrack_all_intel_runtime_pm_wakerefs(rpm);  in intel_runtime_pm_driver_release()
644 void intel_runtime_pm_init_early(struct intel_runtime_pm *rpm)  in intel_runtime_pm_init_early()  argument
647 			container_of(rpm, struct drm_i915_private, runtime_pm);  in intel_runtime_pm_init_early()
651 	rpm->kdev = kdev;  in intel_runtime_pm_init_early()
652 	rpm->available = HAS_RUNTIME_PM(i915);  in intel_runtime_pm_init_early()
654 	init_intel_runtime_pm_wakeref(rpm);  in intel_runtime_pm_init_early()