78 static void init_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm)  in init_intel_runtime_pm_wakeref()  argument
80 	spin_lock_init(&rpm->debug.lock);  in init_intel_runtime_pm_wakeref()
84 track_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm)  in track_intel_runtime_pm_wakeref()  argument
89 	if (!rpm->available)  in track_intel_runtime_pm_wakeref()
96 	spin_lock_irqsave(&rpm->debug.lock, flags);  in track_intel_runtime_pm_wakeref()
98 	if (!rpm->debug.count)  in track_intel_runtime_pm_wakeref()
99 		rpm->debug.last_acquire = stack;  in track_intel_runtime_pm_wakeref()
101 	stacks = krealloc(rpm->debug.owners,  in track_intel_runtime_pm_wakeref()
102 			  (rpm->debug.count + 1) * sizeof(*stacks),  in track_intel_runtime_pm_wakeref()
105 		stacks[rpm->debug.count++] = stack;  in track_intel_runtime_pm_wakeref()
106 		rpm->debug.owners = stacks;  in track_intel_runtime_pm_wakeref()
111 	spin_unlock_irqrestore(&rpm->debug.lock, flags);  in track_intel_runtime_pm_wakeref()
116 static void untrack_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm,  in untrack_intel_runtime_pm_wakeref()  argument
119 	struct drm_i915_private *i915 = container_of(rpm,  in untrack_intel_runtime_pm_wakeref()
128 	spin_lock_irqsave(&rpm->debug.lock, flags);  in untrack_intel_runtime_pm_wakeref()
129 	for (n = rpm->debug.count; n--; ) {  in untrack_intel_runtime_pm_wakeref()
130 		if (rpm->debug.owners[n] == stack) {  in untrack_intel_runtime_pm_wakeref()
131 			memmove(rpm->debug.owners + n,  in untrack_intel_runtime_pm_wakeref()
132 				rpm->debug.owners + n + 1,  in untrack_intel_runtime_pm_wakeref()
133 				(--rpm->debug.count - n) * sizeof(stack));  in untrack_intel_runtime_pm_wakeref()
138 	spin_unlock_irqrestore(&rpm->debug.lock, flags);  in untrack_intel_runtime_pm_wakeref()
142 		     rpm->debug.count, atomic_read(&rpm->wakeref_count))) {  in untrack_intel_runtime_pm_wakeref()
152 		stack = READ_ONCE(rpm->debug.last_release);  in untrack_intel_runtime_pm_wakeref()
237 __intel_wakeref_dec_and_check_tracking(struct intel_runtime_pm *rpm)  in __intel_wakeref_dec_and_check_tracking()  argument
242 	if (!atomic_dec_and_lock_irqsave(&rpm->wakeref_count,  in __intel_wakeref_dec_and_check_tracking()
243 					 &rpm->debug.lock,  in __intel_wakeref_dec_and_check_tracking()
247 	__untrack_all_wakerefs(&rpm->debug, &dbg);  in __intel_wakeref_dec_and_check_tracking()
248 	spin_unlock_irqrestore(&rpm->debug.lock, flags);  in __intel_wakeref_dec_and_check_tracking()
254 untrack_all_intel_runtime_pm_wakerefs(struct intel_runtime_pm *rpm)  in untrack_all_intel_runtime_pm_wakerefs()  argument
259 	spin_lock_irqsave(&rpm->debug.lock, flags);  in untrack_all_intel_runtime_pm_wakerefs()
260 	__untrack_all_wakerefs(&rpm->debug, &dbg);  in untrack_all_intel_runtime_pm_wakerefs()
261 	spin_unlock_irqrestore(&rpm->debug.lock, flags);  in untrack_all_intel_runtime_pm_wakerefs()
266 void print_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm,  in print_intel_runtime_pm_wakeref()  argument
275 		spin_lock_irq(&rpm->debug.lock);  in print_intel_runtime_pm_wakeref()
276 		dbg.count = rpm->debug.count;  in print_intel_runtime_pm_wakeref()
279 			       rpm->debug.owners,  in print_intel_runtime_pm_wakeref()
282 		dbg.last_acquire = rpm->debug.last_acquire;  in print_intel_runtime_pm_wakeref()
283 		dbg.last_release = rpm->debug.last_release;  in print_intel_runtime_pm_wakeref()
284 		spin_unlock_irq(&rpm->debug.lock);  in print_intel_runtime_pm_wakeref()
305 static void init_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm)  in init_intel_runtime_pm_wakeref()  argument
310 track_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm)  in track_intel_runtime_pm_wakeref()  argument
315 static void untrack_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm,  in untrack_intel_runtime_pm_wakeref()  argument
321 __intel_wakeref_dec_and_check_tracking(struct intel_runtime_pm *rpm)  in __intel_wakeref_dec_and_check_tracking()  argument
323 	atomic_dec(&rpm->wakeref_count);  in __intel_wakeref_dec_and_check_tracking()
327 untrack_all_intel_runtime_pm_wakerefs(struct intel_runtime_pm *rpm)  in untrack_all_intel_runtime_pm_wakerefs()  argument
334 intel_runtime_pm_acquire(struct intel_runtime_pm *rpm, bool wakelock)  in intel_runtime_pm_acquire()  argument
337 		atomic_add(1 + INTEL_RPM_WAKELOCK_BIAS, &rpm->wakeref_count);  in intel_runtime_pm_acquire()
338 		assert_rpm_wakelock_held(rpm);  in intel_runtime_pm_acquire()
340 		atomic_inc(&rpm->wakeref_count);  in intel_runtime_pm_acquire()
341 		assert_rpm_raw_wakeref_held(rpm);  in intel_runtime_pm_acquire()
346 intel_runtime_pm_release(struct intel_runtime_pm *rpm, int wakelock)  in intel_runtime_pm_release()  argument
349 		assert_rpm_wakelock_held(rpm);  in intel_runtime_pm_release()
350 		atomic_sub(INTEL_RPM_WAKELOCK_BIAS, &rpm->wakeref_count);  in intel_runtime_pm_release()
352 		assert_rpm_raw_wakeref_held(rpm);  in intel_runtime_pm_release()
355 	__intel_wakeref_dec_and_check_tracking(rpm);  in intel_runtime_pm_release()
358 static intel_wakeref_t __intel_runtime_pm_get(struct intel_runtime_pm *rpm,  in __intel_runtime_pm_get()  argument
361 	struct drm_i915_private *i915 = container_of(rpm,  in __intel_runtime_pm_get()
366 	ret = pm_runtime_get_sync(rpm->kdev);  in __intel_runtime_pm_get()
370 	intel_runtime_pm_acquire(rpm, wakelock);  in __intel_runtime_pm_get()
372 	return track_intel_runtime_pm_wakeref(rpm);  in __intel_runtime_pm_get()
377  * @rpm: the intel_runtime_pm structure
392 intel_wakeref_t intel_runtime_pm_get_raw(struct intel_runtime_pm *rpm)  in intel_runtime_pm_get_raw()  argument
394 	return __intel_runtime_pm_get(rpm, false);  in intel_runtime_pm_get_raw()
399  * @rpm: the intel_runtime_pm structure
409 intel_wakeref_t intel_runtime_pm_get(struct intel_runtime_pm *rpm)  in intel_runtime_pm_get()  argument
411 	return __intel_runtime_pm_get(rpm, true);  in intel_runtime_pm_get()
416  * @rpm: the intel_runtime_pm structure
428 intel_wakeref_t intel_runtime_pm_get_if_in_use(struct intel_runtime_pm *rpm)  in intel_runtime_pm_get_if_in_use()  argument
432 		 * In cases runtime PM is disabled by the RPM core and we get  in intel_runtime_pm_get_if_in_use()
437 		if (pm_runtime_get_if_in_use(rpm->kdev) <= 0)  in intel_runtime_pm_get_if_in_use()
441 	intel_runtime_pm_acquire(rpm, true);  in intel_runtime_pm_get_if_in_use()
443 	return track_intel_runtime_pm_wakeref(rpm);  in intel_runtime_pm_get_if_in_use()
448  * @rpm: the intel_runtime_pm structure
465 intel_wakeref_t intel_runtime_pm_get_noresume(struct intel_runtime_pm *rpm)  in intel_runtime_pm_get_noresume()  argument
467 	assert_rpm_wakelock_held(rpm);  in intel_runtime_pm_get_noresume()
468 	pm_runtime_get_noresume(rpm->kdev);  in intel_runtime_pm_get_noresume()
470 	intel_runtime_pm_acquire(rpm, true);  in intel_runtime_pm_get_noresume()
472 	return track_intel_runtime_pm_wakeref(rpm);  in intel_runtime_pm_get_noresume()
475 static void __intel_runtime_pm_put(struct intel_runtime_pm *rpm,  in __intel_runtime_pm_put()  argument
479 	struct device *kdev = rpm->kdev;  in __intel_runtime_pm_put()
481 	untrack_intel_runtime_pm_wakeref(rpm, wref);  in __intel_runtime_pm_put()
483 	intel_runtime_pm_release(rpm, wakelock);  in __intel_runtime_pm_put()
491  * @rpm: the intel_runtime_pm structure
499 intel_runtime_pm_put_raw(struct intel_runtime_pm *rpm, intel_wakeref_t wref)  in intel_runtime_pm_put_raw()  argument
501 	__intel_runtime_pm_put(rpm, wref, false);  in intel_runtime_pm_put_raw()
506  * @rpm: the intel_runtime_pm structure
516 void intel_runtime_pm_put_unchecked(struct intel_runtime_pm *rpm)  in intel_runtime_pm_put_unchecked()  argument
518 	__intel_runtime_pm_put(rpm, -1, true);  in intel_runtime_pm_put_unchecked()
524  * @rpm: the intel_runtime_pm structure
531 void intel_runtime_pm_put(struct intel_runtime_pm *rpm, intel_wakeref_t wref)  in intel_runtime_pm_put()  argument
533 	__intel_runtime_pm_put(rpm, wref, true);  in intel_runtime_pm_put()
539  * @rpm: the intel_runtime_pm structure
547 void intel_runtime_pm_enable(struct intel_runtime_pm *rpm)  in intel_runtime_pm_enable()  argument
549 	struct drm_i915_private *i915 = container_of(rpm,  in intel_runtime_pm_enable()
552 	struct device *kdev = rpm->kdev;  in intel_runtime_pm_enable()
568 	 * Take a permanent reference to disable the RPM functionality and drop  in intel_runtime_pm_enable()
570 	 * so the driver's own RPM reference tracking asserts also work on  in intel_runtime_pm_enable()
571 	 * platforms without RPM support.  in intel_runtime_pm_enable()
573 	if (!rpm->available) {  in intel_runtime_pm_enable()
585 	 * The core calls the driver load handler with an RPM reference held.  in intel_runtime_pm_enable()
592 void intel_runtime_pm_disable(struct intel_runtime_pm *rpm)  in intel_runtime_pm_disable()  argument
594 	struct drm_i915_private *i915 = container_of(rpm,  in intel_runtime_pm_disable()
597 	struct device *kdev = rpm->kdev;  in intel_runtime_pm_disable()
599 	/* Transfer rpm ownership back to core */  in intel_runtime_pm_disable()
601 		 "Failed to pass rpm ownership back to core\n");  in intel_runtime_pm_disable()
605 	if (!rpm->available)  in intel_runtime_pm_disable()
609 void intel_runtime_pm_driver_release(struct intel_runtime_pm *rpm)  in intel_runtime_pm_driver_release()  argument
611 	struct drm_i915_private *i915 = container_of(rpm,  in intel_runtime_pm_driver_release()
614 	int count = atomic_read(&rpm->wakeref_count);  in intel_runtime_pm_driver_release()
621 	untrack_all_intel_runtime_pm_wakerefs(rpm);  in intel_runtime_pm_driver_release()
624 void intel_runtime_pm_init_early(struct intel_runtime_pm *rpm)  in intel_runtime_pm_init_early()  argument
627 			container_of(rpm, struct drm_i915_private, runtime_pm);  in intel_runtime_pm_init_early()
631 	rpm->kdev = kdev;  in intel_runtime_pm_init_early()
632 	rpm->available = HAS_RUNTIME_PM(i915);  in intel_runtime_pm_init_early()
634 	init_intel_runtime_pm_wakeref(rpm);  in intel_runtime_pm_init_early()