24 static void user_forcewake(struct intel_gt *gt, bool suspend)  in user_forcewake()  argument
26 	int count = atomic_read(&gt->user_wakeref);  in user_forcewake()
32 	intel_gt_pm_get(gt);  in user_forcewake()
34 		GEM_BUG_ON(count > atomic_read(&gt->wakeref.count));  in user_forcewake()
35 		atomic_sub(count, &gt->wakeref.count);  in user_forcewake()
37 		atomic_add(count, &gt->wakeref.count);  in user_forcewake()
39 	intel_gt_pm_put(gt);  in user_forcewake()
44 	struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref);  in __gt_unpark()  local
45 	struct drm_i915_private *i915 = gt->i915;  in __gt_unpark()
47 	GT_TRACE(gt, "\n");  in __gt_unpark()
59 	 * Work around it by grabbing a GT IRQ power domain whilst there is any  in __gt_unpark()
60 	 * GT activity, preventing any DC state transitions.  in __gt_unpark()
62 	gt->awake = intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ);  in __gt_unpark()
63 	GEM_BUG_ON(!gt->awake);  in __gt_unpark()
65 	intel_rc6_unpark(&gt->rc6);  in __gt_unpark()
66 	intel_rps_unpark(&gt->rps);  in __gt_unpark()
69 	intel_gt_unpark_requests(gt);  in __gt_unpark()
76 	struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref);  in __gt_park()  local
77 	intel_wakeref_t wakeref = fetch_and_zero(&gt->awake);  in __gt_park()
78 	struct drm_i915_private *i915 = gt->i915;  in __gt_park()
80 	GT_TRACE(gt, "\n");  in __gt_park()
82 	intel_gt_park_requests(gt);  in __gt_park()
84 	i915_vma_parked(gt);  in __gt_park()
86 	intel_rps_park(&gt->rps);  in __gt_park()
87 	intel_rc6_park(&gt->rc6);  in __gt_park()
106 void intel_gt_pm_init_early(struct intel_gt *gt)  in intel_gt_pm_init_early()  argument
108 	intel_wakeref_init(&gt->wakeref, gt->uncore->rpm, &wf_ops);  in intel_gt_pm_init_early()
111 void intel_gt_pm_init(struct intel_gt *gt)  in intel_gt_pm_init()  argument
118 	intel_rc6_init(&gt->rc6);  in intel_gt_pm_init()
119 	intel_rps_init(&gt->rps);  in intel_gt_pm_init()
122 static bool reset_engines(struct intel_gt *gt)  in reset_engines()  argument
124 	if (INTEL_INFO(gt->i915)->gpu_reset_clobbers_display)  in reset_engines()
127 	return __intel_gt_reset(gt, ALL_ENGINES) == 0;  in reset_engines()
130 static void gt_sanitize(struct intel_gt *gt, bool force)  in gt_sanitize()  argument
136 	GT_TRACE(gt, "force:%s", yesno(force));  in gt_sanitize()
139 	wakeref = intel_runtime_pm_get(gt->uncore->rpm);  in gt_sanitize()
140 	intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);  in gt_sanitize()
142 	intel_gt_check_clock_frequency(gt);  in gt_sanitize()
150 	if (intel_gt_is_wedged(gt))  in gt_sanitize()
151 		intel_gt_unset_wedged(gt);  in gt_sanitize()
153 	intel_uc_sanitize(&gt->uc);  in gt_sanitize()
155 	for_each_engine(engine, gt, id)  in gt_sanitize()
159 	intel_uc_reset_prepare(&gt->uc);  in gt_sanitize()
161 	for_each_engine(engine, gt, id)  in gt_sanitize()
165 	if (reset_engines(gt) || force) {  in gt_sanitize()
166 		for_each_engine(engine, gt, id)  in gt_sanitize()
170 	for_each_engine(engine, gt, id)  in gt_sanitize()
174 	intel_rps_sanitize(&gt->rps);  in gt_sanitize()
176 	intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);  in gt_sanitize()
177 	intel_runtime_pm_put(gt->uncore->rpm, wakeref);  in gt_sanitize()
180 void intel_gt_pm_fini(struct intel_gt *gt)  in intel_gt_pm_fini()  argument
182 	intel_rc6_fini(&gt->rc6);  in intel_gt_pm_fini()
185 int intel_gt_resume(struct intel_gt *gt)  in intel_gt_resume()  argument
191 	err = intel_gt_has_unrecoverable_error(gt);  in intel_gt_resume()
195 	GT_TRACE(gt, "\n");  in intel_gt_resume()
203 	gt_sanitize(gt, true);  in intel_gt_resume()
205 	intel_gt_pm_get(gt);  in intel_gt_resume()
207 	intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);  in intel_gt_resume()
208 	intel_rc6_sanitize(&gt->rc6);  in intel_gt_resume()
209 	if (intel_gt_is_wedged(gt)) {  in intel_gt_resume()
215 	err = intel_gt_init_hw(gt);  in intel_gt_resume()
217 		i915_probe_error(gt->i915,  in intel_gt_resume()
222 	intel_rps_enable(&gt->rps);  in intel_gt_resume()
223 	intel_llc_enable(&gt->llc);  in intel_gt_resume()
225 	for_each_engine(engine, gt, id) {  in intel_gt_resume()
233 			drm_err(&gt->i915->drm,  in intel_gt_resume()
240 	intel_rc6_enable(&gt->rc6);  in intel_gt_resume()
242 	intel_uc_resume(&gt->uc);  in intel_gt_resume()
244 	user_forcewake(gt, false);  in intel_gt_resume()
247 	intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);  in intel_gt_resume()
248 	intel_gt_pm_put(gt);  in intel_gt_resume()
252 	intel_gt_set_wedged(gt);  in intel_gt_resume()
256 static void wait_for_suspend(struct intel_gt *gt)  in wait_for_suspend()  argument
258 	if (!intel_gt_pm_is_awake(gt))  in wait_for_suspend()
261 	if (intel_gt_wait_for_idle(gt, I915_GEM_IDLE_TIMEOUT) == -ETIME) {  in wait_for_suspend()
266 		intel_gt_set_wedged(gt);  in wait_for_suspend()
267 		intel_gt_retire_requests(gt);  in wait_for_suspend()
270 	intel_gt_pm_wait_for_idle(gt);  in wait_for_suspend()
273 void intel_gt_suspend_prepare(struct intel_gt *gt)  in intel_gt_suspend_prepare()  argument
275 	user_forcewake(gt, true);  in intel_gt_suspend_prepare()
276 	wait_for_suspend(gt);  in intel_gt_suspend_prepare()
278 	intel_uc_suspend(&gt->uc);  in intel_gt_suspend_prepare()
290 void intel_gt_suspend_late(struct intel_gt *gt)  in intel_gt_suspend_late()  argument
295 	wait_for_suspend(gt);  in intel_gt_suspend_late()
297 	if (is_mock_gt(gt))  in intel_gt_suspend_late()
300 	GEM_BUG_ON(gt->awake);  in intel_gt_suspend_late()
315 	with_intel_runtime_pm(gt->uncore->rpm, wakeref) {  in intel_gt_suspend_late()
316 		intel_rps_disable(&gt->rps);  in intel_gt_suspend_late()
317 		intel_rc6_disable(&gt->rc6);  in intel_gt_suspend_late()
318 		intel_llc_disable(&gt->llc);  in intel_gt_suspend_late()
321 	gt_sanitize(gt, false);  in intel_gt_suspend_late()
323 	GT_TRACE(gt, "\n");  in intel_gt_suspend_late()
326 void intel_gt_runtime_suspend(struct intel_gt *gt)  in intel_gt_runtime_suspend()  argument
328 	intel_uc_runtime_suspend(&gt->uc);  in intel_gt_runtime_suspend()
330 	GT_TRACE(gt, "\n");  in intel_gt_runtime_suspend()
333 int intel_gt_runtime_resume(struct intel_gt *gt)  in intel_gt_runtime_resume()  argument
335 	GT_TRACE(gt, "\n");  in intel_gt_runtime_resume()
336 	intel_gt_init_swizzling(gt);  in intel_gt_runtime_resume()
337 	intel_ggtt_restore_fences(gt->ggtt);  in intel_gt_runtime_resume()
339 	return intel_uc_runtime_resume(&gt->uc);  in intel_gt_runtime_resume()