111 static int i915_workqueues_init(struct drm_i915_private *dev_priv)  in i915_workqueues_init()  argument
127 	dev_priv->wq = alloc_ordered_workqueue("i915", 0);  in i915_workqueues_init()
128 	if (dev_priv->wq == NULL)  in i915_workqueues_init()
131 	dev_priv->display.hotplug.dp_wq = alloc_ordered_workqueue("i915-dp", 0);  in i915_workqueues_init()
132 	if (dev_priv->display.hotplug.dp_wq == NULL)  in i915_workqueues_init()
141 	dev_priv->unordered_wq = alloc_workqueue("i915-unordered", 0, 0);  in i915_workqueues_init()
142 	if (dev_priv->unordered_wq == NULL)  in i915_workqueues_init()
148 	destroy_workqueue(dev_priv->display.hotplug.dp_wq);  in i915_workqueues_init()
150 	destroy_workqueue(dev_priv->wq);  in i915_workqueues_init()
152 	drm_err(&dev_priv->drm, "Failed to allocate workqueues.\n");  in i915_workqueues_init()
157 static void i915_workqueues_cleanup(struct drm_i915_private *dev_priv)  in i915_workqueues_cleanup()  argument
159 	destroy_workqueue(dev_priv->unordered_wq);  in i915_workqueues_cleanup()
160 	destroy_workqueue(dev_priv->display.hotplug.dp_wq);  in i915_workqueues_cleanup()
161 	destroy_workqueue(dev_priv->wq);  in i915_workqueues_cleanup()
174 static void intel_detect_preproduction_hw(struct drm_i915_private *dev_priv)  in intel_detect_preproduction_hw()  argument
178 	pre |= IS_HASWELL_EARLY_SDV(dev_priv);  in intel_detect_preproduction_hw()
179 	pre |= IS_SKYLAKE(dev_priv) && INTEL_REVID(dev_priv) < 0x6;  in intel_detect_preproduction_hw()
180 	pre |= IS_BROXTON(dev_priv) && INTEL_REVID(dev_priv) < 0xA;  in intel_detect_preproduction_hw()
181 	pre |= IS_KABYLAKE(dev_priv) && INTEL_REVID(dev_priv) < 0x1;  in intel_detect_preproduction_hw()
182 	pre |= IS_GEMINILAKE(dev_priv) && INTEL_REVID(dev_priv) < 0x3;  in intel_detect_preproduction_hw()
183 	pre |= IS_ICELAKE(dev_priv) && INTEL_REVID(dev_priv) < 0x7;  in intel_detect_preproduction_hw()
184 	pre |= IS_TIGERLAKE(dev_priv) && INTEL_REVID(dev_priv) < 0x1;  in intel_detect_preproduction_hw()
185 	pre |= IS_DG1(dev_priv) && INTEL_REVID(dev_priv) < 0x1;  in intel_detect_preproduction_hw()
188 		drm_err(&dev_priv->drm, "This is a pre-production stepping. "  in intel_detect_preproduction_hw()
215 static int i915_driver_early_probe(struct drm_i915_private *dev_priv)  in i915_driver_early_probe()  argument
219 	if (i915_inject_probe_failure(dev_priv))  in i915_driver_early_probe()
222 	intel_device_info_runtime_init_early(dev_priv);  in i915_driver_early_probe()
224 	intel_step_init(dev_priv);  in i915_driver_early_probe()
226 	intel_uncore_mmio_debug_init_early(dev_priv);  in i915_driver_early_probe()
228 	spin_lock_init(&dev_priv->irq_lock);  in i915_driver_early_probe()
229 	spin_lock_init(&dev_priv->gpu_error.lock);  in i915_driver_early_probe()
230 	mutex_init(&dev_priv->display.backlight.lock);  in i915_driver_early_probe()
232 	mutex_init(&dev_priv->sb_lock);  in i915_driver_early_probe()
233 	cpu_latency_qos_add_request(&dev_priv->sb_qos, PM_QOS_DEFAULT_VALUE);  in i915_driver_early_probe()
235 	mutex_init(&dev_priv->display.audio.mutex);  in i915_driver_early_probe()
236 	mutex_init(&dev_priv->display.wm.wm_mutex);  in i915_driver_early_probe()
237 	mutex_init(&dev_priv->display.pps.mutex);  in i915_driver_early_probe()
238 	mutex_init(&dev_priv->display.hdcp.hdcp_mutex);  in i915_driver_early_probe()
240 	i915_memcpy_init_early(dev_priv);  in i915_driver_early_probe()
241 	intel_runtime_pm_init_early(&dev_priv->runtime_pm);  in i915_driver_early_probe()
243 	ret = i915_workqueues_init(dev_priv);  in i915_driver_early_probe()
247 	ret = vlv_suspend_init(dev_priv);  in i915_driver_early_probe()
251 	ret = intel_region_ttm_device_init(dev_priv);  in i915_driver_early_probe()
255 	ret = intel_root_gt_init_early(dev_priv);  in i915_driver_early_probe()
259 	i915_gem_init_early(dev_priv);  in i915_driver_early_probe()
262 	intel_detect_pch(dev_priv);  in i915_driver_early_probe()
264 	intel_irq_init(dev_priv);  in i915_driver_early_probe()
265 	intel_display_driver_early_probe(dev_priv);  in i915_driver_early_probe()
266 	intel_clock_gating_hooks_init(dev_priv);  in i915_driver_early_probe()
268 	intel_detect_preproduction_hw(dev_priv);  in i915_driver_early_probe()
273 	intel_region_ttm_device_fini(dev_priv);  in i915_driver_early_probe()
275 	vlv_suspend_cleanup(dev_priv);  in i915_driver_early_probe()
277 	i915_workqueues_cleanup(dev_priv);  in i915_driver_early_probe()
286 static void i915_driver_late_release(struct drm_i915_private *dev_priv)  in i915_driver_late_release()  argument
288 	intel_irq_fini(dev_priv);  in i915_driver_late_release()
289 	intel_power_domains_cleanup(dev_priv);  in i915_driver_late_release()
290 	i915_gem_cleanup_early(dev_priv);  in i915_driver_late_release()
291 	intel_gt_driver_late_release_all(dev_priv);  in i915_driver_late_release()
292 	intel_region_ttm_device_fini(dev_priv);  in i915_driver_late_release()
293 	vlv_suspend_cleanup(dev_priv);  in i915_driver_late_release()
294 	i915_workqueues_cleanup(dev_priv);  in i915_driver_late_release()
296 	cpu_latency_qos_remove_request(&dev_priv->sb_qos);  in i915_driver_late_release()
297 	mutex_destroy(&dev_priv->sb_lock);  in i915_driver_late_release()
299 	i915_params_free(&dev_priv->params);  in i915_driver_late_release()
311 static int i915_driver_mmio_probe(struct drm_i915_private *dev_priv)  in i915_driver_mmio_probe()  argument
316 	if (i915_inject_probe_failure(dev_priv))  in i915_driver_mmio_probe()
319 	ret = intel_gmch_bridge_setup(dev_priv);  in i915_driver_mmio_probe()
323 	for_each_gt(gt, dev_priv, i) {  in i915_driver_mmio_probe()
328 		ret = drmm_add_action_or_reset(&dev_priv->drm,  in i915_driver_mmio_probe()
336 	intel_gmch_bar_setup(dev_priv);  in i915_driver_mmio_probe()
337 	intel_device_info_runtime_init(dev_priv);  in i915_driver_mmio_probe()
339 	for_each_gt(gt, dev_priv, i) {  in i915_driver_mmio_probe()
346 	sanitize_gpu(dev_priv);  in i915_driver_mmio_probe()
351 	intel_gmch_bar_teardown(dev_priv);  in i915_driver_mmio_probe()
360 static void i915_driver_mmio_release(struct drm_i915_private *dev_priv)  in i915_driver_mmio_release()  argument
362 	intel_gmch_bar_teardown(dev_priv);  in i915_driver_mmio_release()
443 static int i915_driver_hw_probe(struct drm_i915_private *dev_priv)  in i915_driver_hw_probe()  argument
445 	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);  in i915_driver_hw_probe()
448 	if (i915_inject_probe_failure(dev_priv))  in i915_driver_hw_probe()
451 	if (HAS_PPGTT(dev_priv)) {  in i915_driver_hw_probe()
452 		if (intel_vgpu_active(dev_priv) &&  in i915_driver_hw_probe()
453 		    !intel_vgpu_has_full_ppgtt(dev_priv)) {  in i915_driver_hw_probe()
454 			i915_report_error(dev_priv,  in i915_driver_hw_probe()
460 	if (HAS_EXECLISTS(dev_priv)) {  in i915_driver_hw_probe()
466 		if (intel_vgpu_active(dev_priv) &&  in i915_driver_hw_probe()
467 		    !intel_vgpu_has_hwsp_emulation(dev_priv)) {  in i915_driver_hw_probe()
468 			i915_report_error(dev_priv,  in i915_driver_hw_probe()
475 	intel_dram_edram_detect(dev_priv);  in i915_driver_hw_probe()
477 	ret = i915_set_dma_info(dev_priv);  in i915_driver_hw_probe()
481 	ret = i915_perf_init(dev_priv);  in i915_driver_hw_probe()
485 	ret = i915_ggtt_probe_hw(dev_priv);  in i915_driver_hw_probe()
489 	ret = drm_aperture_remove_conflicting_pci_framebuffers(pdev, dev_priv->drm.driver);  in i915_driver_hw_probe()
493 	ret = i915_ggtt_init_hw(dev_priv);  in i915_driver_hw_probe()
501 	ret = intel_gt_tiles_init(dev_priv);  in i915_driver_hw_probe()
505 	ret = intel_memory_regions_hw_probe(dev_priv);  in i915_driver_hw_probe()
509 	ret = i915_ggtt_enable_hw(dev_priv);  in i915_driver_hw_probe()
511 		drm_err(&dev_priv->drm, "failed to enable GGTT\n");  in i915_driver_hw_probe()
536 	if (GRAPHICS_VER(dev_priv) >= 5) {  in i915_driver_hw_probe()
538 			drm_dbg(&dev_priv->drm, "can't enable MSI");  in i915_driver_hw_probe()
541 	ret = intel_gvt_init(dev_priv);  in i915_driver_hw_probe()
545 	intel_opregion_setup(dev_priv);  in i915_driver_hw_probe()
547 	ret = i915_pcode_init(dev_priv);  in i915_driver_hw_probe()
555 	intel_dram_detect(dev_priv);  in i915_driver_hw_probe()
557 	intel_bw_init_hw(dev_priv);  in i915_driver_hw_probe()
562 	intel_opregion_cleanup(dev_priv);  in i915_driver_hw_probe()
567 	intel_memory_regions_driver_release(dev_priv);  in i915_driver_hw_probe()
569 	i915_ggtt_driver_release(dev_priv);  in i915_driver_hw_probe()
570 	i915_gem_drain_freed_objects(dev_priv);  in i915_driver_hw_probe()
571 	i915_ggtt_driver_late_release(dev_priv);  in i915_driver_hw_probe()
573 	i915_perf_fini(dev_priv);  in i915_driver_hw_probe()
581 static void i915_driver_hw_remove(struct drm_i915_private *dev_priv)  in i915_driver_hw_remove()  argument
583 	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);  in i915_driver_hw_remove()
585 	i915_perf_fini(dev_priv);  in i915_driver_hw_remove()
587 	intel_opregion_cleanup(dev_priv);  in i915_driver_hw_remove()
600 static void i915_driver_register(struct drm_i915_private *dev_priv)  in i915_driver_register()  argument
605 	i915_gem_driver_register(dev_priv);  in i915_driver_register()
606 	i915_pmu_register(dev_priv);  in i915_driver_register()
608 	intel_vgpu_register(dev_priv);  in i915_driver_register()
611 	if (drm_dev_register(&dev_priv->drm, 0)) {  in i915_driver_register()
612 		drm_err(&dev_priv->drm,  in i915_driver_register()
617 	i915_debugfs_register(dev_priv);  in i915_driver_register()
618 	i915_setup_sysfs(dev_priv);  in i915_driver_register()
621 	i915_perf_register(dev_priv);  in i915_driver_register()
623 	for_each_gt(gt, dev_priv, i)  in i915_driver_register()
626 	intel_pxp_debugfs_register(dev_priv->pxp);  in i915_driver_register()
628 	i915_hwmon_register(dev_priv);  in i915_driver_register()
630 	intel_display_driver_register(dev_priv);  in i915_driver_register()
632 	intel_power_domains_enable(dev_priv);  in i915_driver_register()
633 	intel_runtime_pm_enable(&dev_priv->runtime_pm);  in i915_driver_register()
637 	if (i915_switcheroo_register(dev_priv))  in i915_driver_register()
638 		drm_err(&dev_priv->drm, "Failed to register vga switcheroo!\n");  in i915_driver_register()
645 static void i915_driver_unregister(struct drm_i915_private *dev_priv)  in i915_driver_unregister()  argument
650 	i915_switcheroo_unregister(dev_priv);  in i915_driver_unregister()
654 	intel_runtime_pm_disable(&dev_priv->runtime_pm);  in i915_driver_unregister()
655 	intel_power_domains_disable(dev_priv);  in i915_driver_unregister()
657 	intel_display_driver_unregister(dev_priv);  in i915_driver_unregister()
659 	intel_pxp_fini(dev_priv);  in i915_driver_unregister()
661 	for_each_gt(gt, dev_priv, i)  in i915_driver_unregister()
664 	i915_hwmon_unregister(dev_priv);  in i915_driver_unregister()
666 	i915_perf_unregister(dev_priv);  in i915_driver_unregister()
667 	i915_pmu_unregister(dev_priv);  in i915_driver_unregister()
669 	i915_teardown_sysfs(dev_priv);  in i915_driver_unregister()
670 	drm_dev_unplug(&dev_priv->drm);  in i915_driver_unregister()
672 	i915_gem_driver_unregister(dev_priv);  in i915_driver_unregister()
682 static void i915_welcome_messages(struct drm_i915_private *dev_priv)  in i915_welcome_messages()  argument
690 			   INTEL_DEVID(dev_priv),  in i915_welcome_messages()
691 			   INTEL_REVID(dev_priv),  in i915_welcome_messages()
692 			   intel_platform_name(INTEL_INFO(dev_priv)->platform),  in i915_welcome_messages()
693 			   intel_subplatform(RUNTIME_INFO(dev_priv),  in i915_welcome_messages()
694 					     INTEL_INFO(dev_priv)->platform),  in i915_welcome_messages()
695 			   GRAPHICS_VER(dev_priv));  in i915_welcome_messages()
697 		intel_device_info_print(INTEL_INFO(dev_priv),  in i915_welcome_messages()
698 					RUNTIME_INFO(dev_priv), &p);  in i915_welcome_messages()
699 		intel_display_device_info_print(DISPLAY_INFO(dev_priv),  in i915_welcome_messages()
700 						DISPLAY_RUNTIME_INFO(dev_priv), &p);  in i915_welcome_messages()
701 		i915_print_iommu_status(dev_priv, &p);  in i915_welcome_messages()
702 		for_each_gt(gt, dev_priv, i)  in i915_welcome_messages()
707 		drm_info(&dev_priv->drm, "DRM_I915_DEBUG enabled\n");  in i915_welcome_messages()
709 		drm_info(&dev_priv->drm, "DRM_I915_DEBUG_GEM enabled\n");  in i915_welcome_messages()
711 		drm_info(&dev_priv->drm,  in i915_welcome_messages()
884 	struct drm_i915_private *dev_priv = to_i915(dev);  in i915_driver_release()  local
885 	struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;  in i915_driver_release()
888 	if (!dev_priv->do_release)  in i915_driver_release()
893 	i915_gem_driver_release(dev_priv);  in i915_driver_release()
895 	intel_memory_regions_driver_release(dev_priv);  in i915_driver_release()
896 	i915_ggtt_driver_release(dev_priv);  in i915_driver_release()
897 	i915_gem_drain_freed_objects(dev_priv);  in i915_driver_release()
898 	i915_ggtt_driver_late_release(dev_priv);  in i915_driver_release()
900 	i915_driver_mmio_release(dev_priv);  in i915_driver_release()
906 	i915_driver_late_release(dev_priv);  in i915_driver_release()
955 static void intel_suspend_encoders(struct drm_i915_private *dev_priv)  in intel_suspend_encoders()  argument
959 	if (!HAS_DISPLAY(dev_priv))  in intel_suspend_encoders()
966 	drm_modeset_lock_all(&dev_priv->drm);  in intel_suspend_encoders()
967 	for_each_intel_encoder(&dev_priv->drm, encoder)  in intel_suspend_encoders()
970 	drm_modeset_unlock_all(&dev_priv->drm);  in intel_suspend_encoders()
972 	for_each_intel_encoder(&dev_priv->drm, encoder)  in intel_suspend_encoders()
977 static void intel_shutdown_encoders(struct drm_i915_private *dev_priv)  in intel_shutdown_encoders()  argument
981 	if (!HAS_DISPLAY(dev_priv))  in intel_shutdown_encoders()
988 	drm_modeset_lock_all(&dev_priv->drm);  in intel_shutdown_encoders()
989 	for_each_intel_encoder(&dev_priv->drm, encoder)  in intel_shutdown_encoders()
992 	drm_modeset_unlock_all(&dev_priv->drm);  in intel_shutdown_encoders()
994 	for_each_intel_encoder(&dev_priv->drm, encoder)  in intel_shutdown_encoders()
1040 static bool suspend_to_idle(struct drm_i915_private *dev_priv)  in suspend_to_idle()  argument
1073 	struct drm_i915_private *dev_priv = to_i915(dev);  in i915_drm_suspend()  local
1074 	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);  in i915_drm_suspend()
1077 	disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);  in i915_drm_suspend()
1081 	intel_power_domains_disable(dev_priv);  in i915_drm_suspend()
1082 	if (HAS_DISPLAY(dev_priv))  in i915_drm_suspend()
1087 	intel_display_driver_suspend(dev_priv);  in i915_drm_suspend()
1089 	intel_dp_mst_suspend(dev_priv);  in i915_drm_suspend()
1091 	intel_runtime_pm_disable_interrupts(dev_priv);  in i915_drm_suspend()
1092 	intel_hpd_cancel_work(dev_priv);  in i915_drm_suspend()
1094 	intel_suspend_encoders(dev_priv);  in i915_drm_suspend()
1097 	intel_dpt_suspend(dev_priv);  in i915_drm_suspend()
1098 	i915_ggtt_suspend(to_gt(dev_priv)->ggtt);  in i915_drm_suspend()
1100 	i915_save_display(dev_priv);  in i915_drm_suspend()
1102 	opregion_target_state = suspend_to_idle(dev_priv) ? PCI_D1 : PCI_D3cold;  in i915_drm_suspend()
1103 	intel_opregion_suspend(dev_priv, opregion_target_state);  in i915_drm_suspend()
1107 	dev_priv->suspend_count++;  in i915_drm_suspend()
1109 	intel_dmc_suspend(dev_priv);  in i915_drm_suspend()
1111 	enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);  in i915_drm_suspend()
1113 	i915_gem_drain_freed_objects(dev_priv);  in i915_drm_suspend()
1120 	struct drm_i915_private *dev_priv = to_i915(dev);  in i915_drm_suspend_late()  local
1121 	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);  in i915_drm_suspend_late()
1122 	struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;  in i915_drm_suspend_late()
1125 	bool s2idle = !hibernation && suspend_to_idle(dev_priv);  in i915_drm_suspend_late()
1129 	intel_pxp_suspend(dev_priv->pxp);  in i915_drm_suspend_late()
1131 	i915_gem_suspend_late(dev_priv);  in i915_drm_suspend_late()
1133 	for_each_gt(gt, dev_priv, i)  in i915_drm_suspend_late()
1136 	intel_power_domains_suspend(dev_priv, s2idle);  in i915_drm_suspend_late()
1138 	intel_display_power_suspend_late(dev_priv);  in i915_drm_suspend_late()
1140 	ret = vlv_suspend_complete(dev_priv);  in i915_drm_suspend_late()
1142 		drm_err(&dev_priv->drm, "Suspend complete failed: %d\n", ret);  in i915_drm_suspend_late()
1143 		intel_power_domains_resume(dev_priv);  in i915_drm_suspend_late()
1161 	if (!(hibernation && GRAPHICS_VER(dev_priv) < 6))  in i915_drm_suspend_late()
1166 	if (!dev_priv->uncore.user_forcewake_count)  in i915_drm_suspend_late()
1193 	struct drm_i915_private *dev_priv = to_i915(dev);  in i915_drm_resume()  local
1197 	disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);  in i915_drm_resume()
1199 	ret = i915_pcode_init(dev_priv);  in i915_drm_resume()
1203 	sanitize_gpu(dev_priv);  in i915_drm_resume()
1205 	ret = i915_ggtt_enable_hw(dev_priv);  in i915_drm_resume()
1207 		drm_err(&dev_priv->drm, "failed to re-enable GGTT\n");  in i915_drm_resume()
1209 	i915_ggtt_resume(to_gt(dev_priv)->ggtt);  in i915_drm_resume()
1211 	for_each_gt(gt, dev_priv, i)  in i915_drm_resume()
1216 	intel_dpt_resume(dev_priv);  in i915_drm_resume()
1218 	intel_dmc_resume(dev_priv);  in i915_drm_resume()
1220 	i915_restore_display(dev_priv);  in i915_drm_resume()
1221 	intel_pps_unlock_regs_wa(dev_priv);  in i915_drm_resume()
1223 	intel_init_pch_refclk(dev_priv);  in i915_drm_resume()
1235 	intel_runtime_pm_enable_interrupts(dev_priv);  in i915_drm_resume()
1237 	if (HAS_DISPLAY(dev_priv))  in i915_drm_resume()
1240 	i915_gem_resume(dev_priv);  in i915_drm_resume()
1242 	intel_display_driver_init_hw(dev_priv);  in i915_drm_resume()
1244 	intel_clock_gating_init(dev_priv);  in i915_drm_resume()
1245 	intel_hpd_init(dev_priv);  in i915_drm_resume()
1248 	intel_dp_mst_resume(dev_priv);  in i915_drm_resume()
1249 	intel_display_driver_resume(dev_priv);  in i915_drm_resume()
1251 	intel_hpd_poll_disable(dev_priv);  in i915_drm_resume()
1252 	if (HAS_DISPLAY(dev_priv))  in i915_drm_resume()
1255 	intel_opregion_resume(dev_priv);  in i915_drm_resume()
1259 	intel_power_domains_enable(dev_priv);  in i915_drm_resume()
1261 	intel_gvt_resume(dev_priv);  in i915_drm_resume()
1263 	enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);  in i915_drm_resume()
1270 	struct drm_i915_private *dev_priv = to_i915(dev);  in i915_drm_resume_early()  local
1271 	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);  in i915_drm_resume_early()
1297 		drm_err(&dev_priv->drm,  in i915_drm_resume_early()
1320 	disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);  in i915_drm_resume_early()
1322 	ret = vlv_resume_prepare(dev_priv, false);  in i915_drm_resume_early()
1324 		drm_err(&dev_priv->drm,  in i915_drm_resume_early()
1327 	for_each_gt(gt, dev_priv, i) {  in i915_drm_resume_early()
1332 	intel_display_power_resume_early(dev_priv);  in i915_drm_resume_early()
1334 	intel_power_domains_resume(dev_priv);  in i915_drm_resume_early()
1336 	enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);  in i915_drm_resume_early()
1505 	struct drm_i915_private *dev_priv = kdev_to_i915(kdev);  in intel_runtime_suspend()  local
1506 	struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;  in intel_runtime_suspend()
1507 	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);  in intel_runtime_suspend()
1512 	if (drm_WARN_ON_ONCE(&dev_priv->drm, !HAS_RUNTIME_PM(dev_priv)))  in intel_runtime_suspend()
1515 	drm_dbg(&dev_priv->drm, "Suspending device\n");  in intel_runtime_suspend()
1523 	i915_gem_runtime_suspend(dev_priv);  in intel_runtime_suspend()
1525 	intel_pxp_runtime_suspend(dev_priv->pxp);  in intel_runtime_suspend()
1527 	for_each_gt(gt, dev_priv, i)  in intel_runtime_suspend()
1530 	intel_runtime_pm_disable_interrupts(dev_priv);  in intel_runtime_suspend()
1532 	for_each_gt(gt, dev_priv, i)  in intel_runtime_suspend()
1535 	intel_display_power_suspend(dev_priv);  in intel_runtime_suspend()
1537 	ret = vlv_suspend_complete(dev_priv);  in intel_runtime_suspend()
1539 		drm_err(&dev_priv->drm,  in intel_runtime_suspend()
1541 		intel_uncore_runtime_resume(&dev_priv->uncore);  in intel_runtime_suspend()
1543 		intel_runtime_pm_enable_interrupts(dev_priv);  in intel_runtime_suspend()
1545 		for_each_gt(gt, dev_priv, i)  in intel_runtime_suspend()
1556 	if (intel_uncore_arm_unclaimed_mmio_detection(&dev_priv->uncore))  in intel_runtime_suspend()
1557 		drm_err(&dev_priv->drm,  in intel_runtime_suspend()
1575 	if (IS_BROADWELL(dev_priv)) {  in intel_runtime_suspend()
1582 		intel_opregion_notify_adapter(dev_priv, PCI_D3hot);  in intel_runtime_suspend()
1591 		intel_opregion_notify_adapter(dev_priv, PCI_D1);  in intel_runtime_suspend()
1594 	assert_forcewakes_inactive(&dev_priv->uncore);  in intel_runtime_suspend()
1596 	if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))  in intel_runtime_suspend()
1597 		intel_hpd_poll_enable(dev_priv);  in intel_runtime_suspend()
1599 	drm_dbg(&dev_priv->drm, "Device suspended\n");  in intel_runtime_suspend()
1605 	struct drm_i915_private *dev_priv = kdev_to_i915(kdev);  in intel_runtime_resume()  local
1606 	struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;  in intel_runtime_resume()
1607 	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);  in intel_runtime_resume()
1612 	if (drm_WARN_ON_ONCE(&dev_priv->drm, !HAS_RUNTIME_PM(dev_priv)))  in intel_runtime_resume()
1615 	drm_dbg(&dev_priv->drm, "Resuming device\n");  in intel_runtime_resume()
1617 	drm_WARN_ON_ONCE(&dev_priv->drm, atomic_read(&rpm->wakeref_count));  in intel_runtime_resume()
1620 	intel_opregion_notify_adapter(dev_priv, PCI_D0);  in intel_runtime_resume()
1627 	if (intel_uncore_unclaimed_mmio(&dev_priv->uncore))  in intel_runtime_resume()
1628 		drm_dbg(&dev_priv->drm,  in intel_runtime_resume()
1631 	intel_display_power_resume(dev_priv);  in intel_runtime_resume()
1633 	ret = vlv_resume_prepare(dev_priv, true);  in intel_runtime_resume()
1635 	for_each_gt(gt, dev_priv, i)  in intel_runtime_resume()
1638 	intel_runtime_pm_enable_interrupts(dev_priv);  in intel_runtime_resume()
1644 	for_each_gt(gt, dev_priv, i)  in intel_runtime_resume()
1647 	intel_pxp_runtime_resume(dev_priv->pxp);  in intel_runtime_resume()
1654 	if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) {  in intel_runtime_resume()
1655 		intel_hpd_init(dev_priv);  in intel_runtime_resume()
1656 		intel_hpd_poll_disable(dev_priv);  in intel_runtime_resume()
1659 	skl_watermark_ipc_update(dev_priv);  in intel_runtime_resume()
1664 		drm_err(&dev_priv->drm,  in intel_runtime_resume()
1667 		drm_dbg(&dev_priv->drm, "Device resumed\n");  in intel_runtime_resume()