91 static int i915_get_bridge_dev(struct drm_i915_private *dev_priv)  in i915_get_bridge_dev()  argument
93 	int domain = pci_domain_nr(dev_priv->drm.pdev->bus);  in i915_get_bridge_dev()
95 	dev_priv->bridge_dev =  in i915_get_bridge_dev()
97 	if (!dev_priv->bridge_dev) {  in i915_get_bridge_dev()
98 		drm_err(&dev_priv->drm, "bridge device not found\n");  in i915_get_bridge_dev()
106 intel_alloc_mchbar_resource(struct drm_i915_private *dev_priv)  in intel_alloc_mchbar_resource()  argument
108 	int reg = INTEL_GEN(dev_priv) >= 4 ? MCHBAR_I965 : MCHBAR_I915;  in intel_alloc_mchbar_resource()
113 	if (INTEL_GEN(dev_priv) >= 4)  in intel_alloc_mchbar_resource()
114 		pci_read_config_dword(dev_priv->bridge_dev, reg + 4, &temp_hi);  in intel_alloc_mchbar_resource()
115 	pci_read_config_dword(dev_priv->bridge_dev, reg, &temp_lo);  in intel_alloc_mchbar_resource()
126 	dev_priv->mch_res.name = "i915 MCHBAR";  in intel_alloc_mchbar_resource()
127 	dev_priv->mch_res.flags = IORESOURCE_MEM;  in intel_alloc_mchbar_resource()
128 	ret = pci_bus_alloc_resource(dev_priv->bridge_dev->bus,  in intel_alloc_mchbar_resource()
129 				     &dev_priv->mch_res,  in intel_alloc_mchbar_resource()
133 				     dev_priv->bridge_dev);  in intel_alloc_mchbar_resource()
135 		drm_dbg(&dev_priv->drm, "failed bus alloc: %d\n", ret);  in intel_alloc_mchbar_resource()
136 		dev_priv->mch_res.start = 0;  in intel_alloc_mchbar_resource()
140 	if (INTEL_GEN(dev_priv) >= 4)  in intel_alloc_mchbar_resource()
141 		pci_write_config_dword(dev_priv->bridge_dev, reg + 4,  in intel_alloc_mchbar_resource()
142 				       upper_32_bits(dev_priv->mch_res.start));  in intel_alloc_mchbar_resource()
144 	pci_write_config_dword(dev_priv->bridge_dev, reg,  in intel_alloc_mchbar_resource()
145 			       lower_32_bits(dev_priv->mch_res.start));  in intel_alloc_mchbar_resource()
151 intel_setup_mchbar(struct drm_i915_private *dev_priv)  in intel_setup_mchbar()  argument
153 	int mchbar_reg = INTEL_GEN(dev_priv) >= 4 ? MCHBAR_I965 : MCHBAR_I915;  in intel_setup_mchbar()
157 	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))  in intel_setup_mchbar()
160 	dev_priv->mchbar_need_disable = false;  in intel_setup_mchbar()
162 	if (IS_I915G(dev_priv) || IS_I915GM(dev_priv)) {  in intel_setup_mchbar()
163 		pci_read_config_dword(dev_priv->bridge_dev, DEVEN, &temp);  in intel_setup_mchbar()
166 		pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, &temp);  in intel_setup_mchbar()
174 	if (intel_alloc_mchbar_resource(dev_priv))  in intel_setup_mchbar()
177 	dev_priv->mchbar_need_disable = true;  in intel_setup_mchbar()
180 	if (IS_I915G(dev_priv) || IS_I915GM(dev_priv)) {  in intel_setup_mchbar()
181 		pci_write_config_dword(dev_priv->bridge_dev, DEVEN,  in intel_setup_mchbar()
184 		pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, &temp);  in intel_setup_mchbar()
185 		pci_write_config_dword(dev_priv->bridge_dev, mchbar_reg, temp | 1);  in intel_setup_mchbar()
190 intel_teardown_mchbar(struct drm_i915_private *dev_priv)  in intel_teardown_mchbar()  argument
192 	int mchbar_reg = INTEL_GEN(dev_priv) >= 4 ? MCHBAR_I965 : MCHBAR_I915;  in intel_teardown_mchbar()
194 	if (dev_priv->mchbar_need_disable) {  in intel_teardown_mchbar()
195 		if (IS_I915G(dev_priv) || IS_I915GM(dev_priv)) {  in intel_teardown_mchbar()
198 			pci_read_config_dword(dev_priv->bridge_dev, DEVEN,  in intel_teardown_mchbar()
201 			pci_write_config_dword(dev_priv->bridge_dev, DEVEN,  in intel_teardown_mchbar()
206 			pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg,  in intel_teardown_mchbar()
209 			pci_write_config_dword(dev_priv->bridge_dev, mchbar_reg,  in intel_teardown_mchbar()
214 	if (dev_priv->mch_res.start)  in intel_teardown_mchbar()
215 		release_resource(&dev_priv->mch_res);  in intel_teardown_mchbar()
218 static int i915_workqueues_init(struct drm_i915_private *dev_priv)  in i915_workqueues_init()  argument
234 	dev_priv->wq = alloc_ordered_workqueue("i915", 0);  in i915_workqueues_init()
235 	if (dev_priv->wq == NULL)  in i915_workqueues_init()
238 	dev_priv->hotplug.dp_wq = alloc_ordered_workqueue("i915-dp", 0);  in i915_workqueues_init()
239 	if (dev_priv->hotplug.dp_wq == NULL)  in i915_workqueues_init()
245 	destroy_workqueue(dev_priv->wq);  in i915_workqueues_init()
247 	drm_err(&dev_priv->drm, "Failed to allocate workqueues.\n");  in i915_workqueues_init()
252 static void i915_workqueues_cleanup(struct drm_i915_private *dev_priv)  in i915_workqueues_cleanup()  argument
254 	destroy_workqueue(dev_priv->hotplug.dp_wq);  in i915_workqueues_cleanup()
255 	destroy_workqueue(dev_priv->wq);  in i915_workqueues_cleanup()
268 static void intel_detect_preproduction_hw(struct drm_i915_private *dev_priv)  in intel_detect_preproduction_hw()  argument
272 	pre |= IS_HSW_EARLY_SDV(dev_priv);  in intel_detect_preproduction_hw()
273 	pre |= IS_SKL_REVID(dev_priv, 0, SKL_REVID_F0);  in intel_detect_preproduction_hw()
274 	pre |= IS_BXT_REVID(dev_priv, 0, BXT_REVID_B_LAST);  in intel_detect_preproduction_hw()
275 	pre |= IS_KBL_GT_REVID(dev_priv, 0, KBL_REVID_A0);  in intel_detect_preproduction_hw()
276 	pre |= IS_GLK_REVID(dev_priv, 0, GLK_REVID_A2);  in intel_detect_preproduction_hw()
279 		drm_err(&dev_priv->drm, "This is a pre-production stepping. "  in intel_detect_preproduction_hw()
301 static int i915_driver_early_probe(struct drm_i915_private *dev_priv)  in i915_driver_early_probe()  argument
305 	if (i915_inject_probe_failure(dev_priv))  in i915_driver_early_probe()
308 	intel_device_info_subplatform_init(dev_priv);  in i915_driver_early_probe()
310 	intel_uncore_mmio_debug_init_early(&dev_priv->mmio_debug);  in i915_driver_early_probe()
311 	intel_uncore_init_early(&dev_priv->uncore, dev_priv);  in i915_driver_early_probe()
313 	spin_lock_init(&dev_priv->irq_lock);  in i915_driver_early_probe()
314 	spin_lock_init(&dev_priv->gpu_error.lock);  in i915_driver_early_probe()
315 	mutex_init(&dev_priv->backlight_lock);  in i915_driver_early_probe()
317 	mutex_init(&dev_priv->sb_lock);  in i915_driver_early_probe()
318 	cpu_latency_qos_add_request(&dev_priv->sb_qos, PM_QOS_DEFAULT_VALUE);  in i915_driver_early_probe()
320 	mutex_init(&dev_priv->av_mutex);  in i915_driver_early_probe()
321 	mutex_init(&dev_priv->wm.wm_mutex);  in i915_driver_early_probe()
322 	mutex_init(&dev_priv->pps_mutex);  in i915_driver_early_probe()
323 	mutex_init(&dev_priv->hdcp_comp_mutex);  in i915_driver_early_probe()
325 	i915_memcpy_init_early(dev_priv);  in i915_driver_early_probe()
326 	intel_runtime_pm_init_early(&dev_priv->runtime_pm);  in i915_driver_early_probe()
328 	ret = i915_workqueues_init(dev_priv);  in i915_driver_early_probe()
332 	ret = vlv_suspend_init(dev_priv);  in i915_driver_early_probe()
336 	intel_wopcm_init_early(&dev_priv->wopcm);  in i915_driver_early_probe()
338 	intel_gt_init_early(&dev_priv->gt, dev_priv);  in i915_driver_early_probe()
340 	i915_gem_init_early(dev_priv);  in i915_driver_early_probe()
343 	intel_detect_pch(dev_priv);  in i915_driver_early_probe()
345 	intel_pm_setup(dev_priv);  in i915_driver_early_probe()
346 	ret = intel_power_domains_init(dev_priv);  in i915_driver_early_probe()
349 	intel_irq_init(dev_priv);  in i915_driver_early_probe()
350 	intel_init_display_hooks(dev_priv);  in i915_driver_early_probe()
351 	intel_init_clock_gating_hooks(dev_priv);  in i915_driver_early_probe()
352 	intel_init_audio_hooks(dev_priv);  in i915_driver_early_probe()
354 	intel_detect_preproduction_hw(dev_priv);  in i915_driver_early_probe()
359 	i915_gem_cleanup_early(dev_priv);  in i915_driver_early_probe()
360 	intel_gt_driver_late_release(&dev_priv->gt);  in i915_driver_early_probe()
361 	vlv_suspend_cleanup(dev_priv);  in i915_driver_early_probe()
363 	i915_workqueues_cleanup(dev_priv);  in i915_driver_early_probe()
372 static void i915_driver_late_release(struct drm_i915_private *dev_priv)  in i915_driver_late_release()  argument
374 	intel_irq_fini(dev_priv);  in i915_driver_late_release()
375 	intel_power_domains_cleanup(dev_priv);  in i915_driver_late_release()
376 	i915_gem_cleanup_early(dev_priv);  in i915_driver_late_release()
377 	intel_gt_driver_late_release(&dev_priv->gt);  in i915_driver_late_release()
378 	vlv_suspend_cleanup(dev_priv);  in i915_driver_late_release()
379 	i915_workqueues_cleanup(dev_priv);  in i915_driver_late_release()
381 	cpu_latency_qos_remove_request(&dev_priv->sb_qos);  in i915_driver_late_release()
382 	mutex_destroy(&dev_priv->sb_lock);  in i915_driver_late_release()
384 	i915_params_free(&dev_priv->params);  in i915_driver_late_release()
396 static int i915_driver_mmio_probe(struct drm_i915_private *dev_priv)  in i915_driver_mmio_probe()  argument
400 	if (i915_inject_probe_failure(dev_priv))  in i915_driver_mmio_probe()
403 	if (i915_get_bridge_dev(dev_priv))  in i915_driver_mmio_probe()
406 	ret = intel_uncore_init_mmio(&dev_priv->uncore);  in i915_driver_mmio_probe()
411 	intel_setup_mchbar(dev_priv);  in i915_driver_mmio_probe()
413 	ret = intel_gt_init_mmio(&dev_priv->gt);  in i915_driver_mmio_probe()
418 	sanitize_gpu(dev_priv);  in i915_driver_mmio_probe()
423 	intel_teardown_mchbar(dev_priv);  in i915_driver_mmio_probe()
424 	intel_uncore_fini_mmio(&dev_priv->uncore);  in i915_driver_mmio_probe()
426 	pci_dev_put(dev_priv->bridge_dev);  in i915_driver_mmio_probe()
435 static void i915_driver_mmio_release(struct drm_i915_private *dev_priv)  in i915_driver_mmio_release()  argument
437 	intel_teardown_mchbar(dev_priv);  in i915_driver_mmio_release()
438 	intel_uncore_fini_mmio(&dev_priv->uncore);  in i915_driver_mmio_release()
439 	pci_dev_put(dev_priv->bridge_dev);  in i915_driver_mmio_release()
442 static void intel_sanitize_options(struct drm_i915_private *dev_priv)  in intel_sanitize_options()  argument
444 	intel_gvt_sanitize_options(dev_priv);  in intel_sanitize_options()
510 static int i915_driver_hw_probe(struct drm_i915_private *dev_priv)  in i915_driver_hw_probe()  argument
512 	struct pci_dev *pdev = dev_priv->drm.pdev;  in i915_driver_hw_probe()
515 	if (i915_inject_probe_failure(dev_priv))  in i915_driver_hw_probe()
518 	intel_device_info_runtime_init(dev_priv);  in i915_driver_hw_probe()
520 	if (HAS_PPGTT(dev_priv)) {  in i915_driver_hw_probe()
521 		if (intel_vgpu_active(dev_priv) &&  in i915_driver_hw_probe()
522 		    !intel_vgpu_has_full_ppgtt(dev_priv)) {  in i915_driver_hw_probe()
523 			i915_report_error(dev_priv,  in i915_driver_hw_probe()
529 	if (HAS_EXECLISTS(dev_priv)) {  in i915_driver_hw_probe()
535 		if (intel_vgpu_active(dev_priv) &&  in i915_driver_hw_probe()
536 		    !intel_vgpu_has_hwsp_emulation(dev_priv)) {  in i915_driver_hw_probe()
537 			i915_report_error(dev_priv,  in i915_driver_hw_probe()
543 	intel_sanitize_options(dev_priv);  in i915_driver_hw_probe()
546 	intel_dram_edram_detect(dev_priv);  in i915_driver_hw_probe()
548 	ret = i915_set_dma_info(dev_priv);  in i915_driver_hw_probe()
552 	i915_perf_init(dev_priv);  in i915_driver_hw_probe()
554 	ret = i915_ggtt_probe_hw(dev_priv);  in i915_driver_hw_probe()
562 	ret = i915_ggtt_init_hw(dev_priv);  in i915_driver_hw_probe()
566 	ret = intel_memory_regions_hw_probe(dev_priv);  in i915_driver_hw_probe()
570 	intel_gt_init_hw_early(&dev_priv->gt, &dev_priv->ggtt);  in i915_driver_hw_probe()
572 	ret = i915_ggtt_enable_hw(dev_priv);  in i915_driver_hw_probe()
574 		drm_err(&dev_priv->drm, "failed to enable GGTT\n");  in i915_driver_hw_probe()
580 	cpu_latency_qos_add_request(&dev_priv->pm_qos, PM_QOS_DEFAULT_VALUE);  in i915_driver_hw_probe()
582 	intel_gt_init_workarounds(dev_priv);  in i915_driver_hw_probe()
603 	if (INTEL_GEN(dev_priv) >= 5) {  in i915_driver_hw_probe()
605 			drm_dbg(&dev_priv->drm, "can't enable MSI");  in i915_driver_hw_probe()
608 	ret = intel_gvt_init(dev_priv);  in i915_driver_hw_probe()
612 	intel_opregion_setup(dev_priv);  in i915_driver_hw_probe()
617 	intel_dram_detect(dev_priv);  in i915_driver_hw_probe()
619 	intel_bw_init_hw(dev_priv);  in i915_driver_hw_probe()
626 	cpu_latency_qos_remove_request(&dev_priv->pm_qos);  in i915_driver_hw_probe()
628 	intel_memory_regions_driver_release(dev_priv);  in i915_driver_hw_probe()
630 	i915_ggtt_driver_release(dev_priv);  in i915_driver_hw_probe()
632 	i915_perf_fini(dev_priv);  in i915_driver_hw_probe()
640 static void i915_driver_hw_remove(struct drm_i915_private *dev_priv)  in i915_driver_hw_remove()  argument
642 	struct pci_dev *pdev = dev_priv->drm.pdev;  in i915_driver_hw_remove()
644 	i915_perf_fini(dev_priv);  in i915_driver_hw_remove()
649 	cpu_latency_qos_remove_request(&dev_priv->pm_qos);  in i915_driver_hw_remove()
659 static void i915_driver_register(struct drm_i915_private *dev_priv)  in i915_driver_register()  argument
661 	struct drm_device *dev = &dev_priv->drm;  in i915_driver_register()
663 	i915_gem_driver_register(dev_priv);  in i915_driver_register()
664 	i915_pmu_register(dev_priv);  in i915_driver_register()
666 	intel_vgpu_register(dev_priv);  in i915_driver_register()
670 		i915_debugfs_register(dev_priv);  in i915_driver_register()
671 		intel_display_debugfs_register(dev_priv);  in i915_driver_register()
672 		i915_setup_sysfs(dev_priv);  in i915_driver_register()
675 		i915_perf_register(dev_priv);  in i915_driver_register()
677 		drm_err(&dev_priv->drm,  in i915_driver_register()
680 	if (HAS_DISPLAY(dev_priv)) {  in i915_driver_register()
682 		intel_opregion_register(dev_priv);  in i915_driver_register()
686 	intel_gt_driver_register(&dev_priv->gt);  in i915_driver_register()
688 	intel_audio_init(dev_priv);  in i915_driver_register()
703 	if (HAS_DISPLAY(dev_priv))  in i915_driver_register()
706 	intel_power_domains_enable(dev_priv);  in i915_driver_register()
707 	intel_runtime_pm_enable(&dev_priv->runtime_pm);  in i915_driver_register()
711 	if (i915_switcheroo_register(dev_priv))  in i915_driver_register()
712 		drm_err(&dev_priv->drm, "Failed to register vga switcheroo!\n");  in i915_driver_register()
719 static void i915_driver_unregister(struct drm_i915_private *dev_priv)  in i915_driver_unregister()  argument
721 	i915_switcheroo_unregister(dev_priv);  in i915_driver_unregister()
725 	intel_runtime_pm_disable(&dev_priv->runtime_pm);  in i915_driver_unregister()
726 	intel_power_domains_disable(dev_priv);  in i915_driver_unregister()
728 	intel_fbdev_unregister(dev_priv);  in i915_driver_unregister()
729 	intel_audio_deinit(dev_priv);  in i915_driver_unregister()
736 	drm_kms_helper_poll_fini(&dev_priv->drm);  in i915_driver_unregister()
738 	intel_gt_driver_unregister(&dev_priv->gt);  in i915_driver_unregister()
740 	intel_opregion_unregister(dev_priv);  in i915_driver_unregister()
742 	i915_perf_unregister(dev_priv);  in i915_driver_unregister()
743 	i915_pmu_unregister(dev_priv);  in i915_driver_unregister()
745 	i915_teardown_sysfs(dev_priv);  in i915_driver_unregister()
746 	drm_dev_unplug(&dev_priv->drm);  in i915_driver_unregister()
748 	i915_gem_driver_unregister(dev_priv);  in i915_driver_unregister()
751 static void i915_welcome_messages(struct drm_i915_private *dev_priv)  in i915_welcome_messages()  argument
757 			   INTEL_DEVID(dev_priv),  in i915_welcome_messages()
758 			   INTEL_REVID(dev_priv),  in i915_welcome_messages()
759 			   intel_platform_name(INTEL_INFO(dev_priv)->platform),  in i915_welcome_messages()
760 			   intel_subplatform(RUNTIME_INFO(dev_priv),  in i915_welcome_messages()
761 					     INTEL_INFO(dev_priv)->platform),  in i915_welcome_messages()
762 			   INTEL_GEN(dev_priv));  in i915_welcome_messages()
764 		intel_device_info_print_static(INTEL_INFO(dev_priv), &p);  in i915_welcome_messages()
765 		intel_device_info_print_runtime(RUNTIME_INFO(dev_priv), &p);  in i915_welcome_messages()
766 		intel_gt_info_print(&dev_priv->gt.info, &p);  in i915_welcome_messages()
770 		drm_info(&dev_priv->drm, "DRM_I915_DEBUG enabled\n");  in i915_welcome_messages()
772 		drm_info(&dev_priv->drm, "DRM_I915_DEBUG_GEM enabled\n");  in i915_welcome_messages()
774 		drm_info(&dev_priv->drm,  in i915_welcome_messages()
963 	struct drm_i915_private *dev_priv = to_i915(dev);  in i915_driver_release()  local
964 	struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;  in i915_driver_release()
966 	if (!dev_priv->do_release)  in i915_driver_release()
971 	i915_gem_driver_release(dev_priv);  in i915_driver_release()
973 	intel_memory_regions_driver_release(dev_priv);  in i915_driver_release()
974 	i915_ggtt_driver_release(dev_priv);  in i915_driver_release()
975 	i915_gem_drain_freed_objects(dev_priv);  in i915_driver_release()
977 	i915_driver_mmio_release(dev_priv);  in i915_driver_release()
982 	i915_driver_late_release(dev_priv);  in i915_driver_release()
1027 static void intel_suspend_encoders(struct drm_i915_private *dev_priv)  in intel_suspend_encoders()  argument
1029 	struct drm_device *dev = &dev_priv->drm;  in intel_suspend_encoders()
1039 static bool suspend_to_idle(struct drm_i915_private *dev_priv)  in suspend_to_idle()  argument
1065 	struct drm_i915_private *dev_priv = to_i915(dev);  in i915_drm_suspend()  local
1066 	struct pci_dev *pdev = dev_priv->drm.pdev;  in i915_drm_suspend()
1069 	disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);  in i915_drm_suspend()
1073 	intel_power_domains_disable(dev_priv);  in i915_drm_suspend()
1081 	intel_dp_mst_suspend(dev_priv);  in i915_drm_suspend()
1083 	intel_runtime_pm_disable_interrupts(dev_priv);  in i915_drm_suspend()
1084 	intel_hpd_cancel_work(dev_priv);  in i915_drm_suspend()
1086 	intel_suspend_encoders(dev_priv);  in i915_drm_suspend()
1088 	intel_suspend_hw(dev_priv);  in i915_drm_suspend()
1090 	i915_ggtt_suspend(&dev_priv->ggtt);  in i915_drm_suspend()
1092 	i915_save_state(dev_priv);  in i915_drm_suspend()
1094 	opregion_target_state = suspend_to_idle(dev_priv) ? PCI_D1 : PCI_D3cold;  in i915_drm_suspend()
1095 	intel_opregion_suspend(dev_priv, opregion_target_state);  in i915_drm_suspend()
1099 	dev_priv->suspend_count++;  in i915_drm_suspend()
1101 	intel_csr_ucode_suspend(dev_priv);  in i915_drm_suspend()
1103 	enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);  in i915_drm_suspend()
1109 get_suspend_mode(struct drm_i915_private *dev_priv, bool hibernate)  in get_suspend_mode()  argument
1114 	if (suspend_to_idle(dev_priv))  in get_suspend_mode()
1122 	struct drm_i915_private *dev_priv = to_i915(dev);  in i915_drm_suspend_late()  local
1123 	struct pci_dev *pdev = dev_priv->drm.pdev;  in i915_drm_suspend_late()
1124 	struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;  in i915_drm_suspend_late()
1129 	i915_gem_suspend_late(dev_priv);  in i915_drm_suspend_late()
1131 	intel_uncore_suspend(&dev_priv->uncore);  in i915_drm_suspend_late()
1133 	intel_power_domains_suspend(dev_priv,  in i915_drm_suspend_late()
1134 				    get_suspend_mode(dev_priv, hibernation));  in i915_drm_suspend_late()
1136 	intel_display_power_suspend_late(dev_priv);  in i915_drm_suspend_late()
1138 	ret = vlv_suspend_complete(dev_priv);  in i915_drm_suspend_late()
1140 		drm_err(&dev_priv->drm, "Suspend complete failed: %d\n", ret);  in i915_drm_suspend_late()
1141 		intel_power_domains_resume(dev_priv);  in i915_drm_suspend_late()
1159 	if (!(hibernation && INTEL_GEN(dev_priv) < 6))  in i915_drm_suspend_late()
1164 	if (!dev_priv->uncore.user_forcewake_count)  in i915_drm_suspend_late()
1190 	struct drm_i915_private *dev_priv = to_i915(dev);  in i915_drm_resume()  local
1193 	disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);  in i915_drm_resume()
1195 	sanitize_gpu(dev_priv);  in i915_drm_resume()
1197 	ret = i915_ggtt_enable_hw(dev_priv);  in i915_drm_resume()
1199 		drm_err(&dev_priv->drm, "failed to re-enable GGTT\n");  in i915_drm_resume()
1201 	i915_ggtt_resume(&dev_priv->ggtt);  in i915_drm_resume()
1203 	intel_csr_ucode_resume(dev_priv);  in i915_drm_resume()
1205 	i915_restore_state(dev_priv);  in i915_drm_resume()
1206 	intel_pps_unlock_regs_wa(dev_priv);  in i915_drm_resume()
1208 	intel_init_pch_refclk(dev_priv);  in i915_drm_resume()
1220 	intel_runtime_pm_enable_interrupts(dev_priv);  in i915_drm_resume()
1224 	i915_gem_resume(dev_priv);  in i915_drm_resume()
1226 	intel_modeset_init_hw(dev_priv);  in i915_drm_resume()
1227 	intel_init_clock_gating(dev_priv);  in i915_drm_resume()
1229 	spin_lock_irq(&dev_priv->irq_lock);  in i915_drm_resume()
1230 	if (dev_priv->display.hpd_irq_setup)  in i915_drm_resume()
1231 		dev_priv->display.hpd_irq_setup(dev_priv);  in i915_drm_resume()
1232 	spin_unlock_irq(&dev_priv->irq_lock);  in i915_drm_resume()
1234 	intel_dp_mst_resume(dev_priv);  in i915_drm_resume()
1246 	intel_hpd_init(dev_priv);  in i915_drm_resume()
1248 	intel_opregion_resume(dev_priv);  in i915_drm_resume()
1252 	intel_power_domains_enable(dev_priv);  in i915_drm_resume()
1254 	enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);  in i915_drm_resume()
1261 	struct drm_i915_private *dev_priv = to_i915(dev);  in i915_drm_resume_early()  local
1262 	struct pci_dev *pdev = dev_priv->drm.pdev;  in i915_drm_resume_early()
1287 		drm_err(&dev_priv->drm,  in i915_drm_resume_early()
1310 	disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);  in i915_drm_resume_early()
1312 	ret = vlv_resume_prepare(dev_priv, false);  in i915_drm_resume_early()
1314 		drm_err(&dev_priv->drm,  in i915_drm_resume_early()
1317 	intel_uncore_resume_early(&dev_priv->uncore);  in i915_drm_resume_early()
1319 	intel_gt_check_and_clear_faults(&dev_priv->gt);  in i915_drm_resume_early()
1321 	intel_display_power_resume_early(dev_priv);  in i915_drm_resume_early()
1323 	intel_power_domains_resume(dev_priv);  in i915_drm_resume_early()
1325 	enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);  in i915_drm_resume_early()
1484 	struct drm_i915_private *dev_priv = kdev_to_i915(kdev);  in intel_runtime_suspend()  local
1485 	struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;  in intel_runtime_suspend()
1488 	if (drm_WARN_ON_ONCE(&dev_priv->drm, !HAS_RUNTIME_PM(dev_priv)))  in intel_runtime_suspend()
1491 	drm_dbg_kms(&dev_priv->drm, "Suspending device\n");  in intel_runtime_suspend()
1499 	i915_gem_runtime_suspend(dev_priv);  in intel_runtime_suspend()
1501 	intel_gt_runtime_suspend(&dev_priv->gt);  in intel_runtime_suspend()
1503 	intel_runtime_pm_disable_interrupts(dev_priv);  in intel_runtime_suspend()
1505 	intel_uncore_suspend(&dev_priv->uncore);  in intel_runtime_suspend()
1507 	intel_display_power_suspend(dev_priv);  in intel_runtime_suspend()
1509 	ret = vlv_suspend_complete(dev_priv);  in intel_runtime_suspend()
1511 		drm_err(&dev_priv->drm,  in intel_runtime_suspend()
1513 		intel_uncore_runtime_resume(&dev_priv->uncore);  in intel_runtime_suspend()
1515 		intel_runtime_pm_enable_interrupts(dev_priv);  in intel_runtime_suspend()
1517 		intel_gt_runtime_resume(&dev_priv->gt);  in intel_runtime_suspend()
1527 	if (intel_uncore_arm_unclaimed_mmio_detection(&dev_priv->uncore))  in intel_runtime_suspend()
1528 		drm_err(&dev_priv->drm,  in intel_runtime_suspend()
1537 	if (IS_BROADWELL(dev_priv)) {  in intel_runtime_suspend()
1544 		intel_opregion_notify_adapter(dev_priv, PCI_D3hot);  in intel_runtime_suspend()
1553 		intel_opregion_notify_adapter(dev_priv, PCI_D1);  in intel_runtime_suspend()
1556 	assert_forcewakes_inactive(&dev_priv->uncore);  in intel_runtime_suspend()
1558 	if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))  in intel_runtime_suspend()
1559 		intel_hpd_poll_init(dev_priv);  in intel_runtime_suspend()
1561 	drm_dbg_kms(&dev_priv->drm, "Device suspended\n");  in intel_runtime_suspend()
1567 	struct drm_i915_private *dev_priv = kdev_to_i915(kdev);  in intel_runtime_resume()  local
1568 	struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;  in intel_runtime_resume()
1571 	if (drm_WARN_ON_ONCE(&dev_priv->drm, !HAS_RUNTIME_PM(dev_priv)))  in intel_runtime_resume()
1574 	drm_dbg_kms(&dev_priv->drm, "Resuming device\n");  in intel_runtime_resume()
1576 	drm_WARN_ON_ONCE(&dev_priv->drm, atomic_read(&rpm->wakeref_count));  in intel_runtime_resume()
1579 	intel_opregion_notify_adapter(dev_priv, PCI_D0);  in intel_runtime_resume()
1581 	if (intel_uncore_unclaimed_mmio(&dev_priv->uncore))  in intel_runtime_resume()
1582 		drm_dbg(&dev_priv->drm,  in intel_runtime_resume()
1585 	intel_display_power_resume(dev_priv);  in intel_runtime_resume()
1587 	ret = vlv_resume_prepare(dev_priv, true);  in intel_runtime_resume()
1589 	intel_uncore_runtime_resume(&dev_priv->uncore);  in intel_runtime_resume()
1591 	intel_runtime_pm_enable_interrupts(dev_priv);  in intel_runtime_resume()
1597 	intel_gt_runtime_resume(&dev_priv->gt);  in intel_runtime_resume()
1604 	if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))  in intel_runtime_resume()
1605 		intel_hpd_init(dev_priv);  in intel_runtime_resume()
1607 	intel_enable_ipc(dev_priv);  in intel_runtime_resume()
1612 		drm_err(&dev_priv->drm,  in intel_runtime_resume()
1615 		drm_dbg_kms(&dev_priv->drm, "Device resumed\n");  in intel_runtime_resume()