75 static bool intel_psr2_enabled(struct drm_i915_private *dev_priv,  in intel_psr2_enabled()  argument
82 	switch (dev_priv->psr.debug & I915_PSR_DEBUG_MODE_MASK) {  in intel_psr2_enabled()
108 void intel_psr_irq_control(struct drm_i915_private *dev_priv, u32 debug)  in intel_psr_irq_control()  argument
114 	if (INTEL_GEN(dev_priv) >= 8)  in intel_psr_irq_control()
121 	for_each_cpu_transcoder_masked(dev_priv, cpu_transcoder, transcoders) {  in intel_psr_irq_control()
172 void intel_psr_irq_handler(struct drm_i915_private *dev_priv, u32 psr_iir)  in intel_psr_irq_handler()  argument
179 	if (INTEL_GEN(dev_priv) >= 8)  in intel_psr_irq_handler()
184 	for_each_cpu_transcoder_masked(dev_priv, cpu_transcoder, transcoders) {  in intel_psr_irq_handler()
191 			dev_priv->psr.irq_aux_error = true;  in intel_psr_irq_handler()
205 			dev_priv->psr.last_entry_attempt = time_ns;  in intel_psr_irq_handler()
211 			dev_priv->psr.last_exit = time_ns;  in intel_psr_irq_handler()
215 			if (INTEL_GEN(dev_priv) >= 9) {  in intel_psr_irq_handler()
217 				bool psr2_enabled = dev_priv->psr.psr2_enabled;  in intel_psr_irq_handler()
229 		schedule_work(&dev_priv->psr.work);  in intel_psr_irq_handler()
283 	struct drm_i915_private *dev_priv =  in intel_psr_init_dpcd()  local
304 	dev_priv->psr.sink_support = true;  in intel_psr_init_dpcd()
305 	dev_priv->psr.sink_sync_latency =  in intel_psr_init_dpcd()
308 	WARN_ON(dev_priv->psr.dp);  in intel_psr_init_dpcd()
309 	dev_priv->psr.dp = intel_dp;  in intel_psr_init_dpcd()
311 	if (INTEL_GEN(dev_priv) >= 9 &&  in intel_psr_init_dpcd()
328 		dev_priv->psr.sink_psr2_support = y_req && alpm;  in intel_psr_init_dpcd()
330 			      dev_priv->psr.sink_psr2_support ? "" : "not ");  in intel_psr_init_dpcd()
332 		if (dev_priv->psr.sink_psr2_support) {  in intel_psr_init_dpcd()
333 			dev_priv->psr.colorimetry_support =  in intel_psr_init_dpcd()
335 			dev_priv->psr.su_x_granularity =  in intel_psr_init_dpcd()
345 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in intel_psr_setup_vsc()  local
348 	if (dev_priv->psr.psr2_enabled) {  in intel_psr_setup_vsc()
353 		if (dev_priv->psr.colorimetry_support) {  in intel_psr_setup_vsc()
376 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in hsw_psr_setup_aux()  local
409 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in intel_psr_enable_sink()  local
413 	if (dev_priv->psr.psr2_enabled) {  in intel_psr_enable_sink()
418 		if (dev_priv->psr.link_standby)  in intel_psr_enable_sink()
421 		if (INTEL_GEN(dev_priv) >= 8)  in intel_psr_enable_sink()
432 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in intel_psr1_get_tp_time()  local
435 	if (INTEL_GEN(dev_priv) >= 11)  in intel_psr1_get_tp_time()
438 	if (dev_priv->vbt.psr.tp1_wakeup_time_us == 0)  in intel_psr1_get_tp_time()
440 	else if (dev_priv->vbt.psr.tp1_wakeup_time_us <= 100)  in intel_psr1_get_tp_time()
442 	else if (dev_priv->vbt.psr.tp1_wakeup_time_us <= 500)  in intel_psr1_get_tp_time()
447 	if (dev_priv->vbt.psr.tp2_tp3_wakeup_time_us == 0)  in intel_psr1_get_tp_time()
449 	else if (dev_priv->vbt.psr.tp2_tp3_wakeup_time_us <= 100)  in intel_psr1_get_tp_time()
451 	else if (dev_priv->vbt.psr.tp2_tp3_wakeup_time_us <= 500)  in intel_psr1_get_tp_time()
467 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in hsw_activate_psr1()  local
474 	int idle_frames = max(6, dev_priv->vbt.psr.idle_frames);  in hsw_activate_psr1()
479 	idle_frames = max(idle_frames, dev_priv->psr.sink_sync_latency + 1);  in hsw_activate_psr1()
483 	if (IS_HASWELL(dev_priv))  in hsw_activate_psr1()
486 	if (dev_priv->psr.link_standby)  in hsw_activate_psr1()
491 	if (INTEL_GEN(dev_priv) >= 8)  in hsw_activate_psr1()
500 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in hsw_activate_psr2()  local
506 	int idle_frames = max(6, dev_priv->vbt.psr.idle_frames);  in hsw_activate_psr2()
508 	idle_frames = max(idle_frames, dev_priv->psr.sink_sync_latency + 1);  in hsw_activate_psr2()
512 	if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))  in hsw_activate_psr2()
515 	val |= EDP_PSR2_FRAME_BEFORE_SU(dev_priv->psr.sink_sync_latency + 1);  in hsw_activate_psr2()
517 	if (dev_priv->vbt.psr.psr2_tp2_tp3_wakeup_time_us >= 0 &&  in hsw_activate_psr2()
518 	    dev_priv->vbt.psr.psr2_tp2_tp3_wakeup_time_us <= 50)  in hsw_activate_psr2()
520 	else if (dev_priv->vbt.psr.psr2_tp2_tp3_wakeup_time_us <= 100)  in hsw_activate_psr2()
522 	else if (dev_priv->vbt.psr.psr2_tp2_tp3_wakeup_time_us <= 500)  in hsw_activate_psr2()
539 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in intel_psr2_config_valid()  local
544 	if (!dev_priv->psr.sink_psr2_support)  in intel_psr2_config_valid()
557 	if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) {  in intel_psr2_config_valid()
560 	} else if (IS_GEN(dev_priv, 9)) {  in intel_psr2_config_valid()
578 	if (crtc_hdisplay % dev_priv->psr.su_x_granularity) {  in intel_psr2_config_valid()
580 			      crtc_hdisplay, dev_priv->psr.su_x_granularity);  in intel_psr2_config_valid()
596 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in intel_psr_compute_config()  local
601 	if (!CAN_PSR(dev_priv))  in intel_psr_compute_config()
604 	if (intel_dp != dev_priv->psr.dp)  in intel_psr_compute_config()
619 	if (dev_priv->psr.sink_not_reliable) {  in intel_psr_compute_config()
649 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in intel_psr_activate()  local
651 	if (INTEL_GEN(dev_priv) >= 9)  in intel_psr_activate()
654 	WARN_ON(dev_priv->psr.active);  in intel_psr_activate()
655 	lockdep_assert_held(&dev_priv->psr.lock);  in intel_psr_activate()
658 	if (dev_priv->psr.psr2_enabled)  in intel_psr_activate()
663 	dev_priv->psr.active = true;  in intel_psr_activate()
666 static i915_reg_t gen9_chicken_trans_reg(struct drm_i915_private *dev_priv,  in gen9_chicken_trans_reg()  argument
676 	WARN_ON(INTEL_GEN(dev_priv) < 9);  in gen9_chicken_trans_reg()
688 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in intel_psr_enable_source()  local
695 	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))  in intel_psr_enable_source()
698 	if (dev_priv->psr.psr2_enabled && (IS_GEN(dev_priv, 9) &&  in intel_psr_enable_source()
699 					   !IS_GEMINILAKE(dev_priv))) {  in intel_psr_enable_source()
700 		i915_reg_t reg = gen9_chicken_trans_reg(dev_priv,  in intel_psr_enable_source()
720 	if (INTEL_GEN(dev_priv) < 11)  in intel_psr_enable_source()
726 static void intel_psr_enable_locked(struct drm_i915_private *dev_priv,  in intel_psr_enable_locked()  argument
729 	struct intel_dp *intel_dp = dev_priv->psr.dp;  in intel_psr_enable_locked()
731 	WARN_ON(dev_priv->psr.enabled);  in intel_psr_enable_locked()
733 	dev_priv->psr.psr2_enabled = intel_psr2_enabled(dev_priv, crtc_state);  in intel_psr_enable_locked()
734 	dev_priv->psr.busy_frontbuffer_bits = 0;  in intel_psr_enable_locked()
735 	dev_priv->psr.pipe = to_intel_crtc(crtc_state->base.crtc)->pipe;  in intel_psr_enable_locked()
738 		      dev_priv->psr.psr2_enabled ? "2" : "1");  in intel_psr_enable_locked()
742 	dev_priv->psr.enabled = true;  in intel_psr_enable_locked()
757 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in intel_psr_enable()  local
762 	if (WARN_ON(!CAN_PSR(dev_priv)))  in intel_psr_enable()
765 	WARN_ON(dev_priv->drrs.dp);  in intel_psr_enable()
767 	mutex_lock(&dev_priv->psr.lock);  in intel_psr_enable()
769 	if (!psr_global_enabled(dev_priv->psr.debug)) {  in intel_psr_enable()
774 	intel_psr_enable_locked(dev_priv, crtc_state);  in intel_psr_enable()
777 	mutex_unlock(&dev_priv->psr.lock);  in intel_psr_enable()
780 static void intel_psr_exit(struct drm_i915_private *dev_priv)  in intel_psr_exit()  argument
784 	if (!dev_priv->psr.active) {  in intel_psr_exit()
785 		if (INTEL_GEN(dev_priv) >= 9)  in intel_psr_exit()
791 	if (dev_priv->psr.psr2_enabled) {  in intel_psr_exit()
800 	dev_priv->psr.active = false;  in intel_psr_exit()
805 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in intel_psr_disable_locked()  local
809 	lockdep_assert_held(&dev_priv->psr.lock);  in intel_psr_disable_locked()
811 	if (!dev_priv->psr.enabled)  in intel_psr_disable_locked()
815 		      dev_priv->psr.psr2_enabled ? "2" : "1");  in intel_psr_disable_locked()
817 	intel_psr_exit(dev_priv);  in intel_psr_disable_locked()
819 	if (dev_priv->psr.psr2_enabled) {  in intel_psr_disable_locked()
828 	if (intel_de_wait_for_clear(dev_priv, psr_status,  in intel_psr_disable_locked()
835 	dev_priv->psr.enabled = false;  in intel_psr_disable_locked()
848 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in intel_psr_disable()  local
853 	if (WARN_ON(!CAN_PSR(dev_priv)))  in intel_psr_disable()
856 	mutex_lock(&dev_priv->psr.lock);  in intel_psr_disable()
860 	mutex_unlock(&dev_priv->psr.lock);  in intel_psr_disable()
861 	cancel_work_sync(&dev_priv->psr.work);  in intel_psr_disable()
864 static void psr_force_hw_tracking_exit(struct drm_i915_private *dev_priv)  in psr_force_hw_tracking_exit()  argument
866 	if (INTEL_GEN(dev_priv) >= 9)  in psr_force_hw_tracking_exit()
876 		I915_WRITE(CURSURFLIVE(dev_priv->psr.pipe), 0);  in psr_force_hw_tracking_exit()
882 		intel_psr_exit(dev_priv);  in psr_force_hw_tracking_exit()
897 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in intel_psr_update()  local
898 	struct i915_psr *psr = &dev_priv->psr;  in intel_psr_update()
901 	if (!CAN_PSR(dev_priv) || READ_ONCE(psr->dp) != intel_dp)  in intel_psr_update()
904 	mutex_lock(&dev_priv->psr.lock);  in intel_psr_update()
907 	psr2_enable = intel_psr2_enabled(dev_priv, crtc_state);  in intel_psr_update()
912 			psr_force_hw_tracking_exit(dev_priv);  in intel_psr_update()
913 		else if (INTEL_GEN(dev_priv) < 9 && psr->enabled) {  in intel_psr_update()
918 			if (!dev_priv->psr.active &&  in intel_psr_update()
919 			    !dev_priv->psr.busy_frontbuffer_bits)  in intel_psr_update()
920 				schedule_work(&dev_priv->psr.work);  in intel_psr_update()
930 		intel_psr_enable_locked(dev_priv, crtc_state);  in intel_psr_update()
933 	mutex_unlock(&dev_priv->psr.lock);  in intel_psr_update()
950 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);  in intel_psr_wait_for_idle()  local
952 	if (!dev_priv->psr.enabled || !new_crtc_state->has_psr)  in intel_psr_wait_for_idle()
956 	if (READ_ONCE(dev_priv->psr.psr2_enabled))  in intel_psr_wait_for_idle()
966 	return __intel_wait_for_register(&dev_priv->uncore, EDP_PSR_STATUS,  in intel_psr_wait_for_idle()
972 static bool __psr_wait_for_idle_locked(struct drm_i915_private *dev_priv)  in __psr_wait_for_idle_locked()  argument
978 	if (!dev_priv->psr.enabled)  in __psr_wait_for_idle_locked()
981 	if (dev_priv->psr.psr2_enabled) {  in __psr_wait_for_idle_locked()
989 	mutex_unlock(&dev_priv->psr.lock);  in __psr_wait_for_idle_locked()
991 	err = intel_de_wait_for_clear(dev_priv, reg, mask, 50);  in __psr_wait_for_idle_locked()
996 	mutex_lock(&dev_priv->psr.lock);  in __psr_wait_for_idle_locked()
997 	return err == 0 && dev_priv->psr.enabled;  in __psr_wait_for_idle_locked()
1000 static int intel_psr_fastset_force(struct drm_i915_private *dev_priv)  in intel_psr_fastset_force()  argument
1002 	struct drm_device *dev = &dev_priv->drm;  in intel_psr_fastset_force()
1052 int intel_psr_debug_set(struct drm_i915_private *dev_priv, u64 val)  in intel_psr_debug_set()  argument
1064 	ret = mutex_lock_interruptible(&dev_priv->psr.lock);  in intel_psr_debug_set()
1068 	old_mode = dev_priv->psr.debug & I915_PSR_DEBUG_MODE_MASK;  in intel_psr_debug_set()
1069 	dev_priv->psr.debug = val;  in intel_psr_debug_set()
1070 	intel_psr_irq_control(dev_priv, dev_priv->psr.debug);  in intel_psr_debug_set()
1072 	mutex_unlock(&dev_priv->psr.lock);  in intel_psr_debug_set()
1075 		ret = intel_psr_fastset_force(dev_priv);  in intel_psr_debug_set()
1080 static void intel_psr_handle_irq(struct drm_i915_private *dev_priv)  in intel_psr_handle_irq()  argument
1082 	struct i915_psr *psr = &dev_priv->psr;  in intel_psr_handle_irq()
1092 	struct drm_i915_private *dev_priv =  in intel_psr_work()  local
1093 		container_of(work, typeof(*dev_priv), psr.work);  in intel_psr_work()
1095 	mutex_lock(&dev_priv->psr.lock);  in intel_psr_work()
1097 	if (!dev_priv->psr.enabled)  in intel_psr_work()
1100 	if (READ_ONCE(dev_priv->psr.irq_aux_error))  in intel_psr_work()
1101 		intel_psr_handle_irq(dev_priv);  in intel_psr_work()
1109 	if (!__psr_wait_for_idle_locked(dev_priv))  in intel_psr_work()
1117 	if (dev_priv->psr.busy_frontbuffer_bits || dev_priv->psr.active)  in intel_psr_work()
1120 	intel_psr_activate(dev_priv->psr.dp);  in intel_psr_work()
1122 	mutex_unlock(&dev_priv->psr.lock);  in intel_psr_work()
1138 void intel_psr_invalidate(struct drm_i915_private *dev_priv,  in intel_psr_invalidate()  argument
1141 	if (!CAN_PSR(dev_priv))  in intel_psr_invalidate()
1147 	mutex_lock(&dev_priv->psr.lock);  in intel_psr_invalidate()
1148 	if (!dev_priv->psr.enabled) {  in intel_psr_invalidate()
1149 		mutex_unlock(&dev_priv->psr.lock);  in intel_psr_invalidate()
1153 	frontbuffer_bits &= INTEL_FRONTBUFFER_ALL_MASK(dev_priv->psr.pipe);  in intel_psr_invalidate()
1154 	dev_priv->psr.busy_frontbuffer_bits |= frontbuffer_bits;  in intel_psr_invalidate()
1157 		intel_psr_exit(dev_priv);  in intel_psr_invalidate()
1159 	mutex_unlock(&dev_priv->psr.lock);  in intel_psr_invalidate()
1175 void intel_psr_flush(struct drm_i915_private *dev_priv,  in intel_psr_flush()  argument
1178 	if (!CAN_PSR(dev_priv))  in intel_psr_flush()
1184 	mutex_lock(&dev_priv->psr.lock);  in intel_psr_flush()
1185 	if (!dev_priv->psr.enabled) {  in intel_psr_flush()
1186 		mutex_unlock(&dev_priv->psr.lock);  in intel_psr_flush()
1190 	frontbuffer_bits &= INTEL_FRONTBUFFER_ALL_MASK(dev_priv->psr.pipe);  in intel_psr_flush()
1191 	dev_priv->psr.busy_frontbuffer_bits &= ~frontbuffer_bits;  in intel_psr_flush()
1195 		psr_force_hw_tracking_exit(dev_priv);  in intel_psr_flush()
1197 	if (!dev_priv->psr.active && !dev_priv->psr.busy_frontbuffer_bits)  in intel_psr_flush()
1198 		schedule_work(&dev_priv->psr.work);  in intel_psr_flush()
1199 	mutex_unlock(&dev_priv->psr.lock);  in intel_psr_flush()
1209 void intel_psr_init(struct drm_i915_private *dev_priv)  in intel_psr_init()  argument
1213 	if (!HAS_PSR(dev_priv))  in intel_psr_init()
1216 	dev_priv->psr_mmio_base = IS_HASWELL(dev_priv) ?  in intel_psr_init()
1219 	if (!dev_priv->psr.sink_support)  in intel_psr_init()
1223 		if (INTEL_GEN(dev_priv) < 9 || !dev_priv->vbt.psr.enable)  in intel_psr_init()
1238 		dev_priv->psr.sink_not_reliable = true;  in intel_psr_init()
1242 	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))  in intel_psr_init()
1244 		dev_priv->psr.link_standby = false;  in intel_psr_init()
1247 		dev_priv->psr.link_standby = dev_priv->vbt.psr.full_link;  in intel_psr_init()
1249 	INIT_WORK(&dev_priv->psr.work, intel_psr_work);  in intel_psr_init()
1250 	mutex_init(&dev_priv->psr.lock);  in intel_psr_init()
1255 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in intel_psr_short_pulse()  local
1256 	struct i915_psr *psr = &dev_priv->psr;  in intel_psr_short_pulse()
1262 	if (!CAN_PSR(dev_priv) || !intel_dp_is_edp(intel_dp))  in intel_psr_short_pulse()
1308 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in intel_psr_enabled()  local
1311 	if (!CAN_PSR(dev_priv) || !intel_dp_is_edp(intel_dp))  in intel_psr_enabled()
1314 	mutex_lock(&dev_priv->psr.lock);  in intel_psr_enabled()
1315 	ret = (dev_priv->psr.dp == intel_dp && dev_priv->psr.enabled);  in intel_psr_enabled()
1316 	mutex_unlock(&dev_priv->psr.lock);  in intel_psr_enabled()