87 static bool psr_global_enabled(struct intel_dp *intel_dp)  in psr_global_enabled()  argument
89 	struct intel_connector *connector = intel_dp->attached_connector;  in psr_global_enabled()
90 	struct drm_i915_private *i915 = dp_to_i915(intel_dp);  in psr_global_enabled()
92 	switch (intel_dp->psr.debug & I915_PSR_DEBUG_MODE_MASK) {  in psr_global_enabled()
104 static bool psr2_global_enabled(struct intel_dp *intel_dp)  in psr2_global_enabled()  argument
106 	struct drm_i915_private *i915 = dp_to_i915(intel_dp);  in psr2_global_enabled()
108 	switch (intel_dp->psr.debug & I915_PSR_DEBUG_MODE_MASK) {  in psr2_global_enabled()
119 static u32 psr_irq_psr_error_bit_get(struct intel_dp *intel_dp)  in psr_irq_psr_error_bit_get()  argument
121 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in psr_irq_psr_error_bit_get()
124 		EDP_PSR_ERROR(intel_dp->psr.transcoder);  in psr_irq_psr_error_bit_get()
127 static u32 psr_irq_post_exit_bit_get(struct intel_dp *intel_dp)  in psr_irq_post_exit_bit_get()  argument
129 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in psr_irq_post_exit_bit_get()
132 		EDP_PSR_POST_EXIT(intel_dp->psr.transcoder);  in psr_irq_post_exit_bit_get()
135 static u32 psr_irq_pre_entry_bit_get(struct intel_dp *intel_dp)  in psr_irq_pre_entry_bit_get()  argument
137 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in psr_irq_pre_entry_bit_get()
140 		EDP_PSR_PRE_ENTRY(intel_dp->psr.transcoder);  in psr_irq_pre_entry_bit_get()
143 static u32 psr_irq_mask_get(struct intel_dp *intel_dp)  in psr_irq_mask_get()  argument
145 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in psr_irq_mask_get()
148 		EDP_PSR_MASK(intel_dp->psr.transcoder);  in psr_irq_mask_get()
151 static void psr_irq_control(struct intel_dp *intel_dp)  in psr_irq_control()  argument
153 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in psr_irq_control()
158 		imr_reg = TRANS_PSR_IMR(intel_dp->psr.transcoder);  in psr_irq_control()
162 	mask = psr_irq_psr_error_bit_get(intel_dp);  in psr_irq_control()
163 	if (intel_dp->psr.debug & I915_PSR_DEBUG_IRQ)  in psr_irq_control()
164 		mask |= psr_irq_post_exit_bit_get(intel_dp) |  in psr_irq_control()
165 			psr_irq_pre_entry_bit_get(intel_dp);  in psr_irq_control()
168 	val &= ~psr_irq_mask_get(intel_dp);  in psr_irq_control()
211 void intel_psr_irq_handler(struct intel_dp *intel_dp, u32 psr_iir)  in intel_psr_irq_handler()  argument
213 	enum transcoder cpu_transcoder = intel_dp->psr.transcoder;  in intel_psr_irq_handler()
214 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in intel_psr_irq_handler()
219 		imr_reg = TRANS_PSR_IMR(intel_dp->psr.transcoder);  in intel_psr_irq_handler()
223 	if (psr_iir & psr_irq_pre_entry_bit_get(intel_dp)) {  in intel_psr_irq_handler()
224 		intel_dp->psr.last_entry_attempt = time_ns;  in intel_psr_irq_handler()
230 	if (psr_iir & psr_irq_post_exit_bit_get(intel_dp)) {  in intel_psr_irq_handler()
231 		intel_dp->psr.last_exit = time_ns;  in intel_psr_irq_handler()
239 			bool psr2_enabled = intel_dp->psr.psr2_enabled;  in intel_psr_irq_handler()
247 	if (psr_iir & psr_irq_psr_error_bit_get(intel_dp)) {  in intel_psr_irq_handler()
253 		intel_dp->psr.irq_aux_error = true;  in intel_psr_irq_handler()
264 		val |= psr_irq_psr_error_bit_get(intel_dp);  in intel_psr_irq_handler()
267 		schedule_work(&intel_dp->psr.work);  in intel_psr_irq_handler()
271 static bool intel_dp_get_alpm_status(struct intel_dp *intel_dp)  in intel_dp_get_alpm_status()  argument
275 	if (drm_dp_dpcd_readb(&intel_dp->aux, DP_RECEIVER_ALPM_CAP,  in intel_dp_get_alpm_status()
281 static u8 intel_dp_get_sink_sync_latency(struct intel_dp *intel_dp)  in intel_dp_get_sink_sync_latency()  argument
283 	struct drm_i915_private *i915 = dp_to_i915(intel_dp);  in intel_dp_get_sink_sync_latency()
286 	if (drm_dp_dpcd_readb(&intel_dp->aux,  in intel_dp_get_sink_sync_latency()
295 static void intel_dp_get_su_granularity(struct intel_dp *intel_dp)  in intel_dp_get_su_granularity()  argument
297 	struct drm_i915_private *i915 = dp_to_i915(intel_dp);  in intel_dp_get_su_granularity()
303 	if (!(intel_dp->psr_dpcd[1] & DP_PSR2_SU_GRANULARITY_REQUIRED)) {  in intel_dp_get_su_granularity()
310 	r = drm_dp_dpcd_read(&intel_dp->aux, DP_PSR2_SU_X_GRANULARITY, &w, 2);  in intel_dp_get_su_granularity()
321 	r = drm_dp_dpcd_read(&intel_dp->aux, DP_PSR2_SU_Y_GRANULARITY, &y, 1);  in intel_dp_get_su_granularity()
331 	intel_dp->psr.su_w_granularity = w;  in intel_dp_get_su_granularity()
332 	intel_dp->psr.su_y_granularity = y;  in intel_dp_get_su_granularity()
335 void intel_psr_init_dpcd(struct intel_dp *intel_dp)  in intel_psr_init_dpcd()  argument
338 		to_i915(dp_to_dig_port(intel_dp)->base.base.dev);  in intel_psr_init_dpcd()
340 	drm_dp_dpcd_read(&intel_dp->aux, DP_PSR_SUPPORT, intel_dp->psr_dpcd,  in intel_psr_init_dpcd()
341 			 sizeof(intel_dp->psr_dpcd));  in intel_psr_init_dpcd()
343 	if (!intel_dp->psr_dpcd[0])  in intel_psr_init_dpcd()
346 		    intel_dp->psr_dpcd[0]);  in intel_psr_init_dpcd()
348 	if (drm_dp_has_quirk(&intel_dp->desc, DP_DPCD_QUIRK_NO_PSR)) {  in intel_psr_init_dpcd()
354 	if (!(intel_dp->edp_dpcd[1] & DP_EDP_SET_POWER_CAP)) {  in intel_psr_init_dpcd()
360 	intel_dp->psr.sink_support = true;  in intel_psr_init_dpcd()
361 	intel_dp->psr.sink_sync_latency =  in intel_psr_init_dpcd()
362 		intel_dp_get_sink_sync_latency(intel_dp);  in intel_psr_init_dpcd()
365 	    (intel_dp->psr_dpcd[0] == DP_PSR2_WITH_Y_COORD_IS_SUPPORTED)) {  in intel_psr_init_dpcd()
366 		bool y_req = intel_dp->psr_dpcd[1] &  in intel_psr_init_dpcd()
368 		bool alpm = intel_dp_get_alpm_status(intel_dp);  in intel_psr_init_dpcd()
381 		intel_dp->psr.sink_psr2_support = y_req && alpm;  in intel_psr_init_dpcd()
383 			    intel_dp->psr.sink_psr2_support ? "" : "not ");  in intel_psr_init_dpcd()
385 		if (intel_dp->psr.sink_psr2_support) {  in intel_psr_init_dpcd()
386 			intel_dp->psr.colorimetry_support =  in intel_psr_init_dpcd()
387 				intel_dp_get_colorimetry_status(intel_dp);  in intel_psr_init_dpcd()
388 			intel_dp_get_su_granularity(intel_dp);  in intel_psr_init_dpcd()
393 static void intel_psr_enable_sink(struct intel_dp *intel_dp)  in intel_psr_enable_sink()  argument
395 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in intel_psr_enable_sink()
399 	if (intel_dp->psr.psr2_enabled) {  in intel_psr_enable_sink()
400 		drm_dp_dpcd_writeb(&intel_dp->aux, DP_RECEIVER_ALPM_CONFIG,  in intel_psr_enable_sink()
406 		if (intel_dp->psr.link_standby)  in intel_psr_enable_sink()
413 	if (intel_dp->psr.req_psr2_sdp_prior_scanline)  in intel_psr_enable_sink()
416 	drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG, dpcd_val);  in intel_psr_enable_sink()
418 	drm_dp_dpcd_writeb(&intel_dp->aux, DP_SET_POWER, DP_SET_POWER_D0);  in intel_psr_enable_sink()
421 static u32 intel_psr1_get_tp_time(struct intel_dp *intel_dp)  in intel_psr1_get_tp_time()  argument
423 	struct intel_connector *connector = intel_dp->attached_connector;  in intel_psr1_get_tp_time()
424 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in intel_psr1_get_tp_time()
456 	    drm_dp_tps3_supported(intel_dp->dpcd))  in intel_psr1_get_tp_time()
464 static u8 psr_compute_idle_frames(struct intel_dp *intel_dp)  in psr_compute_idle_frames()  argument
466 	struct intel_connector *connector = intel_dp->attached_connector;  in psr_compute_idle_frames()
467 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in psr_compute_idle_frames()
474 	idle_frames = max(idle_frames, intel_dp->psr.sink_sync_latency + 1);  in psr_compute_idle_frames()
482 static void hsw_activate_psr1(struct intel_dp *intel_dp)  in hsw_activate_psr1()  argument
484 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in hsw_activate_psr1()
488 	val |= psr_compute_idle_frames(intel_dp) << EDP_PSR_IDLE_FRAME_SHIFT;  in hsw_activate_psr1()
494 	if (intel_dp->psr.link_standby)  in hsw_activate_psr1()
497 	val |= intel_psr1_get_tp_time(intel_dp);  in hsw_activate_psr1()
502 	val |= (intel_de_read(dev_priv, EDP_PSR_CTL(intel_dp->psr.transcoder)) &  in hsw_activate_psr1()
504 	intel_de_write(dev_priv, EDP_PSR_CTL(intel_dp->psr.transcoder), val);  in hsw_activate_psr1()
507 static u32 intel_psr2_get_tp_time(struct intel_dp *intel_dp)  in intel_psr2_get_tp_time()  argument
509 	struct intel_connector *connector = intel_dp->attached_connector;  in intel_psr2_get_tp_time()
510 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in intel_psr2_get_tp_time()
529 static void hsw_activate_psr2(struct intel_dp *intel_dp)  in hsw_activate_psr2()  argument
531 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in hsw_activate_psr2()
534 	val |= psr_compute_idle_frames(intel_dp) << EDP_PSR2_IDLE_FRAME_SHIFT;  in hsw_activate_psr2()
542 	val |= EDP_PSR2_FRAME_BEFORE_SU(max_t(u8, intel_dp->psr.sink_sync_latency + 1, 2));  in hsw_activate_psr2()
543 	val |= intel_psr2_get_tp_time(intel_dp);  in hsw_activate_psr2()
588 	if (intel_dp->psr.req_psr2_sdp_prior_scanline)  in hsw_activate_psr2()
591 	if (intel_dp->psr.psr2_sel_fetch_enabled) {  in hsw_activate_psr2()
600 		tmp = intel_de_read(dev_priv, PSR2_MAN_TRK_CTL(intel_dp->psr.transcoder));  in hsw_activate_psr2()
604 			       PSR2_MAN_TRK_CTL(intel_dp->psr.transcoder), 0);  in hsw_activate_psr2()
611 	intel_de_write(dev_priv, EDP_PSR_CTL(intel_dp->psr.transcoder), 0);  in hsw_activate_psr2()
613 	intel_de_write(dev_priv, EDP_PSR2_CTL(intel_dp->psr.transcoder), val);  in hsw_activate_psr2()
636 static void psr2_program_idle_frames(struct intel_dp *intel_dp,  in psr2_program_idle_frames()  argument
639 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in psr2_program_idle_frames()
643 	val = intel_de_read(dev_priv, EDP_PSR2_CTL(intel_dp->psr.transcoder));  in psr2_program_idle_frames()
646 	intel_de_write(dev_priv, EDP_PSR2_CTL(intel_dp->psr.transcoder), val);  in psr2_program_idle_frames()
649 static void tgl_psr2_enable_dc3co(struct intel_dp *intel_dp)  in tgl_psr2_enable_dc3co()  argument
651 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in tgl_psr2_enable_dc3co()
653 	psr2_program_idle_frames(intel_dp, 0);  in tgl_psr2_enable_dc3co()
657 static void tgl_psr2_disable_dc3co(struct intel_dp *intel_dp)  in tgl_psr2_disable_dc3co()  argument
659 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in tgl_psr2_disable_dc3co()
662 	psr2_program_idle_frames(intel_dp, psr_compute_idle_frames(intel_dp));  in tgl_psr2_disable_dc3co()
667 	struct intel_dp *intel_dp =  in tgl_dc3co_disable_work()  local
668 		container_of(work, typeof(*intel_dp), psr.dc3co_work.work);  in tgl_dc3co_disable_work()
670 	mutex_lock(&intel_dp->psr.lock);  in tgl_dc3co_disable_work()
672 	if (delayed_work_pending(&intel_dp->psr.dc3co_work))  in tgl_dc3co_disable_work()
675 	tgl_psr2_disable_dc3co(intel_dp);  in tgl_dc3co_disable_work()
677 	mutex_unlock(&intel_dp->psr.lock);  in tgl_dc3co_disable_work()
680 static void tgl_disallow_dc3co_on_psr2_exit(struct intel_dp *intel_dp)  in tgl_disallow_dc3co_on_psr2_exit()  argument
682 	if (!intel_dp->psr.dc3co_exitline)  in tgl_disallow_dc3co_on_psr2_exit()
685 	cancel_delayed_work(&intel_dp->psr.dc3co_work);  in tgl_disallow_dc3co_on_psr2_exit()
687 	tgl_psr2_disable_dc3co(intel_dp);  in tgl_disallow_dc3co_on_psr2_exit()
691 dc3co_is_pipe_port_compatible(struct intel_dp *intel_dp,  in dc3co_is_pipe_port_compatible()  argument
694 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);  in dc3co_is_pipe_port_compatible()
696 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in dc3co_is_pipe_port_compatible()
706 tgl_dc3co_exitline_compute_config(struct intel_dp *intel_dp,  in tgl_dc3co_exitline_compute_config()  argument
710 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in tgl_dc3co_exitline_compute_config()
730 	if (!dc3co_is_pipe_port_compatible(intel_dp, crtc_state))  in tgl_dc3co_exitline_compute_config()
750 static bool intel_psr2_sel_fetch_config_valid(struct intel_dp *intel_dp,  in intel_psr2_sel_fetch_config_valid()  argument
753 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in intel_psr2_sel_fetch_config_valid()
756 	    intel_dp->psr.debug != I915_PSR_DEBUG_ENABLE_SEL_FETCH) {  in intel_psr2_sel_fetch_config_valid()
778 static bool psr2_granularity_check(struct intel_dp *intel_dp,  in psr2_granularity_check()  argument
781 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in psr2_granularity_check()
787 	if (crtc_hdisplay % intel_dp->psr.su_w_granularity)  in psr2_granularity_check()
790 	if (crtc_vdisplay % intel_dp->psr.su_y_granularity)  in psr2_granularity_check()
795 		return intel_dp->psr.su_y_granularity == 4;  in psr2_granularity_check()
803 		y_granularity = intel_dp->psr.su_y_granularity;  in psr2_granularity_check()
804 	else if (intel_dp->psr.su_y_granularity <= 2)  in psr2_granularity_check()
806 	else if ((intel_dp->psr.su_y_granularity % 4) == 0)  in psr2_granularity_check()
807 		y_granularity = intel_dp->psr.su_y_granularity;  in psr2_granularity_check()
816 static bool _compute_psr2_sdp_prior_scanline_indication(struct intel_dp *intel_dp,  in _compute_psr2_sdp_prior_scanline_indication()  argument
820 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in _compute_psr2_sdp_prior_scanline_indication()
833 	if (DISPLAY_VER(dev_priv) <= 13 || intel_dp->edp_dpcd[0] < DP_EDP_14b)  in _compute_psr2_sdp_prior_scanline_indication()
840 static bool intel_psr2_config_valid(struct intel_dp *intel_dp,  in intel_psr2_config_valid()  argument
843 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in intel_psr2_config_valid()
848 	if (!intel_dp->psr.sink_psr2_support)  in intel_psr2_config_valid()
876 	if (!psr2_global_enabled(intel_dp)) {  in intel_psr2_config_valid()
927 	if (!_compute_psr2_sdp_prior_scanline_indication(intel_dp, crtc_state)) {  in intel_psr2_config_valid()
934 		if (!intel_psr2_sel_fetch_config_valid(intel_dp, crtc_state) &&  in intel_psr2_config_valid()
949 	if (!psr2_granularity_check(intel_dp, crtc_state)) {  in intel_psr2_config_valid()
963 	tgl_dc3co_exitline_compute_config(intel_dp, crtc_state);  in intel_psr2_config_valid()
971 void intel_psr_compute_config(struct intel_dp *intel_dp,  in intel_psr_compute_config()  argument
975 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in intel_psr_compute_config()
987 	if (!CAN_PSR(intel_dp))  in intel_psr_compute_config()
990 	if (!psr_global_enabled(intel_dp)) {  in intel_psr_compute_config()
995 	if (intel_dp->psr.sink_not_reliable) {  in intel_psr_compute_config()
1007 	psr_setup_time = drm_dp_psr_setup_time(intel_dp->psr_dpcd);  in intel_psr_compute_config()
1011 			    intel_dp->psr_dpcd[1]);  in intel_psr_compute_config()
1024 	crtc_state->has_psr2 = intel_psr2_config_valid(intel_dp, crtc_state);  in intel_psr_compute_config()
1027 	intel_dp_compute_psr_vsc_sdp(intel_dp, crtc_state, conn_state,  in intel_psr_compute_config()
1036 	struct intel_dp *intel_dp;  in intel_psr_get_config()  local
1042 	intel_dp = &dig_port->dp;  in intel_psr_get_config()
1043 	if (!CAN_PSR(intel_dp))  in intel_psr_get_config()
1046 	mutex_lock(&intel_dp->psr.lock);  in intel_psr_get_config()
1047 	if (!intel_dp->psr.enabled)  in intel_psr_get_config()
1055 	pipe_config->has_psr2 = intel_dp->psr.psr2_enabled;  in intel_psr_get_config()
1058 	if (!intel_dp->psr.psr2_enabled)  in intel_psr_get_config()
1062 		val = intel_de_read(dev_priv, PSR2_MAN_TRK_CTL(intel_dp->psr.transcoder));  in intel_psr_get_config()
1068 		val = intel_de_read(dev_priv, EXITLINE(intel_dp->psr.transcoder));  in intel_psr_get_config()
1073 	mutex_unlock(&intel_dp->psr.lock);  in intel_psr_get_config()
1076 static void intel_psr_activate(struct intel_dp *intel_dp)  in intel_psr_activate()  argument
1078 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in intel_psr_activate()
1079 	enum transcoder transcoder = intel_dp->psr.transcoder;  in intel_psr_activate()
1087 	drm_WARN_ON(&dev_priv->drm, intel_dp->psr.active);  in intel_psr_activate()
1088 	lockdep_assert_held(&intel_dp->psr.lock);  in intel_psr_activate()
1091 	if (intel_dp->psr.psr2_enabled)  in intel_psr_activate()
1092 		hsw_activate_psr2(intel_dp);  in intel_psr_activate()
1094 		hsw_activate_psr1(intel_dp);  in intel_psr_activate()
1096 	intel_dp->psr.active = true;  in intel_psr_activate()
1099 static u32 wa_16013835468_bit_get(struct intel_dp *intel_dp)  in wa_16013835468_bit_get()  argument
1101 	switch (intel_dp->psr.pipe) {  in wa_16013835468_bit_get()
1109 		MISSING_CASE(intel_dp->psr.pipe);  in wa_16013835468_bit_get()
1114 static void intel_psr_enable_source(struct intel_dp *intel_dp,  in intel_psr_enable_source()  argument
1117 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in intel_psr_enable_source()
1118 	enum transcoder cpu_transcoder = intel_dp->psr.transcoder;  in intel_psr_enable_source()
1135 	intel_de_write(dev_priv, EDP_PSR_DEBUG(intel_dp->psr.transcoder),  in intel_psr_enable_source()
1138 	psr_irq_control(intel_dp);  in intel_psr_enable_source()
1140 	if (intel_dp->psr.dc3co_exitline) {  in intel_psr_enable_source()
1149 		val |= intel_dp->psr.dc3co_exitline << EXITLINE_SHIFT;  in intel_psr_enable_source()
1156 			     intel_dp->psr.psr2_sel_fetch_enabled ?  in intel_psr_enable_source()
1159 	if (intel_dp->psr.psr2_enabled) {  in intel_psr_enable_source()
1177 				     TRANS_SET_CONTEXT_LATENCY(intel_dp->psr.transcoder),  in intel_psr_enable_source()
1197 					     wa_16013835468_bit_get(intel_dp));  in intel_psr_enable_source()
1202 static bool psr_interrupt_error_check(struct intel_dp *intel_dp)  in psr_interrupt_error_check()  argument
1204 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in psr_interrupt_error_check()
1217 				    TRANS_PSR_IIR(intel_dp->psr.transcoder));  in psr_interrupt_error_check()
1220 	val &= psr_irq_psr_error_bit_get(intel_dp);  in psr_interrupt_error_check()
1222 		intel_dp->psr.sink_not_reliable = true;  in psr_interrupt_error_check()
1231 static void intel_psr_enable_locked(struct intel_dp *intel_dp,  in intel_psr_enable_locked()  argument
1234 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);  in intel_psr_enable_locked()
1235 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in intel_psr_enable_locked()
1240 	drm_WARN_ON(&dev_priv->drm, intel_dp->psr.enabled);  in intel_psr_enable_locked()
1242 	intel_dp->psr.psr2_enabled = crtc_state->has_psr2;  in intel_psr_enable_locked()
1243 	intel_dp->psr.busy_frontbuffer_bits = 0;  in intel_psr_enable_locked()
1244 	intel_dp->psr.pipe = to_intel_crtc(crtc_state->uapi.crtc)->pipe;  in intel_psr_enable_locked()
1245 	intel_dp->psr.transcoder = crtc_state->cpu_transcoder;  in intel_psr_enable_locked()
1248 	intel_dp->psr.dc3co_exit_delay = val;  in intel_psr_enable_locked()
1249 	intel_dp->psr.dc3co_exitline = crtc_state->dc3co_exitline;  in intel_psr_enable_locked()
1250 	intel_dp->psr.psr2_sel_fetch_enabled = crtc_state->enable_psr2_sel_fetch;  in intel_psr_enable_locked()
1251 	intel_dp->psr.psr2_sel_fetch_cff_enabled = false;  in intel_psr_enable_locked()
1252 	intel_dp->psr.req_psr2_sdp_prior_scanline =  in intel_psr_enable_locked()
1255 	if (!psr_interrupt_error_check(intel_dp))  in intel_psr_enable_locked()
1259 		    intel_dp->psr.psr2_enabled ? "2" : "1");  in intel_psr_enable_locked()
1262 	intel_psr_enable_sink(intel_dp);  in intel_psr_enable_locked()
1263 	intel_psr_enable_source(intel_dp, crtc_state);  in intel_psr_enable_locked()
1264 	intel_dp->psr.enabled = true;  in intel_psr_enable_locked()
1265 	intel_dp->psr.paused = false;  in intel_psr_enable_locked()
1267 	intel_psr_activate(intel_dp);  in intel_psr_enable_locked()
1270 static void intel_psr_exit(struct intel_dp *intel_dp)  in intel_psr_exit()  argument
1272 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in intel_psr_exit()
1275 	if (!intel_dp->psr.active) {  in intel_psr_exit()
1276 		if (transcoder_has_psr2(dev_priv, intel_dp->psr.transcoder)) {  in intel_psr_exit()
1278 					    EDP_PSR2_CTL(intel_dp->psr.transcoder));  in intel_psr_exit()
1283 				    EDP_PSR_CTL(intel_dp->psr.transcoder));  in intel_psr_exit()
1289 	if (intel_dp->psr.psr2_enabled) {  in intel_psr_exit()
1290 		tgl_disallow_dc3co_on_psr2_exit(intel_dp);  in intel_psr_exit()
1292 				    EDP_PSR2_CTL(intel_dp->psr.transcoder));  in intel_psr_exit()
1296 			       EDP_PSR2_CTL(intel_dp->psr.transcoder), val);  in intel_psr_exit()
1299 				    EDP_PSR_CTL(intel_dp->psr.transcoder));  in intel_psr_exit()
1303 			       EDP_PSR_CTL(intel_dp->psr.transcoder), val);  in intel_psr_exit()
1305 	intel_dp->psr.active = false;  in intel_psr_exit()
1308 static void intel_psr_wait_exit_locked(struct intel_dp *intel_dp)  in intel_psr_wait_exit_locked()  argument
1310 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in intel_psr_wait_exit_locked()
1314 	if (intel_dp->psr.psr2_enabled) {  in intel_psr_wait_exit_locked()
1315 		psr_status = EDP_PSR2_STATUS(intel_dp->psr.transcoder);  in intel_psr_wait_exit_locked()
1318 		psr_status = EDP_PSR_STATUS(intel_dp->psr.transcoder);  in intel_psr_wait_exit_locked()
1328 static void intel_psr_disable_locked(struct intel_dp *intel_dp)  in intel_psr_disable_locked()  argument
1330 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in intel_psr_disable_locked()
1332 					 dp_to_dig_port(intel_dp)->base.port);  in intel_psr_disable_locked()
1334 	lockdep_assert_held(&intel_dp->psr.lock);  in intel_psr_disable_locked()
1336 	if (!intel_dp->psr.enabled)  in intel_psr_disable_locked()
1340 		    intel_dp->psr.psr2_enabled ? "2" : "1");  in intel_psr_disable_locked()
1342 	intel_psr_exit(intel_dp);  in intel_psr_disable_locked()
1343 	intel_psr_wait_exit_locked(intel_dp);  in intel_psr_disable_locked()
1346 	if (intel_dp->psr.psr2_sel_fetch_enabled &&  in intel_psr_disable_locked()
1351 	if (intel_dp->psr.psr2_enabled) {  in intel_psr_disable_locked()
1355 				     TRANS_SET_CONTEXT_LATENCY(intel_dp->psr.transcoder),  in intel_psr_disable_locked()
1367 				     wa_16013835468_bit_get(intel_dp), 0);  in intel_psr_disable_locked()
1373 	drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG, 0);  in intel_psr_disable_locked()
1375 	if (intel_dp->psr.psr2_enabled)  in intel_psr_disable_locked()
1376 		drm_dp_dpcd_writeb(&intel_dp->aux, DP_RECEIVER_ALPM_CONFIG, 0);  in intel_psr_disable_locked()
1378 	intel_dp->psr.enabled = false;  in intel_psr_disable_locked()
1379 	intel_dp->psr.psr2_enabled = false;  in intel_psr_disable_locked()
1380 	intel_dp->psr.psr2_sel_fetch_enabled = false;  in intel_psr_disable_locked()
1381 	intel_dp->psr.psr2_sel_fetch_cff_enabled = false;  in intel_psr_disable_locked()
1391 void intel_psr_disable(struct intel_dp *intel_dp,  in intel_psr_disable()  argument
1394 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in intel_psr_disable()
1399 	if (drm_WARN_ON(&dev_priv->drm, !CAN_PSR(intel_dp)))  in intel_psr_disable()
1402 	mutex_lock(&intel_dp->psr.lock);  in intel_psr_disable()
1404 	intel_psr_disable_locked(intel_dp);  in intel_psr_disable()
1406 	mutex_unlock(&intel_dp->psr.lock);  in intel_psr_disable()
1407 	cancel_work_sync(&intel_dp->psr.work);  in intel_psr_disable()
1408 	cancel_delayed_work_sync(&intel_dp->psr.dc3co_work);  in intel_psr_disable()
1417 void intel_psr_pause(struct intel_dp *intel_dp)  in intel_psr_pause()  argument
1419 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in intel_psr_pause()
1420 	struct intel_psr *psr = &intel_dp->psr;  in intel_psr_pause()
1422 	if (!CAN_PSR(intel_dp))  in intel_psr_pause()
1435 	intel_psr_exit(intel_dp);  in intel_psr_pause()
1436 	intel_psr_wait_exit_locked(intel_dp);  in intel_psr_pause()
1451 void intel_psr_resume(struct intel_dp *intel_dp)  in intel_psr_resume()  argument
1453 	struct intel_psr *psr = &intel_dp->psr;  in intel_psr_resume()
1455 	if (!CAN_PSR(intel_dp))  in intel_psr_resume()
1464 	intel_psr_activate(intel_dp);  in intel_psr_resume()
1496 static void psr_force_hw_tracking_exit(struct intel_dp *intel_dp)  in psr_force_hw_tracking_exit()  argument
1498 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in psr_force_hw_tracking_exit()
1500 	if (intel_dp->psr.psr2_sel_fetch_enabled)  in psr_force_hw_tracking_exit()
1502 			       PSR2_MAN_TRK_CTL(intel_dp->psr.transcoder),  in psr_force_hw_tracking_exit()
1520 	intel_de_write(dev_priv, CURSURFLIVE(intel_dp->psr.pipe), 0);  in psr_force_hw_tracking_exit()
1596 		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);  in intel_psr2_program_trans_man_trk_ctl()  local
1598 		lockdep_assert_held(&intel_dp->psr.lock);  in intel_psr2_program_trans_man_trk_ctl()
1599 		if (intel_dp->psr.psr2_sel_fetch_cff_enabled)  in intel_psr2_program_trans_man_trk_ctl()
1888 		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);  in intel_psr_pre_plane_update()  local
1889 		struct intel_psr *psr = &intel_dp->psr;  in intel_psr_pre_plane_update()
1906 			intel_psr_disable_locked(intel_dp);  in intel_psr_pre_plane_update()
1923 		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);  in _intel_psr_post_plane_update()  local
1924 		struct intel_psr *psr = &intel_dp->psr;  in _intel_psr_post_plane_update()
1935 			intel_psr_enable_locked(intel_dp, crtc_state);  in _intel_psr_post_plane_update()
1939 			psr_force_hw_tracking_exit(intel_dp);  in _intel_psr_post_plane_update()
1960 static int _psr2_ready_for_pipe_update_locked(struct intel_dp *intel_dp)  in _psr2_ready_for_pipe_update_locked()  argument
1962 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in _psr2_ready_for_pipe_update_locked()
1970 				       EDP_PSR2_STATUS(intel_dp->psr.transcoder),  in _psr2_ready_for_pipe_update_locked()
1974 static int _psr1_ready_for_pipe_update_locked(struct intel_dp *intel_dp)  in _psr1_ready_for_pipe_update_locked()  argument
1976 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in _psr1_ready_for_pipe_update_locked()
1985 				       EDP_PSR_STATUS(intel_dp->psr.transcoder),  in _psr1_ready_for_pipe_update_locked()
2006 		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);  in intel_psr_wait_for_idle_locked()  local
2009 		lockdep_assert_held(&intel_dp->psr.lock);  in intel_psr_wait_for_idle_locked()
2011 		if (!intel_dp->psr.enabled)  in intel_psr_wait_for_idle_locked()
2014 		if (intel_dp->psr.psr2_enabled)  in intel_psr_wait_for_idle_locked()
2015 			ret = _psr2_ready_for_pipe_update_locked(intel_dp);  in intel_psr_wait_for_idle_locked()
2017 			ret = _psr1_ready_for_pipe_update_locked(intel_dp);  in intel_psr_wait_for_idle_locked()
2024 static bool __psr_wait_for_idle_locked(struct intel_dp *intel_dp)  in __psr_wait_for_idle_locked()  argument
2026 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in __psr_wait_for_idle_locked()
2031 	if (!intel_dp->psr.enabled)  in __psr_wait_for_idle_locked()
2034 	if (intel_dp->psr.psr2_enabled) {  in __psr_wait_for_idle_locked()
2035 		reg = EDP_PSR2_STATUS(intel_dp->psr.transcoder);  in __psr_wait_for_idle_locked()
2038 		reg = EDP_PSR_STATUS(intel_dp->psr.transcoder);  in __psr_wait_for_idle_locked()
2042 	mutex_unlock(&intel_dp->psr.lock);  in __psr_wait_for_idle_locked()
2050 	mutex_lock(&intel_dp->psr.lock);  in __psr_wait_for_idle_locked()
2051 	return err == 0 && intel_dp->psr.enabled;  in __psr_wait_for_idle_locked()
2117 int intel_psr_debug_set(struct intel_dp *intel_dp, u64 val)  in intel_psr_debug_set()  argument
2119 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in intel_psr_debug_set()
2130 	ret = mutex_lock_interruptible(&intel_dp->psr.lock);  in intel_psr_debug_set()
2134 	old_mode = intel_dp->psr.debug & I915_PSR_DEBUG_MODE_MASK;  in intel_psr_debug_set()
2135 	intel_dp->psr.debug = val;  in intel_psr_debug_set()
2141 	if (intel_dp->psr.enabled)  in intel_psr_debug_set()
2142 		psr_irq_control(intel_dp);  in intel_psr_debug_set()
2144 	mutex_unlock(&intel_dp->psr.lock);  in intel_psr_debug_set()
2152 static void intel_psr_handle_irq(struct intel_dp *intel_dp)  in intel_psr_handle_irq()  argument
2154 	struct intel_psr *psr = &intel_dp->psr;  in intel_psr_handle_irq()
2156 	intel_psr_disable_locked(intel_dp);  in intel_psr_handle_irq()
2159 	drm_dp_dpcd_writeb(&intel_dp->aux, DP_SET_POWER, DP_SET_POWER_D0);  in intel_psr_handle_irq()
2164 	struct intel_dp *intel_dp =  in intel_psr_work()  local
2165 		container_of(work, typeof(*intel_dp), psr.work);  in intel_psr_work()
2167 	mutex_lock(&intel_dp->psr.lock);  in intel_psr_work()
2169 	if (!intel_dp->psr.enabled)  in intel_psr_work()
2172 	if (READ_ONCE(intel_dp->psr.irq_aux_error))  in intel_psr_work()
2173 		intel_psr_handle_irq(intel_dp);  in intel_psr_work()
2181 	if (!__psr_wait_for_idle_locked(intel_dp))  in intel_psr_work()
2189 	if (intel_dp->psr.busy_frontbuffer_bits || intel_dp->psr.active)  in intel_psr_work()
2192 	intel_psr_activate(intel_dp);  in intel_psr_work()
2194 	mutex_unlock(&intel_dp->psr.lock);  in intel_psr_work()
2197 static void _psr_invalidate_handle(struct intel_dp *intel_dp)  in _psr_invalidate_handle()  argument
2199 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in _psr_invalidate_handle()
2201 	if (intel_dp->psr.psr2_sel_fetch_enabled) {  in _psr_invalidate_handle()
2204 		if (intel_dp->psr.psr2_sel_fetch_cff_enabled) {  in _psr_invalidate_handle()
2206 			intel_de_write(dev_priv, CURSURFLIVE(intel_dp->psr.pipe), 0);  in _psr_invalidate_handle()
2213 		intel_de_write(dev_priv, PSR2_MAN_TRK_CTL(intel_dp->psr.transcoder), val);  in _psr_invalidate_handle()
2214 		intel_de_write(dev_priv, CURSURFLIVE(intel_dp->psr.pipe), 0);  in _psr_invalidate_handle()
2215 		intel_dp->psr.psr2_sel_fetch_cff_enabled = true;  in _psr_invalidate_handle()
2217 		intel_psr_exit(intel_dp);  in _psr_invalidate_handle()
2244 		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);  in intel_psr_invalidate()  local
2246 		mutex_lock(&intel_dp->psr.lock);  in intel_psr_invalidate()
2247 		if (!intel_dp->psr.enabled) {  in intel_psr_invalidate()
2248 			mutex_unlock(&intel_dp->psr.lock);  in intel_psr_invalidate()
2253 			INTEL_FRONTBUFFER_ALL_MASK(intel_dp->psr.pipe);  in intel_psr_invalidate()
2254 		intel_dp->psr.busy_frontbuffer_bits |= pipe_frontbuffer_bits;  in intel_psr_invalidate()
2257 			_psr_invalidate_handle(intel_dp);  in intel_psr_invalidate()
2259 		mutex_unlock(&intel_dp->psr.lock);  in intel_psr_invalidate()
2269 tgl_dc3co_flush_locked(struct intel_dp *intel_dp, unsigned int frontbuffer_bits,  in tgl_dc3co_flush_locked()  argument
2272 	if (!intel_dp->psr.dc3co_exitline || !intel_dp->psr.psr2_enabled ||  in tgl_dc3co_flush_locked()
2273 	    !intel_dp->psr.active)  in tgl_dc3co_flush_locked()
2281 	    INTEL_FRONTBUFFER_ALL_MASK(intel_dp->psr.pipe)))  in tgl_dc3co_flush_locked()
2284 	tgl_psr2_enable_dc3co(intel_dp);  in tgl_dc3co_flush_locked()
2285 	mod_delayed_work(system_wq, &intel_dp->psr.dc3co_work,  in tgl_dc3co_flush_locked()
2286 			 intel_dp->psr.dc3co_exit_delay);  in tgl_dc3co_flush_locked()
2289 static void _psr_flush_handle(struct intel_dp *intel_dp)  in _psr_flush_handle()  argument
2291 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in _psr_flush_handle()
2293 	if (intel_dp->psr.psr2_sel_fetch_enabled) {  in _psr_flush_handle()
2294 		if (intel_dp->psr.psr2_sel_fetch_cff_enabled) {  in _psr_flush_handle()
2296 			if (intel_dp->psr.busy_frontbuffer_bits == 0) {  in _psr_flush_handle()
2305 				intel_de_write(dev_priv, PSR2_MAN_TRK_CTL(intel_dp->psr.transcoder),  in _psr_flush_handle()
2307 				intel_de_write(dev_priv, CURSURFLIVE(intel_dp->psr.pipe), 0);  in _psr_flush_handle()
2308 				intel_dp->psr.psr2_sel_fetch_cff_enabled = false;  in _psr_flush_handle()
2315 			psr_force_hw_tracking_exit(intel_dp);  in _psr_flush_handle()
2318 		psr_force_hw_tracking_exit(intel_dp);  in _psr_flush_handle()
2320 		if (!intel_dp->psr.active && !intel_dp->psr.busy_frontbuffer_bits)  in _psr_flush_handle()
2321 			schedule_work(&intel_dp->psr.work);  in _psr_flush_handle()
2345 		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);  in intel_psr_flush()  local
2347 		mutex_lock(&intel_dp->psr.lock);  in intel_psr_flush()
2348 		if (!intel_dp->psr.enabled) {  in intel_psr_flush()
2349 			mutex_unlock(&intel_dp->psr.lock);  in intel_psr_flush()
2354 			INTEL_FRONTBUFFER_ALL_MASK(intel_dp->psr.pipe);  in intel_psr_flush()
2355 		intel_dp->psr.busy_frontbuffer_bits &= ~pipe_frontbuffer_bits;  in intel_psr_flush()
2362 		if (intel_dp->psr.paused)  in intel_psr_flush()
2367 		     !intel_dp->psr.psr2_sel_fetch_enabled)) {  in intel_psr_flush()
2368 			tgl_dc3co_flush_locked(intel_dp, frontbuffer_bits, origin);  in intel_psr_flush()
2376 		_psr_flush_handle(intel_dp);  in intel_psr_flush()
2378 		mutex_unlock(&intel_dp->psr.lock);  in intel_psr_flush()
2390 void intel_psr_init(struct intel_dp *intel_dp)  in intel_psr_init()  argument
2392 	struct intel_connector *connector = intel_dp->attached_connector;  in intel_psr_init()
2393 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);  in intel_psr_init()
2394 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in intel_psr_init()
2414 	intel_dp->psr.source_support = true;  in intel_psr_init()
2419 		intel_dp->psr.link_standby = connector->panel.vbt.psr.full_link;  in intel_psr_init()
2421 	INIT_WORK(&intel_dp->psr.work, intel_psr_work);  in intel_psr_init()
2422 	INIT_DELAYED_WORK(&intel_dp->psr.dc3co_work, tgl_dc3co_disable_work);  in intel_psr_init()
2423 	mutex_init(&intel_dp->psr.lock);  in intel_psr_init()
2426 static int psr_get_status_and_error_status(struct intel_dp *intel_dp,  in psr_get_status_and_error_status()  argument
2429 	struct drm_dp_aux *aux = &intel_dp->aux;  in psr_get_status_and_error_status()
2445 static void psr_alpm_check(struct intel_dp *intel_dp)  in psr_alpm_check()  argument
2447 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in psr_alpm_check()
2448 	struct drm_dp_aux *aux = &intel_dp->aux;  in psr_alpm_check()
2449 	struct intel_psr *psr = &intel_dp->psr;  in psr_alpm_check()
2463 		intel_psr_disable_locked(intel_dp);  in psr_alpm_check()
2473 static void psr_capability_changed_check(struct intel_dp *intel_dp)  in psr_capability_changed_check()  argument
2475 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in psr_capability_changed_check()
2476 	struct intel_psr *psr = &intel_dp->psr;  in psr_capability_changed_check()
2480 	r = drm_dp_dpcd_readb(&intel_dp->aux, DP_PSR_ESI, &val);  in psr_capability_changed_check()
2487 		intel_psr_disable_locked(intel_dp);  in psr_capability_changed_check()
2493 		drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_ESI, val);  in psr_capability_changed_check()
2497 void intel_psr_short_pulse(struct intel_dp *intel_dp)  in intel_psr_short_pulse()  argument
2499 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);  in intel_psr_short_pulse()
2500 	struct intel_psr *psr = &intel_dp->psr;  in intel_psr_short_pulse()
2506 	if (!CAN_PSR(intel_dp))  in intel_psr_short_pulse()
2514 	if (psr_get_status_and_error_status(intel_dp, &status, &error_status)) {  in intel_psr_short_pulse()
2521 		intel_psr_disable_locked(intel_dp);  in intel_psr_short_pulse()
2543 	drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_ERROR_STATUS, error_status);  in intel_psr_short_pulse()
2545 	psr_alpm_check(intel_dp);  in intel_psr_short_pulse()
2546 	psr_capability_changed_check(intel_dp);  in intel_psr_short_pulse()
2552 bool intel_psr_enabled(struct intel_dp *intel_dp)  in intel_psr_enabled()  argument
2556 	if (!CAN_PSR(intel_dp))  in intel_psr_enabled()
2559 	mutex_lock(&intel_dp->psr.lock);  in intel_psr_enabled()
2560 	ret = intel_dp->psr.enabled;  in intel_psr_enabled()
2561 	mutex_unlock(&intel_dp->psr.lock);  in intel_psr_enabled()
2584 		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);  in intel_psr_lock()  local
2586 		mutex_lock(&intel_dp->psr.lock);  in intel_psr_lock()
2607 		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);  in intel_psr_unlock()  local
2609 		mutex_unlock(&intel_dp->psr.lock);  in intel_psr_unlock()