64 static int intel_fbc_calculate_cfb_size(struct drm_i915_private *dev_priv,  in intel_fbc_calculate_cfb_size()  argument
70 	if (IS_GEN(dev_priv, 7))  in intel_fbc_calculate_cfb_size()
72 	else if (INTEL_GEN(dev_priv) >= 8)  in intel_fbc_calculate_cfb_size()
79 static void i8xx_fbc_deactivate(struct drm_i915_private *dev_priv)  in i8xx_fbc_deactivate()  argument
84 	fbc_ctl = intel_de_read(dev_priv, FBC_CONTROL);  in i8xx_fbc_deactivate()
89 	intel_de_write(dev_priv, FBC_CONTROL, fbc_ctl);  in i8xx_fbc_deactivate()
92 	if (intel_de_wait_for_clear(dev_priv, FBC_STATUS,  in i8xx_fbc_deactivate()
94 		drm_dbg_kms(&dev_priv->drm, "FBC idle timed out\n");  in i8xx_fbc_deactivate()
99 static void i8xx_fbc_activate(struct drm_i915_private *dev_priv)  in i8xx_fbc_activate()  argument
101 	struct intel_fbc_reg_params *params = &dev_priv->fbc.params;  in i8xx_fbc_activate()
112 	if (IS_GEN(dev_priv, 2))  in i8xx_fbc_activate()
119 		intel_de_write(dev_priv, FBC_TAG(i), 0);  in i8xx_fbc_activate()
121 	if (IS_GEN(dev_priv, 4)) {  in i8xx_fbc_activate()
129 		intel_de_write(dev_priv, FBC_CONTROL2, fbc_ctl2);  in i8xx_fbc_activate()
130 		intel_de_write(dev_priv, FBC_FENCE_OFF,  in i8xx_fbc_activate()
137 	if (IS_I945GM(dev_priv))  in i8xx_fbc_activate()
142 	intel_de_write(dev_priv, FBC_CONTROL, fbc_ctl);  in i8xx_fbc_activate()
145 static bool i8xx_fbc_is_active(struct drm_i915_private *dev_priv)  in i8xx_fbc_is_active()  argument
147 	return intel_de_read(dev_priv, FBC_CONTROL) & FBC_CTL_EN;  in i8xx_fbc_is_active()
150 static void g4x_fbc_activate(struct drm_i915_private *dev_priv)  in g4x_fbc_activate()  argument
152 	struct intel_fbc_reg_params *params = &dev_priv->fbc.params;  in g4x_fbc_activate()
163 		intel_de_write(dev_priv, DPFC_FENCE_YOFF,  in g4x_fbc_activate()
166 		intel_de_write(dev_priv, DPFC_FENCE_YOFF, 0);  in g4x_fbc_activate()
170 	intel_de_write(dev_priv, DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN);  in g4x_fbc_activate()
173 static void g4x_fbc_deactivate(struct drm_i915_private *dev_priv)  in g4x_fbc_deactivate()  argument
178 	dpfc_ctl = intel_de_read(dev_priv, DPFC_CONTROL);  in g4x_fbc_deactivate()
181 		intel_de_write(dev_priv, DPFC_CONTROL, dpfc_ctl);  in g4x_fbc_deactivate()
185 static bool g4x_fbc_is_active(struct drm_i915_private *dev_priv)  in g4x_fbc_is_active()  argument
187 	return intel_de_read(dev_priv, DPFC_CONTROL) & DPFC_CTL_EN;  in g4x_fbc_is_active()
190 static void i8xx_fbc_recompress(struct drm_i915_private *dev_priv)  in i8xx_fbc_recompress()  argument
192 	struct intel_fbc_reg_params *params = &dev_priv->fbc.params;  in i8xx_fbc_recompress()
195 	spin_lock_irq(&dev_priv->uncore.lock);  in i8xx_fbc_recompress()
196 	intel_de_write_fw(dev_priv, DSPADDR(i9xx_plane),  in i8xx_fbc_recompress()
197 			  intel_de_read_fw(dev_priv, DSPADDR(i9xx_plane)));  in i8xx_fbc_recompress()
198 	spin_unlock_irq(&dev_priv->uncore.lock);  in i8xx_fbc_recompress()
201 static void i965_fbc_recompress(struct drm_i915_private *dev_priv)  in i965_fbc_recompress()  argument
203 	struct intel_fbc_reg_params *params = &dev_priv->fbc.params;  in i965_fbc_recompress()
206 	spin_lock_irq(&dev_priv->uncore.lock);  in i965_fbc_recompress()
207 	intel_de_write_fw(dev_priv, DSPSURF(i9xx_plane),  in i965_fbc_recompress()
208 			  intel_de_read_fw(dev_priv, DSPSURF(i9xx_plane)));  in i965_fbc_recompress()
209 	spin_unlock_irq(&dev_priv->uncore.lock);  in i965_fbc_recompress()
213 static void snb_fbc_recompress(struct drm_i915_private *dev_priv)  in snb_fbc_recompress()  argument
215 	struct intel_fbc *fbc = &dev_priv->fbc;  in snb_fbc_recompress()
219 	intel_de_write(dev_priv, MSG_FBC_REND_STATE, FBC_REND_NUKE);  in snb_fbc_recompress()
220 	intel_de_posting_read(dev_priv, MSG_FBC_REND_STATE);  in snb_fbc_recompress()
223 static void intel_fbc_recompress(struct drm_i915_private *dev_priv)  in intel_fbc_recompress()  argument
225 	if (INTEL_GEN(dev_priv) >= 6)  in intel_fbc_recompress()
226 		snb_fbc_recompress(dev_priv);  in intel_fbc_recompress()
227 	else if (INTEL_GEN(dev_priv) >= 4)  in intel_fbc_recompress()
228 		i965_fbc_recompress(dev_priv);  in intel_fbc_recompress()
230 		i8xx_fbc_recompress(dev_priv);  in intel_fbc_recompress()
233 static void ilk_fbc_activate(struct drm_i915_private *dev_priv)  in ilk_fbc_activate()  argument
235 	struct intel_fbc_reg_params *params = &dev_priv->fbc.params;  in ilk_fbc_activate()
237 	int threshold = dev_priv->fbc.threshold;  in ilk_fbc_activate()
258 		if (IS_GEN(dev_priv, 5))  in ilk_fbc_activate()
260 		if (IS_GEN(dev_priv, 6)) {  in ilk_fbc_activate()
261 			intel_de_write(dev_priv, SNB_DPFC_CTL_SA,  in ilk_fbc_activate()
263 			intel_de_write(dev_priv, DPFC_CPU_FENCE_OFFSET,  in ilk_fbc_activate()
267 		if (IS_GEN(dev_priv, 6)) {  in ilk_fbc_activate()
268 			intel_de_write(dev_priv, SNB_DPFC_CTL_SA, 0);  in ilk_fbc_activate()
269 			intel_de_write(dev_priv, DPFC_CPU_FENCE_OFFSET, 0);  in ilk_fbc_activate()
273 	intel_de_write(dev_priv, ILK_DPFC_FENCE_YOFF,  in ilk_fbc_activate()
276 	intel_de_write(dev_priv, ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN);  in ilk_fbc_activate()
278 	intel_fbc_recompress(dev_priv);  in ilk_fbc_activate()
281 static void ilk_fbc_deactivate(struct drm_i915_private *dev_priv)  in ilk_fbc_deactivate()  argument
286 	dpfc_ctl = intel_de_read(dev_priv, ILK_DPFC_CONTROL);  in ilk_fbc_deactivate()
289 		intel_de_write(dev_priv, ILK_DPFC_CONTROL, dpfc_ctl);  in ilk_fbc_deactivate()
293 static bool ilk_fbc_is_active(struct drm_i915_private *dev_priv)  in ilk_fbc_is_active()  argument
295 	return intel_de_read(dev_priv, ILK_DPFC_CONTROL) & DPFC_CTL_EN;  in ilk_fbc_is_active()
298 static void gen7_fbc_activate(struct drm_i915_private *dev_priv)  in gen7_fbc_activate()  argument
300 	struct intel_fbc_reg_params *params = &dev_priv->fbc.params;  in gen7_fbc_activate()
302 	int threshold = dev_priv->fbc.threshold;  in gen7_fbc_activate()
305 	if (IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv)) {  in gen7_fbc_activate()
306 		u32 val = intel_de_read(dev_priv, CHICKEN_MISC_4);  in gen7_fbc_activate()
313 		intel_de_write(dev_priv, CHICKEN_MISC_4, val);  in gen7_fbc_activate()
317 	if (IS_IVYBRIDGE(dev_priv))  in gen7_fbc_activate()
338 		intel_de_write(dev_priv, SNB_DPFC_CTL_SA,  in gen7_fbc_activate()
340 		intel_de_write(dev_priv, DPFC_CPU_FENCE_OFFSET,  in gen7_fbc_activate()
342 	} else if (dev_priv->ggtt.num_fences) {  in gen7_fbc_activate()
343 		intel_de_write(dev_priv, SNB_DPFC_CTL_SA, 0);  in gen7_fbc_activate()
344 		intel_de_write(dev_priv, DPFC_CPU_FENCE_OFFSET, 0);  in gen7_fbc_activate()
347 	if (dev_priv->fbc.false_color)  in gen7_fbc_activate()
350 	intel_de_write(dev_priv, ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN);  in gen7_fbc_activate()
352 	intel_fbc_recompress(dev_priv);  in gen7_fbc_activate()
355 static bool intel_fbc_hw_is_active(struct drm_i915_private *dev_priv)  in intel_fbc_hw_is_active()  argument
357 	if (INTEL_GEN(dev_priv) >= 5)  in intel_fbc_hw_is_active()
358 		return ilk_fbc_is_active(dev_priv);  in intel_fbc_hw_is_active()
359 	else if (IS_GM45(dev_priv))  in intel_fbc_hw_is_active()
360 		return g4x_fbc_is_active(dev_priv);  in intel_fbc_hw_is_active()
362 		return i8xx_fbc_is_active(dev_priv);  in intel_fbc_hw_is_active()
365 static void intel_fbc_hw_activate(struct drm_i915_private *dev_priv)  in intel_fbc_hw_activate()  argument
367 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_hw_activate()
374 	if (INTEL_GEN(dev_priv) >= 7)  in intel_fbc_hw_activate()
375 		gen7_fbc_activate(dev_priv);  in intel_fbc_hw_activate()
376 	else if (INTEL_GEN(dev_priv) >= 5)  in intel_fbc_hw_activate()
377 		ilk_fbc_activate(dev_priv);  in intel_fbc_hw_activate()
378 	else if (IS_GM45(dev_priv))  in intel_fbc_hw_activate()
379 		g4x_fbc_activate(dev_priv);  in intel_fbc_hw_activate()
381 		i8xx_fbc_activate(dev_priv);  in intel_fbc_hw_activate()
384 static void intel_fbc_hw_deactivate(struct drm_i915_private *dev_priv)  in intel_fbc_hw_deactivate()  argument
386 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_hw_deactivate()
392 	if (INTEL_GEN(dev_priv) >= 5)  in intel_fbc_hw_deactivate()
393 		ilk_fbc_deactivate(dev_priv);  in intel_fbc_hw_deactivate()
394 	else if (IS_GM45(dev_priv))  in intel_fbc_hw_deactivate()
395 		g4x_fbc_deactivate(dev_priv);  in intel_fbc_hw_deactivate()
397 		i8xx_fbc_deactivate(dev_priv);  in intel_fbc_hw_deactivate()
409 bool intel_fbc_is_active(struct drm_i915_private *dev_priv)  in intel_fbc_is_active()  argument
411 	return dev_priv->fbc.active;  in intel_fbc_is_active()
414 static void intel_fbc_deactivate(struct drm_i915_private *dev_priv,  in intel_fbc_deactivate()  argument
417 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_deactivate()
419 	drm_WARN_ON(&dev_priv->drm, !mutex_is_locked(&fbc->lock));  in intel_fbc_deactivate()
422 		intel_fbc_hw_deactivate(dev_priv);  in intel_fbc_deactivate()
435 static int find_compression_threshold(struct drm_i915_private *dev_priv,  in find_compression_threshold()  argument
448 	if (IS_BROADWELL(dev_priv) || IS_GEN9_BC(dev_priv))  in find_compression_threshold()
449 		end = resource_size(&dev_priv->dsm) - 8 * 1024 * 1024;  in find_compression_threshold()
453 	end = min(end, intel_fbc_cfb_base_max(dev_priv));  in find_compression_threshold()
463 	ret = i915_gem_stolen_insert_node_in_range(dev_priv, node, size <<= 1,  in find_compression_threshold()
474 	ret = i915_gem_stolen_insert_node_in_range(dev_priv, node, size >>= 1,  in find_compression_threshold()
476 	if (ret && INTEL_GEN(dev_priv) <= 4) {  in find_compression_threshold()
486 static int intel_fbc_alloc_cfb(struct drm_i915_private *dev_priv,  in intel_fbc_alloc_cfb()  argument
489 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_alloc_cfb()
493 	drm_WARN_ON(&dev_priv->drm,  in intel_fbc_alloc_cfb()
496 	ret = find_compression_threshold(dev_priv, &fbc->compressed_fb,  in intel_fbc_alloc_cfb()
501 		drm_info_once(&dev_priv->drm,  in intel_fbc_alloc_cfb()
507 	if (INTEL_GEN(dev_priv) >= 5)  in intel_fbc_alloc_cfb()
508 		intel_de_write(dev_priv, ILK_DPFC_CB_BASE,  in intel_fbc_alloc_cfb()
510 	else if (IS_GM45(dev_priv)) {  in intel_fbc_alloc_cfb()
511 		intel_de_write(dev_priv, DPFC_CB_BASE,  in intel_fbc_alloc_cfb()
518 		ret = i915_gem_stolen_insert_node(dev_priv, compressed_llb,  in intel_fbc_alloc_cfb()
525 		GEM_BUG_ON(range_overflows_end_t(u64, dev_priv->dsm.start,  in intel_fbc_alloc_cfb()
528 		GEM_BUG_ON(range_overflows_end_t(u64, dev_priv->dsm.start,  in intel_fbc_alloc_cfb()
531 		intel_de_write(dev_priv, FBC_CFB_BASE,  in intel_fbc_alloc_cfb()
532 			       dev_priv->dsm.start + fbc->compressed_fb.start);  in intel_fbc_alloc_cfb()
533 		intel_de_write(dev_priv, FBC_LL_BASE,  in intel_fbc_alloc_cfb()
534 			       dev_priv->dsm.start + compressed_llb->start);  in intel_fbc_alloc_cfb()
537 	drm_dbg_kms(&dev_priv->drm,  in intel_fbc_alloc_cfb()
545 	i915_gem_stolen_remove_node(dev_priv, &fbc->compressed_fb);  in intel_fbc_alloc_cfb()
547 	if (drm_mm_initialized(&dev_priv->mm.stolen))  in intel_fbc_alloc_cfb()
548 …drm_info_once(&dev_priv->drm, "not enough stolen space for compressed buffer (need %d more bytes),…  in intel_fbc_alloc_cfb()
552 static void __intel_fbc_cleanup_cfb(struct drm_i915_private *dev_priv)  in __intel_fbc_cleanup_cfb()  argument
554 	struct intel_fbc *fbc = &dev_priv->fbc;  in __intel_fbc_cleanup_cfb()
556 	if (WARN_ON(intel_fbc_hw_is_active(dev_priv)))  in __intel_fbc_cleanup_cfb()
563 		i915_gem_stolen_remove_node(dev_priv, fbc->compressed_llb);  in __intel_fbc_cleanup_cfb()
567 	i915_gem_stolen_remove_node(dev_priv, &fbc->compressed_fb);  in __intel_fbc_cleanup_cfb()
570 void intel_fbc_cleanup_cfb(struct drm_i915_private *dev_priv)  in intel_fbc_cleanup_cfb()  argument
572 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_cleanup_cfb()
574 	if (!HAS_FBC(dev_priv))  in intel_fbc_cleanup_cfb()
578 	__intel_fbc_cleanup_cfb(dev_priv);  in intel_fbc_cleanup_cfb()
582 static bool stride_is_valid(struct drm_i915_private *dev_priv,  in stride_is_valid()  argument
586 	if (drm_WARN_ON_ONCE(&dev_priv->drm, (stride & (64 - 1)) != 0))  in stride_is_valid()
593 	if (IS_GEN(dev_priv, 2) || IS_GEN(dev_priv, 3))  in stride_is_valid()
596 	if (IS_GEN(dev_priv, 4) && !IS_G4X(dev_priv) && stride < 2048)  in stride_is_valid()
600 	if (IS_GEN(dev_priv, 9) &&  in stride_is_valid()
610 static bool pixel_format_is_valid(struct drm_i915_private *dev_priv,  in pixel_format_is_valid()  argument
620 		if (IS_GEN(dev_priv, 2))  in pixel_format_is_valid()
623 		if (IS_G4X(dev_priv))  in pixel_format_is_valid()
631 static bool rotation_is_valid(struct drm_i915_private *dev_priv,  in rotation_is_valid()  argument
634 	if (INTEL_GEN(dev_priv) >= 9 && pixel_format == DRM_FORMAT_RGB565 &&  in rotation_is_valid()
637 	else if (INTEL_GEN(dev_priv) <= 4 && !IS_G4X(dev_priv) &&  in rotation_is_valid()
652 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);  in intel_fbc_hw_tracking_covers_screen()  local
653 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_hw_tracking_covers_screen()
656 	if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) {  in intel_fbc_hw_tracking_covers_screen()
659 	} else if (INTEL_GEN(dev_priv) >= 8 || IS_HASWELL(dev_priv)) {  in intel_fbc_hw_tracking_covers_screen()
662 	} else if (IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5) {  in intel_fbc_hw_tracking_covers_screen()
678 static bool tiling_is_valid(struct drm_i915_private *dev_priv,  in tiling_is_valid()  argument
683 		if (INTEL_GEN(dev_priv) >= 9)  in tiling_is_valid()
698 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);  in intel_fbc_update_state_cache()  local
699 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_update_state_cache()
708 	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))  in intel_fbc_update_state_cache()
737 	drm_WARN_ON(&dev_priv->drm, plane_state->flags & PLANE_HAS_FENCE &&  in intel_fbc_update_state_cache()
747 static bool intel_fbc_cfb_size_changed(struct drm_i915_private *dev_priv)  in intel_fbc_cfb_size_changed()  argument
749 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_cfb_size_changed()
751 	return intel_fbc_calculate_cfb_size(dev_priv, &fbc->state_cache) >  in intel_fbc_cfb_size_changed()
755 static u16 intel_fbc_gen9_wa_cfb_stride(struct drm_i915_private *dev_priv)  in intel_fbc_gen9_wa_cfb_stride()  argument
757 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_gen9_wa_cfb_stride()
760 	if ((IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv)) &&  in intel_fbc_gen9_wa_cfb_stride()
767 static bool intel_fbc_gen9_wa_cfb_stride_changed(struct drm_i915_private *dev_priv)  in intel_fbc_gen9_wa_cfb_stride_changed()  argument
769 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_gen9_wa_cfb_stride_changed()
771 	return fbc->params.gen9_wa_cfb_stride != intel_fbc_gen9_wa_cfb_stride(dev_priv);  in intel_fbc_gen9_wa_cfb_stride_changed()
774 static bool intel_fbc_can_enable(struct drm_i915_private *dev_priv)  in intel_fbc_can_enable()  argument
776 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_can_enable()
778 	if (intel_vgpu_active(dev_priv)) {  in intel_fbc_can_enable()
783 	if (!dev_priv->params.enable_fbc) {  in intel_fbc_can_enable()
798 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);  in intel_fbc_can_activate()  local
799 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_can_activate()
802 	if (!intel_fbc_can_enable(dev_priv))  in intel_fbc_can_activate()
845 	if (INTEL_GEN(dev_priv) < 9 && cache->fence_id < 0) {  in intel_fbc_can_activate()
850 	if (!pixel_format_is_valid(dev_priv, cache->fb.format->format)) {  in intel_fbc_can_activate()
855 	if (!rotation_is_valid(dev_priv, cache->fb.format->format,  in intel_fbc_can_activate()
861 	if (!tiling_is_valid(dev_priv, cache->fb.modifier)) {  in intel_fbc_can_activate()
866 	if (!stride_is_valid(dev_priv, cache->fb.modifier, cache->fb.stride)) {  in intel_fbc_can_activate()
878 	if ((IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) &&  in intel_fbc_can_activate()
879 	    cache->crtc.hsw_bdw_pixel_rate >= dev_priv->cdclk.hw.cdclk * 95 / 100) {  in intel_fbc_can_activate()
894 	if (intel_fbc_cfb_size_changed(dev_priv)) {  in intel_fbc_can_activate()
904 	if (INTEL_GEN(dev_priv) >= 9 &&  in intel_fbc_can_activate()
916 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);  in intel_fbc_get_reg_params()  local
917 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_get_reg_params()
937 	params->cfb_size = intel_fbc_calculate_cfb_size(dev_priv, cache);  in intel_fbc_get_reg_params()
947 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);  in intel_fbc_can_flip_nuke()  local
948 	const struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_can_flip_nuke()
970 	if (params->cfb_size != intel_fbc_calculate_cfb_size(dev_priv, cache))  in intel_fbc_can_flip_nuke()
987 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);  in intel_fbc_pre_update()  local
988 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_pre_update()
1004 		intel_fbc_deactivate(dev_priv, reason);  in intel_fbc_pre_update()
1020 		    (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)))  in intel_fbc_pre_update()
1037 static void __intel_fbc_disable(struct drm_i915_private *dev_priv)  in __intel_fbc_disable()  argument
1039 	struct intel_fbc *fbc = &dev_priv->fbc;  in __intel_fbc_disable()
1042 	drm_WARN_ON(&dev_priv->drm, !mutex_is_locked(&fbc->lock));  in __intel_fbc_disable()
1043 	drm_WARN_ON(&dev_priv->drm, !fbc->crtc);  in __intel_fbc_disable()
1044 	drm_WARN_ON(&dev_priv->drm, fbc->active);  in __intel_fbc_disable()
1046 	drm_dbg_kms(&dev_priv->drm, "Disabling FBC on pipe %c\n",  in __intel_fbc_disable()
1049 	__intel_fbc_cleanup_cfb(dev_priv);  in __intel_fbc_disable()
1056 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);  in __intel_fbc_post_update()  local
1057 	struct intel_fbc *fbc = &dev_priv->fbc;  in __intel_fbc_post_update()
1059 	drm_WARN_ON(&dev_priv->drm, !mutex_is_locked(&fbc->lock));  in __intel_fbc_post_update()
1066 	if (!dev_priv->params.enable_fbc) {  in __intel_fbc_post_update()
1067 		intel_fbc_deactivate(dev_priv, "disabled at runtime per module param");  in __intel_fbc_post_update()
1068 		__intel_fbc_disable(dev_priv);  in __intel_fbc_post_update()
1079 		intel_fbc_hw_activate(dev_priv);  in __intel_fbc_post_update()
1081 		intel_fbc_deactivate(dev_priv, "frontbuffer write");  in __intel_fbc_post_update()
1087 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);  in intel_fbc_post_update()  local
1091 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_post_update()
1109 void intel_fbc_invalidate(struct drm_i915_private *dev_priv,  in intel_fbc_invalidate()  argument
1113 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_invalidate()
1115 	if (!HAS_FBC(dev_priv))  in intel_fbc_invalidate()
1126 		intel_fbc_deactivate(dev_priv, "frontbuffer write");  in intel_fbc_invalidate()
1131 void intel_fbc_flush(struct drm_i915_private *dev_priv,  in intel_fbc_flush()  argument
1134 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_flush()
1136 	if (!HAS_FBC(dev_priv))  in intel_fbc_flush()
1157 			intel_fbc_recompress(dev_priv);  in intel_fbc_flush()
1178 void intel_fbc_choose_crtc(struct drm_i915_private *dev_priv,  in intel_fbc_choose_crtc()  argument
1181 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_choose_crtc()
1194 	if (!intel_fbc_can_enable(dev_priv))  in intel_fbc_choose_crtc()
1238 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);  in intel_fbc_enable()  local
1244 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_enable()
1254 		    (!intel_fbc_cfb_size_changed(dev_priv) &&  in intel_fbc_enable()
1255 		     !intel_fbc_gen9_wa_cfb_stride_changed(dev_priv)))  in intel_fbc_enable()
1258 		__intel_fbc_disable(dev_priv);  in intel_fbc_enable()
1261 	drm_WARN_ON(&dev_priv->drm, fbc->active);  in intel_fbc_enable()
1269 	if (intel_fbc_alloc_cfb(dev_priv,  in intel_fbc_enable()
1270 				intel_fbc_calculate_cfb_size(dev_priv, cache),  in intel_fbc_enable()
1277 	cache->gen9_wa_cfb_stride = intel_fbc_gen9_wa_cfb_stride(dev_priv);  in intel_fbc_enable()
1279 	drm_dbg_kms(&dev_priv->drm, "Enabling FBC on pipe %c\n",  in intel_fbc_enable()
1296 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);  in intel_fbc_disable()  local
1298 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_disable()
1305 		__intel_fbc_disable(dev_priv);  in intel_fbc_disable()
1315 void intel_fbc_global_disable(struct drm_i915_private *dev_priv)  in intel_fbc_global_disable()  argument
1317 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_global_disable()
1319 	if (!HAS_FBC(dev_priv))  in intel_fbc_global_disable()
1324 		drm_WARN_ON(&dev_priv->drm, fbc->crtc->active);  in intel_fbc_global_disable()
1325 		__intel_fbc_disable(dev_priv);  in intel_fbc_global_disable()
1332 	struct drm_i915_private *dev_priv =  in intel_fbc_underrun_work_fn()  local
1334 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_underrun_work_fn()
1342 	drm_dbg_kms(&dev_priv->drm, "Disabling FBC due to FIFO underrun.\n");  in intel_fbc_underrun_work_fn()
1345 	intel_fbc_deactivate(dev_priv, "FIFO underrun");  in intel_fbc_underrun_work_fn()
1357 int intel_fbc_reset_underrun(struct drm_i915_private *dev_priv)  in intel_fbc_reset_underrun()  argument
1361 	cancel_work_sync(&dev_priv->fbc.underrun_work);  in intel_fbc_reset_underrun()
1363 	ret = mutex_lock_interruptible(&dev_priv->fbc.lock);  in intel_fbc_reset_underrun()
1367 	if (dev_priv->fbc.underrun_detected) {  in intel_fbc_reset_underrun()
1368 		drm_dbg_kms(&dev_priv->drm,  in intel_fbc_reset_underrun()
1370 		dev_priv->fbc.no_fbc_reason = "FIFO underrun cleared";  in intel_fbc_reset_underrun()
1373 	dev_priv->fbc.underrun_detected = false;  in intel_fbc_reset_underrun()
1374 	mutex_unlock(&dev_priv->fbc.lock);  in intel_fbc_reset_underrun()
1393 void intel_fbc_handle_fifo_underrun_irq(struct drm_i915_private *dev_priv)  in intel_fbc_handle_fifo_underrun_irq()  argument
1395 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_handle_fifo_underrun_irq()
1397 	if (!HAS_FBC(dev_priv))  in intel_fbc_handle_fifo_underrun_irq()
1421 static int intel_sanitize_fbc_option(struct drm_i915_private *dev_priv)  in intel_sanitize_fbc_option()  argument
1423 	if (dev_priv->params.enable_fbc >= 0)  in intel_sanitize_fbc_option()
1424 		return !!dev_priv->params.enable_fbc;  in intel_sanitize_fbc_option()
1426 	if (!HAS_FBC(dev_priv))  in intel_sanitize_fbc_option()
1433 	if (IS_TIGERLAKE(dev_priv))  in intel_sanitize_fbc_option()
1436 	if (IS_BROADWELL(dev_priv) || INTEL_GEN(dev_priv) >= 9)  in intel_sanitize_fbc_option()
1442 static bool need_fbc_vtd_wa(struct drm_i915_private *dev_priv)  in need_fbc_vtd_wa()  argument
1446 	    (IS_SKYLAKE(dev_priv) || IS_BROXTON(dev_priv))) {  in need_fbc_vtd_wa()
1447 		drm_info(&dev_priv->drm,  in need_fbc_vtd_wa()
1461 void intel_fbc_init(struct drm_i915_private *dev_priv)  in intel_fbc_init()  argument
1463 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_init()
1469 	if (!drm_mm_initialized(&dev_priv->mm.stolen))  in intel_fbc_init()
1470 		mkwrite_device_info(dev_priv)->display.has_fbc = false;  in intel_fbc_init()
1472 	if (need_fbc_vtd_wa(dev_priv))  in intel_fbc_init()
1473 		mkwrite_device_info(dev_priv)->display.has_fbc = false;  in intel_fbc_init()
1475 	dev_priv->params.enable_fbc = intel_sanitize_fbc_option(dev_priv);  in intel_fbc_init()
1476 	drm_dbg_kms(&dev_priv->drm, "Sanitized enable_fbc value: %d\n",  in intel_fbc_init()
1477 		    dev_priv->params.enable_fbc);  in intel_fbc_init()
1479 	if (!HAS_FBC(dev_priv)) {  in intel_fbc_init()
1487 	if (intel_fbc_hw_is_active(dev_priv))  in intel_fbc_init()
1488 		intel_fbc_hw_deactivate(dev_priv);  in intel_fbc_init()