65 static int intel_fbc_calculate_cfb_size(struct drm_i915_private *dev_priv,  in intel_fbc_calculate_cfb_size()  argument
71 	if (DISPLAY_VER(dev_priv) == 7)  in intel_fbc_calculate_cfb_size()
73 	else if (DISPLAY_VER(dev_priv) >= 8)  in intel_fbc_calculate_cfb_size()
80 static void i8xx_fbc_deactivate(struct drm_i915_private *dev_priv)  in i8xx_fbc_deactivate()  argument
85 	fbc_ctl = intel_de_read(dev_priv, FBC_CONTROL);  in i8xx_fbc_deactivate()
90 	intel_de_write(dev_priv, FBC_CONTROL, fbc_ctl);  in i8xx_fbc_deactivate()
93 	if (intel_de_wait_for_clear(dev_priv, FBC_STATUS,  in i8xx_fbc_deactivate()
95 		drm_dbg_kms(&dev_priv->drm, "FBC idle timed out\n");  in i8xx_fbc_deactivate()
100 static void i8xx_fbc_activate(struct drm_i915_private *dev_priv)  in i8xx_fbc_activate()  argument
102 	struct intel_fbc_reg_params *params = &dev_priv->fbc.params;  in i8xx_fbc_activate()
113 	if (DISPLAY_VER(dev_priv) == 2)  in i8xx_fbc_activate()
120 		intel_de_write(dev_priv, FBC_TAG(i), 0);  in i8xx_fbc_activate()
122 	if (DISPLAY_VER(dev_priv) == 4) {  in i8xx_fbc_activate()
130 		intel_de_write(dev_priv, FBC_CONTROL2, fbc_ctl2);  in i8xx_fbc_activate()
131 		intel_de_write(dev_priv, FBC_FENCE_OFF,  in i8xx_fbc_activate()
138 	if (IS_I945GM(dev_priv))  in i8xx_fbc_activate()
143 	intel_de_write(dev_priv, FBC_CONTROL, fbc_ctl);  in i8xx_fbc_activate()
146 static bool i8xx_fbc_is_active(struct drm_i915_private *dev_priv)  in i8xx_fbc_is_active()  argument
148 	return intel_de_read(dev_priv, FBC_CONTROL) & FBC_CTL_EN;  in i8xx_fbc_is_active()
172 static void g4x_fbc_activate(struct drm_i915_private *dev_priv)  in g4x_fbc_activate()  argument
174 	struct intel_fbc_reg_params *params = &dev_priv->fbc.params;  in g4x_fbc_activate()
179 	dpfc_ctl |= g4x_dpfc_ctl_limit(dev_priv);  in g4x_fbc_activate()
183 		intel_de_write(dev_priv, DPFC_FENCE_YOFF,  in g4x_fbc_activate()
186 		intel_de_write(dev_priv, DPFC_FENCE_YOFF, 0);  in g4x_fbc_activate()
190 	intel_de_write(dev_priv, DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN);  in g4x_fbc_activate()
193 static void g4x_fbc_deactivate(struct drm_i915_private *dev_priv)  in g4x_fbc_deactivate()  argument
198 	dpfc_ctl = intel_de_read(dev_priv, DPFC_CONTROL);  in g4x_fbc_deactivate()
201 		intel_de_write(dev_priv, DPFC_CONTROL, dpfc_ctl);  in g4x_fbc_deactivate()
205 static bool g4x_fbc_is_active(struct drm_i915_private *dev_priv)  in g4x_fbc_is_active()  argument
207 	return intel_de_read(dev_priv, DPFC_CONTROL) & DPFC_CTL_EN;  in g4x_fbc_is_active()
210 static void i8xx_fbc_recompress(struct drm_i915_private *dev_priv)  in i8xx_fbc_recompress()  argument
212 	struct intel_fbc_reg_params *params = &dev_priv->fbc.params;  in i8xx_fbc_recompress()
215 	spin_lock_irq(&dev_priv->uncore.lock);  in i8xx_fbc_recompress()
216 	intel_de_write_fw(dev_priv, DSPADDR(i9xx_plane),  in i8xx_fbc_recompress()
217 			  intel_de_read_fw(dev_priv, DSPADDR(i9xx_plane)));  in i8xx_fbc_recompress()
218 	spin_unlock_irq(&dev_priv->uncore.lock);  in i8xx_fbc_recompress()
221 static void i965_fbc_recompress(struct drm_i915_private *dev_priv)  in i965_fbc_recompress()  argument
223 	struct intel_fbc_reg_params *params = &dev_priv->fbc.params;  in i965_fbc_recompress()
226 	spin_lock_irq(&dev_priv->uncore.lock);  in i965_fbc_recompress()
227 	intel_de_write_fw(dev_priv, DSPSURF(i9xx_plane),  in i965_fbc_recompress()
228 			  intel_de_read_fw(dev_priv, DSPSURF(i9xx_plane)));  in i965_fbc_recompress()
229 	spin_unlock_irq(&dev_priv->uncore.lock);  in i965_fbc_recompress()
233 static void snb_fbc_recompress(struct drm_i915_private *dev_priv)  in snb_fbc_recompress()  argument
235 	struct intel_fbc *fbc = &dev_priv->fbc;  in snb_fbc_recompress()
239 	intel_de_write(dev_priv, MSG_FBC_REND_STATE, FBC_REND_NUKE);  in snb_fbc_recompress()
240 	intel_de_posting_read(dev_priv, MSG_FBC_REND_STATE);  in snb_fbc_recompress()
243 static void intel_fbc_recompress(struct drm_i915_private *dev_priv)  in intel_fbc_recompress()  argument
245 	if (DISPLAY_VER(dev_priv) >= 6)  in intel_fbc_recompress()
246 		snb_fbc_recompress(dev_priv);  in intel_fbc_recompress()
247 	else if (DISPLAY_VER(dev_priv) >= 4)  in intel_fbc_recompress()
248 		i965_fbc_recompress(dev_priv);  in intel_fbc_recompress()
250 		i8xx_fbc_recompress(dev_priv);  in intel_fbc_recompress()
253 static void ilk_fbc_activate(struct drm_i915_private *dev_priv)  in ilk_fbc_activate()  argument
255 	struct intel_fbc_reg_params *params = &dev_priv->fbc.params;  in ilk_fbc_activate()
260 	dpfc_ctl |= g4x_dpfc_ctl_limit(dev_priv);  in ilk_fbc_activate()
264 		if (IS_IRONLAKE(dev_priv))  in ilk_fbc_activate()
266 		if (IS_SANDYBRIDGE(dev_priv)) {  in ilk_fbc_activate()
267 			intel_de_write(dev_priv, SNB_DPFC_CTL_SA,  in ilk_fbc_activate()
269 			intel_de_write(dev_priv, DPFC_CPU_FENCE_OFFSET,  in ilk_fbc_activate()
273 		if (IS_SANDYBRIDGE(dev_priv)) {  in ilk_fbc_activate()
274 			intel_de_write(dev_priv, SNB_DPFC_CTL_SA, 0);  in ilk_fbc_activate()
275 			intel_de_write(dev_priv, DPFC_CPU_FENCE_OFFSET, 0);  in ilk_fbc_activate()
279 	intel_de_write(dev_priv, ILK_DPFC_FENCE_YOFF,  in ilk_fbc_activate()
282 	intel_de_write(dev_priv, ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN);  in ilk_fbc_activate()
284 	intel_fbc_recompress(dev_priv);  in ilk_fbc_activate()
287 static void ilk_fbc_deactivate(struct drm_i915_private *dev_priv)  in ilk_fbc_deactivate()  argument
292 	dpfc_ctl = intel_de_read(dev_priv, ILK_DPFC_CONTROL);  in ilk_fbc_deactivate()
295 		intel_de_write(dev_priv, ILK_DPFC_CONTROL, dpfc_ctl);  in ilk_fbc_deactivate()
299 static bool ilk_fbc_is_active(struct drm_i915_private *dev_priv)  in ilk_fbc_is_active()  argument
301 	return intel_de_read(dev_priv, ILK_DPFC_CONTROL) & DPFC_CTL_EN;  in ilk_fbc_is_active()
304 static void gen7_fbc_activate(struct drm_i915_private *dev_priv)  in gen7_fbc_activate()  argument
306 	struct intel_fbc_reg_params *params = &dev_priv->fbc.params;  in gen7_fbc_activate()
310 	if (DISPLAY_VER(dev_priv) == 9) {  in gen7_fbc_activate()
311 		u32 val = intel_de_read(dev_priv, CHICKEN_MISC_4);  in gen7_fbc_activate()
318 		intel_de_write(dev_priv, CHICKEN_MISC_4, val);  in gen7_fbc_activate()
322 	if (IS_IVYBRIDGE(dev_priv))  in gen7_fbc_activate()
325 	dpfc_ctl |= g4x_dpfc_ctl_limit(dev_priv);  in gen7_fbc_activate()
329 		intel_de_write(dev_priv, SNB_DPFC_CTL_SA,  in gen7_fbc_activate()
331 		intel_de_write(dev_priv, DPFC_CPU_FENCE_OFFSET,  in gen7_fbc_activate()
333 	} else if (dev_priv->ggtt.num_fences) {  in gen7_fbc_activate()
334 		intel_de_write(dev_priv, SNB_DPFC_CTL_SA, 0);  in gen7_fbc_activate()
335 		intel_de_write(dev_priv, DPFC_CPU_FENCE_OFFSET, 0);  in gen7_fbc_activate()
338 	if (dev_priv->fbc.false_color)  in gen7_fbc_activate()
341 	intel_de_write(dev_priv, ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN);  in gen7_fbc_activate()
343 	intel_fbc_recompress(dev_priv);  in gen7_fbc_activate()
346 static bool intel_fbc_hw_is_active(struct drm_i915_private *dev_priv)  in intel_fbc_hw_is_active()  argument
348 	if (DISPLAY_VER(dev_priv) >= 5)  in intel_fbc_hw_is_active()
349 		return ilk_fbc_is_active(dev_priv);  in intel_fbc_hw_is_active()
350 	else if (IS_GM45(dev_priv))  in intel_fbc_hw_is_active()
351 		return g4x_fbc_is_active(dev_priv);  in intel_fbc_hw_is_active()
353 		return i8xx_fbc_is_active(dev_priv);  in intel_fbc_hw_is_active()
356 static void intel_fbc_hw_activate(struct drm_i915_private *dev_priv)  in intel_fbc_hw_activate()  argument
358 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_hw_activate()
365 	if (DISPLAY_VER(dev_priv) >= 7)  in intel_fbc_hw_activate()
366 		gen7_fbc_activate(dev_priv);  in intel_fbc_hw_activate()
367 	else if (DISPLAY_VER(dev_priv) >= 5)  in intel_fbc_hw_activate()
368 		ilk_fbc_activate(dev_priv);  in intel_fbc_hw_activate()
369 	else if (IS_GM45(dev_priv))  in intel_fbc_hw_activate()
370 		g4x_fbc_activate(dev_priv);  in intel_fbc_hw_activate()
372 		i8xx_fbc_activate(dev_priv);  in intel_fbc_hw_activate()
375 static void intel_fbc_hw_deactivate(struct drm_i915_private *dev_priv)  in intel_fbc_hw_deactivate()  argument
377 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_hw_deactivate()
383 	if (DISPLAY_VER(dev_priv) >= 5)  in intel_fbc_hw_deactivate()
384 		ilk_fbc_deactivate(dev_priv);  in intel_fbc_hw_deactivate()
385 	else if (IS_GM45(dev_priv))  in intel_fbc_hw_deactivate()
386 		g4x_fbc_deactivate(dev_priv);  in intel_fbc_hw_deactivate()
388 		i8xx_fbc_deactivate(dev_priv);  in intel_fbc_hw_deactivate()
400 bool intel_fbc_is_active(struct drm_i915_private *dev_priv)  in intel_fbc_is_active()  argument
402 	return dev_priv->fbc.active;  in intel_fbc_is_active()
405 static void intel_fbc_deactivate(struct drm_i915_private *dev_priv,  in intel_fbc_deactivate()  argument
408 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_deactivate()
410 	drm_WARN_ON(&dev_priv->drm, !mutex_is_locked(&fbc->lock));  in intel_fbc_deactivate()
413 		intel_fbc_hw_deactivate(dev_priv);  in intel_fbc_deactivate()
426 static u64 intel_fbc_stolen_end(struct drm_i915_private *dev_priv)  in intel_fbc_stolen_end()  argument
434 	if (IS_BROADWELL(dev_priv) || (DISPLAY_VER(dev_priv) == 9 &&  in intel_fbc_stolen_end()
435 				       !IS_BROXTON(dev_priv)))  in intel_fbc_stolen_end()
436 		end = resource_size(&dev_priv->dsm) - 8 * 1024 * 1024;  in intel_fbc_stolen_end()
440 	return min(end, intel_fbc_cfb_base_max(dev_priv));  in intel_fbc_stolen_end()
443 static int intel_fbc_max_limit(struct drm_i915_private *dev_priv, int fb_cpp)  in intel_fbc_max_limit()  argument
449 	if (DISPLAY_VER(dev_priv) < 5 && !IS_G4X(dev_priv))  in intel_fbc_max_limit()
453 	if (IS_G4X(dev_priv))  in intel_fbc_max_limit()
460 static int find_compression_limit(struct drm_i915_private *dev_priv,  in find_compression_limit()  argument
464 	struct intel_fbc *fbc = &dev_priv->fbc;  in find_compression_limit()
465 	u64 end = intel_fbc_stolen_end(dev_priv);  in find_compression_limit()
469 	ret = i915_gem_stolen_insert_node_in_range(dev_priv, &fbc->compressed_fb,  in find_compression_limit()
474 	for (; limit <= intel_fbc_max_limit(dev_priv, fb_cpp); limit <<= 1) {  in find_compression_limit()
475 		ret = i915_gem_stolen_insert_node_in_range(dev_priv, &fbc->compressed_fb,  in find_compression_limit()
484 static int intel_fbc_alloc_cfb(struct drm_i915_private *dev_priv,  in intel_fbc_alloc_cfb()  argument
487 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_alloc_cfb()
490 	drm_WARN_ON(&dev_priv->drm,  in intel_fbc_alloc_cfb()
492 	drm_WARN_ON(&dev_priv->drm,  in intel_fbc_alloc_cfb()
495 	if (DISPLAY_VER(dev_priv) < 5 && !IS_G4X(dev_priv)) {  in intel_fbc_alloc_cfb()
496 		ret = i915_gem_stolen_insert_node(dev_priv, &fbc->compressed_llb,  in intel_fbc_alloc_cfb()
502 	ret = find_compression_limit(dev_priv, size, fb_cpp);  in intel_fbc_alloc_cfb()
506 		drm_info_once(&dev_priv->drm,  in intel_fbc_alloc_cfb()
512 	drm_dbg_kms(&dev_priv->drm,  in intel_fbc_alloc_cfb()
520 		i915_gem_stolen_remove_node(dev_priv, &fbc->compressed_llb);  in intel_fbc_alloc_cfb()
522 	if (drm_mm_initialized(&dev_priv->mm.stolen))  in intel_fbc_alloc_cfb()
523 …drm_info_once(&dev_priv->drm, "not enough stolen space for compressed buffer (need %d more bytes),…  in intel_fbc_alloc_cfb()
527 static void intel_fbc_program_cfb(struct drm_i915_private *dev_priv)  in intel_fbc_program_cfb()  argument
529 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_program_cfb()
531 	if (DISPLAY_VER(dev_priv) >= 5) {  in intel_fbc_program_cfb()
532 		intel_de_write(dev_priv, ILK_DPFC_CB_BASE,  in intel_fbc_program_cfb()
534 	} else if (IS_GM45(dev_priv)) {  in intel_fbc_program_cfb()
535 		intel_de_write(dev_priv, DPFC_CB_BASE,  in intel_fbc_program_cfb()
538 		GEM_BUG_ON(range_overflows_end_t(u64, dev_priv->dsm.start,  in intel_fbc_program_cfb()
541 		GEM_BUG_ON(range_overflows_end_t(u64, dev_priv->dsm.start,  in intel_fbc_program_cfb()
545 		intel_de_write(dev_priv, FBC_CFB_BASE,  in intel_fbc_program_cfb()
546 			       dev_priv->dsm.start + fbc->compressed_fb.start);  in intel_fbc_program_cfb()
547 		intel_de_write(dev_priv, FBC_LL_BASE,  in intel_fbc_program_cfb()
548 			       dev_priv->dsm.start + fbc->compressed_llb.start);  in intel_fbc_program_cfb()
552 static void __intel_fbc_cleanup_cfb(struct drm_i915_private *dev_priv)  in __intel_fbc_cleanup_cfb()  argument
554 	struct intel_fbc *fbc = &dev_priv->fbc;  in __intel_fbc_cleanup_cfb()
556 	if (WARN_ON(intel_fbc_hw_is_active(dev_priv)))  in __intel_fbc_cleanup_cfb()
560 		i915_gem_stolen_remove_node(dev_priv, &fbc->compressed_llb);  in __intel_fbc_cleanup_cfb()
562 		i915_gem_stolen_remove_node(dev_priv, &fbc->compressed_fb);  in __intel_fbc_cleanup_cfb()
565 void intel_fbc_cleanup_cfb(struct drm_i915_private *dev_priv)  in intel_fbc_cleanup_cfb()  argument
567 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_cleanup_cfb()
569 	if (!HAS_FBC(dev_priv))  in intel_fbc_cleanup_cfb()
573 	__intel_fbc_cleanup_cfb(dev_priv);  in intel_fbc_cleanup_cfb()
577 static bool stride_is_valid(struct drm_i915_private *dev_priv,  in stride_is_valid()  argument
581 	if (drm_WARN_ON_ONCE(&dev_priv->drm, (stride & (64 - 1)) != 0))  in stride_is_valid()
588 	if (DISPLAY_VER(dev_priv) == 2 || DISPLAY_VER(dev_priv) == 3)  in stride_is_valid()
591 	if (DISPLAY_VER(dev_priv) == 4 && !IS_G4X(dev_priv) && stride < 2048)  in stride_is_valid()
595 	if ((DISPLAY_VER(dev_priv) == 9 || IS_GEMINILAKE(dev_priv)) &&  in stride_is_valid()
605 static bool pixel_format_is_valid(struct drm_i915_private *dev_priv,  in pixel_format_is_valid()  argument
615 		if (DISPLAY_VER(dev_priv) == 2)  in pixel_format_is_valid()
618 		if (IS_G4X(dev_priv))  in pixel_format_is_valid()
626 static bool rotation_is_valid(struct drm_i915_private *dev_priv,  in rotation_is_valid()  argument
629 	if (DISPLAY_VER(dev_priv) >= 9 && pixel_format == DRM_FORMAT_RGB565 &&  in rotation_is_valid()
632 	else if (DISPLAY_VER(dev_priv) <= 4 && !IS_G4X(dev_priv) &&  in rotation_is_valid()
647 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);  in intel_fbc_hw_tracking_covers_screen()  local
648 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_hw_tracking_covers_screen()
651 	if (DISPLAY_VER(dev_priv) >= 10) {  in intel_fbc_hw_tracking_covers_screen()
654 	} else if (DISPLAY_VER(dev_priv) >= 8 || IS_HASWELL(dev_priv)) {  in intel_fbc_hw_tracking_covers_screen()
657 	} else if (IS_G4X(dev_priv) || DISPLAY_VER(dev_priv) >= 5) {  in intel_fbc_hw_tracking_covers_screen()
673 static bool tiling_is_valid(struct drm_i915_private *dev_priv,  in tiling_is_valid()  argument
678 		if (DISPLAY_VER(dev_priv) >= 9)  in tiling_is_valid()
693 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);  in intel_fbc_update_state_cache()  local
694 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_update_state_cache()
703 	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))  in intel_fbc_update_state_cache()
732 	drm_WARN_ON(&dev_priv->drm, plane_state->flags & PLANE_HAS_FENCE &&  in intel_fbc_update_state_cache()
744 static bool intel_fbc_cfb_size_changed(struct drm_i915_private *dev_priv)  in intel_fbc_cfb_size_changed()  argument
746 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_cfb_size_changed()
748 	return intel_fbc_calculate_cfb_size(dev_priv, &fbc->state_cache) >  in intel_fbc_cfb_size_changed()
752 static u16 intel_fbc_gen9_wa_cfb_stride(struct drm_i915_private *dev_priv)  in intel_fbc_gen9_wa_cfb_stride()  argument
754 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_gen9_wa_cfb_stride()
757 	if ((DISPLAY_VER(dev_priv) == 9) &&  in intel_fbc_gen9_wa_cfb_stride()
764 static bool intel_fbc_gen9_wa_cfb_stride_changed(struct drm_i915_private *dev_priv)  in intel_fbc_gen9_wa_cfb_stride_changed()  argument
766 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_gen9_wa_cfb_stride_changed()
768 	return fbc->params.gen9_wa_cfb_stride != intel_fbc_gen9_wa_cfb_stride(dev_priv);  in intel_fbc_gen9_wa_cfb_stride_changed()
771 static bool intel_fbc_can_enable(struct drm_i915_private *dev_priv)  in intel_fbc_can_enable()  argument
773 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_can_enable()
775 	if (intel_vgpu_active(dev_priv)) {  in intel_fbc_can_enable()
780 	if (!dev_priv->params.enable_fbc) {  in intel_fbc_can_enable()
795 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);  in intel_fbc_can_activate()  local
796 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_can_activate()
799 	if (!intel_fbc_can_enable(dev_priv))  in intel_fbc_can_activate()
842 	if (DISPLAY_VER(dev_priv) < 9 && cache->fence_id < 0) {  in intel_fbc_can_activate()
847 	if (!pixel_format_is_valid(dev_priv, cache->fb.format->format)) {  in intel_fbc_can_activate()
852 	if (!rotation_is_valid(dev_priv, cache->fb.format->format,  in intel_fbc_can_activate()
858 	if (!tiling_is_valid(dev_priv, cache->fb.modifier)) {  in intel_fbc_can_activate()
863 	if (!stride_is_valid(dev_priv, cache->fb.modifier, cache->fb.stride)) {  in intel_fbc_can_activate()
875 	if ((IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) &&  in intel_fbc_can_activate()
876 	    cache->crtc.hsw_bdw_pixel_rate >= dev_priv->cdclk.hw.cdclk * 95 / 100) {  in intel_fbc_can_activate()
891 	if (intel_fbc_cfb_size_changed(dev_priv)) {  in intel_fbc_can_activate()
901 	if (DISPLAY_VER(dev_priv) >= 9 &&  in intel_fbc_can_activate()
908 	if (DISPLAY_VER(dev_priv) >= 11 &&  in intel_fbc_can_activate()
919 	if (fbc->state_cache.psr2_active && DISPLAY_VER(dev_priv) >= 12) {  in intel_fbc_can_activate()
930 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);  in intel_fbc_get_reg_params()  local
931 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_get_reg_params()
951 	params->cfb_size = intel_fbc_calculate_cfb_size(dev_priv, cache);  in intel_fbc_get_reg_params()
961 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);  in intel_fbc_can_flip_nuke()  local
962 	const struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_can_flip_nuke()
984 	if (params->cfb_size != intel_fbc_calculate_cfb_size(dev_priv, cache))  in intel_fbc_can_flip_nuke()
1001 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);  in intel_fbc_pre_update()  local
1002 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_pre_update()
1018 		intel_fbc_deactivate(dev_priv, reason);  in intel_fbc_pre_update()
1034 		    DISPLAY_VER(dev_priv) >= 10)  in intel_fbc_pre_update()
1051 static void __intel_fbc_disable(struct drm_i915_private *dev_priv)  in __intel_fbc_disable()  argument
1053 	struct intel_fbc *fbc = &dev_priv->fbc;  in __intel_fbc_disable()
1056 	drm_WARN_ON(&dev_priv->drm, !mutex_is_locked(&fbc->lock));  in __intel_fbc_disable()
1057 	drm_WARN_ON(&dev_priv->drm, !fbc->crtc);  in __intel_fbc_disable()
1058 	drm_WARN_ON(&dev_priv->drm, fbc->active);  in __intel_fbc_disable()
1060 	drm_dbg_kms(&dev_priv->drm, "Disabling FBC on pipe %c\n",  in __intel_fbc_disable()
1063 	__intel_fbc_cleanup_cfb(dev_priv);  in __intel_fbc_disable()
1070 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);  in __intel_fbc_post_update()  local
1071 	struct intel_fbc *fbc = &dev_priv->fbc;  in __intel_fbc_post_update()
1073 	drm_WARN_ON(&dev_priv->drm, !mutex_is_locked(&fbc->lock));  in __intel_fbc_post_update()
1080 	if (!dev_priv->params.enable_fbc) {  in __intel_fbc_post_update()
1081 		intel_fbc_deactivate(dev_priv, "disabled at runtime per module param");  in __intel_fbc_post_update()
1082 		__intel_fbc_disable(dev_priv);  in __intel_fbc_post_update()
1093 		intel_fbc_hw_activate(dev_priv);  in __intel_fbc_post_update()
1095 		intel_fbc_deactivate(dev_priv, "frontbuffer write");  in __intel_fbc_post_update()
1101 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);  in intel_fbc_post_update()  local
1105 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_post_update()
1123 void intel_fbc_invalidate(struct drm_i915_private *dev_priv,  in intel_fbc_invalidate()  argument
1127 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_invalidate()
1129 	if (!HAS_FBC(dev_priv))  in intel_fbc_invalidate()
1140 		intel_fbc_deactivate(dev_priv, "frontbuffer write");  in intel_fbc_invalidate()
1145 void intel_fbc_flush(struct drm_i915_private *dev_priv,  in intel_fbc_flush()  argument
1148 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_flush()
1150 	if (!HAS_FBC(dev_priv))  in intel_fbc_flush()
1171 			intel_fbc_recompress(dev_priv);  in intel_fbc_flush()
1192 void intel_fbc_choose_crtc(struct drm_i915_private *dev_priv,  in intel_fbc_choose_crtc()  argument
1195 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_choose_crtc()
1208 	if (!intel_fbc_can_enable(dev_priv))  in intel_fbc_choose_crtc()
1252 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);  in intel_fbc_enable()  local
1258 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_enable()
1268 		    (!intel_fbc_cfb_size_changed(dev_priv) &&  in intel_fbc_enable()
1269 		     !intel_fbc_gen9_wa_cfb_stride_changed(dev_priv)))  in intel_fbc_enable()
1272 		__intel_fbc_disable(dev_priv);  in intel_fbc_enable()
1275 	drm_WARN_ON(&dev_priv->drm, fbc->active);  in intel_fbc_enable()
1283 	if (intel_fbc_alloc_cfb(dev_priv,  in intel_fbc_enable()
1284 				intel_fbc_calculate_cfb_size(dev_priv, cache),  in intel_fbc_enable()
1291 	cache->gen9_wa_cfb_stride = intel_fbc_gen9_wa_cfb_stride(dev_priv);  in intel_fbc_enable()
1293 	drm_dbg_kms(&dev_priv->drm, "Enabling FBC on pipe %c\n",  in intel_fbc_enable()
1299 	intel_fbc_program_cfb(dev_priv);  in intel_fbc_enable()
1312 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);  in intel_fbc_disable()  local
1314 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_disable()
1321 		__intel_fbc_disable(dev_priv);  in intel_fbc_disable()
1331 void intel_fbc_global_disable(struct drm_i915_private *dev_priv)  in intel_fbc_global_disable()  argument
1333 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_global_disable()
1335 	if (!HAS_FBC(dev_priv))  in intel_fbc_global_disable()
1340 		drm_WARN_ON(&dev_priv->drm, fbc->crtc->active);  in intel_fbc_global_disable()
1341 		__intel_fbc_disable(dev_priv);  in intel_fbc_global_disable()
1348 	struct drm_i915_private *dev_priv =  in intel_fbc_underrun_work_fn()  local
1350 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_underrun_work_fn()
1358 	drm_dbg_kms(&dev_priv->drm, "Disabling FBC due to FIFO underrun.\n");  in intel_fbc_underrun_work_fn()
1361 	intel_fbc_deactivate(dev_priv, "FIFO underrun");  in intel_fbc_underrun_work_fn()
1373 int intel_fbc_reset_underrun(struct drm_i915_private *dev_priv)  in intel_fbc_reset_underrun()  argument
1377 	cancel_work_sync(&dev_priv->fbc.underrun_work);  in intel_fbc_reset_underrun()
1379 	ret = mutex_lock_interruptible(&dev_priv->fbc.lock);  in intel_fbc_reset_underrun()
1383 	if (dev_priv->fbc.underrun_detected) {  in intel_fbc_reset_underrun()
1384 		drm_dbg_kms(&dev_priv->drm,  in intel_fbc_reset_underrun()
1386 		dev_priv->fbc.no_fbc_reason = "FIFO underrun cleared";  in intel_fbc_reset_underrun()
1389 	dev_priv->fbc.underrun_detected = false;  in intel_fbc_reset_underrun()
1390 	mutex_unlock(&dev_priv->fbc.lock);  in intel_fbc_reset_underrun()
1409 void intel_fbc_handle_fifo_underrun_irq(struct drm_i915_private *dev_priv)  in intel_fbc_handle_fifo_underrun_irq()  argument
1411 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_handle_fifo_underrun_irq()
1413 	if (!HAS_FBC(dev_priv))  in intel_fbc_handle_fifo_underrun_irq()
1437 static int intel_sanitize_fbc_option(struct drm_i915_private *dev_priv)  in intel_sanitize_fbc_option()  argument
1439 	if (dev_priv->params.enable_fbc >= 0)  in intel_sanitize_fbc_option()
1440 		return !!dev_priv->params.enable_fbc;  in intel_sanitize_fbc_option()
1442 	if (!HAS_FBC(dev_priv))  in intel_sanitize_fbc_option()
1445 	if (IS_BROADWELL(dev_priv) || DISPLAY_VER(dev_priv) >= 9)  in intel_sanitize_fbc_option()
1451 static bool need_fbc_vtd_wa(struct drm_i915_private *dev_priv)  in need_fbc_vtd_wa()  argument
1455 	    (IS_SKYLAKE(dev_priv) || IS_BROXTON(dev_priv))) {  in need_fbc_vtd_wa()
1456 		drm_info(&dev_priv->drm,  in need_fbc_vtd_wa()
1470 void intel_fbc_init(struct drm_i915_private *dev_priv)  in intel_fbc_init()  argument
1472 	struct intel_fbc *fbc = &dev_priv->fbc;  in intel_fbc_init()
1478 	if (!drm_mm_initialized(&dev_priv->mm.stolen))  in intel_fbc_init()
1479 		mkwrite_device_info(dev_priv)->display.has_fbc = false;  in intel_fbc_init()
1481 	if (need_fbc_vtd_wa(dev_priv))  in intel_fbc_init()
1482 		mkwrite_device_info(dev_priv)->display.has_fbc = false;  in intel_fbc_init()
1484 	dev_priv->params.enable_fbc = intel_sanitize_fbc_option(dev_priv);  in intel_fbc_init()
1485 	drm_dbg_kms(&dev_priv->drm, "Sanitized enable_fbc value: %d\n",  in intel_fbc_init()
1486 		    dev_priv->params.enable_fbc);  in intel_fbc_init()
1488 	if (!HAS_FBC(dev_priv)) {  in intel_fbc_init()
1496 	if (intel_fbc_hw_is_active(dev_priv))  in intel_fbc_init()
1497 		intel_fbc_hw_deactivate(dev_priv);  in intel_fbc_init()