64 		for_each_if((__fbc) = (__dev_priv)->display.fbc[(__fbc_id)])
67 	void (*activate)(struct intel_fbc *fbc);
68 	void (*deactivate)(struct intel_fbc *fbc);
69 	bool (*is_active)(struct intel_fbc *fbc);
70 	bool (*is_compressing)(struct intel_fbc *fbc);
71 	void (*nuke)(struct intel_fbc *fbc);
72 	void (*program_cfb)(struct intel_fbc *fbc);
73 	void (*set_false_color)(struct intel_fbc *fbc, bool enable);
226 static u32 i8xx_fbc_ctl(struct intel_fbc *fbc)  in i8xx_fbc_ctl()  argument
228 	const struct intel_fbc_state *fbc_state = &fbc->state;  in i8xx_fbc_ctl()
229 	struct drm_i915_private *i915 = fbc->i915;  in i8xx_fbc_ctl()
233 	cfb_stride = fbc_state->cfb_stride / fbc->limit;  in i8xx_fbc_ctl()
254 static u32 i965_fbc_ctl2(struct intel_fbc *fbc)  in i965_fbc_ctl2()  argument
256 	const struct intel_fbc_state *fbc_state = &fbc->state;  in i965_fbc_ctl2()
268 static void i8xx_fbc_deactivate(struct intel_fbc *fbc)  in i8xx_fbc_deactivate()  argument
270 	struct drm_i915_private *i915 = fbc->i915;  in i8xx_fbc_deactivate()
289 static void i8xx_fbc_activate(struct intel_fbc *fbc)  in i8xx_fbc_activate()  argument
291 	const struct intel_fbc_state *fbc_state = &fbc->state;  in i8xx_fbc_activate()
292 	struct drm_i915_private *i915 = fbc->i915;  in i8xx_fbc_activate()
301 			       i965_fbc_ctl2(fbc));  in i8xx_fbc_activate()
307 		       FBC_CTL_EN | i8xx_fbc_ctl(fbc));  in i8xx_fbc_activate()
310 static bool i8xx_fbc_is_active(struct intel_fbc *fbc)  in i8xx_fbc_is_active()  argument
312 	return intel_de_read(fbc->i915, FBC_CONTROL) & FBC_CTL_EN;  in i8xx_fbc_is_active()
315 static bool i8xx_fbc_is_compressing(struct intel_fbc *fbc)  in i8xx_fbc_is_compressing()  argument
317 	return intel_de_read(fbc->i915, FBC_STATUS) &  in i8xx_fbc_is_compressing()
321 static void i8xx_fbc_nuke(struct intel_fbc *fbc)  in i8xx_fbc_nuke()  argument
323 	struct intel_fbc_state *fbc_state = &fbc->state;  in i8xx_fbc_nuke()
325 	struct drm_i915_private *dev_priv = fbc->i915;  in i8xx_fbc_nuke()
331 static void i8xx_fbc_program_cfb(struct intel_fbc *fbc)  in i8xx_fbc_program_cfb()  argument
333 	struct drm_i915_private *i915 = fbc->i915;  in i8xx_fbc_program_cfb()
336 					 i915_gem_stolen_node_offset(&fbc->compressed_fb),  in i8xx_fbc_program_cfb()
339 					 i915_gem_stolen_node_offset(&fbc->compressed_llb),  in i8xx_fbc_program_cfb()
342 		       i915_gem_stolen_node_address(i915, &fbc->compressed_fb));  in i8xx_fbc_program_cfb()
344 		       i915_gem_stolen_node_address(i915, &fbc->compressed_llb));  in i8xx_fbc_program_cfb()
356 static void i965_fbc_nuke(struct intel_fbc *fbc)  in i965_fbc_nuke()  argument
358 	struct intel_fbc_state *fbc_state = &fbc->state;  in i965_fbc_nuke()
360 	struct drm_i915_private *dev_priv = fbc->i915;  in i965_fbc_nuke()
375 static u32 g4x_dpfc_ctl_limit(struct intel_fbc *fbc)  in g4x_dpfc_ctl_limit()  argument
377 	switch (fbc->limit) {  in g4x_dpfc_ctl_limit()
379 		MISSING_CASE(fbc->limit);  in g4x_dpfc_ctl_limit()
390 static u32 g4x_dpfc_ctl(struct intel_fbc *fbc)  in g4x_dpfc_ctl()  argument
392 	const struct intel_fbc_state *fbc_state = &fbc->state;  in g4x_dpfc_ctl()
393 	struct drm_i915_private *i915 = fbc->i915;  in g4x_dpfc_ctl()
396 	dpfc_ctl = g4x_dpfc_ctl_limit(fbc) |  in g4x_dpfc_ctl()
412 static void g4x_fbc_activate(struct intel_fbc *fbc)  in g4x_fbc_activate()  argument
414 	const struct intel_fbc_state *fbc_state = &fbc->state;  in g4x_fbc_activate()
415 	struct drm_i915_private *i915 = fbc->i915;  in g4x_fbc_activate()
421 		       DPFC_CTL_EN | g4x_dpfc_ctl(fbc));  in g4x_fbc_activate()
424 static void g4x_fbc_deactivate(struct intel_fbc *fbc)  in g4x_fbc_deactivate()  argument
426 	struct drm_i915_private *i915 = fbc->i915;  in g4x_fbc_deactivate()
437 static bool g4x_fbc_is_active(struct intel_fbc *fbc)  in g4x_fbc_is_active()  argument
439 	return intel_de_read(fbc->i915, DPFC_CONTROL) & DPFC_CTL_EN;  in g4x_fbc_is_active()
442 static bool g4x_fbc_is_compressing(struct intel_fbc *fbc)  in g4x_fbc_is_compressing()  argument
444 	return intel_de_read(fbc->i915, DPFC_STATUS) & DPFC_COMP_SEG_MASK;  in g4x_fbc_is_compressing()
447 static void g4x_fbc_program_cfb(struct intel_fbc *fbc)  in g4x_fbc_program_cfb()  argument
449 	struct drm_i915_private *i915 = fbc->i915;  in g4x_fbc_program_cfb()
452 		       i915_gem_stolen_node_offset(&fbc->compressed_fb));  in g4x_fbc_program_cfb()
464 static void ilk_fbc_activate(struct intel_fbc *fbc)  in ilk_fbc_activate()  argument
466 	struct intel_fbc_state *fbc_state = &fbc->state;  in ilk_fbc_activate()
467 	struct drm_i915_private *i915 = fbc->i915;  in ilk_fbc_activate()
469 	intel_de_write(i915, ILK_DPFC_FENCE_YOFF(fbc->id),  in ilk_fbc_activate()
472 	intel_de_write(i915, ILK_DPFC_CONTROL(fbc->id),  in ilk_fbc_activate()
473 		       DPFC_CTL_EN | g4x_dpfc_ctl(fbc));  in ilk_fbc_activate()
476 static void ilk_fbc_deactivate(struct intel_fbc *fbc)  in ilk_fbc_deactivate()  argument
478 	struct drm_i915_private *i915 = fbc->i915;  in ilk_fbc_deactivate()
482 	dpfc_ctl = intel_de_read(i915, ILK_DPFC_CONTROL(fbc->id));  in ilk_fbc_deactivate()
485 		intel_de_write(i915, ILK_DPFC_CONTROL(fbc->id), dpfc_ctl);  in ilk_fbc_deactivate()
489 static bool ilk_fbc_is_active(struct intel_fbc *fbc)  in ilk_fbc_is_active()  argument
491 	return intel_de_read(fbc->i915, ILK_DPFC_CONTROL(fbc->id)) & DPFC_CTL_EN;  in ilk_fbc_is_active()
494 static bool ilk_fbc_is_compressing(struct intel_fbc *fbc)  in ilk_fbc_is_compressing()  argument
496 	return intel_de_read(fbc->i915, ILK_DPFC_STATUS(fbc->id)) & DPFC_COMP_SEG_MASK;  in ilk_fbc_is_compressing()
499 static void ilk_fbc_program_cfb(struct intel_fbc *fbc)  in ilk_fbc_program_cfb()  argument
501 	struct drm_i915_private *i915 = fbc->i915;  in ilk_fbc_program_cfb()
503 	intel_de_write(i915, ILK_DPFC_CB_BASE(fbc->id),  in ilk_fbc_program_cfb()
504 		       i915_gem_stolen_node_offset(&fbc->compressed_fb));  in ilk_fbc_program_cfb()
516 static void snb_fbc_program_fence(struct intel_fbc *fbc)  in snb_fbc_program_fence()  argument
518 	const struct intel_fbc_state *fbc_state = &fbc->state;  in snb_fbc_program_fence()
519 	struct drm_i915_private *i915 = fbc->i915;  in snb_fbc_program_fence()
529 static void snb_fbc_activate(struct intel_fbc *fbc)  in snb_fbc_activate()  argument
531 	snb_fbc_program_fence(fbc);  in snb_fbc_activate()
533 	ilk_fbc_activate(fbc);  in snb_fbc_activate()
536 static void snb_fbc_nuke(struct intel_fbc *fbc)  in snb_fbc_nuke()  argument
538 	struct drm_i915_private *i915 = fbc->i915;  in snb_fbc_nuke()
540 	intel_de_write(i915, MSG_FBC_REND_STATE(fbc->id), FBC_REND_NUKE);  in snb_fbc_nuke()
541 	intel_de_posting_read(i915, MSG_FBC_REND_STATE(fbc->id));  in snb_fbc_nuke()
553 static void glk_fbc_program_cfb_stride(struct intel_fbc *fbc)  in glk_fbc_program_cfb_stride()  argument
555 	const struct intel_fbc_state *fbc_state = &fbc->state;  in glk_fbc_program_cfb_stride()
556 	struct drm_i915_private *i915 = fbc->i915;  in glk_fbc_program_cfb_stride()
561 			FBC_STRIDE(fbc_state->override_cfb_stride / fbc->limit);  in glk_fbc_program_cfb_stride()
563 	intel_de_write(i915, GLK_FBC_STRIDE(fbc->id), val);  in glk_fbc_program_cfb_stride()
566 static void skl_fbc_program_cfb_stride(struct intel_fbc *fbc)  in skl_fbc_program_cfb_stride()  argument
568 	const struct intel_fbc_state *fbc_state = &fbc->state;  in skl_fbc_program_cfb_stride()
569 	struct drm_i915_private *i915 = fbc->i915;  in skl_fbc_program_cfb_stride()
575 			CHICKEN_FBC_STRIDE(fbc_state->override_cfb_stride / fbc->limit);  in skl_fbc_program_cfb_stride()
582 static u32 ivb_dpfc_ctl(struct intel_fbc *fbc)  in ivb_dpfc_ctl()  argument
584 	const struct intel_fbc_state *fbc_state = &fbc->state;  in ivb_dpfc_ctl()
585 	struct drm_i915_private *i915 = fbc->i915;  in ivb_dpfc_ctl()
588 	dpfc_ctl = g4x_dpfc_ctl_limit(fbc);  in ivb_dpfc_ctl()
596 	if (fbc->false_color)  in ivb_dpfc_ctl()
602 static void ivb_fbc_activate(struct intel_fbc *fbc)  in ivb_fbc_activate()  argument
604 	struct drm_i915_private *i915 = fbc->i915;  in ivb_fbc_activate()
607 		glk_fbc_program_cfb_stride(fbc);  in ivb_fbc_activate()
609 		skl_fbc_program_cfb_stride(fbc);  in ivb_fbc_activate()
612 		snb_fbc_program_fence(fbc);  in ivb_fbc_activate()
614 	intel_de_write(i915, ILK_DPFC_CONTROL(fbc->id),  in ivb_fbc_activate()
615 		       DPFC_CTL_EN | ivb_dpfc_ctl(fbc));  in ivb_fbc_activate()
618 static bool ivb_fbc_is_compressing(struct intel_fbc *fbc)  in ivb_fbc_is_compressing()  argument
620 	return intel_de_read(fbc->i915, ILK_DPFC_STATUS2(fbc->id)) & DPFC_COMP_SEG_MASK_IVB;  in ivb_fbc_is_compressing()
623 static void ivb_fbc_set_false_color(struct intel_fbc *fbc,  in ivb_fbc_set_false_color()  argument
626 	intel_de_rmw(fbc->i915, ILK_DPFC_CONTROL(fbc->id),  in ivb_fbc_set_false_color()
640 static bool intel_fbc_hw_is_active(struct intel_fbc *fbc)  in intel_fbc_hw_is_active()  argument
642 	return fbc->funcs->is_active(fbc);  in intel_fbc_hw_is_active()
645 static void intel_fbc_hw_activate(struct intel_fbc *fbc)  in intel_fbc_hw_activate()  argument
647 	trace_intel_fbc_activate(fbc->state.plane);  in intel_fbc_hw_activate()
649 	fbc->active = true;  in intel_fbc_hw_activate()
650 	fbc->activated = true;  in intel_fbc_hw_activate()
652 	fbc->funcs->activate(fbc);  in intel_fbc_hw_activate()
655 static void intel_fbc_hw_deactivate(struct intel_fbc *fbc)  in intel_fbc_hw_deactivate()  argument
657 	trace_intel_fbc_deactivate(fbc->state.plane);  in intel_fbc_hw_deactivate()
659 	fbc->active = false;  in intel_fbc_hw_deactivate()
661 	fbc->funcs->deactivate(fbc);  in intel_fbc_hw_deactivate()
664 static bool intel_fbc_is_compressing(struct intel_fbc *fbc)  in intel_fbc_is_compressing()  argument
666 	return fbc->funcs->is_compressing(fbc);  in intel_fbc_is_compressing()
669 static void intel_fbc_nuke(struct intel_fbc *fbc)  in intel_fbc_nuke()  argument
671 	struct drm_i915_private *i915 = fbc->i915;  in intel_fbc_nuke()
673 	lockdep_assert_held(&fbc->lock);  in intel_fbc_nuke()
674 	drm_WARN_ON(&i915->drm, fbc->flip_pending);  in intel_fbc_nuke()
676 	trace_intel_fbc_nuke(fbc->state.plane);  in intel_fbc_nuke()
678 	fbc->funcs->nuke(fbc);  in intel_fbc_nuke()
681 static void intel_fbc_activate(struct intel_fbc *fbc)  in intel_fbc_activate()  argument
683 	lockdep_assert_held(&fbc->lock);  in intel_fbc_activate()
685 	intel_fbc_hw_activate(fbc);  in intel_fbc_activate()
686 	intel_fbc_nuke(fbc);  in intel_fbc_activate()
688 	fbc->no_fbc_reason = NULL;  in intel_fbc_activate()
691 static void intel_fbc_deactivate(struct intel_fbc *fbc, const char *reason)  in intel_fbc_deactivate()  argument
693 	lockdep_assert_held(&fbc->lock);  in intel_fbc_deactivate()
695 	if (fbc->active)  in intel_fbc_deactivate()
696 		intel_fbc_hw_deactivate(fbc);  in intel_fbc_deactivate()
698 	fbc->no_fbc_reason = reason;  in intel_fbc_deactivate()
744 static int find_compression_limit(struct intel_fbc *fbc,  in find_compression_limit()  argument
747 	struct drm_i915_private *i915 = fbc->i915;  in find_compression_limit()
754 	ret = i915_gem_stolen_insert_node_in_range(i915, &fbc->compressed_fb,  in find_compression_limit()
760 		ret = i915_gem_stolen_insert_node_in_range(i915, &fbc->compressed_fb,  in find_compression_limit()
769 static int intel_fbc_alloc_cfb(struct intel_fbc *fbc,  in intel_fbc_alloc_cfb()  argument
772 	struct drm_i915_private *i915 = fbc->i915;  in intel_fbc_alloc_cfb()
776 		    i915_gem_stolen_node_allocated(&fbc->compressed_fb));  in intel_fbc_alloc_cfb()
778 		    i915_gem_stolen_node_allocated(&fbc->compressed_llb));  in intel_fbc_alloc_cfb()
781 		ret = i915_gem_stolen_insert_node(i915, &fbc->compressed_llb,  in intel_fbc_alloc_cfb()
787 	ret = find_compression_limit(fbc, size, min_limit);  in intel_fbc_alloc_cfb()
794 	fbc->limit = ret;  in intel_fbc_alloc_cfb()
798 		    i915_gem_stolen_node_size(&fbc->compressed_fb), fbc->limit);  in intel_fbc_alloc_cfb()
802 	if (i915_gem_stolen_node_allocated(&fbc->compressed_llb))  in intel_fbc_alloc_cfb()
803 		i915_gem_stolen_remove_node(i915, &fbc->compressed_llb);  in intel_fbc_alloc_cfb()
810 static void intel_fbc_program_cfb(struct intel_fbc *fbc)  in intel_fbc_program_cfb()  argument
812 	fbc->funcs->program_cfb(fbc);  in intel_fbc_program_cfb()
815 static void intel_fbc_program_workarounds(struct intel_fbc *fbc)  in intel_fbc_program_workarounds()  argument
818 	if (DISPLAY_VER(fbc->i915) >= 11 && !IS_DG2(fbc->i915))  in intel_fbc_program_workarounds()
819 		intel_de_rmw(fbc->i915, ILK_DPFC_CHICKEN(fbc->id), 0,  in intel_fbc_program_workarounds()
823 static void __intel_fbc_cleanup_cfb(struct intel_fbc *fbc)  in __intel_fbc_cleanup_cfb()  argument
825 	struct drm_i915_private *i915 = fbc->i915;  in __intel_fbc_cleanup_cfb()
827 	if (WARN_ON(intel_fbc_hw_is_active(fbc)))  in __intel_fbc_cleanup_cfb()
830 	if (i915_gem_stolen_node_allocated(&fbc->compressed_llb))  in __intel_fbc_cleanup_cfb()
831 		i915_gem_stolen_remove_node(i915, &fbc->compressed_llb);  in __intel_fbc_cleanup_cfb()
832 	if (i915_gem_stolen_node_allocated(&fbc->compressed_fb))  in __intel_fbc_cleanup_cfb()
833 		i915_gem_stolen_remove_node(i915, &fbc->compressed_fb);  in __intel_fbc_cleanup_cfb()
838 	struct intel_fbc *fbc;  in intel_fbc_cleanup()  local
841 	for_each_intel_fbc(i915, fbc, fbc_id) {  in intel_fbc_cleanup()
842 		mutex_lock(&fbc->lock);  in intel_fbc_cleanup()
843 		__intel_fbc_cleanup_cfb(fbc);  in intel_fbc_cleanup()
844 		mutex_unlock(&fbc->lock);  in intel_fbc_cleanup()
846 		kfree(fbc);  in intel_fbc_cleanup()
981 	struct intel_fbc *fbc = plane->fbc;  in intel_fbc_update_state()  local
982 	struct intel_fbc_state *fbc_state = &fbc->state;  in intel_fbc_update_state()
1031 	struct intel_fbc *fbc = plane->fbc;  in intel_fbc_is_cfb_ok()  local
1033 	return intel_fbc_min_limit(plane_state) <= fbc->limit &&  in intel_fbc_is_cfb_ok()
1034 		intel_fbc_cfb_size(plane_state) <= fbc->limit *  in intel_fbc_is_cfb_ok()
1035 			i915_gem_stolen_node_size(&fbc->compressed_fb);  in intel_fbc_is_cfb_ok()
1054 	struct intel_fbc *fbc = plane->fbc;  in intel_fbc_check_plane()  local
1056 	if (!fbc)  in intel_fbc_check_plane()
1229 	struct intel_fbc *fbc = plane->fbc;  in __intel_fbc_pre_update()  local
1232 	lockdep_assert_held(&fbc->lock);  in __intel_fbc_pre_update()
1234 	fbc->flip_pending = true;  in __intel_fbc_pre_update()
1239 	intel_fbc_deactivate(fbc, "update pending");  in __intel_fbc_pre_update()
1254 	if (fbc->activated && DISPLAY_VER(i915) >= 10)  in __intel_fbc_pre_update()
1256 	fbc->activated = false;  in __intel_fbc_pre_update()
1270 		struct intel_fbc *fbc = plane->fbc;  in intel_fbc_pre_update()  local
1272 		if (!fbc || plane->pipe != crtc->pipe)  in intel_fbc_pre_update()
1275 		mutex_lock(&fbc->lock);  in intel_fbc_pre_update()
1277 		if (fbc->state.plane == plane)  in intel_fbc_pre_update()
1280 		mutex_unlock(&fbc->lock);  in intel_fbc_pre_update()
1286 static void __intel_fbc_disable(struct intel_fbc *fbc)  in __intel_fbc_disable()  argument
1288 	struct drm_i915_private *i915 = fbc->i915;  in __intel_fbc_disable()
1289 	struct intel_plane *plane = fbc->state.plane;  in __intel_fbc_disable()
1291 	lockdep_assert_held(&fbc->lock);  in __intel_fbc_disable()
1292 	drm_WARN_ON(&i915->drm, fbc->active);  in __intel_fbc_disable()
1297 	__intel_fbc_cleanup_cfb(fbc);  in __intel_fbc_disable()
1299 	fbc->state.plane = NULL;  in __intel_fbc_disable()
1300 	fbc->flip_pending = false;  in __intel_fbc_disable()
1301 	fbc->busy_bits = 0;  in __intel_fbc_disable()
1304 static void __intel_fbc_post_update(struct intel_fbc *fbc)  in __intel_fbc_post_update()  argument
1306 	lockdep_assert_held(&fbc->lock);  in __intel_fbc_post_update()
1308 	fbc->flip_pending = false;  in __intel_fbc_post_update()
1310 	if (!fbc->busy_bits)  in __intel_fbc_post_update()
1311 		intel_fbc_activate(fbc);  in __intel_fbc_post_update()
1313 		intel_fbc_deactivate(fbc, "frontbuffer write");  in __intel_fbc_post_update()
1324 		struct intel_fbc *fbc = plane->fbc;  in intel_fbc_post_update()  local
1326 		if (!fbc || plane->pipe != crtc->pipe)  in intel_fbc_post_update()
1329 		mutex_lock(&fbc->lock);  in intel_fbc_post_update()
1331 		if (fbc->state.plane == plane)  in intel_fbc_post_update()
1332 			__intel_fbc_post_update(fbc);  in intel_fbc_post_update()
1334 		mutex_unlock(&fbc->lock);  in intel_fbc_post_update()
1338 static unsigned int intel_fbc_get_frontbuffer_bit(struct intel_fbc *fbc)  in intel_fbc_get_frontbuffer_bit()  argument
1340 	if (fbc->state.plane)  in intel_fbc_get_frontbuffer_bit()
1341 		return fbc->state.plane->frontbuffer_bit;  in intel_fbc_get_frontbuffer_bit()
1346 static void __intel_fbc_invalidate(struct intel_fbc *fbc,  in __intel_fbc_invalidate()  argument
1353 	mutex_lock(&fbc->lock);  in __intel_fbc_invalidate()
1355 	frontbuffer_bits &= intel_fbc_get_frontbuffer_bit(fbc);  in __intel_fbc_invalidate()
1359 	fbc->busy_bits |= frontbuffer_bits;  in __intel_fbc_invalidate()
1360 	intel_fbc_deactivate(fbc, "frontbuffer write");  in __intel_fbc_invalidate()
1363 	mutex_unlock(&fbc->lock);  in __intel_fbc_invalidate()
1370 	struct intel_fbc *fbc;  in intel_fbc_invalidate()  local
1373 	for_each_intel_fbc(i915, fbc, fbc_id)  in intel_fbc_invalidate()
1374 		__intel_fbc_invalidate(fbc, frontbuffer_bits, origin);  in intel_fbc_invalidate()
1378 static void __intel_fbc_flush(struct intel_fbc *fbc,  in __intel_fbc_flush()  argument
1382 	mutex_lock(&fbc->lock);  in __intel_fbc_flush()
1384 	frontbuffer_bits &= intel_fbc_get_frontbuffer_bit(fbc);  in __intel_fbc_flush()
1388 	fbc->busy_bits &= ~frontbuffer_bits;  in __intel_fbc_flush()
1393 	if (fbc->busy_bits || fbc->flip_pending)  in __intel_fbc_flush()
1396 	if (fbc->active)  in __intel_fbc_flush()
1397 		intel_fbc_nuke(fbc);  in __intel_fbc_flush()
1399 		intel_fbc_activate(fbc);  in __intel_fbc_flush()
1402 	mutex_unlock(&fbc->lock);  in __intel_fbc_flush()
1409 	struct intel_fbc *fbc;  in intel_fbc_flush()  local
1412 	for_each_intel_fbc(i915, fbc, fbc_id)  in intel_fbc_flush()
1413 		__intel_fbc_flush(fbc, frontbuffer_bits, origin);  in intel_fbc_flush()
1440 	struct intel_fbc *fbc = plane->fbc;  in __intel_fbc_enable()  local
1442 	lockdep_assert_held(&fbc->lock);  in __intel_fbc_enable()
1444 	if (fbc->state.plane) {  in __intel_fbc_enable()
1445 		if (fbc->state.plane != plane)  in __intel_fbc_enable()
1453 		__intel_fbc_disable(fbc);  in __intel_fbc_enable()
1456 	drm_WARN_ON(&i915->drm, fbc->active);  in __intel_fbc_enable()
1458 	fbc->no_fbc_reason = plane_state->no_fbc_reason;  in __intel_fbc_enable()
1459 	if (fbc->no_fbc_reason)  in __intel_fbc_enable()
1463 		fbc->no_fbc_reason = "framebuffer not fenced";  in __intel_fbc_enable()
1467 	if (fbc->underrun_detected) {  in __intel_fbc_enable()
1468 		fbc->no_fbc_reason = "FIFO underrun";  in __intel_fbc_enable()
1472 	if (intel_fbc_alloc_cfb(fbc, intel_fbc_cfb_size(plane_state),  in __intel_fbc_enable()
1474 		fbc->no_fbc_reason = "not enough stolen memory";  in __intel_fbc_enable()
1480 	fbc->no_fbc_reason = "FBC enabled but not active yet\n";  in __intel_fbc_enable()
1484 	intel_fbc_program_workarounds(fbc);  in __intel_fbc_enable()
1485 	intel_fbc_program_cfb(fbc);  in __intel_fbc_enable()
1500 		struct intel_fbc *fbc = plane->fbc;  in intel_fbc_disable()  local
1502 		if (!fbc || plane->pipe != crtc->pipe)  in intel_fbc_disable()
1505 		mutex_lock(&fbc->lock);  in intel_fbc_disable()
1506 		if (fbc->state.plane == plane)  in intel_fbc_disable()
1507 			__intel_fbc_disable(fbc);  in intel_fbc_disable()
1508 		mutex_unlock(&fbc->lock);  in intel_fbc_disable()
1522 		struct intel_fbc *fbc = plane->fbc;  in intel_fbc_update()  local
1524 		if (!fbc || plane->pipe != crtc->pipe)  in intel_fbc_update()
1527 		mutex_lock(&fbc->lock);  in intel_fbc_update()
1531 			if (fbc->state.plane == plane)  in intel_fbc_update()
1532 				__intel_fbc_disable(fbc);  in intel_fbc_update()
1537 		mutex_unlock(&fbc->lock);  in intel_fbc_update()
1543 	struct intel_fbc *fbc = container_of(work, typeof(*fbc), underrun_work);  in intel_fbc_underrun_work_fn()  local
1544 	struct drm_i915_private *i915 = fbc->i915;  in intel_fbc_underrun_work_fn()
1546 	mutex_lock(&fbc->lock);  in intel_fbc_underrun_work_fn()
1549 	if (fbc->underrun_detected || !fbc->state.plane)  in intel_fbc_underrun_work_fn()
1553 	fbc->underrun_detected = true;  in intel_fbc_underrun_work_fn()
1555 	intel_fbc_deactivate(fbc, "FIFO underrun");  in intel_fbc_underrun_work_fn()
1556 	if (!fbc->flip_pending)  in intel_fbc_underrun_work_fn()
1557 		intel_crtc_wait_for_next_vblank(intel_crtc_for_pipe(i915, fbc->state.plane->pipe));  in intel_fbc_underrun_work_fn()
1558 	__intel_fbc_disable(fbc);  in intel_fbc_underrun_work_fn()
1560 	mutex_unlock(&fbc->lock);  in intel_fbc_underrun_work_fn()
1563 static void __intel_fbc_reset_underrun(struct intel_fbc *fbc)  in __intel_fbc_reset_underrun()  argument
1565 	struct drm_i915_private *i915 = fbc->i915;  in __intel_fbc_reset_underrun()
1567 	cancel_work_sync(&fbc->underrun_work);  in __intel_fbc_reset_underrun()
1569 	mutex_lock(&fbc->lock);  in __intel_fbc_reset_underrun()
1571 	if (fbc->underrun_detected) {  in __intel_fbc_reset_underrun()
1574 		fbc->no_fbc_reason = "FIFO underrun cleared";  in __intel_fbc_reset_underrun()
1577 	fbc->underrun_detected = false;  in __intel_fbc_reset_underrun()
1578 	mutex_unlock(&fbc->lock);  in __intel_fbc_reset_underrun()
1590 	struct intel_fbc *fbc;  in intel_fbc_reset_underrun()  local
1593 	for_each_intel_fbc(i915, fbc, fbc_id)  in intel_fbc_reset_underrun()
1594 		__intel_fbc_reset_underrun(fbc);  in intel_fbc_reset_underrun()
1597 static void __intel_fbc_handle_fifo_underrun_irq(struct intel_fbc *fbc)  in __intel_fbc_handle_fifo_underrun_irq()  argument
1607 	if (READ_ONCE(fbc->underrun_detected))  in __intel_fbc_handle_fifo_underrun_irq()
1610 	queue_work(fbc->i915->unordered_wq, &fbc->underrun_work);  in __intel_fbc_handle_fifo_underrun_irq()
1629 	struct intel_fbc *fbc;  in intel_fbc_handle_fifo_underrun_irq()  local
1632 	for_each_intel_fbc(i915, fbc, fbc_id)  in intel_fbc_handle_fifo_underrun_irq()
1633 		__intel_fbc_handle_fifo_underrun_irq(fbc);  in intel_fbc_handle_fifo_underrun_irq()
1672 void intel_fbc_add_plane(struct intel_fbc *fbc, struct intel_plane *plane)  in intel_fbc_add_plane()  argument
1674 	plane->fbc = fbc;  in intel_fbc_add_plane()
1680 	struct intel_fbc *fbc;  in intel_fbc_create()  local
1682 	fbc = kzalloc(sizeof(*fbc), GFP_KERNEL);  in intel_fbc_create()
1683 	if (!fbc)  in intel_fbc_create()
1686 	fbc->id = fbc_id;  in intel_fbc_create()
1687 	fbc->i915 = i915;  in intel_fbc_create()
1688 	INIT_WORK(&fbc->underrun_work, intel_fbc_underrun_work_fn);  in intel_fbc_create()
1689 	mutex_init(&fbc->lock);  in intel_fbc_create()
1692 		fbc->funcs = &ivb_fbc_funcs;  in intel_fbc_create()
1694 		fbc->funcs = &snb_fbc_funcs;  in intel_fbc_create()
1696 		fbc->funcs = &ilk_fbc_funcs;  in intel_fbc_create()
1698 		fbc->funcs = &g4x_fbc_funcs;  in intel_fbc_create()
1700 		fbc->funcs = &i965_fbc_funcs;  in intel_fbc_create()
1702 		fbc->funcs = &i8xx_fbc_funcs;  in intel_fbc_create()
1704 	return fbc;  in intel_fbc_create()
1725 		i915->display.fbc[fbc_id] = intel_fbc_create(i915, fbc_id);  in intel_fbc_init()
1738 	struct intel_fbc *fbc;  in intel_fbc_sanitize()  local
1741 	for_each_intel_fbc(i915, fbc, fbc_id) {  in intel_fbc_sanitize()
1742 		if (intel_fbc_hw_is_active(fbc))  in intel_fbc_sanitize()
1743 			intel_fbc_hw_deactivate(fbc);  in intel_fbc_sanitize()
1749 	struct intel_fbc *fbc = m->private;  in intel_fbc_debugfs_status_show()  local
1750 	struct drm_i915_private *i915 = fbc->i915;  in intel_fbc_debugfs_status_show()
1757 	mutex_lock(&fbc->lock);  in intel_fbc_debugfs_status_show()
1759 	if (fbc->active) {  in intel_fbc_debugfs_status_show()
1762 			   str_yes_no(intel_fbc_is_compressing(fbc)));  in intel_fbc_debugfs_status_show()
1764 		seq_printf(m, "FBC disabled: %s\n", fbc->no_fbc_reason);  in intel_fbc_debugfs_status_show()
1771 		if (plane->fbc != fbc)  in intel_fbc_debugfs_status_show()
1775 			   fbc->state.plane == plane ? '*' : ' ',  in intel_fbc_debugfs_status_show()
1780 	mutex_unlock(&fbc->lock);  in intel_fbc_debugfs_status_show()
1792 	struct intel_fbc *fbc = data;  in intel_fbc_debugfs_false_color_get()  local
1794 	*val = fbc->false_color;  in intel_fbc_debugfs_false_color_get()
1801 	struct intel_fbc *fbc = data;  in intel_fbc_debugfs_false_color_set()  local
1803 	mutex_lock(&fbc->lock);  in intel_fbc_debugfs_false_color_set()
1805 	fbc->false_color = val;  in intel_fbc_debugfs_false_color_set()
1807 	if (fbc->active)  in intel_fbc_debugfs_false_color_set()
1808 		fbc->funcs->set_false_color(fbc, fbc->false_color);  in intel_fbc_debugfs_false_color_set()
1810 	mutex_unlock(&fbc->lock);  in intel_fbc_debugfs_false_color_set()
1820 static void intel_fbc_debugfs_add(struct intel_fbc *fbc,  in intel_fbc_debugfs_add()  argument
1824 			    fbc, &intel_fbc_debugfs_status_fops);  in intel_fbc_debugfs_add()
1826 	if (fbc->funcs->set_false_color)  in intel_fbc_debugfs_add()
1828 					   fbc, &intel_fbc_debugfs_false_color_fops);  in intel_fbc_debugfs_add()
1835 	if (plane->fbc)  in intel_fbc_crtc_debugfs_add()
1836 		intel_fbc_debugfs_add(plane->fbc, crtc->base.debugfs_entry);  in intel_fbc_crtc_debugfs_add()
1843 	struct intel_fbc *fbc;  in intel_fbc_debugfs_register()  local
1845 	fbc = i915->display.fbc[INTEL_FBC_A];  in intel_fbc_debugfs_register()
1846 	if (fbc)  in intel_fbc_debugfs_register()
1847 		intel_fbc_debugfs_add(fbc, minor->debugfs_root);  in intel_fbc_debugfs_register()