62 		for_each_if((__fbc) = (__dev_priv)->display.fbc[(__fbc_id)])
65 	void (*activate)(struct intel_fbc *fbc);
66 	void (*deactivate)(struct intel_fbc *fbc);
67 	bool (*is_active)(struct intel_fbc *fbc);
68 	bool (*is_compressing)(struct intel_fbc *fbc);
69 	void (*nuke)(struct intel_fbc *fbc);
70 	void (*program_cfb)(struct intel_fbc *fbc);
71 	void (*set_false_color)(struct intel_fbc *fbc, bool enable);
225 static u32 i8xx_fbc_ctl(struct intel_fbc *fbc)  in i8xx_fbc_ctl()  argument
227 	const struct intel_fbc_state *fbc_state = &fbc->state;  in i8xx_fbc_ctl()
228 	struct drm_i915_private *i915 = fbc->i915;  in i8xx_fbc_ctl()
232 	cfb_stride = fbc_state->cfb_stride / fbc->limit;  in i8xx_fbc_ctl()
253 static u32 i965_fbc_ctl2(struct intel_fbc *fbc)  in i965_fbc_ctl2()  argument
255 	const struct intel_fbc_state *fbc_state = &fbc->state;  in i965_fbc_ctl2()
267 static void i8xx_fbc_deactivate(struct intel_fbc *fbc)  in i8xx_fbc_deactivate()  argument
269 	struct drm_i915_private *i915 = fbc->i915;  in i8xx_fbc_deactivate()
288 static void i8xx_fbc_activate(struct intel_fbc *fbc)  in i8xx_fbc_activate()  argument
290 	const struct intel_fbc_state *fbc_state = &fbc->state;  in i8xx_fbc_activate()
291 	struct drm_i915_private *i915 = fbc->i915;  in i8xx_fbc_activate()
300 			       i965_fbc_ctl2(fbc));  in i8xx_fbc_activate()
306 		       FBC_CTL_EN | i8xx_fbc_ctl(fbc));  in i8xx_fbc_activate()
309 static bool i8xx_fbc_is_active(struct intel_fbc *fbc)  in i8xx_fbc_is_active()  argument
311 	return intel_de_read(fbc->i915, FBC_CONTROL) & FBC_CTL_EN;  in i8xx_fbc_is_active()
314 static bool i8xx_fbc_is_compressing(struct intel_fbc *fbc)  in i8xx_fbc_is_compressing()  argument
316 	return intel_de_read(fbc->i915, FBC_STATUS) &  in i8xx_fbc_is_compressing()
320 static void i8xx_fbc_nuke(struct intel_fbc *fbc)  in i8xx_fbc_nuke()  argument
322 	struct intel_fbc_state *fbc_state = &fbc->state;  in i8xx_fbc_nuke()
324 	struct drm_i915_private *dev_priv = fbc->i915;  in i8xx_fbc_nuke()
332 static void i8xx_fbc_program_cfb(struct intel_fbc *fbc)  in i8xx_fbc_program_cfb()  argument
334 	struct drm_i915_private *i915 = fbc->i915;  in i8xx_fbc_program_cfb()
337 					 fbc->compressed_fb.start, U32_MAX));  in i8xx_fbc_program_cfb()
339 					 fbc->compressed_llb.start, U32_MAX));  in i8xx_fbc_program_cfb()
342 		       i915->dsm.start + fbc->compressed_fb.start);  in i8xx_fbc_program_cfb()
344 		       i915->dsm.start + fbc->compressed_llb.start);  in i8xx_fbc_program_cfb()
356 static void i965_fbc_nuke(struct intel_fbc *fbc)  in i965_fbc_nuke()  argument
358 	struct intel_fbc_state *fbc_state = &fbc->state;  in i965_fbc_nuke()
360 	struct drm_i915_private *dev_priv = fbc->i915;  in i965_fbc_nuke()
377 static u32 g4x_dpfc_ctl_limit(struct intel_fbc *fbc)  in g4x_dpfc_ctl_limit()  argument
379 	switch (fbc->limit) {  in g4x_dpfc_ctl_limit()
381 		MISSING_CASE(fbc->limit);  in g4x_dpfc_ctl_limit()
392 static u32 g4x_dpfc_ctl(struct intel_fbc *fbc)  in g4x_dpfc_ctl()  argument
394 	const struct intel_fbc_state *fbc_state = &fbc->state;  in g4x_dpfc_ctl()
395 	struct drm_i915_private *i915 = fbc->i915;  in g4x_dpfc_ctl()
398 	dpfc_ctl = g4x_dpfc_ctl_limit(fbc) |  in g4x_dpfc_ctl()
414 static void g4x_fbc_activate(struct intel_fbc *fbc)  in g4x_fbc_activate()  argument
416 	const struct intel_fbc_state *fbc_state = &fbc->state;  in g4x_fbc_activate()
417 	struct drm_i915_private *i915 = fbc->i915;  in g4x_fbc_activate()
423 		       DPFC_CTL_EN | g4x_dpfc_ctl(fbc));  in g4x_fbc_activate()
426 static void g4x_fbc_deactivate(struct intel_fbc *fbc)  in g4x_fbc_deactivate()  argument
428 	struct drm_i915_private *i915 = fbc->i915;  in g4x_fbc_deactivate()
439 static bool g4x_fbc_is_active(struct intel_fbc *fbc)  in g4x_fbc_is_active()  argument
441 	return intel_de_read(fbc->i915, DPFC_CONTROL) & DPFC_CTL_EN;  in g4x_fbc_is_active()
444 static bool g4x_fbc_is_compressing(struct intel_fbc *fbc)  in g4x_fbc_is_compressing()  argument
446 	return intel_de_read(fbc->i915, DPFC_STATUS) & DPFC_COMP_SEG_MASK;  in g4x_fbc_is_compressing()
449 static void g4x_fbc_program_cfb(struct intel_fbc *fbc)  in g4x_fbc_program_cfb()  argument
451 	struct drm_i915_private *i915 = fbc->i915;  in g4x_fbc_program_cfb()
453 	intel_de_write(i915, DPFC_CB_BASE, fbc->compressed_fb.start);  in g4x_fbc_program_cfb()
465 static void ilk_fbc_activate(struct intel_fbc *fbc)  in ilk_fbc_activate()  argument
467 	struct intel_fbc_state *fbc_state = &fbc->state;  in ilk_fbc_activate()
468 	struct drm_i915_private *i915 = fbc->i915;  in ilk_fbc_activate()
470 	intel_de_write(i915, ILK_DPFC_FENCE_YOFF(fbc->id),  in ilk_fbc_activate()
473 	intel_de_write(i915, ILK_DPFC_CONTROL(fbc->id),  in ilk_fbc_activate()
474 		       DPFC_CTL_EN | g4x_dpfc_ctl(fbc));  in ilk_fbc_activate()
477 static void ilk_fbc_deactivate(struct intel_fbc *fbc)  in ilk_fbc_deactivate()  argument
479 	struct drm_i915_private *i915 = fbc->i915;  in ilk_fbc_deactivate()
483 	dpfc_ctl = intel_de_read(i915, ILK_DPFC_CONTROL(fbc->id));  in ilk_fbc_deactivate()
486 		intel_de_write(i915, ILK_DPFC_CONTROL(fbc->id), dpfc_ctl);  in ilk_fbc_deactivate()
490 static bool ilk_fbc_is_active(struct intel_fbc *fbc)  in ilk_fbc_is_active()  argument
492 	return intel_de_read(fbc->i915, ILK_DPFC_CONTROL(fbc->id)) & DPFC_CTL_EN;  in ilk_fbc_is_active()
495 static bool ilk_fbc_is_compressing(struct intel_fbc *fbc)  in ilk_fbc_is_compressing()  argument
497 	return intel_de_read(fbc->i915, ILK_DPFC_STATUS(fbc->id)) & DPFC_COMP_SEG_MASK;  in ilk_fbc_is_compressing()
500 static void ilk_fbc_program_cfb(struct intel_fbc *fbc)  in ilk_fbc_program_cfb()  argument
502 	struct drm_i915_private *i915 = fbc->i915;  in ilk_fbc_program_cfb()
504 	intel_de_write(i915, ILK_DPFC_CB_BASE(fbc->id), fbc->compressed_fb.start);  in ilk_fbc_program_cfb()
516 static void snb_fbc_program_fence(struct intel_fbc *fbc)  in snb_fbc_program_fence()  argument
518 	const struct intel_fbc_state *fbc_state = &fbc->state;  in snb_fbc_program_fence()
519 	struct drm_i915_private *i915 = fbc->i915;  in snb_fbc_program_fence()
529 static void snb_fbc_activate(struct intel_fbc *fbc)  in snb_fbc_activate()  argument
531 	snb_fbc_program_fence(fbc);  in snb_fbc_activate()
533 	ilk_fbc_activate(fbc);  in snb_fbc_activate()
536 static void snb_fbc_nuke(struct intel_fbc *fbc)  in snb_fbc_nuke()  argument
538 	struct drm_i915_private *i915 = fbc->i915;  in snb_fbc_nuke()
540 	intel_de_write(i915, MSG_FBC_REND_STATE(fbc->id), FBC_REND_NUKE);  in snb_fbc_nuke()
541 	intel_de_posting_read(i915, MSG_FBC_REND_STATE(fbc->id));  in snb_fbc_nuke()
553 static void glk_fbc_program_cfb_stride(struct intel_fbc *fbc)  in glk_fbc_program_cfb_stride()  argument
555 	const struct intel_fbc_state *fbc_state = &fbc->state;  in glk_fbc_program_cfb_stride()
556 	struct drm_i915_private *i915 = fbc->i915;  in glk_fbc_program_cfb_stride()
561 			FBC_STRIDE(fbc_state->override_cfb_stride / fbc->limit);  in glk_fbc_program_cfb_stride()
563 	intel_de_write(i915, GLK_FBC_STRIDE(fbc->id), val);  in glk_fbc_program_cfb_stride()
566 static void skl_fbc_program_cfb_stride(struct intel_fbc *fbc)  in skl_fbc_program_cfb_stride()  argument
568 	const struct intel_fbc_state *fbc_state = &fbc->state;  in skl_fbc_program_cfb_stride()
569 	struct drm_i915_private *i915 = fbc->i915;  in skl_fbc_program_cfb_stride()
575 			CHICKEN_FBC_STRIDE(fbc_state->override_cfb_stride / fbc->limit);  in skl_fbc_program_cfb_stride()
582 static u32 ivb_dpfc_ctl(struct intel_fbc *fbc)  in ivb_dpfc_ctl()  argument
584 	const struct intel_fbc_state *fbc_state = &fbc->state;  in ivb_dpfc_ctl()
585 	struct drm_i915_private *i915 = fbc->i915;  in ivb_dpfc_ctl()
588 	dpfc_ctl = g4x_dpfc_ctl_limit(fbc);  in ivb_dpfc_ctl()
596 	if (fbc->false_color)  in ivb_dpfc_ctl()
602 static void ivb_fbc_activate(struct intel_fbc *fbc)  in ivb_fbc_activate()  argument
604 	struct drm_i915_private *i915 = fbc->i915;  in ivb_fbc_activate()
607 		glk_fbc_program_cfb_stride(fbc);  in ivb_fbc_activate()
609 		skl_fbc_program_cfb_stride(fbc);  in ivb_fbc_activate()
612 		snb_fbc_program_fence(fbc);  in ivb_fbc_activate()
614 	intel_de_write(i915, ILK_DPFC_CONTROL(fbc->id),  in ivb_fbc_activate()
615 		       DPFC_CTL_EN | ivb_dpfc_ctl(fbc));  in ivb_fbc_activate()
618 static bool ivb_fbc_is_compressing(struct intel_fbc *fbc)  in ivb_fbc_is_compressing()  argument
620 	return intel_de_read(fbc->i915, ILK_DPFC_STATUS2(fbc->id)) & DPFC_COMP_SEG_MASK_IVB;  in ivb_fbc_is_compressing()
623 static void ivb_fbc_set_false_color(struct intel_fbc *fbc,  in ivb_fbc_set_false_color()  argument
626 	intel_de_rmw(fbc->i915, ILK_DPFC_CONTROL(fbc->id),  in ivb_fbc_set_false_color()
640 static bool intel_fbc_hw_is_active(struct intel_fbc *fbc)  in intel_fbc_hw_is_active()  argument
642 	return fbc->funcs->is_active(fbc);  in intel_fbc_hw_is_active()
645 static void intel_fbc_hw_activate(struct intel_fbc *fbc)  in intel_fbc_hw_activate()  argument
647 	trace_intel_fbc_activate(fbc->state.plane);  in intel_fbc_hw_activate()
649 	fbc->active = true;  in intel_fbc_hw_activate()
650 	fbc->activated = true;  in intel_fbc_hw_activate()
652 	fbc->funcs->activate(fbc);  in intel_fbc_hw_activate()
655 static void intel_fbc_hw_deactivate(struct intel_fbc *fbc)  in intel_fbc_hw_deactivate()  argument
657 	trace_intel_fbc_deactivate(fbc->state.plane);  in intel_fbc_hw_deactivate()
659 	fbc->active = false;  in intel_fbc_hw_deactivate()
661 	fbc->funcs->deactivate(fbc);  in intel_fbc_hw_deactivate()
664 static bool intel_fbc_is_compressing(struct intel_fbc *fbc)  in intel_fbc_is_compressing()  argument
666 	return fbc->funcs->is_compressing(fbc);  in intel_fbc_is_compressing()
669 static void intel_fbc_nuke(struct intel_fbc *fbc)  in intel_fbc_nuke()  argument
671 	struct drm_i915_private *i915 = fbc->i915;  in intel_fbc_nuke()
673 	drm_WARN_ON(&i915->drm, fbc->flip_pending);  in intel_fbc_nuke()
675 	trace_intel_fbc_nuke(fbc->state.plane);  in intel_fbc_nuke()
677 	fbc->funcs->nuke(fbc);  in intel_fbc_nuke()
680 static void intel_fbc_activate(struct intel_fbc *fbc)  in intel_fbc_activate()  argument
682 	intel_fbc_hw_activate(fbc);  in intel_fbc_activate()
683 	intel_fbc_nuke(fbc);  in intel_fbc_activate()
685 	fbc->no_fbc_reason = NULL;  in intel_fbc_activate()
688 static void intel_fbc_deactivate(struct intel_fbc *fbc, const char *reason)  in intel_fbc_deactivate()  argument
690 	struct drm_i915_private *i915 = fbc->i915;  in intel_fbc_deactivate()
692 	drm_WARN_ON(&i915->drm, !mutex_is_locked(&fbc->lock));  in intel_fbc_deactivate()
694 	if (fbc->active)  in intel_fbc_deactivate()
695 		intel_fbc_hw_deactivate(fbc);  in intel_fbc_deactivate()
697 	fbc->no_fbc_reason = reason;  in intel_fbc_deactivate()
743 static int find_compression_limit(struct intel_fbc *fbc,  in find_compression_limit()  argument
746 	struct drm_i915_private *i915 = fbc->i915;  in find_compression_limit()
753 	ret = i915_gem_stolen_insert_node_in_range(i915, &fbc->compressed_fb,  in find_compression_limit()
759 		ret = i915_gem_stolen_insert_node_in_range(i915, &fbc->compressed_fb,  in find_compression_limit()
768 static int intel_fbc_alloc_cfb(struct intel_fbc *fbc,  in intel_fbc_alloc_cfb()  argument
771 	struct drm_i915_private *i915 = fbc->i915;  in intel_fbc_alloc_cfb()
775 		    drm_mm_node_allocated(&fbc->compressed_fb));  in intel_fbc_alloc_cfb()
777 		    drm_mm_node_allocated(&fbc->compressed_llb));  in intel_fbc_alloc_cfb()
780 		ret = i915_gem_stolen_insert_node(i915, &fbc->compressed_llb,  in intel_fbc_alloc_cfb()
786 	ret = find_compression_limit(fbc, size, min_limit);  in intel_fbc_alloc_cfb()
793 	fbc->limit = ret;  in intel_fbc_alloc_cfb()
797 		    fbc->compressed_fb.size, fbc->limit);  in intel_fbc_alloc_cfb()
802 	if (drm_mm_node_allocated(&fbc->compressed_llb))  in intel_fbc_alloc_cfb()
803 		i915_gem_stolen_remove_node(i915, &fbc->compressed_llb);  in intel_fbc_alloc_cfb()
810 static void intel_fbc_program_cfb(struct intel_fbc *fbc)  in intel_fbc_program_cfb()  argument
812 	fbc->funcs->program_cfb(fbc);  in intel_fbc_program_cfb()
815 static void intel_fbc_program_workarounds(struct intel_fbc *fbc)  in intel_fbc_program_workarounds()  argument
818 	if (DISPLAY_VER(fbc->i915) >= 11 && !IS_DG2(fbc->i915))  in intel_fbc_program_workarounds()
819 		intel_de_rmw(fbc->i915, ILK_DPFC_CHICKEN(fbc->id), 0,  in intel_fbc_program_workarounds()
823 static void __intel_fbc_cleanup_cfb(struct intel_fbc *fbc)  in __intel_fbc_cleanup_cfb()  argument
825 	struct drm_i915_private *i915 = fbc->i915;  in __intel_fbc_cleanup_cfb()
827 	if (WARN_ON(intel_fbc_hw_is_active(fbc)))  in __intel_fbc_cleanup_cfb()
830 	if (drm_mm_node_allocated(&fbc->compressed_llb))  in __intel_fbc_cleanup_cfb()
831 		i915_gem_stolen_remove_node(i915, &fbc->compressed_llb);  in __intel_fbc_cleanup_cfb()
832 	if (drm_mm_node_allocated(&fbc->compressed_fb))  in __intel_fbc_cleanup_cfb()
833 		i915_gem_stolen_remove_node(i915, &fbc->compressed_fb);  in __intel_fbc_cleanup_cfb()
838 	struct intel_fbc *fbc;  in intel_fbc_cleanup()  local
841 	for_each_intel_fbc(i915, fbc, fbc_id) {  in intel_fbc_cleanup()
842 		mutex_lock(&fbc->lock);  in intel_fbc_cleanup()
843 		__intel_fbc_cleanup_cfb(fbc);  in intel_fbc_cleanup()
844 		mutex_unlock(&fbc->lock);  in intel_fbc_cleanup()
846 		kfree(fbc);  in intel_fbc_cleanup()
981 	struct intel_fbc *fbc = plane->fbc;  in intel_fbc_update_state()  local
982 	struct intel_fbc_state *fbc_state = &fbc->state;  in intel_fbc_update_state()
1037 	struct intel_fbc *fbc = plane->fbc;  in intel_fbc_is_cfb_ok()  local
1039 	return intel_fbc_min_limit(plane_state) <= fbc->limit &&  in intel_fbc_is_cfb_ok()
1040 		intel_fbc_cfb_size(plane_state) <= fbc->compressed_fb.size * fbc->limit;  in intel_fbc_is_cfb_ok()
1059 	struct intel_fbc *fbc = plane->fbc;  in intel_fbc_check_plane()  local
1061 	if (!fbc)  in intel_fbc_check_plane()
1227 	struct intel_fbc *fbc = plane->fbc;  in __intel_fbc_pre_update()  local
1230 	fbc->flip_pending = true;  in __intel_fbc_pre_update()
1235 	intel_fbc_deactivate(fbc, "update pending");  in __intel_fbc_pre_update()
1250 	if (fbc->activated && DISPLAY_VER(i915) >= 10)  in __intel_fbc_pre_update()
1252 	fbc->activated = false;  in __intel_fbc_pre_update()
1266 		struct intel_fbc *fbc = plane->fbc;  in intel_fbc_pre_update()  local
1268 		if (!fbc || plane->pipe != crtc->pipe)  in intel_fbc_pre_update()
1271 		mutex_lock(&fbc->lock);  in intel_fbc_pre_update()
1273 		if (fbc->state.plane == plane)  in intel_fbc_pre_update()
1276 		mutex_unlock(&fbc->lock);  in intel_fbc_pre_update()
1282 static void __intel_fbc_disable(struct intel_fbc *fbc)  in __intel_fbc_disable()  argument
1284 	struct drm_i915_private *i915 = fbc->i915;  in __intel_fbc_disable()
1285 	struct intel_plane *plane = fbc->state.plane;  in __intel_fbc_disable()
1287 	drm_WARN_ON(&i915->drm, !mutex_is_locked(&fbc->lock));  in __intel_fbc_disable()
1288 	drm_WARN_ON(&i915->drm, fbc->active);  in __intel_fbc_disable()
1293 	__intel_fbc_cleanup_cfb(fbc);  in __intel_fbc_disable()
1295 	fbc->state.plane = NULL;  in __intel_fbc_disable()
1296 	fbc->flip_pending = false;  in __intel_fbc_disable()
1297 	fbc->busy_bits = 0;  in __intel_fbc_disable()
1300 static void __intel_fbc_post_update(struct intel_fbc *fbc)  in __intel_fbc_post_update()  argument
1302 	struct drm_i915_private *i915 = fbc->i915;  in __intel_fbc_post_update()
1304 	drm_WARN_ON(&i915->drm, !mutex_is_locked(&fbc->lock));  in __intel_fbc_post_update()
1306 	if (!fbc->busy_bits)  in __intel_fbc_post_update()
1307 		intel_fbc_activate(fbc);  in __intel_fbc_post_update()
1309 		intel_fbc_deactivate(fbc, "frontbuffer write");  in __intel_fbc_post_update()
1320 		struct intel_fbc *fbc = plane->fbc;  in intel_fbc_post_update()  local
1322 		if (!fbc || plane->pipe != crtc->pipe)  in intel_fbc_post_update()
1325 		mutex_lock(&fbc->lock);  in intel_fbc_post_update()
1327 		if (fbc->state.plane == plane) {  in intel_fbc_post_update()
1328 			fbc->flip_pending = false;  in intel_fbc_post_update()
1329 			__intel_fbc_post_update(fbc);  in intel_fbc_post_update()
1332 		mutex_unlock(&fbc->lock);  in intel_fbc_post_update()
1336 static unsigned int intel_fbc_get_frontbuffer_bit(struct intel_fbc *fbc)  in intel_fbc_get_frontbuffer_bit()  argument
1338 	if (fbc->state.plane)  in intel_fbc_get_frontbuffer_bit()
1339 		return fbc->state.plane->frontbuffer_bit;  in intel_fbc_get_frontbuffer_bit()
1344 static void __intel_fbc_invalidate(struct intel_fbc *fbc,  in __intel_fbc_invalidate()  argument
1351 	mutex_lock(&fbc->lock);  in __intel_fbc_invalidate()
1353 	frontbuffer_bits &= intel_fbc_get_frontbuffer_bit(fbc);  in __intel_fbc_invalidate()
1357 	fbc->busy_bits |= frontbuffer_bits;  in __intel_fbc_invalidate()
1358 	intel_fbc_deactivate(fbc, "frontbuffer write");  in __intel_fbc_invalidate()
1361 	mutex_unlock(&fbc->lock);  in __intel_fbc_invalidate()
1368 	struct intel_fbc *fbc;  in intel_fbc_invalidate()  local
1371 	for_each_intel_fbc(i915, fbc, fbc_id)  in intel_fbc_invalidate()
1372 		__intel_fbc_invalidate(fbc, frontbuffer_bits, origin);  in intel_fbc_invalidate()
1376 static void __intel_fbc_flush(struct intel_fbc *fbc,  in __intel_fbc_flush()  argument
1380 	mutex_lock(&fbc->lock);  in __intel_fbc_flush()
1382 	frontbuffer_bits &= intel_fbc_get_frontbuffer_bit(fbc);  in __intel_fbc_flush()
1386 	fbc->busy_bits &= ~frontbuffer_bits;  in __intel_fbc_flush()
1391 	if (fbc->busy_bits || fbc->flip_pending)  in __intel_fbc_flush()
1394 	if (fbc->active)  in __intel_fbc_flush()
1395 		intel_fbc_nuke(fbc);  in __intel_fbc_flush()
1397 		intel_fbc_activate(fbc);  in __intel_fbc_flush()
1400 	mutex_unlock(&fbc->lock);  in __intel_fbc_flush()
1407 	struct intel_fbc *fbc;  in intel_fbc_flush()  local
1410 	for_each_intel_fbc(i915, fbc, fbc_id)  in intel_fbc_flush()
1411 		__intel_fbc_flush(fbc, frontbuffer_bits, origin);  in intel_fbc_flush()
1438 	struct intel_fbc *fbc = plane->fbc;  in __intel_fbc_enable()  local
1440 	if (fbc->state.plane) {  in __intel_fbc_enable()
1441 		if (fbc->state.plane != plane)  in __intel_fbc_enable()
1449 		__intel_fbc_disable(fbc);  in __intel_fbc_enable()
1452 	drm_WARN_ON(&i915->drm, fbc->active);  in __intel_fbc_enable()
1454 	fbc->no_fbc_reason = plane_state->no_fbc_reason;  in __intel_fbc_enable()
1455 	if (fbc->no_fbc_reason)  in __intel_fbc_enable()
1459 		fbc->no_fbc_reason = "framebuffer not fenced";  in __intel_fbc_enable()
1463 	if (fbc->underrun_detected) {  in __intel_fbc_enable()
1464 		fbc->no_fbc_reason = "FIFO underrun";  in __intel_fbc_enable()
1468 	if (intel_fbc_alloc_cfb(fbc, intel_fbc_cfb_size(plane_state),  in __intel_fbc_enable()
1470 		fbc->no_fbc_reason = "not enough stolen memory";  in __intel_fbc_enable()
1476 	fbc->no_fbc_reason = "FBC enabled but not active yet\n";  in __intel_fbc_enable()
1480 	intel_fbc_program_workarounds(fbc);  in __intel_fbc_enable()
1481 	intel_fbc_program_cfb(fbc);  in __intel_fbc_enable()
1496 		struct intel_fbc *fbc = plane->fbc;  in intel_fbc_disable()  local
1498 		if (!fbc || plane->pipe != crtc->pipe)  in intel_fbc_disable()
1501 		mutex_lock(&fbc->lock);  in intel_fbc_disable()
1502 		if (fbc->state.plane == plane)  in intel_fbc_disable()
1503 			__intel_fbc_disable(fbc);  in intel_fbc_disable()
1504 		mutex_unlock(&fbc->lock);  in intel_fbc_disable()
1518 		struct intel_fbc *fbc = plane->fbc;  in intel_fbc_update()  local
1520 		if (!fbc || plane->pipe != crtc->pipe)  in intel_fbc_update()
1523 		mutex_lock(&fbc->lock);  in intel_fbc_update()
1526 			if (fbc->state.plane == plane)  in intel_fbc_update()
1527 				__intel_fbc_disable(fbc);  in intel_fbc_update()
1532 		mutex_unlock(&fbc->lock);  in intel_fbc_update()
1538 	struct intel_fbc *fbc = container_of(work, typeof(*fbc), underrun_work);  in intel_fbc_underrun_work_fn()  local
1539 	struct drm_i915_private *i915 = fbc->i915;  in intel_fbc_underrun_work_fn()
1541 	mutex_lock(&fbc->lock);  in intel_fbc_underrun_work_fn()
1544 	if (fbc->underrun_detected || !fbc->state.plane)  in intel_fbc_underrun_work_fn()
1548 	fbc->underrun_detected = true;  in intel_fbc_underrun_work_fn()
1550 	intel_fbc_deactivate(fbc, "FIFO underrun");  in intel_fbc_underrun_work_fn()
1551 	if (!fbc->flip_pending)  in intel_fbc_underrun_work_fn()
1552 		intel_crtc_wait_for_next_vblank(intel_crtc_for_pipe(i915, fbc->state.plane->pipe));  in intel_fbc_underrun_work_fn()
1553 	__intel_fbc_disable(fbc);  in intel_fbc_underrun_work_fn()
1555 	mutex_unlock(&fbc->lock);  in intel_fbc_underrun_work_fn()
1558 static void __intel_fbc_reset_underrun(struct intel_fbc *fbc)  in __intel_fbc_reset_underrun()  argument
1560 	struct drm_i915_private *i915 = fbc->i915;  in __intel_fbc_reset_underrun()
1562 	cancel_work_sync(&fbc->underrun_work);  in __intel_fbc_reset_underrun()
1564 	mutex_lock(&fbc->lock);  in __intel_fbc_reset_underrun()
1566 	if (fbc->underrun_detected) {  in __intel_fbc_reset_underrun()
1569 		fbc->no_fbc_reason = "FIFO underrun cleared";  in __intel_fbc_reset_underrun()
1572 	fbc->underrun_detected = false;  in __intel_fbc_reset_underrun()
1573 	mutex_unlock(&fbc->lock);  in __intel_fbc_reset_underrun()
1585 	struct intel_fbc *fbc;  in intel_fbc_reset_underrun()  local
1588 	for_each_intel_fbc(i915, fbc, fbc_id)  in intel_fbc_reset_underrun()
1589 		__intel_fbc_reset_underrun(fbc);  in intel_fbc_reset_underrun()
1592 static void __intel_fbc_handle_fifo_underrun_irq(struct intel_fbc *fbc)  in __intel_fbc_handle_fifo_underrun_irq()  argument
1602 	if (READ_ONCE(fbc->underrun_detected))  in __intel_fbc_handle_fifo_underrun_irq()
1605 	schedule_work(&fbc->underrun_work);  in __intel_fbc_handle_fifo_underrun_irq()
1624 	struct intel_fbc *fbc;  in intel_fbc_handle_fifo_underrun_irq()  local
1627 	for_each_intel_fbc(i915, fbc, fbc_id)  in intel_fbc_handle_fifo_underrun_irq()
1628 		__intel_fbc_handle_fifo_underrun_irq(fbc);  in intel_fbc_handle_fifo_underrun_irq()
1667 void intel_fbc_add_plane(struct intel_fbc *fbc, struct intel_plane *plane)  in intel_fbc_add_plane()  argument
1669 	plane->fbc = fbc;  in intel_fbc_add_plane()
1675 	struct intel_fbc *fbc;  in intel_fbc_create()  local
1677 	fbc = kzalloc(sizeof(*fbc), GFP_KERNEL);  in intel_fbc_create()
1678 	if (!fbc)  in intel_fbc_create()
1681 	fbc->id = fbc_id;  in intel_fbc_create()
1682 	fbc->i915 = i915;  in intel_fbc_create()
1683 	INIT_WORK(&fbc->underrun_work, intel_fbc_underrun_work_fn);  in intel_fbc_create()
1684 	mutex_init(&fbc->lock);  in intel_fbc_create()
1687 		fbc->funcs = &ivb_fbc_funcs;  in intel_fbc_create()
1689 		fbc->funcs = &snb_fbc_funcs;  in intel_fbc_create()
1691 		fbc->funcs = &ilk_fbc_funcs;  in intel_fbc_create()
1693 		fbc->funcs = &g4x_fbc_funcs;  in intel_fbc_create()
1695 		fbc->funcs = &i965_fbc_funcs;  in intel_fbc_create()
1697 		fbc->funcs = &i8xx_fbc_funcs;  in intel_fbc_create()
1699 	return fbc;  in intel_fbc_create()
1723 		i915->display.fbc[fbc_id] = intel_fbc_create(i915, fbc_id);  in intel_fbc_init()
1736 	struct intel_fbc *fbc;  in intel_fbc_sanitize()  local
1739 	for_each_intel_fbc(i915, fbc, fbc_id) {  in intel_fbc_sanitize()
1740 		if (intel_fbc_hw_is_active(fbc))  in intel_fbc_sanitize()
1741 			intel_fbc_hw_deactivate(fbc);  in intel_fbc_sanitize()
1747 	struct intel_fbc *fbc = m->private;  in intel_fbc_debugfs_status_show()  local
1748 	struct drm_i915_private *i915 = fbc->i915;  in intel_fbc_debugfs_status_show()
1755 	mutex_lock(&fbc->lock);  in intel_fbc_debugfs_status_show()
1757 	if (fbc->active) {  in intel_fbc_debugfs_status_show()
1760 			   str_yes_no(intel_fbc_is_compressing(fbc)));  in intel_fbc_debugfs_status_show()
1762 		seq_printf(m, "FBC disabled: %s\n", fbc->no_fbc_reason);  in intel_fbc_debugfs_status_show()
1769 		if (plane->fbc != fbc)  in intel_fbc_debugfs_status_show()
1773 			   fbc->state.plane == plane ? '*' : ' ',  in intel_fbc_debugfs_status_show()
1778 	mutex_unlock(&fbc->lock);  in intel_fbc_debugfs_status_show()
1790 	struct intel_fbc *fbc = data;  in intel_fbc_debugfs_false_color_get()  local
1792 	*val = fbc->false_color;  in intel_fbc_debugfs_false_color_get()
1799 	struct intel_fbc *fbc = data;  in intel_fbc_debugfs_false_color_set()  local
1801 	mutex_lock(&fbc->lock);  in intel_fbc_debugfs_false_color_set()
1803 	fbc->false_color = val;  in intel_fbc_debugfs_false_color_set()
1805 	if (fbc->active)  in intel_fbc_debugfs_false_color_set()
1806 		fbc->funcs->set_false_color(fbc, fbc->false_color);  in intel_fbc_debugfs_false_color_set()
1808 	mutex_unlock(&fbc->lock);  in intel_fbc_debugfs_false_color_set()
1818 static void intel_fbc_debugfs_add(struct intel_fbc *fbc,  in intel_fbc_debugfs_add()  argument
1822 			    fbc, &intel_fbc_debugfs_status_fops);  in intel_fbc_debugfs_add()
1824 	if (fbc->funcs->set_false_color)  in intel_fbc_debugfs_add()
1826 				    fbc, &intel_fbc_debugfs_false_color_fops);  in intel_fbc_debugfs_add()
1833 	if (plane->fbc)  in intel_fbc_crtc_debugfs_add()
1834 		intel_fbc_debugfs_add(plane->fbc, crtc->base.debugfs_entry);  in intel_fbc_crtc_debugfs_add()
1841 	struct intel_fbc *fbc;  in intel_fbc_debugfs_register()  local
1843 	fbc = i915->display.fbc[INTEL_FBC_A];  in intel_fbc_debugfs_register()
1844 	if (fbc)  in intel_fbc_debugfs_register()
1845 		intel_fbc_debugfs_add(fbc, minor->debugfs_root);  in intel_fbc_debugfs_register()