16 	u32 *cs, flags = 0;  in gen8_emit_flush_rcs()  local
58 	cs = intel_ring_begin(rq, len);  in gen8_emit_flush_rcs()
59 	if (IS_ERR(cs))  in gen8_emit_flush_rcs()
60 		return PTR_ERR(cs);  in gen8_emit_flush_rcs()
63 		cs = gen8_emit_pipe_control(cs, 0, 0);  in gen8_emit_flush_rcs()
66 		cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_DC_FLUSH_ENABLE,  in gen8_emit_flush_rcs()
69 	cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR);  in gen8_emit_flush_rcs()
72 		cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_CS_STALL, 0);  in gen8_emit_flush_rcs()
74 	intel_ring_advance(rq, cs);  in gen8_emit_flush_rcs()
81 	u32 cmd, *cs;  in gen8_emit_flush_xcs()  local
83 	cs = intel_ring_begin(rq, 4);  in gen8_emit_flush_xcs()
84 	if (IS_ERR(cs))  in gen8_emit_flush_xcs()
85 		return PTR_ERR(cs);  in gen8_emit_flush_xcs()
103 	*cs++ = cmd;  in gen8_emit_flush_xcs()
104 	*cs++ = LRC_PPHWSP_SCRATCH_ADDR;  in gen8_emit_flush_xcs()
105 	*cs++ = 0; /* upper addr */  in gen8_emit_flush_xcs()
106 	*cs++ = 0; /* value */  in gen8_emit_flush_xcs()
107 	intel_ring_advance(rq, cs);  in gen8_emit_flush_xcs()
115 		u32 *cs;  in gen11_emit_flush_rcs()  local
128 		cs = intel_ring_begin(rq, 6);  in gen11_emit_flush_rcs()
129 		if (IS_ERR(cs))  in gen11_emit_flush_rcs()
130 			return PTR_ERR(cs);  in gen11_emit_flush_rcs()
132 		cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR);  in gen11_emit_flush_rcs()
133 		intel_ring_advance(rq, cs);  in gen11_emit_flush_rcs()
137 		u32 *cs;  in gen11_emit_flush_rcs()  local
152 		cs = intel_ring_begin(rq, 6);  in gen11_emit_flush_rcs()
153 		if (IS_ERR(cs))  in gen11_emit_flush_rcs()
154 			return PTR_ERR(cs);  in gen11_emit_flush_rcs()
156 		cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR);  in gen11_emit_flush_rcs()
157 		intel_ring_advance(rq, cs);  in gen11_emit_flush_rcs()
168 u32 *gen12_emit_aux_table_inv(struct intel_gt *gt, u32 *cs, const i915_reg_t inv_reg)  in gen12_emit_aux_table_inv()  argument
172 	*cs++ = MI_LOAD_REGISTER_IMM(1) | MI_LRI_MMIO_REMAP_EN;  in gen12_emit_aux_table_inv()
173 	*cs++ = i915_mmio_reg_offset(inv_reg) + gsi_offset;  in gen12_emit_aux_table_inv()
174 	*cs++ = AUX_INV;  in gen12_emit_aux_table_inv()
175 	*cs++ = MI_NOOP;  in gen12_emit_aux_table_inv()
177 	return cs;  in gen12_emit_aux_table_inv()
186 		u32 *cs;  in gen12_emit_flush_rcs()  local
207 		cs = intel_ring_begin(rq, 6);  in gen12_emit_flush_rcs()
208 		if (IS_ERR(cs))  in gen12_emit_flush_rcs()
209 			return PTR_ERR(cs);  in gen12_emit_flush_rcs()
211 		cs = gen12_emit_pipe_control(cs,  in gen12_emit_flush_rcs()
214 		intel_ring_advance(rq, cs);  in gen12_emit_flush_rcs()
219 		u32 *cs, count;  in gen12_emit_flush_rcs()  local
244 		cs = intel_ring_begin(rq, count);  in gen12_emit_flush_rcs()
245 		if (IS_ERR(cs))  in gen12_emit_flush_rcs()
246 			return PTR_ERR(cs);  in gen12_emit_flush_rcs()
253 		*cs++ = preparser_disable(true);  in gen12_emit_flush_rcs()
255 		cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR);  in gen12_emit_flush_rcs()
259 			cs = gen12_emit_aux_table_inv(rq->engine->gt,  in gen12_emit_flush_rcs()
260 						      cs, GEN12_GFX_CCS_AUX_NV);  in gen12_emit_flush_rcs()
263 		*cs++ = preparser_disable(false);  in gen12_emit_flush_rcs()
264 		intel_ring_advance(rq, cs);  in gen12_emit_flush_rcs()
273 	u32 cmd, *cs;  in gen12_emit_flush_xcs()  local
289 	cs = intel_ring_begin(rq, cmd);  in gen12_emit_flush_xcs()
290 	if (IS_ERR(cs))  in gen12_emit_flush_xcs()
291 		return PTR_ERR(cs);  in gen12_emit_flush_xcs()
294 		*cs++ = preparser_disable(true);  in gen12_emit_flush_xcs()
312 	*cs++ = cmd;  in gen12_emit_flush_xcs()
313 	*cs++ = LRC_PPHWSP_SCRATCH_ADDR;  in gen12_emit_flush_xcs()
314 	*cs++ = 0; /* upper addr */  in gen12_emit_flush_xcs()
315 	*cs++ = 0; /* value */  in gen12_emit_flush_xcs()
319 			cs = gen12_emit_aux_table_inv(rq->engine->gt,  in gen12_emit_flush_xcs()
320 						      cs, GEN12_VD0_AUX_NV);  in gen12_emit_flush_xcs()
322 			cs = gen12_emit_aux_table_inv(rq->engine->gt,  in gen12_emit_flush_xcs()
323 						      cs, GEN12_VE0_AUX_NV);  in gen12_emit_flush_xcs()
327 		*cs++ = preparser_disable(false);  in gen12_emit_flush_xcs()
329 	intel_ring_advance(rq, cs);  in gen12_emit_flush_xcs()
354 	u32 *cs;  in gen8_emit_init_breadcrumb()  local
360 	cs = intel_ring_begin(rq, 6);  in gen8_emit_init_breadcrumb()
361 	if (IS_ERR(cs))  in gen8_emit_init_breadcrumb()
362 		return PTR_ERR(cs);  in gen8_emit_init_breadcrumb()
364 	*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;  in gen8_emit_init_breadcrumb()
365 	*cs++ = hwsp_offset(rq);  in gen8_emit_init_breadcrumb()
366 	*cs++ = 0;  in gen8_emit_init_breadcrumb()
367 	*cs++ = rq->fence.seqno - 1;  in gen8_emit_init_breadcrumb()
386 	*cs++ = MI_NOOP;  in gen8_emit_init_breadcrumb()
387 	*cs++ = MI_ARB_CHECK;  in gen8_emit_init_breadcrumb()
389 	intel_ring_advance(rq, cs);  in gen8_emit_init_breadcrumb()
392 	rq->infix = intel_ring_offset(rq, cs);  in gen8_emit_init_breadcrumb()
406 	u32 *cs;  in __gen125_emit_bb_start()  local
408 	cs = intel_ring_begin(rq, 12);  in __gen125_emit_bb_start()
409 	if (IS_ERR(cs))  in __gen125_emit_bb_start()
410 		return PTR_ERR(cs);  in __gen125_emit_bb_start()
412 	*cs++ = MI_ARB_ON_OFF | arb;  in __gen125_emit_bb_start()
414 	*cs++ = MI_LOAD_REGISTER_MEM_GEN8 |  in __gen125_emit_bb_start()
417 	*cs++ = i915_mmio_reg_offset(RING_PREDICATE_RESULT(0));  in __gen125_emit_bb_start()
418 	*cs++ = wa_offset + DG2_PREDICATE_RESULT_WA;  in __gen125_emit_bb_start()
419 	*cs++ = 0;  in __gen125_emit_bb_start()
421 	*cs++ = MI_BATCH_BUFFER_START_GEN8 |  in __gen125_emit_bb_start()
423 	*cs++ = lower_32_bits(offset);  in __gen125_emit_bb_start()
424 	*cs++ = upper_32_bits(offset);  in __gen125_emit_bb_start()
427 	*cs++ = MI_BATCH_BUFFER_START_GEN8;  in __gen125_emit_bb_start()
428 	*cs++ = wa_offset + DG2_PREDICATE_RESULT_BB;  in __gen125_emit_bb_start()
429 	*cs++ = 0;  in __gen125_emit_bb_start()
431 	*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;  in __gen125_emit_bb_start()
433 	intel_ring_advance(rq, cs);  in __gen125_emit_bb_start()
456 	u32 *cs;  in gen8_emit_bb_start_noarb()  local
458 	cs = intel_ring_begin(rq, 4);  in gen8_emit_bb_start_noarb()
459 	if (IS_ERR(cs))  in gen8_emit_bb_start_noarb()
460 		return PTR_ERR(cs);  in gen8_emit_bb_start_noarb()
475 	*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;  in gen8_emit_bb_start_noarb()
478 	*cs++ = MI_BATCH_BUFFER_START_GEN8 |  in gen8_emit_bb_start_noarb()
480 	*cs++ = lower_32_bits(offset);  in gen8_emit_bb_start_noarb()
481 	*cs++ = upper_32_bits(offset);  in gen8_emit_bb_start_noarb()
483 	intel_ring_advance(rq, cs);  in gen8_emit_bb_start_noarb()
492 	u32 *cs;  in gen8_emit_bb_start()  local
497 	cs = intel_ring_begin(rq, 6);  in gen8_emit_bb_start()
498 	if (IS_ERR(cs))  in gen8_emit_bb_start()
499 		return PTR_ERR(cs);  in gen8_emit_bb_start()
501 	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;  in gen8_emit_bb_start()
503 	*cs++ = MI_BATCH_BUFFER_START_GEN8 |  in gen8_emit_bb_start()
505 	*cs++ = lower_32_bits(offset);  in gen8_emit_bb_start()
506 	*cs++ = upper_32_bits(offset);  in gen8_emit_bb_start()
508 	*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;  in gen8_emit_bb_start()
509 	*cs++ = MI_NOOP;  in gen8_emit_bb_start()
511 	intel_ring_advance(rq, cs);  in gen8_emit_bb_start()
529 static u32 *gen8_emit_wa_tail(struct i915_request *rq, u32 *cs)  in gen8_emit_wa_tail()  argument
532 	*cs++ = MI_ARB_CHECK;  in gen8_emit_wa_tail()
533 	*cs++ = MI_NOOP;  in gen8_emit_wa_tail()
534 	rq->wa_tail = intel_ring_offset(rq, cs);  in gen8_emit_wa_tail()
539 	return cs;  in gen8_emit_wa_tail()
542 static u32 *emit_preempt_busywait(struct i915_request *rq, u32 *cs)  in emit_preempt_busywait()  argument
544 	*cs++ = MI_ARB_CHECK; /* trigger IDLE->ACTIVE first */  in emit_preempt_busywait()
545 	*cs++ = MI_SEMAPHORE_WAIT |  in emit_preempt_busywait()
549 	*cs++ = 0;  in emit_preempt_busywait()
550 	*cs++ = preempt_address(rq->engine);  in emit_preempt_busywait()
551 	*cs++ = 0;  in emit_preempt_busywait()
552 	*cs++ = MI_NOOP;  in emit_preempt_busywait()
554 	return cs;  in emit_preempt_busywait()
558 gen8_emit_fini_breadcrumb_tail(struct i915_request *rq, u32 *cs)  in gen8_emit_fini_breadcrumb_tail()  argument
560 	*cs++ = MI_USER_INTERRUPT;  in gen8_emit_fini_breadcrumb_tail()
562 	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;  in gen8_emit_fini_breadcrumb_tail()
565 		cs = emit_preempt_busywait(rq, cs);  in gen8_emit_fini_breadcrumb_tail()
567 	rq->tail = intel_ring_offset(rq, cs);  in gen8_emit_fini_breadcrumb_tail()
570 	return gen8_emit_wa_tail(rq, cs);  in gen8_emit_fini_breadcrumb_tail()
573 static u32 *emit_xcs_breadcrumb(struct i915_request *rq, u32 *cs)  in emit_xcs_breadcrumb()  argument
575 	return gen8_emit_ggtt_write(cs, rq->fence.seqno, hwsp_offset(rq), 0);  in emit_xcs_breadcrumb()
578 u32 *gen8_emit_fini_breadcrumb_xcs(struct i915_request *rq, u32 *cs)  in gen8_emit_fini_breadcrumb_xcs()  argument
580 	return gen8_emit_fini_breadcrumb_tail(rq, emit_xcs_breadcrumb(rq, cs));  in gen8_emit_fini_breadcrumb_xcs()
583 u32 *gen8_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs)  in gen8_emit_fini_breadcrumb_rcs()  argument
585 	cs = gen8_emit_pipe_control(cs,  in gen8_emit_fini_breadcrumb_rcs()
592 	cs = gen8_emit_ggtt_write_rcs(cs,  in gen8_emit_fini_breadcrumb_rcs()
598 	return gen8_emit_fini_breadcrumb_tail(rq, cs);  in gen8_emit_fini_breadcrumb_rcs()
601 u32 *gen11_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs)  in gen11_emit_fini_breadcrumb_rcs()  argument
603 	cs = gen8_emit_ggtt_write_rcs(cs,  in gen11_emit_fini_breadcrumb_rcs()
613 	return gen8_emit_fini_breadcrumb_tail(rq, cs);  in gen11_emit_fini_breadcrumb_rcs()
617  * Note that the CS instruction pre-parser will not stall on the breadcrumb
635 static u32 *gen12_emit_preempt_busywait(struct i915_request *rq, u32 *cs)  in gen12_emit_preempt_busywait()  argument
637 	*cs++ = MI_ARB_CHECK; /* trigger IDLE->ACTIVE first */  in gen12_emit_preempt_busywait()
638 	*cs++ = MI_SEMAPHORE_WAIT_TOKEN |  in gen12_emit_preempt_busywait()
642 	*cs++ = 0;  in gen12_emit_preempt_busywait()
643 	*cs++ = preempt_address(rq->engine);  in gen12_emit_preempt_busywait()
644 	*cs++ = 0;  in gen12_emit_preempt_busywait()
645 	*cs++ = 0;  in gen12_emit_preempt_busywait()
647 	return cs;  in gen12_emit_preempt_busywait()
659 static u32 *ccs_emit_wa_busywait(struct i915_request *rq, u32 *cs)  in ccs_emit_wa_busywait()  argument
663 	*cs++ = MI_ATOMIC_INLINE | MI_ATOMIC_GLOBAL_GTT | MI_ATOMIC_CS_STALL |  in ccs_emit_wa_busywait()
665 	*cs++ = ccs_semaphore_offset(rq);  in ccs_emit_wa_busywait()
666 	*cs++ = 0;  in ccs_emit_wa_busywait()
667 	*cs++ = 1;  in ccs_emit_wa_busywait()
674 		*cs++ = 0;  in ccs_emit_wa_busywait()
676 	*cs++ = MI_SEMAPHORE_WAIT |  in ccs_emit_wa_busywait()
680 	*cs++ = 0;  in ccs_emit_wa_busywait()
681 	*cs++ = ccs_semaphore_offset(rq);  in ccs_emit_wa_busywait()
682 	*cs++ = 0;  in ccs_emit_wa_busywait()
684 	return cs;  in ccs_emit_wa_busywait()
688 gen12_emit_fini_breadcrumb_tail(struct i915_request *rq, u32 *cs)  in gen12_emit_fini_breadcrumb_tail()  argument
690 	*cs++ = MI_USER_INTERRUPT;  in gen12_emit_fini_breadcrumb_tail()
692 	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;  in gen12_emit_fini_breadcrumb_tail()
695 		cs = gen12_emit_preempt_busywait(rq, cs);  in gen12_emit_fini_breadcrumb_tail()
699 		cs = ccs_emit_wa_busywait(rq, cs);  in gen12_emit_fini_breadcrumb_tail()
701 	rq->tail = intel_ring_offset(rq, cs);  in gen12_emit_fini_breadcrumb_tail()
704 	return gen8_emit_wa_tail(rq, cs);  in gen12_emit_fini_breadcrumb_tail()
707 u32 *gen12_emit_fini_breadcrumb_xcs(struct i915_request *rq, u32 *cs)  in gen12_emit_fini_breadcrumb_xcs()  argument
710 	cs = emit_xcs_breadcrumb(rq, __gen8_emit_flush_dw(cs, 0, 0, 0));  in gen12_emit_fini_breadcrumb_xcs()
711 	return gen12_emit_fini_breadcrumb_tail(rq, cs);  in gen12_emit_fini_breadcrumb_xcs()
714 u32 *gen12_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs)  in gen12_emit_fini_breadcrumb_rcs()  argument
734 	cs = gen12_emit_ggtt_write_rcs(cs,  in gen12_emit_fini_breadcrumb_rcs()
740 	return gen12_emit_fini_breadcrumb_tail(rq, cs);  in gen12_emit_fini_breadcrumb_rcs()