64 	u32 cmd, *cs;  in gen2_render_ring_flush()  local
73 	cs = intel_ring_begin(rq, 2 + 3 * num_store_dw);  in gen2_render_ring_flush()
74 	if (IS_ERR(cs))  in gen2_render_ring_flush()
75 		return PTR_ERR(cs);  in gen2_render_ring_flush()
77 	*cs++ = cmd;  in gen2_render_ring_flush()
79 		*cs++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL;  in gen2_render_ring_flush()
80 		*cs++ = intel_gt_scratch_offset(rq->engine->gt,  in gen2_render_ring_flush()
82 		*cs++ = 0;  in gen2_render_ring_flush()
84 	*cs++ = MI_FLUSH | MI_NO_WRITE_FLUSH;  in gen2_render_ring_flush()
86 	intel_ring_advance(rq, cs);  in gen2_render_ring_flush()
94 	u32 cmd, *cs;  in gen4_render_ring_flush()  local
136 	cs = intel_ring_begin(rq, i);  in gen4_render_ring_flush()
137 	if (IS_ERR(cs))  in gen4_render_ring_flush()
138 		return PTR_ERR(cs);  in gen4_render_ring_flush()
140 	*cs++ = cmd;  in gen4_render_ring_flush()
153 		*cs++ = GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE;  in gen4_render_ring_flush()
154 		*cs++ = intel_gt_scratch_offset(rq->engine->gt,  in gen4_render_ring_flush()
157 		*cs++ = 0;  in gen4_render_ring_flush()
158 		*cs++ = 0;  in gen4_render_ring_flush()
161 			*cs++ = MI_FLUSH;  in gen4_render_ring_flush()
163 		*cs++ = GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE;  in gen4_render_ring_flush()
164 		*cs++ = intel_gt_scratch_offset(rq->engine->gt,  in gen4_render_ring_flush()
167 		*cs++ = 0;  in gen4_render_ring_flush()
168 		*cs++ = 0;  in gen4_render_ring_flush()
171 	*cs++ = cmd;  in gen4_render_ring_flush()
173 	intel_ring_advance(rq, cs);  in gen4_render_ring_flush()
221 	u32 *cs;  in gen6_emit_post_sync_nonzero_flush()  local
223 	cs = intel_ring_begin(rq, 6);  in gen6_emit_post_sync_nonzero_flush()
224 	if (IS_ERR(cs))  in gen6_emit_post_sync_nonzero_flush()
225 		return PTR_ERR(cs);  in gen6_emit_post_sync_nonzero_flush()
227 	*cs++ = GFX_OP_PIPE_CONTROL(5);  in gen6_emit_post_sync_nonzero_flush()
228 	*cs++ = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD;  in gen6_emit_post_sync_nonzero_flush()
229 	*cs++ = scratch_addr | PIPE_CONTROL_GLOBAL_GTT;  in gen6_emit_post_sync_nonzero_flush()
230 	*cs++ = 0; /* low dword */  in gen6_emit_post_sync_nonzero_flush()
231 	*cs++ = 0; /* high dword */  in gen6_emit_post_sync_nonzero_flush()
232 	*cs++ = MI_NOOP;  in gen6_emit_post_sync_nonzero_flush()
233 	intel_ring_advance(rq, cs);  in gen6_emit_post_sync_nonzero_flush()
235 	cs = intel_ring_begin(rq, 6);  in gen6_emit_post_sync_nonzero_flush()
236 	if (IS_ERR(cs))  in gen6_emit_post_sync_nonzero_flush()
237 		return PTR_ERR(cs);  in gen6_emit_post_sync_nonzero_flush()
239 	*cs++ = GFX_OP_PIPE_CONTROL(5);  in gen6_emit_post_sync_nonzero_flush()
240 	*cs++ = PIPE_CONTROL_QW_WRITE;  in gen6_emit_post_sync_nonzero_flush()
241 	*cs++ = scratch_addr | PIPE_CONTROL_GLOBAL_GTT;  in gen6_emit_post_sync_nonzero_flush()
242 	*cs++ = 0;  in gen6_emit_post_sync_nonzero_flush()
243 	*cs++ = 0;  in gen6_emit_post_sync_nonzero_flush()
244 	*cs++ = MI_NOOP;  in gen6_emit_post_sync_nonzero_flush()
245 	intel_ring_advance(rq, cs);  in gen6_emit_post_sync_nonzero_flush()
256 	u32 *cs, flags = 0;  in gen6_render_ring_flush()  local
290 	cs = intel_ring_begin(rq, 4);  in gen6_render_ring_flush()
291 	if (IS_ERR(cs))  in gen6_render_ring_flush()
292 		return PTR_ERR(cs);  in gen6_render_ring_flush()
294 	*cs++ = GFX_OP_PIPE_CONTROL(4);  in gen6_render_ring_flush()
295 	*cs++ = flags;  in gen6_render_ring_flush()
296 	*cs++ = scratch_addr | PIPE_CONTROL_GLOBAL_GTT;  in gen6_render_ring_flush()
297 	*cs++ = 0;  in gen6_render_ring_flush()
298 	intel_ring_advance(rq, cs);  in gen6_render_ring_flush()
303 static u32 *gen6_rcs_emit_breadcrumb(struct i915_request *rq, u32 *cs)  in gen6_rcs_emit_breadcrumb()  argument
306 	*cs++ = GFX_OP_PIPE_CONTROL(4);  in gen6_rcs_emit_breadcrumb()
307 	*cs++ = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD;  in gen6_rcs_emit_breadcrumb()
308 	*cs++ = 0;  in gen6_rcs_emit_breadcrumb()
309 	*cs++ = 0;  in gen6_rcs_emit_breadcrumb()
311 	*cs++ = GFX_OP_PIPE_CONTROL(4);  in gen6_rcs_emit_breadcrumb()
312 	*cs++ = PIPE_CONTROL_QW_WRITE;  in gen6_rcs_emit_breadcrumb()
313 	*cs++ = intel_gt_scratch_offset(rq->engine->gt,  in gen6_rcs_emit_breadcrumb()
316 	*cs++ = 0;  in gen6_rcs_emit_breadcrumb()
319 	*cs++ = GFX_OP_PIPE_CONTROL(4);  in gen6_rcs_emit_breadcrumb()
320 	*cs++ = (PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH |  in gen6_rcs_emit_breadcrumb()
325 	*cs++ = rq->timeline->hwsp_offset | PIPE_CONTROL_GLOBAL_GTT;  in gen6_rcs_emit_breadcrumb()
326 	*cs++ = rq->fence.seqno;  in gen6_rcs_emit_breadcrumb()
328 	*cs++ = MI_USER_INTERRUPT;  in gen6_rcs_emit_breadcrumb()
329 	*cs++ = MI_NOOP;  in gen6_rcs_emit_breadcrumb()
331 	rq->tail = intel_ring_offset(rq, cs);  in gen6_rcs_emit_breadcrumb()
334 	return cs;  in gen6_rcs_emit_breadcrumb()
340 	u32 *cs;  in gen7_render_ring_cs_stall_wa()  local
342 	cs = intel_ring_begin(rq, 4);  in gen7_render_ring_cs_stall_wa()
343 	if (IS_ERR(cs))  in gen7_render_ring_cs_stall_wa()
344 		return PTR_ERR(cs);  in gen7_render_ring_cs_stall_wa()
346 	*cs++ = GFX_OP_PIPE_CONTROL(4);  in gen7_render_ring_cs_stall_wa()
347 	*cs++ = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD;  in gen7_render_ring_cs_stall_wa()
348 	*cs++ = 0;  in gen7_render_ring_cs_stall_wa()
349 	*cs++ = 0;  in gen7_render_ring_cs_stall_wa()
350 	intel_ring_advance(rq, cs);  in gen7_render_ring_cs_stall_wa()
361 	u32 *cs, flags = 0;  in gen7_render_ring_flush()  local
405 	cs = intel_ring_begin(rq, 4);  in gen7_render_ring_flush()
406 	if (IS_ERR(cs))  in gen7_render_ring_flush()
407 		return PTR_ERR(cs);  in gen7_render_ring_flush()
409 	*cs++ = GFX_OP_PIPE_CONTROL(4);  in gen7_render_ring_flush()
410 	*cs++ = flags;  in gen7_render_ring_flush()
411 	*cs++ = scratch_addr;  in gen7_render_ring_flush()
412 	*cs++ = 0;  in gen7_render_ring_flush()
413 	intel_ring_advance(rq, cs);  in gen7_render_ring_flush()
418 static u32 *gen7_rcs_emit_breadcrumb(struct i915_request *rq, u32 *cs)  in gen7_rcs_emit_breadcrumb()  argument
420 	*cs++ = GFX_OP_PIPE_CONTROL(4);  in gen7_rcs_emit_breadcrumb()
421 	*cs++ = (PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH |  in gen7_rcs_emit_breadcrumb()
428 	*cs++ = rq->timeline->hwsp_offset;  in gen7_rcs_emit_breadcrumb()
429 	*cs++ = rq->fence.seqno;  in gen7_rcs_emit_breadcrumb()
431 	*cs++ = MI_USER_INTERRUPT;  in gen7_rcs_emit_breadcrumb()
432 	*cs++ = MI_NOOP;  in gen7_rcs_emit_breadcrumb()
434 	rq->tail = intel_ring_offset(rq, cs);  in gen7_rcs_emit_breadcrumb()
437 	return cs;  in gen7_rcs_emit_breadcrumb()
440 static u32 *gen6_xcs_emit_breadcrumb(struct i915_request *rq, u32 *cs)  in gen6_xcs_emit_breadcrumb()  argument
445 	*cs++ = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_DW_STORE_INDEX;  in gen6_xcs_emit_breadcrumb()
446 	*cs++ = I915_GEM_HWS_SEQNO_ADDR | MI_FLUSH_DW_USE_GTT;  in gen6_xcs_emit_breadcrumb()
447 	*cs++ = rq->fence.seqno;  in gen6_xcs_emit_breadcrumb()
449 	*cs++ = MI_USER_INTERRUPT;  in gen6_xcs_emit_breadcrumb()
451 	rq->tail = intel_ring_offset(rq, cs);  in gen6_xcs_emit_breadcrumb()
454 	return cs;  in gen6_xcs_emit_breadcrumb()
458 static u32 *gen7_xcs_emit_breadcrumb(struct i915_request *rq, u32 *cs)  in gen7_xcs_emit_breadcrumb()  argument
465 	*cs++ = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_DW_STORE_INDEX;  in gen7_xcs_emit_breadcrumb()
466 	*cs++ = I915_GEM_HWS_SEQNO_ADDR | MI_FLUSH_DW_USE_GTT;  in gen7_xcs_emit_breadcrumb()
467 	*cs++ = rq->fence.seqno;  in gen7_xcs_emit_breadcrumb()
470 		*cs++ = MI_STORE_DWORD_INDEX;  in gen7_xcs_emit_breadcrumb()
471 		*cs++ = I915_GEM_HWS_SEQNO_ADDR;  in gen7_xcs_emit_breadcrumb()
472 		*cs++ = rq->fence.seqno;  in gen7_xcs_emit_breadcrumb()
475 	*cs++ = MI_FLUSH_DW;  in gen7_xcs_emit_breadcrumb()
476 	*cs++ = 0;  in gen7_xcs_emit_breadcrumb()
477 	*cs++ = 0;  in gen7_xcs_emit_breadcrumb()
479 	*cs++ = MI_USER_INTERRUPT;  in gen7_xcs_emit_breadcrumb()
480 	*cs++ = MI_NOOP;  in gen7_xcs_emit_breadcrumb()
482 	rq->tail = intel_ring_offset(rq, cs);  in gen7_xcs_emit_breadcrumb()
485 	return cs;  in gen7_xcs_emit_breadcrumb()
938 static u32 *i9xx_emit_breadcrumb(struct i915_request *rq, u32 *cs)  in i9xx_emit_breadcrumb()  argument
943 	*cs++ = MI_FLUSH;  in i9xx_emit_breadcrumb()
945 	*cs++ = MI_STORE_DWORD_INDEX;  in i9xx_emit_breadcrumb()
946 	*cs++ = I915_GEM_HWS_SEQNO_ADDR;  in i9xx_emit_breadcrumb()
947 	*cs++ = rq->fence.seqno;  in i9xx_emit_breadcrumb()
949 	*cs++ = MI_USER_INTERRUPT;  in i9xx_emit_breadcrumb()
950 	*cs++ = MI_NOOP;  in i9xx_emit_breadcrumb()
952 	rq->tail = intel_ring_offset(rq, cs);  in i9xx_emit_breadcrumb()
955 	return cs;  in i9xx_emit_breadcrumb()
959 static u32 *gen5_emit_breadcrumb(struct i915_request *rq, u32 *cs)  in gen5_emit_breadcrumb()  argument
966 	*cs++ = MI_FLUSH;  in gen5_emit_breadcrumb()
970 		*cs++ = MI_STORE_DWORD_INDEX;  in gen5_emit_breadcrumb()
971 		*cs++ = I915_GEM_HWS_SEQNO_ADDR;  in gen5_emit_breadcrumb()
972 		*cs++ = rq->fence.seqno;  in gen5_emit_breadcrumb()
975 	*cs++ = MI_USER_INTERRUPT;  in gen5_emit_breadcrumb()
977 	rq->tail = intel_ring_offset(rq, cs);  in gen5_emit_breadcrumb()
980 	return cs;  in gen5_emit_breadcrumb()
1033 	u32 *cs;  in bsd_ring_flush()  local
1035 	cs = intel_ring_begin(rq, 2);  in bsd_ring_flush()
1036 	if (IS_ERR(cs))  in bsd_ring_flush()
1037 		return PTR_ERR(cs);  in bsd_ring_flush()
1039 	*cs++ = MI_FLUSH;  in bsd_ring_flush()
1040 	*cs++ = MI_NOOP;  in bsd_ring_flush()
1041 	intel_ring_advance(rq, cs);  in bsd_ring_flush()
1087 	u32 *cs;  in i965_emit_bb_start()  local
1089 	cs = intel_ring_begin(rq, 2);  in i965_emit_bb_start()
1090 	if (IS_ERR(cs))  in i965_emit_bb_start()
1091 		return PTR_ERR(cs);  in i965_emit_bb_start()
1093 	*cs++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT | (dispatch_flags &  in i965_emit_bb_start()
1095 	*cs++ = offset;  in i965_emit_bb_start()
1096 	intel_ring_advance(rq, cs);  in i965_emit_bb_start()
1110 	u32 *cs, cs_offset =  in i830_emit_bb_start()  local
1116 	cs = intel_ring_begin(rq, 6);  in i830_emit_bb_start()
1117 	if (IS_ERR(cs))  in i830_emit_bb_start()
1118 		return PTR_ERR(cs);  in i830_emit_bb_start()
1121 	*cs++ = COLOR_BLT_CMD | BLT_WRITE_RGBA;  in i830_emit_bb_start()
1122 	*cs++ = BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | 4096;  in i830_emit_bb_start()
1123 	*cs++ = I830_TLB_ENTRIES << 16 | 4; /* load each page */  in i830_emit_bb_start()
1124 	*cs++ = cs_offset;  in i830_emit_bb_start()
1125 	*cs++ = 0xdeadbeef;  in i830_emit_bb_start()
1126 	*cs++ = MI_NOOP;  in i830_emit_bb_start()
1127 	intel_ring_advance(rq, cs);  in i830_emit_bb_start()
1133 		cs = intel_ring_begin(rq, 6 + 2);  in i830_emit_bb_start()
1134 		if (IS_ERR(cs))  in i830_emit_bb_start()
1135 			return PTR_ERR(cs);  in i830_emit_bb_start()
1141 		*cs++ = SRC_COPY_BLT_CMD | BLT_WRITE_RGBA | (6 - 2);  in i830_emit_bb_start()
1142 		*cs++ = BLT_DEPTH_32 | BLT_ROP_SRC_COPY | 4096;  in i830_emit_bb_start()
1143 		*cs++ = DIV_ROUND_UP(len, 4096) << 16 | 4096;  in i830_emit_bb_start()
1144 		*cs++ = cs_offset;  in i830_emit_bb_start()
1145 		*cs++ = 4096;  in i830_emit_bb_start()
1146 		*cs++ = offset;  in i830_emit_bb_start()
1148 		*cs++ = MI_FLUSH;  in i830_emit_bb_start()
1149 		*cs++ = MI_NOOP;  in i830_emit_bb_start()
1150 		intel_ring_advance(rq, cs);  in i830_emit_bb_start()
1156 	cs = intel_ring_begin(rq, 2);  in i830_emit_bb_start()
1157 	if (IS_ERR(cs))  in i830_emit_bb_start()
1158 		return PTR_ERR(cs);  in i830_emit_bb_start()
1160 	*cs++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT;  in i830_emit_bb_start()
1161 	*cs++ = offset | (dispatch_flags & I915_DISPATCH_SECURE ? 0 :  in i830_emit_bb_start()
1163 	intel_ring_advance(rq, cs);  in i830_emit_bb_start()
1173 	u32 *cs;  in i915_emit_bb_start()  local
1175 	cs = intel_ring_begin(rq, 2);  in i915_emit_bb_start()
1176 	if (IS_ERR(cs))  in i915_emit_bb_start()
1177 		return PTR_ERR(cs);  in i915_emit_bb_start()
1179 	*cs++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT;  in i915_emit_bb_start()
1180 	*cs++ = offset | (dispatch_flags & I915_DISPATCH_SECURE ? 0 :  in i915_emit_bb_start()
1182 	intel_ring_advance(rq, cs);  in i915_emit_bb_start()
1531 	u32 *cs;  in load_pd_dir()  local
1533 	cs = intel_ring_begin(rq, 6);  in load_pd_dir()
1534 	if (IS_ERR(cs))  in load_pd_dir()
1535 		return PTR_ERR(cs);  in load_pd_dir()
1537 	*cs++ = MI_LOAD_REGISTER_IMM(1);  in load_pd_dir()
1538 	*cs++ = i915_mmio_reg_offset(RING_PP_DIR_DCLV(engine->mmio_base));  in load_pd_dir()
1539 	*cs++ = PP_DIR_DCLV_2G;  in load_pd_dir()
1541 	*cs++ = MI_LOAD_REGISTER_IMM(1);  in load_pd_dir()
1542 	*cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine->mmio_base));  in load_pd_dir()
1543 	*cs++ = px_base(ppgtt->pd)->ggtt_offset << 10;  in load_pd_dir()
1545 	intel_ring_advance(rq, cs);  in load_pd_dir()
1553 	u32 *cs;  in flush_pd_dir()  local
1555 	cs = intel_ring_begin(rq, 4);  in flush_pd_dir()
1556 	if (IS_ERR(cs))  in flush_pd_dir()
1557 		return PTR_ERR(cs);  in flush_pd_dir()
1560 	*cs++ = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;  in flush_pd_dir()
1561 	*cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine->mmio_base));  in flush_pd_dir()
1562 	*cs++ = intel_gt_scratch_offset(rq->engine->gt,  in flush_pd_dir()
1564 	*cs++ = MI_NOOP;  in flush_pd_dir()
1566 	intel_ring_advance(rq, cs);  in flush_pd_dir()
1579 	u32 *cs;  in mi_set_context()  local
1601 	cs = intel_ring_begin(rq, len);  in mi_set_context()
1602 	if (IS_ERR(cs))  in mi_set_context()
1603 		return PTR_ERR(cs);  in mi_set_context()
1607 		*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;  in mi_set_context()
1611 			*cs++ = MI_LOAD_REGISTER_IMM(num_engines);  in mi_set_context()
1616 				*cs++ = i915_mmio_reg_offset(  in mi_set_context()
1618 				*cs++ = _MASKED_BIT_ENABLE(  in mi_set_context()
1629 		*cs++ = MI_SUSPEND_FLUSH | MI_SUSPEND_FLUSH_EN;  in mi_set_context()
1645 		*cs++ = MI_SET_CONTEXT;  in mi_set_context()
1646 		*cs++ = i915_ggtt_offset(engine->kernel_context->state) |  in mi_set_context()
1651 	*cs++ = MI_NOOP;  in mi_set_context()
1652 	*cs++ = MI_SET_CONTEXT;  in mi_set_context()
1653 	*cs++ = i915_ggtt_offset(rq->hw_context->state) | flags;  in mi_set_context()
1658 	*cs++ = MI_NOOP;  in mi_set_context()
1665 			*cs++ = MI_LOAD_REGISTER_IMM(num_engines);  in mi_set_context()
1671 				*cs++ = i915_mmio_reg_offset(last_reg);  in mi_set_context()
1672 				*cs++ = _MASKED_BIT_DISABLE(  in mi_set_context()
1677 			*cs++ = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;  in mi_set_context()
1678 			*cs++ = i915_mmio_reg_offset(last_reg);  in mi_set_context()
1679 			*cs++ = intel_gt_scratch_offset(rq->engine->gt,  in mi_set_context()
1681 			*cs++ = MI_NOOP;  in mi_set_context()
1683 		*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;  in mi_set_context()
1685 		*cs++ = MI_SUSPEND_FLUSH;  in mi_set_context()
1688 	intel_ring_advance(rq, cs);  in mi_set_context()
1695 	u32 *cs, *remap_info = rq->i915->l3_parity.remap_info[slice];  in remap_l3_slice()  local
1701 	cs = intel_ring_begin(rq, GEN7_L3LOG_SIZE/4 * 2 + 2);  in remap_l3_slice()
1702 	if (IS_ERR(cs))  in remap_l3_slice()
1703 		return PTR_ERR(cs);  in remap_l3_slice()
1710 	*cs++ = MI_LOAD_REGISTER_IMM(GEN7_L3LOG_SIZE/4);  in remap_l3_slice()
1712 		*cs++ = i915_mmio_reg_offset(GEN7_L3LOG(slice, i));  in remap_l3_slice()
1713 		*cs++ = remap_info[i];  in remap_l3_slice()
1715 	*cs++ = MI_NOOP;  in remap_l3_slice()
1716 	intel_ring_advance(rq, cs);  in remap_l3_slice()
1911 	u32 *cs;  in intel_ring_begin()  local
1974 	cs = ring->vaddr + ring->emit;  in intel_ring_begin()
1975 	GEM_DEBUG_EXEC(memset32(cs, POISON_INUSE, bytes / sizeof(*cs)));  in intel_ring_begin()
1979 	return cs;  in intel_ring_begin()
1986 	void *cs;  in intel_ring_cacheline_align()  local
1995 	cs = intel_ring_begin(rq, num_dwords);  in intel_ring_cacheline_align()
1996 	if (IS_ERR(cs))  in intel_ring_cacheline_align()
1997 		return PTR_ERR(cs);  in intel_ring_cacheline_align()
1999 	memset64(cs, (u64)MI_NOOP << 32 | MI_NOOP, num_dwords / 2);  in intel_ring_cacheline_align()
2000 	intel_ring_advance(rq, cs);  in intel_ring_cacheline_align()
2045 	u32 cmd, *cs;  in mi_flush_dw()  local
2047 	cs = intel_ring_begin(rq, 4);  in mi_flush_dw()
2048 	if (IS_ERR(cs))  in mi_flush_dw()
2049 		return PTR_ERR(cs);  in mi_flush_dw()
2069 	*cs++ = cmd;  in mi_flush_dw()
2070 	*cs++ = I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT;  in mi_flush_dw()
2071 	*cs++ = 0;  in mi_flush_dw()
2072 	*cs++ = MI_NOOP;  in mi_flush_dw()
2074 	intel_ring_advance(rq, cs);  in mi_flush_dw()
2094 	u32 *cs;  in hsw_emit_bb_start()  local
2096 	cs = intel_ring_begin(rq, 2);  in hsw_emit_bb_start()
2097 	if (IS_ERR(cs))  in hsw_emit_bb_start()
2098 		return PTR_ERR(cs);  in hsw_emit_bb_start()
2100 	*cs++ = MI_BATCH_BUFFER_START | (dispatch_flags & I915_DISPATCH_SECURE ?  in hsw_emit_bb_start()
2103 	*cs++ = offset;  in hsw_emit_bb_start()
2104 	intel_ring_advance(rq, cs);  in hsw_emit_bb_start()
2114 	u32 *cs;  in gen6_emit_bb_start()  local
2116 	cs = intel_ring_begin(rq, 2);  in gen6_emit_bb_start()
2117 	if (IS_ERR(cs))  in gen6_emit_bb_start()
2118 		return PTR_ERR(cs);  in gen6_emit_bb_start()
2120 	*cs++ = MI_BATCH_BUFFER_START | (dispatch_flags & I915_DISPATCH_SECURE ?  in gen6_emit_bb_start()
2123 	*cs++ = offset;  in gen6_emit_bb_start()
2124 	intel_ring_advance(rq, cs);  in gen6_emit_bb_start()