330 	u32 *cs;  in emit_no_arbitration()  local
332 	cs = intel_ring_begin(rq, 2);  in emit_no_arbitration()
333 	if (IS_ERR(cs))  in emit_no_arbitration()
334 		return PTR_ERR(cs);  in emit_no_arbitration()
337 	*cs++ = MI_ARB_ON_OFF;  in emit_no_arbitration()
338 	*cs++ = MI_NOOP;  in emit_no_arbitration()
339 	intel_ring_advance(rq, cs);  in emit_no_arbitration()
358 	u32 *hdr, *cs;  in emit_pte()  local
385 	cs = intel_ring_begin(rq, 6);  in emit_pte()
386 	if (IS_ERR(cs))  in emit_pte()
387 		return PTR_ERR(cs);  in emit_pte()
393 	hdr = cs;  in emit_pte()
394 	*cs++ = MI_STORE_DATA_IMM | REG_BIT(21); /* as qword elements */  in emit_pte()
395 	*cs++ = lower_32_bits(offset);  in emit_pte()
396 	*cs++ = upper_32_bits(offset);  in emit_pte()
399 		if (cs - hdr >= pkt) {  in emit_pte()
402 			*hdr += cs - hdr - 2;  in emit_pte()
403 			*cs++ = MI_NOOP;  in emit_pte()
405 			ring->emit = (void *)cs - ring->vaddr;  in emit_pte()
406 			intel_ring_advance(rq, cs);  in emit_pte()
409 			cs = intel_ring_begin(rq, 6);  in emit_pte()
410 			if (IS_ERR(cs))  in emit_pte()
411 				return PTR_ERR(cs);  in emit_pte()
427 			hdr = cs;  in emit_pte()
428 			*cs++ = MI_STORE_DATA_IMM | REG_BIT(21);  in emit_pte()
429 			*cs++ = lower_32_bits(offset);  in emit_pte()
430 			*cs++ = upper_32_bits(offset);  in emit_pte()
435 		*cs++ = lower_32_bits(encode | it->dma);  in emit_pte()
436 		*cs++ = upper_32_bits(encode | it->dma);  in emit_pte()
452 	*hdr += cs - hdr - 2;  in emit_pte()
453 	*cs++ = MI_NOOP;  in emit_pte()
455 	ring->emit = (void *)cs - ring->vaddr;  in emit_pte()
456 	intel_ring_advance(rq, cs);  in emit_pte()
521 	u32 *cs;  in emit_copy_ccs()  local
523 	cs = intel_ring_begin(rq, 12);  in emit_copy_ccs()
524 	if (IS_ERR(cs))  in emit_copy_ccs()
525 		return PTR_ERR(cs);  in emit_copy_ccs()
530 	cs = i915_flush_dw(cs, MI_FLUSH_DW_LLC | MI_FLUSH_DW_CCS);  in emit_copy_ccs()
546 	*cs++ = XY_CTRL_SURF_COPY_BLT |  in emit_copy_ccs()
550 	*cs++ = src_offset;  in emit_copy_ccs()
551 	*cs++ = rq->engine->instance |  in emit_copy_ccs()
553 	*cs++ = dst_offset;  in emit_copy_ccs()
554 	*cs++ = rq->engine->instance |  in emit_copy_ccs()
557 	cs = i915_flush_dw(cs, MI_FLUSH_DW_LLC | MI_FLUSH_DW_CCS);  in emit_copy_ccs()
558 	*cs++ = MI_NOOP;  in emit_copy_ccs()
560 	intel_ring_advance(rq, cs);  in emit_copy_ccs()
570 	u32 *cs;  in emit_copy()  local
572 	cs = intel_ring_begin(rq, ver >= 8 ? 10 : 6);  in emit_copy()
573 	if (IS_ERR(cs))  in emit_copy()
574 		return PTR_ERR(cs);  in emit_copy()
577 		*cs++ = GEN9_XY_FAST_COPY_BLT_CMD | (10 - 2);  in emit_copy()
578 		*cs++ = BLT_DEPTH_32 | PAGE_SIZE;  in emit_copy()
579 		*cs++ = 0;  in emit_copy()
580 		*cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4;  in emit_copy()
581 		*cs++ = dst_offset;  in emit_copy()
582 		*cs++ = instance;  in emit_copy()
583 		*cs++ = 0;  in emit_copy()
584 		*cs++ = PAGE_SIZE;  in emit_copy()
585 		*cs++ = src_offset;  in emit_copy()
586 		*cs++ = instance;  in emit_copy()
588 		*cs++ = XY_SRC_COPY_BLT_CMD | BLT_WRITE_RGBA | (10 - 2);  in emit_copy()
589 		*cs++ = BLT_DEPTH_32 | BLT_ROP_SRC_COPY | PAGE_SIZE;  in emit_copy()
590 		*cs++ = 0;  in emit_copy()
591 		*cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4;  in emit_copy()
592 		*cs++ = dst_offset;  in emit_copy()
593 		*cs++ = instance;  in emit_copy()
594 		*cs++ = 0;  in emit_copy()
595 		*cs++ = PAGE_SIZE;  in emit_copy()
596 		*cs++ = src_offset;  in emit_copy()
597 		*cs++ = instance;  in emit_copy()
600 		*cs++ = SRC_COPY_BLT_CMD | BLT_WRITE_RGBA | (6 - 2);  in emit_copy()
601 		*cs++ = BLT_DEPTH_32 | BLT_ROP_SRC_COPY | PAGE_SIZE;  in emit_copy()
602 		*cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE;  in emit_copy()
603 		*cs++ = dst_offset;  in emit_copy()
604 		*cs++ = PAGE_SIZE;  in emit_copy()
605 		*cs++ = src_offset;  in emit_copy()
608 	intel_ring_advance(rq, cs);  in emit_copy()
887 	u32 *cs;  in emit_clear()  local
898 	cs = intel_ring_begin(rq, ring_sz);  in emit_clear()
899 	if (IS_ERR(cs))  in emit_clear()
900 		return PTR_ERR(cs);  in emit_clear()
903 		*cs++ = XY_FAST_COLOR_BLT_CMD | XY_FAST_COLOR_BLT_DEPTH_32 |  in emit_clear()
905 		*cs++ = FIELD_PREP(XY_FAST_COLOR_BLT_MOCS_MASK, mocs) |  in emit_clear()
907 		*cs++ = 0;  in emit_clear()
908 		*cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4;  in emit_clear()
909 		*cs++ = offset;  in emit_clear()
910 		*cs++ = rq->engine->instance;  in emit_clear()
911 		*cs++ = !is_lmem << XY_FAST_COLOR_BLT_MEM_TYPE_SHIFT;  in emit_clear()
913 		*cs++ = value;  in emit_clear()
914 		*cs++ = 0;  in emit_clear()
915 		*cs++ = 0;  in emit_clear()
916 		*cs++ = 0;  in emit_clear()
918 		*cs++ = 0;  in emit_clear()
919 		*cs++ = 0;  in emit_clear()
921 		*cs++ = 0;  in emit_clear()
922 		*cs++ = 0;  in emit_clear()
923 		*cs++ = 0;  in emit_clear()
925 		*cs++ = XY_COLOR_BLT_CMD | BLT_WRITE_RGBA | (7 - 2);  in emit_clear()
926 		*cs++ = BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | PAGE_SIZE;  in emit_clear()
927 		*cs++ = 0;  in emit_clear()
928 		*cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4;  in emit_clear()
929 		*cs++ = offset;  in emit_clear()
930 		*cs++ = rq->engine->instance;  in emit_clear()
931 		*cs++ = value;  in emit_clear()
932 		*cs++ = MI_NOOP;  in emit_clear()
934 		*cs++ = XY_COLOR_BLT_CMD | BLT_WRITE_RGBA | (6 - 2);  in emit_clear()
935 		*cs++ = BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | PAGE_SIZE;  in emit_clear()
936 		*cs++ = 0;  in emit_clear()
937 		*cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4;  in emit_clear()
938 		*cs++ = offset;  in emit_clear()
939 		*cs++ = value;  in emit_clear()
942 	intel_ring_advance(rq, cs);  in emit_clear()