16 	u32 *cs, flags = 0;  in gen8_emit_flush_rcs()  local
58 	cs = intel_ring_begin(rq, len);  in gen8_emit_flush_rcs()
59 	if (IS_ERR(cs))  in gen8_emit_flush_rcs()
60 		return PTR_ERR(cs);  in gen8_emit_flush_rcs()
63 		cs = gen8_emit_pipe_control(cs, 0, 0);  in gen8_emit_flush_rcs()
66 		cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_DC_FLUSH_ENABLE,  in gen8_emit_flush_rcs()
69 	cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR);  in gen8_emit_flush_rcs()
72 		cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_CS_STALL, 0);  in gen8_emit_flush_rcs()
74 	intel_ring_advance(rq, cs);  in gen8_emit_flush_rcs()
81 	u32 cmd, *cs;  in gen8_emit_flush_xcs()  local
83 	cs = intel_ring_begin(rq, 4);  in gen8_emit_flush_xcs()
84 	if (IS_ERR(cs))  in gen8_emit_flush_xcs()
85 		return PTR_ERR(cs);  in gen8_emit_flush_xcs()
103 	*cs++ = cmd;  in gen8_emit_flush_xcs()
104 	*cs++ = LRC_PPHWSP_SCRATCH_ADDR;  in gen8_emit_flush_xcs()
105 	*cs++ = 0; /* upper addr */  in gen8_emit_flush_xcs()
106 	*cs++ = 0; /* value */  in gen8_emit_flush_xcs()
107 	intel_ring_advance(rq, cs);  in gen8_emit_flush_xcs()
115 		u32 *cs;  in gen11_emit_flush_rcs()  local
128 		cs = intel_ring_begin(rq, 6);  in gen11_emit_flush_rcs()
129 		if (IS_ERR(cs))  in gen11_emit_flush_rcs()
130 			return PTR_ERR(cs);  in gen11_emit_flush_rcs()
132 		cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR);  in gen11_emit_flush_rcs()
133 		intel_ring_advance(rq, cs);  in gen11_emit_flush_rcs()
137 		u32 *cs;  in gen11_emit_flush_rcs()  local
152 		cs = intel_ring_begin(rq, 6);  in gen11_emit_flush_rcs()
153 		if (IS_ERR(cs))  in gen11_emit_flush_rcs()
154 			return PTR_ERR(cs);  in gen11_emit_flush_rcs()
156 		cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR);  in gen11_emit_flush_rcs()
157 		intel_ring_advance(rq, cs);  in gen11_emit_flush_rcs()
202 u32 *gen12_emit_aux_table_inv(struct intel_engine_cs *engine, u32 *cs)  in gen12_emit_aux_table_inv()  argument
208 		return cs;  in gen12_emit_aux_table_inv()
210 	*cs++ = MI_LOAD_REGISTER_IMM(1) | MI_LRI_MMIO_REMAP_EN;  in gen12_emit_aux_table_inv()
211 	*cs++ = i915_mmio_reg_offset(inv_reg) + gsi_offset;  in gen12_emit_aux_table_inv()
212 	*cs++ = AUX_INV;  in gen12_emit_aux_table_inv()
214 	*cs++ = MI_SEMAPHORE_WAIT_TOKEN |  in gen12_emit_aux_table_inv()
218 	*cs++ = 0;  in gen12_emit_aux_table_inv()
219 	*cs++ = i915_mmio_reg_offset(inv_reg) + gsi_offset;  in gen12_emit_aux_table_inv()
220 	*cs++ = 0;  in gen12_emit_aux_table_inv()
221 	*cs++ = 0;  in gen12_emit_aux_table_inv()
223 	return cs;  in gen12_emit_aux_table_inv()
231 		u32 *cs;  in mtl_dummy_pipe_control()  local
234 		cs = intel_ring_begin(rq, 6);  in mtl_dummy_pipe_control()
235 		if (IS_ERR(cs))  in mtl_dummy_pipe_control()
236 			return PTR_ERR(cs);  in mtl_dummy_pipe_control()
237 		cs = gen12_emit_pipe_control(cs,  in mtl_dummy_pipe_control()
241 		intel_ring_advance(rq, cs);  in mtl_dummy_pipe_control()
259 		u32 *cs;  in gen12_emit_flush_rcs()  local
302 		cs = intel_ring_begin(rq, 6);  in gen12_emit_flush_rcs()
303 		if (IS_ERR(cs))  in gen12_emit_flush_rcs()
304 			return PTR_ERR(cs);  in gen12_emit_flush_rcs()
306 		cs = gen12_emit_pipe_control(cs, bit_group_0, bit_group_1,  in gen12_emit_flush_rcs()
308 		intel_ring_advance(rq, cs);  in gen12_emit_flush_rcs()
313 		u32 *cs, count;  in gen12_emit_flush_rcs()  local
342 		cs = intel_ring_begin(rq, count);  in gen12_emit_flush_rcs()
343 		if (IS_ERR(cs))  in gen12_emit_flush_rcs()
344 			return PTR_ERR(cs);  in gen12_emit_flush_rcs()
351 		*cs++ = preparser_disable(true);  in gen12_emit_flush_rcs()
353 		cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR);  in gen12_emit_flush_rcs()
355 		cs = gen12_emit_aux_table_inv(engine, cs);  in gen12_emit_flush_rcs()
357 		*cs++ = preparser_disable(false);  in gen12_emit_flush_rcs()
358 		intel_ring_advance(rq, cs);  in gen12_emit_flush_rcs()
367 	u32 *cs;  in gen12_emit_flush_xcs()  local
376 	cs = intel_ring_begin(rq, cmd);  in gen12_emit_flush_xcs()
377 	if (IS_ERR(cs))  in gen12_emit_flush_xcs()
378 		return PTR_ERR(cs);  in gen12_emit_flush_xcs()
381 		*cs++ = preparser_disable(true);  in gen12_emit_flush_xcs()
403 	*cs++ = cmd;  in gen12_emit_flush_xcs()
404 	*cs++ = LRC_PPHWSP_SCRATCH_ADDR;  in gen12_emit_flush_xcs()
405 	*cs++ = 0; /* upper addr */  in gen12_emit_flush_xcs()
406 	*cs++ = 0; /* value */  in gen12_emit_flush_xcs()
408 	cs = gen12_emit_aux_table_inv(rq->engine, cs);  in gen12_emit_flush_xcs()
411 		*cs++ = preparser_disable(false);  in gen12_emit_flush_xcs()
413 	intel_ring_advance(rq, cs);  in gen12_emit_flush_xcs()
438 	u32 *cs;  in gen8_emit_init_breadcrumb()  local
444 	cs = intel_ring_begin(rq, 6);  in gen8_emit_init_breadcrumb()
445 	if (IS_ERR(cs))  in gen8_emit_init_breadcrumb()
446 		return PTR_ERR(cs);  in gen8_emit_init_breadcrumb()
448 	*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;  in gen8_emit_init_breadcrumb()
449 	*cs++ = hwsp_offset(rq);  in gen8_emit_init_breadcrumb()
450 	*cs++ = 0;  in gen8_emit_init_breadcrumb()
451 	*cs++ = rq->fence.seqno - 1;  in gen8_emit_init_breadcrumb()
470 	*cs++ = MI_NOOP;  in gen8_emit_init_breadcrumb()
471 	*cs++ = MI_ARB_CHECK;  in gen8_emit_init_breadcrumb()
473 	intel_ring_advance(rq, cs);  in gen8_emit_init_breadcrumb()
476 	rq->infix = intel_ring_offset(rq, cs);  in gen8_emit_init_breadcrumb()
490 	u32 *cs;  in __xehp_emit_bb_start()  local
494 	cs = intel_ring_begin(rq, 12);  in __xehp_emit_bb_start()
495 	if (IS_ERR(cs))  in __xehp_emit_bb_start()
496 		return PTR_ERR(cs);  in __xehp_emit_bb_start()
498 	*cs++ = MI_ARB_ON_OFF | arb;  in __xehp_emit_bb_start()
500 	*cs++ = MI_LOAD_REGISTER_MEM_GEN8 |  in __xehp_emit_bb_start()
503 	*cs++ = i915_mmio_reg_offset(RING_PREDICATE_RESULT(0));  in __xehp_emit_bb_start()
504 	*cs++ = wa_offset + DG2_PREDICATE_RESULT_WA;  in __xehp_emit_bb_start()
505 	*cs++ = 0;  in __xehp_emit_bb_start()
507 	*cs++ = MI_BATCH_BUFFER_START_GEN8 |  in __xehp_emit_bb_start()
509 	*cs++ = lower_32_bits(offset);  in __xehp_emit_bb_start()
510 	*cs++ = upper_32_bits(offset);  in __xehp_emit_bb_start()
513 	*cs++ = MI_BATCH_BUFFER_START_GEN8;  in __xehp_emit_bb_start()
514 	*cs++ = wa_offset + DG2_PREDICATE_RESULT_BB;  in __xehp_emit_bb_start()
515 	*cs++ = 0;  in __xehp_emit_bb_start()
517 	*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;  in __xehp_emit_bb_start()
519 	intel_ring_advance(rq, cs);  in __xehp_emit_bb_start()
542 	u32 *cs;  in gen8_emit_bb_start_noarb()  local
544 	cs = intel_ring_begin(rq, 4);  in gen8_emit_bb_start_noarb()
545 	if (IS_ERR(cs))  in gen8_emit_bb_start_noarb()
546 		return PTR_ERR(cs);  in gen8_emit_bb_start_noarb()
561 	*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;  in gen8_emit_bb_start_noarb()
564 	*cs++ = MI_BATCH_BUFFER_START_GEN8 |  in gen8_emit_bb_start_noarb()
566 	*cs++ = lower_32_bits(offset);  in gen8_emit_bb_start_noarb()
567 	*cs++ = upper_32_bits(offset);  in gen8_emit_bb_start_noarb()
569 	intel_ring_advance(rq, cs);  in gen8_emit_bb_start_noarb()
578 	u32 *cs;  in gen8_emit_bb_start()  local
583 	cs = intel_ring_begin(rq, 6);  in gen8_emit_bb_start()
584 	if (IS_ERR(cs))  in gen8_emit_bb_start()
585 		return PTR_ERR(cs);  in gen8_emit_bb_start()
587 	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;  in gen8_emit_bb_start()
589 	*cs++ = MI_BATCH_BUFFER_START_GEN8 |  in gen8_emit_bb_start()
591 	*cs++ = lower_32_bits(offset);  in gen8_emit_bb_start()
592 	*cs++ = upper_32_bits(offset);  in gen8_emit_bb_start()
594 	*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;  in gen8_emit_bb_start()
595 	*cs++ = MI_NOOP;  in gen8_emit_bb_start()
597 	intel_ring_advance(rq, cs);  in gen8_emit_bb_start()
615 static u32 *gen8_emit_wa_tail(struct i915_request *rq, u32 *cs)  in gen8_emit_wa_tail()  argument
618 	*cs++ = MI_ARB_CHECK;  in gen8_emit_wa_tail()
619 	*cs++ = MI_NOOP;  in gen8_emit_wa_tail()
620 	rq->wa_tail = intel_ring_offset(rq, cs);  in gen8_emit_wa_tail()
625 	return cs;  in gen8_emit_wa_tail()
628 static u32 *emit_preempt_busywait(struct i915_request *rq, u32 *cs)  in emit_preempt_busywait()  argument
630 	*cs++ = MI_ARB_CHECK; /* trigger IDLE->ACTIVE first */  in emit_preempt_busywait()
631 	*cs++ = MI_SEMAPHORE_WAIT |  in emit_preempt_busywait()
635 	*cs++ = 0;  in emit_preempt_busywait()
636 	*cs++ = preempt_address(rq->engine);  in emit_preempt_busywait()
637 	*cs++ = 0;  in emit_preempt_busywait()
638 	*cs++ = MI_NOOP;  in emit_preempt_busywait()
640 	return cs;  in emit_preempt_busywait()
644 gen8_emit_fini_breadcrumb_tail(struct i915_request *rq, u32 *cs)  in gen8_emit_fini_breadcrumb_tail()  argument
646 	*cs++ = MI_USER_INTERRUPT;  in gen8_emit_fini_breadcrumb_tail()
648 	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;  in gen8_emit_fini_breadcrumb_tail()
651 		cs = emit_preempt_busywait(rq, cs);  in gen8_emit_fini_breadcrumb_tail()
653 	rq->tail = intel_ring_offset(rq, cs);  in gen8_emit_fini_breadcrumb_tail()
656 	return gen8_emit_wa_tail(rq, cs);  in gen8_emit_fini_breadcrumb_tail()
659 static u32 *emit_xcs_breadcrumb(struct i915_request *rq, u32 *cs)  in emit_xcs_breadcrumb()  argument
661 	return gen8_emit_ggtt_write(cs, rq->fence.seqno, hwsp_offset(rq), 0);  in emit_xcs_breadcrumb()
664 u32 *gen8_emit_fini_breadcrumb_xcs(struct i915_request *rq, u32 *cs)  in gen8_emit_fini_breadcrumb_xcs()  argument
666 	return gen8_emit_fini_breadcrumb_tail(rq, emit_xcs_breadcrumb(rq, cs));  in gen8_emit_fini_breadcrumb_xcs()
669 u32 *gen8_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs)  in gen8_emit_fini_breadcrumb_rcs()  argument
671 	cs = gen8_emit_pipe_control(cs,  in gen8_emit_fini_breadcrumb_rcs()
680 	cs = gen8_emit_ggtt_write_rcs(cs,  in gen8_emit_fini_breadcrumb_rcs()
686 	return gen8_emit_fini_breadcrumb_tail(rq, cs);  in gen8_emit_fini_breadcrumb_rcs()
689 u32 *gen11_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs)  in gen11_emit_fini_breadcrumb_rcs()  argument
691 	cs = gen8_emit_pipe_control(cs,  in gen11_emit_fini_breadcrumb_rcs()
701 	cs = gen8_emit_ggtt_write_rcs(cs,  in gen11_emit_fini_breadcrumb_rcs()
707 	return gen8_emit_fini_breadcrumb_tail(rq, cs);  in gen11_emit_fini_breadcrumb_rcs()
729 static u32 *gen12_emit_preempt_busywait(struct i915_request *rq, u32 *cs)  in gen12_emit_preempt_busywait()  argument
731 	*cs++ = MI_ARB_CHECK; /* trigger IDLE->ACTIVE first */  in gen12_emit_preempt_busywait()
732 	*cs++ = MI_SEMAPHORE_WAIT_TOKEN |  in gen12_emit_preempt_busywait()
736 	*cs++ = 0;  in gen12_emit_preempt_busywait()
737 	*cs++ = preempt_address(rq->engine);  in gen12_emit_preempt_busywait()
738 	*cs++ = 0;  in gen12_emit_preempt_busywait()
739 	*cs++ = 0;  in gen12_emit_preempt_busywait()
741 	return cs;  in gen12_emit_preempt_busywait()
753 static u32 *ccs_emit_wa_busywait(struct i915_request *rq, u32 *cs)  in ccs_emit_wa_busywait()  argument
757 	*cs++ = MI_ATOMIC_INLINE | MI_ATOMIC_GLOBAL_GTT | MI_ATOMIC_CS_STALL |  in ccs_emit_wa_busywait()
759 	*cs++ = ccs_semaphore_offset(rq);  in ccs_emit_wa_busywait()
760 	*cs++ = 0;  in ccs_emit_wa_busywait()
761 	*cs++ = 1;  in ccs_emit_wa_busywait()
768 		*cs++ = 0;  in ccs_emit_wa_busywait()
770 	*cs++ = MI_SEMAPHORE_WAIT |  in ccs_emit_wa_busywait()
774 	*cs++ = 0;  in ccs_emit_wa_busywait()
775 	*cs++ = ccs_semaphore_offset(rq);  in ccs_emit_wa_busywait()
776 	*cs++ = 0;  in ccs_emit_wa_busywait()
778 	return cs;  in ccs_emit_wa_busywait()
782 gen12_emit_fini_breadcrumb_tail(struct i915_request *rq, u32 *cs)  in gen12_emit_fini_breadcrumb_tail()  argument
784 	*cs++ = MI_USER_INTERRUPT;  in gen12_emit_fini_breadcrumb_tail()
786 	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;  in gen12_emit_fini_breadcrumb_tail()
789 		cs = gen12_emit_preempt_busywait(rq, cs);  in gen12_emit_fini_breadcrumb_tail()
793 		cs = ccs_emit_wa_busywait(rq, cs);  in gen12_emit_fini_breadcrumb_tail()
795 	rq->tail = intel_ring_offset(rq, cs);  in gen12_emit_fini_breadcrumb_tail()
798 	return gen8_emit_wa_tail(rq, cs);  in gen12_emit_fini_breadcrumb_tail()
801 u32 *gen12_emit_fini_breadcrumb_xcs(struct i915_request *rq, u32 *cs)  in gen12_emit_fini_breadcrumb_xcs()  argument
804 	cs = emit_xcs_breadcrumb(rq, __gen8_emit_flush_dw(cs, 0, 0, 0));  in gen12_emit_fini_breadcrumb_xcs()
805 	return gen12_emit_fini_breadcrumb_tail(rq, cs);  in gen12_emit_fini_breadcrumb_xcs()
808 u32 *gen12_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs)  in gen12_emit_fini_breadcrumb_rcs()  argument
824 		cs = gen12_emit_pipe_control(cs, 0,  in gen12_emit_fini_breadcrumb_rcs()
836 	cs = gen12_emit_pipe_control(cs, PIPE_CONTROL0_HDC_PIPELINE_FLUSH, flags, 0);  in gen12_emit_fini_breadcrumb_rcs()
839 	cs = gen12_emit_ggtt_write_rcs(cs,  in gen12_emit_fini_breadcrumb_rcs()
846 	return gen12_emit_fini_breadcrumb_tail(rq, cs);  in gen12_emit_fini_breadcrumb_rcs()