60 	u32 *cs;  in gen6_emit_post_sync_nonzero_flush()  local
62 	cs = intel_ring_begin(rq, 6);  in gen6_emit_post_sync_nonzero_flush()
63 	if (IS_ERR(cs))  in gen6_emit_post_sync_nonzero_flush()
64 		return PTR_ERR(cs);  in gen6_emit_post_sync_nonzero_flush()
66 	*cs++ = GFX_OP_PIPE_CONTROL(5);  in gen6_emit_post_sync_nonzero_flush()
67 	*cs++ = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD;  in gen6_emit_post_sync_nonzero_flush()
68 	*cs++ = scratch_addr | PIPE_CONTROL_GLOBAL_GTT;  in gen6_emit_post_sync_nonzero_flush()
69 	*cs++ = 0; /* low dword */  in gen6_emit_post_sync_nonzero_flush()
70 	*cs++ = 0; /* high dword */  in gen6_emit_post_sync_nonzero_flush()
71 	*cs++ = MI_NOOP;  in gen6_emit_post_sync_nonzero_flush()
72 	intel_ring_advance(rq, cs);  in gen6_emit_post_sync_nonzero_flush()
74 	cs = intel_ring_begin(rq, 6);  in gen6_emit_post_sync_nonzero_flush()
75 	if (IS_ERR(cs))  in gen6_emit_post_sync_nonzero_flush()
76 		return PTR_ERR(cs);  in gen6_emit_post_sync_nonzero_flush()
78 	*cs++ = GFX_OP_PIPE_CONTROL(5);  in gen6_emit_post_sync_nonzero_flush()
79 	*cs++ = PIPE_CONTROL_QW_WRITE;  in gen6_emit_post_sync_nonzero_flush()
80 	*cs++ = scratch_addr | PIPE_CONTROL_GLOBAL_GTT;  in gen6_emit_post_sync_nonzero_flush()
81 	*cs++ = 0;  in gen6_emit_post_sync_nonzero_flush()
82 	*cs++ = 0;  in gen6_emit_post_sync_nonzero_flush()
83 	*cs++ = MI_NOOP;  in gen6_emit_post_sync_nonzero_flush()
84 	intel_ring_advance(rq, cs);  in gen6_emit_post_sync_nonzero_flush()
94 	u32 *cs, flags = 0;  in gen6_emit_flush_rcs()  local
130 	cs = intel_ring_begin(rq, 4);  in gen6_emit_flush_rcs()
131 	if (IS_ERR(cs))  in gen6_emit_flush_rcs()
132 		return PTR_ERR(cs);  in gen6_emit_flush_rcs()
134 	*cs++ = GFX_OP_PIPE_CONTROL(4);  in gen6_emit_flush_rcs()
135 	*cs++ = flags;  in gen6_emit_flush_rcs()
136 	*cs++ = scratch_addr | PIPE_CONTROL_GLOBAL_GTT;  in gen6_emit_flush_rcs()
137 	*cs++ = 0;  in gen6_emit_flush_rcs()
138 	intel_ring_advance(rq, cs);  in gen6_emit_flush_rcs()
143 u32 *gen6_emit_breadcrumb_rcs(struct i915_request *rq, u32 *cs)  in gen6_emit_breadcrumb_rcs()  argument
146 	*cs++ = GFX_OP_PIPE_CONTROL(4);  in gen6_emit_breadcrumb_rcs()
147 	*cs++ = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD;  in gen6_emit_breadcrumb_rcs()
148 	*cs++ = 0;  in gen6_emit_breadcrumb_rcs()
149 	*cs++ = 0;  in gen6_emit_breadcrumb_rcs()
151 	*cs++ = GFX_OP_PIPE_CONTROL(4);  in gen6_emit_breadcrumb_rcs()
152 	*cs++ = PIPE_CONTROL_QW_WRITE;  in gen6_emit_breadcrumb_rcs()
153 	*cs++ = intel_gt_scratch_offset(rq->engine->gt,  in gen6_emit_breadcrumb_rcs()
156 	*cs++ = 0;  in gen6_emit_breadcrumb_rcs()
159 	*cs++ = GFX_OP_PIPE_CONTROL(4);  in gen6_emit_breadcrumb_rcs()
160 	*cs++ = (PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH |  in gen6_emit_breadcrumb_rcs()
165 	*cs++ = i915_request_active_seqno(rq) |  in gen6_emit_breadcrumb_rcs()
167 	*cs++ = rq->fence.seqno;  in gen6_emit_breadcrumb_rcs()
169 	*cs++ = MI_USER_INTERRUPT;  in gen6_emit_breadcrumb_rcs()
170 	*cs++ = MI_NOOP;  in gen6_emit_breadcrumb_rcs()
172 	rq->tail = intel_ring_offset(rq, cs);  in gen6_emit_breadcrumb_rcs()
175 	return cs;  in gen6_emit_breadcrumb_rcs()
180 	u32 cmd, *cs;  in mi_flush_dw()  local
182 	cs = intel_ring_begin(rq, 4);  in mi_flush_dw()
183 	if (IS_ERR(cs))  in mi_flush_dw()
184 		return PTR_ERR(cs);  in mi_flush_dw()
204 	*cs++ = cmd;  in mi_flush_dw()
205 	*cs++ = HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT;  in mi_flush_dw()
206 	*cs++ = 0;  in mi_flush_dw()
207 	*cs++ = MI_NOOP;  in mi_flush_dw()
209 	intel_ring_advance(rq, cs);  in mi_flush_dw()
234 	u32 *cs;  in gen6_emit_bb_start()  local
240 	cs = intel_ring_begin(rq, 2);  in gen6_emit_bb_start()
241 	if (IS_ERR(cs))  in gen6_emit_bb_start()
242 		return PTR_ERR(cs);  in gen6_emit_bb_start()
244 	cs = __gen6_emit_bb_start(cs, offset, security);  in gen6_emit_bb_start()
245 	intel_ring_advance(rq, cs);  in gen6_emit_bb_start()
256 	u32 *cs;  in hsw_emit_bb_start()  local
262 	cs = intel_ring_begin(rq, 2);  in hsw_emit_bb_start()
263 	if (IS_ERR(cs))  in hsw_emit_bb_start()
264 		return PTR_ERR(cs);  in hsw_emit_bb_start()
266 	cs = __gen6_emit_bb_start(cs, offset, security);  in hsw_emit_bb_start()
267 	intel_ring_advance(rq, cs);  in hsw_emit_bb_start()
274 	u32 *cs;  in gen7_stall_cs()  local
276 	cs = intel_ring_begin(rq, 4);  in gen7_stall_cs()
277 	if (IS_ERR(cs))  in gen7_stall_cs()
278 		return PTR_ERR(cs);  in gen7_stall_cs()
280 	*cs++ = GFX_OP_PIPE_CONTROL(4);  in gen7_stall_cs()
281 	*cs++ = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD;  in gen7_stall_cs()
282 	*cs++ = 0;  in gen7_stall_cs()
283 	*cs++ = 0;  in gen7_stall_cs()
284 	intel_ring_advance(rq, cs);  in gen7_stall_cs()
294 	u32 *cs, flags = 0;  in gen7_emit_flush_rcs()  local
340 	cs = intel_ring_begin(rq, 4);  in gen7_emit_flush_rcs()
341 	if (IS_ERR(cs))  in gen7_emit_flush_rcs()
342 		return PTR_ERR(cs);  in gen7_emit_flush_rcs()
344 	*cs++ = GFX_OP_PIPE_CONTROL(4);  in gen7_emit_flush_rcs()
345 	*cs++ = flags;  in gen7_emit_flush_rcs()
346 	*cs++ = scratch_addr;  in gen7_emit_flush_rcs()
347 	*cs++ = 0;  in gen7_emit_flush_rcs()
348 	intel_ring_advance(rq, cs);  in gen7_emit_flush_rcs()
353 u32 *gen7_emit_breadcrumb_rcs(struct i915_request *rq, u32 *cs)  in gen7_emit_breadcrumb_rcs()  argument
355 	*cs++ = GFX_OP_PIPE_CONTROL(4);  in gen7_emit_breadcrumb_rcs()
356 	*cs++ = (PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH |  in gen7_emit_breadcrumb_rcs()
363 	*cs++ = i915_request_active_seqno(rq);  in gen7_emit_breadcrumb_rcs()
364 	*cs++ = rq->fence.seqno;  in gen7_emit_breadcrumb_rcs()
366 	*cs++ = MI_USER_INTERRUPT;  in gen7_emit_breadcrumb_rcs()
367 	*cs++ = MI_NOOP;  in gen7_emit_breadcrumb_rcs()
369 	rq->tail = intel_ring_offset(rq, cs);  in gen7_emit_breadcrumb_rcs()
372 	return cs;  in gen7_emit_breadcrumb_rcs()
375 u32 *gen6_emit_breadcrumb_xcs(struct i915_request *rq, u32 *cs)  in gen6_emit_breadcrumb_xcs()  argument
380 	*cs++ = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_DW_STORE_INDEX;  in gen6_emit_breadcrumb_xcs()
381 	*cs++ = I915_GEM_HWS_SEQNO_ADDR | MI_FLUSH_DW_USE_GTT;  in gen6_emit_breadcrumb_xcs()
382 	*cs++ = rq->fence.seqno;  in gen6_emit_breadcrumb_xcs()
384 	*cs++ = MI_USER_INTERRUPT;  in gen6_emit_breadcrumb_xcs()
386 	rq->tail = intel_ring_offset(rq, cs);  in gen6_emit_breadcrumb_xcs()
389 	return cs;  in gen6_emit_breadcrumb_xcs()
393 u32 *gen7_emit_breadcrumb_xcs(struct i915_request *rq, u32 *cs)  in gen7_emit_breadcrumb_xcs()  argument
400 	*cs++ = MI_FLUSH_DW | MI_INVALIDATE_TLB |  in gen7_emit_breadcrumb_xcs()
402 	*cs++ = I915_GEM_HWS_SEQNO_ADDR | MI_FLUSH_DW_USE_GTT;  in gen7_emit_breadcrumb_xcs()
403 	*cs++ = rq->fence.seqno;  in gen7_emit_breadcrumb_xcs()
406 		*cs++ = MI_STORE_DWORD_INDEX;  in gen7_emit_breadcrumb_xcs()
407 		*cs++ = I915_GEM_HWS_SEQNO_ADDR;  in gen7_emit_breadcrumb_xcs()
408 		*cs++ = rq->fence.seqno;  in gen7_emit_breadcrumb_xcs()
411 	*cs++ = MI_FLUSH_DW;  in gen7_emit_breadcrumb_xcs()
412 	*cs++ = 0;  in gen7_emit_breadcrumb_xcs()
413 	*cs++ = 0;  in gen7_emit_breadcrumb_xcs()
415 	*cs++ = MI_USER_INTERRUPT;  in gen7_emit_breadcrumb_xcs()
416 	*cs++ = MI_NOOP;  in gen7_emit_breadcrumb_xcs()
418 	rq->tail = intel_ring_offset(rq, cs);  in gen7_emit_breadcrumb_xcs()
421 	return cs;  in gen7_emit_breadcrumb_xcs()