84 	u32 *cs;  in emit_semaphore_signal()  local
90 	cs = intel_ring_begin(rq, 4);  in emit_semaphore_signal()
91 	if (IS_ERR(cs)) {  in emit_semaphore_signal()
93 		return PTR_ERR(cs);  in emit_semaphore_signal()
96 	*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;  in emit_semaphore_signal()
97 	*cs++ = offset;  in emit_semaphore_signal()
98 	*cs++ = 0;  in emit_semaphore_signal()
99 	*cs++ = 1;  in emit_semaphore_signal()
101 	intel_ring_advance(rq, cs);  in emit_semaphore_signal()
414 	u32 *cs;  in __live_lrc_state()  local
436 	cs = intel_ring_begin(rq, 4 * MAX_IDX);  in __live_lrc_state()
437 	if (IS_ERR(cs)) {  in __live_lrc_state()
438 		err = PTR_ERR(cs);  in __live_lrc_state()
443 	*cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;  in __live_lrc_state()
444 	*cs++ = i915_mmio_reg_offset(RING_START(engine->mmio_base));  in __live_lrc_state()
445 	*cs++ = i915_ggtt_offset(scratch) + RING_START_IDX * sizeof(u32);  in __live_lrc_state()
446 	*cs++ = 0;  in __live_lrc_state()
450 	*cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;  in __live_lrc_state()
451 	*cs++ = i915_mmio_reg_offset(RING_TAIL(engine->mmio_base));  in __live_lrc_state()
452 	*cs++ = i915_ggtt_offset(scratch) + RING_TAIL_IDX * sizeof(u32);  in __live_lrc_state()
453 	*cs++ = 0;  in __live_lrc_state()
472 	cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);  in __live_lrc_state()
473 	if (IS_ERR(cs)) {  in __live_lrc_state()
474 		err = PTR_ERR(cs);  in __live_lrc_state()
479 		if (cs[n] != expected[n]) {  in __live_lrc_state()
481 			       engine->name, n, cs[n], expected[n]);  in __live_lrc_state()
537 	u32 *cs;  in gpr_make_dirty()  local
544 	cs = intel_ring_begin(rq, 2 * NUM_GPR_DW + 2);  in gpr_make_dirty()
545 	if (IS_ERR(cs)) {  in gpr_make_dirty()
547 		return PTR_ERR(cs);  in gpr_make_dirty()
550 	*cs++ = MI_LOAD_REGISTER_IMM(NUM_GPR_DW);  in gpr_make_dirty()
552 		*cs++ = CS_GPR(ce->engine, n);  in gpr_make_dirty()
553 		*cs++ = STACK_MAGIC;  in gpr_make_dirty()
555 	*cs++ = MI_NOOP;  in gpr_make_dirty()
557 	intel_ring_advance(rq, cs);  in gpr_make_dirty()
572 	u32 *cs;  in __gpr_read()  local
580 	cs = intel_ring_begin(rq, 6 + 4 * NUM_GPR_DW);  in __gpr_read()
581 	if (IS_ERR(cs)) {  in __gpr_read()
583 		return ERR_CAST(cs);  in __gpr_read()
586 	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;  in __gpr_read()
587 	*cs++ = MI_NOOP;  in __gpr_read()
589 	*cs++ = MI_SEMAPHORE_WAIT |  in __gpr_read()
593 	*cs++ = 0;  in __gpr_read()
594 	*cs++ = offset;  in __gpr_read()
595 	*cs++ = 0;  in __gpr_read()
598 		*cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;  in __gpr_read()
599 		*cs++ = CS_GPR(ce->engine, n);  in __gpr_read()
600 		*cs++ = i915_ggtt_offset(scratch) + n * sizeof(u32);  in __gpr_read()
601 		*cs++ = 0;  in __gpr_read()
627 	u32 *cs;  in __live_lrc_gpr()  local
674 	cs = i915_gem_object_pin_map_unlocked(scratch->obj, I915_MAP_WB);  in __live_lrc_gpr()
675 	if (IS_ERR(cs)) {  in __live_lrc_gpr()
676 		err = PTR_ERR(cs);  in __live_lrc_gpr()
681 		if (cs[n]) {  in __live_lrc_gpr()
685 			       cs[n]);  in __live_lrc_gpr()
749 	u32 *cs;  in create_timestamp()  local
756 	cs = intel_ring_begin(rq, 10);  in create_timestamp()
757 	if (IS_ERR(cs)) {  in create_timestamp()
758 		err = PTR_ERR(cs);  in create_timestamp()
762 	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;  in create_timestamp()
763 	*cs++ = MI_NOOP;  in create_timestamp()
765 	*cs++ = MI_SEMAPHORE_WAIT |  in create_timestamp()
769 	*cs++ = 0;  in create_timestamp()
770 	*cs++ = offset;  in create_timestamp()
771 	*cs++ = 0;  in create_timestamp()
773 	*cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;  in create_timestamp()
774 	*cs++ = i915_mmio_reg_offset(RING_CTX_TIMESTAMP(rq->engine->mmio_base));  in create_timestamp()
775 	*cs++ = offset + idx * sizeof(u32);  in create_timestamp()
776 	*cs++ = 0;  in create_timestamp()
778 	intel_ring_advance(rq, cs);  in create_timestamp()
973 	u32 dw, x, *cs, *hw;  in store_context()  local
980 	cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);  in store_context()
981 	if (IS_ERR(cs)) {  in store_context()
983 		return ERR_CAST(cs);  in store_context()
1035 			*cs++ = MI_STORE_REGISTER_MEM_GEN8;  in store_context()
1036 			*cs++ = hw[dw];  in store_context()
1037 			*cs++ = lower_32_bits(scratch->node.start + x);  in store_context()
1038 			*cs++ = upper_32_bits(scratch->node.start + x);  in store_context()
1046 	*cs++ = MI_BATCH_BUFFER_END;  in store_context()
1079 	u32 *cs;  in record_registers()  local
1112 	cs = intel_ring_begin(rq, 14);  in record_registers()
1113 	if (IS_ERR(cs)) {  in record_registers()
1114 		err = PTR_ERR(cs);  in record_registers()
1118 	*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;  in record_registers()
1119 	*cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);  in record_registers()
1120 	*cs++ = lower_32_bits(b_before->node.start);  in record_registers()
1121 	*cs++ = upper_32_bits(b_before->node.start);  in record_registers()
1123 	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;  in record_registers()
1124 	*cs++ = MI_SEMAPHORE_WAIT |  in record_registers()
1128 	*cs++ = 0;  in record_registers()
1129 	*cs++ = i915_ggtt_offset(ce->engine->status_page.vma) +  in record_registers()
1131 	*cs++ = 0;  in record_registers()
1132 	*cs++ = MI_NOOP;  in record_registers()
1134 	*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;  in record_registers()
1135 	*cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);  in record_registers()
1136 	*cs++ = lower_32_bits(b_after->node.start);  in record_registers()
1137 	*cs++ = upper_32_bits(b_after->node.start);  in record_registers()
1139 	intel_ring_advance(rq, cs);  in record_registers()
1159 	u32 dw, *cs, *hw;  in load_context()  local
1166 	cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);  in load_context()
1167 	if (IS_ERR(cs)) {  in load_context()
1169 		return ERR_CAST(cs);  in load_context()
1208 		*cs++ = MI_LOAD_REGISTER_IMM(len);  in load_context()
1210 			*cs++ = hw[dw];  in load_context()
1211 			*cs++ = safe_poison(hw[dw] & get_lri_mask(ce->engine,  in load_context()
1219 	*cs++ = MI_BATCH_BUFFER_END;  in load_context()
1233 	u32 *cs;  in poison_registers()  local
1250 	cs = intel_ring_begin(rq, 8);  in poison_registers()
1251 	if (IS_ERR(cs)) {  in poison_registers()
1252 		err = PTR_ERR(cs);  in poison_registers()
1256 	*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;  in poison_registers()
1257 	*cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);  in poison_registers()
1258 	*cs++ = lower_32_bits(batch->node.start);  in poison_registers()
1259 	*cs++ = upper_32_bits(batch->node.start);  in poison_registers()
1261 	*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;  in poison_registers()
1262 	*cs++ = i915_ggtt_offset(ce->engine->status_page.vma) +  in poison_registers()
1264 	*cs++ = 0;  in poison_registers()
1265 	*cs++ = 1;  in poison_registers()
1267 	intel_ring_advance(rq, cs);  in poison_registers()
1603 emit_indirect_ctx_bb_canary(const struct intel_context *ce, u32 *cs)  in emit_indirect_ctx_bb_canary()  argument
1605 	*cs++ = MI_STORE_REGISTER_MEM_GEN8 |  in emit_indirect_ctx_bb_canary()
1608 	*cs++ = i915_mmio_reg_offset(RING_START(0));  in emit_indirect_ctx_bb_canary()
1609 	*cs++ = i915_ggtt_offset(ce->state) +  in emit_indirect_ctx_bb_canary()
1612 	*cs++ = 0;  in emit_indirect_ctx_bb_canary()
1614 	return cs;  in emit_indirect_ctx_bb_canary()
1620 	u32 *cs = context_indirect_bb(ce);  in indirect_ctx_bb_setup()  local
1622 	cs[CTX_BB_CANARY_INDEX] = 0xdeadf00d;  in indirect_ctx_bb_setup()