1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2018 Intel Corporation
4 */
5
6 #include <linux/prime_numbers.h>
7
8 #include "gem/i915_gem_internal.h"
9
10 #include "i915_selftest.h"
11 #include "intel_engine_heartbeat.h"
12 #include "intel_engine_pm.h"
13 #include "intel_reset.h"
14 #include "intel_ring.h"
15 #include "selftest_engine_heartbeat.h"
16 #include "selftests/i915_random.h"
17 #include "selftests/igt_flush_test.h"
18 #include "selftests/igt_live_test.h"
19 #include "selftests/igt_spinner.h"
20 #include "selftests/lib_sw_fence.h"
21 #include "shmem_utils.h"
22
23 #include "gem/selftests/igt_gem_utils.h"
24 #include "gem/selftests/mock_context.h"
25
26 #define CS_GPR(engine, n) ((engine)->mmio_base + 0x600 + (n) * 4)
27 #define NUM_GPR 16
28 #define NUM_GPR_DW (NUM_GPR * 2) /* each GPR is 2 dwords */
29
create_scratch(struct intel_gt * gt)30 static struct i915_vma *create_scratch(struct intel_gt *gt)
31 {
32 return __vm_create_scratch_for_read_pinned(>->ggtt->vm, PAGE_SIZE);
33 }
34
is_active(struct i915_request * rq)35 static bool is_active(struct i915_request *rq)
36 {
37 if (i915_request_is_active(rq))
38 return true;
39
40 if (i915_request_on_hold(rq))
41 return true;
42
43 if (i915_request_has_initial_breadcrumb(rq) && i915_request_started(rq))
44 return true;
45
46 return false;
47 }
48
wait_for_submit(struct intel_engine_cs * engine,struct i915_request * rq,unsigned long timeout)49 static int wait_for_submit(struct intel_engine_cs *engine,
50 struct i915_request *rq,
51 unsigned long timeout)
52 {
53 /* Ignore our own attempts to suppress excess tasklets */
54 tasklet_hi_schedule(&engine->sched_engine->tasklet);
55
56 timeout += jiffies;
57 do {
58 bool done = time_after(jiffies, timeout);
59
60 if (i915_request_completed(rq)) /* that was quick! */
61 return 0;
62
63 /* Wait until the HW has acknowleged the submission (or err) */
64 intel_engine_flush_submission(engine);
65 if (!READ_ONCE(engine->execlists.pending[0]) && is_active(rq))
66 return 0;
67
68 if (done)
69 return -ETIME;
70
71 cond_resched();
72 } while (1);
73 }
74
emit_semaphore_signal(struct intel_context * ce,void * slot)75 static int emit_semaphore_signal(struct intel_context *ce, void *slot)
76 {
77 const u32 offset =
78 i915_ggtt_offset(ce->engine->status_page.vma) +
79 offset_in_page(slot);
80 struct i915_request *rq;
81 u32 *cs;
82
83 rq = intel_context_create_request(ce);
84 if (IS_ERR(rq))
85 return PTR_ERR(rq);
86
87 cs = intel_ring_begin(rq, 4);
88 if (IS_ERR(cs)) {
89 i915_request_add(rq);
90 return PTR_ERR(cs);
91 }
92
93 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
94 *cs++ = offset;
95 *cs++ = 0;
96 *cs++ = 1;
97
98 intel_ring_advance(rq, cs);
99
100 rq->sched.attr.priority = I915_PRIORITY_BARRIER;
101 i915_request_add(rq);
102 return 0;
103 }
104
context_flush(struct intel_context * ce,long timeout)105 static int context_flush(struct intel_context *ce, long timeout)
106 {
107 struct i915_request *rq;
108 struct dma_fence *fence;
109 int err = 0;
110
111 rq = intel_engine_create_kernel_request(ce->engine);
112 if (IS_ERR(rq))
113 return PTR_ERR(rq);
114
115 fence = i915_active_fence_get(&ce->timeline->last_request);
116 if (fence) {
117 i915_request_await_dma_fence(rq, fence);
118 dma_fence_put(fence);
119 }
120
121 rq = i915_request_get(rq);
122 i915_request_add(rq);
123 if (i915_request_wait(rq, 0, timeout) < 0)
124 err = -ETIME;
125 i915_request_put(rq);
126
127 rmb(); /* We know the request is written, make sure all state is too! */
128 return err;
129 }
130
get_lri_mask(struct intel_engine_cs * engine,u32 lri)131 static int get_lri_mask(struct intel_engine_cs *engine, u32 lri)
132 {
133 if ((lri & MI_LRI_LRM_CS_MMIO) == 0)
134 return ~0u;
135
136 if (GRAPHICS_VER(engine->i915) < 12)
137 return 0xfff;
138
139 switch (engine->class) {
140 default:
141 case RENDER_CLASS:
142 case COMPUTE_CLASS:
143 return 0x07ff;
144 case COPY_ENGINE_CLASS:
145 return 0x0fff;
146 case VIDEO_DECODE_CLASS:
147 case VIDEO_ENHANCEMENT_CLASS:
148 return 0x3fff;
149 }
150 }
151
live_lrc_layout(void * arg)152 static int live_lrc_layout(void *arg)
153 {
154 struct intel_gt *gt = arg;
155 struct intel_engine_cs *engine;
156 enum intel_engine_id id;
157 u32 *lrc;
158 int err;
159
160 /*
161 * Check the registers offsets we use to create the initial reg state
162 * match the layout saved by HW.
163 */
164
165 lrc = (u32 *)__get_free_page(GFP_KERNEL); /* requires page alignment */
166 if (!lrc)
167 return -ENOMEM;
168 GEM_BUG_ON(offset_in_page(lrc));
169
170 err = 0;
171 for_each_engine(engine, gt, id) {
172 u32 *hw;
173 int dw;
174
175 if (!engine->default_state)
176 continue;
177
178 hw = shmem_pin_map(engine->default_state);
179 if (!hw) {
180 err = -ENOMEM;
181 break;
182 }
183 hw += LRC_STATE_OFFSET / sizeof(*hw);
184
185 __lrc_init_regs(memset(lrc, POISON_INUSE, PAGE_SIZE),
186 engine->kernel_context, engine, true);
187
188 dw = 0;
189 do {
190 u32 lri = READ_ONCE(hw[dw]);
191 u32 lri_mask;
192
193 if (lri == 0) {
194 dw++;
195 continue;
196 }
197
198 if (lrc[dw] == 0) {
199 pr_debug("%s: skipped instruction %x at dword %d\n",
200 engine->name, lri, dw);
201 dw++;
202 continue;
203 }
204
205 if ((lri & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
206 pr_err("%s: Expected LRI command at dword %d, found %08x\n",
207 engine->name, dw, lri);
208 err = -EINVAL;
209 break;
210 }
211
212 if (lrc[dw] != lri) {
213 pr_err("%s: LRI command mismatch at dword %d, expected %08x found %08x\n",
214 engine->name, dw, lri, lrc[dw]);
215 err = -EINVAL;
216 break;
217 }
218
219 /*
220 * When bit 19 of MI_LOAD_REGISTER_IMM instruction
221 * opcode is set on Gen12+ devices, HW does not
222 * care about certain register address offsets, and
223 * instead check the following for valid address
224 * ranges on specific engines:
225 * RCS && CCS: BITS(0 - 10)
226 * BCS: BITS(0 - 11)
227 * VECS && VCS: BITS(0 - 13)
228 */
229 lri_mask = get_lri_mask(engine, lri);
230
231 lri &= 0x7f;
232 lri++;
233 dw++;
234
235 while (lri) {
236 u32 offset = READ_ONCE(hw[dw]);
237
238 if ((offset ^ lrc[dw]) & lri_mask) {
239 pr_err("%s: Different registers found at dword %d, expected %x, found %x\n",
240 engine->name, dw, offset, lrc[dw]);
241 err = -EINVAL;
242 break;
243 }
244
245 /*
246 * Skip over the actual register value as we
247 * expect that to differ.
248 */
249 dw += 2;
250 lri -= 2;
251 }
252 } while (!err && (lrc[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
253
254 if (err) {
255 pr_info("%s: HW register image:\n", engine->name);
256 igt_hexdump(hw, PAGE_SIZE);
257
258 pr_info("%s: SW register image:\n", engine->name);
259 igt_hexdump(lrc, PAGE_SIZE);
260 }
261
262 shmem_unpin_map(engine->default_state, hw);
263 if (err)
264 break;
265 }
266
267 free_page((unsigned long)lrc);
268 return err;
269 }
270
find_offset(const u32 * lri,u32 offset)271 static int find_offset(const u32 *lri, u32 offset)
272 {
273 int i;
274
275 for (i = 0; i < PAGE_SIZE / sizeof(u32); i++)
276 if (lri[i] == offset)
277 return i;
278
279 return -1;
280 }
281
live_lrc_fixed(void * arg)282 static int live_lrc_fixed(void *arg)
283 {
284 struct intel_gt *gt = arg;
285 struct intel_engine_cs *engine;
286 enum intel_engine_id id;
287 int err = 0;
288
289 /*
290 * Check the assumed register offsets match the actual locations in
291 * the context image.
292 */
293
294 for_each_engine(engine, gt, id) {
295 const struct {
296 u32 reg;
297 u32 offset;
298 const char *name;
299 } tbl[] = {
300 {
301 i915_mmio_reg_offset(RING_START(engine->mmio_base)),
302 CTX_RING_START - 1,
303 "RING_START"
304 },
305 {
306 i915_mmio_reg_offset(RING_CTL(engine->mmio_base)),
307 CTX_RING_CTL - 1,
308 "RING_CTL"
309 },
310 {
311 i915_mmio_reg_offset(RING_HEAD(engine->mmio_base)),
312 CTX_RING_HEAD - 1,
313 "RING_HEAD"
314 },
315 {
316 i915_mmio_reg_offset(RING_TAIL(engine->mmio_base)),
317 CTX_RING_TAIL - 1,
318 "RING_TAIL"
319 },
320 {
321 i915_mmio_reg_offset(RING_MI_MODE(engine->mmio_base)),
322 lrc_ring_mi_mode(engine),
323 "RING_MI_MODE"
324 },
325 {
326 i915_mmio_reg_offset(RING_BBSTATE(engine->mmio_base)),
327 CTX_BB_STATE - 1,
328 "BB_STATE"
329 },
330 {
331 i915_mmio_reg_offset(RING_BB_PER_CTX_PTR(engine->mmio_base)),
332 lrc_ring_wa_bb_per_ctx(engine),
333 "RING_BB_PER_CTX_PTR"
334 },
335 {
336 i915_mmio_reg_offset(RING_INDIRECT_CTX(engine->mmio_base)),
337 lrc_ring_indirect_ptr(engine),
338 "RING_INDIRECT_CTX_PTR"
339 },
340 {
341 i915_mmio_reg_offset(RING_INDIRECT_CTX_OFFSET(engine->mmio_base)),
342 lrc_ring_indirect_offset(engine),
343 "RING_INDIRECT_CTX_OFFSET"
344 },
345 {
346 i915_mmio_reg_offset(RING_CTX_TIMESTAMP(engine->mmio_base)),
347 CTX_TIMESTAMP - 1,
348 "RING_CTX_TIMESTAMP"
349 },
350 {
351 i915_mmio_reg_offset(GEN8_RING_CS_GPR(engine->mmio_base, 0)),
352 lrc_ring_gpr0(engine),
353 "RING_CS_GPR0"
354 },
355 {
356 i915_mmio_reg_offset(RING_CMD_BUF_CCTL(engine->mmio_base)),
357 lrc_ring_cmd_buf_cctl(engine),
358 "RING_CMD_BUF_CCTL"
359 },
360 { },
361 }, *t;
362 u32 *hw;
363
364 if (!engine->default_state)
365 continue;
366
367 hw = shmem_pin_map(engine->default_state);
368 if (!hw) {
369 err = -ENOMEM;
370 break;
371 }
372 hw += LRC_STATE_OFFSET / sizeof(*hw);
373
374 for (t = tbl; t->name; t++) {
375 int dw = find_offset(hw, t->reg);
376
377 if (dw != t->offset) {
378 pr_err("%s: Offset for %s [0x%x] mismatch, found %x, expected %x\n",
379 engine->name,
380 t->name,
381 t->reg,
382 dw,
383 t->offset);
384 err = -EINVAL;
385 }
386 }
387
388 shmem_unpin_map(engine->default_state, hw);
389 }
390
391 return err;
392 }
393
__live_lrc_state(struct intel_engine_cs * engine,struct i915_vma * scratch)394 static int __live_lrc_state(struct intel_engine_cs *engine,
395 struct i915_vma *scratch)
396 {
397 struct intel_context *ce;
398 struct i915_request *rq;
399 struct i915_gem_ww_ctx ww;
400 enum {
401 RING_START_IDX = 0,
402 RING_TAIL_IDX,
403 MAX_IDX
404 };
405 u32 expected[MAX_IDX];
406 u32 *cs;
407 int err;
408 int n;
409
410 ce = intel_context_create(engine);
411 if (IS_ERR(ce))
412 return PTR_ERR(ce);
413
414 i915_gem_ww_ctx_init(&ww, false);
415 retry:
416 err = i915_gem_object_lock(scratch->obj, &ww);
417 if (!err)
418 err = intel_context_pin_ww(ce, &ww);
419 if (err)
420 goto err_put;
421
422 rq = i915_request_create(ce);
423 if (IS_ERR(rq)) {
424 err = PTR_ERR(rq);
425 goto err_unpin;
426 }
427
428 cs = intel_ring_begin(rq, 4 * MAX_IDX);
429 if (IS_ERR(cs)) {
430 err = PTR_ERR(cs);
431 i915_request_add(rq);
432 goto err_unpin;
433 }
434
435 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
436 *cs++ = i915_mmio_reg_offset(RING_START(engine->mmio_base));
437 *cs++ = i915_ggtt_offset(scratch) + RING_START_IDX * sizeof(u32);
438 *cs++ = 0;
439
440 expected[RING_START_IDX] = i915_ggtt_offset(ce->ring->vma);
441
442 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
443 *cs++ = i915_mmio_reg_offset(RING_TAIL(engine->mmio_base));
444 *cs++ = i915_ggtt_offset(scratch) + RING_TAIL_IDX * sizeof(u32);
445 *cs++ = 0;
446
447 err = i915_request_await_object(rq, scratch->obj, true);
448 if (!err)
449 err = i915_vma_move_to_active(scratch, rq, EXEC_OBJECT_WRITE);
450
451 i915_request_get(rq);
452 i915_request_add(rq);
453 if (err)
454 goto err_rq;
455
456 intel_engine_flush_submission(engine);
457 expected[RING_TAIL_IDX] = ce->ring->tail;
458
459 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
460 err = -ETIME;
461 goto err_rq;
462 }
463
464 cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
465 if (IS_ERR(cs)) {
466 err = PTR_ERR(cs);
467 goto err_rq;
468 }
469
470 for (n = 0; n < MAX_IDX; n++) {
471 if (cs[n] != expected[n]) {
472 pr_err("%s: Stored register[%d] value[0x%x] did not match expected[0x%x]\n",
473 engine->name, n, cs[n], expected[n]);
474 err = -EINVAL;
475 break;
476 }
477 }
478
479 i915_gem_object_unpin_map(scratch->obj);
480
481 err_rq:
482 i915_request_put(rq);
483 err_unpin:
484 intel_context_unpin(ce);
485 err_put:
486 if (err == -EDEADLK) {
487 err = i915_gem_ww_ctx_backoff(&ww);
488 if (!err)
489 goto retry;
490 }
491 i915_gem_ww_ctx_fini(&ww);
492 intel_context_put(ce);
493 return err;
494 }
495
live_lrc_state(void * arg)496 static int live_lrc_state(void *arg)
497 {
498 struct intel_gt *gt = arg;
499 struct intel_engine_cs *engine;
500 struct i915_vma *scratch;
501 enum intel_engine_id id;
502 int err = 0;
503
504 /*
505 * Check the live register state matches what we expect for this
506 * intel_context.
507 */
508
509 scratch = create_scratch(gt);
510 if (IS_ERR(scratch))
511 return PTR_ERR(scratch);
512
513 for_each_engine(engine, gt, id) {
514 err = __live_lrc_state(engine, scratch);
515 if (err)
516 break;
517 }
518
519 if (igt_flush_test(gt->i915))
520 err = -EIO;
521
522 i915_vma_unpin_and_release(&scratch, 0);
523 return err;
524 }
525
gpr_make_dirty(struct intel_context * ce)526 static int gpr_make_dirty(struct intel_context *ce)
527 {
528 struct i915_request *rq;
529 u32 *cs;
530 int n;
531
532 rq = intel_context_create_request(ce);
533 if (IS_ERR(rq))
534 return PTR_ERR(rq);
535
536 cs = intel_ring_begin(rq, 2 * NUM_GPR_DW + 2);
537 if (IS_ERR(cs)) {
538 i915_request_add(rq);
539 return PTR_ERR(cs);
540 }
541
542 *cs++ = MI_LOAD_REGISTER_IMM(NUM_GPR_DW);
543 for (n = 0; n < NUM_GPR_DW; n++) {
544 *cs++ = CS_GPR(ce->engine, n);
545 *cs++ = STACK_MAGIC;
546 }
547 *cs++ = MI_NOOP;
548
549 intel_ring_advance(rq, cs);
550
551 rq->sched.attr.priority = I915_PRIORITY_BARRIER;
552 i915_request_add(rq);
553
554 return 0;
555 }
556
557 static struct i915_request *
__gpr_read(struct intel_context * ce,struct i915_vma * scratch,u32 * slot)558 __gpr_read(struct intel_context *ce, struct i915_vma *scratch, u32 *slot)
559 {
560 const u32 offset =
561 i915_ggtt_offset(ce->engine->status_page.vma) +
562 offset_in_page(slot);
563 struct i915_request *rq;
564 u32 *cs;
565 int err;
566 int n;
567
568 rq = intel_context_create_request(ce);
569 if (IS_ERR(rq))
570 return rq;
571
572 cs = intel_ring_begin(rq, 6 + 4 * NUM_GPR_DW);
573 if (IS_ERR(cs)) {
574 i915_request_add(rq);
575 return ERR_CAST(cs);
576 }
577
578 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
579 *cs++ = MI_NOOP;
580
581 *cs++ = MI_SEMAPHORE_WAIT |
582 MI_SEMAPHORE_GLOBAL_GTT |
583 MI_SEMAPHORE_POLL |
584 MI_SEMAPHORE_SAD_NEQ_SDD;
585 *cs++ = 0;
586 *cs++ = offset;
587 *cs++ = 0;
588
589 for (n = 0; n < NUM_GPR_DW; n++) {
590 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
591 *cs++ = CS_GPR(ce->engine, n);
592 *cs++ = i915_ggtt_offset(scratch) + n * sizeof(u32);
593 *cs++ = 0;
594 }
595
596 i915_vma_lock(scratch);
597 err = i915_request_await_object(rq, scratch->obj, true);
598 if (!err)
599 err = i915_vma_move_to_active(scratch, rq, EXEC_OBJECT_WRITE);
600 i915_vma_unlock(scratch);
601
602 i915_request_get(rq);
603 i915_request_add(rq);
604 if (err) {
605 i915_request_put(rq);
606 rq = ERR_PTR(err);
607 }
608
609 return rq;
610 }
611
__live_lrc_gpr(struct intel_engine_cs * engine,struct i915_vma * scratch,bool preempt)612 static int __live_lrc_gpr(struct intel_engine_cs *engine,
613 struct i915_vma *scratch,
614 bool preempt)
615 {
616 u32 *slot = memset32(engine->status_page.addr + 1000, 0, 4);
617 struct intel_context *ce;
618 struct i915_request *rq;
619 u32 *cs;
620 int err;
621 int n;
622
623 if (GRAPHICS_VER(engine->i915) < 9 && engine->class != RENDER_CLASS)
624 return 0; /* GPR only on rcs0 for gen8 */
625
626 err = gpr_make_dirty(engine->kernel_context);
627 if (err)
628 return err;
629
630 ce = intel_context_create(engine);
631 if (IS_ERR(ce))
632 return PTR_ERR(ce);
633
634 rq = __gpr_read(ce, scratch, slot);
635 if (IS_ERR(rq)) {
636 err = PTR_ERR(rq);
637 goto err_put;
638 }
639
640 err = wait_for_submit(engine, rq, HZ / 2);
641 if (err)
642 goto err_rq;
643
644 if (preempt) {
645 err = gpr_make_dirty(engine->kernel_context);
646 if (err)
647 goto err_rq;
648
649 err = emit_semaphore_signal(engine->kernel_context, slot);
650 if (err)
651 goto err_rq;
652
653 err = wait_for_submit(engine, rq, HZ / 2);
654 if (err)
655 goto err_rq;
656 } else {
657 slot[0] = 1;
658 wmb();
659 }
660
661 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
662 err = -ETIME;
663 goto err_rq;
664 }
665
666 cs = i915_gem_object_pin_map_unlocked(scratch->obj, I915_MAP_WB);
667 if (IS_ERR(cs)) {
668 err = PTR_ERR(cs);
669 goto err_rq;
670 }
671
672 for (n = 0; n < NUM_GPR_DW; n++) {
673 if (cs[n]) {
674 pr_err("%s: GPR[%d].%s was not zero, found 0x%08x!\n",
675 engine->name,
676 n / 2, n & 1 ? "udw" : "ldw",
677 cs[n]);
678 err = -EINVAL;
679 break;
680 }
681 }
682
683 i915_gem_object_unpin_map(scratch->obj);
684
685 err_rq:
686 memset32(&slot[0], -1, 4);
687 wmb();
688 i915_request_put(rq);
689 err_put:
690 intel_context_put(ce);
691 return err;
692 }
693
live_lrc_gpr(void * arg)694 static int live_lrc_gpr(void *arg)
695 {
696 struct intel_gt *gt = arg;
697 struct intel_engine_cs *engine;
698 struct i915_vma *scratch;
699 enum intel_engine_id id;
700 int err = 0;
701
702 /*
703 * Check that GPR registers are cleared in new contexts as we need
704 * to avoid leaking any information from previous contexts.
705 */
706
707 scratch = create_scratch(gt);
708 if (IS_ERR(scratch))
709 return PTR_ERR(scratch);
710
711 for_each_engine(engine, gt, id) {
712 st_engine_heartbeat_disable(engine);
713
714 err = __live_lrc_gpr(engine, scratch, false);
715 if (err)
716 goto err;
717
718 err = __live_lrc_gpr(engine, scratch, true);
719 if (err)
720 goto err;
721
722 err:
723 st_engine_heartbeat_enable(engine);
724 if (igt_flush_test(gt->i915))
725 err = -EIO;
726 if (err)
727 break;
728 }
729
730 i915_vma_unpin_and_release(&scratch, 0);
731 return err;
732 }
733
734 static struct i915_request *
create_timestamp(struct intel_context * ce,void * slot,int idx)735 create_timestamp(struct intel_context *ce, void *slot, int idx)
736 {
737 const u32 offset =
738 i915_ggtt_offset(ce->engine->status_page.vma) +
739 offset_in_page(slot);
740 struct i915_request *rq;
741 u32 *cs;
742 int err;
743
744 rq = intel_context_create_request(ce);
745 if (IS_ERR(rq))
746 return rq;
747
748 cs = intel_ring_begin(rq, 10);
749 if (IS_ERR(cs)) {
750 err = PTR_ERR(cs);
751 goto err;
752 }
753
754 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
755 *cs++ = MI_NOOP;
756
757 *cs++ = MI_SEMAPHORE_WAIT |
758 MI_SEMAPHORE_GLOBAL_GTT |
759 MI_SEMAPHORE_POLL |
760 MI_SEMAPHORE_SAD_NEQ_SDD;
761 *cs++ = 0;
762 *cs++ = offset;
763 *cs++ = 0;
764
765 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
766 *cs++ = i915_mmio_reg_offset(RING_CTX_TIMESTAMP(rq->engine->mmio_base));
767 *cs++ = offset + idx * sizeof(u32);
768 *cs++ = 0;
769
770 intel_ring_advance(rq, cs);
771
772 err = 0;
773 err:
774 i915_request_get(rq);
775 i915_request_add(rq);
776 if (err) {
777 i915_request_put(rq);
778 return ERR_PTR(err);
779 }
780
781 return rq;
782 }
783
784 struct lrc_timestamp {
785 struct intel_engine_cs *engine;
786 struct intel_context *ce[2];
787 u32 poison;
788 };
789
timestamp_advanced(u32 start,u32 end)790 static bool timestamp_advanced(u32 start, u32 end)
791 {
792 return (s32)(end - start) > 0;
793 }
794
__lrc_timestamp(const struct lrc_timestamp * arg,bool preempt)795 static int __lrc_timestamp(const struct lrc_timestamp *arg, bool preempt)
796 {
797 u32 *slot = memset32(arg->engine->status_page.addr + 1000, 0, 4);
798 struct i915_request *rq;
799 u32 timestamp;
800 int err = 0;
801
802 arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP] = arg->poison;
803 rq = create_timestamp(arg->ce[0], slot, 1);
804 if (IS_ERR(rq))
805 return PTR_ERR(rq);
806
807 err = wait_for_submit(rq->engine, rq, HZ / 2);
808 if (err)
809 goto err;
810
811 if (preempt) {
812 arg->ce[1]->lrc_reg_state[CTX_TIMESTAMP] = 0xdeadbeef;
813 err = emit_semaphore_signal(arg->ce[1], slot);
814 if (err)
815 goto err;
816 } else {
817 slot[0] = 1;
818 wmb();
819 }
820
821 /* And wait for switch to kernel (to save our context to memory) */
822 err = context_flush(arg->ce[0], HZ / 2);
823 if (err)
824 goto err;
825
826 if (!timestamp_advanced(arg->poison, slot[1])) {
827 pr_err("%s(%s): invalid timestamp on restore, context:%x, request:%x\n",
828 arg->engine->name, preempt ? "preempt" : "simple",
829 arg->poison, slot[1]);
830 err = -EINVAL;
831 }
832
833 timestamp = READ_ONCE(arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP]);
834 if (!timestamp_advanced(slot[1], timestamp)) {
835 pr_err("%s(%s): invalid timestamp on save, request:%x, context:%x\n",
836 arg->engine->name, preempt ? "preempt" : "simple",
837 slot[1], timestamp);
838 err = -EINVAL;
839 }
840
841 err:
842 memset32(slot, -1, 4);
843 i915_request_put(rq);
844 return err;
845 }
846
live_lrc_timestamp(void * arg)847 static int live_lrc_timestamp(void *arg)
848 {
849 struct lrc_timestamp data = {};
850 struct intel_gt *gt = arg;
851 enum intel_engine_id id;
852 const u32 poison[] = {
853 0,
854 S32_MAX,
855 (u32)S32_MAX + 1,
856 U32_MAX,
857 };
858
859 /*
860 * We want to verify that the timestamp is saved and restore across
861 * context switches and is monotonic.
862 *
863 * So we do this with a little bit of LRC poisoning to check various
864 * boundary conditions, and see what happens if we preempt the context
865 * with a second request (carrying more poison into the timestamp).
866 */
867
868 for_each_engine(data.engine, gt, id) {
869 int i, err = 0;
870
871 st_engine_heartbeat_disable(data.engine);
872
873 for (i = 0; i < ARRAY_SIZE(data.ce); i++) {
874 struct intel_context *tmp;
875
876 tmp = intel_context_create(data.engine);
877 if (IS_ERR(tmp)) {
878 err = PTR_ERR(tmp);
879 goto err;
880 }
881
882 err = intel_context_pin(tmp);
883 if (err) {
884 intel_context_put(tmp);
885 goto err;
886 }
887
888 data.ce[i] = tmp;
889 }
890
891 for (i = 0; i < ARRAY_SIZE(poison); i++) {
892 data.poison = poison[i];
893
894 err = __lrc_timestamp(&data, false);
895 if (err)
896 break;
897
898 err = __lrc_timestamp(&data, true);
899 if (err)
900 break;
901 }
902
903 err:
904 st_engine_heartbeat_enable(data.engine);
905 for (i = 0; i < ARRAY_SIZE(data.ce); i++) {
906 if (!data.ce[i])
907 break;
908
909 intel_context_unpin(data.ce[i]);
910 intel_context_put(data.ce[i]);
911 }
912
913 if (igt_flush_test(gt->i915))
914 err = -EIO;
915 if (err)
916 return err;
917 }
918
919 return 0;
920 }
921
922 static struct i915_vma *
create_user_vma(struct i915_address_space * vm,unsigned long size)923 create_user_vma(struct i915_address_space *vm, unsigned long size)
924 {
925 struct drm_i915_gem_object *obj;
926 struct i915_vma *vma;
927 int err;
928
929 obj = i915_gem_object_create_internal(vm->i915, size);
930 if (IS_ERR(obj))
931 return ERR_CAST(obj);
932
933 vma = i915_vma_instance(obj, vm, NULL);
934 if (IS_ERR(vma)) {
935 i915_gem_object_put(obj);
936 return vma;
937 }
938
939 err = i915_vma_pin(vma, 0, 0, PIN_USER);
940 if (err) {
941 i915_gem_object_put(obj);
942 return ERR_PTR(err);
943 }
944
945 return vma;
946 }
947
safe_poison(u32 offset,u32 poison)948 static u32 safe_poison(u32 offset, u32 poison)
949 {
950 /*
951 * Do not enable predication as it will nop all subsequent commands,
952 * not only disabling the tests (by preventing all the other SRM) but
953 * also preventing the arbitration events at the end of the request.
954 */
955 if (offset == i915_mmio_reg_offset(RING_PREDICATE_RESULT(0)))
956 poison &= ~REG_BIT(0);
957
958 return poison;
959 }
960
961 static struct i915_vma *
store_context(struct intel_context * ce,struct i915_vma * scratch)962 store_context(struct intel_context *ce, struct i915_vma *scratch)
963 {
964 struct i915_vma *batch;
965 u32 dw, x, *cs, *hw;
966 u32 *defaults;
967
968 batch = create_user_vma(ce->vm, SZ_64K);
969 if (IS_ERR(batch))
970 return batch;
971
972 cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);
973 if (IS_ERR(cs)) {
974 i915_vma_put(batch);
975 return ERR_CAST(cs);
976 }
977
978 defaults = shmem_pin_map(ce->engine->default_state);
979 if (!defaults) {
980 i915_gem_object_unpin_map(batch->obj);
981 i915_vma_put(batch);
982 return ERR_PTR(-ENOMEM);
983 }
984
985 x = 0;
986 dw = 0;
987 hw = defaults;
988 hw += LRC_STATE_OFFSET / sizeof(*hw);
989 do {
990 u32 len = hw[dw] & 0x7f;
991
992 if (hw[dw] == 0) {
993 dw++;
994 continue;
995 }
996
997 if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
998 dw += len + 2;
999 continue;
1000 }
1001
1002 dw++;
1003 len = (len + 1) / 2;
1004 while (len--) {
1005 *cs++ = MI_STORE_REGISTER_MEM_GEN8;
1006 *cs++ = hw[dw];
1007 *cs++ = lower_32_bits(scratch->node.start + x);
1008 *cs++ = upper_32_bits(scratch->node.start + x);
1009
1010 dw += 2;
1011 x += 4;
1012 }
1013 } while (dw < PAGE_SIZE / sizeof(u32) &&
1014 (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
1015
1016 *cs++ = MI_BATCH_BUFFER_END;
1017
1018 shmem_unpin_map(ce->engine->default_state, defaults);
1019
1020 i915_gem_object_flush_map(batch->obj);
1021 i915_gem_object_unpin_map(batch->obj);
1022
1023 return batch;
1024 }
1025
move_to_active(struct i915_request * rq,struct i915_vma * vma,unsigned int flags)1026 static int move_to_active(struct i915_request *rq,
1027 struct i915_vma *vma,
1028 unsigned int flags)
1029 {
1030 int err;
1031
1032 i915_vma_lock(vma);
1033 err = i915_request_await_object(rq, vma->obj, flags);
1034 if (!err)
1035 err = i915_vma_move_to_active(vma, rq, flags);
1036 i915_vma_unlock(vma);
1037
1038 return err;
1039 }
1040
1041 static struct i915_request *
record_registers(struct intel_context * ce,struct i915_vma * before,struct i915_vma * after,u32 * sema)1042 record_registers(struct intel_context *ce,
1043 struct i915_vma *before,
1044 struct i915_vma *after,
1045 u32 *sema)
1046 {
1047 struct i915_vma *b_before, *b_after;
1048 struct i915_request *rq;
1049 u32 *cs;
1050 int err;
1051
1052 b_before = store_context(ce, before);
1053 if (IS_ERR(b_before))
1054 return ERR_CAST(b_before);
1055
1056 b_after = store_context(ce, after);
1057 if (IS_ERR(b_after)) {
1058 rq = ERR_CAST(b_after);
1059 goto err_before;
1060 }
1061
1062 rq = intel_context_create_request(ce);
1063 if (IS_ERR(rq))
1064 goto err_after;
1065
1066 err = move_to_active(rq, before, EXEC_OBJECT_WRITE);
1067 if (err)
1068 goto err_rq;
1069
1070 err = move_to_active(rq, b_before, 0);
1071 if (err)
1072 goto err_rq;
1073
1074 err = move_to_active(rq, after, EXEC_OBJECT_WRITE);
1075 if (err)
1076 goto err_rq;
1077
1078 err = move_to_active(rq, b_after, 0);
1079 if (err)
1080 goto err_rq;
1081
1082 cs = intel_ring_begin(rq, 14);
1083 if (IS_ERR(cs)) {
1084 err = PTR_ERR(cs);
1085 goto err_rq;
1086 }
1087
1088 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1089 *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
1090 *cs++ = lower_32_bits(b_before->node.start);
1091 *cs++ = upper_32_bits(b_before->node.start);
1092
1093 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
1094 *cs++ = MI_SEMAPHORE_WAIT |
1095 MI_SEMAPHORE_GLOBAL_GTT |
1096 MI_SEMAPHORE_POLL |
1097 MI_SEMAPHORE_SAD_NEQ_SDD;
1098 *cs++ = 0;
1099 *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) +
1100 offset_in_page(sema);
1101 *cs++ = 0;
1102 *cs++ = MI_NOOP;
1103
1104 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1105 *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
1106 *cs++ = lower_32_bits(b_after->node.start);
1107 *cs++ = upper_32_bits(b_after->node.start);
1108
1109 intel_ring_advance(rq, cs);
1110
1111 WRITE_ONCE(*sema, 0);
1112 i915_request_get(rq);
1113 i915_request_add(rq);
1114 err_after:
1115 i915_vma_put(b_after);
1116 err_before:
1117 i915_vma_put(b_before);
1118 return rq;
1119
1120 err_rq:
1121 i915_request_add(rq);
1122 rq = ERR_PTR(err);
1123 goto err_after;
1124 }
1125
load_context(struct intel_context * ce,u32 poison)1126 static struct i915_vma *load_context(struct intel_context *ce, u32 poison)
1127 {
1128 struct i915_vma *batch;
1129 u32 dw, *cs, *hw;
1130 u32 *defaults;
1131
1132 batch = create_user_vma(ce->vm, SZ_64K);
1133 if (IS_ERR(batch))
1134 return batch;
1135
1136 cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);
1137 if (IS_ERR(cs)) {
1138 i915_vma_put(batch);
1139 return ERR_CAST(cs);
1140 }
1141
1142 defaults = shmem_pin_map(ce->engine->default_state);
1143 if (!defaults) {
1144 i915_gem_object_unpin_map(batch->obj);
1145 i915_vma_put(batch);
1146 return ERR_PTR(-ENOMEM);
1147 }
1148
1149 dw = 0;
1150 hw = defaults;
1151 hw += LRC_STATE_OFFSET / sizeof(*hw);
1152 do {
1153 u32 len = hw[dw] & 0x7f;
1154
1155 if (hw[dw] == 0) {
1156 dw++;
1157 continue;
1158 }
1159
1160 if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
1161 dw += len + 2;
1162 continue;
1163 }
1164
1165 dw++;
1166 len = (len + 1) / 2;
1167 *cs++ = MI_LOAD_REGISTER_IMM(len);
1168 while (len--) {
1169 *cs++ = hw[dw];
1170 *cs++ = safe_poison(hw[dw] & get_lri_mask(ce->engine,
1171 MI_LRI_LRM_CS_MMIO),
1172 poison);
1173 dw += 2;
1174 }
1175 } while (dw < PAGE_SIZE / sizeof(u32) &&
1176 (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
1177
1178 *cs++ = MI_BATCH_BUFFER_END;
1179
1180 shmem_unpin_map(ce->engine->default_state, defaults);
1181
1182 i915_gem_object_flush_map(batch->obj);
1183 i915_gem_object_unpin_map(batch->obj);
1184
1185 return batch;
1186 }
1187
poison_registers(struct intel_context * ce,u32 poison,u32 * sema)1188 static int poison_registers(struct intel_context *ce, u32 poison, u32 *sema)
1189 {
1190 struct i915_request *rq;
1191 struct i915_vma *batch;
1192 u32 *cs;
1193 int err;
1194
1195 batch = load_context(ce, poison);
1196 if (IS_ERR(batch))
1197 return PTR_ERR(batch);
1198
1199 rq = intel_context_create_request(ce);
1200 if (IS_ERR(rq)) {
1201 err = PTR_ERR(rq);
1202 goto err_batch;
1203 }
1204
1205 err = move_to_active(rq, batch, 0);
1206 if (err)
1207 goto err_rq;
1208
1209 cs = intel_ring_begin(rq, 8);
1210 if (IS_ERR(cs)) {
1211 err = PTR_ERR(cs);
1212 goto err_rq;
1213 }
1214
1215 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1216 *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
1217 *cs++ = lower_32_bits(batch->node.start);
1218 *cs++ = upper_32_bits(batch->node.start);
1219
1220 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
1221 *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) +
1222 offset_in_page(sema);
1223 *cs++ = 0;
1224 *cs++ = 1;
1225
1226 intel_ring_advance(rq, cs);
1227
1228 rq->sched.attr.priority = I915_PRIORITY_BARRIER;
1229 err_rq:
1230 i915_request_add(rq);
1231 err_batch:
1232 i915_vma_put(batch);
1233 return err;
1234 }
1235
is_moving(u32 a,u32 b)1236 static bool is_moving(u32 a, u32 b)
1237 {
1238 return a != b;
1239 }
1240
compare_isolation(struct intel_engine_cs * engine,struct i915_vma * ref[2],struct i915_vma * result[2],struct intel_context * ce,u32 poison)1241 static int compare_isolation(struct intel_engine_cs *engine,
1242 struct i915_vma *ref[2],
1243 struct i915_vma *result[2],
1244 struct intel_context *ce,
1245 u32 poison)
1246 {
1247 u32 x, dw, *hw, *lrc;
1248 u32 *A[2], *B[2];
1249 u32 *defaults;
1250 int err = 0;
1251
1252 A[0] = i915_gem_object_pin_map_unlocked(ref[0]->obj, I915_MAP_WC);
1253 if (IS_ERR(A[0]))
1254 return PTR_ERR(A[0]);
1255
1256 A[1] = i915_gem_object_pin_map_unlocked(ref[1]->obj, I915_MAP_WC);
1257 if (IS_ERR(A[1])) {
1258 err = PTR_ERR(A[1]);
1259 goto err_A0;
1260 }
1261
1262 B[0] = i915_gem_object_pin_map_unlocked(result[0]->obj, I915_MAP_WC);
1263 if (IS_ERR(B[0])) {
1264 err = PTR_ERR(B[0]);
1265 goto err_A1;
1266 }
1267
1268 B[1] = i915_gem_object_pin_map_unlocked(result[1]->obj, I915_MAP_WC);
1269 if (IS_ERR(B[1])) {
1270 err = PTR_ERR(B[1]);
1271 goto err_B0;
1272 }
1273
1274 lrc = i915_gem_object_pin_map_unlocked(ce->state->obj,
1275 i915_coherent_map_type(engine->i915,
1276 ce->state->obj,
1277 false));
1278 if (IS_ERR(lrc)) {
1279 err = PTR_ERR(lrc);
1280 goto err_B1;
1281 }
1282 lrc += LRC_STATE_OFFSET / sizeof(*hw);
1283
1284 defaults = shmem_pin_map(ce->engine->default_state);
1285 if (!defaults) {
1286 err = -ENOMEM;
1287 goto err_lrc;
1288 }
1289
1290 x = 0;
1291 dw = 0;
1292 hw = defaults;
1293 hw += LRC_STATE_OFFSET / sizeof(*hw);
1294 do {
1295 u32 len = hw[dw] & 0x7f;
1296
1297 if (hw[dw] == 0) {
1298 dw++;
1299 continue;
1300 }
1301
1302 if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
1303 dw += len + 2;
1304 continue;
1305 }
1306
1307 dw++;
1308 len = (len + 1) / 2;
1309 while (len--) {
1310 if (!is_moving(A[0][x], A[1][x]) &&
1311 (A[0][x] != B[0][x] || A[1][x] != B[1][x])) {
1312 switch (hw[dw] & 4095) {
1313 case 0x30: /* RING_HEAD */
1314 case 0x34: /* RING_TAIL */
1315 break;
1316
1317 default:
1318 pr_err("%s[%d]: Mismatch for register %4x, default %08x, reference %08x, result (%08x, %08x), poison %08x, context %08x\n",
1319 engine->name, dw,
1320 hw[dw], hw[dw + 1],
1321 A[0][x], B[0][x], B[1][x],
1322 poison, lrc[dw + 1]);
1323 err = -EINVAL;
1324 }
1325 }
1326 dw += 2;
1327 x++;
1328 }
1329 } while (dw < PAGE_SIZE / sizeof(u32) &&
1330 (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
1331
1332 shmem_unpin_map(ce->engine->default_state, defaults);
1333 err_lrc:
1334 i915_gem_object_unpin_map(ce->state->obj);
1335 err_B1:
1336 i915_gem_object_unpin_map(result[1]->obj);
1337 err_B0:
1338 i915_gem_object_unpin_map(result[0]->obj);
1339 err_A1:
1340 i915_gem_object_unpin_map(ref[1]->obj);
1341 err_A0:
1342 i915_gem_object_unpin_map(ref[0]->obj);
1343 return err;
1344 }
1345
__lrc_isolation(struct intel_engine_cs * engine,u32 poison)1346 static int __lrc_isolation(struct intel_engine_cs *engine, u32 poison)
1347 {
1348 u32 *sema = memset32(engine->status_page.addr + 1000, 0, 1);
1349 struct i915_vma *ref[2], *result[2];
1350 struct intel_context *A, *B;
1351 struct i915_request *rq;
1352 int err;
1353
1354 A = intel_context_create(engine);
1355 if (IS_ERR(A))
1356 return PTR_ERR(A);
1357
1358 B = intel_context_create(engine);
1359 if (IS_ERR(B)) {
1360 err = PTR_ERR(B);
1361 goto err_A;
1362 }
1363
1364 ref[0] = create_user_vma(A->vm, SZ_64K);
1365 if (IS_ERR(ref[0])) {
1366 err = PTR_ERR(ref[0]);
1367 goto err_B;
1368 }
1369
1370 ref[1] = create_user_vma(A->vm, SZ_64K);
1371 if (IS_ERR(ref[1])) {
1372 err = PTR_ERR(ref[1]);
1373 goto err_ref0;
1374 }
1375
1376 rq = record_registers(A, ref[0], ref[1], sema);
1377 if (IS_ERR(rq)) {
1378 err = PTR_ERR(rq);
1379 goto err_ref1;
1380 }
1381
1382 WRITE_ONCE(*sema, 1);
1383 wmb();
1384
1385 if (i915_request_wait(rq, 0, HZ / 2) < 0) {
1386 i915_request_put(rq);
1387 err = -ETIME;
1388 goto err_ref1;
1389 }
1390 i915_request_put(rq);
1391
1392 result[0] = create_user_vma(A->vm, SZ_64K);
1393 if (IS_ERR(result[0])) {
1394 err = PTR_ERR(result[0]);
1395 goto err_ref1;
1396 }
1397
1398 result[1] = create_user_vma(A->vm, SZ_64K);
1399 if (IS_ERR(result[1])) {
1400 err = PTR_ERR(result[1]);
1401 goto err_result0;
1402 }
1403
1404 rq = record_registers(A, result[0], result[1], sema);
1405 if (IS_ERR(rq)) {
1406 err = PTR_ERR(rq);
1407 goto err_result1;
1408 }
1409
1410 err = poison_registers(B, poison, sema);
1411 if (err) {
1412 WRITE_ONCE(*sema, -1);
1413 i915_request_put(rq);
1414 goto err_result1;
1415 }
1416
1417 if (i915_request_wait(rq, 0, HZ / 2) < 0) {
1418 i915_request_put(rq);
1419 err = -ETIME;
1420 goto err_result1;
1421 }
1422 i915_request_put(rq);
1423
1424 err = compare_isolation(engine, ref, result, A, poison);
1425
1426 err_result1:
1427 i915_vma_put(result[1]);
1428 err_result0:
1429 i915_vma_put(result[0]);
1430 err_ref1:
1431 i915_vma_put(ref[1]);
1432 err_ref0:
1433 i915_vma_put(ref[0]);
1434 err_B:
1435 intel_context_put(B);
1436 err_A:
1437 intel_context_put(A);
1438 return err;
1439 }
1440
skip_isolation(const struct intel_engine_cs * engine)1441 static bool skip_isolation(const struct intel_engine_cs *engine)
1442 {
1443 if (engine->class == COPY_ENGINE_CLASS && GRAPHICS_VER(engine->i915) == 9)
1444 return true;
1445
1446 if (engine->class == RENDER_CLASS && GRAPHICS_VER(engine->i915) == 11)
1447 return true;
1448
1449 return false;
1450 }
1451
live_lrc_isolation(void * arg)1452 static int live_lrc_isolation(void *arg)
1453 {
1454 struct intel_gt *gt = arg;
1455 struct intel_engine_cs *engine;
1456 enum intel_engine_id id;
1457 const u32 poison[] = {
1458 STACK_MAGIC,
1459 0x3a3a3a3a,
1460 0x5c5c5c5c,
1461 0xffffffff,
1462 0xffff0000,
1463 };
1464 int err = 0;
1465
1466 /*
1467 * Our goal is try and verify that per-context state cannot be
1468 * tampered with by another non-privileged client.
1469 *
1470 * We take the list of context registers from the LRI in the default
1471 * context image and attempt to modify that list from a remote context.
1472 */
1473
1474 for_each_engine(engine, gt, id) {
1475 int i;
1476
1477 /* Just don't even ask */
1478 if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN) &&
1479 skip_isolation(engine))
1480 continue;
1481
1482 intel_engine_pm_get(engine);
1483 for (i = 0; i < ARRAY_SIZE(poison); i++) {
1484 int result;
1485
1486 result = __lrc_isolation(engine, poison[i]);
1487 if (result && !err)
1488 err = result;
1489
1490 result = __lrc_isolation(engine, ~poison[i]);
1491 if (result && !err)
1492 err = result;
1493 }
1494 intel_engine_pm_put(engine);
1495 if (igt_flush_test(gt->i915)) {
1496 err = -EIO;
1497 break;
1498 }
1499 }
1500
1501 return err;
1502 }
1503
indirect_ctx_submit_req(struct intel_context * ce)1504 static int indirect_ctx_submit_req(struct intel_context *ce)
1505 {
1506 struct i915_request *rq;
1507 int err = 0;
1508
1509 rq = intel_context_create_request(ce);
1510 if (IS_ERR(rq))
1511 return PTR_ERR(rq);
1512
1513 i915_request_get(rq);
1514 i915_request_add(rq);
1515
1516 if (i915_request_wait(rq, 0, HZ / 5) < 0)
1517 err = -ETIME;
1518
1519 i915_request_put(rq);
1520
1521 return err;
1522 }
1523
1524 #define CTX_BB_CANARY_OFFSET (3 * 1024)
1525 #define CTX_BB_CANARY_INDEX (CTX_BB_CANARY_OFFSET / sizeof(u32))
1526
1527 static u32 *
emit_indirect_ctx_bb_canary(const struct intel_context * ce,u32 * cs)1528 emit_indirect_ctx_bb_canary(const struct intel_context *ce, u32 *cs)
1529 {
1530 *cs++ = MI_STORE_REGISTER_MEM_GEN8 |
1531 MI_SRM_LRM_GLOBAL_GTT |
1532 MI_LRI_LRM_CS_MMIO;
1533 *cs++ = i915_mmio_reg_offset(RING_START(0));
1534 *cs++ = i915_ggtt_offset(ce->state) +
1535 context_wa_bb_offset(ce) +
1536 CTX_BB_CANARY_OFFSET;
1537 *cs++ = 0;
1538
1539 return cs;
1540 }
1541
1542 static void
indirect_ctx_bb_setup(struct intel_context * ce)1543 indirect_ctx_bb_setup(struct intel_context *ce)
1544 {
1545 u32 *cs = context_indirect_bb(ce);
1546
1547 cs[CTX_BB_CANARY_INDEX] = 0xdeadf00d;
1548
1549 setup_indirect_ctx_bb(ce, ce->engine, emit_indirect_ctx_bb_canary);
1550 }
1551
check_ring_start(struct intel_context * ce)1552 static bool check_ring_start(struct intel_context *ce)
1553 {
1554 const u32 * const ctx_bb = (void *)(ce->lrc_reg_state) -
1555 LRC_STATE_OFFSET + context_wa_bb_offset(ce);
1556
1557 if (ctx_bb[CTX_BB_CANARY_INDEX] == ce->lrc_reg_state[CTX_RING_START])
1558 return true;
1559
1560 pr_err("ring start mismatch: canary 0x%08x vs state 0x%08x\n",
1561 ctx_bb[CTX_BB_CANARY_INDEX],
1562 ce->lrc_reg_state[CTX_RING_START]);
1563
1564 return false;
1565 }
1566
indirect_ctx_bb_check(struct intel_context * ce)1567 static int indirect_ctx_bb_check(struct intel_context *ce)
1568 {
1569 int err;
1570
1571 err = indirect_ctx_submit_req(ce);
1572 if (err)
1573 return err;
1574
1575 if (!check_ring_start(ce))
1576 return -EINVAL;
1577
1578 return 0;
1579 }
1580
__live_lrc_indirect_ctx_bb(struct intel_engine_cs * engine)1581 static int __live_lrc_indirect_ctx_bb(struct intel_engine_cs *engine)
1582 {
1583 struct intel_context *a, *b;
1584 int err;
1585
1586 a = intel_context_create(engine);
1587 if (IS_ERR(a))
1588 return PTR_ERR(a);
1589 err = intel_context_pin(a);
1590 if (err)
1591 goto put_a;
1592
1593 b = intel_context_create(engine);
1594 if (IS_ERR(b)) {
1595 err = PTR_ERR(b);
1596 goto unpin_a;
1597 }
1598 err = intel_context_pin(b);
1599 if (err)
1600 goto put_b;
1601
1602 /* We use the already reserved extra page in context state */
1603 if (!a->wa_bb_page) {
1604 GEM_BUG_ON(b->wa_bb_page);
1605 GEM_BUG_ON(GRAPHICS_VER(engine->i915) == 12);
1606 goto unpin_b;
1607 }
1608
1609 /*
1610 * In order to test that our per context bb is truly per context,
1611 * and executes at the intended spot on context restoring process,
1612 * make the batch store the ring start value to memory.
1613 * As ring start is restored apriori of starting the indirect ctx bb and
1614 * as it will be different for each context, it fits to this purpose.
1615 */
1616 indirect_ctx_bb_setup(a);
1617 indirect_ctx_bb_setup(b);
1618
1619 err = indirect_ctx_bb_check(a);
1620 if (err)
1621 goto unpin_b;
1622
1623 err = indirect_ctx_bb_check(b);
1624
1625 unpin_b:
1626 intel_context_unpin(b);
1627 put_b:
1628 intel_context_put(b);
1629 unpin_a:
1630 intel_context_unpin(a);
1631 put_a:
1632 intel_context_put(a);
1633
1634 return err;
1635 }
1636
live_lrc_indirect_ctx_bb(void * arg)1637 static int live_lrc_indirect_ctx_bb(void *arg)
1638 {
1639 struct intel_gt *gt = arg;
1640 struct intel_engine_cs *engine;
1641 enum intel_engine_id id;
1642 int err = 0;
1643
1644 for_each_engine(engine, gt, id) {
1645 intel_engine_pm_get(engine);
1646 err = __live_lrc_indirect_ctx_bb(engine);
1647 intel_engine_pm_put(engine);
1648
1649 if (igt_flush_test(gt->i915))
1650 err = -EIO;
1651
1652 if (err)
1653 break;
1654 }
1655
1656 return err;
1657 }
1658
garbage_reset(struct intel_engine_cs * engine,struct i915_request * rq)1659 static void garbage_reset(struct intel_engine_cs *engine,
1660 struct i915_request *rq)
1661 {
1662 const unsigned int bit = I915_RESET_ENGINE + engine->id;
1663 unsigned long *lock = &engine->gt->reset.flags;
1664
1665 local_bh_disable();
1666 if (!test_and_set_bit(bit, lock)) {
1667 tasklet_disable(&engine->sched_engine->tasklet);
1668
1669 if (!rq->fence.error)
1670 __intel_engine_reset_bh(engine, NULL);
1671
1672 tasklet_enable(&engine->sched_engine->tasklet);
1673 clear_and_wake_up_bit(bit, lock);
1674 }
1675 local_bh_enable();
1676 }
1677
garbage(struct intel_context * ce,struct rnd_state * prng)1678 static struct i915_request *garbage(struct intel_context *ce,
1679 struct rnd_state *prng)
1680 {
1681 struct i915_request *rq;
1682 int err;
1683
1684 err = intel_context_pin(ce);
1685 if (err)
1686 return ERR_PTR(err);
1687
1688 prandom_bytes_state(prng,
1689 ce->lrc_reg_state,
1690 ce->engine->context_size -
1691 LRC_STATE_OFFSET);
1692
1693 rq = intel_context_create_request(ce);
1694 if (IS_ERR(rq)) {
1695 err = PTR_ERR(rq);
1696 goto err_unpin;
1697 }
1698
1699 i915_request_get(rq);
1700 i915_request_add(rq);
1701 return rq;
1702
1703 err_unpin:
1704 intel_context_unpin(ce);
1705 return ERR_PTR(err);
1706 }
1707
__lrc_garbage(struct intel_engine_cs * engine,struct rnd_state * prng)1708 static int __lrc_garbage(struct intel_engine_cs *engine, struct rnd_state *prng)
1709 {
1710 struct intel_context *ce;
1711 struct i915_request *hang;
1712 int err = 0;
1713
1714 ce = intel_context_create(engine);
1715 if (IS_ERR(ce))
1716 return PTR_ERR(ce);
1717
1718 hang = garbage(ce, prng);
1719 if (IS_ERR(hang)) {
1720 err = PTR_ERR(hang);
1721 goto err_ce;
1722 }
1723
1724 if (wait_for_submit(engine, hang, HZ / 2)) {
1725 i915_request_put(hang);
1726 err = -ETIME;
1727 goto err_ce;
1728 }
1729
1730 intel_context_set_banned(ce);
1731 garbage_reset(engine, hang);
1732
1733 intel_engine_flush_submission(engine);
1734 if (!hang->fence.error) {
1735 i915_request_put(hang);
1736 pr_err("%s: corrupted context was not reset\n",
1737 engine->name);
1738 err = -EINVAL;
1739 goto err_ce;
1740 }
1741
1742 if (i915_request_wait(hang, 0, HZ / 2) < 0) {
1743 pr_err("%s: corrupted context did not recover\n",
1744 engine->name);
1745 i915_request_put(hang);
1746 err = -EIO;
1747 goto err_ce;
1748 }
1749 i915_request_put(hang);
1750
1751 err_ce:
1752 intel_context_put(ce);
1753 return err;
1754 }
1755
live_lrc_garbage(void * arg)1756 static int live_lrc_garbage(void *arg)
1757 {
1758 struct intel_gt *gt = arg;
1759 struct intel_engine_cs *engine;
1760 enum intel_engine_id id;
1761
1762 /*
1763 * Verify that we can recover if one context state is completely
1764 * corrupted.
1765 */
1766
1767 if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN))
1768 return 0;
1769
1770 for_each_engine(engine, gt, id) {
1771 I915_RND_STATE(prng);
1772 int err = 0, i;
1773
1774 if (!intel_has_reset_engine(engine->gt))
1775 continue;
1776
1777 intel_engine_pm_get(engine);
1778 for (i = 0; i < 3; i++) {
1779 err = __lrc_garbage(engine, &prng);
1780 if (err)
1781 break;
1782 }
1783 intel_engine_pm_put(engine);
1784
1785 if (igt_flush_test(gt->i915))
1786 err = -EIO;
1787 if (err)
1788 return err;
1789 }
1790
1791 return 0;
1792 }
1793
__live_pphwsp_runtime(struct intel_engine_cs * engine)1794 static int __live_pphwsp_runtime(struct intel_engine_cs *engine)
1795 {
1796 struct intel_context *ce;
1797 struct i915_request *rq;
1798 IGT_TIMEOUT(end_time);
1799 int err;
1800
1801 ce = intel_context_create(engine);
1802 if (IS_ERR(ce))
1803 return PTR_ERR(ce);
1804
1805 ce->stats.runtime.num_underflow = 0;
1806 ce->stats.runtime.max_underflow = 0;
1807
1808 do {
1809 unsigned int loop = 1024;
1810
1811 while (loop) {
1812 rq = intel_context_create_request(ce);
1813 if (IS_ERR(rq)) {
1814 err = PTR_ERR(rq);
1815 goto err_rq;
1816 }
1817
1818 if (--loop == 0)
1819 i915_request_get(rq);
1820
1821 i915_request_add(rq);
1822 }
1823
1824 if (__igt_timeout(end_time, NULL))
1825 break;
1826
1827 i915_request_put(rq);
1828 } while (1);
1829
1830 err = i915_request_wait(rq, 0, HZ / 5);
1831 if (err < 0) {
1832 pr_err("%s: request not completed!\n", engine->name);
1833 goto err_wait;
1834 }
1835
1836 igt_flush_test(engine->i915);
1837
1838 pr_info("%s: pphwsp runtime %lluns, average %lluns\n",
1839 engine->name,
1840 intel_context_get_total_runtime_ns(ce),
1841 intel_context_get_avg_runtime_ns(ce));
1842
1843 err = 0;
1844 if (ce->stats.runtime.num_underflow) {
1845 pr_err("%s: pphwsp underflow %u time(s), max %u cycles!\n",
1846 engine->name,
1847 ce->stats.runtime.num_underflow,
1848 ce->stats.runtime.max_underflow);
1849 GEM_TRACE_DUMP();
1850 err = -EOVERFLOW;
1851 }
1852
1853 err_wait:
1854 i915_request_put(rq);
1855 err_rq:
1856 intel_context_put(ce);
1857 return err;
1858 }
1859
live_pphwsp_runtime(void * arg)1860 static int live_pphwsp_runtime(void *arg)
1861 {
1862 struct intel_gt *gt = arg;
1863 struct intel_engine_cs *engine;
1864 enum intel_engine_id id;
1865 int err = 0;
1866
1867 /*
1868 * Check that cumulative context runtime as stored in the pphwsp[16]
1869 * is monotonic.
1870 */
1871
1872 for_each_engine(engine, gt, id) {
1873 err = __live_pphwsp_runtime(engine);
1874 if (err)
1875 break;
1876 }
1877
1878 if (igt_flush_test(gt->i915))
1879 err = -EIO;
1880
1881 return err;
1882 }
1883
intel_lrc_live_selftests(struct drm_i915_private * i915)1884 int intel_lrc_live_selftests(struct drm_i915_private *i915)
1885 {
1886 static const struct i915_subtest tests[] = {
1887 SUBTEST(live_lrc_layout),
1888 SUBTEST(live_lrc_fixed),
1889 SUBTEST(live_lrc_state),
1890 SUBTEST(live_lrc_gpr),
1891 SUBTEST(live_lrc_isolation),
1892 SUBTEST(live_lrc_timestamp),
1893 SUBTEST(live_lrc_garbage),
1894 SUBTEST(live_pphwsp_runtime),
1895 SUBTEST(live_lrc_indirect_ctx_bb),
1896 };
1897
1898 if (!HAS_LOGICAL_RING_CONTEXTS(i915))
1899 return 0;
1900
1901 return intel_gt_live_subtests(tests, to_gt(i915));
1902 }
1903