1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2016 Intel Corporation
4 */
5
6 #include <linux/string_helpers.h>
7
8 #include <drm/drm_print.h>
9
10 #include "gem/i915_gem_context.h"
11 #include "gem/i915_gem_internal.h"
12 #include "gt/intel_gt_regs.h"
13
14 #include "i915_cmd_parser.h"
15 #include "i915_drv.h"
16 #include "intel_breadcrumbs.h"
17 #include "intel_context.h"
18 #include "intel_engine.h"
19 #include "intel_engine_pm.h"
20 #include "intel_engine_regs.h"
21 #include "intel_engine_user.h"
22 #include "intel_execlists_submission.h"
23 #include "intel_gt.h"
24 #include "intel_gt_requests.h"
25 #include "intel_gt_pm.h"
26 #include "intel_lrc.h"
27 #include "intel_lrc_reg.h"
28 #include "intel_reset.h"
29 #include "intel_ring.h"
30 #include "uc/intel_guc_submission.h"
31
32 /* Haswell does have the CXT_SIZE register however it does not appear to be
33 * valid. Now, docs explain in dwords what is in the context object. The full
34 * size is 70720 bytes, however, the power context and execlist context will
35 * never be saved (power context is stored elsewhere, and execlists don't work
36 * on HSW) - so the final size, including the extra state required for the
37 * Resource Streamer, is 66944 bytes, which rounds to 17 pages.
38 */
39 #define HSW_CXT_TOTAL_SIZE (17 * PAGE_SIZE)
40
41 #define DEFAULT_LR_CONTEXT_RENDER_SIZE (22 * PAGE_SIZE)
42 #define GEN8_LR_CONTEXT_RENDER_SIZE (20 * PAGE_SIZE)
43 #define GEN9_LR_CONTEXT_RENDER_SIZE (22 * PAGE_SIZE)
44 #define GEN11_LR_CONTEXT_RENDER_SIZE (14 * PAGE_SIZE)
45
46 #define GEN8_LR_CONTEXT_OTHER_SIZE ( 2 * PAGE_SIZE)
47
48 #define MAX_MMIO_BASES 3
49 struct engine_info {
50 u8 class;
51 u8 instance;
52 /* mmio bases table *must* be sorted in reverse graphics_ver order */
53 struct engine_mmio_base {
54 u32 graphics_ver : 8;
55 u32 base : 24;
56 } mmio_bases[MAX_MMIO_BASES];
57 };
58
59 static const struct engine_info intel_engines[] = {
60 [RCS0] = {
61 .class = RENDER_CLASS,
62 .instance = 0,
63 .mmio_bases = {
64 { .graphics_ver = 1, .base = RENDER_RING_BASE }
65 },
66 },
67 [BCS0] = {
68 .class = COPY_ENGINE_CLASS,
69 .instance = 0,
70 .mmio_bases = {
71 { .graphics_ver = 6, .base = BLT_RING_BASE }
72 },
73 },
74 [VCS0] = {
75 .class = VIDEO_DECODE_CLASS,
76 .instance = 0,
77 .mmio_bases = {
78 { .graphics_ver = 11, .base = GEN11_BSD_RING_BASE },
79 { .graphics_ver = 6, .base = GEN6_BSD_RING_BASE },
80 { .graphics_ver = 4, .base = BSD_RING_BASE }
81 },
82 },
83 [VCS1] = {
84 .class = VIDEO_DECODE_CLASS,
85 .instance = 1,
86 .mmio_bases = {
87 { .graphics_ver = 11, .base = GEN11_BSD2_RING_BASE },
88 { .graphics_ver = 8, .base = GEN8_BSD2_RING_BASE }
89 },
90 },
91 [VCS2] = {
92 .class = VIDEO_DECODE_CLASS,
93 .instance = 2,
94 .mmio_bases = {
95 { .graphics_ver = 11, .base = GEN11_BSD3_RING_BASE }
96 },
97 },
98 [VCS3] = {
99 .class = VIDEO_DECODE_CLASS,
100 .instance = 3,
101 .mmio_bases = {
102 { .graphics_ver = 11, .base = GEN11_BSD4_RING_BASE }
103 },
104 },
105 [VCS4] = {
106 .class = VIDEO_DECODE_CLASS,
107 .instance = 4,
108 .mmio_bases = {
109 { .graphics_ver = 12, .base = XEHP_BSD5_RING_BASE }
110 },
111 },
112 [VCS5] = {
113 .class = VIDEO_DECODE_CLASS,
114 .instance = 5,
115 .mmio_bases = {
116 { .graphics_ver = 12, .base = XEHP_BSD6_RING_BASE }
117 },
118 },
119 [VCS6] = {
120 .class = VIDEO_DECODE_CLASS,
121 .instance = 6,
122 .mmio_bases = {
123 { .graphics_ver = 12, .base = XEHP_BSD7_RING_BASE }
124 },
125 },
126 [VCS7] = {
127 .class = VIDEO_DECODE_CLASS,
128 .instance = 7,
129 .mmio_bases = {
130 { .graphics_ver = 12, .base = XEHP_BSD8_RING_BASE }
131 },
132 },
133 [VECS0] = {
134 .class = VIDEO_ENHANCEMENT_CLASS,
135 .instance = 0,
136 .mmio_bases = {
137 { .graphics_ver = 11, .base = GEN11_VEBOX_RING_BASE },
138 { .graphics_ver = 7, .base = VEBOX_RING_BASE }
139 },
140 },
141 [VECS1] = {
142 .class = VIDEO_ENHANCEMENT_CLASS,
143 .instance = 1,
144 .mmio_bases = {
145 { .graphics_ver = 11, .base = GEN11_VEBOX2_RING_BASE }
146 },
147 },
148 [VECS2] = {
149 .class = VIDEO_ENHANCEMENT_CLASS,
150 .instance = 2,
151 .mmio_bases = {
152 { .graphics_ver = 12, .base = XEHP_VEBOX3_RING_BASE }
153 },
154 },
155 [VECS3] = {
156 .class = VIDEO_ENHANCEMENT_CLASS,
157 .instance = 3,
158 .mmio_bases = {
159 { .graphics_ver = 12, .base = XEHP_VEBOX4_RING_BASE }
160 },
161 },
162 [CCS0] = {
163 .class = COMPUTE_CLASS,
164 .instance = 0,
165 .mmio_bases = {
166 { .graphics_ver = 12, .base = GEN12_COMPUTE0_RING_BASE }
167 }
168 },
169 [CCS1] = {
170 .class = COMPUTE_CLASS,
171 .instance = 1,
172 .mmio_bases = {
173 { .graphics_ver = 12, .base = GEN12_COMPUTE1_RING_BASE }
174 }
175 },
176 [CCS2] = {
177 .class = COMPUTE_CLASS,
178 .instance = 2,
179 .mmio_bases = {
180 { .graphics_ver = 12, .base = GEN12_COMPUTE2_RING_BASE }
181 }
182 },
183 [CCS3] = {
184 .class = COMPUTE_CLASS,
185 .instance = 3,
186 .mmio_bases = {
187 { .graphics_ver = 12, .base = GEN12_COMPUTE3_RING_BASE }
188 }
189 },
190 };
191
192 /**
193 * intel_engine_context_size() - return the size of the context for an engine
194 * @gt: the gt
195 * @class: engine class
196 *
197 * Each engine class may require a different amount of space for a context
198 * image.
199 *
200 * Return: size (in bytes) of an engine class specific context image
201 *
202 * Note: this size includes the HWSP, which is part of the context image
203 * in LRC mode, but does not include the "shared data page" used with
204 * GuC submission. The caller should account for this if using the GuC.
205 */
intel_engine_context_size(struct intel_gt * gt,u8 class)206 u32 intel_engine_context_size(struct intel_gt *gt, u8 class)
207 {
208 struct intel_uncore *uncore = gt->uncore;
209 u32 cxt_size;
210
211 BUILD_BUG_ON(I915_GTT_PAGE_SIZE != PAGE_SIZE);
212
213 switch (class) {
214 case COMPUTE_CLASS:
215 fallthrough;
216 case RENDER_CLASS:
217 switch (GRAPHICS_VER(gt->i915)) {
218 default:
219 MISSING_CASE(GRAPHICS_VER(gt->i915));
220 return DEFAULT_LR_CONTEXT_RENDER_SIZE;
221 case 12:
222 case 11:
223 return GEN11_LR_CONTEXT_RENDER_SIZE;
224 case 9:
225 return GEN9_LR_CONTEXT_RENDER_SIZE;
226 case 8:
227 return GEN8_LR_CONTEXT_RENDER_SIZE;
228 case 7:
229 if (IS_HASWELL(gt->i915))
230 return HSW_CXT_TOTAL_SIZE;
231
232 cxt_size = intel_uncore_read(uncore, GEN7_CXT_SIZE);
233 return round_up(GEN7_CXT_TOTAL_SIZE(cxt_size) * 64,
234 PAGE_SIZE);
235 case 6:
236 cxt_size = intel_uncore_read(uncore, CXT_SIZE);
237 return round_up(GEN6_CXT_TOTAL_SIZE(cxt_size) * 64,
238 PAGE_SIZE);
239 case 5:
240 case 4:
241 /*
242 * There is a discrepancy here between the size reported
243 * by the register and the size of the context layout
244 * in the docs. Both are described as authorative!
245 *
246 * The discrepancy is on the order of a few cachelines,
247 * but the total is under one page (4k), which is our
248 * minimum allocation anyway so it should all come
249 * out in the wash.
250 */
251 cxt_size = intel_uncore_read(uncore, CXT_SIZE) + 1;
252 drm_dbg(>->i915->drm,
253 "graphics_ver = %d CXT_SIZE = %d bytes [0x%08x]\n",
254 GRAPHICS_VER(gt->i915), cxt_size * 64,
255 cxt_size - 1);
256 return round_up(cxt_size * 64, PAGE_SIZE);
257 case 3:
258 case 2:
259 /* For the special day when i810 gets merged. */
260 case 1:
261 return 0;
262 }
263 break;
264 default:
265 MISSING_CASE(class);
266 fallthrough;
267 case VIDEO_DECODE_CLASS:
268 case VIDEO_ENHANCEMENT_CLASS:
269 case COPY_ENGINE_CLASS:
270 if (GRAPHICS_VER(gt->i915) < 8)
271 return 0;
272 return GEN8_LR_CONTEXT_OTHER_SIZE;
273 }
274 }
275
__engine_mmio_base(struct drm_i915_private * i915,const struct engine_mmio_base * bases)276 static u32 __engine_mmio_base(struct drm_i915_private *i915,
277 const struct engine_mmio_base *bases)
278 {
279 int i;
280
281 for (i = 0; i < MAX_MMIO_BASES; i++)
282 if (GRAPHICS_VER(i915) >= bases[i].graphics_ver)
283 break;
284
285 GEM_BUG_ON(i == MAX_MMIO_BASES);
286 GEM_BUG_ON(!bases[i].base);
287
288 return bases[i].base;
289 }
290
__sprint_engine_name(struct intel_engine_cs * engine)291 static void __sprint_engine_name(struct intel_engine_cs *engine)
292 {
293 /*
294 * Before we know what the uABI name for this engine will be,
295 * we still would like to keep track of this engine in the debug logs.
296 * We throw in a ' here as a reminder that this isn't its final name.
297 */
298 GEM_WARN_ON(snprintf(engine->name, sizeof(engine->name), "%s'%u",
299 intel_engine_class_repr(engine->class),
300 engine->instance) >= sizeof(engine->name));
301 }
302
intel_engine_set_hwsp_writemask(struct intel_engine_cs * engine,u32 mask)303 void intel_engine_set_hwsp_writemask(struct intel_engine_cs *engine, u32 mask)
304 {
305 /*
306 * Though they added more rings on g4x/ilk, they did not add
307 * per-engine HWSTAM until gen6.
308 */
309 if (GRAPHICS_VER(engine->i915) < 6 && engine->class != RENDER_CLASS)
310 return;
311
312 if (GRAPHICS_VER(engine->i915) >= 3)
313 ENGINE_WRITE(engine, RING_HWSTAM, mask);
314 else
315 ENGINE_WRITE16(engine, RING_HWSTAM, mask);
316 }
317
intel_engine_sanitize_mmio(struct intel_engine_cs * engine)318 static void intel_engine_sanitize_mmio(struct intel_engine_cs *engine)
319 {
320 /* Mask off all writes into the unknown HWSP */
321 intel_engine_set_hwsp_writemask(engine, ~0u);
322 }
323
nop_irq_handler(struct intel_engine_cs * engine,u16 iir)324 static void nop_irq_handler(struct intel_engine_cs *engine, u16 iir)
325 {
326 GEM_DEBUG_WARN_ON(iir);
327 }
328
get_reset_domain(u8 ver,enum intel_engine_id id)329 static u32 get_reset_domain(u8 ver, enum intel_engine_id id)
330 {
331 u32 reset_domain;
332
333 if (ver >= 11) {
334 static const u32 engine_reset_domains[] = {
335 [RCS0] = GEN11_GRDOM_RENDER,
336 [BCS0] = GEN11_GRDOM_BLT,
337 [VCS0] = GEN11_GRDOM_MEDIA,
338 [VCS1] = GEN11_GRDOM_MEDIA2,
339 [VCS2] = GEN11_GRDOM_MEDIA3,
340 [VCS3] = GEN11_GRDOM_MEDIA4,
341 [VCS4] = GEN11_GRDOM_MEDIA5,
342 [VCS5] = GEN11_GRDOM_MEDIA6,
343 [VCS6] = GEN11_GRDOM_MEDIA7,
344 [VCS7] = GEN11_GRDOM_MEDIA8,
345 [VECS0] = GEN11_GRDOM_VECS,
346 [VECS1] = GEN11_GRDOM_VECS2,
347 [VECS2] = GEN11_GRDOM_VECS3,
348 [VECS3] = GEN11_GRDOM_VECS4,
349 [CCS0] = GEN11_GRDOM_RENDER,
350 [CCS1] = GEN11_GRDOM_RENDER,
351 [CCS2] = GEN11_GRDOM_RENDER,
352 [CCS3] = GEN11_GRDOM_RENDER,
353 };
354 GEM_BUG_ON(id >= ARRAY_SIZE(engine_reset_domains) ||
355 !engine_reset_domains[id]);
356 reset_domain = engine_reset_domains[id];
357 } else {
358 static const u32 engine_reset_domains[] = {
359 [RCS0] = GEN6_GRDOM_RENDER,
360 [BCS0] = GEN6_GRDOM_BLT,
361 [VCS0] = GEN6_GRDOM_MEDIA,
362 [VCS1] = GEN8_GRDOM_MEDIA2,
363 [VECS0] = GEN6_GRDOM_VECS,
364 };
365 GEM_BUG_ON(id >= ARRAY_SIZE(engine_reset_domains) ||
366 !engine_reset_domains[id]);
367 reset_domain = engine_reset_domains[id];
368 }
369
370 return reset_domain;
371 }
372
intel_engine_setup(struct intel_gt * gt,enum intel_engine_id id,u8 logical_instance)373 static int intel_engine_setup(struct intel_gt *gt, enum intel_engine_id id,
374 u8 logical_instance)
375 {
376 const struct engine_info *info = &intel_engines[id];
377 struct drm_i915_private *i915 = gt->i915;
378 struct intel_engine_cs *engine;
379 u8 guc_class;
380
381 BUILD_BUG_ON(MAX_ENGINE_CLASS >= BIT(GEN11_ENGINE_CLASS_WIDTH));
382 BUILD_BUG_ON(MAX_ENGINE_INSTANCE >= BIT(GEN11_ENGINE_INSTANCE_WIDTH));
383 BUILD_BUG_ON(I915_MAX_VCS > (MAX_ENGINE_INSTANCE + 1));
384 BUILD_BUG_ON(I915_MAX_VECS > (MAX_ENGINE_INSTANCE + 1));
385
386 if (GEM_DEBUG_WARN_ON(id >= ARRAY_SIZE(gt->engine)))
387 return -EINVAL;
388
389 if (GEM_DEBUG_WARN_ON(info->class > MAX_ENGINE_CLASS))
390 return -EINVAL;
391
392 if (GEM_DEBUG_WARN_ON(info->instance > MAX_ENGINE_INSTANCE))
393 return -EINVAL;
394
395 if (GEM_DEBUG_WARN_ON(gt->engine_class[info->class][info->instance]))
396 return -EINVAL;
397
398 engine = kzalloc(sizeof(*engine), GFP_KERNEL);
399 if (!engine)
400 return -ENOMEM;
401
402 BUILD_BUG_ON(BITS_PER_TYPE(engine->mask) < I915_NUM_ENGINES);
403
404 INIT_LIST_HEAD(&engine->pinned_contexts_list);
405 engine->id = id;
406 engine->legacy_idx = INVALID_ENGINE;
407 engine->mask = BIT(id);
408 engine->reset_domain = get_reset_domain(GRAPHICS_VER(gt->i915),
409 id);
410 engine->i915 = i915;
411 engine->gt = gt;
412 engine->uncore = gt->uncore;
413 guc_class = engine_class_to_guc_class(info->class);
414 engine->guc_id = MAKE_GUC_ID(guc_class, info->instance);
415 engine->mmio_base = __engine_mmio_base(i915, info->mmio_bases);
416
417 engine->irq_handler = nop_irq_handler;
418
419 engine->class = info->class;
420 engine->instance = info->instance;
421 engine->logical_mask = BIT(logical_instance);
422 __sprint_engine_name(engine);
423
424 engine->props.heartbeat_interval_ms =
425 CONFIG_DRM_I915_HEARTBEAT_INTERVAL;
426 engine->props.max_busywait_duration_ns =
427 CONFIG_DRM_I915_MAX_REQUEST_BUSYWAIT;
428 engine->props.preempt_timeout_ms =
429 CONFIG_DRM_I915_PREEMPT_TIMEOUT;
430 engine->props.stop_timeout_ms =
431 CONFIG_DRM_I915_STOP_TIMEOUT;
432 engine->props.timeslice_duration_ms =
433 CONFIG_DRM_I915_TIMESLICE_DURATION;
434
435 /* Override to uninterruptible for OpenCL workloads. */
436 if (GRAPHICS_VER(i915) == 12 && engine->class == RENDER_CLASS)
437 engine->props.preempt_timeout_ms = 0;
438
439 if ((engine->class == COMPUTE_CLASS && !RCS_MASK(engine->gt) &&
440 __ffs(CCS_MASK(engine->gt)) == engine->instance) ||
441 engine->class == RENDER_CLASS)
442 engine->flags |= I915_ENGINE_FIRST_RENDER_COMPUTE;
443
444 /* features common between engines sharing EUs */
445 if (engine->class == RENDER_CLASS || engine->class == COMPUTE_CLASS) {
446 engine->flags |= I915_ENGINE_HAS_RCS_REG_STATE;
447 engine->flags |= I915_ENGINE_HAS_EU_PRIORITY;
448 }
449
450 engine->defaults = engine->props; /* never to change again */
451
452 engine->context_size = intel_engine_context_size(gt, engine->class);
453 if (WARN_ON(engine->context_size > BIT(20)))
454 engine->context_size = 0;
455 if (engine->context_size)
456 DRIVER_CAPS(i915)->has_logical_contexts = true;
457
458 ewma__engine_latency_init(&engine->latency);
459 seqcount_init(&engine->stats.execlists.lock);
460
461 ATOMIC_INIT_NOTIFIER_HEAD(&engine->context_status_notifier);
462
463 /* Scrub mmio state on takeover */
464 intel_engine_sanitize_mmio(engine);
465
466 gt->engine_class[info->class][info->instance] = engine;
467 gt->engine[id] = engine;
468
469 return 0;
470 }
471
__setup_engine_capabilities(struct intel_engine_cs * engine)472 static void __setup_engine_capabilities(struct intel_engine_cs *engine)
473 {
474 struct drm_i915_private *i915 = engine->i915;
475
476 if (engine->class == VIDEO_DECODE_CLASS) {
477 /*
478 * HEVC support is present on first engine instance
479 * before Gen11 and on all instances afterwards.
480 */
481 if (GRAPHICS_VER(i915) >= 11 ||
482 (GRAPHICS_VER(i915) >= 9 && engine->instance == 0))
483 engine->uabi_capabilities |=
484 I915_VIDEO_CLASS_CAPABILITY_HEVC;
485
486 /*
487 * SFC block is present only on even logical engine
488 * instances.
489 */
490 if ((GRAPHICS_VER(i915) >= 11 &&
491 (engine->gt->info.vdbox_sfc_access &
492 BIT(engine->instance))) ||
493 (GRAPHICS_VER(i915) >= 9 && engine->instance == 0))
494 engine->uabi_capabilities |=
495 I915_VIDEO_AND_ENHANCE_CLASS_CAPABILITY_SFC;
496 } else if (engine->class == VIDEO_ENHANCEMENT_CLASS) {
497 if (GRAPHICS_VER(i915) >= 9 &&
498 engine->gt->info.sfc_mask & BIT(engine->instance))
499 engine->uabi_capabilities |=
500 I915_VIDEO_AND_ENHANCE_CLASS_CAPABILITY_SFC;
501 }
502 }
503
intel_setup_engine_capabilities(struct intel_gt * gt)504 static void intel_setup_engine_capabilities(struct intel_gt *gt)
505 {
506 struct intel_engine_cs *engine;
507 enum intel_engine_id id;
508
509 for_each_engine(engine, gt, id)
510 __setup_engine_capabilities(engine);
511 }
512
513 /**
514 * intel_engines_release() - free the resources allocated for Command Streamers
515 * @gt: pointer to struct intel_gt
516 */
intel_engines_release(struct intel_gt * gt)517 void intel_engines_release(struct intel_gt *gt)
518 {
519 struct intel_engine_cs *engine;
520 enum intel_engine_id id;
521
522 /*
523 * Before we release the resources held by engine, we must be certain
524 * that the HW is no longer accessing them -- having the GPU scribble
525 * to or read from a page being used for something else causes no end
526 * of fun.
527 *
528 * The GPU should be reset by this point, but assume the worst just
529 * in case we aborted before completely initialising the engines.
530 */
531 GEM_BUG_ON(intel_gt_pm_is_awake(gt));
532 if (!INTEL_INFO(gt->i915)->gpu_reset_clobbers_display)
533 __intel_gt_reset(gt, ALL_ENGINES);
534
535 /* Decouple the backend; but keep the layout for late GPU resets */
536 for_each_engine(engine, gt, id) {
537 if (!engine->release)
538 continue;
539
540 intel_wakeref_wait_for_idle(&engine->wakeref);
541 GEM_BUG_ON(intel_engine_pm_is_awake(engine));
542
543 engine->release(engine);
544 engine->release = NULL;
545
546 memset(&engine->reset, 0, sizeof(engine->reset));
547 }
548 }
549
intel_engine_free_request_pool(struct intel_engine_cs * engine)550 void intel_engine_free_request_pool(struct intel_engine_cs *engine)
551 {
552 if (!engine->request_pool)
553 return;
554
555 kmem_cache_free(i915_request_slab_cache(), engine->request_pool);
556 }
557
intel_engines_free(struct intel_gt * gt)558 void intel_engines_free(struct intel_gt *gt)
559 {
560 struct intel_engine_cs *engine;
561 enum intel_engine_id id;
562
563 /* Free the requests! dma-resv keeps fences around for an eternity */
564 rcu_barrier();
565
566 for_each_engine(engine, gt, id) {
567 intel_engine_free_request_pool(engine);
568 kfree(engine);
569 gt->engine[id] = NULL;
570 }
571 }
572
573 static
gen11_vdbox_has_sfc(struct intel_gt * gt,unsigned int physical_vdbox,unsigned int logical_vdbox,u16 vdbox_mask)574 bool gen11_vdbox_has_sfc(struct intel_gt *gt,
575 unsigned int physical_vdbox,
576 unsigned int logical_vdbox, u16 vdbox_mask)
577 {
578 struct drm_i915_private *i915 = gt->i915;
579
580 /*
581 * In Gen11, only even numbered logical VDBOXes are hooked
582 * up to an SFC (Scaler & Format Converter) unit.
583 * In Gen12, Even numbered physical instance always are connected
584 * to an SFC. Odd numbered physical instances have SFC only if
585 * previous even instance is fused off.
586 *
587 * Starting with Xe_HP, there's also a dedicated SFC_ENABLE field
588 * in the fuse register that tells us whether a specific SFC is present.
589 */
590 if ((gt->info.sfc_mask & BIT(physical_vdbox / 2)) == 0)
591 return false;
592 else if (GRAPHICS_VER(i915) == 12)
593 return (physical_vdbox % 2 == 0) ||
594 !(BIT(physical_vdbox - 1) & vdbox_mask);
595 else if (GRAPHICS_VER(i915) == 11)
596 return logical_vdbox % 2 == 0;
597
598 MISSING_CASE(GRAPHICS_VER(i915));
599 return false;
600 }
601
engine_mask_apply_compute_fuses(struct intel_gt * gt)602 static void engine_mask_apply_compute_fuses(struct intel_gt *gt)
603 {
604 struct drm_i915_private *i915 = gt->i915;
605 struct intel_gt_info *info = >->info;
606 int ss_per_ccs = info->sseu.max_subslices / I915_MAX_CCS;
607 unsigned long ccs_mask;
608 unsigned int i;
609
610 if (GRAPHICS_VER_FULL(i915) < IP_VER(12, 50))
611 return;
612
613 ccs_mask = intel_slicemask_from_dssmask(intel_sseu_get_compute_subslices(&info->sseu),
614 ss_per_ccs);
615 /*
616 * If all DSS in a quadrant are fused off, the corresponding CCS
617 * engine is not available for use.
618 */
619 for_each_clear_bit(i, &ccs_mask, I915_MAX_CCS) {
620 info->engine_mask &= ~BIT(_CCS(i));
621 drm_dbg(&i915->drm, "ccs%u fused off\n", i);
622 }
623 }
624
625 /*
626 * Determine which engines are fused off in our particular hardware.
627 * Note that we have a catch-22 situation where we need to be able to access
628 * the blitter forcewake domain to read the engine fuses, but at the same time
629 * we need to know which engines are available on the system to know which
630 * forcewake domains are present. We solve this by intializing the forcewake
631 * domains based on the full engine mask in the platform capabilities before
632 * calling this function and pruning the domains for fused-off engines
633 * afterwards.
634 */
init_engine_mask(struct intel_gt * gt)635 static intel_engine_mask_t init_engine_mask(struct intel_gt *gt)
636 {
637 struct drm_i915_private *i915 = gt->i915;
638 struct intel_gt_info *info = >->info;
639 struct intel_uncore *uncore = gt->uncore;
640 unsigned int logical_vdbox = 0;
641 unsigned int i;
642 u32 media_fuse, fuse1;
643 u16 vdbox_mask;
644 u16 vebox_mask;
645
646 info->engine_mask = INTEL_INFO(i915)->platform_engine_mask;
647
648 if (GRAPHICS_VER(i915) < 11)
649 return info->engine_mask;
650
651 /*
652 * On newer platforms the fusing register is called 'enable' and has
653 * enable semantics, while on older platforms it is called 'disable'
654 * and bits have disable semantices.
655 */
656 media_fuse = intel_uncore_read(uncore, GEN11_GT_VEBOX_VDBOX_DISABLE);
657 if (GRAPHICS_VER_FULL(i915) < IP_VER(12, 50))
658 media_fuse = ~media_fuse;
659
660 vdbox_mask = media_fuse & GEN11_GT_VDBOX_DISABLE_MASK;
661 vebox_mask = (media_fuse & GEN11_GT_VEBOX_DISABLE_MASK) >>
662 GEN11_GT_VEBOX_DISABLE_SHIFT;
663
664 if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50)) {
665 fuse1 = intel_uncore_read(uncore, HSW_PAVP_FUSE1);
666 gt->info.sfc_mask = REG_FIELD_GET(XEHP_SFC_ENABLE_MASK, fuse1);
667 } else {
668 gt->info.sfc_mask = ~0;
669 }
670
671 for (i = 0; i < I915_MAX_VCS; i++) {
672 if (!HAS_ENGINE(gt, _VCS(i))) {
673 vdbox_mask &= ~BIT(i);
674 continue;
675 }
676
677 if (!(BIT(i) & vdbox_mask)) {
678 info->engine_mask &= ~BIT(_VCS(i));
679 drm_dbg(&i915->drm, "vcs%u fused off\n", i);
680 continue;
681 }
682
683 if (gen11_vdbox_has_sfc(gt, i, logical_vdbox, vdbox_mask))
684 gt->info.vdbox_sfc_access |= BIT(i);
685 logical_vdbox++;
686 }
687 drm_dbg(&i915->drm, "vdbox enable: %04x, instances: %04lx\n",
688 vdbox_mask, VDBOX_MASK(gt));
689 GEM_BUG_ON(vdbox_mask != VDBOX_MASK(gt));
690
691 for (i = 0; i < I915_MAX_VECS; i++) {
692 if (!HAS_ENGINE(gt, _VECS(i))) {
693 vebox_mask &= ~BIT(i);
694 continue;
695 }
696
697 if (!(BIT(i) & vebox_mask)) {
698 info->engine_mask &= ~BIT(_VECS(i));
699 drm_dbg(&i915->drm, "vecs%u fused off\n", i);
700 }
701 }
702 drm_dbg(&i915->drm, "vebox enable: %04x, instances: %04lx\n",
703 vebox_mask, VEBOX_MASK(gt));
704 GEM_BUG_ON(vebox_mask != VEBOX_MASK(gt));
705
706 engine_mask_apply_compute_fuses(gt);
707
708 return info->engine_mask;
709 }
710
populate_logical_ids(struct intel_gt * gt,u8 * logical_ids,u8 class,const u8 * map,u8 num_instances)711 static void populate_logical_ids(struct intel_gt *gt, u8 *logical_ids,
712 u8 class, const u8 *map, u8 num_instances)
713 {
714 int i, j;
715 u8 current_logical_id = 0;
716
717 for (j = 0; j < num_instances; ++j) {
718 for (i = 0; i < ARRAY_SIZE(intel_engines); ++i) {
719 if (!HAS_ENGINE(gt, i) ||
720 intel_engines[i].class != class)
721 continue;
722
723 if (intel_engines[i].instance == map[j]) {
724 logical_ids[intel_engines[i].instance] =
725 current_logical_id++;
726 break;
727 }
728 }
729 }
730 }
731
setup_logical_ids(struct intel_gt * gt,u8 * logical_ids,u8 class)732 static void setup_logical_ids(struct intel_gt *gt, u8 *logical_ids, u8 class)
733 {
734 /*
735 * Logical to physical mapping is needed for proper support
736 * to split-frame feature.
737 */
738 if (MEDIA_VER(gt->i915) >= 11 && class == VIDEO_DECODE_CLASS) {
739 const u8 map[] = { 0, 2, 4, 6, 1, 3, 5, 7 };
740
741 populate_logical_ids(gt, logical_ids, class,
742 map, ARRAY_SIZE(map));
743 } else {
744 int i;
745 u8 map[MAX_ENGINE_INSTANCE + 1];
746
747 for (i = 0; i < MAX_ENGINE_INSTANCE + 1; ++i)
748 map[i] = i;
749 populate_logical_ids(gt, logical_ids, class,
750 map, ARRAY_SIZE(map));
751 }
752 }
753
754 /**
755 * intel_engines_init_mmio() - allocate and prepare the Engine Command Streamers
756 * @gt: pointer to struct intel_gt
757 *
758 * Return: non-zero if the initialization failed.
759 */
intel_engines_init_mmio(struct intel_gt * gt)760 int intel_engines_init_mmio(struct intel_gt *gt)
761 {
762 struct drm_i915_private *i915 = gt->i915;
763 const unsigned int engine_mask = init_engine_mask(gt);
764 unsigned int mask = 0;
765 unsigned int i, class;
766 u8 logical_ids[MAX_ENGINE_INSTANCE + 1];
767 int err;
768
769 drm_WARN_ON(&i915->drm, engine_mask == 0);
770 drm_WARN_ON(&i915->drm, engine_mask &
771 GENMASK(BITS_PER_TYPE(mask) - 1, I915_NUM_ENGINES));
772
773 if (i915_inject_probe_failure(i915))
774 return -ENODEV;
775
776 for (class = 0; class < MAX_ENGINE_CLASS + 1; ++class) {
777 setup_logical_ids(gt, logical_ids, class);
778
779 for (i = 0; i < ARRAY_SIZE(intel_engines); ++i) {
780 u8 instance = intel_engines[i].instance;
781
782 if (intel_engines[i].class != class ||
783 !HAS_ENGINE(gt, i))
784 continue;
785
786 err = intel_engine_setup(gt, i,
787 logical_ids[instance]);
788 if (err)
789 goto cleanup;
790
791 mask |= BIT(i);
792 }
793 }
794
795 /*
796 * Catch failures to update intel_engines table when the new engines
797 * are added to the driver by a warning and disabling the forgotten
798 * engines.
799 */
800 if (drm_WARN_ON(&i915->drm, mask != engine_mask))
801 gt->info.engine_mask = mask;
802
803 gt->info.num_engines = hweight32(mask);
804
805 intel_gt_check_and_clear_faults(gt);
806
807 intel_setup_engine_capabilities(gt);
808
809 intel_uncore_prune_engine_fw_domains(gt->uncore, gt);
810
811 return 0;
812
813 cleanup:
814 intel_engines_free(gt);
815 return err;
816 }
817
intel_engine_init_execlists(struct intel_engine_cs * engine)818 void intel_engine_init_execlists(struct intel_engine_cs *engine)
819 {
820 struct intel_engine_execlists * const execlists = &engine->execlists;
821
822 execlists->port_mask = 1;
823 GEM_BUG_ON(!is_power_of_2(execlists_num_ports(execlists)));
824 GEM_BUG_ON(execlists_num_ports(execlists) > EXECLIST_MAX_PORTS);
825
826 memset(execlists->pending, 0, sizeof(execlists->pending));
827 execlists->active =
828 memset(execlists->inflight, 0, sizeof(execlists->inflight));
829 }
830
cleanup_status_page(struct intel_engine_cs * engine)831 static void cleanup_status_page(struct intel_engine_cs *engine)
832 {
833 struct i915_vma *vma;
834
835 /* Prevent writes into HWSP after returning the page to the system */
836 intel_engine_set_hwsp_writemask(engine, ~0u);
837
838 vma = fetch_and_zero(&engine->status_page.vma);
839 if (!vma)
840 return;
841
842 if (!HWS_NEEDS_PHYSICAL(engine->i915))
843 i915_vma_unpin(vma);
844
845 i915_gem_object_unpin_map(vma->obj);
846 i915_gem_object_put(vma->obj);
847 }
848
pin_ggtt_status_page(struct intel_engine_cs * engine,struct i915_gem_ww_ctx * ww,struct i915_vma * vma)849 static int pin_ggtt_status_page(struct intel_engine_cs *engine,
850 struct i915_gem_ww_ctx *ww,
851 struct i915_vma *vma)
852 {
853 unsigned int flags;
854
855 if (!HAS_LLC(engine->i915) && i915_ggtt_has_aperture(engine->gt->ggtt))
856 /*
857 * On g33, we cannot place HWS above 256MiB, so
858 * restrict its pinning to the low mappable arena.
859 * Though this restriction is not documented for
860 * gen4, gen5, or byt, they also behave similarly
861 * and hang if the HWS is placed at the top of the
862 * GTT. To generalise, it appears that all !llc
863 * platforms have issues with us placing the HWS
864 * above the mappable region (even though we never
865 * actually map it).
866 */
867 flags = PIN_MAPPABLE;
868 else
869 flags = PIN_HIGH;
870
871 return i915_ggtt_pin(vma, ww, 0, flags);
872 }
873
init_status_page(struct intel_engine_cs * engine)874 static int init_status_page(struct intel_engine_cs *engine)
875 {
876 struct drm_i915_gem_object *obj;
877 struct i915_gem_ww_ctx ww;
878 struct i915_vma *vma;
879 void *vaddr;
880 int ret;
881
882 INIT_LIST_HEAD(&engine->status_page.timelines);
883
884 /*
885 * Though the HWS register does support 36bit addresses, historically
886 * we have had hangs and corruption reported due to wild writes if
887 * the HWS is placed above 4G. We only allow objects to be allocated
888 * in GFP_DMA32 for i965, and no earlier physical address users had
889 * access to more than 4G.
890 */
891 obj = i915_gem_object_create_internal(engine->i915, PAGE_SIZE);
892 if (IS_ERR(obj)) {
893 drm_err(&engine->i915->drm,
894 "Failed to allocate status page\n");
895 return PTR_ERR(obj);
896 }
897
898 i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
899
900 vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL);
901 if (IS_ERR(vma)) {
902 ret = PTR_ERR(vma);
903 goto err_put;
904 }
905
906 i915_gem_ww_ctx_init(&ww, true);
907 retry:
908 ret = i915_gem_object_lock(obj, &ww);
909 if (!ret && !HWS_NEEDS_PHYSICAL(engine->i915))
910 ret = pin_ggtt_status_page(engine, &ww, vma);
911 if (ret)
912 goto err;
913
914 vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
915 if (IS_ERR(vaddr)) {
916 ret = PTR_ERR(vaddr);
917 goto err_unpin;
918 }
919
920 engine->status_page.addr = memset(vaddr, 0, PAGE_SIZE);
921 engine->status_page.vma = vma;
922
923 err_unpin:
924 if (ret)
925 i915_vma_unpin(vma);
926 err:
927 if (ret == -EDEADLK) {
928 ret = i915_gem_ww_ctx_backoff(&ww);
929 if (!ret)
930 goto retry;
931 }
932 i915_gem_ww_ctx_fini(&ww);
933 err_put:
934 if (ret)
935 i915_gem_object_put(obj);
936 return ret;
937 }
938
engine_setup_common(struct intel_engine_cs * engine)939 static int engine_setup_common(struct intel_engine_cs *engine)
940 {
941 int err;
942
943 init_llist_head(&engine->barrier_tasks);
944
945 err = init_status_page(engine);
946 if (err)
947 return err;
948
949 engine->breadcrumbs = intel_breadcrumbs_create(engine);
950 if (!engine->breadcrumbs) {
951 err = -ENOMEM;
952 goto err_status;
953 }
954
955 engine->sched_engine = i915_sched_engine_create(ENGINE_PHYSICAL);
956 if (!engine->sched_engine) {
957 err = -ENOMEM;
958 goto err_sched_engine;
959 }
960 engine->sched_engine->private_data = engine;
961
962 err = intel_engine_init_cmd_parser(engine);
963 if (err)
964 goto err_cmd_parser;
965
966 intel_engine_init_execlists(engine);
967 intel_engine_init__pm(engine);
968 intel_engine_init_retire(engine);
969
970 /* Use the whole device by default */
971 engine->sseu =
972 intel_sseu_from_device_info(&engine->gt->info.sseu);
973
974 intel_engine_init_workarounds(engine);
975 intel_engine_init_whitelist(engine);
976 intel_engine_init_ctx_wa(engine);
977
978 if (GRAPHICS_VER(engine->i915) >= 12)
979 engine->flags |= I915_ENGINE_HAS_RELATIVE_MMIO;
980
981 return 0;
982
983 err_cmd_parser:
984 i915_sched_engine_put(engine->sched_engine);
985 err_sched_engine:
986 intel_breadcrumbs_put(engine->breadcrumbs);
987 err_status:
988 cleanup_status_page(engine);
989 return err;
990 }
991
992 struct measure_breadcrumb {
993 struct i915_request rq;
994 struct intel_ring ring;
995 u32 cs[2048];
996 };
997
measure_breadcrumb_dw(struct intel_context * ce)998 static int measure_breadcrumb_dw(struct intel_context *ce)
999 {
1000 struct intel_engine_cs *engine = ce->engine;
1001 struct measure_breadcrumb *frame;
1002 int dw;
1003
1004 GEM_BUG_ON(!engine->gt->scratch);
1005
1006 frame = kzalloc(sizeof(*frame), GFP_KERNEL);
1007 if (!frame)
1008 return -ENOMEM;
1009
1010 frame->rq.engine = engine;
1011 frame->rq.context = ce;
1012 rcu_assign_pointer(frame->rq.timeline, ce->timeline);
1013 frame->rq.hwsp_seqno = ce->timeline->hwsp_seqno;
1014
1015 frame->ring.vaddr = frame->cs;
1016 frame->ring.size = sizeof(frame->cs);
1017 frame->ring.wrap =
1018 BITS_PER_TYPE(frame->ring.size) - ilog2(frame->ring.size);
1019 frame->ring.effective_size = frame->ring.size;
1020 intel_ring_update_space(&frame->ring);
1021 frame->rq.ring = &frame->ring;
1022
1023 mutex_lock(&ce->timeline->mutex);
1024 spin_lock_irq(&engine->sched_engine->lock);
1025
1026 dw = engine->emit_fini_breadcrumb(&frame->rq, frame->cs) - frame->cs;
1027
1028 spin_unlock_irq(&engine->sched_engine->lock);
1029 mutex_unlock(&ce->timeline->mutex);
1030
1031 GEM_BUG_ON(dw & 1); /* RING_TAIL must be qword aligned */
1032
1033 kfree(frame);
1034 return dw;
1035 }
1036
1037 struct intel_context *
intel_engine_create_pinned_context(struct intel_engine_cs * engine,struct i915_address_space * vm,unsigned int ring_size,unsigned int hwsp,struct lock_class_key * key,const char * name)1038 intel_engine_create_pinned_context(struct intel_engine_cs *engine,
1039 struct i915_address_space *vm,
1040 unsigned int ring_size,
1041 unsigned int hwsp,
1042 struct lock_class_key *key,
1043 const char *name)
1044 {
1045 struct intel_context *ce;
1046 int err;
1047
1048 ce = intel_context_create(engine);
1049 if (IS_ERR(ce))
1050 return ce;
1051
1052 __set_bit(CONTEXT_BARRIER_BIT, &ce->flags);
1053 ce->timeline = page_pack_bits(NULL, hwsp);
1054 ce->ring = NULL;
1055 ce->ring_size = ring_size;
1056
1057 i915_vm_put(ce->vm);
1058 ce->vm = i915_vm_get(vm);
1059
1060 err = intel_context_pin(ce); /* perma-pin so it is always available */
1061 if (err) {
1062 intel_context_put(ce);
1063 return ERR_PTR(err);
1064 }
1065
1066 list_add_tail(&ce->pinned_contexts_link, &engine->pinned_contexts_list);
1067
1068 /*
1069 * Give our perma-pinned kernel timelines a separate lockdep class,
1070 * so that we can use them from within the normal user timelines
1071 * should we need to inject GPU operations during their request
1072 * construction.
1073 */
1074 lockdep_set_class_and_name(&ce->timeline->mutex, key, name);
1075
1076 return ce;
1077 }
1078
intel_engine_destroy_pinned_context(struct intel_context * ce)1079 void intel_engine_destroy_pinned_context(struct intel_context *ce)
1080 {
1081 struct intel_engine_cs *engine = ce->engine;
1082 struct i915_vma *hwsp = engine->status_page.vma;
1083
1084 GEM_BUG_ON(ce->timeline->hwsp_ggtt != hwsp);
1085
1086 mutex_lock(&hwsp->vm->mutex);
1087 list_del(&ce->timeline->engine_link);
1088 mutex_unlock(&hwsp->vm->mutex);
1089
1090 list_del(&ce->pinned_contexts_link);
1091 intel_context_unpin(ce);
1092 intel_context_put(ce);
1093 }
1094
1095 static struct intel_context *
create_kernel_context(struct intel_engine_cs * engine)1096 create_kernel_context(struct intel_engine_cs *engine)
1097 {
1098 static struct lock_class_key kernel;
1099
1100 return intel_engine_create_pinned_context(engine, engine->gt->vm, SZ_4K,
1101 I915_GEM_HWS_SEQNO_ADDR,
1102 &kernel, "kernel_context");
1103 }
1104
1105 /**
1106 * intel_engines_init_common - initialize cengine state which might require hw access
1107 * @engine: Engine to initialize.
1108 *
1109 * Initializes @engine@ structure members shared between legacy and execlists
1110 * submission modes which do require hardware access.
1111 *
1112 * Typcally done at later stages of submission mode specific engine setup.
1113 *
1114 * Returns zero on success or an error code on failure.
1115 */
engine_init_common(struct intel_engine_cs * engine)1116 static int engine_init_common(struct intel_engine_cs *engine)
1117 {
1118 struct intel_context *ce;
1119 int ret;
1120
1121 engine->set_default_submission(engine);
1122
1123 /*
1124 * We may need to do things with the shrinker which
1125 * require us to immediately switch back to the default
1126 * context. This can cause a problem as pinning the
1127 * default context also requires GTT space which may not
1128 * be available. To avoid this we always pin the default
1129 * context.
1130 */
1131 ce = create_kernel_context(engine);
1132 if (IS_ERR(ce))
1133 return PTR_ERR(ce);
1134
1135 ret = measure_breadcrumb_dw(ce);
1136 if (ret < 0)
1137 goto err_context;
1138
1139 engine->emit_fini_breadcrumb_dw = ret;
1140 engine->kernel_context = ce;
1141
1142 return 0;
1143
1144 err_context:
1145 intel_engine_destroy_pinned_context(ce);
1146 return ret;
1147 }
1148
intel_engines_init(struct intel_gt * gt)1149 int intel_engines_init(struct intel_gt *gt)
1150 {
1151 int (*setup)(struct intel_engine_cs *engine);
1152 struct intel_engine_cs *engine;
1153 enum intel_engine_id id;
1154 int err;
1155
1156 if (intel_uc_uses_guc_submission(>->uc)) {
1157 gt->submission_method = INTEL_SUBMISSION_GUC;
1158 setup = intel_guc_submission_setup;
1159 } else if (HAS_EXECLISTS(gt->i915)) {
1160 gt->submission_method = INTEL_SUBMISSION_ELSP;
1161 setup = intel_execlists_submission_setup;
1162 } else {
1163 gt->submission_method = INTEL_SUBMISSION_RING;
1164 setup = intel_ring_submission_setup;
1165 }
1166
1167 for_each_engine(engine, gt, id) {
1168 err = engine_setup_common(engine);
1169 if (err)
1170 return err;
1171
1172 err = setup(engine);
1173 if (err)
1174 return err;
1175
1176 err = engine_init_common(engine);
1177 if (err)
1178 return err;
1179
1180 intel_engine_add_user(engine);
1181 }
1182
1183 return 0;
1184 }
1185
1186 /**
1187 * intel_engines_cleanup_common - cleans up the engine state created by
1188 * the common initiailizers.
1189 * @engine: Engine to cleanup.
1190 *
1191 * This cleans up everything created by the common helpers.
1192 */
intel_engine_cleanup_common(struct intel_engine_cs * engine)1193 void intel_engine_cleanup_common(struct intel_engine_cs *engine)
1194 {
1195 GEM_BUG_ON(!list_empty(&engine->sched_engine->requests));
1196
1197 i915_sched_engine_put(engine->sched_engine);
1198 intel_breadcrumbs_put(engine->breadcrumbs);
1199
1200 intel_engine_fini_retire(engine);
1201 intel_engine_cleanup_cmd_parser(engine);
1202
1203 if (engine->default_state)
1204 fput(engine->default_state);
1205
1206 if (engine->kernel_context)
1207 intel_engine_destroy_pinned_context(engine->kernel_context);
1208
1209 GEM_BUG_ON(!llist_empty(&engine->barrier_tasks));
1210 cleanup_status_page(engine);
1211
1212 intel_wa_list_free(&engine->ctx_wa_list);
1213 intel_wa_list_free(&engine->wa_list);
1214 intel_wa_list_free(&engine->whitelist);
1215 }
1216
1217 /**
1218 * intel_engine_resume - re-initializes the HW state of the engine
1219 * @engine: Engine to resume.
1220 *
1221 * Returns zero on success or an error code on failure.
1222 */
intel_engine_resume(struct intel_engine_cs * engine)1223 int intel_engine_resume(struct intel_engine_cs *engine)
1224 {
1225 intel_engine_apply_workarounds(engine);
1226 intel_engine_apply_whitelist(engine);
1227
1228 return engine->resume(engine);
1229 }
1230
intel_engine_get_active_head(const struct intel_engine_cs * engine)1231 u64 intel_engine_get_active_head(const struct intel_engine_cs *engine)
1232 {
1233 struct drm_i915_private *i915 = engine->i915;
1234
1235 u64 acthd;
1236
1237 if (GRAPHICS_VER(i915) >= 8)
1238 acthd = ENGINE_READ64(engine, RING_ACTHD, RING_ACTHD_UDW);
1239 else if (GRAPHICS_VER(i915) >= 4)
1240 acthd = ENGINE_READ(engine, RING_ACTHD);
1241 else
1242 acthd = ENGINE_READ(engine, ACTHD);
1243
1244 return acthd;
1245 }
1246
intel_engine_get_last_batch_head(const struct intel_engine_cs * engine)1247 u64 intel_engine_get_last_batch_head(const struct intel_engine_cs *engine)
1248 {
1249 u64 bbaddr;
1250
1251 if (GRAPHICS_VER(engine->i915) >= 8)
1252 bbaddr = ENGINE_READ64(engine, RING_BBADDR, RING_BBADDR_UDW);
1253 else
1254 bbaddr = ENGINE_READ(engine, RING_BBADDR);
1255
1256 return bbaddr;
1257 }
1258
stop_timeout(const struct intel_engine_cs * engine)1259 static unsigned long stop_timeout(const struct intel_engine_cs *engine)
1260 {
1261 if (in_atomic() || irqs_disabled()) /* inside atomic preempt-reset? */
1262 return 0;
1263
1264 /*
1265 * If we are doing a normal GPU reset, we can take our time and allow
1266 * the engine to quiesce. We've stopped submission to the engine, and
1267 * if we wait long enough an innocent context should complete and
1268 * leave the engine idle. So they should not be caught unaware by
1269 * the forthcoming GPU reset (which usually follows the stop_cs)!
1270 */
1271 return READ_ONCE(engine->props.stop_timeout_ms);
1272 }
1273
__intel_engine_stop_cs(struct intel_engine_cs * engine,int fast_timeout_us,int slow_timeout_ms)1274 static int __intel_engine_stop_cs(struct intel_engine_cs *engine,
1275 int fast_timeout_us,
1276 int slow_timeout_ms)
1277 {
1278 struct intel_uncore *uncore = engine->uncore;
1279 const i915_reg_t mode = RING_MI_MODE(engine->mmio_base);
1280 int err;
1281
1282 intel_uncore_write_fw(uncore, mode, _MASKED_BIT_ENABLE(STOP_RING));
1283
1284 /*
1285 * Wa_22011802037 : gen11, gen12, Prior to doing a reset, ensure CS is
1286 * stopped, set ring stop bit and prefetch disable bit to halt CS
1287 */
1288 if (IS_GRAPHICS_VER(engine->i915, 11, 12))
1289 intel_uncore_write_fw(uncore, RING_MODE_GEN7(engine->mmio_base),
1290 _MASKED_BIT_ENABLE(GEN12_GFX_PREFETCH_DISABLE));
1291
1292 err = __intel_wait_for_register_fw(engine->uncore, mode,
1293 MODE_IDLE, MODE_IDLE,
1294 fast_timeout_us,
1295 slow_timeout_ms,
1296 NULL);
1297
1298 /* A final mmio read to let GPU writes be hopefully flushed to memory */
1299 intel_uncore_posting_read_fw(uncore, mode);
1300 return err;
1301 }
1302
intel_engine_stop_cs(struct intel_engine_cs * engine)1303 int intel_engine_stop_cs(struct intel_engine_cs *engine)
1304 {
1305 int err = 0;
1306
1307 if (GRAPHICS_VER(engine->i915) < 3)
1308 return -ENODEV;
1309
1310 ENGINE_TRACE(engine, "\n");
1311 /*
1312 * TODO: Find out why occasionally stopping the CS times out. Seen
1313 * especially with gem_eio tests.
1314 *
1315 * Occasionally trying to stop the cs times out, but does not adversely
1316 * affect functionality. The timeout is set as a config parameter that
1317 * defaults to 100ms. In most cases the follow up operation is to wait
1318 * for pending MI_FORCE_WAKES. The assumption is that this timeout is
1319 * sufficient for any pending MI_FORCEWAKEs to complete. Once root
1320 * caused, the caller must check and handle the return from this
1321 * function.
1322 */
1323 if (__intel_engine_stop_cs(engine, 1000, stop_timeout(engine))) {
1324 ENGINE_TRACE(engine,
1325 "timed out on STOP_RING -> IDLE; HEAD:%04x, TAIL:%04x\n",
1326 ENGINE_READ_FW(engine, RING_HEAD) & HEAD_ADDR,
1327 ENGINE_READ_FW(engine, RING_TAIL) & TAIL_ADDR);
1328
1329 /*
1330 * Sometimes we observe that the idle flag is not
1331 * set even though the ring is empty. So double
1332 * check before giving up.
1333 */
1334 if ((ENGINE_READ_FW(engine, RING_HEAD) & HEAD_ADDR) !=
1335 (ENGINE_READ_FW(engine, RING_TAIL) & TAIL_ADDR))
1336 err = -ETIMEDOUT;
1337 }
1338
1339 return err;
1340 }
1341
intel_engine_cancel_stop_cs(struct intel_engine_cs * engine)1342 void intel_engine_cancel_stop_cs(struct intel_engine_cs *engine)
1343 {
1344 ENGINE_TRACE(engine, "\n");
1345
1346 ENGINE_WRITE_FW(engine, RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING));
1347 }
1348
__cs_pending_mi_force_wakes(struct intel_engine_cs * engine)1349 static u32 __cs_pending_mi_force_wakes(struct intel_engine_cs *engine)
1350 {
1351 static const i915_reg_t _reg[I915_NUM_ENGINES] = {
1352 [RCS0] = MSG_IDLE_CS,
1353 [BCS0] = MSG_IDLE_BCS,
1354 [VCS0] = MSG_IDLE_VCS0,
1355 [VCS1] = MSG_IDLE_VCS1,
1356 [VCS2] = MSG_IDLE_VCS2,
1357 [VCS3] = MSG_IDLE_VCS3,
1358 [VCS4] = MSG_IDLE_VCS4,
1359 [VCS5] = MSG_IDLE_VCS5,
1360 [VCS6] = MSG_IDLE_VCS6,
1361 [VCS7] = MSG_IDLE_VCS7,
1362 [VECS0] = MSG_IDLE_VECS0,
1363 [VECS1] = MSG_IDLE_VECS1,
1364 [VECS2] = MSG_IDLE_VECS2,
1365 [VECS3] = MSG_IDLE_VECS3,
1366 [CCS0] = MSG_IDLE_CS,
1367 [CCS1] = MSG_IDLE_CS,
1368 [CCS2] = MSG_IDLE_CS,
1369 [CCS3] = MSG_IDLE_CS,
1370 };
1371 u32 val;
1372
1373 if (!_reg[engine->id].reg) {
1374 drm_err(&engine->i915->drm,
1375 "MSG IDLE undefined for engine id %u\n", engine->id);
1376 return 0;
1377 }
1378
1379 val = intel_uncore_read(engine->uncore, _reg[engine->id]);
1380
1381 /* bits[29:25] & bits[13:9] >> shift */
1382 return (val & (val >> 16) & MSG_IDLE_FW_MASK) >> MSG_IDLE_FW_SHIFT;
1383 }
1384
__gpm_wait_for_fw_complete(struct intel_gt * gt,u32 fw_mask)1385 static void __gpm_wait_for_fw_complete(struct intel_gt *gt, u32 fw_mask)
1386 {
1387 int ret;
1388
1389 /* Ensure GPM receives fw up/down after CS is stopped */
1390 udelay(1);
1391
1392 /* Wait for forcewake request to complete in GPM */
1393 ret = __intel_wait_for_register_fw(gt->uncore,
1394 GEN9_PWRGT_DOMAIN_STATUS,
1395 fw_mask, fw_mask, 5000, 0, NULL);
1396
1397 /* Ensure CS receives fw ack from GPM */
1398 udelay(1);
1399
1400 if (ret)
1401 GT_TRACE(gt, "Failed to complete pending forcewake %d\n", ret);
1402 }
1403
1404 /*
1405 * Wa_22011802037:gen12: In addition to stopping the cs, we need to wait for any
1406 * pending MI_FORCE_WAKEUP requests that the CS has initiated to complete. The
1407 * pending status is indicated by bits[13:9] (masked by bits[29:25]) in the
1408 * MSG_IDLE register. There's one MSG_IDLE register per reset domain. Since we
1409 * are concerned only with the gt reset here, we use a logical OR of pending
1410 * forcewakeups from all reset domains and then wait for them to complete by
1411 * querying PWRGT_DOMAIN_STATUS.
1412 */
intel_engine_wait_for_pending_mi_fw(struct intel_engine_cs * engine)1413 void intel_engine_wait_for_pending_mi_fw(struct intel_engine_cs *engine)
1414 {
1415 u32 fw_pending = __cs_pending_mi_force_wakes(engine);
1416
1417 if (fw_pending)
1418 __gpm_wait_for_fw_complete(engine->gt, fw_pending);
1419 }
1420
1421 static u32
read_subslice_reg(const struct intel_engine_cs * engine,int slice,int subslice,i915_reg_t reg)1422 read_subslice_reg(const struct intel_engine_cs *engine,
1423 int slice, int subslice, i915_reg_t reg)
1424 {
1425 return intel_uncore_read_with_mcr_steering(engine->uncore, reg,
1426 slice, subslice);
1427 }
1428
1429 /* NB: please notice the memset */
intel_engine_get_instdone(const struct intel_engine_cs * engine,struct intel_instdone * instdone)1430 void intel_engine_get_instdone(const struct intel_engine_cs *engine,
1431 struct intel_instdone *instdone)
1432 {
1433 struct drm_i915_private *i915 = engine->i915;
1434 const struct sseu_dev_info *sseu = &engine->gt->info.sseu;
1435 struct intel_uncore *uncore = engine->uncore;
1436 u32 mmio_base = engine->mmio_base;
1437 int slice;
1438 int subslice;
1439 int iter;
1440
1441 memset(instdone, 0, sizeof(*instdone));
1442
1443 if (GRAPHICS_VER(i915) >= 8) {
1444 instdone->instdone =
1445 intel_uncore_read(uncore, RING_INSTDONE(mmio_base));
1446
1447 if (engine->id != RCS0)
1448 return;
1449
1450 instdone->slice_common =
1451 intel_uncore_read(uncore, GEN7_SC_INSTDONE);
1452 if (GRAPHICS_VER(i915) >= 12) {
1453 instdone->slice_common_extra[0] =
1454 intel_uncore_read(uncore, GEN12_SC_INSTDONE_EXTRA);
1455 instdone->slice_common_extra[1] =
1456 intel_uncore_read(uncore, GEN12_SC_INSTDONE_EXTRA2);
1457 }
1458
1459 if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50)) {
1460 for_each_instdone_gslice_dss_xehp(i915, sseu, iter, slice, subslice) {
1461 instdone->sampler[slice][subslice] =
1462 read_subslice_reg(engine, slice, subslice,
1463 GEN7_SAMPLER_INSTDONE);
1464 instdone->row[slice][subslice] =
1465 read_subslice_reg(engine, slice, subslice,
1466 GEN7_ROW_INSTDONE);
1467 }
1468 } else {
1469 for_each_instdone_slice_subslice(i915, sseu, slice, subslice) {
1470 instdone->sampler[slice][subslice] =
1471 read_subslice_reg(engine, slice, subslice,
1472 GEN7_SAMPLER_INSTDONE);
1473 instdone->row[slice][subslice] =
1474 read_subslice_reg(engine, slice, subslice,
1475 GEN7_ROW_INSTDONE);
1476 }
1477 }
1478
1479 if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 55)) {
1480 for_each_instdone_gslice_dss_xehp(i915, sseu, iter, slice, subslice)
1481 instdone->geom_svg[slice][subslice] =
1482 read_subslice_reg(engine, slice, subslice,
1483 XEHPG_INSTDONE_GEOM_SVG);
1484 }
1485 } else if (GRAPHICS_VER(i915) >= 7) {
1486 instdone->instdone =
1487 intel_uncore_read(uncore, RING_INSTDONE(mmio_base));
1488
1489 if (engine->id != RCS0)
1490 return;
1491
1492 instdone->slice_common =
1493 intel_uncore_read(uncore, GEN7_SC_INSTDONE);
1494 instdone->sampler[0][0] =
1495 intel_uncore_read(uncore, GEN7_SAMPLER_INSTDONE);
1496 instdone->row[0][0] =
1497 intel_uncore_read(uncore, GEN7_ROW_INSTDONE);
1498 } else if (GRAPHICS_VER(i915) >= 4) {
1499 instdone->instdone =
1500 intel_uncore_read(uncore, RING_INSTDONE(mmio_base));
1501 if (engine->id == RCS0)
1502 /* HACK: Using the wrong struct member */
1503 instdone->slice_common =
1504 intel_uncore_read(uncore, GEN4_INSTDONE1);
1505 } else {
1506 instdone->instdone = intel_uncore_read(uncore, GEN2_INSTDONE);
1507 }
1508 }
1509
ring_is_idle(struct intel_engine_cs * engine)1510 static bool ring_is_idle(struct intel_engine_cs *engine)
1511 {
1512 bool idle = true;
1513
1514 if (I915_SELFTEST_ONLY(!engine->mmio_base))
1515 return true;
1516
1517 if (!intel_engine_pm_get_if_awake(engine))
1518 return true;
1519
1520 /* First check that no commands are left in the ring */
1521 if ((ENGINE_READ(engine, RING_HEAD) & HEAD_ADDR) !=
1522 (ENGINE_READ(engine, RING_TAIL) & TAIL_ADDR))
1523 idle = false;
1524
1525 /* No bit for gen2, so assume the CS parser is idle */
1526 if (GRAPHICS_VER(engine->i915) > 2 &&
1527 !(ENGINE_READ(engine, RING_MI_MODE) & MODE_IDLE))
1528 idle = false;
1529
1530 intel_engine_pm_put(engine);
1531
1532 return idle;
1533 }
1534
__intel_engine_flush_submission(struct intel_engine_cs * engine,bool sync)1535 void __intel_engine_flush_submission(struct intel_engine_cs *engine, bool sync)
1536 {
1537 struct tasklet_struct *t = &engine->sched_engine->tasklet;
1538
1539 if (!t->callback)
1540 return;
1541
1542 local_bh_disable();
1543 if (tasklet_trylock(t)) {
1544 /* Must wait for any GPU reset in progress. */
1545 if (__tasklet_is_enabled(t))
1546 t->callback(t);
1547 tasklet_unlock(t);
1548 }
1549 local_bh_enable();
1550
1551 /* Synchronise and wait for the tasklet on another CPU */
1552 if (sync)
1553 tasklet_unlock_wait(t);
1554 }
1555
1556 /**
1557 * intel_engine_is_idle() - Report if the engine has finished process all work
1558 * @engine: the intel_engine_cs
1559 *
1560 * Return true if there are no requests pending, nothing left to be submitted
1561 * to hardware, and that the engine is idle.
1562 */
intel_engine_is_idle(struct intel_engine_cs * engine)1563 bool intel_engine_is_idle(struct intel_engine_cs *engine)
1564 {
1565 /* More white lies, if wedged, hw state is inconsistent */
1566 if (intel_gt_is_wedged(engine->gt))
1567 return true;
1568
1569 if (!intel_engine_pm_is_awake(engine))
1570 return true;
1571
1572 /* Waiting to drain ELSP? */
1573 intel_synchronize_hardirq(engine->i915);
1574 intel_engine_flush_submission(engine);
1575
1576 /* ELSP is empty, but there are ready requests? E.g. after reset */
1577 if (!i915_sched_engine_is_empty(engine->sched_engine))
1578 return false;
1579
1580 /* Ring stopped? */
1581 return ring_is_idle(engine);
1582 }
1583
intel_engines_are_idle(struct intel_gt * gt)1584 bool intel_engines_are_idle(struct intel_gt *gt)
1585 {
1586 struct intel_engine_cs *engine;
1587 enum intel_engine_id id;
1588
1589 /*
1590 * If the driver is wedged, HW state may be very inconsistent and
1591 * report that it is still busy, even though we have stopped using it.
1592 */
1593 if (intel_gt_is_wedged(gt))
1594 return true;
1595
1596 /* Already parked (and passed an idleness test); must still be idle */
1597 if (!READ_ONCE(gt->awake))
1598 return true;
1599
1600 for_each_engine(engine, gt, id) {
1601 if (!intel_engine_is_idle(engine))
1602 return false;
1603 }
1604
1605 return true;
1606 }
1607
intel_engine_irq_enable(struct intel_engine_cs * engine)1608 bool intel_engine_irq_enable(struct intel_engine_cs *engine)
1609 {
1610 if (!engine->irq_enable)
1611 return false;
1612
1613 /* Caller disables interrupts */
1614 spin_lock(&engine->gt->irq_lock);
1615 engine->irq_enable(engine);
1616 spin_unlock(&engine->gt->irq_lock);
1617
1618 return true;
1619 }
1620
intel_engine_irq_disable(struct intel_engine_cs * engine)1621 void intel_engine_irq_disable(struct intel_engine_cs *engine)
1622 {
1623 if (!engine->irq_disable)
1624 return;
1625
1626 /* Caller disables interrupts */
1627 spin_lock(&engine->gt->irq_lock);
1628 engine->irq_disable(engine);
1629 spin_unlock(&engine->gt->irq_lock);
1630 }
1631
intel_engines_reset_default_submission(struct intel_gt * gt)1632 void intel_engines_reset_default_submission(struct intel_gt *gt)
1633 {
1634 struct intel_engine_cs *engine;
1635 enum intel_engine_id id;
1636
1637 for_each_engine(engine, gt, id) {
1638 if (engine->sanitize)
1639 engine->sanitize(engine);
1640
1641 engine->set_default_submission(engine);
1642 }
1643 }
1644
intel_engine_can_store_dword(struct intel_engine_cs * engine)1645 bool intel_engine_can_store_dword(struct intel_engine_cs *engine)
1646 {
1647 switch (GRAPHICS_VER(engine->i915)) {
1648 case 2:
1649 return false; /* uses physical not virtual addresses */
1650 case 3:
1651 /* maybe only uses physical not virtual addresses */
1652 return !(IS_I915G(engine->i915) || IS_I915GM(engine->i915));
1653 case 4:
1654 return !IS_I965G(engine->i915); /* who knows! */
1655 case 6:
1656 return engine->class != VIDEO_DECODE_CLASS; /* b0rked */
1657 default:
1658 return true;
1659 }
1660 }
1661
get_timeline(struct i915_request * rq)1662 static struct intel_timeline *get_timeline(struct i915_request *rq)
1663 {
1664 struct intel_timeline *tl;
1665
1666 /*
1667 * Even though we are holding the engine->sched_engine->lock here, there
1668 * is no control over the submission queue per-se and we are
1669 * inspecting the active state at a random point in time, with an
1670 * unknown queue. Play safe and make sure the timeline remains valid.
1671 * (Only being used for pretty printing, one extra kref shouldn't
1672 * cause a camel stampede!)
1673 */
1674 rcu_read_lock();
1675 tl = rcu_dereference(rq->timeline);
1676 if (!kref_get_unless_zero(&tl->kref))
1677 tl = NULL;
1678 rcu_read_unlock();
1679
1680 return tl;
1681 }
1682
print_ring(char * buf,int sz,struct i915_request * rq)1683 static int print_ring(char *buf, int sz, struct i915_request *rq)
1684 {
1685 int len = 0;
1686
1687 if (!i915_request_signaled(rq)) {
1688 struct intel_timeline *tl = get_timeline(rq);
1689
1690 len = scnprintf(buf, sz,
1691 "ring:{start:%08x, hwsp:%08x, seqno:%08x, runtime:%llums}, ",
1692 i915_ggtt_offset(rq->ring->vma),
1693 tl ? tl->hwsp_offset : 0,
1694 hwsp_seqno(rq),
1695 DIV_ROUND_CLOSEST_ULL(intel_context_get_total_runtime_ns(rq->context),
1696 1000 * 1000));
1697
1698 if (tl)
1699 intel_timeline_put(tl);
1700 }
1701
1702 return len;
1703 }
1704
hexdump(struct drm_printer * m,const void * buf,size_t len)1705 static void hexdump(struct drm_printer *m, const void *buf, size_t len)
1706 {
1707 const size_t rowsize = 8 * sizeof(u32);
1708 const void *prev = NULL;
1709 bool skip = false;
1710 size_t pos;
1711
1712 for (pos = 0; pos < len; pos += rowsize) {
1713 char line[128];
1714
1715 if (prev && !memcmp(prev, buf + pos, rowsize)) {
1716 if (!skip) {
1717 drm_printf(m, "*\n");
1718 skip = true;
1719 }
1720 continue;
1721 }
1722
1723 WARN_ON_ONCE(hex_dump_to_buffer(buf + pos, len - pos,
1724 rowsize, sizeof(u32),
1725 line, sizeof(line),
1726 false) >= sizeof(line));
1727 drm_printf(m, "[%04zx] %s\n", pos, line);
1728
1729 prev = buf + pos;
1730 skip = false;
1731 }
1732 }
1733
repr_timer(const struct timer_list * t)1734 static const char *repr_timer(const struct timer_list *t)
1735 {
1736 if (!READ_ONCE(t->expires))
1737 return "inactive";
1738
1739 if (timer_pending(t))
1740 return "active";
1741
1742 return "expired";
1743 }
1744
intel_engine_print_registers(struct intel_engine_cs * engine,struct drm_printer * m)1745 static void intel_engine_print_registers(struct intel_engine_cs *engine,
1746 struct drm_printer *m)
1747 {
1748 struct drm_i915_private *dev_priv = engine->i915;
1749 struct intel_engine_execlists * const execlists = &engine->execlists;
1750 u64 addr;
1751
1752 if (engine->id == RENDER_CLASS && IS_GRAPHICS_VER(dev_priv, 4, 7))
1753 drm_printf(m, "\tCCID: 0x%08x\n", ENGINE_READ(engine, CCID));
1754 if (HAS_EXECLISTS(dev_priv)) {
1755 drm_printf(m, "\tEL_STAT_HI: 0x%08x\n",
1756 ENGINE_READ(engine, RING_EXECLIST_STATUS_HI));
1757 drm_printf(m, "\tEL_STAT_LO: 0x%08x\n",
1758 ENGINE_READ(engine, RING_EXECLIST_STATUS_LO));
1759 }
1760 drm_printf(m, "\tRING_START: 0x%08x\n",
1761 ENGINE_READ(engine, RING_START));
1762 drm_printf(m, "\tRING_HEAD: 0x%08x\n",
1763 ENGINE_READ(engine, RING_HEAD) & HEAD_ADDR);
1764 drm_printf(m, "\tRING_TAIL: 0x%08x\n",
1765 ENGINE_READ(engine, RING_TAIL) & TAIL_ADDR);
1766 drm_printf(m, "\tRING_CTL: 0x%08x%s\n",
1767 ENGINE_READ(engine, RING_CTL),
1768 ENGINE_READ(engine, RING_CTL) & (RING_WAIT | RING_WAIT_SEMAPHORE) ? " [waiting]" : "");
1769 if (GRAPHICS_VER(engine->i915) > 2) {
1770 drm_printf(m, "\tRING_MODE: 0x%08x%s\n",
1771 ENGINE_READ(engine, RING_MI_MODE),
1772 ENGINE_READ(engine, RING_MI_MODE) & (MODE_IDLE) ? " [idle]" : "");
1773 }
1774
1775 if (GRAPHICS_VER(dev_priv) >= 6) {
1776 drm_printf(m, "\tRING_IMR: 0x%08x\n",
1777 ENGINE_READ(engine, RING_IMR));
1778 drm_printf(m, "\tRING_ESR: 0x%08x\n",
1779 ENGINE_READ(engine, RING_ESR));
1780 drm_printf(m, "\tRING_EMR: 0x%08x\n",
1781 ENGINE_READ(engine, RING_EMR));
1782 drm_printf(m, "\tRING_EIR: 0x%08x\n",
1783 ENGINE_READ(engine, RING_EIR));
1784 }
1785
1786 addr = intel_engine_get_active_head(engine);
1787 drm_printf(m, "\tACTHD: 0x%08x_%08x\n",
1788 upper_32_bits(addr), lower_32_bits(addr));
1789 addr = intel_engine_get_last_batch_head(engine);
1790 drm_printf(m, "\tBBADDR: 0x%08x_%08x\n",
1791 upper_32_bits(addr), lower_32_bits(addr));
1792 if (GRAPHICS_VER(dev_priv) >= 8)
1793 addr = ENGINE_READ64(engine, RING_DMA_FADD, RING_DMA_FADD_UDW);
1794 else if (GRAPHICS_VER(dev_priv) >= 4)
1795 addr = ENGINE_READ(engine, RING_DMA_FADD);
1796 else
1797 addr = ENGINE_READ(engine, DMA_FADD_I8XX);
1798 drm_printf(m, "\tDMA_FADDR: 0x%08x_%08x\n",
1799 upper_32_bits(addr), lower_32_bits(addr));
1800 if (GRAPHICS_VER(dev_priv) >= 4) {
1801 drm_printf(m, "\tIPEIR: 0x%08x\n",
1802 ENGINE_READ(engine, RING_IPEIR));
1803 drm_printf(m, "\tIPEHR: 0x%08x\n",
1804 ENGINE_READ(engine, RING_IPEHR));
1805 } else {
1806 drm_printf(m, "\tIPEIR: 0x%08x\n", ENGINE_READ(engine, IPEIR));
1807 drm_printf(m, "\tIPEHR: 0x%08x\n", ENGINE_READ(engine, IPEHR));
1808 }
1809
1810 if (HAS_EXECLISTS(dev_priv) && !intel_engine_uses_guc(engine)) {
1811 struct i915_request * const *port, *rq;
1812 const u32 *hws =
1813 &engine->status_page.addr[I915_HWS_CSB_BUF0_INDEX];
1814 const u8 num_entries = execlists->csb_size;
1815 unsigned int idx;
1816 u8 read, write;
1817
1818 drm_printf(m, "\tExeclist tasklet queued? %s (%s), preempt? %s, timeslice? %s\n",
1819 str_yes_no(test_bit(TASKLET_STATE_SCHED, &engine->sched_engine->tasklet.state)),
1820 str_enabled_disabled(!atomic_read(&engine->sched_engine->tasklet.count)),
1821 repr_timer(&engine->execlists.preempt),
1822 repr_timer(&engine->execlists.timer));
1823
1824 read = execlists->csb_head;
1825 write = READ_ONCE(*execlists->csb_write);
1826
1827 drm_printf(m, "\tExeclist status: 0x%08x %08x; CSB read:%d, write:%d, entries:%d\n",
1828 ENGINE_READ(engine, RING_EXECLIST_STATUS_LO),
1829 ENGINE_READ(engine, RING_EXECLIST_STATUS_HI),
1830 read, write, num_entries);
1831
1832 if (read >= num_entries)
1833 read = 0;
1834 if (write >= num_entries)
1835 write = 0;
1836 if (read > write)
1837 write += num_entries;
1838 while (read < write) {
1839 idx = ++read % num_entries;
1840 drm_printf(m, "\tExeclist CSB[%d]: 0x%08x, context: %d\n",
1841 idx, hws[idx * 2], hws[idx * 2 + 1]);
1842 }
1843
1844 i915_sched_engine_active_lock_bh(engine->sched_engine);
1845 rcu_read_lock();
1846 for (port = execlists->active; (rq = *port); port++) {
1847 char hdr[160];
1848 int len;
1849
1850 len = scnprintf(hdr, sizeof(hdr),
1851 "\t\tActive[%d]: ccid:%08x%s%s, ",
1852 (int)(port - execlists->active),
1853 rq->context->lrc.ccid,
1854 intel_context_is_closed(rq->context) ? "!" : "",
1855 intel_context_is_banned(rq->context) ? "*" : "");
1856 len += print_ring(hdr + len, sizeof(hdr) - len, rq);
1857 scnprintf(hdr + len, sizeof(hdr) - len, "rq: ");
1858 i915_request_show(m, rq, hdr, 0);
1859 }
1860 for (port = execlists->pending; (rq = *port); port++) {
1861 char hdr[160];
1862 int len;
1863
1864 len = scnprintf(hdr, sizeof(hdr),
1865 "\t\tPending[%d]: ccid:%08x%s%s, ",
1866 (int)(port - execlists->pending),
1867 rq->context->lrc.ccid,
1868 intel_context_is_closed(rq->context) ? "!" : "",
1869 intel_context_is_banned(rq->context) ? "*" : "");
1870 len += print_ring(hdr + len, sizeof(hdr) - len, rq);
1871 scnprintf(hdr + len, sizeof(hdr) - len, "rq: ");
1872 i915_request_show(m, rq, hdr, 0);
1873 }
1874 rcu_read_unlock();
1875 i915_sched_engine_active_unlock_bh(engine->sched_engine);
1876 } else if (GRAPHICS_VER(dev_priv) > 6) {
1877 drm_printf(m, "\tPP_DIR_BASE: 0x%08x\n",
1878 ENGINE_READ(engine, RING_PP_DIR_BASE));
1879 drm_printf(m, "\tPP_DIR_BASE_READ: 0x%08x\n",
1880 ENGINE_READ(engine, RING_PP_DIR_BASE_READ));
1881 drm_printf(m, "\tPP_DIR_DCLV: 0x%08x\n",
1882 ENGINE_READ(engine, RING_PP_DIR_DCLV));
1883 }
1884 }
1885
print_request_ring(struct drm_printer * m,struct i915_request * rq)1886 static void print_request_ring(struct drm_printer *m, struct i915_request *rq)
1887 {
1888 struct i915_vma_resource *vma_res = rq->batch_res;
1889 void *ring;
1890 int size;
1891
1892 drm_printf(m,
1893 "[head %04x, postfix %04x, tail %04x, batch 0x%08x_%08x]:\n",
1894 rq->head, rq->postfix, rq->tail,
1895 vma_res ? upper_32_bits(vma_res->start) : ~0u,
1896 vma_res ? lower_32_bits(vma_res->start) : ~0u);
1897
1898 size = rq->tail - rq->head;
1899 if (rq->tail < rq->head)
1900 size += rq->ring->size;
1901
1902 ring = kmalloc(size, GFP_ATOMIC);
1903 if (ring) {
1904 const void *vaddr = rq->ring->vaddr;
1905 unsigned int head = rq->head;
1906 unsigned int len = 0;
1907
1908 if (rq->tail < head) {
1909 len = rq->ring->size - head;
1910 memcpy(ring, vaddr + head, len);
1911 head = 0;
1912 }
1913 memcpy(ring + len, vaddr + head, size - len);
1914
1915 hexdump(m, ring, size);
1916 kfree(ring);
1917 }
1918 }
1919
list_count(struct list_head * list)1920 static unsigned long list_count(struct list_head *list)
1921 {
1922 struct list_head *pos;
1923 unsigned long count = 0;
1924
1925 list_for_each(pos, list)
1926 count++;
1927
1928 return count;
1929 }
1930
read_ul(void * p,size_t x)1931 static unsigned long read_ul(void *p, size_t x)
1932 {
1933 return *(unsigned long *)(p + x);
1934 }
1935
print_properties(struct intel_engine_cs * engine,struct drm_printer * m)1936 static void print_properties(struct intel_engine_cs *engine,
1937 struct drm_printer *m)
1938 {
1939 static const struct pmap {
1940 size_t offset;
1941 const char *name;
1942 } props[] = {
1943 #define P(x) { \
1944 .offset = offsetof(typeof(engine->props), x), \
1945 .name = #x \
1946 }
1947 P(heartbeat_interval_ms),
1948 P(max_busywait_duration_ns),
1949 P(preempt_timeout_ms),
1950 P(stop_timeout_ms),
1951 P(timeslice_duration_ms),
1952
1953 {},
1954 #undef P
1955 };
1956 const struct pmap *p;
1957
1958 drm_printf(m, "\tProperties:\n");
1959 for (p = props; p->name; p++)
1960 drm_printf(m, "\t\t%s: %lu [default %lu]\n",
1961 p->name,
1962 read_ul(&engine->props, p->offset),
1963 read_ul(&engine->defaults, p->offset));
1964 }
1965
engine_dump_request(struct i915_request * rq,struct drm_printer * m,const char * msg)1966 static void engine_dump_request(struct i915_request *rq, struct drm_printer *m, const char *msg)
1967 {
1968 struct intel_timeline *tl = get_timeline(rq);
1969
1970 i915_request_show(m, rq, msg, 0);
1971
1972 drm_printf(m, "\t\tring->start: 0x%08x\n",
1973 i915_ggtt_offset(rq->ring->vma));
1974 drm_printf(m, "\t\tring->head: 0x%08x\n",
1975 rq->ring->head);
1976 drm_printf(m, "\t\tring->tail: 0x%08x\n",
1977 rq->ring->tail);
1978 drm_printf(m, "\t\tring->emit: 0x%08x\n",
1979 rq->ring->emit);
1980 drm_printf(m, "\t\tring->space: 0x%08x\n",
1981 rq->ring->space);
1982
1983 if (tl) {
1984 drm_printf(m, "\t\tring->hwsp: 0x%08x\n",
1985 tl->hwsp_offset);
1986 intel_timeline_put(tl);
1987 }
1988
1989 print_request_ring(m, rq);
1990
1991 if (rq->context->lrc_reg_state) {
1992 drm_printf(m, "Logical Ring Context:\n");
1993 hexdump(m, rq->context->lrc_reg_state, PAGE_SIZE);
1994 }
1995 }
1996
intel_engine_dump_active_requests(struct list_head * requests,struct i915_request * hung_rq,struct drm_printer * m)1997 void intel_engine_dump_active_requests(struct list_head *requests,
1998 struct i915_request *hung_rq,
1999 struct drm_printer *m)
2000 {
2001 struct i915_request *rq;
2002 const char *msg;
2003 enum i915_request_state state;
2004
2005 list_for_each_entry(rq, requests, sched.link) {
2006 if (rq == hung_rq)
2007 continue;
2008
2009 state = i915_test_request_state(rq);
2010 if (state < I915_REQUEST_QUEUED)
2011 continue;
2012
2013 if (state == I915_REQUEST_ACTIVE)
2014 msg = "\t\tactive on engine";
2015 else
2016 msg = "\t\tactive in queue";
2017
2018 engine_dump_request(rq, m, msg);
2019 }
2020 }
2021
engine_dump_active_requests(struct intel_engine_cs * engine,struct drm_printer * m)2022 static void engine_dump_active_requests(struct intel_engine_cs *engine, struct drm_printer *m)
2023 {
2024 struct i915_request *hung_rq = NULL;
2025 struct intel_context *ce;
2026 bool guc;
2027
2028 /*
2029 * No need for an engine->irq_seqno_barrier() before the seqno reads.
2030 * The GPU is still running so requests are still executing and any
2031 * hardware reads will be out of date by the time they are reported.
2032 * But the intention here is just to report an instantaneous snapshot
2033 * so that's fine.
2034 */
2035 lockdep_assert_held(&engine->sched_engine->lock);
2036
2037 drm_printf(m, "\tRequests:\n");
2038
2039 guc = intel_uc_uses_guc_submission(&engine->gt->uc);
2040 if (guc) {
2041 ce = intel_engine_get_hung_context(engine);
2042 if (ce)
2043 hung_rq = intel_context_find_active_request(ce);
2044 } else {
2045 hung_rq = intel_engine_execlist_find_hung_request(engine);
2046 }
2047
2048 if (hung_rq)
2049 engine_dump_request(hung_rq, m, "\t\thung");
2050
2051 if (guc)
2052 intel_guc_dump_active_requests(engine, hung_rq, m);
2053 else
2054 intel_engine_dump_active_requests(&engine->sched_engine->requests,
2055 hung_rq, m);
2056 }
2057
intel_engine_dump(struct intel_engine_cs * engine,struct drm_printer * m,const char * header,...)2058 void intel_engine_dump(struct intel_engine_cs *engine,
2059 struct drm_printer *m,
2060 const char *header, ...)
2061 {
2062 struct i915_gpu_error * const error = &engine->i915->gpu_error;
2063 struct i915_request *rq;
2064 intel_wakeref_t wakeref;
2065 unsigned long flags;
2066 ktime_t dummy;
2067
2068 if (header) {
2069 va_list ap;
2070
2071 va_start(ap, header);
2072 drm_vprintf(m, header, &ap);
2073 va_end(ap);
2074 }
2075
2076 if (intel_gt_is_wedged(engine->gt))
2077 drm_printf(m, "*** WEDGED ***\n");
2078
2079 drm_printf(m, "\tAwake? %d\n", atomic_read(&engine->wakeref.count));
2080 drm_printf(m, "\tBarriers?: %s\n",
2081 str_yes_no(!llist_empty(&engine->barrier_tasks)));
2082 drm_printf(m, "\tLatency: %luus\n",
2083 ewma__engine_latency_read(&engine->latency));
2084 if (intel_engine_supports_stats(engine))
2085 drm_printf(m, "\tRuntime: %llums\n",
2086 ktime_to_ms(intel_engine_get_busy_time(engine,
2087 &dummy)));
2088 drm_printf(m, "\tForcewake: %x domains, %d active\n",
2089 engine->fw_domain, READ_ONCE(engine->fw_active));
2090
2091 rcu_read_lock();
2092 rq = READ_ONCE(engine->heartbeat.systole);
2093 if (rq)
2094 drm_printf(m, "\tHeartbeat: %d ms ago\n",
2095 jiffies_to_msecs(jiffies - rq->emitted_jiffies));
2096 rcu_read_unlock();
2097 drm_printf(m, "\tReset count: %d (global %d)\n",
2098 i915_reset_engine_count(error, engine),
2099 i915_reset_count(error));
2100 print_properties(engine, m);
2101
2102 spin_lock_irqsave(&engine->sched_engine->lock, flags);
2103 engine_dump_active_requests(engine, m);
2104
2105 drm_printf(m, "\tOn hold?: %lu\n",
2106 list_count(&engine->sched_engine->hold));
2107 spin_unlock_irqrestore(&engine->sched_engine->lock, flags);
2108
2109 drm_printf(m, "\tMMIO base: 0x%08x\n", engine->mmio_base);
2110 wakeref = intel_runtime_pm_get_if_in_use(engine->uncore->rpm);
2111 if (wakeref) {
2112 intel_engine_print_registers(engine, m);
2113 intel_runtime_pm_put(engine->uncore->rpm, wakeref);
2114 } else {
2115 drm_printf(m, "\tDevice is asleep; skipping register dump\n");
2116 }
2117
2118 intel_execlists_show_requests(engine, m, i915_request_show, 8);
2119
2120 drm_printf(m, "HWSP:\n");
2121 hexdump(m, engine->status_page.addr, PAGE_SIZE);
2122
2123 drm_printf(m, "Idle? %s\n", str_yes_no(intel_engine_is_idle(engine)));
2124
2125 intel_engine_print_breadcrumbs(engine, m);
2126 }
2127
2128 /**
2129 * intel_engine_get_busy_time() - Return current accumulated engine busyness
2130 * @engine: engine to report on
2131 * @now: monotonic timestamp of sampling
2132 *
2133 * Returns accumulated time @engine was busy since engine stats were enabled.
2134 */
intel_engine_get_busy_time(struct intel_engine_cs * engine,ktime_t * now)2135 ktime_t intel_engine_get_busy_time(struct intel_engine_cs *engine, ktime_t *now)
2136 {
2137 return engine->busyness(engine, now);
2138 }
2139
2140 struct intel_context *
intel_engine_create_virtual(struct intel_engine_cs ** siblings,unsigned int count,unsigned long flags)2141 intel_engine_create_virtual(struct intel_engine_cs **siblings,
2142 unsigned int count, unsigned long flags)
2143 {
2144 if (count == 0)
2145 return ERR_PTR(-EINVAL);
2146
2147 if (count == 1 && !(flags & FORCE_VIRTUAL))
2148 return intel_context_create(siblings[0]);
2149
2150 GEM_BUG_ON(!siblings[0]->cops->create_virtual);
2151 return siblings[0]->cops->create_virtual(siblings, count, flags);
2152 }
2153
2154 struct i915_request *
intel_engine_execlist_find_hung_request(struct intel_engine_cs * engine)2155 intel_engine_execlist_find_hung_request(struct intel_engine_cs *engine)
2156 {
2157 struct i915_request *request, *active = NULL;
2158
2159 /*
2160 * This search does not work in GuC submission mode. However, the GuC
2161 * will report the hanging context directly to the driver itself. So
2162 * the driver should never get here when in GuC mode.
2163 */
2164 GEM_BUG_ON(intel_uc_uses_guc_submission(&engine->gt->uc));
2165
2166 /*
2167 * We are called by the error capture, reset and to dump engine
2168 * state at random points in time. In particular, note that neither is
2169 * crucially ordered with an interrupt. After a hang, the GPU is dead
2170 * and we assume that no more writes can happen (we waited long enough
2171 * for all writes that were in transaction to be flushed) - adding an
2172 * extra delay for a recent interrupt is pointless. Hence, we do
2173 * not need an engine->irq_seqno_barrier() before the seqno reads.
2174 * At all other times, we must assume the GPU is still running, but
2175 * we only care about the snapshot of this moment.
2176 */
2177 lockdep_assert_held(&engine->sched_engine->lock);
2178
2179 rcu_read_lock();
2180 request = execlists_active(&engine->execlists);
2181 if (request) {
2182 struct intel_timeline *tl = request->context->timeline;
2183
2184 list_for_each_entry_from_reverse(request, &tl->requests, link) {
2185 if (__i915_request_is_complete(request))
2186 break;
2187
2188 active = request;
2189 }
2190 }
2191 rcu_read_unlock();
2192 if (active)
2193 return active;
2194
2195 list_for_each_entry(request, &engine->sched_engine->requests,
2196 sched.link) {
2197 if (i915_test_request_state(request) != I915_REQUEST_ACTIVE)
2198 continue;
2199
2200 active = request;
2201 break;
2202 }
2203
2204 return active;
2205 }
2206
xehp_enable_ccs_engines(struct intel_engine_cs * engine)2207 void xehp_enable_ccs_engines(struct intel_engine_cs *engine)
2208 {
2209 /*
2210 * If there are any non-fused-off CCS engines, we need to enable CCS
2211 * support in the RCU_MODE register. This only needs to be done once,
2212 * so for simplicity we'll take care of this in the RCS engine's
2213 * resume handler; since the RCS and all CCS engines belong to the
2214 * same reset domain and are reset together, this will also take care
2215 * of re-applying the setting after i915-triggered resets.
2216 */
2217 if (!CCS_MASK(engine->gt))
2218 return;
2219
2220 intel_uncore_write(engine->uncore, GEN12_RCU_MODE,
2221 _MASKED_BIT_ENABLE(GEN12_RCU_MODE_CCS_ENABLE));
2222 }
2223
2224 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
2225 #include "mock_engine.c"
2226 #include "selftest_engine.c"
2227 #include "selftest_engine_cs.c"
2228 #endif
2229