1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2016 Intel Corporation
4  */
5 
6 #include <linux/string_helpers.h>
7 
8 #include <drm/drm_print.h>
9 
10 #include "gem/i915_gem_context.h"
11 #include "gem/i915_gem_internal.h"
12 #include "gt/intel_gt_regs.h"
13 
14 #include "i915_cmd_parser.h"
15 #include "i915_drv.h"
16 #include "intel_breadcrumbs.h"
17 #include "intel_context.h"
18 #include "intel_engine.h"
19 #include "intel_engine_pm.h"
20 #include "intel_engine_regs.h"
21 #include "intel_engine_user.h"
22 #include "intel_execlists_submission.h"
23 #include "intel_gt.h"
24 #include "intel_gt_requests.h"
25 #include "intel_gt_pm.h"
26 #include "intel_lrc.h"
27 #include "intel_lrc_reg.h"
28 #include "intel_reset.h"
29 #include "intel_ring.h"
30 #include "uc/intel_guc_submission.h"
31 
32 /* Haswell does have the CXT_SIZE register however it does not appear to be
33  * valid. Now, docs explain in dwords what is in the context object. The full
34  * size is 70720 bytes, however, the power context and execlist context will
35  * never be saved (power context is stored elsewhere, and execlists don't work
36  * on HSW) - so the final size, including the extra state required for the
37  * Resource Streamer, is 66944 bytes, which rounds to 17 pages.
38  */
39 #define HSW_CXT_TOTAL_SIZE		(17 * PAGE_SIZE)
40 
41 #define DEFAULT_LR_CONTEXT_RENDER_SIZE	(22 * PAGE_SIZE)
42 #define GEN8_LR_CONTEXT_RENDER_SIZE	(20 * PAGE_SIZE)
43 #define GEN9_LR_CONTEXT_RENDER_SIZE	(22 * PAGE_SIZE)
44 #define GEN11_LR_CONTEXT_RENDER_SIZE	(14 * PAGE_SIZE)
45 
46 #define GEN8_LR_CONTEXT_OTHER_SIZE	( 2 * PAGE_SIZE)
47 
48 #define MAX_MMIO_BASES 3
49 struct engine_info {
50 	u8 class;
51 	u8 instance;
52 	/* mmio bases table *must* be sorted in reverse graphics_ver order */
53 	struct engine_mmio_base {
54 		u32 graphics_ver : 8;
55 		u32 base : 24;
56 	} mmio_bases[MAX_MMIO_BASES];
57 };
58 
59 static const struct engine_info intel_engines[] = {
60 	[RCS0] = {
61 		.class = RENDER_CLASS,
62 		.instance = 0,
63 		.mmio_bases = {
64 			{ .graphics_ver = 1, .base = RENDER_RING_BASE }
65 		},
66 	},
67 	[BCS0] = {
68 		.class = COPY_ENGINE_CLASS,
69 		.instance = 0,
70 		.mmio_bases = {
71 			{ .graphics_ver = 6, .base = BLT_RING_BASE }
72 		},
73 	},
74 	[VCS0] = {
75 		.class = VIDEO_DECODE_CLASS,
76 		.instance = 0,
77 		.mmio_bases = {
78 			{ .graphics_ver = 11, .base = GEN11_BSD_RING_BASE },
79 			{ .graphics_ver = 6, .base = GEN6_BSD_RING_BASE },
80 			{ .graphics_ver = 4, .base = BSD_RING_BASE }
81 		},
82 	},
83 	[VCS1] = {
84 		.class = VIDEO_DECODE_CLASS,
85 		.instance = 1,
86 		.mmio_bases = {
87 			{ .graphics_ver = 11, .base = GEN11_BSD2_RING_BASE },
88 			{ .graphics_ver = 8, .base = GEN8_BSD2_RING_BASE }
89 		},
90 	},
91 	[VCS2] = {
92 		.class = VIDEO_DECODE_CLASS,
93 		.instance = 2,
94 		.mmio_bases = {
95 			{ .graphics_ver = 11, .base = GEN11_BSD3_RING_BASE }
96 		},
97 	},
98 	[VCS3] = {
99 		.class = VIDEO_DECODE_CLASS,
100 		.instance = 3,
101 		.mmio_bases = {
102 			{ .graphics_ver = 11, .base = GEN11_BSD4_RING_BASE }
103 		},
104 	},
105 	[VCS4] = {
106 		.class = VIDEO_DECODE_CLASS,
107 		.instance = 4,
108 		.mmio_bases = {
109 			{ .graphics_ver = 12, .base = XEHP_BSD5_RING_BASE }
110 		},
111 	},
112 	[VCS5] = {
113 		.class = VIDEO_DECODE_CLASS,
114 		.instance = 5,
115 		.mmio_bases = {
116 			{ .graphics_ver = 12, .base = XEHP_BSD6_RING_BASE }
117 		},
118 	},
119 	[VCS6] = {
120 		.class = VIDEO_DECODE_CLASS,
121 		.instance = 6,
122 		.mmio_bases = {
123 			{ .graphics_ver = 12, .base = XEHP_BSD7_RING_BASE }
124 		},
125 	},
126 	[VCS7] = {
127 		.class = VIDEO_DECODE_CLASS,
128 		.instance = 7,
129 		.mmio_bases = {
130 			{ .graphics_ver = 12, .base = XEHP_BSD8_RING_BASE }
131 		},
132 	},
133 	[VECS0] = {
134 		.class = VIDEO_ENHANCEMENT_CLASS,
135 		.instance = 0,
136 		.mmio_bases = {
137 			{ .graphics_ver = 11, .base = GEN11_VEBOX_RING_BASE },
138 			{ .graphics_ver = 7, .base = VEBOX_RING_BASE }
139 		},
140 	},
141 	[VECS1] = {
142 		.class = VIDEO_ENHANCEMENT_CLASS,
143 		.instance = 1,
144 		.mmio_bases = {
145 			{ .graphics_ver = 11, .base = GEN11_VEBOX2_RING_BASE }
146 		},
147 	},
148 	[VECS2] = {
149 		.class = VIDEO_ENHANCEMENT_CLASS,
150 		.instance = 2,
151 		.mmio_bases = {
152 			{ .graphics_ver = 12, .base = XEHP_VEBOX3_RING_BASE }
153 		},
154 	},
155 	[VECS3] = {
156 		.class = VIDEO_ENHANCEMENT_CLASS,
157 		.instance = 3,
158 		.mmio_bases = {
159 			{ .graphics_ver = 12, .base = XEHP_VEBOX4_RING_BASE }
160 		},
161 	},
162 	[CCS0] = {
163 		.class = COMPUTE_CLASS,
164 		.instance = 0,
165 		.mmio_bases = {
166 			{ .graphics_ver = 12, .base = GEN12_COMPUTE0_RING_BASE }
167 		}
168 	},
169 	[CCS1] = {
170 		.class = COMPUTE_CLASS,
171 		.instance = 1,
172 		.mmio_bases = {
173 			{ .graphics_ver = 12, .base = GEN12_COMPUTE1_RING_BASE }
174 		}
175 	},
176 	[CCS2] = {
177 		.class = COMPUTE_CLASS,
178 		.instance = 2,
179 		.mmio_bases = {
180 			{ .graphics_ver = 12, .base = GEN12_COMPUTE2_RING_BASE }
181 		}
182 	},
183 	[CCS3] = {
184 		.class = COMPUTE_CLASS,
185 		.instance = 3,
186 		.mmio_bases = {
187 			{ .graphics_ver = 12, .base = GEN12_COMPUTE3_RING_BASE }
188 		}
189 	},
190 };
191 
192 /**
193  * intel_engine_context_size() - return the size of the context for an engine
194  * @gt: the gt
195  * @class: engine class
196  *
197  * Each engine class may require a different amount of space for a context
198  * image.
199  *
200  * Return: size (in bytes) of an engine class specific context image
201  *
202  * Note: this size includes the HWSP, which is part of the context image
203  * in LRC mode, but does not include the "shared data page" used with
204  * GuC submission. The caller should account for this if using the GuC.
205  */
intel_engine_context_size(struct intel_gt * gt,u8 class)206 u32 intel_engine_context_size(struct intel_gt *gt, u8 class)
207 {
208 	struct intel_uncore *uncore = gt->uncore;
209 	u32 cxt_size;
210 
211 	BUILD_BUG_ON(I915_GTT_PAGE_SIZE != PAGE_SIZE);
212 
213 	switch (class) {
214 	case COMPUTE_CLASS:
215 		fallthrough;
216 	case RENDER_CLASS:
217 		switch (GRAPHICS_VER(gt->i915)) {
218 		default:
219 			MISSING_CASE(GRAPHICS_VER(gt->i915));
220 			return DEFAULT_LR_CONTEXT_RENDER_SIZE;
221 		case 12:
222 		case 11:
223 			return GEN11_LR_CONTEXT_RENDER_SIZE;
224 		case 9:
225 			return GEN9_LR_CONTEXT_RENDER_SIZE;
226 		case 8:
227 			return GEN8_LR_CONTEXT_RENDER_SIZE;
228 		case 7:
229 			if (IS_HASWELL(gt->i915))
230 				return HSW_CXT_TOTAL_SIZE;
231 
232 			cxt_size = intel_uncore_read(uncore, GEN7_CXT_SIZE);
233 			return round_up(GEN7_CXT_TOTAL_SIZE(cxt_size) * 64,
234 					PAGE_SIZE);
235 		case 6:
236 			cxt_size = intel_uncore_read(uncore, CXT_SIZE);
237 			return round_up(GEN6_CXT_TOTAL_SIZE(cxt_size) * 64,
238 					PAGE_SIZE);
239 		case 5:
240 		case 4:
241 			/*
242 			 * There is a discrepancy here between the size reported
243 			 * by the register and the size of the context layout
244 			 * in the docs. Both are described as authorative!
245 			 *
246 			 * The discrepancy is on the order of a few cachelines,
247 			 * but the total is under one page (4k), which is our
248 			 * minimum allocation anyway so it should all come
249 			 * out in the wash.
250 			 */
251 			cxt_size = intel_uncore_read(uncore, CXT_SIZE) + 1;
252 			drm_dbg(&gt->i915->drm,
253 				"graphics_ver = %d CXT_SIZE = %d bytes [0x%08x]\n",
254 				GRAPHICS_VER(gt->i915), cxt_size * 64,
255 				cxt_size - 1);
256 			return round_up(cxt_size * 64, PAGE_SIZE);
257 		case 3:
258 		case 2:
259 		/* For the special day when i810 gets merged. */
260 		case 1:
261 			return 0;
262 		}
263 		break;
264 	default:
265 		MISSING_CASE(class);
266 		fallthrough;
267 	case VIDEO_DECODE_CLASS:
268 	case VIDEO_ENHANCEMENT_CLASS:
269 	case COPY_ENGINE_CLASS:
270 		if (GRAPHICS_VER(gt->i915) < 8)
271 			return 0;
272 		return GEN8_LR_CONTEXT_OTHER_SIZE;
273 	}
274 }
275 
__engine_mmio_base(struct drm_i915_private * i915,const struct engine_mmio_base * bases)276 static u32 __engine_mmio_base(struct drm_i915_private *i915,
277 			      const struct engine_mmio_base *bases)
278 {
279 	int i;
280 
281 	for (i = 0; i < MAX_MMIO_BASES; i++)
282 		if (GRAPHICS_VER(i915) >= bases[i].graphics_ver)
283 			break;
284 
285 	GEM_BUG_ON(i == MAX_MMIO_BASES);
286 	GEM_BUG_ON(!bases[i].base);
287 
288 	return bases[i].base;
289 }
290 
__sprint_engine_name(struct intel_engine_cs * engine)291 static void __sprint_engine_name(struct intel_engine_cs *engine)
292 {
293 	/*
294 	 * Before we know what the uABI name for this engine will be,
295 	 * we still would like to keep track of this engine in the debug logs.
296 	 * We throw in a ' here as a reminder that this isn't its final name.
297 	 */
298 	GEM_WARN_ON(snprintf(engine->name, sizeof(engine->name), "%s'%u",
299 			     intel_engine_class_repr(engine->class),
300 			     engine->instance) >= sizeof(engine->name));
301 }
302 
intel_engine_set_hwsp_writemask(struct intel_engine_cs * engine,u32 mask)303 void intel_engine_set_hwsp_writemask(struct intel_engine_cs *engine, u32 mask)
304 {
305 	/*
306 	 * Though they added more rings on g4x/ilk, they did not add
307 	 * per-engine HWSTAM until gen6.
308 	 */
309 	if (GRAPHICS_VER(engine->i915) < 6 && engine->class != RENDER_CLASS)
310 		return;
311 
312 	if (GRAPHICS_VER(engine->i915) >= 3)
313 		ENGINE_WRITE(engine, RING_HWSTAM, mask);
314 	else
315 		ENGINE_WRITE16(engine, RING_HWSTAM, mask);
316 }
317 
intel_engine_sanitize_mmio(struct intel_engine_cs * engine)318 static void intel_engine_sanitize_mmio(struct intel_engine_cs *engine)
319 {
320 	/* Mask off all writes into the unknown HWSP */
321 	intel_engine_set_hwsp_writemask(engine, ~0u);
322 }
323 
nop_irq_handler(struct intel_engine_cs * engine,u16 iir)324 static void nop_irq_handler(struct intel_engine_cs *engine, u16 iir)
325 {
326 	GEM_DEBUG_WARN_ON(iir);
327 }
328 
get_reset_domain(u8 ver,enum intel_engine_id id)329 static u32 get_reset_domain(u8 ver, enum intel_engine_id id)
330 {
331 	u32 reset_domain;
332 
333 	if (ver >= 11) {
334 		static const u32 engine_reset_domains[] = {
335 			[RCS0]  = GEN11_GRDOM_RENDER,
336 			[BCS0]  = GEN11_GRDOM_BLT,
337 			[VCS0]  = GEN11_GRDOM_MEDIA,
338 			[VCS1]  = GEN11_GRDOM_MEDIA2,
339 			[VCS2]  = GEN11_GRDOM_MEDIA3,
340 			[VCS3]  = GEN11_GRDOM_MEDIA4,
341 			[VCS4]  = GEN11_GRDOM_MEDIA5,
342 			[VCS5]  = GEN11_GRDOM_MEDIA6,
343 			[VCS6]  = GEN11_GRDOM_MEDIA7,
344 			[VCS7]  = GEN11_GRDOM_MEDIA8,
345 			[VECS0] = GEN11_GRDOM_VECS,
346 			[VECS1] = GEN11_GRDOM_VECS2,
347 			[VECS2] = GEN11_GRDOM_VECS3,
348 			[VECS3] = GEN11_GRDOM_VECS4,
349 			[CCS0]  = GEN11_GRDOM_RENDER,
350 			[CCS1]  = GEN11_GRDOM_RENDER,
351 			[CCS2]  = GEN11_GRDOM_RENDER,
352 			[CCS3]  = GEN11_GRDOM_RENDER,
353 		};
354 		GEM_BUG_ON(id >= ARRAY_SIZE(engine_reset_domains) ||
355 			   !engine_reset_domains[id]);
356 		reset_domain = engine_reset_domains[id];
357 	} else {
358 		static const u32 engine_reset_domains[] = {
359 			[RCS0]  = GEN6_GRDOM_RENDER,
360 			[BCS0]  = GEN6_GRDOM_BLT,
361 			[VCS0]  = GEN6_GRDOM_MEDIA,
362 			[VCS1]  = GEN8_GRDOM_MEDIA2,
363 			[VECS0] = GEN6_GRDOM_VECS,
364 		};
365 		GEM_BUG_ON(id >= ARRAY_SIZE(engine_reset_domains) ||
366 			   !engine_reset_domains[id]);
367 		reset_domain = engine_reset_domains[id];
368 	}
369 
370 	return reset_domain;
371 }
372 
intel_engine_setup(struct intel_gt * gt,enum intel_engine_id id,u8 logical_instance)373 static int intel_engine_setup(struct intel_gt *gt, enum intel_engine_id id,
374 			      u8 logical_instance)
375 {
376 	const struct engine_info *info = &intel_engines[id];
377 	struct drm_i915_private *i915 = gt->i915;
378 	struct intel_engine_cs *engine;
379 	u8 guc_class;
380 
381 	BUILD_BUG_ON(MAX_ENGINE_CLASS >= BIT(GEN11_ENGINE_CLASS_WIDTH));
382 	BUILD_BUG_ON(MAX_ENGINE_INSTANCE >= BIT(GEN11_ENGINE_INSTANCE_WIDTH));
383 	BUILD_BUG_ON(I915_MAX_VCS > (MAX_ENGINE_INSTANCE + 1));
384 	BUILD_BUG_ON(I915_MAX_VECS > (MAX_ENGINE_INSTANCE + 1));
385 
386 	if (GEM_DEBUG_WARN_ON(id >= ARRAY_SIZE(gt->engine)))
387 		return -EINVAL;
388 
389 	if (GEM_DEBUG_WARN_ON(info->class > MAX_ENGINE_CLASS))
390 		return -EINVAL;
391 
392 	if (GEM_DEBUG_WARN_ON(info->instance > MAX_ENGINE_INSTANCE))
393 		return -EINVAL;
394 
395 	if (GEM_DEBUG_WARN_ON(gt->engine_class[info->class][info->instance]))
396 		return -EINVAL;
397 
398 	engine = kzalloc(sizeof(*engine), GFP_KERNEL);
399 	if (!engine)
400 		return -ENOMEM;
401 
402 	BUILD_BUG_ON(BITS_PER_TYPE(engine->mask) < I915_NUM_ENGINES);
403 
404 	INIT_LIST_HEAD(&engine->pinned_contexts_list);
405 	engine->id = id;
406 	engine->legacy_idx = INVALID_ENGINE;
407 	engine->mask = BIT(id);
408 	engine->reset_domain = get_reset_domain(GRAPHICS_VER(gt->i915),
409 						id);
410 	engine->i915 = i915;
411 	engine->gt = gt;
412 	engine->uncore = gt->uncore;
413 	guc_class = engine_class_to_guc_class(info->class);
414 	engine->guc_id = MAKE_GUC_ID(guc_class, info->instance);
415 	engine->mmio_base = __engine_mmio_base(i915, info->mmio_bases);
416 
417 	engine->irq_handler = nop_irq_handler;
418 
419 	engine->class = info->class;
420 	engine->instance = info->instance;
421 	engine->logical_mask = BIT(logical_instance);
422 	__sprint_engine_name(engine);
423 
424 	engine->props.heartbeat_interval_ms =
425 		CONFIG_DRM_I915_HEARTBEAT_INTERVAL;
426 	engine->props.max_busywait_duration_ns =
427 		CONFIG_DRM_I915_MAX_REQUEST_BUSYWAIT;
428 	engine->props.preempt_timeout_ms =
429 		CONFIG_DRM_I915_PREEMPT_TIMEOUT;
430 	engine->props.stop_timeout_ms =
431 		CONFIG_DRM_I915_STOP_TIMEOUT;
432 	engine->props.timeslice_duration_ms =
433 		CONFIG_DRM_I915_TIMESLICE_DURATION;
434 
435 	/* Override to uninterruptible for OpenCL workloads. */
436 	if (GRAPHICS_VER(i915) == 12 && engine->class == RENDER_CLASS)
437 		engine->props.preempt_timeout_ms = 0;
438 
439 	if ((engine->class == COMPUTE_CLASS && !RCS_MASK(engine->gt) &&
440 	     __ffs(CCS_MASK(engine->gt)) == engine->instance) ||
441 	     engine->class == RENDER_CLASS)
442 		engine->flags |= I915_ENGINE_FIRST_RENDER_COMPUTE;
443 
444 	/* features common between engines sharing EUs */
445 	if (engine->class == RENDER_CLASS || engine->class == COMPUTE_CLASS) {
446 		engine->flags |= I915_ENGINE_HAS_RCS_REG_STATE;
447 		engine->flags |= I915_ENGINE_HAS_EU_PRIORITY;
448 	}
449 
450 	engine->defaults = engine->props; /* never to change again */
451 
452 	engine->context_size = intel_engine_context_size(gt, engine->class);
453 	if (WARN_ON(engine->context_size > BIT(20)))
454 		engine->context_size = 0;
455 	if (engine->context_size)
456 		DRIVER_CAPS(i915)->has_logical_contexts = true;
457 
458 	ewma__engine_latency_init(&engine->latency);
459 	seqcount_init(&engine->stats.execlists.lock);
460 
461 	ATOMIC_INIT_NOTIFIER_HEAD(&engine->context_status_notifier);
462 
463 	/* Scrub mmio state on takeover */
464 	intel_engine_sanitize_mmio(engine);
465 
466 	gt->engine_class[info->class][info->instance] = engine;
467 	gt->engine[id] = engine;
468 
469 	return 0;
470 }
471 
__setup_engine_capabilities(struct intel_engine_cs * engine)472 static void __setup_engine_capabilities(struct intel_engine_cs *engine)
473 {
474 	struct drm_i915_private *i915 = engine->i915;
475 
476 	if (engine->class == VIDEO_DECODE_CLASS) {
477 		/*
478 		 * HEVC support is present on first engine instance
479 		 * before Gen11 and on all instances afterwards.
480 		 */
481 		if (GRAPHICS_VER(i915) >= 11 ||
482 		    (GRAPHICS_VER(i915) >= 9 && engine->instance == 0))
483 			engine->uabi_capabilities |=
484 				I915_VIDEO_CLASS_CAPABILITY_HEVC;
485 
486 		/*
487 		 * SFC block is present only on even logical engine
488 		 * instances.
489 		 */
490 		if ((GRAPHICS_VER(i915) >= 11 &&
491 		     (engine->gt->info.vdbox_sfc_access &
492 		      BIT(engine->instance))) ||
493 		    (GRAPHICS_VER(i915) >= 9 && engine->instance == 0))
494 			engine->uabi_capabilities |=
495 				I915_VIDEO_AND_ENHANCE_CLASS_CAPABILITY_SFC;
496 	} else if (engine->class == VIDEO_ENHANCEMENT_CLASS) {
497 		if (GRAPHICS_VER(i915) >= 9 &&
498 		    engine->gt->info.sfc_mask & BIT(engine->instance))
499 			engine->uabi_capabilities |=
500 				I915_VIDEO_AND_ENHANCE_CLASS_CAPABILITY_SFC;
501 	}
502 }
503 
intel_setup_engine_capabilities(struct intel_gt * gt)504 static void intel_setup_engine_capabilities(struct intel_gt *gt)
505 {
506 	struct intel_engine_cs *engine;
507 	enum intel_engine_id id;
508 
509 	for_each_engine(engine, gt, id)
510 		__setup_engine_capabilities(engine);
511 }
512 
513 /**
514  * intel_engines_release() - free the resources allocated for Command Streamers
515  * @gt: pointer to struct intel_gt
516  */
intel_engines_release(struct intel_gt * gt)517 void intel_engines_release(struct intel_gt *gt)
518 {
519 	struct intel_engine_cs *engine;
520 	enum intel_engine_id id;
521 
522 	/*
523 	 * Before we release the resources held by engine, we must be certain
524 	 * that the HW is no longer accessing them -- having the GPU scribble
525 	 * to or read from a page being used for something else causes no end
526 	 * of fun.
527 	 *
528 	 * The GPU should be reset by this point, but assume the worst just
529 	 * in case we aborted before completely initialising the engines.
530 	 */
531 	GEM_BUG_ON(intel_gt_pm_is_awake(gt));
532 	if (!INTEL_INFO(gt->i915)->gpu_reset_clobbers_display)
533 		__intel_gt_reset(gt, ALL_ENGINES);
534 
535 	/* Decouple the backend; but keep the layout for late GPU resets */
536 	for_each_engine(engine, gt, id) {
537 		if (!engine->release)
538 			continue;
539 
540 		intel_wakeref_wait_for_idle(&engine->wakeref);
541 		GEM_BUG_ON(intel_engine_pm_is_awake(engine));
542 
543 		engine->release(engine);
544 		engine->release = NULL;
545 
546 		memset(&engine->reset, 0, sizeof(engine->reset));
547 	}
548 }
549 
intel_engine_free_request_pool(struct intel_engine_cs * engine)550 void intel_engine_free_request_pool(struct intel_engine_cs *engine)
551 {
552 	if (!engine->request_pool)
553 		return;
554 
555 	kmem_cache_free(i915_request_slab_cache(), engine->request_pool);
556 }
557 
intel_engines_free(struct intel_gt * gt)558 void intel_engines_free(struct intel_gt *gt)
559 {
560 	struct intel_engine_cs *engine;
561 	enum intel_engine_id id;
562 
563 	/* Free the requests! dma-resv keeps fences around for an eternity */
564 	rcu_barrier();
565 
566 	for_each_engine(engine, gt, id) {
567 		intel_engine_free_request_pool(engine);
568 		kfree(engine);
569 		gt->engine[id] = NULL;
570 	}
571 }
572 
573 static
gen11_vdbox_has_sfc(struct intel_gt * gt,unsigned int physical_vdbox,unsigned int logical_vdbox,u16 vdbox_mask)574 bool gen11_vdbox_has_sfc(struct intel_gt *gt,
575 			 unsigned int physical_vdbox,
576 			 unsigned int logical_vdbox, u16 vdbox_mask)
577 {
578 	struct drm_i915_private *i915 = gt->i915;
579 
580 	/*
581 	 * In Gen11, only even numbered logical VDBOXes are hooked
582 	 * up to an SFC (Scaler & Format Converter) unit.
583 	 * In Gen12, Even numbered physical instance always are connected
584 	 * to an SFC. Odd numbered physical instances have SFC only if
585 	 * previous even instance is fused off.
586 	 *
587 	 * Starting with Xe_HP, there's also a dedicated SFC_ENABLE field
588 	 * in the fuse register that tells us whether a specific SFC is present.
589 	 */
590 	if ((gt->info.sfc_mask & BIT(physical_vdbox / 2)) == 0)
591 		return false;
592 	else if (GRAPHICS_VER(i915) == 12)
593 		return (physical_vdbox % 2 == 0) ||
594 			!(BIT(physical_vdbox - 1) & vdbox_mask);
595 	else if (GRAPHICS_VER(i915) == 11)
596 		return logical_vdbox % 2 == 0;
597 
598 	MISSING_CASE(GRAPHICS_VER(i915));
599 	return false;
600 }
601 
engine_mask_apply_compute_fuses(struct intel_gt * gt)602 static void engine_mask_apply_compute_fuses(struct intel_gt *gt)
603 {
604 	struct drm_i915_private *i915 = gt->i915;
605 	struct intel_gt_info *info = &gt->info;
606 	int ss_per_ccs = info->sseu.max_subslices / I915_MAX_CCS;
607 	unsigned long ccs_mask;
608 	unsigned int i;
609 
610 	if (GRAPHICS_VER_FULL(i915) < IP_VER(12, 50))
611 		return;
612 
613 	ccs_mask = intel_slicemask_from_dssmask(intel_sseu_get_compute_subslices(&info->sseu),
614 						ss_per_ccs);
615 	/*
616 	 * If all DSS in a quadrant are fused off, the corresponding CCS
617 	 * engine is not available for use.
618 	 */
619 	for_each_clear_bit(i, &ccs_mask, I915_MAX_CCS) {
620 		info->engine_mask &= ~BIT(_CCS(i));
621 		drm_dbg(&i915->drm, "ccs%u fused off\n", i);
622 	}
623 }
624 
625 /*
626  * Determine which engines are fused off in our particular hardware.
627  * Note that we have a catch-22 situation where we need to be able to access
628  * the blitter forcewake domain to read the engine fuses, but at the same time
629  * we need to know which engines are available on the system to know which
630  * forcewake domains are present. We solve this by intializing the forcewake
631  * domains based on the full engine mask in the platform capabilities before
632  * calling this function and pruning the domains for fused-off engines
633  * afterwards.
634  */
init_engine_mask(struct intel_gt * gt)635 static intel_engine_mask_t init_engine_mask(struct intel_gt *gt)
636 {
637 	struct drm_i915_private *i915 = gt->i915;
638 	struct intel_gt_info *info = &gt->info;
639 	struct intel_uncore *uncore = gt->uncore;
640 	unsigned int logical_vdbox = 0;
641 	unsigned int i;
642 	u32 media_fuse, fuse1;
643 	u16 vdbox_mask;
644 	u16 vebox_mask;
645 
646 	info->engine_mask = INTEL_INFO(i915)->platform_engine_mask;
647 
648 	if (GRAPHICS_VER(i915) < 11)
649 		return info->engine_mask;
650 
651 	/*
652 	 * On newer platforms the fusing register is called 'enable' and has
653 	 * enable semantics, while on older platforms it is called 'disable'
654 	 * and bits have disable semantices.
655 	 */
656 	media_fuse = intel_uncore_read(uncore, GEN11_GT_VEBOX_VDBOX_DISABLE);
657 	if (GRAPHICS_VER_FULL(i915) < IP_VER(12, 50))
658 		media_fuse = ~media_fuse;
659 
660 	vdbox_mask = media_fuse & GEN11_GT_VDBOX_DISABLE_MASK;
661 	vebox_mask = (media_fuse & GEN11_GT_VEBOX_DISABLE_MASK) >>
662 		      GEN11_GT_VEBOX_DISABLE_SHIFT;
663 
664 	if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50)) {
665 		fuse1 = intel_uncore_read(uncore, HSW_PAVP_FUSE1);
666 		gt->info.sfc_mask = REG_FIELD_GET(XEHP_SFC_ENABLE_MASK, fuse1);
667 	} else {
668 		gt->info.sfc_mask = ~0;
669 	}
670 
671 	for (i = 0; i < I915_MAX_VCS; i++) {
672 		if (!HAS_ENGINE(gt, _VCS(i))) {
673 			vdbox_mask &= ~BIT(i);
674 			continue;
675 		}
676 
677 		if (!(BIT(i) & vdbox_mask)) {
678 			info->engine_mask &= ~BIT(_VCS(i));
679 			drm_dbg(&i915->drm, "vcs%u fused off\n", i);
680 			continue;
681 		}
682 
683 		if (gen11_vdbox_has_sfc(gt, i, logical_vdbox, vdbox_mask))
684 			gt->info.vdbox_sfc_access |= BIT(i);
685 		logical_vdbox++;
686 	}
687 	drm_dbg(&i915->drm, "vdbox enable: %04x, instances: %04lx\n",
688 		vdbox_mask, VDBOX_MASK(gt));
689 	GEM_BUG_ON(vdbox_mask != VDBOX_MASK(gt));
690 
691 	for (i = 0; i < I915_MAX_VECS; i++) {
692 		if (!HAS_ENGINE(gt, _VECS(i))) {
693 			vebox_mask &= ~BIT(i);
694 			continue;
695 		}
696 
697 		if (!(BIT(i) & vebox_mask)) {
698 			info->engine_mask &= ~BIT(_VECS(i));
699 			drm_dbg(&i915->drm, "vecs%u fused off\n", i);
700 		}
701 	}
702 	drm_dbg(&i915->drm, "vebox enable: %04x, instances: %04lx\n",
703 		vebox_mask, VEBOX_MASK(gt));
704 	GEM_BUG_ON(vebox_mask != VEBOX_MASK(gt));
705 
706 	engine_mask_apply_compute_fuses(gt);
707 
708 	return info->engine_mask;
709 }
710 
populate_logical_ids(struct intel_gt * gt,u8 * logical_ids,u8 class,const u8 * map,u8 num_instances)711 static void populate_logical_ids(struct intel_gt *gt, u8 *logical_ids,
712 				 u8 class, const u8 *map, u8 num_instances)
713 {
714 	int i, j;
715 	u8 current_logical_id = 0;
716 
717 	for (j = 0; j < num_instances; ++j) {
718 		for (i = 0; i < ARRAY_SIZE(intel_engines); ++i) {
719 			if (!HAS_ENGINE(gt, i) ||
720 			    intel_engines[i].class != class)
721 				continue;
722 
723 			if (intel_engines[i].instance == map[j]) {
724 				logical_ids[intel_engines[i].instance] =
725 					current_logical_id++;
726 				break;
727 			}
728 		}
729 	}
730 }
731 
setup_logical_ids(struct intel_gt * gt,u8 * logical_ids,u8 class)732 static void setup_logical_ids(struct intel_gt *gt, u8 *logical_ids, u8 class)
733 {
734 	/*
735 	 * Logical to physical mapping is needed for proper support
736 	 * to split-frame feature.
737 	 */
738 	if (MEDIA_VER(gt->i915) >= 11 && class == VIDEO_DECODE_CLASS) {
739 		const u8 map[] = { 0, 2, 4, 6, 1, 3, 5, 7 };
740 
741 		populate_logical_ids(gt, logical_ids, class,
742 				     map, ARRAY_SIZE(map));
743 	} else {
744 		int i;
745 		u8 map[MAX_ENGINE_INSTANCE + 1];
746 
747 		for (i = 0; i < MAX_ENGINE_INSTANCE + 1; ++i)
748 			map[i] = i;
749 		populate_logical_ids(gt, logical_ids, class,
750 				     map, ARRAY_SIZE(map));
751 	}
752 }
753 
754 /**
755  * intel_engines_init_mmio() - allocate and prepare the Engine Command Streamers
756  * @gt: pointer to struct intel_gt
757  *
758  * Return: non-zero if the initialization failed.
759  */
intel_engines_init_mmio(struct intel_gt * gt)760 int intel_engines_init_mmio(struct intel_gt *gt)
761 {
762 	struct drm_i915_private *i915 = gt->i915;
763 	const unsigned int engine_mask = init_engine_mask(gt);
764 	unsigned int mask = 0;
765 	unsigned int i, class;
766 	u8 logical_ids[MAX_ENGINE_INSTANCE + 1];
767 	int err;
768 
769 	drm_WARN_ON(&i915->drm, engine_mask == 0);
770 	drm_WARN_ON(&i915->drm, engine_mask &
771 		    GENMASK(BITS_PER_TYPE(mask) - 1, I915_NUM_ENGINES));
772 
773 	if (i915_inject_probe_failure(i915))
774 		return -ENODEV;
775 
776 	for (class = 0; class < MAX_ENGINE_CLASS + 1; ++class) {
777 		setup_logical_ids(gt, logical_ids, class);
778 
779 		for (i = 0; i < ARRAY_SIZE(intel_engines); ++i) {
780 			u8 instance = intel_engines[i].instance;
781 
782 			if (intel_engines[i].class != class ||
783 			    !HAS_ENGINE(gt, i))
784 				continue;
785 
786 			err = intel_engine_setup(gt, i,
787 						 logical_ids[instance]);
788 			if (err)
789 				goto cleanup;
790 
791 			mask |= BIT(i);
792 		}
793 	}
794 
795 	/*
796 	 * Catch failures to update intel_engines table when the new engines
797 	 * are added to the driver by a warning and disabling the forgotten
798 	 * engines.
799 	 */
800 	if (drm_WARN_ON(&i915->drm, mask != engine_mask))
801 		gt->info.engine_mask = mask;
802 
803 	gt->info.num_engines = hweight32(mask);
804 
805 	intel_gt_check_and_clear_faults(gt);
806 
807 	intel_setup_engine_capabilities(gt);
808 
809 	intel_uncore_prune_engine_fw_domains(gt->uncore, gt);
810 
811 	return 0;
812 
813 cleanup:
814 	intel_engines_free(gt);
815 	return err;
816 }
817 
intel_engine_init_execlists(struct intel_engine_cs * engine)818 void intel_engine_init_execlists(struct intel_engine_cs *engine)
819 {
820 	struct intel_engine_execlists * const execlists = &engine->execlists;
821 
822 	execlists->port_mask = 1;
823 	GEM_BUG_ON(!is_power_of_2(execlists_num_ports(execlists)));
824 	GEM_BUG_ON(execlists_num_ports(execlists) > EXECLIST_MAX_PORTS);
825 
826 	memset(execlists->pending, 0, sizeof(execlists->pending));
827 	execlists->active =
828 		memset(execlists->inflight, 0, sizeof(execlists->inflight));
829 }
830 
cleanup_status_page(struct intel_engine_cs * engine)831 static void cleanup_status_page(struct intel_engine_cs *engine)
832 {
833 	struct i915_vma *vma;
834 
835 	/* Prevent writes into HWSP after returning the page to the system */
836 	intel_engine_set_hwsp_writemask(engine, ~0u);
837 
838 	vma = fetch_and_zero(&engine->status_page.vma);
839 	if (!vma)
840 		return;
841 
842 	if (!HWS_NEEDS_PHYSICAL(engine->i915))
843 		i915_vma_unpin(vma);
844 
845 	i915_gem_object_unpin_map(vma->obj);
846 	i915_gem_object_put(vma->obj);
847 }
848 
pin_ggtt_status_page(struct intel_engine_cs * engine,struct i915_gem_ww_ctx * ww,struct i915_vma * vma)849 static int pin_ggtt_status_page(struct intel_engine_cs *engine,
850 				struct i915_gem_ww_ctx *ww,
851 				struct i915_vma *vma)
852 {
853 	unsigned int flags;
854 
855 	if (!HAS_LLC(engine->i915) && i915_ggtt_has_aperture(engine->gt->ggtt))
856 		/*
857 		 * On g33, we cannot place HWS above 256MiB, so
858 		 * restrict its pinning to the low mappable arena.
859 		 * Though this restriction is not documented for
860 		 * gen4, gen5, or byt, they also behave similarly
861 		 * and hang if the HWS is placed at the top of the
862 		 * GTT. To generalise, it appears that all !llc
863 		 * platforms have issues with us placing the HWS
864 		 * above the mappable region (even though we never
865 		 * actually map it).
866 		 */
867 		flags = PIN_MAPPABLE;
868 	else
869 		flags = PIN_HIGH;
870 
871 	return i915_ggtt_pin(vma, ww, 0, flags);
872 }
873 
init_status_page(struct intel_engine_cs * engine)874 static int init_status_page(struct intel_engine_cs *engine)
875 {
876 	struct drm_i915_gem_object *obj;
877 	struct i915_gem_ww_ctx ww;
878 	struct i915_vma *vma;
879 	void *vaddr;
880 	int ret;
881 
882 	INIT_LIST_HEAD(&engine->status_page.timelines);
883 
884 	/*
885 	 * Though the HWS register does support 36bit addresses, historically
886 	 * we have had hangs and corruption reported due to wild writes if
887 	 * the HWS is placed above 4G. We only allow objects to be allocated
888 	 * in GFP_DMA32 for i965, and no earlier physical address users had
889 	 * access to more than 4G.
890 	 */
891 	obj = i915_gem_object_create_internal(engine->i915, PAGE_SIZE);
892 	if (IS_ERR(obj)) {
893 		drm_err(&engine->i915->drm,
894 			"Failed to allocate status page\n");
895 		return PTR_ERR(obj);
896 	}
897 
898 	i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
899 
900 	vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL);
901 	if (IS_ERR(vma)) {
902 		ret = PTR_ERR(vma);
903 		goto err_put;
904 	}
905 
906 	i915_gem_ww_ctx_init(&ww, true);
907 retry:
908 	ret = i915_gem_object_lock(obj, &ww);
909 	if (!ret && !HWS_NEEDS_PHYSICAL(engine->i915))
910 		ret = pin_ggtt_status_page(engine, &ww, vma);
911 	if (ret)
912 		goto err;
913 
914 	vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
915 	if (IS_ERR(vaddr)) {
916 		ret = PTR_ERR(vaddr);
917 		goto err_unpin;
918 	}
919 
920 	engine->status_page.addr = memset(vaddr, 0, PAGE_SIZE);
921 	engine->status_page.vma = vma;
922 
923 err_unpin:
924 	if (ret)
925 		i915_vma_unpin(vma);
926 err:
927 	if (ret == -EDEADLK) {
928 		ret = i915_gem_ww_ctx_backoff(&ww);
929 		if (!ret)
930 			goto retry;
931 	}
932 	i915_gem_ww_ctx_fini(&ww);
933 err_put:
934 	if (ret)
935 		i915_gem_object_put(obj);
936 	return ret;
937 }
938 
engine_setup_common(struct intel_engine_cs * engine)939 static int engine_setup_common(struct intel_engine_cs *engine)
940 {
941 	int err;
942 
943 	init_llist_head(&engine->barrier_tasks);
944 
945 	err = init_status_page(engine);
946 	if (err)
947 		return err;
948 
949 	engine->breadcrumbs = intel_breadcrumbs_create(engine);
950 	if (!engine->breadcrumbs) {
951 		err = -ENOMEM;
952 		goto err_status;
953 	}
954 
955 	engine->sched_engine = i915_sched_engine_create(ENGINE_PHYSICAL);
956 	if (!engine->sched_engine) {
957 		err = -ENOMEM;
958 		goto err_sched_engine;
959 	}
960 	engine->sched_engine->private_data = engine;
961 
962 	err = intel_engine_init_cmd_parser(engine);
963 	if (err)
964 		goto err_cmd_parser;
965 
966 	intel_engine_init_execlists(engine);
967 	intel_engine_init__pm(engine);
968 	intel_engine_init_retire(engine);
969 
970 	/* Use the whole device by default */
971 	engine->sseu =
972 		intel_sseu_from_device_info(&engine->gt->info.sseu);
973 
974 	intel_engine_init_workarounds(engine);
975 	intel_engine_init_whitelist(engine);
976 	intel_engine_init_ctx_wa(engine);
977 
978 	if (GRAPHICS_VER(engine->i915) >= 12)
979 		engine->flags |= I915_ENGINE_HAS_RELATIVE_MMIO;
980 
981 	return 0;
982 
983 err_cmd_parser:
984 	i915_sched_engine_put(engine->sched_engine);
985 err_sched_engine:
986 	intel_breadcrumbs_put(engine->breadcrumbs);
987 err_status:
988 	cleanup_status_page(engine);
989 	return err;
990 }
991 
992 struct measure_breadcrumb {
993 	struct i915_request rq;
994 	struct intel_ring ring;
995 	u32 cs[2048];
996 };
997 
measure_breadcrumb_dw(struct intel_context * ce)998 static int measure_breadcrumb_dw(struct intel_context *ce)
999 {
1000 	struct intel_engine_cs *engine = ce->engine;
1001 	struct measure_breadcrumb *frame;
1002 	int dw;
1003 
1004 	GEM_BUG_ON(!engine->gt->scratch);
1005 
1006 	frame = kzalloc(sizeof(*frame), GFP_KERNEL);
1007 	if (!frame)
1008 		return -ENOMEM;
1009 
1010 	frame->rq.engine = engine;
1011 	frame->rq.context = ce;
1012 	rcu_assign_pointer(frame->rq.timeline, ce->timeline);
1013 	frame->rq.hwsp_seqno = ce->timeline->hwsp_seqno;
1014 
1015 	frame->ring.vaddr = frame->cs;
1016 	frame->ring.size = sizeof(frame->cs);
1017 	frame->ring.wrap =
1018 		BITS_PER_TYPE(frame->ring.size) - ilog2(frame->ring.size);
1019 	frame->ring.effective_size = frame->ring.size;
1020 	intel_ring_update_space(&frame->ring);
1021 	frame->rq.ring = &frame->ring;
1022 
1023 	mutex_lock(&ce->timeline->mutex);
1024 	spin_lock_irq(&engine->sched_engine->lock);
1025 
1026 	dw = engine->emit_fini_breadcrumb(&frame->rq, frame->cs) - frame->cs;
1027 
1028 	spin_unlock_irq(&engine->sched_engine->lock);
1029 	mutex_unlock(&ce->timeline->mutex);
1030 
1031 	GEM_BUG_ON(dw & 1); /* RING_TAIL must be qword aligned */
1032 
1033 	kfree(frame);
1034 	return dw;
1035 }
1036 
1037 struct intel_context *
intel_engine_create_pinned_context(struct intel_engine_cs * engine,struct i915_address_space * vm,unsigned int ring_size,unsigned int hwsp,struct lock_class_key * key,const char * name)1038 intel_engine_create_pinned_context(struct intel_engine_cs *engine,
1039 				   struct i915_address_space *vm,
1040 				   unsigned int ring_size,
1041 				   unsigned int hwsp,
1042 				   struct lock_class_key *key,
1043 				   const char *name)
1044 {
1045 	struct intel_context *ce;
1046 	int err;
1047 
1048 	ce = intel_context_create(engine);
1049 	if (IS_ERR(ce))
1050 		return ce;
1051 
1052 	__set_bit(CONTEXT_BARRIER_BIT, &ce->flags);
1053 	ce->timeline = page_pack_bits(NULL, hwsp);
1054 	ce->ring = NULL;
1055 	ce->ring_size = ring_size;
1056 
1057 	i915_vm_put(ce->vm);
1058 	ce->vm = i915_vm_get(vm);
1059 
1060 	err = intel_context_pin(ce); /* perma-pin so it is always available */
1061 	if (err) {
1062 		intel_context_put(ce);
1063 		return ERR_PTR(err);
1064 	}
1065 
1066 	list_add_tail(&ce->pinned_contexts_link, &engine->pinned_contexts_list);
1067 
1068 	/*
1069 	 * Give our perma-pinned kernel timelines a separate lockdep class,
1070 	 * so that we can use them from within the normal user timelines
1071 	 * should we need to inject GPU operations during their request
1072 	 * construction.
1073 	 */
1074 	lockdep_set_class_and_name(&ce->timeline->mutex, key, name);
1075 
1076 	return ce;
1077 }
1078 
intel_engine_destroy_pinned_context(struct intel_context * ce)1079 void intel_engine_destroy_pinned_context(struct intel_context *ce)
1080 {
1081 	struct intel_engine_cs *engine = ce->engine;
1082 	struct i915_vma *hwsp = engine->status_page.vma;
1083 
1084 	GEM_BUG_ON(ce->timeline->hwsp_ggtt != hwsp);
1085 
1086 	mutex_lock(&hwsp->vm->mutex);
1087 	list_del(&ce->timeline->engine_link);
1088 	mutex_unlock(&hwsp->vm->mutex);
1089 
1090 	list_del(&ce->pinned_contexts_link);
1091 	intel_context_unpin(ce);
1092 	intel_context_put(ce);
1093 }
1094 
1095 static struct intel_context *
create_kernel_context(struct intel_engine_cs * engine)1096 create_kernel_context(struct intel_engine_cs *engine)
1097 {
1098 	static struct lock_class_key kernel;
1099 
1100 	return intel_engine_create_pinned_context(engine, engine->gt->vm, SZ_4K,
1101 						  I915_GEM_HWS_SEQNO_ADDR,
1102 						  &kernel, "kernel_context");
1103 }
1104 
1105 /**
1106  * intel_engines_init_common - initialize cengine state which might require hw access
1107  * @engine: Engine to initialize.
1108  *
1109  * Initializes @engine@ structure members shared between legacy and execlists
1110  * submission modes which do require hardware access.
1111  *
1112  * Typcally done at later stages of submission mode specific engine setup.
1113  *
1114  * Returns zero on success or an error code on failure.
1115  */
engine_init_common(struct intel_engine_cs * engine)1116 static int engine_init_common(struct intel_engine_cs *engine)
1117 {
1118 	struct intel_context *ce;
1119 	int ret;
1120 
1121 	engine->set_default_submission(engine);
1122 
1123 	/*
1124 	 * We may need to do things with the shrinker which
1125 	 * require us to immediately switch back to the default
1126 	 * context. This can cause a problem as pinning the
1127 	 * default context also requires GTT space which may not
1128 	 * be available. To avoid this we always pin the default
1129 	 * context.
1130 	 */
1131 	ce = create_kernel_context(engine);
1132 	if (IS_ERR(ce))
1133 		return PTR_ERR(ce);
1134 
1135 	ret = measure_breadcrumb_dw(ce);
1136 	if (ret < 0)
1137 		goto err_context;
1138 
1139 	engine->emit_fini_breadcrumb_dw = ret;
1140 	engine->kernel_context = ce;
1141 
1142 	return 0;
1143 
1144 err_context:
1145 	intel_engine_destroy_pinned_context(ce);
1146 	return ret;
1147 }
1148 
intel_engines_init(struct intel_gt * gt)1149 int intel_engines_init(struct intel_gt *gt)
1150 {
1151 	int (*setup)(struct intel_engine_cs *engine);
1152 	struct intel_engine_cs *engine;
1153 	enum intel_engine_id id;
1154 	int err;
1155 
1156 	if (intel_uc_uses_guc_submission(&gt->uc)) {
1157 		gt->submission_method = INTEL_SUBMISSION_GUC;
1158 		setup = intel_guc_submission_setup;
1159 	} else if (HAS_EXECLISTS(gt->i915)) {
1160 		gt->submission_method = INTEL_SUBMISSION_ELSP;
1161 		setup = intel_execlists_submission_setup;
1162 	} else {
1163 		gt->submission_method = INTEL_SUBMISSION_RING;
1164 		setup = intel_ring_submission_setup;
1165 	}
1166 
1167 	for_each_engine(engine, gt, id) {
1168 		err = engine_setup_common(engine);
1169 		if (err)
1170 			return err;
1171 
1172 		err = setup(engine);
1173 		if (err)
1174 			return err;
1175 
1176 		err = engine_init_common(engine);
1177 		if (err)
1178 			return err;
1179 
1180 		intel_engine_add_user(engine);
1181 	}
1182 
1183 	return 0;
1184 }
1185 
1186 /**
1187  * intel_engines_cleanup_common - cleans up the engine state created by
1188  *                                the common initiailizers.
1189  * @engine: Engine to cleanup.
1190  *
1191  * This cleans up everything created by the common helpers.
1192  */
intel_engine_cleanup_common(struct intel_engine_cs * engine)1193 void intel_engine_cleanup_common(struct intel_engine_cs *engine)
1194 {
1195 	GEM_BUG_ON(!list_empty(&engine->sched_engine->requests));
1196 
1197 	i915_sched_engine_put(engine->sched_engine);
1198 	intel_breadcrumbs_put(engine->breadcrumbs);
1199 
1200 	intel_engine_fini_retire(engine);
1201 	intel_engine_cleanup_cmd_parser(engine);
1202 
1203 	if (engine->default_state)
1204 		fput(engine->default_state);
1205 
1206 	if (engine->kernel_context)
1207 		intel_engine_destroy_pinned_context(engine->kernel_context);
1208 
1209 	GEM_BUG_ON(!llist_empty(&engine->barrier_tasks));
1210 	cleanup_status_page(engine);
1211 
1212 	intel_wa_list_free(&engine->ctx_wa_list);
1213 	intel_wa_list_free(&engine->wa_list);
1214 	intel_wa_list_free(&engine->whitelist);
1215 }
1216 
1217 /**
1218  * intel_engine_resume - re-initializes the HW state of the engine
1219  * @engine: Engine to resume.
1220  *
1221  * Returns zero on success or an error code on failure.
1222  */
intel_engine_resume(struct intel_engine_cs * engine)1223 int intel_engine_resume(struct intel_engine_cs *engine)
1224 {
1225 	intel_engine_apply_workarounds(engine);
1226 	intel_engine_apply_whitelist(engine);
1227 
1228 	return engine->resume(engine);
1229 }
1230 
intel_engine_get_active_head(const struct intel_engine_cs * engine)1231 u64 intel_engine_get_active_head(const struct intel_engine_cs *engine)
1232 {
1233 	struct drm_i915_private *i915 = engine->i915;
1234 
1235 	u64 acthd;
1236 
1237 	if (GRAPHICS_VER(i915) >= 8)
1238 		acthd = ENGINE_READ64(engine, RING_ACTHD, RING_ACTHD_UDW);
1239 	else if (GRAPHICS_VER(i915) >= 4)
1240 		acthd = ENGINE_READ(engine, RING_ACTHD);
1241 	else
1242 		acthd = ENGINE_READ(engine, ACTHD);
1243 
1244 	return acthd;
1245 }
1246 
intel_engine_get_last_batch_head(const struct intel_engine_cs * engine)1247 u64 intel_engine_get_last_batch_head(const struct intel_engine_cs *engine)
1248 {
1249 	u64 bbaddr;
1250 
1251 	if (GRAPHICS_VER(engine->i915) >= 8)
1252 		bbaddr = ENGINE_READ64(engine, RING_BBADDR, RING_BBADDR_UDW);
1253 	else
1254 		bbaddr = ENGINE_READ(engine, RING_BBADDR);
1255 
1256 	return bbaddr;
1257 }
1258 
stop_timeout(const struct intel_engine_cs * engine)1259 static unsigned long stop_timeout(const struct intel_engine_cs *engine)
1260 {
1261 	if (in_atomic() || irqs_disabled()) /* inside atomic preempt-reset? */
1262 		return 0;
1263 
1264 	/*
1265 	 * If we are doing a normal GPU reset, we can take our time and allow
1266 	 * the engine to quiesce. We've stopped submission to the engine, and
1267 	 * if we wait long enough an innocent context should complete and
1268 	 * leave the engine idle. So they should not be caught unaware by
1269 	 * the forthcoming GPU reset (which usually follows the stop_cs)!
1270 	 */
1271 	return READ_ONCE(engine->props.stop_timeout_ms);
1272 }
1273 
__intel_engine_stop_cs(struct intel_engine_cs * engine,int fast_timeout_us,int slow_timeout_ms)1274 static int __intel_engine_stop_cs(struct intel_engine_cs *engine,
1275 				  int fast_timeout_us,
1276 				  int slow_timeout_ms)
1277 {
1278 	struct intel_uncore *uncore = engine->uncore;
1279 	const i915_reg_t mode = RING_MI_MODE(engine->mmio_base);
1280 	int err;
1281 
1282 	intel_uncore_write_fw(uncore, mode, _MASKED_BIT_ENABLE(STOP_RING));
1283 
1284 	/*
1285 	 * Wa_22011802037 : gen11, gen12, Prior to doing a reset, ensure CS is
1286 	 * stopped, set ring stop bit and prefetch disable bit to halt CS
1287 	 */
1288 	if (IS_GRAPHICS_VER(engine->i915, 11, 12))
1289 		intel_uncore_write_fw(uncore, RING_MODE_GEN7(engine->mmio_base),
1290 				      _MASKED_BIT_ENABLE(GEN12_GFX_PREFETCH_DISABLE));
1291 
1292 	err = __intel_wait_for_register_fw(engine->uncore, mode,
1293 					   MODE_IDLE, MODE_IDLE,
1294 					   fast_timeout_us,
1295 					   slow_timeout_ms,
1296 					   NULL);
1297 
1298 	/* A final mmio read to let GPU writes be hopefully flushed to memory */
1299 	intel_uncore_posting_read_fw(uncore, mode);
1300 	return err;
1301 }
1302 
intel_engine_stop_cs(struct intel_engine_cs * engine)1303 int intel_engine_stop_cs(struct intel_engine_cs *engine)
1304 {
1305 	int err = 0;
1306 
1307 	if (GRAPHICS_VER(engine->i915) < 3)
1308 		return -ENODEV;
1309 
1310 	ENGINE_TRACE(engine, "\n");
1311 	/*
1312 	 * TODO: Find out why occasionally stopping the CS times out. Seen
1313 	 * especially with gem_eio tests.
1314 	 *
1315 	 * Occasionally trying to stop the cs times out, but does not adversely
1316 	 * affect functionality. The timeout is set as a config parameter that
1317 	 * defaults to 100ms. In most cases the follow up operation is to wait
1318 	 * for pending MI_FORCE_WAKES. The assumption is that this timeout is
1319 	 * sufficient for any pending MI_FORCEWAKEs to complete. Once root
1320 	 * caused, the caller must check and handle the return from this
1321 	 * function.
1322 	 */
1323 	if (__intel_engine_stop_cs(engine, 1000, stop_timeout(engine))) {
1324 		ENGINE_TRACE(engine,
1325 			     "timed out on STOP_RING -> IDLE; HEAD:%04x, TAIL:%04x\n",
1326 			     ENGINE_READ_FW(engine, RING_HEAD) & HEAD_ADDR,
1327 			     ENGINE_READ_FW(engine, RING_TAIL) & TAIL_ADDR);
1328 
1329 		/*
1330 		 * Sometimes we observe that the idle flag is not
1331 		 * set even though the ring is empty. So double
1332 		 * check before giving up.
1333 		 */
1334 		if ((ENGINE_READ_FW(engine, RING_HEAD) & HEAD_ADDR) !=
1335 		    (ENGINE_READ_FW(engine, RING_TAIL) & TAIL_ADDR))
1336 			err = -ETIMEDOUT;
1337 	}
1338 
1339 	return err;
1340 }
1341 
intel_engine_cancel_stop_cs(struct intel_engine_cs * engine)1342 void intel_engine_cancel_stop_cs(struct intel_engine_cs *engine)
1343 {
1344 	ENGINE_TRACE(engine, "\n");
1345 
1346 	ENGINE_WRITE_FW(engine, RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING));
1347 }
1348 
__cs_pending_mi_force_wakes(struct intel_engine_cs * engine)1349 static u32 __cs_pending_mi_force_wakes(struct intel_engine_cs *engine)
1350 {
1351 	static const i915_reg_t _reg[I915_NUM_ENGINES] = {
1352 		[RCS0] = MSG_IDLE_CS,
1353 		[BCS0] = MSG_IDLE_BCS,
1354 		[VCS0] = MSG_IDLE_VCS0,
1355 		[VCS1] = MSG_IDLE_VCS1,
1356 		[VCS2] = MSG_IDLE_VCS2,
1357 		[VCS3] = MSG_IDLE_VCS3,
1358 		[VCS4] = MSG_IDLE_VCS4,
1359 		[VCS5] = MSG_IDLE_VCS5,
1360 		[VCS6] = MSG_IDLE_VCS6,
1361 		[VCS7] = MSG_IDLE_VCS7,
1362 		[VECS0] = MSG_IDLE_VECS0,
1363 		[VECS1] = MSG_IDLE_VECS1,
1364 		[VECS2] = MSG_IDLE_VECS2,
1365 		[VECS3] = MSG_IDLE_VECS3,
1366 		[CCS0] = MSG_IDLE_CS,
1367 		[CCS1] = MSG_IDLE_CS,
1368 		[CCS2] = MSG_IDLE_CS,
1369 		[CCS3] = MSG_IDLE_CS,
1370 	};
1371 	u32 val;
1372 
1373 	if (!_reg[engine->id].reg) {
1374 		drm_err(&engine->i915->drm,
1375 			"MSG IDLE undefined for engine id %u\n", engine->id);
1376 		return 0;
1377 	}
1378 
1379 	val = intel_uncore_read(engine->uncore, _reg[engine->id]);
1380 
1381 	/* bits[29:25] & bits[13:9] >> shift */
1382 	return (val & (val >> 16) & MSG_IDLE_FW_MASK) >> MSG_IDLE_FW_SHIFT;
1383 }
1384 
__gpm_wait_for_fw_complete(struct intel_gt * gt,u32 fw_mask)1385 static void __gpm_wait_for_fw_complete(struct intel_gt *gt, u32 fw_mask)
1386 {
1387 	int ret;
1388 
1389 	/* Ensure GPM receives fw up/down after CS is stopped */
1390 	udelay(1);
1391 
1392 	/* Wait for forcewake request to complete in GPM */
1393 	ret =  __intel_wait_for_register_fw(gt->uncore,
1394 					    GEN9_PWRGT_DOMAIN_STATUS,
1395 					    fw_mask, fw_mask, 5000, 0, NULL);
1396 
1397 	/* Ensure CS receives fw ack from GPM */
1398 	udelay(1);
1399 
1400 	if (ret)
1401 		GT_TRACE(gt, "Failed to complete pending forcewake %d\n", ret);
1402 }
1403 
1404 /*
1405  * Wa_22011802037:gen12: In addition to stopping the cs, we need to wait for any
1406  * pending MI_FORCE_WAKEUP requests that the CS has initiated to complete. The
1407  * pending status is indicated by bits[13:9] (masked by bits[29:25]) in the
1408  * MSG_IDLE register. There's one MSG_IDLE register per reset domain. Since we
1409  * are concerned only with the gt reset here, we use a logical OR of pending
1410  * forcewakeups from all reset domains and then wait for them to complete by
1411  * querying PWRGT_DOMAIN_STATUS.
1412  */
intel_engine_wait_for_pending_mi_fw(struct intel_engine_cs * engine)1413 void intel_engine_wait_for_pending_mi_fw(struct intel_engine_cs *engine)
1414 {
1415 	u32 fw_pending = __cs_pending_mi_force_wakes(engine);
1416 
1417 	if (fw_pending)
1418 		__gpm_wait_for_fw_complete(engine->gt, fw_pending);
1419 }
1420 
1421 static u32
read_subslice_reg(const struct intel_engine_cs * engine,int slice,int subslice,i915_reg_t reg)1422 read_subslice_reg(const struct intel_engine_cs *engine,
1423 		  int slice, int subslice, i915_reg_t reg)
1424 {
1425 	return intel_uncore_read_with_mcr_steering(engine->uncore, reg,
1426 						   slice, subslice);
1427 }
1428 
1429 /* NB: please notice the memset */
intel_engine_get_instdone(const struct intel_engine_cs * engine,struct intel_instdone * instdone)1430 void intel_engine_get_instdone(const struct intel_engine_cs *engine,
1431 			       struct intel_instdone *instdone)
1432 {
1433 	struct drm_i915_private *i915 = engine->i915;
1434 	const struct sseu_dev_info *sseu = &engine->gt->info.sseu;
1435 	struct intel_uncore *uncore = engine->uncore;
1436 	u32 mmio_base = engine->mmio_base;
1437 	int slice;
1438 	int subslice;
1439 	int iter;
1440 
1441 	memset(instdone, 0, sizeof(*instdone));
1442 
1443 	if (GRAPHICS_VER(i915) >= 8) {
1444 		instdone->instdone =
1445 			intel_uncore_read(uncore, RING_INSTDONE(mmio_base));
1446 
1447 		if (engine->id != RCS0)
1448 			return;
1449 
1450 		instdone->slice_common =
1451 			intel_uncore_read(uncore, GEN7_SC_INSTDONE);
1452 		if (GRAPHICS_VER(i915) >= 12) {
1453 			instdone->slice_common_extra[0] =
1454 				intel_uncore_read(uncore, GEN12_SC_INSTDONE_EXTRA);
1455 			instdone->slice_common_extra[1] =
1456 				intel_uncore_read(uncore, GEN12_SC_INSTDONE_EXTRA2);
1457 		}
1458 
1459 		if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50)) {
1460 			for_each_instdone_gslice_dss_xehp(i915, sseu, iter, slice, subslice) {
1461 				instdone->sampler[slice][subslice] =
1462 					read_subslice_reg(engine, slice, subslice,
1463 							  GEN7_SAMPLER_INSTDONE);
1464 				instdone->row[slice][subslice] =
1465 					read_subslice_reg(engine, slice, subslice,
1466 							  GEN7_ROW_INSTDONE);
1467 			}
1468 		} else {
1469 			for_each_instdone_slice_subslice(i915, sseu, slice, subslice) {
1470 				instdone->sampler[slice][subslice] =
1471 					read_subslice_reg(engine, slice, subslice,
1472 							  GEN7_SAMPLER_INSTDONE);
1473 				instdone->row[slice][subslice] =
1474 					read_subslice_reg(engine, slice, subslice,
1475 							  GEN7_ROW_INSTDONE);
1476 			}
1477 		}
1478 
1479 		if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 55)) {
1480 			for_each_instdone_gslice_dss_xehp(i915, sseu, iter, slice, subslice)
1481 				instdone->geom_svg[slice][subslice] =
1482 					read_subslice_reg(engine, slice, subslice,
1483 							  XEHPG_INSTDONE_GEOM_SVG);
1484 		}
1485 	} else if (GRAPHICS_VER(i915) >= 7) {
1486 		instdone->instdone =
1487 			intel_uncore_read(uncore, RING_INSTDONE(mmio_base));
1488 
1489 		if (engine->id != RCS0)
1490 			return;
1491 
1492 		instdone->slice_common =
1493 			intel_uncore_read(uncore, GEN7_SC_INSTDONE);
1494 		instdone->sampler[0][0] =
1495 			intel_uncore_read(uncore, GEN7_SAMPLER_INSTDONE);
1496 		instdone->row[0][0] =
1497 			intel_uncore_read(uncore, GEN7_ROW_INSTDONE);
1498 	} else if (GRAPHICS_VER(i915) >= 4) {
1499 		instdone->instdone =
1500 			intel_uncore_read(uncore, RING_INSTDONE(mmio_base));
1501 		if (engine->id == RCS0)
1502 			/* HACK: Using the wrong struct member */
1503 			instdone->slice_common =
1504 				intel_uncore_read(uncore, GEN4_INSTDONE1);
1505 	} else {
1506 		instdone->instdone = intel_uncore_read(uncore, GEN2_INSTDONE);
1507 	}
1508 }
1509 
ring_is_idle(struct intel_engine_cs * engine)1510 static bool ring_is_idle(struct intel_engine_cs *engine)
1511 {
1512 	bool idle = true;
1513 
1514 	if (I915_SELFTEST_ONLY(!engine->mmio_base))
1515 		return true;
1516 
1517 	if (!intel_engine_pm_get_if_awake(engine))
1518 		return true;
1519 
1520 	/* First check that no commands are left in the ring */
1521 	if ((ENGINE_READ(engine, RING_HEAD) & HEAD_ADDR) !=
1522 	    (ENGINE_READ(engine, RING_TAIL) & TAIL_ADDR))
1523 		idle = false;
1524 
1525 	/* No bit for gen2, so assume the CS parser is idle */
1526 	if (GRAPHICS_VER(engine->i915) > 2 &&
1527 	    !(ENGINE_READ(engine, RING_MI_MODE) & MODE_IDLE))
1528 		idle = false;
1529 
1530 	intel_engine_pm_put(engine);
1531 
1532 	return idle;
1533 }
1534 
__intel_engine_flush_submission(struct intel_engine_cs * engine,bool sync)1535 void __intel_engine_flush_submission(struct intel_engine_cs *engine, bool sync)
1536 {
1537 	struct tasklet_struct *t = &engine->sched_engine->tasklet;
1538 
1539 	if (!t->callback)
1540 		return;
1541 
1542 	local_bh_disable();
1543 	if (tasklet_trylock(t)) {
1544 		/* Must wait for any GPU reset in progress. */
1545 		if (__tasklet_is_enabled(t))
1546 			t->callback(t);
1547 		tasklet_unlock(t);
1548 	}
1549 	local_bh_enable();
1550 
1551 	/* Synchronise and wait for the tasklet on another CPU */
1552 	if (sync)
1553 		tasklet_unlock_wait(t);
1554 }
1555 
1556 /**
1557  * intel_engine_is_idle() - Report if the engine has finished process all work
1558  * @engine: the intel_engine_cs
1559  *
1560  * Return true if there are no requests pending, nothing left to be submitted
1561  * to hardware, and that the engine is idle.
1562  */
intel_engine_is_idle(struct intel_engine_cs * engine)1563 bool intel_engine_is_idle(struct intel_engine_cs *engine)
1564 {
1565 	/* More white lies, if wedged, hw state is inconsistent */
1566 	if (intel_gt_is_wedged(engine->gt))
1567 		return true;
1568 
1569 	if (!intel_engine_pm_is_awake(engine))
1570 		return true;
1571 
1572 	/* Waiting to drain ELSP? */
1573 	intel_synchronize_hardirq(engine->i915);
1574 	intel_engine_flush_submission(engine);
1575 
1576 	/* ELSP is empty, but there are ready requests? E.g. after reset */
1577 	if (!i915_sched_engine_is_empty(engine->sched_engine))
1578 		return false;
1579 
1580 	/* Ring stopped? */
1581 	return ring_is_idle(engine);
1582 }
1583 
intel_engines_are_idle(struct intel_gt * gt)1584 bool intel_engines_are_idle(struct intel_gt *gt)
1585 {
1586 	struct intel_engine_cs *engine;
1587 	enum intel_engine_id id;
1588 
1589 	/*
1590 	 * If the driver is wedged, HW state may be very inconsistent and
1591 	 * report that it is still busy, even though we have stopped using it.
1592 	 */
1593 	if (intel_gt_is_wedged(gt))
1594 		return true;
1595 
1596 	/* Already parked (and passed an idleness test); must still be idle */
1597 	if (!READ_ONCE(gt->awake))
1598 		return true;
1599 
1600 	for_each_engine(engine, gt, id) {
1601 		if (!intel_engine_is_idle(engine))
1602 			return false;
1603 	}
1604 
1605 	return true;
1606 }
1607 
intel_engine_irq_enable(struct intel_engine_cs * engine)1608 bool intel_engine_irq_enable(struct intel_engine_cs *engine)
1609 {
1610 	if (!engine->irq_enable)
1611 		return false;
1612 
1613 	/* Caller disables interrupts */
1614 	spin_lock(&engine->gt->irq_lock);
1615 	engine->irq_enable(engine);
1616 	spin_unlock(&engine->gt->irq_lock);
1617 
1618 	return true;
1619 }
1620 
intel_engine_irq_disable(struct intel_engine_cs * engine)1621 void intel_engine_irq_disable(struct intel_engine_cs *engine)
1622 {
1623 	if (!engine->irq_disable)
1624 		return;
1625 
1626 	/* Caller disables interrupts */
1627 	spin_lock(&engine->gt->irq_lock);
1628 	engine->irq_disable(engine);
1629 	spin_unlock(&engine->gt->irq_lock);
1630 }
1631 
intel_engines_reset_default_submission(struct intel_gt * gt)1632 void intel_engines_reset_default_submission(struct intel_gt *gt)
1633 {
1634 	struct intel_engine_cs *engine;
1635 	enum intel_engine_id id;
1636 
1637 	for_each_engine(engine, gt, id) {
1638 		if (engine->sanitize)
1639 			engine->sanitize(engine);
1640 
1641 		engine->set_default_submission(engine);
1642 	}
1643 }
1644 
intel_engine_can_store_dword(struct intel_engine_cs * engine)1645 bool intel_engine_can_store_dword(struct intel_engine_cs *engine)
1646 {
1647 	switch (GRAPHICS_VER(engine->i915)) {
1648 	case 2:
1649 		return false; /* uses physical not virtual addresses */
1650 	case 3:
1651 		/* maybe only uses physical not virtual addresses */
1652 		return !(IS_I915G(engine->i915) || IS_I915GM(engine->i915));
1653 	case 4:
1654 		return !IS_I965G(engine->i915); /* who knows! */
1655 	case 6:
1656 		return engine->class != VIDEO_DECODE_CLASS; /* b0rked */
1657 	default:
1658 		return true;
1659 	}
1660 }
1661 
get_timeline(struct i915_request * rq)1662 static struct intel_timeline *get_timeline(struct i915_request *rq)
1663 {
1664 	struct intel_timeline *tl;
1665 
1666 	/*
1667 	 * Even though we are holding the engine->sched_engine->lock here, there
1668 	 * is no control over the submission queue per-se and we are
1669 	 * inspecting the active state at a random point in time, with an
1670 	 * unknown queue. Play safe and make sure the timeline remains valid.
1671 	 * (Only being used for pretty printing, one extra kref shouldn't
1672 	 * cause a camel stampede!)
1673 	 */
1674 	rcu_read_lock();
1675 	tl = rcu_dereference(rq->timeline);
1676 	if (!kref_get_unless_zero(&tl->kref))
1677 		tl = NULL;
1678 	rcu_read_unlock();
1679 
1680 	return tl;
1681 }
1682 
print_ring(char * buf,int sz,struct i915_request * rq)1683 static int print_ring(char *buf, int sz, struct i915_request *rq)
1684 {
1685 	int len = 0;
1686 
1687 	if (!i915_request_signaled(rq)) {
1688 		struct intel_timeline *tl = get_timeline(rq);
1689 
1690 		len = scnprintf(buf, sz,
1691 				"ring:{start:%08x, hwsp:%08x, seqno:%08x, runtime:%llums}, ",
1692 				i915_ggtt_offset(rq->ring->vma),
1693 				tl ? tl->hwsp_offset : 0,
1694 				hwsp_seqno(rq),
1695 				DIV_ROUND_CLOSEST_ULL(intel_context_get_total_runtime_ns(rq->context),
1696 						      1000 * 1000));
1697 
1698 		if (tl)
1699 			intel_timeline_put(tl);
1700 	}
1701 
1702 	return len;
1703 }
1704 
hexdump(struct drm_printer * m,const void * buf,size_t len)1705 static void hexdump(struct drm_printer *m, const void *buf, size_t len)
1706 {
1707 	const size_t rowsize = 8 * sizeof(u32);
1708 	const void *prev = NULL;
1709 	bool skip = false;
1710 	size_t pos;
1711 
1712 	for (pos = 0; pos < len; pos += rowsize) {
1713 		char line[128];
1714 
1715 		if (prev && !memcmp(prev, buf + pos, rowsize)) {
1716 			if (!skip) {
1717 				drm_printf(m, "*\n");
1718 				skip = true;
1719 			}
1720 			continue;
1721 		}
1722 
1723 		WARN_ON_ONCE(hex_dump_to_buffer(buf + pos, len - pos,
1724 						rowsize, sizeof(u32),
1725 						line, sizeof(line),
1726 						false) >= sizeof(line));
1727 		drm_printf(m, "[%04zx] %s\n", pos, line);
1728 
1729 		prev = buf + pos;
1730 		skip = false;
1731 	}
1732 }
1733 
repr_timer(const struct timer_list * t)1734 static const char *repr_timer(const struct timer_list *t)
1735 {
1736 	if (!READ_ONCE(t->expires))
1737 		return "inactive";
1738 
1739 	if (timer_pending(t))
1740 		return "active";
1741 
1742 	return "expired";
1743 }
1744 
intel_engine_print_registers(struct intel_engine_cs * engine,struct drm_printer * m)1745 static void intel_engine_print_registers(struct intel_engine_cs *engine,
1746 					 struct drm_printer *m)
1747 {
1748 	struct drm_i915_private *dev_priv = engine->i915;
1749 	struct intel_engine_execlists * const execlists = &engine->execlists;
1750 	u64 addr;
1751 
1752 	if (engine->id == RENDER_CLASS && IS_GRAPHICS_VER(dev_priv, 4, 7))
1753 		drm_printf(m, "\tCCID: 0x%08x\n", ENGINE_READ(engine, CCID));
1754 	if (HAS_EXECLISTS(dev_priv)) {
1755 		drm_printf(m, "\tEL_STAT_HI: 0x%08x\n",
1756 			   ENGINE_READ(engine, RING_EXECLIST_STATUS_HI));
1757 		drm_printf(m, "\tEL_STAT_LO: 0x%08x\n",
1758 			   ENGINE_READ(engine, RING_EXECLIST_STATUS_LO));
1759 	}
1760 	drm_printf(m, "\tRING_START: 0x%08x\n",
1761 		   ENGINE_READ(engine, RING_START));
1762 	drm_printf(m, "\tRING_HEAD:  0x%08x\n",
1763 		   ENGINE_READ(engine, RING_HEAD) & HEAD_ADDR);
1764 	drm_printf(m, "\tRING_TAIL:  0x%08x\n",
1765 		   ENGINE_READ(engine, RING_TAIL) & TAIL_ADDR);
1766 	drm_printf(m, "\tRING_CTL:   0x%08x%s\n",
1767 		   ENGINE_READ(engine, RING_CTL),
1768 		   ENGINE_READ(engine, RING_CTL) & (RING_WAIT | RING_WAIT_SEMAPHORE) ? " [waiting]" : "");
1769 	if (GRAPHICS_VER(engine->i915) > 2) {
1770 		drm_printf(m, "\tRING_MODE:  0x%08x%s\n",
1771 			   ENGINE_READ(engine, RING_MI_MODE),
1772 			   ENGINE_READ(engine, RING_MI_MODE) & (MODE_IDLE) ? " [idle]" : "");
1773 	}
1774 
1775 	if (GRAPHICS_VER(dev_priv) >= 6) {
1776 		drm_printf(m, "\tRING_IMR:   0x%08x\n",
1777 			   ENGINE_READ(engine, RING_IMR));
1778 		drm_printf(m, "\tRING_ESR:   0x%08x\n",
1779 			   ENGINE_READ(engine, RING_ESR));
1780 		drm_printf(m, "\tRING_EMR:   0x%08x\n",
1781 			   ENGINE_READ(engine, RING_EMR));
1782 		drm_printf(m, "\tRING_EIR:   0x%08x\n",
1783 			   ENGINE_READ(engine, RING_EIR));
1784 	}
1785 
1786 	addr = intel_engine_get_active_head(engine);
1787 	drm_printf(m, "\tACTHD:  0x%08x_%08x\n",
1788 		   upper_32_bits(addr), lower_32_bits(addr));
1789 	addr = intel_engine_get_last_batch_head(engine);
1790 	drm_printf(m, "\tBBADDR: 0x%08x_%08x\n",
1791 		   upper_32_bits(addr), lower_32_bits(addr));
1792 	if (GRAPHICS_VER(dev_priv) >= 8)
1793 		addr = ENGINE_READ64(engine, RING_DMA_FADD, RING_DMA_FADD_UDW);
1794 	else if (GRAPHICS_VER(dev_priv) >= 4)
1795 		addr = ENGINE_READ(engine, RING_DMA_FADD);
1796 	else
1797 		addr = ENGINE_READ(engine, DMA_FADD_I8XX);
1798 	drm_printf(m, "\tDMA_FADDR: 0x%08x_%08x\n",
1799 		   upper_32_bits(addr), lower_32_bits(addr));
1800 	if (GRAPHICS_VER(dev_priv) >= 4) {
1801 		drm_printf(m, "\tIPEIR: 0x%08x\n",
1802 			   ENGINE_READ(engine, RING_IPEIR));
1803 		drm_printf(m, "\tIPEHR: 0x%08x\n",
1804 			   ENGINE_READ(engine, RING_IPEHR));
1805 	} else {
1806 		drm_printf(m, "\tIPEIR: 0x%08x\n", ENGINE_READ(engine, IPEIR));
1807 		drm_printf(m, "\tIPEHR: 0x%08x\n", ENGINE_READ(engine, IPEHR));
1808 	}
1809 
1810 	if (HAS_EXECLISTS(dev_priv) && !intel_engine_uses_guc(engine)) {
1811 		struct i915_request * const *port, *rq;
1812 		const u32 *hws =
1813 			&engine->status_page.addr[I915_HWS_CSB_BUF0_INDEX];
1814 		const u8 num_entries = execlists->csb_size;
1815 		unsigned int idx;
1816 		u8 read, write;
1817 
1818 		drm_printf(m, "\tExeclist tasklet queued? %s (%s), preempt? %s, timeslice? %s\n",
1819 			   str_yes_no(test_bit(TASKLET_STATE_SCHED, &engine->sched_engine->tasklet.state)),
1820 			   str_enabled_disabled(!atomic_read(&engine->sched_engine->tasklet.count)),
1821 			   repr_timer(&engine->execlists.preempt),
1822 			   repr_timer(&engine->execlists.timer));
1823 
1824 		read = execlists->csb_head;
1825 		write = READ_ONCE(*execlists->csb_write);
1826 
1827 		drm_printf(m, "\tExeclist status: 0x%08x %08x; CSB read:%d, write:%d, entries:%d\n",
1828 			   ENGINE_READ(engine, RING_EXECLIST_STATUS_LO),
1829 			   ENGINE_READ(engine, RING_EXECLIST_STATUS_HI),
1830 			   read, write, num_entries);
1831 
1832 		if (read >= num_entries)
1833 			read = 0;
1834 		if (write >= num_entries)
1835 			write = 0;
1836 		if (read > write)
1837 			write += num_entries;
1838 		while (read < write) {
1839 			idx = ++read % num_entries;
1840 			drm_printf(m, "\tExeclist CSB[%d]: 0x%08x, context: %d\n",
1841 				   idx, hws[idx * 2], hws[idx * 2 + 1]);
1842 		}
1843 
1844 		i915_sched_engine_active_lock_bh(engine->sched_engine);
1845 		rcu_read_lock();
1846 		for (port = execlists->active; (rq = *port); port++) {
1847 			char hdr[160];
1848 			int len;
1849 
1850 			len = scnprintf(hdr, sizeof(hdr),
1851 					"\t\tActive[%d]:  ccid:%08x%s%s, ",
1852 					(int)(port - execlists->active),
1853 					rq->context->lrc.ccid,
1854 					intel_context_is_closed(rq->context) ? "!" : "",
1855 					intel_context_is_banned(rq->context) ? "*" : "");
1856 			len += print_ring(hdr + len, sizeof(hdr) - len, rq);
1857 			scnprintf(hdr + len, sizeof(hdr) - len, "rq: ");
1858 			i915_request_show(m, rq, hdr, 0);
1859 		}
1860 		for (port = execlists->pending; (rq = *port); port++) {
1861 			char hdr[160];
1862 			int len;
1863 
1864 			len = scnprintf(hdr, sizeof(hdr),
1865 					"\t\tPending[%d]: ccid:%08x%s%s, ",
1866 					(int)(port - execlists->pending),
1867 					rq->context->lrc.ccid,
1868 					intel_context_is_closed(rq->context) ? "!" : "",
1869 					intel_context_is_banned(rq->context) ? "*" : "");
1870 			len += print_ring(hdr + len, sizeof(hdr) - len, rq);
1871 			scnprintf(hdr + len, sizeof(hdr) - len, "rq: ");
1872 			i915_request_show(m, rq, hdr, 0);
1873 		}
1874 		rcu_read_unlock();
1875 		i915_sched_engine_active_unlock_bh(engine->sched_engine);
1876 	} else if (GRAPHICS_VER(dev_priv) > 6) {
1877 		drm_printf(m, "\tPP_DIR_BASE: 0x%08x\n",
1878 			   ENGINE_READ(engine, RING_PP_DIR_BASE));
1879 		drm_printf(m, "\tPP_DIR_BASE_READ: 0x%08x\n",
1880 			   ENGINE_READ(engine, RING_PP_DIR_BASE_READ));
1881 		drm_printf(m, "\tPP_DIR_DCLV: 0x%08x\n",
1882 			   ENGINE_READ(engine, RING_PP_DIR_DCLV));
1883 	}
1884 }
1885 
print_request_ring(struct drm_printer * m,struct i915_request * rq)1886 static void print_request_ring(struct drm_printer *m, struct i915_request *rq)
1887 {
1888 	struct i915_vma_resource *vma_res = rq->batch_res;
1889 	void *ring;
1890 	int size;
1891 
1892 	drm_printf(m,
1893 		   "[head %04x, postfix %04x, tail %04x, batch 0x%08x_%08x]:\n",
1894 		   rq->head, rq->postfix, rq->tail,
1895 		   vma_res ? upper_32_bits(vma_res->start) : ~0u,
1896 		   vma_res ? lower_32_bits(vma_res->start) : ~0u);
1897 
1898 	size = rq->tail - rq->head;
1899 	if (rq->tail < rq->head)
1900 		size += rq->ring->size;
1901 
1902 	ring = kmalloc(size, GFP_ATOMIC);
1903 	if (ring) {
1904 		const void *vaddr = rq->ring->vaddr;
1905 		unsigned int head = rq->head;
1906 		unsigned int len = 0;
1907 
1908 		if (rq->tail < head) {
1909 			len = rq->ring->size - head;
1910 			memcpy(ring, vaddr + head, len);
1911 			head = 0;
1912 		}
1913 		memcpy(ring + len, vaddr + head, size - len);
1914 
1915 		hexdump(m, ring, size);
1916 		kfree(ring);
1917 	}
1918 }
1919 
list_count(struct list_head * list)1920 static unsigned long list_count(struct list_head *list)
1921 {
1922 	struct list_head *pos;
1923 	unsigned long count = 0;
1924 
1925 	list_for_each(pos, list)
1926 		count++;
1927 
1928 	return count;
1929 }
1930 
read_ul(void * p,size_t x)1931 static unsigned long read_ul(void *p, size_t x)
1932 {
1933 	return *(unsigned long *)(p + x);
1934 }
1935 
print_properties(struct intel_engine_cs * engine,struct drm_printer * m)1936 static void print_properties(struct intel_engine_cs *engine,
1937 			     struct drm_printer *m)
1938 {
1939 	static const struct pmap {
1940 		size_t offset;
1941 		const char *name;
1942 	} props[] = {
1943 #define P(x) { \
1944 	.offset = offsetof(typeof(engine->props), x), \
1945 	.name = #x \
1946 }
1947 		P(heartbeat_interval_ms),
1948 		P(max_busywait_duration_ns),
1949 		P(preempt_timeout_ms),
1950 		P(stop_timeout_ms),
1951 		P(timeslice_duration_ms),
1952 
1953 		{},
1954 #undef P
1955 	};
1956 	const struct pmap *p;
1957 
1958 	drm_printf(m, "\tProperties:\n");
1959 	for (p = props; p->name; p++)
1960 		drm_printf(m, "\t\t%s: %lu [default %lu]\n",
1961 			   p->name,
1962 			   read_ul(&engine->props, p->offset),
1963 			   read_ul(&engine->defaults, p->offset));
1964 }
1965 
engine_dump_request(struct i915_request * rq,struct drm_printer * m,const char * msg)1966 static void engine_dump_request(struct i915_request *rq, struct drm_printer *m, const char *msg)
1967 {
1968 	struct intel_timeline *tl = get_timeline(rq);
1969 
1970 	i915_request_show(m, rq, msg, 0);
1971 
1972 	drm_printf(m, "\t\tring->start:  0x%08x\n",
1973 		   i915_ggtt_offset(rq->ring->vma));
1974 	drm_printf(m, "\t\tring->head:   0x%08x\n",
1975 		   rq->ring->head);
1976 	drm_printf(m, "\t\tring->tail:   0x%08x\n",
1977 		   rq->ring->tail);
1978 	drm_printf(m, "\t\tring->emit:   0x%08x\n",
1979 		   rq->ring->emit);
1980 	drm_printf(m, "\t\tring->space:  0x%08x\n",
1981 		   rq->ring->space);
1982 
1983 	if (tl) {
1984 		drm_printf(m, "\t\tring->hwsp:   0x%08x\n",
1985 			   tl->hwsp_offset);
1986 		intel_timeline_put(tl);
1987 	}
1988 
1989 	print_request_ring(m, rq);
1990 
1991 	if (rq->context->lrc_reg_state) {
1992 		drm_printf(m, "Logical Ring Context:\n");
1993 		hexdump(m, rq->context->lrc_reg_state, PAGE_SIZE);
1994 	}
1995 }
1996 
intel_engine_dump_active_requests(struct list_head * requests,struct i915_request * hung_rq,struct drm_printer * m)1997 void intel_engine_dump_active_requests(struct list_head *requests,
1998 				       struct i915_request *hung_rq,
1999 				       struct drm_printer *m)
2000 {
2001 	struct i915_request *rq;
2002 	const char *msg;
2003 	enum i915_request_state state;
2004 
2005 	list_for_each_entry(rq, requests, sched.link) {
2006 		if (rq == hung_rq)
2007 			continue;
2008 
2009 		state = i915_test_request_state(rq);
2010 		if (state < I915_REQUEST_QUEUED)
2011 			continue;
2012 
2013 		if (state == I915_REQUEST_ACTIVE)
2014 			msg = "\t\tactive on engine";
2015 		else
2016 			msg = "\t\tactive in queue";
2017 
2018 		engine_dump_request(rq, m, msg);
2019 	}
2020 }
2021 
engine_dump_active_requests(struct intel_engine_cs * engine,struct drm_printer * m)2022 static void engine_dump_active_requests(struct intel_engine_cs *engine, struct drm_printer *m)
2023 {
2024 	struct i915_request *hung_rq = NULL;
2025 	struct intel_context *ce;
2026 	bool guc;
2027 
2028 	/*
2029 	 * No need for an engine->irq_seqno_barrier() before the seqno reads.
2030 	 * The GPU is still running so requests are still executing and any
2031 	 * hardware reads will be out of date by the time they are reported.
2032 	 * But the intention here is just to report an instantaneous snapshot
2033 	 * so that's fine.
2034 	 */
2035 	lockdep_assert_held(&engine->sched_engine->lock);
2036 
2037 	drm_printf(m, "\tRequests:\n");
2038 
2039 	guc = intel_uc_uses_guc_submission(&engine->gt->uc);
2040 	if (guc) {
2041 		ce = intel_engine_get_hung_context(engine);
2042 		if (ce)
2043 			hung_rq = intel_context_find_active_request(ce);
2044 	} else {
2045 		hung_rq = intel_engine_execlist_find_hung_request(engine);
2046 	}
2047 
2048 	if (hung_rq)
2049 		engine_dump_request(hung_rq, m, "\t\thung");
2050 
2051 	if (guc)
2052 		intel_guc_dump_active_requests(engine, hung_rq, m);
2053 	else
2054 		intel_engine_dump_active_requests(&engine->sched_engine->requests,
2055 						  hung_rq, m);
2056 }
2057 
intel_engine_dump(struct intel_engine_cs * engine,struct drm_printer * m,const char * header,...)2058 void intel_engine_dump(struct intel_engine_cs *engine,
2059 		       struct drm_printer *m,
2060 		       const char *header, ...)
2061 {
2062 	struct i915_gpu_error * const error = &engine->i915->gpu_error;
2063 	struct i915_request *rq;
2064 	intel_wakeref_t wakeref;
2065 	unsigned long flags;
2066 	ktime_t dummy;
2067 
2068 	if (header) {
2069 		va_list ap;
2070 
2071 		va_start(ap, header);
2072 		drm_vprintf(m, header, &ap);
2073 		va_end(ap);
2074 	}
2075 
2076 	if (intel_gt_is_wedged(engine->gt))
2077 		drm_printf(m, "*** WEDGED ***\n");
2078 
2079 	drm_printf(m, "\tAwake? %d\n", atomic_read(&engine->wakeref.count));
2080 	drm_printf(m, "\tBarriers?: %s\n",
2081 		   str_yes_no(!llist_empty(&engine->barrier_tasks)));
2082 	drm_printf(m, "\tLatency: %luus\n",
2083 		   ewma__engine_latency_read(&engine->latency));
2084 	if (intel_engine_supports_stats(engine))
2085 		drm_printf(m, "\tRuntime: %llums\n",
2086 			   ktime_to_ms(intel_engine_get_busy_time(engine,
2087 								  &dummy)));
2088 	drm_printf(m, "\tForcewake: %x domains, %d active\n",
2089 		   engine->fw_domain, READ_ONCE(engine->fw_active));
2090 
2091 	rcu_read_lock();
2092 	rq = READ_ONCE(engine->heartbeat.systole);
2093 	if (rq)
2094 		drm_printf(m, "\tHeartbeat: %d ms ago\n",
2095 			   jiffies_to_msecs(jiffies - rq->emitted_jiffies));
2096 	rcu_read_unlock();
2097 	drm_printf(m, "\tReset count: %d (global %d)\n",
2098 		   i915_reset_engine_count(error, engine),
2099 		   i915_reset_count(error));
2100 	print_properties(engine, m);
2101 
2102 	spin_lock_irqsave(&engine->sched_engine->lock, flags);
2103 	engine_dump_active_requests(engine, m);
2104 
2105 	drm_printf(m, "\tOn hold?: %lu\n",
2106 		   list_count(&engine->sched_engine->hold));
2107 	spin_unlock_irqrestore(&engine->sched_engine->lock, flags);
2108 
2109 	drm_printf(m, "\tMMIO base:  0x%08x\n", engine->mmio_base);
2110 	wakeref = intel_runtime_pm_get_if_in_use(engine->uncore->rpm);
2111 	if (wakeref) {
2112 		intel_engine_print_registers(engine, m);
2113 		intel_runtime_pm_put(engine->uncore->rpm, wakeref);
2114 	} else {
2115 		drm_printf(m, "\tDevice is asleep; skipping register dump\n");
2116 	}
2117 
2118 	intel_execlists_show_requests(engine, m, i915_request_show, 8);
2119 
2120 	drm_printf(m, "HWSP:\n");
2121 	hexdump(m, engine->status_page.addr, PAGE_SIZE);
2122 
2123 	drm_printf(m, "Idle? %s\n", str_yes_no(intel_engine_is_idle(engine)));
2124 
2125 	intel_engine_print_breadcrumbs(engine, m);
2126 }
2127 
2128 /**
2129  * intel_engine_get_busy_time() - Return current accumulated engine busyness
2130  * @engine: engine to report on
2131  * @now: monotonic timestamp of sampling
2132  *
2133  * Returns accumulated time @engine was busy since engine stats were enabled.
2134  */
intel_engine_get_busy_time(struct intel_engine_cs * engine,ktime_t * now)2135 ktime_t intel_engine_get_busy_time(struct intel_engine_cs *engine, ktime_t *now)
2136 {
2137 	return engine->busyness(engine, now);
2138 }
2139 
2140 struct intel_context *
intel_engine_create_virtual(struct intel_engine_cs ** siblings,unsigned int count,unsigned long flags)2141 intel_engine_create_virtual(struct intel_engine_cs **siblings,
2142 			    unsigned int count, unsigned long flags)
2143 {
2144 	if (count == 0)
2145 		return ERR_PTR(-EINVAL);
2146 
2147 	if (count == 1 && !(flags & FORCE_VIRTUAL))
2148 		return intel_context_create(siblings[0]);
2149 
2150 	GEM_BUG_ON(!siblings[0]->cops->create_virtual);
2151 	return siblings[0]->cops->create_virtual(siblings, count, flags);
2152 }
2153 
2154 struct i915_request *
intel_engine_execlist_find_hung_request(struct intel_engine_cs * engine)2155 intel_engine_execlist_find_hung_request(struct intel_engine_cs *engine)
2156 {
2157 	struct i915_request *request, *active = NULL;
2158 
2159 	/*
2160 	 * This search does not work in GuC submission mode. However, the GuC
2161 	 * will report the hanging context directly to the driver itself. So
2162 	 * the driver should never get here when in GuC mode.
2163 	 */
2164 	GEM_BUG_ON(intel_uc_uses_guc_submission(&engine->gt->uc));
2165 
2166 	/*
2167 	 * We are called by the error capture, reset and to dump engine
2168 	 * state at random points in time. In particular, note that neither is
2169 	 * crucially ordered with an interrupt. After a hang, the GPU is dead
2170 	 * and we assume that no more writes can happen (we waited long enough
2171 	 * for all writes that were in transaction to be flushed) - adding an
2172 	 * extra delay for a recent interrupt is pointless. Hence, we do
2173 	 * not need an engine->irq_seqno_barrier() before the seqno reads.
2174 	 * At all other times, we must assume the GPU is still running, but
2175 	 * we only care about the snapshot of this moment.
2176 	 */
2177 	lockdep_assert_held(&engine->sched_engine->lock);
2178 
2179 	rcu_read_lock();
2180 	request = execlists_active(&engine->execlists);
2181 	if (request) {
2182 		struct intel_timeline *tl = request->context->timeline;
2183 
2184 		list_for_each_entry_from_reverse(request, &tl->requests, link) {
2185 			if (__i915_request_is_complete(request))
2186 				break;
2187 
2188 			active = request;
2189 		}
2190 	}
2191 	rcu_read_unlock();
2192 	if (active)
2193 		return active;
2194 
2195 	list_for_each_entry(request, &engine->sched_engine->requests,
2196 			    sched.link) {
2197 		if (i915_test_request_state(request) != I915_REQUEST_ACTIVE)
2198 			continue;
2199 
2200 		active = request;
2201 		break;
2202 	}
2203 
2204 	return active;
2205 }
2206 
xehp_enable_ccs_engines(struct intel_engine_cs * engine)2207 void xehp_enable_ccs_engines(struct intel_engine_cs *engine)
2208 {
2209 	/*
2210 	 * If there are any non-fused-off CCS engines, we need to enable CCS
2211 	 * support in the RCU_MODE register.  This only needs to be done once,
2212 	 * so for simplicity we'll take care of this in the RCS engine's
2213 	 * resume handler; since the RCS and all CCS engines belong to the
2214 	 * same reset domain and are reset together, this will also take care
2215 	 * of re-applying the setting after i915-triggered resets.
2216 	 */
2217 	if (!CCS_MASK(engine->gt))
2218 		return;
2219 
2220 	intel_uncore_write(engine->uncore, GEN12_RCU_MODE,
2221 			   _MASKED_BIT_ENABLE(GEN12_RCU_MODE_CCS_ENABLE));
2222 }
2223 
2224 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
2225 #include "mock_engine.c"
2226 #include "selftest_engine.c"
2227 #include "selftest_engine_cs.c"
2228 #endif
2229