1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2014 Intel Corporation
4  */
5 
6 /**
7  * DOC: Logical Rings, Logical Ring Contexts and Execlists
8  *
9  * Motivation:
10  * GEN8 brings an expansion of the HW contexts: "Logical Ring Contexts".
11  * These expanded contexts enable a number of new abilities, especially
12  * "Execlists" (also implemented in this file).
13  *
14  * One of the main differences with the legacy HW contexts is that logical
15  * ring contexts incorporate many more things to the context's state, like
16  * PDPs or ringbuffer control registers:
17  *
18  * The reason why PDPs are included in the context is straightforward: as
19  * PPGTTs (per-process GTTs) are actually per-context, having the PDPs
20  * contained there mean you don't need to do a ppgtt->switch_mm yourself,
21  * instead, the GPU will do it for you on the context switch.
22  *
23  * But, what about the ringbuffer control registers (head, tail, etc..)?
24  * shouldn't we just need a set of those per engine command streamer? This is
25  * where the name "Logical Rings" starts to make sense: by virtualizing the
26  * rings, the engine cs shifts to a new "ring buffer" with every context
27  * switch. When you want to submit a workload to the GPU you: A) choose your
28  * context, B) find its appropriate virtualized ring, C) write commands to it
29  * and then, finally, D) tell the GPU to switch to that context.
30  *
31  * Instead of the legacy MI_SET_CONTEXT, the way you tell the GPU to switch
32  * to a contexts is via a context execution list, ergo "Execlists".
33  *
34  * LRC implementation:
35  * Regarding the creation of contexts, we have:
36  *
37  * - One global default context.
38  * - One local default context for each opened fd.
39  * - One local extra context for each context create ioctl call.
40  *
41  * Now that ringbuffers belong per-context (and not per-engine, like before)
42  * and that contexts are uniquely tied to a given engine (and not reusable,
43  * like before) we need:
44  *
45  * - One ringbuffer per-engine inside each context.
46  * - One backing object per-engine inside each context.
47  *
48  * The global default context starts its life with these new objects fully
49  * allocated and populated. The local default context for each opened fd is
50  * more complex, because we don't know at creation time which engine is going
51  * to use them. To handle this, we have implemented a deferred creation of LR
52  * contexts:
53  *
54  * The local context starts its life as a hollow or blank holder, that only
55  * gets populated for a given engine once we receive an execbuffer. If later
56  * on we receive another execbuffer ioctl for the same context but a different
57  * engine, we allocate/populate a new ringbuffer and context backing object and
58  * so on.
59  *
60  * Finally, regarding local contexts created using the ioctl call: as they are
61  * only allowed with the render ring, we can allocate & populate them right
62  * away (no need to defer anything, at least for now).
63  *
64  * Execlists implementation:
65  * Execlists are the new method by which, on gen8+ hardware, workloads are
66  * submitted for execution (as opposed to the legacy, ringbuffer-based, method).
67  * This method works as follows:
68  *
69  * When a request is committed, its commands (the BB start and any leading or
70  * trailing commands, like the seqno breadcrumbs) are placed in the ringbuffer
71  * for the appropriate context. The tail pointer in the hardware context is not
72  * updated at this time, but instead, kept by the driver in the ringbuffer
73  * structure. A structure representing this request is added to a request queue
74  * for the appropriate engine: this structure contains a copy of the context's
75  * tail after the request was written to the ring buffer and a pointer to the
76  * context itself.
77  *
78  * If the engine's request queue was empty before the request was added, the
79  * queue is processed immediately. Otherwise the queue will be processed during
80  * a context switch interrupt. In any case, elements on the queue will get sent
81  * (in pairs) to the GPU's ExecLists Submit Port (ELSP, for short) with a
82  * globally unique 20-bits submission ID.
83  *
84  * When execution of a request completes, the GPU updates the context status
85  * buffer with a context complete event and generates a context switch interrupt.
86  * During the interrupt handling, the driver examines the events in the buffer:
87  * for each context complete event, if the announced ID matches that on the head
88  * of the request queue, then that request is retired and removed from the queue.
89  *
90  * After processing, if any requests were retired and the queue is not empty
91  * then a new execution list can be submitted. The two requests at the front of
92  * the queue are next to be submitted but since a context may not occur twice in
93  * an execution list, if subsequent requests have the same ID as the first then
94  * the two requests must be combined. This is done simply by discarding requests
95  * at the head of the queue until either only one requests is left (in which case
96  * we use a NULL second context) or the first two requests have unique IDs.
97  *
98  * By always executing the first two requests in the queue the driver ensures
99  * that the GPU is kept as busy as possible. In the case where a single context
100  * completes but a second context is still executing, the request for this second
101  * context will be at the head of the queue when we remove the first one. This
102  * request will then be resubmitted along with a new request for a different context,
103  * which will cause the hardware to continue executing the second request and queue
104  * the new request (the GPU detects the condition of a context getting preempted
105  * with the same context and optimizes the context switch flow by not doing
106  * preemption, but just sampling the new tail pointer).
107  *
108  */
109 #include <linux/interrupt.h>
110 #include <linux/string_helpers.h>
111 
112 #include "i915_drv.h"
113 #include "i915_trace.h"
114 #include "i915_vgpu.h"
115 #include "gen8_engine_cs.h"
116 #include "intel_breadcrumbs.h"
117 #include "intel_context.h"
118 #include "intel_engine_heartbeat.h"
119 #include "intel_engine_pm.h"
120 #include "intel_engine_regs.h"
121 #include "intel_engine_stats.h"
122 #include "intel_execlists_submission.h"
123 #include "intel_gt.h"
124 #include "intel_gt_irq.h"
125 #include "intel_gt_pm.h"
126 #include "intel_gt_regs.h"
127 #include "intel_gt_requests.h"
128 #include "intel_lrc.h"
129 #include "intel_lrc_reg.h"
130 #include "intel_mocs.h"
131 #include "intel_reset.h"
132 #include "intel_ring.h"
133 #include "intel_workarounds.h"
134 #include "shmem_utils.h"
135 
136 #define RING_EXECLIST_QFULL		(1 << 0x2)
137 #define RING_EXECLIST1_VALID		(1 << 0x3)
138 #define RING_EXECLIST0_VALID		(1 << 0x4)
139 #define RING_EXECLIST_ACTIVE_STATUS	(3 << 0xE)
140 #define RING_EXECLIST1_ACTIVE		(1 << 0x11)
141 #define RING_EXECLIST0_ACTIVE		(1 << 0x12)
142 
143 #define GEN8_CTX_STATUS_IDLE_ACTIVE	(1 << 0)
144 #define GEN8_CTX_STATUS_PREEMPTED	(1 << 1)
145 #define GEN8_CTX_STATUS_ELEMENT_SWITCH	(1 << 2)
146 #define GEN8_CTX_STATUS_ACTIVE_IDLE	(1 << 3)
147 #define GEN8_CTX_STATUS_COMPLETE	(1 << 4)
148 #define GEN8_CTX_STATUS_LITE_RESTORE	(1 << 15)
149 
150 #define GEN8_CTX_STATUS_COMPLETED_MASK \
151 	 (GEN8_CTX_STATUS_COMPLETE | GEN8_CTX_STATUS_PREEMPTED)
152 
153 #define GEN12_CTX_STATUS_SWITCHED_TO_NEW_QUEUE	(0x1) /* lower csb dword */
154 #define GEN12_CTX_SWITCH_DETAIL(csb_dw)	((csb_dw) & 0xF) /* upper csb dword */
155 #define GEN12_CSB_SW_CTX_ID_MASK		GENMASK(25, 15)
156 #define GEN12_IDLE_CTX_ID		0x7FF
157 #define GEN12_CSB_CTX_VALID(csb_dw) \
158 	(FIELD_GET(GEN12_CSB_SW_CTX_ID_MASK, csb_dw) != GEN12_IDLE_CTX_ID)
159 
160 #define XEHP_CTX_STATUS_SWITCHED_TO_NEW_QUEUE	BIT(1) /* upper csb dword */
161 #define XEHP_CSB_SW_CTX_ID_MASK			GENMASK(31, 10)
162 #define XEHP_IDLE_CTX_ID			0xFFFF
163 #define XEHP_CSB_CTX_VALID(csb_dw) \
164 	(FIELD_GET(XEHP_CSB_SW_CTX_ID_MASK, csb_dw) != XEHP_IDLE_CTX_ID)
165 
166 /* Typical size of the average request (2 pipecontrols and a MI_BB) */
167 #define EXECLISTS_REQUEST_SIZE 64 /* bytes */
168 
169 struct virtual_engine {
170 	struct intel_engine_cs base;
171 	struct intel_context context;
172 	struct rcu_work rcu;
173 
174 	/*
175 	 * We allow only a single request through the virtual engine at a time
176 	 * (each request in the timeline waits for the completion fence of
177 	 * the previous before being submitted). By restricting ourselves to
178 	 * only submitting a single request, each request is placed on to a
179 	 * physical to maximise load spreading (by virtue of the late greedy
180 	 * scheduling -- each real engine takes the next available request
181 	 * upon idling).
182 	 */
183 	struct i915_request *request;
184 
185 	/*
186 	 * We keep a rbtree of available virtual engines inside each physical
187 	 * engine, sorted by priority. Here we preallocate the nodes we need
188 	 * for the virtual engine, indexed by physical_engine->id.
189 	 */
190 	struct ve_node {
191 		struct rb_node rb;
192 		int prio;
193 	} nodes[I915_NUM_ENGINES];
194 
195 	/* And finally, which physical engines this virtual engine maps onto. */
196 	unsigned int num_siblings;
197 	struct intel_engine_cs *siblings[];
198 };
199 
to_virtual_engine(struct intel_engine_cs * engine)200 static struct virtual_engine *to_virtual_engine(struct intel_engine_cs *engine)
201 {
202 	GEM_BUG_ON(!intel_engine_is_virtual(engine));
203 	return container_of(engine, struct virtual_engine, base);
204 }
205 
206 static struct intel_context *
207 execlists_create_virtual(struct intel_engine_cs **siblings, unsigned int count,
208 			 unsigned long flags);
209 
210 static struct i915_request *
__active_request(const struct intel_timeline * const tl,struct i915_request * rq,int error)211 __active_request(const struct intel_timeline * const tl,
212 		 struct i915_request *rq,
213 		 int error)
214 {
215 	struct i915_request *active = rq;
216 
217 	list_for_each_entry_from_reverse(rq, &tl->requests, link) {
218 		if (__i915_request_is_complete(rq))
219 			break;
220 
221 		if (error) {
222 			i915_request_set_error_once(rq, error);
223 			__i915_request_skip(rq);
224 		}
225 		active = rq;
226 	}
227 
228 	return active;
229 }
230 
231 static struct i915_request *
active_request(const struct intel_timeline * const tl,struct i915_request * rq)232 active_request(const struct intel_timeline * const tl, struct i915_request *rq)
233 {
234 	return __active_request(tl, rq, 0);
235 }
236 
ring_set_paused(const struct intel_engine_cs * engine,int state)237 static void ring_set_paused(const struct intel_engine_cs *engine, int state)
238 {
239 	/*
240 	 * We inspect HWS_PREEMPT with a semaphore inside
241 	 * engine->emit_fini_breadcrumb. If the dword is true,
242 	 * the ring is paused as the semaphore will busywait
243 	 * until the dword is false.
244 	 */
245 	engine->status_page.addr[I915_GEM_HWS_PREEMPT] = state;
246 	if (state)
247 		wmb();
248 }
249 
to_priolist(struct rb_node * rb)250 static struct i915_priolist *to_priolist(struct rb_node *rb)
251 {
252 	return rb_entry(rb, struct i915_priolist, node);
253 }
254 
rq_prio(const struct i915_request * rq)255 static int rq_prio(const struct i915_request *rq)
256 {
257 	return READ_ONCE(rq->sched.attr.priority);
258 }
259 
effective_prio(const struct i915_request * rq)260 static int effective_prio(const struct i915_request *rq)
261 {
262 	int prio = rq_prio(rq);
263 
264 	/*
265 	 * If this request is special and must not be interrupted at any
266 	 * cost, so be it. Note we are only checking the most recent request
267 	 * in the context and so may be masking an earlier vip request. It
268 	 * is hoped that under the conditions where nopreempt is used, this
269 	 * will not matter (i.e. all requests to that context will be
270 	 * nopreempt for as long as desired).
271 	 */
272 	if (i915_request_has_nopreempt(rq))
273 		prio = I915_PRIORITY_UNPREEMPTABLE;
274 
275 	return prio;
276 }
277 
queue_prio(const struct i915_sched_engine * sched_engine)278 static int queue_prio(const struct i915_sched_engine *sched_engine)
279 {
280 	struct rb_node *rb;
281 
282 	rb = rb_first_cached(&sched_engine->queue);
283 	if (!rb)
284 		return INT_MIN;
285 
286 	return to_priolist(rb)->priority;
287 }
288 
virtual_prio(const struct intel_engine_execlists * el)289 static int virtual_prio(const struct intel_engine_execlists *el)
290 {
291 	struct rb_node *rb = rb_first_cached(&el->virtual);
292 
293 	return rb ? rb_entry(rb, struct ve_node, rb)->prio : INT_MIN;
294 }
295 
need_preempt(const struct intel_engine_cs * engine,const struct i915_request * rq)296 static bool need_preempt(const struct intel_engine_cs *engine,
297 			 const struct i915_request *rq)
298 {
299 	int last_prio;
300 
301 	if (!intel_engine_has_semaphores(engine))
302 		return false;
303 
304 	/*
305 	 * Check if the current priority hint merits a preemption attempt.
306 	 *
307 	 * We record the highest value priority we saw during rescheduling
308 	 * prior to this dequeue, therefore we know that if it is strictly
309 	 * less than the current tail of ESLP[0], we do not need to force
310 	 * a preempt-to-idle cycle.
311 	 *
312 	 * However, the priority hint is a mere hint that we may need to
313 	 * preempt. If that hint is stale or we may be trying to preempt
314 	 * ourselves, ignore the request.
315 	 *
316 	 * More naturally we would write
317 	 *      prio >= max(0, last);
318 	 * except that we wish to prevent triggering preemption at the same
319 	 * priority level: the task that is running should remain running
320 	 * to preserve FIFO ordering of dependencies.
321 	 */
322 	last_prio = max(effective_prio(rq), I915_PRIORITY_NORMAL - 1);
323 	if (engine->sched_engine->queue_priority_hint <= last_prio)
324 		return false;
325 
326 	/*
327 	 * Check against the first request in ELSP[1], it will, thanks to the
328 	 * power of PI, be the highest priority of that context.
329 	 */
330 	if (!list_is_last(&rq->sched.link, &engine->sched_engine->requests) &&
331 	    rq_prio(list_next_entry(rq, sched.link)) > last_prio)
332 		return true;
333 
334 	/*
335 	 * If the inflight context did not trigger the preemption, then maybe
336 	 * it was the set of queued requests? Pick the highest priority in
337 	 * the queue (the first active priolist) and see if it deserves to be
338 	 * running instead of ELSP[0].
339 	 *
340 	 * The highest priority request in the queue can not be either
341 	 * ELSP[0] or ELSP[1] as, thanks again to PI, if it was the same
342 	 * context, it's priority would not exceed ELSP[0] aka last_prio.
343 	 */
344 	return max(virtual_prio(&engine->execlists),
345 		   queue_prio(engine->sched_engine)) > last_prio;
346 }
347 
348 __maybe_unused static bool
assert_priority_queue(const struct i915_request * prev,const struct i915_request * next)349 assert_priority_queue(const struct i915_request *prev,
350 		      const struct i915_request *next)
351 {
352 	/*
353 	 * Without preemption, the prev may refer to the still active element
354 	 * which we refuse to let go.
355 	 *
356 	 * Even with preemption, there are times when we think it is better not
357 	 * to preempt and leave an ostensibly lower priority request in flight.
358 	 */
359 	if (i915_request_is_active(prev))
360 		return true;
361 
362 	return rq_prio(prev) >= rq_prio(next);
363 }
364 
365 static struct i915_request *
__unwind_incomplete_requests(struct intel_engine_cs * engine)366 __unwind_incomplete_requests(struct intel_engine_cs *engine)
367 {
368 	struct i915_request *rq, *rn, *active = NULL;
369 	struct list_head *pl;
370 	int prio = I915_PRIORITY_INVALID;
371 
372 	lockdep_assert_held(&engine->sched_engine->lock);
373 
374 	list_for_each_entry_safe_reverse(rq, rn,
375 					 &engine->sched_engine->requests,
376 					 sched.link) {
377 		if (__i915_request_is_complete(rq)) {
378 			list_del_init(&rq->sched.link);
379 			continue;
380 		}
381 
382 		__i915_request_unsubmit(rq);
383 
384 		GEM_BUG_ON(rq_prio(rq) == I915_PRIORITY_INVALID);
385 		if (rq_prio(rq) != prio) {
386 			prio = rq_prio(rq);
387 			pl = i915_sched_lookup_priolist(engine->sched_engine,
388 							prio);
389 		}
390 		GEM_BUG_ON(i915_sched_engine_is_empty(engine->sched_engine));
391 
392 		list_move(&rq->sched.link, pl);
393 		set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);
394 
395 		/* Check in case we rollback so far we wrap [size/2] */
396 		if (intel_ring_direction(rq->ring,
397 					 rq->tail,
398 					 rq->ring->tail + 8) > 0)
399 			rq->context->lrc.desc |= CTX_DESC_FORCE_RESTORE;
400 
401 		active = rq;
402 	}
403 
404 	return active;
405 }
406 
407 struct i915_request *
execlists_unwind_incomplete_requests(struct intel_engine_execlists * execlists)408 execlists_unwind_incomplete_requests(struct intel_engine_execlists *execlists)
409 {
410 	struct intel_engine_cs *engine =
411 		container_of(execlists, typeof(*engine), execlists);
412 
413 	return __unwind_incomplete_requests(engine);
414 }
415 
416 static void
execlists_context_status_change(struct i915_request * rq,unsigned long status)417 execlists_context_status_change(struct i915_request *rq, unsigned long status)
418 {
419 	/*
420 	 * Only used when GVT-g is enabled now. When GVT-g is disabled,
421 	 * The compiler should eliminate this function as dead-code.
422 	 */
423 	if (!IS_ENABLED(CONFIG_DRM_I915_GVT))
424 		return;
425 
426 	atomic_notifier_call_chain(&rq->engine->context_status_notifier,
427 				   status, rq);
428 }
429 
reset_active(struct i915_request * rq,struct intel_engine_cs * engine)430 static void reset_active(struct i915_request *rq,
431 			 struct intel_engine_cs *engine)
432 {
433 	struct intel_context * const ce = rq->context;
434 	u32 head;
435 
436 	/*
437 	 * The executing context has been cancelled. We want to prevent
438 	 * further execution along this context and propagate the error on
439 	 * to anything depending on its results.
440 	 *
441 	 * In __i915_request_submit(), we apply the -EIO and remove the
442 	 * requests' payloads for any banned requests. But first, we must
443 	 * rewind the context back to the start of the incomplete request so
444 	 * that we do not jump back into the middle of the batch.
445 	 *
446 	 * We preserve the breadcrumbs and semaphores of the incomplete
447 	 * requests so that inter-timeline dependencies (i.e other timelines)
448 	 * remain correctly ordered. And we defer to __i915_request_submit()
449 	 * so that all asynchronous waits are correctly handled.
450 	 */
451 	ENGINE_TRACE(engine, "{ reset rq=%llx:%lld }\n",
452 		     rq->fence.context, rq->fence.seqno);
453 
454 	/* On resubmission of the active request, payload will be scrubbed */
455 	if (__i915_request_is_complete(rq))
456 		head = rq->tail;
457 	else
458 		head = __active_request(ce->timeline, rq, -EIO)->head;
459 	head = intel_ring_wrap(ce->ring, head);
460 
461 	/* Scrub the context image to prevent replaying the previous batch */
462 	lrc_init_regs(ce, engine, true);
463 
464 	/* We've switched away, so this should be a no-op, but intent matters */
465 	ce->lrc.lrca = lrc_update_regs(ce, engine, head);
466 }
467 
bad_request(const struct i915_request * rq)468 static bool bad_request(const struct i915_request *rq)
469 {
470 	return rq->fence.error && i915_request_started(rq);
471 }
472 
473 static struct intel_engine_cs *
__execlists_schedule_in(struct i915_request * rq)474 __execlists_schedule_in(struct i915_request *rq)
475 {
476 	struct intel_engine_cs * const engine = rq->engine;
477 	struct intel_context * const ce = rq->context;
478 
479 	intel_context_get(ce);
480 
481 	if (unlikely(intel_context_is_closed(ce) &&
482 		     !intel_engine_has_heartbeat(engine)))
483 		intel_context_set_exiting(ce);
484 
485 	if (unlikely(!intel_context_is_schedulable(ce) || bad_request(rq)))
486 		reset_active(rq, engine);
487 
488 	if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
489 		lrc_check_regs(ce, engine, "before");
490 
491 	if (ce->tag) {
492 		/* Use a fixed tag for OA and friends */
493 		GEM_BUG_ON(ce->tag <= BITS_PER_LONG);
494 		ce->lrc.ccid = ce->tag;
495 	} else if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50)) {
496 		/* We don't need a strict matching tag, just different values */
497 		unsigned int tag = ffs(READ_ONCE(engine->context_tag));
498 
499 		GEM_BUG_ON(tag == 0 || tag >= BITS_PER_LONG);
500 		clear_bit(tag - 1, &engine->context_tag);
501 		ce->lrc.ccid = tag << (XEHP_SW_CTX_ID_SHIFT - 32);
502 
503 		BUILD_BUG_ON(BITS_PER_LONG > GEN12_MAX_CONTEXT_HW_ID);
504 
505 	} else {
506 		/* We don't need a strict matching tag, just different values */
507 		unsigned int tag = __ffs(engine->context_tag);
508 
509 		GEM_BUG_ON(tag >= BITS_PER_LONG);
510 		__clear_bit(tag, &engine->context_tag);
511 		ce->lrc.ccid = (1 + tag) << (GEN11_SW_CTX_ID_SHIFT - 32);
512 
513 		BUILD_BUG_ON(BITS_PER_LONG > GEN12_MAX_CONTEXT_HW_ID);
514 	}
515 
516 	ce->lrc.ccid |= engine->execlists.ccid;
517 
518 	__intel_gt_pm_get(engine->gt);
519 	if (engine->fw_domain && !engine->fw_active++)
520 		intel_uncore_forcewake_get(engine->uncore, engine->fw_domain);
521 	execlists_context_status_change(rq, INTEL_CONTEXT_SCHEDULE_IN);
522 	intel_engine_context_in(engine);
523 
524 	CE_TRACE(ce, "schedule-in, ccid:%x\n", ce->lrc.ccid);
525 
526 	return engine;
527 }
528 
execlists_schedule_in(struct i915_request * rq,int idx)529 static void execlists_schedule_in(struct i915_request *rq, int idx)
530 {
531 	struct intel_context * const ce = rq->context;
532 	struct intel_engine_cs *old;
533 
534 	GEM_BUG_ON(!intel_engine_pm_is_awake(rq->engine));
535 	trace_i915_request_in(rq, idx);
536 
537 	old = ce->inflight;
538 	if (!old)
539 		old = __execlists_schedule_in(rq);
540 	WRITE_ONCE(ce->inflight, ptr_inc(old));
541 
542 	GEM_BUG_ON(intel_context_inflight(ce) != rq->engine);
543 }
544 
545 static void
resubmit_virtual_request(struct i915_request * rq,struct virtual_engine * ve)546 resubmit_virtual_request(struct i915_request *rq, struct virtual_engine *ve)
547 {
548 	struct intel_engine_cs *engine = rq->engine;
549 
550 	spin_lock_irq(&engine->sched_engine->lock);
551 
552 	clear_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);
553 	WRITE_ONCE(rq->engine, &ve->base);
554 	ve->base.submit_request(rq);
555 
556 	spin_unlock_irq(&engine->sched_engine->lock);
557 }
558 
kick_siblings(struct i915_request * rq,struct intel_context * ce)559 static void kick_siblings(struct i915_request *rq, struct intel_context *ce)
560 {
561 	struct virtual_engine *ve = container_of(ce, typeof(*ve), context);
562 	struct intel_engine_cs *engine = rq->engine;
563 
564 	/*
565 	 * After this point, the rq may be transferred to a new sibling, so
566 	 * before we clear ce->inflight make sure that the context has been
567 	 * removed from the b->signalers and furthermore we need to make sure
568 	 * that the concurrent iterator in signal_irq_work is no longer
569 	 * following ce->signal_link.
570 	 */
571 	if (!list_empty(&ce->signals))
572 		intel_context_remove_breadcrumbs(ce, engine->breadcrumbs);
573 
574 	/*
575 	 * This engine is now too busy to run this virtual request, so
576 	 * see if we can find an alternative engine for it to execute on.
577 	 * Once a request has become bonded to this engine, we treat it the
578 	 * same as other native request.
579 	 */
580 	if (i915_request_in_priority_queue(rq) &&
581 	    rq->execution_mask != engine->mask)
582 		resubmit_virtual_request(rq, ve);
583 
584 	if (READ_ONCE(ve->request))
585 		tasklet_hi_schedule(&ve->base.sched_engine->tasklet);
586 }
587 
__execlists_schedule_out(struct i915_request * const rq,struct intel_context * const ce)588 static void __execlists_schedule_out(struct i915_request * const rq,
589 				     struct intel_context * const ce)
590 {
591 	struct intel_engine_cs * const engine = rq->engine;
592 	unsigned int ccid;
593 
594 	/*
595 	 * NB process_csb() is not under the engine->sched_engine->lock and hence
596 	 * schedule_out can race with schedule_in meaning that we should
597 	 * refrain from doing non-trivial work here.
598 	 */
599 
600 	CE_TRACE(ce, "schedule-out, ccid:%x\n", ce->lrc.ccid);
601 	GEM_BUG_ON(ce->inflight != engine);
602 
603 	if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
604 		lrc_check_regs(ce, engine, "after");
605 
606 	/*
607 	 * If we have just completed this context, the engine may now be
608 	 * idle and we want to re-enter powersaving.
609 	 */
610 	if (intel_timeline_is_last(ce->timeline, rq) &&
611 	    __i915_request_is_complete(rq))
612 		intel_engine_add_retire(engine, ce->timeline);
613 
614 	ccid = ce->lrc.ccid;
615 	if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50)) {
616 		ccid >>= XEHP_SW_CTX_ID_SHIFT - 32;
617 		ccid &= XEHP_MAX_CONTEXT_HW_ID;
618 	} else {
619 		ccid >>= GEN11_SW_CTX_ID_SHIFT - 32;
620 		ccid &= GEN12_MAX_CONTEXT_HW_ID;
621 	}
622 
623 	if (ccid < BITS_PER_LONG) {
624 		GEM_BUG_ON(ccid == 0);
625 		GEM_BUG_ON(test_bit(ccid - 1, &engine->context_tag));
626 		__set_bit(ccid - 1, &engine->context_tag);
627 	}
628 	intel_engine_context_out(engine);
629 	execlists_context_status_change(rq, INTEL_CONTEXT_SCHEDULE_OUT);
630 	if (engine->fw_domain && !--engine->fw_active)
631 		intel_uncore_forcewake_put(engine->uncore, engine->fw_domain);
632 	intel_gt_pm_put_async(engine->gt);
633 
634 	/*
635 	 * If this is part of a virtual engine, its next request may
636 	 * have been blocked waiting for access to the active context.
637 	 * We have to kick all the siblings again in case we need to
638 	 * switch (e.g. the next request is not runnable on this
639 	 * engine). Hopefully, we will already have submitted the next
640 	 * request before the tasklet runs and do not need to rebuild
641 	 * each virtual tree and kick everyone again.
642 	 */
643 	if (ce->engine != engine)
644 		kick_siblings(rq, ce);
645 
646 	WRITE_ONCE(ce->inflight, NULL);
647 	intel_context_put(ce);
648 }
649 
execlists_schedule_out(struct i915_request * rq)650 static inline void execlists_schedule_out(struct i915_request *rq)
651 {
652 	struct intel_context * const ce = rq->context;
653 
654 	trace_i915_request_out(rq);
655 
656 	GEM_BUG_ON(!ce->inflight);
657 	ce->inflight = ptr_dec(ce->inflight);
658 	if (!__intel_context_inflight_count(ce->inflight))
659 		__execlists_schedule_out(rq, ce);
660 
661 	i915_request_put(rq);
662 }
663 
map_i915_prio_to_lrc_desc_prio(int prio)664 static u32 map_i915_prio_to_lrc_desc_prio(int prio)
665 {
666 	if (prio > I915_PRIORITY_NORMAL)
667 		return GEN12_CTX_PRIORITY_HIGH;
668 	else if (prio < I915_PRIORITY_NORMAL)
669 		return GEN12_CTX_PRIORITY_LOW;
670 	else
671 		return GEN12_CTX_PRIORITY_NORMAL;
672 }
673 
execlists_update_context(struct i915_request * rq)674 static u64 execlists_update_context(struct i915_request *rq)
675 {
676 	struct intel_context *ce = rq->context;
677 	u64 desc;
678 	u32 tail, prev;
679 
680 	desc = ce->lrc.desc;
681 	if (rq->engine->flags & I915_ENGINE_HAS_EU_PRIORITY)
682 		desc |= map_i915_prio_to_lrc_desc_prio(rq_prio(rq));
683 
684 	/*
685 	 * WaIdleLiteRestore:bdw,skl
686 	 *
687 	 * We should never submit the context with the same RING_TAIL twice
688 	 * just in case we submit an empty ring, which confuses the HW.
689 	 *
690 	 * We append a couple of NOOPs (gen8_emit_wa_tail) after the end of
691 	 * the normal request to be able to always advance the RING_TAIL on
692 	 * subsequent resubmissions (for lite restore). Should that fail us,
693 	 * and we try and submit the same tail again, force the context
694 	 * reload.
695 	 *
696 	 * If we need to return to a preempted context, we need to skip the
697 	 * lite-restore and force it to reload the RING_TAIL. Otherwise, the
698 	 * HW has a tendency to ignore us rewinding the TAIL to the end of
699 	 * an earlier request.
700 	 */
701 	GEM_BUG_ON(ce->lrc_reg_state[CTX_RING_TAIL] != rq->ring->tail);
702 	prev = rq->ring->tail;
703 	tail = intel_ring_set_tail(rq->ring, rq->tail);
704 	if (unlikely(intel_ring_direction(rq->ring, tail, prev) <= 0))
705 		desc |= CTX_DESC_FORCE_RESTORE;
706 	ce->lrc_reg_state[CTX_RING_TAIL] = tail;
707 	rq->tail = rq->wa_tail;
708 
709 	/*
710 	 * Make sure the context image is complete before we submit it to HW.
711 	 *
712 	 * Ostensibly, writes (including the WCB) should be flushed prior to
713 	 * an uncached write such as our mmio register access, the empirical
714 	 * evidence (esp. on Braswell) suggests that the WC write into memory
715 	 * may not be visible to the HW prior to the completion of the UC
716 	 * register write and that we may begin execution from the context
717 	 * before its image is complete leading to invalid PD chasing.
718 	 */
719 	wmb();
720 
721 	ce->lrc.desc &= ~CTX_DESC_FORCE_RESTORE;
722 	return desc;
723 }
724 
write_desc(struct intel_engine_execlists * execlists,u64 desc,u32 port)725 static void write_desc(struct intel_engine_execlists *execlists, u64 desc, u32 port)
726 {
727 	if (execlists->ctrl_reg) {
728 		writel(lower_32_bits(desc), execlists->submit_reg + port * 2);
729 		writel(upper_32_bits(desc), execlists->submit_reg + port * 2 + 1);
730 	} else {
731 		writel(upper_32_bits(desc), execlists->submit_reg);
732 		writel(lower_32_bits(desc), execlists->submit_reg);
733 	}
734 }
735 
736 static __maybe_unused char *
dump_port(char * buf,int buflen,const char * prefix,struct i915_request * rq)737 dump_port(char *buf, int buflen, const char *prefix, struct i915_request *rq)
738 {
739 	if (!rq)
740 		return "";
741 
742 	snprintf(buf, buflen, "%sccid:%x %llx:%lld%s prio %d",
743 		 prefix,
744 		 rq->context->lrc.ccid,
745 		 rq->fence.context, rq->fence.seqno,
746 		 __i915_request_is_complete(rq) ? "!" :
747 		 __i915_request_has_started(rq) ? "*" :
748 		 "",
749 		 rq_prio(rq));
750 
751 	return buf;
752 }
753 
754 static __maybe_unused noinline void
trace_ports(const struct intel_engine_execlists * execlists,const char * msg,struct i915_request * const * ports)755 trace_ports(const struct intel_engine_execlists *execlists,
756 	    const char *msg,
757 	    struct i915_request * const *ports)
758 {
759 	const struct intel_engine_cs *engine =
760 		container_of(execlists, typeof(*engine), execlists);
761 	char __maybe_unused p0[40], p1[40];
762 
763 	if (!ports[0])
764 		return;
765 
766 	ENGINE_TRACE(engine, "%s { %s%s }\n", msg,
767 		     dump_port(p0, sizeof(p0), "", ports[0]),
768 		     dump_port(p1, sizeof(p1), ", ", ports[1]));
769 }
770 
771 static bool
reset_in_progress(const struct intel_engine_cs * engine)772 reset_in_progress(const struct intel_engine_cs *engine)
773 {
774 	return unlikely(!__tasklet_is_enabled(&engine->sched_engine->tasklet));
775 }
776 
777 static __maybe_unused noinline bool
assert_pending_valid(const struct intel_engine_execlists * execlists,const char * msg)778 assert_pending_valid(const struct intel_engine_execlists *execlists,
779 		     const char *msg)
780 {
781 	struct intel_engine_cs *engine =
782 		container_of(execlists, typeof(*engine), execlists);
783 	struct i915_request * const *port, *rq, *prev = NULL;
784 	struct intel_context *ce = NULL;
785 	u32 ccid = -1;
786 
787 	trace_ports(execlists, msg, execlists->pending);
788 
789 	/* We may be messing around with the lists during reset, lalala */
790 	if (reset_in_progress(engine))
791 		return true;
792 
793 	if (!execlists->pending[0]) {
794 		GEM_TRACE_ERR("%s: Nothing pending for promotion!\n",
795 			      engine->name);
796 		return false;
797 	}
798 
799 	if (execlists->pending[execlists_num_ports(execlists)]) {
800 		GEM_TRACE_ERR("%s: Excess pending[%d] for promotion!\n",
801 			      engine->name, execlists_num_ports(execlists));
802 		return false;
803 	}
804 
805 	for (port = execlists->pending; (rq = *port); port++) {
806 		unsigned long flags;
807 		bool ok = true;
808 
809 		GEM_BUG_ON(!kref_read(&rq->fence.refcount));
810 		GEM_BUG_ON(!i915_request_is_active(rq));
811 
812 		if (ce == rq->context) {
813 			GEM_TRACE_ERR("%s: Dup context:%llx in pending[%zd]\n",
814 				      engine->name,
815 				      ce->timeline->fence_context,
816 				      port - execlists->pending);
817 			return false;
818 		}
819 		ce = rq->context;
820 
821 		if (ccid == ce->lrc.ccid) {
822 			GEM_TRACE_ERR("%s: Dup ccid:%x context:%llx in pending[%zd]\n",
823 				      engine->name,
824 				      ccid, ce->timeline->fence_context,
825 				      port - execlists->pending);
826 			return false;
827 		}
828 		ccid = ce->lrc.ccid;
829 
830 		/*
831 		 * Sentinels are supposed to be the last request so they flush
832 		 * the current execution off the HW. Check that they are the only
833 		 * request in the pending submission.
834 		 *
835 		 * NB: Due to the async nature of preempt-to-busy and request
836 		 * cancellation we need to handle the case where request
837 		 * becomes a sentinel in parallel to CSB processing.
838 		 */
839 		if (prev && i915_request_has_sentinel(prev) &&
840 		    !READ_ONCE(prev->fence.error)) {
841 			GEM_TRACE_ERR("%s: context:%llx after sentinel in pending[%zd]\n",
842 				      engine->name,
843 				      ce->timeline->fence_context,
844 				      port - execlists->pending);
845 			return false;
846 		}
847 		prev = rq;
848 
849 		/*
850 		 * We want virtual requests to only be in the first slot so
851 		 * that they are never stuck behind a hog and can be immediately
852 		 * transferred onto the next idle engine.
853 		 */
854 		if (rq->execution_mask != engine->mask &&
855 		    port != execlists->pending) {
856 			GEM_TRACE_ERR("%s: virtual engine:%llx not in prime position[%zd]\n",
857 				      engine->name,
858 				      ce->timeline->fence_context,
859 				      port - execlists->pending);
860 			return false;
861 		}
862 
863 		/* Hold tightly onto the lock to prevent concurrent retires! */
864 		if (!spin_trylock_irqsave(&rq->lock, flags))
865 			continue;
866 
867 		if (__i915_request_is_complete(rq))
868 			goto unlock;
869 
870 		if (i915_active_is_idle(&ce->active) &&
871 		    !intel_context_is_barrier(ce)) {
872 			GEM_TRACE_ERR("%s: Inactive context:%llx in pending[%zd]\n",
873 				      engine->name,
874 				      ce->timeline->fence_context,
875 				      port - execlists->pending);
876 			ok = false;
877 			goto unlock;
878 		}
879 
880 		if (!i915_vma_is_pinned(ce->state)) {
881 			GEM_TRACE_ERR("%s: Unpinned context:%llx in pending[%zd]\n",
882 				      engine->name,
883 				      ce->timeline->fence_context,
884 				      port - execlists->pending);
885 			ok = false;
886 			goto unlock;
887 		}
888 
889 		if (!i915_vma_is_pinned(ce->ring->vma)) {
890 			GEM_TRACE_ERR("%s: Unpinned ring:%llx in pending[%zd]\n",
891 				      engine->name,
892 				      ce->timeline->fence_context,
893 				      port - execlists->pending);
894 			ok = false;
895 			goto unlock;
896 		}
897 
898 unlock:
899 		spin_unlock_irqrestore(&rq->lock, flags);
900 		if (!ok)
901 			return false;
902 	}
903 
904 	return ce;
905 }
906 
execlists_submit_ports(struct intel_engine_cs * engine)907 static void execlists_submit_ports(struct intel_engine_cs *engine)
908 {
909 	struct intel_engine_execlists *execlists = &engine->execlists;
910 	unsigned int n;
911 
912 	GEM_BUG_ON(!assert_pending_valid(execlists, "submit"));
913 
914 	/*
915 	 * We can skip acquiring intel_runtime_pm_get() here as it was taken
916 	 * on our behalf by the request (see i915_gem_mark_busy()) and it will
917 	 * not be relinquished until the device is idle (see
918 	 * i915_gem_idle_work_handler()). As a precaution, we make sure
919 	 * that all ELSP are drained i.e. we have processed the CSB,
920 	 * before allowing ourselves to idle and calling intel_runtime_pm_put().
921 	 */
922 	GEM_BUG_ON(!intel_engine_pm_is_awake(engine));
923 
924 	/*
925 	 * ELSQ note: the submit queue is not cleared after being submitted
926 	 * to the HW so we need to make sure we always clean it up. This is
927 	 * currently ensured by the fact that we always write the same number
928 	 * of elsq entries, keep this in mind before changing the loop below.
929 	 */
930 	for (n = execlists_num_ports(execlists); n--; ) {
931 		struct i915_request *rq = execlists->pending[n];
932 
933 		write_desc(execlists,
934 			   rq ? execlists_update_context(rq) : 0,
935 			   n);
936 	}
937 
938 	/* we need to manually load the submit queue */
939 	if (execlists->ctrl_reg)
940 		writel(EL_CTRL_LOAD, execlists->ctrl_reg);
941 }
942 
ctx_single_port_submission(const struct intel_context * ce)943 static bool ctx_single_port_submission(const struct intel_context *ce)
944 {
945 	return (IS_ENABLED(CONFIG_DRM_I915_GVT) &&
946 		intel_context_force_single_submission(ce));
947 }
948 
can_merge_ctx(const struct intel_context * prev,const struct intel_context * next)949 static bool can_merge_ctx(const struct intel_context *prev,
950 			  const struct intel_context *next)
951 {
952 	if (prev != next)
953 		return false;
954 
955 	if (ctx_single_port_submission(prev))
956 		return false;
957 
958 	return true;
959 }
960 
i915_request_flags(const struct i915_request * rq)961 static unsigned long i915_request_flags(const struct i915_request *rq)
962 {
963 	return READ_ONCE(rq->fence.flags);
964 }
965 
can_merge_rq(const struct i915_request * prev,const struct i915_request * next)966 static bool can_merge_rq(const struct i915_request *prev,
967 			 const struct i915_request *next)
968 {
969 	GEM_BUG_ON(prev == next);
970 	GEM_BUG_ON(!assert_priority_queue(prev, next));
971 
972 	/*
973 	 * We do not submit known completed requests. Therefore if the next
974 	 * request is already completed, we can pretend to merge it in
975 	 * with the previous context (and we will skip updating the ELSP
976 	 * and tracking). Thus hopefully keeping the ELSP full with active
977 	 * contexts, despite the best efforts of preempt-to-busy to confuse
978 	 * us.
979 	 */
980 	if (__i915_request_is_complete(next))
981 		return true;
982 
983 	if (unlikely((i915_request_flags(prev) | i915_request_flags(next)) &
984 		     (BIT(I915_FENCE_FLAG_NOPREEMPT) |
985 		      BIT(I915_FENCE_FLAG_SENTINEL))))
986 		return false;
987 
988 	if (!can_merge_ctx(prev->context, next->context))
989 		return false;
990 
991 	GEM_BUG_ON(i915_seqno_passed(prev->fence.seqno, next->fence.seqno));
992 	return true;
993 }
994 
virtual_matches(const struct virtual_engine * ve,const struct i915_request * rq,const struct intel_engine_cs * engine)995 static bool virtual_matches(const struct virtual_engine *ve,
996 			    const struct i915_request *rq,
997 			    const struct intel_engine_cs *engine)
998 {
999 	const struct intel_engine_cs *inflight;
1000 
1001 	if (!rq)
1002 		return false;
1003 
1004 	if (!(rq->execution_mask & engine->mask)) /* We peeked too soon! */
1005 		return false;
1006 
1007 	/*
1008 	 * We track when the HW has completed saving the context image
1009 	 * (i.e. when we have seen the final CS event switching out of
1010 	 * the context) and must not overwrite the context image before
1011 	 * then. This restricts us to only using the active engine
1012 	 * while the previous virtualized request is inflight (so
1013 	 * we reuse the register offsets). This is a very small
1014 	 * hystersis on the greedy seelction algorithm.
1015 	 */
1016 	inflight = intel_context_inflight(&ve->context);
1017 	if (inflight && inflight != engine)
1018 		return false;
1019 
1020 	return true;
1021 }
1022 
1023 static struct virtual_engine *
first_virtual_engine(struct intel_engine_cs * engine)1024 first_virtual_engine(struct intel_engine_cs *engine)
1025 {
1026 	struct intel_engine_execlists *el = &engine->execlists;
1027 	struct rb_node *rb = rb_first_cached(&el->virtual);
1028 
1029 	while (rb) {
1030 		struct virtual_engine *ve =
1031 			rb_entry(rb, typeof(*ve), nodes[engine->id].rb);
1032 		struct i915_request *rq = READ_ONCE(ve->request);
1033 
1034 		/* lazily cleanup after another engine handled rq */
1035 		if (!rq || !virtual_matches(ve, rq, engine)) {
1036 			rb_erase_cached(rb, &el->virtual);
1037 			RB_CLEAR_NODE(rb);
1038 			rb = rb_first_cached(&el->virtual);
1039 			continue;
1040 		}
1041 
1042 		return ve;
1043 	}
1044 
1045 	return NULL;
1046 }
1047 
virtual_xfer_context(struct virtual_engine * ve,struct intel_engine_cs * engine)1048 static void virtual_xfer_context(struct virtual_engine *ve,
1049 				 struct intel_engine_cs *engine)
1050 {
1051 	unsigned int n;
1052 
1053 	if (likely(engine == ve->siblings[0]))
1054 		return;
1055 
1056 	GEM_BUG_ON(READ_ONCE(ve->context.inflight));
1057 	if (!intel_engine_has_relative_mmio(engine))
1058 		lrc_update_offsets(&ve->context, engine);
1059 
1060 	/*
1061 	 * Move the bound engine to the top of the list for
1062 	 * future execution. We then kick this tasklet first
1063 	 * before checking others, so that we preferentially
1064 	 * reuse this set of bound registers.
1065 	 */
1066 	for (n = 1; n < ve->num_siblings; n++) {
1067 		if (ve->siblings[n] == engine) {
1068 			swap(ve->siblings[n], ve->siblings[0]);
1069 			break;
1070 		}
1071 	}
1072 }
1073 
defer_request(struct i915_request * rq,struct list_head * const pl)1074 static void defer_request(struct i915_request *rq, struct list_head * const pl)
1075 {
1076 	LIST_HEAD(list);
1077 
1078 	/*
1079 	 * We want to move the interrupted request to the back of
1080 	 * the round-robin list (i.e. its priority level), but
1081 	 * in doing so, we must then move all requests that were in
1082 	 * flight and were waiting for the interrupted request to
1083 	 * be run after it again.
1084 	 */
1085 	do {
1086 		struct i915_dependency *p;
1087 
1088 		GEM_BUG_ON(i915_request_is_active(rq));
1089 		list_move_tail(&rq->sched.link, pl);
1090 
1091 		for_each_waiter(p, rq) {
1092 			struct i915_request *w =
1093 				container_of(p->waiter, typeof(*w), sched);
1094 
1095 			if (p->flags & I915_DEPENDENCY_WEAK)
1096 				continue;
1097 
1098 			/* Leave semaphores spinning on the other engines */
1099 			if (w->engine != rq->engine)
1100 				continue;
1101 
1102 			/* No waiter should start before its signaler */
1103 			GEM_BUG_ON(i915_request_has_initial_breadcrumb(w) &&
1104 				   __i915_request_has_started(w) &&
1105 				   !__i915_request_is_complete(rq));
1106 
1107 			if (!i915_request_is_ready(w))
1108 				continue;
1109 
1110 			if (rq_prio(w) < rq_prio(rq))
1111 				continue;
1112 
1113 			GEM_BUG_ON(rq_prio(w) > rq_prio(rq));
1114 			GEM_BUG_ON(i915_request_is_active(w));
1115 			list_move_tail(&w->sched.link, &list);
1116 		}
1117 
1118 		rq = list_first_entry_or_null(&list, typeof(*rq), sched.link);
1119 	} while (rq);
1120 }
1121 
defer_active(struct intel_engine_cs * engine)1122 static void defer_active(struct intel_engine_cs *engine)
1123 {
1124 	struct i915_request *rq;
1125 
1126 	rq = __unwind_incomplete_requests(engine);
1127 	if (!rq)
1128 		return;
1129 
1130 	defer_request(rq, i915_sched_lookup_priolist(engine->sched_engine,
1131 						     rq_prio(rq)));
1132 }
1133 
1134 static bool
timeslice_yield(const struct intel_engine_execlists * el,const struct i915_request * rq)1135 timeslice_yield(const struct intel_engine_execlists *el,
1136 		const struct i915_request *rq)
1137 {
1138 	/*
1139 	 * Once bitten, forever smitten!
1140 	 *
1141 	 * If the active context ever busy-waited on a semaphore,
1142 	 * it will be treated as a hog until the end of its timeslice (i.e.
1143 	 * until it is scheduled out and replaced by a new submission,
1144 	 * possibly even its own lite-restore). The HW only sends an interrupt
1145 	 * on the first miss, and we do know if that semaphore has been
1146 	 * signaled, or even if it is now stuck on another semaphore. Play
1147 	 * safe, yield if it might be stuck -- it will be given a fresh
1148 	 * timeslice in the near future.
1149 	 */
1150 	return rq->context->lrc.ccid == READ_ONCE(el->yield);
1151 }
1152 
needs_timeslice(const struct intel_engine_cs * engine,const struct i915_request * rq)1153 static bool needs_timeslice(const struct intel_engine_cs *engine,
1154 			    const struct i915_request *rq)
1155 {
1156 	if (!intel_engine_has_timeslices(engine))
1157 		return false;
1158 
1159 	/* If not currently active, or about to switch, wait for next event */
1160 	if (!rq || __i915_request_is_complete(rq))
1161 		return false;
1162 
1163 	/* We do not need to start the timeslice until after the ACK */
1164 	if (READ_ONCE(engine->execlists.pending[0]))
1165 		return false;
1166 
1167 	/* If ELSP[1] is occupied, always check to see if worth slicing */
1168 	if (!list_is_last_rcu(&rq->sched.link,
1169 			      &engine->sched_engine->requests)) {
1170 		ENGINE_TRACE(engine, "timeslice required for second inflight context\n");
1171 		return true;
1172 	}
1173 
1174 	/* Otherwise, ELSP[0] is by itself, but may be waiting in the queue */
1175 	if (!i915_sched_engine_is_empty(engine->sched_engine)) {
1176 		ENGINE_TRACE(engine, "timeslice required for queue\n");
1177 		return true;
1178 	}
1179 
1180 	if (!RB_EMPTY_ROOT(&engine->execlists.virtual.rb_root)) {
1181 		ENGINE_TRACE(engine, "timeslice required for virtual\n");
1182 		return true;
1183 	}
1184 
1185 	return false;
1186 }
1187 
1188 static bool
timeslice_expired(struct intel_engine_cs * engine,const struct i915_request * rq)1189 timeslice_expired(struct intel_engine_cs *engine, const struct i915_request *rq)
1190 {
1191 	const struct intel_engine_execlists *el = &engine->execlists;
1192 
1193 	if (i915_request_has_nopreempt(rq) && __i915_request_has_started(rq))
1194 		return false;
1195 
1196 	if (!needs_timeslice(engine, rq))
1197 		return false;
1198 
1199 	return timer_expired(&el->timer) || timeslice_yield(el, rq);
1200 }
1201 
timeslice(const struct intel_engine_cs * engine)1202 static unsigned long timeslice(const struct intel_engine_cs *engine)
1203 {
1204 	return READ_ONCE(engine->props.timeslice_duration_ms);
1205 }
1206 
start_timeslice(struct intel_engine_cs * engine)1207 static void start_timeslice(struct intel_engine_cs *engine)
1208 {
1209 	struct intel_engine_execlists *el = &engine->execlists;
1210 	unsigned long duration;
1211 
1212 	/* Disable the timer if there is nothing to switch to */
1213 	duration = 0;
1214 	if (needs_timeslice(engine, *el->active)) {
1215 		/* Avoid continually prolonging an active timeslice */
1216 		if (timer_active(&el->timer)) {
1217 			/*
1218 			 * If we just submitted a new ELSP after an old
1219 			 * context, that context may have already consumed
1220 			 * its timeslice, so recheck.
1221 			 */
1222 			if (!timer_pending(&el->timer))
1223 				tasklet_hi_schedule(&engine->sched_engine->tasklet);
1224 			return;
1225 		}
1226 
1227 		duration = timeslice(engine);
1228 	}
1229 
1230 	set_timer_ms(&el->timer, duration);
1231 }
1232 
record_preemption(struct intel_engine_execlists * execlists)1233 static void record_preemption(struct intel_engine_execlists *execlists)
1234 {
1235 	(void)I915_SELFTEST_ONLY(execlists->preempt_hang.count++);
1236 }
1237 
active_preempt_timeout(struct intel_engine_cs * engine,const struct i915_request * rq)1238 static unsigned long active_preempt_timeout(struct intel_engine_cs *engine,
1239 					    const struct i915_request *rq)
1240 {
1241 	if (!rq)
1242 		return 0;
1243 
1244 	/* Only allow ourselves to force reset the currently active context */
1245 	engine->execlists.preempt_target = rq;
1246 
1247 	/* Force a fast reset for terminated contexts (ignoring sysfs!) */
1248 	if (unlikely(intel_context_is_banned(rq->context) || bad_request(rq)))
1249 		return INTEL_CONTEXT_BANNED_PREEMPT_TIMEOUT_MS;
1250 
1251 	return READ_ONCE(engine->props.preempt_timeout_ms);
1252 }
1253 
set_preempt_timeout(struct intel_engine_cs * engine,const struct i915_request * rq)1254 static void set_preempt_timeout(struct intel_engine_cs *engine,
1255 				const struct i915_request *rq)
1256 {
1257 	if (!intel_engine_has_preempt_reset(engine))
1258 		return;
1259 
1260 	set_timer_ms(&engine->execlists.preempt,
1261 		     active_preempt_timeout(engine, rq));
1262 }
1263 
completed(const struct i915_request * rq)1264 static bool completed(const struct i915_request *rq)
1265 {
1266 	if (i915_request_has_sentinel(rq))
1267 		return false;
1268 
1269 	return __i915_request_is_complete(rq);
1270 }
1271 
execlists_dequeue(struct intel_engine_cs * engine)1272 static void execlists_dequeue(struct intel_engine_cs *engine)
1273 {
1274 	struct intel_engine_execlists * const execlists = &engine->execlists;
1275 	struct i915_sched_engine * const sched_engine = engine->sched_engine;
1276 	struct i915_request **port = execlists->pending;
1277 	struct i915_request ** const last_port = port + execlists->port_mask;
1278 	struct i915_request *last, * const *active;
1279 	struct virtual_engine *ve;
1280 	struct rb_node *rb;
1281 	bool submit = false;
1282 
1283 	/*
1284 	 * Hardware submission is through 2 ports. Conceptually each port
1285 	 * has a (RING_START, RING_HEAD, RING_TAIL) tuple. RING_START is
1286 	 * static for a context, and unique to each, so we only execute
1287 	 * requests belonging to a single context from each ring. RING_HEAD
1288 	 * is maintained by the CS in the context image, it marks the place
1289 	 * where it got up to last time, and through RING_TAIL we tell the CS
1290 	 * where we want to execute up to this time.
1291 	 *
1292 	 * In this list the requests are in order of execution. Consecutive
1293 	 * requests from the same context are adjacent in the ringbuffer. We
1294 	 * can combine these requests into a single RING_TAIL update:
1295 	 *
1296 	 *              RING_HEAD...req1...req2
1297 	 *                                    ^- RING_TAIL
1298 	 * since to execute req2 the CS must first execute req1.
1299 	 *
1300 	 * Our goal then is to point each port to the end of a consecutive
1301 	 * sequence of requests as being the most optimal (fewest wake ups
1302 	 * and context switches) submission.
1303 	 */
1304 
1305 	spin_lock(&sched_engine->lock);
1306 
1307 	/*
1308 	 * If the queue is higher priority than the last
1309 	 * request in the currently active context, submit afresh.
1310 	 * We will resubmit again afterwards in case we need to split
1311 	 * the active context to interject the preemption request,
1312 	 * i.e. we will retrigger preemption following the ack in case
1313 	 * of trouble.
1314 	 *
1315 	 */
1316 	active = execlists->active;
1317 	while ((last = *active) && completed(last))
1318 		active++;
1319 
1320 	if (last) {
1321 		if (need_preempt(engine, last)) {
1322 			ENGINE_TRACE(engine,
1323 				     "preempting last=%llx:%lld, prio=%d, hint=%d\n",
1324 				     last->fence.context,
1325 				     last->fence.seqno,
1326 				     last->sched.attr.priority,
1327 				     sched_engine->queue_priority_hint);
1328 			record_preemption(execlists);
1329 
1330 			/*
1331 			 * Don't let the RING_HEAD advance past the breadcrumb
1332 			 * as we unwind (and until we resubmit) so that we do
1333 			 * not accidentally tell it to go backwards.
1334 			 */
1335 			ring_set_paused(engine, 1);
1336 
1337 			/*
1338 			 * Note that we have not stopped the GPU at this point,
1339 			 * so we are unwinding the incomplete requests as they
1340 			 * remain inflight and so by the time we do complete
1341 			 * the preemption, some of the unwound requests may
1342 			 * complete!
1343 			 */
1344 			__unwind_incomplete_requests(engine);
1345 
1346 			last = NULL;
1347 		} else if (timeslice_expired(engine, last)) {
1348 			ENGINE_TRACE(engine,
1349 				     "expired:%s last=%llx:%lld, prio=%d, hint=%d, yield?=%s\n",
1350 				     str_yes_no(timer_expired(&execlists->timer)),
1351 				     last->fence.context, last->fence.seqno,
1352 				     rq_prio(last),
1353 				     sched_engine->queue_priority_hint,
1354 				     str_yes_no(timeslice_yield(execlists, last)));
1355 
1356 			/*
1357 			 * Consume this timeslice; ensure we start a new one.
1358 			 *
1359 			 * The timeslice expired, and we will unwind the
1360 			 * running contexts and recompute the next ELSP.
1361 			 * If that submit will be the same pair of contexts
1362 			 * (due to dependency ordering), we will skip the
1363 			 * submission. If we don't cancel the timer now,
1364 			 * we will see that the timer has expired and
1365 			 * reschedule the tasklet; continually until the
1366 			 * next context switch or other preemption event.
1367 			 *
1368 			 * Since we have decided to reschedule based on
1369 			 * consumption of this timeslice, if we submit the
1370 			 * same context again, grant it a full timeslice.
1371 			 */
1372 			cancel_timer(&execlists->timer);
1373 			ring_set_paused(engine, 1);
1374 			defer_active(engine);
1375 
1376 			/*
1377 			 * Unlike for preemption, if we rewind and continue
1378 			 * executing the same context as previously active,
1379 			 * the order of execution will remain the same and
1380 			 * the tail will only advance. We do not need to
1381 			 * force a full context restore, as a lite-restore
1382 			 * is sufficient to resample the monotonic TAIL.
1383 			 *
1384 			 * If we switch to any other context, similarly we
1385 			 * will not rewind TAIL of current context, and
1386 			 * normal save/restore will preserve state and allow
1387 			 * us to later continue executing the same request.
1388 			 */
1389 			last = NULL;
1390 		} else {
1391 			/*
1392 			 * Otherwise if we already have a request pending
1393 			 * for execution after the current one, we can
1394 			 * just wait until the next CS event before
1395 			 * queuing more. In either case we will force a
1396 			 * lite-restore preemption event, but if we wait
1397 			 * we hopefully coalesce several updates into a single
1398 			 * submission.
1399 			 */
1400 			if (active[1]) {
1401 				/*
1402 				 * Even if ELSP[1] is occupied and not worthy
1403 				 * of timeslices, our queue might be.
1404 				 */
1405 				spin_unlock(&sched_engine->lock);
1406 				return;
1407 			}
1408 		}
1409 	}
1410 
1411 	/* XXX virtual is always taking precedence */
1412 	while ((ve = first_virtual_engine(engine))) {
1413 		struct i915_request *rq;
1414 
1415 		spin_lock(&ve->base.sched_engine->lock);
1416 
1417 		rq = ve->request;
1418 		if (unlikely(!virtual_matches(ve, rq, engine)))
1419 			goto unlock; /* lost the race to a sibling */
1420 
1421 		GEM_BUG_ON(rq->engine != &ve->base);
1422 		GEM_BUG_ON(rq->context != &ve->context);
1423 
1424 		if (unlikely(rq_prio(rq) < queue_prio(sched_engine))) {
1425 			spin_unlock(&ve->base.sched_engine->lock);
1426 			break;
1427 		}
1428 
1429 		if (last && !can_merge_rq(last, rq)) {
1430 			spin_unlock(&ve->base.sched_engine->lock);
1431 			spin_unlock(&engine->sched_engine->lock);
1432 			return; /* leave this for another sibling */
1433 		}
1434 
1435 		ENGINE_TRACE(engine,
1436 			     "virtual rq=%llx:%lld%s, new engine? %s\n",
1437 			     rq->fence.context,
1438 			     rq->fence.seqno,
1439 			     __i915_request_is_complete(rq) ? "!" :
1440 			     __i915_request_has_started(rq) ? "*" :
1441 			     "",
1442 			     str_yes_no(engine != ve->siblings[0]));
1443 
1444 		WRITE_ONCE(ve->request, NULL);
1445 		WRITE_ONCE(ve->base.sched_engine->queue_priority_hint, INT_MIN);
1446 
1447 		rb = &ve->nodes[engine->id].rb;
1448 		rb_erase_cached(rb, &execlists->virtual);
1449 		RB_CLEAR_NODE(rb);
1450 
1451 		GEM_BUG_ON(!(rq->execution_mask & engine->mask));
1452 		WRITE_ONCE(rq->engine, engine);
1453 
1454 		if (__i915_request_submit(rq)) {
1455 			/*
1456 			 * Only after we confirm that we will submit
1457 			 * this request (i.e. it has not already
1458 			 * completed), do we want to update the context.
1459 			 *
1460 			 * This serves two purposes. It avoids
1461 			 * unnecessary work if we are resubmitting an
1462 			 * already completed request after timeslicing.
1463 			 * But more importantly, it prevents us altering
1464 			 * ve->siblings[] on an idle context, where
1465 			 * we may be using ve->siblings[] in
1466 			 * virtual_context_enter / virtual_context_exit.
1467 			 */
1468 			virtual_xfer_context(ve, engine);
1469 			GEM_BUG_ON(ve->siblings[0] != engine);
1470 
1471 			submit = true;
1472 			last = rq;
1473 		}
1474 
1475 		i915_request_put(rq);
1476 unlock:
1477 		spin_unlock(&ve->base.sched_engine->lock);
1478 
1479 		/*
1480 		 * Hmm, we have a bunch of virtual engine requests,
1481 		 * but the first one was already completed (thanks
1482 		 * preempt-to-busy!). Keep looking at the veng queue
1483 		 * until we have no more relevant requests (i.e.
1484 		 * the normal submit queue has higher priority).
1485 		 */
1486 		if (submit)
1487 			break;
1488 	}
1489 
1490 	while ((rb = rb_first_cached(&sched_engine->queue))) {
1491 		struct i915_priolist *p = to_priolist(rb);
1492 		struct i915_request *rq, *rn;
1493 
1494 		priolist_for_each_request_consume(rq, rn, p) {
1495 			bool merge = true;
1496 
1497 			/*
1498 			 * Can we combine this request with the current port?
1499 			 * It has to be the same context/ringbuffer and not
1500 			 * have any exceptions (e.g. GVT saying never to
1501 			 * combine contexts).
1502 			 *
1503 			 * If we can combine the requests, we can execute both
1504 			 * by updating the RING_TAIL to point to the end of the
1505 			 * second request, and so we never need to tell the
1506 			 * hardware about the first.
1507 			 */
1508 			if (last && !can_merge_rq(last, rq)) {
1509 				/*
1510 				 * If we are on the second port and cannot
1511 				 * combine this request with the last, then we
1512 				 * are done.
1513 				 */
1514 				if (port == last_port)
1515 					goto done;
1516 
1517 				/*
1518 				 * We must not populate both ELSP[] with the
1519 				 * same LRCA, i.e. we must submit 2 different
1520 				 * contexts if we submit 2 ELSP.
1521 				 */
1522 				if (last->context == rq->context)
1523 					goto done;
1524 
1525 				if (i915_request_has_sentinel(last))
1526 					goto done;
1527 
1528 				/*
1529 				 * We avoid submitting virtual requests into
1530 				 * the secondary ports so that we can migrate
1531 				 * the request immediately to another engine
1532 				 * rather than wait for the primary request.
1533 				 */
1534 				if (rq->execution_mask != engine->mask)
1535 					goto done;
1536 
1537 				/*
1538 				 * If GVT overrides us we only ever submit
1539 				 * port[0], leaving port[1] empty. Note that we
1540 				 * also have to be careful that we don't queue
1541 				 * the same context (even though a different
1542 				 * request) to the second port.
1543 				 */
1544 				if (ctx_single_port_submission(last->context) ||
1545 				    ctx_single_port_submission(rq->context))
1546 					goto done;
1547 
1548 				merge = false;
1549 			}
1550 
1551 			if (__i915_request_submit(rq)) {
1552 				if (!merge) {
1553 					*port++ = i915_request_get(last);
1554 					last = NULL;
1555 				}
1556 
1557 				GEM_BUG_ON(last &&
1558 					   !can_merge_ctx(last->context,
1559 							  rq->context));
1560 				GEM_BUG_ON(last &&
1561 					   i915_seqno_passed(last->fence.seqno,
1562 							     rq->fence.seqno));
1563 
1564 				submit = true;
1565 				last = rq;
1566 			}
1567 		}
1568 
1569 		rb_erase_cached(&p->node, &sched_engine->queue);
1570 		i915_priolist_free(p);
1571 	}
1572 done:
1573 	*port++ = i915_request_get(last);
1574 
1575 	/*
1576 	 * Here be a bit of magic! Or sleight-of-hand, whichever you prefer.
1577 	 *
1578 	 * We choose the priority hint such that if we add a request of greater
1579 	 * priority than this, we kick the submission tasklet to decide on
1580 	 * the right order of submitting the requests to hardware. We must
1581 	 * also be prepared to reorder requests as they are in-flight on the
1582 	 * HW. We derive the priority hint then as the first "hole" in
1583 	 * the HW submission ports and if there are no available slots,
1584 	 * the priority of the lowest executing request, i.e. last.
1585 	 *
1586 	 * When we do receive a higher priority request ready to run from the
1587 	 * user, see queue_request(), the priority hint is bumped to that
1588 	 * request triggering preemption on the next dequeue (or subsequent
1589 	 * interrupt for secondary ports).
1590 	 */
1591 	sched_engine->queue_priority_hint = queue_prio(sched_engine);
1592 	i915_sched_engine_reset_on_empty(sched_engine);
1593 	spin_unlock(&sched_engine->lock);
1594 
1595 	/*
1596 	 * We can skip poking the HW if we ended up with exactly the same set
1597 	 * of requests as currently running, e.g. trying to timeslice a pair
1598 	 * of ordered contexts.
1599 	 */
1600 	if (submit &&
1601 	    memcmp(active,
1602 		   execlists->pending,
1603 		   (port - execlists->pending) * sizeof(*port))) {
1604 		*port = NULL;
1605 		while (port-- != execlists->pending)
1606 			execlists_schedule_in(*port, port - execlists->pending);
1607 
1608 		WRITE_ONCE(execlists->yield, -1);
1609 		set_preempt_timeout(engine, *active);
1610 		execlists_submit_ports(engine);
1611 	} else {
1612 		ring_set_paused(engine, 0);
1613 		while (port-- != execlists->pending)
1614 			i915_request_put(*port);
1615 		*execlists->pending = NULL;
1616 	}
1617 }
1618 
execlists_dequeue_irq(struct intel_engine_cs * engine)1619 static void execlists_dequeue_irq(struct intel_engine_cs *engine)
1620 {
1621 	local_irq_disable(); /* Suspend interrupts across request submission */
1622 	execlists_dequeue(engine);
1623 	local_irq_enable(); /* flush irq_work (e.g. breadcrumb enabling) */
1624 }
1625 
clear_ports(struct i915_request ** ports,int count)1626 static void clear_ports(struct i915_request **ports, int count)
1627 {
1628 	memset_p((void **)ports, NULL, count);
1629 }
1630 
1631 static void
copy_ports(struct i915_request ** dst,struct i915_request ** src,int count)1632 copy_ports(struct i915_request **dst, struct i915_request **src, int count)
1633 {
1634 	/* A memcpy_p() would be very useful here! */
1635 	while (count--)
1636 		WRITE_ONCE(*dst++, *src++); /* avoid write tearing */
1637 }
1638 
1639 static struct i915_request **
cancel_port_requests(struct intel_engine_execlists * const execlists,struct i915_request ** inactive)1640 cancel_port_requests(struct intel_engine_execlists * const execlists,
1641 		     struct i915_request **inactive)
1642 {
1643 	struct i915_request * const *port;
1644 
1645 	for (port = execlists->pending; *port; port++)
1646 		*inactive++ = *port;
1647 	clear_ports(execlists->pending, ARRAY_SIZE(execlists->pending));
1648 
1649 	/* Mark the end of active before we overwrite *active */
1650 	for (port = xchg(&execlists->active, execlists->pending); *port; port++)
1651 		*inactive++ = *port;
1652 	clear_ports(execlists->inflight, ARRAY_SIZE(execlists->inflight));
1653 
1654 	smp_wmb(); /* complete the seqlock for execlists_active() */
1655 	WRITE_ONCE(execlists->active, execlists->inflight);
1656 
1657 	/* Having cancelled all outstanding process_csb(), stop their timers */
1658 	GEM_BUG_ON(execlists->pending[0]);
1659 	cancel_timer(&execlists->timer);
1660 	cancel_timer(&execlists->preempt);
1661 
1662 	return inactive;
1663 }
1664 
1665 /*
1666  * Starting with Gen12, the status has a new format:
1667  *
1668  *     bit  0:     switched to new queue
1669  *     bit  1:     reserved
1670  *     bit  2:     semaphore wait mode (poll or signal), only valid when
1671  *                 switch detail is set to "wait on semaphore"
1672  *     bits 3-5:   engine class
1673  *     bits 6-11:  engine instance
1674  *     bits 12-14: reserved
1675  *     bits 15-25: sw context id of the lrc the GT switched to
1676  *     bits 26-31: sw counter of the lrc the GT switched to
1677  *     bits 32-35: context switch detail
1678  *                  - 0: ctx complete
1679  *                  - 1: wait on sync flip
1680  *                  - 2: wait on vblank
1681  *                  - 3: wait on scanline
1682  *                  - 4: wait on semaphore
1683  *                  - 5: context preempted (not on SEMAPHORE_WAIT or
1684  *                       WAIT_FOR_EVENT)
1685  *     bit  36:    reserved
1686  *     bits 37-43: wait detail (for switch detail 1 to 4)
1687  *     bits 44-46: reserved
1688  *     bits 47-57: sw context id of the lrc the GT switched away from
1689  *     bits 58-63: sw counter of the lrc the GT switched away from
1690  *
1691  * Xe_HP csb shuffles things around compared to TGL:
1692  *
1693  *     bits 0-3:   context switch detail (same possible values as TGL)
1694  *     bits 4-9:   engine instance
1695  *     bits 10-25: sw context id of the lrc the GT switched to
1696  *     bits 26-31: sw counter of the lrc the GT switched to
1697  *     bit  32:    semaphore wait mode (poll or signal), Only valid when
1698  *                 switch detail is set to "wait on semaphore"
1699  *     bit  33:    switched to new queue
1700  *     bits 34-41: wait detail (for switch detail 1 to 4)
1701  *     bits 42-57: sw context id of the lrc the GT switched away from
1702  *     bits 58-63: sw counter of the lrc the GT switched away from
1703  */
1704 static inline bool
__gen12_csb_parse(bool ctx_to_valid,bool ctx_away_valid,bool new_queue,u8 switch_detail)1705 __gen12_csb_parse(bool ctx_to_valid, bool ctx_away_valid, bool new_queue,
1706 		  u8 switch_detail)
1707 {
1708 	/*
1709 	 * The context switch detail is not guaranteed to be 5 when a preemption
1710 	 * occurs, so we can't just check for that. The check below works for
1711 	 * all the cases we care about, including preemptions of WAIT
1712 	 * instructions and lite-restore. Preempt-to-idle via the CTRL register
1713 	 * would require some extra handling, but we don't support that.
1714 	 */
1715 	if (!ctx_away_valid || new_queue) {
1716 		GEM_BUG_ON(!ctx_to_valid);
1717 		return true;
1718 	}
1719 
1720 	/*
1721 	 * switch detail = 5 is covered by the case above and we do not expect a
1722 	 * context switch on an unsuccessful wait instruction since we always
1723 	 * use polling mode.
1724 	 */
1725 	GEM_BUG_ON(switch_detail);
1726 	return false;
1727 }
1728 
xehp_csb_parse(const u64 csb)1729 static bool xehp_csb_parse(const u64 csb)
1730 {
1731 	return __gen12_csb_parse(XEHP_CSB_CTX_VALID(lower_32_bits(csb)), /* cxt to */
1732 				 XEHP_CSB_CTX_VALID(upper_32_bits(csb)), /* cxt away */
1733 				 upper_32_bits(csb) & XEHP_CTX_STATUS_SWITCHED_TO_NEW_QUEUE,
1734 				 GEN12_CTX_SWITCH_DETAIL(lower_32_bits(csb)));
1735 }
1736 
gen12_csb_parse(const u64 csb)1737 static bool gen12_csb_parse(const u64 csb)
1738 {
1739 	return __gen12_csb_parse(GEN12_CSB_CTX_VALID(lower_32_bits(csb)), /* cxt to */
1740 				 GEN12_CSB_CTX_VALID(upper_32_bits(csb)), /* cxt away */
1741 				 lower_32_bits(csb) & GEN12_CTX_STATUS_SWITCHED_TO_NEW_QUEUE,
1742 				 GEN12_CTX_SWITCH_DETAIL(upper_32_bits(csb)));
1743 }
1744 
gen8_csb_parse(const u64 csb)1745 static bool gen8_csb_parse(const u64 csb)
1746 {
1747 	return csb & (GEN8_CTX_STATUS_IDLE_ACTIVE | GEN8_CTX_STATUS_PREEMPTED);
1748 }
1749 
1750 static noinline u64
wa_csb_read(const struct intel_engine_cs * engine,u64 * const csb)1751 wa_csb_read(const struct intel_engine_cs *engine, u64 * const csb)
1752 {
1753 	u64 entry;
1754 
1755 	/*
1756 	 * Reading from the HWSP has one particular advantage: we can detect
1757 	 * a stale entry. Since the write into HWSP is broken, we have no reason
1758 	 * to trust the HW at all, the mmio entry may equally be unordered, so
1759 	 * we prefer the path that is self-checking and as a last resort,
1760 	 * return the mmio value.
1761 	 *
1762 	 * tgl,dg1:HSDES#22011327657
1763 	 */
1764 	preempt_disable();
1765 	if (wait_for_atomic_us((entry = READ_ONCE(*csb)) != -1, 10)) {
1766 		int idx = csb - engine->execlists.csb_status;
1767 		int status;
1768 
1769 		status = GEN8_EXECLISTS_STATUS_BUF;
1770 		if (idx >= 6) {
1771 			status = GEN11_EXECLISTS_STATUS_BUF2;
1772 			idx -= 6;
1773 		}
1774 		status += sizeof(u64) * idx;
1775 
1776 		entry = intel_uncore_read64(engine->uncore,
1777 					    _MMIO(engine->mmio_base + status));
1778 	}
1779 	preempt_enable();
1780 
1781 	return entry;
1782 }
1783 
csb_read(const struct intel_engine_cs * engine,u64 * const csb)1784 static u64 csb_read(const struct intel_engine_cs *engine, u64 * const csb)
1785 {
1786 	u64 entry = READ_ONCE(*csb);
1787 
1788 	/*
1789 	 * Unfortunately, the GPU does not always serialise its write
1790 	 * of the CSB entries before its write of the CSB pointer, at least
1791 	 * from the perspective of the CPU, using what is known as a Global
1792 	 * Observation Point. We may read a new CSB tail pointer, but then
1793 	 * read the stale CSB entries, causing us to misinterpret the
1794 	 * context-switch events, and eventually declare the GPU hung.
1795 	 *
1796 	 * icl:HSDES#1806554093
1797 	 * tgl:HSDES#22011248461
1798 	 */
1799 	if (unlikely(entry == -1))
1800 		entry = wa_csb_read(engine, csb);
1801 
1802 	/* Consume this entry so that we can spot its future reuse. */
1803 	WRITE_ONCE(*csb, -1);
1804 
1805 	/* ELSP is an implicit wmb() before the GPU wraps and overwrites csb */
1806 	return entry;
1807 }
1808 
new_timeslice(struct intel_engine_execlists * el)1809 static void new_timeslice(struct intel_engine_execlists *el)
1810 {
1811 	/* By cancelling, we will start afresh in start_timeslice() */
1812 	cancel_timer(&el->timer);
1813 }
1814 
1815 static struct i915_request **
process_csb(struct intel_engine_cs * engine,struct i915_request ** inactive)1816 process_csb(struct intel_engine_cs *engine, struct i915_request **inactive)
1817 {
1818 	struct intel_engine_execlists * const execlists = &engine->execlists;
1819 	u64 * const buf = execlists->csb_status;
1820 	const u8 num_entries = execlists->csb_size;
1821 	struct i915_request **prev;
1822 	u8 head, tail;
1823 
1824 	/*
1825 	 * As we modify our execlists state tracking we require exclusive
1826 	 * access. Either we are inside the tasklet, or the tasklet is disabled
1827 	 * and we assume that is only inside the reset paths and so serialised.
1828 	 */
1829 	GEM_BUG_ON(!tasklet_is_locked(&engine->sched_engine->tasklet) &&
1830 		   !reset_in_progress(engine));
1831 
1832 	/*
1833 	 * Note that csb_write, csb_status may be either in HWSP or mmio.
1834 	 * When reading from the csb_write mmio register, we have to be
1835 	 * careful to only use the GEN8_CSB_WRITE_PTR portion, which is
1836 	 * the low 4bits. As it happens we know the next 4bits are always
1837 	 * zero and so we can simply masked off the low u8 of the register
1838 	 * and treat it identically to reading from the HWSP (without having
1839 	 * to use explicit shifting and masking, and probably bifurcating
1840 	 * the code to handle the legacy mmio read).
1841 	 */
1842 	head = execlists->csb_head;
1843 	tail = READ_ONCE(*execlists->csb_write);
1844 	if (unlikely(head == tail))
1845 		return inactive;
1846 
1847 	/*
1848 	 * We will consume all events from HW, or at least pretend to.
1849 	 *
1850 	 * The sequence of events from the HW is deterministic, and derived
1851 	 * from our writes to the ELSP, with a smidgen of variability for
1852 	 * the arrival of the asynchronous requests wrt to the inflight
1853 	 * execution. If the HW sends an event that does not correspond with
1854 	 * the one we are expecting, we have to abandon all hope as we lose
1855 	 * all tracking of what the engine is actually executing. We will
1856 	 * only detect we are out of sequence with the HW when we get an
1857 	 * 'impossible' event because we have already drained our own
1858 	 * preemption/promotion queue. If this occurs, we know that we likely
1859 	 * lost track of execution earlier and must unwind and restart, the
1860 	 * simplest way is by stop processing the event queue and force the
1861 	 * engine to reset.
1862 	 */
1863 	execlists->csb_head = tail;
1864 	ENGINE_TRACE(engine, "cs-irq head=%d, tail=%d\n", head, tail);
1865 
1866 	/*
1867 	 * Hopefully paired with a wmb() in HW!
1868 	 *
1869 	 * We must complete the read of the write pointer before any reads
1870 	 * from the CSB, so that we do not see stale values. Without an rmb
1871 	 * (lfence) the HW may speculatively perform the CSB[] reads *before*
1872 	 * we perform the READ_ONCE(*csb_write).
1873 	 */
1874 	rmb();
1875 
1876 	/* Remember who was last running under the timer */
1877 	prev = inactive;
1878 	*prev = NULL;
1879 
1880 	do {
1881 		bool promote;
1882 		u64 csb;
1883 
1884 		if (++head == num_entries)
1885 			head = 0;
1886 
1887 		/*
1888 		 * We are flying near dragons again.
1889 		 *
1890 		 * We hold a reference to the request in execlist_port[]
1891 		 * but no more than that. We are operating in softirq
1892 		 * context and so cannot hold any mutex or sleep. That
1893 		 * prevents us stopping the requests we are processing
1894 		 * in port[] from being retired simultaneously (the
1895 		 * breadcrumb will be complete before we see the
1896 		 * context-switch). As we only hold the reference to the
1897 		 * request, any pointer chasing underneath the request
1898 		 * is subject to a potential use-after-free. Thus we
1899 		 * store all of the bookkeeping within port[] as
1900 		 * required, and avoid using unguarded pointers beneath
1901 		 * request itself. The same applies to the atomic
1902 		 * status notifier.
1903 		 */
1904 
1905 		csb = csb_read(engine, buf + head);
1906 		ENGINE_TRACE(engine, "csb[%d]: status=0x%08x:0x%08x\n",
1907 			     head, upper_32_bits(csb), lower_32_bits(csb));
1908 
1909 		if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50))
1910 			promote = xehp_csb_parse(csb);
1911 		else if (GRAPHICS_VER(engine->i915) >= 12)
1912 			promote = gen12_csb_parse(csb);
1913 		else
1914 			promote = gen8_csb_parse(csb);
1915 		if (promote) {
1916 			struct i915_request * const *old = execlists->active;
1917 
1918 			if (GEM_WARN_ON(!*execlists->pending)) {
1919 				execlists->error_interrupt |= ERROR_CSB;
1920 				break;
1921 			}
1922 
1923 			ring_set_paused(engine, 0);
1924 
1925 			/* Point active to the new ELSP; prevent overwriting */
1926 			WRITE_ONCE(execlists->active, execlists->pending);
1927 			smp_wmb(); /* notify execlists_active() */
1928 
1929 			/* cancel old inflight, prepare for switch */
1930 			trace_ports(execlists, "preempted", old);
1931 			while (*old)
1932 				*inactive++ = *old++;
1933 
1934 			/* switch pending to inflight */
1935 			GEM_BUG_ON(!assert_pending_valid(execlists, "promote"));
1936 			copy_ports(execlists->inflight,
1937 				   execlists->pending,
1938 				   execlists_num_ports(execlists));
1939 			smp_wmb(); /* complete the seqlock */
1940 			WRITE_ONCE(execlists->active, execlists->inflight);
1941 
1942 			/* XXX Magic delay for tgl */
1943 			ENGINE_POSTING_READ(engine, RING_CONTEXT_STATUS_PTR);
1944 
1945 			WRITE_ONCE(execlists->pending[0], NULL);
1946 		} else {
1947 			if (GEM_WARN_ON(!*execlists->active)) {
1948 				execlists->error_interrupt |= ERROR_CSB;
1949 				break;
1950 			}
1951 
1952 			/* port0 completed, advanced to port1 */
1953 			trace_ports(execlists, "completed", execlists->active);
1954 
1955 			/*
1956 			 * We rely on the hardware being strongly
1957 			 * ordered, that the breadcrumb write is
1958 			 * coherent (visible from the CPU) before the
1959 			 * user interrupt is processed. One might assume
1960 			 * that the breadcrumb write being before the
1961 			 * user interrupt and the CS event for the context
1962 			 * switch would therefore be before the CS event
1963 			 * itself...
1964 			 */
1965 			if (GEM_SHOW_DEBUG() &&
1966 			    !__i915_request_is_complete(*execlists->active)) {
1967 				struct i915_request *rq = *execlists->active;
1968 				const u32 *regs __maybe_unused =
1969 					rq->context->lrc_reg_state;
1970 
1971 				ENGINE_TRACE(engine,
1972 					     "context completed before request!\n");
1973 				ENGINE_TRACE(engine,
1974 					     "ring:{start:0x%08x, head:%04x, tail:%04x, ctl:%08x, mode:%08x}\n",
1975 					     ENGINE_READ(engine, RING_START),
1976 					     ENGINE_READ(engine, RING_HEAD) & HEAD_ADDR,
1977 					     ENGINE_READ(engine, RING_TAIL) & TAIL_ADDR,
1978 					     ENGINE_READ(engine, RING_CTL),
1979 					     ENGINE_READ(engine, RING_MI_MODE));
1980 				ENGINE_TRACE(engine,
1981 					     "rq:{start:%08x, head:%04x, tail:%04x, seqno:%llx:%d, hwsp:%d}, ",
1982 					     i915_ggtt_offset(rq->ring->vma),
1983 					     rq->head, rq->tail,
1984 					     rq->fence.context,
1985 					     lower_32_bits(rq->fence.seqno),
1986 					     hwsp_seqno(rq));
1987 				ENGINE_TRACE(engine,
1988 					     "ctx:{start:%08x, head:%04x, tail:%04x}, ",
1989 					     regs[CTX_RING_START],
1990 					     regs[CTX_RING_HEAD],
1991 					     regs[CTX_RING_TAIL]);
1992 			}
1993 
1994 			*inactive++ = *execlists->active++;
1995 
1996 			GEM_BUG_ON(execlists->active - execlists->inflight >
1997 				   execlists_num_ports(execlists));
1998 		}
1999 	} while (head != tail);
2000 
2001 	/*
2002 	 * Gen11 has proven to fail wrt global observation point between
2003 	 * entry and tail update, failing on the ordering and thus
2004 	 * we see an old entry in the context status buffer.
2005 	 *
2006 	 * Forcibly evict out entries for the next gpu csb update,
2007 	 * to increase the odds that we get a fresh entries with non
2008 	 * working hardware. The cost for doing so comes out mostly with
2009 	 * the wash as hardware, working or not, will need to do the
2010 	 * invalidation before.
2011 	 */
2012 	drm_clflush_virt_range(&buf[0], num_entries * sizeof(buf[0]));
2013 
2014 	/*
2015 	 * We assume that any event reflects a change in context flow
2016 	 * and merits a fresh timeslice. We reinstall the timer after
2017 	 * inspecting the queue to see if we need to resumbit.
2018 	 */
2019 	if (*prev != *execlists->active) { /* elide lite-restores */
2020 		/*
2021 		 * Note the inherent discrepancy between the HW runtime,
2022 		 * recorded as part of the context switch, and the CPU
2023 		 * adjustment for active contexts. We have to hope that
2024 		 * the delay in processing the CS event is very small
2025 		 * and consistent. It works to our advantage to have
2026 		 * the CPU adjustment _undershoot_ (i.e. start later than)
2027 		 * the CS timestamp so we never overreport the runtime
2028 		 * and correct overselves later when updating from HW.
2029 		 */
2030 		if (*prev)
2031 			lrc_runtime_stop((*prev)->context);
2032 		if (*execlists->active)
2033 			lrc_runtime_start((*execlists->active)->context);
2034 		new_timeslice(execlists);
2035 	}
2036 
2037 	return inactive;
2038 }
2039 
post_process_csb(struct i915_request ** port,struct i915_request ** last)2040 static void post_process_csb(struct i915_request **port,
2041 			     struct i915_request **last)
2042 {
2043 	while (port != last)
2044 		execlists_schedule_out(*port++);
2045 }
2046 
__execlists_hold(struct i915_request * rq)2047 static void __execlists_hold(struct i915_request *rq)
2048 {
2049 	LIST_HEAD(list);
2050 
2051 	do {
2052 		struct i915_dependency *p;
2053 
2054 		if (i915_request_is_active(rq))
2055 			__i915_request_unsubmit(rq);
2056 
2057 		clear_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);
2058 		list_move_tail(&rq->sched.link,
2059 			       &rq->engine->sched_engine->hold);
2060 		i915_request_set_hold(rq);
2061 		RQ_TRACE(rq, "on hold\n");
2062 
2063 		for_each_waiter(p, rq) {
2064 			struct i915_request *w =
2065 				container_of(p->waiter, typeof(*w), sched);
2066 
2067 			if (p->flags & I915_DEPENDENCY_WEAK)
2068 				continue;
2069 
2070 			/* Leave semaphores spinning on the other engines */
2071 			if (w->engine != rq->engine)
2072 				continue;
2073 
2074 			if (!i915_request_is_ready(w))
2075 				continue;
2076 
2077 			if (__i915_request_is_complete(w))
2078 				continue;
2079 
2080 			if (i915_request_on_hold(w))
2081 				continue;
2082 
2083 			list_move_tail(&w->sched.link, &list);
2084 		}
2085 
2086 		rq = list_first_entry_or_null(&list, typeof(*rq), sched.link);
2087 	} while (rq);
2088 }
2089 
execlists_hold(struct intel_engine_cs * engine,struct i915_request * rq)2090 static bool execlists_hold(struct intel_engine_cs *engine,
2091 			   struct i915_request *rq)
2092 {
2093 	if (i915_request_on_hold(rq))
2094 		return false;
2095 
2096 	spin_lock_irq(&engine->sched_engine->lock);
2097 
2098 	if (__i915_request_is_complete(rq)) { /* too late! */
2099 		rq = NULL;
2100 		goto unlock;
2101 	}
2102 
2103 	/*
2104 	 * Transfer this request onto the hold queue to prevent it
2105 	 * being resumbitted to HW (and potentially completed) before we have
2106 	 * released it. Since we may have already submitted following
2107 	 * requests, we need to remove those as well.
2108 	 */
2109 	GEM_BUG_ON(i915_request_on_hold(rq));
2110 	GEM_BUG_ON(rq->engine != engine);
2111 	__execlists_hold(rq);
2112 	GEM_BUG_ON(list_empty(&engine->sched_engine->hold));
2113 
2114 unlock:
2115 	spin_unlock_irq(&engine->sched_engine->lock);
2116 	return rq;
2117 }
2118 
hold_request(const struct i915_request * rq)2119 static bool hold_request(const struct i915_request *rq)
2120 {
2121 	struct i915_dependency *p;
2122 	bool result = false;
2123 
2124 	/*
2125 	 * If one of our ancestors is on hold, we must also be on hold,
2126 	 * otherwise we will bypass it and execute before it.
2127 	 */
2128 	rcu_read_lock();
2129 	for_each_signaler(p, rq) {
2130 		const struct i915_request *s =
2131 			container_of(p->signaler, typeof(*s), sched);
2132 
2133 		if (s->engine != rq->engine)
2134 			continue;
2135 
2136 		result = i915_request_on_hold(s);
2137 		if (result)
2138 			break;
2139 	}
2140 	rcu_read_unlock();
2141 
2142 	return result;
2143 }
2144 
__execlists_unhold(struct i915_request * rq)2145 static void __execlists_unhold(struct i915_request *rq)
2146 {
2147 	LIST_HEAD(list);
2148 
2149 	do {
2150 		struct i915_dependency *p;
2151 
2152 		RQ_TRACE(rq, "hold release\n");
2153 
2154 		GEM_BUG_ON(!i915_request_on_hold(rq));
2155 		GEM_BUG_ON(!i915_sw_fence_signaled(&rq->submit));
2156 
2157 		i915_request_clear_hold(rq);
2158 		list_move_tail(&rq->sched.link,
2159 			       i915_sched_lookup_priolist(rq->engine->sched_engine,
2160 							  rq_prio(rq)));
2161 		set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);
2162 
2163 		/* Also release any children on this engine that are ready */
2164 		for_each_waiter(p, rq) {
2165 			struct i915_request *w =
2166 				container_of(p->waiter, typeof(*w), sched);
2167 
2168 			if (p->flags & I915_DEPENDENCY_WEAK)
2169 				continue;
2170 
2171 			if (w->engine != rq->engine)
2172 				continue;
2173 
2174 			if (!i915_request_on_hold(w))
2175 				continue;
2176 
2177 			/* Check that no other parents are also on hold */
2178 			if (hold_request(w))
2179 				continue;
2180 
2181 			list_move_tail(&w->sched.link, &list);
2182 		}
2183 
2184 		rq = list_first_entry_or_null(&list, typeof(*rq), sched.link);
2185 	} while (rq);
2186 }
2187 
execlists_unhold(struct intel_engine_cs * engine,struct i915_request * rq)2188 static void execlists_unhold(struct intel_engine_cs *engine,
2189 			     struct i915_request *rq)
2190 {
2191 	spin_lock_irq(&engine->sched_engine->lock);
2192 
2193 	/*
2194 	 * Move this request back to the priority queue, and all of its
2195 	 * children and grandchildren that were suspended along with it.
2196 	 */
2197 	__execlists_unhold(rq);
2198 
2199 	if (rq_prio(rq) > engine->sched_engine->queue_priority_hint) {
2200 		engine->sched_engine->queue_priority_hint = rq_prio(rq);
2201 		tasklet_hi_schedule(&engine->sched_engine->tasklet);
2202 	}
2203 
2204 	spin_unlock_irq(&engine->sched_engine->lock);
2205 }
2206 
2207 struct execlists_capture {
2208 	struct work_struct work;
2209 	struct i915_request *rq;
2210 	struct i915_gpu_coredump *error;
2211 };
2212 
execlists_capture_work(struct work_struct * work)2213 static void execlists_capture_work(struct work_struct *work)
2214 {
2215 	struct execlists_capture *cap = container_of(work, typeof(*cap), work);
2216 	const gfp_t gfp = __GFP_KSWAPD_RECLAIM | __GFP_RETRY_MAYFAIL |
2217 		__GFP_NOWARN;
2218 	struct intel_engine_cs *engine = cap->rq->engine;
2219 	struct intel_gt_coredump *gt = cap->error->gt;
2220 	struct intel_engine_capture_vma *vma;
2221 
2222 	/* Compress all the objects attached to the request, slow! */
2223 	vma = intel_engine_coredump_add_request(gt->engine, cap->rq, gfp);
2224 	if (vma) {
2225 		struct i915_vma_compress *compress =
2226 			i915_vma_capture_prepare(gt);
2227 
2228 		intel_engine_coredump_add_vma(gt->engine, vma, compress);
2229 		i915_vma_capture_finish(gt, compress);
2230 	}
2231 
2232 	gt->simulated = gt->engine->simulated;
2233 	cap->error->simulated = gt->simulated;
2234 
2235 	/* Publish the error state, and announce it to the world */
2236 	i915_error_state_store(cap->error);
2237 	i915_gpu_coredump_put(cap->error);
2238 
2239 	/* Return this request and all that depend upon it for signaling */
2240 	execlists_unhold(engine, cap->rq);
2241 	i915_request_put(cap->rq);
2242 
2243 	kfree(cap);
2244 }
2245 
capture_regs(struct intel_engine_cs * engine)2246 static struct execlists_capture *capture_regs(struct intel_engine_cs *engine)
2247 {
2248 	const gfp_t gfp = GFP_ATOMIC | __GFP_NOWARN;
2249 	struct execlists_capture *cap;
2250 
2251 	cap = kmalloc(sizeof(*cap), gfp);
2252 	if (!cap)
2253 		return NULL;
2254 
2255 	cap->error = i915_gpu_coredump_alloc(engine->i915, gfp);
2256 	if (!cap->error)
2257 		goto err_cap;
2258 
2259 	cap->error->gt = intel_gt_coredump_alloc(engine->gt, gfp, CORE_DUMP_FLAG_NONE);
2260 	if (!cap->error->gt)
2261 		goto err_gpu;
2262 
2263 	cap->error->gt->engine = intel_engine_coredump_alloc(engine, gfp, CORE_DUMP_FLAG_NONE);
2264 	if (!cap->error->gt->engine)
2265 		goto err_gt;
2266 
2267 	cap->error->gt->engine->hung = true;
2268 
2269 	return cap;
2270 
2271 err_gt:
2272 	kfree(cap->error->gt);
2273 err_gpu:
2274 	kfree(cap->error);
2275 err_cap:
2276 	kfree(cap);
2277 	return NULL;
2278 }
2279 
2280 static struct i915_request *
active_context(struct intel_engine_cs * engine,u32 ccid)2281 active_context(struct intel_engine_cs *engine, u32 ccid)
2282 {
2283 	const struct intel_engine_execlists * const el = &engine->execlists;
2284 	struct i915_request * const *port, *rq;
2285 
2286 	/*
2287 	 * Use the most recent result from process_csb(), but just in case
2288 	 * we trigger an error (via interrupt) before the first CS event has
2289 	 * been written, peek at the next submission.
2290 	 */
2291 
2292 	for (port = el->active; (rq = *port); port++) {
2293 		if (rq->context->lrc.ccid == ccid) {
2294 			ENGINE_TRACE(engine,
2295 				     "ccid:%x found at active:%zd\n",
2296 				     ccid, port - el->active);
2297 			return rq;
2298 		}
2299 	}
2300 
2301 	for (port = el->pending; (rq = *port); port++) {
2302 		if (rq->context->lrc.ccid == ccid) {
2303 			ENGINE_TRACE(engine,
2304 				     "ccid:%x found at pending:%zd\n",
2305 				     ccid, port - el->pending);
2306 			return rq;
2307 		}
2308 	}
2309 
2310 	ENGINE_TRACE(engine, "ccid:%x not found\n", ccid);
2311 	return NULL;
2312 }
2313 
active_ccid(struct intel_engine_cs * engine)2314 static u32 active_ccid(struct intel_engine_cs *engine)
2315 {
2316 	return ENGINE_READ_FW(engine, RING_EXECLIST_STATUS_HI);
2317 }
2318 
execlists_capture(struct intel_engine_cs * engine)2319 static void execlists_capture(struct intel_engine_cs *engine)
2320 {
2321 	struct execlists_capture *cap;
2322 
2323 	if (!IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR))
2324 		return;
2325 
2326 	/*
2327 	 * We need to _quickly_ capture the engine state before we reset.
2328 	 * We are inside an atomic section (softirq) here and we are delaying
2329 	 * the forced preemption event.
2330 	 */
2331 	cap = capture_regs(engine);
2332 	if (!cap)
2333 		return;
2334 
2335 	spin_lock_irq(&engine->sched_engine->lock);
2336 	cap->rq = active_context(engine, active_ccid(engine));
2337 	if (cap->rq) {
2338 		cap->rq = active_request(cap->rq->context->timeline, cap->rq);
2339 		cap->rq = i915_request_get_rcu(cap->rq);
2340 	}
2341 	spin_unlock_irq(&engine->sched_engine->lock);
2342 	if (!cap->rq)
2343 		goto err_free;
2344 
2345 	/*
2346 	 * Remove the request from the execlists queue, and take ownership
2347 	 * of the request. We pass it to our worker who will _slowly_ compress
2348 	 * all the pages the _user_ requested for debugging their batch, after
2349 	 * which we return it to the queue for signaling.
2350 	 *
2351 	 * By removing them from the execlists queue, we also remove the
2352 	 * requests from being processed by __unwind_incomplete_requests()
2353 	 * during the intel_engine_reset(), and so they will *not* be replayed
2354 	 * afterwards.
2355 	 *
2356 	 * Note that because we have not yet reset the engine at this point,
2357 	 * it is possible for the request that we have identified as being
2358 	 * guilty, did in fact complete and we will then hit an arbitration
2359 	 * point allowing the outstanding preemption to succeed. The likelihood
2360 	 * of that is very low (as capturing of the engine registers should be
2361 	 * fast enough to run inside an irq-off atomic section!), so we will
2362 	 * simply hold that request accountable for being non-preemptible
2363 	 * long enough to force the reset.
2364 	 */
2365 	if (!execlists_hold(engine, cap->rq))
2366 		goto err_rq;
2367 
2368 	INIT_WORK(&cap->work, execlists_capture_work);
2369 	schedule_work(&cap->work);
2370 	return;
2371 
2372 err_rq:
2373 	i915_request_put(cap->rq);
2374 err_free:
2375 	i915_gpu_coredump_put(cap->error);
2376 	kfree(cap);
2377 }
2378 
execlists_reset(struct intel_engine_cs * engine,const char * msg)2379 static void execlists_reset(struct intel_engine_cs *engine, const char *msg)
2380 {
2381 	const unsigned int bit = I915_RESET_ENGINE + engine->id;
2382 	unsigned long *lock = &engine->gt->reset.flags;
2383 
2384 	if (!intel_has_reset_engine(engine->gt))
2385 		return;
2386 
2387 	if (test_and_set_bit(bit, lock))
2388 		return;
2389 
2390 	ENGINE_TRACE(engine, "reset for %s\n", msg);
2391 
2392 	/* Mark this tasklet as disabled to avoid waiting for it to complete */
2393 	tasklet_disable_nosync(&engine->sched_engine->tasklet);
2394 
2395 	ring_set_paused(engine, 1); /* Freeze the current request in place */
2396 	execlists_capture(engine);
2397 	intel_engine_reset(engine, msg);
2398 
2399 	tasklet_enable(&engine->sched_engine->tasklet);
2400 	clear_and_wake_up_bit(bit, lock);
2401 }
2402 
preempt_timeout(const struct intel_engine_cs * const engine)2403 static bool preempt_timeout(const struct intel_engine_cs *const engine)
2404 {
2405 	const struct timer_list *t = &engine->execlists.preempt;
2406 
2407 	if (!CONFIG_DRM_I915_PREEMPT_TIMEOUT)
2408 		return false;
2409 
2410 	if (!timer_expired(t))
2411 		return false;
2412 
2413 	return engine->execlists.pending[0];
2414 }
2415 
2416 /*
2417  * Check the unread Context Status Buffers and manage the submission of new
2418  * contexts to the ELSP accordingly.
2419  */
execlists_submission_tasklet(struct tasklet_struct * t)2420 static void execlists_submission_tasklet(struct tasklet_struct *t)
2421 {
2422 	struct i915_sched_engine *sched_engine =
2423 		from_tasklet(sched_engine, t, tasklet);
2424 	struct intel_engine_cs * const engine = sched_engine->private_data;
2425 	struct i915_request *post[2 * EXECLIST_MAX_PORTS];
2426 	struct i915_request **inactive;
2427 
2428 	rcu_read_lock();
2429 	inactive = process_csb(engine, post);
2430 	GEM_BUG_ON(inactive - post > ARRAY_SIZE(post));
2431 
2432 	if (unlikely(preempt_timeout(engine))) {
2433 		const struct i915_request *rq = *engine->execlists.active;
2434 
2435 		/*
2436 		 * If after the preempt-timeout expired, we are still on the
2437 		 * same active request/context as before we initiated the
2438 		 * preemption, reset the engine.
2439 		 *
2440 		 * However, if we have processed a CS event to switch contexts,
2441 		 * but not yet processed the CS event for the pending
2442 		 * preemption, reset the timer allowing the new context to
2443 		 * gracefully exit.
2444 		 */
2445 		cancel_timer(&engine->execlists.preempt);
2446 		if (rq == engine->execlists.preempt_target)
2447 			engine->execlists.error_interrupt |= ERROR_PREEMPT;
2448 		else
2449 			set_timer_ms(&engine->execlists.preempt,
2450 				     active_preempt_timeout(engine, rq));
2451 	}
2452 
2453 	if (unlikely(READ_ONCE(engine->execlists.error_interrupt))) {
2454 		const char *msg;
2455 
2456 		/* Generate the error message in priority wrt to the user! */
2457 		if (engine->execlists.error_interrupt & GENMASK(15, 0))
2458 			msg = "CS error"; /* thrown by a user payload */
2459 		else if (engine->execlists.error_interrupt & ERROR_CSB)
2460 			msg = "invalid CSB event";
2461 		else if (engine->execlists.error_interrupt & ERROR_PREEMPT)
2462 			msg = "preemption time out";
2463 		else
2464 			msg = "internal error";
2465 
2466 		engine->execlists.error_interrupt = 0;
2467 		execlists_reset(engine, msg);
2468 	}
2469 
2470 	if (!engine->execlists.pending[0]) {
2471 		execlists_dequeue_irq(engine);
2472 		start_timeslice(engine);
2473 	}
2474 
2475 	post_process_csb(post, inactive);
2476 	rcu_read_unlock();
2477 }
2478 
execlists_irq_handler(struct intel_engine_cs * engine,u16 iir)2479 static void execlists_irq_handler(struct intel_engine_cs *engine, u16 iir)
2480 {
2481 	bool tasklet = false;
2482 
2483 	if (unlikely(iir & GT_CS_MASTER_ERROR_INTERRUPT)) {
2484 		u32 eir;
2485 
2486 		/* Upper 16b are the enabling mask, rsvd for internal errors */
2487 		eir = ENGINE_READ(engine, RING_EIR) & GENMASK(15, 0);
2488 		ENGINE_TRACE(engine, "CS error: %x\n", eir);
2489 
2490 		/* Disable the error interrupt until after the reset */
2491 		if (likely(eir)) {
2492 			ENGINE_WRITE(engine, RING_EMR, ~0u);
2493 			ENGINE_WRITE(engine, RING_EIR, eir);
2494 			WRITE_ONCE(engine->execlists.error_interrupt, eir);
2495 			tasklet = true;
2496 		}
2497 	}
2498 
2499 	if (iir & GT_WAIT_SEMAPHORE_INTERRUPT) {
2500 		WRITE_ONCE(engine->execlists.yield,
2501 			   ENGINE_READ_FW(engine, RING_EXECLIST_STATUS_HI));
2502 		ENGINE_TRACE(engine, "semaphore yield: %08x\n",
2503 			     engine->execlists.yield);
2504 		if (del_timer(&engine->execlists.timer))
2505 			tasklet = true;
2506 	}
2507 
2508 	if (iir & GT_CONTEXT_SWITCH_INTERRUPT)
2509 		tasklet = true;
2510 
2511 	if (iir & GT_RENDER_USER_INTERRUPT)
2512 		intel_engine_signal_breadcrumbs(engine);
2513 
2514 	if (tasklet)
2515 		tasklet_hi_schedule(&engine->sched_engine->tasklet);
2516 }
2517 
__execlists_kick(struct intel_engine_execlists * execlists)2518 static void __execlists_kick(struct intel_engine_execlists *execlists)
2519 {
2520 	struct intel_engine_cs *engine =
2521 		container_of(execlists, typeof(*engine), execlists);
2522 
2523 	/* Kick the tasklet for some interrupt coalescing and reset handling */
2524 	tasklet_hi_schedule(&engine->sched_engine->tasklet);
2525 }
2526 
2527 #define execlists_kick(t, member) \
2528 	__execlists_kick(container_of(t, struct intel_engine_execlists, member))
2529 
execlists_timeslice(struct timer_list * timer)2530 static void execlists_timeslice(struct timer_list *timer)
2531 {
2532 	execlists_kick(timer, timer);
2533 }
2534 
execlists_preempt(struct timer_list * timer)2535 static void execlists_preempt(struct timer_list *timer)
2536 {
2537 	execlists_kick(timer, preempt);
2538 }
2539 
queue_request(struct intel_engine_cs * engine,struct i915_request * rq)2540 static void queue_request(struct intel_engine_cs *engine,
2541 			  struct i915_request *rq)
2542 {
2543 	GEM_BUG_ON(!list_empty(&rq->sched.link));
2544 	list_add_tail(&rq->sched.link,
2545 		      i915_sched_lookup_priolist(engine->sched_engine,
2546 						 rq_prio(rq)));
2547 	set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);
2548 }
2549 
submit_queue(struct intel_engine_cs * engine,const struct i915_request * rq)2550 static bool submit_queue(struct intel_engine_cs *engine,
2551 			 const struct i915_request *rq)
2552 {
2553 	struct i915_sched_engine *sched_engine = engine->sched_engine;
2554 
2555 	if (rq_prio(rq) <= sched_engine->queue_priority_hint)
2556 		return false;
2557 
2558 	sched_engine->queue_priority_hint = rq_prio(rq);
2559 	return true;
2560 }
2561 
ancestor_on_hold(const struct intel_engine_cs * engine,const struct i915_request * rq)2562 static bool ancestor_on_hold(const struct intel_engine_cs *engine,
2563 			     const struct i915_request *rq)
2564 {
2565 	GEM_BUG_ON(i915_request_on_hold(rq));
2566 	return !list_empty(&engine->sched_engine->hold) && hold_request(rq);
2567 }
2568 
execlists_submit_request(struct i915_request * request)2569 static void execlists_submit_request(struct i915_request *request)
2570 {
2571 	struct intel_engine_cs *engine = request->engine;
2572 	unsigned long flags;
2573 
2574 	/* Will be called from irq-context when using foreign fences. */
2575 	spin_lock_irqsave(&engine->sched_engine->lock, flags);
2576 
2577 	if (unlikely(ancestor_on_hold(engine, request))) {
2578 		RQ_TRACE(request, "ancestor on hold\n");
2579 		list_add_tail(&request->sched.link,
2580 			      &engine->sched_engine->hold);
2581 		i915_request_set_hold(request);
2582 	} else {
2583 		queue_request(engine, request);
2584 
2585 		GEM_BUG_ON(i915_sched_engine_is_empty(engine->sched_engine));
2586 		GEM_BUG_ON(list_empty(&request->sched.link));
2587 
2588 		if (submit_queue(engine, request))
2589 			__execlists_kick(&engine->execlists);
2590 	}
2591 
2592 	spin_unlock_irqrestore(&engine->sched_engine->lock, flags);
2593 }
2594 
2595 static int
__execlists_context_pre_pin(struct intel_context * ce,struct intel_engine_cs * engine,struct i915_gem_ww_ctx * ww,void ** vaddr)2596 __execlists_context_pre_pin(struct intel_context *ce,
2597 			    struct intel_engine_cs *engine,
2598 			    struct i915_gem_ww_ctx *ww, void **vaddr)
2599 {
2600 	int err;
2601 
2602 	err = lrc_pre_pin(ce, engine, ww, vaddr);
2603 	if (err)
2604 		return err;
2605 
2606 	if (!__test_and_set_bit(CONTEXT_INIT_BIT, &ce->flags)) {
2607 		lrc_init_state(ce, engine, *vaddr);
2608 
2609 		__i915_gem_object_flush_map(ce->state->obj, 0, engine->context_size);
2610 	}
2611 
2612 	return 0;
2613 }
2614 
execlists_context_pre_pin(struct intel_context * ce,struct i915_gem_ww_ctx * ww,void ** vaddr)2615 static int execlists_context_pre_pin(struct intel_context *ce,
2616 				     struct i915_gem_ww_ctx *ww,
2617 				     void **vaddr)
2618 {
2619 	return __execlists_context_pre_pin(ce, ce->engine, ww, vaddr);
2620 }
2621 
execlists_context_pin(struct intel_context * ce,void * vaddr)2622 static int execlists_context_pin(struct intel_context *ce, void *vaddr)
2623 {
2624 	return lrc_pin(ce, ce->engine, vaddr);
2625 }
2626 
execlists_context_alloc(struct intel_context * ce)2627 static int execlists_context_alloc(struct intel_context *ce)
2628 {
2629 	return lrc_alloc(ce, ce->engine);
2630 }
2631 
execlists_context_cancel_request(struct intel_context * ce,struct i915_request * rq)2632 static void execlists_context_cancel_request(struct intel_context *ce,
2633 					     struct i915_request *rq)
2634 {
2635 	struct intel_engine_cs *engine = NULL;
2636 
2637 	i915_request_active_engine(rq, &engine);
2638 
2639 	if (engine && intel_engine_pulse(engine))
2640 		intel_gt_handle_error(engine->gt, engine->mask, 0,
2641 				      "request cancellation by %s",
2642 				      current->comm);
2643 }
2644 
2645 static struct intel_context *
execlists_create_parallel(struct intel_engine_cs ** engines,unsigned int num_siblings,unsigned int width)2646 execlists_create_parallel(struct intel_engine_cs **engines,
2647 			  unsigned int num_siblings,
2648 			  unsigned int width)
2649 {
2650 	struct intel_context *parent = NULL, *ce, *err;
2651 	int i;
2652 
2653 	GEM_BUG_ON(num_siblings != 1);
2654 
2655 	for (i = 0; i < width; ++i) {
2656 		ce = intel_context_create(engines[i]);
2657 		if (IS_ERR(ce)) {
2658 			err = ce;
2659 			goto unwind;
2660 		}
2661 
2662 		if (i == 0)
2663 			parent = ce;
2664 		else
2665 			intel_context_bind_parent_child(parent, ce);
2666 	}
2667 
2668 	parent->parallel.fence_context = dma_fence_context_alloc(1);
2669 
2670 	intel_context_set_nopreempt(parent);
2671 	for_each_child(parent, ce)
2672 		intel_context_set_nopreempt(ce);
2673 
2674 	return parent;
2675 
2676 unwind:
2677 	if (parent)
2678 		intel_context_put(parent);
2679 	return err;
2680 }
2681 
2682 static const struct intel_context_ops execlists_context_ops = {
2683 	.flags = COPS_HAS_INFLIGHT | COPS_RUNTIME_CYCLES,
2684 
2685 	.alloc = execlists_context_alloc,
2686 
2687 	.cancel_request = execlists_context_cancel_request,
2688 
2689 	.pre_pin = execlists_context_pre_pin,
2690 	.pin = execlists_context_pin,
2691 	.unpin = lrc_unpin,
2692 	.post_unpin = lrc_post_unpin,
2693 
2694 	.enter = intel_context_enter_engine,
2695 	.exit = intel_context_exit_engine,
2696 
2697 	.reset = lrc_reset,
2698 	.destroy = lrc_destroy,
2699 
2700 	.create_parallel = execlists_create_parallel,
2701 	.create_virtual = execlists_create_virtual,
2702 };
2703 
emit_pdps(struct i915_request * rq)2704 static int emit_pdps(struct i915_request *rq)
2705 {
2706 	const struct intel_engine_cs * const engine = rq->engine;
2707 	struct i915_ppgtt * const ppgtt = i915_vm_to_ppgtt(rq->context->vm);
2708 	int err, i;
2709 	u32 *cs;
2710 
2711 	GEM_BUG_ON(intel_vgpu_active(rq->engine->i915));
2712 
2713 	/*
2714 	 * Beware ye of the dragons, this sequence is magic!
2715 	 *
2716 	 * Small changes to this sequence can cause anything from
2717 	 * GPU hangs to forcewake errors and machine lockups!
2718 	 */
2719 
2720 	cs = intel_ring_begin(rq, 2);
2721 	if (IS_ERR(cs))
2722 		return PTR_ERR(cs);
2723 
2724 	*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
2725 	*cs++ = MI_NOOP;
2726 	intel_ring_advance(rq, cs);
2727 
2728 	/* Flush any residual operations from the context load */
2729 	err = engine->emit_flush(rq, EMIT_FLUSH);
2730 	if (err)
2731 		return err;
2732 
2733 	/* Magic required to prevent forcewake errors! */
2734 	err = engine->emit_flush(rq, EMIT_INVALIDATE);
2735 	if (err)
2736 		return err;
2737 
2738 	cs = intel_ring_begin(rq, 4 * GEN8_3LVL_PDPES + 2);
2739 	if (IS_ERR(cs))
2740 		return PTR_ERR(cs);
2741 
2742 	/* Ensure the LRI have landed before we invalidate & continue */
2743 	*cs++ = MI_LOAD_REGISTER_IMM(2 * GEN8_3LVL_PDPES) | MI_LRI_FORCE_POSTED;
2744 	for (i = GEN8_3LVL_PDPES; i--; ) {
2745 		const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i);
2746 		u32 base = engine->mmio_base;
2747 
2748 		*cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(base, i));
2749 		*cs++ = upper_32_bits(pd_daddr);
2750 		*cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(base, i));
2751 		*cs++ = lower_32_bits(pd_daddr);
2752 	}
2753 	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
2754 	intel_ring_advance(rq, cs);
2755 
2756 	intel_ring_advance(rq, cs);
2757 
2758 	return 0;
2759 }
2760 
execlists_request_alloc(struct i915_request * request)2761 static int execlists_request_alloc(struct i915_request *request)
2762 {
2763 	int ret;
2764 
2765 	GEM_BUG_ON(!intel_context_is_pinned(request->context));
2766 
2767 	/*
2768 	 * Flush enough space to reduce the likelihood of waiting after
2769 	 * we start building the request - in which case we will just
2770 	 * have to repeat work.
2771 	 */
2772 	request->reserved_space += EXECLISTS_REQUEST_SIZE;
2773 
2774 	/*
2775 	 * Note that after this point, we have committed to using
2776 	 * this request as it is being used to both track the
2777 	 * state of engine initialisation and liveness of the
2778 	 * golden renderstate above. Think twice before you try
2779 	 * to cancel/unwind this request now.
2780 	 */
2781 
2782 	if (!i915_vm_is_4lvl(request->context->vm)) {
2783 		ret = emit_pdps(request);
2784 		if (ret)
2785 			return ret;
2786 	}
2787 
2788 	/* Unconditionally invalidate GPU caches and TLBs. */
2789 	ret = request->engine->emit_flush(request, EMIT_INVALIDATE);
2790 	if (ret)
2791 		return ret;
2792 
2793 	request->reserved_space -= EXECLISTS_REQUEST_SIZE;
2794 	return 0;
2795 }
2796 
reset_csb_pointers(struct intel_engine_cs * engine)2797 static void reset_csb_pointers(struct intel_engine_cs *engine)
2798 {
2799 	struct intel_engine_execlists * const execlists = &engine->execlists;
2800 	const unsigned int reset_value = execlists->csb_size - 1;
2801 
2802 	ring_set_paused(engine, 0);
2803 
2804 	/*
2805 	 * Sometimes Icelake forgets to reset its pointers on a GPU reset.
2806 	 * Bludgeon them with a mmio update to be sure.
2807 	 */
2808 	ENGINE_WRITE(engine, RING_CONTEXT_STATUS_PTR,
2809 		     0xffff << 16 | reset_value << 8 | reset_value);
2810 	ENGINE_POSTING_READ(engine, RING_CONTEXT_STATUS_PTR);
2811 
2812 	/*
2813 	 * After a reset, the HW starts writing into CSB entry [0]. We
2814 	 * therefore have to set our HEAD pointer back one entry so that
2815 	 * the *first* entry we check is entry 0. To complicate this further,
2816 	 * as we don't wait for the first interrupt after reset, we have to
2817 	 * fake the HW write to point back to the last entry so that our
2818 	 * inline comparison of our cached head position against the last HW
2819 	 * write works even before the first interrupt.
2820 	 */
2821 	execlists->csb_head = reset_value;
2822 	WRITE_ONCE(*execlists->csb_write, reset_value);
2823 	wmb(); /* Make sure this is visible to HW (paranoia?) */
2824 
2825 	/* Check that the GPU does indeed update the CSB entries! */
2826 	memset(execlists->csb_status, -1, (reset_value + 1) * sizeof(u64));
2827 	drm_clflush_virt_range(execlists->csb_status,
2828 			       execlists->csb_size *
2829 			       sizeof(execlists->csb_status));
2830 
2831 	/* Once more for luck and our trusty paranoia */
2832 	ENGINE_WRITE(engine, RING_CONTEXT_STATUS_PTR,
2833 		     0xffff << 16 | reset_value << 8 | reset_value);
2834 	ENGINE_POSTING_READ(engine, RING_CONTEXT_STATUS_PTR);
2835 
2836 	GEM_BUG_ON(READ_ONCE(*execlists->csb_write) != reset_value);
2837 }
2838 
sanitize_hwsp(struct intel_engine_cs * engine)2839 static void sanitize_hwsp(struct intel_engine_cs *engine)
2840 {
2841 	struct intel_timeline *tl;
2842 
2843 	list_for_each_entry(tl, &engine->status_page.timelines, engine_link)
2844 		intel_timeline_reset_seqno(tl);
2845 }
2846 
execlists_sanitize(struct intel_engine_cs * engine)2847 static void execlists_sanitize(struct intel_engine_cs *engine)
2848 {
2849 	GEM_BUG_ON(execlists_active(&engine->execlists));
2850 
2851 	/*
2852 	 * Poison residual state on resume, in case the suspend didn't!
2853 	 *
2854 	 * We have to assume that across suspend/resume (or other loss
2855 	 * of control) that the contents of our pinned buffers has been
2856 	 * lost, replaced by garbage. Since this doesn't always happen,
2857 	 * let's poison such state so that we more quickly spot when
2858 	 * we falsely assume it has been preserved.
2859 	 */
2860 	if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
2861 		memset(engine->status_page.addr, POISON_INUSE, PAGE_SIZE);
2862 
2863 	reset_csb_pointers(engine);
2864 
2865 	/*
2866 	 * The kernel_context HWSP is stored in the status_page. As above,
2867 	 * that may be lost on resume/initialisation, and so we need to
2868 	 * reset the value in the HWSP.
2869 	 */
2870 	sanitize_hwsp(engine);
2871 
2872 	/* And scrub the dirty cachelines for the HWSP */
2873 	drm_clflush_virt_range(engine->status_page.addr, PAGE_SIZE);
2874 
2875 	intel_engine_reset_pinned_contexts(engine);
2876 }
2877 
enable_error_interrupt(struct intel_engine_cs * engine)2878 static void enable_error_interrupt(struct intel_engine_cs *engine)
2879 {
2880 	u32 status;
2881 
2882 	engine->execlists.error_interrupt = 0;
2883 	ENGINE_WRITE(engine, RING_EMR, ~0u);
2884 	ENGINE_WRITE(engine, RING_EIR, ~0u); /* clear all existing errors */
2885 
2886 	status = ENGINE_READ(engine, RING_ESR);
2887 	if (unlikely(status)) {
2888 		drm_err(&engine->i915->drm,
2889 			"engine '%s' resumed still in error: %08x\n",
2890 			engine->name, status);
2891 		__intel_gt_reset(engine->gt, engine->mask);
2892 	}
2893 
2894 	/*
2895 	 * On current gen8+, we have 2 signals to play with
2896 	 *
2897 	 * - I915_ERROR_INSTUCTION (bit 0)
2898 	 *
2899 	 *    Generate an error if the command parser encounters an invalid
2900 	 *    instruction
2901 	 *
2902 	 *    This is a fatal error.
2903 	 *
2904 	 * - CP_PRIV (bit 2)
2905 	 *
2906 	 *    Generate an error on privilege violation (where the CP replaces
2907 	 *    the instruction with a no-op). This also fires for writes into
2908 	 *    read-only scratch pages.
2909 	 *
2910 	 *    This is a non-fatal error, parsing continues.
2911 	 *
2912 	 * * there are a few others defined for odd HW that we do not use
2913 	 *
2914 	 * Since CP_PRIV fires for cases where we have chosen to ignore the
2915 	 * error (as the HW is validating and suppressing the mistakes), we
2916 	 * only unmask the instruction error bit.
2917 	 */
2918 	ENGINE_WRITE(engine, RING_EMR, ~I915_ERROR_INSTRUCTION);
2919 }
2920 
enable_execlists(struct intel_engine_cs * engine)2921 static void enable_execlists(struct intel_engine_cs *engine)
2922 {
2923 	u32 mode;
2924 
2925 	assert_forcewakes_active(engine->uncore, FORCEWAKE_ALL);
2926 
2927 	intel_engine_set_hwsp_writemask(engine, ~0u); /* HWSTAM */
2928 
2929 	if (GRAPHICS_VER(engine->i915) >= 11)
2930 		mode = _MASKED_BIT_ENABLE(GEN11_GFX_DISABLE_LEGACY_MODE);
2931 	else
2932 		mode = _MASKED_BIT_ENABLE(GFX_RUN_LIST_ENABLE);
2933 	ENGINE_WRITE_FW(engine, RING_MODE_GEN7, mode);
2934 
2935 	ENGINE_WRITE_FW(engine, RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING));
2936 
2937 	ENGINE_WRITE_FW(engine,
2938 			RING_HWS_PGA,
2939 			i915_ggtt_offset(engine->status_page.vma));
2940 	ENGINE_POSTING_READ(engine, RING_HWS_PGA);
2941 
2942 	enable_error_interrupt(engine);
2943 }
2944 
execlists_resume(struct intel_engine_cs * engine)2945 static int execlists_resume(struct intel_engine_cs *engine)
2946 {
2947 	intel_mocs_init_engine(engine);
2948 	intel_breadcrumbs_reset(engine->breadcrumbs);
2949 
2950 	enable_execlists(engine);
2951 
2952 	if (engine->flags & I915_ENGINE_FIRST_RENDER_COMPUTE)
2953 		xehp_enable_ccs_engines(engine);
2954 
2955 	return 0;
2956 }
2957 
execlists_reset_prepare(struct intel_engine_cs * engine)2958 static void execlists_reset_prepare(struct intel_engine_cs *engine)
2959 {
2960 	ENGINE_TRACE(engine, "depth<-%d\n",
2961 		     atomic_read(&engine->sched_engine->tasklet.count));
2962 
2963 	/*
2964 	 * Prevent request submission to the hardware until we have
2965 	 * completed the reset in i915_gem_reset_finish(). If a request
2966 	 * is completed by one engine, it may then queue a request
2967 	 * to a second via its execlists->tasklet *just* as we are
2968 	 * calling engine->resume() and also writing the ELSP.
2969 	 * Turning off the execlists->tasklet until the reset is over
2970 	 * prevents the race.
2971 	 */
2972 	__tasklet_disable_sync_once(&engine->sched_engine->tasklet);
2973 	GEM_BUG_ON(!reset_in_progress(engine));
2974 
2975 	/*
2976 	 * We stop engines, otherwise we might get failed reset and a
2977 	 * dead gpu (on elk). Also as modern gpu as kbl can suffer
2978 	 * from system hang if batchbuffer is progressing when
2979 	 * the reset is issued, regardless of READY_TO_RESET ack.
2980 	 * Thus assume it is best to stop engines on all gens
2981 	 * where we have a gpu reset.
2982 	 *
2983 	 * WaKBLVECSSemaphoreWaitPoll:kbl (on ALL_ENGINES)
2984 	 *
2985 	 * FIXME: Wa for more modern gens needs to be validated
2986 	 */
2987 	ring_set_paused(engine, 1);
2988 	intel_engine_stop_cs(engine);
2989 
2990 	/*
2991 	 * Wa_22011802037:gen11/gen12: In addition to stopping the cs, we need
2992 	 * to wait for any pending mi force wakeups
2993 	 */
2994 	if (IS_GRAPHICS_VER(engine->i915, 11, 12))
2995 		intel_engine_wait_for_pending_mi_fw(engine);
2996 
2997 	engine->execlists.reset_ccid = active_ccid(engine);
2998 }
2999 
3000 static struct i915_request **
reset_csb(struct intel_engine_cs * engine,struct i915_request ** inactive)3001 reset_csb(struct intel_engine_cs *engine, struct i915_request **inactive)
3002 {
3003 	struct intel_engine_execlists * const execlists = &engine->execlists;
3004 
3005 	drm_clflush_virt_range(execlists->csb_write,
3006 			       sizeof(execlists->csb_write[0]));
3007 
3008 	inactive = process_csb(engine, inactive); /* drain preemption events */
3009 
3010 	/* Following the reset, we need to reload the CSB read/write pointers */
3011 	reset_csb_pointers(engine);
3012 
3013 	return inactive;
3014 }
3015 
3016 static void
execlists_reset_active(struct intel_engine_cs * engine,bool stalled)3017 execlists_reset_active(struct intel_engine_cs *engine, bool stalled)
3018 {
3019 	struct intel_context *ce;
3020 	struct i915_request *rq;
3021 	u32 head;
3022 
3023 	/*
3024 	 * Save the currently executing context, even if we completed
3025 	 * its request, it was still running at the time of the
3026 	 * reset and will have been clobbered.
3027 	 */
3028 	rq = active_context(engine, engine->execlists.reset_ccid);
3029 	if (!rq)
3030 		return;
3031 
3032 	ce = rq->context;
3033 	GEM_BUG_ON(!i915_vma_is_pinned(ce->state));
3034 
3035 	if (__i915_request_is_complete(rq)) {
3036 		/* Idle context; tidy up the ring so we can restart afresh */
3037 		head = intel_ring_wrap(ce->ring, rq->tail);
3038 		goto out_replay;
3039 	}
3040 
3041 	/* We still have requests in-flight; the engine should be active */
3042 	GEM_BUG_ON(!intel_engine_pm_is_awake(engine));
3043 
3044 	/* Context has requests still in-flight; it should not be idle! */
3045 	GEM_BUG_ON(i915_active_is_idle(&ce->active));
3046 
3047 	rq = active_request(ce->timeline, rq);
3048 	head = intel_ring_wrap(ce->ring, rq->head);
3049 	GEM_BUG_ON(head == ce->ring->tail);
3050 
3051 	/*
3052 	 * If this request hasn't started yet, e.g. it is waiting on a
3053 	 * semaphore, we need to avoid skipping the request or else we
3054 	 * break the signaling chain. However, if the context is corrupt
3055 	 * the request will not restart and we will be stuck with a wedged
3056 	 * device. It is quite often the case that if we issue a reset
3057 	 * while the GPU is loading the context image, that the context
3058 	 * image becomes corrupt.
3059 	 *
3060 	 * Otherwise, if we have not started yet, the request should replay
3061 	 * perfectly and we do not need to flag the result as being erroneous.
3062 	 */
3063 	if (!__i915_request_has_started(rq))
3064 		goto out_replay;
3065 
3066 	/*
3067 	 * If the request was innocent, we leave the request in the ELSP
3068 	 * and will try to replay it on restarting. The context image may
3069 	 * have been corrupted by the reset, in which case we may have
3070 	 * to service a new GPU hang, but more likely we can continue on
3071 	 * without impact.
3072 	 *
3073 	 * If the request was guilty, we presume the context is corrupt
3074 	 * and have to at least restore the RING register in the context
3075 	 * image back to the expected values to skip over the guilty request.
3076 	 */
3077 	__i915_request_reset(rq, stalled);
3078 
3079 	/*
3080 	 * We want a simple context + ring to execute the breadcrumb update.
3081 	 * We cannot rely on the context being intact across the GPU hang,
3082 	 * so clear it and rebuild just what we need for the breadcrumb.
3083 	 * All pending requests for this context will be zapped, and any
3084 	 * future request will be after userspace has had the opportunity
3085 	 * to recreate its own state.
3086 	 */
3087 out_replay:
3088 	ENGINE_TRACE(engine, "replay {head:%04x, tail:%04x}\n",
3089 		     head, ce->ring->tail);
3090 	lrc_reset_regs(ce, engine);
3091 	ce->lrc.lrca = lrc_update_regs(ce, engine, head);
3092 }
3093 
execlists_reset_csb(struct intel_engine_cs * engine,bool stalled)3094 static void execlists_reset_csb(struct intel_engine_cs *engine, bool stalled)
3095 {
3096 	struct intel_engine_execlists * const execlists = &engine->execlists;
3097 	struct i915_request *post[2 * EXECLIST_MAX_PORTS];
3098 	struct i915_request **inactive;
3099 
3100 	rcu_read_lock();
3101 	inactive = reset_csb(engine, post);
3102 
3103 	execlists_reset_active(engine, true);
3104 
3105 	inactive = cancel_port_requests(execlists, inactive);
3106 	post_process_csb(post, inactive);
3107 	rcu_read_unlock();
3108 }
3109 
execlists_reset_rewind(struct intel_engine_cs * engine,bool stalled)3110 static void execlists_reset_rewind(struct intel_engine_cs *engine, bool stalled)
3111 {
3112 	unsigned long flags;
3113 
3114 	ENGINE_TRACE(engine, "\n");
3115 
3116 	/* Process the csb, find the guilty context and throw away */
3117 	execlists_reset_csb(engine, stalled);
3118 
3119 	/* Push back any incomplete requests for replay after the reset. */
3120 	rcu_read_lock();
3121 	spin_lock_irqsave(&engine->sched_engine->lock, flags);
3122 	__unwind_incomplete_requests(engine);
3123 	spin_unlock_irqrestore(&engine->sched_engine->lock, flags);
3124 	rcu_read_unlock();
3125 }
3126 
nop_submission_tasklet(struct tasklet_struct * t)3127 static void nop_submission_tasklet(struct tasklet_struct *t)
3128 {
3129 	struct i915_sched_engine *sched_engine =
3130 		from_tasklet(sched_engine, t, tasklet);
3131 	struct intel_engine_cs * const engine = sched_engine->private_data;
3132 
3133 	/* The driver is wedged; don't process any more events. */
3134 	WRITE_ONCE(engine->sched_engine->queue_priority_hint, INT_MIN);
3135 }
3136 
execlists_reset_cancel(struct intel_engine_cs * engine)3137 static void execlists_reset_cancel(struct intel_engine_cs *engine)
3138 {
3139 	struct intel_engine_execlists * const execlists = &engine->execlists;
3140 	struct i915_sched_engine * const sched_engine = engine->sched_engine;
3141 	struct i915_request *rq, *rn;
3142 	struct rb_node *rb;
3143 	unsigned long flags;
3144 
3145 	ENGINE_TRACE(engine, "\n");
3146 
3147 	/*
3148 	 * Before we call engine->cancel_requests(), we should have exclusive
3149 	 * access to the submission state. This is arranged for us by the
3150 	 * caller disabling the interrupt generation, the tasklet and other
3151 	 * threads that may then access the same state, giving us a free hand
3152 	 * to reset state. However, we still need to let lockdep be aware that
3153 	 * we know this state may be accessed in hardirq context, so we
3154 	 * disable the irq around this manipulation and we want to keep
3155 	 * the spinlock focused on its duties and not accidentally conflate
3156 	 * coverage to the submission's irq state. (Similarly, although we
3157 	 * shouldn't need to disable irq around the manipulation of the
3158 	 * submission's irq state, we also wish to remind ourselves that
3159 	 * it is irq state.)
3160 	 */
3161 	execlists_reset_csb(engine, true);
3162 
3163 	rcu_read_lock();
3164 	spin_lock_irqsave(&engine->sched_engine->lock, flags);
3165 
3166 	/* Mark all executing requests as skipped. */
3167 	list_for_each_entry(rq, &engine->sched_engine->requests, sched.link)
3168 		i915_request_put(i915_request_mark_eio(rq));
3169 	intel_engine_signal_breadcrumbs(engine);
3170 
3171 	/* Flush the queued requests to the timeline list (for retiring). */
3172 	while ((rb = rb_first_cached(&sched_engine->queue))) {
3173 		struct i915_priolist *p = to_priolist(rb);
3174 
3175 		priolist_for_each_request_consume(rq, rn, p) {
3176 			if (i915_request_mark_eio(rq)) {
3177 				__i915_request_submit(rq);
3178 				i915_request_put(rq);
3179 			}
3180 		}
3181 
3182 		rb_erase_cached(&p->node, &sched_engine->queue);
3183 		i915_priolist_free(p);
3184 	}
3185 
3186 	/* On-hold requests will be flushed to timeline upon their release */
3187 	list_for_each_entry(rq, &sched_engine->hold, sched.link)
3188 		i915_request_put(i915_request_mark_eio(rq));
3189 
3190 	/* Cancel all attached virtual engines */
3191 	while ((rb = rb_first_cached(&execlists->virtual))) {
3192 		struct virtual_engine *ve =
3193 			rb_entry(rb, typeof(*ve), nodes[engine->id].rb);
3194 
3195 		rb_erase_cached(rb, &execlists->virtual);
3196 		RB_CLEAR_NODE(rb);
3197 
3198 		spin_lock(&ve->base.sched_engine->lock);
3199 		rq = fetch_and_zero(&ve->request);
3200 		if (rq) {
3201 			if (i915_request_mark_eio(rq)) {
3202 				rq->engine = engine;
3203 				__i915_request_submit(rq);
3204 				i915_request_put(rq);
3205 			}
3206 			i915_request_put(rq);
3207 
3208 			ve->base.sched_engine->queue_priority_hint = INT_MIN;
3209 		}
3210 		spin_unlock(&ve->base.sched_engine->lock);
3211 	}
3212 
3213 	/* Remaining _unready_ requests will be nop'ed when submitted */
3214 
3215 	sched_engine->queue_priority_hint = INT_MIN;
3216 	sched_engine->queue = RB_ROOT_CACHED;
3217 
3218 	GEM_BUG_ON(__tasklet_is_enabled(&engine->sched_engine->tasklet));
3219 	engine->sched_engine->tasklet.callback = nop_submission_tasklet;
3220 
3221 	spin_unlock_irqrestore(&engine->sched_engine->lock, flags);
3222 	rcu_read_unlock();
3223 }
3224 
execlists_reset_finish(struct intel_engine_cs * engine)3225 static void execlists_reset_finish(struct intel_engine_cs *engine)
3226 {
3227 	struct intel_engine_execlists * const execlists = &engine->execlists;
3228 
3229 	/*
3230 	 * After a GPU reset, we may have requests to replay. Do so now while
3231 	 * we still have the forcewake to be sure that the GPU is not allowed
3232 	 * to sleep before we restart and reload a context.
3233 	 *
3234 	 * If the GPU reset fails, the engine may still be alive with requests
3235 	 * inflight. We expect those to complete, or for the device to be
3236 	 * reset as the next level of recovery, and as a final resort we
3237 	 * will declare the device wedged.
3238 	 */
3239 	GEM_BUG_ON(!reset_in_progress(engine));
3240 
3241 	/* And kick in case we missed a new request submission. */
3242 	if (__tasklet_enable(&engine->sched_engine->tasklet))
3243 		__execlists_kick(execlists);
3244 
3245 	ENGINE_TRACE(engine, "depth->%d\n",
3246 		     atomic_read(&engine->sched_engine->tasklet.count));
3247 }
3248 
gen8_logical_ring_enable_irq(struct intel_engine_cs * engine)3249 static void gen8_logical_ring_enable_irq(struct intel_engine_cs *engine)
3250 {
3251 	ENGINE_WRITE(engine, RING_IMR,
3252 		     ~(engine->irq_enable_mask | engine->irq_keep_mask));
3253 	ENGINE_POSTING_READ(engine, RING_IMR);
3254 }
3255 
gen8_logical_ring_disable_irq(struct intel_engine_cs * engine)3256 static void gen8_logical_ring_disable_irq(struct intel_engine_cs *engine)
3257 {
3258 	ENGINE_WRITE(engine, RING_IMR, ~engine->irq_keep_mask);
3259 }
3260 
execlists_park(struct intel_engine_cs * engine)3261 static void execlists_park(struct intel_engine_cs *engine)
3262 {
3263 	cancel_timer(&engine->execlists.timer);
3264 	cancel_timer(&engine->execlists.preempt);
3265 }
3266 
add_to_engine(struct i915_request * rq)3267 static void add_to_engine(struct i915_request *rq)
3268 {
3269 	lockdep_assert_held(&rq->engine->sched_engine->lock);
3270 	list_move_tail(&rq->sched.link, &rq->engine->sched_engine->requests);
3271 }
3272 
remove_from_engine(struct i915_request * rq)3273 static void remove_from_engine(struct i915_request *rq)
3274 {
3275 	struct intel_engine_cs *engine, *locked;
3276 
3277 	/*
3278 	 * Virtual engines complicate acquiring the engine timeline lock,
3279 	 * as their rq->engine pointer is not stable until under that
3280 	 * engine lock. The simple ploy we use is to take the lock then
3281 	 * check that the rq still belongs to the newly locked engine.
3282 	 */
3283 	locked = READ_ONCE(rq->engine);
3284 	spin_lock_irq(&locked->sched_engine->lock);
3285 	while (unlikely(locked != (engine = READ_ONCE(rq->engine)))) {
3286 		spin_unlock(&locked->sched_engine->lock);
3287 		spin_lock(&engine->sched_engine->lock);
3288 		locked = engine;
3289 	}
3290 	list_del_init(&rq->sched.link);
3291 
3292 	clear_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);
3293 	clear_bit(I915_FENCE_FLAG_HOLD, &rq->fence.flags);
3294 
3295 	/* Prevent further __await_execution() registering a cb, then flush */
3296 	set_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags);
3297 
3298 	spin_unlock_irq(&locked->sched_engine->lock);
3299 
3300 	i915_request_notify_execute_cb_imm(rq);
3301 }
3302 
can_preempt(struct intel_engine_cs * engine)3303 static bool can_preempt(struct intel_engine_cs *engine)
3304 {
3305 	if (GRAPHICS_VER(engine->i915) > 8)
3306 		return true;
3307 
3308 	/* GPGPU on bdw requires extra w/a; not implemented */
3309 	return engine->class != RENDER_CLASS;
3310 }
3311 
kick_execlists(const struct i915_request * rq,int prio)3312 static void kick_execlists(const struct i915_request *rq, int prio)
3313 {
3314 	struct intel_engine_cs *engine = rq->engine;
3315 	struct i915_sched_engine *sched_engine = engine->sched_engine;
3316 	const struct i915_request *inflight;
3317 
3318 	/*
3319 	 * We only need to kick the tasklet once for the high priority
3320 	 * new context we add into the queue.
3321 	 */
3322 	if (prio <= sched_engine->queue_priority_hint)
3323 		return;
3324 
3325 	rcu_read_lock();
3326 
3327 	/* Nothing currently active? We're overdue for a submission! */
3328 	inflight = execlists_active(&engine->execlists);
3329 	if (!inflight)
3330 		goto unlock;
3331 
3332 	/*
3333 	 * If we are already the currently executing context, don't
3334 	 * bother evaluating if we should preempt ourselves.
3335 	 */
3336 	if (inflight->context == rq->context)
3337 		goto unlock;
3338 
3339 	ENGINE_TRACE(engine,
3340 		     "bumping queue-priority-hint:%d for rq:%llx:%lld, inflight:%llx:%lld prio %d\n",
3341 		     prio,
3342 		     rq->fence.context, rq->fence.seqno,
3343 		     inflight->fence.context, inflight->fence.seqno,
3344 		     inflight->sched.attr.priority);
3345 
3346 	sched_engine->queue_priority_hint = prio;
3347 
3348 	/*
3349 	 * Allow preemption of low -> normal -> high, but we do
3350 	 * not allow low priority tasks to preempt other low priority
3351 	 * tasks under the impression that latency for low priority
3352 	 * tasks does not matter (as much as background throughput),
3353 	 * so kiss.
3354 	 */
3355 	if (prio >= max(I915_PRIORITY_NORMAL, rq_prio(inflight)))
3356 		tasklet_hi_schedule(&sched_engine->tasklet);
3357 
3358 unlock:
3359 	rcu_read_unlock();
3360 }
3361 
execlists_set_default_submission(struct intel_engine_cs * engine)3362 static void execlists_set_default_submission(struct intel_engine_cs *engine)
3363 {
3364 	engine->submit_request = execlists_submit_request;
3365 	engine->sched_engine->schedule = i915_schedule;
3366 	engine->sched_engine->kick_backend = kick_execlists;
3367 	engine->sched_engine->tasklet.callback = execlists_submission_tasklet;
3368 }
3369 
execlists_shutdown(struct intel_engine_cs * engine)3370 static void execlists_shutdown(struct intel_engine_cs *engine)
3371 {
3372 	/* Synchronise with residual timers and any softirq they raise */
3373 	del_timer_sync(&engine->execlists.timer);
3374 	del_timer_sync(&engine->execlists.preempt);
3375 	tasklet_kill(&engine->sched_engine->tasklet);
3376 }
3377 
execlists_release(struct intel_engine_cs * engine)3378 static void execlists_release(struct intel_engine_cs *engine)
3379 {
3380 	engine->sanitize = NULL; /* no longer in control, nothing to sanitize */
3381 
3382 	execlists_shutdown(engine);
3383 
3384 	intel_engine_cleanup_common(engine);
3385 	lrc_fini_wa_ctx(engine);
3386 }
3387 
__execlists_engine_busyness(struct intel_engine_cs * engine,ktime_t * now)3388 static ktime_t __execlists_engine_busyness(struct intel_engine_cs *engine,
3389 					   ktime_t *now)
3390 {
3391 	struct intel_engine_execlists_stats *stats = &engine->stats.execlists;
3392 	ktime_t total = stats->total;
3393 
3394 	/*
3395 	 * If the engine is executing something at the moment
3396 	 * add it to the total.
3397 	 */
3398 	*now = ktime_get();
3399 	if (READ_ONCE(stats->active))
3400 		total = ktime_add(total, ktime_sub(*now, stats->start));
3401 
3402 	return total;
3403 }
3404 
execlists_engine_busyness(struct intel_engine_cs * engine,ktime_t * now)3405 static ktime_t execlists_engine_busyness(struct intel_engine_cs *engine,
3406 					 ktime_t *now)
3407 {
3408 	struct intel_engine_execlists_stats *stats = &engine->stats.execlists;
3409 	unsigned int seq;
3410 	ktime_t total;
3411 
3412 	do {
3413 		seq = read_seqcount_begin(&stats->lock);
3414 		total = __execlists_engine_busyness(engine, now);
3415 	} while (read_seqcount_retry(&stats->lock, seq));
3416 
3417 	return total;
3418 }
3419 
3420 static void
logical_ring_default_vfuncs(struct intel_engine_cs * engine)3421 logical_ring_default_vfuncs(struct intel_engine_cs *engine)
3422 {
3423 	/* Default vfuncs which can be overridden by each engine. */
3424 
3425 	engine->resume = execlists_resume;
3426 
3427 	engine->cops = &execlists_context_ops;
3428 	engine->request_alloc = execlists_request_alloc;
3429 	engine->add_active_request = add_to_engine;
3430 	engine->remove_active_request = remove_from_engine;
3431 
3432 	engine->reset.prepare = execlists_reset_prepare;
3433 	engine->reset.rewind = execlists_reset_rewind;
3434 	engine->reset.cancel = execlists_reset_cancel;
3435 	engine->reset.finish = execlists_reset_finish;
3436 
3437 	engine->park = execlists_park;
3438 	engine->unpark = NULL;
3439 
3440 	engine->emit_flush = gen8_emit_flush_xcs;
3441 	engine->emit_init_breadcrumb = gen8_emit_init_breadcrumb;
3442 	engine->emit_fini_breadcrumb = gen8_emit_fini_breadcrumb_xcs;
3443 	if (GRAPHICS_VER(engine->i915) >= 12) {
3444 		engine->emit_fini_breadcrumb = gen12_emit_fini_breadcrumb_xcs;
3445 		engine->emit_flush = gen12_emit_flush_xcs;
3446 	}
3447 	engine->set_default_submission = execlists_set_default_submission;
3448 
3449 	if (GRAPHICS_VER(engine->i915) < 11) {
3450 		engine->irq_enable = gen8_logical_ring_enable_irq;
3451 		engine->irq_disable = gen8_logical_ring_disable_irq;
3452 	} else {
3453 		/*
3454 		 * TODO: On Gen11 interrupt masks need to be clear
3455 		 * to allow C6 entry. Keep interrupts enabled at
3456 		 * and take the hit of generating extra interrupts
3457 		 * until a more refined solution exists.
3458 		 */
3459 	}
3460 	intel_engine_set_irq_handler(engine, execlists_irq_handler);
3461 
3462 	engine->flags |= I915_ENGINE_SUPPORTS_STATS;
3463 	if (!intel_vgpu_active(engine->i915)) {
3464 		engine->flags |= I915_ENGINE_HAS_SEMAPHORES;
3465 		if (can_preempt(engine)) {
3466 			engine->flags |= I915_ENGINE_HAS_PREEMPTION;
3467 			if (CONFIG_DRM_I915_TIMESLICE_DURATION)
3468 				engine->flags |= I915_ENGINE_HAS_TIMESLICES;
3469 		}
3470 	}
3471 
3472 	if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50)) {
3473 		if (intel_engine_has_preemption(engine))
3474 			engine->emit_bb_start = gen125_emit_bb_start;
3475 		else
3476 			engine->emit_bb_start = gen125_emit_bb_start_noarb;
3477 	} else {
3478 		if (intel_engine_has_preemption(engine))
3479 			engine->emit_bb_start = gen8_emit_bb_start;
3480 		else
3481 			engine->emit_bb_start = gen8_emit_bb_start_noarb;
3482 	}
3483 
3484 	engine->busyness = execlists_engine_busyness;
3485 }
3486 
logical_ring_default_irqs(struct intel_engine_cs * engine)3487 static void logical_ring_default_irqs(struct intel_engine_cs *engine)
3488 {
3489 	unsigned int shift = 0;
3490 
3491 	if (GRAPHICS_VER(engine->i915) < 11) {
3492 		const u8 irq_shifts[] = {
3493 			[RCS0]  = GEN8_RCS_IRQ_SHIFT,
3494 			[BCS0]  = GEN8_BCS_IRQ_SHIFT,
3495 			[VCS0]  = GEN8_VCS0_IRQ_SHIFT,
3496 			[VCS1]  = GEN8_VCS1_IRQ_SHIFT,
3497 			[VECS0] = GEN8_VECS_IRQ_SHIFT,
3498 		};
3499 
3500 		shift = irq_shifts[engine->id];
3501 	}
3502 
3503 	engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT << shift;
3504 	engine->irq_keep_mask = GT_CONTEXT_SWITCH_INTERRUPT << shift;
3505 	engine->irq_keep_mask |= GT_CS_MASTER_ERROR_INTERRUPT << shift;
3506 	engine->irq_keep_mask |= GT_WAIT_SEMAPHORE_INTERRUPT << shift;
3507 }
3508 
rcs_submission_override(struct intel_engine_cs * engine)3509 static void rcs_submission_override(struct intel_engine_cs *engine)
3510 {
3511 	switch (GRAPHICS_VER(engine->i915)) {
3512 	case 12:
3513 		engine->emit_flush = gen12_emit_flush_rcs;
3514 		engine->emit_fini_breadcrumb = gen12_emit_fini_breadcrumb_rcs;
3515 		break;
3516 	case 11:
3517 		engine->emit_flush = gen11_emit_flush_rcs;
3518 		engine->emit_fini_breadcrumb = gen11_emit_fini_breadcrumb_rcs;
3519 		break;
3520 	default:
3521 		engine->emit_flush = gen8_emit_flush_rcs;
3522 		engine->emit_fini_breadcrumb = gen8_emit_fini_breadcrumb_rcs;
3523 		break;
3524 	}
3525 }
3526 
intel_execlists_submission_setup(struct intel_engine_cs * engine)3527 int intel_execlists_submission_setup(struct intel_engine_cs *engine)
3528 {
3529 	struct intel_engine_execlists * const execlists = &engine->execlists;
3530 	struct drm_i915_private *i915 = engine->i915;
3531 	struct intel_uncore *uncore = engine->uncore;
3532 	u32 base = engine->mmio_base;
3533 
3534 	tasklet_setup(&engine->sched_engine->tasklet, execlists_submission_tasklet);
3535 	timer_setup(&engine->execlists.timer, execlists_timeslice, 0);
3536 	timer_setup(&engine->execlists.preempt, execlists_preempt, 0);
3537 
3538 	logical_ring_default_vfuncs(engine);
3539 	logical_ring_default_irqs(engine);
3540 
3541 	if (engine->flags & I915_ENGINE_HAS_RCS_REG_STATE)
3542 		rcs_submission_override(engine);
3543 
3544 	lrc_init_wa_ctx(engine);
3545 
3546 	if (HAS_LOGICAL_RING_ELSQ(i915)) {
3547 		execlists->submit_reg = uncore->regs +
3548 			i915_mmio_reg_offset(RING_EXECLIST_SQ_CONTENTS(base));
3549 		execlists->ctrl_reg = uncore->regs +
3550 			i915_mmio_reg_offset(RING_EXECLIST_CONTROL(base));
3551 
3552 		engine->fw_domain = intel_uncore_forcewake_for_reg(engine->uncore,
3553 				    RING_EXECLIST_CONTROL(engine->mmio_base),
3554 				    FW_REG_WRITE);
3555 	} else {
3556 		execlists->submit_reg = uncore->regs +
3557 			i915_mmio_reg_offset(RING_ELSP(base));
3558 	}
3559 
3560 	execlists->csb_status =
3561 		(u64 *)&engine->status_page.addr[I915_HWS_CSB_BUF0_INDEX];
3562 
3563 	execlists->csb_write =
3564 		&engine->status_page.addr[INTEL_HWS_CSB_WRITE_INDEX(i915)];
3565 
3566 	if (GRAPHICS_VER(i915) < 11)
3567 		execlists->csb_size = GEN8_CSB_ENTRIES;
3568 	else
3569 		execlists->csb_size = GEN11_CSB_ENTRIES;
3570 
3571 	engine->context_tag = GENMASK(BITS_PER_LONG - 2, 0);
3572 	if (GRAPHICS_VER(engine->i915) >= 11 &&
3573 	    GRAPHICS_VER_FULL(engine->i915) < IP_VER(12, 50)) {
3574 		execlists->ccid |= engine->instance << (GEN11_ENGINE_INSTANCE_SHIFT - 32);
3575 		execlists->ccid |= engine->class << (GEN11_ENGINE_CLASS_SHIFT - 32);
3576 	}
3577 
3578 	/* Finally, take ownership and responsibility for cleanup! */
3579 	engine->sanitize = execlists_sanitize;
3580 	engine->release = execlists_release;
3581 
3582 	return 0;
3583 }
3584 
virtual_queue(struct virtual_engine * ve)3585 static struct list_head *virtual_queue(struct virtual_engine *ve)
3586 {
3587 	return &ve->base.sched_engine->default_priolist.requests;
3588 }
3589 
rcu_virtual_context_destroy(struct work_struct * wrk)3590 static void rcu_virtual_context_destroy(struct work_struct *wrk)
3591 {
3592 	struct virtual_engine *ve =
3593 		container_of(wrk, typeof(*ve), rcu.work);
3594 	unsigned int n;
3595 
3596 	GEM_BUG_ON(ve->context.inflight);
3597 
3598 	/* Preempt-to-busy may leave a stale request behind. */
3599 	if (unlikely(ve->request)) {
3600 		struct i915_request *old;
3601 
3602 		spin_lock_irq(&ve->base.sched_engine->lock);
3603 
3604 		old = fetch_and_zero(&ve->request);
3605 		if (old) {
3606 			GEM_BUG_ON(!__i915_request_is_complete(old));
3607 			__i915_request_submit(old);
3608 			i915_request_put(old);
3609 		}
3610 
3611 		spin_unlock_irq(&ve->base.sched_engine->lock);
3612 	}
3613 
3614 	/*
3615 	 * Flush the tasklet in case it is still running on another core.
3616 	 *
3617 	 * This needs to be done before we remove ourselves from the siblings'
3618 	 * rbtrees as in the case it is running in parallel, it may reinsert
3619 	 * the rb_node into a sibling.
3620 	 */
3621 	tasklet_kill(&ve->base.sched_engine->tasklet);
3622 
3623 	/* Decouple ourselves from the siblings, no more access allowed. */
3624 	for (n = 0; n < ve->num_siblings; n++) {
3625 		struct intel_engine_cs *sibling = ve->siblings[n];
3626 		struct rb_node *node = &ve->nodes[sibling->id].rb;
3627 
3628 		if (RB_EMPTY_NODE(node))
3629 			continue;
3630 
3631 		spin_lock_irq(&sibling->sched_engine->lock);
3632 
3633 		/* Detachment is lazily performed in the sched_engine->tasklet */
3634 		if (!RB_EMPTY_NODE(node))
3635 			rb_erase_cached(node, &sibling->execlists.virtual);
3636 
3637 		spin_unlock_irq(&sibling->sched_engine->lock);
3638 	}
3639 	GEM_BUG_ON(__tasklet_is_scheduled(&ve->base.sched_engine->tasklet));
3640 	GEM_BUG_ON(!list_empty(virtual_queue(ve)));
3641 
3642 	lrc_fini(&ve->context);
3643 	intel_context_fini(&ve->context);
3644 
3645 	if (ve->base.breadcrumbs)
3646 		intel_breadcrumbs_put(ve->base.breadcrumbs);
3647 	if (ve->base.sched_engine)
3648 		i915_sched_engine_put(ve->base.sched_engine);
3649 	intel_engine_free_request_pool(&ve->base);
3650 
3651 	kfree(ve);
3652 }
3653 
virtual_context_destroy(struct kref * kref)3654 static void virtual_context_destroy(struct kref *kref)
3655 {
3656 	struct virtual_engine *ve =
3657 		container_of(kref, typeof(*ve), context.ref);
3658 
3659 	GEM_BUG_ON(!list_empty(&ve->context.signals));
3660 
3661 	/*
3662 	 * When destroying the virtual engine, we have to be aware that
3663 	 * it may still be in use from an hardirq/softirq context causing
3664 	 * the resubmission of a completed request (background completion
3665 	 * due to preempt-to-busy). Before we can free the engine, we need
3666 	 * to flush the submission code and tasklets that are still potentially
3667 	 * accessing the engine. Flushing the tasklets requires process context,
3668 	 * and since we can guard the resubmit onto the engine with an RCU read
3669 	 * lock, we can delegate the free of the engine to an RCU worker.
3670 	 */
3671 	INIT_RCU_WORK(&ve->rcu, rcu_virtual_context_destroy);
3672 	queue_rcu_work(system_wq, &ve->rcu);
3673 }
3674 
virtual_engine_initial_hint(struct virtual_engine * ve)3675 static void virtual_engine_initial_hint(struct virtual_engine *ve)
3676 {
3677 	int swp;
3678 
3679 	/*
3680 	 * Pick a random sibling on starting to help spread the load around.
3681 	 *
3682 	 * New contexts are typically created with exactly the same order
3683 	 * of siblings, and often started in batches. Due to the way we iterate
3684 	 * the array of sibling when submitting requests, sibling[0] is
3685 	 * prioritised for dequeuing. If we make sure that sibling[0] is fairly
3686 	 * randomised across the system, we also help spread the load by the
3687 	 * first engine we inspect being different each time.
3688 	 *
3689 	 * NB This does not force us to execute on this engine, it will just
3690 	 * typically be the first we inspect for submission.
3691 	 */
3692 	swp = prandom_u32_max(ve->num_siblings);
3693 	if (swp)
3694 		swap(ve->siblings[swp], ve->siblings[0]);
3695 }
3696 
virtual_context_alloc(struct intel_context * ce)3697 static int virtual_context_alloc(struct intel_context *ce)
3698 {
3699 	struct virtual_engine *ve = container_of(ce, typeof(*ve), context);
3700 
3701 	return lrc_alloc(ce, ve->siblings[0]);
3702 }
3703 
virtual_context_pre_pin(struct intel_context * ce,struct i915_gem_ww_ctx * ww,void ** vaddr)3704 static int virtual_context_pre_pin(struct intel_context *ce,
3705 				   struct i915_gem_ww_ctx *ww,
3706 				   void **vaddr)
3707 {
3708 	struct virtual_engine *ve = container_of(ce, typeof(*ve), context);
3709 
3710 	 /* Note: we must use a real engine class for setting up reg state */
3711 	return __execlists_context_pre_pin(ce, ve->siblings[0], ww, vaddr);
3712 }
3713 
virtual_context_pin(struct intel_context * ce,void * vaddr)3714 static int virtual_context_pin(struct intel_context *ce, void *vaddr)
3715 {
3716 	struct virtual_engine *ve = container_of(ce, typeof(*ve), context);
3717 
3718 	return lrc_pin(ce, ve->siblings[0], vaddr);
3719 }
3720 
virtual_context_enter(struct intel_context * ce)3721 static void virtual_context_enter(struct intel_context *ce)
3722 {
3723 	struct virtual_engine *ve = container_of(ce, typeof(*ve), context);
3724 	unsigned int n;
3725 
3726 	for (n = 0; n < ve->num_siblings; n++)
3727 		intel_engine_pm_get(ve->siblings[n]);
3728 
3729 	intel_timeline_enter(ce->timeline);
3730 }
3731 
virtual_context_exit(struct intel_context * ce)3732 static void virtual_context_exit(struct intel_context *ce)
3733 {
3734 	struct virtual_engine *ve = container_of(ce, typeof(*ve), context);
3735 	unsigned int n;
3736 
3737 	intel_timeline_exit(ce->timeline);
3738 
3739 	for (n = 0; n < ve->num_siblings; n++)
3740 		intel_engine_pm_put(ve->siblings[n]);
3741 }
3742 
3743 static struct intel_engine_cs *
virtual_get_sibling(struct intel_engine_cs * engine,unsigned int sibling)3744 virtual_get_sibling(struct intel_engine_cs *engine, unsigned int sibling)
3745 {
3746 	struct virtual_engine *ve = to_virtual_engine(engine);
3747 
3748 	if (sibling >= ve->num_siblings)
3749 		return NULL;
3750 
3751 	return ve->siblings[sibling];
3752 }
3753 
3754 static const struct intel_context_ops virtual_context_ops = {
3755 	.flags = COPS_HAS_INFLIGHT | COPS_RUNTIME_CYCLES,
3756 
3757 	.alloc = virtual_context_alloc,
3758 
3759 	.cancel_request = execlists_context_cancel_request,
3760 
3761 	.pre_pin = virtual_context_pre_pin,
3762 	.pin = virtual_context_pin,
3763 	.unpin = lrc_unpin,
3764 	.post_unpin = lrc_post_unpin,
3765 
3766 	.enter = virtual_context_enter,
3767 	.exit = virtual_context_exit,
3768 
3769 	.destroy = virtual_context_destroy,
3770 
3771 	.get_sibling = virtual_get_sibling,
3772 };
3773 
virtual_submission_mask(struct virtual_engine * ve)3774 static intel_engine_mask_t virtual_submission_mask(struct virtual_engine *ve)
3775 {
3776 	struct i915_request *rq;
3777 	intel_engine_mask_t mask;
3778 
3779 	rq = READ_ONCE(ve->request);
3780 	if (!rq)
3781 		return 0;
3782 
3783 	/* The rq is ready for submission; rq->execution_mask is now stable. */
3784 	mask = rq->execution_mask;
3785 	if (unlikely(!mask)) {
3786 		/* Invalid selection, submit to a random engine in error */
3787 		i915_request_set_error_once(rq, -ENODEV);
3788 		mask = ve->siblings[0]->mask;
3789 	}
3790 
3791 	ENGINE_TRACE(&ve->base, "rq=%llx:%lld, mask=%x, prio=%d\n",
3792 		     rq->fence.context, rq->fence.seqno,
3793 		     mask, ve->base.sched_engine->queue_priority_hint);
3794 
3795 	return mask;
3796 }
3797 
virtual_submission_tasklet(struct tasklet_struct * t)3798 static void virtual_submission_tasklet(struct tasklet_struct *t)
3799 {
3800 	struct i915_sched_engine *sched_engine =
3801 		from_tasklet(sched_engine, t, tasklet);
3802 	struct virtual_engine * const ve =
3803 		(struct virtual_engine *)sched_engine->private_data;
3804 	const int prio = READ_ONCE(sched_engine->queue_priority_hint);
3805 	intel_engine_mask_t mask;
3806 	unsigned int n;
3807 
3808 	rcu_read_lock();
3809 	mask = virtual_submission_mask(ve);
3810 	rcu_read_unlock();
3811 	if (unlikely(!mask))
3812 		return;
3813 
3814 	for (n = 0; n < ve->num_siblings; n++) {
3815 		struct intel_engine_cs *sibling = READ_ONCE(ve->siblings[n]);
3816 		struct ve_node * const node = &ve->nodes[sibling->id];
3817 		struct rb_node **parent, *rb;
3818 		bool first;
3819 
3820 		if (!READ_ONCE(ve->request))
3821 			break; /* already handled by a sibling's tasklet */
3822 
3823 		spin_lock_irq(&sibling->sched_engine->lock);
3824 
3825 		if (unlikely(!(mask & sibling->mask))) {
3826 			if (!RB_EMPTY_NODE(&node->rb)) {
3827 				rb_erase_cached(&node->rb,
3828 						&sibling->execlists.virtual);
3829 				RB_CLEAR_NODE(&node->rb);
3830 			}
3831 
3832 			goto unlock_engine;
3833 		}
3834 
3835 		if (unlikely(!RB_EMPTY_NODE(&node->rb))) {
3836 			/*
3837 			 * Cheat and avoid rebalancing the tree if we can
3838 			 * reuse this node in situ.
3839 			 */
3840 			first = rb_first_cached(&sibling->execlists.virtual) ==
3841 				&node->rb;
3842 			if (prio == node->prio || (prio > node->prio && first))
3843 				goto submit_engine;
3844 
3845 			rb_erase_cached(&node->rb, &sibling->execlists.virtual);
3846 		}
3847 
3848 		rb = NULL;
3849 		first = true;
3850 		parent = &sibling->execlists.virtual.rb_root.rb_node;
3851 		while (*parent) {
3852 			struct ve_node *other;
3853 
3854 			rb = *parent;
3855 			other = rb_entry(rb, typeof(*other), rb);
3856 			if (prio > other->prio) {
3857 				parent = &rb->rb_left;
3858 			} else {
3859 				parent = &rb->rb_right;
3860 				first = false;
3861 			}
3862 		}
3863 
3864 		rb_link_node(&node->rb, rb, parent);
3865 		rb_insert_color_cached(&node->rb,
3866 				       &sibling->execlists.virtual,
3867 				       first);
3868 
3869 submit_engine:
3870 		GEM_BUG_ON(RB_EMPTY_NODE(&node->rb));
3871 		node->prio = prio;
3872 		if (first && prio > sibling->sched_engine->queue_priority_hint)
3873 			tasklet_hi_schedule(&sibling->sched_engine->tasklet);
3874 
3875 unlock_engine:
3876 		spin_unlock_irq(&sibling->sched_engine->lock);
3877 
3878 		if (intel_context_inflight(&ve->context))
3879 			break;
3880 	}
3881 }
3882 
virtual_submit_request(struct i915_request * rq)3883 static void virtual_submit_request(struct i915_request *rq)
3884 {
3885 	struct virtual_engine *ve = to_virtual_engine(rq->engine);
3886 	unsigned long flags;
3887 
3888 	ENGINE_TRACE(&ve->base, "rq=%llx:%lld\n",
3889 		     rq->fence.context,
3890 		     rq->fence.seqno);
3891 
3892 	GEM_BUG_ON(ve->base.submit_request != virtual_submit_request);
3893 
3894 	spin_lock_irqsave(&ve->base.sched_engine->lock, flags);
3895 
3896 	/* By the time we resubmit a request, it may be completed */
3897 	if (__i915_request_is_complete(rq)) {
3898 		__i915_request_submit(rq);
3899 		goto unlock;
3900 	}
3901 
3902 	if (ve->request) { /* background completion from preempt-to-busy */
3903 		GEM_BUG_ON(!__i915_request_is_complete(ve->request));
3904 		__i915_request_submit(ve->request);
3905 		i915_request_put(ve->request);
3906 	}
3907 
3908 	ve->base.sched_engine->queue_priority_hint = rq_prio(rq);
3909 	ve->request = i915_request_get(rq);
3910 
3911 	GEM_BUG_ON(!list_empty(virtual_queue(ve)));
3912 	list_move_tail(&rq->sched.link, virtual_queue(ve));
3913 
3914 	tasklet_hi_schedule(&ve->base.sched_engine->tasklet);
3915 
3916 unlock:
3917 	spin_unlock_irqrestore(&ve->base.sched_engine->lock, flags);
3918 }
3919 
3920 static struct intel_context *
execlists_create_virtual(struct intel_engine_cs ** siblings,unsigned int count,unsigned long flags)3921 execlists_create_virtual(struct intel_engine_cs **siblings, unsigned int count,
3922 			 unsigned long flags)
3923 {
3924 	struct virtual_engine *ve;
3925 	unsigned int n;
3926 	int err;
3927 
3928 	ve = kzalloc(struct_size(ve, siblings, count), GFP_KERNEL);
3929 	if (!ve)
3930 		return ERR_PTR(-ENOMEM);
3931 
3932 	ve->base.i915 = siblings[0]->i915;
3933 	ve->base.gt = siblings[0]->gt;
3934 	ve->base.uncore = siblings[0]->uncore;
3935 	ve->base.id = -1;
3936 
3937 	ve->base.class = OTHER_CLASS;
3938 	ve->base.uabi_class = I915_ENGINE_CLASS_INVALID;
3939 	ve->base.instance = I915_ENGINE_CLASS_INVALID_VIRTUAL;
3940 	ve->base.uabi_instance = I915_ENGINE_CLASS_INVALID_VIRTUAL;
3941 
3942 	/*
3943 	 * The decision on whether to submit a request using semaphores
3944 	 * depends on the saturated state of the engine. We only compute
3945 	 * this during HW submission of the request, and we need for this
3946 	 * state to be globally applied to all requests being submitted
3947 	 * to this engine. Virtual engines encompass more than one physical
3948 	 * engine and so we cannot accurately tell in advance if one of those
3949 	 * engines is already saturated and so cannot afford to use a semaphore
3950 	 * and be pessimized in priority for doing so -- if we are the only
3951 	 * context using semaphores after all other clients have stopped, we
3952 	 * will be starved on the saturated system. Such a global switch for
3953 	 * semaphores is less than ideal, but alas is the current compromise.
3954 	 */
3955 	ve->base.saturated = ALL_ENGINES;
3956 
3957 	snprintf(ve->base.name, sizeof(ve->base.name), "virtual");
3958 
3959 	intel_engine_init_execlists(&ve->base);
3960 
3961 	ve->base.sched_engine = i915_sched_engine_create(ENGINE_VIRTUAL);
3962 	if (!ve->base.sched_engine) {
3963 		err = -ENOMEM;
3964 		goto err_put;
3965 	}
3966 	ve->base.sched_engine->private_data = &ve->base;
3967 
3968 	ve->base.cops = &virtual_context_ops;
3969 	ve->base.request_alloc = execlists_request_alloc;
3970 
3971 	ve->base.sched_engine->schedule = i915_schedule;
3972 	ve->base.sched_engine->kick_backend = kick_execlists;
3973 	ve->base.submit_request = virtual_submit_request;
3974 
3975 	INIT_LIST_HEAD(virtual_queue(ve));
3976 	tasklet_setup(&ve->base.sched_engine->tasklet, virtual_submission_tasklet);
3977 
3978 	intel_context_init(&ve->context, &ve->base);
3979 
3980 	ve->base.breadcrumbs = intel_breadcrumbs_create(NULL);
3981 	if (!ve->base.breadcrumbs) {
3982 		err = -ENOMEM;
3983 		goto err_put;
3984 	}
3985 
3986 	for (n = 0; n < count; n++) {
3987 		struct intel_engine_cs *sibling = siblings[n];
3988 
3989 		GEM_BUG_ON(!is_power_of_2(sibling->mask));
3990 		if (sibling->mask & ve->base.mask) {
3991 			DRM_DEBUG("duplicate %s entry in load balancer\n",
3992 				  sibling->name);
3993 			err = -EINVAL;
3994 			goto err_put;
3995 		}
3996 
3997 		/*
3998 		 * The virtual engine implementation is tightly coupled to
3999 		 * the execlists backend -- we push out request directly
4000 		 * into a tree inside each physical engine. We could support
4001 		 * layering if we handle cloning of the requests and
4002 		 * submitting a copy into each backend.
4003 		 */
4004 		if (sibling->sched_engine->tasklet.callback !=
4005 		    execlists_submission_tasklet) {
4006 			err = -ENODEV;
4007 			goto err_put;
4008 		}
4009 
4010 		GEM_BUG_ON(RB_EMPTY_NODE(&ve->nodes[sibling->id].rb));
4011 		RB_CLEAR_NODE(&ve->nodes[sibling->id].rb);
4012 
4013 		ve->siblings[ve->num_siblings++] = sibling;
4014 		ve->base.mask |= sibling->mask;
4015 		ve->base.logical_mask |= sibling->logical_mask;
4016 
4017 		/*
4018 		 * All physical engines must be compatible for their emission
4019 		 * functions (as we build the instructions during request
4020 		 * construction and do not alter them before submission
4021 		 * on the physical engine). We use the engine class as a guide
4022 		 * here, although that could be refined.
4023 		 */
4024 		if (ve->base.class != OTHER_CLASS) {
4025 			if (ve->base.class != sibling->class) {
4026 				DRM_DEBUG("invalid mixing of engine class, sibling %d, already %d\n",
4027 					  sibling->class, ve->base.class);
4028 				err = -EINVAL;
4029 				goto err_put;
4030 			}
4031 			continue;
4032 		}
4033 
4034 		ve->base.class = sibling->class;
4035 		ve->base.uabi_class = sibling->uabi_class;
4036 		snprintf(ve->base.name, sizeof(ve->base.name),
4037 			 "v%dx%d", ve->base.class, count);
4038 		ve->base.context_size = sibling->context_size;
4039 
4040 		ve->base.add_active_request = sibling->add_active_request;
4041 		ve->base.remove_active_request = sibling->remove_active_request;
4042 		ve->base.emit_bb_start = sibling->emit_bb_start;
4043 		ve->base.emit_flush = sibling->emit_flush;
4044 		ve->base.emit_init_breadcrumb = sibling->emit_init_breadcrumb;
4045 		ve->base.emit_fini_breadcrumb = sibling->emit_fini_breadcrumb;
4046 		ve->base.emit_fini_breadcrumb_dw =
4047 			sibling->emit_fini_breadcrumb_dw;
4048 
4049 		ve->base.flags = sibling->flags;
4050 	}
4051 
4052 	ve->base.flags |= I915_ENGINE_IS_VIRTUAL;
4053 
4054 	virtual_engine_initial_hint(ve);
4055 	return &ve->context;
4056 
4057 err_put:
4058 	intel_context_put(&ve->context);
4059 	return ERR_PTR(err);
4060 }
4061 
intel_execlists_show_requests(struct intel_engine_cs * engine,struct drm_printer * m,void (* show_request)(struct drm_printer * m,const struct i915_request * rq,const char * prefix,int indent),unsigned int max)4062 void intel_execlists_show_requests(struct intel_engine_cs *engine,
4063 				   struct drm_printer *m,
4064 				   void (*show_request)(struct drm_printer *m,
4065 							const struct i915_request *rq,
4066 							const char *prefix,
4067 							int indent),
4068 				   unsigned int max)
4069 {
4070 	const struct intel_engine_execlists *execlists = &engine->execlists;
4071 	struct i915_sched_engine *sched_engine = engine->sched_engine;
4072 	struct i915_request *rq, *last;
4073 	unsigned long flags;
4074 	unsigned int count;
4075 	struct rb_node *rb;
4076 
4077 	spin_lock_irqsave(&sched_engine->lock, flags);
4078 
4079 	last = NULL;
4080 	count = 0;
4081 	list_for_each_entry(rq, &sched_engine->requests, sched.link) {
4082 		if (count++ < max - 1)
4083 			show_request(m, rq, "\t\t", 0);
4084 		else
4085 			last = rq;
4086 	}
4087 	if (last) {
4088 		if (count > max) {
4089 			drm_printf(m,
4090 				   "\t\t...skipping %d executing requests...\n",
4091 				   count - max);
4092 		}
4093 		show_request(m, last, "\t\t", 0);
4094 	}
4095 
4096 	if (sched_engine->queue_priority_hint != INT_MIN)
4097 		drm_printf(m, "\t\tQueue priority hint: %d\n",
4098 			   READ_ONCE(sched_engine->queue_priority_hint));
4099 
4100 	last = NULL;
4101 	count = 0;
4102 	for (rb = rb_first_cached(&sched_engine->queue); rb; rb = rb_next(rb)) {
4103 		struct i915_priolist *p = rb_entry(rb, typeof(*p), node);
4104 
4105 		priolist_for_each_request(rq, p) {
4106 			if (count++ < max - 1)
4107 				show_request(m, rq, "\t\t", 0);
4108 			else
4109 				last = rq;
4110 		}
4111 	}
4112 	if (last) {
4113 		if (count > max) {
4114 			drm_printf(m,
4115 				   "\t\t...skipping %d queued requests...\n",
4116 				   count - max);
4117 		}
4118 		show_request(m, last, "\t\t", 0);
4119 	}
4120 
4121 	last = NULL;
4122 	count = 0;
4123 	for (rb = rb_first_cached(&execlists->virtual); rb; rb = rb_next(rb)) {
4124 		struct virtual_engine *ve =
4125 			rb_entry(rb, typeof(*ve), nodes[engine->id].rb);
4126 		struct i915_request *rq = READ_ONCE(ve->request);
4127 
4128 		if (rq) {
4129 			if (count++ < max - 1)
4130 				show_request(m, rq, "\t\t", 0);
4131 			else
4132 				last = rq;
4133 		}
4134 	}
4135 	if (last) {
4136 		if (count > max) {
4137 			drm_printf(m,
4138 				   "\t\t...skipping %d virtual requests...\n",
4139 				   count - max);
4140 		}
4141 		show_request(m, last, "\t\t", 0);
4142 	}
4143 
4144 	spin_unlock_irqrestore(&sched_engine->lock, flags);
4145 }
4146 
4147 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
4148 #include "selftest_execlists.c"
4149 #endif
4150