1 /*
2  * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21  * SOFTWARE.
22  *
23  * Authors:
24  *    Zhiyuan Lv <zhiyuan.lv@intel.com>
25  *    Zhi Wang <zhi.a.wang@intel.com>
26  *
27  * Contributors:
28  *    Min He <min.he@intel.com>
29  *    Bing Niu <bing.niu@intel.com>
30  *    Ping Gao <ping.a.gao@intel.com>
31  *    Tina Zhang <tina.zhang@intel.com>
32  *
33  */
34 
35 #include "i915_drv.h"
36 #include "gvt.h"
37 
38 #define _EL_OFFSET_STATUS       0x234
39 #define _EL_OFFSET_STATUS_BUF   0x370
40 #define _EL_OFFSET_STATUS_PTR   0x3A0
41 
42 #define execlist_ring_mmio(e, offset) ((e)->mmio_base + (offset))
43 
44 #define valid_context(ctx) ((ctx)->valid)
45 #define same_context(a, b) (((a)->context_id == (b)->context_id) && \
46 		((a)->lrca == (b)->lrca))
47 
48 static int context_switch_events[] = {
49 	[RCS0]  = RCS_AS_CONTEXT_SWITCH,
50 	[BCS0]  = BCS_AS_CONTEXT_SWITCH,
51 	[VCS0]  = VCS_AS_CONTEXT_SWITCH,
52 	[VCS1]  = VCS2_AS_CONTEXT_SWITCH,
53 	[VECS0] = VECS_AS_CONTEXT_SWITCH,
54 };
55 
to_context_switch_event(const struct intel_engine_cs * engine)56 static int to_context_switch_event(const struct intel_engine_cs *engine)
57 {
58 	if (WARN_ON(engine->id >= ARRAY_SIZE(context_switch_events)))
59 		return -EINVAL;
60 
61 	return context_switch_events[engine->id];
62 }
63 
switch_virtual_execlist_slot(struct intel_vgpu_execlist * execlist)64 static void switch_virtual_execlist_slot(struct intel_vgpu_execlist *execlist)
65 {
66 	gvt_dbg_el("[before] running slot %d/context %x pending slot %d\n",
67 			execlist->running_slot ?
68 			execlist->running_slot->index : -1,
69 			execlist->running_context ?
70 			execlist->running_context->context_id : 0,
71 			execlist->pending_slot ?
72 			execlist->pending_slot->index : -1);
73 
74 	execlist->running_slot = execlist->pending_slot;
75 	execlist->pending_slot = NULL;
76 	execlist->running_context = execlist->running_context ?
77 		&execlist->running_slot->ctx[0] : NULL;
78 
79 	gvt_dbg_el("[after] running slot %d/context %x pending slot %d\n",
80 			execlist->running_slot ?
81 			execlist->running_slot->index : -1,
82 			execlist->running_context ?
83 			execlist->running_context->context_id : 0,
84 			execlist->pending_slot ?
85 			execlist->pending_slot->index : -1);
86 }
87 
emulate_execlist_status(struct intel_vgpu_execlist * execlist)88 static void emulate_execlist_status(struct intel_vgpu_execlist *execlist)
89 {
90 	struct intel_vgpu_execlist_slot *running = execlist->running_slot;
91 	struct intel_vgpu_execlist_slot *pending = execlist->pending_slot;
92 	struct execlist_ctx_descriptor_format *desc = execlist->running_context;
93 	struct intel_vgpu *vgpu = execlist->vgpu;
94 	struct execlist_status_format status;
95 	u32 status_reg =
96 		execlist_ring_mmio(execlist->engine, _EL_OFFSET_STATUS);
97 
98 	status.ldw = vgpu_vreg(vgpu, status_reg);
99 	status.udw = vgpu_vreg(vgpu, status_reg + 4);
100 
101 	if (running) {
102 		status.current_execlist_pointer = !!running->index;
103 		status.execlist_write_pointer = !!!running->index;
104 		status.execlist_0_active = status.execlist_0_valid =
105 			!!!(running->index);
106 		status.execlist_1_active = status.execlist_1_valid =
107 			!!(running->index);
108 	} else {
109 		status.context_id = 0;
110 		status.execlist_0_active = status.execlist_0_valid = 0;
111 		status.execlist_1_active = status.execlist_1_valid = 0;
112 	}
113 
114 	status.context_id = desc ? desc->context_id : 0;
115 	status.execlist_queue_full = !!(pending);
116 
117 	vgpu_vreg(vgpu, status_reg) = status.ldw;
118 	vgpu_vreg(vgpu, status_reg + 4) = status.udw;
119 
120 	gvt_dbg_el("vgpu%d: status reg offset %x ldw %x udw %x\n",
121 		vgpu->id, status_reg, status.ldw, status.udw);
122 }
123 
emulate_csb_update(struct intel_vgpu_execlist * execlist,struct execlist_context_status_format * status,bool trigger_interrupt_later)124 static void emulate_csb_update(struct intel_vgpu_execlist *execlist,
125 			       struct execlist_context_status_format *status,
126 			       bool trigger_interrupt_later)
127 {
128 	struct intel_vgpu *vgpu = execlist->vgpu;
129 	struct execlist_context_status_pointer_format ctx_status_ptr;
130 	u32 write_pointer;
131 	u32 ctx_status_ptr_reg, ctx_status_buf_reg, offset;
132 	unsigned long hwsp_gpa;
133 
134 	ctx_status_ptr_reg =
135 		execlist_ring_mmio(execlist->engine, _EL_OFFSET_STATUS_PTR);
136 	ctx_status_buf_reg =
137 		execlist_ring_mmio(execlist->engine, _EL_OFFSET_STATUS_BUF);
138 
139 	ctx_status_ptr.dw = vgpu_vreg(vgpu, ctx_status_ptr_reg);
140 
141 	write_pointer = ctx_status_ptr.write_ptr;
142 
143 	if (write_pointer == 0x7)
144 		write_pointer = 0;
145 	else {
146 		++write_pointer;
147 		write_pointer %= 0x6;
148 	}
149 
150 	offset = ctx_status_buf_reg + write_pointer * 8;
151 
152 	vgpu_vreg(vgpu, offset) = status->ldw;
153 	vgpu_vreg(vgpu, offset + 4) = status->udw;
154 
155 	ctx_status_ptr.write_ptr = write_pointer;
156 	vgpu_vreg(vgpu, ctx_status_ptr_reg) = ctx_status_ptr.dw;
157 
158 	/* Update the CSB and CSB write pointer in HWSP */
159 	hwsp_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm,
160 					 vgpu->hws_pga[execlist->engine->id]);
161 	if (hwsp_gpa != INTEL_GVT_INVALID_ADDR) {
162 		intel_gvt_write_gpa(vgpu,
163 			hwsp_gpa + I915_HWS_CSB_BUF0_INDEX * 4 + write_pointer * 8,
164 			status, 8);
165 		intel_gvt_write_gpa(vgpu,
166 			hwsp_gpa + INTEL_HWS_CSB_WRITE_INDEX(execlist->engine->i915) * 4,
167 			&write_pointer, 4);
168 	}
169 
170 	gvt_dbg_el("vgpu%d: w pointer %u reg %x csb l %x csb h %x\n",
171 		   vgpu->id, write_pointer, offset, status->ldw, status->udw);
172 
173 	if (trigger_interrupt_later)
174 		return;
175 
176 	intel_vgpu_trigger_virtual_event(vgpu,
177 					 to_context_switch_event(execlist->engine));
178 }
179 
emulate_execlist_ctx_schedule_out(struct intel_vgpu_execlist * execlist,struct execlist_ctx_descriptor_format * ctx)180 static int emulate_execlist_ctx_schedule_out(
181 		struct intel_vgpu_execlist *execlist,
182 		struct execlist_ctx_descriptor_format *ctx)
183 {
184 	struct intel_vgpu *vgpu = execlist->vgpu;
185 	struct intel_vgpu_execlist_slot *running = execlist->running_slot;
186 	struct intel_vgpu_execlist_slot *pending = execlist->pending_slot;
187 	struct execlist_ctx_descriptor_format *ctx0 = &running->ctx[0];
188 	struct execlist_ctx_descriptor_format *ctx1 = &running->ctx[1];
189 	struct execlist_context_status_format status;
190 
191 	memset(&status, 0, sizeof(status));
192 
193 	gvt_dbg_el("schedule out context id %x\n", ctx->context_id);
194 
195 	if (WARN_ON(!same_context(ctx, execlist->running_context))) {
196 		gvt_vgpu_err("schedule out context is not running context,"
197 				"ctx id %x running ctx id %x\n",
198 				ctx->context_id,
199 				execlist->running_context->context_id);
200 		return -EINVAL;
201 	}
202 
203 	/* ctx1 is valid, ctx0/ctx is scheduled-out -> element switch */
204 	if (valid_context(ctx1) && same_context(ctx0, ctx)) {
205 		gvt_dbg_el("ctx 1 valid, ctx/ctx 0 is scheduled-out\n");
206 
207 		execlist->running_context = ctx1;
208 
209 		emulate_execlist_status(execlist);
210 
211 		status.context_complete = status.element_switch = 1;
212 		status.context_id = ctx->context_id;
213 
214 		emulate_csb_update(execlist, &status, false);
215 		/*
216 		 * ctx1 is not valid, ctx == ctx0
217 		 * ctx1 is valid, ctx1 == ctx
218 		 *	--> last element is finished
219 		 * emulate:
220 		 *	active-to-idle if there is *no* pending execlist
221 		 *	context-complete if there *is* pending execlist
222 		 */
223 	} else if ((!valid_context(ctx1) && same_context(ctx0, ctx))
224 			|| (valid_context(ctx1) && same_context(ctx1, ctx))) {
225 		gvt_dbg_el("need to switch virtual execlist slot\n");
226 
227 		switch_virtual_execlist_slot(execlist);
228 
229 		emulate_execlist_status(execlist);
230 
231 		status.context_complete = status.active_to_idle = 1;
232 		status.context_id = ctx->context_id;
233 
234 		if (!pending) {
235 			emulate_csb_update(execlist, &status, false);
236 		} else {
237 			emulate_csb_update(execlist, &status, true);
238 
239 			memset(&status, 0, sizeof(status));
240 
241 			status.idle_to_active = 1;
242 			status.context_id = 0;
243 
244 			emulate_csb_update(execlist, &status, false);
245 		}
246 	} else {
247 		WARN_ON(1);
248 		return -EINVAL;
249 	}
250 
251 	return 0;
252 }
253 
get_next_execlist_slot(struct intel_vgpu_execlist * execlist)254 static struct intel_vgpu_execlist_slot *get_next_execlist_slot(
255 		struct intel_vgpu_execlist *execlist)
256 {
257 	struct intel_vgpu *vgpu = execlist->vgpu;
258 	u32 status_reg =
259 		execlist_ring_mmio(execlist->engine, _EL_OFFSET_STATUS);
260 	struct execlist_status_format status;
261 
262 	status.ldw = vgpu_vreg(vgpu, status_reg);
263 	status.udw = vgpu_vreg(vgpu, status_reg + 4);
264 
265 	if (status.execlist_queue_full) {
266 		gvt_vgpu_err("virtual execlist slots are full\n");
267 		return NULL;
268 	}
269 
270 	return &execlist->slot[status.execlist_write_pointer];
271 }
272 
emulate_execlist_schedule_in(struct intel_vgpu_execlist * execlist,struct execlist_ctx_descriptor_format ctx[2])273 static int emulate_execlist_schedule_in(struct intel_vgpu_execlist *execlist,
274 		struct execlist_ctx_descriptor_format ctx[2])
275 {
276 	struct intel_vgpu_execlist_slot *running = execlist->running_slot;
277 	struct intel_vgpu_execlist_slot *slot =
278 		get_next_execlist_slot(execlist);
279 
280 	struct execlist_ctx_descriptor_format *ctx0, *ctx1;
281 	struct execlist_context_status_format status;
282 	struct intel_vgpu *vgpu = execlist->vgpu;
283 
284 	gvt_dbg_el("emulate schedule-in\n");
285 
286 	if (!slot) {
287 		gvt_vgpu_err("no available execlist slot\n");
288 		return -EINVAL;
289 	}
290 
291 	memset(&status, 0, sizeof(status));
292 	memset(slot->ctx, 0, sizeof(slot->ctx));
293 
294 	slot->ctx[0] = ctx[0];
295 	slot->ctx[1] = ctx[1];
296 
297 	gvt_dbg_el("alloc slot index %d ctx 0 %x ctx 1 %x\n",
298 			slot->index, ctx[0].context_id,
299 			ctx[1].context_id);
300 
301 	/*
302 	 * no running execlist, make this write bundle as running execlist
303 	 * -> idle-to-active
304 	 */
305 	if (!running) {
306 		gvt_dbg_el("no current running execlist\n");
307 
308 		execlist->running_slot = slot;
309 		execlist->pending_slot = NULL;
310 		execlist->running_context = &slot->ctx[0];
311 
312 		gvt_dbg_el("running slot index %d running context %x\n",
313 				execlist->running_slot->index,
314 				execlist->running_context->context_id);
315 
316 		emulate_execlist_status(execlist);
317 
318 		status.idle_to_active = 1;
319 		status.context_id = 0;
320 
321 		emulate_csb_update(execlist, &status, false);
322 		return 0;
323 	}
324 
325 	ctx0 = &running->ctx[0];
326 	ctx1 = &running->ctx[1];
327 
328 	gvt_dbg_el("current running slot index %d ctx 0 %x ctx 1 %x\n",
329 		running->index, ctx0->context_id, ctx1->context_id);
330 
331 	/*
332 	 * already has an running execlist
333 	 *	a. running ctx1 is valid,
334 	 *	   ctx0 is finished, and running ctx1 == new execlist ctx[0]
335 	 *	b. running ctx1 is not valid,
336 	 *	   ctx0 == new execlist ctx[0]
337 	 * ----> lite-restore + preempted
338 	 */
339 	if ((valid_context(ctx1) && same_context(ctx1, &slot->ctx[0]) &&
340 		/* condition a */
341 		(!same_context(ctx0, execlist->running_context))) ||
342 			(!valid_context(ctx1) &&
343 			 same_context(ctx0, &slot->ctx[0]))) { /* condition b */
344 		gvt_dbg_el("need to switch virtual execlist slot\n");
345 
346 		execlist->pending_slot = slot;
347 		switch_virtual_execlist_slot(execlist);
348 
349 		emulate_execlist_status(execlist);
350 
351 		status.lite_restore = status.preempted = 1;
352 		status.context_id = ctx[0].context_id;
353 
354 		emulate_csb_update(execlist, &status, false);
355 	} else {
356 		gvt_dbg_el("emulate as pending slot\n");
357 		/*
358 		 * otherwise
359 		 * --> emulate pending execlist exist + but no preemption case
360 		 */
361 		execlist->pending_slot = slot;
362 		emulate_execlist_status(execlist);
363 	}
364 	return 0;
365 }
366 
367 #define get_desc_from_elsp_dwords(ed, i) \
368 	((struct execlist_ctx_descriptor_format *)&((ed)->data[i * 2]))
369 
prepare_execlist_workload(struct intel_vgpu_workload * workload)370 static int prepare_execlist_workload(struct intel_vgpu_workload *workload)
371 {
372 	struct intel_vgpu *vgpu = workload->vgpu;
373 	struct intel_vgpu_submission *s = &vgpu->submission;
374 	struct execlist_ctx_descriptor_format ctx[2];
375 	int ret;
376 
377 	if (!workload->emulate_schedule_in)
378 		return 0;
379 
380 	ctx[0] = *get_desc_from_elsp_dwords(&workload->elsp_dwords, 0);
381 	ctx[1] = *get_desc_from_elsp_dwords(&workload->elsp_dwords, 1);
382 
383 	ret = emulate_execlist_schedule_in(&s->execlist[workload->engine->id],
384 					   ctx);
385 	if (ret) {
386 		gvt_vgpu_err("fail to emulate execlist schedule in\n");
387 		return ret;
388 	}
389 	return 0;
390 }
391 
complete_execlist_workload(struct intel_vgpu_workload * workload)392 static int complete_execlist_workload(struct intel_vgpu_workload *workload)
393 {
394 	struct intel_vgpu *vgpu = workload->vgpu;
395 	struct intel_vgpu_submission *s = &vgpu->submission;
396 	struct intel_vgpu_execlist *execlist =
397 		&s->execlist[workload->engine->id];
398 	struct intel_vgpu_workload *next_workload;
399 	struct list_head *next = workload_q_head(vgpu, workload->engine)->next;
400 	bool lite_restore = false;
401 	int ret = 0;
402 
403 	gvt_dbg_el("complete workload %p status %d\n",
404 		   workload, workload->status);
405 
406 	if (workload->status || vgpu->resetting_eng & workload->engine->mask)
407 		goto out;
408 
409 	if (!list_empty(workload_q_head(vgpu, workload->engine))) {
410 		struct execlist_ctx_descriptor_format *this_desc, *next_desc;
411 
412 		next_workload = container_of(next,
413 				struct intel_vgpu_workload, list);
414 		this_desc = &workload->ctx_desc;
415 		next_desc = &next_workload->ctx_desc;
416 
417 		lite_restore = same_context(this_desc, next_desc);
418 	}
419 
420 	if (lite_restore) {
421 		gvt_dbg_el("next context == current - no schedule-out\n");
422 		goto out;
423 	}
424 
425 	ret = emulate_execlist_ctx_schedule_out(execlist, &workload->ctx_desc);
426 out:
427 	return ret;
428 }
429 
submit_context(struct intel_vgpu * vgpu,const struct intel_engine_cs * engine,struct execlist_ctx_descriptor_format * desc,bool emulate_schedule_in)430 static int submit_context(struct intel_vgpu *vgpu,
431 			  const struct intel_engine_cs *engine,
432 			  struct execlist_ctx_descriptor_format *desc,
433 			  bool emulate_schedule_in)
434 {
435 	struct intel_vgpu_submission *s = &vgpu->submission;
436 	struct intel_vgpu_workload *workload = NULL;
437 
438 	workload = intel_vgpu_create_workload(vgpu, engine, desc);
439 	if (IS_ERR(workload))
440 		return PTR_ERR(workload);
441 
442 	workload->prepare = prepare_execlist_workload;
443 	workload->complete = complete_execlist_workload;
444 	workload->emulate_schedule_in = emulate_schedule_in;
445 
446 	if (emulate_schedule_in)
447 		workload->elsp_dwords = s->execlist[engine->id].elsp_dwords;
448 
449 	gvt_dbg_el("workload %p emulate schedule_in %d\n", workload,
450 		   emulate_schedule_in);
451 
452 	intel_vgpu_queue_workload(workload);
453 	return 0;
454 }
455 
intel_vgpu_submit_execlist(struct intel_vgpu * vgpu,const struct intel_engine_cs * engine)456 int intel_vgpu_submit_execlist(struct intel_vgpu *vgpu,
457 			       const struct intel_engine_cs *engine)
458 {
459 	struct intel_vgpu_submission *s = &vgpu->submission;
460 	struct intel_vgpu_execlist *execlist = &s->execlist[engine->id];
461 	struct execlist_ctx_descriptor_format *desc[2];
462 	int i, ret;
463 
464 	desc[0] = get_desc_from_elsp_dwords(&execlist->elsp_dwords, 0);
465 	desc[1] = get_desc_from_elsp_dwords(&execlist->elsp_dwords, 1);
466 
467 	if (!desc[0]->valid) {
468 		gvt_vgpu_err("invalid elsp submission, desc0 is invalid\n");
469 		goto inv_desc;
470 	}
471 
472 	for (i = 0; i < ARRAY_SIZE(desc); i++) {
473 		if (!desc[i]->valid)
474 			continue;
475 		if (!desc[i]->privilege_access) {
476 			gvt_vgpu_err("unexpected GGTT elsp submission\n");
477 			goto inv_desc;
478 		}
479 	}
480 
481 	/* submit workload */
482 	for (i = 0; i < ARRAY_SIZE(desc); i++) {
483 		if (!desc[i]->valid)
484 			continue;
485 		ret = submit_context(vgpu, engine, desc[i], i == 0);
486 		if (ret) {
487 			gvt_vgpu_err("failed to submit desc %d\n", i);
488 			return ret;
489 		}
490 	}
491 
492 	return 0;
493 
494 inv_desc:
495 	gvt_vgpu_err("descriptors content: desc0 %08x %08x desc1 %08x %08x\n",
496 		     desc[0]->udw, desc[0]->ldw, desc[1]->udw, desc[1]->ldw);
497 	return -EINVAL;
498 }
499 
init_vgpu_execlist(struct intel_vgpu * vgpu,const struct intel_engine_cs * engine)500 static void init_vgpu_execlist(struct intel_vgpu *vgpu,
501 			       const struct intel_engine_cs *engine)
502 {
503 	struct intel_vgpu_submission *s = &vgpu->submission;
504 	struct intel_vgpu_execlist *execlist = &s->execlist[engine->id];
505 	struct execlist_context_status_pointer_format ctx_status_ptr;
506 	u32 ctx_status_ptr_reg;
507 
508 	memset(execlist, 0, sizeof(*execlist));
509 
510 	execlist->vgpu = vgpu;
511 	execlist->engine = engine;
512 	execlist->slot[0].index = 0;
513 	execlist->slot[1].index = 1;
514 
515 	ctx_status_ptr_reg = execlist_ring_mmio(engine, _EL_OFFSET_STATUS_PTR);
516 	ctx_status_ptr.dw = vgpu_vreg(vgpu, ctx_status_ptr_reg);
517 	ctx_status_ptr.read_ptr = 0;
518 	ctx_status_ptr.write_ptr = 0x7;
519 	vgpu_vreg(vgpu, ctx_status_ptr_reg) = ctx_status_ptr.dw;
520 }
521 
clean_execlist(struct intel_vgpu * vgpu,intel_engine_mask_t engine_mask)522 static void clean_execlist(struct intel_vgpu *vgpu,
523 			   intel_engine_mask_t engine_mask)
524 {
525 	struct intel_vgpu_submission *s = &vgpu->submission;
526 	struct intel_engine_cs *engine;
527 	intel_engine_mask_t tmp;
528 
529 	for_each_engine_masked(engine, vgpu->gvt->gt, engine_mask, tmp) {
530 		kfree(s->ring_scan_buffer[engine->id]);
531 		s->ring_scan_buffer[engine->id] = NULL;
532 		s->ring_scan_buffer_size[engine->id] = 0;
533 	}
534 }
535 
reset_execlist(struct intel_vgpu * vgpu,intel_engine_mask_t engine_mask)536 static void reset_execlist(struct intel_vgpu *vgpu,
537 			   intel_engine_mask_t engine_mask)
538 {
539 	struct intel_engine_cs *engine;
540 	intel_engine_mask_t tmp;
541 
542 	for_each_engine_masked(engine, vgpu->gvt->gt, engine_mask, tmp)
543 		init_vgpu_execlist(vgpu, engine);
544 }
545 
init_execlist(struct intel_vgpu * vgpu,intel_engine_mask_t engine_mask)546 static int init_execlist(struct intel_vgpu *vgpu,
547 			 intel_engine_mask_t engine_mask)
548 {
549 	reset_execlist(vgpu, engine_mask);
550 	return 0;
551 }
552 
553 const struct intel_vgpu_submission_ops intel_vgpu_execlist_submission_ops = {
554 	.name = "execlist",
555 	.init = init_execlist,
556 	.reset = reset_execlist,
557 	.clean = clean_execlist,
558 };
559