1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2014-2019 Intel Corporation
4 */
5
6 #include <linux/bsearch.h>
7
8 #include "gt/intel_engine_regs.h"
9 #include "gt/intel_gt.h"
10 #include "gt/intel_gt_mcr.h"
11 #include "gt/intel_gt_regs.h"
12 #include "gt/intel_lrc.h"
13 #include "gt/shmem_utils.h"
14 #include "intel_guc_ads.h"
15 #include "intel_guc_capture.h"
16 #include "intel_guc_fwif.h"
17 #include "intel_uc.h"
18 #include "i915_drv.h"
19
20 /*
21 * The Additional Data Struct (ADS) has pointers for different buffers used by
22 * the GuC. One single gem object contains the ADS struct itself (guc_ads) and
23 * all the extra buffers indirectly linked via the ADS struct's entries.
24 *
25 * Layout of the ADS blob allocated for the GuC:
26 *
27 * +---------------------------------------+ <== base
28 * | guc_ads |
29 * +---------------------------------------+
30 * | guc_policies |
31 * +---------------------------------------+
32 * | guc_gt_system_info |
33 * +---------------------------------------+
34 * | guc_engine_usage |
35 * +---------------------------------------+ <== static
36 * | guc_mmio_reg[countA] (engine 0.0) |
37 * | guc_mmio_reg[countB] (engine 0.1) |
38 * | guc_mmio_reg[countC] (engine 1.0) |
39 * | ... |
40 * +---------------------------------------+ <== dynamic
41 * | padding |
42 * +---------------------------------------+ <== 4K aligned
43 * | golden contexts |
44 * +---------------------------------------+
45 * | padding |
46 * +---------------------------------------+ <== 4K aligned
47 * | capture lists |
48 * +---------------------------------------+
49 * | padding |
50 * +---------------------------------------+ <== 4K aligned
51 * | private data |
52 * +---------------------------------------+
53 * | padding |
54 * +---------------------------------------+ <== 4K aligned
55 */
56 struct __guc_ads_blob {
57 struct guc_ads ads;
58 struct guc_policies policies;
59 struct guc_gt_system_info system_info;
60 struct guc_engine_usage engine_usage;
61 /* From here on, location is dynamic! Refer to above diagram. */
62 struct guc_mmio_reg regset[];
63 } __packed;
64
65 #define ads_blob_read(guc_, field_) \
66 iosys_map_rd_field(&(guc_)->ads_map, 0, struct __guc_ads_blob, field_)
67
68 #define ads_blob_write(guc_, field_, val_) \
69 iosys_map_wr_field(&(guc_)->ads_map, 0, struct __guc_ads_blob, \
70 field_, val_)
71
72 #define info_map_write(map_, field_, val_) \
73 iosys_map_wr_field(map_, 0, struct guc_gt_system_info, field_, val_)
74
75 #define info_map_read(map_, field_) \
76 iosys_map_rd_field(map_, 0, struct guc_gt_system_info, field_)
77
guc_ads_regset_size(struct intel_guc * guc)78 static u32 guc_ads_regset_size(struct intel_guc *guc)
79 {
80 GEM_BUG_ON(!guc->ads_regset_size);
81 return guc->ads_regset_size;
82 }
83
guc_ads_golden_ctxt_size(struct intel_guc * guc)84 static u32 guc_ads_golden_ctxt_size(struct intel_guc *guc)
85 {
86 return PAGE_ALIGN(guc->ads_golden_ctxt_size);
87 }
88
guc_ads_capture_size(struct intel_guc * guc)89 static u32 guc_ads_capture_size(struct intel_guc *guc)
90 {
91 return PAGE_ALIGN(guc->ads_capture_size);
92 }
93
guc_ads_private_data_size(struct intel_guc * guc)94 static u32 guc_ads_private_data_size(struct intel_guc *guc)
95 {
96 return PAGE_ALIGN(guc->fw.private_data_size);
97 }
98
guc_ads_regset_offset(struct intel_guc * guc)99 static u32 guc_ads_regset_offset(struct intel_guc *guc)
100 {
101 return offsetof(struct __guc_ads_blob, regset);
102 }
103
guc_ads_golden_ctxt_offset(struct intel_guc * guc)104 static u32 guc_ads_golden_ctxt_offset(struct intel_guc *guc)
105 {
106 u32 offset;
107
108 offset = guc_ads_regset_offset(guc) +
109 guc_ads_regset_size(guc);
110
111 return PAGE_ALIGN(offset);
112 }
113
guc_ads_capture_offset(struct intel_guc * guc)114 static u32 guc_ads_capture_offset(struct intel_guc *guc)
115 {
116 u32 offset;
117
118 offset = guc_ads_golden_ctxt_offset(guc) +
119 guc_ads_golden_ctxt_size(guc);
120
121 return PAGE_ALIGN(offset);
122 }
123
guc_ads_private_data_offset(struct intel_guc * guc)124 static u32 guc_ads_private_data_offset(struct intel_guc *guc)
125 {
126 u32 offset;
127
128 offset = guc_ads_capture_offset(guc) +
129 guc_ads_capture_size(guc);
130
131 return PAGE_ALIGN(offset);
132 }
133
guc_ads_blob_size(struct intel_guc * guc)134 static u32 guc_ads_blob_size(struct intel_guc *guc)
135 {
136 return guc_ads_private_data_offset(guc) +
137 guc_ads_private_data_size(guc);
138 }
139
guc_policies_init(struct intel_guc * guc)140 static void guc_policies_init(struct intel_guc *guc)
141 {
142 struct intel_gt *gt = guc_to_gt(guc);
143 struct drm_i915_private *i915 = gt->i915;
144 u32 global_flags = 0;
145
146 ads_blob_write(guc, policies.dpc_promote_time,
147 GLOBAL_POLICY_DEFAULT_DPC_PROMOTE_TIME_US);
148 ads_blob_write(guc, policies.max_num_work_items,
149 GLOBAL_POLICY_MAX_NUM_WI);
150
151 if (i915->params.reset < 2)
152 global_flags |= GLOBAL_POLICY_DISABLE_ENGINE_RESET;
153
154 ads_blob_write(guc, policies.global_flags, global_flags);
155 ads_blob_write(guc, policies.is_valid, 1);
156 }
157
intel_guc_ads_print_policy_info(struct intel_guc * guc,struct drm_printer * dp)158 void intel_guc_ads_print_policy_info(struct intel_guc *guc,
159 struct drm_printer *dp)
160 {
161 if (unlikely(iosys_map_is_null(&guc->ads_map)))
162 return;
163
164 drm_printf(dp, "Global scheduling policies:\n");
165 drm_printf(dp, " DPC promote time = %u\n",
166 ads_blob_read(guc, policies.dpc_promote_time));
167 drm_printf(dp, " Max num work items = %u\n",
168 ads_blob_read(guc, policies.max_num_work_items));
169 drm_printf(dp, " Flags = %u\n",
170 ads_blob_read(guc, policies.global_flags));
171 }
172
guc_action_policies_update(struct intel_guc * guc,u32 policy_offset)173 static int guc_action_policies_update(struct intel_guc *guc, u32 policy_offset)
174 {
175 u32 action[] = {
176 INTEL_GUC_ACTION_GLOBAL_SCHED_POLICY_CHANGE,
177 policy_offset
178 };
179
180 return intel_guc_send_busy_loop(guc, action, ARRAY_SIZE(action), 0, true);
181 }
182
intel_guc_global_policies_update(struct intel_guc * guc)183 int intel_guc_global_policies_update(struct intel_guc *guc)
184 {
185 struct intel_gt *gt = guc_to_gt(guc);
186 u32 scheduler_policies;
187 intel_wakeref_t wakeref;
188 int ret;
189
190 if (iosys_map_is_null(&guc->ads_map))
191 return -EOPNOTSUPP;
192
193 scheduler_policies = ads_blob_read(guc, ads.scheduler_policies);
194 GEM_BUG_ON(!scheduler_policies);
195
196 guc_policies_init(guc);
197
198 if (!intel_guc_is_ready(guc))
199 return 0;
200
201 with_intel_runtime_pm(>->i915->runtime_pm, wakeref)
202 ret = guc_action_policies_update(guc, scheduler_policies);
203
204 return ret;
205 }
206
guc_mapping_table_init(struct intel_gt * gt,struct iosys_map * info_map)207 static void guc_mapping_table_init(struct intel_gt *gt,
208 struct iosys_map *info_map)
209 {
210 unsigned int i, j;
211 struct intel_engine_cs *engine;
212 enum intel_engine_id id;
213
214 /* Table must be set to invalid values for entries not used */
215 for (i = 0; i < GUC_MAX_ENGINE_CLASSES; ++i)
216 for (j = 0; j < GUC_MAX_INSTANCES_PER_CLASS; ++j)
217 info_map_write(info_map, mapping_table[i][j],
218 GUC_MAX_INSTANCES_PER_CLASS);
219
220 for_each_engine(engine, gt, id) {
221 u8 guc_class = engine_class_to_guc_class(engine->class);
222
223 info_map_write(info_map, mapping_table[guc_class][ilog2(engine->logical_mask)],
224 engine->instance);
225 }
226 }
227
228 /*
229 * The save/restore register list must be pre-calculated to a temporary
230 * buffer before it can be copied inside the ADS.
231 */
232 struct temp_regset {
233 /*
234 * ptr to the section of the storage for the engine currently being
235 * worked on
236 */
237 struct guc_mmio_reg *registers;
238 /* ptr to the base of the allocated storage for all engines */
239 struct guc_mmio_reg *storage;
240 u32 storage_used;
241 u32 storage_max;
242 };
243
guc_mmio_reg_cmp(const void * a,const void * b)244 static int guc_mmio_reg_cmp(const void *a, const void *b)
245 {
246 const struct guc_mmio_reg *ra = a;
247 const struct guc_mmio_reg *rb = b;
248
249 return (int)ra->offset - (int)rb->offset;
250 }
251
252 static struct guc_mmio_reg * __must_check
__mmio_reg_add(struct temp_regset * regset,struct guc_mmio_reg * reg)253 __mmio_reg_add(struct temp_regset *regset, struct guc_mmio_reg *reg)
254 {
255 u32 pos = regset->storage_used;
256 struct guc_mmio_reg *slot;
257
258 if (pos >= regset->storage_max) {
259 size_t size = ALIGN((pos + 1) * sizeof(*slot), PAGE_SIZE);
260 struct guc_mmio_reg *r = krealloc(regset->storage,
261 size, GFP_KERNEL);
262 if (!r) {
263 WARN_ONCE(1, "Incomplete regset list: can't add register (%d)\n",
264 -ENOMEM);
265 return ERR_PTR(-ENOMEM);
266 }
267
268 regset->registers = r + (regset->registers - regset->storage);
269 regset->storage = r;
270 regset->storage_max = size / sizeof(*slot);
271 }
272
273 slot = ®set->storage[pos];
274 regset->storage_used++;
275 *slot = *reg;
276
277 return slot;
278 }
279
280 #define GUC_REGSET_STEERING(group, instance) ( \
281 FIELD_PREP(GUC_REGSET_STEERING_GROUP, (group)) | \
282 FIELD_PREP(GUC_REGSET_STEERING_INSTANCE, (instance)) | \
283 GUC_REGSET_NEEDS_STEERING \
284 )
285
guc_mmio_reg_add(struct intel_gt * gt,struct temp_regset * regset,i915_reg_t reg,u32 flags)286 static long __must_check guc_mmio_reg_add(struct intel_gt *gt,
287 struct temp_regset *regset,
288 i915_reg_t reg, u32 flags)
289 {
290 u32 count = regset->storage_used - (regset->registers - regset->storage);
291 u32 offset = i915_mmio_reg_offset(reg);
292 struct guc_mmio_reg entry = {
293 .offset = offset,
294 .flags = flags,
295 };
296 struct guc_mmio_reg *slot;
297 u8 group, inst;
298
299 /*
300 * The mmio list is built using separate lists within the driver.
301 * It's possible that at some point we may attempt to add the same
302 * register more than once. Do not consider this an error; silently
303 * move on if the register is already in the list.
304 */
305 if (bsearch(&entry, regset->registers, count,
306 sizeof(entry), guc_mmio_reg_cmp))
307 return 0;
308
309 /*
310 * The GuC doesn't have a default steering, so we need to explicitly
311 * steer all registers that need steering. However, we do not keep track
312 * of all the steering ranges, only of those that have a chance of using
313 * a non-default steering from the i915 pov. Instead of adding such
314 * tracking, it is easier to just program the default steering for all
315 * regs that don't need a non-default one.
316 */
317 intel_gt_mcr_get_nonterminated_steering(gt, reg, &group, &inst);
318 entry.flags |= GUC_REGSET_STEERING(group, inst);
319
320 slot = __mmio_reg_add(regset, &entry);
321 if (IS_ERR(slot))
322 return PTR_ERR(slot);
323
324 while (slot-- > regset->registers) {
325 GEM_BUG_ON(slot[0].offset == slot[1].offset);
326 if (slot[1].offset > slot[0].offset)
327 break;
328
329 swap(slot[1], slot[0]);
330 }
331
332 return 0;
333 }
334
335 #define GUC_MMIO_REG_ADD(gt, regset, reg, masked) \
336 guc_mmio_reg_add(gt, \
337 regset, \
338 (reg), \
339 (masked) ? GUC_REGSET_MASKED : 0)
340
guc_mmio_regset_init(struct temp_regset * regset,struct intel_engine_cs * engine)341 static int guc_mmio_regset_init(struct temp_regset *regset,
342 struct intel_engine_cs *engine)
343 {
344 struct intel_gt *gt = engine->gt;
345 const u32 base = engine->mmio_base;
346 struct i915_wa_list *wal = &engine->wa_list;
347 struct i915_wa *wa;
348 unsigned int i;
349 int ret = 0;
350
351 /*
352 * Each engine's registers point to a new start relative to
353 * storage
354 */
355 regset->registers = regset->storage + regset->storage_used;
356
357 ret |= GUC_MMIO_REG_ADD(gt, regset, RING_MODE_GEN7(base), true);
358 ret |= GUC_MMIO_REG_ADD(gt, regset, RING_HWS_PGA(base), false);
359 ret |= GUC_MMIO_REG_ADD(gt, regset, RING_IMR(base), false);
360
361 if ((engine->flags & I915_ENGINE_FIRST_RENDER_COMPUTE) &&
362 CCS_MASK(engine->gt))
363 ret |= GUC_MMIO_REG_ADD(gt, regset, GEN12_RCU_MODE, true);
364
365 for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
366 ret |= GUC_MMIO_REG_ADD(gt, regset, wa->reg, wa->masked_reg);
367
368 /* Be extra paranoid and include all whitelist registers. */
369 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++)
370 ret |= GUC_MMIO_REG_ADD(gt, regset,
371 RING_FORCE_TO_NONPRIV(base, i),
372 false);
373
374 /* add in local MOCS registers */
375 for (i = 0; i < GEN9_LNCFCMOCS_REG_COUNT; i++)
376 ret |= GUC_MMIO_REG_ADD(gt, regset, GEN9_LNCFCMOCS(i), false);
377
378 return ret ? -1 : 0;
379 }
380
guc_mmio_reg_state_create(struct intel_guc * guc)381 static long guc_mmio_reg_state_create(struct intel_guc *guc)
382 {
383 struct intel_gt *gt = guc_to_gt(guc);
384 struct intel_engine_cs *engine;
385 enum intel_engine_id id;
386 struct temp_regset temp_set = {};
387 long total = 0;
388 long ret;
389
390 for_each_engine(engine, gt, id) {
391 u32 used = temp_set.storage_used;
392
393 ret = guc_mmio_regset_init(&temp_set, engine);
394 if (ret < 0)
395 goto fail_regset_init;
396
397 guc->ads_regset_count[id] = temp_set.storage_used - used;
398 total += guc->ads_regset_count[id];
399 }
400
401 guc->ads_regset = temp_set.storage;
402
403 drm_dbg(&guc_to_gt(guc)->i915->drm, "Used %zu KB for temporary ADS regset\n",
404 (temp_set.storage_max * sizeof(struct guc_mmio_reg)) >> 10);
405
406 return total * sizeof(struct guc_mmio_reg);
407
408 fail_regset_init:
409 kfree(temp_set.storage);
410 return ret;
411 }
412
guc_mmio_reg_state_init(struct intel_guc * guc)413 static void guc_mmio_reg_state_init(struct intel_guc *guc)
414 {
415 struct intel_gt *gt = guc_to_gt(guc);
416 struct intel_engine_cs *engine;
417 enum intel_engine_id id;
418 u32 addr_ggtt, offset;
419
420 offset = guc_ads_regset_offset(guc);
421 addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset;
422
423 iosys_map_memcpy_to(&guc->ads_map, offset, guc->ads_regset,
424 guc->ads_regset_size);
425
426 for_each_engine(engine, gt, id) {
427 u32 count = guc->ads_regset_count[id];
428 u8 guc_class;
429
430 /* Class index is checked in class converter */
431 GEM_BUG_ON(engine->instance >= GUC_MAX_INSTANCES_PER_CLASS);
432
433 guc_class = engine_class_to_guc_class(engine->class);
434
435 if (!count) {
436 ads_blob_write(guc,
437 ads.reg_state_list[guc_class][engine->instance].address,
438 0);
439 ads_blob_write(guc,
440 ads.reg_state_list[guc_class][engine->instance].count,
441 0);
442 continue;
443 }
444
445 ads_blob_write(guc,
446 ads.reg_state_list[guc_class][engine->instance].address,
447 addr_ggtt);
448 ads_blob_write(guc,
449 ads.reg_state_list[guc_class][engine->instance].count,
450 count);
451
452 addr_ggtt += count * sizeof(struct guc_mmio_reg);
453 }
454 }
455
fill_engine_enable_masks(struct intel_gt * gt,struct iosys_map * info_map)456 static void fill_engine_enable_masks(struct intel_gt *gt,
457 struct iosys_map *info_map)
458 {
459 info_map_write(info_map, engine_enabled_masks[GUC_RENDER_CLASS], RCS_MASK(gt));
460 info_map_write(info_map, engine_enabled_masks[GUC_COMPUTE_CLASS], CCS_MASK(gt));
461 info_map_write(info_map, engine_enabled_masks[GUC_BLITTER_CLASS], BCS_MASK(gt));
462 info_map_write(info_map, engine_enabled_masks[GUC_VIDEO_CLASS], VDBOX_MASK(gt));
463 info_map_write(info_map, engine_enabled_masks[GUC_VIDEOENHANCE_CLASS], VEBOX_MASK(gt));
464 }
465
466 #define LR_HW_CONTEXT_SIZE (80 * sizeof(u32))
467 #define XEHP_LR_HW_CONTEXT_SIZE (96 * sizeof(u32))
468 #define LR_HW_CONTEXT_SZ(i915) (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50) ? \
469 XEHP_LR_HW_CONTEXT_SIZE : \
470 LR_HW_CONTEXT_SIZE)
471 #define LRC_SKIP_SIZE(i915) (LRC_PPHWSP_SZ * PAGE_SIZE + LR_HW_CONTEXT_SZ(i915))
guc_prep_golden_context(struct intel_guc * guc)472 static int guc_prep_golden_context(struct intel_guc *guc)
473 {
474 struct intel_gt *gt = guc_to_gt(guc);
475 u32 addr_ggtt, offset;
476 u32 total_size = 0, alloc_size, real_size;
477 u8 engine_class, guc_class;
478 struct guc_gt_system_info local_info;
479 struct iosys_map info_map;
480
481 /*
482 * Reserve the memory for the golden contexts and point GuC at it but
483 * leave it empty for now. The context data will be filled in later
484 * once there is something available to put there.
485 *
486 * Note that the HWSP and ring context are not included.
487 *
488 * Note also that the storage must be pinned in the GGTT, so that the
489 * address won't change after GuC has been told where to find it. The
490 * GuC will also validate that the LRC base + size fall within the
491 * allowed GGTT range.
492 */
493 if (!iosys_map_is_null(&guc->ads_map)) {
494 offset = guc_ads_golden_ctxt_offset(guc);
495 addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset;
496 info_map = IOSYS_MAP_INIT_OFFSET(&guc->ads_map,
497 offsetof(struct __guc_ads_blob, system_info));
498 } else {
499 memset(&local_info, 0, sizeof(local_info));
500 iosys_map_set_vaddr(&info_map, &local_info);
501 fill_engine_enable_masks(gt, &info_map);
502 }
503
504 for (engine_class = 0; engine_class <= MAX_ENGINE_CLASS; ++engine_class) {
505 if (engine_class == OTHER_CLASS)
506 continue;
507
508 guc_class = engine_class_to_guc_class(engine_class);
509
510 if (!info_map_read(&info_map, engine_enabled_masks[guc_class]))
511 continue;
512
513 real_size = intel_engine_context_size(gt, engine_class);
514 alloc_size = PAGE_ALIGN(real_size);
515 total_size += alloc_size;
516
517 if (iosys_map_is_null(&guc->ads_map))
518 continue;
519
520 /*
521 * This interface is slightly confusing. We need to pass the
522 * base address of the full golden context and the size of just
523 * the engine state, which is the section of the context image
524 * that starts after the execlists context. This is required to
525 * allow the GuC to restore just the engine state when a
526 * watchdog reset occurs.
527 * We calculate the engine state size by removing the size of
528 * what comes before it in the context image (which is identical
529 * on all engines).
530 */
531 ads_blob_write(guc, ads.eng_state_size[guc_class],
532 real_size - LRC_SKIP_SIZE(gt->i915));
533 ads_blob_write(guc, ads.golden_context_lrca[guc_class],
534 addr_ggtt);
535
536 addr_ggtt += alloc_size;
537 }
538
539 /* Make sure current size matches what we calculated previously */
540 if (guc->ads_golden_ctxt_size)
541 GEM_BUG_ON(guc->ads_golden_ctxt_size != total_size);
542
543 return total_size;
544 }
545
find_engine_state(struct intel_gt * gt,u8 engine_class)546 static struct intel_engine_cs *find_engine_state(struct intel_gt *gt, u8 engine_class)
547 {
548 struct intel_engine_cs *engine;
549 enum intel_engine_id id;
550
551 for_each_engine(engine, gt, id) {
552 if (engine->class != engine_class)
553 continue;
554
555 if (!engine->default_state)
556 continue;
557
558 return engine;
559 }
560
561 return NULL;
562 }
563
guc_init_golden_context(struct intel_guc * guc)564 static void guc_init_golden_context(struct intel_guc *guc)
565 {
566 struct intel_engine_cs *engine;
567 struct intel_gt *gt = guc_to_gt(guc);
568 unsigned long offset;
569 u32 addr_ggtt, total_size = 0, alloc_size, real_size;
570 u8 engine_class, guc_class;
571
572 if (!intel_uc_uses_guc_submission(>->uc))
573 return;
574
575 GEM_BUG_ON(iosys_map_is_null(&guc->ads_map));
576
577 /*
578 * Go back and fill in the golden context data now that it is
579 * available.
580 */
581 offset = guc_ads_golden_ctxt_offset(guc);
582 addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset;
583
584 for (engine_class = 0; engine_class <= MAX_ENGINE_CLASS; ++engine_class) {
585 if (engine_class == OTHER_CLASS)
586 continue;
587
588 guc_class = engine_class_to_guc_class(engine_class);
589 if (!ads_blob_read(guc, system_info.engine_enabled_masks[guc_class]))
590 continue;
591
592 real_size = intel_engine_context_size(gt, engine_class);
593 alloc_size = PAGE_ALIGN(real_size);
594 total_size += alloc_size;
595
596 engine = find_engine_state(gt, engine_class);
597 if (!engine) {
598 drm_err(>->i915->drm, "No engine state recorded for class %d!\n",
599 engine_class);
600 ads_blob_write(guc, ads.eng_state_size[guc_class], 0);
601 ads_blob_write(guc, ads.golden_context_lrca[guc_class], 0);
602 continue;
603 }
604
605 GEM_BUG_ON(ads_blob_read(guc, ads.eng_state_size[guc_class]) !=
606 real_size - LRC_SKIP_SIZE(gt->i915));
607 GEM_BUG_ON(ads_blob_read(guc, ads.golden_context_lrca[guc_class]) != addr_ggtt);
608
609 addr_ggtt += alloc_size;
610
611 shmem_read_to_iosys_map(engine->default_state, 0, &guc->ads_map,
612 offset, real_size);
613 offset += alloc_size;
614 }
615
616 GEM_BUG_ON(guc->ads_golden_ctxt_size != total_size);
617 }
618
619 static int
guc_capture_prep_lists(struct intel_guc * guc)620 guc_capture_prep_lists(struct intel_guc *guc)
621 {
622 struct intel_gt *gt = guc_to_gt(guc);
623 struct drm_i915_private *i915 = guc_to_gt(guc)->i915;
624 u32 ads_ggtt, capture_offset, null_ggtt, total_size = 0;
625 struct guc_gt_system_info local_info;
626 struct iosys_map info_map;
627 bool ads_is_mapped;
628 size_t size = 0;
629 void *ptr;
630 int i, j;
631
632 ads_is_mapped = !iosys_map_is_null(&guc->ads_map);
633 if (ads_is_mapped) {
634 capture_offset = guc_ads_capture_offset(guc);
635 ads_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma);
636 info_map = IOSYS_MAP_INIT_OFFSET(&guc->ads_map,
637 offsetof(struct __guc_ads_blob, system_info));
638 } else {
639 memset(&local_info, 0, sizeof(local_info));
640 iosys_map_set_vaddr(&info_map, &local_info);
641 fill_engine_enable_masks(gt, &info_map);
642 }
643
644 /* first, set aside the first page for a capture_list with zero descriptors */
645 total_size = PAGE_SIZE;
646 if (ads_is_mapped) {
647 if (!intel_guc_capture_getnullheader(guc, &ptr, &size))
648 iosys_map_memcpy_to(&guc->ads_map, capture_offset, ptr, size);
649 null_ggtt = ads_ggtt + capture_offset;
650 capture_offset += PAGE_SIZE;
651 }
652
653 for (i = 0; i < GUC_CAPTURE_LIST_INDEX_MAX; i++) {
654 for (j = 0; j < GUC_MAX_ENGINE_CLASSES; j++) {
655
656 /* null list if we dont have said engine or list */
657 if (!info_map_read(&info_map, engine_enabled_masks[j])) {
658 if (ads_is_mapped) {
659 ads_blob_write(guc, ads.capture_class[i][j], null_ggtt);
660 ads_blob_write(guc, ads.capture_instance[i][j], null_ggtt);
661 }
662 continue;
663 }
664 if (intel_guc_capture_getlistsize(guc, i,
665 GUC_CAPTURE_LIST_TYPE_ENGINE_CLASS,
666 j, &size)) {
667 if (ads_is_mapped)
668 ads_blob_write(guc, ads.capture_class[i][j], null_ggtt);
669 goto engine_instance_list;
670 }
671 total_size += size;
672 if (ads_is_mapped) {
673 if (total_size > guc->ads_capture_size ||
674 intel_guc_capture_getlist(guc, i,
675 GUC_CAPTURE_LIST_TYPE_ENGINE_CLASS,
676 j, &ptr)) {
677 ads_blob_write(guc, ads.capture_class[i][j], null_ggtt);
678 continue;
679 }
680 ads_blob_write(guc, ads.capture_class[i][j], ads_ggtt +
681 capture_offset);
682 iosys_map_memcpy_to(&guc->ads_map, capture_offset, ptr, size);
683 capture_offset += size;
684 }
685 engine_instance_list:
686 if (intel_guc_capture_getlistsize(guc, i,
687 GUC_CAPTURE_LIST_TYPE_ENGINE_INSTANCE,
688 j, &size)) {
689 if (ads_is_mapped)
690 ads_blob_write(guc, ads.capture_instance[i][j], null_ggtt);
691 continue;
692 }
693 total_size += size;
694 if (ads_is_mapped) {
695 if (total_size > guc->ads_capture_size ||
696 intel_guc_capture_getlist(guc, i,
697 GUC_CAPTURE_LIST_TYPE_ENGINE_INSTANCE,
698 j, &ptr)) {
699 ads_blob_write(guc, ads.capture_instance[i][j], null_ggtt);
700 continue;
701 }
702 ads_blob_write(guc, ads.capture_instance[i][j], ads_ggtt +
703 capture_offset);
704 iosys_map_memcpy_to(&guc->ads_map, capture_offset, ptr, size);
705 capture_offset += size;
706 }
707 }
708 if (intel_guc_capture_getlistsize(guc, i, GUC_CAPTURE_LIST_TYPE_GLOBAL, 0, &size)) {
709 if (ads_is_mapped)
710 ads_blob_write(guc, ads.capture_global[i], null_ggtt);
711 continue;
712 }
713 total_size += size;
714 if (ads_is_mapped) {
715 if (total_size > guc->ads_capture_size ||
716 intel_guc_capture_getlist(guc, i, GUC_CAPTURE_LIST_TYPE_GLOBAL, 0,
717 &ptr)) {
718 ads_blob_write(guc, ads.capture_global[i], null_ggtt);
719 continue;
720 }
721 ads_blob_write(guc, ads.capture_global[i], ads_ggtt + capture_offset);
722 iosys_map_memcpy_to(&guc->ads_map, capture_offset, ptr, size);
723 capture_offset += size;
724 }
725 }
726
727 if (guc->ads_capture_size && guc->ads_capture_size != PAGE_ALIGN(total_size))
728 drm_warn(&i915->drm, "GuC->ADS->Capture alloc size changed from %d to %d\n",
729 guc->ads_capture_size, PAGE_ALIGN(total_size));
730
731 return PAGE_ALIGN(total_size);
732 }
733
__guc_ads_init(struct intel_guc * guc)734 static void __guc_ads_init(struct intel_guc *guc)
735 {
736 struct intel_gt *gt = guc_to_gt(guc);
737 struct drm_i915_private *i915 = gt->i915;
738 struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(&guc->ads_map,
739 offsetof(struct __guc_ads_blob, system_info));
740 u32 base;
741
742 /* GuC scheduling policies */
743 guc_policies_init(guc);
744
745 /* System info */
746 fill_engine_enable_masks(gt, &info_map);
747
748 ads_blob_write(guc, system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_SLICE_ENABLED],
749 hweight8(gt->info.sseu.slice_mask));
750 ads_blob_write(guc, system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_VDBOX_SFC_SUPPORT_MASK],
751 gt->info.vdbox_sfc_access);
752
753 if (GRAPHICS_VER(i915) >= 12 && !IS_DGFX(i915)) {
754 u32 distdbreg = intel_uncore_read(gt->uncore,
755 GEN12_DIST_DBS_POPULATED);
756 ads_blob_write(guc,
757 system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_DOORBELL_COUNT_PER_SQIDI],
758 ((distdbreg >> GEN12_DOORBELLS_PER_SQIDI_SHIFT)
759 & GEN12_DOORBELLS_PER_SQIDI) + 1);
760 }
761
762 /* Golden contexts for re-initialising after a watchdog reset */
763 guc_prep_golden_context(guc);
764
765 guc_mapping_table_init(guc_to_gt(guc), &info_map);
766
767 base = intel_guc_ggtt_offset(guc, guc->ads_vma);
768
769 /* Lists for error capture debug */
770 guc_capture_prep_lists(guc);
771
772 /* ADS */
773 ads_blob_write(guc, ads.scheduler_policies, base +
774 offsetof(struct __guc_ads_blob, policies));
775 ads_blob_write(guc, ads.gt_system_info, base +
776 offsetof(struct __guc_ads_blob, system_info));
777
778 /* MMIO save/restore list */
779 guc_mmio_reg_state_init(guc);
780
781 /* Private Data */
782 ads_blob_write(guc, ads.private_data, base +
783 guc_ads_private_data_offset(guc));
784
785 i915_gem_object_flush_map(guc->ads_vma->obj);
786 }
787
788 /**
789 * intel_guc_ads_create() - allocates and initializes GuC ADS.
790 * @guc: intel_guc struct
791 *
792 * GuC needs memory block (Additional Data Struct), where it will store
793 * some data. Allocate and initialize such memory block for GuC use.
794 */
intel_guc_ads_create(struct intel_guc * guc)795 int intel_guc_ads_create(struct intel_guc *guc)
796 {
797 void *ads_blob;
798 u32 size;
799 int ret;
800
801 GEM_BUG_ON(guc->ads_vma);
802
803 /*
804 * Create reg state size dynamically on system memory to be copied to
805 * the final ads blob on gt init/reset
806 */
807 ret = guc_mmio_reg_state_create(guc);
808 if (ret < 0)
809 return ret;
810 guc->ads_regset_size = ret;
811
812 /* Likewise the golden contexts: */
813 ret = guc_prep_golden_context(guc);
814 if (ret < 0)
815 return ret;
816 guc->ads_golden_ctxt_size = ret;
817
818 /* Likewise the capture lists: */
819 ret = guc_capture_prep_lists(guc);
820 if (ret < 0)
821 return ret;
822 guc->ads_capture_size = ret;
823
824 /* Now the total size can be determined: */
825 size = guc_ads_blob_size(guc);
826
827 ret = intel_guc_allocate_and_map_vma(guc, size, &guc->ads_vma,
828 &ads_blob);
829 if (ret)
830 return ret;
831
832 if (i915_gem_object_is_lmem(guc->ads_vma->obj))
833 iosys_map_set_vaddr_iomem(&guc->ads_map, (void __iomem *)ads_blob);
834 else
835 iosys_map_set_vaddr(&guc->ads_map, ads_blob);
836
837 __guc_ads_init(guc);
838
839 return 0;
840 }
841
intel_guc_ads_init_late(struct intel_guc * guc)842 void intel_guc_ads_init_late(struct intel_guc *guc)
843 {
844 /*
845 * The golden context setup requires the saved engine state from
846 * __engines_record_defaults(). However, that requires engines to be
847 * operational which means the ADS must already have been configured.
848 * Fortunately, the golden context state is not needed until a hang
849 * occurs, so it can be filled in during this late init phase.
850 */
851 guc_init_golden_context(guc);
852 }
853
intel_guc_ads_destroy(struct intel_guc * guc)854 void intel_guc_ads_destroy(struct intel_guc *guc)
855 {
856 i915_vma_unpin_and_release(&guc->ads_vma, I915_VMA_RELEASE_MAP);
857 iosys_map_clear(&guc->ads_map);
858 kfree(guc->ads_regset);
859 }
860
guc_ads_private_data_reset(struct intel_guc * guc)861 static void guc_ads_private_data_reset(struct intel_guc *guc)
862 {
863 u32 size;
864
865 size = guc_ads_private_data_size(guc);
866 if (!size)
867 return;
868
869 iosys_map_memset(&guc->ads_map, guc_ads_private_data_offset(guc),
870 0, size);
871 }
872
873 /**
874 * intel_guc_ads_reset() - prepares GuC Additional Data Struct for reuse
875 * @guc: intel_guc struct
876 *
877 * GuC stores some data in ADS, which might be stale after a reset.
878 * Reinitialize whole ADS in case any part of it was corrupted during
879 * previous GuC run.
880 */
intel_guc_ads_reset(struct intel_guc * guc)881 void intel_guc_ads_reset(struct intel_guc *guc)
882 {
883 if (!guc->ads_vma)
884 return;
885
886 __guc_ads_init(guc);
887
888 guc_ads_private_data_reset(guc);
889 }
890
intel_guc_engine_usage_offset(struct intel_guc * guc)891 u32 intel_guc_engine_usage_offset(struct intel_guc *guc)
892 {
893 return intel_guc_ggtt_offset(guc, guc->ads_vma) +
894 offsetof(struct __guc_ads_blob, engine_usage);
895 }
896
intel_guc_engine_usage_record_map(struct intel_engine_cs * engine)897 struct iosys_map intel_guc_engine_usage_record_map(struct intel_engine_cs *engine)
898 {
899 struct intel_guc *guc = &engine->gt->uc.guc;
900 u8 guc_class = engine_class_to_guc_class(engine->class);
901 size_t offset = offsetof(struct __guc_ads_blob,
902 engine_usage.engines[guc_class][ilog2(engine->logical_mask)]);
903
904 return IOSYS_MAP_INIT_OFFSET(&guc->ads_map, offset);
905 }
906