1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2015 Intel Corporation
4 */
5
6 #include "i915_drv.h"
7
8 #include "intel_engine.h"
9 #include "intel_gt.h"
10 #include "intel_gt_regs.h"
11 #include "intel_mocs.h"
12 #include "intel_ring.h"
13
14 /* structures required */
15 struct drm_i915_mocs_entry {
16 u32 control_value;
17 u16 l3cc_value;
18 u16 used;
19 };
20
21 struct drm_i915_mocs_table {
22 unsigned int size;
23 unsigned int n_entries;
24 const struct drm_i915_mocs_entry *table;
25 u8 uc_index;
26 u8 unused_entries_index;
27 };
28
29 /* Defines for the tables (XXX_MOCS_0 - XXX_MOCS_63) */
30 #define _LE_CACHEABILITY(value) ((value) << 0)
31 #define _LE_TGT_CACHE(value) ((value) << 2)
32 #define LE_LRUM(value) ((value) << 4)
33 #define LE_AOM(value) ((value) << 6)
34 #define LE_RSC(value) ((value) << 7)
35 #define LE_SCC(value) ((value) << 8)
36 #define LE_PFM(value) ((value) << 11)
37 #define LE_SCF(value) ((value) << 14)
38 #define LE_COS(value) ((value) << 15)
39 #define LE_SSE(value) ((value) << 17)
40
41 /* Defines for the tables (LNCFMOCS0 - LNCFMOCS31) - two entries per word */
42 #define L3_ESC(value) ((value) << 0)
43 #define L3_SCC(value) ((value) << 1)
44 #define _L3_CACHEABILITY(value) ((value) << 4)
45 #define L3_GLBGO(value) ((value) << 6)
46 #define L3_LKUP(value) ((value) << 7)
47
48 /* Helper defines */
49 #define GEN9_NUM_MOCS_ENTRIES 64 /* 63-64 are reserved, but configured. */
50
51 /* (e)LLC caching options */
52 /*
53 * Note: LE_0_PAGETABLE works only up to Gen11; for newer gens it means
54 * the same as LE_UC
55 */
56 #define LE_0_PAGETABLE _LE_CACHEABILITY(0)
57 #define LE_1_UC _LE_CACHEABILITY(1)
58 #define LE_2_WT _LE_CACHEABILITY(2)
59 #define LE_3_WB _LE_CACHEABILITY(3)
60
61 /* Target cache */
62 #define LE_TC_0_PAGETABLE _LE_TGT_CACHE(0)
63 #define LE_TC_1_LLC _LE_TGT_CACHE(1)
64 #define LE_TC_2_LLC_ELLC _LE_TGT_CACHE(2)
65 #define LE_TC_3_LLC_ELLC_ALT _LE_TGT_CACHE(3)
66
67 /* L3 caching options */
68 #define L3_0_DIRECT _L3_CACHEABILITY(0)
69 #define L3_1_UC _L3_CACHEABILITY(1)
70 #define L3_2_RESERVED _L3_CACHEABILITY(2)
71 #define L3_3_WB _L3_CACHEABILITY(3)
72
73 #define MOCS_ENTRY(__idx, __control_value, __l3cc_value) \
74 [__idx] = { \
75 .control_value = __control_value, \
76 .l3cc_value = __l3cc_value, \
77 .used = 1, \
78 }
79
80 /*
81 * MOCS tables
82 *
83 * These are the MOCS tables that are programmed across all the rings.
84 * The control value is programmed to all the rings that support the
85 * MOCS registers. While the l3cc_values are only programmed to the
86 * LNCFCMOCS0 - LNCFCMOCS32 registers.
87 *
88 * These tables are intended to be kept reasonably consistent across
89 * HW platforms, and for ICL+, be identical across OSes. To achieve
90 * that, for Icelake and above, list of entries is published as part
91 * of bspec.
92 *
93 * Entries not part of the following tables are undefined as far as
94 * userspace is concerned and shouldn't be relied upon. For Gen < 12
95 * they will be initialized to PTE. Gen >= 12 don't have a setting for
96 * PTE and those platforms except TGL/RKL will be initialized L3 WB to
97 * catch accidental use of reserved and unused mocs indexes.
98 *
99 * The last few entries are reserved by the hardware. For ICL+ they
100 * should be initialized according to bspec and never used, for older
101 * platforms they should never be written to.
102 *
103 * NOTE1: These tables are part of bspec and defined as part of hardware
104 * interface for ICL+. For older platforms, they are part of kernel
105 * ABI. It is expected that, for specific hardware platform, existing
106 * entries will remain constant and the table will only be updated by
107 * adding new entries, filling unused positions.
108 *
109 * NOTE2: For GEN >= 12 except TGL and RKL, reserved and unspecified MOCS
110 * indices have been set to L3 WB. These reserved entries should never
111 * be used, they may be changed to low performant variants with better
112 * coherency in the future if more entries are needed.
113 * For TGL/RKL, all the unspecified MOCS indexes are mapped to L3 UC.
114 */
115 #define GEN9_MOCS_ENTRIES \
116 MOCS_ENTRY(I915_MOCS_UNCACHED, \
117 LE_1_UC | LE_TC_2_LLC_ELLC, \
118 L3_1_UC), \
119 MOCS_ENTRY(I915_MOCS_PTE, \
120 LE_0_PAGETABLE | LE_TC_0_PAGETABLE | LE_LRUM(3), \
121 L3_3_WB)
122
123 static const struct drm_i915_mocs_entry skl_mocs_table[] = {
124 GEN9_MOCS_ENTRIES,
125 MOCS_ENTRY(I915_MOCS_CACHED,
126 LE_3_WB | LE_TC_2_LLC_ELLC | LE_LRUM(3),
127 L3_3_WB),
128
129 /*
130 * mocs:63
131 * - used by the L3 for all of its evictions.
132 * Thus it is expected to allow LLC cacheability to enable coherent
133 * flows to be maintained.
134 * - used to force L3 uncachable cycles.
135 * Thus it is expected to make the surface L3 uncacheable.
136 */
137 MOCS_ENTRY(63,
138 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3),
139 L3_1_UC)
140 };
141
142 /* NOTE: the LE_TGT_CACHE is not used on Broxton */
143 static const struct drm_i915_mocs_entry broxton_mocs_table[] = {
144 GEN9_MOCS_ENTRIES,
145 MOCS_ENTRY(I915_MOCS_CACHED,
146 LE_1_UC | LE_TC_2_LLC_ELLC | LE_LRUM(3),
147 L3_3_WB)
148 };
149
150 #define GEN11_MOCS_ENTRIES \
151 /* Entries 0 and 1 are defined per-platform */ \
152 /* Base - L3 + LLC */ \
153 MOCS_ENTRY(2, \
154 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), \
155 L3_3_WB), \
156 /* Base - Uncached */ \
157 MOCS_ENTRY(3, \
158 LE_1_UC | LE_TC_1_LLC, \
159 L3_1_UC), \
160 /* Base - L3 */ \
161 MOCS_ENTRY(4, \
162 LE_1_UC | LE_TC_1_LLC, \
163 L3_3_WB), \
164 /* Base - LLC */ \
165 MOCS_ENTRY(5, \
166 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), \
167 L3_1_UC), \
168 /* Age 0 - LLC */ \
169 MOCS_ENTRY(6, \
170 LE_3_WB | LE_TC_1_LLC | LE_LRUM(1), \
171 L3_1_UC), \
172 /* Age 0 - L3 + LLC */ \
173 MOCS_ENTRY(7, \
174 LE_3_WB | LE_TC_1_LLC | LE_LRUM(1), \
175 L3_3_WB), \
176 /* Age: Don't Chg. - LLC */ \
177 MOCS_ENTRY(8, \
178 LE_3_WB | LE_TC_1_LLC | LE_LRUM(2), \
179 L3_1_UC), \
180 /* Age: Don't Chg. - L3 + LLC */ \
181 MOCS_ENTRY(9, \
182 LE_3_WB | LE_TC_1_LLC | LE_LRUM(2), \
183 L3_3_WB), \
184 /* No AOM - LLC */ \
185 MOCS_ENTRY(10, \
186 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_AOM(1), \
187 L3_1_UC), \
188 /* No AOM - L3 + LLC */ \
189 MOCS_ENTRY(11, \
190 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_AOM(1), \
191 L3_3_WB), \
192 /* No AOM; Age 0 - LLC */ \
193 MOCS_ENTRY(12, \
194 LE_3_WB | LE_TC_1_LLC | LE_LRUM(1) | LE_AOM(1), \
195 L3_1_UC), \
196 /* No AOM; Age 0 - L3 + LLC */ \
197 MOCS_ENTRY(13, \
198 LE_3_WB | LE_TC_1_LLC | LE_LRUM(1) | LE_AOM(1), \
199 L3_3_WB), \
200 /* No AOM; Age:DC - LLC */ \
201 MOCS_ENTRY(14, \
202 LE_3_WB | LE_TC_1_LLC | LE_LRUM(2) | LE_AOM(1), \
203 L3_1_UC), \
204 /* No AOM; Age:DC - L3 + LLC */ \
205 MOCS_ENTRY(15, \
206 LE_3_WB | LE_TC_1_LLC | LE_LRUM(2) | LE_AOM(1), \
207 L3_3_WB), \
208 /* Self-Snoop - L3 + LLC */ \
209 MOCS_ENTRY(18, \
210 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SSE(3), \
211 L3_3_WB), \
212 /* Skip Caching - L3 + LLC(12.5%) */ \
213 MOCS_ENTRY(19, \
214 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(7), \
215 L3_3_WB), \
216 /* Skip Caching - L3 + LLC(25%) */ \
217 MOCS_ENTRY(20, \
218 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(3), \
219 L3_3_WB), \
220 /* Skip Caching - L3 + LLC(50%) */ \
221 MOCS_ENTRY(21, \
222 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(1), \
223 L3_3_WB), \
224 /* Skip Caching - L3 + LLC(75%) */ \
225 MOCS_ENTRY(22, \
226 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_RSC(1) | LE_SCC(3), \
227 L3_3_WB), \
228 /* Skip Caching - L3 + LLC(87.5%) */ \
229 MOCS_ENTRY(23, \
230 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_RSC(1) | LE_SCC(7), \
231 L3_3_WB), \
232 /* HW Reserved - SW program but never use */ \
233 MOCS_ENTRY(62, \
234 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), \
235 L3_1_UC), \
236 /* HW Reserved - SW program but never use */ \
237 MOCS_ENTRY(63, \
238 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), \
239 L3_1_UC)
240
241 static const struct drm_i915_mocs_entry tgl_mocs_table[] = {
242 /*
243 * NOTE:
244 * Reserved and unspecified MOCS indices have been set to (L3 + LCC).
245 * These reserved entries should never be used, they may be changed
246 * to low performant variants with better coherency in the future if
247 * more entries are needed. We are programming index I915_MOCS_PTE(1)
248 * only, __init_mocs_table() take care to program unused index with
249 * this entry.
250 */
251 MOCS_ENTRY(I915_MOCS_PTE,
252 LE_0_PAGETABLE | LE_TC_0_PAGETABLE,
253 L3_1_UC),
254 GEN11_MOCS_ENTRIES,
255
256 /* Implicitly enable L1 - HDC:L1 + L3 + LLC */
257 MOCS_ENTRY(48,
258 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3),
259 L3_3_WB),
260 /* Implicitly enable L1 - HDC:L1 + L3 */
261 MOCS_ENTRY(49,
262 LE_1_UC | LE_TC_1_LLC,
263 L3_3_WB),
264 /* Implicitly enable L1 - HDC:L1 + LLC */
265 MOCS_ENTRY(50,
266 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3),
267 L3_1_UC),
268 /* Implicitly enable L1 - HDC:L1 */
269 MOCS_ENTRY(51,
270 LE_1_UC | LE_TC_1_LLC,
271 L3_1_UC),
272 /* HW Special Case (CCS) */
273 MOCS_ENTRY(60,
274 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3),
275 L3_1_UC),
276 /* HW Special Case (Displayable) */
277 MOCS_ENTRY(61,
278 LE_1_UC | LE_TC_1_LLC,
279 L3_3_WB),
280 };
281
282 static const struct drm_i915_mocs_entry icl_mocs_table[] = {
283 /* Base - Uncached (Deprecated) */
284 MOCS_ENTRY(I915_MOCS_UNCACHED,
285 LE_1_UC | LE_TC_1_LLC,
286 L3_1_UC),
287 /* Base - L3 + LeCC:PAT (Deprecated) */
288 MOCS_ENTRY(I915_MOCS_PTE,
289 LE_0_PAGETABLE | LE_TC_0_PAGETABLE,
290 L3_3_WB),
291
292 GEN11_MOCS_ENTRIES
293 };
294
295 static const struct drm_i915_mocs_entry dg1_mocs_table[] = {
296
297 /* UC */
298 MOCS_ENTRY(1, 0, L3_1_UC),
299 /* WB - L3 */
300 MOCS_ENTRY(5, 0, L3_3_WB),
301 /* WB - L3 50% */
302 MOCS_ENTRY(6, 0, L3_ESC(1) | L3_SCC(1) | L3_3_WB),
303 /* WB - L3 25% */
304 MOCS_ENTRY(7, 0, L3_ESC(1) | L3_SCC(3) | L3_3_WB),
305 /* WB - L3 12.5% */
306 MOCS_ENTRY(8, 0, L3_ESC(1) | L3_SCC(7) | L3_3_WB),
307
308 /* HDC:L1 + L3 */
309 MOCS_ENTRY(48, 0, L3_3_WB),
310 /* HDC:L1 */
311 MOCS_ENTRY(49, 0, L3_1_UC),
312
313 /* HW Reserved */
314 MOCS_ENTRY(60, 0, L3_1_UC),
315 MOCS_ENTRY(61, 0, L3_1_UC),
316 MOCS_ENTRY(62, 0, L3_1_UC),
317 MOCS_ENTRY(63, 0, L3_1_UC),
318 };
319
320 static const struct drm_i915_mocs_entry gen12_mocs_table[] = {
321 GEN11_MOCS_ENTRIES,
322 /* Implicitly enable L1 - HDC:L1 + L3 + LLC */
323 MOCS_ENTRY(48,
324 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3),
325 L3_3_WB),
326 /* Implicitly enable L1 - HDC:L1 + L3 */
327 MOCS_ENTRY(49,
328 LE_1_UC | LE_TC_1_LLC,
329 L3_3_WB),
330 /* Implicitly enable L1 - HDC:L1 + LLC */
331 MOCS_ENTRY(50,
332 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3),
333 L3_1_UC),
334 /* Implicitly enable L1 - HDC:L1 */
335 MOCS_ENTRY(51,
336 LE_1_UC | LE_TC_1_LLC,
337 L3_1_UC),
338 /* HW Special Case (CCS) */
339 MOCS_ENTRY(60,
340 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3),
341 L3_1_UC),
342 /* HW Special Case (Displayable) */
343 MOCS_ENTRY(61,
344 LE_1_UC | LE_TC_1_LLC,
345 L3_3_WB),
346 };
347
348 static const struct drm_i915_mocs_entry xehpsdv_mocs_table[] = {
349 /* wa_1608975824 */
350 MOCS_ENTRY(0, 0, L3_3_WB | L3_LKUP(1)),
351
352 /* UC - Coherent; GO:L3 */
353 MOCS_ENTRY(1, 0, L3_1_UC | L3_LKUP(1)),
354 /* UC - Coherent; GO:Memory */
355 MOCS_ENTRY(2, 0, L3_1_UC | L3_GLBGO(1) | L3_LKUP(1)),
356 /* UC - Non-Coherent; GO:Memory */
357 MOCS_ENTRY(3, 0, L3_1_UC | L3_GLBGO(1)),
358 /* UC - Non-Coherent; GO:L3 */
359 MOCS_ENTRY(4, 0, L3_1_UC),
360
361 /* WB */
362 MOCS_ENTRY(5, 0, L3_3_WB | L3_LKUP(1)),
363
364 /* HW Reserved - SW program but never use. */
365 MOCS_ENTRY(48, 0, L3_3_WB | L3_LKUP(1)),
366 MOCS_ENTRY(49, 0, L3_1_UC | L3_LKUP(1)),
367 MOCS_ENTRY(60, 0, L3_1_UC),
368 MOCS_ENTRY(61, 0, L3_1_UC),
369 MOCS_ENTRY(62, 0, L3_1_UC),
370 MOCS_ENTRY(63, 0, L3_1_UC),
371 };
372
373 static const struct drm_i915_mocs_entry dg2_mocs_table[] = {
374 /* UC - Coherent; GO:L3 */
375 MOCS_ENTRY(0, 0, L3_1_UC | L3_LKUP(1)),
376 /* UC - Coherent; GO:Memory */
377 MOCS_ENTRY(1, 0, L3_1_UC | L3_GLBGO(1) | L3_LKUP(1)),
378 /* UC - Non-Coherent; GO:Memory */
379 MOCS_ENTRY(2, 0, L3_1_UC | L3_GLBGO(1)),
380
381 /* WB - LC */
382 MOCS_ENTRY(3, 0, L3_3_WB | L3_LKUP(1)),
383 };
384
385 static const struct drm_i915_mocs_entry dg2_mocs_table_g10_ax[] = {
386 /* Wa_14011441408: Set Go to Memory for MOCS#0 */
387 MOCS_ENTRY(0, 0, L3_1_UC | L3_GLBGO(1) | L3_LKUP(1)),
388 /* UC - Coherent; GO:Memory */
389 MOCS_ENTRY(1, 0, L3_1_UC | L3_GLBGO(1) | L3_LKUP(1)),
390 /* UC - Non-Coherent; GO:Memory */
391 MOCS_ENTRY(2, 0, L3_1_UC | L3_GLBGO(1)),
392
393 /* WB - LC */
394 MOCS_ENTRY(3, 0, L3_3_WB | L3_LKUP(1)),
395 };
396
397 enum {
398 HAS_GLOBAL_MOCS = BIT(0),
399 HAS_ENGINE_MOCS = BIT(1),
400 HAS_RENDER_L3CC = BIT(2),
401 };
402
has_l3cc(const struct drm_i915_private * i915)403 static bool has_l3cc(const struct drm_i915_private *i915)
404 {
405 return true;
406 }
407
has_global_mocs(const struct drm_i915_private * i915)408 static bool has_global_mocs(const struct drm_i915_private *i915)
409 {
410 return HAS_GLOBAL_MOCS_REGISTERS(i915);
411 }
412
has_mocs(const struct drm_i915_private * i915)413 static bool has_mocs(const struct drm_i915_private *i915)
414 {
415 return !IS_DGFX(i915);
416 }
417
get_mocs_settings(const struct drm_i915_private * i915,struct drm_i915_mocs_table * table)418 static unsigned int get_mocs_settings(const struct drm_i915_private *i915,
419 struct drm_i915_mocs_table *table)
420 {
421 unsigned int flags;
422
423 memset(table, 0, sizeof(struct drm_i915_mocs_table));
424
425 table->unused_entries_index = I915_MOCS_PTE;
426 if (IS_DG2(i915)) {
427 if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_A0, STEP_B0)) {
428 table->size = ARRAY_SIZE(dg2_mocs_table_g10_ax);
429 table->table = dg2_mocs_table_g10_ax;
430 } else {
431 table->size = ARRAY_SIZE(dg2_mocs_table);
432 table->table = dg2_mocs_table;
433 }
434 table->uc_index = 1;
435 table->n_entries = GEN9_NUM_MOCS_ENTRIES;
436 table->unused_entries_index = 3;
437 } else if (IS_XEHPSDV(i915)) {
438 table->size = ARRAY_SIZE(xehpsdv_mocs_table);
439 table->table = xehpsdv_mocs_table;
440 table->uc_index = 2;
441 table->n_entries = GEN9_NUM_MOCS_ENTRIES;
442 table->unused_entries_index = 5;
443 } else if (IS_DG1(i915)) {
444 table->size = ARRAY_SIZE(dg1_mocs_table);
445 table->table = dg1_mocs_table;
446 table->uc_index = 1;
447 table->n_entries = GEN9_NUM_MOCS_ENTRIES;
448 table->uc_index = 1;
449 table->unused_entries_index = 5;
450 } else if (IS_TIGERLAKE(i915) || IS_ROCKETLAKE(i915)) {
451 /* For TGL/RKL, Can't be changed now for ABI reasons */
452 table->size = ARRAY_SIZE(tgl_mocs_table);
453 table->table = tgl_mocs_table;
454 table->n_entries = GEN9_NUM_MOCS_ENTRIES;
455 table->uc_index = 3;
456 } else if (GRAPHICS_VER(i915) >= 12) {
457 table->size = ARRAY_SIZE(gen12_mocs_table);
458 table->table = gen12_mocs_table;
459 table->n_entries = GEN9_NUM_MOCS_ENTRIES;
460 table->uc_index = 3;
461 table->unused_entries_index = 2;
462 } else if (GRAPHICS_VER(i915) == 11) {
463 table->size = ARRAY_SIZE(icl_mocs_table);
464 table->table = icl_mocs_table;
465 table->n_entries = GEN9_NUM_MOCS_ENTRIES;
466 } else if (IS_GEN9_BC(i915)) {
467 table->size = ARRAY_SIZE(skl_mocs_table);
468 table->n_entries = GEN9_NUM_MOCS_ENTRIES;
469 table->table = skl_mocs_table;
470 } else if (IS_GEN9_LP(i915)) {
471 table->size = ARRAY_SIZE(broxton_mocs_table);
472 table->n_entries = GEN9_NUM_MOCS_ENTRIES;
473 table->table = broxton_mocs_table;
474 } else {
475 drm_WARN_ONCE(&i915->drm, GRAPHICS_VER(i915) >= 9,
476 "Platform that should have a MOCS table does not.\n");
477 return 0;
478 }
479
480 if (GEM_DEBUG_WARN_ON(table->size > table->n_entries))
481 return 0;
482
483 /* WaDisableSkipCaching:skl,bxt,kbl,glk */
484 if (GRAPHICS_VER(i915) == 9) {
485 int i;
486
487 for (i = 0; i < table->size; i++)
488 if (GEM_DEBUG_WARN_ON(table->table[i].l3cc_value &
489 (L3_ESC(1) | L3_SCC(0x7))))
490 return 0;
491 }
492
493 flags = 0;
494 if (has_mocs(i915)) {
495 if (has_global_mocs(i915))
496 flags |= HAS_GLOBAL_MOCS;
497 else
498 flags |= HAS_ENGINE_MOCS;
499 }
500 if (has_l3cc(i915))
501 flags |= HAS_RENDER_L3CC;
502
503 return flags;
504 }
505
506 /*
507 * Get control_value from MOCS entry taking into account when it's not used
508 * then if unused_entries_index is non-zero then its value will be returned
509 * otherwise I915_MOCS_PTE's value is returned in this case.
510 */
get_entry_control(const struct drm_i915_mocs_table * table,unsigned int index)511 static u32 get_entry_control(const struct drm_i915_mocs_table *table,
512 unsigned int index)
513 {
514 if (index < table->size && table->table[index].used)
515 return table->table[index].control_value;
516 return table->table[table->unused_entries_index].control_value;
517 }
518
519 #define for_each_mocs(mocs, t, i) \
520 for (i = 0; \
521 i < (t)->n_entries ? (mocs = get_entry_control((t), i)), 1 : 0;\
522 i++)
523
__init_mocs_table(struct intel_uncore * uncore,const struct drm_i915_mocs_table * table,u32 addr)524 static void __init_mocs_table(struct intel_uncore *uncore,
525 const struct drm_i915_mocs_table *table,
526 u32 addr)
527 {
528 unsigned int i;
529 u32 mocs;
530
531 drm_WARN_ONCE(&uncore->i915->drm, !table->unused_entries_index,
532 "Unused entries index should have been defined\n");
533 for_each_mocs(mocs, table, i)
534 intel_uncore_write_fw(uncore, _MMIO(addr + i * 4), mocs);
535 }
536
mocs_offset(const struct intel_engine_cs * engine)537 static u32 mocs_offset(const struct intel_engine_cs *engine)
538 {
539 static const u32 offset[] = {
540 [RCS0] = __GEN9_RCS0_MOCS0,
541 [VCS0] = __GEN9_VCS0_MOCS0,
542 [VCS1] = __GEN9_VCS1_MOCS0,
543 [VECS0] = __GEN9_VECS0_MOCS0,
544 [BCS0] = __GEN9_BCS0_MOCS0,
545 [VCS2] = __GEN11_VCS2_MOCS0,
546 };
547
548 GEM_BUG_ON(engine->id >= ARRAY_SIZE(offset));
549 return offset[engine->id];
550 }
551
init_mocs_table(struct intel_engine_cs * engine,const struct drm_i915_mocs_table * table)552 static void init_mocs_table(struct intel_engine_cs *engine,
553 const struct drm_i915_mocs_table *table)
554 {
555 __init_mocs_table(engine->uncore, table, mocs_offset(engine));
556 }
557
558 /*
559 * Get l3cc_value from MOCS entry taking into account when it's not used
560 * then if unused_entries_index is not zero then its value will be returned
561 * otherwise I915_MOCS_PTE's value is returned in this case.
562 */
get_entry_l3cc(const struct drm_i915_mocs_table * table,unsigned int index)563 static u16 get_entry_l3cc(const struct drm_i915_mocs_table *table,
564 unsigned int index)
565 {
566 if (index < table->size && table->table[index].used)
567 return table->table[index].l3cc_value;
568 return table->table[table->unused_entries_index].l3cc_value;
569 }
570
l3cc_combine(u16 low,u16 high)571 static u32 l3cc_combine(u16 low, u16 high)
572 {
573 return low | (u32)high << 16;
574 }
575
576 #define for_each_l3cc(l3cc, t, i) \
577 for (i = 0; \
578 i < ((t)->n_entries + 1) / 2 ? \
579 (l3cc = l3cc_combine(get_entry_l3cc((t), 2 * i), \
580 get_entry_l3cc((t), 2 * i + 1))), 1 : \
581 0; \
582 i++)
583
init_l3cc_table(struct intel_uncore * uncore,const struct drm_i915_mocs_table * table)584 static void init_l3cc_table(struct intel_uncore *uncore,
585 const struct drm_i915_mocs_table *table)
586 {
587 unsigned int i;
588 u32 l3cc;
589
590 for_each_l3cc(l3cc, table, i)
591 intel_uncore_write_fw(uncore, GEN9_LNCFCMOCS(i), l3cc);
592 }
593
intel_mocs_init_engine(struct intel_engine_cs * engine)594 void intel_mocs_init_engine(struct intel_engine_cs *engine)
595 {
596 struct drm_i915_mocs_table table;
597 unsigned int flags;
598
599 /* Called under a blanket forcewake */
600 assert_forcewakes_active(engine->uncore, FORCEWAKE_ALL);
601
602 flags = get_mocs_settings(engine->i915, &table);
603 if (!flags)
604 return;
605
606 /* Platforms with global MOCS do not need per-engine initialization. */
607 if (flags & HAS_ENGINE_MOCS)
608 init_mocs_table(engine, &table);
609
610 if (flags & HAS_RENDER_L3CC && engine->class == RENDER_CLASS)
611 init_l3cc_table(engine->uncore, &table);
612 }
613
global_mocs_offset(void)614 static u32 global_mocs_offset(void)
615 {
616 return i915_mmio_reg_offset(GEN12_GLOBAL_MOCS(0));
617 }
618
intel_set_mocs_index(struct intel_gt * gt)619 void intel_set_mocs_index(struct intel_gt *gt)
620 {
621 struct drm_i915_mocs_table table;
622
623 get_mocs_settings(gt->i915, &table);
624 gt->mocs.uc_index = table.uc_index;
625 }
626
intel_mocs_init(struct intel_gt * gt)627 void intel_mocs_init(struct intel_gt *gt)
628 {
629 struct drm_i915_mocs_table table;
630 unsigned int flags;
631
632 /*
633 * LLC and eDRAM control values are not applicable to dgfx
634 */
635 flags = get_mocs_settings(gt->i915, &table);
636 if (flags & HAS_GLOBAL_MOCS)
637 __init_mocs_table(gt->uncore, &table, global_mocs_offset());
638
639 /*
640 * Initialize the L3CC table as part of mocs initalization to make
641 * sure the LNCFCMOCSx registers are programmed for the subsequent
642 * memory transactions including guc transactions
643 */
644 if (flags & HAS_RENDER_L3CC)
645 init_l3cc_table(gt->uncore, &table);
646 }
647
648 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
649 #include "selftest_mocs.c"
650 #endif
651