1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2022 Intel Corporation
4  */
5 
6 #include "i915_drv.h"
7 
8 #include "intel_gt_mcr.h"
9 #include "intel_gt_regs.h"
10 
11 /**
12  * DOC: GT Multicast/Replicated (MCR) Register Support
13  *
14  * Some GT registers are designed as "multicast" or "replicated" registers:
15  * multiple instances of the same register share a single MMIO offset.  MCR
16  * registers are generally used when the hardware needs to potentially track
17  * independent values of a register per hardware unit (e.g., per-subslice,
18  * per-L3bank, etc.).  The specific types of replication that exist vary
19  * per-platform.
20  *
21  * MMIO accesses to MCR registers are controlled according to the settings
22  * programmed in the platform's MCR_SELECTOR register(s).  MMIO writes to MCR
23  * registers can be done in either a (i.e., a single write updates all
24  * instances of the register to the same value) or unicast (a write updates only
25  * one specific instance).  Reads of MCR registers always operate in a unicast
26  * manner regardless of how the multicast/unicast bit is set in MCR_SELECTOR.
27  * Selection of a specific MCR instance for unicast operations is referred to
28  * as "steering."
29  *
30  * If MCR register operations are steered toward a hardware unit that is
31  * fused off or currently powered down due to power gating, the MMIO operation
32  * is "terminated" by the hardware.  Terminated read operations will return a
33  * value of zero and terminated unicast write operations will be silently
34  * ignored.
35  */
36 
37 #define HAS_MSLICE_STEERING(dev_priv)	(INTEL_INFO(dev_priv)->has_mslice_steering)
38 
39 static const char * const intel_steering_types[] = {
40 	"L3BANK",
41 	"MSLICE",
42 	"LNCF",
43 	"INSTANCE 0",
44 };
45 
46 static const struct intel_mmio_range icl_l3bank_steering_table[] = {
47 	{ 0x00B100, 0x00B3FF },
48 	{},
49 };
50 
51 static const struct intel_mmio_range xehpsdv_mslice_steering_table[] = {
52 	{ 0x004000, 0x004AFF },
53 	{ 0x00C800, 0x00CFFF },
54 	{ 0x00DD00, 0x00DDFF },
55 	{ 0x00E900, 0x00FFFF }, /* 0xEA00 - OxEFFF is unused */
56 	{},
57 };
58 
59 static const struct intel_mmio_range xehpsdv_lncf_steering_table[] = {
60 	{ 0x00B000, 0x00B0FF },
61 	{ 0x00D800, 0x00D8FF },
62 	{},
63 };
64 
65 static const struct intel_mmio_range dg2_lncf_steering_table[] = {
66 	{ 0x00B000, 0x00B0FF },
67 	{ 0x00D880, 0x00D8FF },
68 	{},
69 };
70 
71 /*
72  * We have several types of MCR registers on PVC where steering to (0,0)
73  * will always provide us with a non-terminated value.  We'll stick them
74  * all in the same table for simplicity.
75  */
76 static const struct intel_mmio_range pvc_instance0_steering_table[] = {
77 	{ 0x004000, 0x004AFF },		/* HALF-BSLICE */
78 	{ 0x008800, 0x00887F },		/* CC */
79 	{ 0x008A80, 0x008AFF },		/* TILEPSMI */
80 	{ 0x00B000, 0x00B0FF },		/* HALF-BSLICE */
81 	{ 0x00B100, 0x00B3FF },		/* L3BANK */
82 	{ 0x00C800, 0x00CFFF },		/* HALF-BSLICE */
83 	{ 0x00D800, 0x00D8FF },		/* HALF-BSLICE */
84 	{ 0x00DD00, 0x00DDFF },		/* BSLICE */
85 	{ 0x00E900, 0x00E9FF },		/* HALF-BSLICE */
86 	{ 0x00EC00, 0x00EEFF },		/* HALF-BSLICE */
87 	{ 0x00F000, 0x00FFFF },		/* HALF-BSLICE */
88 	{ 0x024180, 0x0241FF },		/* HALF-BSLICE */
89 	{},
90 };
91 
intel_gt_mcr_init(struct intel_gt * gt)92 void intel_gt_mcr_init(struct intel_gt *gt)
93 {
94 	struct drm_i915_private *i915 = gt->i915;
95 
96 	/*
97 	 * An mslice is unavailable only if both the meml3 for the slice is
98 	 * disabled *and* all of the DSS in the slice (quadrant) are disabled.
99 	 */
100 	if (HAS_MSLICE_STEERING(i915)) {
101 		gt->info.mslice_mask =
102 			intel_slicemask_from_xehp_dssmask(gt->info.sseu.subslice_mask,
103 							  GEN_DSS_PER_MSLICE);
104 		gt->info.mslice_mask |=
105 			(intel_uncore_read(gt->uncore, GEN10_MIRROR_FUSE3) &
106 			 GEN12_MEML3_EN_MASK);
107 
108 		if (!gt->info.mslice_mask) /* should be impossible! */
109 			drm_warn(&i915->drm, "mslice mask all zero!\n");
110 	}
111 
112 	if (IS_PONTEVECCHIO(i915)) {
113 		gt->steering_table[INSTANCE0] = pvc_instance0_steering_table;
114 	} else if (IS_DG2(i915)) {
115 		gt->steering_table[MSLICE] = xehpsdv_mslice_steering_table;
116 		gt->steering_table[LNCF] = dg2_lncf_steering_table;
117 	} else if (IS_XEHPSDV(i915)) {
118 		gt->steering_table[MSLICE] = xehpsdv_mslice_steering_table;
119 		gt->steering_table[LNCF] = xehpsdv_lncf_steering_table;
120 	} else if (GRAPHICS_VER(i915) >= 11 &&
121 		   GRAPHICS_VER_FULL(i915) < IP_VER(12, 50)) {
122 		gt->steering_table[L3BANK] = icl_l3bank_steering_table;
123 		gt->info.l3bank_mask =
124 			~intel_uncore_read(gt->uncore, GEN10_MIRROR_FUSE3) &
125 			GEN10_L3BANK_MASK;
126 		if (!gt->info.l3bank_mask) /* should be impossible! */
127 			drm_warn(&i915->drm, "L3 bank mask is all zero!\n");
128 	} else if (GRAPHICS_VER(i915) >= 11) {
129 		/*
130 		 * We expect all modern platforms to have at least some
131 		 * type of steering that needs to be initialized.
132 		 */
133 		MISSING_CASE(INTEL_INFO(i915)->platform);
134 	}
135 }
136 
137 /*
138  * rw_with_mcr_steering_fw - Access a register with specific MCR steering
139  * @uncore: pointer to struct intel_uncore
140  * @reg: register being accessed
141  * @rw_flag: FW_REG_READ for read access or FW_REG_WRITE for write access
142  * @group: group number (documented as "sliceid" on older platforms)
143  * @instance: instance number (documented as "subsliceid" on older platforms)
144  * @value: register value to be written (ignored for read)
145  *
146  * Return: 0 for write access. register value for read access.
147  *
148  * Caller needs to make sure the relevant forcewake wells are up.
149  */
rw_with_mcr_steering_fw(struct intel_uncore * uncore,i915_reg_t reg,u8 rw_flag,int group,int instance,u32 value)150 static u32 rw_with_mcr_steering_fw(struct intel_uncore *uncore,
151 				   i915_reg_t reg, u8 rw_flag,
152 				   int group, int instance, u32 value)
153 {
154 	u32 mcr_mask, mcr_ss, mcr, old_mcr, val = 0;
155 
156 	lockdep_assert_held(&uncore->lock);
157 
158 	if (GRAPHICS_VER(uncore->i915) >= 11) {
159 		mcr_mask = GEN11_MCR_SLICE_MASK | GEN11_MCR_SUBSLICE_MASK;
160 		mcr_ss = GEN11_MCR_SLICE(group) | GEN11_MCR_SUBSLICE(instance);
161 
162 		/*
163 		 * Wa_22013088509
164 		 *
165 		 * The setting of the multicast/unicast bit usually wouldn't
166 		 * matter for read operations (which always return the value
167 		 * from a single register instance regardless of how that bit
168 		 * is set), but some platforms have a workaround requiring us
169 		 * to remain in multicast mode for reads.  There's no real
170 		 * downside to this, so we'll just go ahead and do so on all
171 		 * platforms; we'll only clear the multicast bit from the mask
172 		 * when exlicitly doing a write operation.
173 		 */
174 		if (rw_flag == FW_REG_WRITE)
175 			mcr_mask |= GEN11_MCR_MULTICAST;
176 	} else {
177 		mcr_mask = GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK;
178 		mcr_ss = GEN8_MCR_SLICE(group) | GEN8_MCR_SUBSLICE(instance);
179 	}
180 
181 	old_mcr = mcr = intel_uncore_read_fw(uncore, GEN8_MCR_SELECTOR);
182 
183 	mcr &= ~mcr_mask;
184 	mcr |= mcr_ss;
185 	intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr);
186 
187 	if (rw_flag == FW_REG_READ)
188 		val = intel_uncore_read_fw(uncore, reg);
189 	else
190 		intel_uncore_write_fw(uncore, reg, value);
191 
192 	mcr &= ~mcr_mask;
193 	mcr |= old_mcr & mcr_mask;
194 
195 	intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr);
196 
197 	return val;
198 }
199 
rw_with_mcr_steering(struct intel_uncore * uncore,i915_reg_t reg,u8 rw_flag,int group,int instance,u32 value)200 static u32 rw_with_mcr_steering(struct intel_uncore *uncore,
201 				i915_reg_t reg, u8 rw_flag,
202 				int group, int instance,
203 				u32 value)
204 {
205 	enum forcewake_domains fw_domains;
206 	u32 val;
207 
208 	fw_domains = intel_uncore_forcewake_for_reg(uncore, reg,
209 						    rw_flag);
210 	fw_domains |= intel_uncore_forcewake_for_reg(uncore,
211 						     GEN8_MCR_SELECTOR,
212 						     FW_REG_READ | FW_REG_WRITE);
213 
214 	spin_lock_irq(&uncore->lock);
215 	intel_uncore_forcewake_get__locked(uncore, fw_domains);
216 
217 	val = rw_with_mcr_steering_fw(uncore, reg, rw_flag, group, instance, value);
218 
219 	intel_uncore_forcewake_put__locked(uncore, fw_domains);
220 	spin_unlock_irq(&uncore->lock);
221 
222 	return val;
223 }
224 
225 /**
226  * intel_gt_mcr_read - read a specific instance of an MCR register
227  * @gt: GT structure
228  * @reg: the MCR register to read
229  * @group: the MCR group
230  * @instance: the MCR instance
231  *
232  * Returns the value read from an MCR register after steering toward a specific
233  * group/instance.
234  */
intel_gt_mcr_read(struct intel_gt * gt,i915_reg_t reg,int group,int instance)235 u32 intel_gt_mcr_read(struct intel_gt *gt,
236 		      i915_reg_t reg,
237 		      int group, int instance)
238 {
239 	return rw_with_mcr_steering(gt->uncore, reg, FW_REG_READ, group, instance, 0);
240 }
241 
242 /**
243  * intel_gt_mcr_unicast_write - write a specific instance of an MCR register
244  * @gt: GT structure
245  * @reg: the MCR register to write
246  * @value: value to write
247  * @group: the MCR group
248  * @instance: the MCR instance
249  *
250  * Write an MCR register in unicast mode after steering toward a specific
251  * group/instance.
252  */
intel_gt_mcr_unicast_write(struct intel_gt * gt,i915_reg_t reg,u32 value,int group,int instance)253 void intel_gt_mcr_unicast_write(struct intel_gt *gt, i915_reg_t reg, u32 value,
254 				int group, int instance)
255 {
256 	rw_with_mcr_steering(gt->uncore, reg, FW_REG_WRITE, group, instance, value);
257 }
258 
259 /**
260  * intel_gt_mcr_multicast_write - write a value to all instances of an MCR register
261  * @gt: GT structure
262  * @reg: the MCR register to write
263  * @value: value to write
264  *
265  * Write an MCR register in multicast mode to update all instances.
266  */
intel_gt_mcr_multicast_write(struct intel_gt * gt,i915_reg_t reg,u32 value)267 void intel_gt_mcr_multicast_write(struct intel_gt *gt,
268 				i915_reg_t reg, u32 value)
269 {
270 	intel_uncore_write(gt->uncore, reg, value);
271 }
272 
273 /**
274  * intel_gt_mcr_multicast_write_fw - write a value to all instances of an MCR register
275  * @gt: GT structure
276  * @reg: the MCR register to write
277  * @value: value to write
278  *
279  * Write an MCR register in multicast mode to update all instances.  This
280  * function assumes the caller is already holding any necessary forcewake
281  * domains; use intel_gt_mcr_multicast_write() in cases where forcewake should
282  * be obtained automatically.
283  */
intel_gt_mcr_multicast_write_fw(struct intel_gt * gt,i915_reg_t reg,u32 value)284 void intel_gt_mcr_multicast_write_fw(struct intel_gt *gt, i915_reg_t reg, u32 value)
285 {
286 	intel_uncore_write_fw(gt->uncore, reg, value);
287 }
288 
289 /*
290  * reg_needs_read_steering - determine whether a register read requires
291  *     explicit steering
292  * @gt: GT structure
293  * @reg: the register to check steering requirements for
294  * @type: type of multicast steering to check
295  *
296  * Determines whether @reg needs explicit steering of a specific type for
297  * reads.
298  *
299  * Returns false if @reg does not belong to a register range of the given
300  * steering type, or if the default (subslice-based) steering IDs are suitable
301  * for @type steering too.
302  */
reg_needs_read_steering(struct intel_gt * gt,i915_reg_t reg,enum intel_steering_type type)303 static bool reg_needs_read_steering(struct intel_gt *gt,
304 				    i915_reg_t reg,
305 				    enum intel_steering_type type)
306 {
307 	const u32 offset = i915_mmio_reg_offset(reg);
308 	const struct intel_mmio_range *entry;
309 
310 	if (likely(!gt->steering_table[type]))
311 		return false;
312 
313 	for (entry = gt->steering_table[type]; entry->end; entry++) {
314 		if (offset >= entry->start && offset <= entry->end)
315 			return true;
316 	}
317 
318 	return false;
319 }
320 
321 /*
322  * get_nonterminated_steering - determines valid IDs for a class of MCR steering
323  * @gt: GT structure
324  * @type: multicast register type
325  * @group: Group ID returned
326  * @instance: Instance ID returned
327  *
328  * Determines group and instance values that will steer reads of the specified
329  * MCR class to a non-terminated instance.
330  */
get_nonterminated_steering(struct intel_gt * gt,enum intel_steering_type type,u8 * group,u8 * instance)331 static void get_nonterminated_steering(struct intel_gt *gt,
332 				       enum intel_steering_type type,
333 				       u8 *group, u8 *instance)
334 {
335 	switch (type) {
336 	case L3BANK:
337 		*group = 0;		/* unused */
338 		*instance = __ffs(gt->info.l3bank_mask);
339 		break;
340 	case MSLICE:
341 		GEM_WARN_ON(!HAS_MSLICE_STEERING(gt->i915));
342 		*group = __ffs(gt->info.mslice_mask);
343 		*instance = 0;	/* unused */
344 		break;
345 	case LNCF:
346 		/*
347 		 * An LNCF is always present if its mslice is present, so we
348 		 * can safely just steer to LNCF 0 in all cases.
349 		 */
350 		GEM_WARN_ON(!HAS_MSLICE_STEERING(gt->i915));
351 		*group = __ffs(gt->info.mslice_mask) << 1;
352 		*instance = 0;	/* unused */
353 		break;
354 	case INSTANCE0:
355 		/*
356 		 * There are a lot of MCR types for which instance (0, 0)
357 		 * will always provide a non-terminated value.
358 		 */
359 		*group = 0;
360 		*instance = 0;
361 		break;
362 	default:
363 		MISSING_CASE(type);
364 		*group = 0;
365 		*instance = 0;
366 	}
367 }
368 
369 /**
370  * intel_gt_mcr_get_nonterminated_steering - find group/instance values that
371  *    will steer a register to a non-terminated instance
372  * @gt: GT structure
373  * @reg: register for which the steering is required
374  * @group: return variable for group steering
375  * @instance: return variable for instance steering
376  *
377  * This function returns a group/instance pair that is guaranteed to work for
378  * read steering of the given register. Note that a value will be returned even
379  * if the register is not replicated and therefore does not actually require
380  * steering.
381  */
intel_gt_mcr_get_nonterminated_steering(struct intel_gt * gt,i915_reg_t reg,u8 * group,u8 * instance)382 void intel_gt_mcr_get_nonterminated_steering(struct intel_gt *gt,
383 					     i915_reg_t reg,
384 					     u8 *group, u8 *instance)
385 {
386 	int type;
387 
388 	for (type = 0; type < NUM_STEERING_TYPES; type++) {
389 		if (reg_needs_read_steering(gt, reg, type)) {
390 			get_nonterminated_steering(gt, type, group, instance);
391 			return;
392 		}
393 	}
394 
395 	*group = gt->default_steering.groupid;
396 	*instance = gt->default_steering.instanceid;
397 }
398 
399 /**
400  * intel_gt_mcr_read_any_fw - reads one instance of an MCR register
401  * @gt: GT structure
402  * @reg: register to read
403  *
404  * Reads a GT MCR register.  The read will be steered to a non-terminated
405  * instance (i.e., one that isn't fused off or powered down by power gating).
406  * This function assumes the caller is already holding any necessary forcewake
407  * domains; use intel_gt_mcr_read_any() in cases where forcewake should be
408  * obtained automatically.
409  *
410  * Returns the value from a non-terminated instance of @reg.
411  */
intel_gt_mcr_read_any_fw(struct intel_gt * gt,i915_reg_t reg)412 u32 intel_gt_mcr_read_any_fw(struct intel_gt *gt, i915_reg_t reg)
413 {
414 	int type;
415 	u8 group, instance;
416 
417 	for (type = 0; type < NUM_STEERING_TYPES; type++) {
418 		if (reg_needs_read_steering(gt, reg, type)) {
419 			get_nonterminated_steering(gt, type, &group, &instance);
420 			return rw_with_mcr_steering_fw(gt->uncore, reg,
421 						       FW_REG_READ,
422 						       group, instance, 0);
423 		}
424 	}
425 
426 	return intel_uncore_read_fw(gt->uncore, reg);
427 }
428 
429 /**
430  * intel_gt_mcr_read_any - reads one instance of an MCR register
431  * @gt: GT structure
432  * @reg: register to read
433  *
434  * Reads a GT MCR register.  The read will be steered to a non-terminated
435  * instance (i.e., one that isn't fused off or powered down by power gating).
436  *
437  * Returns the value from a non-terminated instance of @reg.
438  */
intel_gt_mcr_read_any(struct intel_gt * gt,i915_reg_t reg)439 u32 intel_gt_mcr_read_any(struct intel_gt *gt, i915_reg_t reg)
440 {
441 	int type;
442 	u8 group, instance;
443 
444 	for (type = 0; type < NUM_STEERING_TYPES; type++) {
445 		if (reg_needs_read_steering(gt, reg, type)) {
446 			get_nonterminated_steering(gt, type, &group, &instance);
447 			return rw_with_mcr_steering(gt->uncore, reg,
448 						    FW_REG_READ,
449 						    group, instance, 0);
450 		}
451 	}
452 
453 	return intel_uncore_read(gt->uncore, reg);
454 }
455 
report_steering_type(struct drm_printer * p,struct intel_gt * gt,enum intel_steering_type type,bool dump_table)456 static void report_steering_type(struct drm_printer *p,
457 				 struct intel_gt *gt,
458 				 enum intel_steering_type type,
459 				 bool dump_table)
460 {
461 	const struct intel_mmio_range *entry;
462 	u8 group, instance;
463 
464 	BUILD_BUG_ON(ARRAY_SIZE(intel_steering_types) != NUM_STEERING_TYPES);
465 
466 	if (!gt->steering_table[type]) {
467 		drm_printf(p, "%s steering: uses default steering\n",
468 			   intel_steering_types[type]);
469 		return;
470 	}
471 
472 	get_nonterminated_steering(gt, type, &group, &instance);
473 	drm_printf(p, "%s steering: group=0x%x, instance=0x%x\n",
474 		   intel_steering_types[type], group, instance);
475 
476 	if (!dump_table)
477 		return;
478 
479 	for (entry = gt->steering_table[type]; entry->end; entry++)
480 		drm_printf(p, "\t0x%06x - 0x%06x\n", entry->start, entry->end);
481 }
482 
intel_gt_mcr_report_steering(struct drm_printer * p,struct intel_gt * gt,bool dump_table)483 void intel_gt_mcr_report_steering(struct drm_printer *p, struct intel_gt *gt,
484 				  bool dump_table)
485 {
486 	drm_printf(p, "Default steering: group=0x%x, instance=0x%x\n",
487 		   gt->default_steering.groupid,
488 		   gt->default_steering.instanceid);
489 
490 	if (IS_PONTEVECCHIO(gt->i915)) {
491 		report_steering_type(p, gt, INSTANCE0, dump_table);
492 	} else if (HAS_MSLICE_STEERING(gt->i915)) {
493 		report_steering_type(p, gt, MSLICE, dump_table);
494 		report_steering_type(p, gt, LNCF, dump_table);
495 	}
496 }
497 
498 /**
499  * intel_gt_mcr_get_ss_steering - returns the group/instance steering for a SS
500  * @gt: GT structure
501  * @dss: DSS ID to obtain steering for
502  * @group: pointer to storage for steering group ID
503  * @instance: pointer to storage for steering instance ID
504  *
505  * Returns the steering IDs (via the @group and @instance parameters) that
506  * correspond to a specific subslice/DSS ID.
507  */
intel_gt_mcr_get_ss_steering(struct intel_gt * gt,unsigned int dss,unsigned int * group,unsigned int * instance)508 void intel_gt_mcr_get_ss_steering(struct intel_gt *gt, unsigned int dss,
509 				   unsigned int *group, unsigned int *instance)
510 {
511 	if (IS_PONTEVECCHIO(gt->i915)) {
512 		*group = dss / GEN_DSS_PER_CSLICE;
513 		*instance = dss % GEN_DSS_PER_CSLICE;
514 	} else if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 50)) {
515 		*group = dss / GEN_DSS_PER_GSLICE;
516 		*instance = dss % GEN_DSS_PER_GSLICE;
517 	} else {
518 		*group = dss / GEN_MAX_SS_PER_HSW_SLICE;
519 		*instance = dss % GEN_MAX_SS_PER_HSW_SLICE;
520 		return;
521 	}
522 }
523