1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2019 Intel Corporation
4  */
5 
6 #include <linux/string_helpers.h>
7 
8 #include "i915_drv.h"
9 #include "i915_perf_types.h"
10 #include "intel_engine_regs.h"
11 #include "intel_gt_regs.h"
12 #include "intel_sseu.h"
13 
intel_sseu_set_info(struct sseu_dev_info * sseu,u8 max_slices,u8 max_subslices,u8 max_eus_per_subslice)14 void intel_sseu_set_info(struct sseu_dev_info *sseu, u8 max_slices,
15 			 u8 max_subslices, u8 max_eus_per_subslice)
16 {
17 	sseu->max_slices = max_slices;
18 	sseu->max_subslices = max_subslices;
19 	sseu->max_eus_per_subslice = max_eus_per_subslice;
20 }
21 
22 unsigned int
intel_sseu_subslice_total(const struct sseu_dev_info * sseu)23 intel_sseu_subslice_total(const struct sseu_dev_info *sseu)
24 {
25 	unsigned int i, total = 0;
26 
27 	if (sseu->has_xehp_dss)
28 		return bitmap_weight(sseu->subslice_mask.xehp,
29 				     XEHP_BITMAP_BITS(sseu->subslice_mask));
30 
31 	for (i = 0; i < ARRAY_SIZE(sseu->subslice_mask.hsw); i++)
32 		total += hweight8(sseu->subslice_mask.hsw[i]);
33 
34 	return total;
35 }
36 
37 unsigned int
intel_sseu_get_hsw_subslices(const struct sseu_dev_info * sseu,u8 slice)38 intel_sseu_get_hsw_subslices(const struct sseu_dev_info *sseu, u8 slice)
39 {
40 	WARN_ON(sseu->has_xehp_dss);
41 	if (WARN_ON(slice >= sseu->max_slices))
42 		return 0;
43 
44 	return sseu->subslice_mask.hsw[slice];
45 }
46 
sseu_get_eus(const struct sseu_dev_info * sseu,int slice,int subslice)47 static u16 sseu_get_eus(const struct sseu_dev_info *sseu, int slice,
48 			int subslice)
49 {
50 	if (sseu->has_xehp_dss) {
51 		WARN_ON(slice > 0);
52 		return sseu->eu_mask.xehp[subslice];
53 	} else {
54 		return sseu->eu_mask.hsw[slice][subslice];
55 	}
56 }
57 
sseu_set_eus(struct sseu_dev_info * sseu,int slice,int subslice,u16 eu_mask)58 static void sseu_set_eus(struct sseu_dev_info *sseu, int slice, int subslice,
59 			 u16 eu_mask)
60 {
61 	GEM_WARN_ON(eu_mask && __fls(eu_mask) >= sseu->max_eus_per_subslice);
62 	if (sseu->has_xehp_dss) {
63 		GEM_WARN_ON(slice > 0);
64 		sseu->eu_mask.xehp[subslice] = eu_mask;
65 	} else {
66 		sseu->eu_mask.hsw[slice][subslice] = eu_mask;
67 	}
68 }
69 
compute_eu_total(const struct sseu_dev_info * sseu)70 static u16 compute_eu_total(const struct sseu_dev_info *sseu)
71 {
72 	int s, ss, total = 0;
73 
74 	for (s = 0; s < sseu->max_slices; s++)
75 		for (ss = 0; ss < sseu->max_subslices; ss++)
76 			if (sseu->has_xehp_dss)
77 				total += hweight16(sseu->eu_mask.xehp[ss]);
78 			else
79 				total += hweight16(sseu->eu_mask.hsw[s][ss]);
80 
81 	return total;
82 }
83 
84 /**
85  * intel_sseu_copy_eumask_to_user - Copy EU mask into a userspace buffer
86  * @to: Pointer to userspace buffer to copy to
87  * @sseu: SSEU structure containing EU mask to copy
88  *
89  * Copies the EU mask to a userspace buffer in the format expected by
90  * the query ioctl's topology queries.
91  *
92  * Returns the result of the copy_to_user() operation.
93  */
intel_sseu_copy_eumask_to_user(void __user * to,const struct sseu_dev_info * sseu)94 int intel_sseu_copy_eumask_to_user(void __user *to,
95 				   const struct sseu_dev_info *sseu)
96 {
97 	u8 eu_mask[GEN_SS_MASK_SIZE * GEN_MAX_EU_STRIDE] = {};
98 	int eu_stride = GEN_SSEU_STRIDE(sseu->max_eus_per_subslice);
99 	int len = sseu->max_slices * sseu->max_subslices * eu_stride;
100 	int s, ss, i;
101 
102 	for (s = 0; s < sseu->max_slices; s++) {
103 		for (ss = 0; ss < sseu->max_subslices; ss++) {
104 			int uapi_offset =
105 				s * sseu->max_subslices * eu_stride +
106 				ss * eu_stride;
107 			u16 mask = sseu_get_eus(sseu, s, ss);
108 
109 			for (i = 0; i < eu_stride; i++)
110 				eu_mask[uapi_offset + i] =
111 					(mask >> (BITS_PER_BYTE * i)) & 0xff;
112 		}
113 	}
114 
115 	return copy_to_user(to, eu_mask, len);
116 }
117 
118 /**
119  * intel_sseu_copy_ssmask_to_user - Copy subslice mask into a userspace buffer
120  * @to: Pointer to userspace buffer to copy to
121  * @sseu: SSEU structure containing subslice mask to copy
122  *
123  * Copies the subslice mask to a userspace buffer in the format expected by
124  * the query ioctl's topology queries.
125  *
126  * Returns the result of the copy_to_user() operation.
127  */
intel_sseu_copy_ssmask_to_user(void __user * to,const struct sseu_dev_info * sseu)128 int intel_sseu_copy_ssmask_to_user(void __user *to,
129 				   const struct sseu_dev_info *sseu)
130 {
131 	u8 ss_mask[GEN_SS_MASK_SIZE] = {};
132 	int ss_stride = GEN_SSEU_STRIDE(sseu->max_subslices);
133 	int len = sseu->max_slices * ss_stride;
134 	int s, ss, i;
135 
136 	for (s = 0; s < sseu->max_slices; s++) {
137 		for (ss = 0; ss < sseu->max_subslices; ss++) {
138 			i = s * ss_stride * BITS_PER_BYTE + ss;
139 
140 			if (!intel_sseu_has_subslice(sseu, s, ss))
141 				continue;
142 
143 			ss_mask[i / BITS_PER_BYTE] |= BIT(i % BITS_PER_BYTE);
144 		}
145 	}
146 
147 	return copy_to_user(to, ss_mask, len);
148 }
149 
gen11_compute_sseu_info(struct sseu_dev_info * sseu,u32 ss_en,u16 eu_en)150 static void gen11_compute_sseu_info(struct sseu_dev_info *sseu,
151 				    u32 ss_en, u16 eu_en)
152 {
153 	u32 valid_ss_mask = GENMASK(sseu->max_subslices - 1, 0);
154 	int ss;
155 
156 	sseu->slice_mask |= BIT(0);
157 	sseu->subslice_mask.hsw[0] = ss_en & valid_ss_mask;
158 
159 	for (ss = 0; ss < sseu->max_subslices; ss++)
160 		if (intel_sseu_has_subslice(sseu, 0, ss))
161 			sseu_set_eus(sseu, 0, ss, eu_en);
162 
163 	sseu->eu_per_subslice = hweight16(eu_en);
164 	sseu->eu_total = compute_eu_total(sseu);
165 }
166 
xehp_compute_sseu_info(struct sseu_dev_info * sseu,u16 eu_en)167 static void xehp_compute_sseu_info(struct sseu_dev_info *sseu,
168 				   u16 eu_en)
169 {
170 	int ss;
171 
172 	sseu->slice_mask |= BIT(0);
173 
174 	bitmap_or(sseu->subslice_mask.xehp,
175 		  sseu->compute_subslice_mask.xehp,
176 		  sseu->geometry_subslice_mask.xehp,
177 		  XEHP_BITMAP_BITS(sseu->subslice_mask));
178 
179 	for (ss = 0; ss < sseu->max_subslices; ss++)
180 		if (intel_sseu_has_subslice(sseu, 0, ss))
181 			sseu_set_eus(sseu, 0, ss, eu_en);
182 
183 	sseu->eu_per_subslice = hweight16(eu_en);
184 	sseu->eu_total = compute_eu_total(sseu);
185 }
186 
187 static void
xehp_load_dss_mask(struct intel_uncore * uncore,intel_sseu_ss_mask_t * ssmask,int numregs,...)188 xehp_load_dss_mask(struct intel_uncore *uncore,
189 		   intel_sseu_ss_mask_t *ssmask,
190 		   int numregs,
191 		   ...)
192 {
193 	va_list argp;
194 	u32 fuse_val[I915_MAX_SS_FUSE_REGS] = {};
195 	int i;
196 
197 	if (WARN_ON(numregs > I915_MAX_SS_FUSE_REGS))
198 		numregs = I915_MAX_SS_FUSE_REGS;
199 
200 	va_start(argp, numregs);
201 	for (i = 0; i < numregs; i++)
202 		fuse_val[i] = intel_uncore_read(uncore, va_arg(argp, i915_reg_t));
203 	va_end(argp);
204 
205 	bitmap_from_arr32(ssmask->xehp, fuse_val, numregs * 32);
206 }
207 
xehp_sseu_info_init(struct intel_gt * gt)208 static void xehp_sseu_info_init(struct intel_gt *gt)
209 {
210 	struct sseu_dev_info *sseu = &gt->info.sseu;
211 	struct intel_uncore *uncore = gt->uncore;
212 	u16 eu_en = 0;
213 	u8 eu_en_fuse;
214 	int num_compute_regs, num_geometry_regs;
215 	int eu;
216 
217 	if (IS_PONTEVECCHIO(gt->i915)) {
218 		num_geometry_regs = 0;
219 		num_compute_regs = 2;
220 	} else {
221 		num_geometry_regs = 1;
222 		num_compute_regs = 1;
223 	}
224 
225 	/*
226 	 * The concept of slice has been removed in Xe_HP.  To be compatible
227 	 * with prior generations, assume a single slice across the entire
228 	 * device. Then calculate out the DSS for each workload type within
229 	 * that software slice.
230 	 */
231 	intel_sseu_set_info(sseu, 1,
232 			    32 * max(num_geometry_regs, num_compute_regs),
233 			    HAS_ONE_EU_PER_FUSE_BIT(gt->i915) ? 8 : 16);
234 	sseu->has_xehp_dss = 1;
235 
236 	xehp_load_dss_mask(uncore, &sseu->geometry_subslice_mask,
237 			   num_geometry_regs,
238 			   GEN12_GT_GEOMETRY_DSS_ENABLE);
239 	xehp_load_dss_mask(uncore, &sseu->compute_subslice_mask,
240 			   num_compute_regs,
241 			   GEN12_GT_COMPUTE_DSS_ENABLE,
242 			   XEHPC_GT_COMPUTE_DSS_ENABLE_EXT);
243 
244 	eu_en_fuse = intel_uncore_read(uncore, XEHP_EU_ENABLE) & XEHP_EU_ENA_MASK;
245 
246 	if (HAS_ONE_EU_PER_FUSE_BIT(gt->i915))
247 		eu_en = eu_en_fuse;
248 	else
249 		for (eu = 0; eu < sseu->max_eus_per_subslice / 2; eu++)
250 			if (eu_en_fuse & BIT(eu))
251 				eu_en |= BIT(eu * 2) | BIT(eu * 2 + 1);
252 
253 	xehp_compute_sseu_info(sseu, eu_en);
254 }
255 
gen12_sseu_info_init(struct intel_gt * gt)256 static void gen12_sseu_info_init(struct intel_gt *gt)
257 {
258 	struct sseu_dev_info *sseu = &gt->info.sseu;
259 	struct intel_uncore *uncore = gt->uncore;
260 	u32 g_dss_en;
261 	u16 eu_en = 0;
262 	u8 eu_en_fuse;
263 	u8 s_en;
264 	int eu;
265 
266 	/*
267 	 * Gen12 has Dual-Subslices, which behave similarly to 2 gen11 SS.
268 	 * Instead of splitting these, provide userspace with an array
269 	 * of DSS to more closely represent the hardware resource.
270 	 */
271 	intel_sseu_set_info(sseu, 1, 6, 16);
272 
273 	/*
274 	 * Although gen12 architecture supported multiple slices, TGL, RKL,
275 	 * DG1, and ADL only had a single slice.
276 	 */
277 	s_en = intel_uncore_read(uncore, GEN11_GT_SLICE_ENABLE) &
278 		GEN11_GT_S_ENA_MASK;
279 	drm_WARN_ON(&gt->i915->drm, s_en != 0x1);
280 
281 	g_dss_en = intel_uncore_read(uncore, GEN12_GT_GEOMETRY_DSS_ENABLE);
282 
283 	/* one bit per pair of EUs */
284 	eu_en_fuse = ~(intel_uncore_read(uncore, GEN11_EU_DISABLE) &
285 		       GEN11_EU_DIS_MASK);
286 
287 	for (eu = 0; eu < sseu->max_eus_per_subslice / 2; eu++)
288 		if (eu_en_fuse & BIT(eu))
289 			eu_en |= BIT(eu * 2) | BIT(eu * 2 + 1);
290 
291 	gen11_compute_sseu_info(sseu, g_dss_en, eu_en);
292 
293 	/* TGL only supports slice-level power gating */
294 	sseu->has_slice_pg = 1;
295 }
296 
gen11_sseu_info_init(struct intel_gt * gt)297 static void gen11_sseu_info_init(struct intel_gt *gt)
298 {
299 	struct sseu_dev_info *sseu = &gt->info.sseu;
300 	struct intel_uncore *uncore = gt->uncore;
301 	u32 ss_en;
302 	u8 eu_en;
303 	u8 s_en;
304 
305 	if (IS_JASPERLAKE(gt->i915) || IS_ELKHARTLAKE(gt->i915))
306 		intel_sseu_set_info(sseu, 1, 4, 8);
307 	else
308 		intel_sseu_set_info(sseu, 1, 8, 8);
309 
310 	/*
311 	 * Although gen11 architecture supported multiple slices, ICL and
312 	 * EHL/JSL only had a single slice in practice.
313 	 */
314 	s_en = intel_uncore_read(uncore, GEN11_GT_SLICE_ENABLE) &
315 		GEN11_GT_S_ENA_MASK;
316 	drm_WARN_ON(&gt->i915->drm, s_en != 0x1);
317 
318 	ss_en = ~intel_uncore_read(uncore, GEN11_GT_SUBSLICE_DISABLE);
319 
320 	eu_en = ~(intel_uncore_read(uncore, GEN11_EU_DISABLE) &
321 		  GEN11_EU_DIS_MASK);
322 
323 	gen11_compute_sseu_info(sseu, ss_en, eu_en);
324 
325 	/* ICL has no power gating restrictions. */
326 	sseu->has_slice_pg = 1;
327 	sseu->has_subslice_pg = 1;
328 	sseu->has_eu_pg = 1;
329 }
330 
cherryview_sseu_info_init(struct intel_gt * gt)331 static void cherryview_sseu_info_init(struct intel_gt *gt)
332 {
333 	struct sseu_dev_info *sseu = &gt->info.sseu;
334 	u32 fuse;
335 
336 	fuse = intel_uncore_read(gt->uncore, CHV_FUSE_GT);
337 
338 	sseu->slice_mask = BIT(0);
339 	intel_sseu_set_info(sseu, 1, 2, 8);
340 
341 	if (!(fuse & CHV_FGT_DISABLE_SS0)) {
342 		u8 disabled_mask =
343 			((fuse & CHV_FGT_EU_DIS_SS0_R0_MASK) >>
344 			 CHV_FGT_EU_DIS_SS0_R0_SHIFT) |
345 			(((fuse & CHV_FGT_EU_DIS_SS0_R1_MASK) >>
346 			  CHV_FGT_EU_DIS_SS0_R1_SHIFT) << 4);
347 
348 		sseu->subslice_mask.hsw[0] |= BIT(0);
349 		sseu_set_eus(sseu, 0, 0, ~disabled_mask & 0xFF);
350 	}
351 
352 	if (!(fuse & CHV_FGT_DISABLE_SS1)) {
353 		u8 disabled_mask =
354 			((fuse & CHV_FGT_EU_DIS_SS1_R0_MASK) >>
355 			 CHV_FGT_EU_DIS_SS1_R0_SHIFT) |
356 			(((fuse & CHV_FGT_EU_DIS_SS1_R1_MASK) >>
357 			  CHV_FGT_EU_DIS_SS1_R1_SHIFT) << 4);
358 
359 		sseu->subslice_mask.hsw[0] |= BIT(1);
360 		sseu_set_eus(sseu, 0, 1, ~disabled_mask & 0xFF);
361 	}
362 
363 	sseu->eu_total = compute_eu_total(sseu);
364 
365 	/*
366 	 * CHV expected to always have a uniform distribution of EU
367 	 * across subslices.
368 	 */
369 	sseu->eu_per_subslice = intel_sseu_subslice_total(sseu) ?
370 		sseu->eu_total /
371 		intel_sseu_subslice_total(sseu) :
372 		0;
373 	/*
374 	 * CHV supports subslice power gating on devices with more than
375 	 * one subslice, and supports EU power gating on devices with
376 	 * more than one EU pair per subslice.
377 	 */
378 	sseu->has_slice_pg = 0;
379 	sseu->has_subslice_pg = intel_sseu_subslice_total(sseu) > 1;
380 	sseu->has_eu_pg = (sseu->eu_per_subslice > 2);
381 }
382 
gen9_sseu_info_init(struct intel_gt * gt)383 static void gen9_sseu_info_init(struct intel_gt *gt)
384 {
385 	struct drm_i915_private *i915 = gt->i915;
386 	struct sseu_dev_info *sseu = &gt->info.sseu;
387 	struct intel_uncore *uncore = gt->uncore;
388 	u32 fuse2, eu_disable, subslice_mask;
389 	const u8 eu_mask = 0xff;
390 	int s, ss;
391 
392 	fuse2 = intel_uncore_read(uncore, GEN8_FUSE2);
393 	sseu->slice_mask = (fuse2 & GEN8_F2_S_ENA_MASK) >> GEN8_F2_S_ENA_SHIFT;
394 
395 	/* BXT has a single slice and at most 3 subslices. */
396 	intel_sseu_set_info(sseu, IS_GEN9_LP(i915) ? 1 : 3,
397 			    IS_GEN9_LP(i915) ? 3 : 4, 8);
398 
399 	/*
400 	 * The subslice disable field is global, i.e. it applies
401 	 * to each of the enabled slices.
402 	 */
403 	subslice_mask = (1 << sseu->max_subslices) - 1;
404 	subslice_mask &= ~((fuse2 & GEN9_F2_SS_DIS_MASK) >>
405 			   GEN9_F2_SS_DIS_SHIFT);
406 
407 	/*
408 	 * Iterate through enabled slices and subslices to
409 	 * count the total enabled EU.
410 	 */
411 	for (s = 0; s < sseu->max_slices; s++) {
412 		if (!(sseu->slice_mask & BIT(s)))
413 			/* skip disabled slice */
414 			continue;
415 
416 		sseu->subslice_mask.hsw[s] = subslice_mask;
417 
418 		eu_disable = intel_uncore_read(uncore, GEN9_EU_DISABLE(s));
419 		for (ss = 0; ss < sseu->max_subslices; ss++) {
420 			int eu_per_ss;
421 			u8 eu_disabled_mask;
422 
423 			if (!intel_sseu_has_subslice(sseu, s, ss))
424 				/* skip disabled subslice */
425 				continue;
426 
427 			eu_disabled_mask = (eu_disable >> (ss * 8)) & eu_mask;
428 
429 			sseu_set_eus(sseu, s, ss, ~eu_disabled_mask & eu_mask);
430 
431 			eu_per_ss = sseu->max_eus_per_subslice -
432 				hweight8(eu_disabled_mask);
433 
434 			/*
435 			 * Record which subslice(s) has(have) 7 EUs. we
436 			 * can tune the hash used to spread work among
437 			 * subslices if they are unbalanced.
438 			 */
439 			if (eu_per_ss == 7)
440 				sseu->subslice_7eu[s] |= BIT(ss);
441 		}
442 	}
443 
444 	sseu->eu_total = compute_eu_total(sseu);
445 
446 	/*
447 	 * SKL is expected to always have a uniform distribution
448 	 * of EU across subslices with the exception that any one
449 	 * EU in any one subslice may be fused off for die
450 	 * recovery. BXT is expected to be perfectly uniform in EU
451 	 * distribution.
452 	 */
453 	sseu->eu_per_subslice =
454 		intel_sseu_subslice_total(sseu) ?
455 		DIV_ROUND_UP(sseu->eu_total, intel_sseu_subslice_total(sseu)) :
456 		0;
457 
458 	/*
459 	 * SKL+ supports slice power gating on devices with more than
460 	 * one slice, and supports EU power gating on devices with
461 	 * more than one EU pair per subslice. BXT+ supports subslice
462 	 * power gating on devices with more than one subslice, and
463 	 * supports EU power gating on devices with more than one EU
464 	 * pair per subslice.
465 	 */
466 	sseu->has_slice_pg =
467 		!IS_GEN9_LP(i915) && hweight8(sseu->slice_mask) > 1;
468 	sseu->has_subslice_pg =
469 		IS_GEN9_LP(i915) && intel_sseu_subslice_total(sseu) > 1;
470 	sseu->has_eu_pg = sseu->eu_per_subslice > 2;
471 
472 	if (IS_GEN9_LP(i915)) {
473 #define IS_SS_DISABLED(ss)	(!(sseu->subslice_mask.hsw[0] & BIT(ss)))
474 		RUNTIME_INFO(i915)->has_pooled_eu = hweight8(sseu->subslice_mask.hsw[0]) == 3;
475 
476 		sseu->min_eu_in_pool = 0;
477 		if (HAS_POOLED_EU(i915)) {
478 			if (IS_SS_DISABLED(2) || IS_SS_DISABLED(0))
479 				sseu->min_eu_in_pool = 3;
480 			else if (IS_SS_DISABLED(1))
481 				sseu->min_eu_in_pool = 6;
482 			else
483 				sseu->min_eu_in_pool = 9;
484 		}
485 #undef IS_SS_DISABLED
486 	}
487 }
488 
bdw_sseu_info_init(struct intel_gt * gt)489 static void bdw_sseu_info_init(struct intel_gt *gt)
490 {
491 	struct sseu_dev_info *sseu = &gt->info.sseu;
492 	struct intel_uncore *uncore = gt->uncore;
493 	int s, ss;
494 	u32 fuse2, subslice_mask, eu_disable[3]; /* s_max */
495 	u32 eu_disable0, eu_disable1, eu_disable2;
496 
497 	fuse2 = intel_uncore_read(uncore, GEN8_FUSE2);
498 	sseu->slice_mask = (fuse2 & GEN8_F2_S_ENA_MASK) >> GEN8_F2_S_ENA_SHIFT;
499 	intel_sseu_set_info(sseu, 3, 3, 8);
500 
501 	/*
502 	 * The subslice disable field is global, i.e. it applies
503 	 * to each of the enabled slices.
504 	 */
505 	subslice_mask = GENMASK(sseu->max_subslices - 1, 0);
506 	subslice_mask &= ~((fuse2 & GEN8_F2_SS_DIS_MASK) >>
507 			   GEN8_F2_SS_DIS_SHIFT);
508 	eu_disable0 = intel_uncore_read(uncore, GEN8_EU_DISABLE0);
509 	eu_disable1 = intel_uncore_read(uncore, GEN8_EU_DISABLE1);
510 	eu_disable2 = intel_uncore_read(uncore, GEN8_EU_DISABLE2);
511 	eu_disable[0] = eu_disable0 & GEN8_EU_DIS0_S0_MASK;
512 	eu_disable[1] = (eu_disable0 >> GEN8_EU_DIS0_S1_SHIFT) |
513 		((eu_disable1 & GEN8_EU_DIS1_S1_MASK) <<
514 		 (32 - GEN8_EU_DIS0_S1_SHIFT));
515 	eu_disable[2] = (eu_disable1 >> GEN8_EU_DIS1_S2_SHIFT) |
516 		((eu_disable2 & GEN8_EU_DIS2_S2_MASK) <<
517 		 (32 - GEN8_EU_DIS1_S2_SHIFT));
518 
519 	/*
520 	 * Iterate through enabled slices and subslices to
521 	 * count the total enabled EU.
522 	 */
523 	for (s = 0; s < sseu->max_slices; s++) {
524 		if (!(sseu->slice_mask & BIT(s)))
525 			/* skip disabled slice */
526 			continue;
527 
528 		sseu->subslice_mask.hsw[s] = subslice_mask;
529 
530 		for (ss = 0; ss < sseu->max_subslices; ss++) {
531 			u8 eu_disabled_mask;
532 			u32 n_disabled;
533 
534 			if (!intel_sseu_has_subslice(sseu, s, ss))
535 				/* skip disabled subslice */
536 				continue;
537 
538 			eu_disabled_mask =
539 				eu_disable[s] >> (ss * sseu->max_eus_per_subslice);
540 
541 			sseu_set_eus(sseu, s, ss, ~eu_disabled_mask & 0xFF);
542 
543 			n_disabled = hweight8(eu_disabled_mask);
544 
545 			/*
546 			 * Record which subslices have 7 EUs.
547 			 */
548 			if (sseu->max_eus_per_subslice - n_disabled == 7)
549 				sseu->subslice_7eu[s] |= 1 << ss;
550 		}
551 	}
552 
553 	sseu->eu_total = compute_eu_total(sseu);
554 
555 	/*
556 	 * BDW is expected to always have a uniform distribution of EU across
557 	 * subslices with the exception that any one EU in any one subslice may
558 	 * be fused off for die recovery.
559 	 */
560 	sseu->eu_per_subslice =
561 		intel_sseu_subslice_total(sseu) ?
562 		DIV_ROUND_UP(sseu->eu_total, intel_sseu_subslice_total(sseu)) :
563 		0;
564 
565 	/*
566 	 * BDW supports slice power gating on devices with more than
567 	 * one slice.
568 	 */
569 	sseu->has_slice_pg = hweight8(sseu->slice_mask) > 1;
570 	sseu->has_subslice_pg = 0;
571 	sseu->has_eu_pg = 0;
572 }
573 
hsw_sseu_info_init(struct intel_gt * gt)574 static void hsw_sseu_info_init(struct intel_gt *gt)
575 {
576 	struct drm_i915_private *i915 = gt->i915;
577 	struct sseu_dev_info *sseu = &gt->info.sseu;
578 	u32 fuse1;
579 	u8 subslice_mask = 0;
580 	int s, ss;
581 
582 	/*
583 	 * There isn't a register to tell us how many slices/subslices. We
584 	 * work off the PCI-ids here.
585 	 */
586 	switch (INTEL_INFO(i915)->gt) {
587 	default:
588 		MISSING_CASE(INTEL_INFO(i915)->gt);
589 		fallthrough;
590 	case 1:
591 		sseu->slice_mask = BIT(0);
592 		subslice_mask = BIT(0);
593 		break;
594 	case 2:
595 		sseu->slice_mask = BIT(0);
596 		subslice_mask = BIT(0) | BIT(1);
597 		break;
598 	case 3:
599 		sseu->slice_mask = BIT(0) | BIT(1);
600 		subslice_mask = BIT(0) | BIT(1);
601 		break;
602 	}
603 
604 	fuse1 = intel_uncore_read(gt->uncore, HSW_PAVP_FUSE1);
605 	switch (REG_FIELD_GET(HSW_F1_EU_DIS_MASK, fuse1)) {
606 	default:
607 		MISSING_CASE(REG_FIELD_GET(HSW_F1_EU_DIS_MASK, fuse1));
608 		fallthrough;
609 	case HSW_F1_EU_DIS_10EUS:
610 		sseu->eu_per_subslice = 10;
611 		break;
612 	case HSW_F1_EU_DIS_8EUS:
613 		sseu->eu_per_subslice = 8;
614 		break;
615 	case HSW_F1_EU_DIS_6EUS:
616 		sseu->eu_per_subslice = 6;
617 		break;
618 	}
619 
620 	intel_sseu_set_info(sseu, hweight8(sseu->slice_mask),
621 			    hweight8(subslice_mask),
622 			    sseu->eu_per_subslice);
623 
624 	for (s = 0; s < sseu->max_slices; s++) {
625 		sseu->subslice_mask.hsw[s] = subslice_mask;
626 
627 		for (ss = 0; ss < sseu->max_subslices; ss++) {
628 			sseu_set_eus(sseu, s, ss,
629 				     (1UL << sseu->eu_per_subslice) - 1);
630 		}
631 	}
632 
633 	sseu->eu_total = compute_eu_total(sseu);
634 
635 	/* No powergating for you. */
636 	sseu->has_slice_pg = 0;
637 	sseu->has_subslice_pg = 0;
638 	sseu->has_eu_pg = 0;
639 }
640 
intel_sseu_info_init(struct intel_gt * gt)641 void intel_sseu_info_init(struct intel_gt *gt)
642 {
643 	struct drm_i915_private *i915 = gt->i915;
644 
645 	if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50))
646 		xehp_sseu_info_init(gt);
647 	else if (GRAPHICS_VER(i915) >= 12)
648 		gen12_sseu_info_init(gt);
649 	else if (GRAPHICS_VER(i915) >= 11)
650 		gen11_sseu_info_init(gt);
651 	else if (GRAPHICS_VER(i915) >= 9)
652 		gen9_sseu_info_init(gt);
653 	else if (IS_BROADWELL(i915))
654 		bdw_sseu_info_init(gt);
655 	else if (IS_CHERRYVIEW(i915))
656 		cherryview_sseu_info_init(gt);
657 	else if (IS_HASWELL(i915))
658 		hsw_sseu_info_init(gt);
659 }
660 
intel_sseu_make_rpcs(struct intel_gt * gt,const struct intel_sseu * req_sseu)661 u32 intel_sseu_make_rpcs(struct intel_gt *gt,
662 			 const struct intel_sseu *req_sseu)
663 {
664 	struct drm_i915_private *i915 = gt->i915;
665 	const struct sseu_dev_info *sseu = &gt->info.sseu;
666 	bool subslice_pg = sseu->has_subslice_pg;
667 	u8 slices, subslices;
668 	u32 rpcs = 0;
669 
670 	/*
671 	 * No explicit RPCS request is needed to ensure full
672 	 * slice/subslice/EU enablement prior to Gen9.
673 	 */
674 	if (GRAPHICS_VER(i915) < 9)
675 		return 0;
676 
677 	/*
678 	 * If i915/perf is active, we want a stable powergating configuration
679 	 * on the system. Use the configuration pinned by i915/perf.
680 	 */
681 	if (gt->perf.group && gt->perf.group[PERF_GROUP_OAG].exclusive_stream)
682 		req_sseu = &gt->perf.sseu;
683 
684 	slices = hweight8(req_sseu->slice_mask);
685 	subslices = hweight8(req_sseu->subslice_mask);
686 
687 	/*
688 	 * Since the SScount bitfield in GEN8_R_PWR_CLK_STATE is only three bits
689 	 * wide and Icelake has up to eight subslices, specfial programming is
690 	 * needed in order to correctly enable all subslices.
691 	 *
692 	 * According to documentation software must consider the configuration
693 	 * as 2x4x8 and hardware will translate this to 1x8x8.
694 	 *
695 	 * Furthemore, even though SScount is three bits, maximum documented
696 	 * value for it is four. From this some rules/restrictions follow:
697 	 *
698 	 * 1.
699 	 * If enabled subslice count is greater than four, two whole slices must
700 	 * be enabled instead.
701 	 *
702 	 * 2.
703 	 * When more than one slice is enabled, hardware ignores the subslice
704 	 * count altogether.
705 	 *
706 	 * From these restrictions it follows that it is not possible to enable
707 	 * a count of subslices between the SScount maximum of four restriction,
708 	 * and the maximum available number on a particular SKU. Either all
709 	 * subslices are enabled, or a count between one and four on the first
710 	 * slice.
711 	 */
712 	if (GRAPHICS_VER(i915) == 11 &&
713 	    slices == 1 &&
714 	    subslices > min_t(u8, 4, hweight8(sseu->subslice_mask.hsw[0]) / 2)) {
715 		GEM_BUG_ON(subslices & 1);
716 
717 		subslice_pg = false;
718 		slices *= 2;
719 	}
720 
721 	/*
722 	 * Starting in Gen9, render power gating can leave
723 	 * slice/subslice/EU in a partially enabled state. We
724 	 * must make an explicit request through RPCS for full
725 	 * enablement.
726 	 */
727 	if (sseu->has_slice_pg) {
728 		u32 mask, val = slices;
729 
730 		if (GRAPHICS_VER(i915) >= 11) {
731 			mask = GEN11_RPCS_S_CNT_MASK;
732 			val <<= GEN11_RPCS_S_CNT_SHIFT;
733 		} else {
734 			mask = GEN8_RPCS_S_CNT_MASK;
735 			val <<= GEN8_RPCS_S_CNT_SHIFT;
736 		}
737 
738 		GEM_BUG_ON(val & ~mask);
739 		val &= mask;
740 
741 		rpcs |= GEN8_RPCS_ENABLE | GEN8_RPCS_S_CNT_ENABLE | val;
742 	}
743 
744 	if (subslice_pg) {
745 		u32 val = subslices;
746 
747 		val <<= GEN8_RPCS_SS_CNT_SHIFT;
748 
749 		GEM_BUG_ON(val & ~GEN8_RPCS_SS_CNT_MASK);
750 		val &= GEN8_RPCS_SS_CNT_MASK;
751 
752 		rpcs |= GEN8_RPCS_ENABLE | GEN8_RPCS_SS_CNT_ENABLE | val;
753 	}
754 
755 	if (sseu->has_eu_pg) {
756 		u32 val;
757 
758 		val = req_sseu->min_eus_per_subslice << GEN8_RPCS_EU_MIN_SHIFT;
759 		GEM_BUG_ON(val & ~GEN8_RPCS_EU_MIN_MASK);
760 		val &= GEN8_RPCS_EU_MIN_MASK;
761 
762 		rpcs |= val;
763 
764 		val = req_sseu->max_eus_per_subslice << GEN8_RPCS_EU_MAX_SHIFT;
765 		GEM_BUG_ON(val & ~GEN8_RPCS_EU_MAX_MASK);
766 		val &= GEN8_RPCS_EU_MAX_MASK;
767 
768 		rpcs |= val;
769 
770 		rpcs |= GEN8_RPCS_ENABLE;
771 	}
772 
773 	return rpcs;
774 }
775 
intel_sseu_dump(const struct sseu_dev_info * sseu,struct drm_printer * p)776 void intel_sseu_dump(const struct sseu_dev_info *sseu, struct drm_printer *p)
777 {
778 	int s;
779 
780 	if (sseu->has_xehp_dss) {
781 		drm_printf(p, "subslice total: %u\n",
782 			   intel_sseu_subslice_total(sseu));
783 		drm_printf(p, "geometry dss mask=%*pb\n",
784 			   XEHP_BITMAP_BITS(sseu->geometry_subslice_mask),
785 			   sseu->geometry_subslice_mask.xehp);
786 		drm_printf(p, "compute dss mask=%*pb\n",
787 			   XEHP_BITMAP_BITS(sseu->compute_subslice_mask),
788 			   sseu->compute_subslice_mask.xehp);
789 	} else {
790 		drm_printf(p, "slice total: %u, mask=%04x\n",
791 			   hweight8(sseu->slice_mask), sseu->slice_mask);
792 		drm_printf(p, "subslice total: %u\n",
793 			   intel_sseu_subslice_total(sseu));
794 
795 		for (s = 0; s < sseu->max_slices; s++) {
796 			u8 ss_mask = sseu->subslice_mask.hsw[s];
797 
798 			drm_printf(p, "slice%d: %u subslices, mask=%08x\n",
799 				   s, hweight8(ss_mask), ss_mask);
800 		}
801 	}
802 
803 	drm_printf(p, "EU total: %u\n", sseu->eu_total);
804 	drm_printf(p, "EU per subslice: %u\n", sseu->eu_per_subslice);
805 	drm_printf(p, "has slice power gating: %s\n",
806 		   str_yes_no(sseu->has_slice_pg));
807 	drm_printf(p, "has subslice power gating: %s\n",
808 		   str_yes_no(sseu->has_subslice_pg));
809 	drm_printf(p, "has EU power gating: %s\n",
810 		   str_yes_no(sseu->has_eu_pg));
811 }
812 
sseu_print_hsw_topology(const struct sseu_dev_info * sseu,struct drm_printer * p)813 static void sseu_print_hsw_topology(const struct sseu_dev_info *sseu,
814 				    struct drm_printer *p)
815 {
816 	int s, ss;
817 
818 	for (s = 0; s < sseu->max_slices; s++) {
819 		u8 ss_mask = sseu->subslice_mask.hsw[s];
820 
821 		drm_printf(p, "slice%d: %u subslice(s) (0x%08x):\n",
822 			   s, hweight8(ss_mask), ss_mask);
823 
824 		for (ss = 0; ss < sseu->max_subslices; ss++) {
825 			u16 enabled_eus = sseu_get_eus(sseu, s, ss);
826 
827 			drm_printf(p, "\tsubslice%d: %u EUs (0x%hx)\n",
828 				   ss, hweight16(enabled_eus), enabled_eus);
829 		}
830 	}
831 }
832 
sseu_print_xehp_topology(const struct sseu_dev_info * sseu,struct drm_printer * p)833 static void sseu_print_xehp_topology(const struct sseu_dev_info *sseu,
834 				     struct drm_printer *p)
835 {
836 	int dss;
837 
838 	for (dss = 0; dss < sseu->max_subslices; dss++) {
839 		u16 enabled_eus = sseu_get_eus(sseu, 0, dss);
840 
841 		drm_printf(p, "DSS_%02d: G:%3s C:%3s, %2u EUs (0x%04hx)\n", dss,
842 			   str_yes_no(test_bit(dss, sseu->geometry_subslice_mask.xehp)),
843 			   str_yes_no(test_bit(dss, sseu->compute_subslice_mask.xehp)),
844 			   hweight16(enabled_eus), enabled_eus);
845 	}
846 }
847 
intel_sseu_print_topology(struct drm_i915_private * i915,const struct sseu_dev_info * sseu,struct drm_printer * p)848 void intel_sseu_print_topology(struct drm_i915_private *i915,
849 			       const struct sseu_dev_info *sseu,
850 			       struct drm_printer *p)
851 {
852 	if (sseu->max_slices == 0) {
853 		drm_printf(p, "Unavailable\n");
854 	} else if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50)) {
855 		sseu_print_xehp_topology(sseu, p);
856 	} else {
857 		sseu_print_hsw_topology(sseu, p);
858 	}
859 }
860 
intel_sseu_print_ss_info(const char * type,const struct sseu_dev_info * sseu,struct seq_file * m)861 void intel_sseu_print_ss_info(const char *type,
862 			      const struct sseu_dev_info *sseu,
863 			      struct seq_file *m)
864 {
865 	int s;
866 
867 	if (sseu->has_xehp_dss) {
868 		seq_printf(m, "  %s Geometry DSS: %u\n", type,
869 			   bitmap_weight(sseu->geometry_subslice_mask.xehp,
870 					 XEHP_BITMAP_BITS(sseu->geometry_subslice_mask)));
871 		seq_printf(m, "  %s Compute DSS: %u\n", type,
872 			   bitmap_weight(sseu->compute_subslice_mask.xehp,
873 					 XEHP_BITMAP_BITS(sseu->compute_subslice_mask)));
874 	} else {
875 		for (s = 0; s < fls(sseu->slice_mask); s++)
876 			seq_printf(m, "  %s Slice%i subslices: %u\n", type,
877 				   s, hweight8(sseu->subslice_mask.hsw[s]));
878 	}
879 }
880 
intel_slicemask_from_xehp_dssmask(intel_sseu_ss_mask_t dss_mask,int dss_per_slice)881 u16 intel_slicemask_from_xehp_dssmask(intel_sseu_ss_mask_t dss_mask,
882 				      int dss_per_slice)
883 {
884 	intel_sseu_ss_mask_t per_slice_mask = {};
885 	unsigned long slice_mask = 0;
886 	int i;
887 
888 	WARN_ON(DIV_ROUND_UP(XEHP_BITMAP_BITS(dss_mask), dss_per_slice) >
889 		8 * sizeof(slice_mask));
890 
891 	bitmap_fill(per_slice_mask.xehp, dss_per_slice);
892 	for (i = 0; !bitmap_empty(dss_mask.xehp, XEHP_BITMAP_BITS(dss_mask)); i++) {
893 		if (bitmap_intersects(dss_mask.xehp, per_slice_mask.xehp, dss_per_slice))
894 			slice_mask |= BIT(i);
895 
896 		bitmap_shift_right(dss_mask.xehp, dss_mask.xehp, dss_per_slice,
897 				   XEHP_BITMAP_BITS(dss_mask));
898 	}
899 
900 	return slice_mask;
901 }
902