1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2019 Intel Corporation
4  */
5 
6 #include <drm/drm_atomic_state_helper.h>
7 
8 #include "i915_reg.h"
9 #include "i915_utils.h"
10 #include "intel_atomic.h"
11 #include "intel_bw.h"
12 #include "intel_cdclk.h"
13 #include "intel_display_types.h"
14 #include "intel_mchbar_regs.h"
15 #include "intel_pcode.h"
16 #include "intel_pm.h"
17 
18 /* Parameters for Qclk Geyserville (QGV) */
19 struct intel_qgv_point {
20 	u16 dclk, t_rp, t_rdpre, t_rc, t_ras, t_rcd;
21 };
22 
23 struct intel_psf_gv_point {
24 	u8 clk; /* clock in multiples of 16.6666 MHz */
25 };
26 
27 struct intel_qgv_info {
28 	struct intel_qgv_point points[I915_NUM_QGV_POINTS];
29 	struct intel_psf_gv_point psf_points[I915_NUM_PSF_GV_POINTS];
30 	u8 num_points;
31 	u8 num_psf_points;
32 	u8 t_bl;
33 	u8 max_numchannels;
34 	u8 channel_width;
35 	u8 deinterleave;
36 };
37 
dg1_mchbar_read_qgv_point_info(struct drm_i915_private * dev_priv,struct intel_qgv_point * sp,int point)38 static int dg1_mchbar_read_qgv_point_info(struct drm_i915_private *dev_priv,
39 					  struct intel_qgv_point *sp,
40 					  int point)
41 {
42 	u32 dclk_ratio, dclk_reference;
43 	u32 val;
44 
45 	val = intel_uncore_read(&dev_priv->uncore, SA_PERF_STATUS_0_0_0_MCHBAR_PC);
46 	dclk_ratio = REG_FIELD_GET(DG1_QCLK_RATIO_MASK, val);
47 	if (val & DG1_QCLK_REFERENCE)
48 		dclk_reference = 6; /* 6 * 16.666 MHz = 100 MHz */
49 	else
50 		dclk_reference = 8; /* 8 * 16.666 MHz = 133 MHz */
51 	sp->dclk = DIV_ROUND_UP((16667 * dclk_ratio * dclk_reference) + 500, 1000);
52 
53 	val = intel_uncore_read(&dev_priv->uncore, SKL_MC_BIOS_DATA_0_0_0_MCHBAR_PCU);
54 	if (val & DG1_GEAR_TYPE)
55 		sp->dclk *= 2;
56 
57 	if (sp->dclk == 0)
58 		return -EINVAL;
59 
60 	val = intel_uncore_read(&dev_priv->uncore, MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR);
61 	sp->t_rp = REG_FIELD_GET(DG1_DRAM_T_RP_MASK, val);
62 	sp->t_rdpre = REG_FIELD_GET(DG1_DRAM_T_RDPRE_MASK, val);
63 
64 	val = intel_uncore_read(&dev_priv->uncore, MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR_HIGH);
65 	sp->t_rcd = REG_FIELD_GET(DG1_DRAM_T_RCD_MASK, val);
66 	sp->t_ras = REG_FIELD_GET(DG1_DRAM_T_RAS_MASK, val);
67 
68 	sp->t_rc = sp->t_rp + sp->t_ras;
69 
70 	return 0;
71 }
72 
icl_pcode_read_qgv_point_info(struct drm_i915_private * dev_priv,struct intel_qgv_point * sp,int point)73 static int icl_pcode_read_qgv_point_info(struct drm_i915_private *dev_priv,
74 					 struct intel_qgv_point *sp,
75 					 int point)
76 {
77 	u32 val = 0, val2 = 0;
78 	u16 dclk;
79 	int ret;
80 
81 	ret = snb_pcode_read(dev_priv, ICL_PCODE_MEM_SUBSYSYSTEM_INFO |
82 			     ICL_PCODE_MEM_SS_READ_QGV_POINT_INFO(point),
83 			     &val, &val2);
84 	if (ret)
85 		return ret;
86 
87 	dclk = val & 0xffff;
88 	sp->dclk = DIV_ROUND_UP((16667 * dclk) + (DISPLAY_VER(dev_priv) > 11 ? 500 : 0), 1000);
89 	sp->t_rp = (val & 0xff0000) >> 16;
90 	sp->t_rcd = (val & 0xff000000) >> 24;
91 
92 	sp->t_rdpre = val2 & 0xff;
93 	sp->t_ras = (val2 & 0xff00) >> 8;
94 
95 	sp->t_rc = sp->t_rp + sp->t_ras;
96 
97 	return 0;
98 }
99 
adls_pcode_read_psf_gv_point_info(struct drm_i915_private * dev_priv,struct intel_psf_gv_point * points)100 static int adls_pcode_read_psf_gv_point_info(struct drm_i915_private *dev_priv,
101 					    struct intel_psf_gv_point *points)
102 {
103 	u32 val = 0;
104 	int ret;
105 	int i;
106 
107 	ret = snb_pcode_read(dev_priv, ICL_PCODE_MEM_SUBSYSYSTEM_INFO |
108 			     ADL_PCODE_MEM_SS_READ_PSF_GV_INFO, &val, NULL);
109 	if (ret)
110 		return ret;
111 
112 	for (i = 0; i < I915_NUM_PSF_GV_POINTS; i++) {
113 		points[i].clk = val & 0xff;
114 		val >>= 8;
115 	}
116 
117 	return 0;
118 }
119 
icl_pcode_restrict_qgv_points(struct drm_i915_private * dev_priv,u32 points_mask)120 int icl_pcode_restrict_qgv_points(struct drm_i915_private *dev_priv,
121 				  u32 points_mask)
122 {
123 	int ret;
124 
125 	/* bspec says to keep retrying for at least 1 ms */
126 	ret = skl_pcode_request(dev_priv, ICL_PCODE_SAGV_DE_MEM_SS_CONFIG,
127 				points_mask,
128 				ICL_PCODE_REP_QGV_MASK | ADLS_PCODE_REP_PSF_MASK,
129 				ICL_PCODE_REP_QGV_SAFE | ADLS_PCODE_REP_PSF_SAFE,
130 				1);
131 
132 	if (ret < 0) {
133 		drm_err(&dev_priv->drm, "Failed to disable qgv points (%d) points: 0x%x\n", ret, points_mask);
134 		return ret;
135 	}
136 
137 	return 0;
138 }
139 
icl_get_qgv_points(struct drm_i915_private * dev_priv,struct intel_qgv_info * qi,bool is_y_tile)140 static int icl_get_qgv_points(struct drm_i915_private *dev_priv,
141 			      struct intel_qgv_info *qi,
142 			      bool is_y_tile)
143 {
144 	const struct dram_info *dram_info = &dev_priv->dram_info;
145 	int i, ret;
146 
147 	qi->num_points = dram_info->num_qgv_points;
148 	qi->num_psf_points = dram_info->num_psf_gv_points;
149 
150 	if (DISPLAY_VER(dev_priv) >= 12)
151 		switch (dram_info->type) {
152 		case INTEL_DRAM_DDR4:
153 			qi->t_bl = is_y_tile ? 8 : 4;
154 			qi->max_numchannels = 2;
155 			qi->channel_width = 64;
156 			qi->deinterleave = is_y_tile ? 1 : 2;
157 			break;
158 		case INTEL_DRAM_DDR5:
159 			qi->t_bl = is_y_tile ? 16 : 8;
160 			qi->max_numchannels = 4;
161 			qi->channel_width = 32;
162 			qi->deinterleave = is_y_tile ? 1 : 2;
163 			break;
164 		case INTEL_DRAM_LPDDR4:
165 			if (IS_ROCKETLAKE(dev_priv)) {
166 				qi->t_bl = 8;
167 				qi->max_numchannels = 4;
168 				qi->channel_width = 32;
169 				qi->deinterleave = 2;
170 				break;
171 			}
172 			fallthrough;
173 		case INTEL_DRAM_LPDDR5:
174 			qi->t_bl = 16;
175 			qi->max_numchannels = 8;
176 			qi->channel_width = 16;
177 			qi->deinterleave = is_y_tile ? 2 : 4;
178 			break;
179 		default:
180 			qi->t_bl = 16;
181 			qi->max_numchannels = 1;
182 			break;
183 		}
184 	else if (DISPLAY_VER(dev_priv) == 11) {
185 		qi->t_bl = dev_priv->dram_info.type == INTEL_DRAM_DDR4 ? 4 : 8;
186 		qi->max_numchannels = 1;
187 	}
188 
189 	if (drm_WARN_ON(&dev_priv->drm,
190 			qi->num_points > ARRAY_SIZE(qi->points)))
191 		qi->num_points = ARRAY_SIZE(qi->points);
192 
193 	for (i = 0; i < qi->num_points; i++) {
194 		struct intel_qgv_point *sp = &qi->points[i];
195 
196 		if (IS_DG1(dev_priv))
197 			ret = dg1_mchbar_read_qgv_point_info(dev_priv, sp, i);
198 		else
199 			ret = icl_pcode_read_qgv_point_info(dev_priv, sp, i);
200 
201 		if (ret)
202 			return ret;
203 
204 		drm_dbg_kms(&dev_priv->drm,
205 			    "QGV %d: DCLK=%d tRP=%d tRDPRE=%d tRAS=%d tRCD=%d tRC=%d\n",
206 			    i, sp->dclk, sp->t_rp, sp->t_rdpre, sp->t_ras,
207 			    sp->t_rcd, sp->t_rc);
208 	}
209 
210 	if (qi->num_psf_points > 0) {
211 		ret = adls_pcode_read_psf_gv_point_info(dev_priv, qi->psf_points);
212 		if (ret) {
213 			drm_err(&dev_priv->drm, "Failed to read PSF point data; PSF points will not be considered in bandwidth calculations.\n");
214 			qi->num_psf_points = 0;
215 		}
216 
217 		for (i = 0; i < qi->num_psf_points; i++)
218 			drm_dbg_kms(&dev_priv->drm,
219 				    "PSF GV %d: CLK=%d \n",
220 				    i, qi->psf_points[i].clk);
221 	}
222 
223 	return 0;
224 }
225 
adl_calc_psf_bw(int clk)226 static int adl_calc_psf_bw(int clk)
227 {
228 	/*
229 	 * clk is multiples of 16.666MHz (100/6)
230 	 * According to BSpec PSF GV bandwidth is
231 	 * calculated as BW = 64 * clk * 16.666Mhz
232 	 */
233 	return DIV_ROUND_CLOSEST(64 * clk * 100, 6);
234 }
235 
icl_sagv_max_dclk(const struct intel_qgv_info * qi)236 static int icl_sagv_max_dclk(const struct intel_qgv_info *qi)
237 {
238 	u16 dclk = 0;
239 	int i;
240 
241 	for (i = 0; i < qi->num_points; i++)
242 		dclk = max(dclk, qi->points[i].dclk);
243 
244 	return dclk;
245 }
246 
247 struct intel_sa_info {
248 	u16 displayrtids;
249 	u8 deburst, deprogbwlimit, derating;
250 };
251 
252 static const struct intel_sa_info icl_sa_info = {
253 	.deburst = 8,
254 	.deprogbwlimit = 25, /* GB/s */
255 	.displayrtids = 128,
256 	.derating = 10,
257 };
258 
259 static const struct intel_sa_info tgl_sa_info = {
260 	.deburst = 16,
261 	.deprogbwlimit = 34, /* GB/s */
262 	.displayrtids = 256,
263 	.derating = 10,
264 };
265 
266 static const struct intel_sa_info rkl_sa_info = {
267 	.deburst = 8,
268 	.deprogbwlimit = 20, /* GB/s */
269 	.displayrtids = 128,
270 	.derating = 10,
271 };
272 
273 static const struct intel_sa_info adls_sa_info = {
274 	.deburst = 16,
275 	.deprogbwlimit = 38, /* GB/s */
276 	.displayrtids = 256,
277 	.derating = 10,
278 };
279 
280 static const struct intel_sa_info adlp_sa_info = {
281 	.deburst = 16,
282 	.deprogbwlimit = 38, /* GB/s */
283 	.displayrtids = 256,
284 	.derating = 20,
285 };
286 
icl_get_bw_info(struct drm_i915_private * dev_priv,const struct intel_sa_info * sa)287 static int icl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel_sa_info *sa)
288 {
289 	struct intel_qgv_info qi = {};
290 	bool is_y_tile = true; /* assume y tile may be used */
291 	int num_channels = max_t(u8, 1, dev_priv->dram_info.num_channels);
292 	int ipqdepth, ipqdepthpch = 16;
293 	int dclk_max;
294 	int maxdebw;
295 	int num_groups = ARRAY_SIZE(dev_priv->max_bw);
296 	int i, ret;
297 
298 	ret = icl_get_qgv_points(dev_priv, &qi, is_y_tile);
299 	if (ret) {
300 		drm_dbg_kms(&dev_priv->drm,
301 			    "Failed to get memory subsystem information, ignoring bandwidth limits");
302 		return ret;
303 	}
304 
305 	dclk_max = icl_sagv_max_dclk(&qi);
306 	maxdebw = min(sa->deprogbwlimit * 1000, dclk_max * 16 * 6 / 10);
307 	ipqdepth = min(ipqdepthpch, sa->displayrtids / num_channels);
308 	qi.deinterleave = DIV_ROUND_UP(num_channels, is_y_tile ? 4 : 2);
309 
310 	for (i = 0; i < num_groups; i++) {
311 		struct intel_bw_info *bi = &dev_priv->max_bw[i];
312 		int clpchgroup;
313 		int j;
314 
315 		clpchgroup = (sa->deburst * qi.deinterleave / num_channels) << i;
316 		bi->num_planes = (ipqdepth - clpchgroup) / clpchgroup + 1;
317 
318 		bi->num_qgv_points = qi.num_points;
319 		bi->num_psf_gv_points = qi.num_psf_points;
320 
321 		for (j = 0; j < qi.num_points; j++) {
322 			const struct intel_qgv_point *sp = &qi.points[j];
323 			int ct, bw;
324 
325 			/*
326 			 * Max row cycle time
327 			 *
328 			 * FIXME what is the logic behind the
329 			 * assumed burst length?
330 			 */
331 			ct = max_t(int, sp->t_rc, sp->t_rp + sp->t_rcd +
332 				   (clpchgroup - 1) * qi.t_bl + sp->t_rdpre);
333 			bw = DIV_ROUND_UP(sp->dclk * clpchgroup * 32 * num_channels, ct);
334 
335 			bi->deratedbw[j] = min(maxdebw,
336 					       bw * (100 - sa->derating) / 100);
337 
338 			drm_dbg_kms(&dev_priv->drm,
339 				    "BW%d / QGV %d: num_planes=%d deratedbw=%u\n",
340 				    i, j, bi->num_planes, bi->deratedbw[j]);
341 		}
342 	}
343 	/*
344 	 * In case if SAGV is disabled in BIOS, we always get 1
345 	 * SAGV point, but we can't send PCode commands to restrict it
346 	 * as it will fail and pointless anyway.
347 	 */
348 	if (qi.num_points == 1)
349 		dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
350 	else
351 		dev_priv->sagv_status = I915_SAGV_ENABLED;
352 
353 	return 0;
354 }
355 
tgl_get_bw_info(struct drm_i915_private * dev_priv,const struct intel_sa_info * sa)356 static int tgl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel_sa_info *sa)
357 {
358 	struct intel_qgv_info qi = {};
359 	const struct dram_info *dram_info = &dev_priv->dram_info;
360 	bool is_y_tile = true; /* assume y tile may be used */
361 	int num_channels = max_t(u8, 1, dev_priv->dram_info.num_channels);
362 	int ipqdepth, ipqdepthpch = 16;
363 	int dclk_max;
364 	int maxdebw, peakbw;
365 	int clperchgroup;
366 	int num_groups = ARRAY_SIZE(dev_priv->max_bw);
367 	int i, ret;
368 
369 	ret = icl_get_qgv_points(dev_priv, &qi, is_y_tile);
370 	if (ret) {
371 		drm_dbg_kms(&dev_priv->drm,
372 			    "Failed to get memory subsystem information, ignoring bandwidth limits");
373 		return ret;
374 	}
375 
376 	if (dram_info->type == INTEL_DRAM_LPDDR4 || dram_info->type == INTEL_DRAM_LPDDR5)
377 		num_channels *= 2;
378 
379 	qi.deinterleave = qi.deinterleave ? : DIV_ROUND_UP(num_channels, is_y_tile ? 4 : 2);
380 
381 	if (num_channels < qi.max_numchannels && DISPLAY_VER(dev_priv) >= 12)
382 		qi.deinterleave = max(DIV_ROUND_UP(qi.deinterleave, 2), 1);
383 
384 	if (DISPLAY_VER(dev_priv) > 11 && num_channels > qi.max_numchannels)
385 		drm_warn(&dev_priv->drm, "Number of channels exceeds max number of channels.");
386 	if (qi.max_numchannels != 0)
387 		num_channels = min_t(u8, num_channels, qi.max_numchannels);
388 
389 	dclk_max = icl_sagv_max_dclk(&qi);
390 
391 	peakbw = num_channels * DIV_ROUND_UP(qi.channel_width, 8) * dclk_max;
392 	maxdebw = min(sa->deprogbwlimit * 1000, peakbw * 6 / 10); /* 60% */
393 
394 	ipqdepth = min(ipqdepthpch, sa->displayrtids / num_channels);
395 	/*
396 	 * clperchgroup = 4kpagespermempage * clperchperblock,
397 	 * clperchperblock = 8 / num_channels * interleave
398 	 */
399 	clperchgroup = 4 * DIV_ROUND_UP(8, num_channels) * qi.deinterleave;
400 
401 	for (i = 0; i < num_groups; i++) {
402 		struct intel_bw_info *bi = &dev_priv->max_bw[i];
403 		struct intel_bw_info *bi_next;
404 		int clpchgroup;
405 		int j;
406 
407 		clpchgroup = (sa->deburst * qi.deinterleave / num_channels) << i;
408 
409 		if (i < num_groups - 1) {
410 			bi_next = &dev_priv->max_bw[i + 1];
411 
412 			if (clpchgroup < clperchgroup)
413 				bi_next->num_planes = (ipqdepth - clpchgroup) /
414 						       clpchgroup + 1;
415 			else
416 				bi_next->num_planes = 0;
417 		}
418 
419 		bi->num_qgv_points = qi.num_points;
420 		bi->num_psf_gv_points = qi.num_psf_points;
421 
422 		for (j = 0; j < qi.num_points; j++) {
423 			const struct intel_qgv_point *sp = &qi.points[j];
424 			int ct, bw;
425 
426 			/*
427 			 * Max row cycle time
428 			 *
429 			 * FIXME what is the logic behind the
430 			 * assumed burst length?
431 			 */
432 			ct = max_t(int, sp->t_rc, sp->t_rp + sp->t_rcd +
433 				   (clpchgroup - 1) * qi.t_bl + sp->t_rdpre);
434 			bw = DIV_ROUND_UP(sp->dclk * clpchgroup * 32 * num_channels, ct);
435 
436 			bi->deratedbw[j] = min(maxdebw,
437 					       bw * (100 - sa->derating) / 100);
438 
439 			drm_dbg_kms(&dev_priv->drm,
440 				    "BW%d / QGV %d: num_planes=%d deratedbw=%u\n",
441 				    i, j, bi->num_planes, bi->deratedbw[j]);
442 		}
443 
444 		for (j = 0; j < qi.num_psf_points; j++) {
445 			const struct intel_psf_gv_point *sp = &qi.psf_points[j];
446 
447 			bi->psf_bw[j] = adl_calc_psf_bw(sp->clk);
448 
449 			drm_dbg_kms(&dev_priv->drm,
450 				    "BW%d / PSF GV %d: num_planes=%d bw=%u\n",
451 				    i, j, bi->num_planes, bi->psf_bw[j]);
452 		}
453 	}
454 
455 	/*
456 	 * In case if SAGV is disabled in BIOS, we always get 1
457 	 * SAGV point, but we can't send PCode commands to restrict it
458 	 * as it will fail and pointless anyway.
459 	 */
460 	if (qi.num_points == 1)
461 		dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
462 	else
463 		dev_priv->sagv_status = I915_SAGV_ENABLED;
464 
465 	return 0;
466 }
467 
dg2_get_bw_info(struct drm_i915_private * i915)468 static void dg2_get_bw_info(struct drm_i915_private *i915)
469 {
470 	unsigned int deratedbw = IS_DG2_G11(i915) ? 38000 : 50000;
471 	int num_groups = ARRAY_SIZE(i915->max_bw);
472 	int i;
473 
474 	/*
475 	 * DG2 doesn't have SAGV or QGV points, just a constant max bandwidth
476 	 * that doesn't depend on the number of planes enabled. So fill all the
477 	 * plane group with constant bw information for uniformity with other
478 	 * platforms. DG2-G10 platforms have a constant 50 GB/s bandwidth,
479 	 * whereas DG2-G11 platforms have 38 GB/s.
480 	 */
481 	for (i = 0; i < num_groups; i++) {
482 		struct intel_bw_info *bi = &i915->max_bw[i];
483 
484 		bi->num_planes = 1;
485 		/* Need only one dummy QGV point per group */
486 		bi->num_qgv_points = 1;
487 		bi->deratedbw[0] = deratedbw;
488 	}
489 
490 	i915->sagv_status = I915_SAGV_NOT_CONTROLLED;
491 }
492 
icl_max_bw(struct drm_i915_private * dev_priv,int num_planes,int qgv_point)493 static unsigned int icl_max_bw(struct drm_i915_private *dev_priv,
494 			       int num_planes, int qgv_point)
495 {
496 	int i;
497 
498 	/*
499 	 * Let's return max bw for 0 planes
500 	 */
501 	num_planes = max(1, num_planes);
502 
503 	for (i = 0; i < ARRAY_SIZE(dev_priv->max_bw); i++) {
504 		const struct intel_bw_info *bi =
505 			&dev_priv->max_bw[i];
506 
507 		/*
508 		 * Pcode will not expose all QGV points when
509 		 * SAGV is forced to off/min/med/max.
510 		 */
511 		if (qgv_point >= bi->num_qgv_points)
512 			return UINT_MAX;
513 
514 		if (num_planes >= bi->num_planes)
515 			return bi->deratedbw[qgv_point];
516 	}
517 
518 	return 0;
519 }
520 
tgl_max_bw(struct drm_i915_private * dev_priv,int num_planes,int qgv_point)521 static unsigned int tgl_max_bw(struct drm_i915_private *dev_priv,
522 			       int num_planes, int qgv_point)
523 {
524 	int i;
525 
526 	/*
527 	 * Let's return max bw for 0 planes
528 	 */
529 	num_planes = max(1, num_planes);
530 
531 	for (i = ARRAY_SIZE(dev_priv->max_bw) - 1; i >= 0; i--) {
532 		const struct intel_bw_info *bi =
533 			&dev_priv->max_bw[i];
534 
535 		/*
536 		 * Pcode will not expose all QGV points when
537 		 * SAGV is forced to off/min/med/max.
538 		 */
539 		if (qgv_point >= bi->num_qgv_points)
540 			return UINT_MAX;
541 
542 		if (num_planes <= bi->num_planes)
543 			return bi->deratedbw[qgv_point];
544 	}
545 
546 	return dev_priv->max_bw[0].deratedbw[qgv_point];
547 }
548 
adl_psf_bw(struct drm_i915_private * dev_priv,int psf_gv_point)549 static unsigned int adl_psf_bw(struct drm_i915_private *dev_priv,
550 			       int psf_gv_point)
551 {
552 	const struct intel_bw_info *bi =
553 			&dev_priv->max_bw[0];
554 
555 	return bi->psf_bw[psf_gv_point];
556 }
557 
intel_bw_init_hw(struct drm_i915_private * dev_priv)558 void intel_bw_init_hw(struct drm_i915_private *dev_priv)
559 {
560 	if (!HAS_DISPLAY(dev_priv))
561 		return;
562 
563 	if (IS_DG2(dev_priv))
564 		dg2_get_bw_info(dev_priv);
565 	else if (IS_ALDERLAKE_P(dev_priv))
566 		tgl_get_bw_info(dev_priv, &adlp_sa_info);
567 	else if (IS_ALDERLAKE_S(dev_priv))
568 		tgl_get_bw_info(dev_priv, &adls_sa_info);
569 	else if (IS_ROCKETLAKE(dev_priv))
570 		tgl_get_bw_info(dev_priv, &rkl_sa_info);
571 	else if (DISPLAY_VER(dev_priv) == 12)
572 		tgl_get_bw_info(dev_priv, &tgl_sa_info);
573 	else if (DISPLAY_VER(dev_priv) == 11)
574 		icl_get_bw_info(dev_priv, &icl_sa_info);
575 }
576 
intel_bw_crtc_num_active_planes(const struct intel_crtc_state * crtc_state)577 static unsigned int intel_bw_crtc_num_active_planes(const struct intel_crtc_state *crtc_state)
578 {
579 	/*
580 	 * We assume cursors are small enough
581 	 * to not not cause bandwidth problems.
582 	 */
583 	return hweight8(crtc_state->active_planes & ~BIT(PLANE_CURSOR));
584 }
585 
intel_bw_crtc_data_rate(const struct intel_crtc_state * crtc_state)586 static unsigned int intel_bw_crtc_data_rate(const struct intel_crtc_state *crtc_state)
587 {
588 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
589 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
590 	unsigned int data_rate = 0;
591 	enum plane_id plane_id;
592 
593 	for_each_plane_id_on_crtc(crtc, plane_id) {
594 		/*
595 		 * We assume cursors are small enough
596 		 * to not not cause bandwidth problems.
597 		 */
598 		if (plane_id == PLANE_CURSOR)
599 			continue;
600 
601 		data_rate += crtc_state->data_rate[plane_id];
602 
603 		if (DISPLAY_VER(i915) < 11)
604 			data_rate += crtc_state->data_rate_y[plane_id];
605 	}
606 
607 	return data_rate;
608 }
609 
610 /* "Maximum Pipe Read Bandwidth" */
intel_bw_crtc_min_cdclk(const struct intel_crtc_state * crtc_state)611 static int intel_bw_crtc_min_cdclk(const struct intel_crtc_state *crtc_state)
612 {
613 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
614 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
615 
616 	if (DISPLAY_VER(i915) < 12)
617 		return 0;
618 
619 	return DIV_ROUND_UP_ULL(mul_u32_u32(intel_bw_crtc_data_rate(crtc_state), 10), 512);
620 }
621 
intel_bw_crtc_update(struct intel_bw_state * bw_state,const struct intel_crtc_state * crtc_state)622 void intel_bw_crtc_update(struct intel_bw_state *bw_state,
623 			  const struct intel_crtc_state *crtc_state)
624 {
625 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
626 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
627 
628 	bw_state->data_rate[crtc->pipe] =
629 		intel_bw_crtc_data_rate(crtc_state);
630 	bw_state->num_active_planes[crtc->pipe] =
631 		intel_bw_crtc_num_active_planes(crtc_state);
632 
633 	drm_dbg_kms(&i915->drm, "pipe %c data rate %u num active planes %u\n",
634 		    pipe_name(crtc->pipe),
635 		    bw_state->data_rate[crtc->pipe],
636 		    bw_state->num_active_planes[crtc->pipe]);
637 }
638 
intel_bw_num_active_planes(struct drm_i915_private * dev_priv,const struct intel_bw_state * bw_state)639 static unsigned int intel_bw_num_active_planes(struct drm_i915_private *dev_priv,
640 					       const struct intel_bw_state *bw_state)
641 {
642 	unsigned int num_active_planes = 0;
643 	enum pipe pipe;
644 
645 	for_each_pipe(dev_priv, pipe)
646 		num_active_planes += bw_state->num_active_planes[pipe];
647 
648 	return num_active_planes;
649 }
650 
intel_bw_data_rate(struct drm_i915_private * dev_priv,const struct intel_bw_state * bw_state)651 static unsigned int intel_bw_data_rate(struct drm_i915_private *dev_priv,
652 				       const struct intel_bw_state *bw_state)
653 {
654 	unsigned int data_rate = 0;
655 	enum pipe pipe;
656 
657 	for_each_pipe(dev_priv, pipe)
658 		data_rate += bw_state->data_rate[pipe];
659 
660 	if (DISPLAY_VER(dev_priv) >= 13 && i915_vtd_active(dev_priv))
661 		data_rate = DIV_ROUND_UP(data_rate * 105, 100);
662 
663 	return data_rate;
664 }
665 
666 struct intel_bw_state *
intel_atomic_get_old_bw_state(struct intel_atomic_state * state)667 intel_atomic_get_old_bw_state(struct intel_atomic_state *state)
668 {
669 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
670 	struct intel_global_state *bw_state;
671 
672 	bw_state = intel_atomic_get_old_global_obj_state(state, &dev_priv->bw_obj);
673 
674 	return to_intel_bw_state(bw_state);
675 }
676 
677 struct intel_bw_state *
intel_atomic_get_new_bw_state(struct intel_atomic_state * state)678 intel_atomic_get_new_bw_state(struct intel_atomic_state *state)
679 {
680 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
681 	struct intel_global_state *bw_state;
682 
683 	bw_state = intel_atomic_get_new_global_obj_state(state, &dev_priv->bw_obj);
684 
685 	return to_intel_bw_state(bw_state);
686 }
687 
688 struct intel_bw_state *
intel_atomic_get_bw_state(struct intel_atomic_state * state)689 intel_atomic_get_bw_state(struct intel_atomic_state *state)
690 {
691 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
692 	struct intel_global_state *bw_state;
693 
694 	bw_state = intel_atomic_get_global_obj_state(state, &dev_priv->bw_obj);
695 	if (IS_ERR(bw_state))
696 		return ERR_CAST(bw_state);
697 
698 	return to_intel_bw_state(bw_state);
699 }
700 
intel_bw_state_changed(struct drm_i915_private * i915,const struct intel_bw_state * old_bw_state,const struct intel_bw_state * new_bw_state)701 static bool intel_bw_state_changed(struct drm_i915_private *i915,
702 				   const struct intel_bw_state *old_bw_state,
703 				   const struct intel_bw_state *new_bw_state)
704 {
705 	enum pipe pipe;
706 
707 	for_each_pipe(i915, pipe) {
708 		const struct intel_dbuf_bw *old_crtc_bw =
709 			&old_bw_state->dbuf_bw[pipe];
710 		const struct intel_dbuf_bw *new_crtc_bw =
711 			&new_bw_state->dbuf_bw[pipe];
712 		enum dbuf_slice slice;
713 
714 		for_each_dbuf_slice(i915, slice) {
715 			if (old_crtc_bw->max_bw[slice] != new_crtc_bw->max_bw[slice] ||
716 			    old_crtc_bw->active_planes[slice] != new_crtc_bw->active_planes[slice])
717 				return true;
718 		}
719 
720 		if (old_bw_state->min_cdclk[pipe] != new_bw_state->min_cdclk[pipe])
721 			return true;
722 	}
723 
724 	return false;
725 }
726 
skl_plane_calc_dbuf_bw(struct intel_bw_state * bw_state,struct intel_crtc * crtc,enum plane_id plane_id,const struct skl_ddb_entry * ddb,unsigned int data_rate)727 static void skl_plane_calc_dbuf_bw(struct intel_bw_state *bw_state,
728 				   struct intel_crtc *crtc,
729 				   enum plane_id plane_id,
730 				   const struct skl_ddb_entry *ddb,
731 				   unsigned int data_rate)
732 {
733 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
734 	struct intel_dbuf_bw *crtc_bw = &bw_state->dbuf_bw[crtc->pipe];
735 	unsigned int dbuf_mask = skl_ddb_dbuf_slice_mask(i915, ddb);
736 	enum dbuf_slice slice;
737 
738 	/*
739 	 * The arbiter can only really guarantee an
740 	 * equal share of the total bw to each plane.
741 	 */
742 	for_each_dbuf_slice_in_mask(i915, slice, dbuf_mask) {
743 		crtc_bw->max_bw[slice] = max(crtc_bw->max_bw[slice], data_rate);
744 		crtc_bw->active_planes[slice] |= BIT(plane_id);
745 	}
746 }
747 
skl_crtc_calc_dbuf_bw(struct intel_bw_state * bw_state,const struct intel_crtc_state * crtc_state)748 static void skl_crtc_calc_dbuf_bw(struct intel_bw_state *bw_state,
749 				  const struct intel_crtc_state *crtc_state)
750 {
751 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
752 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
753 	struct intel_dbuf_bw *crtc_bw = &bw_state->dbuf_bw[crtc->pipe];
754 	enum plane_id plane_id;
755 
756 	memset(crtc_bw, 0, sizeof(*crtc_bw));
757 
758 	if (!crtc_state->hw.active)
759 		return;
760 
761 	for_each_plane_id_on_crtc(crtc, plane_id) {
762 		/*
763 		 * We assume cursors are small enough
764 		 * to not cause bandwidth problems.
765 		 */
766 		if (plane_id == PLANE_CURSOR)
767 			continue;
768 
769 		skl_plane_calc_dbuf_bw(bw_state, crtc, plane_id,
770 				       &crtc_state->wm.skl.plane_ddb[plane_id],
771 				       crtc_state->data_rate[plane_id]);
772 
773 		if (DISPLAY_VER(i915) < 11)
774 			skl_plane_calc_dbuf_bw(bw_state, crtc, plane_id,
775 					       &crtc_state->wm.skl.plane_ddb_y[plane_id],
776 					       crtc_state->data_rate[plane_id]);
777 	}
778 }
779 
780 /* "Maximum Data Buffer Bandwidth" */
781 static int
intel_bw_dbuf_min_cdclk(struct drm_i915_private * i915,const struct intel_bw_state * bw_state)782 intel_bw_dbuf_min_cdclk(struct drm_i915_private *i915,
783 			const struct intel_bw_state *bw_state)
784 {
785 	unsigned int total_max_bw = 0;
786 	enum dbuf_slice slice;
787 
788 	for_each_dbuf_slice(i915, slice) {
789 		int num_active_planes = 0;
790 		unsigned int max_bw = 0;
791 		enum pipe pipe;
792 
793 		/*
794 		 * The arbiter can only really guarantee an
795 		 * equal share of the total bw to each plane.
796 		 */
797 		for_each_pipe(i915, pipe) {
798 			const struct intel_dbuf_bw *crtc_bw = &bw_state->dbuf_bw[pipe];
799 
800 			max_bw = max(crtc_bw->max_bw[slice], max_bw);
801 			num_active_planes += hweight8(crtc_bw->active_planes[slice]);
802 		}
803 		max_bw *= num_active_planes;
804 
805 		total_max_bw = max(total_max_bw, max_bw);
806 	}
807 
808 	return DIV_ROUND_UP(total_max_bw, 64);
809 }
810 
intel_bw_min_cdclk(struct drm_i915_private * i915,const struct intel_bw_state * bw_state)811 int intel_bw_min_cdclk(struct drm_i915_private *i915,
812 		       const struct intel_bw_state *bw_state)
813 {
814 	enum pipe pipe;
815 	int min_cdclk;
816 
817 	min_cdclk = intel_bw_dbuf_min_cdclk(i915, bw_state);
818 
819 	for_each_pipe(i915, pipe)
820 		min_cdclk = max(bw_state->min_cdclk[pipe], min_cdclk);
821 
822 	return min_cdclk;
823 }
824 
intel_bw_calc_min_cdclk(struct intel_atomic_state * state,bool * need_cdclk_calc)825 int intel_bw_calc_min_cdclk(struct intel_atomic_state *state,
826 			    bool *need_cdclk_calc)
827 {
828 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
829 	struct intel_bw_state *new_bw_state = NULL;
830 	const struct intel_bw_state *old_bw_state = NULL;
831 	const struct intel_cdclk_state *cdclk_state;
832 	const struct intel_crtc_state *crtc_state;
833 	int old_min_cdclk, new_min_cdclk;
834 	struct intel_crtc *crtc;
835 	int i;
836 
837 	if (DISPLAY_VER(dev_priv) < 9)
838 		return 0;
839 
840 	for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
841 		new_bw_state = intel_atomic_get_bw_state(state);
842 		if (IS_ERR(new_bw_state))
843 			return PTR_ERR(new_bw_state);
844 
845 		old_bw_state = intel_atomic_get_old_bw_state(state);
846 
847 		skl_crtc_calc_dbuf_bw(new_bw_state, crtc_state);
848 
849 		new_bw_state->min_cdclk[crtc->pipe] =
850 			intel_bw_crtc_min_cdclk(crtc_state);
851 	}
852 
853 	if (!old_bw_state)
854 		return 0;
855 
856 	if (intel_bw_state_changed(dev_priv, old_bw_state, new_bw_state)) {
857 		int ret = intel_atomic_lock_global_state(&new_bw_state->base);
858 		if (ret)
859 			return ret;
860 	}
861 
862 	old_min_cdclk = intel_bw_min_cdclk(dev_priv, old_bw_state);
863 	new_min_cdclk = intel_bw_min_cdclk(dev_priv, new_bw_state);
864 
865 	/*
866 	 * No need to check against the cdclk state if
867 	 * the min cdclk doesn't increase.
868 	 *
869 	 * Ie. we only ever increase the cdclk due to bandwidth
870 	 * requirements. This can reduce back and forth
871 	 * display blinking due to constant cdclk changes.
872 	 */
873 	if (new_min_cdclk <= old_min_cdclk)
874 		return 0;
875 
876 	cdclk_state = intel_atomic_get_cdclk_state(state);
877 	if (IS_ERR(cdclk_state))
878 		return PTR_ERR(cdclk_state);
879 
880 	/*
881 	 * No need to recalculate the cdclk state if
882 	 * the min cdclk doesn't increase.
883 	 *
884 	 * Ie. we only ever increase the cdclk due to bandwidth
885 	 * requirements. This can reduce back and forth
886 	 * display blinking due to constant cdclk changes.
887 	 */
888 	if (new_min_cdclk <= cdclk_state->bw_min_cdclk)
889 		return 0;
890 
891 	drm_dbg_kms(&dev_priv->drm,
892 		    "new bandwidth min cdclk (%d kHz) > old min cdclk (%d kHz)\n",
893 		    new_min_cdclk, cdclk_state->bw_min_cdclk);
894 	*need_cdclk_calc = true;
895 
896 	return 0;
897 }
898 
icl_qgv_points_mask(struct drm_i915_private * i915)899 static u16 icl_qgv_points_mask(struct drm_i915_private *i915)
900 {
901 	unsigned int num_psf_gv_points = i915->max_bw[0].num_psf_gv_points;
902 	unsigned int num_qgv_points = i915->max_bw[0].num_qgv_points;
903 	u16 qgv_points = 0, psf_points = 0;
904 
905 	/*
906 	 * We can _not_ use the whole ADLS_QGV_PT_MASK here, as PCode rejects
907 	 * it with failure if we try masking any unadvertised points.
908 	 * So need to operate only with those returned from PCode.
909 	 */
910 	if (num_qgv_points > 0)
911 		qgv_points = GENMASK(num_qgv_points - 1, 0);
912 
913 	if (num_psf_gv_points > 0)
914 		psf_points = GENMASK(num_psf_gv_points - 1, 0);
915 
916 	return ICL_PCODE_REQ_QGV_PT(qgv_points) | ADLS_PCODE_REQ_PSF_PT(psf_points);
917 }
918 
intel_bw_check_data_rate(struct intel_atomic_state * state,bool * changed)919 static int intel_bw_check_data_rate(struct intel_atomic_state *state, bool *changed)
920 {
921 	struct drm_i915_private *i915 = to_i915(state->base.dev);
922 	const struct intel_crtc_state *new_crtc_state, *old_crtc_state;
923 	struct intel_crtc *crtc;
924 	int i;
925 
926 	for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
927 					    new_crtc_state, i) {
928 		unsigned int old_data_rate =
929 			intel_bw_crtc_data_rate(old_crtc_state);
930 		unsigned int new_data_rate =
931 			intel_bw_crtc_data_rate(new_crtc_state);
932 		unsigned int old_active_planes =
933 			intel_bw_crtc_num_active_planes(old_crtc_state);
934 		unsigned int new_active_planes =
935 			intel_bw_crtc_num_active_planes(new_crtc_state);
936 		struct intel_bw_state *new_bw_state;
937 
938 		/*
939 		 * Avoid locking the bw state when
940 		 * nothing significant has changed.
941 		 */
942 		if (old_data_rate == new_data_rate &&
943 		    old_active_planes == new_active_planes)
944 			continue;
945 
946 		new_bw_state = intel_atomic_get_bw_state(state);
947 		if (IS_ERR(new_bw_state))
948 			return PTR_ERR(new_bw_state);
949 
950 		new_bw_state->data_rate[crtc->pipe] = new_data_rate;
951 		new_bw_state->num_active_planes[crtc->pipe] = new_active_planes;
952 
953 		*changed = true;
954 
955 		drm_dbg_kms(&i915->drm,
956 			    "[CRTC:%d:%s] data rate %u num active planes %u\n",
957 			    crtc->base.base.id, crtc->base.name,
958 			    new_bw_state->data_rate[crtc->pipe],
959 			    new_bw_state->num_active_planes[crtc->pipe]);
960 	}
961 
962 	return 0;
963 }
964 
intel_bw_atomic_check(struct intel_atomic_state * state)965 int intel_bw_atomic_check(struct intel_atomic_state *state)
966 {
967 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
968 	const struct intel_bw_state *old_bw_state;
969 	struct intel_bw_state *new_bw_state;
970 	unsigned int data_rate;
971 	unsigned int num_active_planes;
972 	int i, ret;
973 	u16 qgv_points = 0, psf_points = 0;
974 	unsigned int max_bw_point = 0, max_bw = 0;
975 	unsigned int num_qgv_points = dev_priv->max_bw[0].num_qgv_points;
976 	unsigned int num_psf_gv_points = dev_priv->max_bw[0].num_psf_gv_points;
977 	bool changed = false;
978 
979 	/* FIXME earlier gens need some checks too */
980 	if (DISPLAY_VER(dev_priv) < 11)
981 		return 0;
982 
983 	ret = intel_bw_check_data_rate(state, &changed);
984 	if (ret)
985 		return ret;
986 
987 	old_bw_state = intel_atomic_get_old_bw_state(state);
988 	new_bw_state = intel_atomic_get_new_bw_state(state);
989 
990 	if (new_bw_state &&
991 	    intel_can_enable_sagv(dev_priv, old_bw_state) !=
992 	    intel_can_enable_sagv(dev_priv, new_bw_state))
993 		changed = true;
994 
995 	/*
996 	 * If none of our inputs (data rates, number of active
997 	 * planes, SAGV yes/no) changed then nothing to do here.
998 	 */
999 	if (!changed)
1000 		return 0;
1001 
1002 	ret = intel_atomic_lock_global_state(&new_bw_state->base);
1003 	if (ret)
1004 		return ret;
1005 
1006 	data_rate = intel_bw_data_rate(dev_priv, new_bw_state);
1007 	data_rate = DIV_ROUND_UP(data_rate, 1000);
1008 
1009 	num_active_planes = intel_bw_num_active_planes(dev_priv, new_bw_state);
1010 
1011 	for (i = 0; i < num_qgv_points; i++) {
1012 		unsigned int max_data_rate;
1013 
1014 		if (DISPLAY_VER(dev_priv) > 11)
1015 			max_data_rate = tgl_max_bw(dev_priv, num_active_planes, i);
1016 		else
1017 			max_data_rate = icl_max_bw(dev_priv, num_active_planes, i);
1018 		/*
1019 		 * We need to know which qgv point gives us
1020 		 * maximum bandwidth in order to disable SAGV
1021 		 * if we find that we exceed SAGV block time
1022 		 * with watermarks. By that moment we already
1023 		 * have those, as it is calculated earlier in
1024 		 * intel_atomic_check,
1025 		 */
1026 		if (max_data_rate > max_bw) {
1027 			max_bw_point = i;
1028 			max_bw = max_data_rate;
1029 		}
1030 		if (max_data_rate >= data_rate)
1031 			qgv_points |= BIT(i);
1032 
1033 		drm_dbg_kms(&dev_priv->drm, "QGV point %d: max bw %d required %d\n",
1034 			    i, max_data_rate, data_rate);
1035 	}
1036 
1037 	for (i = 0; i < num_psf_gv_points; i++) {
1038 		unsigned int max_data_rate = adl_psf_bw(dev_priv, i);
1039 
1040 		if (max_data_rate >= data_rate)
1041 			psf_points |= BIT(i);
1042 
1043 		drm_dbg_kms(&dev_priv->drm, "PSF GV point %d: max bw %d"
1044 			    " required %d\n",
1045 			    i, max_data_rate, data_rate);
1046 	}
1047 
1048 	/*
1049 	 * BSpec states that we always should have at least one allowed point
1050 	 * left, so if we couldn't - simply reject the configuration for obvious
1051 	 * reasons.
1052 	 */
1053 	if (qgv_points == 0) {
1054 		drm_dbg_kms(&dev_priv->drm, "No QGV points provide sufficient memory"
1055 			    " bandwidth %d for display configuration(%d active planes).\n",
1056 			    data_rate, num_active_planes);
1057 		return -EINVAL;
1058 	}
1059 
1060 	if (num_psf_gv_points > 0 && psf_points == 0) {
1061 		drm_dbg_kms(&dev_priv->drm, "No PSF GV points provide sufficient memory"
1062 			    " bandwidth %d for display configuration(%d active planes).\n",
1063 			    data_rate, num_active_planes);
1064 		return -EINVAL;
1065 	}
1066 
1067 	/*
1068 	 * Leave only single point with highest bandwidth, if
1069 	 * we can't enable SAGV due to the increased memory latency it may
1070 	 * cause.
1071 	 */
1072 	if (!intel_can_enable_sagv(dev_priv, new_bw_state)) {
1073 		qgv_points = BIT(max_bw_point);
1074 		drm_dbg_kms(&dev_priv->drm, "No SAGV, using single QGV point %d\n",
1075 			    max_bw_point);
1076 	}
1077 
1078 	/*
1079 	 * We store the ones which need to be masked as that is what PCode
1080 	 * actually accepts as a parameter.
1081 	 */
1082 	new_bw_state->qgv_points_mask =
1083 		~(ICL_PCODE_REQ_QGV_PT(qgv_points) |
1084 		  ADLS_PCODE_REQ_PSF_PT(psf_points)) &
1085 		icl_qgv_points_mask(dev_priv);
1086 
1087 	/*
1088 	 * If the actual mask had changed we need to make sure that
1089 	 * the commits are serialized(in case this is a nomodeset, nonblocking)
1090 	 */
1091 	if (new_bw_state->qgv_points_mask != old_bw_state->qgv_points_mask) {
1092 		ret = intel_atomic_serialize_global_state(&new_bw_state->base);
1093 		if (ret)
1094 			return ret;
1095 	}
1096 
1097 	return 0;
1098 }
1099 
1100 static struct intel_global_state *
intel_bw_duplicate_state(struct intel_global_obj * obj)1101 intel_bw_duplicate_state(struct intel_global_obj *obj)
1102 {
1103 	struct intel_bw_state *state;
1104 
1105 	state = kmemdup(obj->state, sizeof(*state), GFP_KERNEL);
1106 	if (!state)
1107 		return NULL;
1108 
1109 	return &state->base;
1110 }
1111 
intel_bw_destroy_state(struct intel_global_obj * obj,struct intel_global_state * state)1112 static void intel_bw_destroy_state(struct intel_global_obj *obj,
1113 				   struct intel_global_state *state)
1114 {
1115 	kfree(state);
1116 }
1117 
1118 static const struct intel_global_state_funcs intel_bw_funcs = {
1119 	.atomic_duplicate_state = intel_bw_duplicate_state,
1120 	.atomic_destroy_state = intel_bw_destroy_state,
1121 };
1122 
intel_bw_init(struct drm_i915_private * dev_priv)1123 int intel_bw_init(struct drm_i915_private *dev_priv)
1124 {
1125 	struct intel_bw_state *state;
1126 
1127 	state = kzalloc(sizeof(*state), GFP_KERNEL);
1128 	if (!state)
1129 		return -ENOMEM;
1130 
1131 	intel_atomic_global_obj_init(dev_priv, &dev_priv->bw_obj,
1132 				     &state->base, &intel_bw_funcs);
1133 
1134 	return 0;
1135 }
1136