1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2019 Intel Corporation
4  */
5 
6 #include <linux/string_helpers.h>
7 
8 #include <drm/i915_drm.h>
9 
10 #include "i915_drv.h"
11 #include "i915_irq.h"
12 #include "intel_breadcrumbs.h"
13 #include "intel_gt.h"
14 #include "intel_gt_clock_utils.h"
15 #include "intel_gt_irq.h"
16 #include "intel_gt_pm_irq.h"
17 #include "intel_gt_regs.h"
18 #include "intel_mchbar_regs.h"
19 #include "intel_pcode.h"
20 #include "intel_rps.h"
21 #include "vlv_sideband.h"
22 #include "../../../platform/x86/intel_ips.h"
23 
24 #define BUSY_MAX_EI	20u /* ms */
25 
26 /*
27  * Lock protecting IPS related data structures
28  */
29 static DEFINE_SPINLOCK(mchdev_lock);
30 
rps_to_gt(struct intel_rps * rps)31 static struct intel_gt *rps_to_gt(struct intel_rps *rps)
32 {
33 	return container_of(rps, struct intel_gt, rps);
34 }
35 
rps_to_i915(struct intel_rps * rps)36 static struct drm_i915_private *rps_to_i915(struct intel_rps *rps)
37 {
38 	return rps_to_gt(rps)->i915;
39 }
40 
rps_to_uncore(struct intel_rps * rps)41 static struct intel_uncore *rps_to_uncore(struct intel_rps *rps)
42 {
43 	return rps_to_gt(rps)->uncore;
44 }
45 
rps_to_slpc(struct intel_rps * rps)46 static struct intel_guc_slpc *rps_to_slpc(struct intel_rps *rps)
47 {
48 	struct intel_gt *gt = rps_to_gt(rps);
49 
50 	return &gt->uc.guc.slpc;
51 }
52 
rps_uses_slpc(struct intel_rps * rps)53 static bool rps_uses_slpc(struct intel_rps *rps)
54 {
55 	struct intel_gt *gt = rps_to_gt(rps);
56 
57 	return intel_uc_uses_guc_slpc(&gt->uc);
58 }
59 
rps_pm_sanitize_mask(struct intel_rps * rps,u32 mask)60 static u32 rps_pm_sanitize_mask(struct intel_rps *rps, u32 mask)
61 {
62 	return mask & ~rps->pm_intrmsk_mbz;
63 }
64 
set(struct intel_uncore * uncore,i915_reg_t reg,u32 val)65 static void set(struct intel_uncore *uncore, i915_reg_t reg, u32 val)
66 {
67 	intel_uncore_write_fw(uncore, reg, val);
68 }
69 
rps_timer(struct timer_list * t)70 static void rps_timer(struct timer_list *t)
71 {
72 	struct intel_rps *rps = from_timer(rps, t, timer);
73 	struct intel_engine_cs *engine;
74 	ktime_t dt, last, timestamp;
75 	enum intel_engine_id id;
76 	s64 max_busy[3] = {};
77 
78 	timestamp = 0;
79 	for_each_engine(engine, rps_to_gt(rps), id) {
80 		s64 busy;
81 		int i;
82 
83 		dt = intel_engine_get_busy_time(engine, &timestamp);
84 		last = engine->stats.rps;
85 		engine->stats.rps = dt;
86 
87 		busy = ktime_to_ns(ktime_sub(dt, last));
88 		for (i = 0; i < ARRAY_SIZE(max_busy); i++) {
89 			if (busy > max_busy[i])
90 				swap(busy, max_busy[i]);
91 		}
92 	}
93 	last = rps->pm_timestamp;
94 	rps->pm_timestamp = timestamp;
95 
96 	if (intel_rps_is_active(rps)) {
97 		s64 busy;
98 		int i;
99 
100 		dt = ktime_sub(timestamp, last);
101 
102 		/*
103 		 * Our goal is to evaluate each engine independently, so we run
104 		 * at the lowest clocks required to sustain the heaviest
105 		 * workload. However, a task may be split into sequential
106 		 * dependent operations across a set of engines, such that
107 		 * the independent contributions do not account for high load,
108 		 * but overall the task is GPU bound. For example, consider
109 		 * video decode on vcs followed by colour post-processing
110 		 * on vecs, followed by general post-processing on rcs.
111 		 * Since multi-engines being active does imply a single
112 		 * continuous workload across all engines, we hedge our
113 		 * bets by only contributing a factor of the distributed
114 		 * load into our busyness calculation.
115 		 */
116 		busy = max_busy[0];
117 		for (i = 1; i < ARRAY_SIZE(max_busy); i++) {
118 			if (!max_busy[i])
119 				break;
120 
121 			busy += div_u64(max_busy[i], 1 << i);
122 		}
123 		GT_TRACE(rps_to_gt(rps),
124 			 "busy:%lld [%d%%], max:[%lld, %lld, %lld], interval:%d\n",
125 			 busy, (int)div64_u64(100 * busy, dt),
126 			 max_busy[0], max_busy[1], max_busy[2],
127 			 rps->pm_interval);
128 
129 		if (100 * busy > rps->power.up_threshold * dt &&
130 		    rps->cur_freq < rps->max_freq_softlimit) {
131 			rps->pm_iir |= GEN6_PM_RP_UP_THRESHOLD;
132 			rps->pm_interval = 1;
133 			schedule_work(&rps->work);
134 		} else if (100 * busy < rps->power.down_threshold * dt &&
135 			   rps->cur_freq > rps->min_freq_softlimit) {
136 			rps->pm_iir |= GEN6_PM_RP_DOWN_THRESHOLD;
137 			rps->pm_interval = 1;
138 			schedule_work(&rps->work);
139 		} else {
140 			rps->last_adj = 0;
141 		}
142 
143 		mod_timer(&rps->timer,
144 			  jiffies + msecs_to_jiffies(rps->pm_interval));
145 		rps->pm_interval = min(rps->pm_interval * 2, BUSY_MAX_EI);
146 	}
147 }
148 
rps_start_timer(struct intel_rps * rps)149 static void rps_start_timer(struct intel_rps *rps)
150 {
151 	rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp);
152 	rps->pm_interval = 1;
153 	mod_timer(&rps->timer, jiffies + 1);
154 }
155 
rps_stop_timer(struct intel_rps * rps)156 static void rps_stop_timer(struct intel_rps *rps)
157 {
158 	del_timer_sync(&rps->timer);
159 	rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp);
160 	cancel_work_sync(&rps->work);
161 }
162 
rps_pm_mask(struct intel_rps * rps,u8 val)163 static u32 rps_pm_mask(struct intel_rps *rps, u8 val)
164 {
165 	u32 mask = 0;
166 
167 	/* We use UP_EI_EXPIRED interrupts for both up/down in manual mode */
168 	if (val > rps->min_freq_softlimit)
169 		mask |= (GEN6_PM_RP_UP_EI_EXPIRED |
170 			 GEN6_PM_RP_DOWN_THRESHOLD |
171 			 GEN6_PM_RP_DOWN_TIMEOUT);
172 
173 	if (val < rps->max_freq_softlimit)
174 		mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD;
175 
176 	mask &= rps->pm_events;
177 
178 	return rps_pm_sanitize_mask(rps, ~mask);
179 }
180 
rps_reset_ei(struct intel_rps * rps)181 static void rps_reset_ei(struct intel_rps *rps)
182 {
183 	memset(&rps->ei, 0, sizeof(rps->ei));
184 }
185 
rps_enable_interrupts(struct intel_rps * rps)186 static void rps_enable_interrupts(struct intel_rps *rps)
187 {
188 	struct intel_gt *gt = rps_to_gt(rps);
189 
190 	GEM_BUG_ON(rps_uses_slpc(rps));
191 
192 	GT_TRACE(gt, "interrupts:on rps->pm_events: %x, rps_pm_mask:%x\n",
193 		 rps->pm_events, rps_pm_mask(rps, rps->last_freq));
194 
195 	rps_reset_ei(rps);
196 
197 	spin_lock_irq(gt->irq_lock);
198 	gen6_gt_pm_enable_irq(gt, rps->pm_events);
199 	spin_unlock_irq(gt->irq_lock);
200 
201 	intel_uncore_write(gt->uncore,
202 			   GEN6_PMINTRMSK, rps_pm_mask(rps, rps->last_freq));
203 }
204 
gen6_rps_reset_interrupts(struct intel_rps * rps)205 static void gen6_rps_reset_interrupts(struct intel_rps *rps)
206 {
207 	gen6_gt_pm_reset_iir(rps_to_gt(rps), GEN6_PM_RPS_EVENTS);
208 }
209 
gen11_rps_reset_interrupts(struct intel_rps * rps)210 static void gen11_rps_reset_interrupts(struct intel_rps *rps)
211 {
212 	while (gen11_gt_reset_one_iir(rps_to_gt(rps), 0, GEN11_GTPM))
213 		;
214 }
215 
rps_reset_interrupts(struct intel_rps * rps)216 static void rps_reset_interrupts(struct intel_rps *rps)
217 {
218 	struct intel_gt *gt = rps_to_gt(rps);
219 
220 	spin_lock_irq(gt->irq_lock);
221 	if (GRAPHICS_VER(gt->i915) >= 11)
222 		gen11_rps_reset_interrupts(rps);
223 	else
224 		gen6_rps_reset_interrupts(rps);
225 
226 	rps->pm_iir = 0;
227 	spin_unlock_irq(gt->irq_lock);
228 }
229 
rps_disable_interrupts(struct intel_rps * rps)230 static void rps_disable_interrupts(struct intel_rps *rps)
231 {
232 	struct intel_gt *gt = rps_to_gt(rps);
233 
234 	intel_uncore_write(gt->uncore,
235 			   GEN6_PMINTRMSK, rps_pm_sanitize_mask(rps, ~0u));
236 
237 	spin_lock_irq(gt->irq_lock);
238 	gen6_gt_pm_disable_irq(gt, GEN6_PM_RPS_EVENTS);
239 	spin_unlock_irq(gt->irq_lock);
240 
241 	intel_synchronize_irq(gt->i915);
242 
243 	/*
244 	 * Now that we will not be generating any more work, flush any
245 	 * outstanding tasks. As we are called on the RPS idle path,
246 	 * we will reset the GPU to minimum frequencies, so the current
247 	 * state of the worker can be discarded.
248 	 */
249 	cancel_work_sync(&rps->work);
250 
251 	rps_reset_interrupts(rps);
252 	GT_TRACE(gt, "interrupts:off\n");
253 }
254 
255 static const struct cparams {
256 	u16 i;
257 	u16 t;
258 	u16 m;
259 	u16 c;
260 } cparams[] = {
261 	{ 1, 1333, 301, 28664 },
262 	{ 1, 1066, 294, 24460 },
263 	{ 1, 800, 294, 25192 },
264 	{ 0, 1333, 276, 27605 },
265 	{ 0, 1066, 276, 27605 },
266 	{ 0, 800, 231, 23784 },
267 };
268 
gen5_rps_init(struct intel_rps * rps)269 static void gen5_rps_init(struct intel_rps *rps)
270 {
271 	struct drm_i915_private *i915 = rps_to_i915(rps);
272 	struct intel_uncore *uncore = rps_to_uncore(rps);
273 	u8 fmax, fmin, fstart;
274 	u32 rgvmodectl;
275 	int c_m, i;
276 
277 	if (i915->fsb_freq <= 3200)
278 		c_m = 0;
279 	else if (i915->fsb_freq <= 4800)
280 		c_m = 1;
281 	else
282 		c_m = 2;
283 
284 	for (i = 0; i < ARRAY_SIZE(cparams); i++) {
285 		if (cparams[i].i == c_m && cparams[i].t == i915->mem_freq) {
286 			rps->ips.m = cparams[i].m;
287 			rps->ips.c = cparams[i].c;
288 			break;
289 		}
290 	}
291 
292 	rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);
293 
294 	/* Set up min, max, and cur for interrupt handling */
295 	fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
296 	fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
297 	fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
298 		MEMMODE_FSTART_SHIFT;
299 	drm_dbg(&i915->drm, "fmax: %d, fmin: %d, fstart: %d\n",
300 		fmax, fmin, fstart);
301 
302 	rps->min_freq = fmax;
303 	rps->efficient_freq = fstart;
304 	rps->max_freq = fmin;
305 }
306 
307 static unsigned long
__ips_chipset_val(struct intel_ips * ips)308 __ips_chipset_val(struct intel_ips *ips)
309 {
310 	struct intel_uncore *uncore =
311 		rps_to_uncore(container_of(ips, struct intel_rps, ips));
312 	unsigned long now = jiffies_to_msecs(jiffies), dt;
313 	unsigned long result;
314 	u64 total, delta;
315 
316 	lockdep_assert_held(&mchdev_lock);
317 
318 	/*
319 	 * Prevent division-by-zero if we are asking too fast.
320 	 * Also, we don't get interesting results if we are polling
321 	 * faster than once in 10ms, so just return the saved value
322 	 * in such cases.
323 	 */
324 	dt = now - ips->last_time1;
325 	if (dt <= 10)
326 		return ips->chipset_power;
327 
328 	/* FIXME: handle per-counter overflow */
329 	total = intel_uncore_read(uncore, DMIEC);
330 	total += intel_uncore_read(uncore, DDREC);
331 	total += intel_uncore_read(uncore, CSIEC);
332 
333 	delta = total - ips->last_count1;
334 
335 	result = div_u64(div_u64(ips->m * delta, dt) + ips->c, 10);
336 
337 	ips->last_count1 = total;
338 	ips->last_time1 = now;
339 
340 	ips->chipset_power = result;
341 
342 	return result;
343 }
344 
ips_mch_val(struct intel_uncore * uncore)345 static unsigned long ips_mch_val(struct intel_uncore *uncore)
346 {
347 	unsigned int m, x, b;
348 	u32 tsfs;
349 
350 	tsfs = intel_uncore_read(uncore, TSFS);
351 	x = intel_uncore_read8(uncore, TR1);
352 
353 	b = tsfs & TSFS_INTR_MASK;
354 	m = (tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT;
355 
356 	return m * x / 127 - b;
357 }
358 
_pxvid_to_vd(u8 pxvid)359 static int _pxvid_to_vd(u8 pxvid)
360 {
361 	if (pxvid == 0)
362 		return 0;
363 
364 	if (pxvid >= 8 && pxvid < 31)
365 		pxvid = 31;
366 
367 	return (pxvid + 2) * 125;
368 }
369 
pvid_to_extvid(struct drm_i915_private * i915,u8 pxvid)370 static u32 pvid_to_extvid(struct drm_i915_private *i915, u8 pxvid)
371 {
372 	const int vd = _pxvid_to_vd(pxvid);
373 
374 	if (INTEL_INFO(i915)->is_mobile)
375 		return max(vd - 1125, 0);
376 
377 	return vd;
378 }
379 
__gen5_ips_update(struct intel_ips * ips)380 static void __gen5_ips_update(struct intel_ips *ips)
381 {
382 	struct intel_uncore *uncore =
383 		rps_to_uncore(container_of(ips, struct intel_rps, ips));
384 	u64 now, delta, dt;
385 	u32 count;
386 
387 	lockdep_assert_held(&mchdev_lock);
388 
389 	now = ktime_get_raw_ns();
390 	dt = now - ips->last_time2;
391 	do_div(dt, NSEC_PER_MSEC);
392 
393 	/* Don't divide by 0 */
394 	if (dt <= 10)
395 		return;
396 
397 	count = intel_uncore_read(uncore, GFXEC);
398 	delta = count - ips->last_count2;
399 
400 	ips->last_count2 = count;
401 	ips->last_time2 = now;
402 
403 	/* More magic constants... */
404 	ips->gfx_power = div_u64(delta * 1181, dt * 10);
405 }
406 
gen5_rps_update(struct intel_rps * rps)407 static void gen5_rps_update(struct intel_rps *rps)
408 {
409 	spin_lock_irq(&mchdev_lock);
410 	__gen5_ips_update(&rps->ips);
411 	spin_unlock_irq(&mchdev_lock);
412 }
413 
gen5_invert_freq(struct intel_rps * rps,unsigned int val)414 static unsigned int gen5_invert_freq(struct intel_rps *rps,
415 				     unsigned int val)
416 {
417 	/* Invert the frequency bin into an ips delay */
418 	val = rps->max_freq - val;
419 	val = rps->min_freq + val;
420 
421 	return val;
422 }
423 
__gen5_rps_set(struct intel_rps * rps,u8 val)424 static int __gen5_rps_set(struct intel_rps *rps, u8 val)
425 {
426 	struct intel_uncore *uncore = rps_to_uncore(rps);
427 	u16 rgvswctl;
428 
429 	lockdep_assert_held(&mchdev_lock);
430 
431 	rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
432 	if (rgvswctl & MEMCTL_CMD_STS) {
433 		DRM_DEBUG("gpu busy, RCS change rejected\n");
434 		return -EBUSY; /* still busy with another command */
435 	}
436 
437 	/* Invert the frequency bin into an ips delay */
438 	val = gen5_invert_freq(rps, val);
439 
440 	rgvswctl =
441 		(MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |
442 		(val << MEMCTL_FREQ_SHIFT) |
443 		MEMCTL_SFCAVM;
444 	intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);
445 	intel_uncore_posting_read16(uncore, MEMSWCTL);
446 
447 	rgvswctl |= MEMCTL_CMD_STS;
448 	intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);
449 
450 	return 0;
451 }
452 
gen5_rps_set(struct intel_rps * rps,u8 val)453 static int gen5_rps_set(struct intel_rps *rps, u8 val)
454 {
455 	int err;
456 
457 	spin_lock_irq(&mchdev_lock);
458 	err = __gen5_rps_set(rps, val);
459 	spin_unlock_irq(&mchdev_lock);
460 
461 	return err;
462 }
463 
intel_pxfreq(u32 vidfreq)464 static unsigned long intel_pxfreq(u32 vidfreq)
465 {
466 	int div = (vidfreq & 0x3f0000) >> 16;
467 	int post = (vidfreq & 0x3000) >> 12;
468 	int pre = (vidfreq & 0x7);
469 
470 	if (!pre)
471 		return 0;
472 
473 	return div * 133333 / (pre << post);
474 }
475 
init_emon(struct intel_uncore * uncore)476 static unsigned int init_emon(struct intel_uncore *uncore)
477 {
478 	u8 pxw[16];
479 	int i;
480 
481 	/* Disable to program */
482 	intel_uncore_write(uncore, ECR, 0);
483 	intel_uncore_posting_read(uncore, ECR);
484 
485 	/* Program energy weights for various events */
486 	intel_uncore_write(uncore, SDEW, 0x15040d00);
487 	intel_uncore_write(uncore, CSIEW0, 0x007f0000);
488 	intel_uncore_write(uncore, CSIEW1, 0x1e220004);
489 	intel_uncore_write(uncore, CSIEW2, 0x04000004);
490 
491 	for (i = 0; i < 5; i++)
492 		intel_uncore_write(uncore, PEW(i), 0);
493 	for (i = 0; i < 3; i++)
494 		intel_uncore_write(uncore, DEW(i), 0);
495 
496 	/* Program P-state weights to account for frequency power adjustment */
497 	for (i = 0; i < 16; i++) {
498 		u32 pxvidfreq = intel_uncore_read(uncore, PXVFREQ(i));
499 		unsigned int freq = intel_pxfreq(pxvidfreq);
500 		unsigned int vid =
501 			(pxvidfreq & PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;
502 		unsigned int val;
503 
504 		val = vid * vid * freq / 1000 * 255;
505 		val /= 127 * 127 * 900;
506 
507 		pxw[i] = val;
508 	}
509 	/* Render standby states get 0 weight */
510 	pxw[14] = 0;
511 	pxw[15] = 0;
512 
513 	for (i = 0; i < 4; i++) {
514 		intel_uncore_write(uncore, PXW(i),
515 				   pxw[i * 4 + 0] << 24 |
516 				   pxw[i * 4 + 1] << 16 |
517 				   pxw[i * 4 + 2] <<  8 |
518 				   pxw[i * 4 + 3] <<  0);
519 	}
520 
521 	/* Adjust magic regs to magic values (more experimental results) */
522 	intel_uncore_write(uncore, OGW0, 0);
523 	intel_uncore_write(uncore, OGW1, 0);
524 	intel_uncore_write(uncore, EG0, 0x00007f00);
525 	intel_uncore_write(uncore, EG1, 0x0000000e);
526 	intel_uncore_write(uncore, EG2, 0x000e0000);
527 	intel_uncore_write(uncore, EG3, 0x68000300);
528 	intel_uncore_write(uncore, EG4, 0x42000000);
529 	intel_uncore_write(uncore, EG5, 0x00140031);
530 	intel_uncore_write(uncore, EG6, 0);
531 	intel_uncore_write(uncore, EG7, 0);
532 
533 	for (i = 0; i < 8; i++)
534 		intel_uncore_write(uncore, PXWL(i), 0);
535 
536 	/* Enable PMON + select events */
537 	intel_uncore_write(uncore, ECR, 0x80000019);
538 
539 	return intel_uncore_read(uncore, LCFUSE02) & LCFUSE_HIV_MASK;
540 }
541 
gen5_rps_enable(struct intel_rps * rps)542 static bool gen5_rps_enable(struct intel_rps *rps)
543 {
544 	struct drm_i915_private *i915 = rps_to_i915(rps);
545 	struct intel_uncore *uncore = rps_to_uncore(rps);
546 	u8 fstart, vstart;
547 	u32 rgvmodectl;
548 
549 	spin_lock_irq(&mchdev_lock);
550 
551 	rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);
552 
553 	/* Enable temp reporting */
554 	intel_uncore_write16(uncore, PMMISC,
555 			     intel_uncore_read16(uncore, PMMISC) | MCPPCE_EN);
556 	intel_uncore_write16(uncore, TSC1,
557 			     intel_uncore_read16(uncore, TSC1) | TSE);
558 
559 	/* 100ms RC evaluation intervals */
560 	intel_uncore_write(uncore, RCUPEI, 100000);
561 	intel_uncore_write(uncore, RCDNEI, 100000);
562 
563 	/* Set max/min thresholds to 90ms and 80ms respectively */
564 	intel_uncore_write(uncore, RCBMAXAVG, 90000);
565 	intel_uncore_write(uncore, RCBMINAVG, 80000);
566 
567 	intel_uncore_write(uncore, MEMIHYST, 1);
568 
569 	/* Set up min, max, and cur for interrupt handling */
570 	fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
571 		MEMMODE_FSTART_SHIFT;
572 
573 	vstart = (intel_uncore_read(uncore, PXVFREQ(fstart)) &
574 		  PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;
575 
576 	intel_uncore_write(uncore,
577 			   MEMINTREN,
578 			   MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
579 
580 	intel_uncore_write(uncore, VIDSTART, vstart);
581 	intel_uncore_posting_read(uncore, VIDSTART);
582 
583 	rgvmodectl |= MEMMODE_SWMODE_EN;
584 	intel_uncore_write(uncore, MEMMODECTL, rgvmodectl);
585 
586 	if (wait_for_atomic((intel_uncore_read(uncore, MEMSWCTL) &
587 			     MEMCTL_CMD_STS) == 0, 10))
588 		drm_err(&uncore->i915->drm,
589 			"stuck trying to change perf mode\n");
590 	mdelay(1);
591 
592 	__gen5_rps_set(rps, rps->cur_freq);
593 
594 	rps->ips.last_count1 = intel_uncore_read(uncore, DMIEC);
595 	rps->ips.last_count1 += intel_uncore_read(uncore, DDREC);
596 	rps->ips.last_count1 += intel_uncore_read(uncore, CSIEC);
597 	rps->ips.last_time1 = jiffies_to_msecs(jiffies);
598 
599 	rps->ips.last_count2 = intel_uncore_read(uncore, GFXEC);
600 	rps->ips.last_time2 = ktime_get_raw_ns();
601 
602 	spin_lock(&i915->irq_lock);
603 	ilk_enable_display_irq(i915, DE_PCU_EVENT);
604 	spin_unlock(&i915->irq_lock);
605 
606 	spin_unlock_irq(&mchdev_lock);
607 
608 	rps->ips.corr = init_emon(uncore);
609 
610 	return true;
611 }
612 
gen5_rps_disable(struct intel_rps * rps)613 static void gen5_rps_disable(struct intel_rps *rps)
614 {
615 	struct drm_i915_private *i915 = rps_to_i915(rps);
616 	struct intel_uncore *uncore = rps_to_uncore(rps);
617 	u16 rgvswctl;
618 
619 	spin_lock_irq(&mchdev_lock);
620 
621 	spin_lock(&i915->irq_lock);
622 	ilk_disable_display_irq(i915, DE_PCU_EVENT);
623 	spin_unlock(&i915->irq_lock);
624 
625 	rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
626 
627 	/* Ack interrupts, disable EFC interrupt */
628 	intel_uncore_write(uncore, MEMINTREN,
629 			   intel_uncore_read(uncore, MEMINTREN) &
630 			   ~MEMINT_EVAL_CHG_EN);
631 	intel_uncore_write(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
632 
633 	/* Go back to the starting frequency */
634 	__gen5_rps_set(rps, rps->idle_freq);
635 	mdelay(1);
636 	rgvswctl |= MEMCTL_CMD_STS;
637 	intel_uncore_write(uncore, MEMSWCTL, rgvswctl);
638 	mdelay(1);
639 
640 	spin_unlock_irq(&mchdev_lock);
641 }
642 
rps_limits(struct intel_rps * rps,u8 val)643 static u32 rps_limits(struct intel_rps *rps, u8 val)
644 {
645 	u32 limits;
646 
647 	/*
648 	 * Only set the down limit when we've reached the lowest level to avoid
649 	 * getting more interrupts, otherwise leave this clear. This prevents a
650 	 * race in the hw when coming out of rc6: There's a tiny window where
651 	 * the hw runs at the minimal clock before selecting the desired
652 	 * frequency, if the down threshold expires in that window we will not
653 	 * receive a down interrupt.
654 	 */
655 	if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) {
656 		limits = rps->max_freq_softlimit << 23;
657 		if (val <= rps->min_freq_softlimit)
658 			limits |= rps->min_freq_softlimit << 14;
659 	} else {
660 		limits = rps->max_freq_softlimit << 24;
661 		if (val <= rps->min_freq_softlimit)
662 			limits |= rps->min_freq_softlimit << 16;
663 	}
664 
665 	return limits;
666 }
667 
rps_set_power(struct intel_rps * rps,int new_power)668 static void rps_set_power(struct intel_rps *rps, int new_power)
669 {
670 	struct intel_gt *gt = rps_to_gt(rps);
671 	struct intel_uncore *uncore = gt->uncore;
672 	u32 threshold_up = 0, threshold_down = 0; /* in % */
673 	u32 ei_up = 0, ei_down = 0;
674 
675 	lockdep_assert_held(&rps->power.mutex);
676 
677 	if (new_power == rps->power.mode)
678 		return;
679 
680 	threshold_up = 95;
681 	threshold_down = 85;
682 
683 	/* Note the units here are not exactly 1us, but 1280ns. */
684 	switch (new_power) {
685 	case LOW_POWER:
686 		ei_up = 16000;
687 		ei_down = 32000;
688 		break;
689 
690 	case BETWEEN:
691 		ei_up = 13000;
692 		ei_down = 32000;
693 		break;
694 
695 	case HIGH_POWER:
696 		ei_up = 10000;
697 		ei_down = 32000;
698 		break;
699 	}
700 
701 	/* When byt can survive without system hang with dynamic
702 	 * sw freq adjustments, this restriction can be lifted.
703 	 */
704 	if (IS_VALLEYVIEW(gt->i915))
705 		goto skip_hw_write;
706 
707 	GT_TRACE(gt,
708 		 "changing power mode [%d], up %d%% @ %dus, down %d%% @ %dus\n",
709 		 new_power, threshold_up, ei_up, threshold_down, ei_down);
710 
711 	set(uncore, GEN6_RP_UP_EI,
712 	    intel_gt_ns_to_pm_interval(gt, ei_up * 1000));
713 	set(uncore, GEN6_RP_UP_THRESHOLD,
714 	    intel_gt_ns_to_pm_interval(gt, ei_up * threshold_up * 10));
715 
716 	set(uncore, GEN6_RP_DOWN_EI,
717 	    intel_gt_ns_to_pm_interval(gt, ei_down * 1000));
718 	set(uncore, GEN6_RP_DOWN_THRESHOLD,
719 	    intel_gt_ns_to_pm_interval(gt, ei_down * threshold_down * 10));
720 
721 	set(uncore, GEN6_RP_CONTROL,
722 	    (GRAPHICS_VER(gt->i915) > 9 ? 0 : GEN6_RP_MEDIA_TURBO) |
723 	    GEN6_RP_MEDIA_HW_NORMAL_MODE |
724 	    GEN6_RP_MEDIA_IS_GFX |
725 	    GEN6_RP_ENABLE |
726 	    GEN6_RP_UP_BUSY_AVG |
727 	    GEN6_RP_DOWN_IDLE_AVG);
728 
729 skip_hw_write:
730 	rps->power.mode = new_power;
731 	rps->power.up_threshold = threshold_up;
732 	rps->power.down_threshold = threshold_down;
733 }
734 
gen6_rps_set_thresholds(struct intel_rps * rps,u8 val)735 static void gen6_rps_set_thresholds(struct intel_rps *rps, u8 val)
736 {
737 	int new_power;
738 
739 	new_power = rps->power.mode;
740 	switch (rps->power.mode) {
741 	case LOW_POWER:
742 		if (val > rps->efficient_freq + 1 &&
743 		    val > rps->cur_freq)
744 			new_power = BETWEEN;
745 		break;
746 
747 	case BETWEEN:
748 		if (val <= rps->efficient_freq &&
749 		    val < rps->cur_freq)
750 			new_power = LOW_POWER;
751 		else if (val >= rps->rp0_freq &&
752 			 val > rps->cur_freq)
753 			new_power = HIGH_POWER;
754 		break;
755 
756 	case HIGH_POWER:
757 		if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 &&
758 		    val < rps->cur_freq)
759 			new_power = BETWEEN;
760 		break;
761 	}
762 	/* Max/min bins are special */
763 	if (val <= rps->min_freq_softlimit)
764 		new_power = LOW_POWER;
765 	if (val >= rps->max_freq_softlimit)
766 		new_power = HIGH_POWER;
767 
768 	mutex_lock(&rps->power.mutex);
769 	if (rps->power.interactive)
770 		new_power = HIGH_POWER;
771 	rps_set_power(rps, new_power);
772 	mutex_unlock(&rps->power.mutex);
773 }
774 
intel_rps_mark_interactive(struct intel_rps * rps,bool interactive)775 void intel_rps_mark_interactive(struct intel_rps *rps, bool interactive)
776 {
777 	GT_TRACE(rps_to_gt(rps), "mark interactive: %s\n",
778 		 str_yes_no(interactive));
779 
780 	mutex_lock(&rps->power.mutex);
781 	if (interactive) {
782 		if (!rps->power.interactive++ && intel_rps_is_active(rps))
783 			rps_set_power(rps, HIGH_POWER);
784 	} else {
785 		GEM_BUG_ON(!rps->power.interactive);
786 		rps->power.interactive--;
787 	}
788 	mutex_unlock(&rps->power.mutex);
789 }
790 
gen6_rps_set(struct intel_rps * rps,u8 val)791 static int gen6_rps_set(struct intel_rps *rps, u8 val)
792 {
793 	struct intel_uncore *uncore = rps_to_uncore(rps);
794 	struct drm_i915_private *i915 = rps_to_i915(rps);
795 	u32 swreq;
796 
797 	GEM_BUG_ON(rps_uses_slpc(rps));
798 
799 	if (GRAPHICS_VER(i915) >= 9)
800 		swreq = GEN9_FREQUENCY(val);
801 	else if (IS_HASWELL(i915) || IS_BROADWELL(i915))
802 		swreq = HSW_FREQUENCY(val);
803 	else
804 		swreq = (GEN6_FREQUENCY(val) |
805 			 GEN6_OFFSET(0) |
806 			 GEN6_AGGRESSIVE_TURBO);
807 	set(uncore, GEN6_RPNSWREQ, swreq);
808 
809 	GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d, swreq:%x\n",
810 		 val, intel_gpu_freq(rps, val), swreq);
811 
812 	return 0;
813 }
814 
vlv_rps_set(struct intel_rps * rps,u8 val)815 static int vlv_rps_set(struct intel_rps *rps, u8 val)
816 {
817 	struct drm_i915_private *i915 = rps_to_i915(rps);
818 	int err;
819 
820 	vlv_punit_get(i915);
821 	err = vlv_punit_write(i915, PUNIT_REG_GPU_FREQ_REQ, val);
822 	vlv_punit_put(i915);
823 
824 	GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d\n",
825 		 val, intel_gpu_freq(rps, val));
826 
827 	return err;
828 }
829 
rps_set(struct intel_rps * rps,u8 val,bool update)830 static int rps_set(struct intel_rps *rps, u8 val, bool update)
831 {
832 	struct drm_i915_private *i915 = rps_to_i915(rps);
833 	int err;
834 
835 	if (val == rps->last_freq)
836 		return 0;
837 
838 	if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
839 		err = vlv_rps_set(rps, val);
840 	else if (GRAPHICS_VER(i915) >= 6)
841 		err = gen6_rps_set(rps, val);
842 	else
843 		err = gen5_rps_set(rps, val);
844 	if (err)
845 		return err;
846 
847 	if (update && GRAPHICS_VER(i915) >= 6)
848 		gen6_rps_set_thresholds(rps, val);
849 	rps->last_freq = val;
850 
851 	return 0;
852 }
853 
intel_rps_unpark(struct intel_rps * rps)854 void intel_rps_unpark(struct intel_rps *rps)
855 {
856 	if (!intel_rps_is_enabled(rps))
857 		return;
858 
859 	GT_TRACE(rps_to_gt(rps), "unpark:%x\n", rps->cur_freq);
860 
861 	/*
862 	 * Use the user's desired frequency as a guide, but for better
863 	 * performance, jump directly to RPe as our starting frequency.
864 	 */
865 	mutex_lock(&rps->lock);
866 
867 	intel_rps_set_active(rps);
868 	intel_rps_set(rps,
869 		      clamp(rps->cur_freq,
870 			    rps->min_freq_softlimit,
871 			    rps->max_freq_softlimit));
872 
873 	mutex_unlock(&rps->lock);
874 
875 	rps->pm_iir = 0;
876 	if (intel_rps_has_interrupts(rps))
877 		rps_enable_interrupts(rps);
878 	if (intel_rps_uses_timer(rps))
879 		rps_start_timer(rps);
880 
881 	if (GRAPHICS_VER(rps_to_i915(rps)) == 5)
882 		gen5_rps_update(rps);
883 }
884 
intel_rps_park(struct intel_rps * rps)885 void intel_rps_park(struct intel_rps *rps)
886 {
887 	int adj;
888 
889 	if (!intel_rps_is_enabled(rps))
890 		return;
891 
892 	if (!intel_rps_clear_active(rps))
893 		return;
894 
895 	if (intel_rps_uses_timer(rps))
896 		rps_stop_timer(rps);
897 	if (intel_rps_has_interrupts(rps))
898 		rps_disable_interrupts(rps);
899 
900 	if (rps->last_freq <= rps->idle_freq)
901 		return;
902 
903 	/*
904 	 * The punit delays the write of the frequency and voltage until it
905 	 * determines the GPU is awake. During normal usage we don't want to
906 	 * waste power changing the frequency if the GPU is sleeping (rc6).
907 	 * However, the GPU and driver is now idle and we do not want to delay
908 	 * switching to minimum voltage (reducing power whilst idle) as we do
909 	 * not expect to be woken in the near future and so must flush the
910 	 * change by waking the device.
911 	 *
912 	 * We choose to take the media powerwell (either would do to trick the
913 	 * punit into committing the voltage change) as that takes a lot less
914 	 * power than the render powerwell.
915 	 */
916 	intel_uncore_forcewake_get(rps_to_uncore(rps), FORCEWAKE_MEDIA);
917 	rps_set(rps, rps->idle_freq, false);
918 	intel_uncore_forcewake_put(rps_to_uncore(rps), FORCEWAKE_MEDIA);
919 
920 	/*
921 	 * Since we will try and restart from the previously requested
922 	 * frequency on unparking, treat this idle point as a downclock
923 	 * interrupt and reduce the frequency for resume. If we park/unpark
924 	 * more frequently than the rps worker can run, we will not respond
925 	 * to any EI and never see a change in frequency.
926 	 *
927 	 * (Note we accommodate Cherryview's limitation of only using an
928 	 * even bin by applying it to all.)
929 	 */
930 	adj = rps->last_adj;
931 	if (adj < 0)
932 		adj *= 2;
933 	else /* CHV needs even encode values */
934 		adj = -2;
935 	rps->last_adj = adj;
936 	rps->cur_freq = max_t(int, rps->cur_freq + adj, rps->min_freq);
937 	if (rps->cur_freq < rps->efficient_freq) {
938 		rps->cur_freq = rps->efficient_freq;
939 		rps->last_adj = 0;
940 	}
941 
942 	GT_TRACE(rps_to_gt(rps), "park:%x\n", rps->cur_freq);
943 }
944 
intel_rps_get_boost_frequency(struct intel_rps * rps)945 u32 intel_rps_get_boost_frequency(struct intel_rps *rps)
946 {
947 	struct intel_guc_slpc *slpc;
948 
949 	if (rps_uses_slpc(rps)) {
950 		slpc = rps_to_slpc(rps);
951 
952 		return slpc->boost_freq;
953 	} else {
954 		return intel_gpu_freq(rps, rps->boost_freq);
955 	}
956 }
957 
rps_set_boost_freq(struct intel_rps * rps,u32 val)958 static int rps_set_boost_freq(struct intel_rps *rps, u32 val)
959 {
960 	bool boost = false;
961 
962 	/* Validate against (static) hardware limits */
963 	val = intel_freq_opcode(rps, val);
964 	if (val < rps->min_freq || val > rps->max_freq)
965 		return -EINVAL;
966 
967 	mutex_lock(&rps->lock);
968 	if (val != rps->boost_freq) {
969 		rps->boost_freq = val;
970 		boost = atomic_read(&rps->num_waiters);
971 	}
972 	mutex_unlock(&rps->lock);
973 	if (boost)
974 		schedule_work(&rps->work);
975 
976 	return 0;
977 }
978 
intel_rps_set_boost_frequency(struct intel_rps * rps,u32 freq)979 int intel_rps_set_boost_frequency(struct intel_rps *rps, u32 freq)
980 {
981 	struct intel_guc_slpc *slpc;
982 
983 	if (rps_uses_slpc(rps)) {
984 		slpc = rps_to_slpc(rps);
985 
986 		return intel_guc_slpc_set_boost_freq(slpc, freq);
987 	} else {
988 		return rps_set_boost_freq(rps, freq);
989 	}
990 }
991 
intel_rps_dec_waiters(struct intel_rps * rps)992 void intel_rps_dec_waiters(struct intel_rps *rps)
993 {
994 	struct intel_guc_slpc *slpc;
995 
996 	if (rps_uses_slpc(rps)) {
997 		slpc = rps_to_slpc(rps);
998 
999 		intel_guc_slpc_dec_waiters(slpc);
1000 	} else {
1001 		atomic_dec(&rps->num_waiters);
1002 	}
1003 }
1004 
intel_rps_boost(struct i915_request * rq)1005 void intel_rps_boost(struct i915_request *rq)
1006 {
1007 	struct intel_guc_slpc *slpc;
1008 
1009 	if (i915_request_signaled(rq) || i915_request_has_waitboost(rq))
1010 		return;
1011 
1012 	/* Serializes with i915_request_retire() */
1013 	if (!test_and_set_bit(I915_FENCE_FLAG_BOOST, &rq->fence.flags)) {
1014 		struct intel_rps *rps = &READ_ONCE(rq->engine)->gt->rps;
1015 
1016 		if (rps_uses_slpc(rps)) {
1017 			slpc = rps_to_slpc(rps);
1018 
1019 			/* Return if old value is non zero */
1020 			if (!atomic_fetch_inc(&slpc->num_waiters))
1021 				schedule_work(&slpc->boost_work);
1022 
1023 			return;
1024 		}
1025 
1026 		if (atomic_fetch_inc(&rps->num_waiters))
1027 			return;
1028 
1029 		if (!intel_rps_is_active(rps))
1030 			return;
1031 
1032 		GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n",
1033 			 rq->fence.context, rq->fence.seqno);
1034 
1035 		if (READ_ONCE(rps->cur_freq) < rps->boost_freq)
1036 			schedule_work(&rps->work);
1037 
1038 		WRITE_ONCE(rps->boosts, rps->boosts + 1); /* debug only */
1039 	}
1040 }
1041 
intel_rps_set(struct intel_rps * rps,u8 val)1042 int intel_rps_set(struct intel_rps *rps, u8 val)
1043 {
1044 	int err;
1045 
1046 	lockdep_assert_held(&rps->lock);
1047 	GEM_BUG_ON(val > rps->max_freq);
1048 	GEM_BUG_ON(val < rps->min_freq);
1049 
1050 	if (intel_rps_is_active(rps)) {
1051 		err = rps_set(rps, val, true);
1052 		if (err)
1053 			return err;
1054 
1055 		/*
1056 		 * Make sure we continue to get interrupts
1057 		 * until we hit the minimum or maximum frequencies.
1058 		 */
1059 		if (intel_rps_has_interrupts(rps)) {
1060 			struct intel_uncore *uncore = rps_to_uncore(rps);
1061 
1062 			set(uncore,
1063 			    GEN6_RP_INTERRUPT_LIMITS, rps_limits(rps, val));
1064 
1065 			set(uncore, GEN6_PMINTRMSK, rps_pm_mask(rps, val));
1066 		}
1067 	}
1068 
1069 	rps->cur_freq = val;
1070 	return 0;
1071 }
1072 
intel_rps_read_state_cap(struct intel_rps * rps)1073 static u32 intel_rps_read_state_cap(struct intel_rps *rps)
1074 {
1075 	struct drm_i915_private *i915 = rps_to_i915(rps);
1076 	struct intel_uncore *uncore = rps_to_uncore(rps);
1077 
1078 	if (IS_PONTEVECCHIO(i915))
1079 		return intel_uncore_read(uncore, PVC_RP_STATE_CAP);
1080 	else if (IS_XEHPSDV(i915))
1081 		return intel_uncore_read(uncore, XEHPSDV_RP_STATE_CAP);
1082 	else if (IS_GEN9_LP(i915))
1083 		return intel_uncore_read(uncore, BXT_RP_STATE_CAP);
1084 	else
1085 		return intel_uncore_read(uncore, GEN6_RP_STATE_CAP);
1086 }
1087 
1088 /**
1089  * gen6_rps_get_freq_caps - Get freq caps exposed by HW
1090  * @rps: the intel_rps structure
1091  * @caps: returned freq caps
1092  *
1093  * Returned "caps" frequencies should be converted to MHz using
1094  * intel_gpu_freq()
1095  */
gen6_rps_get_freq_caps(struct intel_rps * rps,struct intel_rps_freq_caps * caps)1096 void gen6_rps_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps)
1097 {
1098 	struct drm_i915_private *i915 = rps_to_i915(rps);
1099 	u32 rp_state_cap;
1100 
1101 	rp_state_cap = intel_rps_read_state_cap(rps);
1102 
1103 	/* static values from HW: RP0 > RP1 > RPn (min_freq) */
1104 	if (IS_GEN9_LP(i915)) {
1105 		caps->rp0_freq = (rp_state_cap >> 16) & 0xff;
1106 		caps->rp1_freq = (rp_state_cap >>  8) & 0xff;
1107 		caps->min_freq = (rp_state_cap >>  0) & 0xff;
1108 	} else {
1109 		caps->rp0_freq = (rp_state_cap >>  0) & 0xff;
1110 		if (GRAPHICS_VER(i915) >= 10)
1111 			caps->rp1_freq = REG_FIELD_GET(RPE_MASK,
1112 						       intel_uncore_read(to_gt(i915)->uncore,
1113 						       GEN10_FREQ_INFO_REC));
1114 		else
1115 			caps->rp1_freq = (rp_state_cap >>  8) & 0xff;
1116 		caps->min_freq = (rp_state_cap >> 16) & 0xff;
1117 	}
1118 
1119 	if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) {
1120 		/*
1121 		 * In this case rp_state_cap register reports frequencies in
1122 		 * units of 50 MHz. Convert these to the actual "hw unit", i.e.
1123 		 * units of 16.67 MHz
1124 		 */
1125 		caps->rp0_freq *= GEN9_FREQ_SCALER;
1126 		caps->rp1_freq *= GEN9_FREQ_SCALER;
1127 		caps->min_freq *= GEN9_FREQ_SCALER;
1128 	}
1129 }
1130 
gen6_rps_init(struct intel_rps * rps)1131 static void gen6_rps_init(struct intel_rps *rps)
1132 {
1133 	struct drm_i915_private *i915 = rps_to_i915(rps);
1134 	struct intel_rps_freq_caps caps;
1135 
1136 	gen6_rps_get_freq_caps(rps, &caps);
1137 	rps->rp0_freq = caps.rp0_freq;
1138 	rps->rp1_freq = caps.rp1_freq;
1139 	rps->min_freq = caps.min_freq;
1140 
1141 	/* hw_max = RP0 until we check for overclocking */
1142 	rps->max_freq = rps->rp0_freq;
1143 
1144 	rps->efficient_freq = rps->rp1_freq;
1145 	if (IS_HASWELL(i915) || IS_BROADWELL(i915) ||
1146 	    IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) {
1147 		u32 ddcc_status = 0;
1148 		u32 mult = 1;
1149 
1150 		if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11)
1151 			mult = GEN9_FREQ_SCALER;
1152 		if (snb_pcode_read(rps_to_gt(rps)->uncore,
1153 				   HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
1154 				   &ddcc_status, NULL) == 0)
1155 			rps->efficient_freq =
1156 				clamp_t(u32,
1157 					((ddcc_status >> 8) & 0xff) * mult,
1158 					rps->min_freq,
1159 					rps->max_freq);
1160 	}
1161 }
1162 
rps_reset(struct intel_rps * rps)1163 static bool rps_reset(struct intel_rps *rps)
1164 {
1165 	struct drm_i915_private *i915 = rps_to_i915(rps);
1166 
1167 	/* force a reset */
1168 	rps->power.mode = -1;
1169 	rps->last_freq = -1;
1170 
1171 	if (rps_set(rps, rps->min_freq, true)) {
1172 		drm_err(&i915->drm, "Failed to reset RPS to initial values\n");
1173 		return false;
1174 	}
1175 
1176 	rps->cur_freq = rps->min_freq;
1177 	return true;
1178 }
1179 
1180 /* See the Gen9_GT_PM_Programming_Guide doc for the below */
gen9_rps_enable(struct intel_rps * rps)1181 static bool gen9_rps_enable(struct intel_rps *rps)
1182 {
1183 	struct intel_gt *gt = rps_to_gt(rps);
1184 	struct intel_uncore *uncore = gt->uncore;
1185 
1186 	/* Program defaults and thresholds for RPS */
1187 	if (GRAPHICS_VER(gt->i915) == 9)
1188 		intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ,
1189 				      GEN9_FREQUENCY(rps->rp1_freq));
1190 
1191 	intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 0xa);
1192 
1193 	rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD;
1194 
1195 	return rps_reset(rps);
1196 }
1197 
gen8_rps_enable(struct intel_rps * rps)1198 static bool gen8_rps_enable(struct intel_rps *rps)
1199 {
1200 	struct intel_uncore *uncore = rps_to_uncore(rps);
1201 
1202 	intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ,
1203 			      HSW_FREQUENCY(rps->rp1_freq));
1204 
1205 	intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1206 
1207 	rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD;
1208 
1209 	return rps_reset(rps);
1210 }
1211 
gen6_rps_enable(struct intel_rps * rps)1212 static bool gen6_rps_enable(struct intel_rps *rps)
1213 {
1214 	struct intel_uncore *uncore = rps_to_uncore(rps);
1215 
1216 	/* Power down if completely idle for over 50ms */
1217 	intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 50000);
1218 	intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1219 
1220 	rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD |
1221 			  GEN6_PM_RP_DOWN_THRESHOLD |
1222 			  GEN6_PM_RP_DOWN_TIMEOUT);
1223 
1224 	return rps_reset(rps);
1225 }
1226 
chv_rps_max_freq(struct intel_rps * rps)1227 static int chv_rps_max_freq(struct intel_rps *rps)
1228 {
1229 	struct drm_i915_private *i915 = rps_to_i915(rps);
1230 	struct intel_gt *gt = rps_to_gt(rps);
1231 	u32 val;
1232 
1233 	val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE);
1234 
1235 	switch (gt->info.sseu.eu_total) {
1236 	case 8:
1237 		/* (2 * 4) config */
1238 		val >>= FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT;
1239 		break;
1240 	case 12:
1241 		/* (2 * 6) config */
1242 		val >>= FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT;
1243 		break;
1244 	case 16:
1245 		/* (2 * 8) config */
1246 	default:
1247 		/* Setting (2 * 8) Min RP0 for any other combination */
1248 		val >>= FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT;
1249 		break;
1250 	}
1251 
1252 	return val & FB_GFX_FREQ_FUSE_MASK;
1253 }
1254 
chv_rps_rpe_freq(struct intel_rps * rps)1255 static int chv_rps_rpe_freq(struct intel_rps *rps)
1256 {
1257 	struct drm_i915_private *i915 = rps_to_i915(rps);
1258 	u32 val;
1259 
1260 	val = vlv_punit_read(i915, PUNIT_GPU_DUTYCYCLE_REG);
1261 	val >>= PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT;
1262 
1263 	return val & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK;
1264 }
1265 
chv_rps_guar_freq(struct intel_rps * rps)1266 static int chv_rps_guar_freq(struct intel_rps *rps)
1267 {
1268 	struct drm_i915_private *i915 = rps_to_i915(rps);
1269 	u32 val;
1270 
1271 	val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE);
1272 
1273 	return val & FB_GFX_FREQ_FUSE_MASK;
1274 }
1275 
chv_rps_min_freq(struct intel_rps * rps)1276 static u32 chv_rps_min_freq(struct intel_rps *rps)
1277 {
1278 	struct drm_i915_private *i915 = rps_to_i915(rps);
1279 	u32 val;
1280 
1281 	val = vlv_punit_read(i915, FB_GFX_FMIN_AT_VMIN_FUSE);
1282 	val >>= FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT;
1283 
1284 	return val & FB_GFX_FREQ_FUSE_MASK;
1285 }
1286 
chv_rps_enable(struct intel_rps * rps)1287 static bool chv_rps_enable(struct intel_rps *rps)
1288 {
1289 	struct intel_uncore *uncore = rps_to_uncore(rps);
1290 	struct drm_i915_private *i915 = rps_to_i915(rps);
1291 	u32 val;
1292 
1293 	/* 1: Program defaults and thresholds for RPS*/
1294 	intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000);
1295 	intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400);
1296 	intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000);
1297 	intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000);
1298 	intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000);
1299 
1300 	intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1301 
1302 	/* 2: Enable RPS */
1303 	intel_uncore_write_fw(uncore, GEN6_RP_CONTROL,
1304 			      GEN6_RP_MEDIA_HW_NORMAL_MODE |
1305 			      GEN6_RP_MEDIA_IS_GFX |
1306 			      GEN6_RP_ENABLE |
1307 			      GEN6_RP_UP_BUSY_AVG |
1308 			      GEN6_RP_DOWN_IDLE_AVG);
1309 
1310 	rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD |
1311 			  GEN6_PM_RP_DOWN_THRESHOLD |
1312 			  GEN6_PM_RP_DOWN_TIMEOUT);
1313 
1314 	/* Setting Fixed Bias */
1315 	vlv_punit_get(i915);
1316 
1317 	val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | CHV_BIAS_CPU_50_SOC_50;
1318 	vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val);
1319 
1320 	val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1321 
1322 	vlv_punit_put(i915);
1323 
1324 	/* RPS code assumes GPLL is used */
1325 	drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0,
1326 		      "GPLL not enabled\n");
1327 
1328 	drm_dbg(&i915->drm, "GPLL enabled? %s\n",
1329 		str_yes_no(val & GPLLENABLE));
1330 	drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val);
1331 
1332 	return rps_reset(rps);
1333 }
1334 
vlv_rps_guar_freq(struct intel_rps * rps)1335 static int vlv_rps_guar_freq(struct intel_rps *rps)
1336 {
1337 	struct drm_i915_private *i915 = rps_to_i915(rps);
1338 	u32 val, rp1;
1339 
1340 	val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE);
1341 
1342 	rp1 = val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK;
1343 	rp1 >>= FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT;
1344 
1345 	return rp1;
1346 }
1347 
vlv_rps_max_freq(struct intel_rps * rps)1348 static int vlv_rps_max_freq(struct intel_rps *rps)
1349 {
1350 	struct drm_i915_private *i915 = rps_to_i915(rps);
1351 	u32 val, rp0;
1352 
1353 	val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE);
1354 
1355 	rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
1356 	/* Clamp to max */
1357 	rp0 = min_t(u32, rp0, 0xea);
1358 
1359 	return rp0;
1360 }
1361 
vlv_rps_rpe_freq(struct intel_rps * rps)1362 static int vlv_rps_rpe_freq(struct intel_rps *rps)
1363 {
1364 	struct drm_i915_private *i915 = rps_to_i915(rps);
1365 	u32 val, rpe;
1366 
1367 	val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
1368 	rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
1369 	val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
1370 	rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
1371 
1372 	return rpe;
1373 }
1374 
vlv_rps_min_freq(struct intel_rps * rps)1375 static int vlv_rps_min_freq(struct intel_rps *rps)
1376 {
1377 	struct drm_i915_private *i915 = rps_to_i915(rps);
1378 	u32 val;
1379 
1380 	val = vlv_punit_read(i915, PUNIT_REG_GPU_LFM) & 0xff;
1381 	/*
1382 	 * According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value
1383 	 * for the minimum frequency in GPLL mode is 0xc1. Contrary to this on
1384 	 * a BYT-M B0 the above register contains 0xbf. Moreover when setting
1385 	 * a frequency Punit will not allow values below 0xc0. Clamp it 0xc0
1386 	 * to make sure it matches what Punit accepts.
1387 	 */
1388 	return max_t(u32, val, 0xc0);
1389 }
1390 
vlv_rps_enable(struct intel_rps * rps)1391 static bool vlv_rps_enable(struct intel_rps *rps)
1392 {
1393 	struct intel_uncore *uncore = rps_to_uncore(rps);
1394 	struct drm_i915_private *i915 = rps_to_i915(rps);
1395 	u32 val;
1396 
1397 	intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000);
1398 	intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400);
1399 	intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000);
1400 	intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000);
1401 	intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000);
1402 
1403 	intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1404 
1405 	intel_uncore_write_fw(uncore, GEN6_RP_CONTROL,
1406 			      GEN6_RP_MEDIA_TURBO |
1407 			      GEN6_RP_MEDIA_HW_NORMAL_MODE |
1408 			      GEN6_RP_MEDIA_IS_GFX |
1409 			      GEN6_RP_ENABLE |
1410 			      GEN6_RP_UP_BUSY_AVG |
1411 			      GEN6_RP_DOWN_IDLE_CONT);
1412 
1413 	/* WaGsvRC0ResidencyMethod:vlv */
1414 	rps->pm_events = GEN6_PM_RP_UP_EI_EXPIRED;
1415 
1416 	vlv_punit_get(i915);
1417 
1418 	/* Setting Fixed Bias */
1419 	val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | VLV_BIAS_CPU_125_SOC_875;
1420 	vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val);
1421 
1422 	val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1423 
1424 	vlv_punit_put(i915);
1425 
1426 	/* RPS code assumes GPLL is used */
1427 	drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0,
1428 		      "GPLL not enabled\n");
1429 
1430 	drm_dbg(&i915->drm, "GPLL enabled? %s\n",
1431 		str_yes_no(val & GPLLENABLE));
1432 	drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val);
1433 
1434 	return rps_reset(rps);
1435 }
1436 
__ips_gfx_val(struct intel_ips * ips)1437 static unsigned long __ips_gfx_val(struct intel_ips *ips)
1438 {
1439 	struct intel_rps *rps = container_of(ips, typeof(*rps), ips);
1440 	struct intel_uncore *uncore = rps_to_uncore(rps);
1441 	unsigned int t, state1, state2;
1442 	u32 pxvid, ext_v;
1443 	u64 corr, corr2;
1444 
1445 	lockdep_assert_held(&mchdev_lock);
1446 
1447 	pxvid = intel_uncore_read(uncore, PXVFREQ(rps->cur_freq));
1448 	pxvid = (pxvid >> 24) & 0x7f;
1449 	ext_v = pvid_to_extvid(rps_to_i915(rps), pxvid);
1450 
1451 	state1 = ext_v;
1452 
1453 	/* Revel in the empirically derived constants */
1454 
1455 	/* Correction factor in 1/100000 units */
1456 	t = ips_mch_val(uncore);
1457 	if (t > 80)
1458 		corr = t * 2349 + 135940;
1459 	else if (t >= 50)
1460 		corr = t * 964 + 29317;
1461 	else /* < 50 */
1462 		corr = t * 301 + 1004;
1463 
1464 	corr = div_u64(corr * 150142 * state1, 10000) - 78642;
1465 	corr2 = div_u64(corr, 100000) * ips->corr;
1466 
1467 	state2 = div_u64(corr2 * state1, 10000);
1468 	state2 /= 100; /* convert to mW */
1469 
1470 	__gen5_ips_update(ips);
1471 
1472 	return ips->gfx_power + state2;
1473 }
1474 
has_busy_stats(struct intel_rps * rps)1475 static bool has_busy_stats(struct intel_rps *rps)
1476 {
1477 	struct intel_engine_cs *engine;
1478 	enum intel_engine_id id;
1479 
1480 	for_each_engine(engine, rps_to_gt(rps), id) {
1481 		if (!intel_engine_supports_stats(engine))
1482 			return false;
1483 	}
1484 
1485 	return true;
1486 }
1487 
intel_rps_enable(struct intel_rps * rps)1488 void intel_rps_enable(struct intel_rps *rps)
1489 {
1490 	struct drm_i915_private *i915 = rps_to_i915(rps);
1491 	struct intel_uncore *uncore = rps_to_uncore(rps);
1492 	bool enabled = false;
1493 
1494 	if (!HAS_RPS(i915))
1495 		return;
1496 
1497 	if (rps_uses_slpc(rps))
1498 		return;
1499 
1500 	intel_gt_check_clock_frequency(rps_to_gt(rps));
1501 
1502 	intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
1503 	if (rps->max_freq <= rps->min_freq)
1504 		/* leave disabled, no room for dynamic reclocking */;
1505 	else if (IS_CHERRYVIEW(i915))
1506 		enabled = chv_rps_enable(rps);
1507 	else if (IS_VALLEYVIEW(i915))
1508 		enabled = vlv_rps_enable(rps);
1509 	else if (GRAPHICS_VER(i915) >= 9)
1510 		enabled = gen9_rps_enable(rps);
1511 	else if (GRAPHICS_VER(i915) >= 8)
1512 		enabled = gen8_rps_enable(rps);
1513 	else if (GRAPHICS_VER(i915) >= 6)
1514 		enabled = gen6_rps_enable(rps);
1515 	else if (IS_IRONLAKE_M(i915))
1516 		enabled = gen5_rps_enable(rps);
1517 	else
1518 		MISSING_CASE(GRAPHICS_VER(i915));
1519 	intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
1520 	if (!enabled)
1521 		return;
1522 
1523 	GT_TRACE(rps_to_gt(rps),
1524 		 "min:%x, max:%x, freq:[%d, %d]\n",
1525 		 rps->min_freq, rps->max_freq,
1526 		 intel_gpu_freq(rps, rps->min_freq),
1527 		 intel_gpu_freq(rps, rps->max_freq));
1528 
1529 	GEM_BUG_ON(rps->max_freq < rps->min_freq);
1530 	GEM_BUG_ON(rps->idle_freq > rps->max_freq);
1531 
1532 	GEM_BUG_ON(rps->efficient_freq < rps->min_freq);
1533 	GEM_BUG_ON(rps->efficient_freq > rps->max_freq);
1534 
1535 	if (has_busy_stats(rps))
1536 		intel_rps_set_timer(rps);
1537 	else if (GRAPHICS_VER(i915) >= 6 && GRAPHICS_VER(i915) <= 11)
1538 		intel_rps_set_interrupts(rps);
1539 	else
1540 		/* Ironlake currently uses intel_ips.ko */ {}
1541 
1542 	intel_rps_set_enabled(rps);
1543 }
1544 
gen6_rps_disable(struct intel_rps * rps)1545 static void gen6_rps_disable(struct intel_rps *rps)
1546 {
1547 	set(rps_to_uncore(rps), GEN6_RP_CONTROL, 0);
1548 }
1549 
intel_rps_disable(struct intel_rps * rps)1550 void intel_rps_disable(struct intel_rps *rps)
1551 {
1552 	struct drm_i915_private *i915 = rps_to_i915(rps);
1553 
1554 	if (!intel_rps_is_enabled(rps))
1555 		return;
1556 
1557 	intel_rps_clear_enabled(rps);
1558 	intel_rps_clear_interrupts(rps);
1559 	intel_rps_clear_timer(rps);
1560 
1561 	if (GRAPHICS_VER(i915) >= 6)
1562 		gen6_rps_disable(rps);
1563 	else if (IS_IRONLAKE_M(i915))
1564 		gen5_rps_disable(rps);
1565 }
1566 
byt_gpu_freq(struct intel_rps * rps,int val)1567 static int byt_gpu_freq(struct intel_rps *rps, int val)
1568 {
1569 	/*
1570 	 * N = val - 0xb7
1571 	 * Slow = Fast = GPLL ref * N
1572 	 */
1573 	return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000);
1574 }
1575 
byt_freq_opcode(struct intel_rps * rps,int val)1576 static int byt_freq_opcode(struct intel_rps *rps, int val)
1577 {
1578 	return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7;
1579 }
1580 
chv_gpu_freq(struct intel_rps * rps,int val)1581 static int chv_gpu_freq(struct intel_rps *rps, int val)
1582 {
1583 	/*
1584 	 * N = val / 2
1585 	 * CU (slow) = CU2x (fast) / 2 = GPLL ref * N / 2
1586 	 */
1587 	return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000);
1588 }
1589 
chv_freq_opcode(struct intel_rps * rps,int val)1590 static int chv_freq_opcode(struct intel_rps *rps, int val)
1591 {
1592 	/* CHV needs even values */
1593 	return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2;
1594 }
1595 
intel_gpu_freq(struct intel_rps * rps,int val)1596 int intel_gpu_freq(struct intel_rps *rps, int val)
1597 {
1598 	struct drm_i915_private *i915 = rps_to_i915(rps);
1599 
1600 	if (GRAPHICS_VER(i915) >= 9)
1601 		return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER,
1602 					 GEN9_FREQ_SCALER);
1603 	else if (IS_CHERRYVIEW(i915))
1604 		return chv_gpu_freq(rps, val);
1605 	else if (IS_VALLEYVIEW(i915))
1606 		return byt_gpu_freq(rps, val);
1607 	else if (GRAPHICS_VER(i915) >= 6)
1608 		return val * GT_FREQUENCY_MULTIPLIER;
1609 	else
1610 		return val;
1611 }
1612 
intel_freq_opcode(struct intel_rps * rps,int val)1613 int intel_freq_opcode(struct intel_rps *rps, int val)
1614 {
1615 	struct drm_i915_private *i915 = rps_to_i915(rps);
1616 
1617 	if (GRAPHICS_VER(i915) >= 9)
1618 		return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER,
1619 					 GT_FREQUENCY_MULTIPLIER);
1620 	else if (IS_CHERRYVIEW(i915))
1621 		return chv_freq_opcode(rps, val);
1622 	else if (IS_VALLEYVIEW(i915))
1623 		return byt_freq_opcode(rps, val);
1624 	else if (GRAPHICS_VER(i915) >= 6)
1625 		return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER);
1626 	else
1627 		return val;
1628 }
1629 
vlv_init_gpll_ref_freq(struct intel_rps * rps)1630 static void vlv_init_gpll_ref_freq(struct intel_rps *rps)
1631 {
1632 	struct drm_i915_private *i915 = rps_to_i915(rps);
1633 
1634 	rps->gpll_ref_freq =
1635 		vlv_get_cck_clock(i915, "GPLL ref",
1636 				  CCK_GPLL_CLOCK_CONTROL,
1637 				  i915->czclk_freq);
1638 
1639 	drm_dbg(&i915->drm, "GPLL reference freq: %d kHz\n",
1640 		rps->gpll_ref_freq);
1641 }
1642 
vlv_rps_init(struct intel_rps * rps)1643 static void vlv_rps_init(struct intel_rps *rps)
1644 {
1645 	struct drm_i915_private *i915 = rps_to_i915(rps);
1646 	u32 val;
1647 
1648 	vlv_iosf_sb_get(i915,
1649 			BIT(VLV_IOSF_SB_PUNIT) |
1650 			BIT(VLV_IOSF_SB_NC) |
1651 			BIT(VLV_IOSF_SB_CCK));
1652 
1653 	vlv_init_gpll_ref_freq(rps);
1654 
1655 	val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1656 	switch ((val >> 6) & 3) {
1657 	case 0:
1658 	case 1:
1659 		i915->mem_freq = 800;
1660 		break;
1661 	case 2:
1662 		i915->mem_freq = 1066;
1663 		break;
1664 	case 3:
1665 		i915->mem_freq = 1333;
1666 		break;
1667 	}
1668 	drm_dbg(&i915->drm, "DDR speed: %d MHz\n", i915->mem_freq);
1669 
1670 	rps->max_freq = vlv_rps_max_freq(rps);
1671 	rps->rp0_freq = rps->max_freq;
1672 	drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n",
1673 		intel_gpu_freq(rps, rps->max_freq), rps->max_freq);
1674 
1675 	rps->efficient_freq = vlv_rps_rpe_freq(rps);
1676 	drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n",
1677 		intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq);
1678 
1679 	rps->rp1_freq = vlv_rps_guar_freq(rps);
1680 	drm_dbg(&i915->drm, "RP1(Guar Freq) GPU freq: %d MHz (%u)\n",
1681 		intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq);
1682 
1683 	rps->min_freq = vlv_rps_min_freq(rps);
1684 	drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n",
1685 		intel_gpu_freq(rps, rps->min_freq), rps->min_freq);
1686 
1687 	vlv_iosf_sb_put(i915,
1688 			BIT(VLV_IOSF_SB_PUNIT) |
1689 			BIT(VLV_IOSF_SB_NC) |
1690 			BIT(VLV_IOSF_SB_CCK));
1691 }
1692 
chv_rps_init(struct intel_rps * rps)1693 static void chv_rps_init(struct intel_rps *rps)
1694 {
1695 	struct drm_i915_private *i915 = rps_to_i915(rps);
1696 	u32 val;
1697 
1698 	vlv_iosf_sb_get(i915,
1699 			BIT(VLV_IOSF_SB_PUNIT) |
1700 			BIT(VLV_IOSF_SB_NC) |
1701 			BIT(VLV_IOSF_SB_CCK));
1702 
1703 	vlv_init_gpll_ref_freq(rps);
1704 
1705 	val = vlv_cck_read(i915, CCK_FUSE_REG);
1706 
1707 	switch ((val >> 2) & 0x7) {
1708 	case 3:
1709 		i915->mem_freq = 2000;
1710 		break;
1711 	default:
1712 		i915->mem_freq = 1600;
1713 		break;
1714 	}
1715 	drm_dbg(&i915->drm, "DDR speed: %d MHz\n", i915->mem_freq);
1716 
1717 	rps->max_freq = chv_rps_max_freq(rps);
1718 	rps->rp0_freq = rps->max_freq;
1719 	drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n",
1720 		intel_gpu_freq(rps, rps->max_freq), rps->max_freq);
1721 
1722 	rps->efficient_freq = chv_rps_rpe_freq(rps);
1723 	drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n",
1724 		intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq);
1725 
1726 	rps->rp1_freq = chv_rps_guar_freq(rps);
1727 	drm_dbg(&i915->drm, "RP1(Guar) GPU freq: %d MHz (%u)\n",
1728 		intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq);
1729 
1730 	rps->min_freq = chv_rps_min_freq(rps);
1731 	drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n",
1732 		intel_gpu_freq(rps, rps->min_freq), rps->min_freq);
1733 
1734 	vlv_iosf_sb_put(i915,
1735 			BIT(VLV_IOSF_SB_PUNIT) |
1736 			BIT(VLV_IOSF_SB_NC) |
1737 			BIT(VLV_IOSF_SB_CCK));
1738 
1739 	drm_WARN_ONCE(&i915->drm, (rps->max_freq | rps->efficient_freq |
1740 				   rps->rp1_freq | rps->min_freq) & 1,
1741 		      "Odd GPU freq values\n");
1742 }
1743 
vlv_c0_read(struct intel_uncore * uncore,struct intel_rps_ei * ei)1744 static void vlv_c0_read(struct intel_uncore *uncore, struct intel_rps_ei *ei)
1745 {
1746 	ei->ktime = ktime_get_raw();
1747 	ei->render_c0 = intel_uncore_read(uncore, VLV_RENDER_C0_COUNT);
1748 	ei->media_c0 = intel_uncore_read(uncore, VLV_MEDIA_C0_COUNT);
1749 }
1750 
vlv_wa_c0_ei(struct intel_rps * rps,u32 pm_iir)1751 static u32 vlv_wa_c0_ei(struct intel_rps *rps, u32 pm_iir)
1752 {
1753 	struct intel_uncore *uncore = rps_to_uncore(rps);
1754 	const struct intel_rps_ei *prev = &rps->ei;
1755 	struct intel_rps_ei now;
1756 	u32 events = 0;
1757 
1758 	if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0)
1759 		return 0;
1760 
1761 	vlv_c0_read(uncore, &now);
1762 
1763 	if (prev->ktime) {
1764 		u64 time, c0;
1765 		u32 render, media;
1766 
1767 		time = ktime_us_delta(now.ktime, prev->ktime);
1768 
1769 		time *= rps_to_i915(rps)->czclk_freq;
1770 
1771 		/* Workload can be split between render + media,
1772 		 * e.g. SwapBuffers being blitted in X after being rendered in
1773 		 * mesa. To account for this we need to combine both engines
1774 		 * into our activity counter.
1775 		 */
1776 		render = now.render_c0 - prev->render_c0;
1777 		media = now.media_c0 - prev->media_c0;
1778 		c0 = max(render, media);
1779 		c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */
1780 
1781 		if (c0 > time * rps->power.up_threshold)
1782 			events = GEN6_PM_RP_UP_THRESHOLD;
1783 		else if (c0 < time * rps->power.down_threshold)
1784 			events = GEN6_PM_RP_DOWN_THRESHOLD;
1785 	}
1786 
1787 	rps->ei = now;
1788 	return events;
1789 }
1790 
rps_work(struct work_struct * work)1791 static void rps_work(struct work_struct *work)
1792 {
1793 	struct intel_rps *rps = container_of(work, typeof(*rps), work);
1794 	struct intel_gt *gt = rps_to_gt(rps);
1795 	struct drm_i915_private *i915 = rps_to_i915(rps);
1796 	bool client_boost = false;
1797 	int new_freq, adj, min, max;
1798 	u32 pm_iir = 0;
1799 
1800 	spin_lock_irq(gt->irq_lock);
1801 	pm_iir = fetch_and_zero(&rps->pm_iir) & rps->pm_events;
1802 	client_boost = atomic_read(&rps->num_waiters);
1803 	spin_unlock_irq(gt->irq_lock);
1804 
1805 	/* Make sure we didn't queue anything we're not going to process. */
1806 	if (!pm_iir && !client_boost)
1807 		goto out;
1808 
1809 	mutex_lock(&rps->lock);
1810 	if (!intel_rps_is_active(rps)) {
1811 		mutex_unlock(&rps->lock);
1812 		return;
1813 	}
1814 
1815 	pm_iir |= vlv_wa_c0_ei(rps, pm_iir);
1816 
1817 	adj = rps->last_adj;
1818 	new_freq = rps->cur_freq;
1819 	min = rps->min_freq_softlimit;
1820 	max = rps->max_freq_softlimit;
1821 	if (client_boost)
1822 		max = rps->max_freq;
1823 
1824 	GT_TRACE(gt,
1825 		 "pm_iir:%x, client_boost:%s, last:%d, cur:%x, min:%x, max:%x\n",
1826 		 pm_iir, str_yes_no(client_boost),
1827 		 adj, new_freq, min, max);
1828 
1829 	if (client_boost && new_freq < rps->boost_freq) {
1830 		new_freq = rps->boost_freq;
1831 		adj = 0;
1832 	} else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
1833 		if (adj > 0)
1834 			adj *= 2;
1835 		else /* CHV needs even encode values */
1836 			adj = IS_CHERRYVIEW(gt->i915) ? 2 : 1;
1837 
1838 		if (new_freq >= rps->max_freq_softlimit)
1839 			adj = 0;
1840 	} else if (client_boost) {
1841 		adj = 0;
1842 	} else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
1843 		if (rps->cur_freq > rps->efficient_freq)
1844 			new_freq = rps->efficient_freq;
1845 		else if (rps->cur_freq > rps->min_freq_softlimit)
1846 			new_freq = rps->min_freq_softlimit;
1847 		adj = 0;
1848 	} else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
1849 		if (adj < 0)
1850 			adj *= 2;
1851 		else /* CHV needs even encode values */
1852 			adj = IS_CHERRYVIEW(gt->i915) ? -2 : -1;
1853 
1854 		if (new_freq <= rps->min_freq_softlimit)
1855 			adj = 0;
1856 	} else { /* unknown event */
1857 		adj = 0;
1858 	}
1859 
1860 	/*
1861 	 * sysfs frequency limits may have snuck in while
1862 	 * servicing the interrupt
1863 	 */
1864 	new_freq += adj;
1865 	new_freq = clamp_t(int, new_freq, min, max);
1866 
1867 	if (intel_rps_set(rps, new_freq)) {
1868 		drm_dbg(&i915->drm, "Failed to set new GPU frequency\n");
1869 		adj = 0;
1870 	}
1871 	rps->last_adj = adj;
1872 
1873 	mutex_unlock(&rps->lock);
1874 
1875 out:
1876 	spin_lock_irq(gt->irq_lock);
1877 	gen6_gt_pm_unmask_irq(gt, rps->pm_events);
1878 	spin_unlock_irq(gt->irq_lock);
1879 }
1880 
gen11_rps_irq_handler(struct intel_rps * rps,u32 pm_iir)1881 void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
1882 {
1883 	struct intel_gt *gt = rps_to_gt(rps);
1884 	const u32 events = rps->pm_events & pm_iir;
1885 
1886 	lockdep_assert_held(gt->irq_lock);
1887 
1888 	if (unlikely(!events))
1889 		return;
1890 
1891 	GT_TRACE(gt, "irq events:%x\n", events);
1892 
1893 	gen6_gt_pm_mask_irq(gt, events);
1894 
1895 	rps->pm_iir |= events;
1896 	schedule_work(&rps->work);
1897 }
1898 
gen6_rps_irq_handler(struct intel_rps * rps,u32 pm_iir)1899 void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
1900 {
1901 	struct intel_gt *gt = rps_to_gt(rps);
1902 	u32 events;
1903 
1904 	events = pm_iir & rps->pm_events;
1905 	if (events) {
1906 		spin_lock(gt->irq_lock);
1907 
1908 		GT_TRACE(gt, "irq events:%x\n", events);
1909 
1910 		gen6_gt_pm_mask_irq(gt, events);
1911 		rps->pm_iir |= events;
1912 
1913 		schedule_work(&rps->work);
1914 		spin_unlock(gt->irq_lock);
1915 	}
1916 
1917 	if (GRAPHICS_VER(gt->i915) >= 8)
1918 		return;
1919 
1920 	if (pm_iir & PM_VEBOX_USER_INTERRUPT)
1921 		intel_engine_cs_irq(gt->engine[VECS0], pm_iir >> 10);
1922 
1923 	if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
1924 		DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
1925 }
1926 
gen5_rps_irq_handler(struct intel_rps * rps)1927 void gen5_rps_irq_handler(struct intel_rps *rps)
1928 {
1929 	struct intel_uncore *uncore = rps_to_uncore(rps);
1930 	u32 busy_up, busy_down, max_avg, min_avg;
1931 	u8 new_freq;
1932 
1933 	spin_lock(&mchdev_lock);
1934 
1935 	intel_uncore_write16(uncore,
1936 			     MEMINTRSTS,
1937 			     intel_uncore_read(uncore, MEMINTRSTS));
1938 
1939 	intel_uncore_write16(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
1940 	busy_up = intel_uncore_read(uncore, RCPREVBSYTUPAVG);
1941 	busy_down = intel_uncore_read(uncore, RCPREVBSYTDNAVG);
1942 	max_avg = intel_uncore_read(uncore, RCBMAXAVG);
1943 	min_avg = intel_uncore_read(uncore, RCBMINAVG);
1944 
1945 	/* Handle RCS change request from hw */
1946 	new_freq = rps->cur_freq;
1947 	if (busy_up > max_avg)
1948 		new_freq++;
1949 	else if (busy_down < min_avg)
1950 		new_freq--;
1951 	new_freq = clamp(new_freq,
1952 			 rps->min_freq_softlimit,
1953 			 rps->max_freq_softlimit);
1954 
1955 	if (new_freq != rps->cur_freq && !__gen5_rps_set(rps, new_freq))
1956 		rps->cur_freq = new_freq;
1957 
1958 	spin_unlock(&mchdev_lock);
1959 }
1960 
intel_rps_init_early(struct intel_rps * rps)1961 void intel_rps_init_early(struct intel_rps *rps)
1962 {
1963 	mutex_init(&rps->lock);
1964 	mutex_init(&rps->power.mutex);
1965 
1966 	INIT_WORK(&rps->work, rps_work);
1967 	timer_setup(&rps->timer, rps_timer, 0);
1968 
1969 	atomic_set(&rps->num_waiters, 0);
1970 }
1971 
intel_rps_init(struct intel_rps * rps)1972 void intel_rps_init(struct intel_rps *rps)
1973 {
1974 	struct drm_i915_private *i915 = rps_to_i915(rps);
1975 
1976 	if (rps_uses_slpc(rps))
1977 		return;
1978 
1979 	if (IS_CHERRYVIEW(i915))
1980 		chv_rps_init(rps);
1981 	else if (IS_VALLEYVIEW(i915))
1982 		vlv_rps_init(rps);
1983 	else if (GRAPHICS_VER(i915) >= 6)
1984 		gen6_rps_init(rps);
1985 	else if (IS_IRONLAKE_M(i915))
1986 		gen5_rps_init(rps);
1987 
1988 	/* Derive initial user preferences/limits from the hardware limits */
1989 	rps->max_freq_softlimit = rps->max_freq;
1990 	rps_to_gt(rps)->defaults.max_freq = rps->max_freq_softlimit;
1991 	rps->min_freq_softlimit = rps->min_freq;
1992 	rps_to_gt(rps)->defaults.min_freq = rps->min_freq_softlimit;
1993 
1994 	/* After setting max-softlimit, find the overclock max freq */
1995 	if (GRAPHICS_VER(i915) == 6 || IS_IVYBRIDGE(i915) || IS_HASWELL(i915)) {
1996 		u32 params = 0;
1997 
1998 		snb_pcode_read(rps_to_gt(rps)->uncore, GEN6_READ_OC_PARAMS, &params, NULL);
1999 		if (params & BIT(31)) { /* OC supported */
2000 			drm_dbg(&i915->drm,
2001 				"Overclocking supported, max: %dMHz, overclock: %dMHz\n",
2002 				(rps->max_freq & 0xff) * 50,
2003 				(params & 0xff) * 50);
2004 			rps->max_freq = params & 0xff;
2005 		}
2006 	}
2007 
2008 	/* Finally allow us to boost to max by default */
2009 	rps->boost_freq = rps->max_freq;
2010 	rps->idle_freq = rps->min_freq;
2011 
2012 	/* Start in the middle, from here we will autotune based on workload */
2013 	rps->cur_freq = rps->efficient_freq;
2014 
2015 	rps->pm_intrmsk_mbz = 0;
2016 
2017 	/*
2018 	 * SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer
2019 	 * if GEN6_PM_UP_EI_EXPIRED is masked.
2020 	 *
2021 	 * TODO: verify if this can be reproduced on VLV,CHV.
2022 	 */
2023 	if (GRAPHICS_VER(i915) <= 7)
2024 		rps->pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED;
2025 
2026 	if (GRAPHICS_VER(i915) >= 8 && GRAPHICS_VER(i915) < 11)
2027 		rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC;
2028 
2029 	/* GuC needs ARAT expired interrupt unmasked */
2030 	if (intel_uc_uses_guc_submission(&rps_to_gt(rps)->uc))
2031 		rps->pm_intrmsk_mbz |= ARAT_EXPIRED_INTRMSK;
2032 }
2033 
intel_rps_sanitize(struct intel_rps * rps)2034 void intel_rps_sanitize(struct intel_rps *rps)
2035 {
2036 	if (rps_uses_slpc(rps))
2037 		return;
2038 
2039 	if (GRAPHICS_VER(rps_to_i915(rps)) >= 6)
2040 		rps_disable_interrupts(rps);
2041 }
2042 
intel_rps_get_cagf(struct intel_rps * rps,u32 rpstat)2043 u32 intel_rps_get_cagf(struct intel_rps *rps, u32 rpstat)
2044 {
2045 	struct drm_i915_private *i915 = rps_to_i915(rps);
2046 	u32 cagf;
2047 
2048 	if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
2049 		cagf = (rpstat >> 8) & 0xff;
2050 	else if (GRAPHICS_VER(i915) >= 9)
2051 		cagf = (rpstat & GEN9_CAGF_MASK) >> GEN9_CAGF_SHIFT;
2052 	else if (IS_HASWELL(i915) || IS_BROADWELL(i915))
2053 		cagf = (rpstat & HSW_CAGF_MASK) >> HSW_CAGF_SHIFT;
2054 	else if (GRAPHICS_VER(i915) >= 6)
2055 		cagf = (rpstat & GEN6_CAGF_MASK) >> GEN6_CAGF_SHIFT;
2056 	else
2057 		cagf = gen5_invert_freq(rps, (rpstat & MEMSTAT_PSTATE_MASK) >>
2058 					MEMSTAT_PSTATE_SHIFT);
2059 
2060 	return cagf;
2061 }
2062 
read_cagf(struct intel_rps * rps)2063 static u32 read_cagf(struct intel_rps *rps)
2064 {
2065 	struct drm_i915_private *i915 = rps_to_i915(rps);
2066 	struct intel_uncore *uncore = rps_to_uncore(rps);
2067 	u32 freq;
2068 
2069 	if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) {
2070 		vlv_punit_get(i915);
2071 		freq = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
2072 		vlv_punit_put(i915);
2073 	} else if (GRAPHICS_VER(i915) >= 6) {
2074 		freq = intel_uncore_read(uncore, GEN6_RPSTAT1);
2075 	} else {
2076 		freq = intel_uncore_read(uncore, MEMSTAT_ILK);
2077 	}
2078 
2079 	return intel_rps_get_cagf(rps, freq);
2080 }
2081 
intel_rps_read_actual_frequency(struct intel_rps * rps)2082 u32 intel_rps_read_actual_frequency(struct intel_rps *rps)
2083 {
2084 	struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm;
2085 	intel_wakeref_t wakeref;
2086 	u32 freq = 0;
2087 
2088 	with_intel_runtime_pm_if_in_use(rpm, wakeref)
2089 		freq = intel_gpu_freq(rps, read_cagf(rps));
2090 
2091 	return freq;
2092 }
2093 
intel_rps_read_punit_req(struct intel_rps * rps)2094 u32 intel_rps_read_punit_req(struct intel_rps *rps)
2095 {
2096 	struct intel_uncore *uncore = rps_to_uncore(rps);
2097 	struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm;
2098 	intel_wakeref_t wakeref;
2099 	u32 freq = 0;
2100 
2101 	with_intel_runtime_pm_if_in_use(rpm, wakeref)
2102 		freq = intel_uncore_read(uncore, GEN6_RPNSWREQ);
2103 
2104 	return freq;
2105 }
2106 
intel_rps_get_req(u32 pureq)2107 static u32 intel_rps_get_req(u32 pureq)
2108 {
2109 	u32 req = pureq >> GEN9_SW_REQ_UNSLICE_RATIO_SHIFT;
2110 
2111 	return req;
2112 }
2113 
intel_rps_read_punit_req_frequency(struct intel_rps * rps)2114 u32 intel_rps_read_punit_req_frequency(struct intel_rps *rps)
2115 {
2116 	u32 freq = intel_rps_get_req(intel_rps_read_punit_req(rps));
2117 
2118 	return intel_gpu_freq(rps, freq);
2119 }
2120 
intel_rps_get_requested_frequency(struct intel_rps * rps)2121 u32 intel_rps_get_requested_frequency(struct intel_rps *rps)
2122 {
2123 	if (rps_uses_slpc(rps))
2124 		return intel_rps_read_punit_req_frequency(rps);
2125 	else
2126 		return intel_gpu_freq(rps, rps->cur_freq);
2127 }
2128 
intel_rps_get_max_frequency(struct intel_rps * rps)2129 u32 intel_rps_get_max_frequency(struct intel_rps *rps)
2130 {
2131 	struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2132 
2133 	if (rps_uses_slpc(rps))
2134 		return slpc->max_freq_softlimit;
2135 	else
2136 		return intel_gpu_freq(rps, rps->max_freq_softlimit);
2137 }
2138 
2139 /**
2140  * intel_rps_get_max_raw_freq - returns the max frequency in some raw format.
2141  * @rps: the intel_rps structure
2142  *
2143  * Returns the max frequency in a raw format. In newer platforms raw is in
2144  * units of 50 MHz.
2145  */
intel_rps_get_max_raw_freq(struct intel_rps * rps)2146 u32 intel_rps_get_max_raw_freq(struct intel_rps *rps)
2147 {
2148 	struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2149 	u32 freq;
2150 
2151 	if (rps_uses_slpc(rps)) {
2152 		return DIV_ROUND_CLOSEST(slpc->rp0_freq,
2153 					 GT_FREQUENCY_MULTIPLIER);
2154 	} else {
2155 		freq = rps->max_freq;
2156 		if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) {
2157 			/* Convert GT frequency to 50 MHz units */
2158 			freq /= GEN9_FREQ_SCALER;
2159 		}
2160 		return freq;
2161 	}
2162 }
2163 
intel_rps_get_rp0_frequency(struct intel_rps * rps)2164 u32 intel_rps_get_rp0_frequency(struct intel_rps *rps)
2165 {
2166 	struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2167 
2168 	if (rps_uses_slpc(rps))
2169 		return slpc->rp0_freq;
2170 	else
2171 		return intel_gpu_freq(rps, rps->rp0_freq);
2172 }
2173 
intel_rps_get_rp1_frequency(struct intel_rps * rps)2174 u32 intel_rps_get_rp1_frequency(struct intel_rps *rps)
2175 {
2176 	struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2177 
2178 	if (rps_uses_slpc(rps))
2179 		return slpc->rp1_freq;
2180 	else
2181 		return intel_gpu_freq(rps, rps->rp1_freq);
2182 }
2183 
intel_rps_get_rpn_frequency(struct intel_rps * rps)2184 u32 intel_rps_get_rpn_frequency(struct intel_rps *rps)
2185 {
2186 	struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2187 
2188 	if (rps_uses_slpc(rps))
2189 		return slpc->min_freq;
2190 	else
2191 		return intel_gpu_freq(rps, rps->min_freq);
2192 }
2193 
set_max_freq(struct intel_rps * rps,u32 val)2194 static int set_max_freq(struct intel_rps *rps, u32 val)
2195 {
2196 	struct drm_i915_private *i915 = rps_to_i915(rps);
2197 	int ret = 0;
2198 
2199 	mutex_lock(&rps->lock);
2200 
2201 	val = intel_freq_opcode(rps, val);
2202 	if (val < rps->min_freq ||
2203 	    val > rps->max_freq ||
2204 	    val < rps->min_freq_softlimit) {
2205 		ret = -EINVAL;
2206 		goto unlock;
2207 	}
2208 
2209 	if (val > rps->rp0_freq)
2210 		drm_dbg(&i915->drm, "User requested overclocking to %d\n",
2211 			intel_gpu_freq(rps, val));
2212 
2213 	rps->max_freq_softlimit = val;
2214 
2215 	val = clamp_t(int, rps->cur_freq,
2216 		      rps->min_freq_softlimit,
2217 		      rps->max_freq_softlimit);
2218 
2219 	/*
2220 	 * We still need *_set_rps to process the new max_delay and
2221 	 * update the interrupt limits and PMINTRMSK even though
2222 	 * frequency request may be unchanged.
2223 	 */
2224 	intel_rps_set(rps, val);
2225 
2226 unlock:
2227 	mutex_unlock(&rps->lock);
2228 
2229 	return ret;
2230 }
2231 
intel_rps_set_max_frequency(struct intel_rps * rps,u32 val)2232 int intel_rps_set_max_frequency(struct intel_rps *rps, u32 val)
2233 {
2234 	struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2235 
2236 	if (rps_uses_slpc(rps))
2237 		return intel_guc_slpc_set_max_freq(slpc, val);
2238 	else
2239 		return set_max_freq(rps, val);
2240 }
2241 
intel_rps_get_min_frequency(struct intel_rps * rps)2242 u32 intel_rps_get_min_frequency(struct intel_rps *rps)
2243 {
2244 	struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2245 
2246 	if (rps_uses_slpc(rps))
2247 		return slpc->min_freq_softlimit;
2248 	else
2249 		return intel_gpu_freq(rps, rps->min_freq_softlimit);
2250 }
2251 
2252 /**
2253  * intel_rps_get_min_raw_freq - returns the min frequency in some raw format.
2254  * @rps: the intel_rps structure
2255  *
2256  * Returns the min frequency in a raw format. In newer platforms raw is in
2257  * units of 50 MHz.
2258  */
intel_rps_get_min_raw_freq(struct intel_rps * rps)2259 u32 intel_rps_get_min_raw_freq(struct intel_rps *rps)
2260 {
2261 	struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2262 	u32 freq;
2263 
2264 	if (rps_uses_slpc(rps)) {
2265 		return DIV_ROUND_CLOSEST(slpc->min_freq,
2266 					 GT_FREQUENCY_MULTIPLIER);
2267 	} else {
2268 		freq = rps->min_freq;
2269 		if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) {
2270 			/* Convert GT frequency to 50 MHz units */
2271 			freq /= GEN9_FREQ_SCALER;
2272 		}
2273 		return freq;
2274 	}
2275 }
2276 
set_min_freq(struct intel_rps * rps,u32 val)2277 static int set_min_freq(struct intel_rps *rps, u32 val)
2278 {
2279 	int ret = 0;
2280 
2281 	mutex_lock(&rps->lock);
2282 
2283 	val = intel_freq_opcode(rps, val);
2284 	if (val < rps->min_freq ||
2285 	    val > rps->max_freq ||
2286 	    val > rps->max_freq_softlimit) {
2287 		ret = -EINVAL;
2288 		goto unlock;
2289 	}
2290 
2291 	rps->min_freq_softlimit = val;
2292 
2293 	val = clamp_t(int, rps->cur_freq,
2294 		      rps->min_freq_softlimit,
2295 		      rps->max_freq_softlimit);
2296 
2297 	/*
2298 	 * We still need *_set_rps to process the new min_delay and
2299 	 * update the interrupt limits and PMINTRMSK even though
2300 	 * frequency request may be unchanged.
2301 	 */
2302 	intel_rps_set(rps, val);
2303 
2304 unlock:
2305 	mutex_unlock(&rps->lock);
2306 
2307 	return ret;
2308 }
2309 
intel_rps_set_min_frequency(struct intel_rps * rps,u32 val)2310 int intel_rps_set_min_frequency(struct intel_rps *rps, u32 val)
2311 {
2312 	struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2313 
2314 	if (rps_uses_slpc(rps))
2315 		return intel_guc_slpc_set_min_freq(slpc, val);
2316 	else
2317 		return set_min_freq(rps, val);
2318 }
2319 
intel_rps_set_manual(struct intel_rps * rps,bool enable)2320 static void intel_rps_set_manual(struct intel_rps *rps, bool enable)
2321 {
2322 	struct intel_uncore *uncore = rps_to_uncore(rps);
2323 	u32 state = enable ? GEN9_RPSWCTL_ENABLE : GEN9_RPSWCTL_DISABLE;
2324 
2325 	/* Allow punit to process software requests */
2326 	intel_uncore_write(uncore, GEN6_RP_CONTROL, state);
2327 }
2328 
intel_rps_raise_unslice(struct intel_rps * rps)2329 void intel_rps_raise_unslice(struct intel_rps *rps)
2330 {
2331 	struct intel_uncore *uncore = rps_to_uncore(rps);
2332 
2333 	mutex_lock(&rps->lock);
2334 
2335 	if (rps_uses_slpc(rps)) {
2336 		/* RP limits have not been initialized yet for SLPC path */
2337 		struct intel_rps_freq_caps caps;
2338 
2339 		gen6_rps_get_freq_caps(rps, &caps);
2340 
2341 		intel_rps_set_manual(rps, true);
2342 		intel_uncore_write(uncore, GEN6_RPNSWREQ,
2343 				   ((caps.rp0_freq <<
2344 				   GEN9_SW_REQ_UNSLICE_RATIO_SHIFT) |
2345 				   GEN9_IGNORE_SLICE_RATIO));
2346 		intel_rps_set_manual(rps, false);
2347 	} else {
2348 		intel_rps_set(rps, rps->rp0_freq);
2349 	}
2350 
2351 	mutex_unlock(&rps->lock);
2352 }
2353 
intel_rps_lower_unslice(struct intel_rps * rps)2354 void intel_rps_lower_unslice(struct intel_rps *rps)
2355 {
2356 	struct intel_uncore *uncore = rps_to_uncore(rps);
2357 
2358 	mutex_lock(&rps->lock);
2359 
2360 	if (rps_uses_slpc(rps)) {
2361 		/* RP limits have not been initialized yet for SLPC path */
2362 		struct intel_rps_freq_caps caps;
2363 
2364 		gen6_rps_get_freq_caps(rps, &caps);
2365 
2366 		intel_rps_set_manual(rps, true);
2367 		intel_uncore_write(uncore, GEN6_RPNSWREQ,
2368 				   ((caps.min_freq <<
2369 				   GEN9_SW_REQ_UNSLICE_RATIO_SHIFT) |
2370 				   GEN9_IGNORE_SLICE_RATIO));
2371 		intel_rps_set_manual(rps, false);
2372 	} else {
2373 		intel_rps_set(rps, rps->min_freq);
2374 	}
2375 
2376 	mutex_unlock(&rps->lock);
2377 }
2378 
rps_read_mmio(struct intel_rps * rps,i915_reg_t reg32)2379 static u32 rps_read_mmio(struct intel_rps *rps, i915_reg_t reg32)
2380 {
2381 	struct intel_gt *gt = rps_to_gt(rps);
2382 	intel_wakeref_t wakeref;
2383 	u32 val;
2384 
2385 	with_intel_runtime_pm(gt->uncore->rpm, wakeref)
2386 		val = intel_uncore_read(gt->uncore, reg32);
2387 
2388 	return val;
2389 }
2390 
rps_read_mask_mmio(struct intel_rps * rps,i915_reg_t reg32,u32 mask)2391 bool rps_read_mask_mmio(struct intel_rps *rps,
2392 			i915_reg_t reg32, u32 mask)
2393 {
2394 	return rps_read_mmio(rps, reg32) & mask;
2395 }
2396 
2397 /* External interface for intel_ips.ko */
2398 
2399 static struct drm_i915_private __rcu *ips_mchdev;
2400 
2401 /**
2402  * Tells the intel_ips driver that the i915 driver is now loaded, if
2403  * IPS got loaded first.
2404  *
2405  * This awkward dance is so that neither module has to depend on the
2406  * other in order for IPS to do the appropriate communication of
2407  * GPU turbo limits to i915.
2408  */
2409 static void
ips_ping_for_i915_load(void)2410 ips_ping_for_i915_load(void)
2411 {
2412 	void (*link)(void);
2413 
2414 	link = symbol_get(ips_link_to_i915_driver);
2415 	if (link) {
2416 		link();
2417 		symbol_put(ips_link_to_i915_driver);
2418 	}
2419 }
2420 
intel_rps_driver_register(struct intel_rps * rps)2421 void intel_rps_driver_register(struct intel_rps *rps)
2422 {
2423 	struct intel_gt *gt = rps_to_gt(rps);
2424 
2425 	/*
2426 	 * We only register the i915 ips part with intel-ips once everything is
2427 	 * set up, to avoid intel-ips sneaking in and reading bogus values.
2428 	 */
2429 	if (GRAPHICS_VER(gt->i915) == 5) {
2430 		GEM_BUG_ON(ips_mchdev);
2431 		rcu_assign_pointer(ips_mchdev, gt->i915);
2432 		ips_ping_for_i915_load();
2433 	}
2434 }
2435 
intel_rps_driver_unregister(struct intel_rps * rps)2436 void intel_rps_driver_unregister(struct intel_rps *rps)
2437 {
2438 	if (rcu_access_pointer(ips_mchdev) == rps_to_i915(rps))
2439 		rcu_assign_pointer(ips_mchdev, NULL);
2440 }
2441 
mchdev_get(void)2442 static struct drm_i915_private *mchdev_get(void)
2443 {
2444 	struct drm_i915_private *i915;
2445 
2446 	rcu_read_lock();
2447 	i915 = rcu_dereference(ips_mchdev);
2448 	if (i915 && !kref_get_unless_zero(&i915->drm.ref))
2449 		i915 = NULL;
2450 	rcu_read_unlock();
2451 
2452 	return i915;
2453 }
2454 
2455 /**
2456  * i915_read_mch_val - return value for IPS use
2457  *
2458  * Calculate and return a value for the IPS driver to use when deciding whether
2459  * we have thermal and power headroom to increase CPU or GPU power budget.
2460  */
i915_read_mch_val(void)2461 unsigned long i915_read_mch_val(void)
2462 {
2463 	struct drm_i915_private *i915;
2464 	unsigned long chipset_val = 0;
2465 	unsigned long graphics_val = 0;
2466 	intel_wakeref_t wakeref;
2467 
2468 	i915 = mchdev_get();
2469 	if (!i915)
2470 		return 0;
2471 
2472 	with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
2473 		struct intel_ips *ips = &to_gt(i915)->rps.ips;
2474 
2475 		spin_lock_irq(&mchdev_lock);
2476 		chipset_val = __ips_chipset_val(ips);
2477 		graphics_val = __ips_gfx_val(ips);
2478 		spin_unlock_irq(&mchdev_lock);
2479 	}
2480 
2481 	drm_dev_put(&i915->drm);
2482 	return chipset_val + graphics_val;
2483 }
2484 EXPORT_SYMBOL_GPL(i915_read_mch_val);
2485 
2486 /**
2487  * i915_gpu_raise - raise GPU frequency limit
2488  *
2489  * Raise the limit; IPS indicates we have thermal headroom.
2490  */
i915_gpu_raise(void)2491 bool i915_gpu_raise(void)
2492 {
2493 	struct drm_i915_private *i915;
2494 	struct intel_rps *rps;
2495 
2496 	i915 = mchdev_get();
2497 	if (!i915)
2498 		return false;
2499 
2500 	rps = &to_gt(i915)->rps;
2501 
2502 	spin_lock_irq(&mchdev_lock);
2503 	if (rps->max_freq_softlimit < rps->max_freq)
2504 		rps->max_freq_softlimit++;
2505 	spin_unlock_irq(&mchdev_lock);
2506 
2507 	drm_dev_put(&i915->drm);
2508 	return true;
2509 }
2510 EXPORT_SYMBOL_GPL(i915_gpu_raise);
2511 
2512 /**
2513  * i915_gpu_lower - lower GPU frequency limit
2514  *
2515  * IPS indicates we're close to a thermal limit, so throttle back the GPU
2516  * frequency maximum.
2517  */
i915_gpu_lower(void)2518 bool i915_gpu_lower(void)
2519 {
2520 	struct drm_i915_private *i915;
2521 	struct intel_rps *rps;
2522 
2523 	i915 = mchdev_get();
2524 	if (!i915)
2525 		return false;
2526 
2527 	rps = &to_gt(i915)->rps;
2528 
2529 	spin_lock_irq(&mchdev_lock);
2530 	if (rps->max_freq_softlimit > rps->min_freq)
2531 		rps->max_freq_softlimit--;
2532 	spin_unlock_irq(&mchdev_lock);
2533 
2534 	drm_dev_put(&i915->drm);
2535 	return true;
2536 }
2537 EXPORT_SYMBOL_GPL(i915_gpu_lower);
2538 
2539 /**
2540  * i915_gpu_busy - indicate GPU business to IPS
2541  *
2542  * Tell the IPS driver whether or not the GPU is busy.
2543  */
i915_gpu_busy(void)2544 bool i915_gpu_busy(void)
2545 {
2546 	struct drm_i915_private *i915;
2547 	bool ret;
2548 
2549 	i915 = mchdev_get();
2550 	if (!i915)
2551 		return false;
2552 
2553 	ret = to_gt(i915)->awake;
2554 
2555 	drm_dev_put(&i915->drm);
2556 	return ret;
2557 }
2558 EXPORT_SYMBOL_GPL(i915_gpu_busy);
2559 
2560 /**
2561  * i915_gpu_turbo_disable - disable graphics turbo
2562  *
2563  * Disable graphics turbo by resetting the max frequency and setting the
2564  * current frequency to the default.
2565  */
i915_gpu_turbo_disable(void)2566 bool i915_gpu_turbo_disable(void)
2567 {
2568 	struct drm_i915_private *i915;
2569 	struct intel_rps *rps;
2570 	bool ret;
2571 
2572 	i915 = mchdev_get();
2573 	if (!i915)
2574 		return false;
2575 
2576 	rps = &to_gt(i915)->rps;
2577 
2578 	spin_lock_irq(&mchdev_lock);
2579 	rps->max_freq_softlimit = rps->min_freq;
2580 	ret = !__gen5_rps_set(&to_gt(i915)->rps, rps->min_freq);
2581 	spin_unlock_irq(&mchdev_lock);
2582 
2583 	drm_dev_put(&i915->drm);
2584 	return ret;
2585 }
2586 EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
2587 
2588 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
2589 #include "selftest_rps.c"
2590 #include "selftest_slpc.c"
2591 #endif
2592