1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2019 Intel Corporation
4  */
5 
6 #include <linux/string_helpers.h>
7 
8 #include <drm/i915_drm.h>
9 
10 #include "display/intel_display.h"
11 #include "display/intel_display_irq.h"
12 #include "i915_drv.h"
13 #include "i915_irq.h"
14 #include "i915_reg.h"
15 #include "intel_breadcrumbs.h"
16 #include "intel_gt.h"
17 #include "intel_gt_clock_utils.h"
18 #include "intel_gt_irq.h"
19 #include "intel_gt_pm.h"
20 #include "intel_gt_pm_irq.h"
21 #include "intel_gt_print.h"
22 #include "intel_gt_regs.h"
23 #include "intel_mchbar_regs.h"
24 #include "intel_pcode.h"
25 #include "intel_rps.h"
26 #include "vlv_sideband.h"
27 #include "../../../platform/x86/intel_ips.h"
28 
29 #define BUSY_MAX_EI	20u /* ms */
30 
31 /*
32  * Lock protecting IPS related data structures
33  */
34 static DEFINE_SPINLOCK(mchdev_lock);
35 
rps_to_gt(struct intel_rps * rps)36 static struct intel_gt *rps_to_gt(struct intel_rps *rps)
37 {
38 	return container_of(rps, struct intel_gt, rps);
39 }
40 
rps_to_i915(struct intel_rps * rps)41 static struct drm_i915_private *rps_to_i915(struct intel_rps *rps)
42 {
43 	return rps_to_gt(rps)->i915;
44 }
45 
rps_to_uncore(struct intel_rps * rps)46 static struct intel_uncore *rps_to_uncore(struct intel_rps *rps)
47 {
48 	return rps_to_gt(rps)->uncore;
49 }
50 
rps_to_slpc(struct intel_rps * rps)51 static struct intel_guc_slpc *rps_to_slpc(struct intel_rps *rps)
52 {
53 	struct intel_gt *gt = rps_to_gt(rps);
54 
55 	return &gt->uc.guc.slpc;
56 }
57 
rps_uses_slpc(struct intel_rps * rps)58 static bool rps_uses_slpc(struct intel_rps *rps)
59 {
60 	struct intel_gt *gt = rps_to_gt(rps);
61 
62 	return intel_uc_uses_guc_slpc(&gt->uc);
63 }
64 
rps_pm_sanitize_mask(struct intel_rps * rps,u32 mask)65 static u32 rps_pm_sanitize_mask(struct intel_rps *rps, u32 mask)
66 {
67 	return mask & ~rps->pm_intrmsk_mbz;
68 }
69 
set(struct intel_uncore * uncore,i915_reg_t reg,u32 val)70 static void set(struct intel_uncore *uncore, i915_reg_t reg, u32 val)
71 {
72 	intel_uncore_write_fw(uncore, reg, val);
73 }
74 
rps_timer(struct timer_list * t)75 static void rps_timer(struct timer_list *t)
76 {
77 	struct intel_rps *rps = from_timer(rps, t, timer);
78 	struct intel_gt *gt = rps_to_gt(rps);
79 	struct intel_engine_cs *engine;
80 	ktime_t dt, last, timestamp;
81 	enum intel_engine_id id;
82 	s64 max_busy[3] = {};
83 
84 	timestamp = 0;
85 	for_each_engine(engine, gt, id) {
86 		s64 busy;
87 		int i;
88 
89 		dt = intel_engine_get_busy_time(engine, &timestamp);
90 		last = engine->stats.rps;
91 		engine->stats.rps = dt;
92 
93 		busy = ktime_to_ns(ktime_sub(dt, last));
94 		for (i = 0; i < ARRAY_SIZE(max_busy); i++) {
95 			if (busy > max_busy[i])
96 				swap(busy, max_busy[i]);
97 		}
98 	}
99 	last = rps->pm_timestamp;
100 	rps->pm_timestamp = timestamp;
101 
102 	if (intel_rps_is_active(rps)) {
103 		s64 busy;
104 		int i;
105 
106 		dt = ktime_sub(timestamp, last);
107 
108 		/*
109 		 * Our goal is to evaluate each engine independently, so we run
110 		 * at the lowest clocks required to sustain the heaviest
111 		 * workload. However, a task may be split into sequential
112 		 * dependent operations across a set of engines, such that
113 		 * the independent contributions do not account for high load,
114 		 * but overall the task is GPU bound. For example, consider
115 		 * video decode on vcs followed by colour post-processing
116 		 * on vecs, followed by general post-processing on rcs.
117 		 * Since multi-engines being active does imply a single
118 		 * continuous workload across all engines, we hedge our
119 		 * bets by only contributing a factor of the distributed
120 		 * load into our busyness calculation.
121 		 */
122 		busy = max_busy[0];
123 		for (i = 1; i < ARRAY_SIZE(max_busy); i++) {
124 			if (!max_busy[i])
125 				break;
126 
127 			busy += div_u64(max_busy[i], 1 << i);
128 		}
129 		GT_TRACE(gt,
130 			 "busy:%lld [%d%%], max:[%lld, %lld, %lld], interval:%d\n",
131 			 busy, (int)div64_u64(100 * busy, dt),
132 			 max_busy[0], max_busy[1], max_busy[2],
133 			 rps->pm_interval);
134 
135 		if (100 * busy > rps->power.up_threshold * dt &&
136 		    rps->cur_freq < rps->max_freq_softlimit) {
137 			rps->pm_iir |= GEN6_PM_RP_UP_THRESHOLD;
138 			rps->pm_interval = 1;
139 			queue_work(gt->i915->unordered_wq, &rps->work);
140 		} else if (100 * busy < rps->power.down_threshold * dt &&
141 			   rps->cur_freq > rps->min_freq_softlimit) {
142 			rps->pm_iir |= GEN6_PM_RP_DOWN_THRESHOLD;
143 			rps->pm_interval = 1;
144 			queue_work(gt->i915->unordered_wq, &rps->work);
145 		} else {
146 			rps->last_adj = 0;
147 		}
148 
149 		mod_timer(&rps->timer,
150 			  jiffies + msecs_to_jiffies(rps->pm_interval));
151 		rps->pm_interval = min(rps->pm_interval * 2, BUSY_MAX_EI);
152 	}
153 }
154 
rps_start_timer(struct intel_rps * rps)155 static void rps_start_timer(struct intel_rps *rps)
156 {
157 	rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp);
158 	rps->pm_interval = 1;
159 	mod_timer(&rps->timer, jiffies + 1);
160 }
161 
rps_stop_timer(struct intel_rps * rps)162 static void rps_stop_timer(struct intel_rps *rps)
163 {
164 	del_timer_sync(&rps->timer);
165 	rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp);
166 	cancel_work_sync(&rps->work);
167 }
168 
rps_pm_mask(struct intel_rps * rps,u8 val)169 static u32 rps_pm_mask(struct intel_rps *rps, u8 val)
170 {
171 	u32 mask = 0;
172 
173 	/* We use UP_EI_EXPIRED interrupts for both up/down in manual mode */
174 	if (val > rps->min_freq_softlimit)
175 		mask |= (GEN6_PM_RP_UP_EI_EXPIRED |
176 			 GEN6_PM_RP_DOWN_THRESHOLD |
177 			 GEN6_PM_RP_DOWN_TIMEOUT);
178 
179 	if (val < rps->max_freq_softlimit)
180 		mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD;
181 
182 	mask &= rps->pm_events;
183 
184 	return rps_pm_sanitize_mask(rps, ~mask);
185 }
186 
rps_reset_ei(struct intel_rps * rps)187 static void rps_reset_ei(struct intel_rps *rps)
188 {
189 	memset(&rps->ei, 0, sizeof(rps->ei));
190 }
191 
rps_enable_interrupts(struct intel_rps * rps)192 static void rps_enable_interrupts(struct intel_rps *rps)
193 {
194 	struct intel_gt *gt = rps_to_gt(rps);
195 
196 	GEM_BUG_ON(rps_uses_slpc(rps));
197 
198 	GT_TRACE(gt, "interrupts:on rps->pm_events: %x, rps_pm_mask:%x\n",
199 		 rps->pm_events, rps_pm_mask(rps, rps->last_freq));
200 
201 	rps_reset_ei(rps);
202 
203 	spin_lock_irq(gt->irq_lock);
204 	gen6_gt_pm_enable_irq(gt, rps->pm_events);
205 	spin_unlock_irq(gt->irq_lock);
206 
207 	intel_uncore_write(gt->uncore,
208 			   GEN6_PMINTRMSK, rps_pm_mask(rps, rps->last_freq));
209 }
210 
gen6_rps_reset_interrupts(struct intel_rps * rps)211 static void gen6_rps_reset_interrupts(struct intel_rps *rps)
212 {
213 	gen6_gt_pm_reset_iir(rps_to_gt(rps), GEN6_PM_RPS_EVENTS);
214 }
215 
gen11_rps_reset_interrupts(struct intel_rps * rps)216 static void gen11_rps_reset_interrupts(struct intel_rps *rps)
217 {
218 	while (gen11_gt_reset_one_iir(rps_to_gt(rps), 0, GEN11_GTPM))
219 		;
220 }
221 
rps_reset_interrupts(struct intel_rps * rps)222 static void rps_reset_interrupts(struct intel_rps *rps)
223 {
224 	struct intel_gt *gt = rps_to_gt(rps);
225 
226 	spin_lock_irq(gt->irq_lock);
227 	if (GRAPHICS_VER(gt->i915) >= 11)
228 		gen11_rps_reset_interrupts(rps);
229 	else
230 		gen6_rps_reset_interrupts(rps);
231 
232 	rps->pm_iir = 0;
233 	spin_unlock_irq(gt->irq_lock);
234 }
235 
rps_disable_interrupts(struct intel_rps * rps)236 static void rps_disable_interrupts(struct intel_rps *rps)
237 {
238 	struct intel_gt *gt = rps_to_gt(rps);
239 
240 	intel_uncore_write(gt->uncore,
241 			   GEN6_PMINTRMSK, rps_pm_sanitize_mask(rps, ~0u));
242 
243 	spin_lock_irq(gt->irq_lock);
244 	gen6_gt_pm_disable_irq(gt, GEN6_PM_RPS_EVENTS);
245 	spin_unlock_irq(gt->irq_lock);
246 
247 	intel_synchronize_irq(gt->i915);
248 
249 	/*
250 	 * Now that we will not be generating any more work, flush any
251 	 * outstanding tasks. As we are called on the RPS idle path,
252 	 * we will reset the GPU to minimum frequencies, so the current
253 	 * state of the worker can be discarded.
254 	 */
255 	cancel_work_sync(&rps->work);
256 
257 	rps_reset_interrupts(rps);
258 	GT_TRACE(gt, "interrupts:off\n");
259 }
260 
261 static const struct cparams {
262 	u16 i;
263 	u16 t;
264 	u16 m;
265 	u16 c;
266 } cparams[] = {
267 	{ 1, 1333, 301, 28664 },
268 	{ 1, 1066, 294, 24460 },
269 	{ 1, 800, 294, 25192 },
270 	{ 0, 1333, 276, 27605 },
271 	{ 0, 1066, 276, 27605 },
272 	{ 0, 800, 231, 23784 },
273 };
274 
gen5_rps_init(struct intel_rps * rps)275 static void gen5_rps_init(struct intel_rps *rps)
276 {
277 	struct drm_i915_private *i915 = rps_to_i915(rps);
278 	struct intel_uncore *uncore = rps_to_uncore(rps);
279 	u8 fmax, fmin, fstart;
280 	u32 rgvmodectl;
281 	int c_m, i;
282 
283 	if (i915->fsb_freq <= 3200)
284 		c_m = 0;
285 	else if (i915->fsb_freq <= 4800)
286 		c_m = 1;
287 	else
288 		c_m = 2;
289 
290 	for (i = 0; i < ARRAY_SIZE(cparams); i++) {
291 		if (cparams[i].i == c_m && cparams[i].t == i915->mem_freq) {
292 			rps->ips.m = cparams[i].m;
293 			rps->ips.c = cparams[i].c;
294 			break;
295 		}
296 	}
297 
298 	rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);
299 
300 	/* Set up min, max, and cur for interrupt handling */
301 	fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
302 	fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
303 	fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
304 		MEMMODE_FSTART_SHIFT;
305 	drm_dbg(&i915->drm, "fmax: %d, fmin: %d, fstart: %d\n",
306 		fmax, fmin, fstart);
307 
308 	rps->min_freq = fmax;
309 	rps->efficient_freq = fstart;
310 	rps->max_freq = fmin;
311 }
312 
313 static unsigned long
__ips_chipset_val(struct intel_ips * ips)314 __ips_chipset_val(struct intel_ips *ips)
315 {
316 	struct intel_uncore *uncore =
317 		rps_to_uncore(container_of(ips, struct intel_rps, ips));
318 	unsigned long now = jiffies_to_msecs(jiffies), dt;
319 	unsigned long result;
320 	u64 total, delta;
321 
322 	lockdep_assert_held(&mchdev_lock);
323 
324 	/*
325 	 * Prevent division-by-zero if we are asking too fast.
326 	 * Also, we don't get interesting results if we are polling
327 	 * faster than once in 10ms, so just return the saved value
328 	 * in such cases.
329 	 */
330 	dt = now - ips->last_time1;
331 	if (dt <= 10)
332 		return ips->chipset_power;
333 
334 	/* FIXME: handle per-counter overflow */
335 	total = intel_uncore_read(uncore, DMIEC);
336 	total += intel_uncore_read(uncore, DDREC);
337 	total += intel_uncore_read(uncore, CSIEC);
338 
339 	delta = total - ips->last_count1;
340 
341 	result = div_u64(div_u64(ips->m * delta, dt) + ips->c, 10);
342 
343 	ips->last_count1 = total;
344 	ips->last_time1 = now;
345 
346 	ips->chipset_power = result;
347 
348 	return result;
349 }
350 
ips_mch_val(struct intel_uncore * uncore)351 static unsigned long ips_mch_val(struct intel_uncore *uncore)
352 {
353 	unsigned int m, x, b;
354 	u32 tsfs;
355 
356 	tsfs = intel_uncore_read(uncore, TSFS);
357 	x = intel_uncore_read8(uncore, TR1);
358 
359 	b = tsfs & TSFS_INTR_MASK;
360 	m = (tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT;
361 
362 	return m * x / 127 - b;
363 }
364 
_pxvid_to_vd(u8 pxvid)365 static int _pxvid_to_vd(u8 pxvid)
366 {
367 	if (pxvid == 0)
368 		return 0;
369 
370 	if (pxvid >= 8 && pxvid < 31)
371 		pxvid = 31;
372 
373 	return (pxvid + 2) * 125;
374 }
375 
pvid_to_extvid(struct drm_i915_private * i915,u8 pxvid)376 static u32 pvid_to_extvid(struct drm_i915_private *i915, u8 pxvid)
377 {
378 	const int vd = _pxvid_to_vd(pxvid);
379 
380 	if (INTEL_INFO(i915)->is_mobile)
381 		return max(vd - 1125, 0);
382 
383 	return vd;
384 }
385 
__gen5_ips_update(struct intel_ips * ips)386 static void __gen5_ips_update(struct intel_ips *ips)
387 {
388 	struct intel_uncore *uncore =
389 		rps_to_uncore(container_of(ips, struct intel_rps, ips));
390 	u64 now, delta, dt;
391 	u32 count;
392 
393 	lockdep_assert_held(&mchdev_lock);
394 
395 	now = ktime_get_raw_ns();
396 	dt = now - ips->last_time2;
397 	do_div(dt, NSEC_PER_MSEC);
398 
399 	/* Don't divide by 0 */
400 	if (dt <= 10)
401 		return;
402 
403 	count = intel_uncore_read(uncore, GFXEC);
404 	delta = count - ips->last_count2;
405 
406 	ips->last_count2 = count;
407 	ips->last_time2 = now;
408 
409 	/* More magic constants... */
410 	ips->gfx_power = div_u64(delta * 1181, dt * 10);
411 }
412 
gen5_rps_update(struct intel_rps * rps)413 static void gen5_rps_update(struct intel_rps *rps)
414 {
415 	spin_lock_irq(&mchdev_lock);
416 	__gen5_ips_update(&rps->ips);
417 	spin_unlock_irq(&mchdev_lock);
418 }
419 
gen5_invert_freq(struct intel_rps * rps,unsigned int val)420 static unsigned int gen5_invert_freq(struct intel_rps *rps,
421 				     unsigned int val)
422 {
423 	/* Invert the frequency bin into an ips delay */
424 	val = rps->max_freq - val;
425 	val = rps->min_freq + val;
426 
427 	return val;
428 }
429 
__gen5_rps_set(struct intel_rps * rps,u8 val)430 static int __gen5_rps_set(struct intel_rps *rps, u8 val)
431 {
432 	struct intel_uncore *uncore = rps_to_uncore(rps);
433 	u16 rgvswctl;
434 
435 	lockdep_assert_held(&mchdev_lock);
436 
437 	rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
438 	if (rgvswctl & MEMCTL_CMD_STS) {
439 		drm_dbg(&rps_to_i915(rps)->drm,
440 			"gpu busy, RCS change rejected\n");
441 		return -EBUSY; /* still busy with another command */
442 	}
443 
444 	/* Invert the frequency bin into an ips delay */
445 	val = gen5_invert_freq(rps, val);
446 
447 	rgvswctl =
448 		(MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |
449 		(val << MEMCTL_FREQ_SHIFT) |
450 		MEMCTL_SFCAVM;
451 	intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);
452 	intel_uncore_posting_read16(uncore, MEMSWCTL);
453 
454 	rgvswctl |= MEMCTL_CMD_STS;
455 	intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);
456 
457 	return 0;
458 }
459 
gen5_rps_set(struct intel_rps * rps,u8 val)460 static int gen5_rps_set(struct intel_rps *rps, u8 val)
461 {
462 	int err;
463 
464 	spin_lock_irq(&mchdev_lock);
465 	err = __gen5_rps_set(rps, val);
466 	spin_unlock_irq(&mchdev_lock);
467 
468 	return err;
469 }
470 
intel_pxfreq(u32 vidfreq)471 static unsigned long intel_pxfreq(u32 vidfreq)
472 {
473 	int div = (vidfreq & 0x3f0000) >> 16;
474 	int post = (vidfreq & 0x3000) >> 12;
475 	int pre = (vidfreq & 0x7);
476 
477 	if (!pre)
478 		return 0;
479 
480 	return div * 133333 / (pre << post);
481 }
482 
init_emon(struct intel_uncore * uncore)483 static unsigned int init_emon(struct intel_uncore *uncore)
484 {
485 	u8 pxw[16];
486 	int i;
487 
488 	/* Disable to program */
489 	intel_uncore_write(uncore, ECR, 0);
490 	intel_uncore_posting_read(uncore, ECR);
491 
492 	/* Program energy weights for various events */
493 	intel_uncore_write(uncore, SDEW, 0x15040d00);
494 	intel_uncore_write(uncore, CSIEW0, 0x007f0000);
495 	intel_uncore_write(uncore, CSIEW1, 0x1e220004);
496 	intel_uncore_write(uncore, CSIEW2, 0x04000004);
497 
498 	for (i = 0; i < 5; i++)
499 		intel_uncore_write(uncore, PEW(i), 0);
500 	for (i = 0; i < 3; i++)
501 		intel_uncore_write(uncore, DEW(i), 0);
502 
503 	/* Program P-state weights to account for frequency power adjustment */
504 	for (i = 0; i < 16; i++) {
505 		u32 pxvidfreq = intel_uncore_read(uncore, PXVFREQ(i));
506 		unsigned int freq = intel_pxfreq(pxvidfreq);
507 		unsigned int vid =
508 			(pxvidfreq & PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;
509 		unsigned int val;
510 
511 		val = vid * vid * freq / 1000 * 255;
512 		val /= 127 * 127 * 900;
513 
514 		pxw[i] = val;
515 	}
516 	/* Render standby states get 0 weight */
517 	pxw[14] = 0;
518 	pxw[15] = 0;
519 
520 	for (i = 0; i < 4; i++) {
521 		intel_uncore_write(uncore, PXW(i),
522 				   pxw[i * 4 + 0] << 24 |
523 				   pxw[i * 4 + 1] << 16 |
524 				   pxw[i * 4 + 2] <<  8 |
525 				   pxw[i * 4 + 3] <<  0);
526 	}
527 
528 	/* Adjust magic regs to magic values (more experimental results) */
529 	intel_uncore_write(uncore, OGW0, 0);
530 	intel_uncore_write(uncore, OGW1, 0);
531 	intel_uncore_write(uncore, EG0, 0x00007f00);
532 	intel_uncore_write(uncore, EG1, 0x0000000e);
533 	intel_uncore_write(uncore, EG2, 0x000e0000);
534 	intel_uncore_write(uncore, EG3, 0x68000300);
535 	intel_uncore_write(uncore, EG4, 0x42000000);
536 	intel_uncore_write(uncore, EG5, 0x00140031);
537 	intel_uncore_write(uncore, EG6, 0);
538 	intel_uncore_write(uncore, EG7, 0);
539 
540 	for (i = 0; i < 8; i++)
541 		intel_uncore_write(uncore, PXWL(i), 0);
542 
543 	/* Enable PMON + select events */
544 	intel_uncore_write(uncore, ECR, 0x80000019);
545 
546 	return intel_uncore_read(uncore, LCFUSE02) & LCFUSE_HIV_MASK;
547 }
548 
gen5_rps_enable(struct intel_rps * rps)549 static bool gen5_rps_enable(struct intel_rps *rps)
550 {
551 	struct drm_i915_private *i915 = rps_to_i915(rps);
552 	struct intel_uncore *uncore = rps_to_uncore(rps);
553 	u8 fstart, vstart;
554 	u32 rgvmodectl;
555 
556 	spin_lock_irq(&mchdev_lock);
557 
558 	rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);
559 
560 	/* Enable temp reporting */
561 	intel_uncore_write16(uncore, PMMISC,
562 			     intel_uncore_read16(uncore, PMMISC) | MCPPCE_EN);
563 	intel_uncore_write16(uncore, TSC1,
564 			     intel_uncore_read16(uncore, TSC1) | TSE);
565 
566 	/* 100ms RC evaluation intervals */
567 	intel_uncore_write(uncore, RCUPEI, 100000);
568 	intel_uncore_write(uncore, RCDNEI, 100000);
569 
570 	/* Set max/min thresholds to 90ms and 80ms respectively */
571 	intel_uncore_write(uncore, RCBMAXAVG, 90000);
572 	intel_uncore_write(uncore, RCBMINAVG, 80000);
573 
574 	intel_uncore_write(uncore, MEMIHYST, 1);
575 
576 	/* Set up min, max, and cur for interrupt handling */
577 	fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
578 		MEMMODE_FSTART_SHIFT;
579 
580 	vstart = (intel_uncore_read(uncore, PXVFREQ(fstart)) &
581 		  PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;
582 
583 	intel_uncore_write(uncore,
584 			   MEMINTREN,
585 			   MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
586 
587 	intel_uncore_write(uncore, VIDSTART, vstart);
588 	intel_uncore_posting_read(uncore, VIDSTART);
589 
590 	rgvmodectl |= MEMMODE_SWMODE_EN;
591 	intel_uncore_write(uncore, MEMMODECTL, rgvmodectl);
592 
593 	if (wait_for_atomic((intel_uncore_read(uncore, MEMSWCTL) &
594 			     MEMCTL_CMD_STS) == 0, 10))
595 		drm_err(&uncore->i915->drm,
596 			"stuck trying to change perf mode\n");
597 	mdelay(1);
598 
599 	__gen5_rps_set(rps, rps->cur_freq);
600 
601 	rps->ips.last_count1 = intel_uncore_read(uncore, DMIEC);
602 	rps->ips.last_count1 += intel_uncore_read(uncore, DDREC);
603 	rps->ips.last_count1 += intel_uncore_read(uncore, CSIEC);
604 	rps->ips.last_time1 = jiffies_to_msecs(jiffies);
605 
606 	rps->ips.last_count2 = intel_uncore_read(uncore, GFXEC);
607 	rps->ips.last_time2 = ktime_get_raw_ns();
608 
609 	spin_lock(&i915->irq_lock);
610 	ilk_enable_display_irq(i915, DE_PCU_EVENT);
611 	spin_unlock(&i915->irq_lock);
612 
613 	spin_unlock_irq(&mchdev_lock);
614 
615 	rps->ips.corr = init_emon(uncore);
616 
617 	return true;
618 }
619 
gen5_rps_disable(struct intel_rps * rps)620 static void gen5_rps_disable(struct intel_rps *rps)
621 {
622 	struct drm_i915_private *i915 = rps_to_i915(rps);
623 	struct intel_uncore *uncore = rps_to_uncore(rps);
624 	u16 rgvswctl;
625 
626 	spin_lock_irq(&mchdev_lock);
627 
628 	spin_lock(&i915->irq_lock);
629 	ilk_disable_display_irq(i915, DE_PCU_EVENT);
630 	spin_unlock(&i915->irq_lock);
631 
632 	rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
633 
634 	/* Ack interrupts, disable EFC interrupt */
635 	intel_uncore_rmw(uncore, MEMINTREN, MEMINT_EVAL_CHG_EN, 0);
636 	intel_uncore_write(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
637 
638 	/* Go back to the starting frequency */
639 	__gen5_rps_set(rps, rps->idle_freq);
640 	mdelay(1);
641 	rgvswctl |= MEMCTL_CMD_STS;
642 	intel_uncore_write(uncore, MEMSWCTL, rgvswctl);
643 	mdelay(1);
644 
645 	spin_unlock_irq(&mchdev_lock);
646 }
647 
rps_limits(struct intel_rps * rps,u8 val)648 static u32 rps_limits(struct intel_rps *rps, u8 val)
649 {
650 	u32 limits;
651 
652 	/*
653 	 * Only set the down limit when we've reached the lowest level to avoid
654 	 * getting more interrupts, otherwise leave this clear. This prevents a
655 	 * race in the hw when coming out of rc6: There's a tiny window where
656 	 * the hw runs at the minimal clock before selecting the desired
657 	 * frequency, if the down threshold expires in that window we will not
658 	 * receive a down interrupt.
659 	 */
660 	if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) {
661 		limits = rps->max_freq_softlimit << 23;
662 		if (val <= rps->min_freq_softlimit)
663 			limits |= rps->min_freq_softlimit << 14;
664 	} else {
665 		limits = rps->max_freq_softlimit << 24;
666 		if (val <= rps->min_freq_softlimit)
667 			limits |= rps->min_freq_softlimit << 16;
668 	}
669 
670 	return limits;
671 }
672 
rps_set_power(struct intel_rps * rps,int new_power)673 static void rps_set_power(struct intel_rps *rps, int new_power)
674 {
675 	struct intel_gt *gt = rps_to_gt(rps);
676 	struct intel_uncore *uncore = gt->uncore;
677 	u32 ei_up = 0, ei_down = 0;
678 
679 	lockdep_assert_held(&rps->power.mutex);
680 
681 	if (new_power == rps->power.mode)
682 		return;
683 
684 	/* Note the units here are not exactly 1us, but 1280ns. */
685 	switch (new_power) {
686 	case LOW_POWER:
687 		ei_up = 16000;
688 		ei_down = 32000;
689 		break;
690 
691 	case BETWEEN:
692 		ei_up = 13000;
693 		ei_down = 32000;
694 		break;
695 
696 	case HIGH_POWER:
697 		ei_up = 10000;
698 		ei_down = 32000;
699 		break;
700 	}
701 
702 	/* When byt can survive without system hang with dynamic
703 	 * sw freq adjustments, this restriction can be lifted.
704 	 */
705 	if (IS_VALLEYVIEW(gt->i915))
706 		goto skip_hw_write;
707 
708 	GT_TRACE(gt,
709 		 "changing power mode [%d], up %d%% @ %dus, down %d%% @ %dus\n",
710 		 new_power,
711 		 rps->power.up_threshold, ei_up,
712 		 rps->power.down_threshold, ei_down);
713 
714 	set(uncore, GEN6_RP_UP_EI,
715 	    intel_gt_ns_to_pm_interval(gt, ei_up * 1000));
716 	set(uncore, GEN6_RP_UP_THRESHOLD,
717 	    intel_gt_ns_to_pm_interval(gt,
718 				       ei_up * rps->power.up_threshold * 10));
719 
720 	set(uncore, GEN6_RP_DOWN_EI,
721 	    intel_gt_ns_to_pm_interval(gt, ei_down * 1000));
722 	set(uncore, GEN6_RP_DOWN_THRESHOLD,
723 	    intel_gt_ns_to_pm_interval(gt,
724 				       ei_down *
725 				       rps->power.down_threshold * 10));
726 
727 	set(uncore, GEN6_RP_CONTROL,
728 	    (GRAPHICS_VER(gt->i915) > 9 ? 0 : GEN6_RP_MEDIA_TURBO) |
729 	    GEN6_RP_MEDIA_HW_NORMAL_MODE |
730 	    GEN6_RP_MEDIA_IS_GFX |
731 	    GEN6_RP_ENABLE |
732 	    GEN6_RP_UP_BUSY_AVG |
733 	    GEN6_RP_DOWN_IDLE_AVG);
734 
735 skip_hw_write:
736 	rps->power.mode = new_power;
737 }
738 
gen6_rps_set_thresholds(struct intel_rps * rps,u8 val)739 static void gen6_rps_set_thresholds(struct intel_rps *rps, u8 val)
740 {
741 	int new_power;
742 
743 	new_power = rps->power.mode;
744 	switch (rps->power.mode) {
745 	case LOW_POWER:
746 		if (val > rps->efficient_freq + 1 &&
747 		    val > rps->cur_freq)
748 			new_power = BETWEEN;
749 		break;
750 
751 	case BETWEEN:
752 		if (val <= rps->efficient_freq &&
753 		    val < rps->cur_freq)
754 			new_power = LOW_POWER;
755 		else if (val >= rps->rp0_freq &&
756 			 val > rps->cur_freq)
757 			new_power = HIGH_POWER;
758 		break;
759 
760 	case HIGH_POWER:
761 		if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 &&
762 		    val < rps->cur_freq)
763 			new_power = BETWEEN;
764 		break;
765 	}
766 	/* Max/min bins are special */
767 	if (val <= rps->min_freq_softlimit)
768 		new_power = LOW_POWER;
769 	if (val >= rps->max_freq_softlimit)
770 		new_power = HIGH_POWER;
771 
772 	mutex_lock(&rps->power.mutex);
773 	if (rps->power.interactive)
774 		new_power = HIGH_POWER;
775 	rps_set_power(rps, new_power);
776 	mutex_unlock(&rps->power.mutex);
777 }
778 
intel_rps_mark_interactive(struct intel_rps * rps,bool interactive)779 void intel_rps_mark_interactive(struct intel_rps *rps, bool interactive)
780 {
781 	GT_TRACE(rps_to_gt(rps), "mark interactive: %s\n",
782 		 str_yes_no(interactive));
783 
784 	mutex_lock(&rps->power.mutex);
785 	if (interactive) {
786 		if (!rps->power.interactive++ && intel_rps_is_active(rps))
787 			rps_set_power(rps, HIGH_POWER);
788 	} else {
789 		GEM_BUG_ON(!rps->power.interactive);
790 		rps->power.interactive--;
791 	}
792 	mutex_unlock(&rps->power.mutex);
793 }
794 
gen6_rps_set(struct intel_rps * rps,u8 val)795 static int gen6_rps_set(struct intel_rps *rps, u8 val)
796 {
797 	struct intel_uncore *uncore = rps_to_uncore(rps);
798 	struct drm_i915_private *i915 = rps_to_i915(rps);
799 	u32 swreq;
800 
801 	GEM_BUG_ON(rps_uses_slpc(rps));
802 
803 	if (GRAPHICS_VER(i915) >= 9)
804 		swreq = GEN9_FREQUENCY(val);
805 	else if (IS_HASWELL(i915) || IS_BROADWELL(i915))
806 		swreq = HSW_FREQUENCY(val);
807 	else
808 		swreq = (GEN6_FREQUENCY(val) |
809 			 GEN6_OFFSET(0) |
810 			 GEN6_AGGRESSIVE_TURBO);
811 	set(uncore, GEN6_RPNSWREQ, swreq);
812 
813 	GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d, swreq:%x\n",
814 		 val, intel_gpu_freq(rps, val), swreq);
815 
816 	return 0;
817 }
818 
vlv_rps_set(struct intel_rps * rps,u8 val)819 static int vlv_rps_set(struct intel_rps *rps, u8 val)
820 {
821 	struct drm_i915_private *i915 = rps_to_i915(rps);
822 	int err;
823 
824 	vlv_punit_get(i915);
825 	err = vlv_punit_write(i915, PUNIT_REG_GPU_FREQ_REQ, val);
826 	vlv_punit_put(i915);
827 
828 	GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d\n",
829 		 val, intel_gpu_freq(rps, val));
830 
831 	return err;
832 }
833 
rps_set(struct intel_rps * rps,u8 val,bool update)834 static int rps_set(struct intel_rps *rps, u8 val, bool update)
835 {
836 	struct drm_i915_private *i915 = rps_to_i915(rps);
837 	int err;
838 
839 	if (val == rps->last_freq)
840 		return 0;
841 
842 	if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
843 		err = vlv_rps_set(rps, val);
844 	else if (GRAPHICS_VER(i915) >= 6)
845 		err = gen6_rps_set(rps, val);
846 	else
847 		err = gen5_rps_set(rps, val);
848 	if (err)
849 		return err;
850 
851 	if (update && GRAPHICS_VER(i915) >= 6)
852 		gen6_rps_set_thresholds(rps, val);
853 	rps->last_freq = val;
854 
855 	return 0;
856 }
857 
intel_rps_unpark(struct intel_rps * rps)858 void intel_rps_unpark(struct intel_rps *rps)
859 {
860 	if (!intel_rps_is_enabled(rps))
861 		return;
862 
863 	GT_TRACE(rps_to_gt(rps), "unpark:%x\n", rps->cur_freq);
864 
865 	/*
866 	 * Use the user's desired frequency as a guide, but for better
867 	 * performance, jump directly to RPe as our starting frequency.
868 	 */
869 	mutex_lock(&rps->lock);
870 
871 	intel_rps_set_active(rps);
872 	intel_rps_set(rps,
873 		      clamp(rps->cur_freq,
874 			    rps->min_freq_softlimit,
875 			    rps->max_freq_softlimit));
876 
877 	mutex_unlock(&rps->lock);
878 
879 	rps->pm_iir = 0;
880 	if (intel_rps_has_interrupts(rps))
881 		rps_enable_interrupts(rps);
882 	if (intel_rps_uses_timer(rps))
883 		rps_start_timer(rps);
884 
885 	if (GRAPHICS_VER(rps_to_i915(rps)) == 5)
886 		gen5_rps_update(rps);
887 }
888 
intel_rps_park(struct intel_rps * rps)889 void intel_rps_park(struct intel_rps *rps)
890 {
891 	int adj;
892 
893 	if (!intel_rps_is_enabled(rps))
894 		return;
895 
896 	if (!intel_rps_clear_active(rps))
897 		return;
898 
899 	if (intel_rps_uses_timer(rps))
900 		rps_stop_timer(rps);
901 	if (intel_rps_has_interrupts(rps))
902 		rps_disable_interrupts(rps);
903 
904 	if (rps->last_freq <= rps->idle_freq)
905 		return;
906 
907 	/*
908 	 * The punit delays the write of the frequency and voltage until it
909 	 * determines the GPU is awake. During normal usage we don't want to
910 	 * waste power changing the frequency if the GPU is sleeping (rc6).
911 	 * However, the GPU and driver is now idle and we do not want to delay
912 	 * switching to minimum voltage (reducing power whilst idle) as we do
913 	 * not expect to be woken in the near future and so must flush the
914 	 * change by waking the device.
915 	 *
916 	 * We choose to take the media powerwell (either would do to trick the
917 	 * punit into committing the voltage change) as that takes a lot less
918 	 * power than the render powerwell.
919 	 */
920 	intel_uncore_forcewake_get(rps_to_uncore(rps), FORCEWAKE_MEDIA);
921 	rps_set(rps, rps->idle_freq, false);
922 	intel_uncore_forcewake_put(rps_to_uncore(rps), FORCEWAKE_MEDIA);
923 
924 	/*
925 	 * Since we will try and restart from the previously requested
926 	 * frequency on unparking, treat this idle point as a downclock
927 	 * interrupt and reduce the frequency for resume. If we park/unpark
928 	 * more frequently than the rps worker can run, we will not respond
929 	 * to any EI and never see a change in frequency.
930 	 *
931 	 * (Note we accommodate Cherryview's limitation of only using an
932 	 * even bin by applying it to all.)
933 	 */
934 	adj = rps->last_adj;
935 	if (adj < 0)
936 		adj *= 2;
937 	else /* CHV needs even encode values */
938 		adj = -2;
939 	rps->last_adj = adj;
940 	rps->cur_freq = max_t(int, rps->cur_freq + adj, rps->min_freq);
941 	if (rps->cur_freq < rps->efficient_freq) {
942 		rps->cur_freq = rps->efficient_freq;
943 		rps->last_adj = 0;
944 	}
945 
946 	GT_TRACE(rps_to_gt(rps), "park:%x\n", rps->cur_freq);
947 }
948 
intel_rps_get_boost_frequency(struct intel_rps * rps)949 u32 intel_rps_get_boost_frequency(struct intel_rps *rps)
950 {
951 	struct intel_guc_slpc *slpc;
952 
953 	if (rps_uses_slpc(rps)) {
954 		slpc = rps_to_slpc(rps);
955 
956 		return slpc->boost_freq;
957 	} else {
958 		return intel_gpu_freq(rps, rps->boost_freq);
959 	}
960 }
961 
rps_set_boost_freq(struct intel_rps * rps,u32 val)962 static int rps_set_boost_freq(struct intel_rps *rps, u32 val)
963 {
964 	bool boost = false;
965 
966 	/* Validate against (static) hardware limits */
967 	val = intel_freq_opcode(rps, val);
968 	if (val < rps->min_freq || val > rps->max_freq)
969 		return -EINVAL;
970 
971 	mutex_lock(&rps->lock);
972 	if (val != rps->boost_freq) {
973 		rps->boost_freq = val;
974 		boost = atomic_read(&rps->num_waiters);
975 	}
976 	mutex_unlock(&rps->lock);
977 	if (boost)
978 		queue_work(rps_to_gt(rps)->i915->unordered_wq, &rps->work);
979 
980 	return 0;
981 }
982 
intel_rps_set_boost_frequency(struct intel_rps * rps,u32 freq)983 int intel_rps_set_boost_frequency(struct intel_rps *rps, u32 freq)
984 {
985 	struct intel_guc_slpc *slpc;
986 
987 	if (rps_uses_slpc(rps)) {
988 		slpc = rps_to_slpc(rps);
989 
990 		return intel_guc_slpc_set_boost_freq(slpc, freq);
991 	} else {
992 		return rps_set_boost_freq(rps, freq);
993 	}
994 }
995 
intel_rps_dec_waiters(struct intel_rps * rps)996 void intel_rps_dec_waiters(struct intel_rps *rps)
997 {
998 	struct intel_guc_slpc *slpc;
999 
1000 	if (rps_uses_slpc(rps)) {
1001 		slpc = rps_to_slpc(rps);
1002 
1003 		intel_guc_slpc_dec_waiters(slpc);
1004 	} else {
1005 		atomic_dec(&rps->num_waiters);
1006 	}
1007 }
1008 
intel_rps_boost(struct i915_request * rq)1009 void intel_rps_boost(struct i915_request *rq)
1010 {
1011 	struct intel_guc_slpc *slpc;
1012 
1013 	if (i915_request_signaled(rq) || i915_request_has_waitboost(rq))
1014 		return;
1015 
1016 	/* Serializes with i915_request_retire() */
1017 	if (!test_and_set_bit(I915_FENCE_FLAG_BOOST, &rq->fence.flags)) {
1018 		struct intel_rps *rps = &READ_ONCE(rq->engine)->gt->rps;
1019 
1020 		if (rps_uses_slpc(rps)) {
1021 			slpc = rps_to_slpc(rps);
1022 
1023 			if (slpc->min_freq_softlimit >= slpc->boost_freq)
1024 				return;
1025 
1026 			/* Return if old value is non zero */
1027 			if (!atomic_fetch_inc(&slpc->num_waiters)) {
1028 				GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n",
1029 					 rq->fence.context, rq->fence.seqno);
1030 				queue_work(rps_to_gt(rps)->i915->unordered_wq,
1031 					   &slpc->boost_work);
1032 			}
1033 
1034 			return;
1035 		}
1036 
1037 		if (atomic_fetch_inc(&rps->num_waiters))
1038 			return;
1039 
1040 		if (!intel_rps_is_active(rps))
1041 			return;
1042 
1043 		GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n",
1044 			 rq->fence.context, rq->fence.seqno);
1045 
1046 		if (READ_ONCE(rps->cur_freq) < rps->boost_freq)
1047 			queue_work(rps_to_gt(rps)->i915->unordered_wq, &rps->work);
1048 
1049 		WRITE_ONCE(rps->boosts, rps->boosts + 1); /* debug only */
1050 	}
1051 }
1052 
intel_rps_set(struct intel_rps * rps,u8 val)1053 int intel_rps_set(struct intel_rps *rps, u8 val)
1054 {
1055 	int err;
1056 
1057 	lockdep_assert_held(&rps->lock);
1058 	GEM_BUG_ON(val > rps->max_freq);
1059 	GEM_BUG_ON(val < rps->min_freq);
1060 
1061 	if (intel_rps_is_active(rps)) {
1062 		err = rps_set(rps, val, true);
1063 		if (err)
1064 			return err;
1065 
1066 		/*
1067 		 * Make sure we continue to get interrupts
1068 		 * until we hit the minimum or maximum frequencies.
1069 		 */
1070 		if (intel_rps_has_interrupts(rps)) {
1071 			struct intel_uncore *uncore = rps_to_uncore(rps);
1072 
1073 			set(uncore,
1074 			    GEN6_RP_INTERRUPT_LIMITS, rps_limits(rps, val));
1075 
1076 			set(uncore, GEN6_PMINTRMSK, rps_pm_mask(rps, val));
1077 		}
1078 	}
1079 
1080 	rps->cur_freq = val;
1081 	return 0;
1082 }
1083 
intel_rps_read_state_cap(struct intel_rps * rps)1084 static u32 intel_rps_read_state_cap(struct intel_rps *rps)
1085 {
1086 	struct drm_i915_private *i915 = rps_to_i915(rps);
1087 	struct intel_uncore *uncore = rps_to_uncore(rps);
1088 
1089 	if (IS_PONTEVECCHIO(i915))
1090 		return intel_uncore_read(uncore, PVC_RP_STATE_CAP);
1091 	else if (IS_XEHPSDV(i915))
1092 		return intel_uncore_read(uncore, XEHPSDV_RP_STATE_CAP);
1093 	else if (IS_GEN9_LP(i915))
1094 		return intel_uncore_read(uncore, BXT_RP_STATE_CAP);
1095 	else
1096 		return intel_uncore_read(uncore, GEN6_RP_STATE_CAP);
1097 }
1098 
1099 static void
mtl_get_freq_caps(struct intel_rps * rps,struct intel_rps_freq_caps * caps)1100 mtl_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps)
1101 {
1102 	struct intel_uncore *uncore = rps_to_uncore(rps);
1103 	u32 rp_state_cap = rps_to_gt(rps)->type == GT_MEDIA ?
1104 				intel_uncore_read(uncore, MTL_MEDIAP_STATE_CAP) :
1105 				intel_uncore_read(uncore, MTL_RP_STATE_CAP);
1106 	u32 rpe = rps_to_gt(rps)->type == GT_MEDIA ?
1107 			intel_uncore_read(uncore, MTL_MPE_FREQUENCY) :
1108 			intel_uncore_read(uncore, MTL_GT_RPE_FREQUENCY);
1109 
1110 	/* MTL values are in units of 16.67 MHz */
1111 	caps->rp0_freq = REG_FIELD_GET(MTL_RP0_CAP_MASK, rp_state_cap);
1112 	caps->min_freq = REG_FIELD_GET(MTL_RPN_CAP_MASK, rp_state_cap);
1113 	caps->rp1_freq = REG_FIELD_GET(MTL_RPE_MASK, rpe);
1114 }
1115 
1116 static void
__gen6_rps_get_freq_caps(struct intel_rps * rps,struct intel_rps_freq_caps * caps)1117 __gen6_rps_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps)
1118 {
1119 	struct drm_i915_private *i915 = rps_to_i915(rps);
1120 	u32 rp_state_cap;
1121 
1122 	rp_state_cap = intel_rps_read_state_cap(rps);
1123 
1124 	/* static values from HW: RP0 > RP1 > RPn (min_freq) */
1125 	if (IS_GEN9_LP(i915)) {
1126 		caps->rp0_freq = (rp_state_cap >> 16) & 0xff;
1127 		caps->rp1_freq = (rp_state_cap >>  8) & 0xff;
1128 		caps->min_freq = (rp_state_cap >>  0) & 0xff;
1129 	} else {
1130 		caps->rp0_freq = (rp_state_cap >>  0) & 0xff;
1131 		if (GRAPHICS_VER(i915) >= 10)
1132 			caps->rp1_freq = REG_FIELD_GET(RPE_MASK,
1133 						       intel_uncore_read(to_gt(i915)->uncore,
1134 						       GEN10_FREQ_INFO_REC));
1135 		else
1136 			caps->rp1_freq = (rp_state_cap >>  8) & 0xff;
1137 		caps->min_freq = (rp_state_cap >> 16) & 0xff;
1138 	}
1139 
1140 	if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) {
1141 		/*
1142 		 * In this case rp_state_cap register reports frequencies in
1143 		 * units of 50 MHz. Convert these to the actual "hw unit", i.e.
1144 		 * units of 16.67 MHz
1145 		 */
1146 		caps->rp0_freq *= GEN9_FREQ_SCALER;
1147 		caps->rp1_freq *= GEN9_FREQ_SCALER;
1148 		caps->min_freq *= GEN9_FREQ_SCALER;
1149 	}
1150 }
1151 
1152 /**
1153  * gen6_rps_get_freq_caps - Get freq caps exposed by HW
1154  * @rps: the intel_rps structure
1155  * @caps: returned freq caps
1156  *
1157  * Returned "caps" frequencies should be converted to MHz using
1158  * intel_gpu_freq()
1159  */
gen6_rps_get_freq_caps(struct intel_rps * rps,struct intel_rps_freq_caps * caps)1160 void gen6_rps_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps)
1161 {
1162 	struct drm_i915_private *i915 = rps_to_i915(rps);
1163 
1164 	if (IS_METEORLAKE(i915))
1165 		return mtl_get_freq_caps(rps, caps);
1166 	else
1167 		return __gen6_rps_get_freq_caps(rps, caps);
1168 }
1169 
gen6_rps_init(struct intel_rps * rps)1170 static void gen6_rps_init(struct intel_rps *rps)
1171 {
1172 	struct drm_i915_private *i915 = rps_to_i915(rps);
1173 	struct intel_rps_freq_caps caps;
1174 
1175 	gen6_rps_get_freq_caps(rps, &caps);
1176 	rps->rp0_freq = caps.rp0_freq;
1177 	rps->rp1_freq = caps.rp1_freq;
1178 	rps->min_freq = caps.min_freq;
1179 
1180 	/* hw_max = RP0 until we check for overclocking */
1181 	rps->max_freq = rps->rp0_freq;
1182 
1183 	rps->efficient_freq = rps->rp1_freq;
1184 	if (IS_HASWELL(i915) || IS_BROADWELL(i915) ||
1185 	    IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) {
1186 		u32 ddcc_status = 0;
1187 		u32 mult = 1;
1188 
1189 		if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11)
1190 			mult = GEN9_FREQ_SCALER;
1191 		if (snb_pcode_read(rps_to_gt(rps)->uncore,
1192 				   HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
1193 				   &ddcc_status, NULL) == 0)
1194 			rps->efficient_freq =
1195 				clamp_t(u32,
1196 					((ddcc_status >> 8) & 0xff) * mult,
1197 					rps->min_freq,
1198 					rps->max_freq);
1199 	}
1200 }
1201 
rps_reset(struct intel_rps * rps)1202 static bool rps_reset(struct intel_rps *rps)
1203 {
1204 	struct drm_i915_private *i915 = rps_to_i915(rps);
1205 
1206 	/* force a reset */
1207 	rps->power.mode = -1;
1208 	rps->last_freq = -1;
1209 
1210 	if (rps_set(rps, rps->min_freq, true)) {
1211 		drm_err(&i915->drm, "Failed to reset RPS to initial values\n");
1212 		return false;
1213 	}
1214 
1215 	rps->cur_freq = rps->min_freq;
1216 	return true;
1217 }
1218 
1219 /* See the Gen9_GT_PM_Programming_Guide doc for the below */
gen9_rps_enable(struct intel_rps * rps)1220 static bool gen9_rps_enable(struct intel_rps *rps)
1221 {
1222 	struct intel_gt *gt = rps_to_gt(rps);
1223 	struct intel_uncore *uncore = gt->uncore;
1224 
1225 	/* Program defaults and thresholds for RPS */
1226 	if (GRAPHICS_VER(gt->i915) == 9)
1227 		intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ,
1228 				      GEN9_FREQUENCY(rps->rp1_freq));
1229 
1230 	intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 0xa);
1231 
1232 	rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD;
1233 
1234 	return rps_reset(rps);
1235 }
1236 
gen8_rps_enable(struct intel_rps * rps)1237 static bool gen8_rps_enable(struct intel_rps *rps)
1238 {
1239 	struct intel_uncore *uncore = rps_to_uncore(rps);
1240 
1241 	intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ,
1242 			      HSW_FREQUENCY(rps->rp1_freq));
1243 
1244 	intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1245 
1246 	rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD;
1247 
1248 	return rps_reset(rps);
1249 }
1250 
gen6_rps_enable(struct intel_rps * rps)1251 static bool gen6_rps_enable(struct intel_rps *rps)
1252 {
1253 	struct intel_uncore *uncore = rps_to_uncore(rps);
1254 
1255 	/* Power down if completely idle for over 50ms */
1256 	intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 50000);
1257 	intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1258 
1259 	rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD |
1260 			  GEN6_PM_RP_DOWN_THRESHOLD |
1261 			  GEN6_PM_RP_DOWN_TIMEOUT);
1262 
1263 	return rps_reset(rps);
1264 }
1265 
chv_rps_max_freq(struct intel_rps * rps)1266 static int chv_rps_max_freq(struct intel_rps *rps)
1267 {
1268 	struct drm_i915_private *i915 = rps_to_i915(rps);
1269 	struct intel_gt *gt = rps_to_gt(rps);
1270 	u32 val;
1271 
1272 	val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE);
1273 
1274 	switch (gt->info.sseu.eu_total) {
1275 	case 8:
1276 		/* (2 * 4) config */
1277 		val >>= FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT;
1278 		break;
1279 	case 12:
1280 		/* (2 * 6) config */
1281 		val >>= FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT;
1282 		break;
1283 	case 16:
1284 		/* (2 * 8) config */
1285 	default:
1286 		/* Setting (2 * 8) Min RP0 for any other combination */
1287 		val >>= FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT;
1288 		break;
1289 	}
1290 
1291 	return val & FB_GFX_FREQ_FUSE_MASK;
1292 }
1293 
chv_rps_rpe_freq(struct intel_rps * rps)1294 static int chv_rps_rpe_freq(struct intel_rps *rps)
1295 {
1296 	struct drm_i915_private *i915 = rps_to_i915(rps);
1297 	u32 val;
1298 
1299 	val = vlv_punit_read(i915, PUNIT_GPU_DUTYCYCLE_REG);
1300 	val >>= PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT;
1301 
1302 	return val & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK;
1303 }
1304 
chv_rps_guar_freq(struct intel_rps * rps)1305 static int chv_rps_guar_freq(struct intel_rps *rps)
1306 {
1307 	struct drm_i915_private *i915 = rps_to_i915(rps);
1308 	u32 val;
1309 
1310 	val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE);
1311 
1312 	return val & FB_GFX_FREQ_FUSE_MASK;
1313 }
1314 
chv_rps_min_freq(struct intel_rps * rps)1315 static u32 chv_rps_min_freq(struct intel_rps *rps)
1316 {
1317 	struct drm_i915_private *i915 = rps_to_i915(rps);
1318 	u32 val;
1319 
1320 	val = vlv_punit_read(i915, FB_GFX_FMIN_AT_VMIN_FUSE);
1321 	val >>= FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT;
1322 
1323 	return val & FB_GFX_FREQ_FUSE_MASK;
1324 }
1325 
chv_rps_enable(struct intel_rps * rps)1326 static bool chv_rps_enable(struct intel_rps *rps)
1327 {
1328 	struct intel_uncore *uncore = rps_to_uncore(rps);
1329 	struct drm_i915_private *i915 = rps_to_i915(rps);
1330 	u32 val;
1331 
1332 	/* 1: Program defaults and thresholds for RPS*/
1333 	intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000);
1334 	intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400);
1335 	intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000);
1336 	intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000);
1337 	intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000);
1338 
1339 	intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1340 
1341 	/* 2: Enable RPS */
1342 	intel_uncore_write_fw(uncore, GEN6_RP_CONTROL,
1343 			      GEN6_RP_MEDIA_HW_NORMAL_MODE |
1344 			      GEN6_RP_MEDIA_IS_GFX |
1345 			      GEN6_RP_ENABLE |
1346 			      GEN6_RP_UP_BUSY_AVG |
1347 			      GEN6_RP_DOWN_IDLE_AVG);
1348 
1349 	rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD |
1350 			  GEN6_PM_RP_DOWN_THRESHOLD |
1351 			  GEN6_PM_RP_DOWN_TIMEOUT);
1352 
1353 	/* Setting Fixed Bias */
1354 	vlv_punit_get(i915);
1355 
1356 	val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | CHV_BIAS_CPU_50_SOC_50;
1357 	vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val);
1358 
1359 	val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1360 
1361 	vlv_punit_put(i915);
1362 
1363 	/* RPS code assumes GPLL is used */
1364 	drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0,
1365 		      "GPLL not enabled\n");
1366 
1367 	drm_dbg(&i915->drm, "GPLL enabled? %s\n",
1368 		str_yes_no(val & GPLLENABLE));
1369 	drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val);
1370 
1371 	return rps_reset(rps);
1372 }
1373 
vlv_rps_guar_freq(struct intel_rps * rps)1374 static int vlv_rps_guar_freq(struct intel_rps *rps)
1375 {
1376 	struct drm_i915_private *i915 = rps_to_i915(rps);
1377 	u32 val, rp1;
1378 
1379 	val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE);
1380 
1381 	rp1 = val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK;
1382 	rp1 >>= FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT;
1383 
1384 	return rp1;
1385 }
1386 
vlv_rps_max_freq(struct intel_rps * rps)1387 static int vlv_rps_max_freq(struct intel_rps *rps)
1388 {
1389 	struct drm_i915_private *i915 = rps_to_i915(rps);
1390 	u32 val, rp0;
1391 
1392 	val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE);
1393 
1394 	rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
1395 	/* Clamp to max */
1396 	rp0 = min_t(u32, rp0, 0xea);
1397 
1398 	return rp0;
1399 }
1400 
vlv_rps_rpe_freq(struct intel_rps * rps)1401 static int vlv_rps_rpe_freq(struct intel_rps *rps)
1402 {
1403 	struct drm_i915_private *i915 = rps_to_i915(rps);
1404 	u32 val, rpe;
1405 
1406 	val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
1407 	rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
1408 	val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
1409 	rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
1410 
1411 	return rpe;
1412 }
1413 
vlv_rps_min_freq(struct intel_rps * rps)1414 static int vlv_rps_min_freq(struct intel_rps *rps)
1415 {
1416 	struct drm_i915_private *i915 = rps_to_i915(rps);
1417 	u32 val;
1418 
1419 	val = vlv_punit_read(i915, PUNIT_REG_GPU_LFM) & 0xff;
1420 	/*
1421 	 * According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value
1422 	 * for the minimum frequency in GPLL mode is 0xc1. Contrary to this on
1423 	 * a BYT-M B0 the above register contains 0xbf. Moreover when setting
1424 	 * a frequency Punit will not allow values below 0xc0. Clamp it 0xc0
1425 	 * to make sure it matches what Punit accepts.
1426 	 */
1427 	return max_t(u32, val, 0xc0);
1428 }
1429 
vlv_rps_enable(struct intel_rps * rps)1430 static bool vlv_rps_enable(struct intel_rps *rps)
1431 {
1432 	struct intel_uncore *uncore = rps_to_uncore(rps);
1433 	struct drm_i915_private *i915 = rps_to_i915(rps);
1434 	u32 val;
1435 
1436 	intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000);
1437 	intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400);
1438 	intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000);
1439 	intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000);
1440 	intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000);
1441 
1442 	intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1443 
1444 	intel_uncore_write_fw(uncore, GEN6_RP_CONTROL,
1445 			      GEN6_RP_MEDIA_TURBO |
1446 			      GEN6_RP_MEDIA_HW_NORMAL_MODE |
1447 			      GEN6_RP_MEDIA_IS_GFX |
1448 			      GEN6_RP_ENABLE |
1449 			      GEN6_RP_UP_BUSY_AVG |
1450 			      GEN6_RP_DOWN_IDLE_CONT);
1451 
1452 	/* WaGsvRC0ResidencyMethod:vlv */
1453 	rps->pm_events = GEN6_PM_RP_UP_EI_EXPIRED;
1454 
1455 	vlv_punit_get(i915);
1456 
1457 	/* Setting Fixed Bias */
1458 	val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | VLV_BIAS_CPU_125_SOC_875;
1459 	vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val);
1460 
1461 	val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1462 
1463 	vlv_punit_put(i915);
1464 
1465 	/* RPS code assumes GPLL is used */
1466 	drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0,
1467 		      "GPLL not enabled\n");
1468 
1469 	drm_dbg(&i915->drm, "GPLL enabled? %s\n",
1470 		str_yes_no(val & GPLLENABLE));
1471 	drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val);
1472 
1473 	return rps_reset(rps);
1474 }
1475 
__ips_gfx_val(struct intel_ips * ips)1476 static unsigned long __ips_gfx_val(struct intel_ips *ips)
1477 {
1478 	struct intel_rps *rps = container_of(ips, typeof(*rps), ips);
1479 	struct intel_uncore *uncore = rps_to_uncore(rps);
1480 	unsigned int t, state1, state2;
1481 	u32 pxvid, ext_v;
1482 	u64 corr, corr2;
1483 
1484 	lockdep_assert_held(&mchdev_lock);
1485 
1486 	pxvid = intel_uncore_read(uncore, PXVFREQ(rps->cur_freq));
1487 	pxvid = (pxvid >> 24) & 0x7f;
1488 	ext_v = pvid_to_extvid(rps_to_i915(rps), pxvid);
1489 
1490 	state1 = ext_v;
1491 
1492 	/* Revel in the empirically derived constants */
1493 
1494 	/* Correction factor in 1/100000 units */
1495 	t = ips_mch_val(uncore);
1496 	if (t > 80)
1497 		corr = t * 2349 + 135940;
1498 	else if (t >= 50)
1499 		corr = t * 964 + 29317;
1500 	else /* < 50 */
1501 		corr = t * 301 + 1004;
1502 
1503 	corr = div_u64(corr * 150142 * state1, 10000) - 78642;
1504 	corr2 = div_u64(corr, 100000) * ips->corr;
1505 
1506 	state2 = div_u64(corr2 * state1, 10000);
1507 	state2 /= 100; /* convert to mW */
1508 
1509 	__gen5_ips_update(ips);
1510 
1511 	return ips->gfx_power + state2;
1512 }
1513 
has_busy_stats(struct intel_rps * rps)1514 static bool has_busy_stats(struct intel_rps *rps)
1515 {
1516 	struct intel_engine_cs *engine;
1517 	enum intel_engine_id id;
1518 
1519 	for_each_engine(engine, rps_to_gt(rps), id) {
1520 		if (!intel_engine_supports_stats(engine))
1521 			return false;
1522 	}
1523 
1524 	return true;
1525 }
1526 
intel_rps_enable(struct intel_rps * rps)1527 void intel_rps_enable(struct intel_rps *rps)
1528 {
1529 	struct drm_i915_private *i915 = rps_to_i915(rps);
1530 	struct intel_uncore *uncore = rps_to_uncore(rps);
1531 	bool enabled = false;
1532 
1533 	if (!HAS_RPS(i915))
1534 		return;
1535 
1536 	if (rps_uses_slpc(rps))
1537 		return;
1538 
1539 	intel_gt_check_clock_frequency(rps_to_gt(rps));
1540 
1541 	intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
1542 	if (rps->max_freq <= rps->min_freq)
1543 		/* leave disabled, no room for dynamic reclocking */;
1544 	else if (IS_CHERRYVIEW(i915))
1545 		enabled = chv_rps_enable(rps);
1546 	else if (IS_VALLEYVIEW(i915))
1547 		enabled = vlv_rps_enable(rps);
1548 	else if (GRAPHICS_VER(i915) >= 9)
1549 		enabled = gen9_rps_enable(rps);
1550 	else if (GRAPHICS_VER(i915) >= 8)
1551 		enabled = gen8_rps_enable(rps);
1552 	else if (GRAPHICS_VER(i915) >= 6)
1553 		enabled = gen6_rps_enable(rps);
1554 	else if (IS_IRONLAKE_M(i915))
1555 		enabled = gen5_rps_enable(rps);
1556 	else
1557 		MISSING_CASE(GRAPHICS_VER(i915));
1558 	intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
1559 	if (!enabled)
1560 		return;
1561 
1562 	GT_TRACE(rps_to_gt(rps),
1563 		 "min:%x, max:%x, freq:[%d, %d], thresholds:[%u, %u]\n",
1564 		 rps->min_freq, rps->max_freq,
1565 		 intel_gpu_freq(rps, rps->min_freq),
1566 		 intel_gpu_freq(rps, rps->max_freq),
1567 		 rps->power.up_threshold,
1568 		 rps->power.down_threshold);
1569 
1570 	GEM_BUG_ON(rps->max_freq < rps->min_freq);
1571 	GEM_BUG_ON(rps->idle_freq > rps->max_freq);
1572 
1573 	GEM_BUG_ON(rps->efficient_freq < rps->min_freq);
1574 	GEM_BUG_ON(rps->efficient_freq > rps->max_freq);
1575 
1576 	if (has_busy_stats(rps))
1577 		intel_rps_set_timer(rps);
1578 	else if (GRAPHICS_VER(i915) >= 6 && GRAPHICS_VER(i915) <= 11)
1579 		intel_rps_set_interrupts(rps);
1580 	else
1581 		/* Ironlake currently uses intel_ips.ko */ {}
1582 
1583 	intel_rps_set_enabled(rps);
1584 }
1585 
gen6_rps_disable(struct intel_rps * rps)1586 static void gen6_rps_disable(struct intel_rps *rps)
1587 {
1588 	set(rps_to_uncore(rps), GEN6_RP_CONTROL, 0);
1589 }
1590 
intel_rps_disable(struct intel_rps * rps)1591 void intel_rps_disable(struct intel_rps *rps)
1592 {
1593 	struct drm_i915_private *i915 = rps_to_i915(rps);
1594 
1595 	if (!intel_rps_is_enabled(rps))
1596 		return;
1597 
1598 	intel_rps_clear_enabled(rps);
1599 	intel_rps_clear_interrupts(rps);
1600 	intel_rps_clear_timer(rps);
1601 
1602 	if (GRAPHICS_VER(i915) >= 6)
1603 		gen6_rps_disable(rps);
1604 	else if (IS_IRONLAKE_M(i915))
1605 		gen5_rps_disable(rps);
1606 }
1607 
byt_gpu_freq(struct intel_rps * rps,int val)1608 static int byt_gpu_freq(struct intel_rps *rps, int val)
1609 {
1610 	/*
1611 	 * N = val - 0xb7
1612 	 * Slow = Fast = GPLL ref * N
1613 	 */
1614 	return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000);
1615 }
1616 
byt_freq_opcode(struct intel_rps * rps,int val)1617 static int byt_freq_opcode(struct intel_rps *rps, int val)
1618 {
1619 	return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7;
1620 }
1621 
chv_gpu_freq(struct intel_rps * rps,int val)1622 static int chv_gpu_freq(struct intel_rps *rps, int val)
1623 {
1624 	/*
1625 	 * N = val / 2
1626 	 * CU (slow) = CU2x (fast) / 2 = GPLL ref * N / 2
1627 	 */
1628 	return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000);
1629 }
1630 
chv_freq_opcode(struct intel_rps * rps,int val)1631 static int chv_freq_opcode(struct intel_rps *rps, int val)
1632 {
1633 	/* CHV needs even values */
1634 	return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2;
1635 }
1636 
intel_gpu_freq(struct intel_rps * rps,int val)1637 int intel_gpu_freq(struct intel_rps *rps, int val)
1638 {
1639 	struct drm_i915_private *i915 = rps_to_i915(rps);
1640 
1641 	if (GRAPHICS_VER(i915) >= 9)
1642 		return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER,
1643 					 GEN9_FREQ_SCALER);
1644 	else if (IS_CHERRYVIEW(i915))
1645 		return chv_gpu_freq(rps, val);
1646 	else if (IS_VALLEYVIEW(i915))
1647 		return byt_gpu_freq(rps, val);
1648 	else if (GRAPHICS_VER(i915) >= 6)
1649 		return val * GT_FREQUENCY_MULTIPLIER;
1650 	else
1651 		return val;
1652 }
1653 
intel_freq_opcode(struct intel_rps * rps,int val)1654 int intel_freq_opcode(struct intel_rps *rps, int val)
1655 {
1656 	struct drm_i915_private *i915 = rps_to_i915(rps);
1657 
1658 	if (GRAPHICS_VER(i915) >= 9)
1659 		return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER,
1660 					 GT_FREQUENCY_MULTIPLIER);
1661 	else if (IS_CHERRYVIEW(i915))
1662 		return chv_freq_opcode(rps, val);
1663 	else if (IS_VALLEYVIEW(i915))
1664 		return byt_freq_opcode(rps, val);
1665 	else if (GRAPHICS_VER(i915) >= 6)
1666 		return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER);
1667 	else
1668 		return val;
1669 }
1670 
vlv_init_gpll_ref_freq(struct intel_rps * rps)1671 static void vlv_init_gpll_ref_freq(struct intel_rps *rps)
1672 {
1673 	struct drm_i915_private *i915 = rps_to_i915(rps);
1674 
1675 	rps->gpll_ref_freq =
1676 		vlv_get_cck_clock(i915, "GPLL ref",
1677 				  CCK_GPLL_CLOCK_CONTROL,
1678 				  i915->czclk_freq);
1679 
1680 	drm_dbg(&i915->drm, "GPLL reference freq: %d kHz\n",
1681 		rps->gpll_ref_freq);
1682 }
1683 
vlv_rps_init(struct intel_rps * rps)1684 static void vlv_rps_init(struct intel_rps *rps)
1685 {
1686 	struct drm_i915_private *i915 = rps_to_i915(rps);
1687 
1688 	vlv_iosf_sb_get(i915,
1689 			BIT(VLV_IOSF_SB_PUNIT) |
1690 			BIT(VLV_IOSF_SB_NC) |
1691 			BIT(VLV_IOSF_SB_CCK));
1692 
1693 	vlv_init_gpll_ref_freq(rps);
1694 
1695 	rps->max_freq = vlv_rps_max_freq(rps);
1696 	rps->rp0_freq = rps->max_freq;
1697 	drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n",
1698 		intel_gpu_freq(rps, rps->max_freq), rps->max_freq);
1699 
1700 	rps->efficient_freq = vlv_rps_rpe_freq(rps);
1701 	drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n",
1702 		intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq);
1703 
1704 	rps->rp1_freq = vlv_rps_guar_freq(rps);
1705 	drm_dbg(&i915->drm, "RP1(Guar Freq) GPU freq: %d MHz (%u)\n",
1706 		intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq);
1707 
1708 	rps->min_freq = vlv_rps_min_freq(rps);
1709 	drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n",
1710 		intel_gpu_freq(rps, rps->min_freq), rps->min_freq);
1711 
1712 	vlv_iosf_sb_put(i915,
1713 			BIT(VLV_IOSF_SB_PUNIT) |
1714 			BIT(VLV_IOSF_SB_NC) |
1715 			BIT(VLV_IOSF_SB_CCK));
1716 }
1717 
chv_rps_init(struct intel_rps * rps)1718 static void chv_rps_init(struct intel_rps *rps)
1719 {
1720 	struct drm_i915_private *i915 = rps_to_i915(rps);
1721 
1722 	vlv_iosf_sb_get(i915,
1723 			BIT(VLV_IOSF_SB_PUNIT) |
1724 			BIT(VLV_IOSF_SB_NC) |
1725 			BIT(VLV_IOSF_SB_CCK));
1726 
1727 	vlv_init_gpll_ref_freq(rps);
1728 
1729 	rps->max_freq = chv_rps_max_freq(rps);
1730 	rps->rp0_freq = rps->max_freq;
1731 	drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n",
1732 		intel_gpu_freq(rps, rps->max_freq), rps->max_freq);
1733 
1734 	rps->efficient_freq = chv_rps_rpe_freq(rps);
1735 	drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n",
1736 		intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq);
1737 
1738 	rps->rp1_freq = chv_rps_guar_freq(rps);
1739 	drm_dbg(&i915->drm, "RP1(Guar) GPU freq: %d MHz (%u)\n",
1740 		intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq);
1741 
1742 	rps->min_freq = chv_rps_min_freq(rps);
1743 	drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n",
1744 		intel_gpu_freq(rps, rps->min_freq), rps->min_freq);
1745 
1746 	vlv_iosf_sb_put(i915,
1747 			BIT(VLV_IOSF_SB_PUNIT) |
1748 			BIT(VLV_IOSF_SB_NC) |
1749 			BIT(VLV_IOSF_SB_CCK));
1750 
1751 	drm_WARN_ONCE(&i915->drm, (rps->max_freq | rps->efficient_freq |
1752 				   rps->rp1_freq | rps->min_freq) & 1,
1753 		      "Odd GPU freq values\n");
1754 }
1755 
vlv_c0_read(struct intel_uncore * uncore,struct intel_rps_ei * ei)1756 static void vlv_c0_read(struct intel_uncore *uncore, struct intel_rps_ei *ei)
1757 {
1758 	ei->ktime = ktime_get_raw();
1759 	ei->render_c0 = intel_uncore_read(uncore, VLV_RENDER_C0_COUNT);
1760 	ei->media_c0 = intel_uncore_read(uncore, VLV_MEDIA_C0_COUNT);
1761 }
1762 
vlv_wa_c0_ei(struct intel_rps * rps,u32 pm_iir)1763 static u32 vlv_wa_c0_ei(struct intel_rps *rps, u32 pm_iir)
1764 {
1765 	struct intel_uncore *uncore = rps_to_uncore(rps);
1766 	const struct intel_rps_ei *prev = &rps->ei;
1767 	struct intel_rps_ei now;
1768 	u32 events = 0;
1769 
1770 	if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0)
1771 		return 0;
1772 
1773 	vlv_c0_read(uncore, &now);
1774 
1775 	if (prev->ktime) {
1776 		u64 time, c0;
1777 		u32 render, media;
1778 
1779 		time = ktime_us_delta(now.ktime, prev->ktime);
1780 
1781 		time *= rps_to_i915(rps)->czclk_freq;
1782 
1783 		/* Workload can be split between render + media,
1784 		 * e.g. SwapBuffers being blitted in X after being rendered in
1785 		 * mesa. To account for this we need to combine both engines
1786 		 * into our activity counter.
1787 		 */
1788 		render = now.render_c0 - prev->render_c0;
1789 		media = now.media_c0 - prev->media_c0;
1790 		c0 = max(render, media);
1791 		c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */
1792 
1793 		if (c0 > time * rps->power.up_threshold)
1794 			events = GEN6_PM_RP_UP_THRESHOLD;
1795 		else if (c0 < time * rps->power.down_threshold)
1796 			events = GEN6_PM_RP_DOWN_THRESHOLD;
1797 	}
1798 
1799 	rps->ei = now;
1800 	return events;
1801 }
1802 
rps_work(struct work_struct * work)1803 static void rps_work(struct work_struct *work)
1804 {
1805 	struct intel_rps *rps = container_of(work, typeof(*rps), work);
1806 	struct intel_gt *gt = rps_to_gt(rps);
1807 	struct drm_i915_private *i915 = rps_to_i915(rps);
1808 	bool client_boost = false;
1809 	int new_freq, adj, min, max;
1810 	u32 pm_iir = 0;
1811 
1812 	spin_lock_irq(gt->irq_lock);
1813 	pm_iir = fetch_and_zero(&rps->pm_iir) & rps->pm_events;
1814 	client_boost = atomic_read(&rps->num_waiters);
1815 	spin_unlock_irq(gt->irq_lock);
1816 
1817 	/* Make sure we didn't queue anything we're not going to process. */
1818 	if (!pm_iir && !client_boost)
1819 		goto out;
1820 
1821 	mutex_lock(&rps->lock);
1822 	if (!intel_rps_is_active(rps)) {
1823 		mutex_unlock(&rps->lock);
1824 		return;
1825 	}
1826 
1827 	pm_iir |= vlv_wa_c0_ei(rps, pm_iir);
1828 
1829 	adj = rps->last_adj;
1830 	new_freq = rps->cur_freq;
1831 	min = rps->min_freq_softlimit;
1832 	max = rps->max_freq_softlimit;
1833 	if (client_boost)
1834 		max = rps->max_freq;
1835 
1836 	GT_TRACE(gt,
1837 		 "pm_iir:%x, client_boost:%s, last:%d, cur:%x, min:%x, max:%x\n",
1838 		 pm_iir, str_yes_no(client_boost),
1839 		 adj, new_freq, min, max);
1840 
1841 	if (client_boost && new_freq < rps->boost_freq) {
1842 		new_freq = rps->boost_freq;
1843 		adj = 0;
1844 	} else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
1845 		if (adj > 0)
1846 			adj *= 2;
1847 		else /* CHV needs even encode values */
1848 			adj = IS_CHERRYVIEW(gt->i915) ? 2 : 1;
1849 
1850 		if (new_freq >= rps->max_freq_softlimit)
1851 			adj = 0;
1852 	} else if (client_boost) {
1853 		adj = 0;
1854 	} else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
1855 		if (rps->cur_freq > rps->efficient_freq)
1856 			new_freq = rps->efficient_freq;
1857 		else if (rps->cur_freq > rps->min_freq_softlimit)
1858 			new_freq = rps->min_freq_softlimit;
1859 		adj = 0;
1860 	} else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
1861 		if (adj < 0)
1862 			adj *= 2;
1863 		else /* CHV needs even encode values */
1864 			adj = IS_CHERRYVIEW(gt->i915) ? -2 : -1;
1865 
1866 		if (new_freq <= rps->min_freq_softlimit)
1867 			adj = 0;
1868 	} else { /* unknown event */
1869 		adj = 0;
1870 	}
1871 
1872 	/*
1873 	 * sysfs frequency limits may have snuck in while
1874 	 * servicing the interrupt
1875 	 */
1876 	new_freq += adj;
1877 	new_freq = clamp_t(int, new_freq, min, max);
1878 
1879 	if (intel_rps_set(rps, new_freq)) {
1880 		drm_dbg(&i915->drm, "Failed to set new GPU frequency\n");
1881 		adj = 0;
1882 	}
1883 	rps->last_adj = adj;
1884 
1885 	mutex_unlock(&rps->lock);
1886 
1887 out:
1888 	spin_lock_irq(gt->irq_lock);
1889 	gen6_gt_pm_unmask_irq(gt, rps->pm_events);
1890 	spin_unlock_irq(gt->irq_lock);
1891 }
1892 
gen11_rps_irq_handler(struct intel_rps * rps,u32 pm_iir)1893 void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
1894 {
1895 	struct intel_gt *gt = rps_to_gt(rps);
1896 	const u32 events = rps->pm_events & pm_iir;
1897 
1898 	lockdep_assert_held(gt->irq_lock);
1899 
1900 	if (unlikely(!events))
1901 		return;
1902 
1903 	GT_TRACE(gt, "irq events:%x\n", events);
1904 
1905 	gen6_gt_pm_mask_irq(gt, events);
1906 
1907 	rps->pm_iir |= events;
1908 	queue_work(gt->i915->unordered_wq, &rps->work);
1909 }
1910 
gen6_rps_irq_handler(struct intel_rps * rps,u32 pm_iir)1911 void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
1912 {
1913 	struct intel_gt *gt = rps_to_gt(rps);
1914 	u32 events;
1915 
1916 	events = pm_iir & rps->pm_events;
1917 	if (events) {
1918 		spin_lock(gt->irq_lock);
1919 
1920 		GT_TRACE(gt, "irq events:%x\n", events);
1921 
1922 		gen6_gt_pm_mask_irq(gt, events);
1923 		rps->pm_iir |= events;
1924 
1925 		queue_work(gt->i915->unordered_wq, &rps->work);
1926 		spin_unlock(gt->irq_lock);
1927 	}
1928 
1929 	if (GRAPHICS_VER(gt->i915) >= 8)
1930 		return;
1931 
1932 	if (pm_iir & PM_VEBOX_USER_INTERRUPT)
1933 		intel_engine_cs_irq(gt->engine[VECS0], pm_iir >> 10);
1934 
1935 	if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
1936 		drm_dbg(&rps_to_i915(rps)->drm,
1937 			"Command parser error, pm_iir 0x%08x\n", pm_iir);
1938 }
1939 
gen5_rps_irq_handler(struct intel_rps * rps)1940 void gen5_rps_irq_handler(struct intel_rps *rps)
1941 {
1942 	struct intel_uncore *uncore = rps_to_uncore(rps);
1943 	u32 busy_up, busy_down, max_avg, min_avg;
1944 	u8 new_freq;
1945 
1946 	spin_lock(&mchdev_lock);
1947 
1948 	intel_uncore_write16(uncore,
1949 			     MEMINTRSTS,
1950 			     intel_uncore_read(uncore, MEMINTRSTS));
1951 
1952 	intel_uncore_write16(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
1953 	busy_up = intel_uncore_read(uncore, RCPREVBSYTUPAVG);
1954 	busy_down = intel_uncore_read(uncore, RCPREVBSYTDNAVG);
1955 	max_avg = intel_uncore_read(uncore, RCBMAXAVG);
1956 	min_avg = intel_uncore_read(uncore, RCBMINAVG);
1957 
1958 	/* Handle RCS change request from hw */
1959 	new_freq = rps->cur_freq;
1960 	if (busy_up > max_avg)
1961 		new_freq++;
1962 	else if (busy_down < min_avg)
1963 		new_freq--;
1964 	new_freq = clamp(new_freq,
1965 			 rps->min_freq_softlimit,
1966 			 rps->max_freq_softlimit);
1967 
1968 	if (new_freq != rps->cur_freq && !__gen5_rps_set(rps, new_freq))
1969 		rps->cur_freq = new_freq;
1970 
1971 	spin_unlock(&mchdev_lock);
1972 }
1973 
intel_rps_init_early(struct intel_rps * rps)1974 void intel_rps_init_early(struct intel_rps *rps)
1975 {
1976 	mutex_init(&rps->lock);
1977 	mutex_init(&rps->power.mutex);
1978 
1979 	INIT_WORK(&rps->work, rps_work);
1980 	timer_setup(&rps->timer, rps_timer, 0);
1981 
1982 	atomic_set(&rps->num_waiters, 0);
1983 }
1984 
intel_rps_init(struct intel_rps * rps)1985 void intel_rps_init(struct intel_rps *rps)
1986 {
1987 	struct drm_i915_private *i915 = rps_to_i915(rps);
1988 
1989 	if (rps_uses_slpc(rps))
1990 		return;
1991 
1992 	if (IS_CHERRYVIEW(i915))
1993 		chv_rps_init(rps);
1994 	else if (IS_VALLEYVIEW(i915))
1995 		vlv_rps_init(rps);
1996 	else if (GRAPHICS_VER(i915) >= 6)
1997 		gen6_rps_init(rps);
1998 	else if (IS_IRONLAKE_M(i915))
1999 		gen5_rps_init(rps);
2000 
2001 	/* Derive initial user preferences/limits from the hardware limits */
2002 	rps->max_freq_softlimit = rps->max_freq;
2003 	rps_to_gt(rps)->defaults.max_freq = rps->max_freq_softlimit;
2004 	rps->min_freq_softlimit = rps->min_freq;
2005 	rps_to_gt(rps)->defaults.min_freq = rps->min_freq_softlimit;
2006 
2007 	/* After setting max-softlimit, find the overclock max freq */
2008 	if (GRAPHICS_VER(i915) == 6 || IS_IVYBRIDGE(i915) || IS_HASWELL(i915)) {
2009 		u32 params = 0;
2010 
2011 		snb_pcode_read(rps_to_gt(rps)->uncore, GEN6_READ_OC_PARAMS, &params, NULL);
2012 		if (params & BIT(31)) { /* OC supported */
2013 			drm_dbg(&i915->drm,
2014 				"Overclocking supported, max: %dMHz, overclock: %dMHz\n",
2015 				(rps->max_freq & 0xff) * 50,
2016 				(params & 0xff) * 50);
2017 			rps->max_freq = params & 0xff;
2018 		}
2019 	}
2020 
2021 	/* Set default thresholds in % */
2022 	rps->power.up_threshold = 95;
2023 	rps_to_gt(rps)->defaults.rps_up_threshold = rps->power.up_threshold;
2024 	rps->power.down_threshold = 85;
2025 	rps_to_gt(rps)->defaults.rps_down_threshold = rps->power.down_threshold;
2026 
2027 	/* Finally allow us to boost to max by default */
2028 	rps->boost_freq = rps->max_freq;
2029 	rps->idle_freq = rps->min_freq;
2030 
2031 	/* Start in the middle, from here we will autotune based on workload */
2032 	rps->cur_freq = rps->efficient_freq;
2033 
2034 	rps->pm_intrmsk_mbz = 0;
2035 
2036 	/*
2037 	 * SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer
2038 	 * if GEN6_PM_UP_EI_EXPIRED is masked.
2039 	 *
2040 	 * TODO: verify if this can be reproduced on VLV,CHV.
2041 	 */
2042 	if (GRAPHICS_VER(i915) <= 7)
2043 		rps->pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED;
2044 
2045 	if (GRAPHICS_VER(i915) >= 8 && GRAPHICS_VER(i915) < 11)
2046 		rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC;
2047 
2048 	/* GuC needs ARAT expired interrupt unmasked */
2049 	if (intel_uc_uses_guc_submission(&rps_to_gt(rps)->uc))
2050 		rps->pm_intrmsk_mbz |= ARAT_EXPIRED_INTRMSK;
2051 }
2052 
intel_rps_sanitize(struct intel_rps * rps)2053 void intel_rps_sanitize(struct intel_rps *rps)
2054 {
2055 	if (rps_uses_slpc(rps))
2056 		return;
2057 
2058 	if (GRAPHICS_VER(rps_to_i915(rps)) >= 6)
2059 		rps_disable_interrupts(rps);
2060 }
2061 
intel_rps_read_rpstat(struct intel_rps * rps)2062 u32 intel_rps_read_rpstat(struct intel_rps *rps)
2063 {
2064 	struct drm_i915_private *i915 = rps_to_i915(rps);
2065 	i915_reg_t rpstat;
2066 
2067 	rpstat = (GRAPHICS_VER(i915) >= 12) ? GEN12_RPSTAT1 : GEN6_RPSTAT1;
2068 
2069 	return intel_uncore_read(rps_to_gt(rps)->uncore, rpstat);
2070 }
2071 
intel_rps_get_cagf(struct intel_rps * rps,u32 rpstat)2072 static u32 intel_rps_get_cagf(struct intel_rps *rps, u32 rpstat)
2073 {
2074 	struct drm_i915_private *i915 = rps_to_i915(rps);
2075 	u32 cagf;
2076 
2077 	if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70))
2078 		cagf = REG_FIELD_GET(MTL_CAGF_MASK, rpstat);
2079 	else if (GRAPHICS_VER(i915) >= 12)
2080 		cagf = REG_FIELD_GET(GEN12_CAGF_MASK, rpstat);
2081 	else if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
2082 		cagf = REG_FIELD_GET(RPE_MASK, rpstat);
2083 	else if (GRAPHICS_VER(i915) >= 9)
2084 		cagf = REG_FIELD_GET(GEN9_CAGF_MASK, rpstat);
2085 	else if (IS_HASWELL(i915) || IS_BROADWELL(i915))
2086 		cagf = REG_FIELD_GET(HSW_CAGF_MASK, rpstat);
2087 	else if (GRAPHICS_VER(i915) >= 6)
2088 		cagf = REG_FIELD_GET(GEN6_CAGF_MASK, rpstat);
2089 	else
2090 		cagf = gen5_invert_freq(rps, REG_FIELD_GET(MEMSTAT_PSTATE_MASK, rpstat));
2091 
2092 	return cagf;
2093 }
2094 
__read_cagf(struct intel_rps * rps,bool take_fw)2095 static u32 __read_cagf(struct intel_rps *rps, bool take_fw)
2096 {
2097 	struct drm_i915_private *i915 = rps_to_i915(rps);
2098 	struct intel_uncore *uncore = rps_to_uncore(rps);
2099 	i915_reg_t r = INVALID_MMIO_REG;
2100 	u32 freq;
2101 
2102 	/*
2103 	 * For Gen12+ reading freq from HW does not need a forcewake and
2104 	 * registers will return 0 freq when GT is in RC6
2105 	 */
2106 	if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) {
2107 		r = MTL_MIRROR_TARGET_WP1;
2108 	} else if (GRAPHICS_VER(i915) >= 12) {
2109 		r = GEN12_RPSTAT1;
2110 	} else if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) {
2111 		vlv_punit_get(i915);
2112 		freq = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
2113 		vlv_punit_put(i915);
2114 	} else if (GRAPHICS_VER(i915) >= 6) {
2115 		r = GEN6_RPSTAT1;
2116 	} else {
2117 		r = MEMSTAT_ILK;
2118 	}
2119 
2120 	if (i915_mmio_reg_valid(r))
2121 		freq = take_fw ? intel_uncore_read(uncore, r) : intel_uncore_read_fw(uncore, r);
2122 
2123 	return intel_rps_get_cagf(rps, freq);
2124 }
2125 
read_cagf(struct intel_rps * rps)2126 static u32 read_cagf(struct intel_rps *rps)
2127 {
2128 	return __read_cagf(rps, true);
2129 }
2130 
intel_rps_read_actual_frequency(struct intel_rps * rps)2131 u32 intel_rps_read_actual_frequency(struct intel_rps *rps)
2132 {
2133 	struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm;
2134 	intel_wakeref_t wakeref;
2135 	u32 freq = 0;
2136 
2137 	with_intel_runtime_pm_if_in_use(rpm, wakeref)
2138 		freq = intel_gpu_freq(rps, read_cagf(rps));
2139 
2140 	return freq;
2141 }
2142 
intel_rps_read_actual_frequency_fw(struct intel_rps * rps)2143 u32 intel_rps_read_actual_frequency_fw(struct intel_rps *rps)
2144 {
2145 	return intel_gpu_freq(rps, __read_cagf(rps, false));
2146 }
2147 
intel_rps_read_punit_req(struct intel_rps * rps)2148 static u32 intel_rps_read_punit_req(struct intel_rps *rps)
2149 {
2150 	struct intel_uncore *uncore = rps_to_uncore(rps);
2151 	struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm;
2152 	intel_wakeref_t wakeref;
2153 	u32 freq = 0;
2154 
2155 	with_intel_runtime_pm_if_in_use(rpm, wakeref)
2156 		freq = intel_uncore_read(uncore, GEN6_RPNSWREQ);
2157 
2158 	return freq;
2159 }
2160 
intel_rps_get_req(u32 pureq)2161 static u32 intel_rps_get_req(u32 pureq)
2162 {
2163 	u32 req = pureq >> GEN9_SW_REQ_UNSLICE_RATIO_SHIFT;
2164 
2165 	return req;
2166 }
2167 
intel_rps_read_punit_req_frequency(struct intel_rps * rps)2168 u32 intel_rps_read_punit_req_frequency(struct intel_rps *rps)
2169 {
2170 	u32 freq = intel_rps_get_req(intel_rps_read_punit_req(rps));
2171 
2172 	return intel_gpu_freq(rps, freq);
2173 }
2174 
intel_rps_get_requested_frequency(struct intel_rps * rps)2175 u32 intel_rps_get_requested_frequency(struct intel_rps *rps)
2176 {
2177 	if (rps_uses_slpc(rps))
2178 		return intel_rps_read_punit_req_frequency(rps);
2179 	else
2180 		return intel_gpu_freq(rps, rps->cur_freq);
2181 }
2182 
intel_rps_get_max_frequency(struct intel_rps * rps)2183 u32 intel_rps_get_max_frequency(struct intel_rps *rps)
2184 {
2185 	struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2186 
2187 	if (rps_uses_slpc(rps))
2188 		return slpc->max_freq_softlimit;
2189 	else
2190 		return intel_gpu_freq(rps, rps->max_freq_softlimit);
2191 }
2192 
2193 /**
2194  * intel_rps_get_max_raw_freq - returns the max frequency in some raw format.
2195  * @rps: the intel_rps structure
2196  *
2197  * Returns the max frequency in a raw format. In newer platforms raw is in
2198  * units of 50 MHz.
2199  */
intel_rps_get_max_raw_freq(struct intel_rps * rps)2200 u32 intel_rps_get_max_raw_freq(struct intel_rps *rps)
2201 {
2202 	struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2203 	u32 freq;
2204 
2205 	if (rps_uses_slpc(rps)) {
2206 		return DIV_ROUND_CLOSEST(slpc->rp0_freq,
2207 					 GT_FREQUENCY_MULTIPLIER);
2208 	} else {
2209 		freq = rps->max_freq;
2210 		if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) {
2211 			/* Convert GT frequency to 50 MHz units */
2212 			freq /= GEN9_FREQ_SCALER;
2213 		}
2214 		return freq;
2215 	}
2216 }
2217 
intel_rps_get_rp0_frequency(struct intel_rps * rps)2218 u32 intel_rps_get_rp0_frequency(struct intel_rps *rps)
2219 {
2220 	struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2221 
2222 	if (rps_uses_slpc(rps))
2223 		return slpc->rp0_freq;
2224 	else
2225 		return intel_gpu_freq(rps, rps->rp0_freq);
2226 }
2227 
intel_rps_get_rp1_frequency(struct intel_rps * rps)2228 u32 intel_rps_get_rp1_frequency(struct intel_rps *rps)
2229 {
2230 	struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2231 
2232 	if (rps_uses_slpc(rps))
2233 		return slpc->rp1_freq;
2234 	else
2235 		return intel_gpu_freq(rps, rps->rp1_freq);
2236 }
2237 
intel_rps_get_rpn_frequency(struct intel_rps * rps)2238 u32 intel_rps_get_rpn_frequency(struct intel_rps *rps)
2239 {
2240 	struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2241 
2242 	if (rps_uses_slpc(rps))
2243 		return slpc->min_freq;
2244 	else
2245 		return intel_gpu_freq(rps, rps->min_freq);
2246 }
2247 
rps_frequency_dump(struct intel_rps * rps,struct drm_printer * p)2248 static void rps_frequency_dump(struct intel_rps *rps, struct drm_printer *p)
2249 {
2250 	struct intel_gt *gt = rps_to_gt(rps);
2251 	struct drm_i915_private *i915 = gt->i915;
2252 	struct intel_uncore *uncore = gt->uncore;
2253 	struct intel_rps_freq_caps caps;
2254 	u32 rp_state_limits;
2255 	u32 gt_perf_status;
2256 	u32 rpmodectl, rpinclimit, rpdeclimit;
2257 	u32 rpstat, cagf, reqf;
2258 	u32 rpcurupei, rpcurup, rpprevup;
2259 	u32 rpcurdownei, rpcurdown, rpprevdown;
2260 	u32 rpupei, rpupt, rpdownei, rpdownt;
2261 	u32 pm_ier, pm_imr, pm_isr, pm_iir, pm_mask;
2262 
2263 	rp_state_limits = intel_uncore_read(uncore, GEN6_RP_STATE_LIMITS);
2264 	gen6_rps_get_freq_caps(rps, &caps);
2265 	if (IS_GEN9_LP(i915))
2266 		gt_perf_status = intel_uncore_read(uncore, BXT_GT_PERF_STATUS);
2267 	else
2268 		gt_perf_status = intel_uncore_read(uncore, GEN6_GT_PERF_STATUS);
2269 
2270 	/* RPSTAT1 is in the GT power well */
2271 	intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
2272 
2273 	reqf = intel_uncore_read(uncore, GEN6_RPNSWREQ);
2274 	if (GRAPHICS_VER(i915) >= 9) {
2275 		reqf >>= 23;
2276 	} else {
2277 		reqf &= ~GEN6_TURBO_DISABLE;
2278 		if (IS_HASWELL(i915) || IS_BROADWELL(i915))
2279 			reqf >>= 24;
2280 		else
2281 			reqf >>= 25;
2282 	}
2283 	reqf = intel_gpu_freq(rps, reqf);
2284 
2285 	rpmodectl = intel_uncore_read(uncore, GEN6_RP_CONTROL);
2286 	rpinclimit = intel_uncore_read(uncore, GEN6_RP_UP_THRESHOLD);
2287 	rpdeclimit = intel_uncore_read(uncore, GEN6_RP_DOWN_THRESHOLD);
2288 
2289 	rpstat = intel_rps_read_rpstat(rps);
2290 	rpcurupei = intel_uncore_read(uncore, GEN6_RP_CUR_UP_EI) & GEN6_CURICONT_MASK;
2291 	rpcurup = intel_uncore_read(uncore, GEN6_RP_CUR_UP) & GEN6_CURBSYTAVG_MASK;
2292 	rpprevup = intel_uncore_read(uncore, GEN6_RP_PREV_UP) & GEN6_CURBSYTAVG_MASK;
2293 	rpcurdownei = intel_uncore_read(uncore, GEN6_RP_CUR_DOWN_EI) & GEN6_CURIAVG_MASK;
2294 	rpcurdown = intel_uncore_read(uncore, GEN6_RP_CUR_DOWN) & GEN6_CURBSYTAVG_MASK;
2295 	rpprevdown = intel_uncore_read(uncore, GEN6_RP_PREV_DOWN) & GEN6_CURBSYTAVG_MASK;
2296 
2297 	rpupei = intel_uncore_read(uncore, GEN6_RP_UP_EI);
2298 	rpupt = intel_uncore_read(uncore, GEN6_RP_UP_THRESHOLD);
2299 
2300 	rpdownei = intel_uncore_read(uncore, GEN6_RP_DOWN_EI);
2301 	rpdownt = intel_uncore_read(uncore, GEN6_RP_DOWN_THRESHOLD);
2302 
2303 	cagf = intel_rps_read_actual_frequency(rps);
2304 
2305 	intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
2306 
2307 	if (GRAPHICS_VER(i915) >= 11) {
2308 		pm_ier = intel_uncore_read(uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE);
2309 		pm_imr = intel_uncore_read(uncore, GEN11_GPM_WGBOXPERF_INTR_MASK);
2310 		/*
2311 		 * The equivalent to the PM ISR & IIR cannot be read
2312 		 * without affecting the current state of the system
2313 		 */
2314 		pm_isr = 0;
2315 		pm_iir = 0;
2316 	} else if (GRAPHICS_VER(i915) >= 8) {
2317 		pm_ier = intel_uncore_read(uncore, GEN8_GT_IER(2));
2318 		pm_imr = intel_uncore_read(uncore, GEN8_GT_IMR(2));
2319 		pm_isr = intel_uncore_read(uncore, GEN8_GT_ISR(2));
2320 		pm_iir = intel_uncore_read(uncore, GEN8_GT_IIR(2));
2321 	} else {
2322 		pm_ier = intel_uncore_read(uncore, GEN6_PMIER);
2323 		pm_imr = intel_uncore_read(uncore, GEN6_PMIMR);
2324 		pm_isr = intel_uncore_read(uncore, GEN6_PMISR);
2325 		pm_iir = intel_uncore_read(uncore, GEN6_PMIIR);
2326 	}
2327 	pm_mask = intel_uncore_read(uncore, GEN6_PMINTRMSK);
2328 
2329 	drm_printf(p, "Video Turbo Mode: %s\n",
2330 		   str_yes_no(rpmodectl & GEN6_RP_MEDIA_TURBO));
2331 	drm_printf(p, "HW control enabled: %s\n",
2332 		   str_yes_no(rpmodectl & GEN6_RP_ENABLE));
2333 	drm_printf(p, "SW control enabled: %s\n",
2334 		   str_yes_no((rpmodectl & GEN6_RP_MEDIA_MODE_MASK) == GEN6_RP_MEDIA_SW_MODE));
2335 
2336 	drm_printf(p, "PM IER=0x%08x IMR=0x%08x, MASK=0x%08x\n",
2337 		   pm_ier, pm_imr, pm_mask);
2338 	if (GRAPHICS_VER(i915) <= 10)
2339 		drm_printf(p, "PM ISR=0x%08x IIR=0x%08x\n",
2340 			   pm_isr, pm_iir);
2341 	drm_printf(p, "pm_intrmsk_mbz: 0x%08x\n",
2342 		   rps->pm_intrmsk_mbz);
2343 	drm_printf(p, "GT_PERF_STATUS: 0x%08x\n", gt_perf_status);
2344 	drm_printf(p, "Render p-state ratio: %d\n",
2345 		   (gt_perf_status & (GRAPHICS_VER(i915) >= 9 ? 0x1ff00 : 0xff00)) >> 8);
2346 	drm_printf(p, "Render p-state VID: %d\n",
2347 		   gt_perf_status & 0xff);
2348 	drm_printf(p, "Render p-state limit: %d\n",
2349 		   rp_state_limits & 0xff);
2350 	drm_printf(p, "RPSTAT1: 0x%08x\n", rpstat);
2351 	drm_printf(p, "RPMODECTL: 0x%08x\n", rpmodectl);
2352 	drm_printf(p, "RPINCLIMIT: 0x%08x\n", rpinclimit);
2353 	drm_printf(p, "RPDECLIMIT: 0x%08x\n", rpdeclimit);
2354 	drm_printf(p, "RPNSWREQ: %dMHz\n", reqf);
2355 	drm_printf(p, "CAGF: %dMHz\n", cagf);
2356 	drm_printf(p, "RP CUR UP EI: %d (%lldns)\n",
2357 		   rpcurupei,
2358 		   intel_gt_pm_interval_to_ns(gt, rpcurupei));
2359 	drm_printf(p, "RP CUR UP: %d (%lldns)\n",
2360 		   rpcurup, intel_gt_pm_interval_to_ns(gt, rpcurup));
2361 	drm_printf(p, "RP PREV UP: %d (%lldns)\n",
2362 		   rpprevup, intel_gt_pm_interval_to_ns(gt, rpprevup));
2363 	drm_printf(p, "Up threshold: %d%%\n",
2364 		   rps->power.up_threshold);
2365 	drm_printf(p, "RP UP EI: %d (%lldns)\n",
2366 		   rpupei, intel_gt_pm_interval_to_ns(gt, rpupei));
2367 	drm_printf(p, "RP UP THRESHOLD: %d (%lldns)\n",
2368 		   rpupt, intel_gt_pm_interval_to_ns(gt, rpupt));
2369 
2370 	drm_printf(p, "RP CUR DOWN EI: %d (%lldns)\n",
2371 		   rpcurdownei,
2372 		   intel_gt_pm_interval_to_ns(gt, rpcurdownei));
2373 	drm_printf(p, "RP CUR DOWN: %d (%lldns)\n",
2374 		   rpcurdown,
2375 		   intel_gt_pm_interval_to_ns(gt, rpcurdown));
2376 	drm_printf(p, "RP PREV DOWN: %d (%lldns)\n",
2377 		   rpprevdown,
2378 		   intel_gt_pm_interval_to_ns(gt, rpprevdown));
2379 	drm_printf(p, "Down threshold: %d%%\n",
2380 		   rps->power.down_threshold);
2381 	drm_printf(p, "RP DOWN EI: %d (%lldns)\n",
2382 		   rpdownei, intel_gt_pm_interval_to_ns(gt, rpdownei));
2383 	drm_printf(p, "RP DOWN THRESHOLD: %d (%lldns)\n",
2384 		   rpdownt, intel_gt_pm_interval_to_ns(gt, rpdownt));
2385 
2386 	drm_printf(p, "Lowest (RPN) frequency: %dMHz\n",
2387 		   intel_gpu_freq(rps, caps.min_freq));
2388 	drm_printf(p, "Nominal (RP1) frequency: %dMHz\n",
2389 		   intel_gpu_freq(rps, caps.rp1_freq));
2390 	drm_printf(p, "Max non-overclocked (RP0) frequency: %dMHz\n",
2391 		   intel_gpu_freq(rps, caps.rp0_freq));
2392 	drm_printf(p, "Max overclocked frequency: %dMHz\n",
2393 		   intel_gpu_freq(rps, rps->max_freq));
2394 
2395 	drm_printf(p, "Current freq: %d MHz\n",
2396 		   intel_gpu_freq(rps, rps->cur_freq));
2397 	drm_printf(p, "Actual freq: %d MHz\n", cagf);
2398 	drm_printf(p, "Idle freq: %d MHz\n",
2399 		   intel_gpu_freq(rps, rps->idle_freq));
2400 	drm_printf(p, "Min freq: %d MHz\n",
2401 		   intel_gpu_freq(rps, rps->min_freq));
2402 	drm_printf(p, "Boost freq: %d MHz\n",
2403 		   intel_gpu_freq(rps, rps->boost_freq));
2404 	drm_printf(p, "Max freq: %d MHz\n",
2405 		   intel_gpu_freq(rps, rps->max_freq));
2406 	drm_printf(p,
2407 		   "efficient (RPe) frequency: %d MHz\n",
2408 		   intel_gpu_freq(rps, rps->efficient_freq));
2409 }
2410 
slpc_frequency_dump(struct intel_rps * rps,struct drm_printer * p)2411 static void slpc_frequency_dump(struct intel_rps *rps, struct drm_printer *p)
2412 {
2413 	struct intel_gt *gt = rps_to_gt(rps);
2414 	struct intel_uncore *uncore = gt->uncore;
2415 	struct intel_rps_freq_caps caps;
2416 	u32 pm_mask;
2417 
2418 	gen6_rps_get_freq_caps(rps, &caps);
2419 	pm_mask = intel_uncore_read(uncore, GEN6_PMINTRMSK);
2420 
2421 	drm_printf(p, "PM MASK=0x%08x\n", pm_mask);
2422 	drm_printf(p, "pm_intrmsk_mbz: 0x%08x\n",
2423 		   rps->pm_intrmsk_mbz);
2424 	drm_printf(p, "RPSTAT1: 0x%08x\n", intel_rps_read_rpstat(rps));
2425 	drm_printf(p, "RPNSWREQ: %dMHz\n", intel_rps_get_requested_frequency(rps));
2426 	drm_printf(p, "Lowest (RPN) frequency: %dMHz\n",
2427 		   intel_gpu_freq(rps, caps.min_freq));
2428 	drm_printf(p, "Nominal (RP1) frequency: %dMHz\n",
2429 		   intel_gpu_freq(rps, caps.rp1_freq));
2430 	drm_printf(p, "Max non-overclocked (RP0) frequency: %dMHz\n",
2431 		   intel_gpu_freq(rps, caps.rp0_freq));
2432 	drm_printf(p, "Current freq: %d MHz\n",
2433 		   intel_rps_get_requested_frequency(rps));
2434 	drm_printf(p, "Actual freq: %d MHz\n",
2435 		   intel_rps_read_actual_frequency(rps));
2436 	drm_printf(p, "Min freq: %d MHz\n",
2437 		   intel_rps_get_min_frequency(rps));
2438 	drm_printf(p, "Boost freq: %d MHz\n",
2439 		   intel_rps_get_boost_frequency(rps));
2440 	drm_printf(p, "Max freq: %d MHz\n",
2441 		   intel_rps_get_max_frequency(rps));
2442 	drm_printf(p,
2443 		   "efficient (RPe) frequency: %d MHz\n",
2444 		   intel_gpu_freq(rps, caps.rp1_freq));
2445 }
2446 
gen6_rps_frequency_dump(struct intel_rps * rps,struct drm_printer * p)2447 void gen6_rps_frequency_dump(struct intel_rps *rps, struct drm_printer *p)
2448 {
2449 	if (rps_uses_slpc(rps))
2450 		return slpc_frequency_dump(rps, p);
2451 	else
2452 		return rps_frequency_dump(rps, p);
2453 }
2454 
set_max_freq(struct intel_rps * rps,u32 val)2455 static int set_max_freq(struct intel_rps *rps, u32 val)
2456 {
2457 	struct drm_i915_private *i915 = rps_to_i915(rps);
2458 	int ret = 0;
2459 
2460 	mutex_lock(&rps->lock);
2461 
2462 	val = intel_freq_opcode(rps, val);
2463 	if (val < rps->min_freq ||
2464 	    val > rps->max_freq ||
2465 	    val < rps->min_freq_softlimit) {
2466 		ret = -EINVAL;
2467 		goto unlock;
2468 	}
2469 
2470 	if (val > rps->rp0_freq)
2471 		drm_dbg(&i915->drm, "User requested overclocking to %d\n",
2472 			intel_gpu_freq(rps, val));
2473 
2474 	rps->max_freq_softlimit = val;
2475 
2476 	val = clamp_t(int, rps->cur_freq,
2477 		      rps->min_freq_softlimit,
2478 		      rps->max_freq_softlimit);
2479 
2480 	/*
2481 	 * We still need *_set_rps to process the new max_delay and
2482 	 * update the interrupt limits and PMINTRMSK even though
2483 	 * frequency request may be unchanged.
2484 	 */
2485 	intel_rps_set(rps, val);
2486 
2487 unlock:
2488 	mutex_unlock(&rps->lock);
2489 
2490 	return ret;
2491 }
2492 
intel_rps_set_max_frequency(struct intel_rps * rps,u32 val)2493 int intel_rps_set_max_frequency(struct intel_rps *rps, u32 val)
2494 {
2495 	struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2496 
2497 	if (rps_uses_slpc(rps))
2498 		return intel_guc_slpc_set_max_freq(slpc, val);
2499 	else
2500 		return set_max_freq(rps, val);
2501 }
2502 
intel_rps_get_min_frequency(struct intel_rps * rps)2503 u32 intel_rps_get_min_frequency(struct intel_rps *rps)
2504 {
2505 	struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2506 
2507 	if (rps_uses_slpc(rps))
2508 		return slpc->min_freq_softlimit;
2509 	else
2510 		return intel_gpu_freq(rps, rps->min_freq_softlimit);
2511 }
2512 
2513 /**
2514  * intel_rps_get_min_raw_freq - returns the min frequency in some raw format.
2515  * @rps: the intel_rps structure
2516  *
2517  * Returns the min frequency in a raw format. In newer platforms raw is in
2518  * units of 50 MHz.
2519  */
intel_rps_get_min_raw_freq(struct intel_rps * rps)2520 u32 intel_rps_get_min_raw_freq(struct intel_rps *rps)
2521 {
2522 	struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2523 	u32 freq;
2524 
2525 	if (rps_uses_slpc(rps)) {
2526 		return DIV_ROUND_CLOSEST(slpc->min_freq,
2527 					 GT_FREQUENCY_MULTIPLIER);
2528 	} else {
2529 		freq = rps->min_freq;
2530 		if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) {
2531 			/* Convert GT frequency to 50 MHz units */
2532 			freq /= GEN9_FREQ_SCALER;
2533 		}
2534 		return freq;
2535 	}
2536 }
2537 
set_min_freq(struct intel_rps * rps,u32 val)2538 static int set_min_freq(struct intel_rps *rps, u32 val)
2539 {
2540 	int ret = 0;
2541 
2542 	mutex_lock(&rps->lock);
2543 
2544 	val = intel_freq_opcode(rps, val);
2545 	if (val < rps->min_freq ||
2546 	    val > rps->max_freq ||
2547 	    val > rps->max_freq_softlimit) {
2548 		ret = -EINVAL;
2549 		goto unlock;
2550 	}
2551 
2552 	rps->min_freq_softlimit = val;
2553 
2554 	val = clamp_t(int, rps->cur_freq,
2555 		      rps->min_freq_softlimit,
2556 		      rps->max_freq_softlimit);
2557 
2558 	/*
2559 	 * We still need *_set_rps to process the new min_delay and
2560 	 * update the interrupt limits and PMINTRMSK even though
2561 	 * frequency request may be unchanged.
2562 	 */
2563 	intel_rps_set(rps, val);
2564 
2565 unlock:
2566 	mutex_unlock(&rps->lock);
2567 
2568 	return ret;
2569 }
2570 
intel_rps_set_min_frequency(struct intel_rps * rps,u32 val)2571 int intel_rps_set_min_frequency(struct intel_rps *rps, u32 val)
2572 {
2573 	struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2574 
2575 	if (rps_uses_slpc(rps))
2576 		return intel_guc_slpc_set_min_freq(slpc, val);
2577 	else
2578 		return set_min_freq(rps, val);
2579 }
2580 
intel_rps_get_up_threshold(struct intel_rps * rps)2581 u8 intel_rps_get_up_threshold(struct intel_rps *rps)
2582 {
2583 	return rps->power.up_threshold;
2584 }
2585 
rps_set_threshold(struct intel_rps * rps,u8 * threshold,u8 val)2586 static int rps_set_threshold(struct intel_rps *rps, u8 *threshold, u8 val)
2587 {
2588 	int ret;
2589 
2590 	if (val > 100)
2591 		return -EINVAL;
2592 
2593 	ret = mutex_lock_interruptible(&rps->lock);
2594 	if (ret)
2595 		return ret;
2596 
2597 	if (*threshold == val)
2598 		goto out_unlock;
2599 
2600 	*threshold = val;
2601 
2602 	/* Force reset. */
2603 	rps->last_freq = -1;
2604 	mutex_lock(&rps->power.mutex);
2605 	rps->power.mode = -1;
2606 	mutex_unlock(&rps->power.mutex);
2607 
2608 	intel_rps_set(rps, clamp(rps->cur_freq,
2609 				 rps->min_freq_softlimit,
2610 				 rps->max_freq_softlimit));
2611 
2612 out_unlock:
2613 	mutex_unlock(&rps->lock);
2614 
2615 	return ret;
2616 }
2617 
intel_rps_set_up_threshold(struct intel_rps * rps,u8 threshold)2618 int intel_rps_set_up_threshold(struct intel_rps *rps, u8 threshold)
2619 {
2620 	return rps_set_threshold(rps, &rps->power.up_threshold, threshold);
2621 }
2622 
intel_rps_get_down_threshold(struct intel_rps * rps)2623 u8 intel_rps_get_down_threshold(struct intel_rps *rps)
2624 {
2625 	return rps->power.down_threshold;
2626 }
2627 
intel_rps_set_down_threshold(struct intel_rps * rps,u8 threshold)2628 int intel_rps_set_down_threshold(struct intel_rps *rps, u8 threshold)
2629 {
2630 	return rps_set_threshold(rps, &rps->power.down_threshold, threshold);
2631 }
2632 
intel_rps_set_manual(struct intel_rps * rps,bool enable)2633 static void intel_rps_set_manual(struct intel_rps *rps, bool enable)
2634 {
2635 	struct intel_uncore *uncore = rps_to_uncore(rps);
2636 	u32 state = enable ? GEN9_RPSWCTL_ENABLE : GEN9_RPSWCTL_DISABLE;
2637 
2638 	/* Allow punit to process software requests */
2639 	intel_uncore_write(uncore, GEN6_RP_CONTROL, state);
2640 }
2641 
intel_rps_raise_unslice(struct intel_rps * rps)2642 void intel_rps_raise_unslice(struct intel_rps *rps)
2643 {
2644 	struct intel_uncore *uncore = rps_to_uncore(rps);
2645 
2646 	mutex_lock(&rps->lock);
2647 
2648 	if (rps_uses_slpc(rps)) {
2649 		/* RP limits have not been initialized yet for SLPC path */
2650 		struct intel_rps_freq_caps caps;
2651 
2652 		gen6_rps_get_freq_caps(rps, &caps);
2653 
2654 		intel_rps_set_manual(rps, true);
2655 		intel_uncore_write(uncore, GEN6_RPNSWREQ,
2656 				   ((caps.rp0_freq <<
2657 				   GEN9_SW_REQ_UNSLICE_RATIO_SHIFT) |
2658 				   GEN9_IGNORE_SLICE_RATIO));
2659 		intel_rps_set_manual(rps, false);
2660 	} else {
2661 		intel_rps_set(rps, rps->rp0_freq);
2662 	}
2663 
2664 	mutex_unlock(&rps->lock);
2665 }
2666 
intel_rps_lower_unslice(struct intel_rps * rps)2667 void intel_rps_lower_unslice(struct intel_rps *rps)
2668 {
2669 	struct intel_uncore *uncore = rps_to_uncore(rps);
2670 
2671 	mutex_lock(&rps->lock);
2672 
2673 	if (rps_uses_slpc(rps)) {
2674 		/* RP limits have not been initialized yet for SLPC path */
2675 		struct intel_rps_freq_caps caps;
2676 
2677 		gen6_rps_get_freq_caps(rps, &caps);
2678 
2679 		intel_rps_set_manual(rps, true);
2680 		intel_uncore_write(uncore, GEN6_RPNSWREQ,
2681 				   ((caps.min_freq <<
2682 				   GEN9_SW_REQ_UNSLICE_RATIO_SHIFT) |
2683 				   GEN9_IGNORE_SLICE_RATIO));
2684 		intel_rps_set_manual(rps, false);
2685 	} else {
2686 		intel_rps_set(rps, rps->min_freq);
2687 	}
2688 
2689 	mutex_unlock(&rps->lock);
2690 }
2691 
rps_read_mmio(struct intel_rps * rps,i915_reg_t reg32)2692 static u32 rps_read_mmio(struct intel_rps *rps, i915_reg_t reg32)
2693 {
2694 	struct intel_gt *gt = rps_to_gt(rps);
2695 	intel_wakeref_t wakeref;
2696 	u32 val;
2697 
2698 	with_intel_runtime_pm(gt->uncore->rpm, wakeref)
2699 		val = intel_uncore_read(gt->uncore, reg32);
2700 
2701 	return val;
2702 }
2703 
rps_read_mask_mmio(struct intel_rps * rps,i915_reg_t reg32,u32 mask)2704 bool rps_read_mask_mmio(struct intel_rps *rps,
2705 			i915_reg_t reg32, u32 mask)
2706 {
2707 	return rps_read_mmio(rps, reg32) & mask;
2708 }
2709 
2710 /* External interface for intel_ips.ko */
2711 
2712 static struct drm_i915_private __rcu *ips_mchdev;
2713 
2714 /*
2715  * Tells the intel_ips driver that the i915 driver is now loaded, if
2716  * IPS got loaded first.
2717  *
2718  * This awkward dance is so that neither module has to depend on the
2719  * other in order for IPS to do the appropriate communication of
2720  * GPU turbo limits to i915.
2721  */
2722 static void
ips_ping_for_i915_load(void)2723 ips_ping_for_i915_load(void)
2724 {
2725 	void (*link)(void);
2726 
2727 	link = symbol_get(ips_link_to_i915_driver);
2728 	if (link) {
2729 		link();
2730 		symbol_put(ips_link_to_i915_driver);
2731 	}
2732 }
2733 
intel_rps_driver_register(struct intel_rps * rps)2734 void intel_rps_driver_register(struct intel_rps *rps)
2735 {
2736 	struct intel_gt *gt = rps_to_gt(rps);
2737 
2738 	/*
2739 	 * We only register the i915 ips part with intel-ips once everything is
2740 	 * set up, to avoid intel-ips sneaking in and reading bogus values.
2741 	 */
2742 	if (GRAPHICS_VER(gt->i915) == 5) {
2743 		GEM_BUG_ON(ips_mchdev);
2744 		rcu_assign_pointer(ips_mchdev, gt->i915);
2745 		ips_ping_for_i915_load();
2746 	}
2747 }
2748 
intel_rps_driver_unregister(struct intel_rps * rps)2749 void intel_rps_driver_unregister(struct intel_rps *rps)
2750 {
2751 	if (rcu_access_pointer(ips_mchdev) == rps_to_i915(rps))
2752 		rcu_assign_pointer(ips_mchdev, NULL);
2753 }
2754 
mchdev_get(void)2755 static struct drm_i915_private *mchdev_get(void)
2756 {
2757 	struct drm_i915_private *i915;
2758 
2759 	rcu_read_lock();
2760 	i915 = rcu_dereference(ips_mchdev);
2761 	if (i915 && !kref_get_unless_zero(&i915->drm.ref))
2762 		i915 = NULL;
2763 	rcu_read_unlock();
2764 
2765 	return i915;
2766 }
2767 
2768 /**
2769  * i915_read_mch_val - return value for IPS use
2770  *
2771  * Calculate and return a value for the IPS driver to use when deciding whether
2772  * we have thermal and power headroom to increase CPU or GPU power budget.
2773  */
i915_read_mch_val(void)2774 unsigned long i915_read_mch_val(void)
2775 {
2776 	struct drm_i915_private *i915;
2777 	unsigned long chipset_val = 0;
2778 	unsigned long graphics_val = 0;
2779 	intel_wakeref_t wakeref;
2780 
2781 	i915 = mchdev_get();
2782 	if (!i915)
2783 		return 0;
2784 
2785 	with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
2786 		struct intel_ips *ips = &to_gt(i915)->rps.ips;
2787 
2788 		spin_lock_irq(&mchdev_lock);
2789 		chipset_val = __ips_chipset_val(ips);
2790 		graphics_val = __ips_gfx_val(ips);
2791 		spin_unlock_irq(&mchdev_lock);
2792 	}
2793 
2794 	drm_dev_put(&i915->drm);
2795 	return chipset_val + graphics_val;
2796 }
2797 EXPORT_SYMBOL_GPL(i915_read_mch_val);
2798 
2799 /**
2800  * i915_gpu_raise - raise GPU frequency limit
2801  *
2802  * Raise the limit; IPS indicates we have thermal headroom.
2803  */
i915_gpu_raise(void)2804 bool i915_gpu_raise(void)
2805 {
2806 	struct drm_i915_private *i915;
2807 	struct intel_rps *rps;
2808 
2809 	i915 = mchdev_get();
2810 	if (!i915)
2811 		return false;
2812 
2813 	rps = &to_gt(i915)->rps;
2814 
2815 	spin_lock_irq(&mchdev_lock);
2816 	if (rps->max_freq_softlimit < rps->max_freq)
2817 		rps->max_freq_softlimit++;
2818 	spin_unlock_irq(&mchdev_lock);
2819 
2820 	drm_dev_put(&i915->drm);
2821 	return true;
2822 }
2823 EXPORT_SYMBOL_GPL(i915_gpu_raise);
2824 
2825 /**
2826  * i915_gpu_lower - lower GPU frequency limit
2827  *
2828  * IPS indicates we're close to a thermal limit, so throttle back the GPU
2829  * frequency maximum.
2830  */
i915_gpu_lower(void)2831 bool i915_gpu_lower(void)
2832 {
2833 	struct drm_i915_private *i915;
2834 	struct intel_rps *rps;
2835 
2836 	i915 = mchdev_get();
2837 	if (!i915)
2838 		return false;
2839 
2840 	rps = &to_gt(i915)->rps;
2841 
2842 	spin_lock_irq(&mchdev_lock);
2843 	if (rps->max_freq_softlimit > rps->min_freq)
2844 		rps->max_freq_softlimit--;
2845 	spin_unlock_irq(&mchdev_lock);
2846 
2847 	drm_dev_put(&i915->drm);
2848 	return true;
2849 }
2850 EXPORT_SYMBOL_GPL(i915_gpu_lower);
2851 
2852 /**
2853  * i915_gpu_busy - indicate GPU business to IPS
2854  *
2855  * Tell the IPS driver whether or not the GPU is busy.
2856  */
i915_gpu_busy(void)2857 bool i915_gpu_busy(void)
2858 {
2859 	struct drm_i915_private *i915;
2860 	bool ret;
2861 
2862 	i915 = mchdev_get();
2863 	if (!i915)
2864 		return false;
2865 
2866 	ret = to_gt(i915)->awake;
2867 
2868 	drm_dev_put(&i915->drm);
2869 	return ret;
2870 }
2871 EXPORT_SYMBOL_GPL(i915_gpu_busy);
2872 
2873 /**
2874  * i915_gpu_turbo_disable - disable graphics turbo
2875  *
2876  * Disable graphics turbo by resetting the max frequency and setting the
2877  * current frequency to the default.
2878  */
i915_gpu_turbo_disable(void)2879 bool i915_gpu_turbo_disable(void)
2880 {
2881 	struct drm_i915_private *i915;
2882 	struct intel_rps *rps;
2883 	bool ret;
2884 
2885 	i915 = mchdev_get();
2886 	if (!i915)
2887 		return false;
2888 
2889 	rps = &to_gt(i915)->rps;
2890 
2891 	spin_lock_irq(&mchdev_lock);
2892 	rps->max_freq_softlimit = rps->min_freq;
2893 	ret = !__gen5_rps_set(&to_gt(i915)->rps, rps->min_freq);
2894 	spin_unlock_irq(&mchdev_lock);
2895 
2896 	drm_dev_put(&i915->drm);
2897 	return ret;
2898 }
2899 EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
2900 
2901 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
2902 #include "selftest_rps.c"
2903 #include "selftest_slpc.c"
2904 #endif
2905