1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2019 Intel Corporation
4 */
5
6 #include <linux/string_helpers.h>
7
8 #include <drm/i915_drm.h>
9
10 #include "i915_drv.h"
11 #include "i915_irq.h"
12 #include "intel_breadcrumbs.h"
13 #include "intel_gt.h"
14 #include "intel_gt_clock_utils.h"
15 #include "intel_gt_irq.h"
16 #include "intel_gt_pm_irq.h"
17 #include "intel_gt_regs.h"
18 #include "intel_mchbar_regs.h"
19 #include "intel_pcode.h"
20 #include "intel_rps.h"
21 #include "vlv_sideband.h"
22 #include "../../../platform/x86/intel_ips.h"
23
24 #define BUSY_MAX_EI 20u /* ms */
25
26 /*
27 * Lock protecting IPS related data structures
28 */
29 static DEFINE_SPINLOCK(mchdev_lock);
30
rps_to_gt(struct intel_rps * rps)31 static struct intel_gt *rps_to_gt(struct intel_rps *rps)
32 {
33 return container_of(rps, struct intel_gt, rps);
34 }
35
rps_to_i915(struct intel_rps * rps)36 static struct drm_i915_private *rps_to_i915(struct intel_rps *rps)
37 {
38 return rps_to_gt(rps)->i915;
39 }
40
rps_to_uncore(struct intel_rps * rps)41 static struct intel_uncore *rps_to_uncore(struct intel_rps *rps)
42 {
43 return rps_to_gt(rps)->uncore;
44 }
45
rps_to_slpc(struct intel_rps * rps)46 static struct intel_guc_slpc *rps_to_slpc(struct intel_rps *rps)
47 {
48 struct intel_gt *gt = rps_to_gt(rps);
49
50 return >->uc.guc.slpc;
51 }
52
rps_uses_slpc(struct intel_rps * rps)53 static bool rps_uses_slpc(struct intel_rps *rps)
54 {
55 struct intel_gt *gt = rps_to_gt(rps);
56
57 return intel_uc_uses_guc_slpc(>->uc);
58 }
59
rps_pm_sanitize_mask(struct intel_rps * rps,u32 mask)60 static u32 rps_pm_sanitize_mask(struct intel_rps *rps, u32 mask)
61 {
62 return mask & ~rps->pm_intrmsk_mbz;
63 }
64
set(struct intel_uncore * uncore,i915_reg_t reg,u32 val)65 static void set(struct intel_uncore *uncore, i915_reg_t reg, u32 val)
66 {
67 intel_uncore_write_fw(uncore, reg, val);
68 }
69
rps_timer(struct timer_list * t)70 static void rps_timer(struct timer_list *t)
71 {
72 struct intel_rps *rps = from_timer(rps, t, timer);
73 struct intel_engine_cs *engine;
74 ktime_t dt, last, timestamp;
75 enum intel_engine_id id;
76 s64 max_busy[3] = {};
77
78 timestamp = 0;
79 for_each_engine(engine, rps_to_gt(rps), id) {
80 s64 busy;
81 int i;
82
83 dt = intel_engine_get_busy_time(engine, ×tamp);
84 last = engine->stats.rps;
85 engine->stats.rps = dt;
86
87 busy = ktime_to_ns(ktime_sub(dt, last));
88 for (i = 0; i < ARRAY_SIZE(max_busy); i++) {
89 if (busy > max_busy[i])
90 swap(busy, max_busy[i]);
91 }
92 }
93 last = rps->pm_timestamp;
94 rps->pm_timestamp = timestamp;
95
96 if (intel_rps_is_active(rps)) {
97 s64 busy;
98 int i;
99
100 dt = ktime_sub(timestamp, last);
101
102 /*
103 * Our goal is to evaluate each engine independently, so we run
104 * at the lowest clocks required to sustain the heaviest
105 * workload. However, a task may be split into sequential
106 * dependent operations across a set of engines, such that
107 * the independent contributions do not account for high load,
108 * but overall the task is GPU bound. For example, consider
109 * video decode on vcs followed by colour post-processing
110 * on vecs, followed by general post-processing on rcs.
111 * Since multi-engines being active does imply a single
112 * continuous workload across all engines, we hedge our
113 * bets by only contributing a factor of the distributed
114 * load into our busyness calculation.
115 */
116 busy = max_busy[0];
117 for (i = 1; i < ARRAY_SIZE(max_busy); i++) {
118 if (!max_busy[i])
119 break;
120
121 busy += div_u64(max_busy[i], 1 << i);
122 }
123 GT_TRACE(rps_to_gt(rps),
124 "busy:%lld [%d%%], max:[%lld, %lld, %lld], interval:%d\n",
125 busy, (int)div64_u64(100 * busy, dt),
126 max_busy[0], max_busy[1], max_busy[2],
127 rps->pm_interval);
128
129 if (100 * busy > rps->power.up_threshold * dt &&
130 rps->cur_freq < rps->max_freq_softlimit) {
131 rps->pm_iir |= GEN6_PM_RP_UP_THRESHOLD;
132 rps->pm_interval = 1;
133 schedule_work(&rps->work);
134 } else if (100 * busy < rps->power.down_threshold * dt &&
135 rps->cur_freq > rps->min_freq_softlimit) {
136 rps->pm_iir |= GEN6_PM_RP_DOWN_THRESHOLD;
137 rps->pm_interval = 1;
138 schedule_work(&rps->work);
139 } else {
140 rps->last_adj = 0;
141 }
142
143 mod_timer(&rps->timer,
144 jiffies + msecs_to_jiffies(rps->pm_interval));
145 rps->pm_interval = min(rps->pm_interval * 2, BUSY_MAX_EI);
146 }
147 }
148
rps_start_timer(struct intel_rps * rps)149 static void rps_start_timer(struct intel_rps *rps)
150 {
151 rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp);
152 rps->pm_interval = 1;
153 mod_timer(&rps->timer, jiffies + 1);
154 }
155
rps_stop_timer(struct intel_rps * rps)156 static void rps_stop_timer(struct intel_rps *rps)
157 {
158 del_timer_sync(&rps->timer);
159 rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp);
160 cancel_work_sync(&rps->work);
161 }
162
rps_pm_mask(struct intel_rps * rps,u8 val)163 static u32 rps_pm_mask(struct intel_rps *rps, u8 val)
164 {
165 u32 mask = 0;
166
167 /* We use UP_EI_EXPIRED interrupts for both up/down in manual mode */
168 if (val > rps->min_freq_softlimit)
169 mask |= (GEN6_PM_RP_UP_EI_EXPIRED |
170 GEN6_PM_RP_DOWN_THRESHOLD |
171 GEN6_PM_RP_DOWN_TIMEOUT);
172
173 if (val < rps->max_freq_softlimit)
174 mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD;
175
176 mask &= rps->pm_events;
177
178 return rps_pm_sanitize_mask(rps, ~mask);
179 }
180
rps_reset_ei(struct intel_rps * rps)181 static void rps_reset_ei(struct intel_rps *rps)
182 {
183 memset(&rps->ei, 0, sizeof(rps->ei));
184 }
185
rps_enable_interrupts(struct intel_rps * rps)186 static void rps_enable_interrupts(struct intel_rps *rps)
187 {
188 struct intel_gt *gt = rps_to_gt(rps);
189
190 GEM_BUG_ON(rps_uses_slpc(rps));
191
192 GT_TRACE(gt, "interrupts:on rps->pm_events: %x, rps_pm_mask:%x\n",
193 rps->pm_events, rps_pm_mask(rps, rps->last_freq));
194
195 rps_reset_ei(rps);
196
197 spin_lock_irq(gt->irq_lock);
198 gen6_gt_pm_enable_irq(gt, rps->pm_events);
199 spin_unlock_irq(gt->irq_lock);
200
201 intel_uncore_write(gt->uncore,
202 GEN6_PMINTRMSK, rps_pm_mask(rps, rps->last_freq));
203 }
204
gen6_rps_reset_interrupts(struct intel_rps * rps)205 static void gen6_rps_reset_interrupts(struct intel_rps *rps)
206 {
207 gen6_gt_pm_reset_iir(rps_to_gt(rps), GEN6_PM_RPS_EVENTS);
208 }
209
gen11_rps_reset_interrupts(struct intel_rps * rps)210 static void gen11_rps_reset_interrupts(struct intel_rps *rps)
211 {
212 while (gen11_gt_reset_one_iir(rps_to_gt(rps), 0, GEN11_GTPM))
213 ;
214 }
215
rps_reset_interrupts(struct intel_rps * rps)216 static void rps_reset_interrupts(struct intel_rps *rps)
217 {
218 struct intel_gt *gt = rps_to_gt(rps);
219
220 spin_lock_irq(gt->irq_lock);
221 if (GRAPHICS_VER(gt->i915) >= 11)
222 gen11_rps_reset_interrupts(rps);
223 else
224 gen6_rps_reset_interrupts(rps);
225
226 rps->pm_iir = 0;
227 spin_unlock_irq(gt->irq_lock);
228 }
229
rps_disable_interrupts(struct intel_rps * rps)230 static void rps_disable_interrupts(struct intel_rps *rps)
231 {
232 struct intel_gt *gt = rps_to_gt(rps);
233
234 intel_uncore_write(gt->uncore,
235 GEN6_PMINTRMSK, rps_pm_sanitize_mask(rps, ~0u));
236
237 spin_lock_irq(gt->irq_lock);
238 gen6_gt_pm_disable_irq(gt, GEN6_PM_RPS_EVENTS);
239 spin_unlock_irq(gt->irq_lock);
240
241 intel_synchronize_irq(gt->i915);
242
243 /*
244 * Now that we will not be generating any more work, flush any
245 * outstanding tasks. As we are called on the RPS idle path,
246 * we will reset the GPU to minimum frequencies, so the current
247 * state of the worker can be discarded.
248 */
249 cancel_work_sync(&rps->work);
250
251 rps_reset_interrupts(rps);
252 GT_TRACE(gt, "interrupts:off\n");
253 }
254
255 static const struct cparams {
256 u16 i;
257 u16 t;
258 u16 m;
259 u16 c;
260 } cparams[] = {
261 { 1, 1333, 301, 28664 },
262 { 1, 1066, 294, 24460 },
263 { 1, 800, 294, 25192 },
264 { 0, 1333, 276, 27605 },
265 { 0, 1066, 276, 27605 },
266 { 0, 800, 231, 23784 },
267 };
268
gen5_rps_init(struct intel_rps * rps)269 static void gen5_rps_init(struct intel_rps *rps)
270 {
271 struct drm_i915_private *i915 = rps_to_i915(rps);
272 struct intel_uncore *uncore = rps_to_uncore(rps);
273 u8 fmax, fmin, fstart;
274 u32 rgvmodectl;
275 int c_m, i;
276
277 if (i915->fsb_freq <= 3200)
278 c_m = 0;
279 else if (i915->fsb_freq <= 4800)
280 c_m = 1;
281 else
282 c_m = 2;
283
284 for (i = 0; i < ARRAY_SIZE(cparams); i++) {
285 if (cparams[i].i == c_m && cparams[i].t == i915->mem_freq) {
286 rps->ips.m = cparams[i].m;
287 rps->ips.c = cparams[i].c;
288 break;
289 }
290 }
291
292 rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);
293
294 /* Set up min, max, and cur for interrupt handling */
295 fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
296 fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
297 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
298 MEMMODE_FSTART_SHIFT;
299 drm_dbg(&i915->drm, "fmax: %d, fmin: %d, fstart: %d\n",
300 fmax, fmin, fstart);
301
302 rps->min_freq = fmax;
303 rps->efficient_freq = fstart;
304 rps->max_freq = fmin;
305 }
306
307 static unsigned long
__ips_chipset_val(struct intel_ips * ips)308 __ips_chipset_val(struct intel_ips *ips)
309 {
310 struct intel_uncore *uncore =
311 rps_to_uncore(container_of(ips, struct intel_rps, ips));
312 unsigned long now = jiffies_to_msecs(jiffies), dt;
313 unsigned long result;
314 u64 total, delta;
315
316 lockdep_assert_held(&mchdev_lock);
317
318 /*
319 * Prevent division-by-zero if we are asking too fast.
320 * Also, we don't get interesting results if we are polling
321 * faster than once in 10ms, so just return the saved value
322 * in such cases.
323 */
324 dt = now - ips->last_time1;
325 if (dt <= 10)
326 return ips->chipset_power;
327
328 /* FIXME: handle per-counter overflow */
329 total = intel_uncore_read(uncore, DMIEC);
330 total += intel_uncore_read(uncore, DDREC);
331 total += intel_uncore_read(uncore, CSIEC);
332
333 delta = total - ips->last_count1;
334
335 result = div_u64(div_u64(ips->m * delta, dt) + ips->c, 10);
336
337 ips->last_count1 = total;
338 ips->last_time1 = now;
339
340 ips->chipset_power = result;
341
342 return result;
343 }
344
ips_mch_val(struct intel_uncore * uncore)345 static unsigned long ips_mch_val(struct intel_uncore *uncore)
346 {
347 unsigned int m, x, b;
348 u32 tsfs;
349
350 tsfs = intel_uncore_read(uncore, TSFS);
351 x = intel_uncore_read8(uncore, TR1);
352
353 b = tsfs & TSFS_INTR_MASK;
354 m = (tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT;
355
356 return m * x / 127 - b;
357 }
358
_pxvid_to_vd(u8 pxvid)359 static int _pxvid_to_vd(u8 pxvid)
360 {
361 if (pxvid == 0)
362 return 0;
363
364 if (pxvid >= 8 && pxvid < 31)
365 pxvid = 31;
366
367 return (pxvid + 2) * 125;
368 }
369
pvid_to_extvid(struct drm_i915_private * i915,u8 pxvid)370 static u32 pvid_to_extvid(struct drm_i915_private *i915, u8 pxvid)
371 {
372 const int vd = _pxvid_to_vd(pxvid);
373
374 if (INTEL_INFO(i915)->is_mobile)
375 return max(vd - 1125, 0);
376
377 return vd;
378 }
379
__gen5_ips_update(struct intel_ips * ips)380 static void __gen5_ips_update(struct intel_ips *ips)
381 {
382 struct intel_uncore *uncore =
383 rps_to_uncore(container_of(ips, struct intel_rps, ips));
384 u64 now, delta, dt;
385 u32 count;
386
387 lockdep_assert_held(&mchdev_lock);
388
389 now = ktime_get_raw_ns();
390 dt = now - ips->last_time2;
391 do_div(dt, NSEC_PER_MSEC);
392
393 /* Don't divide by 0 */
394 if (dt <= 10)
395 return;
396
397 count = intel_uncore_read(uncore, GFXEC);
398 delta = count - ips->last_count2;
399
400 ips->last_count2 = count;
401 ips->last_time2 = now;
402
403 /* More magic constants... */
404 ips->gfx_power = div_u64(delta * 1181, dt * 10);
405 }
406
gen5_rps_update(struct intel_rps * rps)407 static void gen5_rps_update(struct intel_rps *rps)
408 {
409 spin_lock_irq(&mchdev_lock);
410 __gen5_ips_update(&rps->ips);
411 spin_unlock_irq(&mchdev_lock);
412 }
413
gen5_invert_freq(struct intel_rps * rps,unsigned int val)414 static unsigned int gen5_invert_freq(struct intel_rps *rps,
415 unsigned int val)
416 {
417 /* Invert the frequency bin into an ips delay */
418 val = rps->max_freq - val;
419 val = rps->min_freq + val;
420
421 return val;
422 }
423
__gen5_rps_set(struct intel_rps * rps,u8 val)424 static int __gen5_rps_set(struct intel_rps *rps, u8 val)
425 {
426 struct intel_uncore *uncore = rps_to_uncore(rps);
427 u16 rgvswctl;
428
429 lockdep_assert_held(&mchdev_lock);
430
431 rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
432 if (rgvswctl & MEMCTL_CMD_STS) {
433 DRM_DEBUG("gpu busy, RCS change rejected\n");
434 return -EBUSY; /* still busy with another command */
435 }
436
437 /* Invert the frequency bin into an ips delay */
438 val = gen5_invert_freq(rps, val);
439
440 rgvswctl =
441 (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |
442 (val << MEMCTL_FREQ_SHIFT) |
443 MEMCTL_SFCAVM;
444 intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);
445 intel_uncore_posting_read16(uncore, MEMSWCTL);
446
447 rgvswctl |= MEMCTL_CMD_STS;
448 intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);
449
450 return 0;
451 }
452
gen5_rps_set(struct intel_rps * rps,u8 val)453 static int gen5_rps_set(struct intel_rps *rps, u8 val)
454 {
455 int err;
456
457 spin_lock_irq(&mchdev_lock);
458 err = __gen5_rps_set(rps, val);
459 spin_unlock_irq(&mchdev_lock);
460
461 return err;
462 }
463
intel_pxfreq(u32 vidfreq)464 static unsigned long intel_pxfreq(u32 vidfreq)
465 {
466 int div = (vidfreq & 0x3f0000) >> 16;
467 int post = (vidfreq & 0x3000) >> 12;
468 int pre = (vidfreq & 0x7);
469
470 if (!pre)
471 return 0;
472
473 return div * 133333 / (pre << post);
474 }
475
init_emon(struct intel_uncore * uncore)476 static unsigned int init_emon(struct intel_uncore *uncore)
477 {
478 u8 pxw[16];
479 int i;
480
481 /* Disable to program */
482 intel_uncore_write(uncore, ECR, 0);
483 intel_uncore_posting_read(uncore, ECR);
484
485 /* Program energy weights for various events */
486 intel_uncore_write(uncore, SDEW, 0x15040d00);
487 intel_uncore_write(uncore, CSIEW0, 0x007f0000);
488 intel_uncore_write(uncore, CSIEW1, 0x1e220004);
489 intel_uncore_write(uncore, CSIEW2, 0x04000004);
490
491 for (i = 0; i < 5; i++)
492 intel_uncore_write(uncore, PEW(i), 0);
493 for (i = 0; i < 3; i++)
494 intel_uncore_write(uncore, DEW(i), 0);
495
496 /* Program P-state weights to account for frequency power adjustment */
497 for (i = 0; i < 16; i++) {
498 u32 pxvidfreq = intel_uncore_read(uncore, PXVFREQ(i));
499 unsigned int freq = intel_pxfreq(pxvidfreq);
500 unsigned int vid =
501 (pxvidfreq & PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;
502 unsigned int val;
503
504 val = vid * vid * freq / 1000 * 255;
505 val /= 127 * 127 * 900;
506
507 pxw[i] = val;
508 }
509 /* Render standby states get 0 weight */
510 pxw[14] = 0;
511 pxw[15] = 0;
512
513 for (i = 0; i < 4; i++) {
514 intel_uncore_write(uncore, PXW(i),
515 pxw[i * 4 + 0] << 24 |
516 pxw[i * 4 + 1] << 16 |
517 pxw[i * 4 + 2] << 8 |
518 pxw[i * 4 + 3] << 0);
519 }
520
521 /* Adjust magic regs to magic values (more experimental results) */
522 intel_uncore_write(uncore, OGW0, 0);
523 intel_uncore_write(uncore, OGW1, 0);
524 intel_uncore_write(uncore, EG0, 0x00007f00);
525 intel_uncore_write(uncore, EG1, 0x0000000e);
526 intel_uncore_write(uncore, EG2, 0x000e0000);
527 intel_uncore_write(uncore, EG3, 0x68000300);
528 intel_uncore_write(uncore, EG4, 0x42000000);
529 intel_uncore_write(uncore, EG5, 0x00140031);
530 intel_uncore_write(uncore, EG6, 0);
531 intel_uncore_write(uncore, EG7, 0);
532
533 for (i = 0; i < 8; i++)
534 intel_uncore_write(uncore, PXWL(i), 0);
535
536 /* Enable PMON + select events */
537 intel_uncore_write(uncore, ECR, 0x80000019);
538
539 return intel_uncore_read(uncore, LCFUSE02) & LCFUSE_HIV_MASK;
540 }
541
gen5_rps_enable(struct intel_rps * rps)542 static bool gen5_rps_enable(struct intel_rps *rps)
543 {
544 struct drm_i915_private *i915 = rps_to_i915(rps);
545 struct intel_uncore *uncore = rps_to_uncore(rps);
546 u8 fstart, vstart;
547 u32 rgvmodectl;
548
549 spin_lock_irq(&mchdev_lock);
550
551 rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);
552
553 /* Enable temp reporting */
554 intel_uncore_write16(uncore, PMMISC,
555 intel_uncore_read16(uncore, PMMISC) | MCPPCE_EN);
556 intel_uncore_write16(uncore, TSC1,
557 intel_uncore_read16(uncore, TSC1) | TSE);
558
559 /* 100ms RC evaluation intervals */
560 intel_uncore_write(uncore, RCUPEI, 100000);
561 intel_uncore_write(uncore, RCDNEI, 100000);
562
563 /* Set max/min thresholds to 90ms and 80ms respectively */
564 intel_uncore_write(uncore, RCBMAXAVG, 90000);
565 intel_uncore_write(uncore, RCBMINAVG, 80000);
566
567 intel_uncore_write(uncore, MEMIHYST, 1);
568
569 /* Set up min, max, and cur for interrupt handling */
570 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
571 MEMMODE_FSTART_SHIFT;
572
573 vstart = (intel_uncore_read(uncore, PXVFREQ(fstart)) &
574 PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;
575
576 intel_uncore_write(uncore,
577 MEMINTREN,
578 MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
579
580 intel_uncore_write(uncore, VIDSTART, vstart);
581 intel_uncore_posting_read(uncore, VIDSTART);
582
583 rgvmodectl |= MEMMODE_SWMODE_EN;
584 intel_uncore_write(uncore, MEMMODECTL, rgvmodectl);
585
586 if (wait_for_atomic((intel_uncore_read(uncore, MEMSWCTL) &
587 MEMCTL_CMD_STS) == 0, 10))
588 drm_err(&uncore->i915->drm,
589 "stuck trying to change perf mode\n");
590 mdelay(1);
591
592 __gen5_rps_set(rps, rps->cur_freq);
593
594 rps->ips.last_count1 = intel_uncore_read(uncore, DMIEC);
595 rps->ips.last_count1 += intel_uncore_read(uncore, DDREC);
596 rps->ips.last_count1 += intel_uncore_read(uncore, CSIEC);
597 rps->ips.last_time1 = jiffies_to_msecs(jiffies);
598
599 rps->ips.last_count2 = intel_uncore_read(uncore, GFXEC);
600 rps->ips.last_time2 = ktime_get_raw_ns();
601
602 spin_lock(&i915->irq_lock);
603 ilk_enable_display_irq(i915, DE_PCU_EVENT);
604 spin_unlock(&i915->irq_lock);
605
606 spin_unlock_irq(&mchdev_lock);
607
608 rps->ips.corr = init_emon(uncore);
609
610 return true;
611 }
612
gen5_rps_disable(struct intel_rps * rps)613 static void gen5_rps_disable(struct intel_rps *rps)
614 {
615 struct drm_i915_private *i915 = rps_to_i915(rps);
616 struct intel_uncore *uncore = rps_to_uncore(rps);
617 u16 rgvswctl;
618
619 spin_lock_irq(&mchdev_lock);
620
621 spin_lock(&i915->irq_lock);
622 ilk_disable_display_irq(i915, DE_PCU_EVENT);
623 spin_unlock(&i915->irq_lock);
624
625 rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
626
627 /* Ack interrupts, disable EFC interrupt */
628 intel_uncore_write(uncore, MEMINTREN,
629 intel_uncore_read(uncore, MEMINTREN) &
630 ~MEMINT_EVAL_CHG_EN);
631 intel_uncore_write(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
632
633 /* Go back to the starting frequency */
634 __gen5_rps_set(rps, rps->idle_freq);
635 mdelay(1);
636 rgvswctl |= MEMCTL_CMD_STS;
637 intel_uncore_write(uncore, MEMSWCTL, rgvswctl);
638 mdelay(1);
639
640 spin_unlock_irq(&mchdev_lock);
641 }
642
rps_limits(struct intel_rps * rps,u8 val)643 static u32 rps_limits(struct intel_rps *rps, u8 val)
644 {
645 u32 limits;
646
647 /*
648 * Only set the down limit when we've reached the lowest level to avoid
649 * getting more interrupts, otherwise leave this clear. This prevents a
650 * race in the hw when coming out of rc6: There's a tiny window where
651 * the hw runs at the minimal clock before selecting the desired
652 * frequency, if the down threshold expires in that window we will not
653 * receive a down interrupt.
654 */
655 if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) {
656 limits = rps->max_freq_softlimit << 23;
657 if (val <= rps->min_freq_softlimit)
658 limits |= rps->min_freq_softlimit << 14;
659 } else {
660 limits = rps->max_freq_softlimit << 24;
661 if (val <= rps->min_freq_softlimit)
662 limits |= rps->min_freq_softlimit << 16;
663 }
664
665 return limits;
666 }
667
rps_set_power(struct intel_rps * rps,int new_power)668 static void rps_set_power(struct intel_rps *rps, int new_power)
669 {
670 struct intel_gt *gt = rps_to_gt(rps);
671 struct intel_uncore *uncore = gt->uncore;
672 u32 threshold_up = 0, threshold_down = 0; /* in % */
673 u32 ei_up = 0, ei_down = 0;
674
675 lockdep_assert_held(&rps->power.mutex);
676
677 if (new_power == rps->power.mode)
678 return;
679
680 threshold_up = 95;
681 threshold_down = 85;
682
683 /* Note the units here are not exactly 1us, but 1280ns. */
684 switch (new_power) {
685 case LOW_POWER:
686 ei_up = 16000;
687 ei_down = 32000;
688 break;
689
690 case BETWEEN:
691 ei_up = 13000;
692 ei_down = 32000;
693 break;
694
695 case HIGH_POWER:
696 ei_up = 10000;
697 ei_down = 32000;
698 break;
699 }
700
701 /* When byt can survive without system hang with dynamic
702 * sw freq adjustments, this restriction can be lifted.
703 */
704 if (IS_VALLEYVIEW(gt->i915))
705 goto skip_hw_write;
706
707 GT_TRACE(gt,
708 "changing power mode [%d], up %d%% @ %dus, down %d%% @ %dus\n",
709 new_power, threshold_up, ei_up, threshold_down, ei_down);
710
711 set(uncore, GEN6_RP_UP_EI,
712 intel_gt_ns_to_pm_interval(gt, ei_up * 1000));
713 set(uncore, GEN6_RP_UP_THRESHOLD,
714 intel_gt_ns_to_pm_interval(gt, ei_up * threshold_up * 10));
715
716 set(uncore, GEN6_RP_DOWN_EI,
717 intel_gt_ns_to_pm_interval(gt, ei_down * 1000));
718 set(uncore, GEN6_RP_DOWN_THRESHOLD,
719 intel_gt_ns_to_pm_interval(gt, ei_down * threshold_down * 10));
720
721 set(uncore, GEN6_RP_CONTROL,
722 (GRAPHICS_VER(gt->i915) > 9 ? 0 : GEN6_RP_MEDIA_TURBO) |
723 GEN6_RP_MEDIA_HW_NORMAL_MODE |
724 GEN6_RP_MEDIA_IS_GFX |
725 GEN6_RP_ENABLE |
726 GEN6_RP_UP_BUSY_AVG |
727 GEN6_RP_DOWN_IDLE_AVG);
728
729 skip_hw_write:
730 rps->power.mode = new_power;
731 rps->power.up_threshold = threshold_up;
732 rps->power.down_threshold = threshold_down;
733 }
734
gen6_rps_set_thresholds(struct intel_rps * rps,u8 val)735 static void gen6_rps_set_thresholds(struct intel_rps *rps, u8 val)
736 {
737 int new_power;
738
739 new_power = rps->power.mode;
740 switch (rps->power.mode) {
741 case LOW_POWER:
742 if (val > rps->efficient_freq + 1 &&
743 val > rps->cur_freq)
744 new_power = BETWEEN;
745 break;
746
747 case BETWEEN:
748 if (val <= rps->efficient_freq &&
749 val < rps->cur_freq)
750 new_power = LOW_POWER;
751 else if (val >= rps->rp0_freq &&
752 val > rps->cur_freq)
753 new_power = HIGH_POWER;
754 break;
755
756 case HIGH_POWER:
757 if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 &&
758 val < rps->cur_freq)
759 new_power = BETWEEN;
760 break;
761 }
762 /* Max/min bins are special */
763 if (val <= rps->min_freq_softlimit)
764 new_power = LOW_POWER;
765 if (val >= rps->max_freq_softlimit)
766 new_power = HIGH_POWER;
767
768 mutex_lock(&rps->power.mutex);
769 if (rps->power.interactive)
770 new_power = HIGH_POWER;
771 rps_set_power(rps, new_power);
772 mutex_unlock(&rps->power.mutex);
773 }
774
intel_rps_mark_interactive(struct intel_rps * rps,bool interactive)775 void intel_rps_mark_interactive(struct intel_rps *rps, bool interactive)
776 {
777 GT_TRACE(rps_to_gt(rps), "mark interactive: %s\n",
778 str_yes_no(interactive));
779
780 mutex_lock(&rps->power.mutex);
781 if (interactive) {
782 if (!rps->power.interactive++ && intel_rps_is_active(rps))
783 rps_set_power(rps, HIGH_POWER);
784 } else {
785 GEM_BUG_ON(!rps->power.interactive);
786 rps->power.interactive--;
787 }
788 mutex_unlock(&rps->power.mutex);
789 }
790
gen6_rps_set(struct intel_rps * rps,u8 val)791 static int gen6_rps_set(struct intel_rps *rps, u8 val)
792 {
793 struct intel_uncore *uncore = rps_to_uncore(rps);
794 struct drm_i915_private *i915 = rps_to_i915(rps);
795 u32 swreq;
796
797 GEM_BUG_ON(rps_uses_slpc(rps));
798
799 if (GRAPHICS_VER(i915) >= 9)
800 swreq = GEN9_FREQUENCY(val);
801 else if (IS_HASWELL(i915) || IS_BROADWELL(i915))
802 swreq = HSW_FREQUENCY(val);
803 else
804 swreq = (GEN6_FREQUENCY(val) |
805 GEN6_OFFSET(0) |
806 GEN6_AGGRESSIVE_TURBO);
807 set(uncore, GEN6_RPNSWREQ, swreq);
808
809 GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d, swreq:%x\n",
810 val, intel_gpu_freq(rps, val), swreq);
811
812 return 0;
813 }
814
vlv_rps_set(struct intel_rps * rps,u8 val)815 static int vlv_rps_set(struct intel_rps *rps, u8 val)
816 {
817 struct drm_i915_private *i915 = rps_to_i915(rps);
818 int err;
819
820 vlv_punit_get(i915);
821 err = vlv_punit_write(i915, PUNIT_REG_GPU_FREQ_REQ, val);
822 vlv_punit_put(i915);
823
824 GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d\n",
825 val, intel_gpu_freq(rps, val));
826
827 return err;
828 }
829
rps_set(struct intel_rps * rps,u8 val,bool update)830 static int rps_set(struct intel_rps *rps, u8 val, bool update)
831 {
832 struct drm_i915_private *i915 = rps_to_i915(rps);
833 int err;
834
835 if (val == rps->last_freq)
836 return 0;
837
838 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
839 err = vlv_rps_set(rps, val);
840 else if (GRAPHICS_VER(i915) >= 6)
841 err = gen6_rps_set(rps, val);
842 else
843 err = gen5_rps_set(rps, val);
844 if (err)
845 return err;
846
847 if (update && GRAPHICS_VER(i915) >= 6)
848 gen6_rps_set_thresholds(rps, val);
849 rps->last_freq = val;
850
851 return 0;
852 }
853
intel_rps_unpark(struct intel_rps * rps)854 void intel_rps_unpark(struct intel_rps *rps)
855 {
856 if (!intel_rps_is_enabled(rps))
857 return;
858
859 GT_TRACE(rps_to_gt(rps), "unpark:%x\n", rps->cur_freq);
860
861 /*
862 * Use the user's desired frequency as a guide, but for better
863 * performance, jump directly to RPe as our starting frequency.
864 */
865 mutex_lock(&rps->lock);
866
867 intel_rps_set_active(rps);
868 intel_rps_set(rps,
869 clamp(rps->cur_freq,
870 rps->min_freq_softlimit,
871 rps->max_freq_softlimit));
872
873 mutex_unlock(&rps->lock);
874
875 rps->pm_iir = 0;
876 if (intel_rps_has_interrupts(rps))
877 rps_enable_interrupts(rps);
878 if (intel_rps_uses_timer(rps))
879 rps_start_timer(rps);
880
881 if (GRAPHICS_VER(rps_to_i915(rps)) == 5)
882 gen5_rps_update(rps);
883 }
884
intel_rps_park(struct intel_rps * rps)885 void intel_rps_park(struct intel_rps *rps)
886 {
887 int adj;
888
889 if (!intel_rps_is_enabled(rps))
890 return;
891
892 if (!intel_rps_clear_active(rps))
893 return;
894
895 if (intel_rps_uses_timer(rps))
896 rps_stop_timer(rps);
897 if (intel_rps_has_interrupts(rps))
898 rps_disable_interrupts(rps);
899
900 if (rps->last_freq <= rps->idle_freq)
901 return;
902
903 /*
904 * The punit delays the write of the frequency and voltage until it
905 * determines the GPU is awake. During normal usage we don't want to
906 * waste power changing the frequency if the GPU is sleeping (rc6).
907 * However, the GPU and driver is now idle and we do not want to delay
908 * switching to minimum voltage (reducing power whilst idle) as we do
909 * not expect to be woken in the near future and so must flush the
910 * change by waking the device.
911 *
912 * We choose to take the media powerwell (either would do to trick the
913 * punit into committing the voltage change) as that takes a lot less
914 * power than the render powerwell.
915 */
916 intel_uncore_forcewake_get(rps_to_uncore(rps), FORCEWAKE_MEDIA);
917 rps_set(rps, rps->idle_freq, false);
918 intel_uncore_forcewake_put(rps_to_uncore(rps), FORCEWAKE_MEDIA);
919
920 /*
921 * Since we will try and restart from the previously requested
922 * frequency on unparking, treat this idle point as a downclock
923 * interrupt and reduce the frequency for resume. If we park/unpark
924 * more frequently than the rps worker can run, we will not respond
925 * to any EI and never see a change in frequency.
926 *
927 * (Note we accommodate Cherryview's limitation of only using an
928 * even bin by applying it to all.)
929 */
930 adj = rps->last_adj;
931 if (adj < 0)
932 adj *= 2;
933 else /* CHV needs even encode values */
934 adj = -2;
935 rps->last_adj = adj;
936 rps->cur_freq = max_t(int, rps->cur_freq + adj, rps->min_freq);
937 if (rps->cur_freq < rps->efficient_freq) {
938 rps->cur_freq = rps->efficient_freq;
939 rps->last_adj = 0;
940 }
941
942 GT_TRACE(rps_to_gt(rps), "park:%x\n", rps->cur_freq);
943 }
944
intel_rps_get_boost_frequency(struct intel_rps * rps)945 u32 intel_rps_get_boost_frequency(struct intel_rps *rps)
946 {
947 struct intel_guc_slpc *slpc;
948
949 if (rps_uses_slpc(rps)) {
950 slpc = rps_to_slpc(rps);
951
952 return slpc->boost_freq;
953 } else {
954 return intel_gpu_freq(rps, rps->boost_freq);
955 }
956 }
957
rps_set_boost_freq(struct intel_rps * rps,u32 val)958 static int rps_set_boost_freq(struct intel_rps *rps, u32 val)
959 {
960 bool boost = false;
961
962 /* Validate against (static) hardware limits */
963 val = intel_freq_opcode(rps, val);
964 if (val < rps->min_freq || val > rps->max_freq)
965 return -EINVAL;
966
967 mutex_lock(&rps->lock);
968 if (val != rps->boost_freq) {
969 rps->boost_freq = val;
970 boost = atomic_read(&rps->num_waiters);
971 }
972 mutex_unlock(&rps->lock);
973 if (boost)
974 schedule_work(&rps->work);
975
976 return 0;
977 }
978
intel_rps_set_boost_frequency(struct intel_rps * rps,u32 freq)979 int intel_rps_set_boost_frequency(struct intel_rps *rps, u32 freq)
980 {
981 struct intel_guc_slpc *slpc;
982
983 if (rps_uses_slpc(rps)) {
984 slpc = rps_to_slpc(rps);
985
986 return intel_guc_slpc_set_boost_freq(slpc, freq);
987 } else {
988 return rps_set_boost_freq(rps, freq);
989 }
990 }
991
intel_rps_dec_waiters(struct intel_rps * rps)992 void intel_rps_dec_waiters(struct intel_rps *rps)
993 {
994 struct intel_guc_slpc *slpc;
995
996 if (rps_uses_slpc(rps)) {
997 slpc = rps_to_slpc(rps);
998
999 intel_guc_slpc_dec_waiters(slpc);
1000 } else {
1001 atomic_dec(&rps->num_waiters);
1002 }
1003 }
1004
intel_rps_boost(struct i915_request * rq)1005 void intel_rps_boost(struct i915_request *rq)
1006 {
1007 struct intel_guc_slpc *slpc;
1008
1009 if (i915_request_signaled(rq) || i915_request_has_waitboost(rq))
1010 return;
1011
1012 /* Serializes with i915_request_retire() */
1013 if (!test_and_set_bit(I915_FENCE_FLAG_BOOST, &rq->fence.flags)) {
1014 struct intel_rps *rps = &READ_ONCE(rq->engine)->gt->rps;
1015
1016 if (rps_uses_slpc(rps)) {
1017 slpc = rps_to_slpc(rps);
1018
1019 /* Return if old value is non zero */
1020 if (!atomic_fetch_inc(&slpc->num_waiters))
1021 schedule_work(&slpc->boost_work);
1022
1023 return;
1024 }
1025
1026 if (atomic_fetch_inc(&rps->num_waiters))
1027 return;
1028
1029 if (!intel_rps_is_active(rps))
1030 return;
1031
1032 GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n",
1033 rq->fence.context, rq->fence.seqno);
1034
1035 if (READ_ONCE(rps->cur_freq) < rps->boost_freq)
1036 schedule_work(&rps->work);
1037
1038 WRITE_ONCE(rps->boosts, rps->boosts + 1); /* debug only */
1039 }
1040 }
1041
intel_rps_set(struct intel_rps * rps,u8 val)1042 int intel_rps_set(struct intel_rps *rps, u8 val)
1043 {
1044 int err;
1045
1046 lockdep_assert_held(&rps->lock);
1047 GEM_BUG_ON(val > rps->max_freq);
1048 GEM_BUG_ON(val < rps->min_freq);
1049
1050 if (intel_rps_is_active(rps)) {
1051 err = rps_set(rps, val, true);
1052 if (err)
1053 return err;
1054
1055 /*
1056 * Make sure we continue to get interrupts
1057 * until we hit the minimum or maximum frequencies.
1058 */
1059 if (intel_rps_has_interrupts(rps)) {
1060 struct intel_uncore *uncore = rps_to_uncore(rps);
1061
1062 set(uncore,
1063 GEN6_RP_INTERRUPT_LIMITS, rps_limits(rps, val));
1064
1065 set(uncore, GEN6_PMINTRMSK, rps_pm_mask(rps, val));
1066 }
1067 }
1068
1069 rps->cur_freq = val;
1070 return 0;
1071 }
1072
intel_rps_read_state_cap(struct intel_rps * rps)1073 static u32 intel_rps_read_state_cap(struct intel_rps *rps)
1074 {
1075 struct drm_i915_private *i915 = rps_to_i915(rps);
1076 struct intel_uncore *uncore = rps_to_uncore(rps);
1077
1078 if (IS_PONTEVECCHIO(i915))
1079 return intel_uncore_read(uncore, PVC_RP_STATE_CAP);
1080 else if (IS_XEHPSDV(i915))
1081 return intel_uncore_read(uncore, XEHPSDV_RP_STATE_CAP);
1082 else if (IS_GEN9_LP(i915))
1083 return intel_uncore_read(uncore, BXT_RP_STATE_CAP);
1084 else
1085 return intel_uncore_read(uncore, GEN6_RP_STATE_CAP);
1086 }
1087
1088 /**
1089 * gen6_rps_get_freq_caps - Get freq caps exposed by HW
1090 * @rps: the intel_rps structure
1091 * @caps: returned freq caps
1092 *
1093 * Returned "caps" frequencies should be converted to MHz using
1094 * intel_gpu_freq()
1095 */
gen6_rps_get_freq_caps(struct intel_rps * rps,struct intel_rps_freq_caps * caps)1096 void gen6_rps_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps)
1097 {
1098 struct drm_i915_private *i915 = rps_to_i915(rps);
1099 u32 rp_state_cap;
1100
1101 rp_state_cap = intel_rps_read_state_cap(rps);
1102
1103 /* static values from HW: RP0 > RP1 > RPn (min_freq) */
1104 if (IS_GEN9_LP(i915)) {
1105 caps->rp0_freq = (rp_state_cap >> 16) & 0xff;
1106 caps->rp1_freq = (rp_state_cap >> 8) & 0xff;
1107 caps->min_freq = (rp_state_cap >> 0) & 0xff;
1108 } else {
1109 caps->rp0_freq = (rp_state_cap >> 0) & 0xff;
1110 if (GRAPHICS_VER(i915) >= 10)
1111 caps->rp1_freq = REG_FIELD_GET(RPE_MASK,
1112 intel_uncore_read(to_gt(i915)->uncore,
1113 GEN10_FREQ_INFO_REC));
1114 else
1115 caps->rp1_freq = (rp_state_cap >> 8) & 0xff;
1116 caps->min_freq = (rp_state_cap >> 16) & 0xff;
1117 }
1118
1119 if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) {
1120 /*
1121 * In this case rp_state_cap register reports frequencies in
1122 * units of 50 MHz. Convert these to the actual "hw unit", i.e.
1123 * units of 16.67 MHz
1124 */
1125 caps->rp0_freq *= GEN9_FREQ_SCALER;
1126 caps->rp1_freq *= GEN9_FREQ_SCALER;
1127 caps->min_freq *= GEN9_FREQ_SCALER;
1128 }
1129 }
1130
gen6_rps_init(struct intel_rps * rps)1131 static void gen6_rps_init(struct intel_rps *rps)
1132 {
1133 struct drm_i915_private *i915 = rps_to_i915(rps);
1134 struct intel_rps_freq_caps caps;
1135
1136 gen6_rps_get_freq_caps(rps, &caps);
1137 rps->rp0_freq = caps.rp0_freq;
1138 rps->rp1_freq = caps.rp1_freq;
1139 rps->min_freq = caps.min_freq;
1140
1141 /* hw_max = RP0 until we check for overclocking */
1142 rps->max_freq = rps->rp0_freq;
1143
1144 rps->efficient_freq = rps->rp1_freq;
1145 if (IS_HASWELL(i915) || IS_BROADWELL(i915) ||
1146 IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) {
1147 u32 ddcc_status = 0;
1148 u32 mult = 1;
1149
1150 if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11)
1151 mult = GEN9_FREQ_SCALER;
1152 if (snb_pcode_read(rps_to_gt(rps)->uncore,
1153 HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
1154 &ddcc_status, NULL) == 0)
1155 rps->efficient_freq =
1156 clamp_t(u32,
1157 ((ddcc_status >> 8) & 0xff) * mult,
1158 rps->min_freq,
1159 rps->max_freq);
1160 }
1161 }
1162
rps_reset(struct intel_rps * rps)1163 static bool rps_reset(struct intel_rps *rps)
1164 {
1165 struct drm_i915_private *i915 = rps_to_i915(rps);
1166
1167 /* force a reset */
1168 rps->power.mode = -1;
1169 rps->last_freq = -1;
1170
1171 if (rps_set(rps, rps->min_freq, true)) {
1172 drm_err(&i915->drm, "Failed to reset RPS to initial values\n");
1173 return false;
1174 }
1175
1176 rps->cur_freq = rps->min_freq;
1177 return true;
1178 }
1179
1180 /* See the Gen9_GT_PM_Programming_Guide doc for the below */
gen9_rps_enable(struct intel_rps * rps)1181 static bool gen9_rps_enable(struct intel_rps *rps)
1182 {
1183 struct intel_gt *gt = rps_to_gt(rps);
1184 struct intel_uncore *uncore = gt->uncore;
1185
1186 /* Program defaults and thresholds for RPS */
1187 if (GRAPHICS_VER(gt->i915) == 9)
1188 intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ,
1189 GEN9_FREQUENCY(rps->rp1_freq));
1190
1191 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 0xa);
1192
1193 rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD;
1194
1195 return rps_reset(rps);
1196 }
1197
gen8_rps_enable(struct intel_rps * rps)1198 static bool gen8_rps_enable(struct intel_rps *rps)
1199 {
1200 struct intel_uncore *uncore = rps_to_uncore(rps);
1201
1202 intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ,
1203 HSW_FREQUENCY(rps->rp1_freq));
1204
1205 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1206
1207 rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD;
1208
1209 return rps_reset(rps);
1210 }
1211
gen6_rps_enable(struct intel_rps * rps)1212 static bool gen6_rps_enable(struct intel_rps *rps)
1213 {
1214 struct intel_uncore *uncore = rps_to_uncore(rps);
1215
1216 /* Power down if completely idle for over 50ms */
1217 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 50000);
1218 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1219
1220 rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD |
1221 GEN6_PM_RP_DOWN_THRESHOLD |
1222 GEN6_PM_RP_DOWN_TIMEOUT);
1223
1224 return rps_reset(rps);
1225 }
1226
chv_rps_max_freq(struct intel_rps * rps)1227 static int chv_rps_max_freq(struct intel_rps *rps)
1228 {
1229 struct drm_i915_private *i915 = rps_to_i915(rps);
1230 struct intel_gt *gt = rps_to_gt(rps);
1231 u32 val;
1232
1233 val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE);
1234
1235 switch (gt->info.sseu.eu_total) {
1236 case 8:
1237 /* (2 * 4) config */
1238 val >>= FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT;
1239 break;
1240 case 12:
1241 /* (2 * 6) config */
1242 val >>= FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT;
1243 break;
1244 case 16:
1245 /* (2 * 8) config */
1246 default:
1247 /* Setting (2 * 8) Min RP0 for any other combination */
1248 val >>= FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT;
1249 break;
1250 }
1251
1252 return val & FB_GFX_FREQ_FUSE_MASK;
1253 }
1254
chv_rps_rpe_freq(struct intel_rps * rps)1255 static int chv_rps_rpe_freq(struct intel_rps *rps)
1256 {
1257 struct drm_i915_private *i915 = rps_to_i915(rps);
1258 u32 val;
1259
1260 val = vlv_punit_read(i915, PUNIT_GPU_DUTYCYCLE_REG);
1261 val >>= PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT;
1262
1263 return val & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK;
1264 }
1265
chv_rps_guar_freq(struct intel_rps * rps)1266 static int chv_rps_guar_freq(struct intel_rps *rps)
1267 {
1268 struct drm_i915_private *i915 = rps_to_i915(rps);
1269 u32 val;
1270
1271 val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE);
1272
1273 return val & FB_GFX_FREQ_FUSE_MASK;
1274 }
1275
chv_rps_min_freq(struct intel_rps * rps)1276 static u32 chv_rps_min_freq(struct intel_rps *rps)
1277 {
1278 struct drm_i915_private *i915 = rps_to_i915(rps);
1279 u32 val;
1280
1281 val = vlv_punit_read(i915, FB_GFX_FMIN_AT_VMIN_FUSE);
1282 val >>= FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT;
1283
1284 return val & FB_GFX_FREQ_FUSE_MASK;
1285 }
1286
chv_rps_enable(struct intel_rps * rps)1287 static bool chv_rps_enable(struct intel_rps *rps)
1288 {
1289 struct intel_uncore *uncore = rps_to_uncore(rps);
1290 struct drm_i915_private *i915 = rps_to_i915(rps);
1291 u32 val;
1292
1293 /* 1: Program defaults and thresholds for RPS*/
1294 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000);
1295 intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400);
1296 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000);
1297 intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000);
1298 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000);
1299
1300 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1301
1302 /* 2: Enable RPS */
1303 intel_uncore_write_fw(uncore, GEN6_RP_CONTROL,
1304 GEN6_RP_MEDIA_HW_NORMAL_MODE |
1305 GEN6_RP_MEDIA_IS_GFX |
1306 GEN6_RP_ENABLE |
1307 GEN6_RP_UP_BUSY_AVG |
1308 GEN6_RP_DOWN_IDLE_AVG);
1309
1310 rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD |
1311 GEN6_PM_RP_DOWN_THRESHOLD |
1312 GEN6_PM_RP_DOWN_TIMEOUT);
1313
1314 /* Setting Fixed Bias */
1315 vlv_punit_get(i915);
1316
1317 val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | CHV_BIAS_CPU_50_SOC_50;
1318 vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val);
1319
1320 val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1321
1322 vlv_punit_put(i915);
1323
1324 /* RPS code assumes GPLL is used */
1325 drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0,
1326 "GPLL not enabled\n");
1327
1328 drm_dbg(&i915->drm, "GPLL enabled? %s\n",
1329 str_yes_no(val & GPLLENABLE));
1330 drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val);
1331
1332 return rps_reset(rps);
1333 }
1334
vlv_rps_guar_freq(struct intel_rps * rps)1335 static int vlv_rps_guar_freq(struct intel_rps *rps)
1336 {
1337 struct drm_i915_private *i915 = rps_to_i915(rps);
1338 u32 val, rp1;
1339
1340 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE);
1341
1342 rp1 = val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK;
1343 rp1 >>= FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT;
1344
1345 return rp1;
1346 }
1347
vlv_rps_max_freq(struct intel_rps * rps)1348 static int vlv_rps_max_freq(struct intel_rps *rps)
1349 {
1350 struct drm_i915_private *i915 = rps_to_i915(rps);
1351 u32 val, rp0;
1352
1353 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE);
1354
1355 rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
1356 /* Clamp to max */
1357 rp0 = min_t(u32, rp0, 0xea);
1358
1359 return rp0;
1360 }
1361
vlv_rps_rpe_freq(struct intel_rps * rps)1362 static int vlv_rps_rpe_freq(struct intel_rps *rps)
1363 {
1364 struct drm_i915_private *i915 = rps_to_i915(rps);
1365 u32 val, rpe;
1366
1367 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
1368 rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
1369 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
1370 rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
1371
1372 return rpe;
1373 }
1374
vlv_rps_min_freq(struct intel_rps * rps)1375 static int vlv_rps_min_freq(struct intel_rps *rps)
1376 {
1377 struct drm_i915_private *i915 = rps_to_i915(rps);
1378 u32 val;
1379
1380 val = vlv_punit_read(i915, PUNIT_REG_GPU_LFM) & 0xff;
1381 /*
1382 * According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value
1383 * for the minimum frequency in GPLL mode is 0xc1. Contrary to this on
1384 * a BYT-M B0 the above register contains 0xbf. Moreover when setting
1385 * a frequency Punit will not allow values below 0xc0. Clamp it 0xc0
1386 * to make sure it matches what Punit accepts.
1387 */
1388 return max_t(u32, val, 0xc0);
1389 }
1390
vlv_rps_enable(struct intel_rps * rps)1391 static bool vlv_rps_enable(struct intel_rps *rps)
1392 {
1393 struct intel_uncore *uncore = rps_to_uncore(rps);
1394 struct drm_i915_private *i915 = rps_to_i915(rps);
1395 u32 val;
1396
1397 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000);
1398 intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400);
1399 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000);
1400 intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000);
1401 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000);
1402
1403 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1404
1405 intel_uncore_write_fw(uncore, GEN6_RP_CONTROL,
1406 GEN6_RP_MEDIA_TURBO |
1407 GEN6_RP_MEDIA_HW_NORMAL_MODE |
1408 GEN6_RP_MEDIA_IS_GFX |
1409 GEN6_RP_ENABLE |
1410 GEN6_RP_UP_BUSY_AVG |
1411 GEN6_RP_DOWN_IDLE_CONT);
1412
1413 /* WaGsvRC0ResidencyMethod:vlv */
1414 rps->pm_events = GEN6_PM_RP_UP_EI_EXPIRED;
1415
1416 vlv_punit_get(i915);
1417
1418 /* Setting Fixed Bias */
1419 val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | VLV_BIAS_CPU_125_SOC_875;
1420 vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val);
1421
1422 val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1423
1424 vlv_punit_put(i915);
1425
1426 /* RPS code assumes GPLL is used */
1427 drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0,
1428 "GPLL not enabled\n");
1429
1430 drm_dbg(&i915->drm, "GPLL enabled? %s\n",
1431 str_yes_no(val & GPLLENABLE));
1432 drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val);
1433
1434 return rps_reset(rps);
1435 }
1436
__ips_gfx_val(struct intel_ips * ips)1437 static unsigned long __ips_gfx_val(struct intel_ips *ips)
1438 {
1439 struct intel_rps *rps = container_of(ips, typeof(*rps), ips);
1440 struct intel_uncore *uncore = rps_to_uncore(rps);
1441 unsigned int t, state1, state2;
1442 u32 pxvid, ext_v;
1443 u64 corr, corr2;
1444
1445 lockdep_assert_held(&mchdev_lock);
1446
1447 pxvid = intel_uncore_read(uncore, PXVFREQ(rps->cur_freq));
1448 pxvid = (pxvid >> 24) & 0x7f;
1449 ext_v = pvid_to_extvid(rps_to_i915(rps), pxvid);
1450
1451 state1 = ext_v;
1452
1453 /* Revel in the empirically derived constants */
1454
1455 /* Correction factor in 1/100000 units */
1456 t = ips_mch_val(uncore);
1457 if (t > 80)
1458 corr = t * 2349 + 135940;
1459 else if (t >= 50)
1460 corr = t * 964 + 29317;
1461 else /* < 50 */
1462 corr = t * 301 + 1004;
1463
1464 corr = div_u64(corr * 150142 * state1, 10000) - 78642;
1465 corr2 = div_u64(corr, 100000) * ips->corr;
1466
1467 state2 = div_u64(corr2 * state1, 10000);
1468 state2 /= 100; /* convert to mW */
1469
1470 __gen5_ips_update(ips);
1471
1472 return ips->gfx_power + state2;
1473 }
1474
has_busy_stats(struct intel_rps * rps)1475 static bool has_busy_stats(struct intel_rps *rps)
1476 {
1477 struct intel_engine_cs *engine;
1478 enum intel_engine_id id;
1479
1480 for_each_engine(engine, rps_to_gt(rps), id) {
1481 if (!intel_engine_supports_stats(engine))
1482 return false;
1483 }
1484
1485 return true;
1486 }
1487
intel_rps_enable(struct intel_rps * rps)1488 void intel_rps_enable(struct intel_rps *rps)
1489 {
1490 struct drm_i915_private *i915 = rps_to_i915(rps);
1491 struct intel_uncore *uncore = rps_to_uncore(rps);
1492 bool enabled = false;
1493
1494 if (!HAS_RPS(i915))
1495 return;
1496
1497 if (rps_uses_slpc(rps))
1498 return;
1499
1500 intel_gt_check_clock_frequency(rps_to_gt(rps));
1501
1502 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
1503 if (rps->max_freq <= rps->min_freq)
1504 /* leave disabled, no room for dynamic reclocking */;
1505 else if (IS_CHERRYVIEW(i915))
1506 enabled = chv_rps_enable(rps);
1507 else if (IS_VALLEYVIEW(i915))
1508 enabled = vlv_rps_enable(rps);
1509 else if (GRAPHICS_VER(i915) >= 9)
1510 enabled = gen9_rps_enable(rps);
1511 else if (GRAPHICS_VER(i915) >= 8)
1512 enabled = gen8_rps_enable(rps);
1513 else if (GRAPHICS_VER(i915) >= 6)
1514 enabled = gen6_rps_enable(rps);
1515 else if (IS_IRONLAKE_M(i915))
1516 enabled = gen5_rps_enable(rps);
1517 else
1518 MISSING_CASE(GRAPHICS_VER(i915));
1519 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
1520 if (!enabled)
1521 return;
1522
1523 GT_TRACE(rps_to_gt(rps),
1524 "min:%x, max:%x, freq:[%d, %d]\n",
1525 rps->min_freq, rps->max_freq,
1526 intel_gpu_freq(rps, rps->min_freq),
1527 intel_gpu_freq(rps, rps->max_freq));
1528
1529 GEM_BUG_ON(rps->max_freq < rps->min_freq);
1530 GEM_BUG_ON(rps->idle_freq > rps->max_freq);
1531
1532 GEM_BUG_ON(rps->efficient_freq < rps->min_freq);
1533 GEM_BUG_ON(rps->efficient_freq > rps->max_freq);
1534
1535 if (has_busy_stats(rps))
1536 intel_rps_set_timer(rps);
1537 else if (GRAPHICS_VER(i915) >= 6 && GRAPHICS_VER(i915) <= 11)
1538 intel_rps_set_interrupts(rps);
1539 else
1540 /* Ironlake currently uses intel_ips.ko */ {}
1541
1542 intel_rps_set_enabled(rps);
1543 }
1544
gen6_rps_disable(struct intel_rps * rps)1545 static void gen6_rps_disable(struct intel_rps *rps)
1546 {
1547 set(rps_to_uncore(rps), GEN6_RP_CONTROL, 0);
1548 }
1549
intel_rps_disable(struct intel_rps * rps)1550 void intel_rps_disable(struct intel_rps *rps)
1551 {
1552 struct drm_i915_private *i915 = rps_to_i915(rps);
1553
1554 if (!intel_rps_is_enabled(rps))
1555 return;
1556
1557 intel_rps_clear_enabled(rps);
1558 intel_rps_clear_interrupts(rps);
1559 intel_rps_clear_timer(rps);
1560
1561 if (GRAPHICS_VER(i915) >= 6)
1562 gen6_rps_disable(rps);
1563 else if (IS_IRONLAKE_M(i915))
1564 gen5_rps_disable(rps);
1565 }
1566
byt_gpu_freq(struct intel_rps * rps,int val)1567 static int byt_gpu_freq(struct intel_rps *rps, int val)
1568 {
1569 /*
1570 * N = val - 0xb7
1571 * Slow = Fast = GPLL ref * N
1572 */
1573 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000);
1574 }
1575
byt_freq_opcode(struct intel_rps * rps,int val)1576 static int byt_freq_opcode(struct intel_rps *rps, int val)
1577 {
1578 return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7;
1579 }
1580
chv_gpu_freq(struct intel_rps * rps,int val)1581 static int chv_gpu_freq(struct intel_rps *rps, int val)
1582 {
1583 /*
1584 * N = val / 2
1585 * CU (slow) = CU2x (fast) / 2 = GPLL ref * N / 2
1586 */
1587 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000);
1588 }
1589
chv_freq_opcode(struct intel_rps * rps,int val)1590 static int chv_freq_opcode(struct intel_rps *rps, int val)
1591 {
1592 /* CHV needs even values */
1593 return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2;
1594 }
1595
intel_gpu_freq(struct intel_rps * rps,int val)1596 int intel_gpu_freq(struct intel_rps *rps, int val)
1597 {
1598 struct drm_i915_private *i915 = rps_to_i915(rps);
1599
1600 if (GRAPHICS_VER(i915) >= 9)
1601 return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER,
1602 GEN9_FREQ_SCALER);
1603 else if (IS_CHERRYVIEW(i915))
1604 return chv_gpu_freq(rps, val);
1605 else if (IS_VALLEYVIEW(i915))
1606 return byt_gpu_freq(rps, val);
1607 else if (GRAPHICS_VER(i915) >= 6)
1608 return val * GT_FREQUENCY_MULTIPLIER;
1609 else
1610 return val;
1611 }
1612
intel_freq_opcode(struct intel_rps * rps,int val)1613 int intel_freq_opcode(struct intel_rps *rps, int val)
1614 {
1615 struct drm_i915_private *i915 = rps_to_i915(rps);
1616
1617 if (GRAPHICS_VER(i915) >= 9)
1618 return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER,
1619 GT_FREQUENCY_MULTIPLIER);
1620 else if (IS_CHERRYVIEW(i915))
1621 return chv_freq_opcode(rps, val);
1622 else if (IS_VALLEYVIEW(i915))
1623 return byt_freq_opcode(rps, val);
1624 else if (GRAPHICS_VER(i915) >= 6)
1625 return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER);
1626 else
1627 return val;
1628 }
1629
vlv_init_gpll_ref_freq(struct intel_rps * rps)1630 static void vlv_init_gpll_ref_freq(struct intel_rps *rps)
1631 {
1632 struct drm_i915_private *i915 = rps_to_i915(rps);
1633
1634 rps->gpll_ref_freq =
1635 vlv_get_cck_clock(i915, "GPLL ref",
1636 CCK_GPLL_CLOCK_CONTROL,
1637 i915->czclk_freq);
1638
1639 drm_dbg(&i915->drm, "GPLL reference freq: %d kHz\n",
1640 rps->gpll_ref_freq);
1641 }
1642
vlv_rps_init(struct intel_rps * rps)1643 static void vlv_rps_init(struct intel_rps *rps)
1644 {
1645 struct drm_i915_private *i915 = rps_to_i915(rps);
1646 u32 val;
1647
1648 vlv_iosf_sb_get(i915,
1649 BIT(VLV_IOSF_SB_PUNIT) |
1650 BIT(VLV_IOSF_SB_NC) |
1651 BIT(VLV_IOSF_SB_CCK));
1652
1653 vlv_init_gpll_ref_freq(rps);
1654
1655 val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1656 switch ((val >> 6) & 3) {
1657 case 0:
1658 case 1:
1659 i915->mem_freq = 800;
1660 break;
1661 case 2:
1662 i915->mem_freq = 1066;
1663 break;
1664 case 3:
1665 i915->mem_freq = 1333;
1666 break;
1667 }
1668 drm_dbg(&i915->drm, "DDR speed: %d MHz\n", i915->mem_freq);
1669
1670 rps->max_freq = vlv_rps_max_freq(rps);
1671 rps->rp0_freq = rps->max_freq;
1672 drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n",
1673 intel_gpu_freq(rps, rps->max_freq), rps->max_freq);
1674
1675 rps->efficient_freq = vlv_rps_rpe_freq(rps);
1676 drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n",
1677 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq);
1678
1679 rps->rp1_freq = vlv_rps_guar_freq(rps);
1680 drm_dbg(&i915->drm, "RP1(Guar Freq) GPU freq: %d MHz (%u)\n",
1681 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq);
1682
1683 rps->min_freq = vlv_rps_min_freq(rps);
1684 drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n",
1685 intel_gpu_freq(rps, rps->min_freq), rps->min_freq);
1686
1687 vlv_iosf_sb_put(i915,
1688 BIT(VLV_IOSF_SB_PUNIT) |
1689 BIT(VLV_IOSF_SB_NC) |
1690 BIT(VLV_IOSF_SB_CCK));
1691 }
1692
chv_rps_init(struct intel_rps * rps)1693 static void chv_rps_init(struct intel_rps *rps)
1694 {
1695 struct drm_i915_private *i915 = rps_to_i915(rps);
1696 u32 val;
1697
1698 vlv_iosf_sb_get(i915,
1699 BIT(VLV_IOSF_SB_PUNIT) |
1700 BIT(VLV_IOSF_SB_NC) |
1701 BIT(VLV_IOSF_SB_CCK));
1702
1703 vlv_init_gpll_ref_freq(rps);
1704
1705 val = vlv_cck_read(i915, CCK_FUSE_REG);
1706
1707 switch ((val >> 2) & 0x7) {
1708 case 3:
1709 i915->mem_freq = 2000;
1710 break;
1711 default:
1712 i915->mem_freq = 1600;
1713 break;
1714 }
1715 drm_dbg(&i915->drm, "DDR speed: %d MHz\n", i915->mem_freq);
1716
1717 rps->max_freq = chv_rps_max_freq(rps);
1718 rps->rp0_freq = rps->max_freq;
1719 drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n",
1720 intel_gpu_freq(rps, rps->max_freq), rps->max_freq);
1721
1722 rps->efficient_freq = chv_rps_rpe_freq(rps);
1723 drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n",
1724 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq);
1725
1726 rps->rp1_freq = chv_rps_guar_freq(rps);
1727 drm_dbg(&i915->drm, "RP1(Guar) GPU freq: %d MHz (%u)\n",
1728 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq);
1729
1730 rps->min_freq = chv_rps_min_freq(rps);
1731 drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n",
1732 intel_gpu_freq(rps, rps->min_freq), rps->min_freq);
1733
1734 vlv_iosf_sb_put(i915,
1735 BIT(VLV_IOSF_SB_PUNIT) |
1736 BIT(VLV_IOSF_SB_NC) |
1737 BIT(VLV_IOSF_SB_CCK));
1738
1739 drm_WARN_ONCE(&i915->drm, (rps->max_freq | rps->efficient_freq |
1740 rps->rp1_freq | rps->min_freq) & 1,
1741 "Odd GPU freq values\n");
1742 }
1743
vlv_c0_read(struct intel_uncore * uncore,struct intel_rps_ei * ei)1744 static void vlv_c0_read(struct intel_uncore *uncore, struct intel_rps_ei *ei)
1745 {
1746 ei->ktime = ktime_get_raw();
1747 ei->render_c0 = intel_uncore_read(uncore, VLV_RENDER_C0_COUNT);
1748 ei->media_c0 = intel_uncore_read(uncore, VLV_MEDIA_C0_COUNT);
1749 }
1750
vlv_wa_c0_ei(struct intel_rps * rps,u32 pm_iir)1751 static u32 vlv_wa_c0_ei(struct intel_rps *rps, u32 pm_iir)
1752 {
1753 struct intel_uncore *uncore = rps_to_uncore(rps);
1754 const struct intel_rps_ei *prev = &rps->ei;
1755 struct intel_rps_ei now;
1756 u32 events = 0;
1757
1758 if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0)
1759 return 0;
1760
1761 vlv_c0_read(uncore, &now);
1762
1763 if (prev->ktime) {
1764 u64 time, c0;
1765 u32 render, media;
1766
1767 time = ktime_us_delta(now.ktime, prev->ktime);
1768
1769 time *= rps_to_i915(rps)->czclk_freq;
1770
1771 /* Workload can be split between render + media,
1772 * e.g. SwapBuffers being blitted in X after being rendered in
1773 * mesa. To account for this we need to combine both engines
1774 * into our activity counter.
1775 */
1776 render = now.render_c0 - prev->render_c0;
1777 media = now.media_c0 - prev->media_c0;
1778 c0 = max(render, media);
1779 c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */
1780
1781 if (c0 > time * rps->power.up_threshold)
1782 events = GEN6_PM_RP_UP_THRESHOLD;
1783 else if (c0 < time * rps->power.down_threshold)
1784 events = GEN6_PM_RP_DOWN_THRESHOLD;
1785 }
1786
1787 rps->ei = now;
1788 return events;
1789 }
1790
rps_work(struct work_struct * work)1791 static void rps_work(struct work_struct *work)
1792 {
1793 struct intel_rps *rps = container_of(work, typeof(*rps), work);
1794 struct intel_gt *gt = rps_to_gt(rps);
1795 struct drm_i915_private *i915 = rps_to_i915(rps);
1796 bool client_boost = false;
1797 int new_freq, adj, min, max;
1798 u32 pm_iir = 0;
1799
1800 spin_lock_irq(gt->irq_lock);
1801 pm_iir = fetch_and_zero(&rps->pm_iir) & rps->pm_events;
1802 client_boost = atomic_read(&rps->num_waiters);
1803 spin_unlock_irq(gt->irq_lock);
1804
1805 /* Make sure we didn't queue anything we're not going to process. */
1806 if (!pm_iir && !client_boost)
1807 goto out;
1808
1809 mutex_lock(&rps->lock);
1810 if (!intel_rps_is_active(rps)) {
1811 mutex_unlock(&rps->lock);
1812 return;
1813 }
1814
1815 pm_iir |= vlv_wa_c0_ei(rps, pm_iir);
1816
1817 adj = rps->last_adj;
1818 new_freq = rps->cur_freq;
1819 min = rps->min_freq_softlimit;
1820 max = rps->max_freq_softlimit;
1821 if (client_boost)
1822 max = rps->max_freq;
1823
1824 GT_TRACE(gt,
1825 "pm_iir:%x, client_boost:%s, last:%d, cur:%x, min:%x, max:%x\n",
1826 pm_iir, str_yes_no(client_boost),
1827 adj, new_freq, min, max);
1828
1829 if (client_boost && new_freq < rps->boost_freq) {
1830 new_freq = rps->boost_freq;
1831 adj = 0;
1832 } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
1833 if (adj > 0)
1834 adj *= 2;
1835 else /* CHV needs even encode values */
1836 adj = IS_CHERRYVIEW(gt->i915) ? 2 : 1;
1837
1838 if (new_freq >= rps->max_freq_softlimit)
1839 adj = 0;
1840 } else if (client_boost) {
1841 adj = 0;
1842 } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
1843 if (rps->cur_freq > rps->efficient_freq)
1844 new_freq = rps->efficient_freq;
1845 else if (rps->cur_freq > rps->min_freq_softlimit)
1846 new_freq = rps->min_freq_softlimit;
1847 adj = 0;
1848 } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
1849 if (adj < 0)
1850 adj *= 2;
1851 else /* CHV needs even encode values */
1852 adj = IS_CHERRYVIEW(gt->i915) ? -2 : -1;
1853
1854 if (new_freq <= rps->min_freq_softlimit)
1855 adj = 0;
1856 } else { /* unknown event */
1857 adj = 0;
1858 }
1859
1860 /*
1861 * sysfs frequency limits may have snuck in while
1862 * servicing the interrupt
1863 */
1864 new_freq += adj;
1865 new_freq = clamp_t(int, new_freq, min, max);
1866
1867 if (intel_rps_set(rps, new_freq)) {
1868 drm_dbg(&i915->drm, "Failed to set new GPU frequency\n");
1869 adj = 0;
1870 }
1871 rps->last_adj = adj;
1872
1873 mutex_unlock(&rps->lock);
1874
1875 out:
1876 spin_lock_irq(gt->irq_lock);
1877 gen6_gt_pm_unmask_irq(gt, rps->pm_events);
1878 spin_unlock_irq(gt->irq_lock);
1879 }
1880
gen11_rps_irq_handler(struct intel_rps * rps,u32 pm_iir)1881 void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
1882 {
1883 struct intel_gt *gt = rps_to_gt(rps);
1884 const u32 events = rps->pm_events & pm_iir;
1885
1886 lockdep_assert_held(gt->irq_lock);
1887
1888 if (unlikely(!events))
1889 return;
1890
1891 GT_TRACE(gt, "irq events:%x\n", events);
1892
1893 gen6_gt_pm_mask_irq(gt, events);
1894
1895 rps->pm_iir |= events;
1896 schedule_work(&rps->work);
1897 }
1898
gen6_rps_irq_handler(struct intel_rps * rps,u32 pm_iir)1899 void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
1900 {
1901 struct intel_gt *gt = rps_to_gt(rps);
1902 u32 events;
1903
1904 events = pm_iir & rps->pm_events;
1905 if (events) {
1906 spin_lock(gt->irq_lock);
1907
1908 GT_TRACE(gt, "irq events:%x\n", events);
1909
1910 gen6_gt_pm_mask_irq(gt, events);
1911 rps->pm_iir |= events;
1912
1913 schedule_work(&rps->work);
1914 spin_unlock(gt->irq_lock);
1915 }
1916
1917 if (GRAPHICS_VER(gt->i915) >= 8)
1918 return;
1919
1920 if (pm_iir & PM_VEBOX_USER_INTERRUPT)
1921 intel_engine_cs_irq(gt->engine[VECS0], pm_iir >> 10);
1922
1923 if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
1924 DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
1925 }
1926
gen5_rps_irq_handler(struct intel_rps * rps)1927 void gen5_rps_irq_handler(struct intel_rps *rps)
1928 {
1929 struct intel_uncore *uncore = rps_to_uncore(rps);
1930 u32 busy_up, busy_down, max_avg, min_avg;
1931 u8 new_freq;
1932
1933 spin_lock(&mchdev_lock);
1934
1935 intel_uncore_write16(uncore,
1936 MEMINTRSTS,
1937 intel_uncore_read(uncore, MEMINTRSTS));
1938
1939 intel_uncore_write16(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
1940 busy_up = intel_uncore_read(uncore, RCPREVBSYTUPAVG);
1941 busy_down = intel_uncore_read(uncore, RCPREVBSYTDNAVG);
1942 max_avg = intel_uncore_read(uncore, RCBMAXAVG);
1943 min_avg = intel_uncore_read(uncore, RCBMINAVG);
1944
1945 /* Handle RCS change request from hw */
1946 new_freq = rps->cur_freq;
1947 if (busy_up > max_avg)
1948 new_freq++;
1949 else if (busy_down < min_avg)
1950 new_freq--;
1951 new_freq = clamp(new_freq,
1952 rps->min_freq_softlimit,
1953 rps->max_freq_softlimit);
1954
1955 if (new_freq != rps->cur_freq && !__gen5_rps_set(rps, new_freq))
1956 rps->cur_freq = new_freq;
1957
1958 spin_unlock(&mchdev_lock);
1959 }
1960
intel_rps_init_early(struct intel_rps * rps)1961 void intel_rps_init_early(struct intel_rps *rps)
1962 {
1963 mutex_init(&rps->lock);
1964 mutex_init(&rps->power.mutex);
1965
1966 INIT_WORK(&rps->work, rps_work);
1967 timer_setup(&rps->timer, rps_timer, 0);
1968
1969 atomic_set(&rps->num_waiters, 0);
1970 }
1971
intel_rps_init(struct intel_rps * rps)1972 void intel_rps_init(struct intel_rps *rps)
1973 {
1974 struct drm_i915_private *i915 = rps_to_i915(rps);
1975
1976 if (rps_uses_slpc(rps))
1977 return;
1978
1979 if (IS_CHERRYVIEW(i915))
1980 chv_rps_init(rps);
1981 else if (IS_VALLEYVIEW(i915))
1982 vlv_rps_init(rps);
1983 else if (GRAPHICS_VER(i915) >= 6)
1984 gen6_rps_init(rps);
1985 else if (IS_IRONLAKE_M(i915))
1986 gen5_rps_init(rps);
1987
1988 /* Derive initial user preferences/limits from the hardware limits */
1989 rps->max_freq_softlimit = rps->max_freq;
1990 rps_to_gt(rps)->defaults.max_freq = rps->max_freq_softlimit;
1991 rps->min_freq_softlimit = rps->min_freq;
1992 rps_to_gt(rps)->defaults.min_freq = rps->min_freq_softlimit;
1993
1994 /* After setting max-softlimit, find the overclock max freq */
1995 if (GRAPHICS_VER(i915) == 6 || IS_IVYBRIDGE(i915) || IS_HASWELL(i915)) {
1996 u32 params = 0;
1997
1998 snb_pcode_read(rps_to_gt(rps)->uncore, GEN6_READ_OC_PARAMS, ¶ms, NULL);
1999 if (params & BIT(31)) { /* OC supported */
2000 drm_dbg(&i915->drm,
2001 "Overclocking supported, max: %dMHz, overclock: %dMHz\n",
2002 (rps->max_freq & 0xff) * 50,
2003 (params & 0xff) * 50);
2004 rps->max_freq = params & 0xff;
2005 }
2006 }
2007
2008 /* Finally allow us to boost to max by default */
2009 rps->boost_freq = rps->max_freq;
2010 rps->idle_freq = rps->min_freq;
2011
2012 /* Start in the middle, from here we will autotune based on workload */
2013 rps->cur_freq = rps->efficient_freq;
2014
2015 rps->pm_intrmsk_mbz = 0;
2016
2017 /*
2018 * SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer
2019 * if GEN6_PM_UP_EI_EXPIRED is masked.
2020 *
2021 * TODO: verify if this can be reproduced on VLV,CHV.
2022 */
2023 if (GRAPHICS_VER(i915) <= 7)
2024 rps->pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED;
2025
2026 if (GRAPHICS_VER(i915) >= 8 && GRAPHICS_VER(i915) < 11)
2027 rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC;
2028
2029 /* GuC needs ARAT expired interrupt unmasked */
2030 if (intel_uc_uses_guc_submission(&rps_to_gt(rps)->uc))
2031 rps->pm_intrmsk_mbz |= ARAT_EXPIRED_INTRMSK;
2032 }
2033
intel_rps_sanitize(struct intel_rps * rps)2034 void intel_rps_sanitize(struct intel_rps *rps)
2035 {
2036 if (rps_uses_slpc(rps))
2037 return;
2038
2039 if (GRAPHICS_VER(rps_to_i915(rps)) >= 6)
2040 rps_disable_interrupts(rps);
2041 }
2042
intel_rps_get_cagf(struct intel_rps * rps,u32 rpstat)2043 u32 intel_rps_get_cagf(struct intel_rps *rps, u32 rpstat)
2044 {
2045 struct drm_i915_private *i915 = rps_to_i915(rps);
2046 u32 cagf;
2047
2048 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
2049 cagf = (rpstat >> 8) & 0xff;
2050 else if (GRAPHICS_VER(i915) >= 9)
2051 cagf = (rpstat & GEN9_CAGF_MASK) >> GEN9_CAGF_SHIFT;
2052 else if (IS_HASWELL(i915) || IS_BROADWELL(i915))
2053 cagf = (rpstat & HSW_CAGF_MASK) >> HSW_CAGF_SHIFT;
2054 else if (GRAPHICS_VER(i915) >= 6)
2055 cagf = (rpstat & GEN6_CAGF_MASK) >> GEN6_CAGF_SHIFT;
2056 else
2057 cagf = gen5_invert_freq(rps, (rpstat & MEMSTAT_PSTATE_MASK) >>
2058 MEMSTAT_PSTATE_SHIFT);
2059
2060 return cagf;
2061 }
2062
read_cagf(struct intel_rps * rps)2063 static u32 read_cagf(struct intel_rps *rps)
2064 {
2065 struct drm_i915_private *i915 = rps_to_i915(rps);
2066 struct intel_uncore *uncore = rps_to_uncore(rps);
2067 u32 freq;
2068
2069 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) {
2070 vlv_punit_get(i915);
2071 freq = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
2072 vlv_punit_put(i915);
2073 } else if (GRAPHICS_VER(i915) >= 6) {
2074 freq = intel_uncore_read(uncore, GEN6_RPSTAT1);
2075 } else {
2076 freq = intel_uncore_read(uncore, MEMSTAT_ILK);
2077 }
2078
2079 return intel_rps_get_cagf(rps, freq);
2080 }
2081
intel_rps_read_actual_frequency(struct intel_rps * rps)2082 u32 intel_rps_read_actual_frequency(struct intel_rps *rps)
2083 {
2084 struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm;
2085 intel_wakeref_t wakeref;
2086 u32 freq = 0;
2087
2088 with_intel_runtime_pm_if_in_use(rpm, wakeref)
2089 freq = intel_gpu_freq(rps, read_cagf(rps));
2090
2091 return freq;
2092 }
2093
intel_rps_read_punit_req(struct intel_rps * rps)2094 u32 intel_rps_read_punit_req(struct intel_rps *rps)
2095 {
2096 struct intel_uncore *uncore = rps_to_uncore(rps);
2097 struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm;
2098 intel_wakeref_t wakeref;
2099 u32 freq = 0;
2100
2101 with_intel_runtime_pm_if_in_use(rpm, wakeref)
2102 freq = intel_uncore_read(uncore, GEN6_RPNSWREQ);
2103
2104 return freq;
2105 }
2106
intel_rps_get_req(u32 pureq)2107 static u32 intel_rps_get_req(u32 pureq)
2108 {
2109 u32 req = pureq >> GEN9_SW_REQ_UNSLICE_RATIO_SHIFT;
2110
2111 return req;
2112 }
2113
intel_rps_read_punit_req_frequency(struct intel_rps * rps)2114 u32 intel_rps_read_punit_req_frequency(struct intel_rps *rps)
2115 {
2116 u32 freq = intel_rps_get_req(intel_rps_read_punit_req(rps));
2117
2118 return intel_gpu_freq(rps, freq);
2119 }
2120
intel_rps_get_requested_frequency(struct intel_rps * rps)2121 u32 intel_rps_get_requested_frequency(struct intel_rps *rps)
2122 {
2123 if (rps_uses_slpc(rps))
2124 return intel_rps_read_punit_req_frequency(rps);
2125 else
2126 return intel_gpu_freq(rps, rps->cur_freq);
2127 }
2128
intel_rps_get_max_frequency(struct intel_rps * rps)2129 u32 intel_rps_get_max_frequency(struct intel_rps *rps)
2130 {
2131 struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2132
2133 if (rps_uses_slpc(rps))
2134 return slpc->max_freq_softlimit;
2135 else
2136 return intel_gpu_freq(rps, rps->max_freq_softlimit);
2137 }
2138
2139 /**
2140 * intel_rps_get_max_raw_freq - returns the max frequency in some raw format.
2141 * @rps: the intel_rps structure
2142 *
2143 * Returns the max frequency in a raw format. In newer platforms raw is in
2144 * units of 50 MHz.
2145 */
intel_rps_get_max_raw_freq(struct intel_rps * rps)2146 u32 intel_rps_get_max_raw_freq(struct intel_rps *rps)
2147 {
2148 struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2149 u32 freq;
2150
2151 if (rps_uses_slpc(rps)) {
2152 return DIV_ROUND_CLOSEST(slpc->rp0_freq,
2153 GT_FREQUENCY_MULTIPLIER);
2154 } else {
2155 freq = rps->max_freq;
2156 if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) {
2157 /* Convert GT frequency to 50 MHz units */
2158 freq /= GEN9_FREQ_SCALER;
2159 }
2160 return freq;
2161 }
2162 }
2163
intel_rps_get_rp0_frequency(struct intel_rps * rps)2164 u32 intel_rps_get_rp0_frequency(struct intel_rps *rps)
2165 {
2166 struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2167
2168 if (rps_uses_slpc(rps))
2169 return slpc->rp0_freq;
2170 else
2171 return intel_gpu_freq(rps, rps->rp0_freq);
2172 }
2173
intel_rps_get_rp1_frequency(struct intel_rps * rps)2174 u32 intel_rps_get_rp1_frequency(struct intel_rps *rps)
2175 {
2176 struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2177
2178 if (rps_uses_slpc(rps))
2179 return slpc->rp1_freq;
2180 else
2181 return intel_gpu_freq(rps, rps->rp1_freq);
2182 }
2183
intel_rps_get_rpn_frequency(struct intel_rps * rps)2184 u32 intel_rps_get_rpn_frequency(struct intel_rps *rps)
2185 {
2186 struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2187
2188 if (rps_uses_slpc(rps))
2189 return slpc->min_freq;
2190 else
2191 return intel_gpu_freq(rps, rps->min_freq);
2192 }
2193
set_max_freq(struct intel_rps * rps,u32 val)2194 static int set_max_freq(struct intel_rps *rps, u32 val)
2195 {
2196 struct drm_i915_private *i915 = rps_to_i915(rps);
2197 int ret = 0;
2198
2199 mutex_lock(&rps->lock);
2200
2201 val = intel_freq_opcode(rps, val);
2202 if (val < rps->min_freq ||
2203 val > rps->max_freq ||
2204 val < rps->min_freq_softlimit) {
2205 ret = -EINVAL;
2206 goto unlock;
2207 }
2208
2209 if (val > rps->rp0_freq)
2210 drm_dbg(&i915->drm, "User requested overclocking to %d\n",
2211 intel_gpu_freq(rps, val));
2212
2213 rps->max_freq_softlimit = val;
2214
2215 val = clamp_t(int, rps->cur_freq,
2216 rps->min_freq_softlimit,
2217 rps->max_freq_softlimit);
2218
2219 /*
2220 * We still need *_set_rps to process the new max_delay and
2221 * update the interrupt limits and PMINTRMSK even though
2222 * frequency request may be unchanged.
2223 */
2224 intel_rps_set(rps, val);
2225
2226 unlock:
2227 mutex_unlock(&rps->lock);
2228
2229 return ret;
2230 }
2231
intel_rps_set_max_frequency(struct intel_rps * rps,u32 val)2232 int intel_rps_set_max_frequency(struct intel_rps *rps, u32 val)
2233 {
2234 struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2235
2236 if (rps_uses_slpc(rps))
2237 return intel_guc_slpc_set_max_freq(slpc, val);
2238 else
2239 return set_max_freq(rps, val);
2240 }
2241
intel_rps_get_min_frequency(struct intel_rps * rps)2242 u32 intel_rps_get_min_frequency(struct intel_rps *rps)
2243 {
2244 struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2245
2246 if (rps_uses_slpc(rps))
2247 return slpc->min_freq_softlimit;
2248 else
2249 return intel_gpu_freq(rps, rps->min_freq_softlimit);
2250 }
2251
2252 /**
2253 * intel_rps_get_min_raw_freq - returns the min frequency in some raw format.
2254 * @rps: the intel_rps structure
2255 *
2256 * Returns the min frequency in a raw format. In newer platforms raw is in
2257 * units of 50 MHz.
2258 */
intel_rps_get_min_raw_freq(struct intel_rps * rps)2259 u32 intel_rps_get_min_raw_freq(struct intel_rps *rps)
2260 {
2261 struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2262 u32 freq;
2263
2264 if (rps_uses_slpc(rps)) {
2265 return DIV_ROUND_CLOSEST(slpc->min_freq,
2266 GT_FREQUENCY_MULTIPLIER);
2267 } else {
2268 freq = rps->min_freq;
2269 if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) {
2270 /* Convert GT frequency to 50 MHz units */
2271 freq /= GEN9_FREQ_SCALER;
2272 }
2273 return freq;
2274 }
2275 }
2276
set_min_freq(struct intel_rps * rps,u32 val)2277 static int set_min_freq(struct intel_rps *rps, u32 val)
2278 {
2279 int ret = 0;
2280
2281 mutex_lock(&rps->lock);
2282
2283 val = intel_freq_opcode(rps, val);
2284 if (val < rps->min_freq ||
2285 val > rps->max_freq ||
2286 val > rps->max_freq_softlimit) {
2287 ret = -EINVAL;
2288 goto unlock;
2289 }
2290
2291 rps->min_freq_softlimit = val;
2292
2293 val = clamp_t(int, rps->cur_freq,
2294 rps->min_freq_softlimit,
2295 rps->max_freq_softlimit);
2296
2297 /*
2298 * We still need *_set_rps to process the new min_delay and
2299 * update the interrupt limits and PMINTRMSK even though
2300 * frequency request may be unchanged.
2301 */
2302 intel_rps_set(rps, val);
2303
2304 unlock:
2305 mutex_unlock(&rps->lock);
2306
2307 return ret;
2308 }
2309
intel_rps_set_min_frequency(struct intel_rps * rps,u32 val)2310 int intel_rps_set_min_frequency(struct intel_rps *rps, u32 val)
2311 {
2312 struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2313
2314 if (rps_uses_slpc(rps))
2315 return intel_guc_slpc_set_min_freq(slpc, val);
2316 else
2317 return set_min_freq(rps, val);
2318 }
2319
intel_rps_set_manual(struct intel_rps * rps,bool enable)2320 static void intel_rps_set_manual(struct intel_rps *rps, bool enable)
2321 {
2322 struct intel_uncore *uncore = rps_to_uncore(rps);
2323 u32 state = enable ? GEN9_RPSWCTL_ENABLE : GEN9_RPSWCTL_DISABLE;
2324
2325 /* Allow punit to process software requests */
2326 intel_uncore_write(uncore, GEN6_RP_CONTROL, state);
2327 }
2328
intel_rps_raise_unslice(struct intel_rps * rps)2329 void intel_rps_raise_unslice(struct intel_rps *rps)
2330 {
2331 struct intel_uncore *uncore = rps_to_uncore(rps);
2332
2333 mutex_lock(&rps->lock);
2334
2335 if (rps_uses_slpc(rps)) {
2336 /* RP limits have not been initialized yet for SLPC path */
2337 struct intel_rps_freq_caps caps;
2338
2339 gen6_rps_get_freq_caps(rps, &caps);
2340
2341 intel_rps_set_manual(rps, true);
2342 intel_uncore_write(uncore, GEN6_RPNSWREQ,
2343 ((caps.rp0_freq <<
2344 GEN9_SW_REQ_UNSLICE_RATIO_SHIFT) |
2345 GEN9_IGNORE_SLICE_RATIO));
2346 intel_rps_set_manual(rps, false);
2347 } else {
2348 intel_rps_set(rps, rps->rp0_freq);
2349 }
2350
2351 mutex_unlock(&rps->lock);
2352 }
2353
intel_rps_lower_unslice(struct intel_rps * rps)2354 void intel_rps_lower_unslice(struct intel_rps *rps)
2355 {
2356 struct intel_uncore *uncore = rps_to_uncore(rps);
2357
2358 mutex_lock(&rps->lock);
2359
2360 if (rps_uses_slpc(rps)) {
2361 /* RP limits have not been initialized yet for SLPC path */
2362 struct intel_rps_freq_caps caps;
2363
2364 gen6_rps_get_freq_caps(rps, &caps);
2365
2366 intel_rps_set_manual(rps, true);
2367 intel_uncore_write(uncore, GEN6_RPNSWREQ,
2368 ((caps.min_freq <<
2369 GEN9_SW_REQ_UNSLICE_RATIO_SHIFT) |
2370 GEN9_IGNORE_SLICE_RATIO));
2371 intel_rps_set_manual(rps, false);
2372 } else {
2373 intel_rps_set(rps, rps->min_freq);
2374 }
2375
2376 mutex_unlock(&rps->lock);
2377 }
2378
rps_read_mmio(struct intel_rps * rps,i915_reg_t reg32)2379 static u32 rps_read_mmio(struct intel_rps *rps, i915_reg_t reg32)
2380 {
2381 struct intel_gt *gt = rps_to_gt(rps);
2382 intel_wakeref_t wakeref;
2383 u32 val;
2384
2385 with_intel_runtime_pm(gt->uncore->rpm, wakeref)
2386 val = intel_uncore_read(gt->uncore, reg32);
2387
2388 return val;
2389 }
2390
rps_read_mask_mmio(struct intel_rps * rps,i915_reg_t reg32,u32 mask)2391 bool rps_read_mask_mmio(struct intel_rps *rps,
2392 i915_reg_t reg32, u32 mask)
2393 {
2394 return rps_read_mmio(rps, reg32) & mask;
2395 }
2396
2397 /* External interface for intel_ips.ko */
2398
2399 static struct drm_i915_private __rcu *ips_mchdev;
2400
2401 /**
2402 * Tells the intel_ips driver that the i915 driver is now loaded, if
2403 * IPS got loaded first.
2404 *
2405 * This awkward dance is so that neither module has to depend on the
2406 * other in order for IPS to do the appropriate communication of
2407 * GPU turbo limits to i915.
2408 */
2409 static void
ips_ping_for_i915_load(void)2410 ips_ping_for_i915_load(void)
2411 {
2412 void (*link)(void);
2413
2414 link = symbol_get(ips_link_to_i915_driver);
2415 if (link) {
2416 link();
2417 symbol_put(ips_link_to_i915_driver);
2418 }
2419 }
2420
intel_rps_driver_register(struct intel_rps * rps)2421 void intel_rps_driver_register(struct intel_rps *rps)
2422 {
2423 struct intel_gt *gt = rps_to_gt(rps);
2424
2425 /*
2426 * We only register the i915 ips part with intel-ips once everything is
2427 * set up, to avoid intel-ips sneaking in and reading bogus values.
2428 */
2429 if (GRAPHICS_VER(gt->i915) == 5) {
2430 GEM_BUG_ON(ips_mchdev);
2431 rcu_assign_pointer(ips_mchdev, gt->i915);
2432 ips_ping_for_i915_load();
2433 }
2434 }
2435
intel_rps_driver_unregister(struct intel_rps * rps)2436 void intel_rps_driver_unregister(struct intel_rps *rps)
2437 {
2438 if (rcu_access_pointer(ips_mchdev) == rps_to_i915(rps))
2439 rcu_assign_pointer(ips_mchdev, NULL);
2440 }
2441
mchdev_get(void)2442 static struct drm_i915_private *mchdev_get(void)
2443 {
2444 struct drm_i915_private *i915;
2445
2446 rcu_read_lock();
2447 i915 = rcu_dereference(ips_mchdev);
2448 if (i915 && !kref_get_unless_zero(&i915->drm.ref))
2449 i915 = NULL;
2450 rcu_read_unlock();
2451
2452 return i915;
2453 }
2454
2455 /**
2456 * i915_read_mch_val - return value for IPS use
2457 *
2458 * Calculate and return a value for the IPS driver to use when deciding whether
2459 * we have thermal and power headroom to increase CPU or GPU power budget.
2460 */
i915_read_mch_val(void)2461 unsigned long i915_read_mch_val(void)
2462 {
2463 struct drm_i915_private *i915;
2464 unsigned long chipset_val = 0;
2465 unsigned long graphics_val = 0;
2466 intel_wakeref_t wakeref;
2467
2468 i915 = mchdev_get();
2469 if (!i915)
2470 return 0;
2471
2472 with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
2473 struct intel_ips *ips = &to_gt(i915)->rps.ips;
2474
2475 spin_lock_irq(&mchdev_lock);
2476 chipset_val = __ips_chipset_val(ips);
2477 graphics_val = __ips_gfx_val(ips);
2478 spin_unlock_irq(&mchdev_lock);
2479 }
2480
2481 drm_dev_put(&i915->drm);
2482 return chipset_val + graphics_val;
2483 }
2484 EXPORT_SYMBOL_GPL(i915_read_mch_val);
2485
2486 /**
2487 * i915_gpu_raise - raise GPU frequency limit
2488 *
2489 * Raise the limit; IPS indicates we have thermal headroom.
2490 */
i915_gpu_raise(void)2491 bool i915_gpu_raise(void)
2492 {
2493 struct drm_i915_private *i915;
2494 struct intel_rps *rps;
2495
2496 i915 = mchdev_get();
2497 if (!i915)
2498 return false;
2499
2500 rps = &to_gt(i915)->rps;
2501
2502 spin_lock_irq(&mchdev_lock);
2503 if (rps->max_freq_softlimit < rps->max_freq)
2504 rps->max_freq_softlimit++;
2505 spin_unlock_irq(&mchdev_lock);
2506
2507 drm_dev_put(&i915->drm);
2508 return true;
2509 }
2510 EXPORT_SYMBOL_GPL(i915_gpu_raise);
2511
2512 /**
2513 * i915_gpu_lower - lower GPU frequency limit
2514 *
2515 * IPS indicates we're close to a thermal limit, so throttle back the GPU
2516 * frequency maximum.
2517 */
i915_gpu_lower(void)2518 bool i915_gpu_lower(void)
2519 {
2520 struct drm_i915_private *i915;
2521 struct intel_rps *rps;
2522
2523 i915 = mchdev_get();
2524 if (!i915)
2525 return false;
2526
2527 rps = &to_gt(i915)->rps;
2528
2529 spin_lock_irq(&mchdev_lock);
2530 if (rps->max_freq_softlimit > rps->min_freq)
2531 rps->max_freq_softlimit--;
2532 spin_unlock_irq(&mchdev_lock);
2533
2534 drm_dev_put(&i915->drm);
2535 return true;
2536 }
2537 EXPORT_SYMBOL_GPL(i915_gpu_lower);
2538
2539 /**
2540 * i915_gpu_busy - indicate GPU business to IPS
2541 *
2542 * Tell the IPS driver whether or not the GPU is busy.
2543 */
i915_gpu_busy(void)2544 bool i915_gpu_busy(void)
2545 {
2546 struct drm_i915_private *i915;
2547 bool ret;
2548
2549 i915 = mchdev_get();
2550 if (!i915)
2551 return false;
2552
2553 ret = to_gt(i915)->awake;
2554
2555 drm_dev_put(&i915->drm);
2556 return ret;
2557 }
2558 EXPORT_SYMBOL_GPL(i915_gpu_busy);
2559
2560 /**
2561 * i915_gpu_turbo_disable - disable graphics turbo
2562 *
2563 * Disable graphics turbo by resetting the max frequency and setting the
2564 * current frequency to the default.
2565 */
i915_gpu_turbo_disable(void)2566 bool i915_gpu_turbo_disable(void)
2567 {
2568 struct drm_i915_private *i915;
2569 struct intel_rps *rps;
2570 bool ret;
2571
2572 i915 = mchdev_get();
2573 if (!i915)
2574 return false;
2575
2576 rps = &to_gt(i915)->rps;
2577
2578 spin_lock_irq(&mchdev_lock);
2579 rps->max_freq_softlimit = rps->min_freq;
2580 ret = !__gen5_rps_set(&to_gt(i915)->rps, rps->min_freq);
2581 spin_unlock_irq(&mchdev_lock);
2582
2583 drm_dev_put(&i915->drm);
2584 return ret;
2585 }
2586 EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
2587
2588 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
2589 #include "selftest_rps.c"
2590 #include "selftest_slpc.c"
2591 #endif
2592