1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2016-2018, The Linux Foundation. All rights reserved.
3 */
4
5 #define pr_fmt(fmt) "[drm:%s:%d] " fmt, __func__, __LINE__
6
7 #include <linux/debugfs.h>
8 #include <linux/errno.h>
9 #include <linux/mutex.h>
10 #include <linux/pm_opp.h>
11 #include <linux/sort.h>
12 #include <linux/clk.h>
13 #include <linux/bitmap.h>
14
15 #include "dpu_kms.h"
16 #include "dpu_trace.h"
17 #include "dpu_crtc.h"
18 #include "dpu_core_perf.h"
19
20 /**
21 * enum dpu_perf_mode - performance tuning mode
22 * @DPU_PERF_MODE_NORMAL: performance controlled by user mode client
23 * @DPU_PERF_MODE_MINIMUM: performance bounded by minimum setting
24 * @DPU_PERF_MODE_FIXED: performance bounded by fixed setting
25 * @DPU_PERF_MODE_MAX: maximum value, used for error checking
26 */
27 enum dpu_perf_mode {
28 DPU_PERF_MODE_NORMAL,
29 DPU_PERF_MODE_MINIMUM,
30 DPU_PERF_MODE_FIXED,
31 DPU_PERF_MODE_MAX
32 };
33
34 /**
35 * _dpu_core_perf_calc_bw() - to calculate BW per crtc
36 * @kms: pointer to the dpu_kms
37 * @crtc: pointer to a crtc
38 * Return: returns aggregated BW for all planes in crtc.
39 */
_dpu_core_perf_calc_bw(struct dpu_kms * kms,struct drm_crtc * crtc)40 static u64 _dpu_core_perf_calc_bw(struct dpu_kms *kms,
41 struct drm_crtc *crtc)
42 {
43 struct drm_plane *plane;
44 struct dpu_plane_state *pstate;
45 u64 crtc_plane_bw = 0;
46 u32 bw_factor;
47
48 drm_atomic_crtc_for_each_plane(plane, crtc) {
49 pstate = to_dpu_plane_state(plane->state);
50 if (!pstate)
51 continue;
52
53 crtc_plane_bw += pstate->plane_fetch_bw;
54 }
55
56 bw_factor = kms->catalog->perf->bw_inefficiency_factor;
57 if (bw_factor) {
58 crtc_plane_bw *= bw_factor;
59 do_div(crtc_plane_bw, 100);
60 }
61
62 return crtc_plane_bw;
63 }
64
65 /**
66 * _dpu_core_perf_calc_clk() - to calculate clock per crtc
67 * @kms: pointer to the dpu_kms
68 * @crtc: pointer to a crtc
69 * @state: pointer to a crtc state
70 * Return: returns max clk for all planes in crtc.
71 */
_dpu_core_perf_calc_clk(struct dpu_kms * kms,struct drm_crtc * crtc,struct drm_crtc_state * state)72 static u64 _dpu_core_perf_calc_clk(struct dpu_kms *kms,
73 struct drm_crtc *crtc, struct drm_crtc_state *state)
74 {
75 struct drm_plane *plane;
76 struct dpu_plane_state *pstate;
77 struct drm_display_mode *mode;
78 u64 crtc_clk;
79 u32 clk_factor;
80
81 mode = &state->adjusted_mode;
82
83 crtc_clk = mode->vtotal * mode->hdisplay * drm_mode_vrefresh(mode);
84
85 drm_atomic_crtc_for_each_plane(plane, crtc) {
86 pstate = to_dpu_plane_state(plane->state);
87 if (!pstate)
88 continue;
89
90 crtc_clk = max(pstate->plane_clk, crtc_clk);
91 }
92
93 clk_factor = kms->catalog->perf->clk_inefficiency_factor;
94 if (clk_factor) {
95 crtc_clk *= clk_factor;
96 do_div(crtc_clk, 100);
97 }
98
99 return crtc_clk;
100 }
101
_dpu_crtc_get_kms(struct drm_crtc * crtc)102 static struct dpu_kms *_dpu_crtc_get_kms(struct drm_crtc *crtc)
103 {
104 struct msm_drm_private *priv;
105 priv = crtc->dev->dev_private;
106 return to_dpu_kms(priv->kms);
107 }
108
_dpu_core_perf_calc_crtc(struct dpu_kms * kms,struct drm_crtc * crtc,struct drm_crtc_state * state,struct dpu_core_perf_params * perf)109 static void _dpu_core_perf_calc_crtc(struct dpu_kms *kms,
110 struct drm_crtc *crtc,
111 struct drm_crtc_state *state,
112 struct dpu_core_perf_params *perf)
113 {
114 if (!kms || !kms->catalog || !crtc || !state || !perf) {
115 DPU_ERROR("invalid parameters\n");
116 return;
117 }
118
119 memset(perf, 0, sizeof(struct dpu_core_perf_params));
120
121 if (kms->perf.perf_tune.mode == DPU_PERF_MODE_MINIMUM) {
122 perf->bw_ctl = 0;
123 perf->max_per_pipe_ib = 0;
124 perf->core_clk_rate = 0;
125 } else if (kms->perf.perf_tune.mode == DPU_PERF_MODE_FIXED) {
126 perf->bw_ctl = kms->perf.fix_core_ab_vote;
127 perf->max_per_pipe_ib = kms->perf.fix_core_ib_vote;
128 perf->core_clk_rate = kms->perf.fix_core_clk_rate;
129 } else {
130 perf->bw_ctl = _dpu_core_perf_calc_bw(kms, crtc);
131 perf->max_per_pipe_ib = kms->catalog->perf->min_dram_ib;
132 perf->core_clk_rate = _dpu_core_perf_calc_clk(kms, crtc, state);
133 }
134
135 DRM_DEBUG_ATOMIC(
136 "crtc=%d clk_rate=%llu core_ib=%llu core_ab=%llu\n",
137 crtc->base.id, perf->core_clk_rate,
138 perf->max_per_pipe_ib, perf->bw_ctl);
139 }
140
dpu_core_perf_crtc_check(struct drm_crtc * crtc,struct drm_crtc_state * state)141 int dpu_core_perf_crtc_check(struct drm_crtc *crtc,
142 struct drm_crtc_state *state)
143 {
144 u32 bw, threshold;
145 u64 bw_sum_of_intfs = 0;
146 enum dpu_crtc_client_type curr_client_type;
147 struct dpu_crtc_state *dpu_cstate;
148 struct drm_crtc *tmp_crtc;
149 struct dpu_kms *kms;
150
151 if (!crtc || !state) {
152 DPU_ERROR("invalid crtc\n");
153 return -EINVAL;
154 }
155
156 kms = _dpu_crtc_get_kms(crtc);
157 if (!kms->catalog) {
158 DPU_ERROR("invalid parameters\n");
159 return 0;
160 }
161
162 /* we only need bandwidth check on real-time clients (interfaces) */
163 if (dpu_crtc_get_client_type(crtc) == NRT_CLIENT)
164 return 0;
165
166 dpu_cstate = to_dpu_crtc_state(state);
167
168 /* obtain new values */
169 _dpu_core_perf_calc_crtc(kms, crtc, state, &dpu_cstate->new_perf);
170
171 bw_sum_of_intfs = dpu_cstate->new_perf.bw_ctl;
172 curr_client_type = dpu_crtc_get_client_type(crtc);
173
174 drm_for_each_crtc(tmp_crtc, crtc->dev) {
175 if (tmp_crtc->enabled &&
176 (dpu_crtc_get_client_type(tmp_crtc) ==
177 curr_client_type) && (tmp_crtc != crtc)) {
178 struct dpu_crtc_state *tmp_cstate =
179 to_dpu_crtc_state(tmp_crtc->state);
180
181 DRM_DEBUG_ATOMIC("crtc:%d bw:%llu ctrl:%d\n",
182 tmp_crtc->base.id, tmp_cstate->new_perf.bw_ctl,
183 tmp_cstate->bw_control);
184
185 bw_sum_of_intfs += tmp_cstate->new_perf.bw_ctl;
186 }
187
188 /* convert bandwidth to kb */
189 bw = DIV_ROUND_UP_ULL(bw_sum_of_intfs, 1000);
190 DRM_DEBUG_ATOMIC("calculated bandwidth=%uk\n", bw);
191
192 threshold = kms->catalog->perf->max_bw_high;
193
194 DRM_DEBUG_ATOMIC("final threshold bw limit = %d\n", threshold);
195
196 if (!threshold) {
197 DPU_ERROR("no bandwidth limits specified\n");
198 return -E2BIG;
199 } else if (bw > threshold) {
200 DPU_ERROR("exceeds bandwidth: %ukb > %ukb\n", bw,
201 threshold);
202 return -E2BIG;
203 }
204 }
205
206 return 0;
207 }
208
_dpu_core_perf_crtc_update_bus(struct dpu_kms * kms,struct drm_crtc * crtc)209 static int _dpu_core_perf_crtc_update_bus(struct dpu_kms *kms,
210 struct drm_crtc *crtc)
211 {
212 struct dpu_core_perf_params perf = { 0 };
213 enum dpu_crtc_client_type curr_client_type
214 = dpu_crtc_get_client_type(crtc);
215 struct drm_crtc *tmp_crtc;
216 struct dpu_crtc_state *dpu_cstate;
217 int i, ret = 0;
218 u64 avg_bw;
219
220 drm_for_each_crtc(tmp_crtc, crtc->dev) {
221 if (tmp_crtc->enabled &&
222 curr_client_type ==
223 dpu_crtc_get_client_type(tmp_crtc)) {
224 dpu_cstate = to_dpu_crtc_state(tmp_crtc->state);
225
226 perf.max_per_pipe_ib = max(perf.max_per_pipe_ib,
227 dpu_cstate->new_perf.max_per_pipe_ib);
228
229 perf.bw_ctl += dpu_cstate->new_perf.bw_ctl;
230
231 DRM_DEBUG_ATOMIC("crtc=%d bw=%llu paths:%d\n",
232 tmp_crtc->base.id,
233 dpu_cstate->new_perf.bw_ctl, kms->num_paths);
234 }
235 }
236
237 if (!kms->num_paths)
238 return 0;
239
240 avg_bw = perf.bw_ctl;
241 do_div(avg_bw, (kms->num_paths * 1000)); /*Bps_to_icc*/
242
243 for (i = 0; i < kms->num_paths; i++)
244 icc_set_bw(kms->path[i], avg_bw, perf.max_per_pipe_ib);
245
246 return ret;
247 }
248
249 /**
250 * dpu_core_perf_crtc_release_bw() - request zero bandwidth
251 * @crtc: pointer to a crtc
252 *
253 * Function checks a state variable for the crtc, if all pending commit
254 * requests are done, meaning no more bandwidth is needed, release
255 * bandwidth request.
256 */
dpu_core_perf_crtc_release_bw(struct drm_crtc * crtc)257 void dpu_core_perf_crtc_release_bw(struct drm_crtc *crtc)
258 {
259 struct dpu_crtc *dpu_crtc;
260 struct dpu_kms *kms;
261
262 if (!crtc) {
263 DPU_ERROR("invalid crtc\n");
264 return;
265 }
266
267 kms = _dpu_crtc_get_kms(crtc);
268 if (!kms->catalog) {
269 DPU_ERROR("invalid kms\n");
270 return;
271 }
272
273 dpu_crtc = to_dpu_crtc(crtc);
274
275 if (atomic_dec_return(&kms->bandwidth_ref) > 0)
276 return;
277
278 /* Release the bandwidth */
279 if (kms->perf.enable_bw_release) {
280 trace_dpu_cmd_release_bw(crtc->base.id);
281 DRM_DEBUG_ATOMIC("Release BW crtc=%d\n", crtc->base.id);
282 dpu_crtc->cur_perf.bw_ctl = 0;
283 _dpu_core_perf_crtc_update_bus(kms, crtc);
284 }
285 }
286
_dpu_core_perf_get_core_clk_rate(struct dpu_kms * kms)287 static u64 _dpu_core_perf_get_core_clk_rate(struct dpu_kms *kms)
288 {
289 u64 clk_rate = kms->perf.perf_tune.min_core_clk;
290 struct drm_crtc *crtc;
291 struct dpu_crtc_state *dpu_cstate;
292
293 drm_for_each_crtc(crtc, kms->dev) {
294 if (crtc->enabled) {
295 dpu_cstate = to_dpu_crtc_state(crtc->state);
296 clk_rate = max(dpu_cstate->new_perf.core_clk_rate,
297 clk_rate);
298 clk_rate = clk_round_rate(kms->perf.core_clk,
299 clk_rate);
300 }
301 }
302
303 if (kms->perf.perf_tune.mode == DPU_PERF_MODE_FIXED)
304 clk_rate = kms->perf.fix_core_clk_rate;
305
306 DRM_DEBUG_ATOMIC("clk:%llu\n", clk_rate);
307
308 return clk_rate;
309 }
310
dpu_core_perf_crtc_update(struct drm_crtc * crtc,int params_changed,bool stop_req)311 int dpu_core_perf_crtc_update(struct drm_crtc *crtc,
312 int params_changed, bool stop_req)
313 {
314 struct dpu_core_perf_params *new, *old;
315 bool update_bus = false, update_clk = false;
316 u64 clk_rate = 0;
317 struct dpu_crtc *dpu_crtc;
318 struct dpu_crtc_state *dpu_cstate;
319 struct dpu_kms *kms;
320 int ret;
321
322 if (!crtc) {
323 DPU_ERROR("invalid crtc\n");
324 return -EINVAL;
325 }
326
327 kms = _dpu_crtc_get_kms(crtc);
328 if (!kms->catalog) {
329 DPU_ERROR("invalid kms\n");
330 return -EINVAL;
331 }
332
333 dpu_crtc = to_dpu_crtc(crtc);
334 dpu_cstate = to_dpu_crtc_state(crtc->state);
335
336 DRM_DEBUG_ATOMIC("crtc:%d stop_req:%d core_clk:%llu\n",
337 crtc->base.id, stop_req, kms->perf.core_clk_rate);
338
339 old = &dpu_crtc->cur_perf;
340 new = &dpu_cstate->new_perf;
341
342 if (crtc->enabled && !stop_req) {
343 /*
344 * cases for bus bandwidth update.
345 * 1. new bandwidth vote - "ab or ib vote" is higher
346 * than current vote for update request.
347 * 2. new bandwidth vote - "ab or ib vote" is lower
348 * than current vote at end of commit or stop.
349 */
350 if ((params_changed && ((new->bw_ctl > old->bw_ctl) ||
351 (new->max_per_pipe_ib > old->max_per_pipe_ib))) ||
352 (!params_changed && ((new->bw_ctl < old->bw_ctl) ||
353 (new->max_per_pipe_ib < old->max_per_pipe_ib)))) {
354 DRM_DEBUG_ATOMIC("crtc=%d p=%d new_bw=%llu,old_bw=%llu\n",
355 crtc->base.id, params_changed,
356 new->bw_ctl, old->bw_ctl);
357 old->bw_ctl = new->bw_ctl;
358 old->max_per_pipe_ib = new->max_per_pipe_ib;
359 update_bus = true;
360 }
361
362 if ((params_changed &&
363 (new->core_clk_rate > old->core_clk_rate)) ||
364 (!params_changed &&
365 (new->core_clk_rate < old->core_clk_rate))) {
366 old->core_clk_rate = new->core_clk_rate;
367 update_clk = true;
368 }
369 } else {
370 DRM_DEBUG_ATOMIC("crtc=%d disable\n", crtc->base.id);
371 memset(old, 0, sizeof(*old));
372 update_bus = true;
373 update_clk = true;
374 }
375
376 trace_dpu_perf_crtc_update(crtc->base.id, new->bw_ctl,
377 new->core_clk_rate, stop_req, update_bus, update_clk);
378
379 if (update_bus) {
380 ret = _dpu_core_perf_crtc_update_bus(kms, crtc);
381 if (ret) {
382 DPU_ERROR("crtc-%d: failed to update bus bw vote\n",
383 crtc->base.id);
384 return ret;
385 }
386 }
387
388 /*
389 * Update the clock after bandwidth vote to ensure
390 * bandwidth is available before clock rate is increased.
391 */
392 if (update_clk) {
393 clk_rate = _dpu_core_perf_get_core_clk_rate(kms);
394
395 trace_dpu_core_perf_update_clk(kms->dev, stop_req, clk_rate);
396
397 clk_rate = min(clk_rate, kms->perf.max_core_clk_rate);
398 ret = dev_pm_opp_set_rate(&kms->pdev->dev, clk_rate);
399 if (ret) {
400 DPU_ERROR("failed to set core clock rate %llu\n", clk_rate);
401 return ret;
402 }
403
404 kms->perf.core_clk_rate = clk_rate;
405 DRM_DEBUG_ATOMIC("update clk rate = %lld HZ\n", clk_rate);
406 }
407 return 0;
408 }
409
410 #ifdef CONFIG_DEBUG_FS
411
_dpu_core_perf_mode_write(struct file * file,const char __user * user_buf,size_t count,loff_t * ppos)412 static ssize_t _dpu_core_perf_mode_write(struct file *file,
413 const char __user *user_buf, size_t count, loff_t *ppos)
414 {
415 struct dpu_core_perf *perf = file->private_data;
416 const struct dpu_perf_cfg *cfg = perf->catalog->perf;
417 u32 perf_mode = 0;
418 int ret;
419
420 ret = kstrtouint_from_user(user_buf, count, 0, &perf_mode);
421 if (ret)
422 return ret;
423
424 if (perf_mode >= DPU_PERF_MODE_MAX)
425 return -EINVAL;
426
427 if (perf_mode == DPU_PERF_MODE_FIXED) {
428 DRM_INFO("fix performance mode\n");
429 } else if (perf_mode == DPU_PERF_MODE_MINIMUM) {
430 /* run the driver with max clk and BW vote */
431 perf->perf_tune.min_core_clk = perf->max_core_clk_rate;
432 perf->perf_tune.min_bus_vote =
433 (u64) cfg->max_bw_high * 1000;
434 DRM_INFO("minimum performance mode\n");
435 } else if (perf_mode == DPU_PERF_MODE_NORMAL) {
436 /* reset the perf tune params to 0 */
437 perf->perf_tune.min_core_clk = 0;
438 perf->perf_tune.min_bus_vote = 0;
439 DRM_INFO("normal performance mode\n");
440 }
441 perf->perf_tune.mode = perf_mode;
442
443 return count;
444 }
445
_dpu_core_perf_mode_read(struct file * file,char __user * buff,size_t count,loff_t * ppos)446 static ssize_t _dpu_core_perf_mode_read(struct file *file,
447 char __user *buff, size_t count, loff_t *ppos)
448 {
449 struct dpu_core_perf *perf = file->private_data;
450 int len;
451 char buf[128];
452
453 len = scnprintf(buf, sizeof(buf),
454 "mode %d min_mdp_clk %llu min_bus_vote %llu\n",
455 perf->perf_tune.mode,
456 perf->perf_tune.min_core_clk,
457 perf->perf_tune.min_bus_vote);
458
459 return simple_read_from_buffer(buff, count, ppos, buf, len);
460 }
461
462 static const struct file_operations dpu_core_perf_mode_fops = {
463 .open = simple_open,
464 .read = _dpu_core_perf_mode_read,
465 .write = _dpu_core_perf_mode_write,
466 };
467
dpu_core_perf_debugfs_init(struct dpu_kms * dpu_kms,struct dentry * parent)468 int dpu_core_perf_debugfs_init(struct dpu_kms *dpu_kms, struct dentry *parent)
469 {
470 struct dpu_core_perf *perf = &dpu_kms->perf;
471 const struct dpu_mdss_cfg *catalog = perf->catalog;
472 struct dentry *entry;
473
474 entry = debugfs_create_dir("core_perf", parent);
475
476 debugfs_create_u64("max_core_clk_rate", 0600, entry,
477 &perf->max_core_clk_rate);
478 debugfs_create_u64("core_clk_rate", 0600, entry,
479 &perf->core_clk_rate);
480 debugfs_create_u32("enable_bw_release", 0600, entry,
481 (u32 *)&perf->enable_bw_release);
482 debugfs_create_u32("threshold_low", 0600, entry,
483 (u32 *)&catalog->perf->max_bw_low);
484 debugfs_create_u32("threshold_high", 0600, entry,
485 (u32 *)&catalog->perf->max_bw_high);
486 debugfs_create_u32("min_core_ib", 0600, entry,
487 (u32 *)&catalog->perf->min_core_ib);
488 debugfs_create_u32("min_llcc_ib", 0600, entry,
489 (u32 *)&catalog->perf->min_llcc_ib);
490 debugfs_create_u32("min_dram_ib", 0600, entry,
491 (u32 *)&catalog->perf->min_dram_ib);
492 debugfs_create_file("perf_mode", 0600, entry,
493 (u32 *)perf, &dpu_core_perf_mode_fops);
494 debugfs_create_u64("fix_core_clk_rate", 0600, entry,
495 &perf->fix_core_clk_rate);
496 debugfs_create_u64("fix_core_ib_vote", 0600, entry,
497 &perf->fix_core_ib_vote);
498 debugfs_create_u64("fix_core_ab_vote", 0600, entry,
499 &perf->fix_core_ab_vote);
500
501 return 0;
502 }
503 #endif
504
dpu_core_perf_destroy(struct dpu_core_perf * perf)505 void dpu_core_perf_destroy(struct dpu_core_perf *perf)
506 {
507 if (!perf) {
508 DPU_ERROR("invalid parameters\n");
509 return;
510 }
511
512 perf->max_core_clk_rate = 0;
513 perf->core_clk = NULL;
514 perf->catalog = NULL;
515 perf->dev = NULL;
516 }
517
dpu_core_perf_init(struct dpu_core_perf * perf,struct drm_device * dev,const struct dpu_mdss_cfg * catalog,struct clk * core_clk)518 int dpu_core_perf_init(struct dpu_core_perf *perf,
519 struct drm_device *dev,
520 const struct dpu_mdss_cfg *catalog,
521 struct clk *core_clk)
522 {
523 perf->dev = dev;
524 perf->catalog = catalog;
525 perf->core_clk = core_clk;
526
527 perf->max_core_clk_rate = clk_get_rate(core_clk);
528 if (!perf->max_core_clk_rate) {
529 DPU_DEBUG("optional max core clk rate, use default\n");
530 perf->max_core_clk_rate = DPU_PERF_DEFAULT_MAX_CORE_CLK_RATE;
531 }
532
533 return 0;
534 }
535