1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2016-2018, The Linux Foundation. All rights reserved.
3  */
4 
5 #define pr_fmt(fmt)	"[drm:%s:%d] " fmt, __func__, __LINE__
6 
7 #include <linux/debugfs.h>
8 #include <linux/errno.h>
9 #include <linux/mutex.h>
10 #include <linux/pm_opp.h>
11 #include <linux/sort.h>
12 #include <linux/clk.h>
13 #include <linux/bitmap.h>
14 
15 #include "dpu_kms.h"
16 #include "dpu_trace.h"
17 #include "dpu_crtc.h"
18 #include "dpu_core_perf.h"
19 
20 /**
21  * enum dpu_perf_mode - performance tuning mode
22  * @DPU_PERF_MODE_NORMAL: performance controlled by user mode client
23  * @DPU_PERF_MODE_MINIMUM: performance bounded by minimum setting
24  * @DPU_PERF_MODE_FIXED: performance bounded by fixed setting
25  * @DPU_PERF_MODE_MAX: maximum value, used for error checking
26  */
27 enum dpu_perf_mode {
28 	DPU_PERF_MODE_NORMAL,
29 	DPU_PERF_MODE_MINIMUM,
30 	DPU_PERF_MODE_FIXED,
31 	DPU_PERF_MODE_MAX
32 };
33 
34 /**
35  * _dpu_core_perf_calc_bw() - to calculate BW per crtc
36  * @perf_cfg: performance configuration
37  * @crtc: pointer to a crtc
38  * Return: returns aggregated BW for all planes in crtc.
39  */
_dpu_core_perf_calc_bw(const struct dpu_perf_cfg * perf_cfg,struct drm_crtc * crtc)40 static u64 _dpu_core_perf_calc_bw(const struct dpu_perf_cfg *perf_cfg,
41 		struct drm_crtc *crtc)
42 {
43 	struct drm_plane *plane;
44 	struct dpu_plane_state *pstate;
45 	u64 crtc_plane_bw = 0;
46 	u32 bw_factor;
47 
48 	drm_atomic_crtc_for_each_plane(plane, crtc) {
49 		pstate = to_dpu_plane_state(plane->state);
50 		if (!pstate)
51 			continue;
52 
53 		crtc_plane_bw += pstate->plane_fetch_bw;
54 	}
55 
56 	bw_factor = perf_cfg->bw_inefficiency_factor;
57 	if (bw_factor) {
58 		crtc_plane_bw *= bw_factor;
59 		do_div(crtc_plane_bw, 100);
60 	}
61 
62 	return crtc_plane_bw;
63 }
64 
65 /**
66  * _dpu_core_perf_calc_clk() - to calculate clock per crtc
67  * @perf_cfg: performance configuration
68  * @crtc: pointer to a crtc
69  * @state: pointer to a crtc state
70  * Return: returns max clk for all planes in crtc.
71  */
_dpu_core_perf_calc_clk(const struct dpu_perf_cfg * perf_cfg,struct drm_crtc * crtc,struct drm_crtc_state * state)72 static u64 _dpu_core_perf_calc_clk(const struct dpu_perf_cfg *perf_cfg,
73 		struct drm_crtc *crtc, struct drm_crtc_state *state)
74 {
75 	struct drm_plane *plane;
76 	struct dpu_plane_state *pstate;
77 	struct drm_display_mode *mode;
78 	u64 crtc_clk;
79 	u32 clk_factor;
80 
81 	mode = &state->adjusted_mode;
82 
83 	crtc_clk = mode->vtotal * mode->hdisplay * drm_mode_vrefresh(mode);
84 
85 	drm_atomic_crtc_for_each_plane(plane, crtc) {
86 		pstate = to_dpu_plane_state(plane->state);
87 		if (!pstate)
88 			continue;
89 
90 		crtc_clk = max(pstate->plane_clk, crtc_clk);
91 	}
92 
93 	clk_factor = perf_cfg->clk_inefficiency_factor;
94 	if (clk_factor) {
95 		crtc_clk *= clk_factor;
96 		do_div(crtc_clk, 100);
97 	}
98 
99 	return crtc_clk;
100 }
101 
_dpu_crtc_get_kms(struct drm_crtc * crtc)102 static struct dpu_kms *_dpu_crtc_get_kms(struct drm_crtc *crtc)
103 {
104 	struct msm_drm_private *priv;
105 	priv = crtc->dev->dev_private;
106 	return to_dpu_kms(priv->kms);
107 }
108 
_dpu_core_perf_calc_crtc(const struct dpu_core_perf * core_perf,struct drm_crtc * crtc,struct drm_crtc_state * state,struct dpu_core_perf_params * perf)109 static void _dpu_core_perf_calc_crtc(const struct dpu_core_perf *core_perf,
110 				     struct drm_crtc *crtc,
111 				     struct drm_crtc_state *state,
112 				     struct dpu_core_perf_params *perf)
113 {
114 	const struct dpu_perf_cfg *perf_cfg = core_perf->perf_cfg;
115 
116 	if (!perf_cfg || !crtc || !state || !perf) {
117 		DPU_ERROR("invalid parameters\n");
118 		return;
119 	}
120 
121 	memset(perf, 0, sizeof(struct dpu_core_perf_params));
122 
123 	if (core_perf->perf_tune.mode == DPU_PERF_MODE_MINIMUM) {
124 		perf->bw_ctl = 0;
125 		perf->max_per_pipe_ib = 0;
126 		perf->core_clk_rate = 0;
127 	} else if (core_perf->perf_tune.mode == DPU_PERF_MODE_FIXED) {
128 		perf->bw_ctl = core_perf->fix_core_ab_vote;
129 		perf->max_per_pipe_ib = core_perf->fix_core_ib_vote;
130 		perf->core_clk_rate = core_perf->fix_core_clk_rate;
131 	} else {
132 		perf->bw_ctl = _dpu_core_perf_calc_bw(perf_cfg, crtc);
133 		perf->max_per_pipe_ib = perf_cfg->min_dram_ib;
134 		perf->core_clk_rate = _dpu_core_perf_calc_clk(perf_cfg, crtc, state);
135 	}
136 
137 	DRM_DEBUG_ATOMIC(
138 		"crtc=%d clk_rate=%llu core_ib=%llu core_ab=%llu\n",
139 			crtc->base.id, perf->core_clk_rate,
140 			perf->max_per_pipe_ib, perf->bw_ctl);
141 }
142 
dpu_core_perf_crtc_check(struct drm_crtc * crtc,struct drm_crtc_state * state)143 int dpu_core_perf_crtc_check(struct drm_crtc *crtc,
144 		struct drm_crtc_state *state)
145 {
146 	u32 bw, threshold;
147 	u64 bw_sum_of_intfs = 0;
148 	enum dpu_crtc_client_type curr_client_type;
149 	struct dpu_crtc_state *dpu_cstate;
150 	struct drm_crtc *tmp_crtc;
151 	struct dpu_kms *kms;
152 
153 	if (!crtc || !state) {
154 		DPU_ERROR("invalid crtc\n");
155 		return -EINVAL;
156 	}
157 
158 	kms = _dpu_crtc_get_kms(crtc);
159 
160 	/* we only need bandwidth check on real-time clients (interfaces) */
161 	if (dpu_crtc_get_client_type(crtc) == NRT_CLIENT)
162 		return 0;
163 
164 	dpu_cstate = to_dpu_crtc_state(state);
165 
166 	/* obtain new values */
167 	_dpu_core_perf_calc_crtc(&kms->perf, crtc, state, &dpu_cstate->new_perf);
168 
169 	bw_sum_of_intfs = dpu_cstate->new_perf.bw_ctl;
170 	curr_client_type = dpu_crtc_get_client_type(crtc);
171 
172 	drm_for_each_crtc(tmp_crtc, crtc->dev) {
173 		if (tmp_crtc->enabled &&
174 		    dpu_crtc_get_client_type(tmp_crtc) == curr_client_type &&
175 		    tmp_crtc != crtc) {
176 			struct dpu_crtc_state *tmp_cstate =
177 				to_dpu_crtc_state(tmp_crtc->state);
178 
179 			DRM_DEBUG_ATOMIC("crtc:%d bw:%llu ctrl:%d\n",
180 					 tmp_crtc->base.id, tmp_cstate->new_perf.bw_ctl,
181 					 tmp_cstate->bw_control);
182 
183 			bw_sum_of_intfs += tmp_cstate->new_perf.bw_ctl;
184 		}
185 
186 		/* convert bandwidth to kb */
187 		bw = DIV_ROUND_UP_ULL(bw_sum_of_intfs, 1000);
188 		DRM_DEBUG_ATOMIC("calculated bandwidth=%uk\n", bw);
189 
190 		threshold = kms->perf.perf_cfg->max_bw_high;
191 
192 		DRM_DEBUG_ATOMIC("final threshold bw limit = %d\n", threshold);
193 
194 		if (!threshold) {
195 			DPU_ERROR("no bandwidth limits specified\n");
196 			return -E2BIG;
197 		} else if (bw > threshold) {
198 			DPU_ERROR("exceeds bandwidth: %ukb > %ukb\n", bw,
199 					threshold);
200 			return -E2BIG;
201 		}
202 	}
203 
204 	return 0;
205 }
206 
_dpu_core_perf_crtc_update_bus(struct dpu_kms * kms,struct drm_crtc * crtc)207 static int _dpu_core_perf_crtc_update_bus(struct dpu_kms *kms,
208 		struct drm_crtc *crtc)
209 {
210 	struct dpu_core_perf_params perf = { 0 };
211 	enum dpu_crtc_client_type curr_client_type
212 					= dpu_crtc_get_client_type(crtc);
213 	struct drm_crtc *tmp_crtc;
214 	struct dpu_crtc_state *dpu_cstate;
215 	int i, ret = 0;
216 	u64 avg_bw;
217 
218 	if (!kms->num_paths)
219 		return 0;
220 
221 	drm_for_each_crtc(tmp_crtc, crtc->dev) {
222 		if (tmp_crtc->enabled &&
223 			curr_client_type ==
224 				dpu_crtc_get_client_type(tmp_crtc)) {
225 			dpu_cstate = to_dpu_crtc_state(tmp_crtc->state);
226 
227 			perf.max_per_pipe_ib = max(perf.max_per_pipe_ib,
228 					dpu_cstate->new_perf.max_per_pipe_ib);
229 
230 			perf.bw_ctl += dpu_cstate->new_perf.bw_ctl;
231 
232 			DRM_DEBUG_ATOMIC("crtc=%d bw=%llu paths:%d\n",
233 				  tmp_crtc->base.id,
234 				  dpu_cstate->new_perf.bw_ctl, kms->num_paths);
235 		}
236 	}
237 
238 	avg_bw = perf.bw_ctl;
239 	do_div(avg_bw, (kms->num_paths * 1000)); /*Bps_to_icc*/
240 
241 	for (i = 0; i < kms->num_paths; i++)
242 		icc_set_bw(kms->path[i], avg_bw, perf.max_per_pipe_ib);
243 
244 	return ret;
245 }
246 
247 /**
248  * dpu_core_perf_crtc_release_bw() - request zero bandwidth
249  * @crtc: pointer to a crtc
250  *
251  * Function checks a state variable for the crtc, if all pending commit
252  * requests are done, meaning no more bandwidth is needed, release
253  * bandwidth request.
254  */
dpu_core_perf_crtc_release_bw(struct drm_crtc * crtc)255 void dpu_core_perf_crtc_release_bw(struct drm_crtc *crtc)
256 {
257 	struct dpu_crtc *dpu_crtc;
258 	struct dpu_kms *kms;
259 
260 	if (!crtc) {
261 		DPU_ERROR("invalid crtc\n");
262 		return;
263 	}
264 
265 	kms = _dpu_crtc_get_kms(crtc);
266 	dpu_crtc = to_dpu_crtc(crtc);
267 
268 	if (atomic_dec_return(&kms->bandwidth_ref) > 0)
269 		return;
270 
271 	/* Release the bandwidth */
272 	if (kms->perf.enable_bw_release) {
273 		trace_dpu_cmd_release_bw(crtc->base.id);
274 		DRM_DEBUG_ATOMIC("Release BW crtc=%d\n", crtc->base.id);
275 		dpu_crtc->cur_perf.bw_ctl = 0;
276 		_dpu_core_perf_crtc_update_bus(kms, crtc);
277 	}
278 }
279 
_dpu_core_perf_get_core_clk_rate(struct dpu_kms * kms)280 static u64 _dpu_core_perf_get_core_clk_rate(struct dpu_kms *kms)
281 {
282 	u64 clk_rate;
283 	struct drm_crtc *crtc;
284 	struct dpu_crtc_state *dpu_cstate;
285 
286 	if (kms->perf.perf_tune.mode == DPU_PERF_MODE_FIXED)
287 		return kms->perf.fix_core_clk_rate;
288 
289 	if (kms->perf.perf_tune.mode == DPU_PERF_MODE_MINIMUM)
290 		return kms->perf.max_core_clk_rate;
291 
292 	clk_rate = 0;
293 	drm_for_each_crtc(crtc, kms->dev) {
294 		if (crtc->enabled) {
295 			dpu_cstate = to_dpu_crtc_state(crtc->state);
296 			clk_rate = max(dpu_cstate->new_perf.core_clk_rate,
297 							clk_rate);
298 		}
299 	}
300 
301 	return clk_rate;
302 }
303 
dpu_core_perf_crtc_update(struct drm_crtc * crtc,int params_changed)304 int dpu_core_perf_crtc_update(struct drm_crtc *crtc,
305 			      int params_changed)
306 {
307 	struct dpu_core_perf_params *new, *old;
308 	bool update_bus = false, update_clk = false;
309 	u64 clk_rate = 0;
310 	struct dpu_crtc *dpu_crtc;
311 	struct dpu_crtc_state *dpu_cstate;
312 	struct dpu_kms *kms;
313 	int ret;
314 
315 	if (!crtc) {
316 		DPU_ERROR("invalid crtc\n");
317 		return -EINVAL;
318 	}
319 
320 	kms = _dpu_crtc_get_kms(crtc);
321 
322 	dpu_crtc = to_dpu_crtc(crtc);
323 	dpu_cstate = to_dpu_crtc_state(crtc->state);
324 
325 	DRM_DEBUG_ATOMIC("crtc:%d enabled:%d core_clk:%llu\n",
326 			crtc->base.id, crtc->enabled, kms->perf.core_clk_rate);
327 
328 	old = &dpu_crtc->cur_perf;
329 	new = &dpu_cstate->new_perf;
330 
331 	if (crtc->enabled) {
332 		/*
333 		 * cases for bus bandwidth update.
334 		 * 1. new bandwidth vote - "ab or ib vote" is higher
335 		 *    than current vote for update request.
336 		 * 2. new bandwidth vote - "ab or ib vote" is lower
337 		 *    than current vote at end of commit or stop.
338 		 */
339 		if ((params_changed && ((new->bw_ctl > old->bw_ctl) ||
340 			(new->max_per_pipe_ib > old->max_per_pipe_ib)))	||
341 			(!params_changed && ((new->bw_ctl < old->bw_ctl) ||
342 			(new->max_per_pipe_ib < old->max_per_pipe_ib)))) {
343 			DRM_DEBUG_ATOMIC("crtc=%d p=%d new_bw=%llu,old_bw=%llu\n",
344 				crtc->base.id, params_changed,
345 				new->bw_ctl, old->bw_ctl);
346 			old->bw_ctl = new->bw_ctl;
347 			old->max_per_pipe_ib = new->max_per_pipe_ib;
348 			update_bus = true;
349 		}
350 
351 		if ((params_changed && new->core_clk_rate > old->core_clk_rate) ||
352 		    (!params_changed && new->core_clk_rate < old->core_clk_rate)) {
353 			old->core_clk_rate = new->core_clk_rate;
354 			update_clk = true;
355 		}
356 	} else {
357 		DRM_DEBUG_ATOMIC("crtc=%d disable\n", crtc->base.id);
358 		memset(old, 0, sizeof(*old));
359 		update_bus = true;
360 		update_clk = true;
361 	}
362 
363 	trace_dpu_perf_crtc_update(crtc->base.id, new->bw_ctl,
364 		new->core_clk_rate, !crtc->enabled, update_bus, update_clk);
365 
366 	if (update_bus) {
367 		ret = _dpu_core_perf_crtc_update_bus(kms, crtc);
368 		if (ret) {
369 			DPU_ERROR("crtc-%d: failed to update bus bw vote\n",
370 				  crtc->base.id);
371 			return ret;
372 		}
373 	}
374 
375 	/*
376 	 * Update the clock after bandwidth vote to ensure
377 	 * bandwidth is available before clock rate is increased.
378 	 */
379 	if (update_clk) {
380 		clk_rate = _dpu_core_perf_get_core_clk_rate(kms);
381 
382 		DRM_DEBUG_ATOMIC("clk:%llu\n", clk_rate);
383 
384 		trace_dpu_core_perf_update_clk(kms->dev, !crtc->enabled, clk_rate);
385 
386 		clk_rate = min(clk_rate, kms->perf.max_core_clk_rate);
387 		ret = dev_pm_opp_set_rate(&kms->pdev->dev, clk_rate);
388 		if (ret) {
389 			DPU_ERROR("failed to set core clock rate %llu\n", clk_rate);
390 			return ret;
391 		}
392 
393 		kms->perf.core_clk_rate = clk_rate;
394 		DRM_DEBUG_ATOMIC("update clk rate = %lld HZ\n", clk_rate);
395 	}
396 	return 0;
397 }
398 
399 #ifdef CONFIG_DEBUG_FS
400 
_dpu_core_perf_mode_write(struct file * file,const char __user * user_buf,size_t count,loff_t * ppos)401 static ssize_t _dpu_core_perf_mode_write(struct file *file,
402 		    const char __user *user_buf, size_t count, loff_t *ppos)
403 {
404 	struct dpu_core_perf *perf = file->private_data;
405 	u32 perf_mode = 0;
406 	int ret;
407 
408 	ret = kstrtouint_from_user(user_buf, count, 0, &perf_mode);
409 	if (ret)
410 		return ret;
411 
412 	if (perf_mode >= DPU_PERF_MODE_MAX)
413 		return -EINVAL;
414 
415 	if (perf_mode == DPU_PERF_MODE_FIXED) {
416 		DRM_INFO("fix performance mode\n");
417 	} else if (perf_mode == DPU_PERF_MODE_MINIMUM) {
418 		/* run the driver with max clk and BW vote */
419 		DRM_INFO("minimum performance mode\n");
420 	} else if (perf_mode == DPU_PERF_MODE_NORMAL) {
421 		/* reset the perf tune params to 0 */
422 		DRM_INFO("normal performance mode\n");
423 	}
424 	perf->perf_tune.mode = perf_mode;
425 
426 	return count;
427 }
428 
_dpu_core_perf_mode_read(struct file * file,char __user * buff,size_t count,loff_t * ppos)429 static ssize_t _dpu_core_perf_mode_read(struct file *file,
430 			char __user *buff, size_t count, loff_t *ppos)
431 {
432 	struct dpu_core_perf *perf = file->private_data;
433 	int len;
434 	char buf[128];
435 
436 	len = scnprintf(buf, sizeof(buf),
437 			"mode %d\n",
438 			perf->perf_tune.mode);
439 
440 	return simple_read_from_buffer(buff, count, ppos, buf, len);
441 }
442 
443 static const struct file_operations dpu_core_perf_mode_fops = {
444 	.open = simple_open,
445 	.read = _dpu_core_perf_mode_read,
446 	.write = _dpu_core_perf_mode_write,
447 };
448 
dpu_core_perf_debugfs_init(struct dpu_kms * dpu_kms,struct dentry * parent)449 int dpu_core_perf_debugfs_init(struct dpu_kms *dpu_kms, struct dentry *parent)
450 {
451 	struct dpu_core_perf *perf = &dpu_kms->perf;
452 	struct dentry *entry;
453 
454 	entry = debugfs_create_dir("core_perf", parent);
455 
456 	debugfs_create_u64("max_core_clk_rate", 0600, entry,
457 			&perf->max_core_clk_rate);
458 	debugfs_create_u64("core_clk_rate", 0600, entry,
459 			&perf->core_clk_rate);
460 	debugfs_create_u32("enable_bw_release", 0600, entry,
461 			(u32 *)&perf->enable_bw_release);
462 	debugfs_create_u32("threshold_low", 0600, entry,
463 			(u32 *)&perf->perf_cfg->max_bw_low);
464 	debugfs_create_u32("threshold_high", 0600, entry,
465 			(u32 *)&perf->perf_cfg->max_bw_high);
466 	debugfs_create_u32("min_core_ib", 0600, entry,
467 			(u32 *)&perf->perf_cfg->min_core_ib);
468 	debugfs_create_u32("min_llcc_ib", 0600, entry,
469 			(u32 *)&perf->perf_cfg->min_llcc_ib);
470 	debugfs_create_u32("min_dram_ib", 0600, entry,
471 			(u32 *)&perf->perf_cfg->min_dram_ib);
472 	debugfs_create_file("perf_mode", 0600, entry,
473 			(u32 *)perf, &dpu_core_perf_mode_fops);
474 	debugfs_create_u64("fix_core_clk_rate", 0600, entry,
475 			&perf->fix_core_clk_rate);
476 	debugfs_create_u64("fix_core_ib_vote", 0600, entry,
477 			&perf->fix_core_ib_vote);
478 	debugfs_create_u64("fix_core_ab_vote", 0600, entry,
479 			&perf->fix_core_ab_vote);
480 
481 	return 0;
482 }
483 #endif
484 
dpu_core_perf_init(struct dpu_core_perf * perf,const struct dpu_perf_cfg * perf_cfg,unsigned long max_core_clk_rate)485 int dpu_core_perf_init(struct dpu_core_perf *perf,
486 		const struct dpu_perf_cfg *perf_cfg,
487 		unsigned long max_core_clk_rate)
488 {
489 	perf->perf_cfg = perf_cfg;
490 	perf->max_core_clk_rate = max_core_clk_rate;
491 
492 	return 0;
493 }
494