1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2013 Red Hat
4  * Author: Rob Clark <robdclark@gmail.com>
5  */
6 
7 #include "msm_gpu.h"
8 #include "msm_gpu_trace.h"
9 
10 #include <linux/devfreq.h>
11 #include <linux/devfreq_cooling.h>
12 #include <linux/math64.h>
13 #include <linux/units.h>
14 
15 /*
16  * Power Management:
17  */
18 
msm_devfreq_target(struct device * dev,unsigned long * freq,u32 flags)19 static int msm_devfreq_target(struct device *dev, unsigned long *freq,
20 		u32 flags)
21 {
22 	struct msm_gpu *gpu = dev_to_gpu(dev);
23 	struct msm_gpu_devfreq *df = &gpu->devfreq;
24 	struct dev_pm_opp *opp;
25 
26 	/*
27 	 * Note that devfreq_recommended_opp() can modify the freq
28 	 * to something that actually is in the opp table:
29 	 */
30 	opp = devfreq_recommended_opp(dev, freq, flags);
31 	if (IS_ERR(opp))
32 		return PTR_ERR(opp);
33 
34 	trace_msm_gpu_freq_change(dev_pm_opp_get_freq(opp));
35 
36 	if (gpu->funcs->gpu_set_freq) {
37 		mutex_lock(&df->lock);
38 		gpu->funcs->gpu_set_freq(gpu, opp, df->suspended);
39 		mutex_unlock(&df->lock);
40 	} else {
41 		clk_set_rate(gpu->core_clk, *freq);
42 	}
43 
44 	dev_pm_opp_put(opp);
45 
46 	return 0;
47 }
48 
get_freq(struct msm_gpu * gpu)49 static unsigned long get_freq(struct msm_gpu *gpu)
50 {
51 	if (gpu->funcs->gpu_get_freq)
52 		return gpu->funcs->gpu_get_freq(gpu);
53 
54 	return clk_get_rate(gpu->core_clk);
55 }
56 
get_raw_dev_status(struct msm_gpu * gpu,struct devfreq_dev_status * status)57 static void get_raw_dev_status(struct msm_gpu *gpu,
58 		struct devfreq_dev_status *status)
59 {
60 	struct msm_gpu_devfreq *df = &gpu->devfreq;
61 	u64 busy_cycles, busy_time;
62 	unsigned long sample_rate;
63 	ktime_t time;
64 
65 	mutex_lock(&df->lock);
66 
67 	status->current_frequency = get_freq(gpu);
68 	time = ktime_get();
69 	status->total_time = ktime_us_delta(time, df->time);
70 	df->time = time;
71 
72 	if (df->suspended) {
73 		mutex_unlock(&df->lock);
74 		status->busy_time = 0;
75 		return;
76 	}
77 
78 	busy_cycles = gpu->funcs->gpu_busy(gpu, &sample_rate);
79 	busy_time = busy_cycles - df->busy_cycles;
80 	df->busy_cycles = busy_cycles;
81 
82 	mutex_unlock(&df->lock);
83 
84 	busy_time *= USEC_PER_SEC;
85 	busy_time = div64_ul(busy_time, sample_rate);
86 	if (WARN_ON(busy_time > ~0LU))
87 		busy_time = ~0LU;
88 
89 	status->busy_time = busy_time;
90 }
91 
update_average_dev_status(struct msm_gpu * gpu,const struct devfreq_dev_status * raw)92 static void update_average_dev_status(struct msm_gpu *gpu,
93 		const struct devfreq_dev_status *raw)
94 {
95 	struct msm_gpu_devfreq *df = &gpu->devfreq;
96 	const u32 polling_ms = df->devfreq->profile->polling_ms;
97 	const u32 max_history_ms = polling_ms * 11 / 10;
98 	struct devfreq_dev_status *avg = &df->average_status;
99 	u64 avg_freq;
100 
101 	/* simple_ondemand governor interacts poorly with gpu->clamp_to_idle.
102 	 * When we enforce the constraint on idle, it calls get_dev_status
103 	 * which would normally reset the stats.  When we remove the
104 	 * constraint on active, it calls get_dev_status again where busy_time
105 	 * would be 0.
106 	 *
107 	 * To remedy this, we always return the average load over the past
108 	 * polling_ms.
109 	 */
110 
111 	/* raw is longer than polling_ms or avg has no history */
112 	if (div_u64(raw->total_time, USEC_PER_MSEC) >= polling_ms ||
113 	    !avg->total_time) {
114 		*avg = *raw;
115 		return;
116 	}
117 
118 	/* Truncate the oldest history first.
119 	 *
120 	 * Because we keep the history with a single devfreq_dev_status,
121 	 * rather than a list of devfreq_dev_status, we have to assume freq
122 	 * and load are the same over avg->total_time.  We can scale down
123 	 * avg->busy_time and avg->total_time by the same factor to drop
124 	 * history.
125 	 */
126 	if (div_u64(avg->total_time + raw->total_time, USEC_PER_MSEC) >=
127 			max_history_ms) {
128 		const u32 new_total_time = polling_ms * USEC_PER_MSEC -
129 			raw->total_time;
130 		avg->busy_time = div_u64(
131 				mul_u32_u32(avg->busy_time, new_total_time),
132 				avg->total_time);
133 		avg->total_time = new_total_time;
134 	}
135 
136 	/* compute the average freq over avg->total_time + raw->total_time */
137 	avg_freq = mul_u32_u32(avg->current_frequency, avg->total_time);
138 	avg_freq += mul_u32_u32(raw->current_frequency, raw->total_time);
139 	do_div(avg_freq, avg->total_time + raw->total_time);
140 
141 	avg->current_frequency = avg_freq;
142 	avg->busy_time += raw->busy_time;
143 	avg->total_time += raw->total_time;
144 }
145 
msm_devfreq_get_dev_status(struct device * dev,struct devfreq_dev_status * status)146 static int msm_devfreq_get_dev_status(struct device *dev,
147 		struct devfreq_dev_status *status)
148 {
149 	struct msm_gpu *gpu = dev_to_gpu(dev);
150 	struct devfreq_dev_status raw;
151 
152 	get_raw_dev_status(gpu, &raw);
153 	update_average_dev_status(gpu, &raw);
154 	*status = gpu->devfreq.average_status;
155 
156 	return 0;
157 }
158 
msm_devfreq_get_cur_freq(struct device * dev,unsigned long * freq)159 static int msm_devfreq_get_cur_freq(struct device *dev, unsigned long *freq)
160 {
161 	*freq = get_freq(dev_to_gpu(dev));
162 
163 	return 0;
164 }
165 
166 static struct devfreq_dev_profile msm_devfreq_profile = {
167 	.timer = DEVFREQ_TIMER_DELAYED,
168 	.polling_ms = 50,
169 	.target = msm_devfreq_target,
170 	.get_dev_status = msm_devfreq_get_dev_status,
171 	.get_cur_freq = msm_devfreq_get_cur_freq,
172 };
173 
174 static void msm_devfreq_boost_work(struct kthread_work *work);
175 static void msm_devfreq_idle_work(struct kthread_work *work);
176 
has_devfreq(struct msm_gpu * gpu)177 static bool has_devfreq(struct msm_gpu *gpu)
178 {
179 	struct msm_gpu_devfreq *df = &gpu->devfreq;
180 	return !!df->devfreq;
181 }
182 
msm_devfreq_init(struct msm_gpu * gpu)183 void msm_devfreq_init(struct msm_gpu *gpu)
184 {
185 	struct msm_gpu_devfreq *df = &gpu->devfreq;
186 
187 	/* We need target support to do devfreq */
188 	if (!gpu->funcs->gpu_busy)
189 		return;
190 
191 	mutex_init(&df->lock);
192 
193 	dev_pm_qos_add_request(&gpu->pdev->dev, &df->idle_freq,
194 			       DEV_PM_QOS_MAX_FREQUENCY,
195 			       PM_QOS_MAX_FREQUENCY_DEFAULT_VALUE);
196 	dev_pm_qos_add_request(&gpu->pdev->dev, &df->boost_freq,
197 			       DEV_PM_QOS_MIN_FREQUENCY, 0);
198 
199 	msm_devfreq_profile.initial_freq = gpu->fast_rate;
200 
201 	/*
202 	 * Don't set the freq_table or max_state and let devfreq build the table
203 	 * from OPP
204 	 * After a deferred probe, these may have be left to non-zero values,
205 	 * so set them back to zero before creating the devfreq device
206 	 */
207 	msm_devfreq_profile.freq_table = NULL;
208 	msm_devfreq_profile.max_state = 0;
209 
210 	df->devfreq = devm_devfreq_add_device(&gpu->pdev->dev,
211 			&msm_devfreq_profile, DEVFREQ_GOV_SIMPLE_ONDEMAND,
212 			NULL);
213 
214 	if (IS_ERR(df->devfreq)) {
215 		DRM_DEV_ERROR(&gpu->pdev->dev, "Couldn't initialize GPU devfreq\n");
216 		dev_pm_qos_remove_request(&df->idle_freq);
217 		dev_pm_qos_remove_request(&df->boost_freq);
218 		df->devfreq = NULL;
219 		return;
220 	}
221 
222 	devfreq_suspend_device(df->devfreq);
223 
224 	gpu->cooling = of_devfreq_cooling_register(gpu->pdev->dev.of_node, df->devfreq);
225 	if (IS_ERR(gpu->cooling)) {
226 		DRM_DEV_ERROR(&gpu->pdev->dev,
227 				"Couldn't register GPU cooling device\n");
228 		gpu->cooling = NULL;
229 	}
230 
231 	msm_hrtimer_work_init(&df->boost_work, gpu->worker, msm_devfreq_boost_work,
232 			      CLOCK_MONOTONIC, HRTIMER_MODE_REL);
233 	msm_hrtimer_work_init(&df->idle_work, gpu->worker, msm_devfreq_idle_work,
234 			      CLOCK_MONOTONIC, HRTIMER_MODE_REL);
235 }
236 
cancel_idle_work(struct msm_gpu_devfreq * df)237 static void cancel_idle_work(struct msm_gpu_devfreq *df)
238 {
239 	hrtimer_cancel(&df->idle_work.timer);
240 	kthread_cancel_work_sync(&df->idle_work.work);
241 }
242 
cancel_boost_work(struct msm_gpu_devfreq * df)243 static void cancel_boost_work(struct msm_gpu_devfreq *df)
244 {
245 	hrtimer_cancel(&df->boost_work.timer);
246 	kthread_cancel_work_sync(&df->boost_work.work);
247 }
248 
msm_devfreq_cleanup(struct msm_gpu * gpu)249 void msm_devfreq_cleanup(struct msm_gpu *gpu)
250 {
251 	struct msm_gpu_devfreq *df = &gpu->devfreq;
252 
253 	if (!has_devfreq(gpu))
254 		return;
255 
256 	devfreq_cooling_unregister(gpu->cooling);
257 	dev_pm_qos_remove_request(&df->boost_freq);
258 	dev_pm_qos_remove_request(&df->idle_freq);
259 }
260 
msm_devfreq_resume(struct msm_gpu * gpu)261 void msm_devfreq_resume(struct msm_gpu *gpu)
262 {
263 	struct msm_gpu_devfreq *df = &gpu->devfreq;
264 	unsigned long sample_rate;
265 
266 	if (!has_devfreq(gpu))
267 		return;
268 
269 	mutex_lock(&df->lock);
270 	df->busy_cycles = gpu->funcs->gpu_busy(gpu, &sample_rate);
271 	df->time = ktime_get();
272 	df->suspended = false;
273 	mutex_unlock(&df->lock);
274 
275 	devfreq_resume_device(df->devfreq);
276 }
277 
msm_devfreq_suspend(struct msm_gpu * gpu)278 void msm_devfreq_suspend(struct msm_gpu *gpu)
279 {
280 	struct msm_gpu_devfreq *df = &gpu->devfreq;
281 
282 	if (!has_devfreq(gpu))
283 		return;
284 
285 	mutex_lock(&df->lock);
286 	df->suspended = true;
287 	mutex_unlock(&df->lock);
288 
289 	devfreq_suspend_device(df->devfreq);
290 
291 	cancel_idle_work(df);
292 	cancel_boost_work(df);
293 }
294 
msm_devfreq_boost_work(struct kthread_work * work)295 static void msm_devfreq_boost_work(struct kthread_work *work)
296 {
297 	struct msm_gpu_devfreq *df = container_of(work,
298 			struct msm_gpu_devfreq, boost_work.work);
299 
300 	dev_pm_qos_update_request(&df->boost_freq, 0);
301 }
302 
msm_devfreq_boost(struct msm_gpu * gpu,unsigned factor)303 void msm_devfreq_boost(struct msm_gpu *gpu, unsigned factor)
304 {
305 	struct msm_gpu_devfreq *df = &gpu->devfreq;
306 	uint64_t freq;
307 
308 	if (!has_devfreq(gpu))
309 		return;
310 
311 	freq = get_freq(gpu);
312 	freq *= factor;
313 
314 	/*
315 	 * A nice little trap is that PM QoS operates in terms of KHz,
316 	 * while devfreq operates in terms of Hz:
317 	 */
318 	do_div(freq, HZ_PER_KHZ);
319 
320 	dev_pm_qos_update_request(&df->boost_freq, freq);
321 
322 	msm_hrtimer_queue_work(&df->boost_work,
323 			       ms_to_ktime(msm_devfreq_profile.polling_ms),
324 			       HRTIMER_MODE_REL);
325 }
326 
msm_devfreq_active(struct msm_gpu * gpu)327 void msm_devfreq_active(struct msm_gpu *gpu)
328 {
329 	struct msm_gpu_devfreq *df = &gpu->devfreq;
330 	unsigned int idle_time;
331 
332 	if (!has_devfreq(gpu))
333 		return;
334 
335 	/*
336 	 * Cancel any pending transition to idle frequency:
337 	 */
338 	cancel_idle_work(df);
339 
340 	idle_time = ktime_to_ms(ktime_sub(ktime_get(), df->idle_time));
341 
342 	/*
343 	 * If we've been idle for a significant fraction of a polling
344 	 * interval, then we won't meet the threshold of busyness for
345 	 * the governor to ramp up the freq.. so give some boost
346 	 */
347 	if (idle_time > msm_devfreq_profile.polling_ms) {
348 		msm_devfreq_boost(gpu, 2);
349 	}
350 
351 	dev_pm_qos_update_request(&df->idle_freq,
352 				  PM_QOS_MAX_FREQUENCY_DEFAULT_VALUE);
353 }
354 
355 
msm_devfreq_idle_work(struct kthread_work * work)356 static void msm_devfreq_idle_work(struct kthread_work *work)
357 {
358 	struct msm_gpu_devfreq *df = container_of(work,
359 			struct msm_gpu_devfreq, idle_work.work);
360 	struct msm_gpu *gpu = container_of(df, struct msm_gpu, devfreq);
361 
362 	df->idle_time = ktime_get();
363 
364 	if (gpu->clamp_to_idle)
365 		dev_pm_qos_update_request(&df->idle_freq, 0);
366 }
367 
msm_devfreq_idle(struct msm_gpu * gpu)368 void msm_devfreq_idle(struct msm_gpu *gpu)
369 {
370 	struct msm_gpu_devfreq *df = &gpu->devfreq;
371 
372 	if (!has_devfreq(gpu))
373 		return;
374 
375 	msm_hrtimer_queue_work(&df->idle_work, ms_to_ktime(1),
376 			       HRTIMER_MODE_REL);
377 }
378