1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (c) 2016, Fuzhou Rockchip Electronics Co., Ltd.
4  * Author: Lin Huang <hl@rock-chips.com>
5  */
6 
7 #include <linux/arm-smccc.h>
8 #include <linux/bitfield.h>
9 #include <linux/clk.h>
10 #include <linux/delay.h>
11 #include <linux/devfreq.h>
12 #include <linux/devfreq-event.h>
13 #include <linux/interrupt.h>
14 #include <linux/mfd/syscon.h>
15 #include <linux/module.h>
16 #include <linux/of.h>
17 #include <linux/platform_device.h>
18 #include <linux/pm_opp.h>
19 #include <linux/regmap.h>
20 #include <linux/regulator/consumer.h>
21 #include <linux/rwsem.h>
22 #include <linux/suspend.h>
23 
24 #include <soc/rockchip/pm_domains.h>
25 #include <soc/rockchip/rk3399_grf.h>
26 #include <soc/rockchip/rockchip_sip.h>
27 
28 #define NS_TO_CYCLE(NS, MHz)				(((NS) * (MHz)) / NSEC_PER_USEC)
29 
30 #define RK3399_SET_ODT_PD_0_SR_IDLE			GENMASK(7, 0)
31 #define RK3399_SET_ODT_PD_0_SR_MC_GATE_IDLE		GENMASK(15, 8)
32 #define RK3399_SET_ODT_PD_0_STANDBY_IDLE		GENMASK(31, 16)
33 
34 #define RK3399_SET_ODT_PD_1_PD_IDLE			GENMASK(11, 0)
35 #define RK3399_SET_ODT_PD_1_SRPD_LITE_IDLE		GENMASK(27, 16)
36 
37 #define RK3399_SET_ODT_PD_2_ODT_ENABLE			BIT(0)
38 
39 struct rk3399_dmcfreq {
40 	struct device *dev;
41 	struct devfreq *devfreq;
42 	struct devfreq_dev_profile profile;
43 	struct devfreq_simple_ondemand_data ondemand_data;
44 	struct clk *dmc_clk;
45 	struct devfreq_event_dev *edev;
46 	struct mutex lock;
47 	struct regulator *vdd_center;
48 	struct regmap *regmap_pmu;
49 	unsigned long rate, target_rate;
50 	unsigned long volt, target_volt;
51 	unsigned int odt_dis_freq;
52 
53 	unsigned int pd_idle_ns;
54 	unsigned int sr_idle_ns;
55 	unsigned int sr_mc_gate_idle_ns;
56 	unsigned int srpd_lite_idle_ns;
57 	unsigned int standby_idle_ns;
58 	unsigned int ddr3_odt_dis_freq;
59 	unsigned int lpddr3_odt_dis_freq;
60 	unsigned int lpddr4_odt_dis_freq;
61 
62 	unsigned int pd_idle_dis_freq;
63 	unsigned int sr_idle_dis_freq;
64 	unsigned int sr_mc_gate_idle_dis_freq;
65 	unsigned int srpd_lite_idle_dis_freq;
66 	unsigned int standby_idle_dis_freq;
67 };
68 
rk3399_dmcfreq_target(struct device * dev,unsigned long * freq,u32 flags)69 static int rk3399_dmcfreq_target(struct device *dev, unsigned long *freq,
70 				 u32 flags)
71 {
72 	struct rk3399_dmcfreq *dmcfreq = dev_get_drvdata(dev);
73 	struct dev_pm_opp *opp;
74 	unsigned long old_clk_rate = dmcfreq->rate;
75 	unsigned long target_volt, target_rate;
76 	unsigned int ddrcon_mhz;
77 	struct arm_smccc_res res;
78 	int err;
79 
80 	u32 odt_pd_arg0 = 0;
81 	u32 odt_pd_arg1 = 0;
82 	u32 odt_pd_arg2 = 0;
83 
84 	opp = devfreq_recommended_opp(dev, freq, flags);
85 	if (IS_ERR(opp))
86 		return PTR_ERR(opp);
87 
88 	target_rate = dev_pm_opp_get_freq(opp);
89 	target_volt = dev_pm_opp_get_voltage(opp);
90 	dev_pm_opp_put(opp);
91 
92 	if (dmcfreq->rate == target_rate)
93 		return 0;
94 
95 	mutex_lock(&dmcfreq->lock);
96 
97 	/*
98 	 * Ensure power-domain transitions don't interfere with ARM Trusted
99 	 * Firmware power-domain idling.
100 	 */
101 	err = rockchip_pmu_block();
102 	if (err) {
103 		dev_err(dev, "Failed to block PMU: %d\n", err);
104 		goto out_unlock;
105 	}
106 
107 	/*
108 	 * Some idle parameters may be based on the DDR controller clock, which
109 	 * is half of the DDR frequency.
110 	 * pd_idle and standby_idle are based on the controller clock cycle.
111 	 * sr_idle_cycle, sr_mc_gate_idle_cycle, and srpd_lite_idle_cycle
112 	 * are based on the 1024 controller clock cycle
113 	 */
114 	ddrcon_mhz = target_rate / USEC_PER_SEC / 2;
115 
116 	u32p_replace_bits(&odt_pd_arg1,
117 			  NS_TO_CYCLE(dmcfreq->pd_idle_ns, ddrcon_mhz),
118 			  RK3399_SET_ODT_PD_1_PD_IDLE);
119 	u32p_replace_bits(&odt_pd_arg0,
120 			  NS_TO_CYCLE(dmcfreq->standby_idle_ns, ddrcon_mhz),
121 			  RK3399_SET_ODT_PD_0_STANDBY_IDLE);
122 	u32p_replace_bits(&odt_pd_arg0,
123 			  DIV_ROUND_UP(NS_TO_CYCLE(dmcfreq->sr_idle_ns,
124 						   ddrcon_mhz), 1024),
125 			  RK3399_SET_ODT_PD_0_SR_IDLE);
126 	u32p_replace_bits(&odt_pd_arg0,
127 			  DIV_ROUND_UP(NS_TO_CYCLE(dmcfreq->sr_mc_gate_idle_ns,
128 						   ddrcon_mhz), 1024),
129 			  RK3399_SET_ODT_PD_0_SR_MC_GATE_IDLE);
130 	u32p_replace_bits(&odt_pd_arg1,
131 			  DIV_ROUND_UP(NS_TO_CYCLE(dmcfreq->srpd_lite_idle_ns,
132 						   ddrcon_mhz), 1024),
133 			  RK3399_SET_ODT_PD_1_SRPD_LITE_IDLE);
134 
135 	if (dmcfreq->regmap_pmu) {
136 		if (target_rate >= dmcfreq->sr_idle_dis_freq)
137 			odt_pd_arg0 &= ~RK3399_SET_ODT_PD_0_SR_IDLE;
138 
139 		if (target_rate >= dmcfreq->sr_mc_gate_idle_dis_freq)
140 			odt_pd_arg0 &= ~RK3399_SET_ODT_PD_0_SR_MC_GATE_IDLE;
141 
142 		if (target_rate >= dmcfreq->standby_idle_dis_freq)
143 			odt_pd_arg0 &= ~RK3399_SET_ODT_PD_0_STANDBY_IDLE;
144 
145 		if (target_rate >= dmcfreq->pd_idle_dis_freq)
146 			odt_pd_arg1 &= ~RK3399_SET_ODT_PD_1_PD_IDLE;
147 
148 		if (target_rate >= dmcfreq->srpd_lite_idle_dis_freq)
149 			odt_pd_arg1 &= ~RK3399_SET_ODT_PD_1_SRPD_LITE_IDLE;
150 
151 		if (target_rate >= dmcfreq->odt_dis_freq)
152 			odt_pd_arg2 |= RK3399_SET_ODT_PD_2_ODT_ENABLE;
153 
154 		/*
155 		 * This makes a SMC call to the TF-A to set the DDR PD
156 		 * (power-down) timings and to enable or disable the
157 		 * ODT (on-die termination) resistors.
158 		 */
159 		arm_smccc_smc(ROCKCHIP_SIP_DRAM_FREQ, odt_pd_arg0, odt_pd_arg1,
160 			      ROCKCHIP_SIP_CONFIG_DRAM_SET_ODT_PD, odt_pd_arg2,
161 			      0, 0, 0, &res);
162 	}
163 
164 	/*
165 	 * If frequency scaling from low to high, adjust voltage first.
166 	 * If frequency scaling from high to low, adjust frequency first.
167 	 */
168 	if (old_clk_rate < target_rate) {
169 		err = regulator_set_voltage(dmcfreq->vdd_center, target_volt,
170 					    target_volt);
171 		if (err) {
172 			dev_err(dev, "Cannot set voltage %lu uV\n",
173 				target_volt);
174 			goto out;
175 		}
176 	}
177 
178 	err = clk_set_rate(dmcfreq->dmc_clk, target_rate);
179 	if (err) {
180 		dev_err(dev, "Cannot set frequency %lu (%d)\n", target_rate,
181 			err);
182 		regulator_set_voltage(dmcfreq->vdd_center, dmcfreq->volt,
183 				      dmcfreq->volt);
184 		goto out;
185 	}
186 
187 	/*
188 	 * Check the dpll rate,
189 	 * There only two result we will get,
190 	 * 1. Ddr frequency scaling fail, we still get the old rate.
191 	 * 2. Ddr frequency scaling sucessful, we get the rate we set.
192 	 */
193 	dmcfreq->rate = clk_get_rate(dmcfreq->dmc_clk);
194 
195 	/* If get the incorrect rate, set voltage to old value. */
196 	if (dmcfreq->rate != target_rate) {
197 		dev_err(dev, "Got wrong frequency, Request %lu, Current %lu\n",
198 			target_rate, dmcfreq->rate);
199 		regulator_set_voltage(dmcfreq->vdd_center, dmcfreq->volt,
200 				      dmcfreq->volt);
201 		goto out;
202 	} else if (old_clk_rate > target_rate)
203 		err = regulator_set_voltage(dmcfreq->vdd_center, target_volt,
204 					    target_volt);
205 	if (err)
206 		dev_err(dev, "Cannot set voltage %lu uV\n", target_volt);
207 
208 	dmcfreq->rate = target_rate;
209 	dmcfreq->volt = target_volt;
210 
211 out:
212 	rockchip_pmu_unblock();
213 out_unlock:
214 	mutex_unlock(&dmcfreq->lock);
215 	return err;
216 }
217 
rk3399_dmcfreq_get_dev_status(struct device * dev,struct devfreq_dev_status * stat)218 static int rk3399_dmcfreq_get_dev_status(struct device *dev,
219 					 struct devfreq_dev_status *stat)
220 {
221 	struct rk3399_dmcfreq *dmcfreq = dev_get_drvdata(dev);
222 	struct devfreq_event_data edata;
223 	int ret = 0;
224 
225 	ret = devfreq_event_get_event(dmcfreq->edev, &edata);
226 	if (ret < 0)
227 		return ret;
228 
229 	stat->current_frequency = dmcfreq->rate;
230 	stat->busy_time = edata.load_count;
231 	stat->total_time = edata.total_count;
232 
233 	return ret;
234 }
235 
rk3399_dmcfreq_get_cur_freq(struct device * dev,unsigned long * freq)236 static int rk3399_dmcfreq_get_cur_freq(struct device *dev, unsigned long *freq)
237 {
238 	struct rk3399_dmcfreq *dmcfreq = dev_get_drvdata(dev);
239 
240 	*freq = dmcfreq->rate;
241 
242 	return 0;
243 }
244 
rk3399_dmcfreq_suspend(struct device * dev)245 static __maybe_unused int rk3399_dmcfreq_suspend(struct device *dev)
246 {
247 	struct rk3399_dmcfreq *dmcfreq = dev_get_drvdata(dev);
248 	int ret = 0;
249 
250 	ret = devfreq_event_disable_edev(dmcfreq->edev);
251 	if (ret < 0) {
252 		dev_err(dev, "failed to disable the devfreq-event devices\n");
253 		return ret;
254 	}
255 
256 	ret = devfreq_suspend_device(dmcfreq->devfreq);
257 	if (ret < 0) {
258 		dev_err(dev, "failed to suspend the devfreq devices\n");
259 		return ret;
260 	}
261 
262 	return 0;
263 }
264 
rk3399_dmcfreq_resume(struct device * dev)265 static __maybe_unused int rk3399_dmcfreq_resume(struct device *dev)
266 {
267 	struct rk3399_dmcfreq *dmcfreq = dev_get_drvdata(dev);
268 	int ret = 0;
269 
270 	ret = devfreq_event_enable_edev(dmcfreq->edev);
271 	if (ret < 0) {
272 		dev_err(dev, "failed to enable the devfreq-event devices\n");
273 		return ret;
274 	}
275 
276 	ret = devfreq_resume_device(dmcfreq->devfreq);
277 	if (ret < 0) {
278 		dev_err(dev, "failed to resume the devfreq devices\n");
279 		return ret;
280 	}
281 	return ret;
282 }
283 
284 static SIMPLE_DEV_PM_OPS(rk3399_dmcfreq_pm, rk3399_dmcfreq_suspend,
285 			 rk3399_dmcfreq_resume);
286 
rk3399_dmcfreq_of_props(struct rk3399_dmcfreq * data,struct device_node * np)287 static int rk3399_dmcfreq_of_props(struct rk3399_dmcfreq *data,
288 				   struct device_node *np)
289 {
290 	int ret = 0;
291 
292 	/*
293 	 * These are all optional, and serve as minimum bounds. Give them large
294 	 * (i.e., never "disabled") values if the DT doesn't specify one.
295 	 */
296 	data->pd_idle_dis_freq =
297 		data->sr_idle_dis_freq =
298 		data->sr_mc_gate_idle_dis_freq =
299 		data->srpd_lite_idle_dis_freq =
300 		data->standby_idle_dis_freq = UINT_MAX;
301 
302 	ret |= of_property_read_u32(np, "rockchip,pd-idle-ns",
303 				    &data->pd_idle_ns);
304 	ret |= of_property_read_u32(np, "rockchip,sr-idle-ns",
305 				    &data->sr_idle_ns);
306 	ret |= of_property_read_u32(np, "rockchip,sr-mc-gate-idle-ns",
307 				    &data->sr_mc_gate_idle_ns);
308 	ret |= of_property_read_u32(np, "rockchip,srpd-lite-idle-ns",
309 				    &data->srpd_lite_idle_ns);
310 	ret |= of_property_read_u32(np, "rockchip,standby-idle-ns",
311 				    &data->standby_idle_ns);
312 	ret |= of_property_read_u32(np, "rockchip,ddr3_odt_dis_freq",
313 				    &data->ddr3_odt_dis_freq);
314 	ret |= of_property_read_u32(np, "rockchip,lpddr3_odt_dis_freq",
315 				    &data->lpddr3_odt_dis_freq);
316 	ret |= of_property_read_u32(np, "rockchip,lpddr4_odt_dis_freq",
317 				    &data->lpddr4_odt_dis_freq);
318 
319 	ret |= of_property_read_u32(np, "rockchip,pd-idle-dis-freq-hz",
320 				    &data->pd_idle_dis_freq);
321 	ret |= of_property_read_u32(np, "rockchip,sr-idle-dis-freq-hz",
322 				    &data->sr_idle_dis_freq);
323 	ret |= of_property_read_u32(np, "rockchip,sr-mc-gate-idle-dis-freq-hz",
324 				    &data->sr_mc_gate_idle_dis_freq);
325 	ret |= of_property_read_u32(np, "rockchip,srpd-lite-idle-dis-freq-hz",
326 				    &data->srpd_lite_idle_dis_freq);
327 	ret |= of_property_read_u32(np, "rockchip,standby-idle-dis-freq-hz",
328 				    &data->standby_idle_dis_freq);
329 
330 	return ret;
331 }
332 
rk3399_dmcfreq_probe(struct platform_device * pdev)333 static int rk3399_dmcfreq_probe(struct platform_device *pdev)
334 {
335 	struct arm_smccc_res res;
336 	struct device *dev = &pdev->dev;
337 	struct device_node *np = pdev->dev.of_node, *node;
338 	struct rk3399_dmcfreq *data;
339 	int ret;
340 	struct dev_pm_opp *opp;
341 	u32 ddr_type;
342 	u32 val;
343 
344 	data = devm_kzalloc(dev, sizeof(struct rk3399_dmcfreq), GFP_KERNEL);
345 	if (!data)
346 		return -ENOMEM;
347 
348 	mutex_init(&data->lock);
349 
350 	data->vdd_center = devm_regulator_get(dev, "center");
351 	if (IS_ERR(data->vdd_center))
352 		return dev_err_probe(dev, PTR_ERR(data->vdd_center),
353 				     "Cannot get the regulator \"center\"\n");
354 
355 	data->dmc_clk = devm_clk_get(dev, "dmc_clk");
356 	if (IS_ERR(data->dmc_clk))
357 		return dev_err_probe(dev, PTR_ERR(data->dmc_clk),
358 				     "Cannot get the clk dmc_clk\n");
359 
360 	data->edev = devfreq_event_get_edev_by_phandle(dev, "devfreq-events", 0);
361 	if (IS_ERR(data->edev))
362 		return -EPROBE_DEFER;
363 
364 	ret = devfreq_event_enable_edev(data->edev);
365 	if (ret < 0) {
366 		dev_err(dev, "failed to enable devfreq-event devices\n");
367 		return ret;
368 	}
369 
370 	rk3399_dmcfreq_of_props(data, np);
371 
372 	node = of_parse_phandle(np, "rockchip,pmu", 0);
373 	if (!node)
374 		goto no_pmu;
375 
376 	data->regmap_pmu = syscon_node_to_regmap(node);
377 	of_node_put(node);
378 	if (IS_ERR(data->regmap_pmu)) {
379 		ret = PTR_ERR(data->regmap_pmu);
380 		goto err_edev;
381 	}
382 
383 	regmap_read(data->regmap_pmu, RK3399_PMUGRF_OS_REG2, &val);
384 	ddr_type = (val >> RK3399_PMUGRF_DDRTYPE_SHIFT) &
385 		    RK3399_PMUGRF_DDRTYPE_MASK;
386 
387 	switch (ddr_type) {
388 	case RK3399_PMUGRF_DDRTYPE_DDR3:
389 		data->odt_dis_freq = data->ddr3_odt_dis_freq;
390 		break;
391 	case RK3399_PMUGRF_DDRTYPE_LPDDR3:
392 		data->odt_dis_freq = data->lpddr3_odt_dis_freq;
393 		break;
394 	case RK3399_PMUGRF_DDRTYPE_LPDDR4:
395 		data->odt_dis_freq = data->lpddr4_odt_dis_freq;
396 		break;
397 	default:
398 		ret = -EINVAL;
399 		goto err_edev;
400 	}
401 
402 no_pmu:
403 	arm_smccc_smc(ROCKCHIP_SIP_DRAM_FREQ, 0, 0,
404 		      ROCKCHIP_SIP_CONFIG_DRAM_INIT,
405 		      0, 0, 0, 0, &res);
406 
407 	/*
408 	 * We add a devfreq driver to our parent since it has a device tree node
409 	 * with operating points.
410 	 */
411 	if (devm_pm_opp_of_add_table(dev)) {
412 		dev_err(dev, "Invalid operating-points in device tree.\n");
413 		ret = -EINVAL;
414 		goto err_edev;
415 	}
416 
417 	data->ondemand_data.upthreshold = 25;
418 	data->ondemand_data.downdifferential = 15;
419 
420 	data->rate = clk_get_rate(data->dmc_clk);
421 
422 	opp = devfreq_recommended_opp(dev, &data->rate, 0);
423 	if (IS_ERR(opp)) {
424 		ret = PTR_ERR(opp);
425 		goto err_edev;
426 	}
427 
428 	data->rate = dev_pm_opp_get_freq(opp);
429 	data->volt = dev_pm_opp_get_voltage(opp);
430 	dev_pm_opp_put(opp);
431 
432 	data->profile = (struct devfreq_dev_profile) {
433 		.polling_ms	= 200,
434 		.target		= rk3399_dmcfreq_target,
435 		.get_dev_status	= rk3399_dmcfreq_get_dev_status,
436 		.get_cur_freq	= rk3399_dmcfreq_get_cur_freq,
437 		.initial_freq	= data->rate,
438 	};
439 
440 	data->devfreq = devm_devfreq_add_device(dev,
441 					   &data->profile,
442 					   DEVFREQ_GOV_SIMPLE_ONDEMAND,
443 					   &data->ondemand_data);
444 	if (IS_ERR(data->devfreq)) {
445 		ret = PTR_ERR(data->devfreq);
446 		goto err_edev;
447 	}
448 
449 	devm_devfreq_register_opp_notifier(dev, data->devfreq);
450 
451 	data->dev = dev;
452 	platform_set_drvdata(pdev, data);
453 
454 	return 0;
455 
456 err_edev:
457 	devfreq_event_disable_edev(data->edev);
458 
459 	return ret;
460 }
461 
rk3399_dmcfreq_remove(struct platform_device * pdev)462 static int rk3399_dmcfreq_remove(struct platform_device *pdev)
463 {
464 	struct rk3399_dmcfreq *dmcfreq = dev_get_drvdata(&pdev->dev);
465 
466 	devfreq_event_disable_edev(dmcfreq->edev);
467 
468 	return 0;
469 }
470 
471 static const struct of_device_id rk3399dmc_devfreq_of_match[] = {
472 	{ .compatible = "rockchip,rk3399-dmc" },
473 	{ },
474 };
475 MODULE_DEVICE_TABLE(of, rk3399dmc_devfreq_of_match);
476 
477 static struct platform_driver rk3399_dmcfreq_driver = {
478 	.probe	= rk3399_dmcfreq_probe,
479 	.remove = rk3399_dmcfreq_remove,
480 	.driver = {
481 		.name	= "rk3399-dmc-freq",
482 		.pm	= &rk3399_dmcfreq_pm,
483 		.of_match_table = rk3399dmc_devfreq_of_match,
484 	},
485 };
486 module_platform_driver(rk3399_dmcfreq_driver);
487 
488 MODULE_LICENSE("GPL v2");
489 MODULE_AUTHOR("Lin Huang <hl@rock-chips.com>");
490 MODULE_DESCRIPTION("RK3399 dmcfreq driver with devfreq framework");
491