1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2 /* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
3 
4 #include <linux/regulator/consumer.h>
5 #include <linux/reset.h>
6 #include <linux/clk.h>
7 #include <linux/slab.h>
8 #include <linux/dma-mapping.h>
9 #include <linux/platform_device.h>
10 
11 #include "lima_device.h"
12 #include "lima_gp.h"
13 #include "lima_pp.h"
14 #include "lima_mmu.h"
15 #include "lima_pmu.h"
16 #include "lima_l2_cache.h"
17 #include "lima_dlbu.h"
18 #include "lima_bcast.h"
19 #include "lima_vm.h"
20 
21 struct lima_ip_desc {
22 	char *name;
23 	char *irq_name;
24 	bool must_have[lima_gpu_num];
25 	int offset[lima_gpu_num];
26 
27 	int (*init)(struct lima_ip *ip);
28 	void (*fini)(struct lima_ip *ip);
29 	int (*resume)(struct lima_ip *ip);
30 	void (*suspend)(struct lima_ip *ip);
31 };
32 
33 #define LIMA_IP_DESC(ipname, mst0, mst1, off0, off1, func, irq) \
34 	[lima_ip_##ipname] = { \
35 		.name = #ipname, \
36 		.irq_name = irq, \
37 		.must_have = { \
38 			[lima_gpu_mali400] = mst0, \
39 			[lima_gpu_mali450] = mst1, \
40 		}, \
41 		.offset = { \
42 			[lima_gpu_mali400] = off0, \
43 			[lima_gpu_mali450] = off1, \
44 		}, \
45 		.init = lima_##func##_init, \
46 		.fini = lima_##func##_fini, \
47 		.resume = lima_##func##_resume, \
48 		.suspend = lima_##func##_suspend, \
49 	}
50 
51 static struct lima_ip_desc lima_ip_desc[lima_ip_num] = {
52 	LIMA_IP_DESC(pmu,         false, false, 0x02000, 0x02000, pmu,      "pmu"),
53 	LIMA_IP_DESC(l2_cache0,   true,  true,  0x01000, 0x10000, l2_cache, NULL),
54 	LIMA_IP_DESC(l2_cache1,   false, true,  -1,      0x01000, l2_cache, NULL),
55 	LIMA_IP_DESC(l2_cache2,   false, false, -1,      0x11000, l2_cache, NULL),
56 	LIMA_IP_DESC(gp,          true,  true,  0x00000, 0x00000, gp,       "gp"),
57 	LIMA_IP_DESC(pp0,         true,  true,  0x08000, 0x08000, pp,       "pp0"),
58 	LIMA_IP_DESC(pp1,         false, false, 0x0A000, 0x0A000, pp,       "pp1"),
59 	LIMA_IP_DESC(pp2,         false, false, 0x0C000, 0x0C000, pp,       "pp2"),
60 	LIMA_IP_DESC(pp3,         false, false, 0x0E000, 0x0E000, pp,       "pp3"),
61 	LIMA_IP_DESC(pp4,         false, false, -1,      0x28000, pp,       "pp4"),
62 	LIMA_IP_DESC(pp5,         false, false, -1,      0x2A000, pp,       "pp5"),
63 	LIMA_IP_DESC(pp6,         false, false, -1,      0x2C000, pp,       "pp6"),
64 	LIMA_IP_DESC(pp7,         false, false, -1,      0x2E000, pp,       "pp7"),
65 	LIMA_IP_DESC(gpmmu,       true,  true,  0x03000, 0x03000, mmu,      "gpmmu"),
66 	LIMA_IP_DESC(ppmmu0,      true,  true,  0x04000, 0x04000, mmu,      "ppmmu0"),
67 	LIMA_IP_DESC(ppmmu1,      false, false, 0x05000, 0x05000, mmu,      "ppmmu1"),
68 	LIMA_IP_DESC(ppmmu2,      false, false, 0x06000, 0x06000, mmu,      "ppmmu2"),
69 	LIMA_IP_DESC(ppmmu3,      false, false, 0x07000, 0x07000, mmu,      "ppmmu3"),
70 	LIMA_IP_DESC(ppmmu4,      false, false, -1,      0x1C000, mmu,      "ppmmu4"),
71 	LIMA_IP_DESC(ppmmu5,      false, false, -1,      0x1D000, mmu,      "ppmmu5"),
72 	LIMA_IP_DESC(ppmmu6,      false, false, -1,      0x1E000, mmu,      "ppmmu6"),
73 	LIMA_IP_DESC(ppmmu7,      false, false, -1,      0x1F000, mmu,      "ppmmu7"),
74 	LIMA_IP_DESC(dlbu,        false, true,  -1,      0x14000, dlbu,     NULL),
75 	LIMA_IP_DESC(bcast,       false, true,  -1,      0x13000, bcast,    NULL),
76 	LIMA_IP_DESC(pp_bcast,    false, true,  -1,      0x16000, pp_bcast, "pp"),
77 	LIMA_IP_DESC(ppmmu_bcast, false, true,  -1,      0x15000, mmu,      NULL),
78 };
79 
lima_ip_name(struct lima_ip * ip)80 const char *lima_ip_name(struct lima_ip *ip)
81 {
82 	return lima_ip_desc[ip->id].name;
83 }
84 
lima_clk_enable(struct lima_device * dev)85 static int lima_clk_enable(struct lima_device *dev)
86 {
87 	int err;
88 
89 	err = clk_prepare_enable(dev->clk_bus);
90 	if (err)
91 		return err;
92 
93 	err = clk_prepare_enable(dev->clk_gpu);
94 	if (err)
95 		goto error_out0;
96 
97 	if (dev->reset) {
98 		err = reset_control_deassert(dev->reset);
99 		if (err) {
100 			dev_err(dev->dev,
101 				"reset controller deassert failed %d\n", err);
102 			goto error_out1;
103 		}
104 	}
105 
106 	return 0;
107 
108 error_out1:
109 	clk_disable_unprepare(dev->clk_gpu);
110 error_out0:
111 	clk_disable_unprepare(dev->clk_bus);
112 	return err;
113 }
114 
lima_clk_disable(struct lima_device * dev)115 static void lima_clk_disable(struct lima_device *dev)
116 {
117 	if (dev->reset)
118 		reset_control_assert(dev->reset);
119 	clk_disable_unprepare(dev->clk_gpu);
120 	clk_disable_unprepare(dev->clk_bus);
121 }
122 
lima_clk_init(struct lima_device * dev)123 static int lima_clk_init(struct lima_device *dev)
124 {
125 	int err;
126 
127 	dev->clk_bus = devm_clk_get(dev->dev, "bus");
128 	if (IS_ERR(dev->clk_bus)) {
129 		err = PTR_ERR(dev->clk_bus);
130 		if (err != -EPROBE_DEFER)
131 			dev_err(dev->dev, "get bus clk failed %d\n", err);
132 		dev->clk_bus = NULL;
133 		return err;
134 	}
135 
136 	dev->clk_gpu = devm_clk_get(dev->dev, "core");
137 	if (IS_ERR(dev->clk_gpu)) {
138 		err = PTR_ERR(dev->clk_gpu);
139 		if (err != -EPROBE_DEFER)
140 			dev_err(dev->dev, "get core clk failed %d\n", err);
141 		dev->clk_gpu = NULL;
142 		return err;
143 	}
144 
145 	dev->reset = devm_reset_control_array_get_optional_shared(dev->dev);
146 	if (IS_ERR(dev->reset)) {
147 		err = PTR_ERR(dev->reset);
148 		if (err != -EPROBE_DEFER)
149 			dev_err(dev->dev, "get reset controller failed %d\n",
150 				err);
151 		dev->reset = NULL;
152 		return err;
153 	}
154 
155 	return lima_clk_enable(dev);
156 }
157 
lima_clk_fini(struct lima_device * dev)158 static void lima_clk_fini(struct lima_device *dev)
159 {
160 	lima_clk_disable(dev);
161 }
162 
lima_regulator_enable(struct lima_device * dev)163 static int lima_regulator_enable(struct lima_device *dev)
164 {
165 	int ret;
166 
167 	if (!dev->regulator)
168 		return 0;
169 
170 	ret = regulator_enable(dev->regulator);
171 	if (ret < 0) {
172 		dev_err(dev->dev, "failed to enable regulator: %d\n", ret);
173 		return ret;
174 	}
175 
176 	return 0;
177 }
178 
lima_regulator_disable(struct lima_device * dev)179 static void lima_regulator_disable(struct lima_device *dev)
180 {
181 	if (dev->regulator)
182 		regulator_disable(dev->regulator);
183 }
184 
lima_regulator_init(struct lima_device * dev)185 static int lima_regulator_init(struct lima_device *dev)
186 {
187 	int ret;
188 
189 	dev->regulator = devm_regulator_get_optional(dev->dev, "mali");
190 	if (IS_ERR(dev->regulator)) {
191 		ret = PTR_ERR(dev->regulator);
192 		dev->regulator = NULL;
193 		if (ret == -ENODEV)
194 			return 0;
195 		if (ret != -EPROBE_DEFER)
196 			dev_err(dev->dev, "failed to get regulator: %d\n", ret);
197 		return ret;
198 	}
199 
200 	return lima_regulator_enable(dev);
201 }
202 
lima_regulator_fini(struct lima_device * dev)203 static void lima_regulator_fini(struct lima_device *dev)
204 {
205 	lima_regulator_disable(dev);
206 }
207 
lima_init_ip(struct lima_device * dev,int index)208 static int lima_init_ip(struct lima_device *dev, int index)
209 {
210 	struct platform_device *pdev = to_platform_device(dev->dev);
211 	struct lima_ip_desc *desc = lima_ip_desc + index;
212 	struct lima_ip *ip = dev->ip + index;
213 	const char *irq_name = desc->irq_name;
214 	int offset = desc->offset[dev->id];
215 	bool must = desc->must_have[dev->id];
216 	int err;
217 
218 	if (offset < 0)
219 		return 0;
220 
221 	ip->dev = dev;
222 	ip->id = index;
223 	ip->iomem = dev->iomem + offset;
224 	if (irq_name) {
225 		err = must ? platform_get_irq_byname(pdev, irq_name) :
226 			     platform_get_irq_byname_optional(pdev, irq_name);
227 		if (err < 0)
228 			goto out;
229 		ip->irq = err;
230 	}
231 
232 	err = desc->init(ip);
233 	if (!err) {
234 		ip->present = true;
235 		return 0;
236 	}
237 
238 out:
239 	return must ? err : 0;
240 }
241 
lima_fini_ip(struct lima_device * ldev,int index)242 static void lima_fini_ip(struct lima_device *ldev, int index)
243 {
244 	struct lima_ip_desc *desc = lima_ip_desc + index;
245 	struct lima_ip *ip = ldev->ip + index;
246 
247 	if (ip->present)
248 		desc->fini(ip);
249 }
250 
lima_resume_ip(struct lima_device * ldev,int index)251 static int lima_resume_ip(struct lima_device *ldev, int index)
252 {
253 	struct lima_ip_desc *desc = lima_ip_desc + index;
254 	struct lima_ip *ip = ldev->ip + index;
255 	int ret = 0;
256 
257 	if (ip->present)
258 		ret = desc->resume(ip);
259 
260 	return ret;
261 }
262 
lima_suspend_ip(struct lima_device * ldev,int index)263 static void lima_suspend_ip(struct lima_device *ldev, int index)
264 {
265 	struct lima_ip_desc *desc = lima_ip_desc + index;
266 	struct lima_ip *ip = ldev->ip + index;
267 
268 	if (ip->present)
269 		desc->suspend(ip);
270 }
271 
lima_init_gp_pipe(struct lima_device * dev)272 static int lima_init_gp_pipe(struct lima_device *dev)
273 {
274 	struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_gp;
275 	int err;
276 
277 	pipe->ldev = dev;
278 
279 	err = lima_sched_pipe_init(pipe, "gp");
280 	if (err)
281 		return err;
282 
283 	pipe->l2_cache[pipe->num_l2_cache++] = dev->ip + lima_ip_l2_cache0;
284 	pipe->mmu[pipe->num_mmu++] = dev->ip + lima_ip_gpmmu;
285 	pipe->processor[pipe->num_processor++] = dev->ip + lima_ip_gp;
286 
287 	err = lima_gp_pipe_init(dev);
288 	if (err) {
289 		lima_sched_pipe_fini(pipe);
290 		return err;
291 	}
292 
293 	return 0;
294 }
295 
lima_fini_gp_pipe(struct lima_device * dev)296 static void lima_fini_gp_pipe(struct lima_device *dev)
297 {
298 	struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_gp;
299 
300 	lima_gp_pipe_fini(dev);
301 	lima_sched_pipe_fini(pipe);
302 }
303 
lima_init_pp_pipe(struct lima_device * dev)304 static int lima_init_pp_pipe(struct lima_device *dev)
305 {
306 	struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp;
307 	int err, i;
308 
309 	pipe->ldev = dev;
310 
311 	err = lima_sched_pipe_init(pipe, "pp");
312 	if (err)
313 		return err;
314 
315 	for (i = 0; i < LIMA_SCHED_PIPE_MAX_PROCESSOR; i++) {
316 		struct lima_ip *pp = dev->ip + lima_ip_pp0 + i;
317 		struct lima_ip *ppmmu = dev->ip + lima_ip_ppmmu0 + i;
318 		struct lima_ip *l2_cache;
319 
320 		if (dev->id == lima_gpu_mali400)
321 			l2_cache = dev->ip + lima_ip_l2_cache0;
322 		else
323 			l2_cache = dev->ip + lima_ip_l2_cache1 + (i >> 2);
324 
325 		if (pp->present && ppmmu->present && l2_cache->present) {
326 			pipe->mmu[pipe->num_mmu++] = ppmmu;
327 			pipe->processor[pipe->num_processor++] = pp;
328 			if (!pipe->l2_cache[i >> 2])
329 				pipe->l2_cache[pipe->num_l2_cache++] = l2_cache;
330 		}
331 	}
332 
333 	if (dev->ip[lima_ip_bcast].present) {
334 		pipe->bcast_processor = dev->ip + lima_ip_pp_bcast;
335 		pipe->bcast_mmu = dev->ip + lima_ip_ppmmu_bcast;
336 	}
337 
338 	err = lima_pp_pipe_init(dev);
339 	if (err) {
340 		lima_sched_pipe_fini(pipe);
341 		return err;
342 	}
343 
344 	return 0;
345 }
346 
lima_fini_pp_pipe(struct lima_device * dev)347 static void lima_fini_pp_pipe(struct lima_device *dev)
348 {
349 	struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp;
350 
351 	lima_pp_pipe_fini(dev);
352 	lima_sched_pipe_fini(pipe);
353 }
354 
lima_device_init(struct lima_device * ldev)355 int lima_device_init(struct lima_device *ldev)
356 {
357 	struct platform_device *pdev = to_platform_device(ldev->dev);
358 	int err, i;
359 
360 	dma_set_coherent_mask(ldev->dev, DMA_BIT_MASK(32));
361 	dma_set_max_seg_size(ldev->dev, UINT_MAX);
362 
363 	err = lima_clk_init(ldev);
364 	if (err)
365 		return err;
366 
367 	err = lima_regulator_init(ldev);
368 	if (err)
369 		goto err_out0;
370 
371 	ldev->empty_vm = lima_vm_create(ldev);
372 	if (!ldev->empty_vm) {
373 		err = -ENOMEM;
374 		goto err_out1;
375 	}
376 
377 	ldev->va_start = 0;
378 	if (ldev->id == lima_gpu_mali450) {
379 		ldev->va_end = LIMA_VA_RESERVE_START;
380 		ldev->dlbu_cpu = dma_alloc_wc(
381 			ldev->dev, LIMA_PAGE_SIZE,
382 			&ldev->dlbu_dma, GFP_KERNEL | __GFP_NOWARN);
383 		if (!ldev->dlbu_cpu) {
384 			err = -ENOMEM;
385 			goto err_out2;
386 		}
387 	} else
388 		ldev->va_end = LIMA_VA_RESERVE_END;
389 
390 	ldev->iomem = devm_platform_ioremap_resource(pdev, 0);
391 	if (IS_ERR(ldev->iomem)) {
392 		dev_err(ldev->dev, "fail to ioremap iomem\n");
393 		err = PTR_ERR(ldev->iomem);
394 		goto err_out3;
395 	}
396 
397 	for (i = 0; i < lima_ip_num; i++) {
398 		err = lima_init_ip(ldev, i);
399 		if (err)
400 			goto err_out4;
401 	}
402 
403 	err = lima_init_gp_pipe(ldev);
404 	if (err)
405 		goto err_out4;
406 
407 	err = lima_init_pp_pipe(ldev);
408 	if (err)
409 		goto err_out5;
410 
411 	ldev->dump.magic = LIMA_DUMP_MAGIC;
412 	ldev->dump.version_major = LIMA_DUMP_MAJOR;
413 	ldev->dump.version_minor = LIMA_DUMP_MINOR;
414 	INIT_LIST_HEAD(&ldev->error_task_list);
415 	mutex_init(&ldev->error_task_list_lock);
416 
417 	dev_info(ldev->dev, "bus rate = %lu\n", clk_get_rate(ldev->clk_bus));
418 	dev_info(ldev->dev, "mod rate = %lu", clk_get_rate(ldev->clk_gpu));
419 
420 	return 0;
421 
422 err_out5:
423 	lima_fini_gp_pipe(ldev);
424 err_out4:
425 	while (--i >= 0)
426 		lima_fini_ip(ldev, i);
427 err_out3:
428 	if (ldev->dlbu_cpu)
429 		dma_free_wc(ldev->dev, LIMA_PAGE_SIZE,
430 			    ldev->dlbu_cpu, ldev->dlbu_dma);
431 err_out2:
432 	lima_vm_put(ldev->empty_vm);
433 err_out1:
434 	lima_regulator_fini(ldev);
435 err_out0:
436 	lima_clk_fini(ldev);
437 	return err;
438 }
439 
lima_device_fini(struct lima_device * ldev)440 void lima_device_fini(struct lima_device *ldev)
441 {
442 	int i;
443 	struct lima_sched_error_task *et, *tmp;
444 
445 	list_for_each_entry_safe(et, tmp, &ldev->error_task_list, list) {
446 		list_del(&et->list);
447 		kvfree(et);
448 	}
449 	mutex_destroy(&ldev->error_task_list_lock);
450 
451 	lima_fini_pp_pipe(ldev);
452 	lima_fini_gp_pipe(ldev);
453 
454 	for (i = lima_ip_num - 1; i >= 0; i--)
455 		lima_fini_ip(ldev, i);
456 
457 	if (ldev->dlbu_cpu)
458 		dma_free_wc(ldev->dev, LIMA_PAGE_SIZE,
459 			    ldev->dlbu_cpu, ldev->dlbu_dma);
460 
461 	lima_vm_put(ldev->empty_vm);
462 
463 	lima_regulator_fini(ldev);
464 
465 	lima_clk_fini(ldev);
466 }
467 
lima_device_resume(struct device * dev)468 int lima_device_resume(struct device *dev)
469 {
470 	struct lima_device *ldev = dev_get_drvdata(dev);
471 	int i, err;
472 
473 	err = lima_clk_enable(ldev);
474 	if (err) {
475 		dev_err(dev, "resume clk fail %d\n", err);
476 		return err;
477 	}
478 
479 	err = lima_regulator_enable(ldev);
480 	if (err) {
481 		dev_err(dev, "resume regulator fail %d\n", err);
482 		goto err_out0;
483 	}
484 
485 	for (i = 0; i < lima_ip_num; i++) {
486 		err = lima_resume_ip(ldev, i);
487 		if (err) {
488 			dev_err(dev, "resume ip %d fail\n", i);
489 			goto err_out1;
490 		}
491 	}
492 
493 	err = lima_devfreq_resume(&ldev->devfreq);
494 	if (err) {
495 		dev_err(dev, "devfreq resume fail\n");
496 		goto err_out1;
497 	}
498 
499 	return 0;
500 
501 err_out1:
502 	while (--i >= 0)
503 		lima_suspend_ip(ldev, i);
504 	lima_regulator_disable(ldev);
505 err_out0:
506 	lima_clk_disable(ldev);
507 	return err;
508 }
509 
lima_device_suspend(struct device * dev)510 int lima_device_suspend(struct device *dev)
511 {
512 	struct lima_device *ldev = dev_get_drvdata(dev);
513 	int i, err;
514 
515 	/* check any task running */
516 	for (i = 0; i < lima_pipe_num; i++) {
517 		if (atomic_read(&ldev->pipe[i].base.hw_rq_count))
518 			return -EBUSY;
519 	}
520 
521 	err = lima_devfreq_suspend(&ldev->devfreq);
522 	if (err) {
523 		dev_err(dev, "devfreq suspend fail\n");
524 		return err;
525 	}
526 
527 	for (i = lima_ip_num - 1; i >= 0; i--)
528 		lima_suspend_ip(ldev, i);
529 
530 	lima_regulator_disable(ldev);
531 
532 	lima_clk_disable(ldev);
533 
534 	return 0;
535 }
536