1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Support for Medifield PNW Camera Imaging ISP subsystem.
4  *
5  * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
6  *
7  * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
8  *
9  * This program is free software; you can redistribute it and/or
10  * modify it under the terms of the GNU General Public License version
11  * 2 as published by the Free Software Foundation.
12  *
13  * This program is distributed in the hope that it will be useful,
14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
16  * GNU General Public License for more details.
17  *
18  *
19  */
20 /*
21  * ISP MMU management wrap code
22  */
23 #include <linux/kernel.h>
24 #include <linux/types.h>
25 #include <linux/gfp.h>
26 #include <linux/mm.h>		/* for GFP_ATOMIC */
27 #include <linux/slab.h>		/* for kmalloc */
28 #include <linux/list.h>
29 #include <linux/io.h>
30 #include <linux/module.h>
31 #include <linux/moduleparam.h>
32 #include <linux/string.h>
33 #include <linux/errno.h>
34 #include <linux/sizes.h>
35 
36 #ifdef CONFIG_X86
37 #include <asm/set_memory.h>
38 #endif
39 
40 #include "atomisp_internal.h"
41 #include "mmu/isp_mmu.h"
42 
43 /*
44  * 64-bit x86 processor physical address layout:
45  * 0		- 0x7fffffff		DDR RAM	(2GB)
46  * 0x80000000	- 0xffffffff		MMIO	(2GB)
47  * 0x100000000	- 0x3fffffffffff	DDR RAM	(64TB)
48  * So if the system has more than 2GB DDR memory, the lower 2GB occupies the
49  * physical address 0 - 0x7fffffff and the rest will start from 0x100000000.
50  * We have to make sure memory is allocated from the lower 2GB for devices
51  * that are only 32-bit capable(e.g. the ISP MMU).
52  *
53  * For any confusion, contact bin.gao@intel.com.
54  */
55 #define NR_PAGES_2GB	(SZ_2G / PAGE_SIZE)
56 
57 static void free_mmu_map(struct isp_mmu *mmu, unsigned int start_isp_virt,
58 			 unsigned int end_isp_virt);
59 
atomisp_get_pte(phys_addr_t pt,unsigned int idx)60 static unsigned int atomisp_get_pte(phys_addr_t pt, unsigned int idx)
61 {
62 	unsigned int *pt_virt = phys_to_virt(pt);
63 
64 	return *(pt_virt + idx);
65 }
66 
atomisp_set_pte(phys_addr_t pt,unsigned int idx,unsigned int pte)67 static void atomisp_set_pte(phys_addr_t pt,
68 			    unsigned int idx, unsigned int pte)
69 {
70 	unsigned int *pt_virt = phys_to_virt(pt);
71 	*(pt_virt + idx) = pte;
72 }
73 
isp_pt_phys_to_virt(phys_addr_t phys)74 static void *isp_pt_phys_to_virt(phys_addr_t phys)
75 {
76 	return phys_to_virt(phys);
77 }
78 
isp_pte_to_pgaddr(struct isp_mmu * mmu,unsigned int pte)79 static phys_addr_t isp_pte_to_pgaddr(struct isp_mmu *mmu,
80 				     unsigned int pte)
81 {
82 	return mmu->driver->pte_to_phys(mmu, pte);
83 }
84 
isp_pgaddr_to_pte_valid(struct isp_mmu * mmu,phys_addr_t phys)85 static unsigned int isp_pgaddr_to_pte_valid(struct isp_mmu *mmu,
86 	phys_addr_t phys)
87 {
88 	unsigned int pte = mmu->driver->phys_to_pte(mmu, phys);
89 
90 	return (unsigned int)(pte | ISP_PTE_VALID_MASK(mmu));
91 }
92 
93 /*
94  * allocate a uncacheable page table.
95  * return physical address.
96  */
alloc_page_table(struct isp_mmu * mmu)97 static phys_addr_t alloc_page_table(struct isp_mmu *mmu)
98 {
99 	int i;
100 	phys_addr_t page;
101 	void *virt;
102 
103 	virt = (void *)__get_free_page(GFP_KERNEL | GFP_DMA32);
104 
105 	if (!virt)
106 		return (phys_addr_t)NULL_PAGE;
107 
108 	/*
109 	 * we need a uncacheable page table.
110 	 */
111 #ifdef	CONFIG_X86
112 	set_memory_uc((unsigned long)virt, 1);
113 #endif
114 
115 	page = virt_to_phys(virt);
116 
117 	for (i = 0; i < 1024; i++) {
118 		/* NEED CHECK */
119 		atomisp_set_pte(page, i, mmu->driver->null_pte);
120 	}
121 
122 	return page;
123 }
124 
free_page_table(struct isp_mmu * mmu,phys_addr_t page)125 static void free_page_table(struct isp_mmu *mmu, phys_addr_t page)
126 {
127 	void *virt;
128 
129 	page &= ISP_PAGE_MASK;
130 	/*
131 	 * reset the page to write back before free
132 	 */
133 	virt = phys_to_virt(page);
134 
135 #ifdef	CONFIG_X86
136 	set_memory_wb((unsigned long)virt, 1);
137 #endif
138 
139 	free_page((unsigned long)virt);
140 }
141 
mmu_remap_error(struct isp_mmu * mmu,phys_addr_t l1_pt,unsigned int l1_idx,phys_addr_t l2_pt,unsigned int l2_idx,unsigned int isp_virt,phys_addr_t old_phys,phys_addr_t new_phys)142 static void mmu_remap_error(struct isp_mmu *mmu,
143 			    phys_addr_t l1_pt, unsigned int l1_idx,
144 			    phys_addr_t l2_pt, unsigned int l2_idx,
145 			    unsigned int isp_virt, phys_addr_t old_phys,
146 			    phys_addr_t new_phys)
147 {
148 	dev_err(atomisp_dev, "address remap:\n\n"
149 		"\tL1 PT: virt = %p, phys = 0x%llx, idx = %d\n"
150 		"\tL2 PT: virt = %p, phys = 0x%llx, idx = %d\n"
151 		"\told: isp_virt = 0x%x, phys = 0x%llx\n"
152 		"\tnew: isp_virt = 0x%x, phys = 0x%llx\n",
153 		isp_pt_phys_to_virt(l1_pt),
154 		(u64)l1_pt, l1_idx,
155 		isp_pt_phys_to_virt(l2_pt),
156 		(u64)l2_pt, l2_idx, isp_virt,
157 		(u64)old_phys, isp_virt,
158 		(u64)new_phys);
159 }
160 
mmu_unmap_l2_pte_error(struct isp_mmu * mmu,phys_addr_t l1_pt,unsigned int l1_idx,phys_addr_t l2_pt,unsigned int l2_idx,unsigned int isp_virt,unsigned int pte)161 static void mmu_unmap_l2_pte_error(struct isp_mmu *mmu,
162 				   phys_addr_t l1_pt, unsigned int l1_idx,
163 				   phys_addr_t l2_pt, unsigned int l2_idx,
164 				   unsigned int isp_virt, unsigned int pte)
165 {
166 	dev_err(atomisp_dev, "unmap invalid L2 pte:\n\n"
167 		"\tL1 PT: virt = %p, phys = 0x%llx, idx = %d\n"
168 		"\tL2 PT: virt = %p, phys = 0x%llx, idx = %d\n"
169 		"\tisp_virt = 0x%x, pte(page phys) = 0x%x\n",
170 		isp_pt_phys_to_virt(l1_pt),
171 		(u64)l1_pt, l1_idx,
172 		isp_pt_phys_to_virt(l2_pt),
173 		(u64)l2_pt, l2_idx, isp_virt,
174 		pte);
175 }
176 
mmu_unmap_l1_pte_error(struct isp_mmu * mmu,phys_addr_t l1_pt,unsigned int l1_idx,unsigned int isp_virt,unsigned int pte)177 static void mmu_unmap_l1_pte_error(struct isp_mmu *mmu,
178 				   phys_addr_t l1_pt, unsigned int l1_idx,
179 				   unsigned int isp_virt, unsigned int pte)
180 {
181 	dev_err(atomisp_dev, "unmap invalid L1 pte (L2 PT):\n\n"
182 		"\tL1 PT: virt = %p, phys = 0x%llx, idx = %d\n"
183 		"\tisp_virt = 0x%x, l1_pte(L2 PT) = 0x%x\n",
184 		isp_pt_phys_to_virt(l1_pt),
185 		(u64)l1_pt, l1_idx, (unsigned int)isp_virt,
186 		pte);
187 }
188 
mmu_unmap_l1_pt_error(struct isp_mmu * mmu,unsigned int pte)189 static void mmu_unmap_l1_pt_error(struct isp_mmu *mmu, unsigned int pte)
190 {
191 	dev_err(atomisp_dev, "unmap invalid L1PT:\n\n"
192 		"L1PT = 0x%x\n", (unsigned int)pte);
193 }
194 
195 /*
196  * Update L2 page table according to isp virtual address and page physical
197  * address
198  */
mmu_l2_map(struct isp_mmu * mmu,phys_addr_t l1_pt,unsigned int l1_idx,phys_addr_t l2_pt,unsigned int start,unsigned int end,phys_addr_t phys)199 static int mmu_l2_map(struct isp_mmu *mmu, phys_addr_t l1_pt,
200 		      unsigned int l1_idx, phys_addr_t l2_pt,
201 		      unsigned int start, unsigned int end, phys_addr_t phys)
202 {
203 	unsigned int ptr;
204 	unsigned int idx;
205 	unsigned int pte;
206 
207 	l2_pt &= ISP_PAGE_MASK;
208 
209 	start = start & ISP_PAGE_MASK;
210 	end = ISP_PAGE_ALIGN(end);
211 	phys &= ISP_PAGE_MASK;
212 
213 	ptr = start;
214 	do {
215 		idx = ISP_PTR_TO_L2_IDX(ptr);
216 
217 		pte = atomisp_get_pte(l2_pt, idx);
218 
219 		if (ISP_PTE_VALID(mmu, pte)) {
220 			mmu_remap_error(mmu, l1_pt, l1_idx,
221 					l2_pt, idx, ptr, pte, phys);
222 
223 			/* free all mapped pages */
224 			free_mmu_map(mmu, start, ptr);
225 
226 			return -EINVAL;
227 		}
228 
229 		pte = isp_pgaddr_to_pte_valid(mmu, phys);
230 
231 		atomisp_set_pte(l2_pt, idx, pte);
232 		mmu->l2_pgt_refcount[l1_idx]++;
233 		ptr += (1U << ISP_L2PT_OFFSET);
234 		phys += (1U << ISP_L2PT_OFFSET);
235 	} while (ptr < end && idx < ISP_L2PT_PTES - 1);
236 
237 	return 0;
238 }
239 
240 /*
241  * Update L1 page table according to isp virtual address and page physical
242  * address
243  */
mmu_l1_map(struct isp_mmu * mmu,phys_addr_t l1_pt,unsigned int start,unsigned int end,phys_addr_t phys)244 static int mmu_l1_map(struct isp_mmu *mmu, phys_addr_t l1_pt,
245 		      unsigned int start, unsigned int end,
246 		      phys_addr_t phys)
247 {
248 	phys_addr_t l2_pt;
249 	unsigned int ptr, l1_aligned;
250 	unsigned int idx;
251 	unsigned int l2_pte;
252 	int ret;
253 
254 	l1_pt &= ISP_PAGE_MASK;
255 
256 	start = start & ISP_PAGE_MASK;
257 	end = ISP_PAGE_ALIGN(end);
258 	phys &= ISP_PAGE_MASK;
259 
260 	ptr = start;
261 	do {
262 		idx = ISP_PTR_TO_L1_IDX(ptr);
263 
264 		l2_pte = atomisp_get_pte(l1_pt, idx);
265 
266 		if (!ISP_PTE_VALID(mmu, l2_pte)) {
267 			l2_pt = alloc_page_table(mmu);
268 			if (l2_pt == NULL_PAGE) {
269 				dev_err(atomisp_dev,
270 					"alloc page table fail.\n");
271 
272 				/* free all mapped pages */
273 				free_mmu_map(mmu, start, ptr);
274 
275 				return -ENOMEM;
276 			}
277 
278 			l2_pte = isp_pgaddr_to_pte_valid(mmu, l2_pt);
279 
280 			atomisp_set_pte(l1_pt, idx, l2_pte);
281 			mmu->l2_pgt_refcount[idx] = 0;
282 		}
283 
284 		l2_pt = isp_pte_to_pgaddr(mmu, l2_pte);
285 
286 		l1_aligned = (ptr & ISP_PAGE_MASK) + (1U << ISP_L1PT_OFFSET);
287 
288 		if (l1_aligned < end) {
289 			ret = mmu_l2_map(mmu, l1_pt, idx,
290 					 l2_pt, ptr, l1_aligned, phys);
291 			phys += (l1_aligned - ptr);
292 			ptr = l1_aligned;
293 		} else {
294 			ret = mmu_l2_map(mmu, l1_pt, idx,
295 					 l2_pt, ptr, end, phys);
296 			phys += (end - ptr);
297 			ptr = end;
298 		}
299 
300 		if (ret) {
301 			dev_err(atomisp_dev, "setup mapping in L2PT fail.\n");
302 
303 			/* free all mapped pages */
304 			free_mmu_map(mmu, start, ptr);
305 
306 			return -EINVAL;
307 		}
308 	} while (ptr < end && idx < ISP_L1PT_PTES);
309 
310 	return 0;
311 }
312 
313 /*
314  * Update page table according to isp virtual address and page physical
315  * address
316  */
mmu_map(struct isp_mmu * mmu,unsigned int isp_virt,phys_addr_t phys,unsigned int pgnr)317 static int mmu_map(struct isp_mmu *mmu, unsigned int isp_virt,
318 		   phys_addr_t phys, unsigned int pgnr)
319 {
320 	unsigned int start, end;
321 	phys_addr_t l1_pt;
322 	int ret;
323 
324 	mutex_lock(&mmu->pt_mutex);
325 	if (!ISP_PTE_VALID(mmu, mmu->l1_pte)) {
326 		/*
327 		 * allocate 1 new page for L1 page table
328 		 */
329 		l1_pt = alloc_page_table(mmu);
330 		if (l1_pt == NULL_PAGE) {
331 			dev_err(atomisp_dev, "alloc page table fail.\n");
332 			mutex_unlock(&mmu->pt_mutex);
333 			return -ENOMEM;
334 		}
335 
336 		/*
337 		 * setup L1 page table physical addr to MMU
338 		 */
339 		mmu->base_address = l1_pt;
340 		mmu->l1_pte = isp_pgaddr_to_pte_valid(mmu, l1_pt);
341 		memset(mmu->l2_pgt_refcount, 0, sizeof(int) * ISP_L1PT_PTES);
342 	}
343 
344 	l1_pt = isp_pte_to_pgaddr(mmu, mmu->l1_pte);
345 
346 	start = (isp_virt) & ISP_PAGE_MASK;
347 	end = start + (pgnr << ISP_PAGE_OFFSET);
348 	phys &= ISP_PAGE_MASK;
349 
350 	ret = mmu_l1_map(mmu, l1_pt, start, end, phys);
351 
352 	if (ret)
353 		dev_err(atomisp_dev, "setup mapping in L1PT fail.\n");
354 
355 	mutex_unlock(&mmu->pt_mutex);
356 	return ret;
357 }
358 
359 /*
360  * Free L2 page table according to isp virtual address and page physical
361  * address
362  */
mmu_l2_unmap(struct isp_mmu * mmu,phys_addr_t l1_pt,unsigned int l1_idx,phys_addr_t l2_pt,unsigned int start,unsigned int end)363 static void mmu_l2_unmap(struct isp_mmu *mmu, phys_addr_t l1_pt,
364 			 unsigned int l1_idx, phys_addr_t l2_pt,
365 			 unsigned int start, unsigned int end)
366 {
367 	unsigned int ptr;
368 	unsigned int idx;
369 	unsigned int pte;
370 
371 	l2_pt &= ISP_PAGE_MASK;
372 
373 	start = start & ISP_PAGE_MASK;
374 	end = ISP_PAGE_ALIGN(end);
375 
376 	ptr = start;
377 	do {
378 		idx = ISP_PTR_TO_L2_IDX(ptr);
379 
380 		pte = atomisp_get_pte(l2_pt, idx);
381 
382 		if (!ISP_PTE_VALID(mmu, pte))
383 			mmu_unmap_l2_pte_error(mmu, l1_pt, l1_idx,
384 					       l2_pt, idx, ptr, pte);
385 
386 		atomisp_set_pte(l2_pt, idx, mmu->driver->null_pte);
387 		mmu->l2_pgt_refcount[l1_idx]--;
388 		ptr += (1U << ISP_L2PT_OFFSET);
389 	} while (ptr < end && idx < ISP_L2PT_PTES - 1);
390 
391 	if (mmu->l2_pgt_refcount[l1_idx] == 0) {
392 		free_page_table(mmu, l2_pt);
393 		atomisp_set_pte(l1_pt, l1_idx, mmu->driver->null_pte);
394 	}
395 }
396 
397 /*
398  * Free L1 page table according to isp virtual address and page physical
399  * address
400  */
mmu_l1_unmap(struct isp_mmu * mmu,phys_addr_t l1_pt,unsigned int start,unsigned int end)401 static void mmu_l1_unmap(struct isp_mmu *mmu, phys_addr_t l1_pt,
402 			 unsigned int start, unsigned int end)
403 {
404 	phys_addr_t l2_pt;
405 	unsigned int ptr, l1_aligned;
406 	unsigned int idx;
407 	unsigned int l2_pte;
408 
409 	l1_pt &= ISP_PAGE_MASK;
410 
411 	start = start & ISP_PAGE_MASK;
412 	end = ISP_PAGE_ALIGN(end);
413 
414 	ptr = start;
415 	do {
416 		idx = ISP_PTR_TO_L1_IDX(ptr);
417 
418 		l2_pte = atomisp_get_pte(l1_pt, idx);
419 
420 		if (!ISP_PTE_VALID(mmu, l2_pte)) {
421 			mmu_unmap_l1_pte_error(mmu, l1_pt, idx, ptr, l2_pte);
422 			continue;
423 		}
424 
425 		l2_pt = isp_pte_to_pgaddr(mmu, l2_pte);
426 
427 		l1_aligned = (ptr & ISP_PAGE_MASK) + (1U << ISP_L1PT_OFFSET);
428 
429 		if (l1_aligned < end) {
430 			mmu_l2_unmap(mmu, l1_pt, idx, l2_pt, ptr, l1_aligned);
431 			ptr = l1_aligned;
432 		} else {
433 			mmu_l2_unmap(mmu, l1_pt, idx, l2_pt, ptr, end);
434 			ptr = end;
435 		}
436 		/*
437 		 * use the same L2 page next time, so we don't
438 		 * need to invalidate and free this PT.
439 		 */
440 		/*      atomisp_set_pte(l1_pt, idx, NULL_PTE); */
441 	} while (ptr < end && idx < ISP_L1PT_PTES);
442 }
443 
444 /*
445  * Free page table according to isp virtual address and page physical
446  * address
447  */
mmu_unmap(struct isp_mmu * mmu,unsigned int isp_virt,unsigned int pgnr)448 static void mmu_unmap(struct isp_mmu *mmu, unsigned int isp_virt,
449 		      unsigned int pgnr)
450 {
451 	unsigned int start, end;
452 	phys_addr_t l1_pt;
453 
454 	mutex_lock(&mmu->pt_mutex);
455 	if (!ISP_PTE_VALID(mmu, mmu->l1_pte)) {
456 		mmu_unmap_l1_pt_error(mmu, mmu->l1_pte);
457 		mutex_unlock(&mmu->pt_mutex);
458 		return;
459 	}
460 
461 	l1_pt = isp_pte_to_pgaddr(mmu, mmu->l1_pte);
462 
463 	start = (isp_virt) & ISP_PAGE_MASK;
464 	end = start + (pgnr << ISP_PAGE_OFFSET);
465 
466 	mmu_l1_unmap(mmu, l1_pt, start, end);
467 	mutex_unlock(&mmu->pt_mutex);
468 }
469 
470 /*
471  * Free page tables according to isp start virtual address and end virtual
472  * address.
473  */
free_mmu_map(struct isp_mmu * mmu,unsigned int start_isp_virt,unsigned int end_isp_virt)474 static void free_mmu_map(struct isp_mmu *mmu, unsigned int start_isp_virt,
475 			 unsigned int end_isp_virt)
476 {
477 	unsigned int pgnr;
478 	unsigned int start, end;
479 
480 	start = (start_isp_virt) & ISP_PAGE_MASK;
481 	end = (end_isp_virt) & ISP_PAGE_MASK;
482 	pgnr = (end - start) >> ISP_PAGE_OFFSET;
483 	mmu_unmap(mmu, start, pgnr);
484 }
485 
isp_mmu_map(struct isp_mmu * mmu,unsigned int isp_virt,phys_addr_t phys,unsigned int pgnr)486 int isp_mmu_map(struct isp_mmu *mmu, unsigned int isp_virt,
487 		phys_addr_t phys, unsigned int pgnr)
488 {
489 	return mmu_map(mmu, isp_virt, phys, pgnr);
490 }
491 
isp_mmu_unmap(struct isp_mmu * mmu,unsigned int isp_virt,unsigned int pgnr)492 void isp_mmu_unmap(struct isp_mmu *mmu, unsigned int isp_virt,
493 		   unsigned int pgnr)
494 {
495 	mmu_unmap(mmu, isp_virt, pgnr);
496 }
497 
isp_mmu_flush_tlb_range_default(struct isp_mmu * mmu,unsigned int start,unsigned int size)498 static void isp_mmu_flush_tlb_range_default(struct isp_mmu *mmu,
499 	unsigned int start,
500 	unsigned int size)
501 {
502 	isp_mmu_flush_tlb(mmu);
503 }
504 
505 /*MMU init for internal structure*/
isp_mmu_init(struct isp_mmu * mmu,struct isp_mmu_client * driver)506 int isp_mmu_init(struct isp_mmu *mmu, struct isp_mmu_client *driver)
507 {
508 	if (!mmu)		/* error */
509 		return -EINVAL;
510 	if (!driver)		/* error */
511 		return -EINVAL;
512 
513 	if (!driver->name)
514 		dev_warn(atomisp_dev, "NULL name for MMU driver...\n");
515 
516 	mmu->driver = driver;
517 
518 	if (!driver->tlb_flush_all) {
519 		dev_err(atomisp_dev, "tlb_flush_all operation not provided.\n");
520 		return -EINVAL;
521 	}
522 
523 	if (!driver->tlb_flush_range)
524 		driver->tlb_flush_range = isp_mmu_flush_tlb_range_default;
525 
526 	if (!driver->pte_valid_mask) {
527 		dev_err(atomisp_dev, "PTE_MASK is missing from mmu driver\n");
528 		return -EINVAL;
529 	}
530 
531 	mmu->l1_pte = driver->null_pte;
532 
533 	mutex_init(&mmu->pt_mutex);
534 
535 	return 0;
536 }
537 
538 /*Free L1 and L2 page table*/
isp_mmu_exit(struct isp_mmu * mmu)539 void isp_mmu_exit(struct isp_mmu *mmu)
540 {
541 	unsigned int idx;
542 	unsigned int pte;
543 	phys_addr_t l1_pt, l2_pt;
544 
545 	if (!mmu)
546 		return;
547 
548 	if (!ISP_PTE_VALID(mmu, mmu->l1_pte)) {
549 		dev_warn(atomisp_dev, "invalid L1PT: pte = 0x%x\n",
550 			 (unsigned int)mmu->l1_pte);
551 		return;
552 	}
553 
554 	l1_pt = isp_pte_to_pgaddr(mmu, mmu->l1_pte);
555 
556 	for (idx = 0; idx < ISP_L1PT_PTES; idx++) {
557 		pte = atomisp_get_pte(l1_pt, idx);
558 
559 		if (ISP_PTE_VALID(mmu, pte)) {
560 			l2_pt = isp_pte_to_pgaddr(mmu, pte);
561 
562 			free_page_table(mmu, l2_pt);
563 		}
564 	}
565 
566 	free_page_table(mmu, l1_pt);
567 }
568