1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * This file contains common routines for dealing with free of page tables
4  * Along with common page table handling code
5  *
6  *  Derived from arch/powerpc/mm/tlb_64.c:
7  *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
8  *
9  *  Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au)
10  *  and Cort Dougan (PReP) (cort@cs.nmt.edu)
11  *    Copyright (C) 1996 Paul Mackerras
12  *
13  *  Derived from "arch/i386/mm/init.c"
14  *    Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
15  *
16  *  Dave Engebretsen <engebret@us.ibm.com>
17  *      Rework for PPC64 port.
18  */
19 
20 #include <linux/kernel.h>
21 #include <linux/gfp.h>
22 #include <linux/mm.h>
23 #include <linux/percpu.h>
24 #include <linux/hardirq.h>
25 #include <linux/hugetlb.h>
26 #include <asm/tlbflush.h>
27 #include <asm/tlb.h>
28 #include <asm/hugetlb.h>
29 #include <asm/pte-walk.h>
30 
31 #ifdef CONFIG_PPC64
32 #define PGD_ALIGN (sizeof(pgd_t) * MAX_PTRS_PER_PGD)
33 #else
34 #define PGD_ALIGN PAGE_SIZE
35 #endif
36 
37 pgd_t swapper_pg_dir[MAX_PTRS_PER_PGD] __section(".bss..page_aligned") __aligned(PGD_ALIGN);
38 
is_exec_fault(void)39 static inline int is_exec_fault(void)
40 {
41 	return current->thread.regs && TRAP(current->thread.regs) == 0x400;
42 }
43 
44 /* We only try to do i/d cache coherency on stuff that looks like
45  * reasonably "normal" PTEs. We currently require a PTE to be present
46  * and we avoid _PAGE_SPECIAL and cache inhibited pte. We also only do that
47  * on userspace PTEs
48  */
pte_looks_normal(pte_t pte)49 static inline int pte_looks_normal(pte_t pte)
50 {
51 
52 	if (pte_present(pte) && !pte_special(pte)) {
53 		if (pte_ci(pte))
54 			return 0;
55 		if (pte_user(pte))
56 			return 1;
57 	}
58 	return 0;
59 }
60 
maybe_pte_to_page(pte_t pte)61 static struct page *maybe_pte_to_page(pte_t pte)
62 {
63 	unsigned long pfn = pte_pfn(pte);
64 	struct page *page;
65 
66 	if (unlikely(!pfn_valid(pfn)))
67 		return NULL;
68 	page = pfn_to_page(pfn);
69 	if (PageReserved(page))
70 		return NULL;
71 	return page;
72 }
73 
74 #ifdef CONFIG_PPC_BOOK3S
75 
76 /* Server-style MMU handles coherency when hashing if HW exec permission
77  * is supposed per page (currently 64-bit only). If not, then, we always
78  * flush the cache for valid PTEs in set_pte. Embedded CPU without HW exec
79  * support falls into the same category.
80  */
81 
set_pte_filter_hash(pte_t pte)82 static pte_t set_pte_filter_hash(pte_t pte)
83 {
84 	pte = __pte(pte_val(pte) & ~_PAGE_HPTEFLAGS);
85 	if (pte_looks_normal(pte) && !(cpu_has_feature(CPU_FTR_COHERENT_ICACHE) ||
86 				       cpu_has_feature(CPU_FTR_NOEXECUTE))) {
87 		struct page *pg = maybe_pte_to_page(pte);
88 		if (!pg)
89 			return pte;
90 		if (!test_bit(PG_dcache_clean, &pg->flags)) {
91 			flush_dcache_icache_page(pg);
92 			set_bit(PG_dcache_clean, &pg->flags);
93 		}
94 	}
95 	return pte;
96 }
97 
98 #else /* CONFIG_PPC_BOOK3S */
99 
set_pte_filter_hash(pte_t pte)100 static pte_t set_pte_filter_hash(pte_t pte) { return pte; }
101 
102 #endif /* CONFIG_PPC_BOOK3S */
103 
104 /* Embedded type MMU with HW exec support. This is a bit more complicated
105  * as we don't have two bits to spare for _PAGE_EXEC and _PAGE_HWEXEC so
106  * instead we "filter out" the exec permission for non clean pages.
107  */
set_pte_filter(pte_t pte)108 static inline pte_t set_pte_filter(pte_t pte)
109 {
110 	struct page *pg;
111 
112 	if (radix_enabled())
113 		return pte;
114 
115 	if (mmu_has_feature(MMU_FTR_HPTE_TABLE))
116 		return set_pte_filter_hash(pte);
117 
118 	/* No exec permission in the first place, move on */
119 	if (!pte_exec(pte) || !pte_looks_normal(pte))
120 		return pte;
121 
122 	/* If you set _PAGE_EXEC on weird pages you're on your own */
123 	pg = maybe_pte_to_page(pte);
124 	if (unlikely(!pg))
125 		return pte;
126 
127 	/* If the page clean, we move on */
128 	if (test_bit(PG_dcache_clean, &pg->flags))
129 		return pte;
130 
131 	/* If it's an exec fault, we flush the cache and make it clean */
132 	if (is_exec_fault()) {
133 		flush_dcache_icache_page(pg);
134 		set_bit(PG_dcache_clean, &pg->flags);
135 		return pte;
136 	}
137 
138 	/* Else, we filter out _PAGE_EXEC */
139 	return pte_exprotect(pte);
140 }
141 
set_access_flags_filter(pte_t pte,struct vm_area_struct * vma,int dirty)142 static pte_t set_access_flags_filter(pte_t pte, struct vm_area_struct *vma,
143 				     int dirty)
144 {
145 	struct page *pg;
146 
147 	if (IS_ENABLED(CONFIG_PPC_BOOK3S_64))
148 		return pte;
149 
150 	if (mmu_has_feature(MMU_FTR_HPTE_TABLE))
151 		return pte;
152 
153 	/* So here, we only care about exec faults, as we use them
154 	 * to recover lost _PAGE_EXEC and perform I$/D$ coherency
155 	 * if necessary. Also if _PAGE_EXEC is already set, same deal,
156 	 * we just bail out
157 	 */
158 	if (dirty || pte_exec(pte) || !is_exec_fault())
159 		return pte;
160 
161 #ifdef CONFIG_DEBUG_VM
162 	/* So this is an exec fault, _PAGE_EXEC is not set. If it was
163 	 * an error we would have bailed out earlier in do_page_fault()
164 	 * but let's make sure of it
165 	 */
166 	if (WARN_ON(!(vma->vm_flags & VM_EXEC)))
167 		return pte;
168 #endif /* CONFIG_DEBUG_VM */
169 
170 	/* If you set _PAGE_EXEC on weird pages you're on your own */
171 	pg = maybe_pte_to_page(pte);
172 	if (unlikely(!pg))
173 		goto bail;
174 
175 	/* If the page is already clean, we move on */
176 	if (test_bit(PG_dcache_clean, &pg->flags))
177 		goto bail;
178 
179 	/* Clean the page and set PG_dcache_clean */
180 	flush_dcache_icache_page(pg);
181 	set_bit(PG_dcache_clean, &pg->flags);
182 
183  bail:
184 	return pte_mkexec(pte);
185 }
186 
187 /*
188  * set_pte stores a linux PTE into the linux page table.
189  */
set_pte_at(struct mm_struct * mm,unsigned long addr,pte_t * ptep,pte_t pte)190 void set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep,
191 		pte_t pte)
192 {
193 	/*
194 	 * Make sure hardware valid bit is not set. We don't do
195 	 * tlb flush for this update.
196 	 */
197 	VM_WARN_ON(pte_hw_valid(*ptep) && !pte_protnone(*ptep));
198 
199 	/* Note: mm->context.id might not yet have been assigned as
200 	 * this context might not have been activated yet when this
201 	 * is called.
202 	 */
203 	pte = set_pte_filter(pte);
204 
205 	/* Perform the setting of the PTE */
206 	__set_pte_at(mm, addr, ptep, pte, 0);
207 }
208 
unmap_kernel_page(unsigned long va)209 void unmap_kernel_page(unsigned long va)
210 {
211 	pmd_t *pmdp = pmd_off_k(va);
212 	pte_t *ptep = pte_offset_kernel(pmdp, va);
213 
214 	pte_clear(&init_mm, va, ptep);
215 	flush_tlb_kernel_range(va, va + PAGE_SIZE);
216 }
217 
218 /*
219  * This is called when relaxing access to a PTE. It's also called in the page
220  * fault path when we don't hit any of the major fault cases, ie, a minor
221  * update of _PAGE_ACCESSED, _PAGE_DIRTY, etc... The generic code will have
222  * handled those two for us, we additionally deal with missing execute
223  * permission here on some processors
224  */
ptep_set_access_flags(struct vm_area_struct * vma,unsigned long address,pte_t * ptep,pte_t entry,int dirty)225 int ptep_set_access_flags(struct vm_area_struct *vma, unsigned long address,
226 			  pte_t *ptep, pte_t entry, int dirty)
227 {
228 	int changed;
229 	entry = set_access_flags_filter(entry, vma, dirty);
230 	changed = !pte_same(*(ptep), entry);
231 	if (changed) {
232 		assert_pte_locked(vma->vm_mm, address);
233 		__ptep_set_access_flags(vma, ptep, entry,
234 					address, mmu_virtual_psize);
235 	}
236 	return changed;
237 }
238 
239 #ifdef CONFIG_HUGETLB_PAGE
huge_ptep_set_access_flags(struct vm_area_struct * vma,unsigned long addr,pte_t * ptep,pte_t pte,int dirty)240 int huge_ptep_set_access_flags(struct vm_area_struct *vma,
241 			       unsigned long addr, pte_t *ptep,
242 			       pte_t pte, int dirty)
243 {
244 #ifdef HUGETLB_NEED_PRELOAD
245 	/*
246 	 * The "return 1" forces a call of update_mmu_cache, which will write a
247 	 * TLB entry.  Without this, platforms that don't do a write of the TLB
248 	 * entry in the TLB miss handler asm will fault ad infinitum.
249 	 */
250 	ptep_set_access_flags(vma, addr, ptep, pte, dirty);
251 	return 1;
252 #else
253 	int changed, psize;
254 
255 	pte = set_access_flags_filter(pte, vma, dirty);
256 	changed = !pte_same(*(ptep), pte);
257 	if (changed) {
258 
259 #ifdef CONFIG_PPC_BOOK3S_64
260 		struct hstate *h = hstate_vma(vma);
261 
262 		psize = hstate_get_psize(h);
263 #ifdef CONFIG_DEBUG_VM
264 		assert_spin_locked(huge_pte_lockptr(h, vma->vm_mm, ptep));
265 #endif
266 
267 #else
268 		/*
269 		 * Not used on non book3s64 platforms.
270 		 * 8xx compares it with mmu_virtual_psize to
271 		 * know if it is a huge page or not.
272 		 */
273 		psize = MMU_PAGE_COUNT;
274 #endif
275 		__ptep_set_access_flags(vma, ptep, pte, addr, psize);
276 	}
277 	return changed;
278 #endif
279 }
280 
281 #if defined(CONFIG_PPC_8xx)
set_huge_pte_at(struct mm_struct * mm,unsigned long addr,pte_t * ptep,pte_t pte)282 void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte)
283 {
284 	pmd_t *pmd = pmd_off(mm, addr);
285 	pte_basic_t val;
286 	pte_basic_t *entry = (pte_basic_t *)ptep;
287 	int num, i;
288 
289 	/*
290 	 * Make sure hardware valid bit is not set. We don't do
291 	 * tlb flush for this update.
292 	 */
293 	VM_WARN_ON(pte_hw_valid(*ptep) && !pte_protnone(*ptep));
294 
295 	pte = set_pte_filter(pte);
296 
297 	val = pte_val(pte);
298 
299 	num = number_of_cells_per_pte(pmd, val, 1);
300 
301 	for (i = 0; i < num; i++, entry++, val += SZ_4K)
302 		*entry = val;
303 }
304 #endif
305 #endif /* CONFIG_HUGETLB_PAGE */
306 
307 #ifdef CONFIG_DEBUG_VM
assert_pte_locked(struct mm_struct * mm,unsigned long addr)308 void assert_pte_locked(struct mm_struct *mm, unsigned long addr)
309 {
310 	pgd_t *pgd;
311 	p4d_t *p4d;
312 	pud_t *pud;
313 	pmd_t *pmd;
314 
315 	if (mm == &init_mm)
316 		return;
317 	pgd = mm->pgd + pgd_index(addr);
318 	BUG_ON(pgd_none(*pgd));
319 	p4d = p4d_offset(pgd, addr);
320 	BUG_ON(p4d_none(*p4d));
321 	pud = pud_offset(p4d, addr);
322 	BUG_ON(pud_none(*pud));
323 	pmd = pmd_offset(pud, addr);
324 	/*
325 	 * khugepaged to collapse normal pages to hugepage, first set
326 	 * pmd to none to force page fault/gup to take mmap_lock. After
327 	 * pmd is set to none, we do a pte_clear which does this assertion
328 	 * so if we find pmd none, return.
329 	 */
330 	if (pmd_none(*pmd))
331 		return;
332 	BUG_ON(!pmd_present(*pmd));
333 	assert_spin_locked(pte_lockptr(mm, pmd));
334 }
335 #endif /* CONFIG_DEBUG_VM */
336 
vmalloc_to_phys(void * va)337 unsigned long vmalloc_to_phys(void *va)
338 {
339 	unsigned long pfn = vmalloc_to_pfn(va);
340 
341 	BUG_ON(!pfn);
342 	return __pa(pfn_to_kaddr(pfn)) + offset_in_page(va);
343 }
344 EXPORT_SYMBOL_GPL(vmalloc_to_phys);
345 
346 /*
347  * We have 4 cases for pgds and pmds:
348  * (1) invalid (all zeroes)
349  * (2) pointer to next table, as normal; bottom 6 bits == 0
350  * (3) leaf pte for huge page _PAGE_PTE set
351  * (4) hugepd pointer, _PAGE_PTE = 0 and bits [2..6] indicate size of table
352  *
353  * So long as we atomically load page table pointers we are safe against teardown,
354  * we can follow the address down to the page and take a ref on it.
355  * This function need to be called with interrupts disabled. We use this variant
356  * when we have MSR[EE] = 0 but the paca->irq_soft_mask = IRQS_ENABLED
357  */
__find_linux_pte(pgd_t * pgdir,unsigned long ea,bool * is_thp,unsigned * hpage_shift)358 pte_t *__find_linux_pte(pgd_t *pgdir, unsigned long ea,
359 			bool *is_thp, unsigned *hpage_shift)
360 {
361 	pgd_t *pgdp;
362 	p4d_t p4d, *p4dp;
363 	pud_t pud, *pudp;
364 	pmd_t pmd, *pmdp;
365 	pte_t *ret_pte;
366 	hugepd_t *hpdp = NULL;
367 	unsigned pdshift;
368 
369 	if (hpage_shift)
370 		*hpage_shift = 0;
371 
372 	if (is_thp)
373 		*is_thp = false;
374 
375 	/*
376 	 * Always operate on the local stack value. This make sure the
377 	 * value don't get updated by a parallel THP split/collapse,
378 	 * page fault or a page unmap. The return pte_t * is still not
379 	 * stable. So should be checked there for above conditions.
380 	 * Top level is an exception because it is folded into p4d.
381 	 */
382 	pgdp = pgdir + pgd_index(ea);
383 	p4dp = p4d_offset(pgdp, ea);
384 	p4d  = READ_ONCE(*p4dp);
385 	pdshift = P4D_SHIFT;
386 
387 	if (p4d_none(p4d))
388 		return NULL;
389 
390 	if (p4d_is_leaf(p4d)) {
391 		ret_pte = (pte_t *)p4dp;
392 		goto out;
393 	}
394 
395 	if (is_hugepd(__hugepd(p4d_val(p4d)))) {
396 		hpdp = (hugepd_t *)&p4d;
397 		goto out_huge;
398 	}
399 
400 	/*
401 	 * Even if we end up with an unmap, the pgtable will not
402 	 * be freed, because we do an rcu free and here we are
403 	 * irq disabled
404 	 */
405 	pdshift = PUD_SHIFT;
406 	pudp = pud_offset(&p4d, ea);
407 	pud  = READ_ONCE(*pudp);
408 
409 	if (pud_none(pud))
410 		return NULL;
411 
412 	if (pud_is_leaf(pud)) {
413 		ret_pte = (pte_t *)pudp;
414 		goto out;
415 	}
416 
417 	if (is_hugepd(__hugepd(pud_val(pud)))) {
418 		hpdp = (hugepd_t *)&pud;
419 		goto out_huge;
420 	}
421 
422 	pdshift = PMD_SHIFT;
423 	pmdp = pmd_offset(&pud, ea);
424 	pmd  = READ_ONCE(*pmdp);
425 
426 	/*
427 	 * A hugepage collapse is captured by this condition, see
428 	 * pmdp_collapse_flush.
429 	 */
430 	if (pmd_none(pmd))
431 		return NULL;
432 
433 #ifdef CONFIG_PPC_BOOK3S_64
434 	/*
435 	 * A hugepage split is captured by this condition, see
436 	 * pmdp_invalidate.
437 	 *
438 	 * Huge page modification can be caught here too.
439 	 */
440 	if (pmd_is_serializing(pmd))
441 		return NULL;
442 #endif
443 
444 	if (pmd_trans_huge(pmd) || pmd_devmap(pmd)) {
445 		if (is_thp)
446 			*is_thp = true;
447 		ret_pte = (pte_t *)pmdp;
448 		goto out;
449 	}
450 
451 	if (pmd_is_leaf(pmd)) {
452 		ret_pte = (pte_t *)pmdp;
453 		goto out;
454 	}
455 
456 	if (is_hugepd(__hugepd(pmd_val(pmd)))) {
457 		hpdp = (hugepd_t *)&pmd;
458 		goto out_huge;
459 	}
460 
461 	return pte_offset_kernel(&pmd, ea);
462 
463 out_huge:
464 	if (!hpdp)
465 		return NULL;
466 
467 	ret_pte = hugepte_offset(*hpdp, ea, pdshift);
468 	pdshift = hugepd_shift(*hpdp);
469 out:
470 	if (hpage_shift)
471 		*hpage_shift = pdshift;
472 	return ret_pte;
473 }
474 EXPORT_SYMBOL_GPL(__find_linux_pte);
475