1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  *  linux/mm/memory.c
4  *
5  *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
6  */
7 
8 /*
9  * demand-loading started 01.12.91 - seems it is high on the list of
10  * things wanted, and it should be easy to implement. - Linus
11  */
12 
13 /*
14  * Ok, demand-loading was easy, shared pages a little bit tricker. Shared
15  * pages started 02.12.91, seems to work. - Linus.
16  *
17  * Tested sharing by executing about 30 /bin/sh: under the old kernel it
18  * would have taken more than the 6M I have free, but it worked well as
19  * far as I could see.
20  *
21  * Also corrected some "invalidate()"s - I wasn't doing enough of them.
22  */
23 
24 /*
25  * Real VM (paging to/from disk) started 18.12.91. Much more work and
26  * thought has to go into this. Oh, well..
27  * 19.12.91  -  works, somewhat. Sometimes I get faults, don't know why.
28  *		Found it. Everything seems to work now.
29  * 20.12.91  -  Ok, making the swap-device changeable like the root.
30  */
31 
32 /*
33  * 05.04.94  -  Multi-page memory management added for v1.1.
34  *              Idea by Alex Bligh (alex@cconcepts.co.uk)
35  *
36  * 16.07.99  -  Support of BIGMEM added by Gerhard Wichert, Siemens AG
37  *		(Gerhard.Wichert@pdb.siemens.de)
38  *
39  * Aug/Sep 2004 Changed to four level page tables (Andi Kleen)
40  */
41 
42 #include <linux/kernel_stat.h>
43 #include <linux/mm.h>
44 #include <linux/mm_inline.h>
45 #include <linux/sched/mm.h>
46 #include <linux/sched/coredump.h>
47 #include <linux/sched/numa_balancing.h>
48 #include <linux/sched/task.h>
49 #include <linux/hugetlb.h>
50 #include <linux/mman.h>
51 #include <linux/swap.h>
52 #include <linux/highmem.h>
53 #include <linux/pagemap.h>
54 #include <linux/memremap.h>
55 #include <linux/kmsan.h>
56 #include <linux/ksm.h>
57 #include <linux/rmap.h>
58 #include <linux/export.h>
59 #include <linux/delayacct.h>
60 #include <linux/init.h>
61 #include <linux/pfn_t.h>
62 #include <linux/writeback.h>
63 #include <linux/memcontrol.h>
64 #include <linux/mmu_notifier.h>
65 #include <linux/swapops.h>
66 #include <linux/elf.h>
67 #include <linux/gfp.h>
68 #include <linux/migrate.h>
69 #include <linux/string.h>
70 #include <linux/memory-tiers.h>
71 #include <linux/debugfs.h>
72 #include <linux/userfaultfd_k.h>
73 #include <linux/dax.h>
74 #include <linux/oom.h>
75 #include <linux/numa.h>
76 #include <linux/perf_event.h>
77 #include <linux/ptrace.h>
78 #include <linux/vmalloc.h>
79 #include <linux/sched/sysctl.h>
80 
81 #include <trace/events/kmem.h>
82 
83 #include <asm/io.h>
84 #include <asm/mmu_context.h>
85 #include <asm/pgalloc.h>
86 #include <linux/uaccess.h>
87 #include <asm/tlb.h>
88 #include <asm/tlbflush.h>
89 
90 #include "pgalloc-track.h"
91 #include "internal.h"
92 #include "swap.h"
93 
94 #if defined(LAST_CPUPID_NOT_IN_PAGE_FLAGS) && !defined(CONFIG_COMPILE_TEST)
95 #warning Unfortunate NUMA and NUMA Balancing config, growing page-frame for last_cpupid.
96 #endif
97 
98 #ifndef CONFIG_NUMA
99 unsigned long max_mapnr;
100 EXPORT_SYMBOL(max_mapnr);
101 
102 struct page *mem_map;
103 EXPORT_SYMBOL(mem_map);
104 #endif
105 
106 static vm_fault_t do_fault(struct vm_fault *vmf);
107 static vm_fault_t do_anonymous_page(struct vm_fault *vmf);
108 static bool vmf_pte_changed(struct vm_fault *vmf);
109 
110 /*
111  * Return true if the original pte was a uffd-wp pte marker (so the pte was
112  * wr-protected).
113  */
vmf_orig_pte_uffd_wp(struct vm_fault * vmf)114 static bool vmf_orig_pte_uffd_wp(struct vm_fault *vmf)
115 {
116 	if (!(vmf->flags & FAULT_FLAG_ORIG_PTE_VALID))
117 		return false;
118 
119 	return pte_marker_uffd_wp(vmf->orig_pte);
120 }
121 
122 /*
123  * A number of key systems in x86 including ioremap() rely on the assumption
124  * that high_memory defines the upper bound on direct map memory, then end
125  * of ZONE_NORMAL.  Under CONFIG_DISCONTIG this means that max_low_pfn and
126  * highstart_pfn must be the same; there must be no gap between ZONE_NORMAL
127  * and ZONE_HIGHMEM.
128  */
129 void *high_memory;
130 EXPORT_SYMBOL(high_memory);
131 
132 /*
133  * Randomize the address space (stacks, mmaps, brk, etc.).
134  *
135  * ( When CONFIG_COMPAT_BRK=y we exclude brk from randomization,
136  *   as ancient (libc5 based) binaries can segfault. )
137  */
138 int randomize_va_space __read_mostly =
139 #ifdef CONFIG_COMPAT_BRK
140 					1;
141 #else
142 					2;
143 #endif
144 
145 #ifndef arch_wants_old_prefaulted_pte
arch_wants_old_prefaulted_pte(void)146 static inline bool arch_wants_old_prefaulted_pte(void)
147 {
148 	/*
149 	 * Transitioning a PTE from 'old' to 'young' can be expensive on
150 	 * some architectures, even if it's performed in hardware. By
151 	 * default, "false" means prefaulted entries will be 'young'.
152 	 */
153 	return false;
154 }
155 #endif
156 
disable_randmaps(char * s)157 static int __init disable_randmaps(char *s)
158 {
159 	randomize_va_space = 0;
160 	return 1;
161 }
162 __setup("norandmaps", disable_randmaps);
163 
164 unsigned long zero_pfn __read_mostly;
165 EXPORT_SYMBOL(zero_pfn);
166 
167 unsigned long highest_memmap_pfn __read_mostly;
168 
169 /*
170  * CONFIG_MMU architectures set up ZERO_PAGE in their paging_init()
171  */
init_zero_pfn(void)172 static int __init init_zero_pfn(void)
173 {
174 	zero_pfn = page_to_pfn(ZERO_PAGE(0));
175 	return 0;
176 }
177 early_initcall(init_zero_pfn);
178 
mm_trace_rss_stat(struct mm_struct * mm,int member)179 void mm_trace_rss_stat(struct mm_struct *mm, int member)
180 {
181 	trace_rss_stat(mm, member);
182 }
183 
184 /*
185  * Note: this doesn't free the actual pages themselves. That
186  * has been handled earlier when unmapping all the memory regions.
187  */
free_pte_range(struct mmu_gather * tlb,pmd_t * pmd,unsigned long addr)188 static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd,
189 			   unsigned long addr)
190 {
191 	pgtable_t token = pmd_pgtable(*pmd);
192 	pmd_clear(pmd);
193 	pte_free_tlb(tlb, token, addr);
194 	mm_dec_nr_ptes(tlb->mm);
195 }
196 
free_pmd_range(struct mmu_gather * tlb,pud_t * pud,unsigned long addr,unsigned long end,unsigned long floor,unsigned long ceiling)197 static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
198 				unsigned long addr, unsigned long end,
199 				unsigned long floor, unsigned long ceiling)
200 {
201 	pmd_t *pmd;
202 	unsigned long next;
203 	unsigned long start;
204 
205 	start = addr;
206 	pmd = pmd_offset(pud, addr);
207 	do {
208 		next = pmd_addr_end(addr, end);
209 		if (pmd_none_or_clear_bad(pmd))
210 			continue;
211 		free_pte_range(tlb, pmd, addr);
212 	} while (pmd++, addr = next, addr != end);
213 
214 	start &= PUD_MASK;
215 	if (start < floor)
216 		return;
217 	if (ceiling) {
218 		ceiling &= PUD_MASK;
219 		if (!ceiling)
220 			return;
221 	}
222 	if (end - 1 > ceiling - 1)
223 		return;
224 
225 	pmd = pmd_offset(pud, start);
226 	pud_clear(pud);
227 	pmd_free_tlb(tlb, pmd, start);
228 	mm_dec_nr_pmds(tlb->mm);
229 }
230 
free_pud_range(struct mmu_gather * tlb,p4d_t * p4d,unsigned long addr,unsigned long end,unsigned long floor,unsigned long ceiling)231 static inline void free_pud_range(struct mmu_gather *tlb, p4d_t *p4d,
232 				unsigned long addr, unsigned long end,
233 				unsigned long floor, unsigned long ceiling)
234 {
235 	pud_t *pud;
236 	unsigned long next;
237 	unsigned long start;
238 
239 	start = addr;
240 	pud = pud_offset(p4d, addr);
241 	do {
242 		next = pud_addr_end(addr, end);
243 		if (pud_none_or_clear_bad(pud))
244 			continue;
245 		free_pmd_range(tlb, pud, addr, next, floor, ceiling);
246 	} while (pud++, addr = next, addr != end);
247 
248 	start &= P4D_MASK;
249 	if (start < floor)
250 		return;
251 	if (ceiling) {
252 		ceiling &= P4D_MASK;
253 		if (!ceiling)
254 			return;
255 	}
256 	if (end - 1 > ceiling - 1)
257 		return;
258 
259 	pud = pud_offset(p4d, start);
260 	p4d_clear(p4d);
261 	pud_free_tlb(tlb, pud, start);
262 	mm_dec_nr_puds(tlb->mm);
263 }
264 
free_p4d_range(struct mmu_gather * tlb,pgd_t * pgd,unsigned long addr,unsigned long end,unsigned long floor,unsigned long ceiling)265 static inline void free_p4d_range(struct mmu_gather *tlb, pgd_t *pgd,
266 				unsigned long addr, unsigned long end,
267 				unsigned long floor, unsigned long ceiling)
268 {
269 	p4d_t *p4d;
270 	unsigned long next;
271 	unsigned long start;
272 
273 	start = addr;
274 	p4d = p4d_offset(pgd, addr);
275 	do {
276 		next = p4d_addr_end(addr, end);
277 		if (p4d_none_or_clear_bad(p4d))
278 			continue;
279 		free_pud_range(tlb, p4d, addr, next, floor, ceiling);
280 	} while (p4d++, addr = next, addr != end);
281 
282 	start &= PGDIR_MASK;
283 	if (start < floor)
284 		return;
285 	if (ceiling) {
286 		ceiling &= PGDIR_MASK;
287 		if (!ceiling)
288 			return;
289 	}
290 	if (end - 1 > ceiling - 1)
291 		return;
292 
293 	p4d = p4d_offset(pgd, start);
294 	pgd_clear(pgd);
295 	p4d_free_tlb(tlb, p4d, start);
296 }
297 
298 /*
299  * This function frees user-level page tables of a process.
300  */
free_pgd_range(struct mmu_gather * tlb,unsigned long addr,unsigned long end,unsigned long floor,unsigned long ceiling)301 void free_pgd_range(struct mmu_gather *tlb,
302 			unsigned long addr, unsigned long end,
303 			unsigned long floor, unsigned long ceiling)
304 {
305 	pgd_t *pgd;
306 	unsigned long next;
307 
308 	/*
309 	 * The next few lines have given us lots of grief...
310 	 *
311 	 * Why are we testing PMD* at this top level?  Because often
312 	 * there will be no work to do at all, and we'd prefer not to
313 	 * go all the way down to the bottom just to discover that.
314 	 *
315 	 * Why all these "- 1"s?  Because 0 represents both the bottom
316 	 * of the address space and the top of it (using -1 for the
317 	 * top wouldn't help much: the masks would do the wrong thing).
318 	 * The rule is that addr 0 and floor 0 refer to the bottom of
319 	 * the address space, but end 0 and ceiling 0 refer to the top
320 	 * Comparisons need to use "end - 1" and "ceiling - 1" (though
321 	 * that end 0 case should be mythical).
322 	 *
323 	 * Wherever addr is brought up or ceiling brought down, we must
324 	 * be careful to reject "the opposite 0" before it confuses the
325 	 * subsequent tests.  But what about where end is brought down
326 	 * by PMD_SIZE below? no, end can't go down to 0 there.
327 	 *
328 	 * Whereas we round start (addr) and ceiling down, by different
329 	 * masks at different levels, in order to test whether a table
330 	 * now has no other vmas using it, so can be freed, we don't
331 	 * bother to round floor or end up - the tests don't need that.
332 	 */
333 
334 	addr &= PMD_MASK;
335 	if (addr < floor) {
336 		addr += PMD_SIZE;
337 		if (!addr)
338 			return;
339 	}
340 	if (ceiling) {
341 		ceiling &= PMD_MASK;
342 		if (!ceiling)
343 			return;
344 	}
345 	if (end - 1 > ceiling - 1)
346 		end -= PMD_SIZE;
347 	if (addr > end - 1)
348 		return;
349 	/*
350 	 * We add page table cache pages with PAGE_SIZE,
351 	 * (see pte_free_tlb()), flush the tlb if we need
352 	 */
353 	tlb_change_page_size(tlb, PAGE_SIZE);
354 	pgd = pgd_offset(tlb->mm, addr);
355 	do {
356 		next = pgd_addr_end(addr, end);
357 		if (pgd_none_or_clear_bad(pgd))
358 			continue;
359 		free_p4d_range(tlb, pgd, addr, next, floor, ceiling);
360 	} while (pgd++, addr = next, addr != end);
361 }
362 
free_pgtables(struct mmu_gather * tlb,struct ma_state * mas,struct vm_area_struct * vma,unsigned long floor,unsigned long ceiling,bool mm_wr_locked)363 void free_pgtables(struct mmu_gather *tlb, struct ma_state *mas,
364 		   struct vm_area_struct *vma, unsigned long floor,
365 		   unsigned long ceiling, bool mm_wr_locked)
366 {
367 	do {
368 		unsigned long addr = vma->vm_start;
369 		struct vm_area_struct *next;
370 
371 		/*
372 		 * Note: USER_PGTABLES_CEILING may be passed as ceiling and may
373 		 * be 0.  This will underflow and is okay.
374 		 */
375 		next = mas_find(mas, ceiling - 1);
376 
377 		/*
378 		 * Hide vma from rmap and truncate_pagecache before freeing
379 		 * pgtables
380 		 */
381 		if (mm_wr_locked)
382 			vma_start_write(vma);
383 		unlink_anon_vmas(vma);
384 		unlink_file_vma(vma);
385 
386 		if (is_vm_hugetlb_page(vma)) {
387 			hugetlb_free_pgd_range(tlb, addr, vma->vm_end,
388 				floor, next ? next->vm_start : ceiling);
389 		} else {
390 			/*
391 			 * Optimization: gather nearby vmas into one call down
392 			 */
393 			while (next && next->vm_start <= vma->vm_end + PMD_SIZE
394 			       && !is_vm_hugetlb_page(next)) {
395 				vma = next;
396 				next = mas_find(mas, ceiling - 1);
397 				if (mm_wr_locked)
398 					vma_start_write(vma);
399 				unlink_anon_vmas(vma);
400 				unlink_file_vma(vma);
401 			}
402 			free_pgd_range(tlb, addr, vma->vm_end,
403 				floor, next ? next->vm_start : ceiling);
404 		}
405 		vma = next;
406 	} while (vma);
407 }
408 
pmd_install(struct mm_struct * mm,pmd_t * pmd,pgtable_t * pte)409 void pmd_install(struct mm_struct *mm, pmd_t *pmd, pgtable_t *pte)
410 {
411 	spinlock_t *ptl = pmd_lock(mm, pmd);
412 
413 	if (likely(pmd_none(*pmd))) {	/* Has another populated it ? */
414 		mm_inc_nr_ptes(mm);
415 		/*
416 		 * Ensure all pte setup (eg. pte page lock and page clearing) are
417 		 * visible before the pte is made visible to other CPUs by being
418 		 * put into page tables.
419 		 *
420 		 * The other side of the story is the pointer chasing in the page
421 		 * table walking code (when walking the page table without locking;
422 		 * ie. most of the time). Fortunately, these data accesses consist
423 		 * of a chain of data-dependent loads, meaning most CPUs (alpha
424 		 * being the notable exception) will already guarantee loads are
425 		 * seen in-order. See the alpha page table accessors for the
426 		 * smp_rmb() barriers in page table walking code.
427 		 */
428 		smp_wmb(); /* Could be smp_wmb__xxx(before|after)_spin_lock */
429 		pmd_populate(mm, pmd, *pte);
430 		*pte = NULL;
431 	}
432 	spin_unlock(ptl);
433 }
434 
__pte_alloc(struct mm_struct * mm,pmd_t * pmd)435 int __pte_alloc(struct mm_struct *mm, pmd_t *pmd)
436 {
437 	pgtable_t new = pte_alloc_one(mm);
438 	if (!new)
439 		return -ENOMEM;
440 
441 	pmd_install(mm, pmd, &new);
442 	if (new)
443 		pte_free(mm, new);
444 	return 0;
445 }
446 
__pte_alloc_kernel(pmd_t * pmd)447 int __pte_alloc_kernel(pmd_t *pmd)
448 {
449 	pte_t *new = pte_alloc_one_kernel(&init_mm);
450 	if (!new)
451 		return -ENOMEM;
452 
453 	spin_lock(&init_mm.page_table_lock);
454 	if (likely(pmd_none(*pmd))) {	/* Has another populated it ? */
455 		smp_wmb(); /* See comment in pmd_install() */
456 		pmd_populate_kernel(&init_mm, pmd, new);
457 		new = NULL;
458 	}
459 	spin_unlock(&init_mm.page_table_lock);
460 	if (new)
461 		pte_free_kernel(&init_mm, new);
462 	return 0;
463 }
464 
init_rss_vec(int * rss)465 static inline void init_rss_vec(int *rss)
466 {
467 	memset(rss, 0, sizeof(int) * NR_MM_COUNTERS);
468 }
469 
add_mm_rss_vec(struct mm_struct * mm,int * rss)470 static inline void add_mm_rss_vec(struct mm_struct *mm, int *rss)
471 {
472 	int i;
473 
474 	if (current->mm == mm)
475 		sync_mm_rss(mm);
476 	for (i = 0; i < NR_MM_COUNTERS; i++)
477 		if (rss[i])
478 			add_mm_counter(mm, i, rss[i]);
479 }
480 
481 /*
482  * This function is called to print an error when a bad pte
483  * is found. For example, we might have a PFN-mapped pte in
484  * a region that doesn't allow it.
485  *
486  * The calling function must still handle the error.
487  */
print_bad_pte(struct vm_area_struct * vma,unsigned long addr,pte_t pte,struct page * page)488 static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr,
489 			  pte_t pte, struct page *page)
490 {
491 	pgd_t *pgd = pgd_offset(vma->vm_mm, addr);
492 	p4d_t *p4d = p4d_offset(pgd, addr);
493 	pud_t *pud = pud_offset(p4d, addr);
494 	pmd_t *pmd = pmd_offset(pud, addr);
495 	struct address_space *mapping;
496 	pgoff_t index;
497 	static unsigned long resume;
498 	static unsigned long nr_shown;
499 	static unsigned long nr_unshown;
500 
501 	/*
502 	 * Allow a burst of 60 reports, then keep quiet for that minute;
503 	 * or allow a steady drip of one report per second.
504 	 */
505 	if (nr_shown == 60) {
506 		if (time_before(jiffies, resume)) {
507 			nr_unshown++;
508 			return;
509 		}
510 		if (nr_unshown) {
511 			pr_alert("BUG: Bad page map: %lu messages suppressed\n",
512 				 nr_unshown);
513 			nr_unshown = 0;
514 		}
515 		nr_shown = 0;
516 	}
517 	if (nr_shown++ == 0)
518 		resume = jiffies + 60 * HZ;
519 
520 	mapping = vma->vm_file ? vma->vm_file->f_mapping : NULL;
521 	index = linear_page_index(vma, addr);
522 
523 	pr_alert("BUG: Bad page map in process %s  pte:%08llx pmd:%08llx\n",
524 		 current->comm,
525 		 (long long)pte_val(pte), (long long)pmd_val(*pmd));
526 	if (page)
527 		dump_page(page, "bad pte");
528 	pr_alert("addr:%px vm_flags:%08lx anon_vma:%px mapping:%px index:%lx\n",
529 		 (void *)addr, vma->vm_flags, vma->anon_vma, mapping, index);
530 	pr_alert("file:%pD fault:%ps mmap:%ps read_folio:%ps\n",
531 		 vma->vm_file,
532 		 vma->vm_ops ? vma->vm_ops->fault : NULL,
533 		 vma->vm_file ? vma->vm_file->f_op->mmap : NULL,
534 		 mapping ? mapping->a_ops->read_folio : NULL);
535 	dump_stack();
536 	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
537 }
538 
539 /*
540  * vm_normal_page -- This function gets the "struct page" associated with a pte.
541  *
542  * "Special" mappings do not wish to be associated with a "struct page" (either
543  * it doesn't exist, or it exists but they don't want to touch it). In this
544  * case, NULL is returned here. "Normal" mappings do have a struct page.
545  *
546  * There are 2 broad cases. Firstly, an architecture may define a pte_special()
547  * pte bit, in which case this function is trivial. Secondly, an architecture
548  * may not have a spare pte bit, which requires a more complicated scheme,
549  * described below.
550  *
551  * A raw VM_PFNMAP mapping (ie. one that is not COWed) is always considered a
552  * special mapping (even if there are underlying and valid "struct pages").
553  * COWed pages of a VM_PFNMAP are always normal.
554  *
555  * The way we recognize COWed pages within VM_PFNMAP mappings is through the
556  * rules set up by "remap_pfn_range()": the vma will have the VM_PFNMAP bit
557  * set, and the vm_pgoff will point to the first PFN mapped: thus every special
558  * mapping will always honor the rule
559  *
560  *	pfn_of_page == vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT)
561  *
562  * And for normal mappings this is false.
563  *
564  * This restricts such mappings to be a linear translation from virtual address
565  * to pfn. To get around this restriction, we allow arbitrary mappings so long
566  * as the vma is not a COW mapping; in that case, we know that all ptes are
567  * special (because none can have been COWed).
568  *
569  *
570  * In order to support COW of arbitrary special mappings, we have VM_MIXEDMAP.
571  *
572  * VM_MIXEDMAP mappings can likewise contain memory with or without "struct
573  * page" backing, however the difference is that _all_ pages with a struct
574  * page (that is, those where pfn_valid is true) are refcounted and considered
575  * normal pages by the VM. The disadvantage is that pages are refcounted
576  * (which can be slower and simply not an option for some PFNMAP users). The
577  * advantage is that we don't have to follow the strict linearity rule of
578  * PFNMAP mappings in order to support COWable mappings.
579  *
580  */
vm_normal_page(struct vm_area_struct * vma,unsigned long addr,pte_t pte)581 struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
582 			    pte_t pte)
583 {
584 	unsigned long pfn = pte_pfn(pte);
585 
586 	if (IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL)) {
587 		if (likely(!pte_special(pte)))
588 			goto check_pfn;
589 		if (vma->vm_ops && vma->vm_ops->find_special_page)
590 			return vma->vm_ops->find_special_page(vma, addr);
591 		if (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP))
592 			return NULL;
593 		if (is_zero_pfn(pfn))
594 			return NULL;
595 		if (pte_devmap(pte))
596 		/*
597 		 * NOTE: New users of ZONE_DEVICE will not set pte_devmap()
598 		 * and will have refcounts incremented on their struct pages
599 		 * when they are inserted into PTEs, thus they are safe to
600 		 * return here. Legacy ZONE_DEVICE pages that set pte_devmap()
601 		 * do not have refcounts. Example of legacy ZONE_DEVICE is
602 		 * MEMORY_DEVICE_FS_DAX type in pmem or virtio_fs drivers.
603 		 */
604 			return NULL;
605 
606 		print_bad_pte(vma, addr, pte, NULL);
607 		return NULL;
608 	}
609 
610 	/* !CONFIG_ARCH_HAS_PTE_SPECIAL case follows: */
611 
612 	if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) {
613 		if (vma->vm_flags & VM_MIXEDMAP) {
614 			if (!pfn_valid(pfn))
615 				return NULL;
616 			goto out;
617 		} else {
618 			unsigned long off;
619 			off = (addr - vma->vm_start) >> PAGE_SHIFT;
620 			if (pfn == vma->vm_pgoff + off)
621 				return NULL;
622 			if (!is_cow_mapping(vma->vm_flags))
623 				return NULL;
624 		}
625 	}
626 
627 	if (is_zero_pfn(pfn))
628 		return NULL;
629 
630 check_pfn:
631 	if (unlikely(pfn > highest_memmap_pfn)) {
632 		print_bad_pte(vma, addr, pte, NULL);
633 		return NULL;
634 	}
635 
636 	/*
637 	 * NOTE! We still have PageReserved() pages in the page tables.
638 	 * eg. VDSO mappings can cause them to exist.
639 	 */
640 out:
641 	return pfn_to_page(pfn);
642 }
643 
vm_normal_folio(struct vm_area_struct * vma,unsigned long addr,pte_t pte)644 struct folio *vm_normal_folio(struct vm_area_struct *vma, unsigned long addr,
645 			    pte_t pte)
646 {
647 	struct page *page = vm_normal_page(vma, addr, pte);
648 
649 	if (page)
650 		return page_folio(page);
651 	return NULL;
652 }
653 
654 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
vm_normal_page_pmd(struct vm_area_struct * vma,unsigned long addr,pmd_t pmd)655 struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
656 				pmd_t pmd)
657 {
658 	unsigned long pfn = pmd_pfn(pmd);
659 
660 	/*
661 	 * There is no pmd_special() but there may be special pmds, e.g.
662 	 * in a direct-access (dax) mapping, so let's just replicate the
663 	 * !CONFIG_ARCH_HAS_PTE_SPECIAL case from vm_normal_page() here.
664 	 */
665 	if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) {
666 		if (vma->vm_flags & VM_MIXEDMAP) {
667 			if (!pfn_valid(pfn))
668 				return NULL;
669 			goto out;
670 		} else {
671 			unsigned long off;
672 			off = (addr - vma->vm_start) >> PAGE_SHIFT;
673 			if (pfn == vma->vm_pgoff + off)
674 				return NULL;
675 			if (!is_cow_mapping(vma->vm_flags))
676 				return NULL;
677 		}
678 	}
679 
680 	if (pmd_devmap(pmd))
681 		return NULL;
682 	if (is_huge_zero_pmd(pmd))
683 		return NULL;
684 	if (unlikely(pfn > highest_memmap_pfn))
685 		return NULL;
686 
687 	/*
688 	 * NOTE! We still have PageReserved() pages in the page tables.
689 	 * eg. VDSO mappings can cause them to exist.
690 	 */
691 out:
692 	return pfn_to_page(pfn);
693 }
694 #endif
695 
restore_exclusive_pte(struct vm_area_struct * vma,struct page * page,unsigned long address,pte_t * ptep)696 static void restore_exclusive_pte(struct vm_area_struct *vma,
697 				  struct page *page, unsigned long address,
698 				  pte_t *ptep)
699 {
700 	pte_t orig_pte;
701 	pte_t pte;
702 	swp_entry_t entry;
703 
704 	orig_pte = ptep_get(ptep);
705 	pte = pte_mkold(mk_pte(page, READ_ONCE(vma->vm_page_prot)));
706 	if (pte_swp_soft_dirty(orig_pte))
707 		pte = pte_mksoft_dirty(pte);
708 
709 	entry = pte_to_swp_entry(orig_pte);
710 	if (pte_swp_uffd_wp(orig_pte))
711 		pte = pte_mkuffd_wp(pte);
712 	else if (is_writable_device_exclusive_entry(entry))
713 		pte = maybe_mkwrite(pte_mkdirty(pte), vma);
714 
715 	VM_BUG_ON(pte_write(pte) && !(PageAnon(page) && PageAnonExclusive(page)));
716 
717 	/*
718 	 * No need to take a page reference as one was already
719 	 * created when the swap entry was made.
720 	 */
721 	if (PageAnon(page))
722 		page_add_anon_rmap(page, vma, address, RMAP_NONE);
723 	else
724 		/*
725 		 * Currently device exclusive access only supports anonymous
726 		 * memory so the entry shouldn't point to a filebacked page.
727 		 */
728 		WARN_ON_ONCE(1);
729 
730 	set_pte_at(vma->vm_mm, address, ptep, pte);
731 
732 	/*
733 	 * No need to invalidate - it was non-present before. However
734 	 * secondary CPUs may have mappings that need invalidating.
735 	 */
736 	update_mmu_cache(vma, address, ptep);
737 }
738 
739 /*
740  * Tries to restore an exclusive pte if the page lock can be acquired without
741  * sleeping.
742  */
743 static int
try_restore_exclusive_pte(pte_t * src_pte,struct vm_area_struct * vma,unsigned long addr)744 try_restore_exclusive_pte(pte_t *src_pte, struct vm_area_struct *vma,
745 			unsigned long addr)
746 {
747 	swp_entry_t entry = pte_to_swp_entry(ptep_get(src_pte));
748 	struct page *page = pfn_swap_entry_to_page(entry);
749 
750 	if (trylock_page(page)) {
751 		restore_exclusive_pte(vma, page, addr, src_pte);
752 		unlock_page(page);
753 		return 0;
754 	}
755 
756 	return -EBUSY;
757 }
758 
759 /*
760  * copy one vm_area from one task to the other. Assumes the page tables
761  * already present in the new task to be cleared in the whole range
762  * covered by this vma.
763  */
764 
765 static unsigned long
copy_nonpresent_pte(struct mm_struct * dst_mm,struct mm_struct * src_mm,pte_t * dst_pte,pte_t * src_pte,struct vm_area_struct * dst_vma,struct vm_area_struct * src_vma,unsigned long addr,int * rss)766 copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
767 		pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *dst_vma,
768 		struct vm_area_struct *src_vma, unsigned long addr, int *rss)
769 {
770 	unsigned long vm_flags = dst_vma->vm_flags;
771 	pte_t orig_pte = ptep_get(src_pte);
772 	pte_t pte = orig_pte;
773 	struct page *page;
774 	swp_entry_t entry = pte_to_swp_entry(orig_pte);
775 
776 	if (likely(!non_swap_entry(entry))) {
777 		if (swap_duplicate(entry) < 0)
778 			return -EIO;
779 
780 		/* make sure dst_mm is on swapoff's mmlist. */
781 		if (unlikely(list_empty(&dst_mm->mmlist))) {
782 			spin_lock(&mmlist_lock);
783 			if (list_empty(&dst_mm->mmlist))
784 				list_add(&dst_mm->mmlist,
785 						&src_mm->mmlist);
786 			spin_unlock(&mmlist_lock);
787 		}
788 		/* Mark the swap entry as shared. */
789 		if (pte_swp_exclusive(orig_pte)) {
790 			pte = pte_swp_clear_exclusive(orig_pte);
791 			set_pte_at(src_mm, addr, src_pte, pte);
792 		}
793 		rss[MM_SWAPENTS]++;
794 	} else if (is_migration_entry(entry)) {
795 		page = pfn_swap_entry_to_page(entry);
796 
797 		rss[mm_counter(page)]++;
798 
799 		if (!is_readable_migration_entry(entry) &&
800 				is_cow_mapping(vm_flags)) {
801 			/*
802 			 * COW mappings require pages in both parent and child
803 			 * to be set to read. A previously exclusive entry is
804 			 * now shared.
805 			 */
806 			entry = make_readable_migration_entry(
807 							swp_offset(entry));
808 			pte = swp_entry_to_pte(entry);
809 			if (pte_swp_soft_dirty(orig_pte))
810 				pte = pte_swp_mksoft_dirty(pte);
811 			if (pte_swp_uffd_wp(orig_pte))
812 				pte = pte_swp_mkuffd_wp(pte);
813 			set_pte_at(src_mm, addr, src_pte, pte);
814 		}
815 	} else if (is_device_private_entry(entry)) {
816 		page = pfn_swap_entry_to_page(entry);
817 
818 		/*
819 		 * Update rss count even for unaddressable pages, as
820 		 * they should treated just like normal pages in this
821 		 * respect.
822 		 *
823 		 * We will likely want to have some new rss counters
824 		 * for unaddressable pages, at some point. But for now
825 		 * keep things as they are.
826 		 */
827 		get_page(page);
828 		rss[mm_counter(page)]++;
829 		/* Cannot fail as these pages cannot get pinned. */
830 		BUG_ON(page_try_dup_anon_rmap(page, false, src_vma));
831 
832 		/*
833 		 * We do not preserve soft-dirty information, because so
834 		 * far, checkpoint/restore is the only feature that
835 		 * requires that. And checkpoint/restore does not work
836 		 * when a device driver is involved (you cannot easily
837 		 * save and restore device driver state).
838 		 */
839 		if (is_writable_device_private_entry(entry) &&
840 		    is_cow_mapping(vm_flags)) {
841 			entry = make_readable_device_private_entry(
842 							swp_offset(entry));
843 			pte = swp_entry_to_pte(entry);
844 			if (pte_swp_uffd_wp(orig_pte))
845 				pte = pte_swp_mkuffd_wp(pte);
846 			set_pte_at(src_mm, addr, src_pte, pte);
847 		}
848 	} else if (is_device_exclusive_entry(entry)) {
849 		/*
850 		 * Make device exclusive entries present by restoring the
851 		 * original entry then copying as for a present pte. Device
852 		 * exclusive entries currently only support private writable
853 		 * (ie. COW) mappings.
854 		 */
855 		VM_BUG_ON(!is_cow_mapping(src_vma->vm_flags));
856 		if (try_restore_exclusive_pte(src_pte, src_vma, addr))
857 			return -EBUSY;
858 		return -ENOENT;
859 	} else if (is_pte_marker_entry(entry)) {
860 		pte_marker marker = copy_pte_marker(entry, dst_vma);
861 
862 		if (marker)
863 			set_pte_at(dst_mm, addr, dst_pte,
864 				   make_pte_marker(marker));
865 		return 0;
866 	}
867 	if (!userfaultfd_wp(dst_vma))
868 		pte = pte_swp_clear_uffd_wp(pte);
869 	set_pte_at(dst_mm, addr, dst_pte, pte);
870 	return 0;
871 }
872 
873 /*
874  * Copy a present and normal page.
875  *
876  * NOTE! The usual case is that this isn't required;
877  * instead, the caller can just increase the page refcount
878  * and re-use the pte the traditional way.
879  *
880  * And if we need a pre-allocated page but don't yet have
881  * one, return a negative error to let the preallocation
882  * code know so that it can do so outside the page table
883  * lock.
884  */
885 static inline int
copy_present_page(struct vm_area_struct * dst_vma,struct vm_area_struct * src_vma,pte_t * dst_pte,pte_t * src_pte,unsigned long addr,int * rss,struct folio ** prealloc,struct page * page)886 copy_present_page(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
887 		  pte_t *dst_pte, pte_t *src_pte, unsigned long addr, int *rss,
888 		  struct folio **prealloc, struct page *page)
889 {
890 	struct folio *new_folio;
891 	pte_t pte;
892 
893 	new_folio = *prealloc;
894 	if (!new_folio)
895 		return -EAGAIN;
896 
897 	/*
898 	 * We have a prealloc page, all good!  Take it
899 	 * over and copy the page & arm it.
900 	 */
901 	*prealloc = NULL;
902 	copy_user_highpage(&new_folio->page, page, addr, src_vma);
903 	__folio_mark_uptodate(new_folio);
904 	folio_add_new_anon_rmap(new_folio, dst_vma, addr);
905 	folio_add_lru_vma(new_folio, dst_vma);
906 	rss[MM_ANONPAGES]++;
907 
908 	/* All done, just insert the new page copy in the child */
909 	pte = mk_pte(&new_folio->page, dst_vma->vm_page_prot);
910 	pte = maybe_mkwrite(pte_mkdirty(pte), dst_vma);
911 	if (userfaultfd_pte_wp(dst_vma, ptep_get(src_pte)))
912 		/* Uffd-wp needs to be delivered to dest pte as well */
913 		pte = pte_mkuffd_wp(pte);
914 	set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte);
915 	return 0;
916 }
917 
918 /*
919  * Copy one pte.  Returns 0 if succeeded, or -EAGAIN if one preallocated page
920  * is required to copy this pte.
921  */
922 static inline int
copy_present_pte(struct vm_area_struct * dst_vma,struct vm_area_struct * src_vma,pte_t * dst_pte,pte_t * src_pte,unsigned long addr,int * rss,struct folio ** prealloc)923 copy_present_pte(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
924 		 pte_t *dst_pte, pte_t *src_pte, unsigned long addr, int *rss,
925 		 struct folio **prealloc)
926 {
927 	struct mm_struct *src_mm = src_vma->vm_mm;
928 	unsigned long vm_flags = src_vma->vm_flags;
929 	pte_t pte = ptep_get(src_pte);
930 	struct page *page;
931 	struct folio *folio;
932 
933 	page = vm_normal_page(src_vma, addr, pte);
934 	if (page)
935 		folio = page_folio(page);
936 	if (page && folio_test_anon(folio)) {
937 		/*
938 		 * If this page may have been pinned by the parent process,
939 		 * copy the page immediately for the child so that we'll always
940 		 * guarantee the pinned page won't be randomly replaced in the
941 		 * future.
942 		 */
943 		folio_get(folio);
944 		if (unlikely(page_try_dup_anon_rmap(page, false, src_vma))) {
945 			/* Page may be pinned, we have to copy. */
946 			folio_put(folio);
947 			return copy_present_page(dst_vma, src_vma, dst_pte, src_pte,
948 						 addr, rss, prealloc, page);
949 		}
950 		rss[MM_ANONPAGES]++;
951 	} else if (page) {
952 		folio_get(folio);
953 		page_dup_file_rmap(page, false);
954 		rss[mm_counter_file(page)]++;
955 	}
956 
957 	/*
958 	 * If it's a COW mapping, write protect it both
959 	 * in the parent and the child
960 	 */
961 	if (is_cow_mapping(vm_flags) && pte_write(pte)) {
962 		ptep_set_wrprotect(src_mm, addr, src_pte);
963 		pte = pte_wrprotect(pte);
964 	}
965 	VM_BUG_ON(page && folio_test_anon(folio) && PageAnonExclusive(page));
966 
967 	/*
968 	 * If it's a shared mapping, mark it clean in
969 	 * the child
970 	 */
971 	if (vm_flags & VM_SHARED)
972 		pte = pte_mkclean(pte);
973 	pte = pte_mkold(pte);
974 
975 	if (!userfaultfd_wp(dst_vma))
976 		pte = pte_clear_uffd_wp(pte);
977 
978 	set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte);
979 	return 0;
980 }
981 
page_copy_prealloc(struct mm_struct * src_mm,struct vm_area_struct * vma,unsigned long addr)982 static inline struct folio *page_copy_prealloc(struct mm_struct *src_mm,
983 		struct vm_area_struct *vma, unsigned long addr)
984 {
985 	struct folio *new_folio;
986 
987 	new_folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 0, vma, addr, false);
988 	if (!new_folio)
989 		return NULL;
990 
991 	if (mem_cgroup_charge(new_folio, src_mm, GFP_KERNEL)) {
992 		folio_put(new_folio);
993 		return NULL;
994 	}
995 	folio_throttle_swaprate(new_folio, GFP_KERNEL);
996 
997 	return new_folio;
998 }
999 
1000 static int
copy_pte_range(struct vm_area_struct * dst_vma,struct vm_area_struct * src_vma,pmd_t * dst_pmd,pmd_t * src_pmd,unsigned long addr,unsigned long end)1001 copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
1002 	       pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
1003 	       unsigned long end)
1004 {
1005 	struct mm_struct *dst_mm = dst_vma->vm_mm;
1006 	struct mm_struct *src_mm = src_vma->vm_mm;
1007 	pte_t *orig_src_pte, *orig_dst_pte;
1008 	pte_t *src_pte, *dst_pte;
1009 	pte_t ptent;
1010 	spinlock_t *src_ptl, *dst_ptl;
1011 	int progress, ret = 0;
1012 	int rss[NR_MM_COUNTERS];
1013 	swp_entry_t entry = (swp_entry_t){0};
1014 	struct folio *prealloc = NULL;
1015 
1016 again:
1017 	progress = 0;
1018 	init_rss_vec(rss);
1019 
1020 	/*
1021 	 * copy_pmd_range()'s prior pmd_none_or_clear_bad(src_pmd), and the
1022 	 * error handling here, assume that exclusive mmap_lock on dst and src
1023 	 * protects anon from unexpected THP transitions; with shmem and file
1024 	 * protected by mmap_lock-less collapse skipping areas with anon_vma
1025 	 * (whereas vma_needs_copy() skips areas without anon_vma).  A rework
1026 	 * can remove such assumptions later, but this is good enough for now.
1027 	 */
1028 	dst_pte = pte_alloc_map_lock(dst_mm, dst_pmd, addr, &dst_ptl);
1029 	if (!dst_pte) {
1030 		ret = -ENOMEM;
1031 		goto out;
1032 	}
1033 	src_pte = pte_offset_map_nolock(src_mm, src_pmd, addr, &src_ptl);
1034 	if (!src_pte) {
1035 		pte_unmap_unlock(dst_pte, dst_ptl);
1036 		/* ret == 0 */
1037 		goto out;
1038 	}
1039 	spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
1040 	orig_src_pte = src_pte;
1041 	orig_dst_pte = dst_pte;
1042 	arch_enter_lazy_mmu_mode();
1043 
1044 	do {
1045 		/*
1046 		 * We are holding two locks at this point - either of them
1047 		 * could generate latencies in another task on another CPU.
1048 		 */
1049 		if (progress >= 32) {
1050 			progress = 0;
1051 			if (need_resched() ||
1052 			    spin_needbreak(src_ptl) || spin_needbreak(dst_ptl))
1053 				break;
1054 		}
1055 		ptent = ptep_get(src_pte);
1056 		if (pte_none(ptent)) {
1057 			progress++;
1058 			continue;
1059 		}
1060 		if (unlikely(!pte_present(ptent))) {
1061 			ret = copy_nonpresent_pte(dst_mm, src_mm,
1062 						  dst_pte, src_pte,
1063 						  dst_vma, src_vma,
1064 						  addr, rss);
1065 			if (ret == -EIO) {
1066 				entry = pte_to_swp_entry(ptep_get(src_pte));
1067 				break;
1068 			} else if (ret == -EBUSY) {
1069 				break;
1070 			} else if (!ret) {
1071 				progress += 8;
1072 				continue;
1073 			}
1074 
1075 			/*
1076 			 * Device exclusive entry restored, continue by copying
1077 			 * the now present pte.
1078 			 */
1079 			WARN_ON_ONCE(ret != -ENOENT);
1080 		}
1081 		/* copy_present_pte() will clear `*prealloc' if consumed */
1082 		ret = copy_present_pte(dst_vma, src_vma, dst_pte, src_pte,
1083 				       addr, rss, &prealloc);
1084 		/*
1085 		 * If we need a pre-allocated page for this pte, drop the
1086 		 * locks, allocate, and try again.
1087 		 */
1088 		if (unlikely(ret == -EAGAIN))
1089 			break;
1090 		if (unlikely(prealloc)) {
1091 			/*
1092 			 * pre-alloc page cannot be reused by next time so as
1093 			 * to strictly follow mempolicy (e.g., alloc_page_vma()
1094 			 * will allocate page according to address).  This
1095 			 * could only happen if one pinned pte changed.
1096 			 */
1097 			folio_put(prealloc);
1098 			prealloc = NULL;
1099 		}
1100 		progress += 8;
1101 	} while (dst_pte++, src_pte++, addr += PAGE_SIZE, addr != end);
1102 
1103 	arch_leave_lazy_mmu_mode();
1104 	pte_unmap_unlock(orig_src_pte, src_ptl);
1105 	add_mm_rss_vec(dst_mm, rss);
1106 	pte_unmap_unlock(orig_dst_pte, dst_ptl);
1107 	cond_resched();
1108 
1109 	if (ret == -EIO) {
1110 		VM_WARN_ON_ONCE(!entry.val);
1111 		if (add_swap_count_continuation(entry, GFP_KERNEL) < 0) {
1112 			ret = -ENOMEM;
1113 			goto out;
1114 		}
1115 		entry.val = 0;
1116 	} else if (ret == -EBUSY) {
1117 		goto out;
1118 	} else if (ret ==  -EAGAIN) {
1119 		prealloc = page_copy_prealloc(src_mm, src_vma, addr);
1120 		if (!prealloc)
1121 			return -ENOMEM;
1122 	} else if (ret) {
1123 		VM_WARN_ON_ONCE(1);
1124 	}
1125 
1126 	/* We've captured and resolved the error. Reset, try again. */
1127 	ret = 0;
1128 
1129 	if (addr != end)
1130 		goto again;
1131 out:
1132 	if (unlikely(prealloc))
1133 		folio_put(prealloc);
1134 	return ret;
1135 }
1136 
1137 static inline int
copy_pmd_range(struct vm_area_struct * dst_vma,struct vm_area_struct * src_vma,pud_t * dst_pud,pud_t * src_pud,unsigned long addr,unsigned long end)1138 copy_pmd_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
1139 	       pud_t *dst_pud, pud_t *src_pud, unsigned long addr,
1140 	       unsigned long end)
1141 {
1142 	struct mm_struct *dst_mm = dst_vma->vm_mm;
1143 	struct mm_struct *src_mm = src_vma->vm_mm;
1144 	pmd_t *src_pmd, *dst_pmd;
1145 	unsigned long next;
1146 
1147 	dst_pmd = pmd_alloc(dst_mm, dst_pud, addr);
1148 	if (!dst_pmd)
1149 		return -ENOMEM;
1150 	src_pmd = pmd_offset(src_pud, addr);
1151 	do {
1152 		next = pmd_addr_end(addr, end);
1153 		if (is_swap_pmd(*src_pmd) || pmd_trans_huge(*src_pmd)
1154 			|| pmd_devmap(*src_pmd)) {
1155 			int err;
1156 			VM_BUG_ON_VMA(next-addr != HPAGE_PMD_SIZE, src_vma);
1157 			err = copy_huge_pmd(dst_mm, src_mm, dst_pmd, src_pmd,
1158 					    addr, dst_vma, src_vma);
1159 			if (err == -ENOMEM)
1160 				return -ENOMEM;
1161 			if (!err)
1162 				continue;
1163 			/* fall through */
1164 		}
1165 		if (pmd_none_or_clear_bad(src_pmd))
1166 			continue;
1167 		if (copy_pte_range(dst_vma, src_vma, dst_pmd, src_pmd,
1168 				   addr, next))
1169 			return -ENOMEM;
1170 	} while (dst_pmd++, src_pmd++, addr = next, addr != end);
1171 	return 0;
1172 }
1173 
1174 static inline int
copy_pud_range(struct vm_area_struct * dst_vma,struct vm_area_struct * src_vma,p4d_t * dst_p4d,p4d_t * src_p4d,unsigned long addr,unsigned long end)1175 copy_pud_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
1176 	       p4d_t *dst_p4d, p4d_t *src_p4d, unsigned long addr,
1177 	       unsigned long end)
1178 {
1179 	struct mm_struct *dst_mm = dst_vma->vm_mm;
1180 	struct mm_struct *src_mm = src_vma->vm_mm;
1181 	pud_t *src_pud, *dst_pud;
1182 	unsigned long next;
1183 
1184 	dst_pud = pud_alloc(dst_mm, dst_p4d, addr);
1185 	if (!dst_pud)
1186 		return -ENOMEM;
1187 	src_pud = pud_offset(src_p4d, addr);
1188 	do {
1189 		next = pud_addr_end(addr, end);
1190 		if (pud_trans_huge(*src_pud) || pud_devmap(*src_pud)) {
1191 			int err;
1192 
1193 			VM_BUG_ON_VMA(next-addr != HPAGE_PUD_SIZE, src_vma);
1194 			err = copy_huge_pud(dst_mm, src_mm,
1195 					    dst_pud, src_pud, addr, src_vma);
1196 			if (err == -ENOMEM)
1197 				return -ENOMEM;
1198 			if (!err)
1199 				continue;
1200 			/* fall through */
1201 		}
1202 		if (pud_none_or_clear_bad(src_pud))
1203 			continue;
1204 		if (copy_pmd_range(dst_vma, src_vma, dst_pud, src_pud,
1205 				   addr, next))
1206 			return -ENOMEM;
1207 	} while (dst_pud++, src_pud++, addr = next, addr != end);
1208 	return 0;
1209 }
1210 
1211 static inline int
copy_p4d_range(struct vm_area_struct * dst_vma,struct vm_area_struct * src_vma,pgd_t * dst_pgd,pgd_t * src_pgd,unsigned long addr,unsigned long end)1212 copy_p4d_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
1213 	       pgd_t *dst_pgd, pgd_t *src_pgd, unsigned long addr,
1214 	       unsigned long end)
1215 {
1216 	struct mm_struct *dst_mm = dst_vma->vm_mm;
1217 	p4d_t *src_p4d, *dst_p4d;
1218 	unsigned long next;
1219 
1220 	dst_p4d = p4d_alloc(dst_mm, dst_pgd, addr);
1221 	if (!dst_p4d)
1222 		return -ENOMEM;
1223 	src_p4d = p4d_offset(src_pgd, addr);
1224 	do {
1225 		next = p4d_addr_end(addr, end);
1226 		if (p4d_none_or_clear_bad(src_p4d))
1227 			continue;
1228 		if (copy_pud_range(dst_vma, src_vma, dst_p4d, src_p4d,
1229 				   addr, next))
1230 			return -ENOMEM;
1231 	} while (dst_p4d++, src_p4d++, addr = next, addr != end);
1232 	return 0;
1233 }
1234 
1235 /*
1236  * Return true if the vma needs to copy the pgtable during this fork().  Return
1237  * false when we can speed up fork() by allowing lazy page faults later until
1238  * when the child accesses the memory range.
1239  */
1240 static bool
vma_needs_copy(struct vm_area_struct * dst_vma,struct vm_area_struct * src_vma)1241 vma_needs_copy(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma)
1242 {
1243 	/*
1244 	 * Always copy pgtables when dst_vma has uffd-wp enabled even if it's
1245 	 * file-backed (e.g. shmem). Because when uffd-wp is enabled, pgtable
1246 	 * contains uffd-wp protection information, that's something we can't
1247 	 * retrieve from page cache, and skip copying will lose those info.
1248 	 */
1249 	if (userfaultfd_wp(dst_vma))
1250 		return true;
1251 
1252 	if (src_vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP))
1253 		return true;
1254 
1255 	if (src_vma->anon_vma)
1256 		return true;
1257 
1258 	/*
1259 	 * Don't copy ptes where a page fault will fill them correctly.  Fork
1260 	 * becomes much lighter when there are big shared or private readonly
1261 	 * mappings. The tradeoff is that copy_page_range is more efficient
1262 	 * than faulting.
1263 	 */
1264 	return false;
1265 }
1266 
1267 int
copy_page_range(struct vm_area_struct * dst_vma,struct vm_area_struct * src_vma)1268 copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma)
1269 {
1270 	pgd_t *src_pgd, *dst_pgd;
1271 	unsigned long next;
1272 	unsigned long addr = src_vma->vm_start;
1273 	unsigned long end = src_vma->vm_end;
1274 	struct mm_struct *dst_mm = dst_vma->vm_mm;
1275 	struct mm_struct *src_mm = src_vma->vm_mm;
1276 	struct mmu_notifier_range range;
1277 	bool is_cow;
1278 	int ret;
1279 
1280 	if (!vma_needs_copy(dst_vma, src_vma))
1281 		return 0;
1282 
1283 	if (is_vm_hugetlb_page(src_vma))
1284 		return copy_hugetlb_page_range(dst_mm, src_mm, dst_vma, src_vma);
1285 
1286 	if (unlikely(src_vma->vm_flags & VM_PFNMAP)) {
1287 		/*
1288 		 * We do not free on error cases below as remove_vma
1289 		 * gets called on error from higher level routine
1290 		 */
1291 		ret = track_pfn_copy(src_vma);
1292 		if (ret)
1293 			return ret;
1294 	}
1295 
1296 	/*
1297 	 * We need to invalidate the secondary MMU mappings only when
1298 	 * there could be a permission downgrade on the ptes of the
1299 	 * parent mm. And a permission downgrade will only happen if
1300 	 * is_cow_mapping() returns true.
1301 	 */
1302 	is_cow = is_cow_mapping(src_vma->vm_flags);
1303 
1304 	if (is_cow) {
1305 		mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_PAGE,
1306 					0, src_mm, addr, end);
1307 		mmu_notifier_invalidate_range_start(&range);
1308 		/*
1309 		 * Disabling preemption is not needed for the write side, as
1310 		 * the read side doesn't spin, but goes to the mmap_lock.
1311 		 *
1312 		 * Use the raw variant of the seqcount_t write API to avoid
1313 		 * lockdep complaining about preemptibility.
1314 		 */
1315 		vma_assert_write_locked(src_vma);
1316 		raw_write_seqcount_begin(&src_mm->write_protect_seq);
1317 	}
1318 
1319 	ret = 0;
1320 	dst_pgd = pgd_offset(dst_mm, addr);
1321 	src_pgd = pgd_offset(src_mm, addr);
1322 	do {
1323 		next = pgd_addr_end(addr, end);
1324 		if (pgd_none_or_clear_bad(src_pgd))
1325 			continue;
1326 		if (unlikely(copy_p4d_range(dst_vma, src_vma, dst_pgd, src_pgd,
1327 					    addr, next))) {
1328 			untrack_pfn_clear(dst_vma);
1329 			ret = -ENOMEM;
1330 			break;
1331 		}
1332 	} while (dst_pgd++, src_pgd++, addr = next, addr != end);
1333 
1334 	if (is_cow) {
1335 		raw_write_seqcount_end(&src_mm->write_protect_seq);
1336 		mmu_notifier_invalidate_range_end(&range);
1337 	}
1338 	return ret;
1339 }
1340 
1341 /* Whether we should zap all COWed (private) pages too */
should_zap_cows(struct zap_details * details)1342 static inline bool should_zap_cows(struct zap_details *details)
1343 {
1344 	/* By default, zap all pages */
1345 	if (!details)
1346 		return true;
1347 
1348 	/* Or, we zap COWed pages only if the caller wants to */
1349 	return details->even_cows;
1350 }
1351 
1352 /* Decides whether we should zap this page with the page pointer specified */
should_zap_page(struct zap_details * details,struct page * page)1353 static inline bool should_zap_page(struct zap_details *details, struct page *page)
1354 {
1355 	/* If we can make a decision without *page.. */
1356 	if (should_zap_cows(details))
1357 		return true;
1358 
1359 	/* E.g. the caller passes NULL for the case of a zero page */
1360 	if (!page)
1361 		return true;
1362 
1363 	/* Otherwise we should only zap non-anon pages */
1364 	return !PageAnon(page);
1365 }
1366 
zap_drop_file_uffd_wp(struct zap_details * details)1367 static inline bool zap_drop_file_uffd_wp(struct zap_details *details)
1368 {
1369 	if (!details)
1370 		return false;
1371 
1372 	return details->zap_flags & ZAP_FLAG_DROP_MARKER;
1373 }
1374 
1375 /*
1376  * This function makes sure that we'll replace the none pte with an uffd-wp
1377  * swap special pte marker when necessary. Must be with the pgtable lock held.
1378  */
1379 static inline void
zap_install_uffd_wp_if_needed(struct vm_area_struct * vma,unsigned long addr,pte_t * pte,struct zap_details * details,pte_t pteval)1380 zap_install_uffd_wp_if_needed(struct vm_area_struct *vma,
1381 			      unsigned long addr, pte_t *pte,
1382 			      struct zap_details *details, pte_t pteval)
1383 {
1384 	/* Zap on anonymous always means dropping everything */
1385 	if (vma_is_anonymous(vma))
1386 		return;
1387 
1388 	if (zap_drop_file_uffd_wp(details))
1389 		return;
1390 
1391 	pte_install_uffd_wp_if_needed(vma, addr, pte, pteval);
1392 }
1393 
zap_pte_range(struct mmu_gather * tlb,struct vm_area_struct * vma,pmd_t * pmd,unsigned long addr,unsigned long end,struct zap_details * details)1394 static unsigned long zap_pte_range(struct mmu_gather *tlb,
1395 				struct vm_area_struct *vma, pmd_t *pmd,
1396 				unsigned long addr, unsigned long end,
1397 				struct zap_details *details)
1398 {
1399 	struct mm_struct *mm = tlb->mm;
1400 	int force_flush = 0;
1401 	int rss[NR_MM_COUNTERS];
1402 	spinlock_t *ptl;
1403 	pte_t *start_pte;
1404 	pte_t *pte;
1405 	swp_entry_t entry;
1406 
1407 	tlb_change_page_size(tlb, PAGE_SIZE);
1408 	init_rss_vec(rss);
1409 	start_pte = pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
1410 	if (!pte)
1411 		return addr;
1412 
1413 	flush_tlb_batched_pending(mm);
1414 	arch_enter_lazy_mmu_mode();
1415 	do {
1416 		pte_t ptent = ptep_get(pte);
1417 		struct page *page;
1418 
1419 		if (pte_none(ptent))
1420 			continue;
1421 
1422 		if (need_resched())
1423 			break;
1424 
1425 		if (pte_present(ptent)) {
1426 			unsigned int delay_rmap;
1427 
1428 			page = vm_normal_page(vma, addr, ptent);
1429 			if (unlikely(!should_zap_page(details, page)))
1430 				continue;
1431 			ptent = ptep_get_and_clear_full(mm, addr, pte,
1432 							tlb->fullmm);
1433 			arch_check_zapped_pte(vma, ptent);
1434 			tlb_remove_tlb_entry(tlb, pte, addr);
1435 			zap_install_uffd_wp_if_needed(vma, addr, pte, details,
1436 						      ptent);
1437 			if (unlikely(!page)) {
1438 				ksm_might_unmap_zero_page(mm, ptent);
1439 				continue;
1440 			}
1441 
1442 			delay_rmap = 0;
1443 			if (!PageAnon(page)) {
1444 				if (pte_dirty(ptent)) {
1445 					set_page_dirty(page);
1446 					if (tlb_delay_rmap(tlb)) {
1447 						delay_rmap = 1;
1448 						force_flush = 1;
1449 					}
1450 				}
1451 				if (pte_young(ptent) && likely(vma_has_recency(vma)))
1452 					mark_page_accessed(page);
1453 			}
1454 			rss[mm_counter(page)]--;
1455 			if (!delay_rmap) {
1456 				page_remove_rmap(page, vma, false);
1457 				if (unlikely(page_mapcount(page) < 0))
1458 					print_bad_pte(vma, addr, ptent, page);
1459 			}
1460 			if (unlikely(__tlb_remove_page(tlb, page, delay_rmap))) {
1461 				force_flush = 1;
1462 				addr += PAGE_SIZE;
1463 				break;
1464 			}
1465 			continue;
1466 		}
1467 
1468 		entry = pte_to_swp_entry(ptent);
1469 		if (is_device_private_entry(entry) ||
1470 		    is_device_exclusive_entry(entry)) {
1471 			page = pfn_swap_entry_to_page(entry);
1472 			if (unlikely(!should_zap_page(details, page)))
1473 				continue;
1474 			/*
1475 			 * Both device private/exclusive mappings should only
1476 			 * work with anonymous page so far, so we don't need to
1477 			 * consider uffd-wp bit when zap. For more information,
1478 			 * see zap_install_uffd_wp_if_needed().
1479 			 */
1480 			WARN_ON_ONCE(!vma_is_anonymous(vma));
1481 			rss[mm_counter(page)]--;
1482 			if (is_device_private_entry(entry))
1483 				page_remove_rmap(page, vma, false);
1484 			put_page(page);
1485 		} else if (!non_swap_entry(entry)) {
1486 			/* Genuine swap entry, hence a private anon page */
1487 			if (!should_zap_cows(details))
1488 				continue;
1489 			rss[MM_SWAPENTS]--;
1490 			if (unlikely(!free_swap_and_cache(entry)))
1491 				print_bad_pte(vma, addr, ptent, NULL);
1492 		} else if (is_migration_entry(entry)) {
1493 			page = pfn_swap_entry_to_page(entry);
1494 			if (!should_zap_page(details, page))
1495 				continue;
1496 			rss[mm_counter(page)]--;
1497 		} else if (pte_marker_entry_uffd_wp(entry)) {
1498 			/*
1499 			 * For anon: always drop the marker; for file: only
1500 			 * drop the marker if explicitly requested.
1501 			 */
1502 			if (!vma_is_anonymous(vma) &&
1503 			    !zap_drop_file_uffd_wp(details))
1504 				continue;
1505 		} else if (is_hwpoison_entry(entry) ||
1506 			   is_poisoned_swp_entry(entry)) {
1507 			if (!should_zap_cows(details))
1508 				continue;
1509 		} else {
1510 			/* We should have covered all the swap entry types */
1511 			WARN_ON_ONCE(1);
1512 		}
1513 		pte_clear_not_present_full(mm, addr, pte, tlb->fullmm);
1514 		zap_install_uffd_wp_if_needed(vma, addr, pte, details, ptent);
1515 	} while (pte++, addr += PAGE_SIZE, addr != end);
1516 
1517 	add_mm_rss_vec(mm, rss);
1518 	arch_leave_lazy_mmu_mode();
1519 
1520 	/* Do the actual TLB flush before dropping ptl */
1521 	if (force_flush) {
1522 		tlb_flush_mmu_tlbonly(tlb);
1523 		tlb_flush_rmaps(tlb, vma);
1524 	}
1525 	pte_unmap_unlock(start_pte, ptl);
1526 
1527 	/*
1528 	 * If we forced a TLB flush (either due to running out of
1529 	 * batch buffers or because we needed to flush dirty TLB
1530 	 * entries before releasing the ptl), free the batched
1531 	 * memory too. Come back again if we didn't do everything.
1532 	 */
1533 	if (force_flush)
1534 		tlb_flush_mmu(tlb);
1535 
1536 	return addr;
1537 }
1538 
zap_pmd_range(struct mmu_gather * tlb,struct vm_area_struct * vma,pud_t * pud,unsigned long addr,unsigned long end,struct zap_details * details)1539 static inline unsigned long zap_pmd_range(struct mmu_gather *tlb,
1540 				struct vm_area_struct *vma, pud_t *pud,
1541 				unsigned long addr, unsigned long end,
1542 				struct zap_details *details)
1543 {
1544 	pmd_t *pmd;
1545 	unsigned long next;
1546 
1547 	pmd = pmd_offset(pud, addr);
1548 	do {
1549 		next = pmd_addr_end(addr, end);
1550 		if (is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) || pmd_devmap(*pmd)) {
1551 			if (next - addr != HPAGE_PMD_SIZE)
1552 				__split_huge_pmd(vma, pmd, addr, false, NULL);
1553 			else if (zap_huge_pmd(tlb, vma, pmd, addr)) {
1554 				addr = next;
1555 				continue;
1556 			}
1557 			/* fall through */
1558 		} else if (details && details->single_folio &&
1559 			   folio_test_pmd_mappable(details->single_folio) &&
1560 			   next - addr == HPAGE_PMD_SIZE && pmd_none(*pmd)) {
1561 			spinlock_t *ptl = pmd_lock(tlb->mm, pmd);
1562 			/*
1563 			 * Take and drop THP pmd lock so that we cannot return
1564 			 * prematurely, while zap_huge_pmd() has cleared *pmd,
1565 			 * but not yet decremented compound_mapcount().
1566 			 */
1567 			spin_unlock(ptl);
1568 		}
1569 		if (pmd_none(*pmd)) {
1570 			addr = next;
1571 			continue;
1572 		}
1573 		addr = zap_pte_range(tlb, vma, pmd, addr, next, details);
1574 		if (addr != next)
1575 			pmd--;
1576 	} while (pmd++, cond_resched(), addr != end);
1577 
1578 	return addr;
1579 }
1580 
zap_pud_range(struct mmu_gather * tlb,struct vm_area_struct * vma,p4d_t * p4d,unsigned long addr,unsigned long end,struct zap_details * details)1581 static inline unsigned long zap_pud_range(struct mmu_gather *tlb,
1582 				struct vm_area_struct *vma, p4d_t *p4d,
1583 				unsigned long addr, unsigned long end,
1584 				struct zap_details *details)
1585 {
1586 	pud_t *pud;
1587 	unsigned long next;
1588 
1589 	pud = pud_offset(p4d, addr);
1590 	do {
1591 		next = pud_addr_end(addr, end);
1592 		if (pud_trans_huge(*pud) || pud_devmap(*pud)) {
1593 			if (next - addr != HPAGE_PUD_SIZE) {
1594 				mmap_assert_locked(tlb->mm);
1595 				split_huge_pud(vma, pud, addr);
1596 			} else if (zap_huge_pud(tlb, vma, pud, addr))
1597 				goto next;
1598 			/* fall through */
1599 		}
1600 		if (pud_none_or_clear_bad(pud))
1601 			continue;
1602 		next = zap_pmd_range(tlb, vma, pud, addr, next, details);
1603 next:
1604 		cond_resched();
1605 	} while (pud++, addr = next, addr != end);
1606 
1607 	return addr;
1608 }
1609 
zap_p4d_range(struct mmu_gather * tlb,struct vm_area_struct * vma,pgd_t * pgd,unsigned long addr,unsigned long end,struct zap_details * details)1610 static inline unsigned long zap_p4d_range(struct mmu_gather *tlb,
1611 				struct vm_area_struct *vma, pgd_t *pgd,
1612 				unsigned long addr, unsigned long end,
1613 				struct zap_details *details)
1614 {
1615 	p4d_t *p4d;
1616 	unsigned long next;
1617 
1618 	p4d = p4d_offset(pgd, addr);
1619 	do {
1620 		next = p4d_addr_end(addr, end);
1621 		if (p4d_none_or_clear_bad(p4d))
1622 			continue;
1623 		next = zap_pud_range(tlb, vma, p4d, addr, next, details);
1624 	} while (p4d++, addr = next, addr != end);
1625 
1626 	return addr;
1627 }
1628 
unmap_page_range(struct mmu_gather * tlb,struct vm_area_struct * vma,unsigned long addr,unsigned long end,struct zap_details * details)1629 void unmap_page_range(struct mmu_gather *tlb,
1630 			     struct vm_area_struct *vma,
1631 			     unsigned long addr, unsigned long end,
1632 			     struct zap_details *details)
1633 {
1634 	pgd_t *pgd;
1635 	unsigned long next;
1636 
1637 	BUG_ON(addr >= end);
1638 	tlb_start_vma(tlb, vma);
1639 	pgd = pgd_offset(vma->vm_mm, addr);
1640 	do {
1641 		next = pgd_addr_end(addr, end);
1642 		if (pgd_none_or_clear_bad(pgd))
1643 			continue;
1644 		next = zap_p4d_range(tlb, vma, pgd, addr, next, details);
1645 	} while (pgd++, addr = next, addr != end);
1646 	tlb_end_vma(tlb, vma);
1647 }
1648 
1649 
unmap_single_vma(struct mmu_gather * tlb,struct vm_area_struct * vma,unsigned long start_addr,unsigned long end_addr,struct zap_details * details,bool mm_wr_locked)1650 static void unmap_single_vma(struct mmu_gather *tlb,
1651 		struct vm_area_struct *vma, unsigned long start_addr,
1652 		unsigned long end_addr,
1653 		struct zap_details *details, bool mm_wr_locked)
1654 {
1655 	unsigned long start = max(vma->vm_start, start_addr);
1656 	unsigned long end;
1657 
1658 	if (start >= vma->vm_end)
1659 		return;
1660 	end = min(vma->vm_end, end_addr);
1661 	if (end <= vma->vm_start)
1662 		return;
1663 
1664 	if (vma->vm_file)
1665 		uprobe_munmap(vma, start, end);
1666 
1667 	if (unlikely(vma->vm_flags & VM_PFNMAP))
1668 		untrack_pfn(vma, 0, 0, mm_wr_locked);
1669 
1670 	if (start != end) {
1671 		if (unlikely(is_vm_hugetlb_page(vma))) {
1672 			/*
1673 			 * It is undesirable to test vma->vm_file as it
1674 			 * should be non-null for valid hugetlb area.
1675 			 * However, vm_file will be NULL in the error
1676 			 * cleanup path of mmap_region. When
1677 			 * hugetlbfs ->mmap method fails,
1678 			 * mmap_region() nullifies vma->vm_file
1679 			 * before calling this function to clean up.
1680 			 * Since no pte has actually been setup, it is
1681 			 * safe to do nothing in this case.
1682 			 */
1683 			if (vma->vm_file) {
1684 				zap_flags_t zap_flags = details ?
1685 				    details->zap_flags : 0;
1686 				__unmap_hugepage_range(tlb, vma, start, end,
1687 							     NULL, zap_flags);
1688 			}
1689 		} else
1690 			unmap_page_range(tlb, vma, start, end, details);
1691 	}
1692 }
1693 
1694 /**
1695  * unmap_vmas - unmap a range of memory covered by a list of vma's
1696  * @tlb: address of the caller's struct mmu_gather
1697  * @mas: the maple state
1698  * @vma: the starting vma
1699  * @start_addr: virtual address at which to start unmapping
1700  * @end_addr: virtual address at which to end unmapping
1701  * @tree_end: The maximum index to check
1702  * @mm_wr_locked: lock flag
1703  *
1704  * Unmap all pages in the vma list.
1705  *
1706  * Only addresses between `start' and `end' will be unmapped.
1707  *
1708  * The VMA list must be sorted in ascending virtual address order.
1709  *
1710  * unmap_vmas() assumes that the caller will flush the whole unmapped address
1711  * range after unmap_vmas() returns.  So the only responsibility here is to
1712  * ensure that any thus-far unmapped pages are flushed before unmap_vmas()
1713  * drops the lock and schedules.
1714  */
unmap_vmas(struct mmu_gather * tlb,struct ma_state * mas,struct vm_area_struct * vma,unsigned long start_addr,unsigned long end_addr,unsigned long tree_end,bool mm_wr_locked)1715 void unmap_vmas(struct mmu_gather *tlb, struct ma_state *mas,
1716 		struct vm_area_struct *vma, unsigned long start_addr,
1717 		unsigned long end_addr, unsigned long tree_end,
1718 		bool mm_wr_locked)
1719 {
1720 	struct mmu_notifier_range range;
1721 	struct zap_details details = {
1722 		.zap_flags = ZAP_FLAG_DROP_MARKER | ZAP_FLAG_UNMAP,
1723 		/* Careful - we need to zap private pages too! */
1724 		.even_cows = true,
1725 	};
1726 
1727 	mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma->vm_mm,
1728 				start_addr, end_addr);
1729 	mmu_notifier_invalidate_range_start(&range);
1730 	do {
1731 		unsigned long start = start_addr;
1732 		unsigned long end = end_addr;
1733 		hugetlb_zap_begin(vma, &start, &end);
1734 		unmap_single_vma(tlb, vma, start, end, &details,
1735 				 mm_wr_locked);
1736 		hugetlb_zap_end(vma, &details);
1737 	} while ((vma = mas_find(mas, tree_end - 1)) != NULL);
1738 	mmu_notifier_invalidate_range_end(&range);
1739 }
1740 
1741 /**
1742  * zap_page_range_single - remove user pages in a given range
1743  * @vma: vm_area_struct holding the applicable pages
1744  * @address: starting address of pages to zap
1745  * @size: number of bytes to zap
1746  * @details: details of shared cache invalidation
1747  *
1748  * The range must fit into one VMA.
1749  */
zap_page_range_single(struct vm_area_struct * vma,unsigned long address,unsigned long size,struct zap_details * details)1750 void zap_page_range_single(struct vm_area_struct *vma, unsigned long address,
1751 		unsigned long size, struct zap_details *details)
1752 {
1753 	const unsigned long end = address + size;
1754 	struct mmu_notifier_range range;
1755 	struct mmu_gather tlb;
1756 
1757 	lru_add_drain();
1758 	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,
1759 				address, end);
1760 	hugetlb_zap_begin(vma, &range.start, &range.end);
1761 	tlb_gather_mmu(&tlb, vma->vm_mm);
1762 	update_hiwater_rss(vma->vm_mm);
1763 	mmu_notifier_invalidate_range_start(&range);
1764 	/*
1765 	 * unmap 'address-end' not 'range.start-range.end' as range
1766 	 * could have been expanded for hugetlb pmd sharing.
1767 	 */
1768 	unmap_single_vma(&tlb, vma, address, end, details, false);
1769 	mmu_notifier_invalidate_range_end(&range);
1770 	tlb_finish_mmu(&tlb);
1771 	hugetlb_zap_end(vma, details);
1772 }
1773 
1774 /**
1775  * zap_vma_ptes - remove ptes mapping the vma
1776  * @vma: vm_area_struct holding ptes to be zapped
1777  * @address: starting address of pages to zap
1778  * @size: number of bytes to zap
1779  *
1780  * This function only unmaps ptes assigned to VM_PFNMAP vmas.
1781  *
1782  * The entire address range must be fully contained within the vma.
1783  *
1784  */
zap_vma_ptes(struct vm_area_struct * vma,unsigned long address,unsigned long size)1785 void zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
1786 		unsigned long size)
1787 {
1788 	if (!range_in_vma(vma, address, address + size) ||
1789 	    		!(vma->vm_flags & VM_PFNMAP))
1790 		return;
1791 
1792 	zap_page_range_single(vma, address, size, NULL);
1793 }
1794 EXPORT_SYMBOL_GPL(zap_vma_ptes);
1795 
walk_to_pmd(struct mm_struct * mm,unsigned long addr)1796 static pmd_t *walk_to_pmd(struct mm_struct *mm, unsigned long addr)
1797 {
1798 	pgd_t *pgd;
1799 	p4d_t *p4d;
1800 	pud_t *pud;
1801 	pmd_t *pmd;
1802 
1803 	pgd = pgd_offset(mm, addr);
1804 	p4d = p4d_alloc(mm, pgd, addr);
1805 	if (!p4d)
1806 		return NULL;
1807 	pud = pud_alloc(mm, p4d, addr);
1808 	if (!pud)
1809 		return NULL;
1810 	pmd = pmd_alloc(mm, pud, addr);
1811 	if (!pmd)
1812 		return NULL;
1813 
1814 	VM_BUG_ON(pmd_trans_huge(*pmd));
1815 	return pmd;
1816 }
1817 
__get_locked_pte(struct mm_struct * mm,unsigned long addr,spinlock_t ** ptl)1818 pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr,
1819 			spinlock_t **ptl)
1820 {
1821 	pmd_t *pmd = walk_to_pmd(mm, addr);
1822 
1823 	if (!pmd)
1824 		return NULL;
1825 	return pte_alloc_map_lock(mm, pmd, addr, ptl);
1826 }
1827 
validate_page_before_insert(struct page * page)1828 static int validate_page_before_insert(struct page *page)
1829 {
1830 	if (PageAnon(page) || PageSlab(page) || page_has_type(page))
1831 		return -EINVAL;
1832 	flush_dcache_page(page);
1833 	return 0;
1834 }
1835 
insert_page_into_pte_locked(struct vm_area_struct * vma,pte_t * pte,unsigned long addr,struct page * page,pgprot_t prot)1836 static int insert_page_into_pte_locked(struct vm_area_struct *vma, pte_t *pte,
1837 			unsigned long addr, struct page *page, pgprot_t prot)
1838 {
1839 	if (!pte_none(ptep_get(pte)))
1840 		return -EBUSY;
1841 	/* Ok, finally just insert the thing.. */
1842 	get_page(page);
1843 	inc_mm_counter(vma->vm_mm, mm_counter_file(page));
1844 	page_add_file_rmap(page, vma, false);
1845 	set_pte_at(vma->vm_mm, addr, pte, mk_pte(page, prot));
1846 	return 0;
1847 }
1848 
1849 /*
1850  * This is the old fallback for page remapping.
1851  *
1852  * For historical reasons, it only allows reserved pages. Only
1853  * old drivers should use this, and they needed to mark their
1854  * pages reserved for the old functions anyway.
1855  */
insert_page(struct vm_area_struct * vma,unsigned long addr,struct page * page,pgprot_t prot)1856 static int insert_page(struct vm_area_struct *vma, unsigned long addr,
1857 			struct page *page, pgprot_t prot)
1858 {
1859 	int retval;
1860 	pte_t *pte;
1861 	spinlock_t *ptl;
1862 
1863 	retval = validate_page_before_insert(page);
1864 	if (retval)
1865 		goto out;
1866 	retval = -ENOMEM;
1867 	pte = get_locked_pte(vma->vm_mm, addr, &ptl);
1868 	if (!pte)
1869 		goto out;
1870 	retval = insert_page_into_pte_locked(vma, pte, addr, page, prot);
1871 	pte_unmap_unlock(pte, ptl);
1872 out:
1873 	return retval;
1874 }
1875 
insert_page_in_batch_locked(struct vm_area_struct * vma,pte_t * pte,unsigned long addr,struct page * page,pgprot_t prot)1876 static int insert_page_in_batch_locked(struct vm_area_struct *vma, pte_t *pte,
1877 			unsigned long addr, struct page *page, pgprot_t prot)
1878 {
1879 	int err;
1880 
1881 	if (!page_count(page))
1882 		return -EINVAL;
1883 	err = validate_page_before_insert(page);
1884 	if (err)
1885 		return err;
1886 	return insert_page_into_pte_locked(vma, pte, addr, page, prot);
1887 }
1888 
1889 /* insert_pages() amortizes the cost of spinlock operations
1890  * when inserting pages in a loop.
1891  */
insert_pages(struct vm_area_struct * vma,unsigned long addr,struct page ** pages,unsigned long * num,pgprot_t prot)1892 static int insert_pages(struct vm_area_struct *vma, unsigned long addr,
1893 			struct page **pages, unsigned long *num, pgprot_t prot)
1894 {
1895 	pmd_t *pmd = NULL;
1896 	pte_t *start_pte, *pte;
1897 	spinlock_t *pte_lock;
1898 	struct mm_struct *const mm = vma->vm_mm;
1899 	unsigned long curr_page_idx = 0;
1900 	unsigned long remaining_pages_total = *num;
1901 	unsigned long pages_to_write_in_pmd;
1902 	int ret;
1903 more:
1904 	ret = -EFAULT;
1905 	pmd = walk_to_pmd(mm, addr);
1906 	if (!pmd)
1907 		goto out;
1908 
1909 	pages_to_write_in_pmd = min_t(unsigned long,
1910 		remaining_pages_total, PTRS_PER_PTE - pte_index(addr));
1911 
1912 	/* Allocate the PTE if necessary; takes PMD lock once only. */
1913 	ret = -ENOMEM;
1914 	if (pte_alloc(mm, pmd))
1915 		goto out;
1916 
1917 	while (pages_to_write_in_pmd) {
1918 		int pte_idx = 0;
1919 		const int batch_size = min_t(int, pages_to_write_in_pmd, 8);
1920 
1921 		start_pte = pte_offset_map_lock(mm, pmd, addr, &pte_lock);
1922 		if (!start_pte) {
1923 			ret = -EFAULT;
1924 			goto out;
1925 		}
1926 		for (pte = start_pte; pte_idx < batch_size; ++pte, ++pte_idx) {
1927 			int err = insert_page_in_batch_locked(vma, pte,
1928 				addr, pages[curr_page_idx], prot);
1929 			if (unlikely(err)) {
1930 				pte_unmap_unlock(start_pte, pte_lock);
1931 				ret = err;
1932 				remaining_pages_total -= pte_idx;
1933 				goto out;
1934 			}
1935 			addr += PAGE_SIZE;
1936 			++curr_page_idx;
1937 		}
1938 		pte_unmap_unlock(start_pte, pte_lock);
1939 		pages_to_write_in_pmd -= batch_size;
1940 		remaining_pages_total -= batch_size;
1941 	}
1942 	if (remaining_pages_total)
1943 		goto more;
1944 	ret = 0;
1945 out:
1946 	*num = remaining_pages_total;
1947 	return ret;
1948 }
1949 
1950 /**
1951  * vm_insert_pages - insert multiple pages into user vma, batching the pmd lock.
1952  * @vma: user vma to map to
1953  * @addr: target start user address of these pages
1954  * @pages: source kernel pages
1955  * @num: in: number of pages to map. out: number of pages that were *not*
1956  * mapped. (0 means all pages were successfully mapped).
1957  *
1958  * Preferred over vm_insert_page() when inserting multiple pages.
1959  *
1960  * In case of error, we may have mapped a subset of the provided
1961  * pages. It is the caller's responsibility to account for this case.
1962  *
1963  * The same restrictions apply as in vm_insert_page().
1964  */
vm_insert_pages(struct vm_area_struct * vma,unsigned long addr,struct page ** pages,unsigned long * num)1965 int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr,
1966 			struct page **pages, unsigned long *num)
1967 {
1968 	const unsigned long end_addr = addr + (*num * PAGE_SIZE) - 1;
1969 
1970 	if (addr < vma->vm_start || end_addr >= vma->vm_end)
1971 		return -EFAULT;
1972 	if (!(vma->vm_flags & VM_MIXEDMAP)) {
1973 		BUG_ON(mmap_read_trylock(vma->vm_mm));
1974 		BUG_ON(vma->vm_flags & VM_PFNMAP);
1975 		vm_flags_set(vma, VM_MIXEDMAP);
1976 	}
1977 	/* Defer page refcount checking till we're about to map that page. */
1978 	return insert_pages(vma, addr, pages, num, vma->vm_page_prot);
1979 }
1980 EXPORT_SYMBOL(vm_insert_pages);
1981 
1982 /**
1983  * vm_insert_page - insert single page into user vma
1984  * @vma: user vma to map to
1985  * @addr: target user address of this page
1986  * @page: source kernel page
1987  *
1988  * This allows drivers to insert individual pages they've allocated
1989  * into a user vma.
1990  *
1991  * The page has to be a nice clean _individual_ kernel allocation.
1992  * If you allocate a compound page, you need to have marked it as
1993  * such (__GFP_COMP), or manually just split the page up yourself
1994  * (see split_page()).
1995  *
1996  * NOTE! Traditionally this was done with "remap_pfn_range()" which
1997  * took an arbitrary page protection parameter. This doesn't allow
1998  * that. Your vma protection will have to be set up correctly, which
1999  * means that if you want a shared writable mapping, you'd better
2000  * ask for a shared writable mapping!
2001  *
2002  * The page does not need to be reserved.
2003  *
2004  * Usually this function is called from f_op->mmap() handler
2005  * under mm->mmap_lock write-lock, so it can change vma->vm_flags.
2006  * Caller must set VM_MIXEDMAP on vma if it wants to call this
2007  * function from other places, for example from page-fault handler.
2008  *
2009  * Return: %0 on success, negative error code otherwise.
2010  */
vm_insert_page(struct vm_area_struct * vma,unsigned long addr,struct page * page)2011 int vm_insert_page(struct vm_area_struct *vma, unsigned long addr,
2012 			struct page *page)
2013 {
2014 	if (addr < vma->vm_start || addr >= vma->vm_end)
2015 		return -EFAULT;
2016 	if (!page_count(page))
2017 		return -EINVAL;
2018 	if (!(vma->vm_flags & VM_MIXEDMAP)) {
2019 		BUG_ON(mmap_read_trylock(vma->vm_mm));
2020 		BUG_ON(vma->vm_flags & VM_PFNMAP);
2021 		vm_flags_set(vma, VM_MIXEDMAP);
2022 	}
2023 	return insert_page(vma, addr, page, vma->vm_page_prot);
2024 }
2025 EXPORT_SYMBOL(vm_insert_page);
2026 
2027 /*
2028  * __vm_map_pages - maps range of kernel pages into user vma
2029  * @vma: user vma to map to
2030  * @pages: pointer to array of source kernel pages
2031  * @num: number of pages in page array
2032  * @offset: user's requested vm_pgoff
2033  *
2034  * This allows drivers to map range of kernel pages into a user vma.
2035  *
2036  * Return: 0 on success and error code otherwise.
2037  */
__vm_map_pages(struct vm_area_struct * vma,struct page ** pages,unsigned long num,unsigned long offset)2038 static int __vm_map_pages(struct vm_area_struct *vma, struct page **pages,
2039 				unsigned long num, unsigned long offset)
2040 {
2041 	unsigned long count = vma_pages(vma);
2042 	unsigned long uaddr = vma->vm_start;
2043 	int ret, i;
2044 
2045 	/* Fail if the user requested offset is beyond the end of the object */
2046 	if (offset >= num)
2047 		return -ENXIO;
2048 
2049 	/* Fail if the user requested size exceeds available object size */
2050 	if (count > num - offset)
2051 		return -ENXIO;
2052 
2053 	for (i = 0; i < count; i++) {
2054 		ret = vm_insert_page(vma, uaddr, pages[offset + i]);
2055 		if (ret < 0)
2056 			return ret;
2057 		uaddr += PAGE_SIZE;
2058 	}
2059 
2060 	return 0;
2061 }
2062 
2063 /**
2064  * vm_map_pages - maps range of kernel pages starts with non zero offset
2065  * @vma: user vma to map to
2066  * @pages: pointer to array of source kernel pages
2067  * @num: number of pages in page array
2068  *
2069  * Maps an object consisting of @num pages, catering for the user's
2070  * requested vm_pgoff
2071  *
2072  * If we fail to insert any page into the vma, the function will return
2073  * immediately leaving any previously inserted pages present.  Callers
2074  * from the mmap handler may immediately return the error as their caller
2075  * will destroy the vma, removing any successfully inserted pages. Other
2076  * callers should make their own arrangements for calling unmap_region().
2077  *
2078  * Context: Process context. Called by mmap handlers.
2079  * Return: 0 on success and error code otherwise.
2080  */
vm_map_pages(struct vm_area_struct * vma,struct page ** pages,unsigned long num)2081 int vm_map_pages(struct vm_area_struct *vma, struct page **pages,
2082 				unsigned long num)
2083 {
2084 	return __vm_map_pages(vma, pages, num, vma->vm_pgoff);
2085 }
2086 EXPORT_SYMBOL(vm_map_pages);
2087 
2088 /**
2089  * vm_map_pages_zero - map range of kernel pages starts with zero offset
2090  * @vma: user vma to map to
2091  * @pages: pointer to array of source kernel pages
2092  * @num: number of pages in page array
2093  *
2094  * Similar to vm_map_pages(), except that it explicitly sets the offset
2095  * to 0. This function is intended for the drivers that did not consider
2096  * vm_pgoff.
2097  *
2098  * Context: Process context. Called by mmap handlers.
2099  * Return: 0 on success and error code otherwise.
2100  */
vm_map_pages_zero(struct vm_area_struct * vma,struct page ** pages,unsigned long num)2101 int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages,
2102 				unsigned long num)
2103 {
2104 	return __vm_map_pages(vma, pages, num, 0);
2105 }
2106 EXPORT_SYMBOL(vm_map_pages_zero);
2107 
insert_pfn(struct vm_area_struct * vma,unsigned long addr,pfn_t pfn,pgprot_t prot,bool mkwrite)2108 static vm_fault_t insert_pfn(struct vm_area_struct *vma, unsigned long addr,
2109 			pfn_t pfn, pgprot_t prot, bool mkwrite)
2110 {
2111 	struct mm_struct *mm = vma->vm_mm;
2112 	pte_t *pte, entry;
2113 	spinlock_t *ptl;
2114 
2115 	pte = get_locked_pte(mm, addr, &ptl);
2116 	if (!pte)
2117 		return VM_FAULT_OOM;
2118 	entry = ptep_get(pte);
2119 	if (!pte_none(entry)) {
2120 		if (mkwrite) {
2121 			/*
2122 			 * For read faults on private mappings the PFN passed
2123 			 * in may not match the PFN we have mapped if the
2124 			 * mapped PFN is a writeable COW page.  In the mkwrite
2125 			 * case we are creating a writable PTE for a shared
2126 			 * mapping and we expect the PFNs to match. If they
2127 			 * don't match, we are likely racing with block
2128 			 * allocation and mapping invalidation so just skip the
2129 			 * update.
2130 			 */
2131 			if (pte_pfn(entry) != pfn_t_to_pfn(pfn)) {
2132 				WARN_ON_ONCE(!is_zero_pfn(pte_pfn(entry)));
2133 				goto out_unlock;
2134 			}
2135 			entry = pte_mkyoung(entry);
2136 			entry = maybe_mkwrite(pte_mkdirty(entry), vma);
2137 			if (ptep_set_access_flags(vma, addr, pte, entry, 1))
2138 				update_mmu_cache(vma, addr, pte);
2139 		}
2140 		goto out_unlock;
2141 	}
2142 
2143 	/* Ok, finally just insert the thing.. */
2144 	if (pfn_t_devmap(pfn))
2145 		entry = pte_mkdevmap(pfn_t_pte(pfn, prot));
2146 	else
2147 		entry = pte_mkspecial(pfn_t_pte(pfn, prot));
2148 
2149 	if (mkwrite) {
2150 		entry = pte_mkyoung(entry);
2151 		entry = maybe_mkwrite(pte_mkdirty(entry), vma);
2152 	}
2153 
2154 	set_pte_at(mm, addr, pte, entry);
2155 	update_mmu_cache(vma, addr, pte); /* XXX: why not for insert_page? */
2156 
2157 out_unlock:
2158 	pte_unmap_unlock(pte, ptl);
2159 	return VM_FAULT_NOPAGE;
2160 }
2161 
2162 /**
2163  * vmf_insert_pfn_prot - insert single pfn into user vma with specified pgprot
2164  * @vma: user vma to map to
2165  * @addr: target user address of this page
2166  * @pfn: source kernel pfn
2167  * @pgprot: pgprot flags for the inserted page
2168  *
2169  * This is exactly like vmf_insert_pfn(), except that it allows drivers
2170  * to override pgprot on a per-page basis.
2171  *
2172  * This only makes sense for IO mappings, and it makes no sense for
2173  * COW mappings.  In general, using multiple vmas is preferable;
2174  * vmf_insert_pfn_prot should only be used if using multiple VMAs is
2175  * impractical.
2176  *
2177  * pgprot typically only differs from @vma->vm_page_prot when drivers set
2178  * caching- and encryption bits different than those of @vma->vm_page_prot,
2179  * because the caching- or encryption mode may not be known at mmap() time.
2180  *
2181  * This is ok as long as @vma->vm_page_prot is not used by the core vm
2182  * to set caching and encryption bits for those vmas (except for COW pages).
2183  * This is ensured by core vm only modifying these page table entries using
2184  * functions that don't touch caching- or encryption bits, using pte_modify()
2185  * if needed. (See for example mprotect()).
2186  *
2187  * Also when new page-table entries are created, this is only done using the
2188  * fault() callback, and never using the value of vma->vm_page_prot,
2189  * except for page-table entries that point to anonymous pages as the result
2190  * of COW.
2191  *
2192  * Context: Process context.  May allocate using %GFP_KERNEL.
2193  * Return: vm_fault_t value.
2194  */
vmf_insert_pfn_prot(struct vm_area_struct * vma,unsigned long addr,unsigned long pfn,pgprot_t pgprot)2195 vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr,
2196 			unsigned long pfn, pgprot_t pgprot)
2197 {
2198 	/*
2199 	 * Technically, architectures with pte_special can avoid all these
2200 	 * restrictions (same for remap_pfn_range).  However we would like
2201 	 * consistency in testing and feature parity among all, so we should
2202 	 * try to keep these invariants in place for everybody.
2203 	 */
2204 	BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)));
2205 	BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
2206 						(VM_PFNMAP|VM_MIXEDMAP));
2207 	BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));
2208 	BUG_ON((vma->vm_flags & VM_MIXEDMAP) && pfn_valid(pfn));
2209 
2210 	if (addr < vma->vm_start || addr >= vma->vm_end)
2211 		return VM_FAULT_SIGBUS;
2212 
2213 	if (!pfn_modify_allowed(pfn, pgprot))
2214 		return VM_FAULT_SIGBUS;
2215 
2216 	track_pfn_insert(vma, &pgprot, __pfn_to_pfn_t(pfn, PFN_DEV));
2217 
2218 	return insert_pfn(vma, addr, __pfn_to_pfn_t(pfn, PFN_DEV), pgprot,
2219 			false);
2220 }
2221 EXPORT_SYMBOL(vmf_insert_pfn_prot);
2222 
2223 /**
2224  * vmf_insert_pfn - insert single pfn into user vma
2225  * @vma: user vma to map to
2226  * @addr: target user address of this page
2227  * @pfn: source kernel pfn
2228  *
2229  * Similar to vm_insert_page, this allows drivers to insert individual pages
2230  * they've allocated into a user vma. Same comments apply.
2231  *
2232  * This function should only be called from a vm_ops->fault handler, and
2233  * in that case the handler should return the result of this function.
2234  *
2235  * vma cannot be a COW mapping.
2236  *
2237  * As this is called only for pages that do not currently exist, we
2238  * do not need to flush old virtual caches or the TLB.
2239  *
2240  * Context: Process context.  May allocate using %GFP_KERNEL.
2241  * Return: vm_fault_t value.
2242  */
vmf_insert_pfn(struct vm_area_struct * vma,unsigned long addr,unsigned long pfn)2243 vm_fault_t vmf_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
2244 			unsigned long pfn)
2245 {
2246 	return vmf_insert_pfn_prot(vma, addr, pfn, vma->vm_page_prot);
2247 }
2248 EXPORT_SYMBOL(vmf_insert_pfn);
2249 
vm_mixed_ok(struct vm_area_struct * vma,pfn_t pfn)2250 static bool vm_mixed_ok(struct vm_area_struct *vma, pfn_t pfn)
2251 {
2252 	/* these checks mirror the abort conditions in vm_normal_page */
2253 	if (vma->vm_flags & VM_MIXEDMAP)
2254 		return true;
2255 	if (pfn_t_devmap(pfn))
2256 		return true;
2257 	if (pfn_t_special(pfn))
2258 		return true;
2259 	if (is_zero_pfn(pfn_t_to_pfn(pfn)))
2260 		return true;
2261 	return false;
2262 }
2263 
__vm_insert_mixed(struct vm_area_struct * vma,unsigned long addr,pfn_t pfn,bool mkwrite)2264 static vm_fault_t __vm_insert_mixed(struct vm_area_struct *vma,
2265 		unsigned long addr, pfn_t pfn, bool mkwrite)
2266 {
2267 	pgprot_t pgprot = vma->vm_page_prot;
2268 	int err;
2269 
2270 	BUG_ON(!vm_mixed_ok(vma, pfn));
2271 
2272 	if (addr < vma->vm_start || addr >= vma->vm_end)
2273 		return VM_FAULT_SIGBUS;
2274 
2275 	track_pfn_insert(vma, &pgprot, pfn);
2276 
2277 	if (!pfn_modify_allowed(pfn_t_to_pfn(pfn), pgprot))
2278 		return VM_FAULT_SIGBUS;
2279 
2280 	/*
2281 	 * If we don't have pte special, then we have to use the pfn_valid()
2282 	 * based VM_MIXEDMAP scheme (see vm_normal_page), and thus we *must*
2283 	 * refcount the page if pfn_valid is true (hence insert_page rather
2284 	 * than insert_pfn).  If a zero_pfn were inserted into a VM_MIXEDMAP
2285 	 * without pte special, it would there be refcounted as a normal page.
2286 	 */
2287 	if (!IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL) &&
2288 	    !pfn_t_devmap(pfn) && pfn_t_valid(pfn)) {
2289 		struct page *page;
2290 
2291 		/*
2292 		 * At this point we are committed to insert_page()
2293 		 * regardless of whether the caller specified flags that
2294 		 * result in pfn_t_has_page() == false.
2295 		 */
2296 		page = pfn_to_page(pfn_t_to_pfn(pfn));
2297 		err = insert_page(vma, addr, page, pgprot);
2298 	} else {
2299 		return insert_pfn(vma, addr, pfn, pgprot, mkwrite);
2300 	}
2301 
2302 	if (err == -ENOMEM)
2303 		return VM_FAULT_OOM;
2304 	if (err < 0 && err != -EBUSY)
2305 		return VM_FAULT_SIGBUS;
2306 
2307 	return VM_FAULT_NOPAGE;
2308 }
2309 
vmf_insert_mixed(struct vm_area_struct * vma,unsigned long addr,pfn_t pfn)2310 vm_fault_t vmf_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
2311 		pfn_t pfn)
2312 {
2313 	return __vm_insert_mixed(vma, addr, pfn, false);
2314 }
2315 EXPORT_SYMBOL(vmf_insert_mixed);
2316 
2317 /*
2318  *  If the insertion of PTE failed because someone else already added a
2319  *  different entry in the mean time, we treat that as success as we assume
2320  *  the same entry was actually inserted.
2321  */
vmf_insert_mixed_mkwrite(struct vm_area_struct * vma,unsigned long addr,pfn_t pfn)2322 vm_fault_t vmf_insert_mixed_mkwrite(struct vm_area_struct *vma,
2323 		unsigned long addr, pfn_t pfn)
2324 {
2325 	return __vm_insert_mixed(vma, addr, pfn, true);
2326 }
2327 EXPORT_SYMBOL(vmf_insert_mixed_mkwrite);
2328 
2329 /*
2330  * maps a range of physical memory into the requested pages. the old
2331  * mappings are removed. any references to nonexistent pages results
2332  * in null mappings (currently treated as "copy-on-access")
2333  */
remap_pte_range(struct mm_struct * mm,pmd_t * pmd,unsigned long addr,unsigned long end,unsigned long pfn,pgprot_t prot)2334 static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd,
2335 			unsigned long addr, unsigned long end,
2336 			unsigned long pfn, pgprot_t prot)
2337 {
2338 	pte_t *pte, *mapped_pte;
2339 	spinlock_t *ptl;
2340 	int err = 0;
2341 
2342 	mapped_pte = pte = pte_alloc_map_lock(mm, pmd, addr, &ptl);
2343 	if (!pte)
2344 		return -ENOMEM;
2345 	arch_enter_lazy_mmu_mode();
2346 	do {
2347 		BUG_ON(!pte_none(ptep_get(pte)));
2348 		if (!pfn_modify_allowed(pfn, prot)) {
2349 			err = -EACCES;
2350 			break;
2351 		}
2352 		set_pte_at(mm, addr, pte, pte_mkspecial(pfn_pte(pfn, prot)));
2353 		pfn++;
2354 	} while (pte++, addr += PAGE_SIZE, addr != end);
2355 	arch_leave_lazy_mmu_mode();
2356 	pte_unmap_unlock(mapped_pte, ptl);
2357 	return err;
2358 }
2359 
remap_pmd_range(struct mm_struct * mm,pud_t * pud,unsigned long addr,unsigned long end,unsigned long pfn,pgprot_t prot)2360 static inline int remap_pmd_range(struct mm_struct *mm, pud_t *pud,
2361 			unsigned long addr, unsigned long end,
2362 			unsigned long pfn, pgprot_t prot)
2363 {
2364 	pmd_t *pmd;
2365 	unsigned long next;
2366 	int err;
2367 
2368 	pfn -= addr >> PAGE_SHIFT;
2369 	pmd = pmd_alloc(mm, pud, addr);
2370 	if (!pmd)
2371 		return -ENOMEM;
2372 	VM_BUG_ON(pmd_trans_huge(*pmd));
2373 	do {
2374 		next = pmd_addr_end(addr, end);
2375 		err = remap_pte_range(mm, pmd, addr, next,
2376 				pfn + (addr >> PAGE_SHIFT), prot);
2377 		if (err)
2378 			return err;
2379 	} while (pmd++, addr = next, addr != end);
2380 	return 0;
2381 }
2382 
remap_pud_range(struct mm_struct * mm,p4d_t * p4d,unsigned long addr,unsigned long end,unsigned long pfn,pgprot_t prot)2383 static inline int remap_pud_range(struct mm_struct *mm, p4d_t *p4d,
2384 			unsigned long addr, unsigned long end,
2385 			unsigned long pfn, pgprot_t prot)
2386 {
2387 	pud_t *pud;
2388 	unsigned long next;
2389 	int err;
2390 
2391 	pfn -= addr >> PAGE_SHIFT;
2392 	pud = pud_alloc(mm, p4d, addr);
2393 	if (!pud)
2394 		return -ENOMEM;
2395 	do {
2396 		next = pud_addr_end(addr, end);
2397 		err = remap_pmd_range(mm, pud, addr, next,
2398 				pfn + (addr >> PAGE_SHIFT), prot);
2399 		if (err)
2400 			return err;
2401 	} while (pud++, addr = next, addr != end);
2402 	return 0;
2403 }
2404 
remap_p4d_range(struct mm_struct * mm,pgd_t * pgd,unsigned long addr,unsigned long end,unsigned long pfn,pgprot_t prot)2405 static inline int remap_p4d_range(struct mm_struct *mm, pgd_t *pgd,
2406 			unsigned long addr, unsigned long end,
2407 			unsigned long pfn, pgprot_t prot)
2408 {
2409 	p4d_t *p4d;
2410 	unsigned long next;
2411 	int err;
2412 
2413 	pfn -= addr >> PAGE_SHIFT;
2414 	p4d = p4d_alloc(mm, pgd, addr);
2415 	if (!p4d)
2416 		return -ENOMEM;
2417 	do {
2418 		next = p4d_addr_end(addr, end);
2419 		err = remap_pud_range(mm, p4d, addr, next,
2420 				pfn + (addr >> PAGE_SHIFT), prot);
2421 		if (err)
2422 			return err;
2423 	} while (p4d++, addr = next, addr != end);
2424 	return 0;
2425 }
2426 
2427 /*
2428  * Variant of remap_pfn_range that does not call track_pfn_remap.  The caller
2429  * must have pre-validated the caching bits of the pgprot_t.
2430  */
remap_pfn_range_notrack(struct vm_area_struct * vma,unsigned long addr,unsigned long pfn,unsigned long size,pgprot_t prot)2431 int remap_pfn_range_notrack(struct vm_area_struct *vma, unsigned long addr,
2432 		unsigned long pfn, unsigned long size, pgprot_t prot)
2433 {
2434 	pgd_t *pgd;
2435 	unsigned long next;
2436 	unsigned long end = addr + PAGE_ALIGN(size);
2437 	struct mm_struct *mm = vma->vm_mm;
2438 	int err;
2439 
2440 	if (WARN_ON_ONCE(!PAGE_ALIGNED(addr)))
2441 		return -EINVAL;
2442 
2443 	/*
2444 	 * Physically remapped pages are special. Tell the
2445 	 * rest of the world about it:
2446 	 *   VM_IO tells people not to look at these pages
2447 	 *	(accesses can have side effects).
2448 	 *   VM_PFNMAP tells the core MM that the base pages are just
2449 	 *	raw PFN mappings, and do not have a "struct page" associated
2450 	 *	with them.
2451 	 *   VM_DONTEXPAND
2452 	 *      Disable vma merging and expanding with mremap().
2453 	 *   VM_DONTDUMP
2454 	 *      Omit vma from core dump, even when VM_IO turned off.
2455 	 *
2456 	 * There's a horrible special case to handle copy-on-write
2457 	 * behaviour that some programs depend on. We mark the "original"
2458 	 * un-COW'ed pages by matching them up with "vma->vm_pgoff".
2459 	 * See vm_normal_page() for details.
2460 	 */
2461 	if (is_cow_mapping(vma->vm_flags)) {
2462 		if (addr != vma->vm_start || end != vma->vm_end)
2463 			return -EINVAL;
2464 		vma->vm_pgoff = pfn;
2465 	}
2466 
2467 	vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP);
2468 
2469 	BUG_ON(addr >= end);
2470 	pfn -= addr >> PAGE_SHIFT;
2471 	pgd = pgd_offset(mm, addr);
2472 	flush_cache_range(vma, addr, end);
2473 	do {
2474 		next = pgd_addr_end(addr, end);
2475 		err = remap_p4d_range(mm, pgd, addr, next,
2476 				pfn + (addr >> PAGE_SHIFT), prot);
2477 		if (err)
2478 			return err;
2479 	} while (pgd++, addr = next, addr != end);
2480 
2481 	return 0;
2482 }
2483 
2484 /**
2485  * remap_pfn_range - remap kernel memory to userspace
2486  * @vma: user vma to map to
2487  * @addr: target page aligned user address to start at
2488  * @pfn: page frame number of kernel physical memory address
2489  * @size: size of mapping area
2490  * @prot: page protection flags for this mapping
2491  *
2492  * Note: this is only safe if the mm semaphore is held when called.
2493  *
2494  * Return: %0 on success, negative error code otherwise.
2495  */
remap_pfn_range(struct vm_area_struct * vma,unsigned long addr,unsigned long pfn,unsigned long size,pgprot_t prot)2496 int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
2497 		    unsigned long pfn, unsigned long size, pgprot_t prot)
2498 {
2499 	int err;
2500 
2501 	err = track_pfn_remap(vma, &prot, pfn, addr, PAGE_ALIGN(size));
2502 	if (err)
2503 		return -EINVAL;
2504 
2505 	err = remap_pfn_range_notrack(vma, addr, pfn, size, prot);
2506 	if (err)
2507 		untrack_pfn(vma, pfn, PAGE_ALIGN(size), true);
2508 	return err;
2509 }
2510 EXPORT_SYMBOL(remap_pfn_range);
2511 
2512 /**
2513  * vm_iomap_memory - remap memory to userspace
2514  * @vma: user vma to map to
2515  * @start: start of the physical memory to be mapped
2516  * @len: size of area
2517  *
2518  * This is a simplified io_remap_pfn_range() for common driver use. The
2519  * driver just needs to give us the physical memory range to be mapped,
2520  * we'll figure out the rest from the vma information.
2521  *
2522  * NOTE! Some drivers might want to tweak vma->vm_page_prot first to get
2523  * whatever write-combining details or similar.
2524  *
2525  * Return: %0 on success, negative error code otherwise.
2526  */
vm_iomap_memory(struct vm_area_struct * vma,phys_addr_t start,unsigned long len)2527 int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len)
2528 {
2529 	unsigned long vm_len, pfn, pages;
2530 
2531 	/* Check that the physical memory area passed in looks valid */
2532 	if (start + len < start)
2533 		return -EINVAL;
2534 	/*
2535 	 * You *really* shouldn't map things that aren't page-aligned,
2536 	 * but we've historically allowed it because IO memory might
2537 	 * just have smaller alignment.
2538 	 */
2539 	len += start & ~PAGE_MASK;
2540 	pfn = start >> PAGE_SHIFT;
2541 	pages = (len + ~PAGE_MASK) >> PAGE_SHIFT;
2542 	if (pfn + pages < pfn)
2543 		return -EINVAL;
2544 
2545 	/* We start the mapping 'vm_pgoff' pages into the area */
2546 	if (vma->vm_pgoff > pages)
2547 		return -EINVAL;
2548 	pfn += vma->vm_pgoff;
2549 	pages -= vma->vm_pgoff;
2550 
2551 	/* Can we fit all of the mapping? */
2552 	vm_len = vma->vm_end - vma->vm_start;
2553 	if (vm_len >> PAGE_SHIFT > pages)
2554 		return -EINVAL;
2555 
2556 	/* Ok, let it rip */
2557 	return io_remap_pfn_range(vma, vma->vm_start, pfn, vm_len, vma->vm_page_prot);
2558 }
2559 EXPORT_SYMBOL(vm_iomap_memory);
2560 
apply_to_pte_range(struct mm_struct * mm,pmd_t * pmd,unsigned long addr,unsigned long end,pte_fn_t fn,void * data,bool create,pgtbl_mod_mask * mask)2561 static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd,
2562 				     unsigned long addr, unsigned long end,
2563 				     pte_fn_t fn, void *data, bool create,
2564 				     pgtbl_mod_mask *mask)
2565 {
2566 	pte_t *pte, *mapped_pte;
2567 	int err = 0;
2568 	spinlock_t *ptl;
2569 
2570 	if (create) {
2571 		mapped_pte = pte = (mm == &init_mm) ?
2572 			pte_alloc_kernel_track(pmd, addr, mask) :
2573 			pte_alloc_map_lock(mm, pmd, addr, &ptl);
2574 		if (!pte)
2575 			return -ENOMEM;
2576 	} else {
2577 		mapped_pte = pte = (mm == &init_mm) ?
2578 			pte_offset_kernel(pmd, addr) :
2579 			pte_offset_map_lock(mm, pmd, addr, &ptl);
2580 		if (!pte)
2581 			return -EINVAL;
2582 	}
2583 
2584 	arch_enter_lazy_mmu_mode();
2585 
2586 	if (fn) {
2587 		do {
2588 			if (create || !pte_none(ptep_get(pte))) {
2589 				err = fn(pte++, addr, data);
2590 				if (err)
2591 					break;
2592 			}
2593 		} while (addr += PAGE_SIZE, addr != end);
2594 	}
2595 	*mask |= PGTBL_PTE_MODIFIED;
2596 
2597 	arch_leave_lazy_mmu_mode();
2598 
2599 	if (mm != &init_mm)
2600 		pte_unmap_unlock(mapped_pte, ptl);
2601 	return err;
2602 }
2603 
apply_to_pmd_range(struct mm_struct * mm,pud_t * pud,unsigned long addr,unsigned long end,pte_fn_t fn,void * data,bool create,pgtbl_mod_mask * mask)2604 static int apply_to_pmd_range(struct mm_struct *mm, pud_t *pud,
2605 				     unsigned long addr, unsigned long end,
2606 				     pte_fn_t fn, void *data, bool create,
2607 				     pgtbl_mod_mask *mask)
2608 {
2609 	pmd_t *pmd;
2610 	unsigned long next;
2611 	int err = 0;
2612 
2613 	BUG_ON(pud_huge(*pud));
2614 
2615 	if (create) {
2616 		pmd = pmd_alloc_track(mm, pud, addr, mask);
2617 		if (!pmd)
2618 			return -ENOMEM;
2619 	} else {
2620 		pmd = pmd_offset(pud, addr);
2621 	}
2622 	do {
2623 		next = pmd_addr_end(addr, end);
2624 		if (pmd_none(*pmd) && !create)
2625 			continue;
2626 		if (WARN_ON_ONCE(pmd_leaf(*pmd)))
2627 			return -EINVAL;
2628 		if (!pmd_none(*pmd) && WARN_ON_ONCE(pmd_bad(*pmd))) {
2629 			if (!create)
2630 				continue;
2631 			pmd_clear_bad(pmd);
2632 		}
2633 		err = apply_to_pte_range(mm, pmd, addr, next,
2634 					 fn, data, create, mask);
2635 		if (err)
2636 			break;
2637 	} while (pmd++, addr = next, addr != end);
2638 
2639 	return err;
2640 }
2641 
apply_to_pud_range(struct mm_struct * mm,p4d_t * p4d,unsigned long addr,unsigned long end,pte_fn_t fn,void * data,bool create,pgtbl_mod_mask * mask)2642 static int apply_to_pud_range(struct mm_struct *mm, p4d_t *p4d,
2643 				     unsigned long addr, unsigned long end,
2644 				     pte_fn_t fn, void *data, bool create,
2645 				     pgtbl_mod_mask *mask)
2646 {
2647 	pud_t *pud;
2648 	unsigned long next;
2649 	int err = 0;
2650 
2651 	if (create) {
2652 		pud = pud_alloc_track(mm, p4d, addr, mask);
2653 		if (!pud)
2654 			return -ENOMEM;
2655 	} else {
2656 		pud = pud_offset(p4d, addr);
2657 	}
2658 	do {
2659 		next = pud_addr_end(addr, end);
2660 		if (pud_none(*pud) && !create)
2661 			continue;
2662 		if (WARN_ON_ONCE(pud_leaf(*pud)))
2663 			return -EINVAL;
2664 		if (!pud_none(*pud) && WARN_ON_ONCE(pud_bad(*pud))) {
2665 			if (!create)
2666 				continue;
2667 			pud_clear_bad(pud);
2668 		}
2669 		err = apply_to_pmd_range(mm, pud, addr, next,
2670 					 fn, data, create, mask);
2671 		if (err)
2672 			break;
2673 	} while (pud++, addr = next, addr != end);
2674 
2675 	return err;
2676 }
2677 
apply_to_p4d_range(struct mm_struct * mm,pgd_t * pgd,unsigned long addr,unsigned long end,pte_fn_t fn,void * data,bool create,pgtbl_mod_mask * mask)2678 static int apply_to_p4d_range(struct mm_struct *mm, pgd_t *pgd,
2679 				     unsigned long addr, unsigned long end,
2680 				     pte_fn_t fn, void *data, bool create,
2681 				     pgtbl_mod_mask *mask)
2682 {
2683 	p4d_t *p4d;
2684 	unsigned long next;
2685 	int err = 0;
2686 
2687 	if (create) {
2688 		p4d = p4d_alloc_track(mm, pgd, addr, mask);
2689 		if (!p4d)
2690 			return -ENOMEM;
2691 	} else {
2692 		p4d = p4d_offset(pgd, addr);
2693 	}
2694 	do {
2695 		next = p4d_addr_end(addr, end);
2696 		if (p4d_none(*p4d) && !create)
2697 			continue;
2698 		if (WARN_ON_ONCE(p4d_leaf(*p4d)))
2699 			return -EINVAL;
2700 		if (!p4d_none(*p4d) && WARN_ON_ONCE(p4d_bad(*p4d))) {
2701 			if (!create)
2702 				continue;
2703 			p4d_clear_bad(p4d);
2704 		}
2705 		err = apply_to_pud_range(mm, p4d, addr, next,
2706 					 fn, data, create, mask);
2707 		if (err)
2708 			break;
2709 	} while (p4d++, addr = next, addr != end);
2710 
2711 	return err;
2712 }
2713 
__apply_to_page_range(struct mm_struct * mm,unsigned long addr,unsigned long size,pte_fn_t fn,void * data,bool create)2714 static int __apply_to_page_range(struct mm_struct *mm, unsigned long addr,
2715 				 unsigned long size, pte_fn_t fn,
2716 				 void *data, bool create)
2717 {
2718 	pgd_t *pgd;
2719 	unsigned long start = addr, next;
2720 	unsigned long end = addr + size;
2721 	pgtbl_mod_mask mask = 0;
2722 	int err = 0;
2723 
2724 	if (WARN_ON(addr >= end))
2725 		return -EINVAL;
2726 
2727 	pgd = pgd_offset(mm, addr);
2728 	do {
2729 		next = pgd_addr_end(addr, end);
2730 		if (pgd_none(*pgd) && !create)
2731 			continue;
2732 		if (WARN_ON_ONCE(pgd_leaf(*pgd)))
2733 			return -EINVAL;
2734 		if (!pgd_none(*pgd) && WARN_ON_ONCE(pgd_bad(*pgd))) {
2735 			if (!create)
2736 				continue;
2737 			pgd_clear_bad(pgd);
2738 		}
2739 		err = apply_to_p4d_range(mm, pgd, addr, next,
2740 					 fn, data, create, &mask);
2741 		if (err)
2742 			break;
2743 	} while (pgd++, addr = next, addr != end);
2744 
2745 	if (mask & ARCH_PAGE_TABLE_SYNC_MASK)
2746 		arch_sync_kernel_mappings(start, start + size);
2747 
2748 	return err;
2749 }
2750 
2751 /*
2752  * Scan a region of virtual memory, filling in page tables as necessary
2753  * and calling a provided function on each leaf page table.
2754  */
apply_to_page_range(struct mm_struct * mm,unsigned long addr,unsigned long size,pte_fn_t fn,void * data)2755 int apply_to_page_range(struct mm_struct *mm, unsigned long addr,
2756 			unsigned long size, pte_fn_t fn, void *data)
2757 {
2758 	return __apply_to_page_range(mm, addr, size, fn, data, true);
2759 }
2760 EXPORT_SYMBOL_GPL(apply_to_page_range);
2761 
2762 /*
2763  * Scan a region of virtual memory, calling a provided function on
2764  * each leaf page table where it exists.
2765  *
2766  * Unlike apply_to_page_range, this does _not_ fill in page tables
2767  * where they are absent.
2768  */
apply_to_existing_page_range(struct mm_struct * mm,unsigned long addr,unsigned long size,pte_fn_t fn,void * data)2769 int apply_to_existing_page_range(struct mm_struct *mm, unsigned long addr,
2770 				 unsigned long size, pte_fn_t fn, void *data)
2771 {
2772 	return __apply_to_page_range(mm, addr, size, fn, data, false);
2773 }
2774 EXPORT_SYMBOL_GPL(apply_to_existing_page_range);
2775 
2776 /*
2777  * handle_pte_fault chooses page fault handler according to an entry which was
2778  * read non-atomically.  Before making any commitment, on those architectures
2779  * or configurations (e.g. i386 with PAE) which might give a mix of unmatched
2780  * parts, do_swap_page must check under lock before unmapping the pte and
2781  * proceeding (but do_wp_page is only called after already making such a check;
2782  * and do_anonymous_page can safely check later on).
2783  */
pte_unmap_same(struct vm_fault * vmf)2784 static inline int pte_unmap_same(struct vm_fault *vmf)
2785 {
2786 	int same = 1;
2787 #if defined(CONFIG_SMP) || defined(CONFIG_PREEMPTION)
2788 	if (sizeof(pte_t) > sizeof(unsigned long)) {
2789 		spin_lock(vmf->ptl);
2790 		same = pte_same(ptep_get(vmf->pte), vmf->orig_pte);
2791 		spin_unlock(vmf->ptl);
2792 	}
2793 #endif
2794 	pte_unmap(vmf->pte);
2795 	vmf->pte = NULL;
2796 	return same;
2797 }
2798 
2799 /*
2800  * Return:
2801  *	0:		copied succeeded
2802  *	-EHWPOISON:	copy failed due to hwpoison in source page
2803  *	-EAGAIN:	copied failed (some other reason)
2804  */
__wp_page_copy_user(struct page * dst,struct page * src,struct vm_fault * vmf)2805 static inline int __wp_page_copy_user(struct page *dst, struct page *src,
2806 				      struct vm_fault *vmf)
2807 {
2808 	int ret;
2809 	void *kaddr;
2810 	void __user *uaddr;
2811 	struct vm_area_struct *vma = vmf->vma;
2812 	struct mm_struct *mm = vma->vm_mm;
2813 	unsigned long addr = vmf->address;
2814 
2815 	if (likely(src)) {
2816 		if (copy_mc_user_highpage(dst, src, addr, vma)) {
2817 			memory_failure_queue(page_to_pfn(src), 0);
2818 			return -EHWPOISON;
2819 		}
2820 		return 0;
2821 	}
2822 
2823 	/*
2824 	 * If the source page was a PFN mapping, we don't have
2825 	 * a "struct page" for it. We do a best-effort copy by
2826 	 * just copying from the original user address. If that
2827 	 * fails, we just zero-fill it. Live with it.
2828 	 */
2829 	kaddr = kmap_atomic(dst);
2830 	uaddr = (void __user *)(addr & PAGE_MASK);
2831 
2832 	/*
2833 	 * On architectures with software "accessed" bits, we would
2834 	 * take a double page fault, so mark it accessed here.
2835 	 */
2836 	vmf->pte = NULL;
2837 	if (!arch_has_hw_pte_young() && !pte_young(vmf->orig_pte)) {
2838 		pte_t entry;
2839 
2840 		vmf->pte = pte_offset_map_lock(mm, vmf->pmd, addr, &vmf->ptl);
2841 		if (unlikely(!vmf->pte || !pte_same(ptep_get(vmf->pte), vmf->orig_pte))) {
2842 			/*
2843 			 * Other thread has already handled the fault
2844 			 * and update local tlb only
2845 			 */
2846 			if (vmf->pte)
2847 				update_mmu_tlb(vma, addr, vmf->pte);
2848 			ret = -EAGAIN;
2849 			goto pte_unlock;
2850 		}
2851 
2852 		entry = pte_mkyoung(vmf->orig_pte);
2853 		if (ptep_set_access_flags(vma, addr, vmf->pte, entry, 0))
2854 			update_mmu_cache_range(vmf, vma, addr, vmf->pte, 1);
2855 	}
2856 
2857 	/*
2858 	 * This really shouldn't fail, because the page is there
2859 	 * in the page tables. But it might just be unreadable,
2860 	 * in which case we just give up and fill the result with
2861 	 * zeroes.
2862 	 */
2863 	if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE)) {
2864 		if (vmf->pte)
2865 			goto warn;
2866 
2867 		/* Re-validate under PTL if the page is still mapped */
2868 		vmf->pte = pte_offset_map_lock(mm, vmf->pmd, addr, &vmf->ptl);
2869 		if (unlikely(!vmf->pte || !pte_same(ptep_get(vmf->pte), vmf->orig_pte))) {
2870 			/* The PTE changed under us, update local tlb */
2871 			if (vmf->pte)
2872 				update_mmu_tlb(vma, addr, vmf->pte);
2873 			ret = -EAGAIN;
2874 			goto pte_unlock;
2875 		}
2876 
2877 		/*
2878 		 * The same page can be mapped back since last copy attempt.
2879 		 * Try to copy again under PTL.
2880 		 */
2881 		if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE)) {
2882 			/*
2883 			 * Give a warn in case there can be some obscure
2884 			 * use-case
2885 			 */
2886 warn:
2887 			WARN_ON_ONCE(1);
2888 			clear_page(kaddr);
2889 		}
2890 	}
2891 
2892 	ret = 0;
2893 
2894 pte_unlock:
2895 	if (vmf->pte)
2896 		pte_unmap_unlock(vmf->pte, vmf->ptl);
2897 	kunmap_atomic(kaddr);
2898 	flush_dcache_page(dst);
2899 
2900 	return ret;
2901 }
2902 
__get_fault_gfp_mask(struct vm_area_struct * vma)2903 static gfp_t __get_fault_gfp_mask(struct vm_area_struct *vma)
2904 {
2905 	struct file *vm_file = vma->vm_file;
2906 
2907 	if (vm_file)
2908 		return mapping_gfp_mask(vm_file->f_mapping) | __GFP_FS | __GFP_IO;
2909 
2910 	/*
2911 	 * Special mappings (e.g. VDSO) do not have any file so fake
2912 	 * a default GFP_KERNEL for them.
2913 	 */
2914 	return GFP_KERNEL;
2915 }
2916 
2917 /*
2918  * Notify the address space that the page is about to become writable so that
2919  * it can prohibit this or wait for the page to get into an appropriate state.
2920  *
2921  * We do this without the lock held, so that it can sleep if it needs to.
2922  */
do_page_mkwrite(struct vm_fault * vmf,struct folio * folio)2923 static vm_fault_t do_page_mkwrite(struct vm_fault *vmf, struct folio *folio)
2924 {
2925 	vm_fault_t ret;
2926 	unsigned int old_flags = vmf->flags;
2927 
2928 	vmf->flags = FAULT_FLAG_WRITE|FAULT_FLAG_MKWRITE;
2929 
2930 	if (vmf->vma->vm_file &&
2931 	    IS_SWAPFILE(vmf->vma->vm_file->f_mapping->host))
2932 		return VM_FAULT_SIGBUS;
2933 
2934 	ret = vmf->vma->vm_ops->page_mkwrite(vmf);
2935 	/* Restore original flags so that caller is not surprised */
2936 	vmf->flags = old_flags;
2937 	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))
2938 		return ret;
2939 	if (unlikely(!(ret & VM_FAULT_LOCKED))) {
2940 		folio_lock(folio);
2941 		if (!folio->mapping) {
2942 			folio_unlock(folio);
2943 			return 0; /* retry */
2944 		}
2945 		ret |= VM_FAULT_LOCKED;
2946 	} else
2947 		VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
2948 	return ret;
2949 }
2950 
2951 /*
2952  * Handle dirtying of a page in shared file mapping on a write fault.
2953  *
2954  * The function expects the page to be locked and unlocks it.
2955  */
fault_dirty_shared_page(struct vm_fault * vmf)2956 static vm_fault_t fault_dirty_shared_page(struct vm_fault *vmf)
2957 {
2958 	struct vm_area_struct *vma = vmf->vma;
2959 	struct address_space *mapping;
2960 	struct folio *folio = page_folio(vmf->page);
2961 	bool dirtied;
2962 	bool page_mkwrite = vma->vm_ops && vma->vm_ops->page_mkwrite;
2963 
2964 	dirtied = folio_mark_dirty(folio);
2965 	VM_BUG_ON_FOLIO(folio_test_anon(folio), folio);
2966 	/*
2967 	 * Take a local copy of the address_space - folio.mapping may be zeroed
2968 	 * by truncate after folio_unlock().   The address_space itself remains
2969 	 * pinned by vma->vm_file's reference.  We rely on folio_unlock()'s
2970 	 * release semantics to prevent the compiler from undoing this copying.
2971 	 */
2972 	mapping = folio_raw_mapping(folio);
2973 	folio_unlock(folio);
2974 
2975 	if (!page_mkwrite)
2976 		file_update_time(vma->vm_file);
2977 
2978 	/*
2979 	 * Throttle page dirtying rate down to writeback speed.
2980 	 *
2981 	 * mapping may be NULL here because some device drivers do not
2982 	 * set page.mapping but still dirty their pages
2983 	 *
2984 	 * Drop the mmap_lock before waiting on IO, if we can. The file
2985 	 * is pinning the mapping, as per above.
2986 	 */
2987 	if ((dirtied || page_mkwrite) && mapping) {
2988 		struct file *fpin;
2989 
2990 		fpin = maybe_unlock_mmap_for_io(vmf, NULL);
2991 		balance_dirty_pages_ratelimited(mapping);
2992 		if (fpin) {
2993 			fput(fpin);
2994 			return VM_FAULT_COMPLETED;
2995 		}
2996 	}
2997 
2998 	return 0;
2999 }
3000 
3001 /*
3002  * Handle write page faults for pages that can be reused in the current vma
3003  *
3004  * This can happen either due to the mapping being with the VM_SHARED flag,
3005  * or due to us being the last reference standing to the page. In either
3006  * case, all we need to do here is to mark the page as writable and update
3007  * any related book-keeping.
3008  */
wp_page_reuse(struct vm_fault * vmf)3009 static inline void wp_page_reuse(struct vm_fault *vmf)
3010 	__releases(vmf->ptl)
3011 {
3012 	struct vm_area_struct *vma = vmf->vma;
3013 	struct page *page = vmf->page;
3014 	pte_t entry;
3015 
3016 	VM_BUG_ON(!(vmf->flags & FAULT_FLAG_WRITE));
3017 	VM_BUG_ON(page && PageAnon(page) && !PageAnonExclusive(page));
3018 
3019 	/*
3020 	 * Clear the pages cpupid information as the existing
3021 	 * information potentially belongs to a now completely
3022 	 * unrelated process.
3023 	 */
3024 	if (page)
3025 		page_cpupid_xchg_last(page, (1 << LAST_CPUPID_SHIFT) - 1);
3026 
3027 	flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte));
3028 	entry = pte_mkyoung(vmf->orig_pte);
3029 	entry = maybe_mkwrite(pte_mkdirty(entry), vma);
3030 	if (ptep_set_access_flags(vma, vmf->address, vmf->pte, entry, 1))
3031 		update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1);
3032 	pte_unmap_unlock(vmf->pte, vmf->ptl);
3033 	count_vm_event(PGREUSE);
3034 }
3035 
3036 /*
3037  * Handle the case of a page which we actually need to copy to a new page,
3038  * either due to COW or unsharing.
3039  *
3040  * Called with mmap_lock locked and the old page referenced, but
3041  * without the ptl held.
3042  *
3043  * High level logic flow:
3044  *
3045  * - Allocate a page, copy the content of the old page to the new one.
3046  * - Handle book keeping and accounting - cgroups, mmu-notifiers, etc.
3047  * - Take the PTL. If the pte changed, bail out and release the allocated page
3048  * - If the pte is still the way we remember it, update the page table and all
3049  *   relevant references. This includes dropping the reference the page-table
3050  *   held to the old page, as well as updating the rmap.
3051  * - In any case, unlock the PTL and drop the reference we took to the old page.
3052  */
wp_page_copy(struct vm_fault * vmf)3053 static vm_fault_t wp_page_copy(struct vm_fault *vmf)
3054 {
3055 	const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
3056 	struct vm_area_struct *vma = vmf->vma;
3057 	struct mm_struct *mm = vma->vm_mm;
3058 	struct folio *old_folio = NULL;
3059 	struct folio *new_folio = NULL;
3060 	pte_t entry;
3061 	int page_copied = 0;
3062 	struct mmu_notifier_range range;
3063 	int ret;
3064 
3065 	delayacct_wpcopy_start();
3066 
3067 	if (vmf->page)
3068 		old_folio = page_folio(vmf->page);
3069 	if (unlikely(anon_vma_prepare(vma)))
3070 		goto oom;
3071 
3072 	if (is_zero_pfn(pte_pfn(vmf->orig_pte))) {
3073 		new_folio = vma_alloc_zeroed_movable_folio(vma, vmf->address);
3074 		if (!new_folio)
3075 			goto oom;
3076 	} else {
3077 		new_folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 0, vma,
3078 				vmf->address, false);
3079 		if (!new_folio)
3080 			goto oom;
3081 
3082 		ret = __wp_page_copy_user(&new_folio->page, vmf->page, vmf);
3083 		if (ret) {
3084 			/*
3085 			 * COW failed, if the fault was solved by other,
3086 			 * it's fine. If not, userspace would re-fault on
3087 			 * the same address and we will handle the fault
3088 			 * from the second attempt.
3089 			 * The -EHWPOISON case will not be retried.
3090 			 */
3091 			folio_put(new_folio);
3092 			if (old_folio)
3093 				folio_put(old_folio);
3094 
3095 			delayacct_wpcopy_end();
3096 			return ret == -EHWPOISON ? VM_FAULT_HWPOISON : 0;
3097 		}
3098 		kmsan_copy_page_meta(&new_folio->page, vmf->page);
3099 	}
3100 
3101 	if (mem_cgroup_charge(new_folio, mm, GFP_KERNEL))
3102 		goto oom_free_new;
3103 	folio_throttle_swaprate(new_folio, GFP_KERNEL);
3104 
3105 	__folio_mark_uptodate(new_folio);
3106 
3107 	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm,
3108 				vmf->address & PAGE_MASK,
3109 				(vmf->address & PAGE_MASK) + PAGE_SIZE);
3110 	mmu_notifier_invalidate_range_start(&range);
3111 
3112 	/*
3113 	 * Re-check the pte - we dropped the lock
3114 	 */
3115 	vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl);
3116 	if (likely(vmf->pte && pte_same(ptep_get(vmf->pte), vmf->orig_pte))) {
3117 		if (old_folio) {
3118 			if (!folio_test_anon(old_folio)) {
3119 				dec_mm_counter(mm, mm_counter_file(&old_folio->page));
3120 				inc_mm_counter(mm, MM_ANONPAGES);
3121 			}
3122 		} else {
3123 			ksm_might_unmap_zero_page(mm, vmf->orig_pte);
3124 			inc_mm_counter(mm, MM_ANONPAGES);
3125 		}
3126 		flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte));
3127 		entry = mk_pte(&new_folio->page, vma->vm_page_prot);
3128 		entry = pte_sw_mkyoung(entry);
3129 		if (unlikely(unshare)) {
3130 			if (pte_soft_dirty(vmf->orig_pte))
3131 				entry = pte_mksoft_dirty(entry);
3132 			if (pte_uffd_wp(vmf->orig_pte))
3133 				entry = pte_mkuffd_wp(entry);
3134 		} else {
3135 			entry = maybe_mkwrite(pte_mkdirty(entry), vma);
3136 		}
3137 
3138 		/*
3139 		 * Clear the pte entry and flush it first, before updating the
3140 		 * pte with the new entry, to keep TLBs on different CPUs in
3141 		 * sync. This code used to set the new PTE then flush TLBs, but
3142 		 * that left a window where the new PTE could be loaded into
3143 		 * some TLBs while the old PTE remains in others.
3144 		 */
3145 		ptep_clear_flush(vma, vmf->address, vmf->pte);
3146 		folio_add_new_anon_rmap(new_folio, vma, vmf->address);
3147 		folio_add_lru_vma(new_folio, vma);
3148 		/*
3149 		 * We call the notify macro here because, when using secondary
3150 		 * mmu page tables (such as kvm shadow page tables), we want the
3151 		 * new page to be mapped directly into the secondary page table.
3152 		 */
3153 		BUG_ON(unshare && pte_write(entry));
3154 		set_pte_at_notify(mm, vmf->address, vmf->pte, entry);
3155 		update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1);
3156 		if (old_folio) {
3157 			/*
3158 			 * Only after switching the pte to the new page may
3159 			 * we remove the mapcount here. Otherwise another
3160 			 * process may come and find the rmap count decremented
3161 			 * before the pte is switched to the new page, and
3162 			 * "reuse" the old page writing into it while our pte
3163 			 * here still points into it and can be read by other
3164 			 * threads.
3165 			 *
3166 			 * The critical issue is to order this
3167 			 * page_remove_rmap with the ptp_clear_flush above.
3168 			 * Those stores are ordered by (if nothing else,)
3169 			 * the barrier present in the atomic_add_negative
3170 			 * in page_remove_rmap.
3171 			 *
3172 			 * Then the TLB flush in ptep_clear_flush ensures that
3173 			 * no process can access the old page before the
3174 			 * decremented mapcount is visible. And the old page
3175 			 * cannot be reused until after the decremented
3176 			 * mapcount is visible. So transitively, TLBs to
3177 			 * old page will be flushed before it can be reused.
3178 			 */
3179 			page_remove_rmap(vmf->page, vma, false);
3180 		}
3181 
3182 		/* Free the old page.. */
3183 		new_folio = old_folio;
3184 		page_copied = 1;
3185 		pte_unmap_unlock(vmf->pte, vmf->ptl);
3186 	} else if (vmf->pte) {
3187 		update_mmu_tlb(vma, vmf->address, vmf->pte);
3188 		pte_unmap_unlock(vmf->pte, vmf->ptl);
3189 	}
3190 
3191 	mmu_notifier_invalidate_range_end(&range);
3192 
3193 	if (new_folio)
3194 		folio_put(new_folio);
3195 	if (old_folio) {
3196 		if (page_copied)
3197 			free_swap_cache(&old_folio->page);
3198 		folio_put(old_folio);
3199 	}
3200 
3201 	delayacct_wpcopy_end();
3202 	return 0;
3203 oom_free_new:
3204 	folio_put(new_folio);
3205 oom:
3206 	if (old_folio)
3207 		folio_put(old_folio);
3208 
3209 	delayacct_wpcopy_end();
3210 	return VM_FAULT_OOM;
3211 }
3212 
3213 /**
3214  * finish_mkwrite_fault - finish page fault for a shared mapping, making PTE
3215  *			  writeable once the page is prepared
3216  *
3217  * @vmf: structure describing the fault
3218  *
3219  * This function handles all that is needed to finish a write page fault in a
3220  * shared mapping due to PTE being read-only once the mapped page is prepared.
3221  * It handles locking of PTE and modifying it.
3222  *
3223  * The function expects the page to be locked or other protection against
3224  * concurrent faults / writeback (such as DAX radix tree locks).
3225  *
3226  * Return: %0 on success, %VM_FAULT_NOPAGE when PTE got changed before
3227  * we acquired PTE lock.
3228  */
finish_mkwrite_fault(struct vm_fault * vmf)3229 vm_fault_t finish_mkwrite_fault(struct vm_fault *vmf)
3230 {
3231 	WARN_ON_ONCE(!(vmf->vma->vm_flags & VM_SHARED));
3232 	vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd, vmf->address,
3233 				       &vmf->ptl);
3234 	if (!vmf->pte)
3235 		return VM_FAULT_NOPAGE;
3236 	/*
3237 	 * We might have raced with another page fault while we released the
3238 	 * pte_offset_map_lock.
3239 	 */
3240 	if (!pte_same(ptep_get(vmf->pte), vmf->orig_pte)) {
3241 		update_mmu_tlb(vmf->vma, vmf->address, vmf->pte);
3242 		pte_unmap_unlock(vmf->pte, vmf->ptl);
3243 		return VM_FAULT_NOPAGE;
3244 	}
3245 	wp_page_reuse(vmf);
3246 	return 0;
3247 }
3248 
3249 /*
3250  * Handle write page faults for VM_MIXEDMAP or VM_PFNMAP for a VM_SHARED
3251  * mapping
3252  */
wp_pfn_shared(struct vm_fault * vmf)3253 static vm_fault_t wp_pfn_shared(struct vm_fault *vmf)
3254 {
3255 	struct vm_area_struct *vma = vmf->vma;
3256 
3257 	if (vma->vm_ops && vma->vm_ops->pfn_mkwrite) {
3258 		vm_fault_t ret;
3259 
3260 		pte_unmap_unlock(vmf->pte, vmf->ptl);
3261 		if (vmf->flags & FAULT_FLAG_VMA_LOCK) {
3262 			vma_end_read(vmf->vma);
3263 			return VM_FAULT_RETRY;
3264 		}
3265 
3266 		vmf->flags |= FAULT_FLAG_MKWRITE;
3267 		ret = vma->vm_ops->pfn_mkwrite(vmf);
3268 		if (ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))
3269 			return ret;
3270 		return finish_mkwrite_fault(vmf);
3271 	}
3272 	wp_page_reuse(vmf);
3273 	return 0;
3274 }
3275 
wp_page_shared(struct vm_fault * vmf,struct folio * folio)3276 static vm_fault_t wp_page_shared(struct vm_fault *vmf, struct folio *folio)
3277 	__releases(vmf->ptl)
3278 {
3279 	struct vm_area_struct *vma = vmf->vma;
3280 	vm_fault_t ret = 0;
3281 
3282 	folio_get(folio);
3283 
3284 	if (vma->vm_ops && vma->vm_ops->page_mkwrite) {
3285 		vm_fault_t tmp;
3286 
3287 		pte_unmap_unlock(vmf->pte, vmf->ptl);
3288 		if (vmf->flags & FAULT_FLAG_VMA_LOCK) {
3289 			folio_put(folio);
3290 			vma_end_read(vmf->vma);
3291 			return VM_FAULT_RETRY;
3292 		}
3293 
3294 		tmp = do_page_mkwrite(vmf, folio);
3295 		if (unlikely(!tmp || (tmp &
3296 				      (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) {
3297 			folio_put(folio);
3298 			return tmp;
3299 		}
3300 		tmp = finish_mkwrite_fault(vmf);
3301 		if (unlikely(tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))) {
3302 			folio_unlock(folio);
3303 			folio_put(folio);
3304 			return tmp;
3305 		}
3306 	} else {
3307 		wp_page_reuse(vmf);
3308 		folio_lock(folio);
3309 	}
3310 	ret |= fault_dirty_shared_page(vmf);
3311 	folio_put(folio);
3312 
3313 	return ret;
3314 }
3315 
3316 /*
3317  * This routine handles present pages, when
3318  * * users try to write to a shared page (FAULT_FLAG_WRITE)
3319  * * GUP wants to take a R/O pin on a possibly shared anonymous page
3320  *   (FAULT_FLAG_UNSHARE)
3321  *
3322  * It is done by copying the page to a new address and decrementing the
3323  * shared-page counter for the old page.
3324  *
3325  * Note that this routine assumes that the protection checks have been
3326  * done by the caller (the low-level page fault routine in most cases).
3327  * Thus, with FAULT_FLAG_WRITE, we can safely just mark it writable once we've
3328  * done any necessary COW.
3329  *
3330  * In case of FAULT_FLAG_WRITE, we also mark the page dirty at this point even
3331  * though the page will change only once the write actually happens. This
3332  * avoids a few races, and potentially makes it more efficient.
3333  *
3334  * We enter with non-exclusive mmap_lock (to exclude vma changes,
3335  * but allow concurrent faults), with pte both mapped and locked.
3336  * We return with mmap_lock still held, but pte unmapped and unlocked.
3337  */
do_wp_page(struct vm_fault * vmf)3338 static vm_fault_t do_wp_page(struct vm_fault *vmf)
3339 	__releases(vmf->ptl)
3340 {
3341 	const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
3342 	struct vm_area_struct *vma = vmf->vma;
3343 	struct folio *folio = NULL;
3344 
3345 	if (likely(!unshare)) {
3346 		if (userfaultfd_pte_wp(vma, ptep_get(vmf->pte))) {
3347 			pte_unmap_unlock(vmf->pte, vmf->ptl);
3348 			return handle_userfault(vmf, VM_UFFD_WP);
3349 		}
3350 
3351 		/*
3352 		 * Userfaultfd write-protect can defer flushes. Ensure the TLB
3353 		 * is flushed in this case before copying.
3354 		 */
3355 		if (unlikely(userfaultfd_wp(vmf->vma) &&
3356 			     mm_tlb_flush_pending(vmf->vma->vm_mm)))
3357 			flush_tlb_page(vmf->vma, vmf->address);
3358 	}
3359 
3360 	vmf->page = vm_normal_page(vma, vmf->address, vmf->orig_pte);
3361 
3362 	if (vmf->page)
3363 		folio = page_folio(vmf->page);
3364 
3365 	/*
3366 	 * Shared mapping: we are guaranteed to have VM_WRITE and
3367 	 * FAULT_FLAG_WRITE set at this point.
3368 	 */
3369 	if (vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) {
3370 		/*
3371 		 * VM_MIXEDMAP !pfn_valid() case, or VM_SOFTDIRTY clear on a
3372 		 * VM_PFNMAP VMA.
3373 		 *
3374 		 * We should not cow pages in a shared writeable mapping.
3375 		 * Just mark the pages writable and/or call ops->pfn_mkwrite.
3376 		 */
3377 		if (!vmf->page)
3378 			return wp_pfn_shared(vmf);
3379 		return wp_page_shared(vmf, folio);
3380 	}
3381 
3382 	/*
3383 	 * Private mapping: create an exclusive anonymous page copy if reuse
3384 	 * is impossible. We might miss VM_WRITE for FOLL_FORCE handling.
3385 	 */
3386 	if (folio && folio_test_anon(folio)) {
3387 		/*
3388 		 * If the page is exclusive to this process we must reuse the
3389 		 * page without further checks.
3390 		 */
3391 		if (PageAnonExclusive(vmf->page))
3392 			goto reuse;
3393 
3394 		/*
3395 		 * We have to verify under folio lock: these early checks are
3396 		 * just an optimization to avoid locking the folio and freeing
3397 		 * the swapcache if there is little hope that we can reuse.
3398 		 *
3399 		 * KSM doesn't necessarily raise the folio refcount.
3400 		 */
3401 		if (folio_test_ksm(folio) || folio_ref_count(folio) > 3)
3402 			goto copy;
3403 		if (!folio_test_lru(folio))
3404 			/*
3405 			 * We cannot easily detect+handle references from
3406 			 * remote LRU caches or references to LRU folios.
3407 			 */
3408 			lru_add_drain();
3409 		if (folio_ref_count(folio) > 1 + folio_test_swapcache(folio))
3410 			goto copy;
3411 		if (!folio_trylock(folio))
3412 			goto copy;
3413 		if (folio_test_swapcache(folio))
3414 			folio_free_swap(folio);
3415 		if (folio_test_ksm(folio) || folio_ref_count(folio) != 1) {
3416 			folio_unlock(folio);
3417 			goto copy;
3418 		}
3419 		/*
3420 		 * Ok, we've got the only folio reference from our mapping
3421 		 * and the folio is locked, it's dark out, and we're wearing
3422 		 * sunglasses. Hit it.
3423 		 */
3424 		page_move_anon_rmap(vmf->page, vma);
3425 		folio_unlock(folio);
3426 reuse:
3427 		if (unlikely(unshare)) {
3428 			pte_unmap_unlock(vmf->pte, vmf->ptl);
3429 			return 0;
3430 		}
3431 		wp_page_reuse(vmf);
3432 		return 0;
3433 	}
3434 copy:
3435 	if ((vmf->flags & FAULT_FLAG_VMA_LOCK) && !vma->anon_vma) {
3436 		pte_unmap_unlock(vmf->pte, vmf->ptl);
3437 		vma_end_read(vmf->vma);
3438 		return VM_FAULT_RETRY;
3439 	}
3440 
3441 	/*
3442 	 * Ok, we need to copy. Oh, well..
3443 	 */
3444 	if (folio)
3445 		folio_get(folio);
3446 
3447 	pte_unmap_unlock(vmf->pte, vmf->ptl);
3448 #ifdef CONFIG_KSM
3449 	if (folio && folio_test_ksm(folio))
3450 		count_vm_event(COW_KSM);
3451 #endif
3452 	return wp_page_copy(vmf);
3453 }
3454 
unmap_mapping_range_vma(struct vm_area_struct * vma,unsigned long start_addr,unsigned long end_addr,struct zap_details * details)3455 static void unmap_mapping_range_vma(struct vm_area_struct *vma,
3456 		unsigned long start_addr, unsigned long end_addr,
3457 		struct zap_details *details)
3458 {
3459 	zap_page_range_single(vma, start_addr, end_addr - start_addr, details);
3460 }
3461 
unmap_mapping_range_tree(struct rb_root_cached * root,pgoff_t first_index,pgoff_t last_index,struct zap_details * details)3462 static inline void unmap_mapping_range_tree(struct rb_root_cached *root,
3463 					    pgoff_t first_index,
3464 					    pgoff_t last_index,
3465 					    struct zap_details *details)
3466 {
3467 	struct vm_area_struct *vma;
3468 	pgoff_t vba, vea, zba, zea;
3469 
3470 	vma_interval_tree_foreach(vma, root, first_index, last_index) {
3471 		vba = vma->vm_pgoff;
3472 		vea = vba + vma_pages(vma) - 1;
3473 		zba = max(first_index, vba);
3474 		zea = min(last_index, vea);
3475 
3476 		unmap_mapping_range_vma(vma,
3477 			((zba - vba) << PAGE_SHIFT) + vma->vm_start,
3478 			((zea - vba + 1) << PAGE_SHIFT) + vma->vm_start,
3479 				details);
3480 	}
3481 }
3482 
3483 /**
3484  * unmap_mapping_folio() - Unmap single folio from processes.
3485  * @folio: The locked folio to be unmapped.
3486  *
3487  * Unmap this folio from any userspace process which still has it mmaped.
3488  * Typically, for efficiency, the range of nearby pages has already been
3489  * unmapped by unmap_mapping_pages() or unmap_mapping_range().  But once
3490  * truncation or invalidation holds the lock on a folio, it may find that
3491  * the page has been remapped again: and then uses unmap_mapping_folio()
3492  * to unmap it finally.
3493  */
unmap_mapping_folio(struct folio * folio)3494 void unmap_mapping_folio(struct folio *folio)
3495 {
3496 	struct address_space *mapping = folio->mapping;
3497 	struct zap_details details = { };
3498 	pgoff_t	first_index;
3499 	pgoff_t	last_index;
3500 
3501 	VM_BUG_ON(!folio_test_locked(folio));
3502 
3503 	first_index = folio->index;
3504 	last_index = folio_next_index(folio) - 1;
3505 
3506 	details.even_cows = false;
3507 	details.single_folio = folio;
3508 	details.zap_flags = ZAP_FLAG_DROP_MARKER;
3509 
3510 	i_mmap_lock_read(mapping);
3511 	if (unlikely(!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root)))
3512 		unmap_mapping_range_tree(&mapping->i_mmap, first_index,
3513 					 last_index, &details);
3514 	i_mmap_unlock_read(mapping);
3515 }
3516 
3517 /**
3518  * unmap_mapping_pages() - Unmap pages from processes.
3519  * @mapping: The address space containing pages to be unmapped.
3520  * @start: Index of first page to be unmapped.
3521  * @nr: Number of pages to be unmapped.  0 to unmap to end of file.
3522  * @even_cows: Whether to unmap even private COWed pages.
3523  *
3524  * Unmap the pages in this address space from any userspace process which
3525  * has them mmaped.  Generally, you want to remove COWed pages as well when
3526  * a file is being truncated, but not when invalidating pages from the page
3527  * cache.
3528  */
unmap_mapping_pages(struct address_space * mapping,pgoff_t start,pgoff_t nr,bool even_cows)3529 void unmap_mapping_pages(struct address_space *mapping, pgoff_t start,
3530 		pgoff_t nr, bool even_cows)
3531 {
3532 	struct zap_details details = { };
3533 	pgoff_t	first_index = start;
3534 	pgoff_t	last_index = start + nr - 1;
3535 
3536 	details.even_cows = even_cows;
3537 	if (last_index < first_index)
3538 		last_index = ULONG_MAX;
3539 
3540 	i_mmap_lock_read(mapping);
3541 	if (unlikely(!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root)))
3542 		unmap_mapping_range_tree(&mapping->i_mmap, first_index,
3543 					 last_index, &details);
3544 	i_mmap_unlock_read(mapping);
3545 }
3546 EXPORT_SYMBOL_GPL(unmap_mapping_pages);
3547 
3548 /**
3549  * unmap_mapping_range - unmap the portion of all mmaps in the specified
3550  * address_space corresponding to the specified byte range in the underlying
3551  * file.
3552  *
3553  * @mapping: the address space containing mmaps to be unmapped.
3554  * @holebegin: byte in first page to unmap, relative to the start of
3555  * the underlying file.  This will be rounded down to a PAGE_SIZE
3556  * boundary.  Note that this is different from truncate_pagecache(), which
3557  * must keep the partial page.  In contrast, we must get rid of
3558  * partial pages.
3559  * @holelen: size of prospective hole in bytes.  This will be rounded
3560  * up to a PAGE_SIZE boundary.  A holelen of zero truncates to the
3561  * end of the file.
3562  * @even_cows: 1 when truncating a file, unmap even private COWed pages;
3563  * but 0 when invalidating pagecache, don't throw away private data.
3564  */
unmap_mapping_range(struct address_space * mapping,loff_t const holebegin,loff_t const holelen,int even_cows)3565 void unmap_mapping_range(struct address_space *mapping,
3566 		loff_t const holebegin, loff_t const holelen, int even_cows)
3567 {
3568 	pgoff_t hba = (pgoff_t)(holebegin) >> PAGE_SHIFT;
3569 	pgoff_t hlen = ((pgoff_t)(holelen) + PAGE_SIZE - 1) >> PAGE_SHIFT;
3570 
3571 	/* Check for overflow. */
3572 	if (sizeof(holelen) > sizeof(hlen)) {
3573 		long long holeend =
3574 			(holebegin + holelen + PAGE_SIZE - 1) >> PAGE_SHIFT;
3575 		if (holeend & ~(long long)ULONG_MAX)
3576 			hlen = ULONG_MAX - hba + 1;
3577 	}
3578 
3579 	unmap_mapping_pages(mapping, hba, hlen, even_cows);
3580 }
3581 EXPORT_SYMBOL(unmap_mapping_range);
3582 
3583 /*
3584  * Restore a potential device exclusive pte to a working pte entry
3585  */
remove_device_exclusive_entry(struct vm_fault * vmf)3586 static vm_fault_t remove_device_exclusive_entry(struct vm_fault *vmf)
3587 {
3588 	struct folio *folio = page_folio(vmf->page);
3589 	struct vm_area_struct *vma = vmf->vma;
3590 	struct mmu_notifier_range range;
3591 	vm_fault_t ret;
3592 
3593 	/*
3594 	 * We need a reference to lock the folio because we don't hold
3595 	 * the PTL so a racing thread can remove the device-exclusive
3596 	 * entry and unmap it. If the folio is free the entry must
3597 	 * have been removed already. If it happens to have already
3598 	 * been re-allocated after being freed all we do is lock and
3599 	 * unlock it.
3600 	 */
3601 	if (!folio_try_get(folio))
3602 		return 0;
3603 
3604 	ret = folio_lock_or_retry(folio, vmf);
3605 	if (ret) {
3606 		folio_put(folio);
3607 		return ret;
3608 	}
3609 	mmu_notifier_range_init_owner(&range, MMU_NOTIFY_EXCLUSIVE, 0,
3610 				vma->vm_mm, vmf->address & PAGE_MASK,
3611 				(vmf->address & PAGE_MASK) + PAGE_SIZE, NULL);
3612 	mmu_notifier_invalidate_range_start(&range);
3613 
3614 	vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
3615 				&vmf->ptl);
3616 	if (likely(vmf->pte && pte_same(ptep_get(vmf->pte), vmf->orig_pte)))
3617 		restore_exclusive_pte(vma, vmf->page, vmf->address, vmf->pte);
3618 
3619 	if (vmf->pte)
3620 		pte_unmap_unlock(vmf->pte, vmf->ptl);
3621 	folio_unlock(folio);
3622 	folio_put(folio);
3623 
3624 	mmu_notifier_invalidate_range_end(&range);
3625 	return 0;
3626 }
3627 
should_try_to_free_swap(struct folio * folio,struct vm_area_struct * vma,unsigned int fault_flags)3628 static inline bool should_try_to_free_swap(struct folio *folio,
3629 					   struct vm_area_struct *vma,
3630 					   unsigned int fault_flags)
3631 {
3632 	if (!folio_test_swapcache(folio))
3633 		return false;
3634 	if (mem_cgroup_swap_full(folio) || (vma->vm_flags & VM_LOCKED) ||
3635 	    folio_test_mlocked(folio))
3636 		return true;
3637 	/*
3638 	 * If we want to map a page that's in the swapcache writable, we
3639 	 * have to detect via the refcount if we're really the exclusive
3640 	 * user. Try freeing the swapcache to get rid of the swapcache
3641 	 * reference only in case it's likely that we'll be the exlusive user.
3642 	 */
3643 	return (fault_flags & FAULT_FLAG_WRITE) && !folio_test_ksm(folio) &&
3644 		folio_ref_count(folio) == 2;
3645 }
3646 
pte_marker_clear(struct vm_fault * vmf)3647 static vm_fault_t pte_marker_clear(struct vm_fault *vmf)
3648 {
3649 	vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd,
3650 				       vmf->address, &vmf->ptl);
3651 	if (!vmf->pte)
3652 		return 0;
3653 	/*
3654 	 * Be careful so that we will only recover a special uffd-wp pte into a
3655 	 * none pte.  Otherwise it means the pte could have changed, so retry.
3656 	 *
3657 	 * This should also cover the case where e.g. the pte changed
3658 	 * quickly from a PTE_MARKER_UFFD_WP into PTE_MARKER_POISONED.
3659 	 * So is_pte_marker() check is not enough to safely drop the pte.
3660 	 */
3661 	if (pte_same(vmf->orig_pte, ptep_get(vmf->pte)))
3662 		pte_clear(vmf->vma->vm_mm, vmf->address, vmf->pte);
3663 	pte_unmap_unlock(vmf->pte, vmf->ptl);
3664 	return 0;
3665 }
3666 
do_pte_missing(struct vm_fault * vmf)3667 static vm_fault_t do_pte_missing(struct vm_fault *vmf)
3668 {
3669 	if (vma_is_anonymous(vmf->vma))
3670 		return do_anonymous_page(vmf);
3671 	else
3672 		return do_fault(vmf);
3673 }
3674 
3675 /*
3676  * This is actually a page-missing access, but with uffd-wp special pte
3677  * installed.  It means this pte was wr-protected before being unmapped.
3678  */
pte_marker_handle_uffd_wp(struct vm_fault * vmf)3679 static vm_fault_t pte_marker_handle_uffd_wp(struct vm_fault *vmf)
3680 {
3681 	/*
3682 	 * Just in case there're leftover special ptes even after the region
3683 	 * got unregistered - we can simply clear them.
3684 	 */
3685 	if (unlikely(!userfaultfd_wp(vmf->vma)))
3686 		return pte_marker_clear(vmf);
3687 
3688 	return do_pte_missing(vmf);
3689 }
3690 
handle_pte_marker(struct vm_fault * vmf)3691 static vm_fault_t handle_pte_marker(struct vm_fault *vmf)
3692 {
3693 	swp_entry_t entry = pte_to_swp_entry(vmf->orig_pte);
3694 	unsigned long marker = pte_marker_get(entry);
3695 
3696 	/*
3697 	 * PTE markers should never be empty.  If anything weird happened,
3698 	 * the best thing to do is to kill the process along with its mm.
3699 	 */
3700 	if (WARN_ON_ONCE(!marker))
3701 		return VM_FAULT_SIGBUS;
3702 
3703 	/* Higher priority than uffd-wp when data corrupted */
3704 	if (marker & PTE_MARKER_POISONED)
3705 		return VM_FAULT_HWPOISON;
3706 
3707 	if (pte_marker_entry_uffd_wp(entry))
3708 		return pte_marker_handle_uffd_wp(vmf);
3709 
3710 	/* This is an unknown pte marker */
3711 	return VM_FAULT_SIGBUS;
3712 }
3713 
3714 /*
3715  * We enter with non-exclusive mmap_lock (to exclude vma changes,
3716  * but allow concurrent faults), and pte mapped but not yet locked.
3717  * We return with pte unmapped and unlocked.
3718  *
3719  * We return with the mmap_lock locked or unlocked in the same cases
3720  * as does filemap_fault().
3721  */
do_swap_page(struct vm_fault * vmf)3722 vm_fault_t do_swap_page(struct vm_fault *vmf)
3723 {
3724 	struct vm_area_struct *vma = vmf->vma;
3725 	struct folio *swapcache, *folio = NULL;
3726 	struct page *page;
3727 	struct swap_info_struct *si = NULL;
3728 	rmap_t rmap_flags = RMAP_NONE;
3729 	bool need_clear_cache = false;
3730 	bool exclusive = false;
3731 	swp_entry_t entry;
3732 	pte_t pte;
3733 	vm_fault_t ret = 0;
3734 	void *shadow = NULL;
3735 
3736 	if (!pte_unmap_same(vmf))
3737 		goto out;
3738 
3739 	entry = pte_to_swp_entry(vmf->orig_pte);
3740 	if (unlikely(non_swap_entry(entry))) {
3741 		if (is_migration_entry(entry)) {
3742 			migration_entry_wait(vma->vm_mm, vmf->pmd,
3743 					     vmf->address);
3744 		} else if (is_device_exclusive_entry(entry)) {
3745 			vmf->page = pfn_swap_entry_to_page(entry);
3746 			ret = remove_device_exclusive_entry(vmf);
3747 		} else if (is_device_private_entry(entry)) {
3748 			if (vmf->flags & FAULT_FLAG_VMA_LOCK) {
3749 				/*
3750 				 * migrate_to_ram is not yet ready to operate
3751 				 * under VMA lock.
3752 				 */
3753 				vma_end_read(vma);
3754 				ret = VM_FAULT_RETRY;
3755 				goto out;
3756 			}
3757 
3758 			vmf->page = pfn_swap_entry_to_page(entry);
3759 			vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
3760 					vmf->address, &vmf->ptl);
3761 			if (unlikely(!vmf->pte ||
3762 				     !pte_same(ptep_get(vmf->pte),
3763 							vmf->orig_pte)))
3764 				goto unlock;
3765 
3766 			/*
3767 			 * Get a page reference while we know the page can't be
3768 			 * freed.
3769 			 */
3770 			get_page(vmf->page);
3771 			pte_unmap_unlock(vmf->pte, vmf->ptl);
3772 			ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
3773 			put_page(vmf->page);
3774 		} else if (is_hwpoison_entry(entry)) {
3775 			ret = VM_FAULT_HWPOISON;
3776 		} else if (is_pte_marker_entry(entry)) {
3777 			ret = handle_pte_marker(vmf);
3778 		} else {
3779 			print_bad_pte(vma, vmf->address, vmf->orig_pte, NULL);
3780 			ret = VM_FAULT_SIGBUS;
3781 		}
3782 		goto out;
3783 	}
3784 
3785 	/* Prevent swapoff from happening to us. */
3786 	si = get_swap_device(entry);
3787 	if (unlikely(!si))
3788 		goto out;
3789 
3790 	folio = swap_cache_get_folio(entry, vma, vmf->address);
3791 	if (folio)
3792 		page = folio_file_page(folio, swp_offset(entry));
3793 	swapcache = folio;
3794 
3795 	if (!folio) {
3796 		if (data_race(si->flags & SWP_SYNCHRONOUS_IO) &&
3797 		    __swap_count(entry) == 1) {
3798 			/*
3799 			 * Prevent parallel swapin from proceeding with
3800 			 * the cache flag. Otherwise, another thread may
3801 			 * finish swapin first, free the entry, and swapout
3802 			 * reusing the same entry. It's undetectable as
3803 			 * pte_same() returns true due to entry reuse.
3804 			 */
3805 			if (swapcache_prepare(entry)) {
3806 				/* Relax a bit to prevent rapid repeated page faults */
3807 				schedule_timeout_uninterruptible(1);
3808 				goto out;
3809 			}
3810 			need_clear_cache = true;
3811 
3812 			/* skip swapcache */
3813 			folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 0,
3814 						vma, vmf->address, false);
3815 			page = &folio->page;
3816 			if (folio) {
3817 				__folio_set_locked(folio);
3818 				__folio_set_swapbacked(folio);
3819 
3820 				if (mem_cgroup_swapin_charge_folio(folio,
3821 							vma->vm_mm, GFP_KERNEL,
3822 							entry)) {
3823 					ret = VM_FAULT_OOM;
3824 					goto out_page;
3825 				}
3826 				mem_cgroup_swapin_uncharge_swap(entry);
3827 
3828 				shadow = get_shadow_from_swap_cache(entry);
3829 				if (shadow)
3830 					workingset_refault(folio, shadow);
3831 
3832 				folio_add_lru(folio);
3833 
3834 				/* To provide entry to swap_readpage() */
3835 				folio->swap = entry;
3836 				swap_readpage(page, true, NULL);
3837 				folio->private = NULL;
3838 			}
3839 		} else {
3840 			page = swapin_readahead(entry, GFP_HIGHUSER_MOVABLE,
3841 						vmf);
3842 			if (page)
3843 				folio = page_folio(page);
3844 			swapcache = folio;
3845 		}
3846 
3847 		if (!folio) {
3848 			/*
3849 			 * Back out if somebody else faulted in this pte
3850 			 * while we released the pte lock.
3851 			 */
3852 			vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
3853 					vmf->address, &vmf->ptl);
3854 			if (likely(vmf->pte &&
3855 				   pte_same(ptep_get(vmf->pte), vmf->orig_pte)))
3856 				ret = VM_FAULT_OOM;
3857 			goto unlock;
3858 		}
3859 
3860 		/* Had to read the page from swap area: Major fault */
3861 		ret = VM_FAULT_MAJOR;
3862 		count_vm_event(PGMAJFAULT);
3863 		count_memcg_event_mm(vma->vm_mm, PGMAJFAULT);
3864 	} else if (PageHWPoison(page)) {
3865 		/*
3866 		 * hwpoisoned dirty swapcache pages are kept for killing
3867 		 * owner processes (which may be unknown at hwpoison time)
3868 		 */
3869 		ret = VM_FAULT_HWPOISON;
3870 		goto out_release;
3871 	}
3872 
3873 	ret |= folio_lock_or_retry(folio, vmf);
3874 	if (ret & VM_FAULT_RETRY)
3875 		goto out_release;
3876 
3877 	if (swapcache) {
3878 		/*
3879 		 * Make sure folio_free_swap() or swapoff did not release the
3880 		 * swapcache from under us.  The page pin, and pte_same test
3881 		 * below, are not enough to exclude that.  Even if it is still
3882 		 * swapcache, we need to check that the page's swap has not
3883 		 * changed.
3884 		 */
3885 		if (unlikely(!folio_test_swapcache(folio) ||
3886 			     page_swap_entry(page).val != entry.val))
3887 			goto out_page;
3888 
3889 		/*
3890 		 * KSM sometimes has to copy on read faults, for example, if
3891 		 * page->index of !PageKSM() pages would be nonlinear inside the
3892 		 * anon VMA -- PageKSM() is lost on actual swapout.
3893 		 */
3894 		page = ksm_might_need_to_copy(page, vma, vmf->address);
3895 		if (unlikely(!page)) {
3896 			ret = VM_FAULT_OOM;
3897 			goto out_page;
3898 		} else if (unlikely(PTR_ERR(page) == -EHWPOISON)) {
3899 			ret = VM_FAULT_HWPOISON;
3900 			goto out_page;
3901 		}
3902 		folio = page_folio(page);
3903 
3904 		/*
3905 		 * If we want to map a page that's in the swapcache writable, we
3906 		 * have to detect via the refcount if we're really the exclusive
3907 		 * owner. Try removing the extra reference from the local LRU
3908 		 * caches if required.
3909 		 */
3910 		if ((vmf->flags & FAULT_FLAG_WRITE) && folio == swapcache &&
3911 		    !folio_test_ksm(folio) && !folio_test_lru(folio))
3912 			lru_add_drain();
3913 	}
3914 
3915 	folio_throttle_swaprate(folio, GFP_KERNEL);
3916 
3917 	/*
3918 	 * Back out if somebody else already faulted in this pte.
3919 	 */
3920 	vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
3921 			&vmf->ptl);
3922 	if (unlikely(!vmf->pte || !pte_same(ptep_get(vmf->pte), vmf->orig_pte)))
3923 		goto out_nomap;
3924 
3925 	if (unlikely(!folio_test_uptodate(folio))) {
3926 		ret = VM_FAULT_SIGBUS;
3927 		goto out_nomap;
3928 	}
3929 
3930 	/*
3931 	 * PG_anon_exclusive reuses PG_mappedtodisk for anon pages. A swap pte
3932 	 * must never point at an anonymous page in the swapcache that is
3933 	 * PG_anon_exclusive. Sanity check that this holds and especially, that
3934 	 * no filesystem set PG_mappedtodisk on a page in the swapcache. Sanity
3935 	 * check after taking the PT lock and making sure that nobody
3936 	 * concurrently faulted in this page and set PG_anon_exclusive.
3937 	 */
3938 	BUG_ON(!folio_test_anon(folio) && folio_test_mappedtodisk(folio));
3939 	BUG_ON(folio_test_anon(folio) && PageAnonExclusive(page));
3940 
3941 	/*
3942 	 * Check under PT lock (to protect against concurrent fork() sharing
3943 	 * the swap entry concurrently) for certainly exclusive pages.
3944 	 */
3945 	if (!folio_test_ksm(folio)) {
3946 		exclusive = pte_swp_exclusive(vmf->orig_pte);
3947 		if (folio != swapcache) {
3948 			/*
3949 			 * We have a fresh page that is not exposed to the
3950 			 * swapcache -> certainly exclusive.
3951 			 */
3952 			exclusive = true;
3953 		} else if (exclusive && folio_test_writeback(folio) &&
3954 			  data_race(si->flags & SWP_STABLE_WRITES)) {
3955 			/*
3956 			 * This is tricky: not all swap backends support
3957 			 * concurrent page modifications while under writeback.
3958 			 *
3959 			 * So if we stumble over such a page in the swapcache
3960 			 * we must not set the page exclusive, otherwise we can
3961 			 * map it writable without further checks and modify it
3962 			 * while still under writeback.
3963 			 *
3964 			 * For these problematic swap backends, simply drop the
3965 			 * exclusive marker: this is perfectly fine as we start
3966 			 * writeback only if we fully unmapped the page and
3967 			 * there are no unexpected references on the page after
3968 			 * unmapping succeeded. After fully unmapped, no
3969 			 * further GUP references (FOLL_GET and FOLL_PIN) can
3970 			 * appear, so dropping the exclusive marker and mapping
3971 			 * it only R/O is fine.
3972 			 */
3973 			exclusive = false;
3974 		}
3975 	}
3976 
3977 	/*
3978 	 * Some architectures may have to restore extra metadata to the page
3979 	 * when reading from swap. This metadata may be indexed by swap entry
3980 	 * so this must be called before swap_free().
3981 	 */
3982 	arch_swap_restore(entry, folio);
3983 
3984 	/*
3985 	 * Remove the swap entry and conditionally try to free up the swapcache.
3986 	 * We're already holding a reference on the page but haven't mapped it
3987 	 * yet.
3988 	 */
3989 	swap_free(entry);
3990 	if (should_try_to_free_swap(folio, vma, vmf->flags))
3991 		folio_free_swap(folio);
3992 
3993 	inc_mm_counter(vma->vm_mm, MM_ANONPAGES);
3994 	dec_mm_counter(vma->vm_mm, MM_SWAPENTS);
3995 	pte = mk_pte(page, vma->vm_page_prot);
3996 
3997 	/*
3998 	 * Same logic as in do_wp_page(); however, optimize for pages that are
3999 	 * certainly not shared either because we just allocated them without
4000 	 * exposing them to the swapcache or because the swap entry indicates
4001 	 * exclusivity.
4002 	 */
4003 	if (!folio_test_ksm(folio) &&
4004 	    (exclusive || folio_ref_count(folio) == 1)) {
4005 		if (vmf->flags & FAULT_FLAG_WRITE) {
4006 			pte = maybe_mkwrite(pte_mkdirty(pte), vma);
4007 			vmf->flags &= ~FAULT_FLAG_WRITE;
4008 		}
4009 		rmap_flags |= RMAP_EXCLUSIVE;
4010 	}
4011 	flush_icache_page(vma, page);
4012 	if (pte_swp_soft_dirty(vmf->orig_pte))
4013 		pte = pte_mksoft_dirty(pte);
4014 	if (pte_swp_uffd_wp(vmf->orig_pte))
4015 		pte = pte_mkuffd_wp(pte);
4016 	vmf->orig_pte = pte;
4017 
4018 	/* ksm created a completely new copy */
4019 	if (unlikely(folio != swapcache && swapcache)) {
4020 		page_add_new_anon_rmap(page, vma, vmf->address);
4021 		folio_add_lru_vma(folio, vma);
4022 	} else {
4023 		page_add_anon_rmap(page, vma, vmf->address, rmap_flags);
4024 	}
4025 
4026 	VM_BUG_ON(!folio_test_anon(folio) ||
4027 			(pte_write(pte) && !PageAnonExclusive(page)));
4028 	set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte);
4029 	arch_do_swap_page(vma->vm_mm, vma, vmf->address, pte, vmf->orig_pte);
4030 
4031 	folio_unlock(folio);
4032 	if (folio != swapcache && swapcache) {
4033 		/*
4034 		 * Hold the lock to avoid the swap entry to be reused
4035 		 * until we take the PT lock for the pte_same() check
4036 		 * (to avoid false positives from pte_same). For
4037 		 * further safety release the lock after the swap_free
4038 		 * so that the swap count won't change under a
4039 		 * parallel locked swapcache.
4040 		 */
4041 		folio_unlock(swapcache);
4042 		folio_put(swapcache);
4043 	}
4044 
4045 	if (vmf->flags & FAULT_FLAG_WRITE) {
4046 		ret |= do_wp_page(vmf);
4047 		if (ret & VM_FAULT_ERROR)
4048 			ret &= VM_FAULT_ERROR;
4049 		goto out;
4050 	}
4051 
4052 	/* No need to invalidate - it was non-present before */
4053 	update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1);
4054 unlock:
4055 	if (vmf->pte)
4056 		pte_unmap_unlock(vmf->pte, vmf->ptl);
4057 out:
4058 	/* Clear the swap cache pin for direct swapin after PTL unlock */
4059 	if (need_clear_cache)
4060 		swapcache_clear(si, entry);
4061 	if (si)
4062 		put_swap_device(si);
4063 	return ret;
4064 out_nomap:
4065 	if (vmf->pte)
4066 		pte_unmap_unlock(vmf->pte, vmf->ptl);
4067 out_page:
4068 	folio_unlock(folio);
4069 out_release:
4070 	folio_put(folio);
4071 	if (folio != swapcache && swapcache) {
4072 		folio_unlock(swapcache);
4073 		folio_put(swapcache);
4074 	}
4075 	if (need_clear_cache)
4076 		swapcache_clear(si, entry);
4077 	if (si)
4078 		put_swap_device(si);
4079 	return ret;
4080 }
4081 
4082 /*
4083  * We enter with non-exclusive mmap_lock (to exclude vma changes,
4084  * but allow concurrent faults), and pte mapped but not yet locked.
4085  * We return with mmap_lock still held, but pte unmapped and unlocked.
4086  */
do_anonymous_page(struct vm_fault * vmf)4087 static vm_fault_t do_anonymous_page(struct vm_fault *vmf)
4088 {
4089 	bool uffd_wp = vmf_orig_pte_uffd_wp(vmf);
4090 	struct vm_area_struct *vma = vmf->vma;
4091 	struct folio *folio;
4092 	vm_fault_t ret = 0;
4093 	pte_t entry;
4094 
4095 	/* File mapping without ->vm_ops ? */
4096 	if (vma->vm_flags & VM_SHARED)
4097 		return VM_FAULT_SIGBUS;
4098 
4099 	/*
4100 	 * Use pte_alloc() instead of pte_alloc_map(), so that OOM can
4101 	 * be distinguished from a transient failure of pte_offset_map().
4102 	 */
4103 	if (pte_alloc(vma->vm_mm, vmf->pmd))
4104 		return VM_FAULT_OOM;
4105 
4106 	/* Use the zero-page for reads */
4107 	if (!(vmf->flags & FAULT_FLAG_WRITE) &&
4108 			!mm_forbids_zeropage(vma->vm_mm)) {
4109 		entry = pte_mkspecial(pfn_pte(my_zero_pfn(vmf->address),
4110 						vma->vm_page_prot));
4111 		vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
4112 				vmf->address, &vmf->ptl);
4113 		if (!vmf->pte)
4114 			goto unlock;
4115 		if (vmf_pte_changed(vmf)) {
4116 			update_mmu_tlb(vma, vmf->address, vmf->pte);
4117 			goto unlock;
4118 		}
4119 		ret = check_stable_address_space(vma->vm_mm);
4120 		if (ret)
4121 			goto unlock;
4122 		/* Deliver the page fault to userland, check inside PT lock */
4123 		if (userfaultfd_missing(vma)) {
4124 			pte_unmap_unlock(vmf->pte, vmf->ptl);
4125 			return handle_userfault(vmf, VM_UFFD_MISSING);
4126 		}
4127 		goto setpte;
4128 	}
4129 
4130 	/* Allocate our own private page. */
4131 	if (unlikely(anon_vma_prepare(vma)))
4132 		goto oom;
4133 	folio = vma_alloc_zeroed_movable_folio(vma, vmf->address);
4134 	if (!folio)
4135 		goto oom;
4136 
4137 	if (mem_cgroup_charge(folio, vma->vm_mm, GFP_KERNEL))
4138 		goto oom_free_page;
4139 	folio_throttle_swaprate(folio, GFP_KERNEL);
4140 
4141 	/*
4142 	 * The memory barrier inside __folio_mark_uptodate makes sure that
4143 	 * preceding stores to the page contents become visible before
4144 	 * the set_pte_at() write.
4145 	 */
4146 	__folio_mark_uptodate(folio);
4147 
4148 	entry = mk_pte(&folio->page, vma->vm_page_prot);
4149 	entry = pte_sw_mkyoung(entry);
4150 	if (vma->vm_flags & VM_WRITE)
4151 		entry = pte_mkwrite(pte_mkdirty(entry), vma);
4152 
4153 	vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
4154 			&vmf->ptl);
4155 	if (!vmf->pte)
4156 		goto release;
4157 	if (vmf_pte_changed(vmf)) {
4158 		update_mmu_tlb(vma, vmf->address, vmf->pte);
4159 		goto release;
4160 	}
4161 
4162 	ret = check_stable_address_space(vma->vm_mm);
4163 	if (ret)
4164 		goto release;
4165 
4166 	/* Deliver the page fault to userland, check inside PT lock */
4167 	if (userfaultfd_missing(vma)) {
4168 		pte_unmap_unlock(vmf->pte, vmf->ptl);
4169 		folio_put(folio);
4170 		return handle_userfault(vmf, VM_UFFD_MISSING);
4171 	}
4172 
4173 	inc_mm_counter(vma->vm_mm, MM_ANONPAGES);
4174 	folio_add_new_anon_rmap(folio, vma, vmf->address);
4175 	folio_add_lru_vma(folio, vma);
4176 setpte:
4177 	if (uffd_wp)
4178 		entry = pte_mkuffd_wp(entry);
4179 	set_pte_at(vma->vm_mm, vmf->address, vmf->pte, entry);
4180 
4181 	/* No need to invalidate - it was non-present before */
4182 	update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1);
4183 unlock:
4184 	if (vmf->pte)
4185 		pte_unmap_unlock(vmf->pte, vmf->ptl);
4186 	return ret;
4187 release:
4188 	folio_put(folio);
4189 	goto unlock;
4190 oom_free_page:
4191 	folio_put(folio);
4192 oom:
4193 	return VM_FAULT_OOM;
4194 }
4195 
4196 /*
4197  * The mmap_lock must have been held on entry, and may have been
4198  * released depending on flags and vma->vm_ops->fault() return value.
4199  * See filemap_fault() and __lock_page_retry().
4200  */
__do_fault(struct vm_fault * vmf)4201 static vm_fault_t __do_fault(struct vm_fault *vmf)
4202 {
4203 	struct vm_area_struct *vma = vmf->vma;
4204 	vm_fault_t ret;
4205 
4206 	/*
4207 	 * Preallocate pte before we take page_lock because this might lead to
4208 	 * deadlocks for memcg reclaim which waits for pages under writeback:
4209 	 *				lock_page(A)
4210 	 *				SetPageWriteback(A)
4211 	 *				unlock_page(A)
4212 	 * lock_page(B)
4213 	 *				lock_page(B)
4214 	 * pte_alloc_one
4215 	 *   shrink_page_list
4216 	 *     wait_on_page_writeback(A)
4217 	 *				SetPageWriteback(B)
4218 	 *				unlock_page(B)
4219 	 *				# flush A, B to clear the writeback
4220 	 */
4221 	if (pmd_none(*vmf->pmd) && !vmf->prealloc_pte) {
4222 		vmf->prealloc_pte = pte_alloc_one(vma->vm_mm);
4223 		if (!vmf->prealloc_pte)
4224 			return VM_FAULT_OOM;
4225 	}
4226 
4227 	ret = vma->vm_ops->fault(vmf);
4228 	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY |
4229 			    VM_FAULT_DONE_COW)))
4230 		return ret;
4231 
4232 	if (unlikely(PageHWPoison(vmf->page))) {
4233 		struct page *page = vmf->page;
4234 		vm_fault_t poisonret = VM_FAULT_HWPOISON;
4235 		if (ret & VM_FAULT_LOCKED) {
4236 			if (page_mapped(page))
4237 				unmap_mapping_pages(page_mapping(page),
4238 						    page->index, 1, false);
4239 			/* Retry if a clean page was removed from the cache. */
4240 			if (invalidate_inode_page(page))
4241 				poisonret = VM_FAULT_NOPAGE;
4242 			unlock_page(page);
4243 		}
4244 		put_page(page);
4245 		vmf->page = NULL;
4246 		return poisonret;
4247 	}
4248 
4249 	if (unlikely(!(ret & VM_FAULT_LOCKED)))
4250 		lock_page(vmf->page);
4251 	else
4252 		VM_BUG_ON_PAGE(!PageLocked(vmf->page), vmf->page);
4253 
4254 	return ret;
4255 }
4256 
4257 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
deposit_prealloc_pte(struct vm_fault * vmf)4258 static void deposit_prealloc_pte(struct vm_fault *vmf)
4259 {
4260 	struct vm_area_struct *vma = vmf->vma;
4261 
4262 	pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, vmf->prealloc_pte);
4263 	/*
4264 	 * We are going to consume the prealloc table,
4265 	 * count that as nr_ptes.
4266 	 */
4267 	mm_inc_nr_ptes(vma->vm_mm);
4268 	vmf->prealloc_pte = NULL;
4269 }
4270 
do_set_pmd(struct vm_fault * vmf,struct page * page)4271 vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page)
4272 {
4273 	struct vm_area_struct *vma = vmf->vma;
4274 	bool write = vmf->flags & FAULT_FLAG_WRITE;
4275 	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
4276 	pmd_t entry;
4277 	vm_fault_t ret = VM_FAULT_FALLBACK;
4278 
4279 	if (!transhuge_vma_suitable(vma, haddr))
4280 		return ret;
4281 
4282 	page = compound_head(page);
4283 	if (compound_order(page) != HPAGE_PMD_ORDER)
4284 		return ret;
4285 
4286 	/*
4287 	 * Just backoff if any subpage of a THP is corrupted otherwise
4288 	 * the corrupted page may mapped by PMD silently to escape the
4289 	 * check.  This kind of THP just can be PTE mapped.  Access to
4290 	 * the corrupted subpage should trigger SIGBUS as expected.
4291 	 */
4292 	if (unlikely(PageHasHWPoisoned(page)))
4293 		return ret;
4294 
4295 	/*
4296 	 * Archs like ppc64 need additional space to store information
4297 	 * related to pte entry. Use the preallocated table for that.
4298 	 */
4299 	if (arch_needs_pgtable_deposit() && !vmf->prealloc_pte) {
4300 		vmf->prealloc_pte = pte_alloc_one(vma->vm_mm);
4301 		if (!vmf->prealloc_pte)
4302 			return VM_FAULT_OOM;
4303 	}
4304 
4305 	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
4306 	if (unlikely(!pmd_none(*vmf->pmd)))
4307 		goto out;
4308 
4309 	flush_icache_pages(vma, page, HPAGE_PMD_NR);
4310 
4311 	entry = mk_huge_pmd(page, vma->vm_page_prot);
4312 	if (write)
4313 		entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
4314 
4315 	add_mm_counter(vma->vm_mm, mm_counter_file(page), HPAGE_PMD_NR);
4316 	page_add_file_rmap(page, vma, true);
4317 
4318 	/*
4319 	 * deposit and withdraw with pmd lock held
4320 	 */
4321 	if (arch_needs_pgtable_deposit())
4322 		deposit_prealloc_pte(vmf);
4323 
4324 	set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);
4325 
4326 	update_mmu_cache_pmd(vma, haddr, vmf->pmd);
4327 
4328 	/* fault is handled */
4329 	ret = 0;
4330 	count_vm_event(THP_FILE_MAPPED);
4331 out:
4332 	spin_unlock(vmf->ptl);
4333 	return ret;
4334 }
4335 #else
do_set_pmd(struct vm_fault * vmf,struct page * page)4336 vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page)
4337 {
4338 	return VM_FAULT_FALLBACK;
4339 }
4340 #endif
4341 
4342 /**
4343  * set_pte_range - Set a range of PTEs to point to pages in a folio.
4344  * @vmf: Fault decription.
4345  * @folio: The folio that contains @page.
4346  * @page: The first page to create a PTE for.
4347  * @nr: The number of PTEs to create.
4348  * @addr: The first address to create a PTE for.
4349  */
set_pte_range(struct vm_fault * vmf,struct folio * folio,struct page * page,unsigned int nr,unsigned long addr)4350 void set_pte_range(struct vm_fault *vmf, struct folio *folio,
4351 		struct page *page, unsigned int nr, unsigned long addr)
4352 {
4353 	struct vm_area_struct *vma = vmf->vma;
4354 	bool uffd_wp = vmf_orig_pte_uffd_wp(vmf);
4355 	bool write = vmf->flags & FAULT_FLAG_WRITE;
4356 	bool prefault = in_range(vmf->address, addr, nr * PAGE_SIZE);
4357 	pte_t entry;
4358 
4359 	flush_icache_pages(vma, page, nr);
4360 	entry = mk_pte(page, vma->vm_page_prot);
4361 
4362 	if (prefault && arch_wants_old_prefaulted_pte())
4363 		entry = pte_mkold(entry);
4364 	else
4365 		entry = pte_sw_mkyoung(entry);
4366 
4367 	if (write)
4368 		entry = maybe_mkwrite(pte_mkdirty(entry), vma);
4369 	if (unlikely(uffd_wp))
4370 		entry = pte_mkuffd_wp(entry);
4371 	/* copy-on-write page */
4372 	if (write && !(vma->vm_flags & VM_SHARED)) {
4373 		add_mm_counter(vma->vm_mm, MM_ANONPAGES, nr);
4374 		VM_BUG_ON_FOLIO(nr != 1, folio);
4375 		folio_add_new_anon_rmap(folio, vma, addr);
4376 		folio_add_lru_vma(folio, vma);
4377 	} else {
4378 		add_mm_counter(vma->vm_mm, mm_counter_file(page), nr);
4379 		folio_add_file_rmap_range(folio, page, nr, vma, false);
4380 	}
4381 	set_ptes(vma->vm_mm, addr, vmf->pte, entry, nr);
4382 
4383 	/* no need to invalidate: a not-present page won't be cached */
4384 	update_mmu_cache_range(vmf, vma, addr, vmf->pte, nr);
4385 }
4386 
vmf_pte_changed(struct vm_fault * vmf)4387 static bool vmf_pte_changed(struct vm_fault *vmf)
4388 {
4389 	if (vmf->flags & FAULT_FLAG_ORIG_PTE_VALID)
4390 		return !pte_same(ptep_get(vmf->pte), vmf->orig_pte);
4391 
4392 	return !pte_none(ptep_get(vmf->pte));
4393 }
4394 
4395 /**
4396  * finish_fault - finish page fault once we have prepared the page to fault
4397  *
4398  * @vmf: structure describing the fault
4399  *
4400  * This function handles all that is needed to finish a page fault once the
4401  * page to fault in is prepared. It handles locking of PTEs, inserts PTE for
4402  * given page, adds reverse page mapping, handles memcg charges and LRU
4403  * addition.
4404  *
4405  * The function expects the page to be locked and on success it consumes a
4406  * reference of a page being mapped (for the PTE which maps it).
4407  *
4408  * Return: %0 on success, %VM_FAULT_ code in case of error.
4409  */
finish_fault(struct vm_fault * vmf)4410 vm_fault_t finish_fault(struct vm_fault *vmf)
4411 {
4412 	struct vm_area_struct *vma = vmf->vma;
4413 	struct page *page;
4414 	vm_fault_t ret;
4415 
4416 	/* Did we COW the page? */
4417 	if ((vmf->flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED))
4418 		page = vmf->cow_page;
4419 	else
4420 		page = vmf->page;
4421 
4422 	/*
4423 	 * check even for read faults because we might have lost our CoWed
4424 	 * page
4425 	 */
4426 	if (!(vma->vm_flags & VM_SHARED)) {
4427 		ret = check_stable_address_space(vma->vm_mm);
4428 		if (ret)
4429 			return ret;
4430 	}
4431 
4432 	if (pmd_none(*vmf->pmd)) {
4433 		if (PageTransCompound(page)) {
4434 			ret = do_set_pmd(vmf, page);
4435 			if (ret != VM_FAULT_FALLBACK)
4436 				return ret;
4437 		}
4438 
4439 		if (vmf->prealloc_pte)
4440 			pmd_install(vma->vm_mm, vmf->pmd, &vmf->prealloc_pte);
4441 		else if (unlikely(pte_alloc(vma->vm_mm, vmf->pmd)))
4442 			return VM_FAULT_OOM;
4443 	}
4444 
4445 	vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
4446 				      vmf->address, &vmf->ptl);
4447 	if (!vmf->pte)
4448 		return VM_FAULT_NOPAGE;
4449 
4450 	/* Re-check under ptl */
4451 	if (likely(!vmf_pte_changed(vmf))) {
4452 		struct folio *folio = page_folio(page);
4453 
4454 		set_pte_range(vmf, folio, page, 1, vmf->address);
4455 		ret = 0;
4456 	} else {
4457 		update_mmu_tlb(vma, vmf->address, vmf->pte);
4458 		ret = VM_FAULT_NOPAGE;
4459 	}
4460 
4461 	pte_unmap_unlock(vmf->pte, vmf->ptl);
4462 	return ret;
4463 }
4464 
4465 static unsigned long fault_around_pages __read_mostly =
4466 	65536 >> PAGE_SHIFT;
4467 
4468 #ifdef CONFIG_DEBUG_FS
fault_around_bytes_get(void * data,u64 * val)4469 static int fault_around_bytes_get(void *data, u64 *val)
4470 {
4471 	*val = fault_around_pages << PAGE_SHIFT;
4472 	return 0;
4473 }
4474 
4475 /*
4476  * fault_around_bytes must be rounded down to the nearest page order as it's
4477  * what do_fault_around() expects to see.
4478  */
fault_around_bytes_set(void * data,u64 val)4479 static int fault_around_bytes_set(void *data, u64 val)
4480 {
4481 	if (val / PAGE_SIZE > PTRS_PER_PTE)
4482 		return -EINVAL;
4483 
4484 	/*
4485 	 * The minimum value is 1 page, however this results in no fault-around
4486 	 * at all. See should_fault_around().
4487 	 */
4488 	fault_around_pages = max(rounddown_pow_of_two(val) >> PAGE_SHIFT, 1UL);
4489 
4490 	return 0;
4491 }
4492 DEFINE_DEBUGFS_ATTRIBUTE(fault_around_bytes_fops,
4493 		fault_around_bytes_get, fault_around_bytes_set, "%llu\n");
4494 
fault_around_debugfs(void)4495 static int __init fault_around_debugfs(void)
4496 {
4497 	debugfs_create_file_unsafe("fault_around_bytes", 0644, NULL, NULL,
4498 				   &fault_around_bytes_fops);
4499 	return 0;
4500 }
4501 late_initcall(fault_around_debugfs);
4502 #endif
4503 
4504 /*
4505  * do_fault_around() tries to map few pages around the fault address. The hope
4506  * is that the pages will be needed soon and this will lower the number of
4507  * faults to handle.
4508  *
4509  * It uses vm_ops->map_pages() to map the pages, which skips the page if it's
4510  * not ready to be mapped: not up-to-date, locked, etc.
4511  *
4512  * This function doesn't cross VMA or page table boundaries, in order to call
4513  * map_pages() and acquire a PTE lock only once.
4514  *
4515  * fault_around_pages defines how many pages we'll try to map.
4516  * do_fault_around() expects it to be set to a power of two less than or equal
4517  * to PTRS_PER_PTE.
4518  *
4519  * The virtual address of the area that we map is naturally aligned to
4520  * fault_around_pages * PAGE_SIZE rounded down to the machine page size
4521  * (and therefore to page order).  This way it's easier to guarantee
4522  * that we don't cross page table boundaries.
4523  */
do_fault_around(struct vm_fault * vmf)4524 static vm_fault_t do_fault_around(struct vm_fault *vmf)
4525 {
4526 	pgoff_t nr_pages = READ_ONCE(fault_around_pages);
4527 	pgoff_t pte_off = pte_index(vmf->address);
4528 	/* The page offset of vmf->address within the VMA. */
4529 	pgoff_t vma_off = vmf->pgoff - vmf->vma->vm_pgoff;
4530 	pgoff_t from_pte, to_pte;
4531 	vm_fault_t ret;
4532 
4533 	/* The PTE offset of the start address, clamped to the VMA. */
4534 	from_pte = max(ALIGN_DOWN(pte_off, nr_pages),
4535 		       pte_off - min(pte_off, vma_off));
4536 
4537 	/* The PTE offset of the end address, clamped to the VMA and PTE. */
4538 	to_pte = min3(from_pte + nr_pages, (pgoff_t)PTRS_PER_PTE,
4539 		      pte_off + vma_pages(vmf->vma) - vma_off) - 1;
4540 
4541 	if (pmd_none(*vmf->pmd)) {
4542 		vmf->prealloc_pte = pte_alloc_one(vmf->vma->vm_mm);
4543 		if (!vmf->prealloc_pte)
4544 			return VM_FAULT_OOM;
4545 	}
4546 
4547 	rcu_read_lock();
4548 	ret = vmf->vma->vm_ops->map_pages(vmf,
4549 			vmf->pgoff + from_pte - pte_off,
4550 			vmf->pgoff + to_pte - pte_off);
4551 	rcu_read_unlock();
4552 
4553 	return ret;
4554 }
4555 
4556 /* Return true if we should do read fault-around, false otherwise */
should_fault_around(struct vm_fault * vmf)4557 static inline bool should_fault_around(struct vm_fault *vmf)
4558 {
4559 	/* No ->map_pages?  No way to fault around... */
4560 	if (!vmf->vma->vm_ops->map_pages)
4561 		return false;
4562 
4563 	if (uffd_disable_fault_around(vmf->vma))
4564 		return false;
4565 
4566 	/* A single page implies no faulting 'around' at all. */
4567 	return fault_around_pages > 1;
4568 }
4569 
do_read_fault(struct vm_fault * vmf)4570 static vm_fault_t do_read_fault(struct vm_fault *vmf)
4571 {
4572 	vm_fault_t ret = 0;
4573 	struct folio *folio;
4574 
4575 	/*
4576 	 * Let's call ->map_pages() first and use ->fault() as fallback
4577 	 * if page by the offset is not ready to be mapped (cold cache or
4578 	 * something).
4579 	 */
4580 	if (should_fault_around(vmf)) {
4581 		ret = do_fault_around(vmf);
4582 		if (ret)
4583 			return ret;
4584 	}
4585 
4586 	if (vmf->flags & FAULT_FLAG_VMA_LOCK) {
4587 		vma_end_read(vmf->vma);
4588 		return VM_FAULT_RETRY;
4589 	}
4590 
4591 	ret = __do_fault(vmf);
4592 	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
4593 		return ret;
4594 
4595 	ret |= finish_fault(vmf);
4596 	folio = page_folio(vmf->page);
4597 	folio_unlock(folio);
4598 	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
4599 		folio_put(folio);
4600 	return ret;
4601 }
4602 
do_cow_fault(struct vm_fault * vmf)4603 static vm_fault_t do_cow_fault(struct vm_fault *vmf)
4604 {
4605 	struct vm_area_struct *vma = vmf->vma;
4606 	vm_fault_t ret;
4607 
4608 	if (vmf->flags & FAULT_FLAG_VMA_LOCK) {
4609 		vma_end_read(vma);
4610 		return VM_FAULT_RETRY;
4611 	}
4612 
4613 	if (unlikely(anon_vma_prepare(vma)))
4614 		return VM_FAULT_OOM;
4615 
4616 	vmf->cow_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vmf->address);
4617 	if (!vmf->cow_page)
4618 		return VM_FAULT_OOM;
4619 
4620 	if (mem_cgroup_charge(page_folio(vmf->cow_page), vma->vm_mm,
4621 				GFP_KERNEL)) {
4622 		put_page(vmf->cow_page);
4623 		return VM_FAULT_OOM;
4624 	}
4625 	folio_throttle_swaprate(page_folio(vmf->cow_page), GFP_KERNEL);
4626 
4627 	ret = __do_fault(vmf);
4628 	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
4629 		goto uncharge_out;
4630 	if (ret & VM_FAULT_DONE_COW)
4631 		return ret;
4632 
4633 	copy_user_highpage(vmf->cow_page, vmf->page, vmf->address, vma);
4634 	__SetPageUptodate(vmf->cow_page);
4635 
4636 	ret |= finish_fault(vmf);
4637 	unlock_page(vmf->page);
4638 	put_page(vmf->page);
4639 	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
4640 		goto uncharge_out;
4641 	return ret;
4642 uncharge_out:
4643 	put_page(vmf->cow_page);
4644 	return ret;
4645 }
4646 
do_shared_fault(struct vm_fault * vmf)4647 static vm_fault_t do_shared_fault(struct vm_fault *vmf)
4648 {
4649 	struct vm_area_struct *vma = vmf->vma;
4650 	vm_fault_t ret, tmp;
4651 	struct folio *folio;
4652 
4653 	if (vmf->flags & FAULT_FLAG_VMA_LOCK) {
4654 		vma_end_read(vma);
4655 		return VM_FAULT_RETRY;
4656 	}
4657 
4658 	ret = __do_fault(vmf);
4659 	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
4660 		return ret;
4661 
4662 	folio = page_folio(vmf->page);
4663 
4664 	/*
4665 	 * Check if the backing address space wants to know that the page is
4666 	 * about to become writable
4667 	 */
4668 	if (vma->vm_ops->page_mkwrite) {
4669 		folio_unlock(folio);
4670 		tmp = do_page_mkwrite(vmf, folio);
4671 		if (unlikely(!tmp ||
4672 				(tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) {
4673 			folio_put(folio);
4674 			return tmp;
4675 		}
4676 	}
4677 
4678 	ret |= finish_fault(vmf);
4679 	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE |
4680 					VM_FAULT_RETRY))) {
4681 		folio_unlock(folio);
4682 		folio_put(folio);
4683 		return ret;
4684 	}
4685 
4686 	ret |= fault_dirty_shared_page(vmf);
4687 	return ret;
4688 }
4689 
4690 /*
4691  * We enter with non-exclusive mmap_lock (to exclude vma changes,
4692  * but allow concurrent faults).
4693  * The mmap_lock may have been released depending on flags and our
4694  * return value.  See filemap_fault() and __folio_lock_or_retry().
4695  * If mmap_lock is released, vma may become invalid (for example
4696  * by other thread calling munmap()).
4697  */
do_fault(struct vm_fault * vmf)4698 static vm_fault_t do_fault(struct vm_fault *vmf)
4699 {
4700 	struct vm_area_struct *vma = vmf->vma;
4701 	struct mm_struct *vm_mm = vma->vm_mm;
4702 	vm_fault_t ret;
4703 
4704 	/*
4705 	 * The VMA was not fully populated on mmap() or missing VM_DONTEXPAND
4706 	 */
4707 	if (!vma->vm_ops->fault) {
4708 		vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd,
4709 					       vmf->address, &vmf->ptl);
4710 		if (unlikely(!vmf->pte))
4711 			ret = VM_FAULT_SIGBUS;
4712 		else {
4713 			/*
4714 			 * Make sure this is not a temporary clearing of pte
4715 			 * by holding ptl and checking again. A R/M/W update
4716 			 * of pte involves: take ptl, clearing the pte so that
4717 			 * we don't have concurrent modification by hardware
4718 			 * followed by an update.
4719 			 */
4720 			if (unlikely(pte_none(ptep_get(vmf->pte))))
4721 				ret = VM_FAULT_SIGBUS;
4722 			else
4723 				ret = VM_FAULT_NOPAGE;
4724 
4725 			pte_unmap_unlock(vmf->pte, vmf->ptl);
4726 		}
4727 	} else if (!(vmf->flags & FAULT_FLAG_WRITE))
4728 		ret = do_read_fault(vmf);
4729 	else if (!(vma->vm_flags & VM_SHARED))
4730 		ret = do_cow_fault(vmf);
4731 	else
4732 		ret = do_shared_fault(vmf);
4733 
4734 	/* preallocated pagetable is unused: free it */
4735 	if (vmf->prealloc_pte) {
4736 		pte_free(vm_mm, vmf->prealloc_pte);
4737 		vmf->prealloc_pte = NULL;
4738 	}
4739 	return ret;
4740 }
4741 
numa_migrate_prep(struct page * page,struct vm_area_struct * vma,unsigned long addr,int page_nid,int * flags)4742 int numa_migrate_prep(struct page *page, struct vm_area_struct *vma,
4743 		      unsigned long addr, int page_nid, int *flags)
4744 {
4745 	get_page(page);
4746 
4747 	/* Record the current PID acceesing VMA */
4748 	vma_set_access_pid_bit(vma);
4749 
4750 	count_vm_numa_event(NUMA_HINT_FAULTS);
4751 	if (page_nid == numa_node_id()) {
4752 		count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL);
4753 		*flags |= TNF_FAULT_LOCAL;
4754 	}
4755 
4756 	return mpol_misplaced(page, vma, addr);
4757 }
4758 
do_numa_page(struct vm_fault * vmf)4759 static vm_fault_t do_numa_page(struct vm_fault *vmf)
4760 {
4761 	struct vm_area_struct *vma = vmf->vma;
4762 	struct page *page = NULL;
4763 	int page_nid = NUMA_NO_NODE;
4764 	bool writable = false;
4765 	int last_cpupid;
4766 	int target_nid;
4767 	pte_t pte, old_pte;
4768 	int flags = 0;
4769 
4770 	/*
4771 	 * The "pte" at this point cannot be used safely without
4772 	 * validation through pte_unmap_same(). It's of NUMA type but
4773 	 * the pfn may be screwed if the read is non atomic.
4774 	 */
4775 	spin_lock(vmf->ptl);
4776 	if (unlikely(!pte_same(ptep_get(vmf->pte), vmf->orig_pte))) {
4777 		pte_unmap_unlock(vmf->pte, vmf->ptl);
4778 		goto out;
4779 	}
4780 
4781 	/* Get the normal PTE  */
4782 	old_pte = ptep_get(vmf->pte);
4783 	pte = pte_modify(old_pte, vma->vm_page_prot);
4784 
4785 	/*
4786 	 * Detect now whether the PTE could be writable; this information
4787 	 * is only valid while holding the PT lock.
4788 	 */
4789 	writable = pte_write(pte);
4790 	if (!writable && vma_wants_manual_pte_write_upgrade(vma) &&
4791 	    can_change_pte_writable(vma, vmf->address, pte))
4792 		writable = true;
4793 
4794 	page = vm_normal_page(vma, vmf->address, pte);
4795 	if (!page || is_zone_device_page(page))
4796 		goto out_map;
4797 
4798 	/* TODO: handle PTE-mapped THP */
4799 	if (PageCompound(page))
4800 		goto out_map;
4801 
4802 	/*
4803 	 * Avoid grouping on RO pages in general. RO pages shouldn't hurt as
4804 	 * much anyway since they can be in shared cache state. This misses
4805 	 * the case where a mapping is writable but the process never writes
4806 	 * to it but pte_write gets cleared during protection updates and
4807 	 * pte_dirty has unpredictable behaviour between PTE scan updates,
4808 	 * background writeback, dirty balancing and application behaviour.
4809 	 */
4810 	if (!writable)
4811 		flags |= TNF_NO_GROUP;
4812 
4813 	/*
4814 	 * Flag if the page is shared between multiple address spaces. This
4815 	 * is later used when determining whether to group tasks together
4816 	 */
4817 	if (page_mapcount(page) > 1 && (vma->vm_flags & VM_SHARED))
4818 		flags |= TNF_SHARED;
4819 
4820 	page_nid = page_to_nid(page);
4821 	/*
4822 	 * For memory tiering mode, cpupid of slow memory page is used
4823 	 * to record page access time.  So use default value.
4824 	 */
4825 	if ((sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) &&
4826 	    !node_is_toptier(page_nid))
4827 		last_cpupid = (-1 & LAST_CPUPID_MASK);
4828 	else
4829 		last_cpupid = page_cpupid_last(page);
4830 	target_nid = numa_migrate_prep(page, vma, vmf->address, page_nid,
4831 			&flags);
4832 	if (target_nid == NUMA_NO_NODE) {
4833 		put_page(page);
4834 		goto out_map;
4835 	}
4836 	pte_unmap_unlock(vmf->pte, vmf->ptl);
4837 	writable = false;
4838 
4839 	/* Migrate to the requested node */
4840 	if (migrate_misplaced_page(page, vma, target_nid)) {
4841 		page_nid = target_nid;
4842 		flags |= TNF_MIGRATED;
4843 	} else {
4844 		flags |= TNF_MIGRATE_FAIL;
4845 		vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
4846 					       vmf->address, &vmf->ptl);
4847 		if (unlikely(!vmf->pte))
4848 			goto out;
4849 		if (unlikely(!pte_same(ptep_get(vmf->pte), vmf->orig_pte))) {
4850 			pte_unmap_unlock(vmf->pte, vmf->ptl);
4851 			goto out;
4852 		}
4853 		goto out_map;
4854 	}
4855 
4856 out:
4857 	if (page_nid != NUMA_NO_NODE)
4858 		task_numa_fault(last_cpupid, page_nid, 1, flags);
4859 	return 0;
4860 out_map:
4861 	/*
4862 	 * Make it present again, depending on how arch implements
4863 	 * non-accessible ptes, some can allow access by kernel mode.
4864 	 */
4865 	old_pte = ptep_modify_prot_start(vma, vmf->address, vmf->pte);
4866 	pte = pte_modify(old_pte, vma->vm_page_prot);
4867 	pte = pte_mkyoung(pte);
4868 	if (writable)
4869 		pte = pte_mkwrite(pte, vma);
4870 	ptep_modify_prot_commit(vma, vmf->address, vmf->pte, old_pte, pte);
4871 	update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1);
4872 	pte_unmap_unlock(vmf->pte, vmf->ptl);
4873 	goto out;
4874 }
4875 
create_huge_pmd(struct vm_fault * vmf)4876 static inline vm_fault_t create_huge_pmd(struct vm_fault *vmf)
4877 {
4878 	struct vm_area_struct *vma = vmf->vma;
4879 	if (vma_is_anonymous(vma))
4880 		return do_huge_pmd_anonymous_page(vmf);
4881 	if (vma->vm_ops->huge_fault)
4882 		return vma->vm_ops->huge_fault(vmf, PMD_ORDER);
4883 	return VM_FAULT_FALLBACK;
4884 }
4885 
4886 /* `inline' is required to avoid gcc 4.1.2 build error */
wp_huge_pmd(struct vm_fault * vmf)4887 static inline vm_fault_t wp_huge_pmd(struct vm_fault *vmf)
4888 {
4889 	struct vm_area_struct *vma = vmf->vma;
4890 	const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
4891 	vm_fault_t ret;
4892 
4893 	if (vma_is_anonymous(vma)) {
4894 		if (likely(!unshare) &&
4895 		    userfaultfd_huge_pmd_wp(vma, vmf->orig_pmd))
4896 			return handle_userfault(vmf, VM_UFFD_WP);
4897 		return do_huge_pmd_wp_page(vmf);
4898 	}
4899 
4900 	if (vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) {
4901 		if (vma->vm_ops->huge_fault) {
4902 			ret = vma->vm_ops->huge_fault(vmf, PMD_ORDER);
4903 			if (!(ret & VM_FAULT_FALLBACK))
4904 				return ret;
4905 		}
4906 	}
4907 
4908 	/* COW or write-notify handled on pte level: split pmd. */
4909 	__split_huge_pmd(vma, vmf->pmd, vmf->address, false, NULL);
4910 
4911 	return VM_FAULT_FALLBACK;
4912 }
4913 
create_huge_pud(struct vm_fault * vmf)4914 static vm_fault_t create_huge_pud(struct vm_fault *vmf)
4915 {
4916 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) &&			\
4917 	defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
4918 	struct vm_area_struct *vma = vmf->vma;
4919 	/* No support for anonymous transparent PUD pages yet */
4920 	if (vma_is_anonymous(vma))
4921 		return VM_FAULT_FALLBACK;
4922 	if (vma->vm_ops->huge_fault)
4923 		return vma->vm_ops->huge_fault(vmf, PUD_ORDER);
4924 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
4925 	return VM_FAULT_FALLBACK;
4926 }
4927 
wp_huge_pud(struct vm_fault * vmf,pud_t orig_pud)4928 static vm_fault_t wp_huge_pud(struct vm_fault *vmf, pud_t orig_pud)
4929 {
4930 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) &&			\
4931 	defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
4932 	struct vm_area_struct *vma = vmf->vma;
4933 	vm_fault_t ret;
4934 
4935 	/* No support for anonymous transparent PUD pages yet */
4936 	if (vma_is_anonymous(vma))
4937 		goto split;
4938 	if (vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) {
4939 		if (vma->vm_ops->huge_fault) {
4940 			ret = vma->vm_ops->huge_fault(vmf, PUD_ORDER);
4941 			if (!(ret & VM_FAULT_FALLBACK))
4942 				return ret;
4943 		}
4944 	}
4945 split:
4946 	/* COW or write-notify not handled on PUD level: split pud.*/
4947 	__split_huge_pud(vma, vmf->pud, vmf->address);
4948 #endif /* CONFIG_TRANSPARENT_HUGEPAGE && CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
4949 	return VM_FAULT_FALLBACK;
4950 }
4951 
4952 /*
4953  * These routines also need to handle stuff like marking pages dirty
4954  * and/or accessed for architectures that don't do it in hardware (most
4955  * RISC architectures).  The early dirtying is also good on the i386.
4956  *
4957  * There is also a hook called "update_mmu_cache()" that architectures
4958  * with external mmu caches can use to update those (ie the Sparc or
4959  * PowerPC hashed page tables that act as extended TLBs).
4960  *
4961  * We enter with non-exclusive mmap_lock (to exclude vma changes, but allow
4962  * concurrent faults).
4963  *
4964  * The mmap_lock may have been released depending on flags and our return value.
4965  * See filemap_fault() and __folio_lock_or_retry().
4966  */
handle_pte_fault(struct vm_fault * vmf)4967 static vm_fault_t handle_pte_fault(struct vm_fault *vmf)
4968 {
4969 	pte_t entry;
4970 
4971 	if (unlikely(pmd_none(*vmf->pmd))) {
4972 		/*
4973 		 * Leave __pte_alloc() until later: because vm_ops->fault may
4974 		 * want to allocate huge page, and if we expose page table
4975 		 * for an instant, it will be difficult to retract from
4976 		 * concurrent faults and from rmap lookups.
4977 		 */
4978 		vmf->pte = NULL;
4979 		vmf->flags &= ~FAULT_FLAG_ORIG_PTE_VALID;
4980 	} else {
4981 		/*
4982 		 * A regular pmd is established and it can't morph into a huge
4983 		 * pmd by anon khugepaged, since that takes mmap_lock in write
4984 		 * mode; but shmem or file collapse to THP could still morph
4985 		 * it into a huge pmd: just retry later if so.
4986 		 */
4987 		vmf->pte = pte_offset_map_nolock(vmf->vma->vm_mm, vmf->pmd,
4988 						 vmf->address, &vmf->ptl);
4989 		if (unlikely(!vmf->pte))
4990 			return 0;
4991 		vmf->orig_pte = ptep_get_lockless(vmf->pte);
4992 		vmf->flags |= FAULT_FLAG_ORIG_PTE_VALID;
4993 
4994 		if (pte_none(vmf->orig_pte)) {
4995 			pte_unmap(vmf->pte);
4996 			vmf->pte = NULL;
4997 		}
4998 	}
4999 
5000 	if (!vmf->pte)
5001 		return do_pte_missing(vmf);
5002 
5003 	if (!pte_present(vmf->orig_pte))
5004 		return do_swap_page(vmf);
5005 
5006 	if (pte_protnone(vmf->orig_pte) && vma_is_accessible(vmf->vma))
5007 		return do_numa_page(vmf);
5008 
5009 	spin_lock(vmf->ptl);
5010 	entry = vmf->orig_pte;
5011 	if (unlikely(!pte_same(ptep_get(vmf->pte), entry))) {
5012 		update_mmu_tlb(vmf->vma, vmf->address, vmf->pte);
5013 		goto unlock;
5014 	}
5015 	if (vmf->flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) {
5016 		if (!pte_write(entry))
5017 			return do_wp_page(vmf);
5018 		else if (likely(vmf->flags & FAULT_FLAG_WRITE))
5019 			entry = pte_mkdirty(entry);
5020 	}
5021 	entry = pte_mkyoung(entry);
5022 	if (ptep_set_access_flags(vmf->vma, vmf->address, vmf->pte, entry,
5023 				vmf->flags & FAULT_FLAG_WRITE)) {
5024 		update_mmu_cache_range(vmf, vmf->vma, vmf->address,
5025 				vmf->pte, 1);
5026 	} else {
5027 		/* Skip spurious TLB flush for retried page fault */
5028 		if (vmf->flags & FAULT_FLAG_TRIED)
5029 			goto unlock;
5030 		/*
5031 		 * This is needed only for protection faults but the arch code
5032 		 * is not yet telling us if this is a protection fault or not.
5033 		 * This still avoids useless tlb flushes for .text page faults
5034 		 * with threads.
5035 		 */
5036 		if (vmf->flags & FAULT_FLAG_WRITE)
5037 			flush_tlb_fix_spurious_fault(vmf->vma, vmf->address,
5038 						     vmf->pte);
5039 	}
5040 unlock:
5041 	pte_unmap_unlock(vmf->pte, vmf->ptl);
5042 	return 0;
5043 }
5044 
5045 /*
5046  * On entry, we hold either the VMA lock or the mmap_lock
5047  * (FAULT_FLAG_VMA_LOCK tells you which).  If VM_FAULT_RETRY is set in
5048  * the result, the mmap_lock is not held on exit.  See filemap_fault()
5049  * and __folio_lock_or_retry().
5050  */
__handle_mm_fault(struct vm_area_struct * vma,unsigned long address,unsigned int flags)5051 static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma,
5052 		unsigned long address, unsigned int flags)
5053 {
5054 	struct vm_fault vmf = {
5055 		.vma = vma,
5056 		.address = address & PAGE_MASK,
5057 		.real_address = address,
5058 		.flags = flags,
5059 		.pgoff = linear_page_index(vma, address),
5060 		.gfp_mask = __get_fault_gfp_mask(vma),
5061 	};
5062 	struct mm_struct *mm = vma->vm_mm;
5063 	unsigned long vm_flags = vma->vm_flags;
5064 	pgd_t *pgd;
5065 	p4d_t *p4d;
5066 	vm_fault_t ret;
5067 
5068 	pgd = pgd_offset(mm, address);
5069 	p4d = p4d_alloc(mm, pgd, address);
5070 	if (!p4d)
5071 		return VM_FAULT_OOM;
5072 
5073 	vmf.pud = pud_alloc(mm, p4d, address);
5074 	if (!vmf.pud)
5075 		return VM_FAULT_OOM;
5076 retry_pud:
5077 	if (pud_none(*vmf.pud) &&
5078 	    hugepage_vma_check(vma, vm_flags, false, true, true)) {
5079 		ret = create_huge_pud(&vmf);
5080 		if (!(ret & VM_FAULT_FALLBACK))
5081 			return ret;
5082 	} else {
5083 		pud_t orig_pud = *vmf.pud;
5084 
5085 		barrier();
5086 		if (pud_trans_huge(orig_pud) || pud_devmap(orig_pud)) {
5087 
5088 			/*
5089 			 * TODO once we support anonymous PUDs: NUMA case and
5090 			 * FAULT_FLAG_UNSHARE handling.
5091 			 */
5092 			if ((flags & FAULT_FLAG_WRITE) && !pud_write(orig_pud)) {
5093 				ret = wp_huge_pud(&vmf, orig_pud);
5094 				if (!(ret & VM_FAULT_FALLBACK))
5095 					return ret;
5096 			} else {
5097 				huge_pud_set_accessed(&vmf, orig_pud);
5098 				return 0;
5099 			}
5100 		}
5101 	}
5102 
5103 	vmf.pmd = pmd_alloc(mm, vmf.pud, address);
5104 	if (!vmf.pmd)
5105 		return VM_FAULT_OOM;
5106 
5107 	/* Huge pud page fault raced with pmd_alloc? */
5108 	if (pud_trans_unstable(vmf.pud))
5109 		goto retry_pud;
5110 
5111 	if (pmd_none(*vmf.pmd) &&
5112 	    hugepage_vma_check(vma, vm_flags, false, true, true)) {
5113 		ret = create_huge_pmd(&vmf);
5114 		if (!(ret & VM_FAULT_FALLBACK))
5115 			return ret;
5116 	} else {
5117 		vmf.orig_pmd = pmdp_get_lockless(vmf.pmd);
5118 
5119 		if (unlikely(is_swap_pmd(vmf.orig_pmd))) {
5120 			VM_BUG_ON(thp_migration_supported() &&
5121 					  !is_pmd_migration_entry(vmf.orig_pmd));
5122 			if (is_pmd_migration_entry(vmf.orig_pmd))
5123 				pmd_migration_entry_wait(mm, vmf.pmd);
5124 			return 0;
5125 		}
5126 		if (pmd_trans_huge(vmf.orig_pmd) || pmd_devmap(vmf.orig_pmd)) {
5127 			if (pmd_protnone(vmf.orig_pmd) && vma_is_accessible(vma))
5128 				return do_huge_pmd_numa_page(&vmf);
5129 
5130 			if ((flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) &&
5131 			    !pmd_write(vmf.orig_pmd)) {
5132 				ret = wp_huge_pmd(&vmf);
5133 				if (!(ret & VM_FAULT_FALLBACK))
5134 					return ret;
5135 			} else {
5136 				huge_pmd_set_accessed(&vmf);
5137 				return 0;
5138 			}
5139 		}
5140 	}
5141 
5142 	return handle_pte_fault(&vmf);
5143 }
5144 
5145 /**
5146  * mm_account_fault - Do page fault accounting
5147  * @mm: mm from which memcg should be extracted. It can be NULL.
5148  * @regs: the pt_regs struct pointer.  When set to NULL, will skip accounting
5149  *        of perf event counters, but we'll still do the per-task accounting to
5150  *        the task who triggered this page fault.
5151  * @address: the faulted address.
5152  * @flags: the fault flags.
5153  * @ret: the fault retcode.
5154  *
5155  * This will take care of most of the page fault accounting.  Meanwhile, it
5156  * will also include the PERF_COUNT_SW_PAGE_FAULTS_[MAJ|MIN] perf counter
5157  * updates.  However, note that the handling of PERF_COUNT_SW_PAGE_FAULTS should
5158  * still be in per-arch page fault handlers at the entry of page fault.
5159  */
mm_account_fault(struct mm_struct * mm,struct pt_regs * regs,unsigned long address,unsigned int flags,vm_fault_t ret)5160 static inline void mm_account_fault(struct mm_struct *mm, struct pt_regs *regs,
5161 				    unsigned long address, unsigned int flags,
5162 				    vm_fault_t ret)
5163 {
5164 	bool major;
5165 
5166 	/* Incomplete faults will be accounted upon completion. */
5167 	if (ret & VM_FAULT_RETRY)
5168 		return;
5169 
5170 	/*
5171 	 * To preserve the behavior of older kernels, PGFAULT counters record
5172 	 * both successful and failed faults, as opposed to perf counters,
5173 	 * which ignore failed cases.
5174 	 */
5175 	count_vm_event(PGFAULT);
5176 	count_memcg_event_mm(mm, PGFAULT);
5177 
5178 	/*
5179 	 * Do not account for unsuccessful faults (e.g. when the address wasn't
5180 	 * valid).  That includes arch_vma_access_permitted() failing before
5181 	 * reaching here. So this is not a "this many hardware page faults"
5182 	 * counter.  We should use the hw profiling for that.
5183 	 */
5184 	if (ret & VM_FAULT_ERROR)
5185 		return;
5186 
5187 	/*
5188 	 * We define the fault as a major fault when the final successful fault
5189 	 * is VM_FAULT_MAJOR, or if it retried (which implies that we couldn't
5190 	 * handle it immediately previously).
5191 	 */
5192 	major = (ret & VM_FAULT_MAJOR) || (flags & FAULT_FLAG_TRIED);
5193 
5194 	if (major)
5195 		current->maj_flt++;
5196 	else
5197 		current->min_flt++;
5198 
5199 	/*
5200 	 * If the fault is done for GUP, regs will be NULL.  We only do the
5201 	 * accounting for the per thread fault counters who triggered the
5202 	 * fault, and we skip the perf event updates.
5203 	 */
5204 	if (!regs)
5205 		return;
5206 
5207 	if (major)
5208 		perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, address);
5209 	else
5210 		perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, address);
5211 }
5212 
5213 #ifdef CONFIG_LRU_GEN
lru_gen_enter_fault(struct vm_area_struct * vma)5214 static void lru_gen_enter_fault(struct vm_area_struct *vma)
5215 {
5216 	/* the LRU algorithm only applies to accesses with recency */
5217 	current->in_lru_fault = vma_has_recency(vma);
5218 }
5219 
lru_gen_exit_fault(void)5220 static void lru_gen_exit_fault(void)
5221 {
5222 	current->in_lru_fault = false;
5223 }
5224 #else
lru_gen_enter_fault(struct vm_area_struct * vma)5225 static void lru_gen_enter_fault(struct vm_area_struct *vma)
5226 {
5227 }
5228 
lru_gen_exit_fault(void)5229 static void lru_gen_exit_fault(void)
5230 {
5231 }
5232 #endif /* CONFIG_LRU_GEN */
5233 
sanitize_fault_flags(struct vm_area_struct * vma,unsigned int * flags)5234 static vm_fault_t sanitize_fault_flags(struct vm_area_struct *vma,
5235 				       unsigned int *flags)
5236 {
5237 	if (unlikely(*flags & FAULT_FLAG_UNSHARE)) {
5238 		if (WARN_ON_ONCE(*flags & FAULT_FLAG_WRITE))
5239 			return VM_FAULT_SIGSEGV;
5240 		/*
5241 		 * FAULT_FLAG_UNSHARE only applies to COW mappings. Let's
5242 		 * just treat it like an ordinary read-fault otherwise.
5243 		 */
5244 		if (!is_cow_mapping(vma->vm_flags))
5245 			*flags &= ~FAULT_FLAG_UNSHARE;
5246 	} else if (*flags & FAULT_FLAG_WRITE) {
5247 		/* Write faults on read-only mappings are impossible ... */
5248 		if (WARN_ON_ONCE(!(vma->vm_flags & VM_MAYWRITE)))
5249 			return VM_FAULT_SIGSEGV;
5250 		/* ... and FOLL_FORCE only applies to COW mappings. */
5251 		if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE) &&
5252 				 !is_cow_mapping(vma->vm_flags)))
5253 			return VM_FAULT_SIGSEGV;
5254 	}
5255 #ifdef CONFIG_PER_VMA_LOCK
5256 	/*
5257 	 * Per-VMA locks can't be used with FAULT_FLAG_RETRY_NOWAIT because of
5258 	 * the assumption that lock is dropped on VM_FAULT_RETRY.
5259 	 */
5260 	if (WARN_ON_ONCE((*flags &
5261 			(FAULT_FLAG_VMA_LOCK | FAULT_FLAG_RETRY_NOWAIT)) ==
5262 			(FAULT_FLAG_VMA_LOCK | FAULT_FLAG_RETRY_NOWAIT)))
5263 		return VM_FAULT_SIGSEGV;
5264 #endif
5265 
5266 	return 0;
5267 }
5268 
5269 /*
5270  * By the time we get here, we already hold the mm semaphore
5271  *
5272  * The mmap_lock may have been released depending on flags and our
5273  * return value.  See filemap_fault() and __folio_lock_or_retry().
5274  */
handle_mm_fault(struct vm_area_struct * vma,unsigned long address,unsigned int flags,struct pt_regs * regs)5275 vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
5276 			   unsigned int flags, struct pt_regs *regs)
5277 {
5278 	/* If the fault handler drops the mmap_lock, vma may be freed */
5279 	struct mm_struct *mm = vma->vm_mm;
5280 	vm_fault_t ret;
5281 
5282 	__set_current_state(TASK_RUNNING);
5283 
5284 	ret = sanitize_fault_flags(vma, &flags);
5285 	if (ret)
5286 		goto out;
5287 
5288 	if (!arch_vma_access_permitted(vma, flags & FAULT_FLAG_WRITE,
5289 					    flags & FAULT_FLAG_INSTRUCTION,
5290 					    flags & FAULT_FLAG_REMOTE)) {
5291 		ret = VM_FAULT_SIGSEGV;
5292 		goto out;
5293 	}
5294 
5295 	/*
5296 	 * Enable the memcg OOM handling for faults triggered in user
5297 	 * space.  Kernel faults are handled more gracefully.
5298 	 */
5299 	if (flags & FAULT_FLAG_USER)
5300 		mem_cgroup_enter_user_fault();
5301 
5302 	lru_gen_enter_fault(vma);
5303 
5304 	if (unlikely(is_vm_hugetlb_page(vma)))
5305 		ret = hugetlb_fault(vma->vm_mm, vma, address, flags);
5306 	else
5307 		ret = __handle_mm_fault(vma, address, flags);
5308 
5309 	lru_gen_exit_fault();
5310 
5311 	if (flags & FAULT_FLAG_USER) {
5312 		mem_cgroup_exit_user_fault();
5313 		/*
5314 		 * The task may have entered a memcg OOM situation but
5315 		 * if the allocation error was handled gracefully (no
5316 		 * VM_FAULT_OOM), there is no need to kill anything.
5317 		 * Just clean up the OOM state peacefully.
5318 		 */
5319 		if (task_in_memcg_oom(current) && !(ret & VM_FAULT_OOM))
5320 			mem_cgroup_oom_synchronize(false);
5321 	}
5322 out:
5323 	mm_account_fault(mm, regs, address, flags, ret);
5324 
5325 	return ret;
5326 }
5327 EXPORT_SYMBOL_GPL(handle_mm_fault);
5328 
5329 #ifdef CONFIG_LOCK_MM_AND_FIND_VMA
5330 #include <linux/extable.h>
5331 
get_mmap_lock_carefully(struct mm_struct * mm,struct pt_regs * regs)5332 static inline bool get_mmap_lock_carefully(struct mm_struct *mm, struct pt_regs *regs)
5333 {
5334 	if (likely(mmap_read_trylock(mm)))
5335 		return true;
5336 
5337 	if (regs && !user_mode(regs)) {
5338 		unsigned long ip = exception_ip(regs);
5339 		if (!search_exception_tables(ip))
5340 			return false;
5341 	}
5342 
5343 	return !mmap_read_lock_killable(mm);
5344 }
5345 
mmap_upgrade_trylock(struct mm_struct * mm)5346 static inline bool mmap_upgrade_trylock(struct mm_struct *mm)
5347 {
5348 	/*
5349 	 * We don't have this operation yet.
5350 	 *
5351 	 * It should be easy enough to do: it's basically a
5352 	 *    atomic_long_try_cmpxchg_acquire()
5353 	 * from RWSEM_READER_BIAS -> RWSEM_WRITER_LOCKED, but
5354 	 * it also needs the proper lockdep magic etc.
5355 	 */
5356 	return false;
5357 }
5358 
upgrade_mmap_lock_carefully(struct mm_struct * mm,struct pt_regs * regs)5359 static inline bool upgrade_mmap_lock_carefully(struct mm_struct *mm, struct pt_regs *regs)
5360 {
5361 	mmap_read_unlock(mm);
5362 	if (regs && !user_mode(regs)) {
5363 		unsigned long ip = exception_ip(regs);
5364 		if (!search_exception_tables(ip))
5365 			return false;
5366 	}
5367 	return !mmap_write_lock_killable(mm);
5368 }
5369 
5370 /*
5371  * Helper for page fault handling.
5372  *
5373  * This is kind of equivalend to "mmap_read_lock()" followed
5374  * by "find_extend_vma()", except it's a lot more careful about
5375  * the locking (and will drop the lock on failure).
5376  *
5377  * For example, if we have a kernel bug that causes a page
5378  * fault, we don't want to just use mmap_read_lock() to get
5379  * the mm lock, because that would deadlock if the bug were
5380  * to happen while we're holding the mm lock for writing.
5381  *
5382  * So this checks the exception tables on kernel faults in
5383  * order to only do this all for instructions that are actually
5384  * expected to fault.
5385  *
5386  * We can also actually take the mm lock for writing if we
5387  * need to extend the vma, which helps the VM layer a lot.
5388  */
lock_mm_and_find_vma(struct mm_struct * mm,unsigned long addr,struct pt_regs * regs)5389 struct vm_area_struct *lock_mm_and_find_vma(struct mm_struct *mm,
5390 			unsigned long addr, struct pt_regs *regs)
5391 {
5392 	struct vm_area_struct *vma;
5393 
5394 	if (!get_mmap_lock_carefully(mm, regs))
5395 		return NULL;
5396 
5397 	vma = find_vma(mm, addr);
5398 	if (likely(vma && (vma->vm_start <= addr)))
5399 		return vma;
5400 
5401 	/*
5402 	 * Well, dang. We might still be successful, but only
5403 	 * if we can extend a vma to do so.
5404 	 */
5405 	if (!vma || !(vma->vm_flags & VM_GROWSDOWN)) {
5406 		mmap_read_unlock(mm);
5407 		return NULL;
5408 	}
5409 
5410 	/*
5411 	 * We can try to upgrade the mmap lock atomically,
5412 	 * in which case we can continue to use the vma
5413 	 * we already looked up.
5414 	 *
5415 	 * Otherwise we'll have to drop the mmap lock and
5416 	 * re-take it, and also look up the vma again,
5417 	 * re-checking it.
5418 	 */
5419 	if (!mmap_upgrade_trylock(mm)) {
5420 		if (!upgrade_mmap_lock_carefully(mm, regs))
5421 			return NULL;
5422 
5423 		vma = find_vma(mm, addr);
5424 		if (!vma)
5425 			goto fail;
5426 		if (vma->vm_start <= addr)
5427 			goto success;
5428 		if (!(vma->vm_flags & VM_GROWSDOWN))
5429 			goto fail;
5430 	}
5431 
5432 	if (expand_stack_locked(vma, addr))
5433 		goto fail;
5434 
5435 success:
5436 	mmap_write_downgrade(mm);
5437 	return vma;
5438 
5439 fail:
5440 	mmap_write_unlock(mm);
5441 	return NULL;
5442 }
5443 #endif
5444 
5445 #ifdef CONFIG_PER_VMA_LOCK
5446 /*
5447  * Lookup and lock a VMA under RCU protection. Returned VMA is guaranteed to be
5448  * stable and not isolated. If the VMA is not found or is being modified the
5449  * function returns NULL.
5450  */
lock_vma_under_rcu(struct mm_struct * mm,unsigned long address)5451 struct vm_area_struct *lock_vma_under_rcu(struct mm_struct *mm,
5452 					  unsigned long address)
5453 {
5454 	MA_STATE(mas, &mm->mm_mt, address, address);
5455 	struct vm_area_struct *vma;
5456 
5457 	rcu_read_lock();
5458 retry:
5459 	vma = mas_walk(&mas);
5460 	if (!vma)
5461 		goto inval;
5462 
5463 	if (!vma_start_read(vma))
5464 		goto inval;
5465 
5466 	/*
5467 	 * find_mergeable_anon_vma uses adjacent vmas which are not locked.
5468 	 * This check must happen after vma_start_read(); otherwise, a
5469 	 * concurrent mremap() with MREMAP_DONTUNMAP could dissociate the VMA
5470 	 * from its anon_vma.
5471 	 */
5472 	if (unlikely(vma_is_anonymous(vma) && !vma->anon_vma))
5473 		goto inval_end_read;
5474 
5475 	/* Check since vm_start/vm_end might change before we lock the VMA */
5476 	if (unlikely(address < vma->vm_start || address >= vma->vm_end))
5477 		goto inval_end_read;
5478 
5479 	/* Check if the VMA got isolated after we found it */
5480 	if (vma->detached) {
5481 		vma_end_read(vma);
5482 		count_vm_vma_lock_event(VMA_LOCK_MISS);
5483 		/* The area was replaced with another one */
5484 		goto retry;
5485 	}
5486 
5487 	rcu_read_unlock();
5488 	return vma;
5489 
5490 inval_end_read:
5491 	vma_end_read(vma);
5492 inval:
5493 	rcu_read_unlock();
5494 	count_vm_vma_lock_event(VMA_LOCK_ABORT);
5495 	return NULL;
5496 }
5497 #endif /* CONFIG_PER_VMA_LOCK */
5498 
5499 #ifndef __PAGETABLE_P4D_FOLDED
5500 /*
5501  * Allocate p4d page table.
5502  * We've already handled the fast-path in-line.
5503  */
__p4d_alloc(struct mm_struct * mm,pgd_t * pgd,unsigned long address)5504 int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
5505 {
5506 	p4d_t *new = p4d_alloc_one(mm, address);
5507 	if (!new)
5508 		return -ENOMEM;
5509 
5510 	spin_lock(&mm->page_table_lock);
5511 	if (pgd_present(*pgd)) {	/* Another has populated it */
5512 		p4d_free(mm, new);
5513 	} else {
5514 		smp_wmb(); /* See comment in pmd_install() */
5515 		pgd_populate(mm, pgd, new);
5516 	}
5517 	spin_unlock(&mm->page_table_lock);
5518 	return 0;
5519 }
5520 #endif /* __PAGETABLE_P4D_FOLDED */
5521 
5522 #ifndef __PAGETABLE_PUD_FOLDED
5523 /*
5524  * Allocate page upper directory.
5525  * We've already handled the fast-path in-line.
5526  */
__pud_alloc(struct mm_struct * mm,p4d_t * p4d,unsigned long address)5527 int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address)
5528 {
5529 	pud_t *new = pud_alloc_one(mm, address);
5530 	if (!new)
5531 		return -ENOMEM;
5532 
5533 	spin_lock(&mm->page_table_lock);
5534 	if (!p4d_present(*p4d)) {
5535 		mm_inc_nr_puds(mm);
5536 		smp_wmb(); /* See comment in pmd_install() */
5537 		p4d_populate(mm, p4d, new);
5538 	} else	/* Another has populated it */
5539 		pud_free(mm, new);
5540 	spin_unlock(&mm->page_table_lock);
5541 	return 0;
5542 }
5543 #endif /* __PAGETABLE_PUD_FOLDED */
5544 
5545 #ifndef __PAGETABLE_PMD_FOLDED
5546 /*
5547  * Allocate page middle directory.
5548  * We've already handled the fast-path in-line.
5549  */
__pmd_alloc(struct mm_struct * mm,pud_t * pud,unsigned long address)5550 int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
5551 {
5552 	spinlock_t *ptl;
5553 	pmd_t *new = pmd_alloc_one(mm, address);
5554 	if (!new)
5555 		return -ENOMEM;
5556 
5557 	ptl = pud_lock(mm, pud);
5558 	if (!pud_present(*pud)) {
5559 		mm_inc_nr_pmds(mm);
5560 		smp_wmb(); /* See comment in pmd_install() */
5561 		pud_populate(mm, pud, new);
5562 	} else {	/* Another has populated it */
5563 		pmd_free(mm, new);
5564 	}
5565 	spin_unlock(ptl);
5566 	return 0;
5567 }
5568 #endif /* __PAGETABLE_PMD_FOLDED */
5569 
5570 /**
5571  * follow_pte - look up PTE at a user virtual address
5572  * @mm: the mm_struct of the target address space
5573  * @address: user virtual address
5574  * @ptepp: location to store found PTE
5575  * @ptlp: location to store the lock for the PTE
5576  *
5577  * On a successful return, the pointer to the PTE is stored in @ptepp;
5578  * the corresponding lock is taken and its location is stored in @ptlp.
5579  * The contents of the PTE are only stable until @ptlp is released;
5580  * any further use, if any, must be protected against invalidation
5581  * with MMU notifiers.
5582  *
5583  * Only IO mappings and raw PFN mappings are allowed.  The mmap semaphore
5584  * should be taken for read.
5585  *
5586  * KVM uses this function.  While it is arguably less bad than ``follow_pfn``,
5587  * it is not a good general-purpose API.
5588  *
5589  * Return: zero on success, -ve otherwise.
5590  */
follow_pte(struct mm_struct * mm,unsigned long address,pte_t ** ptepp,spinlock_t ** ptlp)5591 int follow_pte(struct mm_struct *mm, unsigned long address,
5592 	       pte_t **ptepp, spinlock_t **ptlp)
5593 {
5594 	pgd_t *pgd;
5595 	p4d_t *p4d;
5596 	pud_t *pud;
5597 	pmd_t *pmd;
5598 	pte_t *ptep;
5599 
5600 	pgd = pgd_offset(mm, address);
5601 	if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
5602 		goto out;
5603 
5604 	p4d = p4d_offset(pgd, address);
5605 	if (p4d_none(*p4d) || unlikely(p4d_bad(*p4d)))
5606 		goto out;
5607 
5608 	pud = pud_offset(p4d, address);
5609 	if (pud_none(*pud) || unlikely(pud_bad(*pud)))
5610 		goto out;
5611 
5612 	pmd = pmd_offset(pud, address);
5613 	VM_BUG_ON(pmd_trans_huge(*pmd));
5614 
5615 	ptep = pte_offset_map_lock(mm, pmd, address, ptlp);
5616 	if (!ptep)
5617 		goto out;
5618 	if (!pte_present(ptep_get(ptep)))
5619 		goto unlock;
5620 	*ptepp = ptep;
5621 	return 0;
5622 unlock:
5623 	pte_unmap_unlock(ptep, *ptlp);
5624 out:
5625 	return -EINVAL;
5626 }
5627 EXPORT_SYMBOL_GPL(follow_pte);
5628 
5629 /**
5630  * follow_pfn - look up PFN at a user virtual address
5631  * @vma: memory mapping
5632  * @address: user virtual address
5633  * @pfn: location to store found PFN
5634  *
5635  * Only IO mappings and raw PFN mappings are allowed.
5636  *
5637  * This function does not allow the caller to read the permissions
5638  * of the PTE.  Do not use it.
5639  *
5640  * Return: zero and the pfn at @pfn on success, -ve otherwise.
5641  */
follow_pfn(struct vm_area_struct * vma,unsigned long address,unsigned long * pfn)5642 int follow_pfn(struct vm_area_struct *vma, unsigned long address,
5643 	unsigned long *pfn)
5644 {
5645 	int ret = -EINVAL;
5646 	spinlock_t *ptl;
5647 	pte_t *ptep;
5648 
5649 	if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
5650 		return ret;
5651 
5652 	ret = follow_pte(vma->vm_mm, address, &ptep, &ptl);
5653 	if (ret)
5654 		return ret;
5655 	*pfn = pte_pfn(ptep_get(ptep));
5656 	pte_unmap_unlock(ptep, ptl);
5657 	return 0;
5658 }
5659 EXPORT_SYMBOL(follow_pfn);
5660 
5661 #ifdef CONFIG_HAVE_IOREMAP_PROT
follow_phys(struct vm_area_struct * vma,unsigned long address,unsigned int flags,unsigned long * prot,resource_size_t * phys)5662 int follow_phys(struct vm_area_struct *vma,
5663 		unsigned long address, unsigned int flags,
5664 		unsigned long *prot, resource_size_t *phys)
5665 {
5666 	int ret = -EINVAL;
5667 	pte_t *ptep, pte;
5668 	spinlock_t *ptl;
5669 
5670 	if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
5671 		goto out;
5672 
5673 	if (follow_pte(vma->vm_mm, address, &ptep, &ptl))
5674 		goto out;
5675 	pte = ptep_get(ptep);
5676 
5677 	if ((flags & FOLL_WRITE) && !pte_write(pte))
5678 		goto unlock;
5679 
5680 	*prot = pgprot_val(pte_pgprot(pte));
5681 	*phys = (resource_size_t)pte_pfn(pte) << PAGE_SHIFT;
5682 
5683 	ret = 0;
5684 unlock:
5685 	pte_unmap_unlock(ptep, ptl);
5686 out:
5687 	return ret;
5688 }
5689 
5690 /**
5691  * generic_access_phys - generic implementation for iomem mmap access
5692  * @vma: the vma to access
5693  * @addr: userspace address, not relative offset within @vma
5694  * @buf: buffer to read/write
5695  * @len: length of transfer
5696  * @write: set to FOLL_WRITE when writing, otherwise reading
5697  *
5698  * This is a generic implementation for &vm_operations_struct.access for an
5699  * iomem mapping. This callback is used by access_process_vm() when the @vma is
5700  * not page based.
5701  */
generic_access_phys(struct vm_area_struct * vma,unsigned long addr,void * buf,int len,int write)5702 int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
5703 			void *buf, int len, int write)
5704 {
5705 	resource_size_t phys_addr;
5706 	unsigned long prot = 0;
5707 	void __iomem *maddr;
5708 	pte_t *ptep, pte;
5709 	spinlock_t *ptl;
5710 	int offset = offset_in_page(addr);
5711 	int ret = -EINVAL;
5712 
5713 	if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
5714 		return -EINVAL;
5715 
5716 retry:
5717 	if (follow_pte(vma->vm_mm, addr, &ptep, &ptl))
5718 		return -EINVAL;
5719 	pte = ptep_get(ptep);
5720 	pte_unmap_unlock(ptep, ptl);
5721 
5722 	prot = pgprot_val(pte_pgprot(pte));
5723 	phys_addr = (resource_size_t)pte_pfn(pte) << PAGE_SHIFT;
5724 
5725 	if ((write & FOLL_WRITE) && !pte_write(pte))
5726 		return -EINVAL;
5727 
5728 	maddr = ioremap_prot(phys_addr, PAGE_ALIGN(len + offset), prot);
5729 	if (!maddr)
5730 		return -ENOMEM;
5731 
5732 	if (follow_pte(vma->vm_mm, addr, &ptep, &ptl))
5733 		goto out_unmap;
5734 
5735 	if (!pte_same(pte, ptep_get(ptep))) {
5736 		pte_unmap_unlock(ptep, ptl);
5737 		iounmap(maddr);
5738 
5739 		goto retry;
5740 	}
5741 
5742 	if (write)
5743 		memcpy_toio(maddr + offset, buf, len);
5744 	else
5745 		memcpy_fromio(buf, maddr + offset, len);
5746 	ret = len;
5747 	pte_unmap_unlock(ptep, ptl);
5748 out_unmap:
5749 	iounmap(maddr);
5750 
5751 	return ret;
5752 }
5753 EXPORT_SYMBOL_GPL(generic_access_phys);
5754 #endif
5755 
5756 /*
5757  * Access another process' address space as given in mm.
5758  */
__access_remote_vm(struct mm_struct * mm,unsigned long addr,void * buf,int len,unsigned int gup_flags)5759 int __access_remote_vm(struct mm_struct *mm, unsigned long addr, void *buf,
5760 		       int len, unsigned int gup_flags)
5761 {
5762 	void *old_buf = buf;
5763 	int write = gup_flags & FOLL_WRITE;
5764 
5765 	if (mmap_read_lock_killable(mm))
5766 		return 0;
5767 
5768 	/* Untag the address before looking up the VMA */
5769 	addr = untagged_addr_remote(mm, addr);
5770 
5771 	/* Avoid triggering the temporary warning in __get_user_pages */
5772 	if (!vma_lookup(mm, addr) && !expand_stack(mm, addr))
5773 		return 0;
5774 
5775 	/* ignore errors, just check how much was successfully transferred */
5776 	while (len) {
5777 		int bytes, offset;
5778 		void *maddr;
5779 		struct vm_area_struct *vma = NULL;
5780 		struct page *page = get_user_page_vma_remote(mm, addr,
5781 							     gup_flags, &vma);
5782 
5783 		if (IS_ERR_OR_NULL(page)) {
5784 			/* We might need to expand the stack to access it */
5785 			vma = vma_lookup(mm, addr);
5786 			if (!vma) {
5787 				vma = expand_stack(mm, addr);
5788 
5789 				/* mmap_lock was dropped on failure */
5790 				if (!vma)
5791 					return buf - old_buf;
5792 
5793 				/* Try again if stack expansion worked */
5794 				continue;
5795 			}
5796 
5797 
5798 			/*
5799 			 * Check if this is a VM_IO | VM_PFNMAP VMA, which
5800 			 * we can access using slightly different code.
5801 			 */
5802 			bytes = 0;
5803 #ifdef CONFIG_HAVE_IOREMAP_PROT
5804 			if (vma->vm_ops && vma->vm_ops->access)
5805 				bytes = vma->vm_ops->access(vma, addr, buf,
5806 							    len, write);
5807 #endif
5808 			if (bytes <= 0)
5809 				break;
5810 		} else {
5811 			bytes = len;
5812 			offset = addr & (PAGE_SIZE-1);
5813 			if (bytes > PAGE_SIZE-offset)
5814 				bytes = PAGE_SIZE-offset;
5815 
5816 			maddr = kmap(page);
5817 			if (write) {
5818 				copy_to_user_page(vma, page, addr,
5819 						  maddr + offset, buf, bytes);
5820 				set_page_dirty_lock(page);
5821 			} else {
5822 				copy_from_user_page(vma, page, addr,
5823 						    buf, maddr + offset, bytes);
5824 			}
5825 			kunmap(page);
5826 			put_page(page);
5827 		}
5828 		len -= bytes;
5829 		buf += bytes;
5830 		addr += bytes;
5831 	}
5832 	mmap_read_unlock(mm);
5833 
5834 	return buf - old_buf;
5835 }
5836 
5837 /**
5838  * access_remote_vm - access another process' address space
5839  * @mm:		the mm_struct of the target address space
5840  * @addr:	start address to access
5841  * @buf:	source or destination buffer
5842  * @len:	number of bytes to transfer
5843  * @gup_flags:	flags modifying lookup behaviour
5844  *
5845  * The caller must hold a reference on @mm.
5846  *
5847  * Return: number of bytes copied from source to destination.
5848  */
access_remote_vm(struct mm_struct * mm,unsigned long addr,void * buf,int len,unsigned int gup_flags)5849 int access_remote_vm(struct mm_struct *mm, unsigned long addr,
5850 		void *buf, int len, unsigned int gup_flags)
5851 {
5852 	return __access_remote_vm(mm, addr, buf, len, gup_flags);
5853 }
5854 
5855 /*
5856  * Access another process' address space.
5857  * Source/target buffer must be kernel space,
5858  * Do not walk the page table directly, use get_user_pages
5859  */
access_process_vm(struct task_struct * tsk,unsigned long addr,void * buf,int len,unsigned int gup_flags)5860 int access_process_vm(struct task_struct *tsk, unsigned long addr,
5861 		void *buf, int len, unsigned int gup_flags)
5862 {
5863 	struct mm_struct *mm;
5864 	int ret;
5865 
5866 	mm = get_task_mm(tsk);
5867 	if (!mm)
5868 		return 0;
5869 
5870 	ret = __access_remote_vm(mm, addr, buf, len, gup_flags);
5871 
5872 	mmput(mm);
5873 
5874 	return ret;
5875 }
5876 EXPORT_SYMBOL_GPL(access_process_vm);
5877 
5878 /*
5879  * Print the name of a VMA.
5880  */
print_vma_addr(char * prefix,unsigned long ip)5881 void print_vma_addr(char *prefix, unsigned long ip)
5882 {
5883 	struct mm_struct *mm = current->mm;
5884 	struct vm_area_struct *vma;
5885 
5886 	/*
5887 	 * we might be running from an atomic context so we cannot sleep
5888 	 */
5889 	if (!mmap_read_trylock(mm))
5890 		return;
5891 
5892 	vma = find_vma(mm, ip);
5893 	if (vma && vma->vm_file) {
5894 		struct file *f = vma->vm_file;
5895 		char *buf = (char *)__get_free_page(GFP_NOWAIT);
5896 		if (buf) {
5897 			char *p;
5898 
5899 			p = file_path(f, buf, PAGE_SIZE);
5900 			if (IS_ERR(p))
5901 				p = "?";
5902 			printk("%s%s[%lx+%lx]", prefix, kbasename(p),
5903 					vma->vm_start,
5904 					vma->vm_end - vma->vm_start);
5905 			free_page((unsigned long)buf);
5906 		}
5907 	}
5908 	mmap_read_unlock(mm);
5909 }
5910 
5911 #if defined(CONFIG_PROVE_LOCKING) || defined(CONFIG_DEBUG_ATOMIC_SLEEP)
__might_fault(const char * file,int line)5912 void __might_fault(const char *file, int line)
5913 {
5914 	if (pagefault_disabled())
5915 		return;
5916 	__might_sleep(file, line);
5917 #if defined(CONFIG_DEBUG_ATOMIC_SLEEP)
5918 	if (current->mm)
5919 		might_lock_read(&current->mm->mmap_lock);
5920 #endif
5921 }
5922 EXPORT_SYMBOL(__might_fault);
5923 #endif
5924 
5925 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS)
5926 /*
5927  * Process all subpages of the specified huge page with the specified
5928  * operation.  The target subpage will be processed last to keep its
5929  * cache lines hot.
5930  */
process_huge_page(unsigned long addr_hint,unsigned int pages_per_huge_page,int (* process_subpage)(unsigned long addr,int idx,void * arg),void * arg)5931 static inline int process_huge_page(
5932 	unsigned long addr_hint, unsigned int pages_per_huge_page,
5933 	int (*process_subpage)(unsigned long addr, int idx, void *arg),
5934 	void *arg)
5935 {
5936 	int i, n, base, l, ret;
5937 	unsigned long addr = addr_hint &
5938 		~(((unsigned long)pages_per_huge_page << PAGE_SHIFT) - 1);
5939 
5940 	/* Process target subpage last to keep its cache lines hot */
5941 	might_sleep();
5942 	n = (addr_hint - addr) / PAGE_SIZE;
5943 	if (2 * n <= pages_per_huge_page) {
5944 		/* If target subpage in first half of huge page */
5945 		base = 0;
5946 		l = n;
5947 		/* Process subpages at the end of huge page */
5948 		for (i = pages_per_huge_page - 1; i >= 2 * n; i--) {
5949 			cond_resched();
5950 			ret = process_subpage(addr + i * PAGE_SIZE, i, arg);
5951 			if (ret)
5952 				return ret;
5953 		}
5954 	} else {
5955 		/* If target subpage in second half of huge page */
5956 		base = pages_per_huge_page - 2 * (pages_per_huge_page - n);
5957 		l = pages_per_huge_page - n;
5958 		/* Process subpages at the begin of huge page */
5959 		for (i = 0; i < base; i++) {
5960 			cond_resched();
5961 			ret = process_subpage(addr + i * PAGE_SIZE, i, arg);
5962 			if (ret)
5963 				return ret;
5964 		}
5965 	}
5966 	/*
5967 	 * Process remaining subpages in left-right-left-right pattern
5968 	 * towards the target subpage
5969 	 */
5970 	for (i = 0; i < l; i++) {
5971 		int left_idx = base + i;
5972 		int right_idx = base + 2 * l - 1 - i;
5973 
5974 		cond_resched();
5975 		ret = process_subpage(addr + left_idx * PAGE_SIZE, left_idx, arg);
5976 		if (ret)
5977 			return ret;
5978 		cond_resched();
5979 		ret = process_subpage(addr + right_idx * PAGE_SIZE, right_idx, arg);
5980 		if (ret)
5981 			return ret;
5982 	}
5983 	return 0;
5984 }
5985 
clear_gigantic_page(struct page * page,unsigned long addr,unsigned int pages_per_huge_page)5986 static void clear_gigantic_page(struct page *page,
5987 				unsigned long addr,
5988 				unsigned int pages_per_huge_page)
5989 {
5990 	int i;
5991 	struct page *p;
5992 
5993 	might_sleep();
5994 	for (i = 0; i < pages_per_huge_page; i++) {
5995 		p = nth_page(page, i);
5996 		cond_resched();
5997 		clear_user_highpage(p, addr + i * PAGE_SIZE);
5998 	}
5999 }
6000 
clear_subpage(unsigned long addr,int idx,void * arg)6001 static int clear_subpage(unsigned long addr, int idx, void *arg)
6002 {
6003 	struct page *page = arg;
6004 
6005 	clear_user_highpage(page + idx, addr);
6006 	return 0;
6007 }
6008 
clear_huge_page(struct page * page,unsigned long addr_hint,unsigned int pages_per_huge_page)6009 void clear_huge_page(struct page *page,
6010 		     unsigned long addr_hint, unsigned int pages_per_huge_page)
6011 {
6012 	unsigned long addr = addr_hint &
6013 		~(((unsigned long)pages_per_huge_page << PAGE_SHIFT) - 1);
6014 
6015 	if (unlikely(pages_per_huge_page > MAX_ORDER_NR_PAGES)) {
6016 		clear_gigantic_page(page, addr, pages_per_huge_page);
6017 		return;
6018 	}
6019 
6020 	process_huge_page(addr_hint, pages_per_huge_page, clear_subpage, page);
6021 }
6022 
copy_user_gigantic_page(struct folio * dst,struct folio * src,unsigned long addr,struct vm_area_struct * vma,unsigned int pages_per_huge_page)6023 static int copy_user_gigantic_page(struct folio *dst, struct folio *src,
6024 				     unsigned long addr,
6025 				     struct vm_area_struct *vma,
6026 				     unsigned int pages_per_huge_page)
6027 {
6028 	int i;
6029 	struct page *dst_page;
6030 	struct page *src_page;
6031 
6032 	for (i = 0; i < pages_per_huge_page; i++) {
6033 		dst_page = folio_page(dst, i);
6034 		src_page = folio_page(src, i);
6035 
6036 		cond_resched();
6037 		if (copy_mc_user_highpage(dst_page, src_page,
6038 					  addr + i*PAGE_SIZE, vma)) {
6039 			memory_failure_queue(page_to_pfn(src_page), 0);
6040 			return -EHWPOISON;
6041 		}
6042 	}
6043 	return 0;
6044 }
6045 
6046 struct copy_subpage_arg {
6047 	struct page *dst;
6048 	struct page *src;
6049 	struct vm_area_struct *vma;
6050 };
6051 
copy_subpage(unsigned long addr,int idx,void * arg)6052 static int copy_subpage(unsigned long addr, int idx, void *arg)
6053 {
6054 	struct copy_subpage_arg *copy_arg = arg;
6055 
6056 	if (copy_mc_user_highpage(copy_arg->dst + idx, copy_arg->src + idx,
6057 				  addr, copy_arg->vma)) {
6058 		memory_failure_queue(page_to_pfn(copy_arg->src + idx), 0);
6059 		return -EHWPOISON;
6060 	}
6061 	return 0;
6062 }
6063 
copy_user_large_folio(struct folio * dst,struct folio * src,unsigned long addr_hint,struct vm_area_struct * vma)6064 int copy_user_large_folio(struct folio *dst, struct folio *src,
6065 			  unsigned long addr_hint, struct vm_area_struct *vma)
6066 {
6067 	unsigned int pages_per_huge_page = folio_nr_pages(dst);
6068 	unsigned long addr = addr_hint &
6069 		~(((unsigned long)pages_per_huge_page << PAGE_SHIFT) - 1);
6070 	struct copy_subpage_arg arg = {
6071 		.dst = &dst->page,
6072 		.src = &src->page,
6073 		.vma = vma,
6074 	};
6075 
6076 	if (unlikely(pages_per_huge_page > MAX_ORDER_NR_PAGES))
6077 		return copy_user_gigantic_page(dst, src, addr, vma,
6078 					       pages_per_huge_page);
6079 
6080 	return process_huge_page(addr_hint, pages_per_huge_page, copy_subpage, &arg);
6081 }
6082 
copy_folio_from_user(struct folio * dst_folio,const void __user * usr_src,bool allow_pagefault)6083 long copy_folio_from_user(struct folio *dst_folio,
6084 			   const void __user *usr_src,
6085 			   bool allow_pagefault)
6086 {
6087 	void *kaddr;
6088 	unsigned long i, rc = 0;
6089 	unsigned int nr_pages = folio_nr_pages(dst_folio);
6090 	unsigned long ret_val = nr_pages * PAGE_SIZE;
6091 	struct page *subpage;
6092 
6093 	for (i = 0; i < nr_pages; i++) {
6094 		subpage = folio_page(dst_folio, i);
6095 		kaddr = kmap_local_page(subpage);
6096 		if (!allow_pagefault)
6097 			pagefault_disable();
6098 		rc = copy_from_user(kaddr, usr_src + i * PAGE_SIZE, PAGE_SIZE);
6099 		if (!allow_pagefault)
6100 			pagefault_enable();
6101 		kunmap_local(kaddr);
6102 
6103 		ret_val -= (PAGE_SIZE - rc);
6104 		if (rc)
6105 			break;
6106 
6107 		flush_dcache_page(subpage);
6108 
6109 		cond_resched();
6110 	}
6111 	return ret_val;
6112 }
6113 #endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */
6114 
6115 #if USE_SPLIT_PTE_PTLOCKS && ALLOC_SPLIT_PTLOCKS
6116 
6117 static struct kmem_cache *page_ptl_cachep;
6118 
ptlock_cache_init(void)6119 void __init ptlock_cache_init(void)
6120 {
6121 	page_ptl_cachep = kmem_cache_create("page->ptl", sizeof(spinlock_t), 0,
6122 			SLAB_PANIC, NULL);
6123 }
6124 
ptlock_alloc(struct ptdesc * ptdesc)6125 bool ptlock_alloc(struct ptdesc *ptdesc)
6126 {
6127 	spinlock_t *ptl;
6128 
6129 	ptl = kmem_cache_alloc(page_ptl_cachep, GFP_KERNEL);
6130 	if (!ptl)
6131 		return false;
6132 	ptdesc->ptl = ptl;
6133 	return true;
6134 }
6135 
ptlock_free(struct ptdesc * ptdesc)6136 void ptlock_free(struct ptdesc *ptdesc)
6137 {
6138 	kmem_cache_free(page_ptl_cachep, ptdesc->ptl);
6139 }
6140 #endif
6141