42 static inline bool can_change_pte_writable(struct vm_area_struct *vma,  in can_change_pte_writable()  argument
47 	VM_BUG_ON(!(vma->vm_flags & VM_WRITE) || pte_write(pte));  in can_change_pte_writable()
53 	if (vma_soft_dirty_enabled(vma) && !pte_soft_dirty(pte))  in can_change_pte_writable()
57 	if (userfaultfd_pte_wp(vma, pte))  in can_change_pte_writable()
60 	if (!(vma->vm_flags & VM_SHARED)) {  in can_change_pte_writable()
67 		page = vm_normal_page(vma, addr, pte);  in can_change_pte_writable()
76 		struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr,  in change_pte_range()  argument
103 	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);  in change_pte_range()
106 	if (prot_numa && !(vma->vm_flags & VM_SHARED) &&  in change_pte_range()
107 	    atomic_read(&vma->vm_mm->mm_users) == 1)  in change_pte_range()
110 	flush_tlb_batched_pending(vma->vm_mm);  in change_pte_range()
131 				page = vm_normal_page(vma, addr, oldpte);  in change_pte_range()
136 				if (is_cow_mapping(vma->vm_flags) &&  in change_pte_range()
170 			oldpte = ptep_modify_prot_start(vma, addr, pte);  in change_pte_range()
197 			    can_change_pte_writable(vma, addr, ptent))  in change_pte_range()
200 			ptep_modify_prot_commit(vma, addr, pte, oldpte, ptent);  in change_pte_range()
250 					pte_clear(vma->vm_mm, addr, pte);  in change_pte_range()
264 				set_pte_at(vma->vm_mm, addr, pte, newpte);  in change_pte_range()
271 			if (unlikely(uffd_wp && !vma_is_anonymous(vma))) {  in change_pte_range()
278 				set_pte_at(vma->vm_mm, addr, pte,  in change_pte_range()
318 uffd_wp_protect_file(struct vm_area_struct *vma, unsigned long cp_flags)  in uffd_wp_protect_file()  argument
320 	return (cp_flags & MM_CP_UFFD_WP) && !vma_is_anonymous(vma);  in uffd_wp_protect_file()
328 #define  change_pmd_prepare(vma, pmd, cp_flags)				\  argument
330 		if (unlikely(uffd_wp_protect_file(vma, cp_flags))) {	\
331 			if (WARN_ON_ONCE(pte_alloc(vma->vm_mm, pmd)))	\
340 #define  change_prepare(vma, high, low, addr, cp_flags)			\  argument
342 		if (unlikely(uffd_wp_protect_file(vma, cp_flags))) {	\
343 			low##_t *p = low##_alloc(vma->vm_mm, high, addr); \
350 		struct vm_area_struct *vma, pud_t *pud, unsigned long addr,  in change_pmd_range()  argument
367 		change_pmd_prepare(vma, pmd, cp_flags);  in change_pmd_range()
384 				vma, vma->vm_mm, addr, end);  in change_pmd_range()
390 			    uffd_wp_protect_file(vma, cp_flags)) {  in change_pmd_range()
391 				__split_huge_pmd(vma, pmd, addr, false, NULL);  in change_pmd_range()
397 				change_pmd_prepare(vma, pmd, cp_flags);  in change_pmd_range()
403 				int nr_ptes = change_huge_pmd(tlb, vma, pmd,  in change_pmd_range()
418 		this_pages = change_pte_range(tlb, vma, pmd, addr, next,  in change_pmd_range()
434 		struct vm_area_struct *vma, p4d_t *p4d, unsigned long addr,  in change_pud_range()  argument
444 		change_prepare(vma, pud, pmd, addr, cp_flags);  in change_pud_range()
447 		pages += change_pmd_range(tlb, vma, pud, addr, next, newprot,  in change_pud_range()
455 		struct vm_area_struct *vma, pgd_t *pgd, unsigned long addr,  in change_p4d_range()  argument
465 		change_prepare(vma, p4d, pud, addr, cp_flags);  in change_p4d_range()
468 		pages += change_pud_range(tlb, vma, p4d, addr, next, newprot,  in change_p4d_range()
476 		struct vm_area_struct *vma, unsigned long addr,  in change_protection_range()  argument
479 	struct mm_struct *mm = vma->vm_mm;  in change_protection_range()
486 	tlb_start_vma(tlb, vma);  in change_protection_range()
489 		change_prepare(vma, pgd, p4d, addr, cp_flags);  in change_protection_range()
492 		pages += change_p4d_range(tlb, vma, pgd, addr, next, newprot,  in change_protection_range()
496 	tlb_end_vma(tlb, vma);  in change_protection_range()
502 		       struct vm_area_struct *vma, unsigned long start,  in change_protection()  argument
510 	if (is_vm_hugetlb_page(vma))  in change_protection()
511 		pages = hugetlb_change_protection(vma, start, end, newprot,  in change_protection()
514 		pages = change_protection_range(tlb, vma, start, end, newprot,  in change_protection()
548 mprotect_fixup(struct mmu_gather *tlb, struct vm_area_struct *vma,  in mprotect_fixup()  argument
552 	struct mm_struct *mm = vma->vm_mm;  in mprotect_fixup()
553 	unsigned long oldflags = vma->vm_flags;  in mprotect_fixup()
561 		*pprev = vma;  in mprotect_fixup()
571 	    (vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) &&  in mprotect_fixup()
604 	pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);  in mprotect_fixup()
606 			   vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma),  in mprotect_fixup()
607 			   vma->vm_userfaultfd_ctx, anon_vma_name(vma));  in mprotect_fixup()
609 		vma = *pprev;  in mprotect_fixup()
610 		VM_WARN_ON((vma->vm_flags ^ newflags) & ~VM_SOFTDIRTY);  in mprotect_fixup()
614 	*pprev = vma;  in mprotect_fixup()
616 	if (start != vma->vm_start) {  in mprotect_fixup()
617 		error = split_vma(mm, vma, start, 1);  in mprotect_fixup()
622 	if (end != vma->vm_end) {  in mprotect_fixup()
623 		error = split_vma(mm, vma, end, 0);  in mprotect_fixup()
633 	vma->vm_flags = newflags;  in mprotect_fixup()
640 	if (vma->vm_flags & VM_SHARED)  in mprotect_fixup()
641 		try_change_writable = vma_wants_writenotify(vma, vma->vm_page_prot);  in mprotect_fixup()
643 		try_change_writable = !!(vma->vm_flags & VM_WRITE);  in mprotect_fixup()
644 	vma_set_page_prot(vma);  in mprotect_fixup()
646 	change_protection(tlb, vma, start, end, vma->vm_page_prot,  in mprotect_fixup()
655 		populate_vma_page_range(vma, start, end, NULL);  in mprotect_fixup()
660 	perf_event_mmap(vma);  in mprotect_fixup()
675 	struct vm_area_struct *vma, *prev;  in do_mprotect_pkey()  local
714 	vma = mas_find(&mas, ULONG_MAX);  in do_mprotect_pkey()
716 	if (!vma)  in do_mprotect_pkey()
720 		if (vma->vm_start >= end)  in do_mprotect_pkey()
722 		start = vma->vm_start;  in do_mprotect_pkey()
724 		if (!(vma->vm_flags & VM_GROWSDOWN))  in do_mprotect_pkey()
727 		if (vma->vm_start > start)  in do_mprotect_pkey()
730 			end = vma->vm_end;  in do_mprotect_pkey()
732 			if (!(vma->vm_flags & VM_GROWSUP))  in do_mprotect_pkey()
737 	if (start > vma->vm_start)  in do_mprotect_pkey()
738 		prev = vma;  in do_mprotect_pkey()
751 		if (rier && (vma->vm_flags & VM_MAYEXEC))  in do_mprotect_pkey()
762 		new_vma_pkey = arch_override_mprotect_pkey(vma, prot, pkey);  in do_mprotect_pkey()
764 		newflags |= (vma->vm_flags & ~mask_off_old_flags);  in do_mprotect_pkey()
778 		error = security_file_mprotect(vma, reqprot, prot);  in do_mprotect_pkey()
782 		tmp = vma->vm_end;  in do_mprotect_pkey()
786 		if (vma->vm_ops && vma->vm_ops->mprotect) {  in do_mprotect_pkey()
787 			error = vma->vm_ops->mprotect(vma, nstart, tmp, newflags);  in do_mprotect_pkey()
792 		error = mprotect_fixup(&tlb, vma, &prev, nstart, tmp, newflags);  in do_mprotect_pkey()
803 		vma = find_vma(current->mm, prev->vm_end);  in do_mprotect_pkey()
804 		if (!vma || vma->vm_start != nstart) {  in do_mprotect_pkey()