Lines Matching refs:vma
72 static pud_t *alloc_new_pud(struct mm_struct *mm, struct vm_area_struct *vma, in alloc_new_pud() argument
86 static pmd_t *alloc_new_pmd(struct mm_struct *mm, struct vm_area_struct *vma, in alloc_new_pmd() argument
92 pud = alloc_new_pud(mm, vma, addr); in alloc_new_pmd()
105 static void take_rmap_locks(struct vm_area_struct *vma) in take_rmap_locks() argument
107 if (vma->vm_file) in take_rmap_locks()
108 i_mmap_lock_write(vma->vm_file->f_mapping); in take_rmap_locks()
109 if (vma->anon_vma) in take_rmap_locks()
110 anon_vma_lock_write(vma->anon_vma); in take_rmap_locks()
113 static void drop_rmap_locks(struct vm_area_struct *vma) in drop_rmap_locks() argument
115 if (vma->anon_vma) in drop_rmap_locks()
116 anon_vma_unlock_write(vma->anon_vma); in drop_rmap_locks()
117 if (vma->vm_file) in drop_rmap_locks()
118 i_mmap_unlock_write(vma->vm_file->f_mapping); in drop_rmap_locks()
136 static void move_ptes(struct vm_area_struct *vma, pmd_t *old_pmd, in move_ptes() argument
141 struct mm_struct *mm = vma->vm_mm; in move_ptes()
166 take_rmap_locks(vma); in move_ptes()
177 flush_tlb_batched_pending(vma->vm_mm); in move_ptes()
206 flush_tlb_range(vma, old_end - len, old_end); in move_ptes()
212 drop_rmap_locks(vma); in move_ptes()
225 static bool move_normal_pmd(struct vm_area_struct *vma, unsigned long old_addr, in move_normal_pmd() argument
229 struct mm_struct *mm = vma->vm_mm; in move_normal_pmd()
264 old_ptl = pmd_lock(vma->vm_mm, old_pmd); in move_normal_pmd()
276 flush_tlb_range(vma, old_addr, old_addr + PMD_SIZE); in move_normal_pmd()
284 static inline bool move_normal_pmd(struct vm_area_struct *vma, in move_normal_pmd() argument
293 static bool move_normal_pud(struct vm_area_struct *vma, unsigned long old_addr, in move_normal_pud() argument
297 struct mm_struct *mm = vma->vm_mm; in move_normal_pud()
313 old_ptl = pud_lock(vma->vm_mm, old_pud); in move_normal_pud()
325 flush_tlb_range(vma, old_addr, old_addr + PUD_SIZE); in move_normal_pud()
333 static inline bool move_normal_pud(struct vm_area_struct *vma, in move_normal_pud() argument
342 static bool move_huge_pud(struct vm_area_struct *vma, unsigned long old_addr, in move_huge_pud() argument
346 struct mm_struct *mm = vma->vm_mm; in move_huge_pud()
360 old_ptl = pud_lock(vma->vm_mm, old_pud); in move_huge_pud()
374 flush_pud_tlb_range(vma, old_addr, old_addr + HPAGE_PUD_SIZE); in move_huge_pud()
382 static bool move_huge_pud(struct vm_area_struct *vma, unsigned long old_addr, in move_huge_pud() argument
440 static bool move_pgt_entry(enum pgt_entry entry, struct vm_area_struct *vma, in move_pgt_entry() argument
448 take_rmap_locks(vma); in move_pgt_entry()
452 moved = move_normal_pmd(vma, old_addr, new_addr, old_entry, in move_pgt_entry()
456 moved = move_normal_pud(vma, old_addr, new_addr, old_entry, in move_pgt_entry()
461 move_huge_pmd(vma, old_addr, new_addr, old_entry, in move_pgt_entry()
466 move_huge_pud(vma, old_addr, new_addr, old_entry, in move_pgt_entry()
476 drop_rmap_locks(vma); in move_pgt_entry()
481 unsigned long move_page_tables(struct vm_area_struct *vma, in move_page_tables() argument
496 if (is_vm_hugetlb_page(vma)) in move_page_tables()
497 return move_hugetlb_page_tables(vma, new_vma, old_addr, in move_page_tables()
500 flush_cache_range(vma, old_addr, old_end); in move_page_tables()
501 mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma, vma->vm_mm, in move_page_tables()
513 old_pud = get_old_pud(vma->vm_mm, old_addr); in move_page_tables()
516 new_pud = alloc_new_pud(vma->vm_mm, vma, new_addr); in move_page_tables()
521 move_pgt_entry(HPAGE_PUD, vma, old_addr, new_addr, in move_page_tables()
528 if (move_pgt_entry(NORMAL_PUD, vma, old_addr, new_addr, in move_page_tables()
534 old_pmd = get_old_pmd(vma->vm_mm, old_addr); in move_page_tables()
537 new_pmd = alloc_new_pmd(vma->vm_mm, vma, new_addr); in move_page_tables()
543 move_pgt_entry(HPAGE_PMD, vma, old_addr, new_addr, in move_page_tables()
546 split_huge_pmd(vma, old_pmd, old_addr); in move_page_tables()
555 if (move_pgt_entry(NORMAL_PMD, vma, old_addr, new_addr, in move_page_tables()
562 move_ptes(vma, old_pmd, old_addr, old_addr + extent, new_vma, in move_page_tables()
571 static unsigned long move_vma(struct vm_area_struct *vma, in move_vma() argument
578 struct mm_struct *mm = vma->vm_mm; in move_vma()
580 unsigned long vm_flags = vma->vm_flags; in move_vma()
599 if (vma->vm_ops && vma->vm_ops->may_split) { in move_vma()
600 if (vma->vm_start != old_addr) in move_vma()
601 err = vma->vm_ops->may_split(vma, old_addr); in move_vma()
602 if (!err && vma->vm_end != old_addr + old_len) in move_vma()
603 err = vma->vm_ops->may_split(vma, old_addr + old_len); in move_vma()
615 err = ksm_madvise(vma, old_addr, old_addr + old_len, in move_vma()
625 new_pgoff = vma->vm_pgoff + ((old_addr - vma->vm_start) >> PAGE_SHIFT); in move_vma()
626 new_vma = copy_vma(&vma, new_addr, new_len, new_pgoff, in move_vma()
634 moved_len = move_page_tables(vma, old_addr, new_vma, new_addr, old_len, in move_vma()
638 } else if (vma->vm_ops && vma->vm_ops->mremap) { in move_vma()
639 err = vma->vm_ops->mremap(new_vma); in move_vma()
648 move_page_tables(new_vma, new_addr, vma, old_addr, moved_len, in move_vma()
650 vma = new_vma; in move_vma()
658 if (is_vm_hugetlb_page(vma)) { in move_vma()
659 clear_vma_resv_huge_pages(vma); in move_vma()
664 vma->vm_flags &= ~VM_ACCOUNT; in move_vma()
665 excess = vma->vm_end - vma->vm_start - old_len; in move_vma()
666 if (old_addr > vma->vm_start && in move_vma()
667 old_addr + old_len < vma->vm_end) in move_vma()
681 vm_stat_account(mm, vma->vm_flags, new_len >> PAGE_SHIFT); in move_vma()
684 if (unlikely(vma->vm_flags & VM_PFNMAP)) in move_vma()
685 untrack_pfn_moved(vma); in move_vma()
689 vma->vm_flags &= VM_LOCKED_CLEAR_MASK; in move_vma()
695 if (new_vma != vma && vma->vm_start == old_addr && in move_vma()
696 vma->vm_end == (old_addr + old_len)) in move_vma()
697 unlink_anon_vmas(vma); in move_vma()
719 vma->vm_flags |= VM_ACCOUNT; in move_vma()
721 find_vma(mm, vma->vm_end)->vm_flags |= VM_ACCOUNT; in move_vma()
731 struct vm_area_struct *vma; in vma_to_resize() local
734 vma = vma_lookup(mm, addr); in vma_to_resize()
735 if (!vma) in vma_to_resize()
746 if (!old_len && !(vma->vm_flags & (VM_SHARED | VM_MAYSHARE))) { in vma_to_resize()
752 (vma->vm_flags & (VM_DONTEXPAND | VM_PFNMAP))) in vma_to_resize()
756 if (old_len > vma->vm_end - addr) in vma_to_resize()
760 return vma; in vma_to_resize()
763 pgoff = (addr - vma->vm_start) >> PAGE_SHIFT; in vma_to_resize()
764 pgoff += vma->vm_pgoff; in vma_to_resize()
768 if (vma->vm_flags & (VM_DONTEXPAND | VM_PFNMAP)) in vma_to_resize()
771 if (mlock_future_check(mm, vma->vm_flags, new_len - old_len)) in vma_to_resize()
774 if (!may_expand_vm(mm, vma->vm_flags, in vma_to_resize()
778 return vma; in vma_to_resize()
788 struct vm_area_struct *vma; in mremap_to() local
832 vma = vma_to_resize(addr, old_len, new_len, flags); in mremap_to()
833 if (IS_ERR(vma)) { in mremap_to()
834 ret = PTR_ERR(vma); in mremap_to()
840 !may_expand_vm(mm, vma->vm_flags, old_len >> PAGE_SHIFT)) { in mremap_to()
848 if (vma->vm_flags & VM_MAYSHARE) in mremap_to()
851 ret = get_unmapped_area(vma->vm_file, new_addr, new_len, vma->vm_pgoff + in mremap_to()
852 ((addr - vma->vm_start) >> PAGE_SHIFT), in mremap_to()
861 ret = move_vma(vma, addr, old_len, new_len, new_addr, locked, flags, uf, in mremap_to()
868 static int vma_expandable(struct vm_area_struct *vma, unsigned long delta) in vma_expandable() argument
870 unsigned long end = vma->vm_end + delta; in vma_expandable()
872 if (end < vma->vm_end) /* overflow */ in vma_expandable()
874 if (find_vma_intersection(vma->vm_mm, vma->vm_end, end)) in vma_expandable()
876 if (get_unmapped_area(NULL, vma->vm_start, end - vma->vm_start, in vma_expandable()
894 struct vm_area_struct *vma; in SYSCALL_DEFINE5() local
945 vma = vma_lookup(mm, addr); in SYSCALL_DEFINE5()
946 if (!vma) { in SYSCALL_DEFINE5()
951 if (is_vm_hugetlb_page(vma)) { in SYSCALL_DEFINE5()
952 struct hstate *h __maybe_unused = hstate_vma(vma); in SYSCALL_DEFINE5()
1005 vma = vma_to_resize(addr, old_len, new_len, flags); in SYSCALL_DEFINE5()
1006 if (IS_ERR(vma)) { in SYSCALL_DEFINE5()
1007 ret = PTR_ERR(vma); in SYSCALL_DEFINE5()
1013 if (old_len == vma->vm_end - addr) { in SYSCALL_DEFINE5()
1015 if (vma_expandable(vma, new_len - old_len)) { in SYSCALL_DEFINE5()
1019 pgoff_t extension_pgoff = vma->vm_pgoff + in SYSCALL_DEFINE5()
1020 ((extension_start - vma->vm_start) >> PAGE_SHIFT); in SYSCALL_DEFINE5()
1022 if (vma->vm_flags & VM_ACCOUNT) { in SYSCALL_DEFINE5()
1036 vma = vma_merge(mm, vma, extension_start, extension_end, in SYSCALL_DEFINE5()
1037 vma->vm_flags, vma->anon_vma, vma->vm_file, in SYSCALL_DEFINE5()
1038 extension_pgoff, vma_policy(vma), in SYSCALL_DEFINE5()
1039 vma->vm_userfaultfd_ctx, anon_vma_name(vma)); in SYSCALL_DEFINE5()
1040 if (!vma) { in SYSCALL_DEFINE5()
1046 vm_stat_account(mm, vma->vm_flags, pages); in SYSCALL_DEFINE5()
1047 if (vma->vm_flags & VM_LOCKED) { in SYSCALL_DEFINE5()
1064 if (vma->vm_flags & VM_MAYSHARE) in SYSCALL_DEFINE5()
1067 new_addr = get_unmapped_area(vma->vm_file, 0, new_len, in SYSCALL_DEFINE5()
1068 vma->vm_pgoff + in SYSCALL_DEFINE5()
1069 ((addr - vma->vm_start) >> PAGE_SHIFT), in SYSCALL_DEFINE5()
1076 ret = move_vma(vma, addr, old_len, new_len, new_addr, in SYSCALL_DEFINE5()