/linux-6.1.9/arch/s390/mm/ |
D | pgtable.c | 48 pte_t *ptep, int nodat) in ptep_ipte_local() argument 61 __ptep_ipte(addr, ptep, opt, asce, IPTE_LOCAL); in ptep_ipte_local() 63 __ptep_ipte(addr, ptep, 0, 0, IPTE_LOCAL); in ptep_ipte_local() 68 pte_t *ptep, int nodat) in ptep_ipte_global() argument 81 __ptep_ipte(addr, ptep, opt, asce, IPTE_GLOBAL); in ptep_ipte_global() 83 __ptep_ipte(addr, ptep, 0, 0, IPTE_GLOBAL); in ptep_ipte_global() 88 unsigned long addr, pte_t *ptep, in ptep_flush_direct() argument 93 old = *ptep; in ptep_flush_direct() 99 ptep_ipte_local(mm, addr, ptep, nodat); in ptep_flush_direct() 101 ptep_ipte_global(mm, addr, ptep, nodat); in ptep_flush_direct() [all …]
|
/linux-6.1.9/arch/arm64/mm/ |
D | hugetlbpage.c | 104 pte_t *ptep, size_t *pgsize) in find_num_contig() argument 115 if ((pte_t *)pmdp == ptep) { in find_num_contig() 151 pte_t huge_ptep_get(pte_t *ptep) in huge_ptep_get() argument 155 pte_t orig_pte = ptep_get(ptep); in huge_ptep_get() 161 for (i = 0; i < ncontig; i++, ptep++) { in huge_ptep_get() 162 pte_t pte = ptep_get(ptep); in huge_ptep_get() 183 pte_t *ptep, in get_clear_contig() argument 187 pte_t orig_pte = ptep_get(ptep); in get_clear_contig() 190 for (i = 0; i < ncontig; i++, addr += pgsize, ptep++) { in get_clear_contig() 191 pte_t pte = ptep_get_and_clear(mm, addr, ptep); in get_clear_contig() [all …]
|
/linux-6.1.9/arch/s390/include/asm/ |
D | hugetlb.h | 19 pte_t *ptep, pte_t pte); 20 pte_t huge_ptep_get(pte_t *ptep); 22 unsigned long addr, pte_t *ptep); 47 pte_t *ptep, unsigned long sz) in huge_pte_clear() argument 49 if ((pte_val(*ptep) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) in huge_pte_clear() 50 set_pte(ptep, __pte(_REGION3_ENTRY_EMPTY)); in huge_pte_clear() 52 set_pte(ptep, __pte(_SEGMENT_ENTRY_EMPTY)); in huge_pte_clear() 56 unsigned long address, pte_t *ptep) in huge_ptep_clear_flush() argument 58 return huge_ptep_get_and_clear(vma->vm_mm, address, ptep); in huge_ptep_clear_flush() 62 unsigned long addr, pte_t *ptep, in huge_ptep_set_access_flags() argument [all …]
|
/linux-6.1.9/drivers/iommu/ |
D | io-pgtable-dart.c | 126 dart_iopte *ptep) in dart_init_pte() argument 133 if (ptep[i] & APPLE_DART_PTE_VALID) { in dart_init_pte() 135 WARN_ON(ptep[i] & APPLE_DART_PTE_VALID); in dart_init_pte() 147 ptep[i] = pte | paddr_to_iopte(paddr + i * sz, data); in dart_init_pte() 153 dart_iopte *ptep, in dart_install_table() argument 168 old = cmpxchg64_relaxed(ptep, curr, new); in dart_install_table() 195 dart_iopte pte, *ptep; in dart_get_l2() local 198 ptep = data->pgd[tbl]; in dart_get_l2() 199 if (!ptep) in dart_get_l2() 202 ptep += dart_get_l1_index(data, iova); in dart_get_l2() [all …]
|
/linux-6.1.9/arch/arm64/kvm/hyp/ |
D | pgtable.c | 140 static void kvm_clear_pte(kvm_pte_t *ptep) in kvm_clear_pte() argument 142 WRITE_ONCE(*ptep, 0); in kvm_clear_pte() 145 static void kvm_set_table_pte(kvm_pte_t *ptep, kvm_pte_t *childp, in kvm_set_table_pte() argument 148 kvm_pte_t old = *ptep, pte = kvm_phys_to_pte(mm_ops->virt_to_phys(childp)); in kvm_set_table_pte() 154 smp_store_release(ptep, pte); in kvm_set_table_pte() 176 u32 level, kvm_pte_t *ptep, in kvm_pgtable_visitor_cb() argument 180 return walker->cb(addr, data->end, level, ptep, flag, walker->arg); in kvm_pgtable_visitor_cb() 187 kvm_pte_t *ptep, u32 level) in __kvm_pgtable_visit() argument 191 kvm_pte_t *childp, pte = *ptep; in __kvm_pgtable_visit() 196 ret = kvm_pgtable_visitor_cb(data, addr, level, ptep, in __kvm_pgtable_visit() [all …]
|
/linux-6.1.9/tools/testing/selftests/kvm/lib/aarch64/ |
D | processor.c | 92 uint64_t *ptep; in _virt_pg_map() local 108 ptep = addr_gpa2hva(vm, vm->pgd) + pgd_index(vm, vaddr) * 8; in _virt_pg_map() 109 if (!*ptep) in _virt_pg_map() 110 *ptep = vm_alloc_page_table(vm) | 3; in _virt_pg_map() 114 ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pud_index(vm, vaddr) * 8; in _virt_pg_map() 115 if (!*ptep) in _virt_pg_map() 116 *ptep = vm_alloc_page_table(vm) | 3; in _virt_pg_map() 119 ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pmd_index(vm, vaddr) * 8; in _virt_pg_map() 120 if (!*ptep) in _virt_pg_map() 121 *ptep = vm_alloc_page_table(vm) | 3; in _virt_pg_map() [all …]
|
/linux-6.1.9/include/asm-generic/ |
D | hugetlb.h | 55 pte_t *ptep, unsigned long sz) in huge_pte_clear() argument 57 pte_clear(mm, addr, ptep); in huge_pte_clear() 72 pte_t *ptep, pte_t pte) in set_huge_pte_at() argument 74 set_pte_at(mm, addr, ptep, pte); in set_huge_pte_at() 80 unsigned long addr, pte_t *ptep) in huge_ptep_get_and_clear() argument 82 return ptep_get_and_clear(mm, addr, ptep); in huge_ptep_get_and_clear() 88 unsigned long addr, pte_t *ptep) in huge_ptep_clear_flush() argument 90 return ptep_clear_flush(vma, addr, ptep); in huge_ptep_clear_flush() 131 unsigned long addr, pte_t *ptep) in huge_ptep_set_wrprotect() argument 133 ptep_set_wrprotect(mm, addr, ptep); in huge_ptep_set_wrprotect() [all …]
|
/linux-6.1.9/arch/sparc/include/asm/ |
D | hugetlb.h | 17 pte_t *ptep, pte_t pte); 21 pte_t *ptep); 25 unsigned long addr, pte_t *ptep) in huge_ptep_clear_flush() argument 27 return *ptep; in huge_ptep_clear_flush() 32 unsigned long addr, pte_t *ptep) in huge_ptep_set_wrprotect() argument 34 pte_t old_pte = *ptep; in huge_ptep_set_wrprotect() 35 set_huge_pte_at(mm, addr, ptep, pte_wrprotect(old_pte)); in huge_ptep_set_wrprotect() 40 unsigned long addr, pte_t *ptep, in huge_ptep_set_access_flags() argument 43 int changed = !pte_same(*ptep, pte); in huge_ptep_set_access_flags() 45 set_huge_pte_at(vma->vm_mm, addr, ptep, pte); in huge_ptep_set_access_flags()
|
/linux-6.1.9/arch/xtensa/include/asm/ |
D | pgalloc.h | 24 #define pmd_populate_kernel(mm, pmdp, ptep) \ argument 25 (pmd_val(*(pmdp)) = ((unsigned long)ptep)) 35 static inline void ptes_clear(pte_t *ptep) in ptes_clear() argument 40 pte_clear(NULL, 0, ptep + i); in ptes_clear() 45 pte_t *ptep; in pte_alloc_one_kernel() local 47 ptep = (pte_t *)__pte_alloc_one_kernel(mm); in pte_alloc_one_kernel() 48 if (!ptep) in pte_alloc_one_kernel() 50 ptes_clear(ptep); in pte_alloc_one_kernel() 51 return ptep; in pte_alloc_one_kernel()
|
D | pgtable.h | 239 #define pte_clear(mm,addr,ptep) \ argument 240 do { update_pte(ptep, __pte(_PAGE_CA_INVALID | _PAGE_USER)); } while (0) 289 static inline void update_pte(pte_t *ptep, pte_t pteval) in update_pte() argument 291 *ptep = pteval; in update_pte() 293 __asm__ __volatile__ ("dhwb %0, 0" :: "a" (ptep)); in update_pte() 301 set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pteval) in set_pte_at() argument 303 update_pte(ptep, pteval); in set_pte_at() 306 static inline void set_pte(pte_t *ptep, pte_t pteval) in set_pte() argument 308 update_pte(ptep, pteval); in set_pte() 321 pte_t *ptep) in ptep_test_and_clear_young() argument [all …]
|
/linux-6.1.9/arch/riscv/kvm/ |
D | mmu.c | 100 pte_t *ptep; in gstage_get_leaf_entry() local 104 ptep = (pte_t *)kvm->arch.pgd; in gstage_get_leaf_entry() 105 ptep = &ptep[gstage_pte_index(addr, current_level)]; in gstage_get_leaf_entry() 106 while (ptep && pte_val(*ptep)) { in gstage_get_leaf_entry() 107 if (gstage_pte_leaf(ptep)) { in gstage_get_leaf_entry() 109 *ptepp = ptep; in gstage_get_leaf_entry() 116 ptep = (pte_t *)gstage_pte_page_vaddr(*ptep); in gstage_get_leaf_entry() 117 ptep = &ptep[gstage_pte_index(addr, current_level)]; in gstage_get_leaf_entry() 119 ptep = NULL; in gstage_get_leaf_entry() 143 pte_t *ptep = &next_ptep[gstage_pte_index(addr, current_level)]; in gstage_set_pte() local [all …]
|
/linux-6.1.9/arch/parisc/mm/ |
D | hugetlbpage.c | 123 pte_t *ptep, pte_t entry) in __set_huge_pte_at() argument 132 set_pte(ptep, entry); in __set_huge_pte_at() 133 ptep++; in __set_huge_pte_at() 143 pte_t *ptep, pte_t entry) in set_huge_pte_at() argument 145 __set_huge_pte_at(mm, addr, ptep, entry); in set_huge_pte_at() 150 pte_t *ptep) in huge_ptep_get_and_clear() argument 154 entry = *ptep; in huge_ptep_get_and_clear() 155 __set_huge_pte_at(mm, addr, ptep, __pte(0)); in huge_ptep_get_and_clear() 162 unsigned long addr, pte_t *ptep) in huge_ptep_set_wrprotect() argument 166 old_pte = *ptep; in huge_ptep_set_wrprotect() [all …]
|
/linux-6.1.9/arch/powerpc/include/asm/nohash/64/ |
D | pgtable.h | 176 pte_t *ptep, unsigned long clr, in pte_update() argument 180 unsigned long old = pte_val(*ptep); in pte_update() 181 *ptep = __pte((old & ~clr) | set); in pte_update() 196 unsigned long addr, pte_t *ptep) in __ptep_test_and_clear_young() argument 200 if (pte_young(*ptep)) in __ptep_test_and_clear_young() 202 old = pte_update(mm, addr, ptep, _PAGE_ACCESSED, 0, 0); in __ptep_test_and_clear_young() 215 pte_t *ptep) in ptep_set_wrprotect() argument 218 if ((pte_val(*ptep) & _PAGE_RW) == 0) in ptep_set_wrprotect() 221 pte_update(mm, addr, ptep, _PAGE_RW, 0, 0); in ptep_set_wrprotect() 226 unsigned long addr, pte_t *ptep) in huge_ptep_set_wrprotect() argument [all …]
|
/linux-6.1.9/arch/powerpc/mm/ |
D | pageattr.c | 18 static pte_basic_t pte_update_delta(pte_t *ptep, unsigned long addr, in pte_update_delta() argument 21 return pte_update(&init_mm, addr, ptep, old & ~new, new & ~old, 0); in pte_update_delta() 30 static int change_page_attr(pte_t *ptep, unsigned long addr, void *data) in change_page_attr() argument 39 pte_update_delta(ptep, addr, _PAGE_KERNEL_RW & ~_PAGE_DIRTY, _PAGE_KERNEL_RO); in change_page_attr() 42 pte_update_delta(ptep, addr, _PAGE_KERNEL_RO, _PAGE_KERNEL_RW); in change_page_attr() 45 pte_update_delta(ptep, addr, _PAGE_KERNEL_ROX, _PAGE_KERNEL_RO); in change_page_attr() 48 pte_update_delta(ptep, addr, _PAGE_KERNEL_RO, _PAGE_KERNEL_ROX); in change_page_attr() 51 pte_update(&init_mm, addr, ptep, _PAGE_PRESENT, 0, 0); in change_page_attr() 54 pte_update(&init_mm, addr, ptep, 0, _PAGE_PRESENT, 0); in change_page_attr()
|
/linux-6.1.9/arch/powerpc/mm/book3s64/ |
D | hugetlbpage.c | 21 pte_t *ptep, unsigned long trap, unsigned long flags, in __hash_page_huge() argument 49 old_pte = pte_val(*ptep); in __hash_page_huge() 64 } while(!pte_xchg(ptep, __pte(old_pte), __pte(new_pte))); in __hash_page_huge() 75 rpte = __real_pte(__pte(old_pte), ptep, offset); in __hash_page_huge() 111 *ptep = __pte(old_pte); in __hash_page_huge() 117 new_pte |= pte_set_hidx(ptep, rpte, 0, slot, offset); in __hash_page_huge() 123 *ptep = __pte(new_pte & ~H_PAGE_BUSY); in __hash_page_huge() 129 unsigned long addr, pte_t *ptep) in huge_ptep_modify_prot_start() argument 137 pte_val = pte_update(vma->vm_mm, addr, ptep, in huge_ptep_modify_prot_start() 144 pte_t *ptep, pte_t old_pte, pte_t pte) in huge_ptep_modify_prot_commit() argument [all …]
|
D | hash_64k.c | 37 pte_t *ptep, unsigned long trap, unsigned long flags, in __hash_page_4K() argument 52 pte_t pte = READ_ONCE(*ptep); in __hash_page_4K() 69 } while (!pte_xchg(ptep, __pte(old_pte), __pte(new_pte))); in __hash_page_4K() 89 rpte = __real_pte(__pte(old_pte), ptep, PTRS_PER_PTE); in __hash_page_4K() 129 *ptep = __pte(new_pte & ~H_PAGE_BUSY); in __hash_page_4K() 210 *ptep = __pte(old_pte); in __hash_page_4K() 216 new_pte |= pte_set_hidx(ptep, rpte, subpg_index, slot, PTRS_PER_PTE); in __hash_page_4K() 219 *ptep = __pte(new_pte & ~H_PAGE_BUSY); in __hash_page_4K() 224 unsigned long vsid, pte_t *ptep, unsigned long trap, in __hash_page_64K() argument 238 pte_t pte = READ_ONCE(*ptep); in __hash_page_64K() [all …]
|
/linux-6.1.9/arch/powerpc/include/asm/book3s/32/ |
D | pgtable.h | 234 #define pte_clear(mm, addr, ptep) \ argument 235 do { pte_update(mm, addr, ptep, ~_PAGE_HASHPTE, 0, 0); } while (0) 258 static inline void flush_hash_entry(struct mm_struct *mm, pte_t *ptep, unsigned long addr) in flush_hash_entry() argument 261 unsigned long ptephys = __pa(ptep) & PAGE_MASK; in flush_hash_entry() 320 unsigned long addr, pte_t *ptep) in __ptep_test_and_clear_young() argument 323 old = pte_update(mm, addr, ptep, _PAGE_ACCESSED, 0, 0); in __ptep_test_and_clear_young() 325 flush_hash_entry(mm, ptep, addr); in __ptep_test_and_clear_young() 334 pte_t *ptep) in ptep_get_and_clear() argument 336 return __pte(pte_update(mm, addr, ptep, ~_PAGE_HASHPTE, 0, 0)); in ptep_get_and_clear() 341 pte_t *ptep) in ptep_set_wrprotect() argument [all …]
|
/linux-6.1.9/arch/x86/xen/ |
D | p2m.c | 220 pte_t *ptep; in xen_build_mfn_list_list() local 248 ptep = lookup_address((unsigned long)(xen_p2m_addr + pfn), in xen_build_mfn_list_list() 250 BUG_ON(!ptep || level != PG_LEVEL_4K); in xen_build_mfn_list_list() 251 mfn = pte_mfn(*ptep); in xen_build_mfn_list_list() 252 ptep = (pte_t *)((unsigned long)ptep & ~(PAGE_SIZE - 1)); in xen_build_mfn_list_list() 258 if (ptep == p2m_missing_pte || ptep == p2m_identity_pte) { in xen_build_mfn_list_list() 336 pte_t *ptep; in xen_rebuild_p2m_list() local 384 ptep = populate_extra_pte((unsigned long)(p2m + pfn)); in xen_rebuild_p2m_list() 385 set_pte(ptep, in xen_rebuild_p2m_list() 394 ptep = populate_extra_pte((unsigned long)(p2m + pfn)); in xen_rebuild_p2m_list() [all …]
|
/linux-6.1.9/tools/testing/selftests/kvm/lib/riscv/ |
D | processor.c | 69 uint64_t *ptep, next_ppn; in virt_arch_pg_map() local 86 ptep = addr_gpa2hva(vm, vm->pgd) + pte_index(vm, vaddr, level) * 8; in virt_arch_pg_map() 87 if (!*ptep) { in virt_arch_pg_map() 89 *ptep = (next_ppn << PGTBL_PTE_ADDR_SHIFT) | in virt_arch_pg_map() 95 ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + in virt_arch_pg_map() 97 if (!*ptep && level > 0) { in virt_arch_pg_map() 100 *ptep = (next_ppn << PGTBL_PTE_ADDR_SHIFT) | in virt_arch_pg_map() 107 *ptep = (paddr << PGTBL_PTE_ADDR_SHIFT) | in virt_arch_pg_map() 113 uint64_t *ptep; in addr_arch_gva2gpa() local 119 ptep = addr_gpa2hva(vm, vm->pgd) + pte_index(vm, gva, level) * 8; in addr_arch_gva2gpa() [all …]
|
/linux-6.1.9/arch/powerpc/include/asm/book3s/64/ |
D | hash.h | 147 pte_t *ptep, unsigned long pte, int huge); 152 pte_t *ptep, unsigned long clr, in hash__pte_update() argument 167 : "=&r" (old_be), "=&r" (tmp_be), "=m" (*ptep) in hash__pte_update() 168 : "r" (ptep), "r" (cpu_to_be64(clr)), "m" (*ptep), in hash__pte_update() 177 hpte_need_flush(mm, addr, ptep, old, huge); in hash__pte_update() 185 static inline void hash__ptep_set_access_flags(pte_t *ptep, pte_t entry) in hash__ptep_set_access_flags() argument 201 :"=&r" (old), "=&r" (tmp), "=m" (*ptep) in hash__ptep_set_access_flags() 202 :"r" (val), "r" (ptep), "m" (*ptep), "r" (cpu_to_be64(H_PAGE_BUSY)) in hash__ptep_set_access_flags() 225 pte_t *ptep, pte_t pte, int percpu) in hash__set_pte_at() argument 231 *ptep = pte; in hash__set_pte_at()
|
D | radix.h | 146 extern void radix__ptep_set_access_flags(struct vm_area_struct *vma, pte_t *ptep, 151 unsigned long addr, pte_t *ptep, 154 static inline unsigned long __radix_pte_update(pte_t *ptep, unsigned long clr, in __radix_pte_update() argument 165 : "=&r" (old_be), "=&r" (tmp_be), "=m" (*ptep) in __radix_pte_update() 166 : "r" (ptep), "r" (cpu_to_be64(set)), "r" (cpu_to_be64(clr)) in __radix_pte_update() 174 pte_t *ptep, unsigned long clr, in radix__pte_update() argument 180 old_pte = __radix_pte_update(ptep, clr, set); in radix__pte_update() 189 pte_t *ptep, int full) in radix__ptep_get_and_clear_full() argument 194 old_pte = pte_val(*ptep); in radix__ptep_get_and_clear_full() 195 *ptep = __pte(0); in radix__ptep_get_and_clear_full() [all …]
|
/linux-6.1.9/arch/mips/include/asm/ |
D | hugetlb.h | 35 unsigned long addr, pte_t *ptep) in huge_ptep_get_and_clear() argument 38 pte_t pte = *ptep; in huge_ptep_get_and_clear() 41 set_pte_at(mm, addr, ptep, clear); in huge_ptep_get_and_clear() 47 unsigned long addr, pte_t *ptep) in huge_ptep_clear_flush() argument 56 pte = huge_ptep_get_and_clear(vma->vm_mm, addr, ptep); in huge_ptep_clear_flush() 71 pte_t *ptep, pte_t pte, in huge_ptep_set_access_flags() argument 74 int changed = !pte_same(*ptep, pte); in huge_ptep_set_access_flags() 77 set_pte_at(vma->vm_mm, addr, ptep, pte); in huge_ptep_set_access_flags()
|
/linux-6.1.9/arch/powerpc/kvm/ |
D | book3s_64_mmu_radix.c | 368 static unsigned long kvmppc_radix_update_pte(struct kvm *kvm, pte_t *ptep, in kvmppc_radix_update_pte() argument 372 return __radix_pte_update(ptep, clr, set); in kvmppc_radix_update_pte() 376 pte_t *ptep, pte_t pte) in kvmppc_radix_set_pte_at() argument 378 radix__set_pte_at(kvm->mm, addr, ptep, pte, 0); in kvmppc_radix_set_pte_at() 395 static void kvmppc_pte_free(pte_t *ptep) in kvmppc_pte_free() argument 397 kmem_cache_free(kvm_pte_cache, ptep); in kvmppc_pte_free() 618 pte_t *ptep, *new_ptep = NULL; in kvmppc_create_pte() local 757 ptep = pte_offset_kernel(pmd, gpa); in kvmppc_create_pte() 758 if (pte_present(*ptep)) { in kvmppc_create_pte() 760 if (pte_raw(*ptep) == pte_raw(pte)) { in kvmppc_create_pte() [all …]
|
/linux-6.1.9/arch/loongarch/include/asm/ |
D | hugetlb.h | 34 unsigned long addr, pte_t *ptep) in huge_ptep_get_and_clear() argument 37 pte_t pte = *ptep; in huge_ptep_get_and_clear() 40 set_pte_at(mm, addr, ptep, clear); in huge_ptep_get_and_clear() 46 unsigned long addr, pte_t *ptep) in huge_ptep_clear_flush() argument 50 pte = huge_ptep_get_and_clear(vma->vm_mm, addr, ptep); in huge_ptep_clear_flush() 65 pte_t *ptep, pte_t pte, in huge_ptep_set_access_flags() argument 68 int changed = !pte_same(*ptep, pte); in huge_ptep_set_access_flags() 71 set_pte_at(vma->vm_mm, addr, ptep, pte); in huge_ptep_set_access_flags()
|
/linux-6.1.9/arch/arm64/include/asm/ |
D | hugetlb.h | 30 pte_t *ptep, pte_t pte); 33 unsigned long addr, pte_t *ptep, 37 unsigned long addr, pte_t *ptep); 40 unsigned long addr, pte_t *ptep); 43 unsigned long addr, pte_t *ptep); 46 pte_t *ptep, unsigned long sz); 48 extern pte_t huge_ptep_get(pte_t *ptep);
|