| /linux-6.6.21/arch/s390/mm/ |
| D | pgtable.c | 48 pte_t *ptep, int nodat) in ptep_ipte_local() argument
61 __ptep_ipte(addr, ptep, opt, asce, IPTE_LOCAL); in ptep_ipte_local()
63 __ptep_ipte(addr, ptep, 0, 0, IPTE_LOCAL); in ptep_ipte_local()
68 pte_t *ptep, int nodat) in ptep_ipte_global() argument
81 __ptep_ipte(addr, ptep, opt, asce, IPTE_GLOBAL); in ptep_ipte_global()
83 __ptep_ipte(addr, ptep, 0, 0, IPTE_GLOBAL); in ptep_ipte_global()
88 unsigned long addr, pte_t *ptep, in ptep_flush_direct() argument
93 old = *ptep; in ptep_flush_direct()
99 ptep_ipte_local(mm, addr, ptep, nodat); in ptep_flush_direct()
101 ptep_ipte_global(mm, addr, ptep, nodat); in ptep_flush_direct()
[all …]
|
| /linux-6.6.21/arch/arm64/mm/ |
| D | hugetlbpage.c | 104 pte_t *ptep, size_t *pgsize) in find_num_contig() argument
115 if ((pte_t *)pmdp == ptep) { in find_num_contig()
151 pte_t huge_ptep_get(pte_t *ptep) in huge_ptep_get() argument
155 pte_t orig_pte = ptep_get(ptep); in huge_ptep_get()
161 for (i = 0; i < ncontig; i++, ptep++) { in huge_ptep_get()
162 pte_t pte = ptep_get(ptep); in huge_ptep_get()
183 pte_t *ptep, in get_clear_contig() argument
187 pte_t orig_pte = ptep_get(ptep); in get_clear_contig()
190 for (i = 0; i < ncontig; i++, addr += pgsize, ptep++) { in get_clear_contig()
191 pte_t pte = ptep_get_and_clear(mm, addr, ptep); in get_clear_contig()
[all …]
|
| /linux-6.6.21/arch/riscv/mm/ |
| D | hugetlbpage.c | 6 pte_t huge_ptep_get(pte_t *ptep) in huge_ptep_get() argument
10 pte_t orig_pte = ptep_get(ptep); in huge_ptep_get()
17 for (i = 0; i < pte_num; i++, ptep++) { in huge_ptep_get()
18 pte_t pte = ptep_get(ptep); in huge_ptep_get()
150 pte_t *ptep, in get_clear_contig() argument
153 pte_t orig_pte = ptep_get(ptep); in get_clear_contig()
156 for (i = 0; i < pte_num; i++, addr += PAGE_SIZE, ptep++) { in get_clear_contig()
157 pte_t pte = ptep_get_and_clear(mm, addr, ptep); in get_clear_contig()
171 pte_t *ptep, in get_clear_contig_flush() argument
174 pte_t orig_pte = get_clear_contig(mm, addr, ptep, pte_num); in get_clear_contig_flush()
[all …]
|
| /linux-6.6.21/arch/s390/include/asm/ |
| D | hugetlb.h | 19 pte_t *ptep, pte_t pte, unsigned long sz);
21 pte_t *ptep, pte_t pte);
22 pte_t huge_ptep_get(pte_t *ptep);
24 unsigned long addr, pte_t *ptep);
49 pte_t *ptep, unsigned long sz) in huge_pte_clear() argument
51 if ((pte_val(*ptep) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) in huge_pte_clear()
52 set_pte(ptep, __pte(_REGION3_ENTRY_EMPTY)); in huge_pte_clear()
54 set_pte(ptep, __pte(_SEGMENT_ENTRY_EMPTY)); in huge_pte_clear()
58 unsigned long address, pte_t *ptep) in huge_ptep_clear_flush() argument
60 return huge_ptep_get_and_clear(vma->vm_mm, address, ptep); in huge_ptep_clear_flush()
[all …]
|
| /linux-6.6.21/drivers/iommu/ |
| D | io-pgtable-dart.c | 126 dart_iopte *ptep) in dart_init_pte() argument
133 if (ptep[i] & APPLE_DART_PTE_VALID) { in dart_init_pte()
135 WARN_ON(ptep[i] & APPLE_DART_PTE_VALID); in dart_init_pte()
147 ptep[i] = pte | paddr_to_iopte(paddr + i * sz, data); in dart_init_pte()
153 dart_iopte *ptep, in dart_install_table() argument
168 old = cmpxchg64_relaxed(ptep, curr, new); in dart_install_table()
195 dart_iopte pte, *ptep; in dart_get_l2() local
198 ptep = data->pgd[tbl]; in dart_get_l2()
199 if (!ptep) in dart_get_l2()
202 ptep += dart_get_l1_index(data, iova); in dart_get_l2()
[all …]
|
| /linux-6.6.21/arch/sparc/include/asm/ |
| D | hugetlb.h | 17 pte_t *ptep, pte_t pte, unsigned long sz);
19 pte_t *ptep, pte_t pte);
23 pte_t *ptep);
27 unsigned long addr, pte_t *ptep) in huge_ptep_clear_flush() argument
29 return *ptep; in huge_ptep_clear_flush()
34 unsigned long addr, pte_t *ptep) in huge_ptep_set_wrprotect() argument
36 pte_t old_pte = *ptep; in huge_ptep_set_wrprotect()
37 __set_huge_pte_at(mm, addr, ptep, pte_wrprotect(old_pte)); in huge_ptep_set_wrprotect()
42 unsigned long addr, pte_t *ptep, in huge_ptep_set_access_flags() argument
45 int changed = !pte_same(*ptep, pte); in huge_ptep_set_access_flags()
[all …]
|
| /linux-6.6.21/tools/testing/selftests/kvm/lib/aarch64/ |
| D | processor.c | 112 uint64_t *ptep; in _virt_pg_map() local
128 ptep = addr_gpa2hva(vm, vm->pgd) + pgd_index(vm, vaddr) * 8; in _virt_pg_map()
129 if (!*ptep) in _virt_pg_map()
130 *ptep = addr_pte(vm, vm_alloc_page_table(vm), 3); in _virt_pg_map()
134 ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pud_index(vm, vaddr) * 8; in _virt_pg_map()
135 if (!*ptep) in _virt_pg_map()
136 *ptep = addr_pte(vm, vm_alloc_page_table(vm), 3); in _virt_pg_map()
139 ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pmd_index(vm, vaddr) * 8; in _virt_pg_map()
140 if (!*ptep) in _virt_pg_map()
141 *ptep = addr_pte(vm, vm_alloc_page_table(vm), 3); in _virt_pg_map()
[all …]
|
| /linux-6.6.21/include/asm-generic/ |
| D | hugetlb.h | 62 pte_t *ptep, unsigned long sz) in huge_pte_clear() argument
64 pte_clear(mm, addr, ptep); in huge_pte_clear()
79 pte_t *ptep, pte_t pte, unsigned long sz) in set_huge_pte_at() argument
81 set_pte_at(mm, addr, ptep, pte); in set_huge_pte_at()
87 unsigned long addr, pte_t *ptep) in huge_ptep_get_and_clear() argument
89 return ptep_get_and_clear(mm, addr, ptep); in huge_ptep_get_and_clear()
95 unsigned long addr, pte_t *ptep) in huge_ptep_clear_flush() argument
97 return ptep_clear_flush(vma, addr, ptep); in huge_ptep_clear_flush()
131 unsigned long addr, pte_t *ptep) in huge_ptep_set_wrprotect() argument
133 ptep_set_wrprotect(mm, addr, ptep); in huge_ptep_set_wrprotect()
[all …]
|
| /linux-6.6.21/arch/xtensa/include/asm/ |
| D | pgalloc.h | 24 #define pmd_populate_kernel(mm, pmdp, ptep) \ argument
25 (pmd_val(*(pmdp)) = ((unsigned long)ptep))
35 static inline void ptes_clear(pte_t *ptep) in ptes_clear() argument
40 pte_clear(NULL, 0, ptep + i); in ptes_clear()
45 pte_t *ptep; in pte_alloc_one_kernel() local
47 ptep = (pte_t *)__pte_alloc_one_kernel(mm); in pte_alloc_one_kernel()
48 if (!ptep) in pte_alloc_one_kernel()
50 ptes_clear(ptep); in pte_alloc_one_kernel()
51 return ptep; in pte_alloc_one_kernel()
|
| D | pgtable.h | 243 #define pte_clear(mm,addr,ptep) \ argument
244 do { update_pte(ptep, __pte(_PAGE_CA_INVALID | _PAGE_USER)); } while (0)
294 static inline void update_pte(pte_t *ptep, pte_t pteval) in update_pte() argument
296 *ptep = pteval; in update_pte()
298 __asm__ __volatile__ ("dhwb %0, 0" :: "a" (ptep)); in update_pte()
305 static inline void set_pte(pte_t *ptep, pte_t pte) in set_pte() argument
307 update_pte(ptep, pte); in set_pte()
320 pte_t *ptep) in ptep_test_and_clear_young() argument
322 pte_t pte = *ptep; in ptep_test_and_clear_young()
325 update_pte(ptep, pte_mkold(pte)); in ptep_test_and_clear_young()
[all …]
|
| /linux-6.6.21/arch/riscv/kvm/ |
| D | mmu.c | 100 pte_t *ptep; in gstage_get_leaf_entry() local
104 ptep = (pte_t *)kvm->arch.pgd; in gstage_get_leaf_entry()
105 ptep = &ptep[gstage_pte_index(addr, current_level)]; in gstage_get_leaf_entry()
106 while (ptep && pte_val(*ptep)) { in gstage_get_leaf_entry()
107 if (gstage_pte_leaf(ptep)) { in gstage_get_leaf_entry()
109 *ptepp = ptep; in gstage_get_leaf_entry()
116 ptep = (pte_t *)gstage_pte_page_vaddr(*ptep); in gstage_get_leaf_entry()
117 ptep = &ptep[gstage_pte_index(addr, current_level)]; in gstage_get_leaf_entry()
119 ptep = NULL; in gstage_get_leaf_entry()
143 pte_t *ptep = &next_ptep[gstage_pte_index(addr, current_level)]; in gstage_set_pte() local
[all …]
|
| /linux-6.6.21/arch/parisc/mm/ |
| D | hugetlbpage.c | 123 pte_t *ptep, pte_t entry) in __set_huge_pte_at() argument
132 set_pte(ptep, entry); in __set_huge_pte_at()
133 ptep++; in __set_huge_pte_at()
143 pte_t *ptep, pte_t entry, unsigned long sz) in set_huge_pte_at() argument
145 __set_huge_pte_at(mm, addr, ptep, entry); in set_huge_pte_at()
150 pte_t *ptep) in huge_ptep_get_and_clear() argument
154 entry = *ptep; in huge_ptep_get_and_clear()
155 __set_huge_pte_at(mm, addr, ptep, __pte(0)); in huge_ptep_get_and_clear()
162 unsigned long addr, pte_t *ptep) in huge_ptep_set_wrprotect() argument
166 old_pte = *ptep; in huge_ptep_set_wrprotect()
[all …]
|
| /linux-6.6.21/arch/powerpc/include/asm/nohash/64/ |
| D | pgtable.h | 176 pte_t *ptep, unsigned long clr, in pte_update() argument
180 unsigned long old = pte_val(*ptep); in pte_update()
181 *ptep = __pte((old & ~clr) | set); in pte_update()
196 unsigned long addr, pte_t *ptep) in __ptep_test_and_clear_young() argument
200 if (!pte_young(*ptep)) in __ptep_test_and_clear_young()
202 old = pte_update(mm, addr, ptep, _PAGE_ACCESSED, 0, 0); in __ptep_test_and_clear_young()
215 pte_t *ptep) in ptep_set_wrprotect() argument
218 if ((pte_val(*ptep) & _PAGE_RW) == 0) in ptep_set_wrprotect()
221 pte_update(mm, addr, ptep, _PAGE_RW, 0, 0); in ptep_set_wrprotect()
226 unsigned long addr, pte_t *ptep) in huge_ptep_set_wrprotect() argument
[all …]
|
| /linux-6.6.21/arch/powerpc/mm/ |
| D | pageattr.c | 18 static pte_basic_t pte_update_delta(pte_t *ptep, unsigned long addr, in pte_update_delta() argument
21 return pte_update(&init_mm, addr, ptep, old & ~new, new & ~old, 0); in pte_update_delta()
30 static int change_page_attr(pte_t *ptep, unsigned long addr, void *data) in change_page_attr() argument
39 pte_update_delta(ptep, addr, _PAGE_KERNEL_RW & ~_PAGE_DIRTY, _PAGE_KERNEL_RO); in change_page_attr()
42 pte_update_delta(ptep, addr, _PAGE_KERNEL_RO, _PAGE_KERNEL_RW); in change_page_attr()
45 pte_update_delta(ptep, addr, _PAGE_KERNEL_ROX, _PAGE_KERNEL_RO); in change_page_attr()
48 pte_update_delta(ptep, addr, _PAGE_KERNEL_RO, _PAGE_KERNEL_ROX); in change_page_attr()
51 pte_update(&init_mm, addr, ptep, _PAGE_PRESENT, 0, 0); in change_page_attr()
54 pte_update(&init_mm, addr, ptep, 0, _PAGE_PRESENT, 0); in change_page_attr()
|
| D | pgtable.c | 192 void set_ptes(struct mm_struct *mm, unsigned long addr, pte_t *ptep, in set_ptes() argument
215 VM_WARN_ON(pte_hw_valid(*ptep) && !pte_protnone(*ptep)); in set_ptes()
218 __set_pte_at(mm, addr, ptep, pte, 0); in set_ptes()
221 ptep++; in set_ptes()
233 pte_t *ptep = pte_offset_kernel(pmdp, va); in unmap_kernel_page() local
235 pte_clear(&init_mm, va, ptep); in unmap_kernel_page()
247 pte_t *ptep, pte_t entry, int dirty) in ptep_set_access_flags() argument
251 changed = !pte_same(*(ptep), entry); in ptep_set_access_flags()
254 __ptep_set_access_flags(vma, ptep, entry, in ptep_set_access_flags()
262 unsigned long addr, pte_t *ptep, in huge_ptep_set_access_flags() argument
[all …]
|
| /linux-6.6.21/arch/powerpc/mm/book3s64/ |
| D | hugetlbpage.c | 21 pte_t *ptep, unsigned long trap, unsigned long flags, in __hash_page_huge() argument
49 old_pte = pte_val(*ptep); in __hash_page_huge()
64 } while(!pte_xchg(ptep, __pte(old_pte), __pte(new_pte))); in __hash_page_huge()
75 rpte = __real_pte(__pte(old_pte), ptep, offset); in __hash_page_huge()
111 *ptep = __pte(old_pte); in __hash_page_huge()
117 new_pte |= pte_set_hidx(ptep, rpte, 0, slot, offset); in __hash_page_huge()
123 *ptep = __pte(new_pte & ~H_PAGE_BUSY); in __hash_page_huge()
129 unsigned long addr, pte_t *ptep) in huge_ptep_modify_prot_start() argument
137 pte_val = pte_update(vma->vm_mm, addr, ptep, in huge_ptep_modify_prot_start()
144 pte_t *ptep, pte_t old_pte, pte_t pte) in huge_ptep_modify_prot_commit() argument
[all …]
|
| /linux-6.6.21/arch/x86/xen/ |
| D | p2m.c | 215 pte_t *ptep; in xen_build_mfn_list_list() local
243 ptep = lookup_address((unsigned long)(xen_p2m_addr + pfn), in xen_build_mfn_list_list()
245 BUG_ON(!ptep || level != PG_LEVEL_4K); in xen_build_mfn_list_list()
246 mfn = pte_mfn(*ptep); in xen_build_mfn_list_list()
247 ptep = (pte_t *)((unsigned long)ptep & ~(PAGE_SIZE - 1)); in xen_build_mfn_list_list()
253 if (ptep == p2m_missing_pte || ptep == p2m_identity_pte) { in xen_build_mfn_list_list()
331 pte_t *ptep; in xen_rebuild_p2m_list() local
379 ptep = populate_extra_pte((unsigned long)(p2m + pfn)); in xen_rebuild_p2m_list()
380 set_pte(ptep, in xen_rebuild_p2m_list()
389 ptep = populate_extra_pte((unsigned long)(p2m + pfn)); in xen_rebuild_p2m_list()
[all …]
|
| /linux-6.6.21/tools/testing/selftests/kvm/lib/riscv/ |
| D | processor.c | 71 uint64_t *ptep, next_ppn; in virt_arch_pg_map() local
88 ptep = addr_gpa2hva(vm, vm->pgd) + pte_index(vm, vaddr, level) * 8; in virt_arch_pg_map()
89 if (!*ptep) { in virt_arch_pg_map()
91 *ptep = (next_ppn << PGTBL_PTE_ADDR_SHIFT) | in virt_arch_pg_map()
97 ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + in virt_arch_pg_map()
99 if (!*ptep && level > 0) { in virt_arch_pg_map()
102 *ptep = (next_ppn << PGTBL_PTE_ADDR_SHIFT) | in virt_arch_pg_map()
109 *ptep = (paddr << PGTBL_PTE_ADDR_SHIFT) | in virt_arch_pg_map()
115 uint64_t *ptep; in addr_arch_gva2gpa() local
121 ptep = addr_gpa2hva(vm, vm->pgd) + pte_index(vm, gva, level) * 8; in addr_arch_gva2gpa()
[all …]
|
| /linux-6.6.21/arch/arm64/include/asm/ |
| D | hugetlb.h | 31 pte_t *ptep, pte_t pte, unsigned long sz);
34 unsigned long addr, pte_t *ptep,
38 unsigned long addr, pte_t *ptep);
41 unsigned long addr, pte_t *ptep);
44 unsigned long addr, pte_t *ptep);
47 pte_t *ptep, unsigned long sz);
49 extern pte_t huge_ptep_get(pte_t *ptep);
55 unsigned long addr, pte_t *ptep);
59 unsigned long addr, pte_t *ptep,
|
| /linux-6.6.21/arch/mips/include/asm/ |
| D | hugetlb.h | 35 unsigned long addr, pte_t *ptep) in huge_ptep_get_and_clear() argument
38 pte_t pte = *ptep; in huge_ptep_get_and_clear()
41 set_pte_at(mm, addr, ptep, clear); in huge_ptep_get_and_clear()
47 unsigned long addr, pte_t *ptep) in huge_ptep_clear_flush() argument
56 pte = huge_ptep_get_and_clear(vma->vm_mm, addr, ptep); in huge_ptep_clear_flush()
71 pte_t *ptep, pte_t pte, in huge_ptep_set_access_flags() argument
74 int changed = !pte_same(*ptep, pte); in huge_ptep_set_access_flags()
77 set_pte_at(vma->vm_mm, addr, ptep, pte); in huge_ptep_set_access_flags()
|
| D | pgtable.h | 119 static inline void set_pte(pte_t *ptep, pte_t pte) argument
121 ptep->pte_high = pte.pte_high;
123 ptep->pte_low = pte.pte_low;
130 pte_t *buddy = ptep_buddy(ptep);
143 static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) argument
150 if (ptep_buddy(ptep)->pte_high & _PAGE_GLOBAL)
153 if (ptep_buddy(ptep)->pte_low & _PAGE_GLOBAL)
157 set_pte(ptep, null);
171 static inline void set_pte(pte_t *ptep, pte_t pteval) argument
173 *ptep = pteval;
[all …]
|
| /linux-6.6.21/include/linux/ |
| D | pgtable.h | 232 pte_t *ptep, pte_t pte, unsigned int nr) in set_ptes() argument
234 page_table_check_ptes_set(mm, ptep, pte, nr); in set_ptes()
238 set_pte(ptep, pte); in set_ptes()
241 ptep++; in set_ptes()
247 #define set_pte_at(mm, addr, ptep, pte) set_ptes(mm, addr, ptep, pte, 1) argument
251 unsigned long address, pte_t *ptep,
282 static inline pte_t ptep_get(pte_t *ptep) in ptep_get() argument
284 return READ_ONCE(*ptep); in ptep_get()
298 pte_t *ptep) in ptep_test_and_clear_young() argument
300 pte_t pte = ptep_get(ptep); in ptep_test_and_clear_young()
[all …]
|
| /linux-6.6.21/arch/powerpc/include/asm/book3s/64/ |
| D | hash.h | 161 pte_t *ptep, unsigned long pte, int huge);
166 pte_t *ptep, unsigned long clr, in hash__pte_update() argument
181 : "=&r" (old_be), "=&r" (tmp_be), "=m" (*ptep) in hash__pte_update()
182 : "r" (ptep), "r" (cpu_to_be64(clr)), "m" (*ptep), in hash__pte_update()
191 hpte_need_flush(mm, addr, ptep, old, huge); in hash__pte_update()
199 static inline void hash__ptep_set_access_flags(pte_t *ptep, pte_t entry) in hash__ptep_set_access_flags() argument
215 :"=&r" (old), "=&r" (tmp), "=m" (*ptep) in hash__ptep_set_access_flags()
216 :"r" (val), "r" (ptep), "m" (*ptep), "r" (cpu_to_be64(H_PAGE_BUSY)) in hash__ptep_set_access_flags()
239 pte_t *ptep, pte_t pte, int percpu) in hash__set_pte_at() argument
245 *ptep = pte; in hash__set_pte_at()
|
| /linux-6.6.21/arch/loongarch/include/asm/ |
| D | hugetlb.h | 34 unsigned long addr, pte_t *ptep) in huge_ptep_get_and_clear() argument
37 pte_t pte = *ptep; in huge_ptep_get_and_clear()
40 set_pte_at(mm, addr, ptep, clear); in huge_ptep_get_and_clear()
46 unsigned long addr, pte_t *ptep) in huge_ptep_clear_flush() argument
50 pte = huge_ptep_get_and_clear(vma->vm_mm, addr, ptep); in huge_ptep_clear_flush()
65 pte_t *ptep, pte_t pte, in huge_ptep_set_access_flags() argument
68 int changed = !pte_same(*ptep, pte); in huge_ptep_set_access_flags()
71 set_pte_at(vma->vm_mm, addr, ptep, pte); in huge_ptep_set_access_flags()
|
| /linux-6.6.21/arch/riscv/include/asm/ |
| D | hugetlb.h | 22 pte_t *ptep, unsigned long sz);
26 unsigned long addr, pte_t *ptep, pte_t pte,
31 unsigned long addr, pte_t *ptep);
35 unsigned long addr, pte_t *ptep);
39 unsigned long addr, pte_t *ptep);
43 unsigned long addr, pte_t *ptep,
47 pte_t huge_ptep_get(pte_t *ptep);
|