/linux-6.6.21/arch/arm64/include/asm/ |
D | pgalloc.h | 23 static inline void __pud_populate(pud_t *pudp, phys_addr_t pmdp, pudval_t prot) in __pud_populate() argument 25 set_pud(pudp, __pud(__phys_to_pud_val(pmdp) | prot)); in __pud_populate() 28 static inline void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmdp) in pud_populate() argument 33 __pud_populate(pudp, __pa(pmdp), pudval); in pud_populate() 36 static inline void __pud_populate(pud_t *pudp, phys_addr_t pmdp, pudval_t prot) in __pud_populate() argument 44 static inline void __p4d_populate(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot) in __p4d_populate() argument 46 set_p4d(p4dp, __p4d(__phys_to_p4d_val(pudp) | prot)); in __p4d_populate() 49 static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4dp, pud_t *pudp) in p4d_populate() argument 54 __p4d_populate(p4dp, __pa(pudp), p4dval); in p4d_populate() 57 static inline void __p4d_populate(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot) in __p4d_populate() argument
|
/linux-6.6.21/arch/powerpc/mm/nohash/ |
D | book3e_pgtable.c | 78 pud_t *pudp; in map_kernel_page() local 86 pudp = pud_alloc(&init_mm, p4dp, ea); in map_kernel_page() 87 if (!pudp) in map_kernel_page() 89 pmdp = pmd_alloc(&init_mm, pudp, ea); in map_kernel_page() 99 pudp = early_alloc_pgtable(PUD_TABLE_SIZE); in map_kernel_page() 100 p4d_populate(&init_mm, p4dp, pudp); in map_kernel_page() 102 pudp = pud_offset(p4dp, ea); in map_kernel_page() 103 if (pud_none(*pudp)) { in map_kernel_page() 105 pud_populate(&init_mm, pudp, pmdp); in map_kernel_page() 107 pmdp = pmd_offset(pudp, ea); in map_kernel_page()
|
/linux-6.6.21/arch/arm64/mm/ |
D | mmu.c | 235 static void init_pmd(pud_t *pudp, unsigned long addr, unsigned long end, in init_pmd() argument 242 pmdp = pmd_set_fixmap_offset(pudp, addr); in init_pmd() 272 static void alloc_init_cont_pmd(pud_t *pudp, unsigned long addr, in alloc_init_cont_pmd() argument 278 pud_t pud = READ_ONCE(*pudp); in alloc_init_cont_pmd() 292 __pud_populate(pudp, pmd_phys, pudval); in alloc_init_cont_pmd() 293 pud = READ_ONCE(*pudp); in alloc_init_cont_pmd() 307 init_pmd(pudp, addr, next, phys, __prot, pgtable_alloc, flags); in alloc_init_cont_pmd() 319 pud_t *pudp; in alloc_init_pud() local 336 pudp = pud_set_fixmap_offset(p4dp, addr); in alloc_init_pud() 338 pud_t old_pud = READ_ONCE(*pudp); in alloc_init_pud() [all …]
|
D | hugetlbpage.c | 108 pud_t *pudp; in find_num_contig() local 113 pudp = pud_offset(p4dp, addr); in find_num_contig() 114 pmdp = pmd_offset(pudp, addr); in find_num_contig() 281 pud_t *pudp; in huge_pte_alloc() local 287 pudp = pud_alloc(mm, p4dp, addr); in huge_pte_alloc() 288 if (!pudp) in huge_pte_alloc() 292 ptep = (pte_t *)pudp; in huge_pte_alloc() 294 pmdp = pmd_alloc(mm, pudp, addr); in huge_pte_alloc() 301 if (want_pmd_share(vma, addr) && pud_none(READ_ONCE(*pudp))) in huge_pte_alloc() 302 ptep = huge_pmd_share(mm, vma, addr, pudp); in huge_pte_alloc() [all …]
|
D | kasan_init.c | 76 static pmd_t *__init kasan_pmd_offset(pud_t *pudp, unsigned long addr, int node, in kasan_pmd_offset() argument 79 if (pud_none(READ_ONCE(*pudp))) { in kasan_pmd_offset() 83 __pud_populate(pudp, pmd_phys, PUD_TYPE_TABLE); in kasan_pmd_offset() 86 return early ? pmd_offset_kimg(pudp, addr) : pmd_offset(pudp, addr); in kasan_pmd_offset() 119 static void __init kasan_pmd_populate(pud_t *pudp, unsigned long addr, in kasan_pmd_populate() argument 123 pmd_t *pmdp = kasan_pmd_offset(pudp, addr, node, early); in kasan_pmd_populate() 135 pud_t *pudp = kasan_pud_offset(p4dp, addr, node, early); in kasan_pud_populate() local 139 kasan_pmd_populate(pudp, addr, next, node, early); in kasan_pud_populate() 140 } while (pudp++, addr = next, addr != end && pud_none(READ_ONCE(*pudp))); in kasan_pud_populate()
|
D | fixmap.c | 51 static void __init early_fixmap_init_pmd(pud_t *pudp, unsigned long addr, in early_fixmap_init_pmd() argument 55 pud_t pud = READ_ONCE(*pudp); in early_fixmap_init_pmd() 59 __pud_populate(pudp, __pa_symbol(bm_pmd), PUD_TYPE_TABLE); in early_fixmap_init_pmd() 61 pmdp = pmd_offset_kimg(pudp, addr); in early_fixmap_init_pmd() 73 pud_t *pudp; in early_fixmap_init_pud() local 88 pudp = pud_offset_kimg(p4dp, addr); in early_fixmap_init_pud() 89 early_fixmap_init_pmd(pudp, addr, end); in early_fixmap_init_pud()
|
D | pageattr.c | 217 pud_t *pudp, pud; in kernel_page_present() local 233 pudp = pud_offset(p4dp, addr); in kernel_page_present() 234 pud = READ_ONCE(*pudp); in kernel_page_present() 240 pmdp = pmd_offset(pudp, addr); in kernel_page_present()
|
/linux-6.6.21/arch/s390/mm/ |
D | pageattr.c | 175 static int walk_pmd_level(pud_t *pudp, unsigned long addr, unsigned long end, in walk_pmd_level() argument 183 pmdp = pmd_offset(pudp, addr); in walk_pmd_level() 211 static int split_pud_page(pud_t *pudp, unsigned long addr) in split_pud_page() argument 221 pmd_addr = pud_pfn(*pudp) << PAGE_SHIFT; in split_pud_page() 222 ro = !!(pud_val(*pudp) & _REGION_ENTRY_PROTECT); in split_pud_page() 223 nx = !!(pud_val(*pudp) & _REGION_ENTRY_NOEXEC); in split_pud_page() 234 pgt_set((unsigned long *)pudp, pud_val(new), addr, CRDTE_DTT_REGION3); in split_pud_page() 240 static void modify_pud_page(pud_t *pudp, unsigned long addr, in modify_pud_page() argument 243 pud_t new = *pudp; in modify_pud_page() 261 pgt_set((unsigned long *)pudp, pud_val(new), addr, CRDTE_DTT_REGION3); in modify_pud_page() [all …]
|
D | hugetlbpage.c | 182 pud_t *pudp = (pud_t *) ptep; in huge_ptep_get_and_clear() local 185 pudp_xchg_direct(mm, addr, pudp, __pud(_REGION3_ENTRY_EMPTY)); in huge_ptep_get_and_clear() 196 pud_t *pudp; in huge_pte_alloc() local 202 pudp = pud_alloc(mm, p4dp, addr); in huge_pte_alloc() 203 if (pudp) { in huge_pte_alloc() 205 return (pte_t *) pudp; in huge_pte_alloc() 207 pmdp = pmd_alloc(mm, pudp, addr); in huge_pte_alloc() 218 pud_t *pudp; in huge_pte_offset() local 225 pudp = pud_offset(p4dp, addr); in huge_pte_offset() 226 if (pud_present(*pudp)) { in huge_pte_offset() [all …]
|
/linux-6.6.21/arch/powerpc/mm/kasan/ |
D | init_book3e_64.c | 36 pud_t *pudp; in kasan_map_kernel_page() local 43 pudp = memblock_alloc(PUD_TABLE_SIZE, PUD_TABLE_SIZE); in kasan_map_kernel_page() 44 memcpy(pudp, kasan_early_shadow_pud, PUD_TABLE_SIZE); in kasan_map_kernel_page() 45 p4d_populate(&init_mm, p4dp, pudp); in kasan_map_kernel_page() 47 pudp = pud_offset(p4dp, ea); in kasan_map_kernel_page() 48 if (kasan_pmd_table(*pudp)) { in kasan_map_kernel_page() 51 pud_populate(&init_mm, pudp, pmdp); in kasan_map_kernel_page() 53 pmdp = pmd_offset(pudp, ea); in kasan_map_kernel_page()
|
/linux-6.6.21/arch/x86/include/asm/ |
D | pgtable-3level.h | 53 static inline void native_set_pud(pud_t *pudp, pud_t pud) in native_set_pud() argument 56 pud.p4d.pgd = pti_set_user_pgtbl(&pudp->p4d.pgd, pud.p4d.pgd); in native_set_pud() 58 pxx_xchg64(pud, pudp, native_pud_val(pud)); in native_set_pud() 81 static inline void native_pud_clear(pud_t *pudp) in native_pud_clear() argument 85 static inline void pud_clear(pud_t *pudp) in pud_clear() argument 87 set_pud(pudp, __pud(0)); in pud_clear() 113 static inline pud_t native_pudp_get_and_clear(pud_t *pudp) in native_pudp_get_and_clear() argument 115 return pxx_xchg64(pud, pudp, 0ULL); in native_pudp_get_and_clear()
|
/linux-6.6.21/arch/powerpc/mm/book3s64/ |
D | pgtable.c | 69 pud_t *pudp, pud_t entry, int dirty) in pudp_set_access_flags() argument 73 WARN_ON(!pud_devmap(*pudp)); in pudp_set_access_flags() 74 assert_spin_locked(pud_lockptr(vma->vm_mm, pudp)); in pudp_set_access_flags() 76 changed = !pud_same(*(pudp), entry); in pudp_set_access_flags() 82 __ptep_set_access_flags(vma, pudp_ptep(pudp), in pudp_set_access_flags() 96 unsigned long address, pud_t *pudp) in pudp_test_and_clear_young() argument 98 return __pudp_test_and_clear_young(vma->vm_mm, address, pudp); in pudp_test_and_clear_young() 123 pud_t *pudp, pud_t pud) in set_pud_at() argument 131 WARN_ON(pte_hw_valid(pud_pte(*pudp))); in set_pud_at() 132 assert_spin_locked(pud_lockptr(mm, pudp)); in set_pud_at() [all …]
|
D | radix_pgtable.c | 78 pud_t *pudp; in early_map_kernel_page() local 85 pudp = early_alloc_pgtable(PAGE_SIZE, nid, in early_map_kernel_page() 87 p4d_populate(&init_mm, p4dp, pudp); in early_map_kernel_page() 89 pudp = pud_offset(p4dp, ea); in early_map_kernel_page() 91 ptep = (pte_t *)pudp; in early_map_kernel_page() 94 if (pud_none(*pudp)) { in early_map_kernel_page() 97 pud_populate(&init_mm, pudp, pmdp); in early_map_kernel_page() 99 pmdp = pmd_offset(pudp, ea); in early_map_kernel_page() 130 pud_t *pudp; in __map_kernel_page() local 153 pudp = pud_alloc(&init_mm, p4dp, ea); in __map_kernel_page() [all …]
|
/linux-6.6.21/arch/riscv/mm/ |
D | kasan_init.c | 83 pud_t *pudp, *p; in kasan_populate_pud() local 91 pudp = pud_offset(p4d, vaddr); in kasan_populate_pud() 96 if (pud_none(*pudp) && IS_ALIGNED(vaddr, PUD_SIZE) && (next - vaddr) >= PUD_SIZE) { in kasan_populate_pud() 99 set_pud(pudp, pfn_pud(PFN_DOWN(phys_addr), PAGE_KERNEL)); in kasan_populate_pud() 105 kasan_populate_pmd(pudp, vaddr, next); in kasan_populate_pud() 106 } while (pudp++, vaddr = next, vaddr != end); in kasan_populate_pud() 165 pud_t *pudp, *base_pud; in kasan_early_clear_pud() local 169 pudp = (pud_t *)p4dp; in kasan_early_clear_pud() 172 pudp = base_pud + pud_index(vaddr); in kasan_early_clear_pud() 179 pud_clear(pudp); in kasan_early_clear_pud() [all …]
|
D | pageattr.c | 96 static int __split_linear_mapping_pmd(pud_t *pudp, in __split_linear_mapping_pmd() argument 102 pmdp = pmd_offset(pudp, vaddr); in __split_linear_mapping_pmd() 138 pud_t *pudp; in __split_linear_mapping_pud() local 142 pudp = pud_offset(p4dp, vaddr); in __split_linear_mapping_pud() 151 if (pud_leaf(*pudp)) { in __split_linear_mapping_pud() 153 unsigned long pfn = _pud_pfn(*pudp); in __split_linear_mapping_pud() 154 pgprot_t prot = __pgprot(pud_val(*pudp) & ~_PAGE_PFN_MASK); in __split_linear_mapping_pud() 169 set_pud(pudp, pfn_pud(page_to_pfn(pmd_page), PAGE_TABLE)); in __split_linear_mapping_pud() 172 ret = __split_linear_mapping_pmd(pudp, vaddr, next); in __split_linear_mapping_pud() 175 } while (pudp++, vaddr = next, vaddr != end); in __split_linear_mapping_pud()
|
/linux-6.6.21/arch/loongarch/mm/ |
D | kasan_init.c | 119 static pmd_t *__init kasan_pmd_offset(pud_t *pudp, unsigned long addr, int node, bool early) in kasan_pmd_offset() argument 121 if (__pud_none(early, READ_ONCE(*pudp))) { in kasan_pmd_offset() 126 pud_populate(&init_mm, pudp, (pmd_t *)__va(pmd_phys)); in kasan_pmd_offset() 129 return pmd_offset(pudp, addr); in kasan_pmd_offset() 160 static void __init kasan_pmd_populate(pud_t *pudp, unsigned long addr, in kasan_pmd_populate() argument 164 pmd_t *pmdp = kasan_pmd_offset(pudp, addr, node, early); in kasan_pmd_populate() 176 pud_t *pudp = kasan_pud_offset(p4dp, addr, node, early); in kasan_pud_populate() local 180 kasan_pmd_populate(pudp, addr, next, node, early); in kasan_pud_populate() 181 } while (pudp++, addr = next, addr != end); in kasan_pud_populate()
|
/linux-6.6.21/mm/ |
D | debug_vm_pgtable.c | 66 pud_t *pudp; member 371 set_pud_at(args->mm, vaddr, args->pudp, pud); in pud_advanced_tests() 373 pudp_set_wrprotect(args->mm, vaddr, args->pudp); in pud_advanced_tests() 374 pud = READ_ONCE(*args->pudp); in pud_advanced_tests() 378 pudp_huge_get_and_clear(args->mm, vaddr, args->pudp); in pud_advanced_tests() 379 pud = READ_ONCE(*args->pudp); in pud_advanced_tests() 386 set_pud_at(args->mm, vaddr, args->pudp, pud); in pud_advanced_tests() 390 pudp_set_access_flags(args->vma, vaddr, args->pudp, pud, 1); in pud_advanced_tests() 391 pud = READ_ONCE(*args->pudp); in pud_advanced_tests() 395 pudp_huge_get_and_clear_full(args->vma, vaddr, args->pudp, 1); in pud_advanced_tests() [all …]
|
/linux-6.6.21/arch/arc/include/asm/ |
D | pgalloc.h | 75 static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4dp, pud_t *pudp) in p4d_populate() argument 77 set_p4d(p4dp, __p4d((unsigned long)pudp)); in p4d_populate() 86 static inline void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmdp) in pud_populate() argument 88 set_pud(pudp, __pud((unsigned long)pmdp)); in pud_populate()
|
/linux-6.6.21/include/trace/events/ |
D | xen.h | 207 TP_PROTO(pud_t *pudp, pud_t pudval), 208 TP_ARGS(pudp, pudval), 210 __field(pud_t *, pudp) 213 TP_fast_assign(__entry->pudp = pudp; 216 __entry->pudp, 242 TP_PROTO(pud_t *pudp, pud_t pudval), 243 TP_ARGS(pudp, pudval), 245 __field(pud_t *, pudp) 248 TP_fast_assign(__entry->pudp = pudp; 251 __entry->pudp,
|
/linux-6.6.21/arch/arm/include/asm/ |
D | pgtable-3level.h | 124 #define pud_clear(pudp) \ argument 126 *pudp = __pud(0); \ 127 clean_pmd_entry(pudp); \ 130 #define set_pud(pudp, pud) \ argument 132 *pudp = pud; \ 133 flush_pmd_entry(pudp); \
|
/linux-6.6.21/include/linux/ |
D | page_table_check.h | 23 void __page_table_check_pud_set(struct mm_struct *mm, pud_t *pudp, pud_t pud); 86 static inline void page_table_check_pud_set(struct mm_struct *mm, pud_t *pudp, in page_table_check_pud_set() argument 92 __page_table_check_pud_set(mm, pudp, pud); in page_table_check_pud_set() 137 static inline void page_table_check_pud_set(struct mm_struct *mm, pud_t *pudp, in page_table_check_pud_set() argument
|
D | pgtable.h | 261 unsigned long address, pud_t *pudp, 272 unsigned long address, pud_t *pudp, in pudp_set_access_flags() argument 514 pud_t *pudp) in pudp_huge_get_and_clear() argument 516 pud_t pud = *pudp; in pudp_huge_get_and_clear() 518 pud_clear(pudp); in pudp_huge_get_and_clear() 538 unsigned long address, pud_t *pudp, in pudp_huge_get_and_clear_full() argument 541 return pudp_huge_get_and_clear(vma->vm_mm, address, pudp); in pudp_huge_get_and_clear_full() 599 pud_t *pudp); 661 unsigned long address, pud_t *pudp) in pudp_set_wrprotect() argument 663 pud_t old_pud = *pudp; in pudp_set_wrprotect() [all …]
|
/linux-6.6.21/include/asm-generic/ |
D | tlb.h | 644 #define __tlb_remove_pud_tlb_entry(tlb, pudp, address) do {} while (0) argument 647 #define tlb_remove_pud_tlb_entry(tlb, pudp, address) \ argument 650 __tlb_remove_pud_tlb_entry(tlb, pudp, address); \ 690 #define pud_free_tlb(tlb, pudp, address) \ argument 694 __pud_free_tlb(tlb, pudp, address); \ 699 #define p4d_free_tlb(tlb, pudp, address) \ argument 703 __p4d_free_tlb(tlb, pudp, address); \
|
/linux-6.6.21/arch/arm/mm/ |
D | kasan_init.c | 87 static void __init kasan_pmd_populate(pud_t *pudp, unsigned long addr, in kasan_pmd_populate() argument 91 pmd_t *pmdp = pmd_offset(pudp, addr); in kasan_pmd_populate() 123 pud_t *pudp; in kasan_pgd_populate() local 150 pudp = pud_offset(p4dp, addr); in kasan_pgd_populate() 152 kasan_pmd_populate(pudp, addr, next, early); in kasan_pgd_populate()
|
/linux-6.6.21/arch/powerpc/include/asm/book3s/64/ |
D | pgtable.h | 887 static inline void pud_clear(pud_t *pudp) in pud_clear() argument 894 WARN_ON((pud_val(*pudp) & (H_PAGE_HASHPTE | _PAGE_PTE)) == (H_PAGE_HASHPTE | _PAGE_PTE)); in pud_clear() 896 *pudp = __pud(0); in pud_clear() 1141 pud_t *pudp, pud_t pud); 1180 pud_hugepage_update(struct mm_struct *mm, unsigned long addr, pud_t *pudp, in pud_hugepage_update() argument 1184 return radix__pud_hugepage_update(mm, addr, pudp, clr, set); in pud_hugepage_update() 1186 return pud_val(*pudp); in pud_hugepage_update() 1219 unsigned long addr, pud_t *pudp) in __pudp_test_and_clear_young() argument 1223 if ((pud_raw(*pudp) & cpu_to_be64(_PAGE_ACCESSED | H_PAGE_HASHPTE)) == 0) in __pudp_test_and_clear_young() 1225 old = pud_hugepage_update(mm, addr, pudp, _PAGE_ACCESSED, 0); in __pudp_test_and_clear_young() [all …]
|