/linux-6.6.21/include/asm-generic/ |
D | pgtable-nopud.h | 16 typedef struct { p4d_t p4d; } pud_t; member 28 static inline int p4d_none(p4d_t p4d) { return 0; } in p4d_none() argument 29 static inline int p4d_bad(p4d_t p4d) { return 0; } in p4d_bad() argument 30 static inline int p4d_present(p4d_t p4d) { return 1; } in p4d_present() argument 31 static inline void p4d_clear(p4d_t *p4d) { } in p4d_clear() argument 32 #define pud_ERROR(pud) (p4d_ERROR((pud).p4d)) 34 #define p4d_populate(mm, p4d, pud) do { } while (0) argument 35 #define p4d_populate_safe(mm, p4d, pud) do { } while (0) argument 42 static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address) in pud_offset() argument 44 return (pud_t *)p4d; in pud_offset() [all …]
|
/linux-6.6.21/mm/kasan/ |
D | init.c | 45 static inline bool kasan_pud_table(p4d_t p4d) in kasan_pud_table() argument 47 return p4d_page(p4d) == virt_to_page(lm_alias(kasan_early_shadow_pud)); in kasan_pud_table() 50 static inline bool kasan_pud_table(p4d_t p4d) in kasan_pud_table() argument 146 static int __ref zero_pud_populate(p4d_t *p4d, unsigned long addr, in zero_pud_populate() argument 149 pud_t *pud = pud_offset(p4d, addr); in zero_pud_populate() 191 p4d_t *p4d = p4d_offset(pgd, addr); in zero_p4d_populate() local 200 p4d_populate(&init_mm, p4d, in zero_p4d_populate() 202 pud = pud_offset(p4d, addr); in zero_p4d_populate() 211 if (p4d_none(*p4d)) { in zero_p4d_populate() 215 p = pud_alloc(&init_mm, p4d, addr); in zero_p4d_populate() [all …]
|
/linux-6.6.21/arch/riscv/include/asm/ |
D | pgtable-64.h | 48 unsigned long p4d; member 51 #define p4d_val(x) ((x).p4d) 274 static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) in set_p4d() argument 277 *p4dp = p4d; in set_p4d() 279 set_pud((pud_t *)p4dp, (pud_t){ p4d_val(p4d) }); in set_p4d() 282 static inline int p4d_none(p4d_t p4d) in p4d_none() argument 285 return (p4d_val(p4d) == 0); in p4d_none() 290 static inline int p4d_present(p4d_t p4d) in p4d_present() argument 293 return (p4d_val(p4d) & _PAGE_PRESENT); in p4d_present() 298 static inline int p4d_bad(p4d_t p4d) in p4d_bad() argument [all …]
|
D | pgalloc.h | 42 static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud) in p4d_populate() argument 47 set_p4d(p4d, __p4d((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE)); in p4d_populate() 51 static inline void p4d_populate_safe(struct mm_struct *mm, p4d_t *p4d, in p4d_populate_safe() argument 57 set_p4d_safe(p4d, in p4d_populate_safe() 62 static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d) in pgd_populate() argument 65 unsigned long pfn = virt_to_pfn(p4d); in pgd_populate() 72 p4d_t *p4d) in pgd_populate_safe() argument 75 unsigned long pfn = virt_to_pfn(p4d); in pgd_populate_safe() 114 static inline void __p4d_free(struct mm_struct *mm, p4d_t *p4d) in __p4d_free() argument 116 BUG_ON((unsigned long)p4d & (PAGE_SIZE-1)); in __p4d_free() [all …]
|
/linux-6.6.21/arch/powerpc/include/asm/nohash/64/ |
D | pgtable-4k.h | 56 #define p4d_none(p4d) (!p4d_val(p4d)) argument 57 #define p4d_bad(p4d) (p4d_val(p4d) == 0) argument 58 #define p4d_present(p4d) (p4d_val(p4d) != 0) argument 62 static inline pud_t *p4d_pgtable(p4d_t p4d) in p4d_pgtable() argument 64 return (pud_t *) (p4d_val(p4d) & ~P4D_MASKED_BITS); in p4d_pgtable() 72 static inline pte_t p4d_pte(p4d_t p4d) in p4d_pte() argument 74 return __pte(p4d_val(p4d)); in p4d_pte() 81 extern struct page *p4d_page(p4d_t p4d);
|
/linux-6.6.21/arch/x86/include/asm/ |
D | pgalloc.h | 113 static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud) in p4d_populate() argument 116 set_p4d(p4d, __p4d(_PAGE_TABLE | __pa(pud))); in p4d_populate() 119 static inline void p4d_populate_safe(struct mm_struct *mm, p4d_t *p4d, pud_t *pud) in p4d_populate_safe() argument 122 set_p4d_safe(p4d, __p4d(_PAGE_TABLE | __pa(pud))); in p4d_populate_safe() 134 static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d) in pgd_populate() argument 138 paravirt_alloc_p4d(mm, __pa(p4d) >> PAGE_SHIFT); in pgd_populate() 139 set_pgd(pgd, __pgd(_PAGE_TABLE | __pa(p4d))); in pgd_populate() 142 static inline void pgd_populate_safe(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d) in pgd_populate_safe() argument 146 paravirt_alloc_p4d(mm, __pa(p4d) >> PAGE_SHIFT); in pgd_populate_safe() 147 set_pgd_safe(pgd, __pgd(_PAGE_TABLE | __pa(p4d))); in pgd_populate_safe() [all …]
|
D | pgtable_types.h | 341 typedef struct { p4dval_t p4d; } p4d_t; member 348 static inline p4dval_t native_p4d_val(p4d_t p4d) in native_p4d_val() argument 350 return p4d.p4d; in native_p4d_val() 360 static inline p4dval_t native_p4d_val(p4d_t p4d) in native_p4d_val() argument 362 return native_pgd_val(p4d.pgd); in native_p4d_val() 383 return (pud_t) { .p4d.pgd = native_make_pgd(val) }; in native_make_pud() 388 return native_pgd_val(pud.p4d.pgd); in native_pud_val() 407 return (pmd_t) { .pud.p4d.pgd = native_make_pgd(val) }; in native_make_pmd() 412 return native_pgd_val(pmd.pud.p4d.pgd); in native_pmd_val() 416 static inline p4dval_t p4d_pfn_mask(p4d_t p4d) in p4d_pfn_mask() argument [all …]
|
/linux-6.6.21/arch/x86/mm/ |
D | init_64.c | 73 DEFINE_POPULATE(p4d_populate, p4d, pud, init) in DEFINE_POPULATE() argument 74 DEFINE_POPULATE(pgd_populate, pgd, p4d, init) in DEFINE_POPULATE() 88 DEFINE_ENTRY(p4d, p4d, init) 192 p4d_t *p4d; in sync_global_pgds_l4() local 196 p4d = p4d_offset(pgd, addr); in sync_global_pgds_l4() 201 if (!p4d_none(*p4d_ref) && !p4d_none(*p4d)) in sync_global_pgds_l4() 202 BUG_ON(p4d_pgtable(*p4d) in sync_global_pgds_l4() 205 if (p4d_none(*p4d)) in sync_global_pgds_l4() 206 set_p4d(p4d, *p4d_ref); in sync_global_pgds_l4() 252 p4d_t *p4d = (p4d_t *)spp_getpage(); in fill_p4d() local [all …]
|
D | ident_map.c | 81 p4d_t *p4d = p4d_page + p4d_index(addr); in ident_p4d_init() local 88 if (p4d_present(*p4d)) { in ident_p4d_init() 89 pud = pud_offset(p4d, 0); in ident_p4d_init() 104 set_p4d(p4d, __p4d(__pa(pud) | info->kernpg_flag)); in ident_p4d_init() 127 p4d_t *p4d; in kernel_ident_mapping_init() local 134 p4d = p4d_offset(pgd, 0); in kernel_ident_mapping_init() 135 result = ident_p4d_init(info, p4d, addr, next); in kernel_ident_mapping_init() 141 p4d = (p4d_t *)info->alloc_pgt_page(info->context); in kernel_ident_mapping_init() 142 if (!p4d) in kernel_ident_mapping_init() 144 result = ident_p4d_init(info, p4d, addr, next); in kernel_ident_mapping_init() [all …]
|
D | kasan_init_64.c | 103 static void __init kasan_populate_p4d(p4d_t *p4d, unsigned long addr, in kasan_populate_p4d() argument 109 if (p4d_none(*p4d)) { in kasan_populate_p4d() 112 p4d_populate(&init_mm, p4d, p); in kasan_populate_p4d() 115 pud = pud_offset(p4d, addr); in kasan_populate_p4d() 127 p4d_t *p4d; in kasan_populate_pgd() local 135 p4d = p4d_offset(pgd, addr); in kasan_populate_pgd() 138 kasan_populate_p4d(p4d, addr, next, nid); in kasan_populate_pgd() 139 } while (p4d++, addr = next, addr != end); in kasan_populate_pgd() 194 unsigned long p4d; in early_p4d_offset() local 199 p4d = pgd_val(*pgd) & PTE_PFN_MASK; in early_p4d_offset() [all …]
|
D | pti.c | 202 p4d_t *p4d; in pti_user_pagetable_walk_pmd() local 205 p4d = pti_user_pagetable_walk_p4d(address); in pti_user_pagetable_walk_pmd() 206 if (!p4d) in pti_user_pagetable_walk_pmd() 209 BUILD_BUG_ON(p4d_large(*p4d) != 0); in pti_user_pagetable_walk_pmd() 210 if (p4d_none(*p4d)) { in pti_user_pagetable_walk_pmd() 215 set_p4d(p4d, __p4d(_KERNPG_TABLE | __pa(new_pud_page))); in pti_user_pagetable_walk_pmd() 218 pud = pud_offset(p4d, address); in pti_user_pagetable_walk_pmd() 316 p4d_t *p4d; in pti_clone_pgtable() local 326 p4d = p4d_offset(pgd, addr); in pti_clone_pgtable() 327 if (WARN_ON(p4d_none(*p4d))) in pti_clone_pgtable() [all …]
|
/linux-6.6.21/arch/sh/mm/ |
D | hugetlbpage.c | 28 p4d_t *p4d; in huge_pte_alloc() local 35 p4d = p4d_alloc(mm, pgd, addr); in huge_pte_alloc() 36 if (p4d) { in huge_pte_alloc() 37 pud = pud_alloc(mm, p4d, addr); in huge_pte_alloc() 53 p4d_t *p4d; in huge_pte_offset() local 60 p4d = p4d_offset(pgd, addr); in huge_pte_offset() 61 if (p4d) { in huge_pte_offset() 62 pud = pud_offset(p4d, addr); in huge_pte_offset()
|
/linux-6.6.21/arch/s390/mm/ |
D | vmem.c | 308 static int modify_pud_table(p4d_t *p4d, unsigned long addr, unsigned long end, in modify_pud_table() argument 319 pud = pud_offset(p4d, addr); in modify_pud_table() 362 static void try_free_pud_table(p4d_t *p4d, unsigned long start) in try_free_pud_table() argument 367 pud = pud_offset(p4d, start); in try_free_pud_table() 372 vmem_free_pages(p4d_deref(*p4d), CRST_ALLOC_ORDER); in try_free_pud_table() 373 p4d_clear(p4d); in try_free_pud_table() 381 p4d_t *p4d; in modify_p4d_table() local 384 p4d = p4d_offset(pgd, addr); in modify_p4d_table() 385 for (; addr < end; addr = next, p4d++) { in modify_p4d_table() 388 if (p4d_none(*p4d)) in modify_p4d_table() [all …]
|
D | page-states.c | 110 static void mark_kernel_pud(p4d_t *p4d, unsigned long addr, unsigned long end) in mark_kernel_pud() argument 117 pud = pud_offset(p4d, addr); in mark_kernel_pud() 135 p4d_t *p4d; in mark_kernel_p4d() local 138 p4d = p4d_offset(pgd, addr); in mark_kernel_p4d() 141 if (p4d_none(*p4d)) in mark_kernel_p4d() 143 if (!p4d_folded(*p4d)) { in mark_kernel_p4d() 144 page = phys_to_page(p4d_val(*p4d)); in mark_kernel_p4d() 148 mark_kernel_pud(p4d, addr, next); in mark_kernel_p4d() 149 } while (p4d++, addr = next, addr != end); in mark_kernel_p4d()
|
/linux-6.6.21/arch/mips/include/asm/ |
D | pgtable-64.h | 184 static inline int p4d_none(p4d_t p4d) in p4d_none() argument 186 return p4d_val(p4d) == (unsigned long)invalid_pud_table; in p4d_none() 189 static inline int p4d_bad(p4d_t p4d) in p4d_bad() argument 191 if (unlikely(p4d_val(p4d) & ~PAGE_MASK)) in p4d_bad() 197 static inline int p4d_present(p4d_t p4d) in p4d_present() argument 199 return p4d_val(p4d) != (unsigned long)invalid_pud_table; in p4d_present() 207 static inline pud_t *p4d_pgtable(p4d_t p4d) in p4d_pgtable() argument 209 return (pud_t *)p4d_val(p4d); in p4d_pgtable() 212 #define p4d_phys(p4d) virt_to_phys((void *)p4d_val(p4d)) argument 213 #define p4d_page(p4d) (pfn_to_page(p4d_phys(p4d) >> PAGE_SHIFT)) argument [all …]
|
/linux-6.6.21/arch/mips/mm/ |
D | hugetlbpage.c | 28 p4d_t *p4d; in huge_pte_alloc() local 33 p4d = p4d_alloc(mm, pgd, addr); in huge_pte_alloc() 34 pud = pud_alloc(mm, p4d, addr); in huge_pte_alloc() 45 p4d_t *p4d; in huge_pte_offset() local 51 p4d = p4d_offset(pgd, addr); in huge_pte_offset() 52 if (p4d_present(*p4d)) { in huge_pte_offset() 53 pud = pud_offset(p4d, addr); in huge_pte_offset()
|
/linux-6.6.21/arch/arm/mm/ |
D | pgd.c | 145 p4d_t *p4d; in pgd_free() local 157 p4d = p4d_offset(pgd, 0); in pgd_free() 158 if (p4d_none_or_clear_bad(p4d)) in pgd_free() 161 pud = pud_offset(p4d, 0); in pgd_free() 178 p4d_clear(p4d); in pgd_free() 182 p4d_free(mm, p4d); in pgd_free() 193 p4d = p4d_offset(pgd, 0); in pgd_free() 194 if (p4d_none_or_clear_bad(p4d)) in pgd_free() 196 pud = pud_offset(p4d, 0); in pgd_free() 203 p4d_clear(p4d); in pgd_free() [all …]
|
/linux-6.6.21/arch/loongarch/mm/ |
D | hugetlbpage.c | 20 p4d_t *p4d; in huge_pte_alloc() local 25 p4d = p4d_alloc(mm, pgd, addr); in huge_pte_alloc() 26 pud = pud_alloc(mm, p4d, addr); in huge_pte_alloc() 37 p4d_t *p4d; in huge_pte_offset() local 43 p4d = p4d_offset(pgd, addr); in huge_pte_offset() 44 if (p4d_present(*p4d)) { in huge_pte_offset() 45 pud = pud_offset(p4d, addr); in huge_pte_offset()
|
/linux-6.6.21/arch/x86/power/ |
D | hibernate_64.c | 32 p4d_t *p4d = NULL; in set_up_temporary_text_mapping() local 55 p4d = (p4d_t *)get_safe_page(GFP_ATOMIC); in set_up_temporary_text_mapping() 56 if (!p4d) in set_up_temporary_text_mapping() 72 if (p4d) { in set_up_temporary_text_mapping() 74 pgd_t new_pgd = __pgd(__pa(p4d) | pgprot_val(pgtable_prot)); in set_up_temporary_text_mapping() 76 set_p4d(p4d + p4d_index(restore_jump_address), new_p4d); in set_up_temporary_text_mapping()
|
/linux-6.6.21/mm/ |
D | sparse-vmemmap.c | 203 pud_t * __meminit vmemmap_pud_populate(p4d_t *p4d, unsigned long addr, int node) in vmemmap_pud_populate() argument 205 pud_t *pud = pud_offset(p4d, addr); in vmemmap_pud_populate() 222 p4d_t *p4d = p4d_offset(pgd, addr); in vmemmap_p4d_populate() local 223 if (p4d_none(*p4d)) { in vmemmap_p4d_populate() 228 p4d_populate(&init_mm, p4d, p); in vmemmap_p4d_populate() 230 return p4d; in vmemmap_p4d_populate() 250 p4d_t *p4d; in vmemmap_populate_address() local 258 p4d = vmemmap_p4d_populate(pgd, addr, node); in vmemmap_populate_address() 259 if (!p4d) in vmemmap_populate_address() 261 pud = vmemmap_pud_populate(p4d, addr, node); in vmemmap_populate_address() [all …]
|
/linux-6.6.21/arch/s390/boot/ |
D | vmem.c | 134 static bool kasan_p4d_populate_zero_shadow(p4d_t *p4d, unsigned long addr, in kasan_p4d_populate_zero_shadow() argument 139 p4d_populate(&init_mm, p4d, kasan_early_shadow_pud); in kasan_p4d_populate_zero_shadow() 187 static inline bool kasan_p4d_populate_zero_shadow(p4d_t *p4d, unsigned long addr, in kasan_p4d_populate_zero_shadow() argument 339 static void pgtable_pud_populate(p4d_t *p4d, unsigned long addr, unsigned long end, in pgtable_pud_populate() argument 346 pud = pud_offset(p4d, addr); in pgtable_pud_populate() 376 p4d_t *p4d; in pgtable_p4d_populate() local 379 p4d = p4d_offset(pgd, addr); in pgtable_p4d_populate() 380 for (; addr < end; addr = next, p4d++) { in pgtable_p4d_populate() 382 if (p4d_none(*p4d)) { in pgtable_p4d_populate() 383 if (kasan_p4d_populate_zero_shadow(p4d, addr, next, mode)) in pgtable_p4d_populate() [all …]
|
/linux-6.6.21/arch/ia64/mm/ |
D | hugetlbpage.c | 33 p4d_t *p4d; in huge_pte_alloc() local 39 p4d = p4d_offset(pgd, taddr); in huge_pte_alloc() 40 pud = pud_alloc(mm, p4d, taddr); in huge_pte_alloc() 54 p4d_t *p4d; in huge_pte_offset() local 61 p4d = p4d_offset(pgd, taddr); in huge_pte_offset() 62 if (p4d_present(*p4d)) { in huge_pte_offset() 63 pud = pud_offset(p4d, taddr); in huge_pte_offset()
|
/linux-6.6.21/arch/sparc/mm/ |
D | hugetlbpage.c | 285 p4d_t *p4d; in huge_pte_alloc() local 290 p4d = p4d_offset(pgd, addr); in huge_pte_alloc() 291 pud = pud_alloc(mm, p4d, addr); in huge_pte_alloc() 308 p4d_t *p4d; in huge_pte_offset() local 315 p4d = p4d_offset(pgd, addr); in huge_pte_offset() 316 if (p4d_none(*p4d)) in huge_pte_offset() 318 pud = pud_offset(p4d, addr); in huge_pte_offset() 469 static void hugetlb_free_pud_range(struct mmu_gather *tlb, p4d_t *p4d, in hugetlb_free_pud_range() argument 478 pud = pud_offset(p4d, addr); in hugetlb_free_pud_range() 501 pud = pud_offset(p4d, start); in hugetlb_free_pud_range() [all …]
|
/linux-6.6.21/arch/parisc/mm/ |
D | hugetlbpage.c | 51 p4d_t *p4d; in huge_pte_alloc() local 64 p4d = p4d_offset(pgd, addr); in huge_pte_alloc() 65 pud = pud_alloc(mm, p4d, addr); in huge_pte_alloc() 78 p4d_t *p4d; in huge_pte_offset() local 87 p4d = p4d_offset(pgd, addr); in huge_pte_offset() 88 if (!p4d_none(*p4d)) { in huge_pte_offset() 89 pud = pud_offset(p4d, addr); in huge_pte_offset()
|
/linux-6.6.21/arch/powerpc/mm/ |
D | pgtable_64.c | 101 struct page *p4d_page(p4d_t p4d) in p4d_page() argument 103 if (p4d_is_leaf(p4d)) { in p4d_page() 105 VM_WARN_ON(!p4d_huge(p4d)); in p4d_page() 106 return pte_page(p4d_pte(p4d)); in p4d_page() 108 return virt_to_page(p4d_pgtable(p4d)); in p4d_page()
|