| /linux-6.6.21/mm/ |
| D | pgtable-generic.c | 110 unsigned long address, pmd_t *pmdp, in pmdp_set_access_flags() argument
113 int changed = !pmd_same(*pmdp, entry); in pmdp_set_access_flags()
116 set_pmd_at(vma->vm_mm, address, pmdp, entry); in pmdp_set_access_flags()
125 unsigned long address, pmd_t *pmdp) in pmdp_clear_flush_young() argument
129 young = pmdp_test_and_clear_young(vma, address, pmdp); in pmdp_clear_flush_young()
138 pmd_t *pmdp) in pmdp_huge_clear_flush() argument
142 VM_BUG_ON(pmd_present(*pmdp) && !pmd_trans_huge(*pmdp) && in pmdp_huge_clear_flush()
143 !pmd_devmap(*pmdp)); in pmdp_huge_clear_flush()
144 pmd = pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp); in pmdp_huge_clear_flush()
165 void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, in pgtable_trans_huge_deposit() argument
[all …]
|
| /linux-6.6.21/arch/powerpc/mm/book3s64/ |
| D | hash_pgtable.c | 152 pmd_t *pmdp; in hash__map_kernel_page() local
162 pmdp = pmd_alloc(&init_mm, pudp, ea); in hash__map_kernel_page()
163 if (!pmdp) in hash__map_kernel_page()
165 ptep = pte_alloc_kernel(pmdp, ea); in hash__map_kernel_page()
191 pmd_t *pmdp, unsigned long clr, in hash__pmd_hugepage_update() argument
198 WARN_ON(!hash__pmd_trans_huge(*pmdp) && !pmd_devmap(*pmdp)); in hash__pmd_hugepage_update()
199 assert_spin_locked(pmd_lockptr(mm, pmdp)); in hash__pmd_hugepage_update()
210 : "=&r" (old_be), "=&r" (tmp), "=m" (*pmdp) in hash__pmd_hugepage_update()
211 : "r" (pmdp), "r" (cpu_to_be64(clr)), "m" (*pmdp), in hash__pmd_hugepage_update()
219 hpte_do_hugepage_flush(mm, addr, pmdp, old); in hash__pmd_hugepage_update()
[all …]
|
| D | radix_pgtable.c | 79 pmd_t *pmdp; in early_map_kernel_page() local
95 pmdp = early_alloc_pgtable(PAGE_SIZE, nid, region_start, in early_map_kernel_page()
97 pud_populate(&init_mm, pudp, pmdp); in early_map_kernel_page()
99 pmdp = pmd_offset(pudp, ea); in early_map_kernel_page()
101 ptep = pmdp_ptep(pmdp); in early_map_kernel_page()
104 if (!pmd_present(*pmdp)) { in early_map_kernel_page()
107 pmd_populate_kernel(&init_mm, pmdp, ptep); in early_map_kernel_page()
109 ptep = pte_offset_kernel(pmdp, ea); in early_map_kernel_page()
131 pmd_t *pmdp; in __map_kernel_page() local
160 pmdp = pmd_alloc(&init_mm, pudp, ea); in __map_kernel_page()
[all …]
|
| D | pgtable.c | 49 pmd_t *pmdp, pmd_t entry, int dirty) in pmdp_set_access_flags() argument
53 WARN_ON(!pmd_trans_huge(*pmdp) && !pmd_devmap(*pmdp)); in pmdp_set_access_flags()
54 assert_spin_locked(pmd_lockptr(vma->vm_mm, pmdp)); in pmdp_set_access_flags()
56 changed = !pmd_same(*(pmdp), entry); in pmdp_set_access_flags()
62 __ptep_set_access_flags(vma, pmdp_ptep(pmdp), in pmdp_set_access_flags()
90 unsigned long address, pmd_t *pmdp) in pmdp_test_and_clear_young() argument
92 return __pmdp_test_and_clear_young(vma->vm_mm, address, pmdp); in pmdp_test_and_clear_young()
106 pmd_t *pmdp, pmd_t pmd) in set_pmd_at() argument
114 WARN_ON(pte_hw_valid(pmd_pte(*pmdp)) && !pte_protnone(pmd_pte(*pmdp))); in set_pmd_at()
115 assert_spin_locked(pmd_lockptr(mm, pmdp)); in set_pmd_at()
[all …]
|
| D | hash_hugepage.c | 22 pmd_t *pmdp, unsigned long trap, unsigned long flags, in __hash_page_thp() argument
36 pmd_t pmd = READ_ONCE(*pmdp); in __hash_page_thp()
52 } while (!pmd_xchg(pmdp, __pmd(old_pmd), __pmd(new_pmd))); in __hash_page_thp()
80 hpte_slot_array = get_hpte_slot_array(pmdp); in __hash_page_thp()
88 flush_hash_hugepage(vsid, ea, pmdp, MMU_PAGE_64K, in __hash_page_thp()
165 *pmdp = __pmd(old_pmd); in __hash_page_thp()
189 *pmdp = __pmd(new_pmd & ~H_PAGE_BUSY); in __hash_page_thp()
|
| /linux-6.6.21/arch/arm64/include/asm/ |
| D | pgalloc.h | 23 static inline void __pud_populate(pud_t *pudp, phys_addr_t pmdp, pudval_t prot) in __pud_populate() argument
25 set_pud(pudp, __pud(__phys_to_pud_val(pmdp) | prot)); in __pud_populate()
28 static inline void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmdp) in pud_populate() argument
33 __pud_populate(pudp, __pa(pmdp), pudval); in pud_populate()
36 static inline void __pud_populate(pud_t *pudp, phys_addr_t pmdp, pudval_t prot) in __pud_populate() argument
66 static inline void __pmd_populate(pmd_t *pmdp, phys_addr_t ptep, in __pmd_populate() argument
69 set_pmd(pmdp, __pmd(__phys_to_pmd_val(ptep) | prot)); in __pmd_populate()
77 pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, pte_t *ptep) in pmd_populate_kernel() argument
80 __pmd_populate(pmdp, __pa(ptep), PMD_TYPE_TABLE | PMD_TABLE_UXN); in pmd_populate_kernel()
84 pmd_populate(struct mm_struct *mm, pmd_t *pmdp, pgtable_t ptep) in pmd_populate() argument
[all …]
|
| /linux-6.6.21/arch/powerpc/mm/nohash/ |
| D | 40x.c | 106 pmd_t *pmdp; in mmu_mapin_ram() local
109 pmdp = pmd_off_k(v); in mmu_mapin_ram()
110 *pmdp++ = __pmd(val); in mmu_mapin_ram()
111 *pmdp++ = __pmd(val); in mmu_mapin_ram()
112 *pmdp++ = __pmd(val); in mmu_mapin_ram()
113 *pmdp++ = __pmd(val); in mmu_mapin_ram()
121 pmd_t *pmdp; in mmu_mapin_ram() local
124 pmdp = pmd_off_k(v); in mmu_mapin_ram()
125 *pmdp = __pmd(val); in mmu_mapin_ram()
|
| D | book3e_pgtable.c | 79 pmd_t *pmdp; in map_kernel_page() local
89 pmdp = pmd_alloc(&init_mm, pudp, ea); in map_kernel_page()
90 if (!pmdp) in map_kernel_page()
92 ptep = pte_alloc_kernel(pmdp, ea); in map_kernel_page()
104 pmdp = early_alloc_pgtable(PMD_TABLE_SIZE); in map_kernel_page()
105 pud_populate(&init_mm, pudp, pmdp); in map_kernel_page()
107 pmdp = pmd_offset(pudp, ea); in map_kernel_page()
108 if (!pmd_present(*pmdp)) { in map_kernel_page()
110 pmd_populate_kernel(&init_mm, pmdp, ptep); in map_kernel_page()
112 ptep = pte_offset_kernel(pmdp, ea); in map_kernel_page()
|
| D | 8xx.c | 49 static pte_t __init *early_hugepd_alloc_kernel(hugepd_t *pmdp, unsigned long va) in early_hugepd_alloc_kernel() argument
51 if (hpd_val(*pmdp) == 0) { in early_hugepd_alloc_kernel()
57 hugepd_populate_kernel((hugepd_t *)pmdp, ptep, PAGE_SHIFT_8M); in early_hugepd_alloc_kernel()
58 hugepd_populate_kernel((hugepd_t *)pmdp + 1, ptep, PAGE_SHIFT_8M); in early_hugepd_alloc_kernel()
60 return hugepte_offset(*(hugepd_t *)pmdp, va, PGDIR_SHIFT); in early_hugepd_alloc_kernel()
66 pmd_t *pmdp = pmd_off_k(va); in __early_map_kernel_hugepage() local
77 ptep = early_pte_alloc_kernel(pmdp, va); in __early_map_kernel_hugepage()
79 ptep = early_hugepd_alloc_kernel((hugepd_t *)pmdp, va); in __early_map_kernel_hugepage()
82 ptep = pte_offset_kernel(pmdp, va); in __early_map_kernel_hugepage()
84 ptep = hugepte_offset(*(hugepd_t *)pmdp, va, PGDIR_SHIFT); in __early_map_kernel_hugepage()
|
| /linux-6.6.21/arch/sparc/mm/ |
| D | tlb.c | 223 pmd_t *pmdp, pmd_t pmd) in set_pmd_at() argument
225 pmd_t orig = *pmdp; in set_pmd_at()
227 *pmdp = pmd; in set_pmd_at()
232 unsigned long address, pmd_t *pmdp, pmd_t pmd) in pmdp_establish() argument
237 old = *pmdp; in pmdp_establish()
238 } while (cmpxchg64(&pmdp->pmd, old.pmd, pmd.pmd) != old.pmd); in pmdp_establish()
248 pmd_t *pmdp) in pmdp_invalidate() argument
252 entry = __pmd(pmd_val(*pmdp) & ~_PAGE_VALID); in pmdp_invalidate()
253 old = pmdp_establish(vma, address, pmdp, entry); in pmdp_invalidate()
268 void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, in pgtable_trans_huge_deposit() argument
[all …]
|
| /linux-6.6.21/arch/s390/mm/ |
| D | pgtable.c | 386 unsigned long addr, pmd_t *pmdp) in pmdp_idte_local() argument
389 __pmdp_idte(addr, pmdp, IDTE_NODAT | IDTE_GUEST_ASCE, in pmdp_idte_local()
392 __pmdp_idte(addr, pmdp, 0, 0, IDTE_LOCAL); in pmdp_idte_local()
398 unsigned long addr, pmd_t *pmdp) in pmdp_idte_global() argument
401 __pmdp_idte(addr, pmdp, IDTE_NODAT | IDTE_GUEST_ASCE, in pmdp_idte_global()
406 __pmdp_idte(addr, pmdp, 0, 0, IDTE_GLOBAL); in pmdp_idte_global()
410 __pmdp_csp(pmdp); in pmdp_idte_global()
417 unsigned long addr, pmd_t *pmdp) in pmdp_flush_direct() argument
421 old = *pmdp; in pmdp_flush_direct()
427 pmdp_idte_local(mm, addr, pmdp); in pmdp_flush_direct()
[all …]
|
| D | pageattr.c | 86 static int walk_pte_level(pmd_t *pmdp, unsigned long addr, unsigned long end, in walk_pte_level() argument
93 ptep = pte_offset_kernel(pmdp, addr); in walk_pte_level()
122 static int split_pmd_page(pmd_t *pmdp, unsigned long addr) in split_pmd_page() argument
132 pte_addr = pmd_pfn(*pmdp) << PAGE_SHIFT; in split_pmd_page()
133 ro = !!(pmd_val(*pmdp) & _SEGMENT_ENTRY_PROTECT); in split_pmd_page()
134 nx = !!(pmd_val(*pmdp) & _SEGMENT_ENTRY_NOEXEC); in split_pmd_page()
145 pgt_set((unsigned long *)pmdp, pmd_val(new), addr, CRDTE_DTT_SEGMENT); in split_pmd_page()
151 static void modify_pmd_page(pmd_t *pmdp, unsigned long addr, in modify_pmd_page() argument
154 pmd_t new = *pmdp; in modify_pmd_page()
172 pgt_set((unsigned long *)pmdp, pmd_val(new), addr, CRDTE_DTT_SEGMENT); in modify_pmd_page()
[all …]
|
| D | gmap.c | 928 pmd_t *pmdp; in gmap_pmd_op_walk() local
931 pmdp = (pmd_t *) gmap_table_walk(gmap, gaddr, 1); in gmap_pmd_op_walk()
932 if (!pmdp) in gmap_pmd_op_walk()
937 return pmd_none(*pmdp) ? NULL : pmdp; in gmap_pmd_op_walk()
940 if (pmd_none(*pmdp)) { in gmap_pmd_op_walk()
946 if (!pmd_large(*pmdp)) in gmap_pmd_op_walk()
948 return pmdp; in gmap_pmd_op_walk()
956 static inline void gmap_pmd_op_end(struct gmap *gmap, pmd_t *pmdp) in gmap_pmd_op_end() argument
958 if (pmd_large(*pmdp)) in gmap_pmd_op_end()
977 pmd_t *pmdp, int prot, unsigned long bits) in gmap_protect_pmd() argument
[all …]
|
| /linux-6.6.21/arch/x86/include/asm/ |
| D | pgtable-3level.h | 48 static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd) in native_set_pmd() argument
50 pxx_xchg64(pmd, pmdp, native_pmd_val(pmd)); in native_set_pmd()
74 static inline void native_pmd_clear(pmd_t *pmdp) in native_pmd_clear() argument
76 WRITE_ONCE(pmdp->pmd_low, 0); in native_pmd_clear()
78 WRITE_ONCE(pmdp->pmd_high, 0); in native_pmd_clear()
108 static inline pmd_t native_pmdp_get_and_clear(pmd_t *pmdp) in native_pmdp_get_and_clear() argument
110 return pxx_xchg64(pmd, pmdp, 0ULL); in native_pmdp_get_and_clear()
126 unsigned long address, pmd_t *pmdp, pmd_t pmd) in pmdp_establish() argument
137 old.pmd_low = xchg(&pmdp->pmd_low, pmd.pmd_low); in pmdp_establish()
138 old.pmd_high = READ_ONCE(pmdp->pmd_high); in pmdp_establish()
[all …]
|
| /linux-6.6.21/arch/arm64/mm/ |
| D | mmu.c | 172 static void init_pte(pmd_t *pmdp, unsigned long addr, unsigned long end, in init_pte() argument
177 ptep = pte_set_fixmap_offset(pmdp, addr); in init_pte()
196 static void alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr, in alloc_init_cont_pte() argument
203 pmd_t pmd = READ_ONCE(*pmdp); in alloc_init_cont_pte()
214 __pmd_populate(pmdp, pte_phys, pmdval); in alloc_init_cont_pte()
215 pmd = READ_ONCE(*pmdp); in alloc_init_cont_pte()
229 init_pte(pmdp, addr, next, phys, __prot); in alloc_init_cont_pte()
240 pmd_t *pmdp; in init_pmd() local
242 pmdp = pmd_set_fixmap_offset(pudp, addr); in init_pmd()
244 pmd_t old_pmd = READ_ONCE(*pmdp); in init_pmd()
[all …]
|
| D | kasan_init.c | 62 static pte_t *__init kasan_pte_offset(pmd_t *pmdp, unsigned long addr, int node, in kasan_pte_offset() argument
65 if (pmd_none(READ_ONCE(*pmdp))) { in kasan_pte_offset()
69 __pmd_populate(pmdp, pte_phys, PMD_TYPE_TABLE); in kasan_pte_offset()
72 return early ? pte_offset_kimg(pmdp, addr) in kasan_pte_offset()
73 : pte_offset_kernel(pmdp, addr); in kasan_pte_offset()
102 static void __init kasan_pte_populate(pmd_t *pmdp, unsigned long addr, in kasan_pte_populate() argument
106 pte_t *ptep = kasan_pte_offset(pmdp, addr, node, early); in kasan_pte_populate()
123 pmd_t *pmdp = kasan_pmd_offset(pudp, addr, node, early); in kasan_pmd_populate() local
127 kasan_pte_populate(pmdp, addr, next, node, early); in kasan_pmd_populate()
128 } while (pmdp++, addr = next, addr != end && pmd_none(READ_ONCE(*pmdp))); in kasan_pmd_populate()
|
| /linux-6.6.21/arch/powerpc/include/asm/nohash/32/ |
| D | pgalloc.h | 17 static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, in pmd_populate_kernel() argument
21 *pmdp = __pmd((unsigned long)pte | _PMD_PRESENT); in pmd_populate_kernel()
23 *pmdp = __pmd(__pa(pte) | _PMD_PRESENT); in pmd_populate_kernel()
26 static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmdp, in pmd_populate() argument
30 *pmdp = __pmd((unsigned long)pte_page | _PMD_PRESENT); in pmd_populate()
32 *pmdp = __pmd(__pa(pte_page) | _PMD_USER | _PMD_PRESENT); in pmd_populate()
|
| /linux-6.6.21/arch/arm/include/asm/ |
| D | pgalloc.h | 107 static inline void __pmd_populate(pmd_t *pmdp, phys_addr_t pte, in __pmd_populate() argument
111 pmdp[0] = __pmd(pmdval); in __pmd_populate()
113 pmdp[1] = __pmd(pmdval + 256 * sizeof(pte_t)); in __pmd_populate()
115 flush_pmd_entry(pmdp); in __pmd_populate()
125 pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, pte_t *ptep) in pmd_populate_kernel() argument
130 __pmd_populate(pmdp, __pa(ptep), _PAGE_KERNEL_TABLE); in pmd_populate_kernel()
134 pmd_populate(struct mm_struct *mm, pmd_t *pmdp, pgtable_t ptep) in pmd_populate() argument
144 __pmd_populate(pmdp, page_to_phys(ptep), prot); in pmd_populate()
|
| /linux-6.6.21/include/linux/ |
| D | pgtable.h | 258 unsigned long address, pmd_t *pmdp,
265 unsigned long address, pmd_t *pmdp, in pmdp_set_access_flags() argument
289 static inline pmd_t pmdp_get(pmd_t *pmdp) in pmdp_get() argument
291 return READ_ONCE(*pmdp); in pmdp_get()
314 pmd_t *pmdp) in pmdp_test_and_clear_young() argument
316 pmd_t pmd = *pmdp; in pmdp_test_and_clear_young()
321 set_pmd_at(vma->vm_mm, address, pmdp, pmd_mkold(pmd)); in pmdp_test_and_clear_young()
327 pmd_t *pmdp) in pmdp_test_and_clear_young() argument
343 unsigned long address, pmd_t *pmdp);
350 unsigned long address, pmd_t *pmdp) in pmdp_clear_flush_young() argument
[all …]
|
| /linux-6.6.21/arch/powerpc/mm/kasan/ |
| D | init_book3e_64.c | 37 pmd_t *pmdp; in kasan_map_kernel_page() local
49 pmdp = memblock_alloc(PMD_TABLE_SIZE, PMD_TABLE_SIZE); in kasan_map_kernel_page()
50 memcpy(pmdp, kasan_early_shadow_pmd, PMD_TABLE_SIZE); in kasan_map_kernel_page()
51 pud_populate(&init_mm, pudp, pmdp); in kasan_map_kernel_page()
53 pmdp = pmd_offset(pudp, ea); in kasan_map_kernel_page()
54 if (kasan_pte_table(*pmdp)) { in kasan_map_kernel_page()
57 pmd_populate_kernel(&init_mm, pmdp, ptep); in kasan_map_kernel_page()
59 ptep = pte_offset_kernel(pmdp, ea); in kasan_map_kernel_page()
|
| /linux-6.6.21/arch/powerpc/mm/ |
| D | pgtable_32.c | 42 pmd_t *pmdp = pmd_off_k(addr); in early_ioremap_init() local
45 addr += PGDIR_SIZE, ptep += PTRS_PER_PTE, pmdp++) in early_ioremap_init()
46 pmd_populate_kernel(&init_mm, pmdp, ptep); in early_ioremap_init()
62 pte_t __init *early_pte_alloc_kernel(pmd_t *pmdp, unsigned long va) in early_pte_alloc_kernel() argument
64 if (pmd_none(*pmdp)) { in early_pte_alloc_kernel()
67 pmd_populate_kernel(&init_mm, pmdp, ptep); in early_pte_alloc_kernel()
69 return pte_offset_kernel(pmdp, va); in early_pte_alloc_kernel()
|
| /linux-6.6.21/arch/powerpc/include/asm/book3s/64/ |
| D | hash-4k.h | 109 static inline char *get_hpte_slot_array(pmd_t *pmdp) in get_hpte_slot_array() argument
146 unsigned long addr, pmd_t *pmdp,
149 unsigned long address, pmd_t *pmdp);
150 extern void hash__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
152 extern pgtable_t hash__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);
154 unsigned long addr, pmd_t *pmdp);
|
| /linux-6.6.21/arch/riscv/include/asm/ |
| D | pgtable.h | 249 static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) in set_pmd() argument
251 *pmdp = pmd; in set_pmd()
254 static inline void pmd_clear(pmd_t *pmdp) in pmd_clear() argument
256 set_pmd(pmdp, __pmd(0)); in pmd_clear()
496 unsigned long address, pmd_t *pmdp) in update_mmu_cache_pmd() argument
498 pte_t *ptep = (pte_t *)pmdp; in update_mmu_cache_pmd()
726 pmd_t *pmdp, pmd_t pmd) in set_pmd_at() argument
728 page_table_check_pmd_set(mm, pmdp, pmd); in set_pmd_at()
729 return __set_pte_at((pte_t *)pmdp, pmd_pte(pmd)); in set_pmd_at()
764 unsigned long address, pmd_t *pmdp, in pmdp_set_access_flags() argument
[all …]
|
| /linux-6.6.21/arch/s390/include/asm/ |
| D | pgtable.h | 937 static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) in set_pmd() argument
939 WRITE_ONCE(*pmdp, pmd); in set_pmd()
965 static inline void pmd_clear(pmd_t *pmdp) in pmd_clear() argument
967 set_pmd(pmdp, __pmd(_SEGMENT_ENTRY_EMPTY)); in pmd_clear()
1608 static inline void __pmdp_csp(pmd_t *pmdp) in __pmdp_csp() argument
1610 csp((unsigned int *)pmdp + 1, pmd_val(*pmdp), in __pmdp_csp()
1611 pmd_val(*pmdp) | _SEGMENT_ENTRY_INVALID); in __pmdp_csp()
1621 static __always_inline void __pmdp_idte(unsigned long addr, pmd_t *pmdp, in __pmdp_idte() argument
1627 sto = __pa(pmdp) - pmd_index(addr) * sizeof(pmd_t); in __pmdp_idte()
1632 : "+m" (*pmdp) in __pmdp_idte()
[all …]
|
| /linux-6.6.21/arch/arm/mm/ |
| D | kasan_init.c | 37 static void __init kasan_pte_populate(pmd_t *pmdp, unsigned long addr, in kasan_pte_populate() argument
41 pte_t *ptep = pte_offset_kernel(pmdp, addr); in kasan_pte_populate()
91 pmd_t *pmdp = pmd_offset(pudp, addr); in kasan_pmd_populate() local
94 if (pmd_none(*pmdp)) { in kasan_pmd_populate()
108 pmd_populate_kernel(&init_mm, pmdp, p); in kasan_pmd_populate()
109 flush_pmd_entry(pmdp); in kasan_pmd_populate()
113 kasan_pte_populate(pmdp, addr, next, early); in kasan_pmd_populate()
114 } while (pmdp++, addr = next, addr != end); in kasan_pmd_populate()
|