| /linux-6.1.9/include/asm-generic/ |
| D | tlb.h | 204 extern void tlb_remove_table(struct mmu_gather *tlb, void *table);
212 #define tlb_remove_table(tlb, page) tlb_remove_page((tlb), (page)) argument
263 extern bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page,
325 void tlb_flush_mmu(struct mmu_gather *tlb);
327 static inline void __tlb_adjust_range(struct mmu_gather *tlb, in __tlb_adjust_range() argument
331 tlb->start = min(tlb->start, address); in __tlb_adjust_range()
332 tlb->end = max(tlb->end, address + range_size); in __tlb_adjust_range()
335 static inline void __tlb_reset_range(struct mmu_gather *tlb) in __tlb_reset_range() argument
337 if (tlb->fullmm) { in __tlb_reset_range()
338 tlb->start = tlb->end = ~0; in __tlb_reset_range()
[all …]
|
| /linux-6.1.9/mm/ |
| D | mmu_gather.c | 18 static bool tlb_next_batch(struct mmu_gather *tlb) in tlb_next_batch() argument
22 batch = tlb->active; in tlb_next_batch()
24 tlb->active = batch->next; in tlb_next_batch()
28 if (tlb->batch_count == MAX_GATHER_BATCH_COUNT) in tlb_next_batch()
35 tlb->batch_count++; in tlb_next_batch()
40 tlb->active->next = batch; in tlb_next_batch()
41 tlb->active = batch; in tlb_next_batch()
46 static void tlb_batch_pages_flush(struct mmu_gather *tlb) in tlb_batch_pages_flush() argument
50 for (batch = &tlb->local; batch && batch->nr; batch = batch->next) { in tlb_batch_pages_flush()
66 tlb->active = &tlb->local; in tlb_batch_pages_flush()
[all …]
|
| D | madvise.c | 41 struct mmu_gather *tlb; member
329 struct mmu_gather *tlb = private->tlb; in madvise_cold_or_pageout_pte_range() local
331 struct mm_struct *mm = tlb->mm; in madvise_cold_or_pageout_pte_range()
346 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); in madvise_cold_or_pageout_pte_range()
386 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); in madvise_cold_or_pageout_pte_range()
411 tlb_change_page_size(tlb, PAGE_SIZE); in madvise_cold_or_pageout_pte_range()
466 tlb->fullmm); in madvise_cold_or_pageout_pte_range()
469 tlb_remove_tlb_entry(tlb, pte, addr); in madvise_cold_or_pageout_pte_range()
504 static void madvise_cold_page_range(struct mmu_gather *tlb, in madvise_cold_page_range() argument
510 .tlb = tlb, in madvise_cold_page_range()
[all …]
|
| D | mprotect.c | 75 static unsigned long change_pte_range(struct mmu_gather *tlb, in change_pte_range() argument
87 tlb_change_page_size(tlb, PAGE_SIZE); in change_pte_range()
202 tlb_flush_pte_range(tlb, addr, PAGE_SIZE); in change_pte_range()
349 static inline unsigned long change_pmd_range(struct mmu_gather *tlb, in change_pmd_range() argument
403 int nr_ptes = change_huge_pmd(tlb, vma, pmd, in change_pmd_range()
418 this_pages = change_pte_range(tlb, vma, pmd, addr, next, in change_pmd_range()
433 static inline unsigned long change_pud_range(struct mmu_gather *tlb, in change_pud_range() argument
447 pages += change_pmd_range(tlb, vma, pud, addr, next, newprot, in change_pud_range()
454 static inline unsigned long change_p4d_range(struct mmu_gather *tlb, in change_p4d_range() argument
468 pages += change_pud_range(tlb, vma, p4d, addr, next, newprot, in change_p4d_range()
[all …]
|
| /linux-6.1.9/arch/arm64/include/asm/ |
| D | tlb.h | 20 static void tlb_flush(struct mmu_gather *tlb);
29 static inline int tlb_get_level(struct mmu_gather *tlb) in tlb_get_level() argument
32 if (tlb->freed_tables) in tlb_get_level()
35 if (tlb->cleared_ptes && !(tlb->cleared_pmds || in tlb_get_level()
36 tlb->cleared_puds || in tlb_get_level()
37 tlb->cleared_p4ds)) in tlb_get_level()
40 if (tlb->cleared_pmds && !(tlb->cleared_ptes || in tlb_get_level()
41 tlb->cleared_puds || in tlb_get_level()
42 tlb->cleared_p4ds)) in tlb_get_level()
45 if (tlb->cleared_puds && !(tlb->cleared_ptes || in tlb_get_level()
[all …]
|
| /linux-6.1.9/arch/s390/include/asm/ |
| D | tlb.h | 26 static inline void tlb_flush(struct mmu_gather *tlb);
27 static inline bool __tlb_remove_page_size(struct mmu_gather *tlb,
44 static inline bool __tlb_remove_page_size(struct mmu_gather *tlb, in __tlb_remove_page_size() argument
51 static inline void tlb_flush(struct mmu_gather *tlb) in tlb_flush() argument
53 __tlb_flush_mm_lazy(tlb->mm); in tlb_flush()
60 static inline void pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, in pte_free_tlb() argument
63 __tlb_adjust_range(tlb, address, PAGE_SIZE); in pte_free_tlb()
64 tlb->mm->context.flush_mm = 1; in pte_free_tlb()
65 tlb->freed_tables = 1; in pte_free_tlb()
66 tlb->cleared_pmds = 1; in pte_free_tlb()
[all …]
|
| /linux-6.1.9/arch/x86/include/asm/ |
| D | tlb.h | 6 static inline void tlb_flush(struct mmu_gather *tlb);
10 static inline void tlb_flush(struct mmu_gather *tlb) in tlb_flush() argument
13 unsigned int stride_shift = tlb_get_unmap_shift(tlb); in tlb_flush()
15 if (!tlb->fullmm && !tlb->need_flush_all) { in tlb_flush()
16 start = tlb->start; in tlb_flush()
17 end = tlb->end; in tlb_flush()
20 flush_tlb_mm_range(tlb->mm, start, end, stride_shift, tlb->freed_tables); in tlb_flush()
|
| D | pgalloc.h | 56 extern void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte);
58 static inline void __pte_free_tlb(struct mmu_gather *tlb, struct page *pte, in __pte_free_tlb() argument
61 ___pte_free_tlb(tlb, pte); in __pte_free_tlb()
88 extern void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd);
90 static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd, in __pmd_free_tlb() argument
93 ___pmd_free_tlb(tlb, pmd); in __pmd_free_tlb()
125 extern void ___pud_free_tlb(struct mmu_gather *tlb, pud_t *pud);
127 static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud, in __pud_free_tlb() argument
130 ___pud_free_tlb(tlb, pud); in __pud_free_tlb()
168 extern void ___p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d);
[all …]
|
| /linux-6.1.9/arch/sh/mm/ |
| D | Makefile | 30 debugfs-$(CONFIG_CPU_SH4) += tlb-debugfs.o
31 tlb-$(CONFIG_CPU_SH3) := tlb-sh3.o
32 tlb-$(CONFIG_CPU_SH4) := tlb-sh4.o tlb-urb.o
33 tlb-$(CONFIG_CPU_HAS_PTEAEX) := tlb-pteaex.o tlb-urb.o
34 obj-y += $(tlb-y)
|
| /linux-6.1.9/arch/arm/include/asm/ |
| D | tlb.h | 23 #define tlb_flush(tlb) ((void) tlb) argument
40 __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, unsigned long addr) in __pte_free_tlb() argument
50 __tlb_adjust_range(tlb, addr - PAGE_SIZE, 2 * PAGE_SIZE); in __pte_free_tlb()
53 tlb_remove_table(tlb, pte); in __pte_free_tlb()
57 __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp, unsigned long addr) in __pmd_free_tlb() argument
63 tlb_remove_table(tlb, page); in __pmd_free_tlb()
|
| /linux-6.1.9/arch/powerpc/include/asm/nohash/ |
| D | pgalloc.h | 8 extern void tlb_remove_table(struct mmu_gather *tlb, void *table);
10 extern void tlb_flush_pgtable(struct mmu_gather *tlb, unsigned long address);
13 static inline void tlb_flush_pgtable(struct mmu_gather *tlb, in tlb_flush_pgtable() argument
49 static inline void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift) in pgtable_free_tlb() argument
55 tlb_remove_table(tlb, (void *)pgf); in pgtable_free_tlb()
66 static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table, in __pte_free_tlb() argument
69 tlb_flush_pgtable(tlb, address); in __pte_free_tlb()
70 pgtable_free_tlb(tlb, table, 0); in __pte_free_tlb()
|
| /linux-6.1.9/arch/loongarch/include/asm/ |
| D | tlb.h | 140 #define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0) argument
142 static void tlb_flush(struct mmu_gather *tlb);
147 static inline void tlb_flush(struct mmu_gather *tlb) in tlb_flush() argument
151 vma.vm_mm = tlb->mm; in tlb_flush()
153 if (tlb->fullmm) { in tlb_flush()
154 flush_tlb_mm(tlb->mm); in tlb_flush()
158 flush_tlb_range(&vma, tlb->start, tlb->end); in tlb_flush()
|
| D | pgalloc.h | 56 #define __pte_free_tlb(tlb, pte, address) \ argument
59 tlb_remove_page((tlb), pte); \
83 #define __pmd_free_tlb(tlb, x, addr) pmd_free((tlb)->mm, x) argument
99 #define __pud_free_tlb(tlb, x, addr) pud_free((tlb)->mm, x) argument
|
| /linux-6.1.9/arch/sparc/include/asm/ |
| D | pgalloc_64.h | 78 static inline void pgtable_free_tlb(struct mmu_gather *tlb, void *table, bool is_page) in pgtable_free_tlb() argument
83 tlb_remove_table(tlb, (void *)pgf); in pgtable_free_tlb()
96 static inline void pgtable_free_tlb(struct mmu_gather *tlb, void *table, bool is_page) in pgtable_free_tlb() argument
102 static inline void __pte_free_tlb(struct mmu_gather *tlb, pte_t *pte, in __pte_free_tlb() argument
105 pgtable_free_tlb(tlb, pte, true); in __pte_free_tlb()
108 #define __pmd_free_tlb(tlb, pmd, addr) \ argument
109 pgtable_free_tlb(tlb, pmd, false)
111 #define __pud_free_tlb(tlb, pud, addr) \ argument
112 pgtable_free_tlb(tlb, pud, false)
|
| /linux-6.1.9/arch/powerpc/mm/ |
| D | hugetlbpage.c | 275 static void hugepd_free(struct mmu_gather *tlb, void *hugepte) in hugepd_free() argument
281 if (atomic_read(&tlb->mm->mm_users) < 2 || in hugepd_free()
282 mm_is_thread_local(tlb->mm)) { in hugepd_free()
301 static inline void hugepd_free(struct mmu_gather *tlb, void *hugepte) {} in hugepd_free() argument
319 static void free_hugepd_range(struct mmu_gather *tlb, hugepd_t *hpdp, int pdshift, in free_hugepd_range() argument
341 hugepd_free(tlb, hugepte); in free_hugepd_range()
343 pgtable_free_tlb(tlb, hugepte, in free_hugepd_range()
347 static void hugetlb_free_pte_range(struct mmu_gather *tlb, pmd_t *pmd, in hugetlb_free_pte_range() argument
357 pte_free_tlb(tlb, token, addr); in hugetlb_free_pte_range()
358 mm_dec_nr_ptes(tlb->mm); in hugetlb_free_pte_range()
[all …]
|
| /linux-6.1.9/arch/riscv/boot/dts/sifive/ |
| D | fu540-c000.dtsi | 45 d-tlb-sets = <1>;
46 d-tlb-size = <32>;
51 i-tlb-sets = <1>;
52 i-tlb-size = <32>;
56 tlb-split;
69 d-tlb-sets = <1>;
70 d-tlb-size = <32>;
75 i-tlb-sets = <1>;
76 i-tlb-size = <32>;
80 tlb-split;
[all …]
|
| D | fu740-c000.dtsi | 46 d-tlb-sets = <1>;
47 d-tlb-size = <40>;
52 i-tlb-sets = <1>;
53 i-tlb-size = <40>;
58 tlb-split;
70 d-tlb-sets = <1>;
71 d-tlb-size = <40>;
76 i-tlb-sets = <1>;
77 i-tlb-size = <40>;
82 tlb-split;
[all …]
|
| /linux-6.1.9/include/linux/ |
| D | io-pgtable.h | 100 const struct iommu_flush_ops *tlb; member
225 if (iop->cfg.tlb && iop->cfg.tlb->tlb_flush_all) in io_pgtable_tlb_flush_all()
226 iop->cfg.tlb->tlb_flush_all(iop->cookie); in io_pgtable_tlb_flush_all()
233 if (iop->cfg.tlb && iop->cfg.tlb->tlb_flush_walk) in io_pgtable_tlb_flush_walk()
234 iop->cfg.tlb->tlb_flush_walk(iova, size, granule, iop->cookie); in io_pgtable_tlb_flush_walk()
242 if (iop->cfg.tlb && iop->cfg.tlb->tlb_add_page) in io_pgtable_tlb_add_page()
243 iop->cfg.tlb->tlb_add_page(gather, iova, granule, iop->cookie); in io_pgtable_tlb_add_page()
|
| /linux-6.1.9/arch/ia64/include/asm/ |
| D | pgalloc.h | 39 #define __pud_free_tlb(tlb, pud, address) pud_free((tlb)->mm, pud) argument
48 #define __pmd_free_tlb(tlb, pmd, address) pmd_free((tlb)->mm, pmd) argument
62 #define __pte_free_tlb(tlb, pte, address) pte_free((tlb)->mm, pte) argument
|
| /linux-6.1.9/arch/arc/include/asm/ |
| D | pgalloc.h | 80 #define __pud_free_tlb(tlb, pmd, addr) pud_free((tlb)->mm, pmd) argument
91 #define __pmd_free_tlb(tlb, pmd, addr) pmd_free((tlb)->mm, pmd) argument
95 #define __pte_free_tlb(tlb, pte, addr) pte_free((tlb)->mm, pte) argument
|
| /linux-6.1.9/arch/riscv/include/asm/ |
| D | pgalloc.h | 98 #define __pud_free_tlb(tlb, pud, addr) pud_free((tlb)->mm, pud) argument
127 #define __p4d_free_tlb(tlb, p4d, addr) p4d_free((tlb)->mm, p4d) argument
152 #define __pmd_free_tlb(tlb, pmd, addr) pmd_free((tlb)->mm, pmd) argument
156 #define __pte_free_tlb(tlb, pte, buf) \ argument
159 tlb_remove_page((tlb), pte); \
|
| /linux-6.1.9/arch/mips/include/asm/ |
| D | pgalloc.h | 57 #define __pte_free_tlb(tlb,pte,address) \ argument
60 tlb_remove_page((tlb), pte); \
84 #define __pmd_free_tlb(tlb, x, addr) pmd_free((tlb)->mm, x) argument
105 #define __pud_free_tlb(tlb, x, addr) pud_free((tlb)->mm, x) argument
|
| /linux-6.1.9/arch/alpha/include/asm/ |
| D | tlb.h | 7 #define __pte_free_tlb(tlb, pte, address) pte_free((tlb)->mm, pte) argument
8 #define __pmd_free_tlb(tlb, pmd, address) pmd_free((tlb)->mm, pmd) argument
|
| /linux-6.1.9/arch/parisc/include/asm/ |
| D | tlb.h | 8 #define __pmd_free_tlb(tlb, pmd, addr) pmd_free((tlb)->mm, pmd) argument
10 #define __pte_free_tlb(tlb, pte, addr) pte_free((tlb)->mm, pte) argument
|
| /linux-6.1.9/arch/riscv/boot/dts/microchip/ |
| D | mpfs.dtsi | 40 d-tlb-sets = <1>;
41 d-tlb-size = <32>;
46 i-tlb-sets = <1>;
47 i-tlb-size = <32>;
52 tlb-split;
68 d-tlb-sets = <1>;
69 d-tlb-size = <32>;
74 i-tlb-sets = <1>;
75 i-tlb-size = <32>;
80 tlb-split;
[all …]
|