1 /* SPDX-License-Identifier: GPL-2.0 */
2
3 /*
4 * Copyright (c) 2021, Google LLC.
5 * Pasha Tatashin <pasha.tatashin@soleen.com>
6 */
7 #ifndef __LINUX_PAGE_TABLE_CHECK_H
8 #define __LINUX_PAGE_TABLE_CHECK_H
9
10 #ifdef CONFIG_PAGE_TABLE_CHECK
11 #include <linux/jump_label.h>
12
13 extern struct static_key_true page_table_check_disabled;
14 extern struct page_ext_operations page_table_check_ops;
15
16 void __page_table_check_zero(struct page *page, unsigned int order);
17 void __page_table_check_pte_clear(struct mm_struct *mm, pte_t pte);
18 void __page_table_check_pmd_clear(struct mm_struct *mm, pmd_t pmd);
19 void __page_table_check_pud_clear(struct mm_struct *mm, pud_t pud);
20 void __page_table_check_ptes_set(struct mm_struct *mm, pte_t *ptep, pte_t pte,
21 unsigned int nr);
22 void __page_table_check_pmd_set(struct mm_struct *mm, pmd_t *pmdp, pmd_t pmd);
23 void __page_table_check_pud_set(struct mm_struct *mm, pud_t *pudp, pud_t pud);
24 void __page_table_check_pte_clear_range(struct mm_struct *mm,
25 unsigned long addr,
26 pmd_t pmd);
27
page_table_check_alloc(struct page * page,unsigned int order)28 static inline void page_table_check_alloc(struct page *page, unsigned int order)
29 {
30 if (static_branch_likely(&page_table_check_disabled))
31 return;
32
33 __page_table_check_zero(page, order);
34 }
35
page_table_check_free(struct page * page,unsigned int order)36 static inline void page_table_check_free(struct page *page, unsigned int order)
37 {
38 if (static_branch_likely(&page_table_check_disabled))
39 return;
40
41 __page_table_check_zero(page, order);
42 }
43
page_table_check_pte_clear(struct mm_struct * mm,pte_t pte)44 static inline void page_table_check_pte_clear(struct mm_struct *mm, pte_t pte)
45 {
46 if (static_branch_likely(&page_table_check_disabled))
47 return;
48
49 __page_table_check_pte_clear(mm, pte);
50 }
51
page_table_check_pmd_clear(struct mm_struct * mm,pmd_t pmd)52 static inline void page_table_check_pmd_clear(struct mm_struct *mm, pmd_t pmd)
53 {
54 if (static_branch_likely(&page_table_check_disabled))
55 return;
56
57 __page_table_check_pmd_clear(mm, pmd);
58 }
59
page_table_check_pud_clear(struct mm_struct * mm,pud_t pud)60 static inline void page_table_check_pud_clear(struct mm_struct *mm, pud_t pud)
61 {
62 if (static_branch_likely(&page_table_check_disabled))
63 return;
64
65 __page_table_check_pud_clear(mm, pud);
66 }
67
page_table_check_ptes_set(struct mm_struct * mm,pte_t * ptep,pte_t pte,unsigned int nr)68 static inline void page_table_check_ptes_set(struct mm_struct *mm,
69 pte_t *ptep, pte_t pte, unsigned int nr)
70 {
71 if (static_branch_likely(&page_table_check_disabled))
72 return;
73
74 __page_table_check_ptes_set(mm, ptep, pte, nr);
75 }
76
page_table_check_pmd_set(struct mm_struct * mm,pmd_t * pmdp,pmd_t pmd)77 static inline void page_table_check_pmd_set(struct mm_struct *mm, pmd_t *pmdp,
78 pmd_t pmd)
79 {
80 if (static_branch_likely(&page_table_check_disabled))
81 return;
82
83 __page_table_check_pmd_set(mm, pmdp, pmd);
84 }
85
page_table_check_pud_set(struct mm_struct * mm,pud_t * pudp,pud_t pud)86 static inline void page_table_check_pud_set(struct mm_struct *mm, pud_t *pudp,
87 pud_t pud)
88 {
89 if (static_branch_likely(&page_table_check_disabled))
90 return;
91
92 __page_table_check_pud_set(mm, pudp, pud);
93 }
94
page_table_check_pte_clear_range(struct mm_struct * mm,unsigned long addr,pmd_t pmd)95 static inline void page_table_check_pte_clear_range(struct mm_struct *mm,
96 unsigned long addr,
97 pmd_t pmd)
98 {
99 if (static_branch_likely(&page_table_check_disabled))
100 return;
101
102 __page_table_check_pte_clear_range(mm, addr, pmd);
103 }
104
105 #else
106
page_table_check_alloc(struct page * page,unsigned int order)107 static inline void page_table_check_alloc(struct page *page, unsigned int order)
108 {
109 }
110
page_table_check_free(struct page * page,unsigned int order)111 static inline void page_table_check_free(struct page *page, unsigned int order)
112 {
113 }
114
page_table_check_pte_clear(struct mm_struct * mm,pte_t pte)115 static inline void page_table_check_pte_clear(struct mm_struct *mm, pte_t pte)
116 {
117 }
118
page_table_check_pmd_clear(struct mm_struct * mm,pmd_t pmd)119 static inline void page_table_check_pmd_clear(struct mm_struct *mm, pmd_t pmd)
120 {
121 }
122
page_table_check_pud_clear(struct mm_struct * mm,pud_t pud)123 static inline void page_table_check_pud_clear(struct mm_struct *mm, pud_t pud)
124 {
125 }
126
page_table_check_ptes_set(struct mm_struct * mm,pte_t * ptep,pte_t pte,unsigned int nr)127 static inline void page_table_check_ptes_set(struct mm_struct *mm,
128 pte_t *ptep, pte_t pte, unsigned int nr)
129 {
130 }
131
page_table_check_pmd_set(struct mm_struct * mm,pmd_t * pmdp,pmd_t pmd)132 static inline void page_table_check_pmd_set(struct mm_struct *mm, pmd_t *pmdp,
133 pmd_t pmd)
134 {
135 }
136
page_table_check_pud_set(struct mm_struct * mm,pud_t * pudp,pud_t pud)137 static inline void page_table_check_pud_set(struct mm_struct *mm, pud_t *pudp,
138 pud_t pud)
139 {
140 }
141
page_table_check_pte_clear_range(struct mm_struct * mm,unsigned long addr,pmd_t pmd)142 static inline void page_table_check_pte_clear_range(struct mm_struct *mm,
143 unsigned long addr,
144 pmd_t pmd)
145 {
146 }
147
148 #endif /* CONFIG_PAGE_TABLE_CHECK */
149 #endif /* __LINUX_PAGE_TABLE_CHECK_H */
150