1 /* SPDX-License-Identifier: GPL-2.0 */
2
3 /*
4 * Copyright (c) 2021, Google LLC.
5 * Pasha Tatashin <pasha.tatashin@soleen.com>
6 */
7 #ifndef __LINUX_PAGE_TABLE_CHECK_H
8 #define __LINUX_PAGE_TABLE_CHECK_H
9
10 #ifdef CONFIG_PAGE_TABLE_CHECK
11 #include <linux/jump_label.h>
12
13 extern struct static_key_true page_table_check_disabled;
14 extern struct page_ext_operations page_table_check_ops;
15
16 void __page_table_check_zero(struct page *page, unsigned int order);
17 void __page_table_check_pte_clear(struct mm_struct *mm, unsigned long addr,
18 pte_t pte);
19 void __page_table_check_pmd_clear(struct mm_struct *mm, unsigned long addr,
20 pmd_t pmd);
21 void __page_table_check_pud_clear(struct mm_struct *mm, unsigned long addr,
22 pud_t pud);
23 void __page_table_check_pte_set(struct mm_struct *mm, unsigned long addr,
24 pte_t *ptep, pte_t pte);
25 void __page_table_check_pmd_set(struct mm_struct *mm, unsigned long addr,
26 pmd_t *pmdp, pmd_t pmd);
27 void __page_table_check_pud_set(struct mm_struct *mm, unsigned long addr,
28 pud_t *pudp, pud_t pud);
29 void __page_table_check_pte_clear_range(struct mm_struct *mm,
30 unsigned long addr,
31 pmd_t pmd);
32
page_table_check_alloc(struct page * page,unsigned int order)33 static inline void page_table_check_alloc(struct page *page, unsigned int order)
34 {
35 if (static_branch_likely(&page_table_check_disabled))
36 return;
37
38 __page_table_check_zero(page, order);
39 }
40
page_table_check_free(struct page * page,unsigned int order)41 static inline void page_table_check_free(struct page *page, unsigned int order)
42 {
43 if (static_branch_likely(&page_table_check_disabled))
44 return;
45
46 __page_table_check_zero(page, order);
47 }
48
page_table_check_pte_clear(struct mm_struct * mm,unsigned long addr,pte_t pte)49 static inline void page_table_check_pte_clear(struct mm_struct *mm,
50 unsigned long addr, pte_t pte)
51 {
52 if (static_branch_likely(&page_table_check_disabled))
53 return;
54
55 __page_table_check_pte_clear(mm, addr, pte);
56 }
57
page_table_check_pmd_clear(struct mm_struct * mm,unsigned long addr,pmd_t pmd)58 static inline void page_table_check_pmd_clear(struct mm_struct *mm,
59 unsigned long addr, pmd_t pmd)
60 {
61 if (static_branch_likely(&page_table_check_disabled))
62 return;
63
64 __page_table_check_pmd_clear(mm, addr, pmd);
65 }
66
page_table_check_pud_clear(struct mm_struct * mm,unsigned long addr,pud_t pud)67 static inline void page_table_check_pud_clear(struct mm_struct *mm,
68 unsigned long addr, pud_t pud)
69 {
70 if (static_branch_likely(&page_table_check_disabled))
71 return;
72
73 __page_table_check_pud_clear(mm, addr, pud);
74 }
75
page_table_check_pte_set(struct mm_struct * mm,unsigned long addr,pte_t * ptep,pte_t pte)76 static inline void page_table_check_pte_set(struct mm_struct *mm,
77 unsigned long addr, pte_t *ptep,
78 pte_t pte)
79 {
80 if (static_branch_likely(&page_table_check_disabled))
81 return;
82
83 __page_table_check_pte_set(mm, addr, ptep, pte);
84 }
85
page_table_check_pmd_set(struct mm_struct * mm,unsigned long addr,pmd_t * pmdp,pmd_t pmd)86 static inline void page_table_check_pmd_set(struct mm_struct *mm,
87 unsigned long addr, pmd_t *pmdp,
88 pmd_t pmd)
89 {
90 if (static_branch_likely(&page_table_check_disabled))
91 return;
92
93 __page_table_check_pmd_set(mm, addr, pmdp, pmd);
94 }
95
page_table_check_pud_set(struct mm_struct * mm,unsigned long addr,pud_t * pudp,pud_t pud)96 static inline void page_table_check_pud_set(struct mm_struct *mm,
97 unsigned long addr, pud_t *pudp,
98 pud_t pud)
99 {
100 if (static_branch_likely(&page_table_check_disabled))
101 return;
102
103 __page_table_check_pud_set(mm, addr, pudp, pud);
104 }
105
page_table_check_pte_clear_range(struct mm_struct * mm,unsigned long addr,pmd_t pmd)106 static inline void page_table_check_pte_clear_range(struct mm_struct *mm,
107 unsigned long addr,
108 pmd_t pmd)
109 {
110 if (static_branch_likely(&page_table_check_disabled))
111 return;
112
113 __page_table_check_pte_clear_range(mm, addr, pmd);
114 }
115
116 #else
117
page_table_check_alloc(struct page * page,unsigned int order)118 static inline void page_table_check_alloc(struct page *page, unsigned int order)
119 {
120 }
121
page_table_check_free(struct page * page,unsigned int order)122 static inline void page_table_check_free(struct page *page, unsigned int order)
123 {
124 }
125
page_table_check_pte_clear(struct mm_struct * mm,unsigned long addr,pte_t pte)126 static inline void page_table_check_pte_clear(struct mm_struct *mm,
127 unsigned long addr, pte_t pte)
128 {
129 }
130
page_table_check_pmd_clear(struct mm_struct * mm,unsigned long addr,pmd_t pmd)131 static inline void page_table_check_pmd_clear(struct mm_struct *mm,
132 unsigned long addr, pmd_t pmd)
133 {
134 }
135
page_table_check_pud_clear(struct mm_struct * mm,unsigned long addr,pud_t pud)136 static inline void page_table_check_pud_clear(struct mm_struct *mm,
137 unsigned long addr, pud_t pud)
138 {
139 }
140
page_table_check_pte_set(struct mm_struct * mm,unsigned long addr,pte_t * ptep,pte_t pte)141 static inline void page_table_check_pte_set(struct mm_struct *mm,
142 unsigned long addr, pte_t *ptep,
143 pte_t pte)
144 {
145 }
146
page_table_check_pmd_set(struct mm_struct * mm,unsigned long addr,pmd_t * pmdp,pmd_t pmd)147 static inline void page_table_check_pmd_set(struct mm_struct *mm,
148 unsigned long addr, pmd_t *pmdp,
149 pmd_t pmd)
150 {
151 }
152
page_table_check_pud_set(struct mm_struct * mm,unsigned long addr,pud_t * pudp,pud_t pud)153 static inline void page_table_check_pud_set(struct mm_struct *mm,
154 unsigned long addr, pud_t *pudp,
155 pud_t pud)
156 {
157 }
158
page_table_check_pte_clear_range(struct mm_struct * mm,unsigned long addr,pmd_t pmd)159 static inline void page_table_check_pte_clear_range(struct mm_struct *mm,
160 unsigned long addr,
161 pmd_t pmd)
162 {
163 }
164
165 #endif /* CONFIG_PAGE_TABLE_CHECK */
166 #endif /* __LINUX_PAGE_TABLE_CHECK_H */
167