1 // SPDX-License-Identifier: GPL-2.0-only
2
3 #ifndef KVM_X86_MMU_SPTE_H
4 #define KVM_X86_MMU_SPTE_H
5
6 #include "mmu_internal.h"
7
8 /*
9 * A MMU present SPTE is backed by actual memory and may or may not be present
10 * in hardware. E.g. MMIO SPTEs are not considered present. Use bit 11, as it
11 * is ignored by all flavors of SPTEs and checking a low bit often generates
12 * better code than for a high bit, e.g. 56+. MMU present checks are pervasive
13 * enough that the improved code generation is noticeable in KVM's footprint.
14 */
15 #define SPTE_MMU_PRESENT_MASK BIT_ULL(11)
16
17 /*
18 * TDP SPTES (more specifically, EPT SPTEs) may not have A/D bits, and may also
19 * be restricted to using write-protection (for L2 when CPU dirty logging, i.e.
20 * PML, is enabled). Use bits 52 and 53 to hold the type of A/D tracking that
21 * is must be employed for a given TDP SPTE.
22 *
23 * Note, the "enabled" mask must be '0', as bits 62:52 are _reserved_ for PAE
24 * paging, including NPT PAE. This scheme works because legacy shadow paging
25 * is guaranteed to have A/D bits and write-protection is forced only for
26 * TDP with CPU dirty logging (PML). If NPT ever gains PML-like support, it
27 * must be restricted to 64-bit KVM.
28 */
29 #define SPTE_TDP_AD_SHIFT 52
30 #define SPTE_TDP_AD_MASK (3ULL << SPTE_TDP_AD_SHIFT)
31 #define SPTE_TDP_AD_ENABLED_MASK (0ULL << SPTE_TDP_AD_SHIFT)
32 #define SPTE_TDP_AD_DISABLED_MASK (1ULL << SPTE_TDP_AD_SHIFT)
33 #define SPTE_TDP_AD_WRPROT_ONLY_MASK (2ULL << SPTE_TDP_AD_SHIFT)
34 static_assert(SPTE_TDP_AD_ENABLED_MASK == 0);
35
36 #ifdef CONFIG_DYNAMIC_PHYSICAL_MASK
37 #define SPTE_BASE_ADDR_MASK (physical_mask & ~(u64)(PAGE_SIZE-1))
38 #else
39 #define SPTE_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
40 #endif
41
42 #define SPTE_PERM_MASK (PT_PRESENT_MASK | PT_WRITABLE_MASK | shadow_user_mask \
43 | shadow_x_mask | shadow_nx_mask | shadow_me_mask)
44
45 #define ACC_EXEC_MASK 1
46 #define ACC_WRITE_MASK PT_WRITABLE_MASK
47 #define ACC_USER_MASK PT_USER_MASK
48 #define ACC_ALL (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK)
49
50 /* The mask for the R/X bits in EPT PTEs */
51 #define SPTE_EPT_READABLE_MASK 0x1ull
52 #define SPTE_EPT_EXECUTABLE_MASK 0x4ull
53
54 #define SPTE_LEVEL_BITS 9
55 #define SPTE_LEVEL_SHIFT(level) __PT_LEVEL_SHIFT(level, SPTE_LEVEL_BITS)
56 #define SPTE_INDEX(address, level) __PT_INDEX(address, level, SPTE_LEVEL_BITS)
57 #define SPTE_ENT_PER_PAGE __PT_ENT_PER_PAGE(SPTE_LEVEL_BITS)
58
59 /*
60 * The mask/shift to use for saving the original R/X bits when marking the PTE
61 * as not-present for access tracking purposes. We do not save the W bit as the
62 * PTEs being access tracked also need to be dirty tracked, so the W bit will be
63 * restored only when a write is attempted to the page. This mask obviously
64 * must not overlap the A/D type mask.
65 */
66 #define SHADOW_ACC_TRACK_SAVED_BITS_MASK (SPTE_EPT_READABLE_MASK | \
67 SPTE_EPT_EXECUTABLE_MASK)
68 #define SHADOW_ACC_TRACK_SAVED_BITS_SHIFT 54
69 #define SHADOW_ACC_TRACK_SAVED_MASK (SHADOW_ACC_TRACK_SAVED_BITS_MASK << \
70 SHADOW_ACC_TRACK_SAVED_BITS_SHIFT)
71 static_assert(!(SPTE_TDP_AD_MASK & SHADOW_ACC_TRACK_SAVED_MASK));
72
73 /*
74 * {DEFAULT,EPT}_SPTE_{HOST,MMU}_WRITABLE are used to keep track of why a given
75 * SPTE is write-protected. See is_writable_pte() for details.
76 */
77
78 /* Bits 9 and 10 are ignored by all non-EPT PTEs. */
79 #define DEFAULT_SPTE_HOST_WRITABLE BIT_ULL(9)
80 #define DEFAULT_SPTE_MMU_WRITABLE BIT_ULL(10)
81
82 /*
83 * Low ignored bits are at a premium for EPT, use high ignored bits, taking care
84 * to not overlap the A/D type mask or the saved access bits of access-tracked
85 * SPTEs when A/D bits are disabled.
86 */
87 #define EPT_SPTE_HOST_WRITABLE BIT_ULL(57)
88 #define EPT_SPTE_MMU_WRITABLE BIT_ULL(58)
89
90 static_assert(!(EPT_SPTE_HOST_WRITABLE & SPTE_TDP_AD_MASK));
91 static_assert(!(EPT_SPTE_MMU_WRITABLE & SPTE_TDP_AD_MASK));
92 static_assert(!(EPT_SPTE_HOST_WRITABLE & SHADOW_ACC_TRACK_SAVED_MASK));
93 static_assert(!(EPT_SPTE_MMU_WRITABLE & SHADOW_ACC_TRACK_SAVED_MASK));
94
95 /* Defined only to keep the above static asserts readable. */
96 #undef SHADOW_ACC_TRACK_SAVED_MASK
97
98 /*
99 * Due to limited space in PTEs, the MMIO generation is a 19 bit subset of
100 * the memslots generation and is derived as follows:
101 *
102 * Bits 0-7 of the MMIO generation are propagated to spte bits 3-10
103 * Bits 8-18 of the MMIO generation are propagated to spte bits 52-62
104 *
105 * The KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS flag is intentionally not included in
106 * the MMIO generation number, as doing so would require stealing a bit from
107 * the "real" generation number and thus effectively halve the maximum number
108 * of MMIO generations that can be handled before encountering a wrap (which
109 * requires a full MMU zap). The flag is instead explicitly queried when
110 * checking for MMIO spte cache hits.
111 */
112
113 #define MMIO_SPTE_GEN_LOW_START 3
114 #define MMIO_SPTE_GEN_LOW_END 10
115
116 #define MMIO_SPTE_GEN_HIGH_START 52
117 #define MMIO_SPTE_GEN_HIGH_END 62
118
119 #define MMIO_SPTE_GEN_LOW_MASK GENMASK_ULL(MMIO_SPTE_GEN_LOW_END, \
120 MMIO_SPTE_GEN_LOW_START)
121 #define MMIO_SPTE_GEN_HIGH_MASK GENMASK_ULL(MMIO_SPTE_GEN_HIGH_END, \
122 MMIO_SPTE_GEN_HIGH_START)
123 static_assert(!(SPTE_MMU_PRESENT_MASK &
124 (MMIO_SPTE_GEN_LOW_MASK | MMIO_SPTE_GEN_HIGH_MASK)));
125
126 /*
127 * The SPTE MMIO mask must NOT overlap the MMIO generation bits or the
128 * MMU-present bit. The generation obviously co-exists with the magic MMIO
129 * mask/value, and MMIO SPTEs are considered !MMU-present.
130 *
131 * The SPTE MMIO mask is allowed to use hardware "present" bits (i.e. all EPT
132 * RWX bits), all physical address bits (legal PA bits are used for "fast" MMIO
133 * and so they're off-limits for generation; additional checks ensure the mask
134 * doesn't overlap legal PA bits), and bit 63 (carved out for future usage).
135 */
136 #define SPTE_MMIO_ALLOWED_MASK (BIT_ULL(63) | GENMASK_ULL(51, 12) | GENMASK_ULL(2, 0))
137 static_assert(!(SPTE_MMIO_ALLOWED_MASK &
138 (SPTE_MMU_PRESENT_MASK | MMIO_SPTE_GEN_LOW_MASK | MMIO_SPTE_GEN_HIGH_MASK)));
139
140 #define MMIO_SPTE_GEN_LOW_BITS (MMIO_SPTE_GEN_LOW_END - MMIO_SPTE_GEN_LOW_START + 1)
141 #define MMIO_SPTE_GEN_HIGH_BITS (MMIO_SPTE_GEN_HIGH_END - MMIO_SPTE_GEN_HIGH_START + 1)
142
143 /* remember to adjust the comment above as well if you change these */
144 static_assert(MMIO_SPTE_GEN_LOW_BITS == 8 && MMIO_SPTE_GEN_HIGH_BITS == 11);
145
146 #define MMIO_SPTE_GEN_LOW_SHIFT (MMIO_SPTE_GEN_LOW_START - 0)
147 #define MMIO_SPTE_GEN_HIGH_SHIFT (MMIO_SPTE_GEN_HIGH_START - MMIO_SPTE_GEN_LOW_BITS)
148
149 #define MMIO_SPTE_GEN_MASK GENMASK_ULL(MMIO_SPTE_GEN_LOW_BITS + MMIO_SPTE_GEN_HIGH_BITS - 1, 0)
150
151 extern u64 __read_mostly shadow_host_writable_mask;
152 extern u64 __read_mostly shadow_mmu_writable_mask;
153 extern u64 __read_mostly shadow_nx_mask;
154 extern u64 __read_mostly shadow_x_mask; /* mutual exclusive with nx_mask */
155 extern u64 __read_mostly shadow_user_mask;
156 extern u64 __read_mostly shadow_accessed_mask;
157 extern u64 __read_mostly shadow_dirty_mask;
158 extern u64 __read_mostly shadow_mmio_value;
159 extern u64 __read_mostly shadow_mmio_mask;
160 extern u64 __read_mostly shadow_mmio_access_mask;
161 extern u64 __read_mostly shadow_present_mask;
162 extern u64 __read_mostly shadow_memtype_mask;
163 extern u64 __read_mostly shadow_me_value;
164 extern u64 __read_mostly shadow_me_mask;
165
166 /*
167 * SPTEs in MMUs without A/D bits are marked with SPTE_TDP_AD_DISABLED_MASK;
168 * shadow_acc_track_mask is the set of bits to be cleared in non-accessed
169 * pages.
170 */
171 extern u64 __read_mostly shadow_acc_track_mask;
172
173 /*
174 * This mask must be set on all non-zero Non-Present or Reserved SPTEs in order
175 * to guard against L1TF attacks.
176 */
177 extern u64 __read_mostly shadow_nonpresent_or_rsvd_mask;
178
179 /*
180 * The number of high-order 1 bits to use in the mask above.
181 */
182 #define SHADOW_NONPRESENT_OR_RSVD_MASK_LEN 5
183
184 /*
185 * If a thread running without exclusive control of the MMU lock must perform a
186 * multi-part operation on an SPTE, it can set the SPTE to REMOVED_SPTE as a
187 * non-present intermediate value. Other threads which encounter this value
188 * should not modify the SPTE.
189 *
190 * Use a semi-arbitrary value that doesn't set RWX bits, i.e. is not-present on
191 * bot AMD and Intel CPUs, and doesn't set PFN bits, i.e. doesn't create a L1TF
192 * vulnerability. Use only low bits to avoid 64-bit immediates.
193 *
194 * Only used by the TDP MMU.
195 */
196 #define REMOVED_SPTE 0x5a0ULL
197
198 /* Removed SPTEs must not be misconstrued as shadow present PTEs. */
199 static_assert(!(REMOVED_SPTE & SPTE_MMU_PRESENT_MASK));
200
is_removed_spte(u64 spte)201 static inline bool is_removed_spte(u64 spte)
202 {
203 return spte == REMOVED_SPTE;
204 }
205
206 /* Get an SPTE's index into its parent's page table (and the spt array). */
spte_index(u64 * sptep)207 static inline int spte_index(u64 *sptep)
208 {
209 return ((unsigned long)sptep / sizeof(*sptep)) & (SPTE_ENT_PER_PAGE - 1);
210 }
211
212 /*
213 * In some cases, we need to preserve the GFN of a non-present or reserved
214 * SPTE when we usurp the upper five bits of the physical address space to
215 * defend against L1TF, e.g. for MMIO SPTEs. To preserve the GFN, we'll
216 * shift bits of the GFN that overlap with shadow_nonpresent_or_rsvd_mask
217 * left into the reserved bits, i.e. the GFN in the SPTE will be split into
218 * high and low parts. This mask covers the lower bits of the GFN.
219 */
220 extern u64 __read_mostly shadow_nonpresent_or_rsvd_lower_gfn_mask;
221
is_mmio_spte(u64 spte)222 static inline bool is_mmio_spte(u64 spte)
223 {
224 return (spte & shadow_mmio_mask) == shadow_mmio_value &&
225 likely(enable_mmio_caching);
226 }
227
is_shadow_present_pte(u64 pte)228 static inline bool is_shadow_present_pte(u64 pte)
229 {
230 return !!(pte & SPTE_MMU_PRESENT_MASK);
231 }
232
233 /*
234 * Returns true if A/D bits are supported in hardware and are enabled by KVM.
235 * When enabled, KVM uses A/D bits for all non-nested MMUs. Because L1 can
236 * disable A/D bits in EPTP12, SP and SPTE variants are needed to handle the
237 * scenario where KVM is using A/D bits for L1, but not L2.
238 */
kvm_ad_enabled(void)239 static inline bool kvm_ad_enabled(void)
240 {
241 return !!shadow_accessed_mask;
242 }
243
sp_ad_disabled(struct kvm_mmu_page * sp)244 static inline bool sp_ad_disabled(struct kvm_mmu_page *sp)
245 {
246 return sp->role.ad_disabled;
247 }
248
spte_ad_enabled(u64 spte)249 static inline bool spte_ad_enabled(u64 spte)
250 {
251 MMU_WARN_ON(!is_shadow_present_pte(spte));
252 return (spte & SPTE_TDP_AD_MASK) != SPTE_TDP_AD_DISABLED_MASK;
253 }
254
spte_ad_need_write_protect(u64 spte)255 static inline bool spte_ad_need_write_protect(u64 spte)
256 {
257 MMU_WARN_ON(!is_shadow_present_pte(spte));
258 /*
259 * This is benign for non-TDP SPTEs as SPTE_TDP_AD_ENABLED_MASK is '0',
260 * and non-TDP SPTEs will never set these bits. Optimize for 64-bit
261 * TDP and do the A/D type check unconditionally.
262 */
263 return (spte & SPTE_TDP_AD_MASK) != SPTE_TDP_AD_ENABLED_MASK;
264 }
265
spte_shadow_accessed_mask(u64 spte)266 static inline u64 spte_shadow_accessed_mask(u64 spte)
267 {
268 MMU_WARN_ON(!is_shadow_present_pte(spte));
269 return spte_ad_enabled(spte) ? shadow_accessed_mask : 0;
270 }
271
spte_shadow_dirty_mask(u64 spte)272 static inline u64 spte_shadow_dirty_mask(u64 spte)
273 {
274 MMU_WARN_ON(!is_shadow_present_pte(spte));
275 return spte_ad_enabled(spte) ? shadow_dirty_mask : 0;
276 }
277
is_access_track_spte(u64 spte)278 static inline bool is_access_track_spte(u64 spte)
279 {
280 return !spte_ad_enabled(spte) && (spte & shadow_acc_track_mask) == 0;
281 }
282
is_large_pte(u64 pte)283 static inline bool is_large_pte(u64 pte)
284 {
285 return pte & PT_PAGE_SIZE_MASK;
286 }
287
is_last_spte(u64 pte,int level)288 static inline bool is_last_spte(u64 pte, int level)
289 {
290 return (level == PG_LEVEL_4K) || is_large_pte(pte);
291 }
292
is_executable_pte(u64 spte)293 static inline bool is_executable_pte(u64 spte)
294 {
295 return (spte & (shadow_x_mask | shadow_nx_mask)) == shadow_x_mask;
296 }
297
spte_to_pfn(u64 pte)298 static inline kvm_pfn_t spte_to_pfn(u64 pte)
299 {
300 return (pte & SPTE_BASE_ADDR_MASK) >> PAGE_SHIFT;
301 }
302
is_accessed_spte(u64 spte)303 static inline bool is_accessed_spte(u64 spte)
304 {
305 u64 accessed_mask = spte_shadow_accessed_mask(spte);
306
307 return accessed_mask ? spte & accessed_mask
308 : !is_access_track_spte(spte);
309 }
310
is_dirty_spte(u64 spte)311 static inline bool is_dirty_spte(u64 spte)
312 {
313 u64 dirty_mask = spte_shadow_dirty_mask(spte);
314
315 return dirty_mask ? spte & dirty_mask : spte & PT_WRITABLE_MASK;
316 }
317
get_rsvd_bits(struct rsvd_bits_validate * rsvd_check,u64 pte,int level)318 static inline u64 get_rsvd_bits(struct rsvd_bits_validate *rsvd_check, u64 pte,
319 int level)
320 {
321 int bit7 = (pte >> 7) & 1;
322
323 return rsvd_check->rsvd_bits_mask[bit7][level-1];
324 }
325
__is_rsvd_bits_set(struct rsvd_bits_validate * rsvd_check,u64 pte,int level)326 static inline bool __is_rsvd_bits_set(struct rsvd_bits_validate *rsvd_check,
327 u64 pte, int level)
328 {
329 return pte & get_rsvd_bits(rsvd_check, pte, level);
330 }
331
__is_bad_mt_xwr(struct rsvd_bits_validate * rsvd_check,u64 pte)332 static inline bool __is_bad_mt_xwr(struct rsvd_bits_validate *rsvd_check,
333 u64 pte)
334 {
335 return rsvd_check->bad_mt_xwr & BIT_ULL(pte & 0x3f);
336 }
337
is_rsvd_spte(struct rsvd_bits_validate * rsvd_check,u64 spte,int level)338 static __always_inline bool is_rsvd_spte(struct rsvd_bits_validate *rsvd_check,
339 u64 spte, int level)
340 {
341 return __is_bad_mt_xwr(rsvd_check, spte) ||
342 __is_rsvd_bits_set(rsvd_check, spte, level);
343 }
344
345 /*
346 * A shadow-present leaf SPTE may be non-writable for 4 possible reasons:
347 *
348 * 1. To intercept writes for dirty logging. KVM write-protects huge pages
349 * so that they can be split be split down into the dirty logging
350 * granularity (4KiB) whenever the guest writes to them. KVM also
351 * write-protects 4KiB pages so that writes can be recorded in the dirty log
352 * (e.g. if not using PML). SPTEs are write-protected for dirty logging
353 * during the VM-iotcls that enable dirty logging.
354 *
355 * 2. To intercept writes to guest page tables that KVM is shadowing. When a
356 * guest writes to its page table the corresponding shadow page table will
357 * be marked "unsync". That way KVM knows which shadow page tables need to
358 * be updated on the next TLB flush, INVLPG, etc. and which do not.
359 *
360 * 3. To prevent guest writes to read-only memory, such as for memory in a
361 * read-only memslot or guest memory backed by a read-only VMA. Writes to
362 * such pages are disallowed entirely.
363 *
364 * 4. To emulate the Accessed bit for SPTEs without A/D bits. Note, in this
365 * case, the SPTE is access-protected, not just write-protected!
366 *
367 * For cases #1 and #4, KVM can safely make such SPTEs writable without taking
368 * mmu_lock as capturing the Accessed/Dirty state doesn't require taking it.
369 * To differentiate #1 and #4 from #2 and #3, KVM uses two software-only bits
370 * in the SPTE:
371 *
372 * shadow_mmu_writable_mask, aka MMU-writable -
373 * Cleared on SPTEs that KVM is currently write-protecting for shadow paging
374 * purposes (case 2 above).
375 *
376 * shadow_host_writable_mask, aka Host-writable -
377 * Cleared on SPTEs that are not host-writable (case 3 above)
378 *
379 * Note, not all possible combinations of PT_WRITABLE_MASK,
380 * shadow_mmu_writable_mask, and shadow_host_writable_mask are valid. A given
381 * SPTE can be in only one of the following states, which map to the
382 * aforementioned 3 cases:
383 *
384 * shadow_host_writable_mask | shadow_mmu_writable_mask | PT_WRITABLE_MASK
385 * ------------------------- | ------------------------ | ----------------
386 * 1 | 1 | 1 (writable)
387 * 1 | 1 | 0 (case 1)
388 * 1 | 0 | 0 (case 2)
389 * 0 | 0 | 0 (case 3)
390 *
391 * The valid combinations of these bits are checked by
392 * check_spte_writable_invariants() whenever an SPTE is modified.
393 *
394 * Clearing the MMU-writable bit is always done under the MMU lock and always
395 * accompanied by a TLB flush before dropping the lock to avoid corrupting the
396 * shadow page tables between vCPUs. Write-protecting an SPTE for dirty logging
397 * (which does not clear the MMU-writable bit), does not flush TLBs before
398 * dropping the lock, as it only needs to synchronize guest writes with the
399 * dirty bitmap. Similarly, making the SPTE inaccessible (and non-writable) for
400 * access-tracking via the clear_young() MMU notifier also does not flush TLBs.
401 *
402 * So, there is the problem: clearing the MMU-writable bit can encounter a
403 * write-protected SPTE while CPUs still have writable mappings for that SPTE
404 * cached in their TLB. To address this, KVM always flushes TLBs when
405 * write-protecting SPTEs if the MMU-writable bit is set on the old SPTE.
406 *
407 * The Host-writable bit is not modified on present SPTEs, it is only set or
408 * cleared when an SPTE is first faulted in from non-present and then remains
409 * immutable.
410 */
is_writable_pte(unsigned long pte)411 static inline bool is_writable_pte(unsigned long pte)
412 {
413 return pte & PT_WRITABLE_MASK;
414 }
415
416 /* Note: spte must be a shadow-present leaf SPTE. */
check_spte_writable_invariants(u64 spte)417 static inline void check_spte_writable_invariants(u64 spte)
418 {
419 if (spte & shadow_mmu_writable_mask)
420 WARN_ONCE(!(spte & shadow_host_writable_mask),
421 "kvm: MMU-writable SPTE is not Host-writable: %llx",
422 spte);
423 else
424 WARN_ONCE(is_writable_pte(spte),
425 "kvm: Writable SPTE is not MMU-writable: %llx", spte);
426 }
427
is_mmu_writable_spte(u64 spte)428 static inline bool is_mmu_writable_spte(u64 spte)
429 {
430 return spte & shadow_mmu_writable_mask;
431 }
432
get_mmio_spte_generation(u64 spte)433 static inline u64 get_mmio_spte_generation(u64 spte)
434 {
435 u64 gen;
436
437 gen = (spte & MMIO_SPTE_GEN_LOW_MASK) >> MMIO_SPTE_GEN_LOW_SHIFT;
438 gen |= (spte & MMIO_SPTE_GEN_HIGH_MASK) >> MMIO_SPTE_GEN_HIGH_SHIFT;
439 return gen;
440 }
441
442 bool spte_has_volatile_bits(u64 spte);
443
444 bool make_spte(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
445 const struct kvm_memory_slot *slot,
446 unsigned int pte_access, gfn_t gfn, kvm_pfn_t pfn,
447 u64 old_spte, bool prefetch, bool can_unsync,
448 bool host_writable, u64 *new_spte);
449 u64 make_huge_page_split_spte(struct kvm *kvm, u64 huge_spte,
450 union kvm_mmu_page_role role, int index);
451 u64 make_nonleaf_spte(u64 *child_pt, bool ad_disabled);
452 u64 make_mmio_spte(struct kvm_vcpu *vcpu, u64 gfn, unsigned int access);
453 u64 mark_spte_for_access_track(u64 spte);
454
455 /* Restore an acc-track PTE back to a regular PTE */
restore_acc_track_spte(u64 spte)456 static inline u64 restore_acc_track_spte(u64 spte)
457 {
458 u64 saved_bits = (spte >> SHADOW_ACC_TRACK_SAVED_BITS_SHIFT)
459 & SHADOW_ACC_TRACK_SAVED_BITS_MASK;
460
461 spte &= ~shadow_acc_track_mask;
462 spte &= ~(SHADOW_ACC_TRACK_SAVED_BITS_MASK <<
463 SHADOW_ACC_TRACK_SAVED_BITS_SHIFT);
464 spte |= saved_bits;
465
466 return spte;
467 }
468
469 u64 kvm_mmu_changed_pte_notifier_make_spte(u64 old_spte, kvm_pfn_t new_pfn);
470
471 void __init kvm_mmu_spte_module_init(void);
472 void kvm_mmu_reset_all_pte_masks(void);
473
474 #endif
475