1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Kernel-based Virtual Machine driver for Linux
4  *
5  * This module enables machines with Intel VT-x extensions to run virtual
6  * machines without emulation or binary translation.
7  *
8  * MMU support
9  *
10  * Copyright (C) 2006 Qumranet, Inc.
11  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
12  *
13  * Authors:
14  *   Yaniv Kamay  <yaniv@qumranet.com>
15  *   Avi Kivity   <avi@qumranet.com>
16  */
17 
18 #include "irq.h"
19 #include "ioapic.h"
20 #include "mmu.h"
21 #include "mmu_internal.h"
22 #include "tdp_mmu.h"
23 #include "x86.h"
24 #include "kvm_cache_regs.h"
25 #include "kvm_emulate.h"
26 #include "cpuid.h"
27 #include "spte.h"
28 
29 #include <linux/kvm_host.h>
30 #include <linux/types.h>
31 #include <linux/string.h>
32 #include <linux/mm.h>
33 #include <linux/highmem.h>
34 #include <linux/moduleparam.h>
35 #include <linux/export.h>
36 #include <linux/swap.h>
37 #include <linux/hugetlb.h>
38 #include <linux/compiler.h>
39 #include <linux/srcu.h>
40 #include <linux/slab.h>
41 #include <linux/sched/signal.h>
42 #include <linux/uaccess.h>
43 #include <linux/hash.h>
44 #include <linux/kern_levels.h>
45 #include <linux/kthread.h>
46 
47 #include <asm/page.h>
48 #include <asm/memtype.h>
49 #include <asm/cmpxchg.h>
50 #include <asm/io.h>
51 #include <asm/set_memory.h>
52 #include <asm/vmx.h>
53 #include <asm/kvm_page_track.h>
54 #include "trace.h"
55 
56 #include "paging.h"
57 
58 extern bool itlb_multihit_kvm_mitigation;
59 
60 int __read_mostly nx_huge_pages = -1;
61 static uint __read_mostly nx_huge_pages_recovery_period_ms;
62 #ifdef CONFIG_PREEMPT_RT
63 /* Recovery can cause latency spikes, disable it for PREEMPT_RT.  */
64 static uint __read_mostly nx_huge_pages_recovery_ratio = 0;
65 #else
66 static uint __read_mostly nx_huge_pages_recovery_ratio = 60;
67 #endif
68 
69 static int set_nx_huge_pages(const char *val, const struct kernel_param *kp);
70 static int set_nx_huge_pages_recovery_param(const char *val, const struct kernel_param *kp);
71 
72 static const struct kernel_param_ops nx_huge_pages_ops = {
73 	.set = set_nx_huge_pages,
74 	.get = param_get_bool,
75 };
76 
77 static const struct kernel_param_ops nx_huge_pages_recovery_param_ops = {
78 	.set = set_nx_huge_pages_recovery_param,
79 	.get = param_get_uint,
80 };
81 
82 module_param_cb(nx_huge_pages, &nx_huge_pages_ops, &nx_huge_pages, 0644);
83 __MODULE_PARM_TYPE(nx_huge_pages, "bool");
84 module_param_cb(nx_huge_pages_recovery_ratio, &nx_huge_pages_recovery_param_ops,
85 		&nx_huge_pages_recovery_ratio, 0644);
86 __MODULE_PARM_TYPE(nx_huge_pages_recovery_ratio, "uint");
87 module_param_cb(nx_huge_pages_recovery_period_ms, &nx_huge_pages_recovery_param_ops,
88 		&nx_huge_pages_recovery_period_ms, 0644);
89 __MODULE_PARM_TYPE(nx_huge_pages_recovery_period_ms, "uint");
90 
91 static bool __read_mostly force_flush_and_sync_on_reuse;
92 module_param_named(flush_on_reuse, force_flush_and_sync_on_reuse, bool, 0644);
93 
94 /*
95  * When setting this variable to true it enables Two-Dimensional-Paging
96  * where the hardware walks 2 page tables:
97  * 1. the guest-virtual to guest-physical
98  * 2. while doing 1. it walks guest-physical to host-physical
99  * If the hardware supports that we don't need to do shadow paging.
100  */
101 bool tdp_enabled = false;
102 
103 static int max_huge_page_level __read_mostly;
104 static int tdp_root_level __read_mostly;
105 static int max_tdp_level __read_mostly;
106 
107 #ifdef MMU_DEBUG
108 bool dbg = 0;
109 module_param(dbg, bool, 0644);
110 #endif
111 
112 #define PTE_PREFETCH_NUM		8
113 
114 #define PT32_LEVEL_BITS 10
115 
116 #define PT32_LEVEL_SHIFT(level) \
117 		(PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS)
118 
119 #define PT32_LVL_OFFSET_MASK(level) \
120 	(PT32_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
121 						* PT32_LEVEL_BITS))) - 1))
122 
123 #define PT32_INDEX(address, level)\
124 	(((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))
125 
126 
127 #define PT32_BASE_ADDR_MASK PAGE_MASK
128 #define PT32_DIR_BASE_ADDR_MASK \
129 	(PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))
130 #define PT32_LVL_ADDR_MASK(level) \
131 	(PAGE_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
132 					    * PT32_LEVEL_BITS))) - 1))
133 
134 #include <trace/events/kvm.h>
135 
136 /* make pte_list_desc fit well in cache lines */
137 #define PTE_LIST_EXT 14
138 
139 /*
140  * Slight optimization of cacheline layout, by putting `more' and `spte_count'
141  * at the start; then accessing it will only use one single cacheline for
142  * either full (entries==PTE_LIST_EXT) case or entries<=6.
143  */
144 struct pte_list_desc {
145 	struct pte_list_desc *more;
146 	/*
147 	 * Stores number of entries stored in the pte_list_desc.  No need to be
148 	 * u64 but just for easier alignment.  When PTE_LIST_EXT, means full.
149 	 */
150 	u64 spte_count;
151 	u64 *sptes[PTE_LIST_EXT];
152 };
153 
154 struct kvm_shadow_walk_iterator {
155 	u64 addr;
156 	hpa_t shadow_addr;
157 	u64 *sptep;
158 	int level;
159 	unsigned index;
160 };
161 
162 #define for_each_shadow_entry_using_root(_vcpu, _root, _addr, _walker)     \
163 	for (shadow_walk_init_using_root(&(_walker), (_vcpu),              \
164 					 (_root), (_addr));                \
165 	     shadow_walk_okay(&(_walker));			           \
166 	     shadow_walk_next(&(_walker)))
167 
168 #define for_each_shadow_entry(_vcpu, _addr, _walker)            \
169 	for (shadow_walk_init(&(_walker), _vcpu, _addr);	\
170 	     shadow_walk_okay(&(_walker));			\
171 	     shadow_walk_next(&(_walker)))
172 
173 #define for_each_shadow_entry_lockless(_vcpu, _addr, _walker, spte)	\
174 	for (shadow_walk_init(&(_walker), _vcpu, _addr);		\
175 	     shadow_walk_okay(&(_walker)) &&				\
176 		({ spte = mmu_spte_get_lockless(_walker.sptep); 1; });	\
177 	     __shadow_walk_next(&(_walker), spte))
178 
179 static struct kmem_cache *pte_list_desc_cache;
180 struct kmem_cache *mmu_page_header_cache;
181 static struct percpu_counter kvm_total_used_mmu_pages;
182 
183 static void mmu_spte_set(u64 *sptep, u64 spte);
184 
185 struct kvm_mmu_role_regs {
186 	const unsigned long cr0;
187 	const unsigned long cr4;
188 	const u64 efer;
189 };
190 
191 #define CREATE_TRACE_POINTS
192 #include "mmutrace.h"
193 
194 /*
195  * Yes, lot's of underscores.  They're a hint that you probably shouldn't be
196  * reading from the role_regs.  Once the root_role is constructed, it becomes
197  * the single source of truth for the MMU's state.
198  */
199 #define BUILD_MMU_ROLE_REGS_ACCESSOR(reg, name, flag)			\
200 static inline bool __maybe_unused					\
201 ____is_##reg##_##name(const struct kvm_mmu_role_regs *regs)		\
202 {									\
203 	return !!(regs->reg & flag);					\
204 }
205 BUILD_MMU_ROLE_REGS_ACCESSOR(cr0, pg, X86_CR0_PG);
206 BUILD_MMU_ROLE_REGS_ACCESSOR(cr0, wp, X86_CR0_WP);
207 BUILD_MMU_ROLE_REGS_ACCESSOR(cr4, pse, X86_CR4_PSE);
208 BUILD_MMU_ROLE_REGS_ACCESSOR(cr4, pae, X86_CR4_PAE);
209 BUILD_MMU_ROLE_REGS_ACCESSOR(cr4, smep, X86_CR4_SMEP);
210 BUILD_MMU_ROLE_REGS_ACCESSOR(cr4, smap, X86_CR4_SMAP);
211 BUILD_MMU_ROLE_REGS_ACCESSOR(cr4, pke, X86_CR4_PKE);
212 BUILD_MMU_ROLE_REGS_ACCESSOR(cr4, la57, X86_CR4_LA57);
213 BUILD_MMU_ROLE_REGS_ACCESSOR(efer, nx, EFER_NX);
214 BUILD_MMU_ROLE_REGS_ACCESSOR(efer, lma, EFER_LMA);
215 
216 /*
217  * The MMU itself (with a valid role) is the single source of truth for the
218  * MMU.  Do not use the regs used to build the MMU/role, nor the vCPU.  The
219  * regs don't account for dependencies, e.g. clearing CR4 bits if CR0.PG=1,
220  * and the vCPU may be incorrect/irrelevant.
221  */
222 #define BUILD_MMU_ROLE_ACCESSOR(base_or_ext, reg, name)		\
223 static inline bool __maybe_unused is_##reg##_##name(struct kvm_mmu *mmu)	\
224 {								\
225 	return !!(mmu->cpu_role. base_or_ext . reg##_##name);	\
226 }
227 BUILD_MMU_ROLE_ACCESSOR(base, cr0, wp);
228 BUILD_MMU_ROLE_ACCESSOR(ext,  cr4, pse);
229 BUILD_MMU_ROLE_ACCESSOR(ext,  cr4, smep);
230 BUILD_MMU_ROLE_ACCESSOR(ext,  cr4, smap);
231 BUILD_MMU_ROLE_ACCESSOR(ext,  cr4, pke);
232 BUILD_MMU_ROLE_ACCESSOR(ext,  cr4, la57);
233 BUILD_MMU_ROLE_ACCESSOR(base, efer, nx);
234 BUILD_MMU_ROLE_ACCESSOR(ext,  efer, lma);
235 
is_cr0_pg(struct kvm_mmu * mmu)236 static inline bool is_cr0_pg(struct kvm_mmu *mmu)
237 {
238         return mmu->cpu_role.base.level > 0;
239 }
240 
is_cr4_pae(struct kvm_mmu * mmu)241 static inline bool is_cr4_pae(struct kvm_mmu *mmu)
242 {
243         return !mmu->cpu_role.base.has_4_byte_gpte;
244 }
245 
vcpu_to_role_regs(struct kvm_vcpu * vcpu)246 static struct kvm_mmu_role_regs vcpu_to_role_regs(struct kvm_vcpu *vcpu)
247 {
248 	struct kvm_mmu_role_regs regs = {
249 		.cr0 = kvm_read_cr0_bits(vcpu, KVM_MMU_CR0_ROLE_BITS),
250 		.cr4 = kvm_read_cr4_bits(vcpu, KVM_MMU_CR4_ROLE_BITS),
251 		.efer = vcpu->arch.efer,
252 	};
253 
254 	return regs;
255 }
256 
kvm_available_flush_tlb_with_range(void)257 static inline bool kvm_available_flush_tlb_with_range(void)
258 {
259 	return kvm_x86_ops.tlb_remote_flush_with_range;
260 }
261 
kvm_flush_remote_tlbs_with_range(struct kvm * kvm,struct kvm_tlb_range * range)262 static void kvm_flush_remote_tlbs_with_range(struct kvm *kvm,
263 		struct kvm_tlb_range *range)
264 {
265 	int ret = -ENOTSUPP;
266 
267 	if (range && kvm_x86_ops.tlb_remote_flush_with_range)
268 		ret = static_call(kvm_x86_tlb_remote_flush_with_range)(kvm, range);
269 
270 	if (ret)
271 		kvm_flush_remote_tlbs(kvm);
272 }
273 
kvm_flush_remote_tlbs_with_address(struct kvm * kvm,u64 start_gfn,u64 pages)274 void kvm_flush_remote_tlbs_with_address(struct kvm *kvm,
275 		u64 start_gfn, u64 pages)
276 {
277 	struct kvm_tlb_range range;
278 
279 	range.start_gfn = start_gfn;
280 	range.pages = pages;
281 
282 	kvm_flush_remote_tlbs_with_range(kvm, &range);
283 }
284 
mark_mmio_spte(struct kvm_vcpu * vcpu,u64 * sptep,u64 gfn,unsigned int access)285 static void mark_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, u64 gfn,
286 			   unsigned int access)
287 {
288 	u64 spte = make_mmio_spte(vcpu, gfn, access);
289 
290 	trace_mark_mmio_spte(sptep, gfn, spte);
291 	mmu_spte_set(sptep, spte);
292 }
293 
get_mmio_spte_gfn(u64 spte)294 static gfn_t get_mmio_spte_gfn(u64 spte)
295 {
296 	u64 gpa = spte & shadow_nonpresent_or_rsvd_lower_gfn_mask;
297 
298 	gpa |= (spte >> SHADOW_NONPRESENT_OR_RSVD_MASK_LEN)
299 	       & shadow_nonpresent_or_rsvd_mask;
300 
301 	return gpa >> PAGE_SHIFT;
302 }
303 
get_mmio_spte_access(u64 spte)304 static unsigned get_mmio_spte_access(u64 spte)
305 {
306 	return spte & shadow_mmio_access_mask;
307 }
308 
check_mmio_spte(struct kvm_vcpu * vcpu,u64 spte)309 static bool check_mmio_spte(struct kvm_vcpu *vcpu, u64 spte)
310 {
311 	u64 kvm_gen, spte_gen, gen;
312 
313 	gen = kvm_vcpu_memslots(vcpu)->generation;
314 	if (unlikely(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS))
315 		return false;
316 
317 	kvm_gen = gen & MMIO_SPTE_GEN_MASK;
318 	spte_gen = get_mmio_spte_generation(spte);
319 
320 	trace_check_mmio_spte(spte, kvm_gen, spte_gen);
321 	return likely(kvm_gen == spte_gen);
322 }
323 
is_cpuid_PSE36(void)324 static int is_cpuid_PSE36(void)
325 {
326 	return 1;
327 }
328 
pse36_gfn_delta(u32 gpte)329 static gfn_t pse36_gfn_delta(u32 gpte)
330 {
331 	int shift = 32 - PT32_DIR_PSE36_SHIFT - PAGE_SHIFT;
332 
333 	return (gpte & PT32_DIR_PSE36_MASK) << shift;
334 }
335 
336 #ifdef CONFIG_X86_64
__set_spte(u64 * sptep,u64 spte)337 static void __set_spte(u64 *sptep, u64 spte)
338 {
339 	WRITE_ONCE(*sptep, spte);
340 }
341 
__update_clear_spte_fast(u64 * sptep,u64 spte)342 static void __update_clear_spte_fast(u64 *sptep, u64 spte)
343 {
344 	WRITE_ONCE(*sptep, spte);
345 }
346 
__update_clear_spte_slow(u64 * sptep,u64 spte)347 static u64 __update_clear_spte_slow(u64 *sptep, u64 spte)
348 {
349 	return xchg(sptep, spte);
350 }
351 
__get_spte_lockless(u64 * sptep)352 static u64 __get_spte_lockless(u64 *sptep)
353 {
354 	return READ_ONCE(*sptep);
355 }
356 #else
357 union split_spte {
358 	struct {
359 		u32 spte_low;
360 		u32 spte_high;
361 	};
362 	u64 spte;
363 };
364 
count_spte_clear(u64 * sptep,u64 spte)365 static void count_spte_clear(u64 *sptep, u64 spte)
366 {
367 	struct kvm_mmu_page *sp =  sptep_to_sp(sptep);
368 
369 	if (is_shadow_present_pte(spte))
370 		return;
371 
372 	/* Ensure the spte is completely set before we increase the count */
373 	smp_wmb();
374 	sp->clear_spte_count++;
375 }
376 
__set_spte(u64 * sptep,u64 spte)377 static void __set_spte(u64 *sptep, u64 spte)
378 {
379 	union split_spte *ssptep, sspte;
380 
381 	ssptep = (union split_spte *)sptep;
382 	sspte = (union split_spte)spte;
383 
384 	ssptep->spte_high = sspte.spte_high;
385 
386 	/*
387 	 * If we map the spte from nonpresent to present, We should store
388 	 * the high bits firstly, then set present bit, so cpu can not
389 	 * fetch this spte while we are setting the spte.
390 	 */
391 	smp_wmb();
392 
393 	WRITE_ONCE(ssptep->spte_low, sspte.spte_low);
394 }
395 
__update_clear_spte_fast(u64 * sptep,u64 spte)396 static void __update_clear_spte_fast(u64 *sptep, u64 spte)
397 {
398 	union split_spte *ssptep, sspte;
399 
400 	ssptep = (union split_spte *)sptep;
401 	sspte = (union split_spte)spte;
402 
403 	WRITE_ONCE(ssptep->spte_low, sspte.spte_low);
404 
405 	/*
406 	 * If we map the spte from present to nonpresent, we should clear
407 	 * present bit firstly to avoid vcpu fetch the old high bits.
408 	 */
409 	smp_wmb();
410 
411 	ssptep->spte_high = sspte.spte_high;
412 	count_spte_clear(sptep, spte);
413 }
414 
__update_clear_spte_slow(u64 * sptep,u64 spte)415 static u64 __update_clear_spte_slow(u64 *sptep, u64 spte)
416 {
417 	union split_spte *ssptep, sspte, orig;
418 
419 	ssptep = (union split_spte *)sptep;
420 	sspte = (union split_spte)spte;
421 
422 	/* xchg acts as a barrier before the setting of the high bits */
423 	orig.spte_low = xchg(&ssptep->spte_low, sspte.spte_low);
424 	orig.spte_high = ssptep->spte_high;
425 	ssptep->spte_high = sspte.spte_high;
426 	count_spte_clear(sptep, spte);
427 
428 	return orig.spte;
429 }
430 
431 /*
432  * The idea using the light way get the spte on x86_32 guest is from
433  * gup_get_pte (mm/gup.c).
434  *
435  * An spte tlb flush may be pending, because kvm_set_pte_rmapp
436  * coalesces them and we are running out of the MMU lock.  Therefore
437  * we need to protect against in-progress updates of the spte.
438  *
439  * Reading the spte while an update is in progress may get the old value
440  * for the high part of the spte.  The race is fine for a present->non-present
441  * change (because the high part of the spte is ignored for non-present spte),
442  * but for a present->present change we must reread the spte.
443  *
444  * All such changes are done in two steps (present->non-present and
445  * non-present->present), hence it is enough to count the number of
446  * present->non-present updates: if it changed while reading the spte,
447  * we might have hit the race.  This is done using clear_spte_count.
448  */
__get_spte_lockless(u64 * sptep)449 static u64 __get_spte_lockless(u64 *sptep)
450 {
451 	struct kvm_mmu_page *sp =  sptep_to_sp(sptep);
452 	union split_spte spte, *orig = (union split_spte *)sptep;
453 	int count;
454 
455 retry:
456 	count = sp->clear_spte_count;
457 	smp_rmb();
458 
459 	spte.spte_low = orig->spte_low;
460 	smp_rmb();
461 
462 	spte.spte_high = orig->spte_high;
463 	smp_rmb();
464 
465 	if (unlikely(spte.spte_low != orig->spte_low ||
466 	      count != sp->clear_spte_count))
467 		goto retry;
468 
469 	return spte.spte;
470 }
471 #endif
472 
473 /* Rules for using mmu_spte_set:
474  * Set the sptep from nonpresent to present.
475  * Note: the sptep being assigned *must* be either not present
476  * or in a state where the hardware will not attempt to update
477  * the spte.
478  */
mmu_spte_set(u64 * sptep,u64 new_spte)479 static void mmu_spte_set(u64 *sptep, u64 new_spte)
480 {
481 	WARN_ON(is_shadow_present_pte(*sptep));
482 	__set_spte(sptep, new_spte);
483 }
484 
485 /*
486  * Update the SPTE (excluding the PFN), but do not track changes in its
487  * accessed/dirty status.
488  */
mmu_spte_update_no_track(u64 * sptep,u64 new_spte)489 static u64 mmu_spte_update_no_track(u64 *sptep, u64 new_spte)
490 {
491 	u64 old_spte = *sptep;
492 
493 	WARN_ON(!is_shadow_present_pte(new_spte));
494 	check_spte_writable_invariants(new_spte);
495 
496 	if (!is_shadow_present_pte(old_spte)) {
497 		mmu_spte_set(sptep, new_spte);
498 		return old_spte;
499 	}
500 
501 	if (!spte_has_volatile_bits(old_spte))
502 		__update_clear_spte_fast(sptep, new_spte);
503 	else
504 		old_spte = __update_clear_spte_slow(sptep, new_spte);
505 
506 	WARN_ON(spte_to_pfn(old_spte) != spte_to_pfn(new_spte));
507 
508 	return old_spte;
509 }
510 
511 /* Rules for using mmu_spte_update:
512  * Update the state bits, it means the mapped pfn is not changed.
513  *
514  * Whenever an MMU-writable SPTE is overwritten with a read-only SPTE, remote
515  * TLBs must be flushed. Otherwise rmap_write_protect will find a read-only
516  * spte, even though the writable spte might be cached on a CPU's TLB.
517  *
518  * Returns true if the TLB needs to be flushed
519  */
mmu_spte_update(u64 * sptep,u64 new_spte)520 static bool mmu_spte_update(u64 *sptep, u64 new_spte)
521 {
522 	bool flush = false;
523 	u64 old_spte = mmu_spte_update_no_track(sptep, new_spte);
524 
525 	if (!is_shadow_present_pte(old_spte))
526 		return false;
527 
528 	/*
529 	 * For the spte updated out of mmu-lock is safe, since
530 	 * we always atomically update it, see the comments in
531 	 * spte_has_volatile_bits().
532 	 */
533 	if (is_mmu_writable_spte(old_spte) &&
534 	      !is_writable_pte(new_spte))
535 		flush = true;
536 
537 	/*
538 	 * Flush TLB when accessed/dirty states are changed in the page tables,
539 	 * to guarantee consistency between TLB and page tables.
540 	 */
541 
542 	if (is_accessed_spte(old_spte) && !is_accessed_spte(new_spte)) {
543 		flush = true;
544 		kvm_set_pfn_accessed(spte_to_pfn(old_spte));
545 	}
546 
547 	if (is_dirty_spte(old_spte) && !is_dirty_spte(new_spte)) {
548 		flush = true;
549 		kvm_set_pfn_dirty(spte_to_pfn(old_spte));
550 	}
551 
552 	return flush;
553 }
554 
555 /*
556  * Rules for using mmu_spte_clear_track_bits:
557  * It sets the sptep from present to nonpresent, and track the
558  * state bits, it is used to clear the last level sptep.
559  * Returns the old PTE.
560  */
mmu_spte_clear_track_bits(struct kvm * kvm,u64 * sptep)561 static int mmu_spte_clear_track_bits(struct kvm *kvm, u64 *sptep)
562 {
563 	kvm_pfn_t pfn;
564 	u64 old_spte = *sptep;
565 	int level = sptep_to_sp(sptep)->role.level;
566 
567 	if (!is_shadow_present_pte(old_spte) ||
568 	    !spte_has_volatile_bits(old_spte))
569 		__update_clear_spte_fast(sptep, 0ull);
570 	else
571 		old_spte = __update_clear_spte_slow(sptep, 0ull);
572 
573 	if (!is_shadow_present_pte(old_spte))
574 		return old_spte;
575 
576 	kvm_update_page_stats(kvm, level, -1);
577 
578 	pfn = spte_to_pfn(old_spte);
579 
580 	/*
581 	 * KVM does not hold the refcount of the page used by
582 	 * kvm mmu, before reclaiming the page, we should
583 	 * unmap it from mmu first.
584 	 */
585 	WARN_ON(!kvm_is_reserved_pfn(pfn) && !page_count(pfn_to_page(pfn)));
586 
587 	if (is_accessed_spte(old_spte))
588 		kvm_set_pfn_accessed(pfn);
589 
590 	if (is_dirty_spte(old_spte))
591 		kvm_set_pfn_dirty(pfn);
592 
593 	return old_spte;
594 }
595 
596 /*
597  * Rules for using mmu_spte_clear_no_track:
598  * Directly clear spte without caring the state bits of sptep,
599  * it is used to set the upper level spte.
600  */
mmu_spte_clear_no_track(u64 * sptep)601 static void mmu_spte_clear_no_track(u64 *sptep)
602 {
603 	__update_clear_spte_fast(sptep, 0ull);
604 }
605 
mmu_spte_get_lockless(u64 * sptep)606 static u64 mmu_spte_get_lockless(u64 *sptep)
607 {
608 	return __get_spte_lockless(sptep);
609 }
610 
611 /* Returns the Accessed status of the PTE and resets it at the same time. */
mmu_spte_age(u64 * sptep)612 static bool mmu_spte_age(u64 *sptep)
613 {
614 	u64 spte = mmu_spte_get_lockless(sptep);
615 
616 	if (!is_accessed_spte(spte))
617 		return false;
618 
619 	if (spte_ad_enabled(spte)) {
620 		clear_bit((ffs(shadow_accessed_mask) - 1),
621 			  (unsigned long *)sptep);
622 	} else {
623 		/*
624 		 * Capture the dirty status of the page, so that it doesn't get
625 		 * lost when the SPTE is marked for access tracking.
626 		 */
627 		if (is_writable_pte(spte))
628 			kvm_set_pfn_dirty(spte_to_pfn(spte));
629 
630 		spte = mark_spte_for_access_track(spte);
631 		mmu_spte_update_no_track(sptep, spte);
632 	}
633 
634 	return true;
635 }
636 
walk_shadow_page_lockless_begin(struct kvm_vcpu * vcpu)637 static void walk_shadow_page_lockless_begin(struct kvm_vcpu *vcpu)
638 {
639 	if (is_tdp_mmu(vcpu->arch.mmu)) {
640 		kvm_tdp_mmu_walk_lockless_begin();
641 	} else {
642 		/*
643 		 * Prevent page table teardown by making any free-er wait during
644 		 * kvm_flush_remote_tlbs() IPI to all active vcpus.
645 		 */
646 		local_irq_disable();
647 
648 		/*
649 		 * Make sure a following spte read is not reordered ahead of the write
650 		 * to vcpu->mode.
651 		 */
652 		smp_store_mb(vcpu->mode, READING_SHADOW_PAGE_TABLES);
653 	}
654 }
655 
walk_shadow_page_lockless_end(struct kvm_vcpu * vcpu)656 static void walk_shadow_page_lockless_end(struct kvm_vcpu *vcpu)
657 {
658 	if (is_tdp_mmu(vcpu->arch.mmu)) {
659 		kvm_tdp_mmu_walk_lockless_end();
660 	} else {
661 		/*
662 		 * Make sure the write to vcpu->mode is not reordered in front of
663 		 * reads to sptes.  If it does, kvm_mmu_commit_zap_page() can see us
664 		 * OUTSIDE_GUEST_MODE and proceed to free the shadow page table.
665 		 */
666 		smp_store_release(&vcpu->mode, OUTSIDE_GUEST_MODE);
667 		local_irq_enable();
668 	}
669 }
670 
mmu_topup_memory_caches(struct kvm_vcpu * vcpu,bool maybe_indirect)671 static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu, bool maybe_indirect)
672 {
673 	int r;
674 
675 	/* 1 rmap, 1 parent PTE per level, and the prefetched rmaps. */
676 	r = kvm_mmu_topup_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache,
677 				       1 + PT64_ROOT_MAX_LEVEL + PTE_PREFETCH_NUM);
678 	if (r)
679 		return r;
680 	r = kvm_mmu_topup_memory_cache(&vcpu->arch.mmu_shadow_page_cache,
681 				       PT64_ROOT_MAX_LEVEL);
682 	if (r)
683 		return r;
684 	if (maybe_indirect) {
685 		r = kvm_mmu_topup_memory_cache(&vcpu->arch.mmu_gfn_array_cache,
686 					       PT64_ROOT_MAX_LEVEL);
687 		if (r)
688 			return r;
689 	}
690 	return kvm_mmu_topup_memory_cache(&vcpu->arch.mmu_page_header_cache,
691 					  PT64_ROOT_MAX_LEVEL);
692 }
693 
mmu_free_memory_caches(struct kvm_vcpu * vcpu)694 static void mmu_free_memory_caches(struct kvm_vcpu *vcpu)
695 {
696 	kvm_mmu_free_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache);
697 	kvm_mmu_free_memory_cache(&vcpu->arch.mmu_shadow_page_cache);
698 	kvm_mmu_free_memory_cache(&vcpu->arch.mmu_gfn_array_cache);
699 	kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_header_cache);
700 }
701 
mmu_alloc_pte_list_desc(struct kvm_vcpu * vcpu)702 static struct pte_list_desc *mmu_alloc_pte_list_desc(struct kvm_vcpu *vcpu)
703 {
704 	return kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_pte_list_desc_cache);
705 }
706 
mmu_free_pte_list_desc(struct pte_list_desc * pte_list_desc)707 static void mmu_free_pte_list_desc(struct pte_list_desc *pte_list_desc)
708 {
709 	kmem_cache_free(pte_list_desc_cache, pte_list_desc);
710 }
711 
kvm_mmu_page_get_gfn(struct kvm_mmu_page * sp,int index)712 static gfn_t kvm_mmu_page_get_gfn(struct kvm_mmu_page *sp, int index)
713 {
714 	if (sp->role.passthrough)
715 		return sp->gfn;
716 
717 	if (!sp->role.direct)
718 		return sp->gfns[index];
719 
720 	return sp->gfn + (index << ((sp->role.level - 1) * PT64_LEVEL_BITS));
721 }
722 
kvm_mmu_page_set_gfn(struct kvm_mmu_page * sp,int index,gfn_t gfn)723 static void kvm_mmu_page_set_gfn(struct kvm_mmu_page *sp, int index, gfn_t gfn)
724 {
725 	if (sp->role.passthrough) {
726 		WARN_ON_ONCE(gfn != sp->gfn);
727 		return;
728 	}
729 
730 	if (!sp->role.direct) {
731 		sp->gfns[index] = gfn;
732 		return;
733 	}
734 
735 	if (WARN_ON(gfn != kvm_mmu_page_get_gfn(sp, index)))
736 		pr_err_ratelimited("gfn mismatch under direct page %llx "
737 				   "(expected %llx, got %llx)\n",
738 				   sp->gfn,
739 				   kvm_mmu_page_get_gfn(sp, index), gfn);
740 }
741 
742 /*
743  * Return the pointer to the large page information for a given gfn,
744  * handling slots that are not large page aligned.
745  */
lpage_info_slot(gfn_t gfn,const struct kvm_memory_slot * slot,int level)746 static struct kvm_lpage_info *lpage_info_slot(gfn_t gfn,
747 		const struct kvm_memory_slot *slot, int level)
748 {
749 	unsigned long idx;
750 
751 	idx = gfn_to_index(gfn, slot->base_gfn, level);
752 	return &slot->arch.lpage_info[level - 2][idx];
753 }
754 
update_gfn_disallow_lpage_count(const struct kvm_memory_slot * slot,gfn_t gfn,int count)755 static void update_gfn_disallow_lpage_count(const struct kvm_memory_slot *slot,
756 					    gfn_t gfn, int count)
757 {
758 	struct kvm_lpage_info *linfo;
759 	int i;
760 
761 	for (i = PG_LEVEL_2M; i <= KVM_MAX_HUGEPAGE_LEVEL; ++i) {
762 		linfo = lpage_info_slot(gfn, slot, i);
763 		linfo->disallow_lpage += count;
764 		WARN_ON(linfo->disallow_lpage < 0);
765 	}
766 }
767 
kvm_mmu_gfn_disallow_lpage(const struct kvm_memory_slot * slot,gfn_t gfn)768 void kvm_mmu_gfn_disallow_lpage(const struct kvm_memory_slot *slot, gfn_t gfn)
769 {
770 	update_gfn_disallow_lpage_count(slot, gfn, 1);
771 }
772 
kvm_mmu_gfn_allow_lpage(const struct kvm_memory_slot * slot,gfn_t gfn)773 void kvm_mmu_gfn_allow_lpage(const struct kvm_memory_slot *slot, gfn_t gfn)
774 {
775 	update_gfn_disallow_lpage_count(slot, gfn, -1);
776 }
777 
account_shadowed(struct kvm * kvm,struct kvm_mmu_page * sp)778 static void account_shadowed(struct kvm *kvm, struct kvm_mmu_page *sp)
779 {
780 	struct kvm_memslots *slots;
781 	struct kvm_memory_slot *slot;
782 	gfn_t gfn;
783 
784 	kvm->arch.indirect_shadow_pages++;
785 	gfn = sp->gfn;
786 	slots = kvm_memslots_for_spte_role(kvm, sp->role);
787 	slot = __gfn_to_memslot(slots, gfn);
788 
789 	/* the non-leaf shadow pages are keeping readonly. */
790 	if (sp->role.level > PG_LEVEL_4K)
791 		return kvm_slot_page_track_add_page(kvm, slot, gfn,
792 						    KVM_PAGE_TRACK_WRITE);
793 
794 	kvm_mmu_gfn_disallow_lpage(slot, gfn);
795 }
796 
account_huge_nx_page(struct kvm * kvm,struct kvm_mmu_page * sp)797 void account_huge_nx_page(struct kvm *kvm, struct kvm_mmu_page *sp)
798 {
799 	if (sp->lpage_disallowed)
800 		return;
801 
802 	++kvm->stat.nx_lpage_splits;
803 	list_add_tail(&sp->lpage_disallowed_link,
804 		      &kvm->arch.lpage_disallowed_mmu_pages);
805 	sp->lpage_disallowed = true;
806 }
807 
unaccount_shadowed(struct kvm * kvm,struct kvm_mmu_page * sp)808 static void unaccount_shadowed(struct kvm *kvm, struct kvm_mmu_page *sp)
809 {
810 	struct kvm_memslots *slots;
811 	struct kvm_memory_slot *slot;
812 	gfn_t gfn;
813 
814 	kvm->arch.indirect_shadow_pages--;
815 	gfn = sp->gfn;
816 	slots = kvm_memslots_for_spte_role(kvm, sp->role);
817 	slot = __gfn_to_memslot(slots, gfn);
818 	if (sp->role.level > PG_LEVEL_4K)
819 		return kvm_slot_page_track_remove_page(kvm, slot, gfn,
820 						       KVM_PAGE_TRACK_WRITE);
821 
822 	kvm_mmu_gfn_allow_lpage(slot, gfn);
823 }
824 
unaccount_huge_nx_page(struct kvm * kvm,struct kvm_mmu_page * sp)825 void unaccount_huge_nx_page(struct kvm *kvm, struct kvm_mmu_page *sp)
826 {
827 	--kvm->stat.nx_lpage_splits;
828 	sp->lpage_disallowed = false;
829 	list_del(&sp->lpage_disallowed_link);
830 }
831 
832 static struct kvm_memory_slot *
gfn_to_memslot_dirty_bitmap(struct kvm_vcpu * vcpu,gfn_t gfn,bool no_dirty_log)833 gfn_to_memslot_dirty_bitmap(struct kvm_vcpu *vcpu, gfn_t gfn,
834 			    bool no_dirty_log)
835 {
836 	struct kvm_memory_slot *slot;
837 
838 	slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
839 	if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
840 		return NULL;
841 	if (no_dirty_log && kvm_slot_dirty_track_enabled(slot))
842 		return NULL;
843 
844 	return slot;
845 }
846 
847 /*
848  * About rmap_head encoding:
849  *
850  * If the bit zero of rmap_head->val is clear, then it points to the only spte
851  * in this rmap chain. Otherwise, (rmap_head->val & ~1) points to a struct
852  * pte_list_desc containing more mappings.
853  */
854 
855 /*
856  * Returns the number of pointers in the rmap chain, not counting the new one.
857  */
pte_list_add(struct kvm_vcpu * vcpu,u64 * spte,struct kvm_rmap_head * rmap_head)858 static int pte_list_add(struct kvm_vcpu *vcpu, u64 *spte,
859 			struct kvm_rmap_head *rmap_head)
860 {
861 	struct pte_list_desc *desc;
862 	int count = 0;
863 
864 	if (!rmap_head->val) {
865 		rmap_printk("%p %llx 0->1\n", spte, *spte);
866 		rmap_head->val = (unsigned long)spte;
867 	} else if (!(rmap_head->val & 1)) {
868 		rmap_printk("%p %llx 1->many\n", spte, *spte);
869 		desc = mmu_alloc_pte_list_desc(vcpu);
870 		desc->sptes[0] = (u64 *)rmap_head->val;
871 		desc->sptes[1] = spte;
872 		desc->spte_count = 2;
873 		rmap_head->val = (unsigned long)desc | 1;
874 		++count;
875 	} else {
876 		rmap_printk("%p %llx many->many\n", spte, *spte);
877 		desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
878 		while (desc->spte_count == PTE_LIST_EXT) {
879 			count += PTE_LIST_EXT;
880 			if (!desc->more) {
881 				desc->more = mmu_alloc_pte_list_desc(vcpu);
882 				desc = desc->more;
883 				desc->spte_count = 0;
884 				break;
885 			}
886 			desc = desc->more;
887 		}
888 		count += desc->spte_count;
889 		desc->sptes[desc->spte_count++] = spte;
890 	}
891 	return count;
892 }
893 
894 static void
pte_list_desc_remove_entry(struct kvm_rmap_head * rmap_head,struct pte_list_desc * desc,int i,struct pte_list_desc * prev_desc)895 pte_list_desc_remove_entry(struct kvm_rmap_head *rmap_head,
896 			   struct pte_list_desc *desc, int i,
897 			   struct pte_list_desc *prev_desc)
898 {
899 	int j = desc->spte_count - 1;
900 
901 	desc->sptes[i] = desc->sptes[j];
902 	desc->sptes[j] = NULL;
903 	desc->spte_count--;
904 	if (desc->spte_count)
905 		return;
906 	if (!prev_desc && !desc->more)
907 		rmap_head->val = 0;
908 	else
909 		if (prev_desc)
910 			prev_desc->more = desc->more;
911 		else
912 			rmap_head->val = (unsigned long)desc->more | 1;
913 	mmu_free_pte_list_desc(desc);
914 }
915 
__pte_list_remove(u64 * spte,struct kvm_rmap_head * rmap_head)916 static void __pte_list_remove(u64 *spte, struct kvm_rmap_head *rmap_head)
917 {
918 	struct pte_list_desc *desc;
919 	struct pte_list_desc *prev_desc;
920 	int i;
921 
922 	if (!rmap_head->val) {
923 		pr_err("%s: %p 0->BUG\n", __func__, spte);
924 		BUG();
925 	} else if (!(rmap_head->val & 1)) {
926 		rmap_printk("%p 1->0\n", spte);
927 		if ((u64 *)rmap_head->val != spte) {
928 			pr_err("%s:  %p 1->BUG\n", __func__, spte);
929 			BUG();
930 		}
931 		rmap_head->val = 0;
932 	} else {
933 		rmap_printk("%p many->many\n", spte);
934 		desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
935 		prev_desc = NULL;
936 		while (desc) {
937 			for (i = 0; i < desc->spte_count; ++i) {
938 				if (desc->sptes[i] == spte) {
939 					pte_list_desc_remove_entry(rmap_head,
940 							desc, i, prev_desc);
941 					return;
942 				}
943 			}
944 			prev_desc = desc;
945 			desc = desc->more;
946 		}
947 		pr_err("%s: %p many->many\n", __func__, spte);
948 		BUG();
949 	}
950 }
951 
pte_list_remove(struct kvm * kvm,struct kvm_rmap_head * rmap_head,u64 * sptep)952 static void pte_list_remove(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
953 			    u64 *sptep)
954 {
955 	mmu_spte_clear_track_bits(kvm, sptep);
956 	__pte_list_remove(sptep, rmap_head);
957 }
958 
959 /* Return true if rmap existed, false otherwise */
pte_list_destroy(struct kvm * kvm,struct kvm_rmap_head * rmap_head)960 static bool pte_list_destroy(struct kvm *kvm, struct kvm_rmap_head *rmap_head)
961 {
962 	struct pte_list_desc *desc, *next;
963 	int i;
964 
965 	if (!rmap_head->val)
966 		return false;
967 
968 	if (!(rmap_head->val & 1)) {
969 		mmu_spte_clear_track_bits(kvm, (u64 *)rmap_head->val);
970 		goto out;
971 	}
972 
973 	desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
974 
975 	for (; desc; desc = next) {
976 		for (i = 0; i < desc->spte_count; i++)
977 			mmu_spte_clear_track_bits(kvm, desc->sptes[i]);
978 		next = desc->more;
979 		mmu_free_pte_list_desc(desc);
980 	}
981 out:
982 	/* rmap_head is meaningless now, remember to reset it */
983 	rmap_head->val = 0;
984 	return true;
985 }
986 
pte_list_count(struct kvm_rmap_head * rmap_head)987 unsigned int pte_list_count(struct kvm_rmap_head *rmap_head)
988 {
989 	struct pte_list_desc *desc;
990 	unsigned int count = 0;
991 
992 	if (!rmap_head->val)
993 		return 0;
994 	else if (!(rmap_head->val & 1))
995 		return 1;
996 
997 	desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
998 
999 	while (desc) {
1000 		count += desc->spte_count;
1001 		desc = desc->more;
1002 	}
1003 
1004 	return count;
1005 }
1006 
gfn_to_rmap(gfn_t gfn,int level,const struct kvm_memory_slot * slot)1007 static struct kvm_rmap_head *gfn_to_rmap(gfn_t gfn, int level,
1008 					 const struct kvm_memory_slot *slot)
1009 {
1010 	unsigned long idx;
1011 
1012 	idx = gfn_to_index(gfn, slot->base_gfn, level);
1013 	return &slot->arch.rmap[level - PG_LEVEL_4K][idx];
1014 }
1015 
rmap_can_add(struct kvm_vcpu * vcpu)1016 static bool rmap_can_add(struct kvm_vcpu *vcpu)
1017 {
1018 	struct kvm_mmu_memory_cache *mc;
1019 
1020 	mc = &vcpu->arch.mmu_pte_list_desc_cache;
1021 	return kvm_mmu_memory_cache_nr_free_objects(mc);
1022 }
1023 
rmap_remove(struct kvm * kvm,u64 * spte)1024 static void rmap_remove(struct kvm *kvm, u64 *spte)
1025 {
1026 	struct kvm_memslots *slots;
1027 	struct kvm_memory_slot *slot;
1028 	struct kvm_mmu_page *sp;
1029 	gfn_t gfn;
1030 	struct kvm_rmap_head *rmap_head;
1031 
1032 	sp = sptep_to_sp(spte);
1033 	gfn = kvm_mmu_page_get_gfn(sp, spte - sp->spt);
1034 
1035 	/*
1036 	 * Unlike rmap_add, rmap_remove does not run in the context of a vCPU
1037 	 * so we have to determine which memslots to use based on context
1038 	 * information in sp->role.
1039 	 */
1040 	slots = kvm_memslots_for_spte_role(kvm, sp->role);
1041 
1042 	slot = __gfn_to_memslot(slots, gfn);
1043 	rmap_head = gfn_to_rmap(gfn, sp->role.level, slot);
1044 
1045 	__pte_list_remove(spte, rmap_head);
1046 }
1047 
1048 /*
1049  * Used by the following functions to iterate through the sptes linked by a
1050  * rmap.  All fields are private and not assumed to be used outside.
1051  */
1052 struct rmap_iterator {
1053 	/* private fields */
1054 	struct pte_list_desc *desc;	/* holds the sptep if not NULL */
1055 	int pos;			/* index of the sptep */
1056 };
1057 
1058 /*
1059  * Iteration must be started by this function.  This should also be used after
1060  * removing/dropping sptes from the rmap link because in such cases the
1061  * information in the iterator may not be valid.
1062  *
1063  * Returns sptep if found, NULL otherwise.
1064  */
rmap_get_first(struct kvm_rmap_head * rmap_head,struct rmap_iterator * iter)1065 static u64 *rmap_get_first(struct kvm_rmap_head *rmap_head,
1066 			   struct rmap_iterator *iter)
1067 {
1068 	u64 *sptep;
1069 
1070 	if (!rmap_head->val)
1071 		return NULL;
1072 
1073 	if (!(rmap_head->val & 1)) {
1074 		iter->desc = NULL;
1075 		sptep = (u64 *)rmap_head->val;
1076 		goto out;
1077 	}
1078 
1079 	iter->desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
1080 	iter->pos = 0;
1081 	sptep = iter->desc->sptes[iter->pos];
1082 out:
1083 	BUG_ON(!is_shadow_present_pte(*sptep));
1084 	return sptep;
1085 }
1086 
1087 /*
1088  * Must be used with a valid iterator: e.g. after rmap_get_first().
1089  *
1090  * Returns sptep if found, NULL otherwise.
1091  */
rmap_get_next(struct rmap_iterator * iter)1092 static u64 *rmap_get_next(struct rmap_iterator *iter)
1093 {
1094 	u64 *sptep;
1095 
1096 	if (iter->desc) {
1097 		if (iter->pos < PTE_LIST_EXT - 1) {
1098 			++iter->pos;
1099 			sptep = iter->desc->sptes[iter->pos];
1100 			if (sptep)
1101 				goto out;
1102 		}
1103 
1104 		iter->desc = iter->desc->more;
1105 
1106 		if (iter->desc) {
1107 			iter->pos = 0;
1108 			/* desc->sptes[0] cannot be NULL */
1109 			sptep = iter->desc->sptes[iter->pos];
1110 			goto out;
1111 		}
1112 	}
1113 
1114 	return NULL;
1115 out:
1116 	BUG_ON(!is_shadow_present_pte(*sptep));
1117 	return sptep;
1118 }
1119 
1120 #define for_each_rmap_spte(_rmap_head_, _iter_, _spte_)			\
1121 	for (_spte_ = rmap_get_first(_rmap_head_, _iter_);		\
1122 	     _spte_; _spte_ = rmap_get_next(_iter_))
1123 
drop_spte(struct kvm * kvm,u64 * sptep)1124 static void drop_spte(struct kvm *kvm, u64 *sptep)
1125 {
1126 	u64 old_spte = mmu_spte_clear_track_bits(kvm, sptep);
1127 
1128 	if (is_shadow_present_pte(old_spte))
1129 		rmap_remove(kvm, sptep);
1130 }
1131 
1132 
__drop_large_spte(struct kvm * kvm,u64 * sptep)1133 static bool __drop_large_spte(struct kvm *kvm, u64 *sptep)
1134 {
1135 	if (is_large_pte(*sptep)) {
1136 		WARN_ON(sptep_to_sp(sptep)->role.level == PG_LEVEL_4K);
1137 		drop_spte(kvm, sptep);
1138 		return true;
1139 	}
1140 
1141 	return false;
1142 }
1143 
drop_large_spte(struct kvm_vcpu * vcpu,u64 * sptep)1144 static void drop_large_spte(struct kvm_vcpu *vcpu, u64 *sptep)
1145 {
1146 	if (__drop_large_spte(vcpu->kvm, sptep)) {
1147 		struct kvm_mmu_page *sp = sptep_to_sp(sptep);
1148 
1149 		kvm_flush_remote_tlbs_with_address(vcpu->kvm, sp->gfn,
1150 			KVM_PAGES_PER_HPAGE(sp->role.level));
1151 	}
1152 }
1153 
1154 /*
1155  * Write-protect on the specified @sptep, @pt_protect indicates whether
1156  * spte write-protection is caused by protecting shadow page table.
1157  *
1158  * Note: write protection is difference between dirty logging and spte
1159  * protection:
1160  * - for dirty logging, the spte can be set to writable at anytime if
1161  *   its dirty bitmap is properly set.
1162  * - for spte protection, the spte can be writable only after unsync-ing
1163  *   shadow page.
1164  *
1165  * Return true if tlb need be flushed.
1166  */
spte_write_protect(u64 * sptep,bool pt_protect)1167 static bool spte_write_protect(u64 *sptep, bool pt_protect)
1168 {
1169 	u64 spte = *sptep;
1170 
1171 	if (!is_writable_pte(spte) &&
1172 	    !(pt_protect && is_mmu_writable_spte(spte)))
1173 		return false;
1174 
1175 	rmap_printk("spte %p %llx\n", sptep, *sptep);
1176 
1177 	if (pt_protect)
1178 		spte &= ~shadow_mmu_writable_mask;
1179 	spte = spte & ~PT_WRITABLE_MASK;
1180 
1181 	return mmu_spte_update(sptep, spte);
1182 }
1183 
rmap_write_protect(struct kvm_rmap_head * rmap_head,bool pt_protect)1184 static bool rmap_write_protect(struct kvm_rmap_head *rmap_head,
1185 			       bool pt_protect)
1186 {
1187 	u64 *sptep;
1188 	struct rmap_iterator iter;
1189 	bool flush = false;
1190 
1191 	for_each_rmap_spte(rmap_head, &iter, sptep)
1192 		flush |= spte_write_protect(sptep, pt_protect);
1193 
1194 	return flush;
1195 }
1196 
spte_clear_dirty(u64 * sptep)1197 static bool spte_clear_dirty(u64 *sptep)
1198 {
1199 	u64 spte = *sptep;
1200 
1201 	rmap_printk("spte %p %llx\n", sptep, *sptep);
1202 
1203 	MMU_WARN_ON(!spte_ad_enabled(spte));
1204 	spte &= ~shadow_dirty_mask;
1205 	return mmu_spte_update(sptep, spte);
1206 }
1207 
spte_wrprot_for_clear_dirty(u64 * sptep)1208 static bool spte_wrprot_for_clear_dirty(u64 *sptep)
1209 {
1210 	bool was_writable = test_and_clear_bit(PT_WRITABLE_SHIFT,
1211 					       (unsigned long *)sptep);
1212 	if (was_writable && !spte_ad_enabled(*sptep))
1213 		kvm_set_pfn_dirty(spte_to_pfn(*sptep));
1214 
1215 	return was_writable;
1216 }
1217 
1218 /*
1219  * Gets the GFN ready for another round of dirty logging by clearing the
1220  *	- D bit on ad-enabled SPTEs, and
1221  *	- W bit on ad-disabled SPTEs.
1222  * Returns true iff any D or W bits were cleared.
1223  */
__rmap_clear_dirty(struct kvm * kvm,struct kvm_rmap_head * rmap_head,const struct kvm_memory_slot * slot)1224 static bool __rmap_clear_dirty(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1225 			       const struct kvm_memory_slot *slot)
1226 {
1227 	u64 *sptep;
1228 	struct rmap_iterator iter;
1229 	bool flush = false;
1230 
1231 	for_each_rmap_spte(rmap_head, &iter, sptep)
1232 		if (spte_ad_need_write_protect(*sptep))
1233 			flush |= spte_wrprot_for_clear_dirty(sptep);
1234 		else
1235 			flush |= spte_clear_dirty(sptep);
1236 
1237 	return flush;
1238 }
1239 
1240 /**
1241  * kvm_mmu_write_protect_pt_masked - write protect selected PT level pages
1242  * @kvm: kvm instance
1243  * @slot: slot to protect
1244  * @gfn_offset: start of the BITS_PER_LONG pages we care about
1245  * @mask: indicates which pages we should protect
1246  *
1247  * Used when we do not need to care about huge page mappings.
1248  */
kvm_mmu_write_protect_pt_masked(struct kvm * kvm,struct kvm_memory_slot * slot,gfn_t gfn_offset,unsigned long mask)1249 static void kvm_mmu_write_protect_pt_masked(struct kvm *kvm,
1250 				     struct kvm_memory_slot *slot,
1251 				     gfn_t gfn_offset, unsigned long mask)
1252 {
1253 	struct kvm_rmap_head *rmap_head;
1254 
1255 	if (is_tdp_mmu_enabled(kvm))
1256 		kvm_tdp_mmu_clear_dirty_pt_masked(kvm, slot,
1257 				slot->base_gfn + gfn_offset, mask, true);
1258 
1259 	if (!kvm_memslots_have_rmaps(kvm))
1260 		return;
1261 
1262 	while (mask) {
1263 		rmap_head = gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask),
1264 					PG_LEVEL_4K, slot);
1265 		rmap_write_protect(rmap_head, false);
1266 
1267 		/* clear the first set bit */
1268 		mask &= mask - 1;
1269 	}
1270 }
1271 
1272 /**
1273  * kvm_mmu_clear_dirty_pt_masked - clear MMU D-bit for PT level pages, or write
1274  * protect the page if the D-bit isn't supported.
1275  * @kvm: kvm instance
1276  * @slot: slot to clear D-bit
1277  * @gfn_offset: start of the BITS_PER_LONG pages we care about
1278  * @mask: indicates which pages we should clear D-bit
1279  *
1280  * Used for PML to re-log the dirty GPAs after userspace querying dirty_bitmap.
1281  */
kvm_mmu_clear_dirty_pt_masked(struct kvm * kvm,struct kvm_memory_slot * slot,gfn_t gfn_offset,unsigned long mask)1282 static void kvm_mmu_clear_dirty_pt_masked(struct kvm *kvm,
1283 					 struct kvm_memory_slot *slot,
1284 					 gfn_t gfn_offset, unsigned long mask)
1285 {
1286 	struct kvm_rmap_head *rmap_head;
1287 
1288 	if (is_tdp_mmu_enabled(kvm))
1289 		kvm_tdp_mmu_clear_dirty_pt_masked(kvm, slot,
1290 				slot->base_gfn + gfn_offset, mask, false);
1291 
1292 	if (!kvm_memslots_have_rmaps(kvm))
1293 		return;
1294 
1295 	while (mask) {
1296 		rmap_head = gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask),
1297 					PG_LEVEL_4K, slot);
1298 		__rmap_clear_dirty(kvm, rmap_head, slot);
1299 
1300 		/* clear the first set bit */
1301 		mask &= mask - 1;
1302 	}
1303 }
1304 
1305 /**
1306  * kvm_arch_mmu_enable_log_dirty_pt_masked - enable dirty logging for selected
1307  * PT level pages.
1308  *
1309  * It calls kvm_mmu_write_protect_pt_masked to write protect selected pages to
1310  * enable dirty logging for them.
1311  *
1312  * We need to care about huge page mappings: e.g. during dirty logging we may
1313  * have such mappings.
1314  */
kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm * kvm,struct kvm_memory_slot * slot,gfn_t gfn_offset,unsigned long mask)1315 void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
1316 				struct kvm_memory_slot *slot,
1317 				gfn_t gfn_offset, unsigned long mask)
1318 {
1319 	/*
1320 	 * Huge pages are NOT write protected when we start dirty logging in
1321 	 * initially-all-set mode; must write protect them here so that they
1322 	 * are split to 4K on the first write.
1323 	 *
1324 	 * The gfn_offset is guaranteed to be aligned to 64, but the base_gfn
1325 	 * of memslot has no such restriction, so the range can cross two large
1326 	 * pages.
1327 	 */
1328 	if (kvm_dirty_log_manual_protect_and_init_set(kvm)) {
1329 		gfn_t start = slot->base_gfn + gfn_offset + __ffs(mask);
1330 		gfn_t end = slot->base_gfn + gfn_offset + __fls(mask);
1331 
1332 		if (READ_ONCE(eager_page_split))
1333 			kvm_mmu_try_split_huge_pages(kvm, slot, start, end, PG_LEVEL_4K);
1334 
1335 		kvm_mmu_slot_gfn_write_protect(kvm, slot, start, PG_LEVEL_2M);
1336 
1337 		/* Cross two large pages? */
1338 		if (ALIGN(start << PAGE_SHIFT, PMD_SIZE) !=
1339 		    ALIGN(end << PAGE_SHIFT, PMD_SIZE))
1340 			kvm_mmu_slot_gfn_write_protect(kvm, slot, end,
1341 						       PG_LEVEL_2M);
1342 	}
1343 
1344 	/* Now handle 4K PTEs.  */
1345 	if (kvm_x86_ops.cpu_dirty_log_size)
1346 		kvm_mmu_clear_dirty_pt_masked(kvm, slot, gfn_offset, mask);
1347 	else
1348 		kvm_mmu_write_protect_pt_masked(kvm, slot, gfn_offset, mask);
1349 }
1350 
kvm_cpu_dirty_log_size(void)1351 int kvm_cpu_dirty_log_size(void)
1352 {
1353 	return kvm_x86_ops.cpu_dirty_log_size;
1354 }
1355 
kvm_mmu_slot_gfn_write_protect(struct kvm * kvm,struct kvm_memory_slot * slot,u64 gfn,int min_level)1356 bool kvm_mmu_slot_gfn_write_protect(struct kvm *kvm,
1357 				    struct kvm_memory_slot *slot, u64 gfn,
1358 				    int min_level)
1359 {
1360 	struct kvm_rmap_head *rmap_head;
1361 	int i;
1362 	bool write_protected = false;
1363 
1364 	if (kvm_memslots_have_rmaps(kvm)) {
1365 		for (i = min_level; i <= KVM_MAX_HUGEPAGE_LEVEL; ++i) {
1366 			rmap_head = gfn_to_rmap(gfn, i, slot);
1367 			write_protected |= rmap_write_protect(rmap_head, true);
1368 		}
1369 	}
1370 
1371 	if (is_tdp_mmu_enabled(kvm))
1372 		write_protected |=
1373 			kvm_tdp_mmu_write_protect_gfn(kvm, slot, gfn, min_level);
1374 
1375 	return write_protected;
1376 }
1377 
kvm_vcpu_write_protect_gfn(struct kvm_vcpu * vcpu,u64 gfn)1378 static bool kvm_vcpu_write_protect_gfn(struct kvm_vcpu *vcpu, u64 gfn)
1379 {
1380 	struct kvm_memory_slot *slot;
1381 
1382 	slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1383 	return kvm_mmu_slot_gfn_write_protect(vcpu->kvm, slot, gfn, PG_LEVEL_4K);
1384 }
1385 
kvm_zap_rmapp(struct kvm * kvm,struct kvm_rmap_head * rmap_head,const struct kvm_memory_slot * slot)1386 static bool kvm_zap_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1387 			  const struct kvm_memory_slot *slot)
1388 {
1389 	return pte_list_destroy(kvm, rmap_head);
1390 }
1391 
kvm_unmap_rmapp(struct kvm * kvm,struct kvm_rmap_head * rmap_head,struct kvm_memory_slot * slot,gfn_t gfn,int level,pte_t unused)1392 static bool kvm_unmap_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1393 			    struct kvm_memory_slot *slot, gfn_t gfn, int level,
1394 			    pte_t unused)
1395 {
1396 	return kvm_zap_rmapp(kvm, rmap_head, slot);
1397 }
1398 
kvm_set_pte_rmapp(struct kvm * kvm,struct kvm_rmap_head * rmap_head,struct kvm_memory_slot * slot,gfn_t gfn,int level,pte_t pte)1399 static bool kvm_set_pte_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1400 			      struct kvm_memory_slot *slot, gfn_t gfn, int level,
1401 			      pte_t pte)
1402 {
1403 	u64 *sptep;
1404 	struct rmap_iterator iter;
1405 	bool need_flush = false;
1406 	u64 new_spte;
1407 	kvm_pfn_t new_pfn;
1408 
1409 	WARN_ON(pte_huge(pte));
1410 	new_pfn = pte_pfn(pte);
1411 
1412 restart:
1413 	for_each_rmap_spte(rmap_head, &iter, sptep) {
1414 		rmap_printk("spte %p %llx gfn %llx (%d)\n",
1415 			    sptep, *sptep, gfn, level);
1416 
1417 		need_flush = true;
1418 
1419 		if (pte_write(pte)) {
1420 			pte_list_remove(kvm, rmap_head, sptep);
1421 			goto restart;
1422 		} else {
1423 			new_spte = kvm_mmu_changed_pte_notifier_make_spte(
1424 					*sptep, new_pfn);
1425 
1426 			mmu_spte_clear_track_bits(kvm, sptep);
1427 			mmu_spte_set(sptep, new_spte);
1428 		}
1429 	}
1430 
1431 	if (need_flush && kvm_available_flush_tlb_with_range()) {
1432 		kvm_flush_remote_tlbs_with_address(kvm, gfn, 1);
1433 		return false;
1434 	}
1435 
1436 	return need_flush;
1437 }
1438 
1439 struct slot_rmap_walk_iterator {
1440 	/* input fields. */
1441 	const struct kvm_memory_slot *slot;
1442 	gfn_t start_gfn;
1443 	gfn_t end_gfn;
1444 	int start_level;
1445 	int end_level;
1446 
1447 	/* output fields. */
1448 	gfn_t gfn;
1449 	struct kvm_rmap_head *rmap;
1450 	int level;
1451 
1452 	/* private field. */
1453 	struct kvm_rmap_head *end_rmap;
1454 };
1455 
1456 static void
rmap_walk_init_level(struct slot_rmap_walk_iterator * iterator,int level)1457 rmap_walk_init_level(struct slot_rmap_walk_iterator *iterator, int level)
1458 {
1459 	iterator->level = level;
1460 	iterator->gfn = iterator->start_gfn;
1461 	iterator->rmap = gfn_to_rmap(iterator->gfn, level, iterator->slot);
1462 	iterator->end_rmap = gfn_to_rmap(iterator->end_gfn, level, iterator->slot);
1463 }
1464 
1465 static void
slot_rmap_walk_init(struct slot_rmap_walk_iterator * iterator,const struct kvm_memory_slot * slot,int start_level,int end_level,gfn_t start_gfn,gfn_t end_gfn)1466 slot_rmap_walk_init(struct slot_rmap_walk_iterator *iterator,
1467 		    const struct kvm_memory_slot *slot, int start_level,
1468 		    int end_level, gfn_t start_gfn, gfn_t end_gfn)
1469 {
1470 	iterator->slot = slot;
1471 	iterator->start_level = start_level;
1472 	iterator->end_level = end_level;
1473 	iterator->start_gfn = start_gfn;
1474 	iterator->end_gfn = end_gfn;
1475 
1476 	rmap_walk_init_level(iterator, iterator->start_level);
1477 }
1478 
slot_rmap_walk_okay(struct slot_rmap_walk_iterator * iterator)1479 static bool slot_rmap_walk_okay(struct slot_rmap_walk_iterator *iterator)
1480 {
1481 	return !!iterator->rmap;
1482 }
1483 
slot_rmap_walk_next(struct slot_rmap_walk_iterator * iterator)1484 static void slot_rmap_walk_next(struct slot_rmap_walk_iterator *iterator)
1485 {
1486 	while (++iterator->rmap <= iterator->end_rmap) {
1487 		iterator->gfn += (1UL << KVM_HPAGE_GFN_SHIFT(iterator->level));
1488 
1489 		if (iterator->rmap->val)
1490 			return;
1491 	}
1492 
1493 	if (++iterator->level > iterator->end_level) {
1494 		iterator->rmap = NULL;
1495 		return;
1496 	}
1497 
1498 	rmap_walk_init_level(iterator, iterator->level);
1499 }
1500 
1501 #define for_each_slot_rmap_range(_slot_, _start_level_, _end_level_,	\
1502 	   _start_gfn, _end_gfn, _iter_)				\
1503 	for (slot_rmap_walk_init(_iter_, _slot_, _start_level_,		\
1504 				 _end_level_, _start_gfn, _end_gfn);	\
1505 	     slot_rmap_walk_okay(_iter_);				\
1506 	     slot_rmap_walk_next(_iter_))
1507 
1508 typedef bool (*rmap_handler_t)(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1509 			       struct kvm_memory_slot *slot, gfn_t gfn,
1510 			       int level, pte_t pte);
1511 
kvm_handle_gfn_range(struct kvm * kvm,struct kvm_gfn_range * range,rmap_handler_t handler)1512 static __always_inline bool kvm_handle_gfn_range(struct kvm *kvm,
1513 						 struct kvm_gfn_range *range,
1514 						 rmap_handler_t handler)
1515 {
1516 	struct slot_rmap_walk_iterator iterator;
1517 	bool ret = false;
1518 
1519 	for_each_slot_rmap_range(range->slot, PG_LEVEL_4K, KVM_MAX_HUGEPAGE_LEVEL,
1520 				 range->start, range->end - 1, &iterator)
1521 		ret |= handler(kvm, iterator.rmap, range->slot, iterator.gfn,
1522 			       iterator.level, range->pte);
1523 
1524 	return ret;
1525 }
1526 
kvm_unmap_gfn_range(struct kvm * kvm,struct kvm_gfn_range * range)1527 bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range)
1528 {
1529 	bool flush = false;
1530 
1531 	if (kvm_memslots_have_rmaps(kvm))
1532 		flush = kvm_handle_gfn_range(kvm, range, kvm_unmap_rmapp);
1533 
1534 	if (is_tdp_mmu_enabled(kvm))
1535 		flush = kvm_tdp_mmu_unmap_gfn_range(kvm, range, flush);
1536 
1537 	return flush;
1538 }
1539 
kvm_set_spte_gfn(struct kvm * kvm,struct kvm_gfn_range * range)1540 bool kvm_set_spte_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
1541 {
1542 	bool flush = false;
1543 
1544 	if (kvm_memslots_have_rmaps(kvm))
1545 		flush = kvm_handle_gfn_range(kvm, range, kvm_set_pte_rmapp);
1546 
1547 	if (is_tdp_mmu_enabled(kvm))
1548 		flush |= kvm_tdp_mmu_set_spte_gfn(kvm, range);
1549 
1550 	return flush;
1551 }
1552 
kvm_age_rmapp(struct kvm * kvm,struct kvm_rmap_head * rmap_head,struct kvm_memory_slot * slot,gfn_t gfn,int level,pte_t unused)1553 static bool kvm_age_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1554 			  struct kvm_memory_slot *slot, gfn_t gfn, int level,
1555 			  pte_t unused)
1556 {
1557 	u64 *sptep;
1558 	struct rmap_iterator iter;
1559 	int young = 0;
1560 
1561 	for_each_rmap_spte(rmap_head, &iter, sptep)
1562 		young |= mmu_spte_age(sptep);
1563 
1564 	return young;
1565 }
1566 
kvm_test_age_rmapp(struct kvm * kvm,struct kvm_rmap_head * rmap_head,struct kvm_memory_slot * slot,gfn_t gfn,int level,pte_t unused)1567 static bool kvm_test_age_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1568 			       struct kvm_memory_slot *slot, gfn_t gfn,
1569 			       int level, pte_t unused)
1570 {
1571 	u64 *sptep;
1572 	struct rmap_iterator iter;
1573 
1574 	for_each_rmap_spte(rmap_head, &iter, sptep)
1575 		if (is_accessed_spte(*sptep))
1576 			return true;
1577 	return false;
1578 }
1579 
1580 #define RMAP_RECYCLE_THRESHOLD 1000
1581 
rmap_add(struct kvm_vcpu * vcpu,struct kvm_memory_slot * slot,u64 * spte,gfn_t gfn)1582 static void rmap_add(struct kvm_vcpu *vcpu, struct kvm_memory_slot *slot,
1583 		     u64 *spte, gfn_t gfn)
1584 {
1585 	struct kvm_mmu_page *sp;
1586 	struct kvm_rmap_head *rmap_head;
1587 	int rmap_count;
1588 
1589 	sp = sptep_to_sp(spte);
1590 	kvm_mmu_page_set_gfn(sp, spte - sp->spt, gfn);
1591 	rmap_head = gfn_to_rmap(gfn, sp->role.level, slot);
1592 	rmap_count = pte_list_add(vcpu, spte, rmap_head);
1593 
1594 	if (rmap_count > RMAP_RECYCLE_THRESHOLD) {
1595 		kvm_unmap_rmapp(vcpu->kvm, rmap_head, NULL, gfn, sp->role.level, __pte(0));
1596 		kvm_flush_remote_tlbs_with_address(
1597 				vcpu->kvm, sp->gfn, KVM_PAGES_PER_HPAGE(sp->role.level));
1598 	}
1599 }
1600 
kvm_age_gfn(struct kvm * kvm,struct kvm_gfn_range * range)1601 bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
1602 {
1603 	bool young = false;
1604 
1605 	if (kvm_memslots_have_rmaps(kvm))
1606 		young = kvm_handle_gfn_range(kvm, range, kvm_age_rmapp);
1607 
1608 	if (is_tdp_mmu_enabled(kvm))
1609 		young |= kvm_tdp_mmu_age_gfn_range(kvm, range);
1610 
1611 	return young;
1612 }
1613 
kvm_test_age_gfn(struct kvm * kvm,struct kvm_gfn_range * range)1614 bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
1615 {
1616 	bool young = false;
1617 
1618 	if (kvm_memslots_have_rmaps(kvm))
1619 		young = kvm_handle_gfn_range(kvm, range, kvm_test_age_rmapp);
1620 
1621 	if (is_tdp_mmu_enabled(kvm))
1622 		young |= kvm_tdp_mmu_test_age_gfn(kvm, range);
1623 
1624 	return young;
1625 }
1626 
1627 #ifdef MMU_DEBUG
is_empty_shadow_page(u64 * spt)1628 static int is_empty_shadow_page(u64 *spt)
1629 {
1630 	u64 *pos;
1631 	u64 *end;
1632 
1633 	for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++)
1634 		if (is_shadow_present_pte(*pos)) {
1635 			printk(KERN_ERR "%s: %p %llx\n", __func__,
1636 			       pos, *pos);
1637 			return 0;
1638 		}
1639 	return 1;
1640 }
1641 #endif
1642 
1643 /*
1644  * This value is the sum of all of the kvm instances's
1645  * kvm->arch.n_used_mmu_pages values.  We need a global,
1646  * aggregate version in order to make the slab shrinker
1647  * faster
1648  */
kvm_mod_used_mmu_pages(struct kvm * kvm,long nr)1649 static inline void kvm_mod_used_mmu_pages(struct kvm *kvm, long nr)
1650 {
1651 	kvm->arch.n_used_mmu_pages += nr;
1652 	percpu_counter_add(&kvm_total_used_mmu_pages, nr);
1653 }
1654 
kvm_mmu_free_page(struct kvm_mmu_page * sp)1655 static void kvm_mmu_free_page(struct kvm_mmu_page *sp)
1656 {
1657 	MMU_WARN_ON(!is_empty_shadow_page(sp->spt));
1658 	hlist_del(&sp->hash_link);
1659 	list_del(&sp->link);
1660 	free_page((unsigned long)sp->spt);
1661 	if (!sp->role.direct)
1662 		free_page((unsigned long)sp->gfns);
1663 	kmem_cache_free(mmu_page_header_cache, sp);
1664 }
1665 
kvm_page_table_hashfn(gfn_t gfn)1666 static unsigned kvm_page_table_hashfn(gfn_t gfn)
1667 {
1668 	return hash_64(gfn, KVM_MMU_HASH_SHIFT);
1669 }
1670 
mmu_page_add_parent_pte(struct kvm_vcpu * vcpu,struct kvm_mmu_page * sp,u64 * parent_pte)1671 static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu,
1672 				    struct kvm_mmu_page *sp, u64 *parent_pte)
1673 {
1674 	if (!parent_pte)
1675 		return;
1676 
1677 	pte_list_add(vcpu, parent_pte, &sp->parent_ptes);
1678 }
1679 
mmu_page_remove_parent_pte(struct kvm_mmu_page * sp,u64 * parent_pte)1680 static void mmu_page_remove_parent_pte(struct kvm_mmu_page *sp,
1681 				       u64 *parent_pte)
1682 {
1683 	__pte_list_remove(parent_pte, &sp->parent_ptes);
1684 }
1685 
drop_parent_pte(struct kvm_mmu_page * sp,u64 * parent_pte)1686 static void drop_parent_pte(struct kvm_mmu_page *sp,
1687 			    u64 *parent_pte)
1688 {
1689 	mmu_page_remove_parent_pte(sp, parent_pte);
1690 	mmu_spte_clear_no_track(parent_pte);
1691 }
1692 
kvm_mmu_alloc_page(struct kvm_vcpu * vcpu,int direct)1693 static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu, int direct)
1694 {
1695 	struct kvm_mmu_page *sp;
1696 
1697 	sp = kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache);
1698 	sp->spt = kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_shadow_page_cache);
1699 	if (!direct)
1700 		sp->gfns = kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_gfn_array_cache);
1701 	set_page_private(virt_to_page(sp->spt), (unsigned long)sp);
1702 
1703 	/*
1704 	 * active_mmu_pages must be a FIFO list, as kvm_zap_obsolete_pages()
1705 	 * depends on valid pages being added to the head of the list.  See
1706 	 * comments in kvm_zap_obsolete_pages().
1707 	 */
1708 	sp->mmu_valid_gen = vcpu->kvm->arch.mmu_valid_gen;
1709 	list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages);
1710 	kvm_mod_used_mmu_pages(vcpu->kvm, +1);
1711 	return sp;
1712 }
1713 
1714 static void mark_unsync(u64 *spte);
kvm_mmu_mark_parents_unsync(struct kvm_mmu_page * sp)1715 static void kvm_mmu_mark_parents_unsync(struct kvm_mmu_page *sp)
1716 {
1717 	u64 *sptep;
1718 	struct rmap_iterator iter;
1719 
1720 	for_each_rmap_spte(&sp->parent_ptes, &iter, sptep) {
1721 		mark_unsync(sptep);
1722 	}
1723 }
1724 
mark_unsync(u64 * spte)1725 static void mark_unsync(u64 *spte)
1726 {
1727 	struct kvm_mmu_page *sp;
1728 	unsigned int index;
1729 
1730 	sp = sptep_to_sp(spte);
1731 	index = spte - sp->spt;
1732 	if (__test_and_set_bit(index, sp->unsync_child_bitmap))
1733 		return;
1734 	if (sp->unsync_children++)
1735 		return;
1736 	kvm_mmu_mark_parents_unsync(sp);
1737 }
1738 
nonpaging_sync_page(struct kvm_vcpu * vcpu,struct kvm_mmu_page * sp)1739 static int nonpaging_sync_page(struct kvm_vcpu *vcpu,
1740 			       struct kvm_mmu_page *sp)
1741 {
1742 	return -1;
1743 }
1744 
1745 #define KVM_PAGE_ARRAY_NR 16
1746 
1747 struct kvm_mmu_pages {
1748 	struct mmu_page_and_offset {
1749 		struct kvm_mmu_page *sp;
1750 		unsigned int idx;
1751 	} page[KVM_PAGE_ARRAY_NR];
1752 	unsigned int nr;
1753 };
1754 
mmu_pages_add(struct kvm_mmu_pages * pvec,struct kvm_mmu_page * sp,int idx)1755 static int mmu_pages_add(struct kvm_mmu_pages *pvec, struct kvm_mmu_page *sp,
1756 			 int idx)
1757 {
1758 	int i;
1759 
1760 	if (sp->unsync)
1761 		for (i=0; i < pvec->nr; i++)
1762 			if (pvec->page[i].sp == sp)
1763 				return 0;
1764 
1765 	pvec->page[pvec->nr].sp = sp;
1766 	pvec->page[pvec->nr].idx = idx;
1767 	pvec->nr++;
1768 	return (pvec->nr == KVM_PAGE_ARRAY_NR);
1769 }
1770 
clear_unsync_child_bit(struct kvm_mmu_page * sp,int idx)1771 static inline void clear_unsync_child_bit(struct kvm_mmu_page *sp, int idx)
1772 {
1773 	--sp->unsync_children;
1774 	WARN_ON((int)sp->unsync_children < 0);
1775 	__clear_bit(idx, sp->unsync_child_bitmap);
1776 }
1777 
__mmu_unsync_walk(struct kvm_mmu_page * sp,struct kvm_mmu_pages * pvec)1778 static int __mmu_unsync_walk(struct kvm_mmu_page *sp,
1779 			   struct kvm_mmu_pages *pvec)
1780 {
1781 	int i, ret, nr_unsync_leaf = 0;
1782 
1783 	for_each_set_bit(i, sp->unsync_child_bitmap, 512) {
1784 		struct kvm_mmu_page *child;
1785 		u64 ent = sp->spt[i];
1786 
1787 		if (!is_shadow_present_pte(ent) || is_large_pte(ent)) {
1788 			clear_unsync_child_bit(sp, i);
1789 			continue;
1790 		}
1791 
1792 		child = to_shadow_page(ent & PT64_BASE_ADDR_MASK);
1793 
1794 		if (child->unsync_children) {
1795 			if (mmu_pages_add(pvec, child, i))
1796 				return -ENOSPC;
1797 
1798 			ret = __mmu_unsync_walk(child, pvec);
1799 			if (!ret) {
1800 				clear_unsync_child_bit(sp, i);
1801 				continue;
1802 			} else if (ret > 0) {
1803 				nr_unsync_leaf += ret;
1804 			} else
1805 				return ret;
1806 		} else if (child->unsync) {
1807 			nr_unsync_leaf++;
1808 			if (mmu_pages_add(pvec, child, i))
1809 				return -ENOSPC;
1810 		} else
1811 			clear_unsync_child_bit(sp, i);
1812 	}
1813 
1814 	return nr_unsync_leaf;
1815 }
1816 
1817 #define INVALID_INDEX (-1)
1818 
mmu_unsync_walk(struct kvm_mmu_page * sp,struct kvm_mmu_pages * pvec)1819 static int mmu_unsync_walk(struct kvm_mmu_page *sp,
1820 			   struct kvm_mmu_pages *pvec)
1821 {
1822 	pvec->nr = 0;
1823 	if (!sp->unsync_children)
1824 		return 0;
1825 
1826 	mmu_pages_add(pvec, sp, INVALID_INDEX);
1827 	return __mmu_unsync_walk(sp, pvec);
1828 }
1829 
kvm_unlink_unsync_page(struct kvm * kvm,struct kvm_mmu_page * sp)1830 static void kvm_unlink_unsync_page(struct kvm *kvm, struct kvm_mmu_page *sp)
1831 {
1832 	WARN_ON(!sp->unsync);
1833 	trace_kvm_mmu_sync_page(sp);
1834 	sp->unsync = 0;
1835 	--kvm->stat.mmu_unsync;
1836 }
1837 
1838 static bool kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
1839 				     struct list_head *invalid_list);
1840 static void kvm_mmu_commit_zap_page(struct kvm *kvm,
1841 				    struct list_head *invalid_list);
1842 
sp_has_gptes(struct kvm_mmu_page * sp)1843 static bool sp_has_gptes(struct kvm_mmu_page *sp)
1844 {
1845 	if (sp->role.direct)
1846 		return false;
1847 
1848 	if (sp->role.passthrough)
1849 		return false;
1850 
1851 	return true;
1852 }
1853 
1854 #define for_each_valid_sp(_kvm, _sp, _list)				\
1855 	hlist_for_each_entry(_sp, _list, hash_link)			\
1856 		if (is_obsolete_sp((_kvm), (_sp))) {			\
1857 		} else
1858 
1859 #define for_each_gfn_valid_sp_with_gptes(_kvm, _sp, _gfn)		\
1860 	for_each_valid_sp(_kvm, _sp,					\
1861 	  &(_kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(_gfn)])	\
1862 		if ((_sp)->gfn != (_gfn) || !sp_has_gptes(_sp)) {} else
1863 
kvm_sync_page(struct kvm_vcpu * vcpu,struct kvm_mmu_page * sp,struct list_head * invalid_list)1864 static int kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
1865 			 struct list_head *invalid_list)
1866 {
1867 	int ret = vcpu->arch.mmu->sync_page(vcpu, sp);
1868 
1869 	if (ret < 0)
1870 		kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);
1871 	return ret;
1872 }
1873 
kvm_mmu_remote_flush_or_zap(struct kvm * kvm,struct list_head * invalid_list,bool remote_flush)1874 static bool kvm_mmu_remote_flush_or_zap(struct kvm *kvm,
1875 					struct list_head *invalid_list,
1876 					bool remote_flush)
1877 {
1878 	if (!remote_flush && list_empty(invalid_list))
1879 		return false;
1880 
1881 	if (!list_empty(invalid_list))
1882 		kvm_mmu_commit_zap_page(kvm, invalid_list);
1883 	else
1884 		kvm_flush_remote_tlbs(kvm);
1885 	return true;
1886 }
1887 
is_obsolete_sp(struct kvm * kvm,struct kvm_mmu_page * sp)1888 static bool is_obsolete_sp(struct kvm *kvm, struct kvm_mmu_page *sp)
1889 {
1890 	if (sp->role.invalid)
1891 		return true;
1892 
1893 	/* TDP MMU pages due not use the MMU generation. */
1894 	return !sp->tdp_mmu_page &&
1895 	       unlikely(sp->mmu_valid_gen != kvm->arch.mmu_valid_gen);
1896 }
1897 
1898 struct mmu_page_path {
1899 	struct kvm_mmu_page *parent[PT64_ROOT_MAX_LEVEL];
1900 	unsigned int idx[PT64_ROOT_MAX_LEVEL];
1901 };
1902 
1903 #define for_each_sp(pvec, sp, parents, i)			\
1904 		for (i = mmu_pages_first(&pvec, &parents);	\
1905 			i < pvec.nr && ({ sp = pvec.page[i].sp; 1;});	\
1906 			i = mmu_pages_next(&pvec, &parents, i))
1907 
mmu_pages_next(struct kvm_mmu_pages * pvec,struct mmu_page_path * parents,int i)1908 static int mmu_pages_next(struct kvm_mmu_pages *pvec,
1909 			  struct mmu_page_path *parents,
1910 			  int i)
1911 {
1912 	int n;
1913 
1914 	for (n = i+1; n < pvec->nr; n++) {
1915 		struct kvm_mmu_page *sp = pvec->page[n].sp;
1916 		unsigned idx = pvec->page[n].idx;
1917 		int level = sp->role.level;
1918 
1919 		parents->idx[level-1] = idx;
1920 		if (level == PG_LEVEL_4K)
1921 			break;
1922 
1923 		parents->parent[level-2] = sp;
1924 	}
1925 
1926 	return n;
1927 }
1928 
mmu_pages_first(struct kvm_mmu_pages * pvec,struct mmu_page_path * parents)1929 static int mmu_pages_first(struct kvm_mmu_pages *pvec,
1930 			   struct mmu_page_path *parents)
1931 {
1932 	struct kvm_mmu_page *sp;
1933 	int level;
1934 
1935 	if (pvec->nr == 0)
1936 		return 0;
1937 
1938 	WARN_ON(pvec->page[0].idx != INVALID_INDEX);
1939 
1940 	sp = pvec->page[0].sp;
1941 	level = sp->role.level;
1942 	WARN_ON(level == PG_LEVEL_4K);
1943 
1944 	parents->parent[level-2] = sp;
1945 
1946 	/* Also set up a sentinel.  Further entries in pvec are all
1947 	 * children of sp, so this element is never overwritten.
1948 	 */
1949 	parents->parent[level-1] = NULL;
1950 	return mmu_pages_next(pvec, parents, 0);
1951 }
1952 
mmu_pages_clear_parents(struct mmu_page_path * parents)1953 static void mmu_pages_clear_parents(struct mmu_page_path *parents)
1954 {
1955 	struct kvm_mmu_page *sp;
1956 	unsigned int level = 0;
1957 
1958 	do {
1959 		unsigned int idx = parents->idx[level];
1960 		sp = parents->parent[level];
1961 		if (!sp)
1962 			return;
1963 
1964 		WARN_ON(idx == INVALID_INDEX);
1965 		clear_unsync_child_bit(sp, idx);
1966 		level++;
1967 	} while (!sp->unsync_children);
1968 }
1969 
mmu_sync_children(struct kvm_vcpu * vcpu,struct kvm_mmu_page * parent,bool can_yield)1970 static int mmu_sync_children(struct kvm_vcpu *vcpu,
1971 			     struct kvm_mmu_page *parent, bool can_yield)
1972 {
1973 	int i;
1974 	struct kvm_mmu_page *sp;
1975 	struct mmu_page_path parents;
1976 	struct kvm_mmu_pages pages;
1977 	LIST_HEAD(invalid_list);
1978 	bool flush = false;
1979 
1980 	while (mmu_unsync_walk(parent, &pages)) {
1981 		bool protected = false;
1982 
1983 		for_each_sp(pages, sp, parents, i)
1984 			protected |= kvm_vcpu_write_protect_gfn(vcpu, sp->gfn);
1985 
1986 		if (protected) {
1987 			kvm_mmu_remote_flush_or_zap(vcpu->kvm, &invalid_list, true);
1988 			flush = false;
1989 		}
1990 
1991 		for_each_sp(pages, sp, parents, i) {
1992 			kvm_unlink_unsync_page(vcpu->kvm, sp);
1993 			flush |= kvm_sync_page(vcpu, sp, &invalid_list) > 0;
1994 			mmu_pages_clear_parents(&parents);
1995 		}
1996 		if (need_resched() || rwlock_needbreak(&vcpu->kvm->mmu_lock)) {
1997 			kvm_mmu_remote_flush_or_zap(vcpu->kvm, &invalid_list, flush);
1998 			if (!can_yield) {
1999 				kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);
2000 				return -EINTR;
2001 			}
2002 
2003 			cond_resched_rwlock_write(&vcpu->kvm->mmu_lock);
2004 			flush = false;
2005 		}
2006 	}
2007 
2008 	kvm_mmu_remote_flush_or_zap(vcpu->kvm, &invalid_list, flush);
2009 	return 0;
2010 }
2011 
__clear_sp_write_flooding_count(struct kvm_mmu_page * sp)2012 static void __clear_sp_write_flooding_count(struct kvm_mmu_page *sp)
2013 {
2014 	atomic_set(&sp->write_flooding_count,  0);
2015 }
2016 
clear_sp_write_flooding_count(u64 * spte)2017 static void clear_sp_write_flooding_count(u64 *spte)
2018 {
2019 	__clear_sp_write_flooding_count(sptep_to_sp(spte));
2020 }
2021 
kvm_mmu_get_page(struct kvm_vcpu * vcpu,gfn_t gfn,gva_t gaddr,unsigned level,int direct,unsigned int access)2022 static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
2023 					     gfn_t gfn,
2024 					     gva_t gaddr,
2025 					     unsigned level,
2026 					     int direct,
2027 					     unsigned int access)
2028 {
2029 	bool direct_mmu = vcpu->arch.mmu->root_role.direct;
2030 	union kvm_mmu_page_role role;
2031 	struct hlist_head *sp_list;
2032 	unsigned quadrant;
2033 	struct kvm_mmu_page *sp;
2034 	int ret;
2035 	int collisions = 0;
2036 	LIST_HEAD(invalid_list);
2037 
2038 	role = vcpu->arch.mmu->root_role;
2039 	role.level = level;
2040 	role.direct = direct;
2041 	role.access = access;
2042 	if (role.has_4_byte_gpte) {
2043 		quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level));
2044 		quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
2045 		role.quadrant = quadrant;
2046 	}
2047 	if (level <= vcpu->arch.mmu->cpu_role.base.level)
2048 		role.passthrough = 0;
2049 
2050 	sp_list = &vcpu->kvm->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)];
2051 	for_each_valid_sp(vcpu->kvm, sp, sp_list) {
2052 		if (sp->gfn != gfn) {
2053 			collisions++;
2054 			continue;
2055 		}
2056 
2057 		if (sp->role.word != role.word) {
2058 			/*
2059 			 * If the guest is creating an upper-level page, zap
2060 			 * unsync pages for the same gfn.  While it's possible
2061 			 * the guest is using recursive page tables, in all
2062 			 * likelihood the guest has stopped using the unsync
2063 			 * page and is installing a completely unrelated page.
2064 			 * Unsync pages must not be left as is, because the new
2065 			 * upper-level page will be write-protected.
2066 			 */
2067 			if (level > PG_LEVEL_4K && sp->unsync)
2068 				kvm_mmu_prepare_zap_page(vcpu->kvm, sp,
2069 							 &invalid_list);
2070 			continue;
2071 		}
2072 
2073 		if (direct_mmu)
2074 			goto trace_get_page;
2075 
2076 		if (sp->unsync) {
2077 			/*
2078 			 * The page is good, but is stale.  kvm_sync_page does
2079 			 * get the latest guest state, but (unlike mmu_unsync_children)
2080 			 * it doesn't write-protect the page or mark it synchronized!
2081 			 * This way the validity of the mapping is ensured, but the
2082 			 * overhead of write protection is not incurred until the
2083 			 * guest invalidates the TLB mapping.  This allows multiple
2084 			 * SPs for a single gfn to be unsync.
2085 			 *
2086 			 * If the sync fails, the page is zapped.  If so, break
2087 			 * in order to rebuild it.
2088 			 */
2089 			ret = kvm_sync_page(vcpu, sp, &invalid_list);
2090 			if (ret < 0)
2091 				break;
2092 
2093 			WARN_ON(!list_empty(&invalid_list));
2094 			if (ret > 0)
2095 				kvm_flush_remote_tlbs(vcpu->kvm);
2096 		}
2097 
2098 		__clear_sp_write_flooding_count(sp);
2099 
2100 trace_get_page:
2101 		trace_kvm_mmu_get_page(sp, false);
2102 		goto out;
2103 	}
2104 
2105 	++vcpu->kvm->stat.mmu_cache_miss;
2106 
2107 	sp = kvm_mmu_alloc_page(vcpu, direct);
2108 
2109 	sp->gfn = gfn;
2110 	sp->role = role;
2111 	hlist_add_head(&sp->hash_link, sp_list);
2112 	if (sp_has_gptes(sp)) {
2113 		account_shadowed(vcpu->kvm, sp);
2114 		if (level == PG_LEVEL_4K && kvm_vcpu_write_protect_gfn(vcpu, gfn))
2115 			kvm_flush_remote_tlbs_with_address(vcpu->kvm, gfn, 1);
2116 	}
2117 	trace_kvm_mmu_get_page(sp, true);
2118 out:
2119 	kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
2120 
2121 	if (collisions > vcpu->kvm->stat.max_mmu_page_hash_collisions)
2122 		vcpu->kvm->stat.max_mmu_page_hash_collisions = collisions;
2123 	return sp;
2124 }
2125 
shadow_walk_init_using_root(struct kvm_shadow_walk_iterator * iterator,struct kvm_vcpu * vcpu,hpa_t root,u64 addr)2126 static void shadow_walk_init_using_root(struct kvm_shadow_walk_iterator *iterator,
2127 					struct kvm_vcpu *vcpu, hpa_t root,
2128 					u64 addr)
2129 {
2130 	iterator->addr = addr;
2131 	iterator->shadow_addr = root;
2132 	iterator->level = vcpu->arch.mmu->root_role.level;
2133 
2134 	if (iterator->level >= PT64_ROOT_4LEVEL &&
2135 	    vcpu->arch.mmu->cpu_role.base.level < PT64_ROOT_4LEVEL &&
2136 	    !vcpu->arch.mmu->root_role.direct)
2137 		iterator->level = PT32E_ROOT_LEVEL;
2138 
2139 	if (iterator->level == PT32E_ROOT_LEVEL) {
2140 		/*
2141 		 * prev_root is currently only used for 64-bit hosts. So only
2142 		 * the active root_hpa is valid here.
2143 		 */
2144 		BUG_ON(root != vcpu->arch.mmu->root.hpa);
2145 
2146 		iterator->shadow_addr
2147 			= vcpu->arch.mmu->pae_root[(addr >> 30) & 3];
2148 		iterator->shadow_addr &= PT64_BASE_ADDR_MASK;
2149 		--iterator->level;
2150 		if (!iterator->shadow_addr)
2151 			iterator->level = 0;
2152 	}
2153 }
2154 
shadow_walk_init(struct kvm_shadow_walk_iterator * iterator,struct kvm_vcpu * vcpu,u64 addr)2155 static void shadow_walk_init(struct kvm_shadow_walk_iterator *iterator,
2156 			     struct kvm_vcpu *vcpu, u64 addr)
2157 {
2158 	shadow_walk_init_using_root(iterator, vcpu, vcpu->arch.mmu->root.hpa,
2159 				    addr);
2160 }
2161 
shadow_walk_okay(struct kvm_shadow_walk_iterator * iterator)2162 static bool shadow_walk_okay(struct kvm_shadow_walk_iterator *iterator)
2163 {
2164 	if (iterator->level < PG_LEVEL_4K)
2165 		return false;
2166 
2167 	iterator->index = SHADOW_PT_INDEX(iterator->addr, iterator->level);
2168 	iterator->sptep	= ((u64 *)__va(iterator->shadow_addr)) + iterator->index;
2169 	return true;
2170 }
2171 
__shadow_walk_next(struct kvm_shadow_walk_iterator * iterator,u64 spte)2172 static void __shadow_walk_next(struct kvm_shadow_walk_iterator *iterator,
2173 			       u64 spte)
2174 {
2175 	if (!is_shadow_present_pte(spte) || is_last_spte(spte, iterator->level)) {
2176 		iterator->level = 0;
2177 		return;
2178 	}
2179 
2180 	iterator->shadow_addr = spte & PT64_BASE_ADDR_MASK;
2181 	--iterator->level;
2182 }
2183 
shadow_walk_next(struct kvm_shadow_walk_iterator * iterator)2184 static void shadow_walk_next(struct kvm_shadow_walk_iterator *iterator)
2185 {
2186 	__shadow_walk_next(iterator, *iterator->sptep);
2187 }
2188 
link_shadow_page(struct kvm_vcpu * vcpu,u64 * sptep,struct kvm_mmu_page * sp)2189 static void link_shadow_page(struct kvm_vcpu *vcpu, u64 *sptep,
2190 			     struct kvm_mmu_page *sp)
2191 {
2192 	u64 spte;
2193 
2194 	BUILD_BUG_ON(VMX_EPT_WRITABLE_MASK != PT_WRITABLE_MASK);
2195 
2196 	spte = make_nonleaf_spte(sp->spt, sp_ad_disabled(sp));
2197 
2198 	mmu_spte_set(sptep, spte);
2199 
2200 	mmu_page_add_parent_pte(vcpu, sp, sptep);
2201 
2202 	if (sp->unsync_children || sp->unsync)
2203 		mark_unsync(sptep);
2204 }
2205 
validate_direct_spte(struct kvm_vcpu * vcpu,u64 * sptep,unsigned direct_access)2206 static void validate_direct_spte(struct kvm_vcpu *vcpu, u64 *sptep,
2207 				   unsigned direct_access)
2208 {
2209 	if (is_shadow_present_pte(*sptep) && !is_large_pte(*sptep)) {
2210 		struct kvm_mmu_page *child;
2211 
2212 		/*
2213 		 * For the direct sp, if the guest pte's dirty bit
2214 		 * changed form clean to dirty, it will corrupt the
2215 		 * sp's access: allow writable in the read-only sp,
2216 		 * so we should update the spte at this point to get
2217 		 * a new sp with the correct access.
2218 		 */
2219 		child = to_shadow_page(*sptep & PT64_BASE_ADDR_MASK);
2220 		if (child->role.access == direct_access)
2221 			return;
2222 
2223 		drop_parent_pte(child, sptep);
2224 		kvm_flush_remote_tlbs_with_address(vcpu->kvm, child->gfn, 1);
2225 	}
2226 }
2227 
2228 /* Returns the number of zapped non-leaf child shadow pages. */
mmu_page_zap_pte(struct kvm * kvm,struct kvm_mmu_page * sp,u64 * spte,struct list_head * invalid_list)2229 static int mmu_page_zap_pte(struct kvm *kvm, struct kvm_mmu_page *sp,
2230 			    u64 *spte, struct list_head *invalid_list)
2231 {
2232 	u64 pte;
2233 	struct kvm_mmu_page *child;
2234 
2235 	pte = *spte;
2236 	if (is_shadow_present_pte(pte)) {
2237 		if (is_last_spte(pte, sp->role.level)) {
2238 			drop_spte(kvm, spte);
2239 		} else {
2240 			child = to_shadow_page(pte & PT64_BASE_ADDR_MASK);
2241 			drop_parent_pte(child, spte);
2242 
2243 			/*
2244 			 * Recursively zap nested TDP SPs, parentless SPs are
2245 			 * unlikely to be used again in the near future.  This
2246 			 * avoids retaining a large number of stale nested SPs.
2247 			 */
2248 			if (tdp_enabled && invalid_list &&
2249 			    child->role.guest_mode && !child->parent_ptes.val)
2250 				return kvm_mmu_prepare_zap_page(kvm, child,
2251 								invalid_list);
2252 		}
2253 	} else if (is_mmio_spte(pte)) {
2254 		mmu_spte_clear_no_track(spte);
2255 	}
2256 	return 0;
2257 }
2258 
kvm_mmu_page_unlink_children(struct kvm * kvm,struct kvm_mmu_page * sp,struct list_head * invalid_list)2259 static int kvm_mmu_page_unlink_children(struct kvm *kvm,
2260 					struct kvm_mmu_page *sp,
2261 					struct list_head *invalid_list)
2262 {
2263 	int zapped = 0;
2264 	unsigned i;
2265 
2266 	for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
2267 		zapped += mmu_page_zap_pte(kvm, sp, sp->spt + i, invalid_list);
2268 
2269 	return zapped;
2270 }
2271 
kvm_mmu_unlink_parents(struct kvm_mmu_page * sp)2272 static void kvm_mmu_unlink_parents(struct kvm_mmu_page *sp)
2273 {
2274 	u64 *sptep;
2275 	struct rmap_iterator iter;
2276 
2277 	while ((sptep = rmap_get_first(&sp->parent_ptes, &iter)))
2278 		drop_parent_pte(sp, sptep);
2279 }
2280 
mmu_zap_unsync_children(struct kvm * kvm,struct kvm_mmu_page * parent,struct list_head * invalid_list)2281 static int mmu_zap_unsync_children(struct kvm *kvm,
2282 				   struct kvm_mmu_page *parent,
2283 				   struct list_head *invalid_list)
2284 {
2285 	int i, zapped = 0;
2286 	struct mmu_page_path parents;
2287 	struct kvm_mmu_pages pages;
2288 
2289 	if (parent->role.level == PG_LEVEL_4K)
2290 		return 0;
2291 
2292 	while (mmu_unsync_walk(parent, &pages)) {
2293 		struct kvm_mmu_page *sp;
2294 
2295 		for_each_sp(pages, sp, parents, i) {
2296 			kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
2297 			mmu_pages_clear_parents(&parents);
2298 			zapped++;
2299 		}
2300 	}
2301 
2302 	return zapped;
2303 }
2304 
__kvm_mmu_prepare_zap_page(struct kvm * kvm,struct kvm_mmu_page * sp,struct list_head * invalid_list,int * nr_zapped)2305 static bool __kvm_mmu_prepare_zap_page(struct kvm *kvm,
2306 				       struct kvm_mmu_page *sp,
2307 				       struct list_head *invalid_list,
2308 				       int *nr_zapped)
2309 {
2310 	bool list_unstable, zapped_root = false;
2311 
2312 	trace_kvm_mmu_prepare_zap_page(sp);
2313 	++kvm->stat.mmu_shadow_zapped;
2314 	*nr_zapped = mmu_zap_unsync_children(kvm, sp, invalid_list);
2315 	*nr_zapped += kvm_mmu_page_unlink_children(kvm, sp, invalid_list);
2316 	kvm_mmu_unlink_parents(sp);
2317 
2318 	/* Zapping children means active_mmu_pages has become unstable. */
2319 	list_unstable = *nr_zapped;
2320 
2321 	if (!sp->role.invalid && sp_has_gptes(sp))
2322 		unaccount_shadowed(kvm, sp);
2323 
2324 	if (sp->unsync)
2325 		kvm_unlink_unsync_page(kvm, sp);
2326 	if (!sp->root_count) {
2327 		/* Count self */
2328 		(*nr_zapped)++;
2329 
2330 		/*
2331 		 * Already invalid pages (previously active roots) are not on
2332 		 * the active page list.  See list_del() in the "else" case of
2333 		 * !sp->root_count.
2334 		 */
2335 		if (sp->role.invalid)
2336 			list_add(&sp->link, invalid_list);
2337 		else
2338 			list_move(&sp->link, invalid_list);
2339 		kvm_mod_used_mmu_pages(kvm, -1);
2340 	} else {
2341 		/*
2342 		 * Remove the active root from the active page list, the root
2343 		 * will be explicitly freed when the root_count hits zero.
2344 		 */
2345 		list_del(&sp->link);
2346 
2347 		/*
2348 		 * Obsolete pages cannot be used on any vCPUs, see the comment
2349 		 * in kvm_mmu_zap_all_fast().  Note, is_obsolete_sp() also
2350 		 * treats invalid shadow pages as being obsolete.
2351 		 */
2352 		zapped_root = !is_obsolete_sp(kvm, sp);
2353 	}
2354 
2355 	if (sp->lpage_disallowed)
2356 		unaccount_huge_nx_page(kvm, sp);
2357 
2358 	sp->role.invalid = 1;
2359 
2360 	/*
2361 	 * Make the request to free obsolete roots after marking the root
2362 	 * invalid, otherwise other vCPUs may not see it as invalid.
2363 	 */
2364 	if (zapped_root)
2365 		kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_FREE_OBSOLETE_ROOTS);
2366 	return list_unstable;
2367 }
2368 
kvm_mmu_prepare_zap_page(struct kvm * kvm,struct kvm_mmu_page * sp,struct list_head * invalid_list)2369 static bool kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
2370 				     struct list_head *invalid_list)
2371 {
2372 	int nr_zapped;
2373 
2374 	__kvm_mmu_prepare_zap_page(kvm, sp, invalid_list, &nr_zapped);
2375 	return nr_zapped;
2376 }
2377 
kvm_mmu_commit_zap_page(struct kvm * kvm,struct list_head * invalid_list)2378 static void kvm_mmu_commit_zap_page(struct kvm *kvm,
2379 				    struct list_head *invalid_list)
2380 {
2381 	struct kvm_mmu_page *sp, *nsp;
2382 
2383 	if (list_empty(invalid_list))
2384 		return;
2385 
2386 	/*
2387 	 * We need to make sure everyone sees our modifications to
2388 	 * the page tables and see changes to vcpu->mode here. The barrier
2389 	 * in the kvm_flush_remote_tlbs() achieves this. This pairs
2390 	 * with vcpu_enter_guest and walk_shadow_page_lockless_begin/end.
2391 	 *
2392 	 * In addition, kvm_flush_remote_tlbs waits for all vcpus to exit
2393 	 * guest mode and/or lockless shadow page table walks.
2394 	 */
2395 	kvm_flush_remote_tlbs(kvm);
2396 
2397 	list_for_each_entry_safe(sp, nsp, invalid_list, link) {
2398 		WARN_ON(!sp->role.invalid || sp->root_count);
2399 		kvm_mmu_free_page(sp);
2400 	}
2401 }
2402 
kvm_mmu_zap_oldest_mmu_pages(struct kvm * kvm,unsigned long nr_to_zap)2403 static unsigned long kvm_mmu_zap_oldest_mmu_pages(struct kvm *kvm,
2404 						  unsigned long nr_to_zap)
2405 {
2406 	unsigned long total_zapped = 0;
2407 	struct kvm_mmu_page *sp, *tmp;
2408 	LIST_HEAD(invalid_list);
2409 	bool unstable;
2410 	int nr_zapped;
2411 
2412 	if (list_empty(&kvm->arch.active_mmu_pages))
2413 		return 0;
2414 
2415 restart:
2416 	list_for_each_entry_safe_reverse(sp, tmp, &kvm->arch.active_mmu_pages, link) {
2417 		/*
2418 		 * Don't zap active root pages, the page itself can't be freed
2419 		 * and zapping it will just force vCPUs to realloc and reload.
2420 		 */
2421 		if (sp->root_count)
2422 			continue;
2423 
2424 		unstable = __kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list,
2425 						      &nr_zapped);
2426 		total_zapped += nr_zapped;
2427 		if (total_zapped >= nr_to_zap)
2428 			break;
2429 
2430 		if (unstable)
2431 			goto restart;
2432 	}
2433 
2434 	kvm_mmu_commit_zap_page(kvm, &invalid_list);
2435 
2436 	kvm->stat.mmu_recycled += total_zapped;
2437 	return total_zapped;
2438 }
2439 
kvm_mmu_available_pages(struct kvm * kvm)2440 static inline unsigned long kvm_mmu_available_pages(struct kvm *kvm)
2441 {
2442 	if (kvm->arch.n_max_mmu_pages > kvm->arch.n_used_mmu_pages)
2443 		return kvm->arch.n_max_mmu_pages -
2444 			kvm->arch.n_used_mmu_pages;
2445 
2446 	return 0;
2447 }
2448 
make_mmu_pages_available(struct kvm_vcpu * vcpu)2449 static int make_mmu_pages_available(struct kvm_vcpu *vcpu)
2450 {
2451 	unsigned long avail = kvm_mmu_available_pages(vcpu->kvm);
2452 
2453 	if (likely(avail >= KVM_MIN_FREE_MMU_PAGES))
2454 		return 0;
2455 
2456 	kvm_mmu_zap_oldest_mmu_pages(vcpu->kvm, KVM_REFILL_PAGES - avail);
2457 
2458 	/*
2459 	 * Note, this check is intentionally soft, it only guarantees that one
2460 	 * page is available, while the caller may end up allocating as many as
2461 	 * four pages, e.g. for PAE roots or for 5-level paging.  Temporarily
2462 	 * exceeding the (arbitrary by default) limit will not harm the host,
2463 	 * being too aggressive may unnecessarily kill the guest, and getting an
2464 	 * exact count is far more trouble than it's worth, especially in the
2465 	 * page fault paths.
2466 	 */
2467 	if (!kvm_mmu_available_pages(vcpu->kvm))
2468 		return -ENOSPC;
2469 	return 0;
2470 }
2471 
2472 /*
2473  * Changing the number of mmu pages allocated to the vm
2474  * Note: if goal_nr_mmu_pages is too small, you will get dead lock
2475  */
kvm_mmu_change_mmu_pages(struct kvm * kvm,unsigned long goal_nr_mmu_pages)2476 void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned long goal_nr_mmu_pages)
2477 {
2478 	write_lock(&kvm->mmu_lock);
2479 
2480 	if (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages) {
2481 		kvm_mmu_zap_oldest_mmu_pages(kvm, kvm->arch.n_used_mmu_pages -
2482 						  goal_nr_mmu_pages);
2483 
2484 		goal_nr_mmu_pages = kvm->arch.n_used_mmu_pages;
2485 	}
2486 
2487 	kvm->arch.n_max_mmu_pages = goal_nr_mmu_pages;
2488 
2489 	write_unlock(&kvm->mmu_lock);
2490 }
2491 
kvm_mmu_unprotect_page(struct kvm * kvm,gfn_t gfn)2492 int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn)
2493 {
2494 	struct kvm_mmu_page *sp;
2495 	LIST_HEAD(invalid_list);
2496 	int r;
2497 
2498 	pgprintk("%s: looking for gfn %llx\n", __func__, gfn);
2499 	r = 0;
2500 	write_lock(&kvm->mmu_lock);
2501 	for_each_gfn_valid_sp_with_gptes(kvm, sp, gfn) {
2502 		pgprintk("%s: gfn %llx role %x\n", __func__, gfn,
2503 			 sp->role.word);
2504 		r = 1;
2505 		kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);
2506 	}
2507 	kvm_mmu_commit_zap_page(kvm, &invalid_list);
2508 	write_unlock(&kvm->mmu_lock);
2509 
2510 	return r;
2511 }
2512 
kvm_mmu_unprotect_page_virt(struct kvm_vcpu * vcpu,gva_t gva)2513 static int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
2514 {
2515 	gpa_t gpa;
2516 	int r;
2517 
2518 	if (vcpu->arch.mmu->root_role.direct)
2519 		return 0;
2520 
2521 	gpa = kvm_mmu_gva_to_gpa_read(vcpu, gva, NULL);
2522 
2523 	r = kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT);
2524 
2525 	return r;
2526 }
2527 
kvm_unsync_page(struct kvm * kvm,struct kvm_mmu_page * sp)2528 static void kvm_unsync_page(struct kvm *kvm, struct kvm_mmu_page *sp)
2529 {
2530 	trace_kvm_mmu_unsync_page(sp);
2531 	++kvm->stat.mmu_unsync;
2532 	sp->unsync = 1;
2533 
2534 	kvm_mmu_mark_parents_unsync(sp);
2535 }
2536 
2537 /*
2538  * Attempt to unsync any shadow pages that can be reached by the specified gfn,
2539  * KVM is creating a writable mapping for said gfn.  Returns 0 if all pages
2540  * were marked unsync (or if there is no shadow page), -EPERM if the SPTE must
2541  * be write-protected.
2542  */
mmu_try_to_unsync_pages(struct kvm * kvm,const struct kvm_memory_slot * slot,gfn_t gfn,bool can_unsync,bool prefetch)2543 int mmu_try_to_unsync_pages(struct kvm *kvm, const struct kvm_memory_slot *slot,
2544 			    gfn_t gfn, bool can_unsync, bool prefetch)
2545 {
2546 	struct kvm_mmu_page *sp;
2547 	bool locked = false;
2548 
2549 	/*
2550 	 * Force write-protection if the page is being tracked.  Note, the page
2551 	 * track machinery is used to write-protect upper-level shadow pages,
2552 	 * i.e. this guards the role.level == 4K assertion below!
2553 	 */
2554 	if (kvm_slot_page_track_is_active(kvm, slot, gfn, KVM_PAGE_TRACK_WRITE))
2555 		return -EPERM;
2556 
2557 	/*
2558 	 * The page is not write-tracked, mark existing shadow pages unsync
2559 	 * unless KVM is synchronizing an unsync SP (can_unsync = false).  In
2560 	 * that case, KVM must complete emulation of the guest TLB flush before
2561 	 * allowing shadow pages to become unsync (writable by the guest).
2562 	 */
2563 	for_each_gfn_valid_sp_with_gptes(kvm, sp, gfn) {
2564 		if (!can_unsync)
2565 			return -EPERM;
2566 
2567 		if (sp->unsync)
2568 			continue;
2569 
2570 		if (prefetch)
2571 			return -EEXIST;
2572 
2573 		/*
2574 		 * TDP MMU page faults require an additional spinlock as they
2575 		 * run with mmu_lock held for read, not write, and the unsync
2576 		 * logic is not thread safe.  Take the spinklock regardless of
2577 		 * the MMU type to avoid extra conditionals/parameters, there's
2578 		 * no meaningful penalty if mmu_lock is held for write.
2579 		 */
2580 		if (!locked) {
2581 			locked = true;
2582 			spin_lock(&kvm->arch.mmu_unsync_pages_lock);
2583 
2584 			/*
2585 			 * Recheck after taking the spinlock, a different vCPU
2586 			 * may have since marked the page unsync.  A false
2587 			 * positive on the unprotected check above is not
2588 			 * possible as clearing sp->unsync _must_ hold mmu_lock
2589 			 * for write, i.e. unsync cannot transition from 0->1
2590 			 * while this CPU holds mmu_lock for read (or write).
2591 			 */
2592 			if (READ_ONCE(sp->unsync))
2593 				continue;
2594 		}
2595 
2596 		WARN_ON(sp->role.level != PG_LEVEL_4K);
2597 		kvm_unsync_page(kvm, sp);
2598 	}
2599 	if (locked)
2600 		spin_unlock(&kvm->arch.mmu_unsync_pages_lock);
2601 
2602 	/*
2603 	 * We need to ensure that the marking of unsync pages is visible
2604 	 * before the SPTE is updated to allow writes because
2605 	 * kvm_mmu_sync_roots() checks the unsync flags without holding
2606 	 * the MMU lock and so can race with this. If the SPTE was updated
2607 	 * before the page had been marked as unsync-ed, something like the
2608 	 * following could happen:
2609 	 *
2610 	 * CPU 1                    CPU 2
2611 	 * ---------------------------------------------------------------------
2612 	 * 1.2 Host updates SPTE
2613 	 *     to be writable
2614 	 *                      2.1 Guest writes a GPTE for GVA X.
2615 	 *                          (GPTE being in the guest page table shadowed
2616 	 *                           by the SP from CPU 1.)
2617 	 *                          This reads SPTE during the page table walk.
2618 	 *                          Since SPTE.W is read as 1, there is no
2619 	 *                          fault.
2620 	 *
2621 	 *                      2.2 Guest issues TLB flush.
2622 	 *                          That causes a VM Exit.
2623 	 *
2624 	 *                      2.3 Walking of unsync pages sees sp->unsync is
2625 	 *                          false and skips the page.
2626 	 *
2627 	 *                      2.4 Guest accesses GVA X.
2628 	 *                          Since the mapping in the SP was not updated,
2629 	 *                          so the old mapping for GVA X incorrectly
2630 	 *                          gets used.
2631 	 * 1.1 Host marks SP
2632 	 *     as unsync
2633 	 *     (sp->unsync = true)
2634 	 *
2635 	 * The write barrier below ensures that 1.1 happens before 1.2 and thus
2636 	 * the situation in 2.4 does not arise.  It pairs with the read barrier
2637 	 * in is_unsync_root(), placed between 2.1's load of SPTE.W and 2.3.
2638 	 */
2639 	smp_wmb();
2640 
2641 	return 0;
2642 }
2643 
mmu_set_spte(struct kvm_vcpu * vcpu,struct kvm_memory_slot * slot,u64 * sptep,unsigned int pte_access,gfn_t gfn,kvm_pfn_t pfn,struct kvm_page_fault * fault)2644 static int mmu_set_spte(struct kvm_vcpu *vcpu, struct kvm_memory_slot *slot,
2645 			u64 *sptep, unsigned int pte_access, gfn_t gfn,
2646 			kvm_pfn_t pfn, struct kvm_page_fault *fault)
2647 {
2648 	struct kvm_mmu_page *sp = sptep_to_sp(sptep);
2649 	int level = sp->role.level;
2650 	int was_rmapped = 0;
2651 	int ret = RET_PF_FIXED;
2652 	bool flush = false;
2653 	bool wrprot;
2654 	u64 spte;
2655 
2656 	/* Prefetching always gets a writable pfn.  */
2657 	bool host_writable = !fault || fault->map_writable;
2658 	bool prefetch = !fault || fault->prefetch;
2659 	bool write_fault = fault && fault->write;
2660 
2661 	pgprintk("%s: spte %llx write_fault %d gfn %llx\n", __func__,
2662 		 *sptep, write_fault, gfn);
2663 
2664 	if (unlikely(is_noslot_pfn(pfn))) {
2665 		vcpu->stat.pf_mmio_spte_created++;
2666 		mark_mmio_spte(vcpu, sptep, gfn, pte_access);
2667 		return RET_PF_EMULATE;
2668 	}
2669 
2670 	if (is_shadow_present_pte(*sptep)) {
2671 		/*
2672 		 * If we overwrite a PTE page pointer with a 2MB PMD, unlink
2673 		 * the parent of the now unreachable PTE.
2674 		 */
2675 		if (level > PG_LEVEL_4K && !is_large_pte(*sptep)) {
2676 			struct kvm_mmu_page *child;
2677 			u64 pte = *sptep;
2678 
2679 			child = to_shadow_page(pte & PT64_BASE_ADDR_MASK);
2680 			drop_parent_pte(child, sptep);
2681 			flush = true;
2682 		} else if (pfn != spte_to_pfn(*sptep)) {
2683 			pgprintk("hfn old %llx new %llx\n",
2684 				 spte_to_pfn(*sptep), pfn);
2685 			drop_spte(vcpu->kvm, sptep);
2686 			flush = true;
2687 		} else
2688 			was_rmapped = 1;
2689 	}
2690 
2691 	wrprot = make_spte(vcpu, sp, slot, pte_access, gfn, pfn, *sptep, prefetch,
2692 			   true, host_writable, &spte);
2693 
2694 	if (*sptep == spte) {
2695 		ret = RET_PF_SPURIOUS;
2696 	} else {
2697 		flush |= mmu_spte_update(sptep, spte);
2698 		trace_kvm_mmu_set_spte(level, gfn, sptep);
2699 	}
2700 
2701 	if (wrprot) {
2702 		if (write_fault)
2703 			ret = RET_PF_EMULATE;
2704 	}
2705 
2706 	if (flush)
2707 		kvm_flush_remote_tlbs_with_address(vcpu->kvm, gfn,
2708 				KVM_PAGES_PER_HPAGE(level));
2709 
2710 	pgprintk("%s: setting spte %llx\n", __func__, *sptep);
2711 
2712 	if (!was_rmapped) {
2713 		WARN_ON_ONCE(ret == RET_PF_SPURIOUS);
2714 		kvm_update_page_stats(vcpu->kvm, level, 1);
2715 		rmap_add(vcpu, slot, sptep, gfn);
2716 	}
2717 
2718 	return ret;
2719 }
2720 
direct_pte_prefetch_many(struct kvm_vcpu * vcpu,struct kvm_mmu_page * sp,u64 * start,u64 * end)2721 static int direct_pte_prefetch_many(struct kvm_vcpu *vcpu,
2722 				    struct kvm_mmu_page *sp,
2723 				    u64 *start, u64 *end)
2724 {
2725 	struct page *pages[PTE_PREFETCH_NUM];
2726 	struct kvm_memory_slot *slot;
2727 	unsigned int access = sp->role.access;
2728 	int i, ret;
2729 	gfn_t gfn;
2730 
2731 	gfn = kvm_mmu_page_get_gfn(sp, start - sp->spt);
2732 	slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, access & ACC_WRITE_MASK);
2733 	if (!slot)
2734 		return -1;
2735 
2736 	ret = gfn_to_page_many_atomic(slot, gfn, pages, end - start);
2737 	if (ret <= 0)
2738 		return -1;
2739 
2740 	for (i = 0; i < ret; i++, gfn++, start++) {
2741 		mmu_set_spte(vcpu, slot, start, access, gfn,
2742 			     page_to_pfn(pages[i]), NULL);
2743 		put_page(pages[i]);
2744 	}
2745 
2746 	return 0;
2747 }
2748 
__direct_pte_prefetch(struct kvm_vcpu * vcpu,struct kvm_mmu_page * sp,u64 * sptep)2749 static void __direct_pte_prefetch(struct kvm_vcpu *vcpu,
2750 				  struct kvm_mmu_page *sp, u64 *sptep)
2751 {
2752 	u64 *spte, *start = NULL;
2753 	int i;
2754 
2755 	WARN_ON(!sp->role.direct);
2756 
2757 	i = (sptep - sp->spt) & ~(PTE_PREFETCH_NUM - 1);
2758 	spte = sp->spt + i;
2759 
2760 	for (i = 0; i < PTE_PREFETCH_NUM; i++, spte++) {
2761 		if (is_shadow_present_pte(*spte) || spte == sptep) {
2762 			if (!start)
2763 				continue;
2764 			if (direct_pte_prefetch_many(vcpu, sp, start, spte) < 0)
2765 				return;
2766 			start = NULL;
2767 		} else if (!start)
2768 			start = spte;
2769 	}
2770 	if (start)
2771 		direct_pte_prefetch_many(vcpu, sp, start, spte);
2772 }
2773 
direct_pte_prefetch(struct kvm_vcpu * vcpu,u64 * sptep)2774 static void direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep)
2775 {
2776 	struct kvm_mmu_page *sp;
2777 
2778 	sp = sptep_to_sp(sptep);
2779 
2780 	/*
2781 	 * Without accessed bits, there's no way to distinguish between
2782 	 * actually accessed translations and prefetched, so disable pte
2783 	 * prefetch if accessed bits aren't available.
2784 	 */
2785 	if (sp_ad_disabled(sp))
2786 		return;
2787 
2788 	if (sp->role.level > PG_LEVEL_4K)
2789 		return;
2790 
2791 	/*
2792 	 * If addresses are being invalidated, skip prefetching to avoid
2793 	 * accidentally prefetching those addresses.
2794 	 */
2795 	if (unlikely(vcpu->kvm->mmu_notifier_count))
2796 		return;
2797 
2798 	__direct_pte_prefetch(vcpu, sp, sptep);
2799 }
2800 
host_pfn_mapping_level(struct kvm * kvm,gfn_t gfn,kvm_pfn_t pfn,const struct kvm_memory_slot * slot)2801 static int host_pfn_mapping_level(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn,
2802 				  const struct kvm_memory_slot *slot)
2803 {
2804 	unsigned long hva;
2805 	unsigned long flags;
2806 	int level = PG_LEVEL_4K;
2807 	pgd_t pgd;
2808 	p4d_t p4d;
2809 	pud_t pud;
2810 	pmd_t pmd;
2811 
2812 	if (!PageCompound(pfn_to_page(pfn)) && !kvm_is_zone_device_pfn(pfn))
2813 		return PG_LEVEL_4K;
2814 
2815 	/*
2816 	 * Note, using the already-retrieved memslot and __gfn_to_hva_memslot()
2817 	 * is not solely for performance, it's also necessary to avoid the
2818 	 * "writable" check in __gfn_to_hva_many(), which will always fail on
2819 	 * read-only memslots due to gfn_to_hva() assuming writes.  Earlier
2820 	 * page fault steps have already verified the guest isn't writing a
2821 	 * read-only memslot.
2822 	 */
2823 	hva = __gfn_to_hva_memslot(slot, gfn);
2824 
2825 	/*
2826 	 * Lookup the mapping level in the current mm.  The information
2827 	 * may become stale soon, but it is safe to use as long as
2828 	 * 1) mmu_notifier_retry was checked after taking mmu_lock, and
2829 	 * 2) mmu_lock is taken now.
2830 	 *
2831 	 * We still need to disable IRQs to prevent concurrent tear down
2832 	 * of page tables.
2833 	 */
2834 	local_irq_save(flags);
2835 
2836 	pgd = READ_ONCE(*pgd_offset(kvm->mm, hva));
2837 	if (pgd_none(pgd))
2838 		goto out;
2839 
2840 	p4d = READ_ONCE(*p4d_offset(&pgd, hva));
2841 	if (p4d_none(p4d) || !p4d_present(p4d))
2842 		goto out;
2843 
2844 	pud = READ_ONCE(*pud_offset(&p4d, hva));
2845 	if (pud_none(pud) || !pud_present(pud))
2846 		goto out;
2847 
2848 	if (pud_large(pud)) {
2849 		level = PG_LEVEL_1G;
2850 		goto out;
2851 	}
2852 
2853 	pmd = READ_ONCE(*pmd_offset(&pud, hva));
2854 	if (pmd_none(pmd) || !pmd_present(pmd))
2855 		goto out;
2856 
2857 	if (pmd_large(pmd))
2858 		level = PG_LEVEL_2M;
2859 
2860 out:
2861 	local_irq_restore(flags);
2862 	return level;
2863 }
2864 
kvm_mmu_max_mapping_level(struct kvm * kvm,const struct kvm_memory_slot * slot,gfn_t gfn,kvm_pfn_t pfn,int max_level)2865 int kvm_mmu_max_mapping_level(struct kvm *kvm,
2866 			      const struct kvm_memory_slot *slot, gfn_t gfn,
2867 			      kvm_pfn_t pfn, int max_level)
2868 {
2869 	struct kvm_lpage_info *linfo;
2870 	int host_level;
2871 
2872 	max_level = min(max_level, max_huge_page_level);
2873 	for ( ; max_level > PG_LEVEL_4K; max_level--) {
2874 		linfo = lpage_info_slot(gfn, slot, max_level);
2875 		if (!linfo->disallow_lpage)
2876 			break;
2877 	}
2878 
2879 	if (max_level == PG_LEVEL_4K)
2880 		return PG_LEVEL_4K;
2881 
2882 	host_level = host_pfn_mapping_level(kvm, gfn, pfn, slot);
2883 	return min(host_level, max_level);
2884 }
2885 
kvm_mmu_hugepage_adjust(struct kvm_vcpu * vcpu,struct kvm_page_fault * fault)2886 void kvm_mmu_hugepage_adjust(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
2887 {
2888 	struct kvm_memory_slot *slot = fault->slot;
2889 	kvm_pfn_t mask;
2890 
2891 	fault->huge_page_disallowed = fault->exec && fault->nx_huge_page_workaround_enabled;
2892 
2893 	if (unlikely(fault->max_level == PG_LEVEL_4K))
2894 		return;
2895 
2896 	if (is_error_noslot_pfn(fault->pfn) || kvm_is_reserved_pfn(fault->pfn))
2897 		return;
2898 
2899 	if (kvm_slot_dirty_track_enabled(slot))
2900 		return;
2901 
2902 	/*
2903 	 * Enforce the iTLB multihit workaround after capturing the requested
2904 	 * level, which will be used to do precise, accurate accounting.
2905 	 */
2906 	fault->req_level = kvm_mmu_max_mapping_level(vcpu->kvm, slot,
2907 						     fault->gfn, fault->pfn,
2908 						     fault->max_level);
2909 	if (fault->req_level == PG_LEVEL_4K || fault->huge_page_disallowed)
2910 		return;
2911 
2912 	/*
2913 	 * mmu_notifier_retry() was successful and mmu_lock is held, so
2914 	 * the pmd can't be split from under us.
2915 	 */
2916 	fault->goal_level = fault->req_level;
2917 	mask = KVM_PAGES_PER_HPAGE(fault->goal_level) - 1;
2918 	VM_BUG_ON((fault->gfn & mask) != (fault->pfn & mask));
2919 	fault->pfn &= ~mask;
2920 }
2921 
disallowed_hugepage_adjust(struct kvm_page_fault * fault,u64 spte,int cur_level)2922 void disallowed_hugepage_adjust(struct kvm_page_fault *fault, u64 spte, int cur_level)
2923 {
2924 	if (cur_level > PG_LEVEL_4K &&
2925 	    cur_level == fault->goal_level &&
2926 	    is_shadow_present_pte(spte) &&
2927 	    !is_large_pte(spte)) {
2928 		/*
2929 		 * A small SPTE exists for this pfn, but FNAME(fetch)
2930 		 * and __direct_map would like to create a large PTE
2931 		 * instead: just force them to go down another level,
2932 		 * patching back for them into pfn the next 9 bits of
2933 		 * the address.
2934 		 */
2935 		u64 page_mask = KVM_PAGES_PER_HPAGE(cur_level) -
2936 				KVM_PAGES_PER_HPAGE(cur_level - 1);
2937 		fault->pfn |= fault->gfn & page_mask;
2938 		fault->goal_level--;
2939 	}
2940 }
2941 
__direct_map(struct kvm_vcpu * vcpu,struct kvm_page_fault * fault)2942 static int __direct_map(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
2943 {
2944 	struct kvm_shadow_walk_iterator it;
2945 	struct kvm_mmu_page *sp;
2946 	int ret;
2947 	gfn_t base_gfn = fault->gfn;
2948 
2949 	kvm_mmu_hugepage_adjust(vcpu, fault);
2950 
2951 	trace_kvm_mmu_spte_requested(fault);
2952 	for_each_shadow_entry(vcpu, fault->addr, it) {
2953 		/*
2954 		 * We cannot overwrite existing page tables with an NX
2955 		 * large page, as the leaf could be executable.
2956 		 */
2957 		if (fault->nx_huge_page_workaround_enabled)
2958 			disallowed_hugepage_adjust(fault, *it.sptep, it.level);
2959 
2960 		base_gfn = fault->gfn & ~(KVM_PAGES_PER_HPAGE(it.level) - 1);
2961 		if (it.level == fault->goal_level)
2962 			break;
2963 
2964 		drop_large_spte(vcpu, it.sptep);
2965 		if (is_shadow_present_pte(*it.sptep))
2966 			continue;
2967 
2968 		sp = kvm_mmu_get_page(vcpu, base_gfn, it.addr,
2969 				      it.level - 1, true, ACC_ALL);
2970 
2971 		link_shadow_page(vcpu, it.sptep, sp);
2972 		if (fault->is_tdp && fault->huge_page_disallowed &&
2973 		    fault->req_level >= it.level)
2974 			account_huge_nx_page(vcpu->kvm, sp);
2975 	}
2976 
2977 	if (WARN_ON_ONCE(it.level != fault->goal_level))
2978 		return -EFAULT;
2979 
2980 	ret = mmu_set_spte(vcpu, fault->slot, it.sptep, ACC_ALL,
2981 			   base_gfn, fault->pfn, fault);
2982 	if (ret == RET_PF_SPURIOUS)
2983 		return ret;
2984 
2985 	direct_pte_prefetch(vcpu, it.sptep);
2986 	return ret;
2987 }
2988 
kvm_send_hwpoison_signal(unsigned long address,struct task_struct * tsk)2989 static void kvm_send_hwpoison_signal(unsigned long address, struct task_struct *tsk)
2990 {
2991 	send_sig_mceerr(BUS_MCEERR_AR, (void __user *)address, PAGE_SHIFT, tsk);
2992 }
2993 
kvm_handle_bad_page(struct kvm_vcpu * vcpu,gfn_t gfn,kvm_pfn_t pfn)2994 static int kvm_handle_bad_page(struct kvm_vcpu *vcpu, gfn_t gfn, kvm_pfn_t pfn)
2995 {
2996 	/*
2997 	 * Do not cache the mmio info caused by writing the readonly gfn
2998 	 * into the spte otherwise read access on readonly gfn also can
2999 	 * caused mmio page fault and treat it as mmio access.
3000 	 */
3001 	if (pfn == KVM_PFN_ERR_RO_FAULT)
3002 		return RET_PF_EMULATE;
3003 
3004 	if (pfn == KVM_PFN_ERR_HWPOISON) {
3005 		kvm_send_hwpoison_signal(kvm_vcpu_gfn_to_hva(vcpu, gfn), current);
3006 		return RET_PF_RETRY;
3007 	}
3008 
3009 	return -EFAULT;
3010 }
3011 
handle_abnormal_pfn(struct kvm_vcpu * vcpu,struct kvm_page_fault * fault,unsigned int access)3012 static int handle_abnormal_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault,
3013 			       unsigned int access)
3014 {
3015 	/* The pfn is invalid, report the error! */
3016 	if (unlikely(is_error_pfn(fault->pfn)))
3017 		return kvm_handle_bad_page(vcpu, fault->gfn, fault->pfn);
3018 
3019 	if (unlikely(!fault->slot)) {
3020 		gva_t gva = fault->is_tdp ? 0 : fault->addr;
3021 
3022 		vcpu_cache_mmio_info(vcpu, gva, fault->gfn,
3023 				     access & shadow_mmio_access_mask);
3024 		/*
3025 		 * If MMIO caching is disabled, emulate immediately without
3026 		 * touching the shadow page tables as attempting to install an
3027 		 * MMIO SPTE will just be an expensive nop.  Do not cache MMIO
3028 		 * whose gfn is greater than host.MAXPHYADDR, any guest that
3029 		 * generates such gfns is running nested and is being tricked
3030 		 * by L0 userspace (you can observe gfn > L1.MAXPHYADDR if
3031 		 * and only if L1's MAXPHYADDR is inaccurate with respect to
3032 		 * the hardware's).
3033 		 */
3034 		if (unlikely(!enable_mmio_caching) ||
3035 		    unlikely(fault->gfn > kvm_mmu_max_gfn()))
3036 			return RET_PF_EMULATE;
3037 	}
3038 
3039 	return RET_PF_CONTINUE;
3040 }
3041 
page_fault_can_be_fast(struct kvm_page_fault * fault)3042 static bool page_fault_can_be_fast(struct kvm_page_fault *fault)
3043 {
3044 	/*
3045 	 * Page faults with reserved bits set, i.e. faults on MMIO SPTEs, only
3046 	 * reach the common page fault handler if the SPTE has an invalid MMIO
3047 	 * generation number.  Refreshing the MMIO generation needs to go down
3048 	 * the slow path.  Note, EPT Misconfigs do NOT set the PRESENT flag!
3049 	 */
3050 	if (fault->rsvd)
3051 		return false;
3052 
3053 	/*
3054 	 * #PF can be fast if:
3055 	 *
3056 	 * 1. The shadow page table entry is not present and A/D bits are
3057 	 *    disabled _by KVM_, which could mean that the fault is potentially
3058 	 *    caused by access tracking (if enabled).  If A/D bits are enabled
3059 	 *    by KVM, but disabled by L1 for L2, KVM is forced to disable A/D
3060 	 *    bits for L2 and employ access tracking, but the fast page fault
3061 	 *    mechanism only supports direct MMUs.
3062 	 * 2. The shadow page table entry is present, the access is a write,
3063 	 *    and no reserved bits are set (MMIO SPTEs cannot be "fixed"), i.e.
3064 	 *    the fault was caused by a write-protection violation.  If the
3065 	 *    SPTE is MMU-writable (determined later), the fault can be fixed
3066 	 *    by setting the Writable bit, which can be done out of mmu_lock.
3067 	 */
3068 	if (!fault->present)
3069 		return !kvm_ad_enabled();
3070 
3071 	/*
3072 	 * Note, instruction fetches and writes are mutually exclusive, ignore
3073 	 * the "exec" flag.
3074 	 */
3075 	return fault->write;
3076 }
3077 
3078 /*
3079  * Returns true if the SPTE was fixed successfully. Otherwise,
3080  * someone else modified the SPTE from its original value.
3081  */
3082 static bool
fast_pf_fix_direct_spte(struct kvm_vcpu * vcpu,struct kvm_page_fault * fault,u64 * sptep,u64 old_spte,u64 new_spte)3083 fast_pf_fix_direct_spte(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault,
3084 			u64 *sptep, u64 old_spte, u64 new_spte)
3085 {
3086 	/*
3087 	 * Theoretically we could also set dirty bit (and flush TLB) here in
3088 	 * order to eliminate unnecessary PML logging. See comments in
3089 	 * set_spte. But fast_page_fault is very unlikely to happen with PML
3090 	 * enabled, so we do not do this. This might result in the same GPA
3091 	 * to be logged in PML buffer again when the write really happens, and
3092 	 * eventually to be called by mark_page_dirty twice. But it's also no
3093 	 * harm. This also avoids the TLB flush needed after setting dirty bit
3094 	 * so non-PML cases won't be impacted.
3095 	 *
3096 	 * Compare with set_spte where instead shadow_dirty_mask is set.
3097 	 */
3098 	if (cmpxchg64(sptep, old_spte, new_spte) != old_spte)
3099 		return false;
3100 
3101 	if (is_writable_pte(new_spte) && !is_writable_pte(old_spte))
3102 		mark_page_dirty_in_slot(vcpu->kvm, fault->slot, fault->gfn);
3103 
3104 	return true;
3105 }
3106 
is_access_allowed(struct kvm_page_fault * fault,u64 spte)3107 static bool is_access_allowed(struct kvm_page_fault *fault, u64 spte)
3108 {
3109 	if (fault->exec)
3110 		return is_executable_pte(spte);
3111 
3112 	if (fault->write)
3113 		return is_writable_pte(spte);
3114 
3115 	/* Fault was on Read access */
3116 	return spte & PT_PRESENT_MASK;
3117 }
3118 
3119 /*
3120  * Returns the last level spte pointer of the shadow page walk for the given
3121  * gpa, and sets *spte to the spte value. This spte may be non-preset. If no
3122  * walk could be performed, returns NULL and *spte does not contain valid data.
3123  *
3124  * Contract:
3125  *  - Must be called between walk_shadow_page_lockless_{begin,end}.
3126  *  - The returned sptep must not be used after walk_shadow_page_lockless_end.
3127  */
fast_pf_get_last_sptep(struct kvm_vcpu * vcpu,gpa_t gpa,u64 * spte)3128 static u64 *fast_pf_get_last_sptep(struct kvm_vcpu *vcpu, gpa_t gpa, u64 *spte)
3129 {
3130 	struct kvm_shadow_walk_iterator iterator;
3131 	u64 old_spte;
3132 	u64 *sptep = NULL;
3133 
3134 	for_each_shadow_entry_lockless(vcpu, gpa, iterator, old_spte) {
3135 		sptep = iterator.sptep;
3136 		*spte = old_spte;
3137 	}
3138 
3139 	return sptep;
3140 }
3141 
3142 /*
3143  * Returns one of RET_PF_INVALID, RET_PF_FIXED or RET_PF_SPURIOUS.
3144  */
fast_page_fault(struct kvm_vcpu * vcpu,struct kvm_page_fault * fault)3145 static int fast_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
3146 {
3147 	struct kvm_mmu_page *sp;
3148 	int ret = RET_PF_INVALID;
3149 	u64 spte = 0ull;
3150 	u64 *sptep = NULL;
3151 	uint retry_count = 0;
3152 
3153 	if (!page_fault_can_be_fast(fault))
3154 		return ret;
3155 
3156 	walk_shadow_page_lockless_begin(vcpu);
3157 
3158 	do {
3159 		u64 new_spte;
3160 
3161 		if (is_tdp_mmu(vcpu->arch.mmu))
3162 			sptep = kvm_tdp_mmu_fast_pf_get_last_sptep(vcpu, fault->addr, &spte);
3163 		else
3164 			sptep = fast_pf_get_last_sptep(vcpu, fault->addr, &spte);
3165 
3166 		if (!is_shadow_present_pte(spte))
3167 			break;
3168 
3169 		sp = sptep_to_sp(sptep);
3170 		if (!is_last_spte(spte, sp->role.level))
3171 			break;
3172 
3173 		/*
3174 		 * Check whether the memory access that caused the fault would
3175 		 * still cause it if it were to be performed right now. If not,
3176 		 * then this is a spurious fault caused by TLB lazily flushed,
3177 		 * or some other CPU has already fixed the PTE after the
3178 		 * current CPU took the fault.
3179 		 *
3180 		 * Need not check the access of upper level table entries since
3181 		 * they are always ACC_ALL.
3182 		 */
3183 		if (is_access_allowed(fault, spte)) {
3184 			ret = RET_PF_SPURIOUS;
3185 			break;
3186 		}
3187 
3188 		new_spte = spte;
3189 
3190 		/*
3191 		 * KVM only supports fixing page faults outside of MMU lock for
3192 		 * direct MMUs, nested MMUs are always indirect, and KVM always
3193 		 * uses A/D bits for non-nested MMUs.  Thus, if A/D bits are
3194 		 * enabled, the SPTE can't be an access-tracked SPTE.
3195 		 */
3196 		if (unlikely(!kvm_ad_enabled()) && is_access_track_spte(spte))
3197 			new_spte = restore_acc_track_spte(new_spte);
3198 
3199 		/*
3200 		 * To keep things simple, only SPTEs that are MMU-writable can
3201 		 * be made fully writable outside of mmu_lock, e.g. only SPTEs
3202 		 * that were write-protected for dirty-logging or access
3203 		 * tracking are handled here.  Don't bother checking if the
3204 		 * SPTE is writable to prioritize running with A/D bits enabled.
3205 		 * The is_access_allowed() check above handles the common case
3206 		 * of the fault being spurious, and the SPTE is known to be
3207 		 * shadow-present, i.e. except for access tracking restoration
3208 		 * making the new SPTE writable, the check is wasteful.
3209 		 */
3210 		if (fault->write && is_mmu_writable_spte(spte)) {
3211 			new_spte |= PT_WRITABLE_MASK;
3212 
3213 			/*
3214 			 * Do not fix write-permission on the large spte when
3215 			 * dirty logging is enabled. Since we only dirty the
3216 			 * first page into the dirty-bitmap in
3217 			 * fast_pf_fix_direct_spte(), other pages are missed
3218 			 * if its slot has dirty logging enabled.
3219 			 *
3220 			 * Instead, we let the slow page fault path create a
3221 			 * normal spte to fix the access.
3222 			 */
3223 			if (sp->role.level > PG_LEVEL_4K &&
3224 			    kvm_slot_dirty_track_enabled(fault->slot))
3225 				break;
3226 		}
3227 
3228 		/* Verify that the fault can be handled in the fast path */
3229 		if (new_spte == spte ||
3230 		    !is_access_allowed(fault, new_spte))
3231 			break;
3232 
3233 		/*
3234 		 * Currently, fast page fault only works for direct mapping
3235 		 * since the gfn is not stable for indirect shadow page. See
3236 		 * Documentation/virt/kvm/locking.rst to get more detail.
3237 		 */
3238 		if (fast_pf_fix_direct_spte(vcpu, fault, sptep, spte, new_spte)) {
3239 			ret = RET_PF_FIXED;
3240 			break;
3241 		}
3242 
3243 		if (++retry_count > 4) {
3244 			printk_once(KERN_WARNING
3245 				"kvm: Fast #PF retrying more than 4 times.\n");
3246 			break;
3247 		}
3248 
3249 	} while (true);
3250 
3251 	trace_fast_page_fault(vcpu, fault, sptep, spte, ret);
3252 	walk_shadow_page_lockless_end(vcpu);
3253 
3254 	if (ret != RET_PF_INVALID)
3255 		vcpu->stat.pf_fast++;
3256 
3257 	return ret;
3258 }
3259 
mmu_free_root_page(struct kvm * kvm,hpa_t * root_hpa,struct list_head * invalid_list)3260 static void mmu_free_root_page(struct kvm *kvm, hpa_t *root_hpa,
3261 			       struct list_head *invalid_list)
3262 {
3263 	struct kvm_mmu_page *sp;
3264 
3265 	if (!VALID_PAGE(*root_hpa))
3266 		return;
3267 
3268 	sp = to_shadow_page(*root_hpa & PT64_BASE_ADDR_MASK);
3269 	if (WARN_ON(!sp))
3270 		return;
3271 
3272 	if (is_tdp_mmu_page(sp))
3273 		kvm_tdp_mmu_put_root(kvm, sp, false);
3274 	else if (!--sp->root_count && sp->role.invalid)
3275 		kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
3276 
3277 	*root_hpa = INVALID_PAGE;
3278 }
3279 
3280 /* roots_to_free must be some combination of the KVM_MMU_ROOT_* flags */
kvm_mmu_free_roots(struct kvm * kvm,struct kvm_mmu * mmu,ulong roots_to_free)3281 void kvm_mmu_free_roots(struct kvm *kvm, struct kvm_mmu *mmu,
3282 			ulong roots_to_free)
3283 {
3284 	int i;
3285 	LIST_HEAD(invalid_list);
3286 	bool free_active_root;
3287 
3288 	BUILD_BUG_ON(KVM_MMU_NUM_PREV_ROOTS >= BITS_PER_LONG);
3289 
3290 	/* Before acquiring the MMU lock, see if we need to do any real work. */
3291 	free_active_root = (roots_to_free & KVM_MMU_ROOT_CURRENT)
3292 		&& VALID_PAGE(mmu->root.hpa);
3293 
3294 	if (!free_active_root) {
3295 		for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
3296 			if ((roots_to_free & KVM_MMU_ROOT_PREVIOUS(i)) &&
3297 			    VALID_PAGE(mmu->prev_roots[i].hpa))
3298 				break;
3299 
3300 		if (i == KVM_MMU_NUM_PREV_ROOTS)
3301 			return;
3302 	}
3303 
3304 	write_lock(&kvm->mmu_lock);
3305 
3306 	for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
3307 		if (roots_to_free & KVM_MMU_ROOT_PREVIOUS(i))
3308 			mmu_free_root_page(kvm, &mmu->prev_roots[i].hpa,
3309 					   &invalid_list);
3310 
3311 	if (free_active_root) {
3312 		if (to_shadow_page(mmu->root.hpa)) {
3313 			mmu_free_root_page(kvm, &mmu->root.hpa, &invalid_list);
3314 		} else if (mmu->pae_root) {
3315 			for (i = 0; i < 4; ++i) {
3316 				if (!IS_VALID_PAE_ROOT(mmu->pae_root[i]))
3317 					continue;
3318 
3319 				mmu_free_root_page(kvm, &mmu->pae_root[i],
3320 						   &invalid_list);
3321 				mmu->pae_root[i] = INVALID_PAE_ROOT;
3322 			}
3323 		}
3324 		mmu->root.hpa = INVALID_PAGE;
3325 		mmu->root.pgd = 0;
3326 	}
3327 
3328 	kvm_mmu_commit_zap_page(kvm, &invalid_list);
3329 	write_unlock(&kvm->mmu_lock);
3330 }
3331 EXPORT_SYMBOL_GPL(kvm_mmu_free_roots);
3332 
kvm_mmu_free_guest_mode_roots(struct kvm * kvm,struct kvm_mmu * mmu)3333 void kvm_mmu_free_guest_mode_roots(struct kvm *kvm, struct kvm_mmu *mmu)
3334 {
3335 	unsigned long roots_to_free = 0;
3336 	hpa_t root_hpa;
3337 	int i;
3338 
3339 	/*
3340 	 * This should not be called while L2 is active, L2 can't invalidate
3341 	 * _only_ its own roots, e.g. INVVPID unconditionally exits.
3342 	 */
3343 	WARN_ON_ONCE(mmu->root_role.guest_mode);
3344 
3345 	for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) {
3346 		root_hpa = mmu->prev_roots[i].hpa;
3347 		if (!VALID_PAGE(root_hpa))
3348 			continue;
3349 
3350 		if (!to_shadow_page(root_hpa) ||
3351 			to_shadow_page(root_hpa)->role.guest_mode)
3352 			roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i);
3353 	}
3354 
3355 	kvm_mmu_free_roots(kvm, mmu, roots_to_free);
3356 }
3357 EXPORT_SYMBOL_GPL(kvm_mmu_free_guest_mode_roots);
3358 
3359 
mmu_check_root(struct kvm_vcpu * vcpu,gfn_t root_gfn)3360 static int mmu_check_root(struct kvm_vcpu *vcpu, gfn_t root_gfn)
3361 {
3362 	int ret = 0;
3363 
3364 	if (!kvm_vcpu_is_visible_gfn(vcpu, root_gfn)) {
3365 		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
3366 		ret = 1;
3367 	}
3368 
3369 	return ret;
3370 }
3371 
mmu_alloc_root(struct kvm_vcpu * vcpu,gfn_t gfn,gva_t gva,u8 level,bool direct)3372 static hpa_t mmu_alloc_root(struct kvm_vcpu *vcpu, gfn_t gfn, gva_t gva,
3373 			    u8 level, bool direct)
3374 {
3375 	struct kvm_mmu_page *sp;
3376 
3377 	sp = kvm_mmu_get_page(vcpu, gfn, gva, level, direct, ACC_ALL);
3378 	++sp->root_count;
3379 
3380 	return __pa(sp->spt);
3381 }
3382 
mmu_alloc_direct_roots(struct kvm_vcpu * vcpu)3383 static int mmu_alloc_direct_roots(struct kvm_vcpu *vcpu)
3384 {
3385 	struct kvm_mmu *mmu = vcpu->arch.mmu;
3386 	u8 shadow_root_level = mmu->root_role.level;
3387 	hpa_t root;
3388 	unsigned i;
3389 	int r;
3390 
3391 	write_lock(&vcpu->kvm->mmu_lock);
3392 	r = make_mmu_pages_available(vcpu);
3393 	if (r < 0)
3394 		goto out_unlock;
3395 
3396 	if (is_tdp_mmu_enabled(vcpu->kvm)) {
3397 		root = kvm_tdp_mmu_get_vcpu_root_hpa(vcpu);
3398 		mmu->root.hpa = root;
3399 	} else if (shadow_root_level >= PT64_ROOT_4LEVEL) {
3400 		root = mmu_alloc_root(vcpu, 0, 0, shadow_root_level, true);
3401 		mmu->root.hpa = root;
3402 	} else if (shadow_root_level == PT32E_ROOT_LEVEL) {
3403 		if (WARN_ON_ONCE(!mmu->pae_root)) {
3404 			r = -EIO;
3405 			goto out_unlock;
3406 		}
3407 
3408 		for (i = 0; i < 4; ++i) {
3409 			WARN_ON_ONCE(IS_VALID_PAE_ROOT(mmu->pae_root[i]));
3410 
3411 			root = mmu_alloc_root(vcpu, i << (30 - PAGE_SHIFT),
3412 					      i << 30, PT32_ROOT_LEVEL, true);
3413 			mmu->pae_root[i] = root | PT_PRESENT_MASK |
3414 					   shadow_me_value;
3415 		}
3416 		mmu->root.hpa = __pa(mmu->pae_root);
3417 	} else {
3418 		WARN_ONCE(1, "Bad TDP root level = %d\n", shadow_root_level);
3419 		r = -EIO;
3420 		goto out_unlock;
3421 	}
3422 
3423 	/* root.pgd is ignored for direct MMUs. */
3424 	mmu->root.pgd = 0;
3425 out_unlock:
3426 	write_unlock(&vcpu->kvm->mmu_lock);
3427 	return r;
3428 }
3429 
mmu_first_shadow_root_alloc(struct kvm * kvm)3430 static int mmu_first_shadow_root_alloc(struct kvm *kvm)
3431 {
3432 	struct kvm_memslots *slots;
3433 	struct kvm_memory_slot *slot;
3434 	int r = 0, i, bkt;
3435 
3436 	/*
3437 	 * Check if this is the first shadow root being allocated before
3438 	 * taking the lock.
3439 	 */
3440 	if (kvm_shadow_root_allocated(kvm))
3441 		return 0;
3442 
3443 	mutex_lock(&kvm->slots_arch_lock);
3444 
3445 	/* Recheck, under the lock, whether this is the first shadow root. */
3446 	if (kvm_shadow_root_allocated(kvm))
3447 		goto out_unlock;
3448 
3449 	/*
3450 	 * Check if anything actually needs to be allocated, e.g. all metadata
3451 	 * will be allocated upfront if TDP is disabled.
3452 	 */
3453 	if (kvm_memslots_have_rmaps(kvm) &&
3454 	    kvm_page_track_write_tracking_enabled(kvm))
3455 		goto out_success;
3456 
3457 	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
3458 		slots = __kvm_memslots(kvm, i);
3459 		kvm_for_each_memslot(slot, bkt, slots) {
3460 			/*
3461 			 * Both of these functions are no-ops if the target is
3462 			 * already allocated, so unconditionally calling both
3463 			 * is safe.  Intentionally do NOT free allocations on
3464 			 * failure to avoid having to track which allocations
3465 			 * were made now versus when the memslot was created.
3466 			 * The metadata is guaranteed to be freed when the slot
3467 			 * is freed, and will be kept/used if userspace retries
3468 			 * KVM_RUN instead of killing the VM.
3469 			 */
3470 			r = memslot_rmap_alloc(slot, slot->npages);
3471 			if (r)
3472 				goto out_unlock;
3473 			r = kvm_page_track_write_tracking_alloc(slot);
3474 			if (r)
3475 				goto out_unlock;
3476 		}
3477 	}
3478 
3479 	/*
3480 	 * Ensure that shadow_root_allocated becomes true strictly after
3481 	 * all the related pointers are set.
3482 	 */
3483 out_success:
3484 	smp_store_release(&kvm->arch.shadow_root_allocated, true);
3485 
3486 out_unlock:
3487 	mutex_unlock(&kvm->slots_arch_lock);
3488 	return r;
3489 }
3490 
mmu_alloc_shadow_roots(struct kvm_vcpu * vcpu)3491 static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
3492 {
3493 	struct kvm_mmu *mmu = vcpu->arch.mmu;
3494 	u64 pdptrs[4], pm_mask;
3495 	gfn_t root_gfn, root_pgd;
3496 	hpa_t root;
3497 	unsigned i;
3498 	int r;
3499 
3500 	root_pgd = mmu->get_guest_pgd(vcpu);
3501 	root_gfn = root_pgd >> PAGE_SHIFT;
3502 
3503 	if (mmu_check_root(vcpu, root_gfn))
3504 		return 1;
3505 
3506 	/*
3507 	 * On SVM, reading PDPTRs might access guest memory, which might fault
3508 	 * and thus might sleep.  Grab the PDPTRs before acquiring mmu_lock.
3509 	 */
3510 	if (mmu->cpu_role.base.level == PT32E_ROOT_LEVEL) {
3511 		for (i = 0; i < 4; ++i) {
3512 			pdptrs[i] = mmu->get_pdptr(vcpu, i);
3513 			if (!(pdptrs[i] & PT_PRESENT_MASK))
3514 				continue;
3515 
3516 			if (mmu_check_root(vcpu, pdptrs[i] >> PAGE_SHIFT))
3517 				return 1;
3518 		}
3519 	}
3520 
3521 	r = mmu_first_shadow_root_alloc(vcpu->kvm);
3522 	if (r)
3523 		return r;
3524 
3525 	write_lock(&vcpu->kvm->mmu_lock);
3526 	r = make_mmu_pages_available(vcpu);
3527 	if (r < 0)
3528 		goto out_unlock;
3529 
3530 	/*
3531 	 * Do we shadow a long mode page table? If so we need to
3532 	 * write-protect the guests page table root.
3533 	 */
3534 	if (mmu->cpu_role.base.level >= PT64_ROOT_4LEVEL) {
3535 		root = mmu_alloc_root(vcpu, root_gfn, 0,
3536 				      mmu->root_role.level, false);
3537 		mmu->root.hpa = root;
3538 		goto set_root_pgd;
3539 	}
3540 
3541 	if (WARN_ON_ONCE(!mmu->pae_root)) {
3542 		r = -EIO;
3543 		goto out_unlock;
3544 	}
3545 
3546 	/*
3547 	 * We shadow a 32 bit page table. This may be a legacy 2-level
3548 	 * or a PAE 3-level page table. In either case we need to be aware that
3549 	 * the shadow page table may be a PAE or a long mode page table.
3550 	 */
3551 	pm_mask = PT_PRESENT_MASK | shadow_me_value;
3552 	if (mmu->root_role.level >= PT64_ROOT_4LEVEL) {
3553 		pm_mask |= PT_ACCESSED_MASK | PT_WRITABLE_MASK | PT_USER_MASK;
3554 
3555 		if (WARN_ON_ONCE(!mmu->pml4_root)) {
3556 			r = -EIO;
3557 			goto out_unlock;
3558 		}
3559 		mmu->pml4_root[0] = __pa(mmu->pae_root) | pm_mask;
3560 
3561 		if (mmu->root_role.level == PT64_ROOT_5LEVEL) {
3562 			if (WARN_ON_ONCE(!mmu->pml5_root)) {
3563 				r = -EIO;
3564 				goto out_unlock;
3565 			}
3566 			mmu->pml5_root[0] = __pa(mmu->pml4_root) | pm_mask;
3567 		}
3568 	}
3569 
3570 	for (i = 0; i < 4; ++i) {
3571 		WARN_ON_ONCE(IS_VALID_PAE_ROOT(mmu->pae_root[i]));
3572 
3573 		if (mmu->cpu_role.base.level == PT32E_ROOT_LEVEL) {
3574 			if (!(pdptrs[i] & PT_PRESENT_MASK)) {
3575 				mmu->pae_root[i] = INVALID_PAE_ROOT;
3576 				continue;
3577 			}
3578 			root_gfn = pdptrs[i] >> PAGE_SHIFT;
3579 		}
3580 
3581 		root = mmu_alloc_root(vcpu, root_gfn, i << 30,
3582 				      PT32_ROOT_LEVEL, false);
3583 		mmu->pae_root[i] = root | pm_mask;
3584 	}
3585 
3586 	if (mmu->root_role.level == PT64_ROOT_5LEVEL)
3587 		mmu->root.hpa = __pa(mmu->pml5_root);
3588 	else if (mmu->root_role.level == PT64_ROOT_4LEVEL)
3589 		mmu->root.hpa = __pa(mmu->pml4_root);
3590 	else
3591 		mmu->root.hpa = __pa(mmu->pae_root);
3592 
3593 set_root_pgd:
3594 	mmu->root.pgd = root_pgd;
3595 out_unlock:
3596 	write_unlock(&vcpu->kvm->mmu_lock);
3597 
3598 	return r;
3599 }
3600 
mmu_alloc_special_roots(struct kvm_vcpu * vcpu)3601 static int mmu_alloc_special_roots(struct kvm_vcpu *vcpu)
3602 {
3603 	struct kvm_mmu *mmu = vcpu->arch.mmu;
3604 	bool need_pml5 = mmu->root_role.level > PT64_ROOT_4LEVEL;
3605 	u64 *pml5_root = NULL;
3606 	u64 *pml4_root = NULL;
3607 	u64 *pae_root;
3608 
3609 	/*
3610 	 * When shadowing 32-bit or PAE NPT with 64-bit NPT, the PML4 and PDP
3611 	 * tables are allocated and initialized at root creation as there is no
3612 	 * equivalent level in the guest's NPT to shadow.  Allocate the tables
3613 	 * on demand, as running a 32-bit L1 VMM on 64-bit KVM is very rare.
3614 	 */
3615 	if (mmu->root_role.direct ||
3616 	    mmu->cpu_role.base.level >= PT64_ROOT_4LEVEL ||
3617 	    mmu->root_role.level < PT64_ROOT_4LEVEL)
3618 		return 0;
3619 
3620 	/*
3621 	 * NPT, the only paging mode that uses this horror, uses a fixed number
3622 	 * of levels for the shadow page tables, e.g. all MMUs are 4-level or
3623 	 * all MMus are 5-level.  Thus, this can safely require that pml5_root
3624 	 * is allocated if the other roots are valid and pml5 is needed, as any
3625 	 * prior MMU would also have required pml5.
3626 	 */
3627 	if (mmu->pae_root && mmu->pml4_root && (!need_pml5 || mmu->pml5_root))
3628 		return 0;
3629 
3630 	/*
3631 	 * The special roots should always be allocated in concert.  Yell and
3632 	 * bail if KVM ends up in a state where only one of the roots is valid.
3633 	 */
3634 	if (WARN_ON_ONCE(!tdp_enabled || mmu->pae_root || mmu->pml4_root ||
3635 			 (need_pml5 && mmu->pml5_root)))
3636 		return -EIO;
3637 
3638 	/*
3639 	 * Unlike 32-bit NPT, the PDP table doesn't need to be in low mem, and
3640 	 * doesn't need to be decrypted.
3641 	 */
3642 	pae_root = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT);
3643 	if (!pae_root)
3644 		return -ENOMEM;
3645 
3646 #ifdef CONFIG_X86_64
3647 	pml4_root = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT);
3648 	if (!pml4_root)
3649 		goto err_pml4;
3650 
3651 	if (need_pml5) {
3652 		pml5_root = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT);
3653 		if (!pml5_root)
3654 			goto err_pml5;
3655 	}
3656 #endif
3657 
3658 	mmu->pae_root = pae_root;
3659 	mmu->pml4_root = pml4_root;
3660 	mmu->pml5_root = pml5_root;
3661 
3662 	return 0;
3663 
3664 #ifdef CONFIG_X86_64
3665 err_pml5:
3666 	free_page((unsigned long)pml4_root);
3667 err_pml4:
3668 	free_page((unsigned long)pae_root);
3669 	return -ENOMEM;
3670 #endif
3671 }
3672 
is_unsync_root(hpa_t root)3673 static bool is_unsync_root(hpa_t root)
3674 {
3675 	struct kvm_mmu_page *sp;
3676 
3677 	if (!VALID_PAGE(root))
3678 		return false;
3679 
3680 	/*
3681 	 * The read barrier orders the CPU's read of SPTE.W during the page table
3682 	 * walk before the reads of sp->unsync/sp->unsync_children here.
3683 	 *
3684 	 * Even if another CPU was marking the SP as unsync-ed simultaneously,
3685 	 * any guest page table changes are not guaranteed to be visible anyway
3686 	 * until this VCPU issues a TLB flush strictly after those changes are
3687 	 * made.  We only need to ensure that the other CPU sets these flags
3688 	 * before any actual changes to the page tables are made.  The comments
3689 	 * in mmu_try_to_unsync_pages() describe what could go wrong if this
3690 	 * requirement isn't satisfied.
3691 	 */
3692 	smp_rmb();
3693 	sp = to_shadow_page(root);
3694 
3695 	/*
3696 	 * PAE roots (somewhat arbitrarily) aren't backed by shadow pages, the
3697 	 * PDPTEs for a given PAE root need to be synchronized individually.
3698 	 */
3699 	if (WARN_ON_ONCE(!sp))
3700 		return false;
3701 
3702 	if (sp->unsync || sp->unsync_children)
3703 		return true;
3704 
3705 	return false;
3706 }
3707 
kvm_mmu_sync_roots(struct kvm_vcpu * vcpu)3708 void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu)
3709 {
3710 	int i;
3711 	struct kvm_mmu_page *sp;
3712 
3713 	if (vcpu->arch.mmu->root_role.direct)
3714 		return;
3715 
3716 	if (!VALID_PAGE(vcpu->arch.mmu->root.hpa))
3717 		return;
3718 
3719 	vcpu_clear_mmio_info(vcpu, MMIO_GVA_ANY);
3720 
3721 	if (vcpu->arch.mmu->cpu_role.base.level >= PT64_ROOT_4LEVEL) {
3722 		hpa_t root = vcpu->arch.mmu->root.hpa;
3723 		sp = to_shadow_page(root);
3724 
3725 		if (!is_unsync_root(root))
3726 			return;
3727 
3728 		write_lock(&vcpu->kvm->mmu_lock);
3729 		mmu_sync_children(vcpu, sp, true);
3730 		write_unlock(&vcpu->kvm->mmu_lock);
3731 		return;
3732 	}
3733 
3734 	write_lock(&vcpu->kvm->mmu_lock);
3735 
3736 	for (i = 0; i < 4; ++i) {
3737 		hpa_t root = vcpu->arch.mmu->pae_root[i];
3738 
3739 		if (IS_VALID_PAE_ROOT(root)) {
3740 			root &= PT64_BASE_ADDR_MASK;
3741 			sp = to_shadow_page(root);
3742 			mmu_sync_children(vcpu, sp, true);
3743 		}
3744 	}
3745 
3746 	write_unlock(&vcpu->kvm->mmu_lock);
3747 }
3748 
kvm_mmu_sync_prev_roots(struct kvm_vcpu * vcpu)3749 void kvm_mmu_sync_prev_roots(struct kvm_vcpu *vcpu)
3750 {
3751 	unsigned long roots_to_free = 0;
3752 	int i;
3753 
3754 	for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
3755 		if (is_unsync_root(vcpu->arch.mmu->prev_roots[i].hpa))
3756 			roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i);
3757 
3758 	/* sync prev_roots by simply freeing them */
3759 	kvm_mmu_free_roots(vcpu->kvm, vcpu->arch.mmu, roots_to_free);
3760 }
3761 
nonpaging_gva_to_gpa(struct kvm_vcpu * vcpu,struct kvm_mmu * mmu,gpa_t vaddr,u64 access,struct x86_exception * exception)3762 static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
3763 				  gpa_t vaddr, u64 access,
3764 				  struct x86_exception *exception)
3765 {
3766 	if (exception)
3767 		exception->error_code = 0;
3768 	return kvm_translate_gpa(vcpu, mmu, vaddr, access, exception);
3769 }
3770 
mmio_info_in_cache(struct kvm_vcpu * vcpu,u64 addr,bool direct)3771 static bool mmio_info_in_cache(struct kvm_vcpu *vcpu, u64 addr, bool direct)
3772 {
3773 	/*
3774 	 * A nested guest cannot use the MMIO cache if it is using nested
3775 	 * page tables, because cr2 is a nGPA while the cache stores GPAs.
3776 	 */
3777 	if (mmu_is_nested(vcpu))
3778 		return false;
3779 
3780 	if (direct)
3781 		return vcpu_match_mmio_gpa(vcpu, addr);
3782 
3783 	return vcpu_match_mmio_gva(vcpu, addr);
3784 }
3785 
3786 /*
3787  * Return the level of the lowest level SPTE added to sptes.
3788  * That SPTE may be non-present.
3789  *
3790  * Must be called between walk_shadow_page_lockless_{begin,end}.
3791  */
get_walk(struct kvm_vcpu * vcpu,u64 addr,u64 * sptes,int * root_level)3792 static int get_walk(struct kvm_vcpu *vcpu, u64 addr, u64 *sptes, int *root_level)
3793 {
3794 	struct kvm_shadow_walk_iterator iterator;
3795 	int leaf = -1;
3796 	u64 spte;
3797 
3798 	for (shadow_walk_init(&iterator, vcpu, addr),
3799 	     *root_level = iterator.level;
3800 	     shadow_walk_okay(&iterator);
3801 	     __shadow_walk_next(&iterator, spte)) {
3802 		leaf = iterator.level;
3803 		spte = mmu_spte_get_lockless(iterator.sptep);
3804 
3805 		sptes[leaf] = spte;
3806 	}
3807 
3808 	return leaf;
3809 }
3810 
3811 /* return true if reserved bit(s) are detected on a valid, non-MMIO SPTE. */
get_mmio_spte(struct kvm_vcpu * vcpu,u64 addr,u64 * sptep)3812 static bool get_mmio_spte(struct kvm_vcpu *vcpu, u64 addr, u64 *sptep)
3813 {
3814 	u64 sptes[PT64_ROOT_MAX_LEVEL + 1];
3815 	struct rsvd_bits_validate *rsvd_check;
3816 	int root, leaf, level;
3817 	bool reserved = false;
3818 
3819 	walk_shadow_page_lockless_begin(vcpu);
3820 
3821 	if (is_tdp_mmu(vcpu->arch.mmu))
3822 		leaf = kvm_tdp_mmu_get_walk(vcpu, addr, sptes, &root);
3823 	else
3824 		leaf = get_walk(vcpu, addr, sptes, &root);
3825 
3826 	walk_shadow_page_lockless_end(vcpu);
3827 
3828 	if (unlikely(leaf < 0)) {
3829 		*sptep = 0ull;
3830 		return reserved;
3831 	}
3832 
3833 	*sptep = sptes[leaf];
3834 
3835 	/*
3836 	 * Skip reserved bits checks on the terminal leaf if it's not a valid
3837 	 * SPTE.  Note, this also (intentionally) skips MMIO SPTEs, which, by
3838 	 * design, always have reserved bits set.  The purpose of the checks is
3839 	 * to detect reserved bits on non-MMIO SPTEs. i.e. buggy SPTEs.
3840 	 */
3841 	if (!is_shadow_present_pte(sptes[leaf]))
3842 		leaf++;
3843 
3844 	rsvd_check = &vcpu->arch.mmu->shadow_zero_check;
3845 
3846 	for (level = root; level >= leaf; level--)
3847 		reserved |= is_rsvd_spte(rsvd_check, sptes[level], level);
3848 
3849 	if (reserved) {
3850 		pr_err("%s: reserved bits set on MMU-present spte, addr 0x%llx, hierarchy:\n",
3851 		       __func__, addr);
3852 		for (level = root; level >= leaf; level--)
3853 			pr_err("------ spte = 0x%llx level = %d, rsvd bits = 0x%llx",
3854 			       sptes[level], level,
3855 			       get_rsvd_bits(rsvd_check, sptes[level], level));
3856 	}
3857 
3858 	return reserved;
3859 }
3860 
handle_mmio_page_fault(struct kvm_vcpu * vcpu,u64 addr,bool direct)3861 static int handle_mmio_page_fault(struct kvm_vcpu *vcpu, u64 addr, bool direct)
3862 {
3863 	u64 spte;
3864 	bool reserved;
3865 
3866 	if (mmio_info_in_cache(vcpu, addr, direct))
3867 		return RET_PF_EMULATE;
3868 
3869 	reserved = get_mmio_spte(vcpu, addr, &spte);
3870 	if (WARN_ON(reserved))
3871 		return -EINVAL;
3872 
3873 	if (is_mmio_spte(spte)) {
3874 		gfn_t gfn = get_mmio_spte_gfn(spte);
3875 		unsigned int access = get_mmio_spte_access(spte);
3876 
3877 		if (!check_mmio_spte(vcpu, spte))
3878 			return RET_PF_INVALID;
3879 
3880 		if (direct)
3881 			addr = 0;
3882 
3883 		trace_handle_mmio_page_fault(addr, gfn, access);
3884 		vcpu_cache_mmio_info(vcpu, addr, gfn, access);
3885 		return RET_PF_EMULATE;
3886 	}
3887 
3888 	/*
3889 	 * If the page table is zapped by other cpus, let CPU fault again on
3890 	 * the address.
3891 	 */
3892 	return RET_PF_RETRY;
3893 }
3894 
page_fault_handle_page_track(struct kvm_vcpu * vcpu,struct kvm_page_fault * fault)3895 static bool page_fault_handle_page_track(struct kvm_vcpu *vcpu,
3896 					 struct kvm_page_fault *fault)
3897 {
3898 	if (unlikely(fault->rsvd))
3899 		return false;
3900 
3901 	if (!fault->present || !fault->write)
3902 		return false;
3903 
3904 	/*
3905 	 * guest is writing the page which is write tracked which can
3906 	 * not be fixed by page fault handler.
3907 	 */
3908 	if (kvm_slot_page_track_is_active(vcpu->kvm, fault->slot, fault->gfn, KVM_PAGE_TRACK_WRITE))
3909 		return true;
3910 
3911 	return false;
3912 }
3913 
shadow_page_table_clear_flood(struct kvm_vcpu * vcpu,gva_t addr)3914 static void shadow_page_table_clear_flood(struct kvm_vcpu *vcpu, gva_t addr)
3915 {
3916 	struct kvm_shadow_walk_iterator iterator;
3917 	u64 spte;
3918 
3919 	walk_shadow_page_lockless_begin(vcpu);
3920 	for_each_shadow_entry_lockless(vcpu, addr, iterator, spte)
3921 		clear_sp_write_flooding_count(iterator.sptep);
3922 	walk_shadow_page_lockless_end(vcpu);
3923 }
3924 
alloc_apf_token(struct kvm_vcpu * vcpu)3925 static u32 alloc_apf_token(struct kvm_vcpu *vcpu)
3926 {
3927 	/* make sure the token value is not 0 */
3928 	u32 id = vcpu->arch.apf.id;
3929 
3930 	if (id << 12 == 0)
3931 		vcpu->arch.apf.id = 1;
3932 
3933 	return (vcpu->arch.apf.id++ << 12) | vcpu->vcpu_id;
3934 }
3935 
kvm_arch_setup_async_pf(struct kvm_vcpu * vcpu,gpa_t cr2_or_gpa,gfn_t gfn)3936 static bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
3937 				    gfn_t gfn)
3938 {
3939 	struct kvm_arch_async_pf arch;
3940 
3941 	arch.token = alloc_apf_token(vcpu);
3942 	arch.gfn = gfn;
3943 	arch.direct_map = vcpu->arch.mmu->root_role.direct;
3944 	arch.cr3 = vcpu->arch.mmu->get_guest_pgd(vcpu);
3945 
3946 	return kvm_setup_async_pf(vcpu, cr2_or_gpa,
3947 				  kvm_vcpu_gfn_to_hva(vcpu, gfn), &arch);
3948 }
3949 
kvm_arch_async_page_ready(struct kvm_vcpu * vcpu,struct kvm_async_pf * work)3950 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work)
3951 {
3952 	int r;
3953 
3954 	if ((vcpu->arch.mmu->root_role.direct != work->arch.direct_map) ||
3955 	      work->wakeup_all)
3956 		return;
3957 
3958 	r = kvm_mmu_reload(vcpu);
3959 	if (unlikely(r))
3960 		return;
3961 
3962 	if (!vcpu->arch.mmu->root_role.direct &&
3963 	      work->arch.cr3 != vcpu->arch.mmu->get_guest_pgd(vcpu))
3964 		return;
3965 
3966 	kvm_mmu_do_page_fault(vcpu, work->cr2_or_gpa, 0, true);
3967 }
3968 
kvm_faultin_pfn(struct kvm_vcpu * vcpu,struct kvm_page_fault * fault)3969 static int kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
3970 {
3971 	struct kvm_memory_slot *slot = fault->slot;
3972 	bool async;
3973 
3974 	/*
3975 	 * Retry the page fault if the gfn hit a memslot that is being deleted
3976 	 * or moved.  This ensures any existing SPTEs for the old memslot will
3977 	 * be zapped before KVM inserts a new MMIO SPTE for the gfn.
3978 	 */
3979 	if (slot && (slot->flags & KVM_MEMSLOT_INVALID))
3980 		return RET_PF_RETRY;
3981 
3982 	if (!kvm_is_visible_memslot(slot)) {
3983 		/* Don't expose private memslots to L2. */
3984 		if (is_guest_mode(vcpu)) {
3985 			fault->slot = NULL;
3986 			fault->pfn = KVM_PFN_NOSLOT;
3987 			fault->map_writable = false;
3988 			return RET_PF_CONTINUE;
3989 		}
3990 		/*
3991 		 * If the APIC access page exists but is disabled, go directly
3992 		 * to emulation without caching the MMIO access or creating a
3993 		 * MMIO SPTE.  That way the cache doesn't need to be purged
3994 		 * when the AVIC is re-enabled.
3995 		 */
3996 		if (slot && slot->id == APIC_ACCESS_PAGE_PRIVATE_MEMSLOT &&
3997 		    !kvm_apicv_activated(vcpu->kvm))
3998 			return RET_PF_EMULATE;
3999 	}
4000 
4001 	async = false;
4002 	fault->pfn = __gfn_to_pfn_memslot(slot, fault->gfn, false, &async,
4003 					  fault->write, &fault->map_writable,
4004 					  &fault->hva);
4005 	if (!async)
4006 		return RET_PF_CONTINUE; /* *pfn has correct page already */
4007 
4008 	if (!fault->prefetch && kvm_can_do_async_pf(vcpu)) {
4009 		trace_kvm_try_async_get_page(fault->addr, fault->gfn);
4010 		if (kvm_find_async_pf_gfn(vcpu, fault->gfn)) {
4011 			trace_kvm_async_pf_doublefault(fault->addr, fault->gfn);
4012 			kvm_make_request(KVM_REQ_APF_HALT, vcpu);
4013 			return RET_PF_RETRY;
4014 		} else if (kvm_arch_setup_async_pf(vcpu, fault->addr, fault->gfn)) {
4015 			return RET_PF_RETRY;
4016 		}
4017 	}
4018 
4019 	fault->pfn = __gfn_to_pfn_memslot(slot, fault->gfn, false, NULL,
4020 					  fault->write, &fault->map_writable,
4021 					  &fault->hva);
4022 	return RET_PF_CONTINUE;
4023 }
4024 
4025 /*
4026  * Returns true if the page fault is stale and needs to be retried, i.e. if the
4027  * root was invalidated by a memslot update or a relevant mmu_notifier fired.
4028  */
is_page_fault_stale(struct kvm_vcpu * vcpu,struct kvm_page_fault * fault,int mmu_seq)4029 static bool is_page_fault_stale(struct kvm_vcpu *vcpu,
4030 				struct kvm_page_fault *fault, int mmu_seq)
4031 {
4032 	struct kvm_mmu_page *sp = to_shadow_page(vcpu->arch.mmu->root.hpa);
4033 
4034 	/* Special roots, e.g. pae_root, are not backed by shadow pages. */
4035 	if (sp && is_obsolete_sp(vcpu->kvm, sp))
4036 		return true;
4037 
4038 	/*
4039 	 * Roots without an associated shadow page are considered invalid if
4040 	 * there is a pending request to free obsolete roots.  The request is
4041 	 * only a hint that the current root _may_ be obsolete and needs to be
4042 	 * reloaded, e.g. if the guest frees a PGD that KVM is tracking as a
4043 	 * previous root, then __kvm_mmu_prepare_zap_page() signals all vCPUs
4044 	 * to reload even if no vCPU is actively using the root.
4045 	 */
4046 	if (!sp && kvm_test_request(KVM_REQ_MMU_FREE_OBSOLETE_ROOTS, vcpu))
4047 		return true;
4048 
4049 	return fault->slot &&
4050 	       mmu_notifier_retry_hva(vcpu->kvm, mmu_seq, fault->hva);
4051 }
4052 
direct_page_fault(struct kvm_vcpu * vcpu,struct kvm_page_fault * fault)4053 static int direct_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
4054 {
4055 	bool is_tdp_mmu_fault = is_tdp_mmu(vcpu->arch.mmu);
4056 
4057 	unsigned long mmu_seq;
4058 	int r;
4059 
4060 	fault->gfn = fault->addr >> PAGE_SHIFT;
4061 	fault->slot = kvm_vcpu_gfn_to_memslot(vcpu, fault->gfn);
4062 
4063 	if (page_fault_handle_page_track(vcpu, fault))
4064 		return RET_PF_EMULATE;
4065 
4066 	r = fast_page_fault(vcpu, fault);
4067 	if (r != RET_PF_INVALID)
4068 		return r;
4069 
4070 	r = mmu_topup_memory_caches(vcpu, false);
4071 	if (r)
4072 		return r;
4073 
4074 	mmu_seq = vcpu->kvm->mmu_notifier_seq;
4075 	smp_rmb();
4076 
4077 	r = kvm_faultin_pfn(vcpu, fault);
4078 	if (r != RET_PF_CONTINUE)
4079 		return r;
4080 
4081 	r = handle_abnormal_pfn(vcpu, fault, ACC_ALL);
4082 	if (r != RET_PF_CONTINUE)
4083 		return r;
4084 
4085 	r = RET_PF_RETRY;
4086 
4087 	if (is_tdp_mmu_fault)
4088 		read_lock(&vcpu->kvm->mmu_lock);
4089 	else
4090 		write_lock(&vcpu->kvm->mmu_lock);
4091 
4092 	if (is_page_fault_stale(vcpu, fault, mmu_seq))
4093 		goto out_unlock;
4094 
4095 	r = make_mmu_pages_available(vcpu);
4096 	if (r)
4097 		goto out_unlock;
4098 
4099 	if (is_tdp_mmu_fault)
4100 		r = kvm_tdp_mmu_map(vcpu, fault);
4101 	else
4102 		r = __direct_map(vcpu, fault);
4103 
4104 out_unlock:
4105 	if (is_tdp_mmu_fault)
4106 		read_unlock(&vcpu->kvm->mmu_lock);
4107 	else
4108 		write_unlock(&vcpu->kvm->mmu_lock);
4109 	kvm_release_pfn_clean(fault->pfn);
4110 	return r;
4111 }
4112 
nonpaging_page_fault(struct kvm_vcpu * vcpu,struct kvm_page_fault * fault)4113 static int nonpaging_page_fault(struct kvm_vcpu *vcpu,
4114 				struct kvm_page_fault *fault)
4115 {
4116 	pgprintk("%s: gva %lx error %x\n", __func__, fault->addr, fault->error_code);
4117 
4118 	/* This path builds a PAE pagetable, we can map 2mb pages at maximum. */
4119 	fault->max_level = PG_LEVEL_2M;
4120 	return direct_page_fault(vcpu, fault);
4121 }
4122 
kvm_handle_page_fault(struct kvm_vcpu * vcpu,u64 error_code,u64 fault_address,char * insn,int insn_len)4123 int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code,
4124 				u64 fault_address, char *insn, int insn_len)
4125 {
4126 	int r = 1;
4127 	u32 flags = vcpu->arch.apf.host_apf_flags;
4128 
4129 #ifndef CONFIG_X86_64
4130 	/* A 64-bit CR2 should be impossible on 32-bit KVM. */
4131 	if (WARN_ON_ONCE(fault_address >> 32))
4132 		return -EFAULT;
4133 #endif
4134 
4135 	vcpu->arch.l1tf_flush_l1d = true;
4136 	if (!flags) {
4137 		trace_kvm_page_fault(fault_address, error_code);
4138 
4139 		if (kvm_event_needs_reinjection(vcpu))
4140 			kvm_mmu_unprotect_page_virt(vcpu, fault_address);
4141 		r = kvm_mmu_page_fault(vcpu, fault_address, error_code, insn,
4142 				insn_len);
4143 	} else if (flags & KVM_PV_REASON_PAGE_NOT_PRESENT) {
4144 		vcpu->arch.apf.host_apf_flags = 0;
4145 		local_irq_disable();
4146 		kvm_async_pf_task_wait_schedule(fault_address);
4147 		local_irq_enable();
4148 	} else {
4149 		WARN_ONCE(1, "Unexpected host async PF flags: %x\n", flags);
4150 	}
4151 
4152 	return r;
4153 }
4154 EXPORT_SYMBOL_GPL(kvm_handle_page_fault);
4155 
kvm_tdp_page_fault(struct kvm_vcpu * vcpu,struct kvm_page_fault * fault)4156 int kvm_tdp_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
4157 {
4158 	while (fault->max_level > PG_LEVEL_4K) {
4159 		int page_num = KVM_PAGES_PER_HPAGE(fault->max_level);
4160 		gfn_t base = (fault->addr >> PAGE_SHIFT) & ~(page_num - 1);
4161 
4162 		if (kvm_mtrr_check_gfn_range_consistency(vcpu, base, page_num))
4163 			break;
4164 
4165 		--fault->max_level;
4166 	}
4167 
4168 	return direct_page_fault(vcpu, fault);
4169 }
4170 
nonpaging_init_context(struct kvm_mmu * context)4171 static void nonpaging_init_context(struct kvm_mmu *context)
4172 {
4173 	context->page_fault = nonpaging_page_fault;
4174 	context->gva_to_gpa = nonpaging_gva_to_gpa;
4175 	context->sync_page = nonpaging_sync_page;
4176 	context->invlpg = NULL;
4177 }
4178 
is_root_usable(struct kvm_mmu_root_info * root,gpa_t pgd,union kvm_mmu_page_role role)4179 static inline bool is_root_usable(struct kvm_mmu_root_info *root, gpa_t pgd,
4180 				  union kvm_mmu_page_role role)
4181 {
4182 	return (role.direct || pgd == root->pgd) &&
4183 	       VALID_PAGE(root->hpa) &&
4184 	       role.word == to_shadow_page(root->hpa)->role.word;
4185 }
4186 
4187 /*
4188  * Find out if a previously cached root matching the new pgd/role is available,
4189  * and insert the current root as the MRU in the cache.
4190  * If a matching root is found, it is assigned to kvm_mmu->root and
4191  * true is returned.
4192  * If no match is found, kvm_mmu->root is left invalid, the LRU root is
4193  * evicted to make room for the current root, and false is returned.
4194  */
cached_root_find_and_keep_current(struct kvm * kvm,struct kvm_mmu * mmu,gpa_t new_pgd,union kvm_mmu_page_role new_role)4195 static bool cached_root_find_and_keep_current(struct kvm *kvm, struct kvm_mmu *mmu,
4196 					      gpa_t new_pgd,
4197 					      union kvm_mmu_page_role new_role)
4198 {
4199 	uint i;
4200 
4201 	if (is_root_usable(&mmu->root, new_pgd, new_role))
4202 		return true;
4203 
4204 	for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) {
4205 		/*
4206 		 * The swaps end up rotating the cache like this:
4207 		 *   C   0 1 2 3   (on entry to the function)
4208 		 *   0   C 1 2 3
4209 		 *   1   C 0 2 3
4210 		 *   2   C 0 1 3
4211 		 *   3   C 0 1 2   (on exit from the loop)
4212 		 */
4213 		swap(mmu->root, mmu->prev_roots[i]);
4214 		if (is_root_usable(&mmu->root, new_pgd, new_role))
4215 			return true;
4216 	}
4217 
4218 	kvm_mmu_free_roots(kvm, mmu, KVM_MMU_ROOT_CURRENT);
4219 	return false;
4220 }
4221 
4222 /*
4223  * Find out if a previously cached root matching the new pgd/role is available.
4224  * On entry, mmu->root is invalid.
4225  * If a matching root is found, it is assigned to kvm_mmu->root, the LRU entry
4226  * of the cache becomes invalid, and true is returned.
4227  * If no match is found, kvm_mmu->root is left invalid and false is returned.
4228  */
cached_root_find_without_current(struct kvm * kvm,struct kvm_mmu * mmu,gpa_t new_pgd,union kvm_mmu_page_role new_role)4229 static bool cached_root_find_without_current(struct kvm *kvm, struct kvm_mmu *mmu,
4230 					     gpa_t new_pgd,
4231 					     union kvm_mmu_page_role new_role)
4232 {
4233 	uint i;
4234 
4235 	for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
4236 		if (is_root_usable(&mmu->prev_roots[i], new_pgd, new_role))
4237 			goto hit;
4238 
4239 	return false;
4240 
4241 hit:
4242 	swap(mmu->root, mmu->prev_roots[i]);
4243 	/* Bubble up the remaining roots.  */
4244 	for (; i < KVM_MMU_NUM_PREV_ROOTS - 1; i++)
4245 		mmu->prev_roots[i] = mmu->prev_roots[i + 1];
4246 	mmu->prev_roots[i].hpa = INVALID_PAGE;
4247 	return true;
4248 }
4249 
fast_pgd_switch(struct kvm * kvm,struct kvm_mmu * mmu,gpa_t new_pgd,union kvm_mmu_page_role new_role)4250 static bool fast_pgd_switch(struct kvm *kvm, struct kvm_mmu *mmu,
4251 			    gpa_t new_pgd, union kvm_mmu_page_role new_role)
4252 {
4253 	/*
4254 	 * For now, limit the caching to 64-bit hosts+VMs in order to avoid
4255 	 * having to deal with PDPTEs. We may add support for 32-bit hosts/VMs
4256 	 * later if necessary.
4257 	 */
4258 	if (VALID_PAGE(mmu->root.hpa) && !to_shadow_page(mmu->root.hpa))
4259 		kvm_mmu_free_roots(kvm, mmu, KVM_MMU_ROOT_CURRENT);
4260 
4261 	if (VALID_PAGE(mmu->root.hpa))
4262 		return cached_root_find_and_keep_current(kvm, mmu, new_pgd, new_role);
4263 	else
4264 		return cached_root_find_without_current(kvm, mmu, new_pgd, new_role);
4265 }
4266 
kvm_mmu_new_pgd(struct kvm_vcpu * vcpu,gpa_t new_pgd)4267 void kvm_mmu_new_pgd(struct kvm_vcpu *vcpu, gpa_t new_pgd)
4268 {
4269 	struct kvm_mmu *mmu = vcpu->arch.mmu;
4270 	union kvm_mmu_page_role new_role = mmu->root_role;
4271 
4272 	if (!fast_pgd_switch(vcpu->kvm, mmu, new_pgd, new_role)) {
4273 		/* kvm_mmu_ensure_valid_pgd will set up a new root.  */
4274 		return;
4275 	}
4276 
4277 	/*
4278 	 * It's possible that the cached previous root page is obsolete because
4279 	 * of a change in the MMU generation number. However, changing the
4280 	 * generation number is accompanied by KVM_REQ_MMU_FREE_OBSOLETE_ROOTS,
4281 	 * which will free the root set here and allocate a new one.
4282 	 */
4283 	kvm_make_request(KVM_REQ_LOAD_MMU_PGD, vcpu);
4284 
4285 	if (force_flush_and_sync_on_reuse) {
4286 		kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);
4287 		kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
4288 	}
4289 
4290 	/*
4291 	 * The last MMIO access's GVA and GPA are cached in the VCPU. When
4292 	 * switching to a new CR3, that GVA->GPA mapping may no longer be
4293 	 * valid. So clear any cached MMIO info even when we don't need to sync
4294 	 * the shadow page tables.
4295 	 */
4296 	vcpu_clear_mmio_info(vcpu, MMIO_GVA_ANY);
4297 
4298 	/*
4299 	 * If this is a direct root page, it doesn't have a write flooding
4300 	 * count. Otherwise, clear the write flooding count.
4301 	 */
4302 	if (!new_role.direct)
4303 		__clear_sp_write_flooding_count(
4304 				to_shadow_page(vcpu->arch.mmu->root.hpa));
4305 }
4306 EXPORT_SYMBOL_GPL(kvm_mmu_new_pgd);
4307 
get_cr3(struct kvm_vcpu * vcpu)4308 static unsigned long get_cr3(struct kvm_vcpu *vcpu)
4309 {
4310 	return kvm_read_cr3(vcpu);
4311 }
4312 
sync_mmio_spte(struct kvm_vcpu * vcpu,u64 * sptep,gfn_t gfn,unsigned int access)4313 static bool sync_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn,
4314 			   unsigned int access)
4315 {
4316 	if (unlikely(is_mmio_spte(*sptep))) {
4317 		if (gfn != get_mmio_spte_gfn(*sptep)) {
4318 			mmu_spte_clear_no_track(sptep);
4319 			return true;
4320 		}
4321 
4322 		mark_mmio_spte(vcpu, sptep, gfn, access);
4323 		return true;
4324 	}
4325 
4326 	return false;
4327 }
4328 
4329 #define PTTYPE_EPT 18 /* arbitrary */
4330 #define PTTYPE PTTYPE_EPT
4331 #include "paging_tmpl.h"
4332 #undef PTTYPE
4333 
4334 #define PTTYPE 64
4335 #include "paging_tmpl.h"
4336 #undef PTTYPE
4337 
4338 #define PTTYPE 32
4339 #include "paging_tmpl.h"
4340 #undef PTTYPE
4341 
4342 static void
__reset_rsvds_bits_mask(struct rsvd_bits_validate * rsvd_check,u64 pa_bits_rsvd,int level,bool nx,bool gbpages,bool pse,bool amd)4343 __reset_rsvds_bits_mask(struct rsvd_bits_validate *rsvd_check,
4344 			u64 pa_bits_rsvd, int level, bool nx, bool gbpages,
4345 			bool pse, bool amd)
4346 {
4347 	u64 gbpages_bit_rsvd = 0;
4348 	u64 nonleaf_bit8_rsvd = 0;
4349 	u64 high_bits_rsvd;
4350 
4351 	rsvd_check->bad_mt_xwr = 0;
4352 
4353 	if (!gbpages)
4354 		gbpages_bit_rsvd = rsvd_bits(7, 7);
4355 
4356 	if (level == PT32E_ROOT_LEVEL)
4357 		high_bits_rsvd = pa_bits_rsvd & rsvd_bits(0, 62);
4358 	else
4359 		high_bits_rsvd = pa_bits_rsvd & rsvd_bits(0, 51);
4360 
4361 	/* Note, NX doesn't exist in PDPTEs, this is handled below. */
4362 	if (!nx)
4363 		high_bits_rsvd |= rsvd_bits(63, 63);
4364 
4365 	/*
4366 	 * Non-leaf PML4Es and PDPEs reserve bit 8 (which would be the G bit for
4367 	 * leaf entries) on AMD CPUs only.
4368 	 */
4369 	if (amd)
4370 		nonleaf_bit8_rsvd = rsvd_bits(8, 8);
4371 
4372 	switch (level) {
4373 	case PT32_ROOT_LEVEL:
4374 		/* no rsvd bits for 2 level 4K page table entries */
4375 		rsvd_check->rsvd_bits_mask[0][1] = 0;
4376 		rsvd_check->rsvd_bits_mask[0][0] = 0;
4377 		rsvd_check->rsvd_bits_mask[1][0] =
4378 			rsvd_check->rsvd_bits_mask[0][0];
4379 
4380 		if (!pse) {
4381 			rsvd_check->rsvd_bits_mask[1][1] = 0;
4382 			break;
4383 		}
4384 
4385 		if (is_cpuid_PSE36())
4386 			/* 36bits PSE 4MB page */
4387 			rsvd_check->rsvd_bits_mask[1][1] = rsvd_bits(17, 21);
4388 		else
4389 			/* 32 bits PSE 4MB page */
4390 			rsvd_check->rsvd_bits_mask[1][1] = rsvd_bits(13, 21);
4391 		break;
4392 	case PT32E_ROOT_LEVEL:
4393 		rsvd_check->rsvd_bits_mask[0][2] = rsvd_bits(63, 63) |
4394 						   high_bits_rsvd |
4395 						   rsvd_bits(5, 8) |
4396 						   rsvd_bits(1, 2);	/* PDPTE */
4397 		rsvd_check->rsvd_bits_mask[0][1] = high_bits_rsvd;	/* PDE */
4398 		rsvd_check->rsvd_bits_mask[0][0] = high_bits_rsvd;	/* PTE */
4399 		rsvd_check->rsvd_bits_mask[1][1] = high_bits_rsvd |
4400 						   rsvd_bits(13, 20);	/* large page */
4401 		rsvd_check->rsvd_bits_mask[1][0] =
4402 			rsvd_check->rsvd_bits_mask[0][0];
4403 		break;
4404 	case PT64_ROOT_5LEVEL:
4405 		rsvd_check->rsvd_bits_mask[0][4] = high_bits_rsvd |
4406 						   nonleaf_bit8_rsvd |
4407 						   rsvd_bits(7, 7);
4408 		rsvd_check->rsvd_bits_mask[1][4] =
4409 			rsvd_check->rsvd_bits_mask[0][4];
4410 		fallthrough;
4411 	case PT64_ROOT_4LEVEL:
4412 		rsvd_check->rsvd_bits_mask[0][3] = high_bits_rsvd |
4413 						   nonleaf_bit8_rsvd |
4414 						   rsvd_bits(7, 7);
4415 		rsvd_check->rsvd_bits_mask[0][2] = high_bits_rsvd |
4416 						   gbpages_bit_rsvd;
4417 		rsvd_check->rsvd_bits_mask[0][1] = high_bits_rsvd;
4418 		rsvd_check->rsvd_bits_mask[0][0] = high_bits_rsvd;
4419 		rsvd_check->rsvd_bits_mask[1][3] =
4420 			rsvd_check->rsvd_bits_mask[0][3];
4421 		rsvd_check->rsvd_bits_mask[1][2] = high_bits_rsvd |
4422 						   gbpages_bit_rsvd |
4423 						   rsvd_bits(13, 29);
4424 		rsvd_check->rsvd_bits_mask[1][1] = high_bits_rsvd |
4425 						   rsvd_bits(13, 20); /* large page */
4426 		rsvd_check->rsvd_bits_mask[1][0] =
4427 			rsvd_check->rsvd_bits_mask[0][0];
4428 		break;
4429 	}
4430 }
4431 
guest_can_use_gbpages(struct kvm_vcpu * vcpu)4432 static bool guest_can_use_gbpages(struct kvm_vcpu *vcpu)
4433 {
4434 	/*
4435 	 * If TDP is enabled, let the guest use GBPAGES if they're supported in
4436 	 * hardware.  The hardware page walker doesn't let KVM disable GBPAGES,
4437 	 * i.e. won't treat them as reserved, and KVM doesn't redo the GVA->GPA
4438 	 * walk for performance and complexity reasons.  Not to mention KVM
4439 	 * _can't_ solve the problem because GVA->GPA walks aren't visible to
4440 	 * KVM once a TDP translation is installed.  Mimic hardware behavior so
4441 	 * that KVM's is at least consistent, i.e. doesn't randomly inject #PF.
4442 	 */
4443 	return tdp_enabled ? boot_cpu_has(X86_FEATURE_GBPAGES) :
4444 			     guest_cpuid_has(vcpu, X86_FEATURE_GBPAGES);
4445 }
4446 
reset_guest_rsvds_bits_mask(struct kvm_vcpu * vcpu,struct kvm_mmu * context)4447 static void reset_guest_rsvds_bits_mask(struct kvm_vcpu *vcpu,
4448 					struct kvm_mmu *context)
4449 {
4450 	__reset_rsvds_bits_mask(&context->guest_rsvd_check,
4451 				vcpu->arch.reserved_gpa_bits,
4452 				context->cpu_role.base.level, is_efer_nx(context),
4453 				guest_can_use_gbpages(vcpu),
4454 				is_cr4_pse(context),
4455 				guest_cpuid_is_amd_or_hygon(vcpu));
4456 }
4457 
4458 static void
__reset_rsvds_bits_mask_ept(struct rsvd_bits_validate * rsvd_check,u64 pa_bits_rsvd,bool execonly,int huge_page_level)4459 __reset_rsvds_bits_mask_ept(struct rsvd_bits_validate *rsvd_check,
4460 			    u64 pa_bits_rsvd, bool execonly, int huge_page_level)
4461 {
4462 	u64 high_bits_rsvd = pa_bits_rsvd & rsvd_bits(0, 51);
4463 	u64 large_1g_rsvd = 0, large_2m_rsvd = 0;
4464 	u64 bad_mt_xwr;
4465 
4466 	if (huge_page_level < PG_LEVEL_1G)
4467 		large_1g_rsvd = rsvd_bits(7, 7);
4468 	if (huge_page_level < PG_LEVEL_2M)
4469 		large_2m_rsvd = rsvd_bits(7, 7);
4470 
4471 	rsvd_check->rsvd_bits_mask[0][4] = high_bits_rsvd | rsvd_bits(3, 7);
4472 	rsvd_check->rsvd_bits_mask[0][3] = high_bits_rsvd | rsvd_bits(3, 7);
4473 	rsvd_check->rsvd_bits_mask[0][2] = high_bits_rsvd | rsvd_bits(3, 6) | large_1g_rsvd;
4474 	rsvd_check->rsvd_bits_mask[0][1] = high_bits_rsvd | rsvd_bits(3, 6) | large_2m_rsvd;
4475 	rsvd_check->rsvd_bits_mask[0][0] = high_bits_rsvd;
4476 
4477 	/* large page */
4478 	rsvd_check->rsvd_bits_mask[1][4] = rsvd_check->rsvd_bits_mask[0][4];
4479 	rsvd_check->rsvd_bits_mask[1][3] = rsvd_check->rsvd_bits_mask[0][3];
4480 	rsvd_check->rsvd_bits_mask[1][2] = high_bits_rsvd | rsvd_bits(12, 29) | large_1g_rsvd;
4481 	rsvd_check->rsvd_bits_mask[1][1] = high_bits_rsvd | rsvd_bits(12, 20) | large_2m_rsvd;
4482 	rsvd_check->rsvd_bits_mask[1][0] = rsvd_check->rsvd_bits_mask[0][0];
4483 
4484 	bad_mt_xwr = 0xFFull << (2 * 8);	/* bits 3..5 must not be 2 */
4485 	bad_mt_xwr |= 0xFFull << (3 * 8);	/* bits 3..5 must not be 3 */
4486 	bad_mt_xwr |= 0xFFull << (7 * 8);	/* bits 3..5 must not be 7 */
4487 	bad_mt_xwr |= REPEAT_BYTE(1ull << 2);	/* bits 0..2 must not be 010 */
4488 	bad_mt_xwr |= REPEAT_BYTE(1ull << 6);	/* bits 0..2 must not be 110 */
4489 	if (!execonly) {
4490 		/* bits 0..2 must not be 100 unless VMX capabilities allow it */
4491 		bad_mt_xwr |= REPEAT_BYTE(1ull << 4);
4492 	}
4493 	rsvd_check->bad_mt_xwr = bad_mt_xwr;
4494 }
4495 
reset_rsvds_bits_mask_ept(struct kvm_vcpu * vcpu,struct kvm_mmu * context,bool execonly,int huge_page_level)4496 static void reset_rsvds_bits_mask_ept(struct kvm_vcpu *vcpu,
4497 		struct kvm_mmu *context, bool execonly, int huge_page_level)
4498 {
4499 	__reset_rsvds_bits_mask_ept(&context->guest_rsvd_check,
4500 				    vcpu->arch.reserved_gpa_bits, execonly,
4501 				    huge_page_level);
4502 }
4503 
reserved_hpa_bits(void)4504 static inline u64 reserved_hpa_bits(void)
4505 {
4506 	return rsvd_bits(shadow_phys_bits, 63);
4507 }
4508 
4509 /*
4510  * the page table on host is the shadow page table for the page
4511  * table in guest or amd nested guest, its mmu features completely
4512  * follow the features in guest.
4513  */
reset_shadow_zero_bits_mask(struct kvm_vcpu * vcpu,struct kvm_mmu * context)4514 static void reset_shadow_zero_bits_mask(struct kvm_vcpu *vcpu,
4515 					struct kvm_mmu *context)
4516 {
4517 	/* @amd adds a check on bit of SPTEs, which KVM shouldn't use anyways. */
4518 	bool is_amd = true;
4519 	/* KVM doesn't use 2-level page tables for the shadow MMU. */
4520 	bool is_pse = false;
4521 	struct rsvd_bits_validate *shadow_zero_check;
4522 	int i;
4523 
4524 	WARN_ON_ONCE(context->root_role.level < PT32E_ROOT_LEVEL);
4525 
4526 	shadow_zero_check = &context->shadow_zero_check;
4527 	__reset_rsvds_bits_mask(shadow_zero_check, reserved_hpa_bits(),
4528 				context->root_role.level,
4529 				context->root_role.efer_nx,
4530 				guest_can_use_gbpages(vcpu), is_pse, is_amd);
4531 
4532 	if (!shadow_me_mask)
4533 		return;
4534 
4535 	for (i = context->root_role.level; --i >= 0;) {
4536 		/*
4537 		 * So far shadow_me_value is a constant during KVM's life
4538 		 * time.  Bits in shadow_me_value are allowed to be set.
4539 		 * Bits in shadow_me_mask but not in shadow_me_value are
4540 		 * not allowed to be set.
4541 		 */
4542 		shadow_zero_check->rsvd_bits_mask[0][i] |= shadow_me_mask;
4543 		shadow_zero_check->rsvd_bits_mask[1][i] |= shadow_me_mask;
4544 		shadow_zero_check->rsvd_bits_mask[0][i] &= ~shadow_me_value;
4545 		shadow_zero_check->rsvd_bits_mask[1][i] &= ~shadow_me_value;
4546 	}
4547 
4548 }
4549 
boot_cpu_is_amd(void)4550 static inline bool boot_cpu_is_amd(void)
4551 {
4552 	WARN_ON_ONCE(!tdp_enabled);
4553 	return shadow_x_mask == 0;
4554 }
4555 
4556 /*
4557  * the direct page table on host, use as much mmu features as
4558  * possible, however, kvm currently does not do execution-protection.
4559  */
4560 static void
reset_tdp_shadow_zero_bits_mask(struct kvm_mmu * context)4561 reset_tdp_shadow_zero_bits_mask(struct kvm_mmu *context)
4562 {
4563 	struct rsvd_bits_validate *shadow_zero_check;
4564 	int i;
4565 
4566 	shadow_zero_check = &context->shadow_zero_check;
4567 
4568 	if (boot_cpu_is_amd())
4569 		__reset_rsvds_bits_mask(shadow_zero_check, reserved_hpa_bits(),
4570 					context->root_role.level, true,
4571 					boot_cpu_has(X86_FEATURE_GBPAGES),
4572 					false, true);
4573 	else
4574 		__reset_rsvds_bits_mask_ept(shadow_zero_check,
4575 					    reserved_hpa_bits(), false,
4576 					    max_huge_page_level);
4577 
4578 	if (!shadow_me_mask)
4579 		return;
4580 
4581 	for (i = context->root_role.level; --i >= 0;) {
4582 		shadow_zero_check->rsvd_bits_mask[0][i] &= ~shadow_me_mask;
4583 		shadow_zero_check->rsvd_bits_mask[1][i] &= ~shadow_me_mask;
4584 	}
4585 }
4586 
4587 /*
4588  * as the comments in reset_shadow_zero_bits_mask() except it
4589  * is the shadow page table for intel nested guest.
4590  */
4591 static void
reset_ept_shadow_zero_bits_mask(struct kvm_mmu * context,bool execonly)4592 reset_ept_shadow_zero_bits_mask(struct kvm_mmu *context, bool execonly)
4593 {
4594 	__reset_rsvds_bits_mask_ept(&context->shadow_zero_check,
4595 				    reserved_hpa_bits(), execonly,
4596 				    max_huge_page_level);
4597 }
4598 
4599 #define BYTE_MASK(access) \
4600 	((1 & (access) ? 2 : 0) | \
4601 	 (2 & (access) ? 4 : 0) | \
4602 	 (3 & (access) ? 8 : 0) | \
4603 	 (4 & (access) ? 16 : 0) | \
4604 	 (5 & (access) ? 32 : 0) | \
4605 	 (6 & (access) ? 64 : 0) | \
4606 	 (7 & (access) ? 128 : 0))
4607 
4608 
update_permission_bitmask(struct kvm_mmu * mmu,bool ept)4609 static void update_permission_bitmask(struct kvm_mmu *mmu, bool ept)
4610 {
4611 	unsigned byte;
4612 
4613 	const u8 x = BYTE_MASK(ACC_EXEC_MASK);
4614 	const u8 w = BYTE_MASK(ACC_WRITE_MASK);
4615 	const u8 u = BYTE_MASK(ACC_USER_MASK);
4616 
4617 	bool cr4_smep = is_cr4_smep(mmu);
4618 	bool cr4_smap = is_cr4_smap(mmu);
4619 	bool cr0_wp = is_cr0_wp(mmu);
4620 	bool efer_nx = is_efer_nx(mmu);
4621 
4622 	for (byte = 0; byte < ARRAY_SIZE(mmu->permissions); ++byte) {
4623 		unsigned pfec = byte << 1;
4624 
4625 		/*
4626 		 * Each "*f" variable has a 1 bit for each UWX value
4627 		 * that causes a fault with the given PFEC.
4628 		 */
4629 
4630 		/* Faults from writes to non-writable pages */
4631 		u8 wf = (pfec & PFERR_WRITE_MASK) ? (u8)~w : 0;
4632 		/* Faults from user mode accesses to supervisor pages */
4633 		u8 uf = (pfec & PFERR_USER_MASK) ? (u8)~u : 0;
4634 		/* Faults from fetches of non-executable pages*/
4635 		u8 ff = (pfec & PFERR_FETCH_MASK) ? (u8)~x : 0;
4636 		/* Faults from kernel mode fetches of user pages */
4637 		u8 smepf = 0;
4638 		/* Faults from kernel mode accesses of user pages */
4639 		u8 smapf = 0;
4640 
4641 		if (!ept) {
4642 			/* Faults from kernel mode accesses to user pages */
4643 			u8 kf = (pfec & PFERR_USER_MASK) ? 0 : u;
4644 
4645 			/* Not really needed: !nx will cause pte.nx to fault */
4646 			if (!efer_nx)
4647 				ff = 0;
4648 
4649 			/* Allow supervisor writes if !cr0.wp */
4650 			if (!cr0_wp)
4651 				wf = (pfec & PFERR_USER_MASK) ? wf : 0;
4652 
4653 			/* Disallow supervisor fetches of user code if cr4.smep */
4654 			if (cr4_smep)
4655 				smepf = (pfec & PFERR_FETCH_MASK) ? kf : 0;
4656 
4657 			/*
4658 			 * SMAP:kernel-mode data accesses from user-mode
4659 			 * mappings should fault. A fault is considered
4660 			 * as a SMAP violation if all of the following
4661 			 * conditions are true:
4662 			 *   - X86_CR4_SMAP is set in CR4
4663 			 *   - A user page is accessed
4664 			 *   - The access is not a fetch
4665 			 *   - The access is supervisor mode
4666 			 *   - If implicit supervisor access or X86_EFLAGS_AC is clear
4667 			 *
4668 			 * Here, we cover the first four conditions.
4669 			 * The fifth is computed dynamically in permission_fault();
4670 			 * PFERR_RSVD_MASK bit will be set in PFEC if the access is
4671 			 * *not* subject to SMAP restrictions.
4672 			 */
4673 			if (cr4_smap)
4674 				smapf = (pfec & (PFERR_RSVD_MASK|PFERR_FETCH_MASK)) ? 0 : kf;
4675 		}
4676 
4677 		mmu->permissions[byte] = ff | uf | wf | smepf | smapf;
4678 	}
4679 }
4680 
4681 /*
4682 * PKU is an additional mechanism by which the paging controls access to
4683 * user-mode addresses based on the value in the PKRU register.  Protection
4684 * key violations are reported through a bit in the page fault error code.
4685 * Unlike other bits of the error code, the PK bit is not known at the
4686 * call site of e.g. gva_to_gpa; it must be computed directly in
4687 * permission_fault based on two bits of PKRU, on some machine state (CR4,
4688 * CR0, EFER, CPL), and on other bits of the error code and the page tables.
4689 *
4690 * In particular the following conditions come from the error code, the
4691 * page tables and the machine state:
4692 * - PK is always zero unless CR4.PKE=1 and EFER.LMA=1
4693 * - PK is always zero if RSVD=1 (reserved bit set) or F=1 (instruction fetch)
4694 * - PK is always zero if U=0 in the page tables
4695 * - PKRU.WD is ignored if CR0.WP=0 and the access is a supervisor access.
4696 *
4697 * The PKRU bitmask caches the result of these four conditions.  The error
4698 * code (minus the P bit) and the page table's U bit form an index into the
4699 * PKRU bitmask.  Two bits of the PKRU bitmask are then extracted and ANDed
4700 * with the two bits of the PKRU register corresponding to the protection key.
4701 * For the first three conditions above the bits will be 00, thus masking
4702 * away both AD and WD.  For all reads or if the last condition holds, WD
4703 * only will be masked away.
4704 */
update_pkru_bitmask(struct kvm_mmu * mmu)4705 static void update_pkru_bitmask(struct kvm_mmu *mmu)
4706 {
4707 	unsigned bit;
4708 	bool wp;
4709 
4710 	mmu->pkru_mask = 0;
4711 
4712 	if (!is_cr4_pke(mmu))
4713 		return;
4714 
4715 	wp = is_cr0_wp(mmu);
4716 
4717 	for (bit = 0; bit < ARRAY_SIZE(mmu->permissions); ++bit) {
4718 		unsigned pfec, pkey_bits;
4719 		bool check_pkey, check_write, ff, uf, wf, pte_user;
4720 
4721 		pfec = bit << 1;
4722 		ff = pfec & PFERR_FETCH_MASK;
4723 		uf = pfec & PFERR_USER_MASK;
4724 		wf = pfec & PFERR_WRITE_MASK;
4725 
4726 		/* PFEC.RSVD is replaced by ACC_USER_MASK. */
4727 		pte_user = pfec & PFERR_RSVD_MASK;
4728 
4729 		/*
4730 		 * Only need to check the access which is not an
4731 		 * instruction fetch and is to a user page.
4732 		 */
4733 		check_pkey = (!ff && pte_user);
4734 		/*
4735 		 * write access is controlled by PKRU if it is a
4736 		 * user access or CR0.WP = 1.
4737 		 */
4738 		check_write = check_pkey && wf && (uf || wp);
4739 
4740 		/* PKRU.AD stops both read and write access. */
4741 		pkey_bits = !!check_pkey;
4742 		/* PKRU.WD stops write access. */
4743 		pkey_bits |= (!!check_write) << 1;
4744 
4745 		mmu->pkru_mask |= (pkey_bits & 3) << pfec;
4746 	}
4747 }
4748 
reset_guest_paging_metadata(struct kvm_vcpu * vcpu,struct kvm_mmu * mmu)4749 static void reset_guest_paging_metadata(struct kvm_vcpu *vcpu,
4750 					struct kvm_mmu *mmu)
4751 {
4752 	if (!is_cr0_pg(mmu))
4753 		return;
4754 
4755 	reset_guest_rsvds_bits_mask(vcpu, mmu);
4756 	update_permission_bitmask(mmu, false);
4757 	update_pkru_bitmask(mmu);
4758 }
4759 
paging64_init_context(struct kvm_mmu * context)4760 static void paging64_init_context(struct kvm_mmu *context)
4761 {
4762 	context->page_fault = paging64_page_fault;
4763 	context->gva_to_gpa = paging64_gva_to_gpa;
4764 	context->sync_page = paging64_sync_page;
4765 	context->invlpg = paging64_invlpg;
4766 }
4767 
paging32_init_context(struct kvm_mmu * context)4768 static void paging32_init_context(struct kvm_mmu *context)
4769 {
4770 	context->page_fault = paging32_page_fault;
4771 	context->gva_to_gpa = paging32_gva_to_gpa;
4772 	context->sync_page = paging32_sync_page;
4773 	context->invlpg = paging32_invlpg;
4774 }
4775 
4776 static union kvm_cpu_role
kvm_calc_cpu_role(struct kvm_vcpu * vcpu,const struct kvm_mmu_role_regs * regs)4777 kvm_calc_cpu_role(struct kvm_vcpu *vcpu, const struct kvm_mmu_role_regs *regs)
4778 {
4779 	union kvm_cpu_role role = {0};
4780 
4781 	role.base.access = ACC_ALL;
4782 	role.base.smm = is_smm(vcpu);
4783 	role.base.guest_mode = is_guest_mode(vcpu);
4784 	role.ext.valid = 1;
4785 
4786 	if (!____is_cr0_pg(regs)) {
4787 		role.base.direct = 1;
4788 		return role;
4789 	}
4790 
4791 	role.base.efer_nx = ____is_efer_nx(regs);
4792 	role.base.cr0_wp = ____is_cr0_wp(regs);
4793 	role.base.smep_andnot_wp = ____is_cr4_smep(regs) && !____is_cr0_wp(regs);
4794 	role.base.smap_andnot_wp = ____is_cr4_smap(regs) && !____is_cr0_wp(regs);
4795 	role.base.has_4_byte_gpte = !____is_cr4_pae(regs);
4796 
4797 	if (____is_efer_lma(regs))
4798 		role.base.level = ____is_cr4_la57(regs) ? PT64_ROOT_5LEVEL
4799 							: PT64_ROOT_4LEVEL;
4800 	else if (____is_cr4_pae(regs))
4801 		role.base.level = PT32E_ROOT_LEVEL;
4802 	else
4803 		role.base.level = PT32_ROOT_LEVEL;
4804 
4805 	role.ext.cr4_smep = ____is_cr4_smep(regs);
4806 	role.ext.cr4_smap = ____is_cr4_smap(regs);
4807 	role.ext.cr4_pse = ____is_cr4_pse(regs);
4808 
4809 	/* PKEY and LA57 are active iff long mode is active. */
4810 	role.ext.cr4_pke = ____is_efer_lma(regs) && ____is_cr4_pke(regs);
4811 	role.ext.cr4_la57 = ____is_efer_lma(regs) && ____is_cr4_la57(regs);
4812 	role.ext.efer_lma = ____is_efer_lma(regs);
4813 	return role;
4814 }
4815 
kvm_mmu_get_tdp_level(struct kvm_vcpu * vcpu)4816 static inline int kvm_mmu_get_tdp_level(struct kvm_vcpu *vcpu)
4817 {
4818 	/* tdp_root_level is architecture forced level, use it if nonzero */
4819 	if (tdp_root_level)
4820 		return tdp_root_level;
4821 
4822 	/* Use 5-level TDP if and only if it's useful/necessary. */
4823 	if (max_tdp_level == 5 && cpuid_maxphyaddr(vcpu) <= 48)
4824 		return 4;
4825 
4826 	return max_tdp_level;
4827 }
4828 
4829 static union kvm_mmu_page_role
kvm_calc_tdp_mmu_root_page_role(struct kvm_vcpu * vcpu,union kvm_cpu_role cpu_role)4830 kvm_calc_tdp_mmu_root_page_role(struct kvm_vcpu *vcpu,
4831 				union kvm_cpu_role cpu_role)
4832 {
4833 	union kvm_mmu_page_role role = {0};
4834 
4835 	role.access = ACC_ALL;
4836 	role.cr0_wp = true;
4837 	role.efer_nx = true;
4838 	role.smm = cpu_role.base.smm;
4839 	role.guest_mode = cpu_role.base.guest_mode;
4840 	role.ad_disabled = !kvm_ad_enabled();
4841 	role.level = kvm_mmu_get_tdp_level(vcpu);
4842 	role.direct = true;
4843 	role.has_4_byte_gpte = false;
4844 
4845 	return role;
4846 }
4847 
init_kvm_tdp_mmu(struct kvm_vcpu * vcpu,union kvm_cpu_role cpu_role)4848 static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu,
4849 			     union kvm_cpu_role cpu_role)
4850 {
4851 	struct kvm_mmu *context = &vcpu->arch.root_mmu;
4852 	union kvm_mmu_page_role root_role = kvm_calc_tdp_mmu_root_page_role(vcpu, cpu_role);
4853 
4854 	if (cpu_role.as_u64 == context->cpu_role.as_u64 &&
4855 	    root_role.word == context->root_role.word)
4856 		return;
4857 
4858 	context->cpu_role.as_u64 = cpu_role.as_u64;
4859 	context->root_role.word = root_role.word;
4860 	context->page_fault = kvm_tdp_page_fault;
4861 	context->sync_page = nonpaging_sync_page;
4862 	context->invlpg = NULL;
4863 	context->get_guest_pgd = get_cr3;
4864 	context->get_pdptr = kvm_pdptr_read;
4865 	context->inject_page_fault = kvm_inject_page_fault;
4866 
4867 	if (!is_cr0_pg(context))
4868 		context->gva_to_gpa = nonpaging_gva_to_gpa;
4869 	else if (is_cr4_pae(context))
4870 		context->gva_to_gpa = paging64_gva_to_gpa;
4871 	else
4872 		context->gva_to_gpa = paging32_gva_to_gpa;
4873 
4874 	reset_guest_paging_metadata(vcpu, context);
4875 	reset_tdp_shadow_zero_bits_mask(context);
4876 }
4877 
shadow_mmu_init_context(struct kvm_vcpu * vcpu,struct kvm_mmu * context,union kvm_cpu_role cpu_role,union kvm_mmu_page_role root_role)4878 static void shadow_mmu_init_context(struct kvm_vcpu *vcpu, struct kvm_mmu *context,
4879 				    union kvm_cpu_role cpu_role,
4880 				    union kvm_mmu_page_role root_role)
4881 {
4882 	if (cpu_role.as_u64 == context->cpu_role.as_u64 &&
4883 	    root_role.word == context->root_role.word)
4884 		return;
4885 
4886 	context->cpu_role.as_u64 = cpu_role.as_u64;
4887 	context->root_role.word = root_role.word;
4888 
4889 	if (!is_cr0_pg(context))
4890 		nonpaging_init_context(context);
4891 	else if (is_cr4_pae(context))
4892 		paging64_init_context(context);
4893 	else
4894 		paging32_init_context(context);
4895 
4896 	reset_guest_paging_metadata(vcpu, context);
4897 	reset_shadow_zero_bits_mask(vcpu, context);
4898 }
4899 
kvm_init_shadow_mmu(struct kvm_vcpu * vcpu,union kvm_cpu_role cpu_role)4900 static void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu,
4901 				union kvm_cpu_role cpu_role)
4902 {
4903 	struct kvm_mmu *context = &vcpu->arch.root_mmu;
4904 	union kvm_mmu_page_role root_role;
4905 
4906 	root_role = cpu_role.base;
4907 
4908 	/* KVM uses PAE paging whenever the guest isn't using 64-bit paging. */
4909 	root_role.level = max_t(u32, root_role.level, PT32E_ROOT_LEVEL);
4910 
4911 	/*
4912 	 * KVM forces EFER.NX=1 when TDP is disabled, reflect it in the MMU role.
4913 	 * KVM uses NX when TDP is disabled to handle a variety of scenarios,
4914 	 * notably for huge SPTEs if iTLB multi-hit mitigation is enabled and
4915 	 * to generate correct permissions for CR0.WP=0/CR4.SMEP=1/EFER.NX=0.
4916 	 * The iTLB multi-hit workaround can be toggled at any time, so assume
4917 	 * NX can be used by any non-nested shadow MMU to avoid having to reset
4918 	 * MMU contexts.
4919 	 */
4920 	root_role.efer_nx = true;
4921 
4922 	shadow_mmu_init_context(vcpu, context, cpu_role, root_role);
4923 }
4924 
kvm_init_shadow_npt_mmu(struct kvm_vcpu * vcpu,unsigned long cr0,unsigned long cr4,u64 efer,gpa_t nested_cr3)4925 void kvm_init_shadow_npt_mmu(struct kvm_vcpu *vcpu, unsigned long cr0,
4926 			     unsigned long cr4, u64 efer, gpa_t nested_cr3)
4927 {
4928 	struct kvm_mmu *context = &vcpu->arch.guest_mmu;
4929 	struct kvm_mmu_role_regs regs = {
4930 		.cr0 = cr0,
4931 		.cr4 = cr4 & ~X86_CR4_PKE,
4932 		.efer = efer,
4933 	};
4934 	union kvm_cpu_role cpu_role = kvm_calc_cpu_role(vcpu, &regs);
4935 	union kvm_mmu_page_role root_role;
4936 
4937 	/* NPT requires CR0.PG=1. */
4938 	WARN_ON_ONCE(cpu_role.base.direct);
4939 
4940 	root_role = cpu_role.base;
4941 	root_role.level = kvm_mmu_get_tdp_level(vcpu);
4942 	if (root_role.level == PT64_ROOT_5LEVEL &&
4943 	    cpu_role.base.level == PT64_ROOT_4LEVEL)
4944 		root_role.passthrough = 1;
4945 
4946 	shadow_mmu_init_context(vcpu, context, cpu_role, root_role);
4947 	kvm_mmu_new_pgd(vcpu, nested_cr3);
4948 }
4949 EXPORT_SYMBOL_GPL(kvm_init_shadow_npt_mmu);
4950 
4951 static union kvm_cpu_role
kvm_calc_shadow_ept_root_page_role(struct kvm_vcpu * vcpu,bool accessed_dirty,bool execonly,u8 level)4952 kvm_calc_shadow_ept_root_page_role(struct kvm_vcpu *vcpu, bool accessed_dirty,
4953 				   bool execonly, u8 level)
4954 {
4955 	union kvm_cpu_role role = {0};
4956 
4957 	/*
4958 	 * KVM does not support SMM transfer monitors, and consequently does not
4959 	 * support the "entry to SMM" control either.  role.base.smm is always 0.
4960 	 */
4961 	WARN_ON_ONCE(is_smm(vcpu));
4962 	role.base.level = level;
4963 	role.base.has_4_byte_gpte = false;
4964 	role.base.direct = false;
4965 	role.base.ad_disabled = !accessed_dirty;
4966 	role.base.guest_mode = true;
4967 	role.base.access = ACC_ALL;
4968 
4969 	role.ext.word = 0;
4970 	role.ext.execonly = execonly;
4971 	role.ext.valid = 1;
4972 
4973 	return role;
4974 }
4975 
kvm_init_shadow_ept_mmu(struct kvm_vcpu * vcpu,bool execonly,int huge_page_level,bool accessed_dirty,gpa_t new_eptp)4976 void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly,
4977 			     int huge_page_level, bool accessed_dirty,
4978 			     gpa_t new_eptp)
4979 {
4980 	struct kvm_mmu *context = &vcpu->arch.guest_mmu;
4981 	u8 level = vmx_eptp_page_walk_level(new_eptp);
4982 	union kvm_cpu_role new_mode =
4983 		kvm_calc_shadow_ept_root_page_role(vcpu, accessed_dirty,
4984 						   execonly, level);
4985 
4986 	if (new_mode.as_u64 != context->cpu_role.as_u64) {
4987 		/* EPT, and thus nested EPT, does not consume CR0, CR4, nor EFER. */
4988 		context->cpu_role.as_u64 = new_mode.as_u64;
4989 		context->root_role.word = new_mode.base.word;
4990 
4991 		context->page_fault = ept_page_fault;
4992 		context->gva_to_gpa = ept_gva_to_gpa;
4993 		context->sync_page = ept_sync_page;
4994 		context->invlpg = ept_invlpg;
4995 
4996 		update_permission_bitmask(context, true);
4997 		context->pkru_mask = 0;
4998 		reset_rsvds_bits_mask_ept(vcpu, context, execonly, huge_page_level);
4999 		reset_ept_shadow_zero_bits_mask(context, execonly);
5000 	}
5001 
5002 	kvm_mmu_new_pgd(vcpu, new_eptp);
5003 }
5004 EXPORT_SYMBOL_GPL(kvm_init_shadow_ept_mmu);
5005 
init_kvm_softmmu(struct kvm_vcpu * vcpu,union kvm_cpu_role cpu_role)5006 static void init_kvm_softmmu(struct kvm_vcpu *vcpu,
5007 			     union kvm_cpu_role cpu_role)
5008 {
5009 	struct kvm_mmu *context = &vcpu->arch.root_mmu;
5010 
5011 	kvm_init_shadow_mmu(vcpu, cpu_role);
5012 
5013 	context->get_guest_pgd     = get_cr3;
5014 	context->get_pdptr         = kvm_pdptr_read;
5015 	context->inject_page_fault = kvm_inject_page_fault;
5016 }
5017 
init_kvm_nested_mmu(struct kvm_vcpu * vcpu,union kvm_cpu_role new_mode)5018 static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu,
5019 				union kvm_cpu_role new_mode)
5020 {
5021 	struct kvm_mmu *g_context = &vcpu->arch.nested_mmu;
5022 
5023 	if (new_mode.as_u64 == g_context->cpu_role.as_u64)
5024 		return;
5025 
5026 	g_context->cpu_role.as_u64   = new_mode.as_u64;
5027 	g_context->get_guest_pgd     = get_cr3;
5028 	g_context->get_pdptr         = kvm_pdptr_read;
5029 	g_context->inject_page_fault = kvm_inject_page_fault;
5030 
5031 	/*
5032 	 * L2 page tables are never shadowed, so there is no need to sync
5033 	 * SPTEs.
5034 	 */
5035 	g_context->invlpg            = NULL;
5036 
5037 	/*
5038 	 * Note that arch.mmu->gva_to_gpa translates l2_gpa to l1_gpa using
5039 	 * L1's nested page tables (e.g. EPT12). The nested translation
5040 	 * of l2_gva to l1_gpa is done by arch.nested_mmu.gva_to_gpa using
5041 	 * L2's page tables as the first level of translation and L1's
5042 	 * nested page tables as the second level of translation. Basically
5043 	 * the gva_to_gpa functions between mmu and nested_mmu are swapped.
5044 	 */
5045 	if (!is_paging(vcpu))
5046 		g_context->gva_to_gpa = nonpaging_gva_to_gpa;
5047 	else if (is_long_mode(vcpu))
5048 		g_context->gva_to_gpa = paging64_gva_to_gpa;
5049 	else if (is_pae(vcpu))
5050 		g_context->gva_to_gpa = paging64_gva_to_gpa;
5051 	else
5052 		g_context->gva_to_gpa = paging32_gva_to_gpa;
5053 
5054 	reset_guest_paging_metadata(vcpu, g_context);
5055 }
5056 
kvm_init_mmu(struct kvm_vcpu * vcpu)5057 void kvm_init_mmu(struct kvm_vcpu *vcpu)
5058 {
5059 	struct kvm_mmu_role_regs regs = vcpu_to_role_regs(vcpu);
5060 	union kvm_cpu_role cpu_role = kvm_calc_cpu_role(vcpu, &regs);
5061 
5062 	if (mmu_is_nested(vcpu))
5063 		init_kvm_nested_mmu(vcpu, cpu_role);
5064 	else if (tdp_enabled)
5065 		init_kvm_tdp_mmu(vcpu, cpu_role);
5066 	else
5067 		init_kvm_softmmu(vcpu, cpu_role);
5068 }
5069 EXPORT_SYMBOL_GPL(kvm_init_mmu);
5070 
kvm_mmu_after_set_cpuid(struct kvm_vcpu * vcpu)5071 void kvm_mmu_after_set_cpuid(struct kvm_vcpu *vcpu)
5072 {
5073 	/*
5074 	 * Invalidate all MMU roles to force them to reinitialize as CPUID
5075 	 * information is factored into reserved bit calculations.
5076 	 *
5077 	 * Correctly handling multiple vCPU models with respect to paging and
5078 	 * physical address properties) in a single VM would require tracking
5079 	 * all relevant CPUID information in kvm_mmu_page_role. That is very
5080 	 * undesirable as it would increase the memory requirements for
5081 	 * gfn_track (see struct kvm_mmu_page_role comments).  For now that
5082 	 * problem is swept under the rug; KVM's CPUID API is horrific and
5083 	 * it's all but impossible to solve it without introducing a new API.
5084 	 */
5085 	vcpu->arch.root_mmu.root_role.word = 0;
5086 	vcpu->arch.guest_mmu.root_role.word = 0;
5087 	vcpu->arch.nested_mmu.root_role.word = 0;
5088 	vcpu->arch.root_mmu.cpu_role.ext.valid = 0;
5089 	vcpu->arch.guest_mmu.cpu_role.ext.valid = 0;
5090 	vcpu->arch.nested_mmu.cpu_role.ext.valid = 0;
5091 	kvm_mmu_reset_context(vcpu);
5092 
5093 	/*
5094 	 * Changing guest CPUID after KVM_RUN is forbidden, see the comment in
5095 	 * kvm_arch_vcpu_ioctl().
5096 	 */
5097 	KVM_BUG_ON(vcpu->arch.last_vmentry_cpu != -1, vcpu->kvm);
5098 }
5099 
kvm_mmu_reset_context(struct kvm_vcpu * vcpu)5100 void kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
5101 {
5102 	kvm_mmu_unload(vcpu);
5103 	kvm_init_mmu(vcpu);
5104 }
5105 EXPORT_SYMBOL_GPL(kvm_mmu_reset_context);
5106 
kvm_mmu_load(struct kvm_vcpu * vcpu)5107 int kvm_mmu_load(struct kvm_vcpu *vcpu)
5108 {
5109 	int r;
5110 
5111 	r = mmu_topup_memory_caches(vcpu, !vcpu->arch.mmu->root_role.direct);
5112 	if (r)
5113 		goto out;
5114 	r = mmu_alloc_special_roots(vcpu);
5115 	if (r)
5116 		goto out;
5117 	if (vcpu->arch.mmu->root_role.direct)
5118 		r = mmu_alloc_direct_roots(vcpu);
5119 	else
5120 		r = mmu_alloc_shadow_roots(vcpu);
5121 	if (r)
5122 		goto out;
5123 
5124 	kvm_mmu_sync_roots(vcpu);
5125 
5126 	kvm_mmu_load_pgd(vcpu);
5127 
5128 	/*
5129 	 * Flush any TLB entries for the new root, the provenance of the root
5130 	 * is unknown.  Even if KVM ensures there are no stale TLB entries
5131 	 * for a freed root, in theory another hypervisor could have left
5132 	 * stale entries.  Flushing on alloc also allows KVM to skip the TLB
5133 	 * flush when freeing a root (see kvm_tdp_mmu_put_root()).
5134 	 */
5135 	static_call(kvm_x86_flush_tlb_current)(vcpu);
5136 out:
5137 	return r;
5138 }
5139 
kvm_mmu_unload(struct kvm_vcpu * vcpu)5140 void kvm_mmu_unload(struct kvm_vcpu *vcpu)
5141 {
5142 	struct kvm *kvm = vcpu->kvm;
5143 
5144 	kvm_mmu_free_roots(kvm, &vcpu->arch.root_mmu, KVM_MMU_ROOTS_ALL);
5145 	WARN_ON(VALID_PAGE(vcpu->arch.root_mmu.root.hpa));
5146 	kvm_mmu_free_roots(kvm, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL);
5147 	WARN_ON(VALID_PAGE(vcpu->arch.guest_mmu.root.hpa));
5148 	vcpu_clear_mmio_info(vcpu, MMIO_GVA_ANY);
5149 }
5150 
is_obsolete_root(struct kvm * kvm,hpa_t root_hpa)5151 static bool is_obsolete_root(struct kvm *kvm, hpa_t root_hpa)
5152 {
5153 	struct kvm_mmu_page *sp;
5154 
5155 	if (!VALID_PAGE(root_hpa))
5156 		return false;
5157 
5158 	/*
5159 	 * When freeing obsolete roots, treat roots as obsolete if they don't
5160 	 * have an associated shadow page.  This does mean KVM will get false
5161 	 * positives and free roots that don't strictly need to be freed, but
5162 	 * such false positives are relatively rare:
5163 	 *
5164 	 *  (a) only PAE paging and nested NPT has roots without shadow pages
5165 	 *  (b) remote reloads due to a memslot update obsoletes _all_ roots
5166 	 *  (c) KVM doesn't track previous roots for PAE paging, and the guest
5167 	 *      is unlikely to zap an in-use PGD.
5168 	 */
5169 	sp = to_shadow_page(root_hpa);
5170 	return !sp || is_obsolete_sp(kvm, sp);
5171 }
5172 
__kvm_mmu_free_obsolete_roots(struct kvm * kvm,struct kvm_mmu * mmu)5173 static void __kvm_mmu_free_obsolete_roots(struct kvm *kvm, struct kvm_mmu *mmu)
5174 {
5175 	unsigned long roots_to_free = 0;
5176 	int i;
5177 
5178 	if (is_obsolete_root(kvm, mmu->root.hpa))
5179 		roots_to_free |= KVM_MMU_ROOT_CURRENT;
5180 
5181 	for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) {
5182 		if (is_obsolete_root(kvm, mmu->prev_roots[i].hpa))
5183 			roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i);
5184 	}
5185 
5186 	if (roots_to_free)
5187 		kvm_mmu_free_roots(kvm, mmu, roots_to_free);
5188 }
5189 
kvm_mmu_free_obsolete_roots(struct kvm_vcpu * vcpu)5190 void kvm_mmu_free_obsolete_roots(struct kvm_vcpu *vcpu)
5191 {
5192 	__kvm_mmu_free_obsolete_roots(vcpu->kvm, &vcpu->arch.root_mmu);
5193 	__kvm_mmu_free_obsolete_roots(vcpu->kvm, &vcpu->arch.guest_mmu);
5194 }
5195 
need_remote_flush(u64 old,u64 new)5196 static bool need_remote_flush(u64 old, u64 new)
5197 {
5198 	if (!is_shadow_present_pte(old))
5199 		return false;
5200 	if (!is_shadow_present_pte(new))
5201 		return true;
5202 	if ((old ^ new) & PT64_BASE_ADDR_MASK)
5203 		return true;
5204 	old ^= shadow_nx_mask;
5205 	new ^= shadow_nx_mask;
5206 	return (old & ~new & PT64_PERM_MASK) != 0;
5207 }
5208 
mmu_pte_write_fetch_gpte(struct kvm_vcpu * vcpu,gpa_t * gpa,int * bytes)5209 static u64 mmu_pte_write_fetch_gpte(struct kvm_vcpu *vcpu, gpa_t *gpa,
5210 				    int *bytes)
5211 {
5212 	u64 gentry = 0;
5213 	int r;
5214 
5215 	/*
5216 	 * Assume that the pte write on a page table of the same type
5217 	 * as the current vcpu paging mode since we update the sptes only
5218 	 * when they have the same mode.
5219 	 */
5220 	if (is_pae(vcpu) && *bytes == 4) {
5221 		/* Handle a 32-bit guest writing two halves of a 64-bit gpte */
5222 		*gpa &= ~(gpa_t)7;
5223 		*bytes = 8;
5224 	}
5225 
5226 	if (*bytes == 4 || *bytes == 8) {
5227 		r = kvm_vcpu_read_guest_atomic(vcpu, *gpa, &gentry, *bytes);
5228 		if (r)
5229 			gentry = 0;
5230 	}
5231 
5232 	return gentry;
5233 }
5234 
5235 /*
5236  * If we're seeing too many writes to a page, it may no longer be a page table,
5237  * or we may be forking, in which case it is better to unmap the page.
5238  */
detect_write_flooding(struct kvm_mmu_page * sp)5239 static bool detect_write_flooding(struct kvm_mmu_page *sp)
5240 {
5241 	/*
5242 	 * Skip write-flooding detected for the sp whose level is 1, because
5243 	 * it can become unsync, then the guest page is not write-protected.
5244 	 */
5245 	if (sp->role.level == PG_LEVEL_4K)
5246 		return false;
5247 
5248 	atomic_inc(&sp->write_flooding_count);
5249 	return atomic_read(&sp->write_flooding_count) >= 3;
5250 }
5251 
5252 /*
5253  * Misaligned accesses are too much trouble to fix up; also, they usually
5254  * indicate a page is not used as a page table.
5255  */
detect_write_misaligned(struct kvm_mmu_page * sp,gpa_t gpa,int bytes)5256 static bool detect_write_misaligned(struct kvm_mmu_page *sp, gpa_t gpa,
5257 				    int bytes)
5258 {
5259 	unsigned offset, pte_size, misaligned;
5260 
5261 	pgprintk("misaligned: gpa %llx bytes %d role %x\n",
5262 		 gpa, bytes, sp->role.word);
5263 
5264 	offset = offset_in_page(gpa);
5265 	pte_size = sp->role.has_4_byte_gpte ? 4 : 8;
5266 
5267 	/*
5268 	 * Sometimes, the OS only writes the last one bytes to update status
5269 	 * bits, for example, in linux, andb instruction is used in clear_bit().
5270 	 */
5271 	if (!(offset & (pte_size - 1)) && bytes == 1)
5272 		return false;
5273 
5274 	misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1);
5275 	misaligned |= bytes < 4;
5276 
5277 	return misaligned;
5278 }
5279 
get_written_sptes(struct kvm_mmu_page * sp,gpa_t gpa,int * nspte)5280 static u64 *get_written_sptes(struct kvm_mmu_page *sp, gpa_t gpa, int *nspte)
5281 {
5282 	unsigned page_offset, quadrant;
5283 	u64 *spte;
5284 	int level;
5285 
5286 	page_offset = offset_in_page(gpa);
5287 	level = sp->role.level;
5288 	*nspte = 1;
5289 	if (sp->role.has_4_byte_gpte) {
5290 		page_offset <<= 1;	/* 32->64 */
5291 		/*
5292 		 * A 32-bit pde maps 4MB while the shadow pdes map
5293 		 * only 2MB.  So we need to double the offset again
5294 		 * and zap two pdes instead of one.
5295 		 */
5296 		if (level == PT32_ROOT_LEVEL) {
5297 			page_offset &= ~7; /* kill rounding error */
5298 			page_offset <<= 1;
5299 			*nspte = 2;
5300 		}
5301 		quadrant = page_offset >> PAGE_SHIFT;
5302 		page_offset &= ~PAGE_MASK;
5303 		if (quadrant != sp->role.quadrant)
5304 			return NULL;
5305 	}
5306 
5307 	spte = &sp->spt[page_offset / sizeof(*spte)];
5308 	return spte;
5309 }
5310 
kvm_mmu_pte_write(struct kvm_vcpu * vcpu,gpa_t gpa,const u8 * new,int bytes,struct kvm_page_track_notifier_node * node)5311 static void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
5312 			      const u8 *new, int bytes,
5313 			      struct kvm_page_track_notifier_node *node)
5314 {
5315 	gfn_t gfn = gpa >> PAGE_SHIFT;
5316 	struct kvm_mmu_page *sp;
5317 	LIST_HEAD(invalid_list);
5318 	u64 entry, gentry, *spte;
5319 	int npte;
5320 	bool flush = false;
5321 
5322 	/*
5323 	 * If we don't have indirect shadow pages, it means no page is
5324 	 * write-protected, so we can exit simply.
5325 	 */
5326 	if (!READ_ONCE(vcpu->kvm->arch.indirect_shadow_pages))
5327 		return;
5328 
5329 	pgprintk("%s: gpa %llx bytes %d\n", __func__, gpa, bytes);
5330 
5331 	/*
5332 	 * No need to care whether allocation memory is successful
5333 	 * or not since pte prefetch is skipped if it does not have
5334 	 * enough objects in the cache.
5335 	 */
5336 	mmu_topup_memory_caches(vcpu, true);
5337 
5338 	write_lock(&vcpu->kvm->mmu_lock);
5339 
5340 	gentry = mmu_pte_write_fetch_gpte(vcpu, &gpa, &bytes);
5341 
5342 	++vcpu->kvm->stat.mmu_pte_write;
5343 
5344 	for_each_gfn_valid_sp_with_gptes(vcpu->kvm, sp, gfn) {
5345 		if (detect_write_misaligned(sp, gpa, bytes) ||
5346 		      detect_write_flooding(sp)) {
5347 			kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list);
5348 			++vcpu->kvm->stat.mmu_flooded;
5349 			continue;
5350 		}
5351 
5352 		spte = get_written_sptes(sp, gpa, &npte);
5353 		if (!spte)
5354 			continue;
5355 
5356 		while (npte--) {
5357 			entry = *spte;
5358 			mmu_page_zap_pte(vcpu->kvm, sp, spte, NULL);
5359 			if (gentry && sp->role.level != PG_LEVEL_4K)
5360 				++vcpu->kvm->stat.mmu_pde_zapped;
5361 			if (need_remote_flush(entry, *spte))
5362 				flush = true;
5363 			++spte;
5364 		}
5365 	}
5366 	kvm_mmu_remote_flush_or_zap(vcpu->kvm, &invalid_list, flush);
5367 	write_unlock(&vcpu->kvm->mmu_lock);
5368 }
5369 
kvm_mmu_page_fault(struct kvm_vcpu * vcpu,gpa_t cr2_or_gpa,u64 error_code,void * insn,int insn_len)5370 int noinline kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 error_code,
5371 		       void *insn, int insn_len)
5372 {
5373 	int r, emulation_type = EMULTYPE_PF;
5374 	bool direct = vcpu->arch.mmu->root_role.direct;
5375 
5376 	if (WARN_ON(!VALID_PAGE(vcpu->arch.mmu->root.hpa)))
5377 		return RET_PF_RETRY;
5378 
5379 	r = RET_PF_INVALID;
5380 	if (unlikely(error_code & PFERR_RSVD_MASK)) {
5381 		r = handle_mmio_page_fault(vcpu, cr2_or_gpa, direct);
5382 		if (r == RET_PF_EMULATE)
5383 			goto emulate;
5384 	}
5385 
5386 	if (r == RET_PF_INVALID) {
5387 		r = kvm_mmu_do_page_fault(vcpu, cr2_or_gpa,
5388 					  lower_32_bits(error_code), false);
5389 		if (KVM_BUG_ON(r == RET_PF_INVALID, vcpu->kvm))
5390 			return -EIO;
5391 	}
5392 
5393 	if (r < 0)
5394 		return r;
5395 	if (r != RET_PF_EMULATE)
5396 		return 1;
5397 
5398 	/*
5399 	 * Before emulating the instruction, check if the error code
5400 	 * was due to a RO violation while translating the guest page.
5401 	 * This can occur when using nested virtualization with nested
5402 	 * paging in both guests. If true, we simply unprotect the page
5403 	 * and resume the guest.
5404 	 */
5405 	if (vcpu->arch.mmu->root_role.direct &&
5406 	    (error_code & PFERR_NESTED_GUEST_PAGE) == PFERR_NESTED_GUEST_PAGE) {
5407 		kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(cr2_or_gpa));
5408 		return 1;
5409 	}
5410 
5411 	/*
5412 	 * vcpu->arch.mmu.page_fault returned RET_PF_EMULATE, but we can still
5413 	 * optimistically try to just unprotect the page and let the processor
5414 	 * re-execute the instruction that caused the page fault.  Do not allow
5415 	 * retrying MMIO emulation, as it's not only pointless but could also
5416 	 * cause us to enter an infinite loop because the processor will keep
5417 	 * faulting on the non-existent MMIO address.  Retrying an instruction
5418 	 * from a nested guest is also pointless and dangerous as we are only
5419 	 * explicitly shadowing L1's page tables, i.e. unprotecting something
5420 	 * for L1 isn't going to magically fix whatever issue cause L2 to fail.
5421 	 */
5422 	if (!mmio_info_in_cache(vcpu, cr2_or_gpa, direct) && !is_guest_mode(vcpu))
5423 		emulation_type |= EMULTYPE_ALLOW_RETRY_PF;
5424 emulate:
5425 	return x86_emulate_instruction(vcpu, cr2_or_gpa, emulation_type, insn,
5426 				       insn_len);
5427 }
5428 EXPORT_SYMBOL_GPL(kvm_mmu_page_fault);
5429 
kvm_mmu_invalidate_gva(struct kvm_vcpu * vcpu,struct kvm_mmu * mmu,gva_t gva,hpa_t root_hpa)5430 void kvm_mmu_invalidate_gva(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
5431 			    gva_t gva, hpa_t root_hpa)
5432 {
5433 	int i;
5434 
5435 	/* It's actually a GPA for vcpu->arch.guest_mmu.  */
5436 	if (mmu != &vcpu->arch.guest_mmu) {
5437 		/* INVLPG on a non-canonical address is a NOP according to the SDM.  */
5438 		if (is_noncanonical_address(gva, vcpu))
5439 			return;
5440 
5441 		static_call(kvm_x86_flush_tlb_gva)(vcpu, gva);
5442 	}
5443 
5444 	if (!mmu->invlpg)
5445 		return;
5446 
5447 	if (root_hpa == INVALID_PAGE) {
5448 		mmu->invlpg(vcpu, gva, mmu->root.hpa);
5449 
5450 		/*
5451 		 * INVLPG is required to invalidate any global mappings for the VA,
5452 		 * irrespective of PCID. Since it would take us roughly similar amount
5453 		 * of work to determine whether any of the prev_root mappings of the VA
5454 		 * is marked global, or to just sync it blindly, so we might as well
5455 		 * just always sync it.
5456 		 *
5457 		 * Mappings not reachable via the current cr3 or the prev_roots will be
5458 		 * synced when switching to that cr3, so nothing needs to be done here
5459 		 * for them.
5460 		 */
5461 		for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
5462 			if (VALID_PAGE(mmu->prev_roots[i].hpa))
5463 				mmu->invlpg(vcpu, gva, mmu->prev_roots[i].hpa);
5464 	} else {
5465 		mmu->invlpg(vcpu, gva, root_hpa);
5466 	}
5467 }
5468 
kvm_mmu_invlpg(struct kvm_vcpu * vcpu,gva_t gva)5469 void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
5470 {
5471 	kvm_mmu_invalidate_gva(vcpu, vcpu->arch.walk_mmu, gva, INVALID_PAGE);
5472 	++vcpu->stat.invlpg;
5473 }
5474 EXPORT_SYMBOL_GPL(kvm_mmu_invlpg);
5475 
5476 
kvm_mmu_invpcid_gva(struct kvm_vcpu * vcpu,gva_t gva,unsigned long pcid)5477 void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid)
5478 {
5479 	struct kvm_mmu *mmu = vcpu->arch.mmu;
5480 	bool tlb_flush = false;
5481 	uint i;
5482 
5483 	if (pcid == kvm_get_active_pcid(vcpu)) {
5484 		if (mmu->invlpg)
5485 			mmu->invlpg(vcpu, gva, mmu->root.hpa);
5486 		tlb_flush = true;
5487 	}
5488 
5489 	for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) {
5490 		if (VALID_PAGE(mmu->prev_roots[i].hpa) &&
5491 		    pcid == kvm_get_pcid(vcpu, mmu->prev_roots[i].pgd)) {
5492 			if (mmu->invlpg)
5493 				mmu->invlpg(vcpu, gva, mmu->prev_roots[i].hpa);
5494 			tlb_flush = true;
5495 		}
5496 	}
5497 
5498 	if (tlb_flush)
5499 		static_call(kvm_x86_flush_tlb_gva)(vcpu, gva);
5500 
5501 	++vcpu->stat.invlpg;
5502 
5503 	/*
5504 	 * Mappings not reachable via the current cr3 or the prev_roots will be
5505 	 * synced when switching to that cr3, so nothing needs to be done here
5506 	 * for them.
5507 	 */
5508 }
5509 
kvm_configure_mmu(bool enable_tdp,int tdp_forced_root_level,int tdp_max_root_level,int tdp_huge_page_level)5510 void kvm_configure_mmu(bool enable_tdp, int tdp_forced_root_level,
5511 		       int tdp_max_root_level, int tdp_huge_page_level)
5512 {
5513 	tdp_enabled = enable_tdp;
5514 	tdp_root_level = tdp_forced_root_level;
5515 	max_tdp_level = tdp_max_root_level;
5516 
5517 	/*
5518 	 * max_huge_page_level reflects KVM's MMU capabilities irrespective
5519 	 * of kernel support, e.g. KVM may be capable of using 1GB pages when
5520 	 * the kernel is not.  But, KVM never creates a page size greater than
5521 	 * what is used by the kernel for any given HVA, i.e. the kernel's
5522 	 * capabilities are ultimately consulted by kvm_mmu_hugepage_adjust().
5523 	 */
5524 	if (tdp_enabled)
5525 		max_huge_page_level = tdp_huge_page_level;
5526 	else if (boot_cpu_has(X86_FEATURE_GBPAGES))
5527 		max_huge_page_level = PG_LEVEL_1G;
5528 	else
5529 		max_huge_page_level = PG_LEVEL_2M;
5530 }
5531 EXPORT_SYMBOL_GPL(kvm_configure_mmu);
5532 
5533 /* The return value indicates if tlb flush on all vcpus is needed. */
5534 typedef bool (*slot_level_handler) (struct kvm *kvm,
5535 				    struct kvm_rmap_head *rmap_head,
5536 				    const struct kvm_memory_slot *slot);
5537 
5538 /* The caller should hold mmu-lock before calling this function. */
5539 static __always_inline bool
slot_handle_level_range(struct kvm * kvm,const struct kvm_memory_slot * memslot,slot_level_handler fn,int start_level,int end_level,gfn_t start_gfn,gfn_t end_gfn,bool flush_on_yield,bool flush)5540 slot_handle_level_range(struct kvm *kvm, const struct kvm_memory_slot *memslot,
5541 			slot_level_handler fn, int start_level, int end_level,
5542 			gfn_t start_gfn, gfn_t end_gfn, bool flush_on_yield,
5543 			bool flush)
5544 {
5545 	struct slot_rmap_walk_iterator iterator;
5546 
5547 	for_each_slot_rmap_range(memslot, start_level, end_level, start_gfn,
5548 			end_gfn, &iterator) {
5549 		if (iterator.rmap)
5550 			flush |= fn(kvm, iterator.rmap, memslot);
5551 
5552 		if (need_resched() || rwlock_needbreak(&kvm->mmu_lock)) {
5553 			if (flush && flush_on_yield) {
5554 				kvm_flush_remote_tlbs_with_address(kvm,
5555 						start_gfn,
5556 						iterator.gfn - start_gfn + 1);
5557 				flush = false;
5558 			}
5559 			cond_resched_rwlock_write(&kvm->mmu_lock);
5560 		}
5561 	}
5562 
5563 	return flush;
5564 }
5565 
5566 static __always_inline bool
slot_handle_level(struct kvm * kvm,const struct kvm_memory_slot * memslot,slot_level_handler fn,int start_level,int end_level,bool flush_on_yield)5567 slot_handle_level(struct kvm *kvm, const struct kvm_memory_slot *memslot,
5568 		  slot_level_handler fn, int start_level, int end_level,
5569 		  bool flush_on_yield)
5570 {
5571 	return slot_handle_level_range(kvm, memslot, fn, start_level,
5572 			end_level, memslot->base_gfn,
5573 			memslot->base_gfn + memslot->npages - 1,
5574 			flush_on_yield, false);
5575 }
5576 
5577 static __always_inline bool
slot_handle_level_4k(struct kvm * kvm,const struct kvm_memory_slot * memslot,slot_level_handler fn,bool flush_on_yield)5578 slot_handle_level_4k(struct kvm *kvm, const struct kvm_memory_slot *memslot,
5579 		     slot_level_handler fn, bool flush_on_yield)
5580 {
5581 	return slot_handle_level(kvm, memslot, fn, PG_LEVEL_4K,
5582 				 PG_LEVEL_4K, flush_on_yield);
5583 }
5584 
free_mmu_pages(struct kvm_mmu * mmu)5585 static void free_mmu_pages(struct kvm_mmu *mmu)
5586 {
5587 	if (!tdp_enabled && mmu->pae_root)
5588 		set_memory_encrypted((unsigned long)mmu->pae_root, 1);
5589 	free_page((unsigned long)mmu->pae_root);
5590 	free_page((unsigned long)mmu->pml4_root);
5591 	free_page((unsigned long)mmu->pml5_root);
5592 }
5593 
__kvm_mmu_create(struct kvm_vcpu * vcpu,struct kvm_mmu * mmu)5594 static int __kvm_mmu_create(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu)
5595 {
5596 	struct page *page;
5597 	int i;
5598 
5599 	mmu->root.hpa = INVALID_PAGE;
5600 	mmu->root.pgd = 0;
5601 	for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
5602 		mmu->prev_roots[i] = KVM_MMU_ROOT_INFO_INVALID;
5603 
5604 	/* vcpu->arch.guest_mmu isn't used when !tdp_enabled. */
5605 	if (!tdp_enabled && mmu == &vcpu->arch.guest_mmu)
5606 		return 0;
5607 
5608 	/*
5609 	 * When using PAE paging, the four PDPTEs are treated as 'root' pages,
5610 	 * while the PDP table is a per-vCPU construct that's allocated at MMU
5611 	 * creation.  When emulating 32-bit mode, cr3 is only 32 bits even on
5612 	 * x86_64.  Therefore we need to allocate the PDP table in the first
5613 	 * 4GB of memory, which happens to fit the DMA32 zone.  TDP paging
5614 	 * generally doesn't use PAE paging and can skip allocating the PDP
5615 	 * table.  The main exception, handled here, is SVM's 32-bit NPT.  The
5616 	 * other exception is for shadowing L1's 32-bit or PAE NPT on 64-bit
5617 	 * KVM; that horror is handled on-demand by mmu_alloc_special_roots().
5618 	 */
5619 	if (tdp_enabled && kvm_mmu_get_tdp_level(vcpu) > PT32E_ROOT_LEVEL)
5620 		return 0;
5621 
5622 	page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_DMA32);
5623 	if (!page)
5624 		return -ENOMEM;
5625 
5626 	mmu->pae_root = page_address(page);
5627 
5628 	/*
5629 	 * CR3 is only 32 bits when PAE paging is used, thus it's impossible to
5630 	 * get the CPU to treat the PDPTEs as encrypted.  Decrypt the page so
5631 	 * that KVM's writes and the CPU's reads get along.  Note, this is
5632 	 * only necessary when using shadow paging, as 64-bit NPT can get at
5633 	 * the C-bit even when shadowing 32-bit NPT, and SME isn't supported
5634 	 * by 32-bit kernels (when KVM itself uses 32-bit NPT).
5635 	 */
5636 	if (!tdp_enabled)
5637 		set_memory_decrypted((unsigned long)mmu->pae_root, 1);
5638 	else
5639 		WARN_ON_ONCE(shadow_me_value);
5640 
5641 	for (i = 0; i < 4; ++i)
5642 		mmu->pae_root[i] = INVALID_PAE_ROOT;
5643 
5644 	return 0;
5645 }
5646 
kvm_mmu_create(struct kvm_vcpu * vcpu)5647 int kvm_mmu_create(struct kvm_vcpu *vcpu)
5648 {
5649 	int ret;
5650 
5651 	vcpu->arch.mmu_pte_list_desc_cache.kmem_cache = pte_list_desc_cache;
5652 	vcpu->arch.mmu_pte_list_desc_cache.gfp_zero = __GFP_ZERO;
5653 
5654 	vcpu->arch.mmu_page_header_cache.kmem_cache = mmu_page_header_cache;
5655 	vcpu->arch.mmu_page_header_cache.gfp_zero = __GFP_ZERO;
5656 
5657 	vcpu->arch.mmu_shadow_page_cache.gfp_zero = __GFP_ZERO;
5658 
5659 	vcpu->arch.mmu = &vcpu->arch.root_mmu;
5660 	vcpu->arch.walk_mmu = &vcpu->arch.root_mmu;
5661 
5662 	ret = __kvm_mmu_create(vcpu, &vcpu->arch.guest_mmu);
5663 	if (ret)
5664 		return ret;
5665 
5666 	ret = __kvm_mmu_create(vcpu, &vcpu->arch.root_mmu);
5667 	if (ret)
5668 		goto fail_allocate_root;
5669 
5670 	return ret;
5671  fail_allocate_root:
5672 	free_mmu_pages(&vcpu->arch.guest_mmu);
5673 	return ret;
5674 }
5675 
5676 #define BATCH_ZAP_PAGES	10
kvm_zap_obsolete_pages(struct kvm * kvm)5677 static void kvm_zap_obsolete_pages(struct kvm *kvm)
5678 {
5679 	struct kvm_mmu_page *sp, *node;
5680 	int nr_zapped, batch = 0;
5681 	bool unstable;
5682 
5683 restart:
5684 	list_for_each_entry_safe_reverse(sp, node,
5685 	      &kvm->arch.active_mmu_pages, link) {
5686 		/*
5687 		 * No obsolete valid page exists before a newly created page
5688 		 * since active_mmu_pages is a FIFO list.
5689 		 */
5690 		if (!is_obsolete_sp(kvm, sp))
5691 			break;
5692 
5693 		/*
5694 		 * Invalid pages should never land back on the list of active
5695 		 * pages.  Skip the bogus page, otherwise we'll get stuck in an
5696 		 * infinite loop if the page gets put back on the list (again).
5697 		 */
5698 		if (WARN_ON(sp->role.invalid))
5699 			continue;
5700 
5701 		/*
5702 		 * No need to flush the TLB since we're only zapping shadow
5703 		 * pages with an obsolete generation number and all vCPUS have
5704 		 * loaded a new root, i.e. the shadow pages being zapped cannot
5705 		 * be in active use by the guest.
5706 		 */
5707 		if (batch >= BATCH_ZAP_PAGES &&
5708 		    cond_resched_rwlock_write(&kvm->mmu_lock)) {
5709 			batch = 0;
5710 			goto restart;
5711 		}
5712 
5713 		unstable = __kvm_mmu_prepare_zap_page(kvm, sp,
5714 				&kvm->arch.zapped_obsolete_pages, &nr_zapped);
5715 		batch += nr_zapped;
5716 
5717 		if (unstable)
5718 			goto restart;
5719 	}
5720 
5721 	/*
5722 	 * Kick all vCPUs (via remote TLB flush) before freeing the page tables
5723 	 * to ensure KVM is not in the middle of a lockless shadow page table
5724 	 * walk, which may reference the pages.  The remote TLB flush itself is
5725 	 * not required and is simply a convenient way to kick vCPUs as needed.
5726 	 * KVM performs a local TLB flush when allocating a new root (see
5727 	 * kvm_mmu_load()), and the reload in the caller ensure no vCPUs are
5728 	 * running with an obsolete MMU.
5729 	 */
5730 	kvm_mmu_commit_zap_page(kvm, &kvm->arch.zapped_obsolete_pages);
5731 }
5732 
5733 /*
5734  * Fast invalidate all shadow pages and use lock-break technique
5735  * to zap obsolete pages.
5736  *
5737  * It's required when memslot is being deleted or VM is being
5738  * destroyed, in these cases, we should ensure that KVM MMU does
5739  * not use any resource of the being-deleted slot or all slots
5740  * after calling the function.
5741  */
kvm_mmu_zap_all_fast(struct kvm * kvm)5742 static void kvm_mmu_zap_all_fast(struct kvm *kvm)
5743 {
5744 	lockdep_assert_held(&kvm->slots_lock);
5745 
5746 	write_lock(&kvm->mmu_lock);
5747 	trace_kvm_mmu_zap_all_fast(kvm);
5748 
5749 	/*
5750 	 * Toggle mmu_valid_gen between '0' and '1'.  Because slots_lock is
5751 	 * held for the entire duration of zapping obsolete pages, it's
5752 	 * impossible for there to be multiple invalid generations associated
5753 	 * with *valid* shadow pages at any given time, i.e. there is exactly
5754 	 * one valid generation and (at most) one invalid generation.
5755 	 */
5756 	kvm->arch.mmu_valid_gen = kvm->arch.mmu_valid_gen ? 0 : 1;
5757 
5758 	/*
5759 	 * In order to ensure all vCPUs drop their soon-to-be invalid roots,
5760 	 * invalidating TDP MMU roots must be done while holding mmu_lock for
5761 	 * write and in the same critical section as making the reload request,
5762 	 * e.g. before kvm_zap_obsolete_pages() could drop mmu_lock and yield.
5763 	 */
5764 	if (is_tdp_mmu_enabled(kvm))
5765 		kvm_tdp_mmu_invalidate_all_roots(kvm);
5766 
5767 	/*
5768 	 * Notify all vcpus to reload its shadow page table and flush TLB.
5769 	 * Then all vcpus will switch to new shadow page table with the new
5770 	 * mmu_valid_gen.
5771 	 *
5772 	 * Note: we need to do this under the protection of mmu_lock,
5773 	 * otherwise, vcpu would purge shadow page but miss tlb flush.
5774 	 */
5775 	kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_FREE_OBSOLETE_ROOTS);
5776 
5777 	kvm_zap_obsolete_pages(kvm);
5778 
5779 	write_unlock(&kvm->mmu_lock);
5780 
5781 	/*
5782 	 * Zap the invalidated TDP MMU roots, all SPTEs must be dropped before
5783 	 * returning to the caller, e.g. if the zap is in response to a memslot
5784 	 * deletion, mmu_notifier callbacks will be unable to reach the SPTEs
5785 	 * associated with the deleted memslot once the update completes, and
5786 	 * Deferring the zap until the final reference to the root is put would
5787 	 * lead to use-after-free.
5788 	 */
5789 	if (is_tdp_mmu_enabled(kvm))
5790 		kvm_tdp_mmu_zap_invalidated_roots(kvm);
5791 }
5792 
kvm_has_zapped_obsolete_pages(struct kvm * kvm)5793 static bool kvm_has_zapped_obsolete_pages(struct kvm *kvm)
5794 {
5795 	return unlikely(!list_empty_careful(&kvm->arch.zapped_obsolete_pages));
5796 }
5797 
kvm_mmu_invalidate_zap_pages_in_memslot(struct kvm * kvm,struct kvm_memory_slot * slot,struct kvm_page_track_notifier_node * node)5798 static void kvm_mmu_invalidate_zap_pages_in_memslot(struct kvm *kvm,
5799 			struct kvm_memory_slot *slot,
5800 			struct kvm_page_track_notifier_node *node)
5801 {
5802 	kvm_mmu_zap_all_fast(kvm);
5803 }
5804 
kvm_mmu_init_vm(struct kvm * kvm)5805 int kvm_mmu_init_vm(struct kvm *kvm)
5806 {
5807 	struct kvm_page_track_notifier_node *node = &kvm->arch.mmu_sp_tracker;
5808 	int r;
5809 
5810 	INIT_LIST_HEAD(&kvm->arch.active_mmu_pages);
5811 	INIT_LIST_HEAD(&kvm->arch.zapped_obsolete_pages);
5812 	INIT_LIST_HEAD(&kvm->arch.lpage_disallowed_mmu_pages);
5813 	spin_lock_init(&kvm->arch.mmu_unsync_pages_lock);
5814 
5815 	r = kvm_mmu_init_tdp_mmu(kvm);
5816 	if (r < 0)
5817 		return r;
5818 
5819 	node->track_write = kvm_mmu_pte_write;
5820 	node->track_flush_slot = kvm_mmu_invalidate_zap_pages_in_memslot;
5821 	kvm_page_track_register_notifier(kvm, node);
5822 	return 0;
5823 }
5824 
kvm_mmu_uninit_vm(struct kvm * kvm)5825 void kvm_mmu_uninit_vm(struct kvm *kvm)
5826 {
5827 	struct kvm_page_track_notifier_node *node = &kvm->arch.mmu_sp_tracker;
5828 
5829 	kvm_page_track_unregister_notifier(kvm, node);
5830 
5831 	kvm_mmu_uninit_tdp_mmu(kvm);
5832 }
5833 
__kvm_zap_rmaps(struct kvm * kvm,gfn_t gfn_start,gfn_t gfn_end)5834 static bool __kvm_zap_rmaps(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end)
5835 {
5836 	const struct kvm_memory_slot *memslot;
5837 	struct kvm_memslots *slots;
5838 	struct kvm_memslot_iter iter;
5839 	bool flush = false;
5840 	gfn_t start, end;
5841 	int i;
5842 
5843 	if (!kvm_memslots_have_rmaps(kvm))
5844 		return flush;
5845 
5846 	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
5847 		slots = __kvm_memslots(kvm, i);
5848 
5849 		kvm_for_each_memslot_in_gfn_range(&iter, slots, gfn_start, gfn_end) {
5850 			memslot = iter.slot;
5851 			start = max(gfn_start, memslot->base_gfn);
5852 			end = min(gfn_end, memslot->base_gfn + memslot->npages);
5853 			if (WARN_ON_ONCE(start >= end))
5854 				continue;
5855 
5856 			flush = slot_handle_level_range(kvm, memslot, kvm_zap_rmapp,
5857 
5858 							PG_LEVEL_4K, KVM_MAX_HUGEPAGE_LEVEL,
5859 							start, end - 1, true, flush);
5860 		}
5861 	}
5862 
5863 	return flush;
5864 }
5865 
5866 /*
5867  * Invalidate (zap) SPTEs that cover GFNs from gfn_start and up to gfn_end
5868  * (not including it)
5869  */
kvm_zap_gfn_range(struct kvm * kvm,gfn_t gfn_start,gfn_t gfn_end)5870 void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end)
5871 {
5872 	bool flush;
5873 	int i;
5874 
5875 	if (WARN_ON_ONCE(gfn_end <= gfn_start))
5876 		return;
5877 
5878 	write_lock(&kvm->mmu_lock);
5879 
5880 	kvm_inc_notifier_count(kvm, gfn_start, gfn_end);
5881 
5882 	flush = __kvm_zap_rmaps(kvm, gfn_start, gfn_end);
5883 
5884 	if (is_tdp_mmu_enabled(kvm)) {
5885 		for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++)
5886 			flush = kvm_tdp_mmu_zap_leafs(kvm, i, gfn_start,
5887 						      gfn_end, true, flush);
5888 	}
5889 
5890 	if (flush)
5891 		kvm_flush_remote_tlbs_with_address(kvm, gfn_start,
5892 						   gfn_end - gfn_start);
5893 
5894 	kvm_dec_notifier_count(kvm, gfn_start, gfn_end);
5895 
5896 	write_unlock(&kvm->mmu_lock);
5897 }
5898 
slot_rmap_write_protect(struct kvm * kvm,struct kvm_rmap_head * rmap_head,const struct kvm_memory_slot * slot)5899 static bool slot_rmap_write_protect(struct kvm *kvm,
5900 				    struct kvm_rmap_head *rmap_head,
5901 				    const struct kvm_memory_slot *slot)
5902 {
5903 	return rmap_write_protect(rmap_head, false);
5904 }
5905 
kvm_mmu_slot_remove_write_access(struct kvm * kvm,const struct kvm_memory_slot * memslot,int start_level)5906 void kvm_mmu_slot_remove_write_access(struct kvm *kvm,
5907 				      const struct kvm_memory_slot *memslot,
5908 				      int start_level)
5909 {
5910 	if (kvm_memslots_have_rmaps(kvm)) {
5911 		write_lock(&kvm->mmu_lock);
5912 		slot_handle_level(kvm, memslot, slot_rmap_write_protect,
5913 				  start_level, KVM_MAX_HUGEPAGE_LEVEL, false);
5914 		write_unlock(&kvm->mmu_lock);
5915 	}
5916 
5917 	if (is_tdp_mmu_enabled(kvm)) {
5918 		read_lock(&kvm->mmu_lock);
5919 		kvm_tdp_mmu_wrprot_slot(kvm, memslot, start_level);
5920 		read_unlock(&kvm->mmu_lock);
5921 	}
5922 }
5923 
5924 /* Must be called with the mmu_lock held in write-mode. */
kvm_mmu_try_split_huge_pages(struct kvm * kvm,const struct kvm_memory_slot * memslot,u64 start,u64 end,int target_level)5925 void kvm_mmu_try_split_huge_pages(struct kvm *kvm,
5926 				   const struct kvm_memory_slot *memslot,
5927 				   u64 start, u64 end,
5928 				   int target_level)
5929 {
5930 	if (is_tdp_mmu_enabled(kvm))
5931 		kvm_tdp_mmu_try_split_huge_pages(kvm, memslot, start, end,
5932 						 target_level, false);
5933 
5934 	/*
5935 	 * A TLB flush is unnecessary at this point for the same resons as in
5936 	 * kvm_mmu_slot_try_split_huge_pages().
5937 	 */
5938 }
5939 
kvm_mmu_slot_try_split_huge_pages(struct kvm * kvm,const struct kvm_memory_slot * memslot,int target_level)5940 void kvm_mmu_slot_try_split_huge_pages(struct kvm *kvm,
5941 					const struct kvm_memory_slot *memslot,
5942 					int target_level)
5943 {
5944 	u64 start = memslot->base_gfn;
5945 	u64 end = start + memslot->npages;
5946 
5947 	if (is_tdp_mmu_enabled(kvm)) {
5948 		read_lock(&kvm->mmu_lock);
5949 		kvm_tdp_mmu_try_split_huge_pages(kvm, memslot, start, end, target_level, true);
5950 		read_unlock(&kvm->mmu_lock);
5951 	}
5952 
5953 	/*
5954 	 * No TLB flush is necessary here. KVM will flush TLBs after
5955 	 * write-protecting and/or clearing dirty on the newly split SPTEs to
5956 	 * ensure that guest writes are reflected in the dirty log before the
5957 	 * ioctl to enable dirty logging on this memslot completes. Since the
5958 	 * split SPTEs retain the write and dirty bits of the huge SPTE, it is
5959 	 * safe for KVM to decide if a TLB flush is necessary based on the split
5960 	 * SPTEs.
5961 	 */
5962 }
5963 
kvm_mmu_zap_collapsible_spte(struct kvm * kvm,struct kvm_rmap_head * rmap_head,const struct kvm_memory_slot * slot)5964 static bool kvm_mmu_zap_collapsible_spte(struct kvm *kvm,
5965 					 struct kvm_rmap_head *rmap_head,
5966 					 const struct kvm_memory_slot *slot)
5967 {
5968 	u64 *sptep;
5969 	struct rmap_iterator iter;
5970 	int need_tlb_flush = 0;
5971 	kvm_pfn_t pfn;
5972 	struct kvm_mmu_page *sp;
5973 
5974 restart:
5975 	for_each_rmap_spte(rmap_head, &iter, sptep) {
5976 		sp = sptep_to_sp(sptep);
5977 		pfn = spte_to_pfn(*sptep);
5978 
5979 		/*
5980 		 * We cannot do huge page mapping for indirect shadow pages,
5981 		 * which are found on the last rmap (level = 1) when not using
5982 		 * tdp; such shadow pages are synced with the page table in
5983 		 * the guest, and the guest page table is using 4K page size
5984 		 * mapping if the indirect sp has level = 1.
5985 		 */
5986 		if (sp->role.direct && !kvm_is_reserved_pfn(pfn) &&
5987 		    sp->role.level < kvm_mmu_max_mapping_level(kvm, slot, sp->gfn,
5988 							       pfn, PG_LEVEL_NUM)) {
5989 			pte_list_remove(kvm, rmap_head, sptep);
5990 
5991 			if (kvm_available_flush_tlb_with_range())
5992 				kvm_flush_remote_tlbs_with_address(kvm, sp->gfn,
5993 					KVM_PAGES_PER_HPAGE(sp->role.level));
5994 			else
5995 				need_tlb_flush = 1;
5996 
5997 			goto restart;
5998 		}
5999 	}
6000 
6001 	return need_tlb_flush;
6002 }
6003 
kvm_mmu_zap_collapsible_sptes(struct kvm * kvm,const struct kvm_memory_slot * slot)6004 void kvm_mmu_zap_collapsible_sptes(struct kvm *kvm,
6005 				   const struct kvm_memory_slot *slot)
6006 {
6007 	if (kvm_memslots_have_rmaps(kvm)) {
6008 		write_lock(&kvm->mmu_lock);
6009 		/*
6010 		 * Zap only 4k SPTEs since the legacy MMU only supports dirty
6011 		 * logging at a 4k granularity and never creates collapsible
6012 		 * 2m SPTEs during dirty logging.
6013 		 */
6014 		if (slot_handle_level_4k(kvm, slot, kvm_mmu_zap_collapsible_spte, true))
6015 			kvm_arch_flush_remote_tlbs_memslot(kvm, slot);
6016 		write_unlock(&kvm->mmu_lock);
6017 	}
6018 
6019 	if (is_tdp_mmu_enabled(kvm)) {
6020 		read_lock(&kvm->mmu_lock);
6021 		kvm_tdp_mmu_zap_collapsible_sptes(kvm, slot);
6022 		read_unlock(&kvm->mmu_lock);
6023 	}
6024 }
6025 
kvm_arch_flush_remote_tlbs_memslot(struct kvm * kvm,const struct kvm_memory_slot * memslot)6026 void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm,
6027 					const struct kvm_memory_slot *memslot)
6028 {
6029 	/*
6030 	 * All current use cases for flushing the TLBs for a specific memslot
6031 	 * related to dirty logging, and many do the TLB flush out of mmu_lock.
6032 	 * The interaction between the various operations on memslot must be
6033 	 * serialized by slots_locks to ensure the TLB flush from one operation
6034 	 * is observed by any other operation on the same memslot.
6035 	 */
6036 	lockdep_assert_held(&kvm->slots_lock);
6037 	kvm_flush_remote_tlbs_with_address(kvm, memslot->base_gfn,
6038 					   memslot->npages);
6039 }
6040 
kvm_mmu_slot_leaf_clear_dirty(struct kvm * kvm,const struct kvm_memory_slot * memslot)6041 void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm,
6042 				   const struct kvm_memory_slot *memslot)
6043 {
6044 	if (kvm_memslots_have_rmaps(kvm)) {
6045 		write_lock(&kvm->mmu_lock);
6046 		/*
6047 		 * Clear dirty bits only on 4k SPTEs since the legacy MMU only
6048 		 * support dirty logging at a 4k granularity.
6049 		 */
6050 		slot_handle_level_4k(kvm, memslot, __rmap_clear_dirty, false);
6051 		write_unlock(&kvm->mmu_lock);
6052 	}
6053 
6054 	if (is_tdp_mmu_enabled(kvm)) {
6055 		read_lock(&kvm->mmu_lock);
6056 		kvm_tdp_mmu_clear_dirty_slot(kvm, memslot);
6057 		read_unlock(&kvm->mmu_lock);
6058 	}
6059 
6060 	/*
6061 	 * The caller will flush the TLBs after this function returns.
6062 	 *
6063 	 * It's also safe to flush TLBs out of mmu lock here as currently this
6064 	 * function is only used for dirty logging, in which case flushing TLB
6065 	 * out of mmu lock also guarantees no dirty pages will be lost in
6066 	 * dirty_bitmap.
6067 	 */
6068 }
6069 
kvm_mmu_zap_all(struct kvm * kvm)6070 void kvm_mmu_zap_all(struct kvm *kvm)
6071 {
6072 	struct kvm_mmu_page *sp, *node;
6073 	LIST_HEAD(invalid_list);
6074 	int ign;
6075 
6076 	write_lock(&kvm->mmu_lock);
6077 restart:
6078 	list_for_each_entry_safe(sp, node, &kvm->arch.active_mmu_pages, link) {
6079 		if (WARN_ON(sp->role.invalid))
6080 			continue;
6081 		if (__kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list, &ign))
6082 			goto restart;
6083 		if (cond_resched_rwlock_write(&kvm->mmu_lock))
6084 			goto restart;
6085 	}
6086 
6087 	kvm_mmu_commit_zap_page(kvm, &invalid_list);
6088 
6089 	if (is_tdp_mmu_enabled(kvm))
6090 		kvm_tdp_mmu_zap_all(kvm);
6091 
6092 	write_unlock(&kvm->mmu_lock);
6093 }
6094 
kvm_mmu_invalidate_mmio_sptes(struct kvm * kvm,u64 gen)6095 void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen)
6096 {
6097 	WARN_ON(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS);
6098 
6099 	gen &= MMIO_SPTE_GEN_MASK;
6100 
6101 	/*
6102 	 * Generation numbers are incremented in multiples of the number of
6103 	 * address spaces in order to provide unique generations across all
6104 	 * address spaces.  Strip what is effectively the address space
6105 	 * modifier prior to checking for a wrap of the MMIO generation so
6106 	 * that a wrap in any address space is detected.
6107 	 */
6108 	gen &= ~((u64)KVM_ADDRESS_SPACE_NUM - 1);
6109 
6110 	/*
6111 	 * The very rare case: if the MMIO generation number has wrapped,
6112 	 * zap all shadow pages.
6113 	 */
6114 	if (unlikely(gen == 0)) {
6115 		kvm_debug_ratelimited("kvm: zapping shadow pages for mmio generation wraparound\n");
6116 		kvm_mmu_zap_all_fast(kvm);
6117 	}
6118 }
6119 
6120 static unsigned long
mmu_shrink_scan(struct shrinker * shrink,struct shrink_control * sc)6121 mmu_shrink_scan(struct shrinker *shrink, struct shrink_control *sc)
6122 {
6123 	struct kvm *kvm;
6124 	int nr_to_scan = sc->nr_to_scan;
6125 	unsigned long freed = 0;
6126 
6127 	mutex_lock(&kvm_lock);
6128 
6129 	list_for_each_entry(kvm, &vm_list, vm_list) {
6130 		int idx;
6131 		LIST_HEAD(invalid_list);
6132 
6133 		/*
6134 		 * Never scan more than sc->nr_to_scan VM instances.
6135 		 * Will not hit this condition practically since we do not try
6136 		 * to shrink more than one VM and it is very unlikely to see
6137 		 * !n_used_mmu_pages so many times.
6138 		 */
6139 		if (!nr_to_scan--)
6140 			break;
6141 		/*
6142 		 * n_used_mmu_pages is accessed without holding kvm->mmu_lock
6143 		 * here. We may skip a VM instance errorneosly, but we do not
6144 		 * want to shrink a VM that only started to populate its MMU
6145 		 * anyway.
6146 		 */
6147 		if (!kvm->arch.n_used_mmu_pages &&
6148 		    !kvm_has_zapped_obsolete_pages(kvm))
6149 			continue;
6150 
6151 		idx = srcu_read_lock(&kvm->srcu);
6152 		write_lock(&kvm->mmu_lock);
6153 
6154 		if (kvm_has_zapped_obsolete_pages(kvm)) {
6155 			kvm_mmu_commit_zap_page(kvm,
6156 			      &kvm->arch.zapped_obsolete_pages);
6157 			goto unlock;
6158 		}
6159 
6160 		freed = kvm_mmu_zap_oldest_mmu_pages(kvm, sc->nr_to_scan);
6161 
6162 unlock:
6163 		write_unlock(&kvm->mmu_lock);
6164 		srcu_read_unlock(&kvm->srcu, idx);
6165 
6166 		/*
6167 		 * unfair on small ones
6168 		 * per-vm shrinkers cry out
6169 		 * sadness comes quickly
6170 		 */
6171 		list_move_tail(&kvm->vm_list, &vm_list);
6172 		break;
6173 	}
6174 
6175 	mutex_unlock(&kvm_lock);
6176 	return freed;
6177 }
6178 
6179 static unsigned long
mmu_shrink_count(struct shrinker * shrink,struct shrink_control * sc)6180 mmu_shrink_count(struct shrinker *shrink, struct shrink_control *sc)
6181 {
6182 	return percpu_counter_read_positive(&kvm_total_used_mmu_pages);
6183 }
6184 
6185 static struct shrinker mmu_shrinker = {
6186 	.count_objects = mmu_shrink_count,
6187 	.scan_objects = mmu_shrink_scan,
6188 	.seeks = DEFAULT_SEEKS * 10,
6189 };
6190 
mmu_destroy_caches(void)6191 static void mmu_destroy_caches(void)
6192 {
6193 	kmem_cache_destroy(pte_list_desc_cache);
6194 	kmem_cache_destroy(mmu_page_header_cache);
6195 }
6196 
get_nx_auto_mode(void)6197 static bool get_nx_auto_mode(void)
6198 {
6199 	/* Return true when CPU has the bug, and mitigations are ON */
6200 	return boot_cpu_has_bug(X86_BUG_ITLB_MULTIHIT) && !cpu_mitigations_off();
6201 }
6202 
__set_nx_huge_pages(bool val)6203 static void __set_nx_huge_pages(bool val)
6204 {
6205 	nx_huge_pages = itlb_multihit_kvm_mitigation = val;
6206 }
6207 
set_nx_huge_pages(const char * val,const struct kernel_param * kp)6208 static int set_nx_huge_pages(const char *val, const struct kernel_param *kp)
6209 {
6210 	bool old_val = nx_huge_pages;
6211 	bool new_val;
6212 
6213 	/* In "auto" mode deploy workaround only if CPU has the bug. */
6214 	if (sysfs_streq(val, "off"))
6215 		new_val = 0;
6216 	else if (sysfs_streq(val, "force"))
6217 		new_val = 1;
6218 	else if (sysfs_streq(val, "auto"))
6219 		new_val = get_nx_auto_mode();
6220 	else if (strtobool(val, &new_val) < 0)
6221 		return -EINVAL;
6222 
6223 	__set_nx_huge_pages(new_val);
6224 
6225 	if (new_val != old_val) {
6226 		struct kvm *kvm;
6227 
6228 		mutex_lock(&kvm_lock);
6229 
6230 		list_for_each_entry(kvm, &vm_list, vm_list) {
6231 			mutex_lock(&kvm->slots_lock);
6232 			kvm_mmu_zap_all_fast(kvm);
6233 			mutex_unlock(&kvm->slots_lock);
6234 
6235 			wake_up_process(kvm->arch.nx_lpage_recovery_thread);
6236 		}
6237 		mutex_unlock(&kvm_lock);
6238 	}
6239 
6240 	return 0;
6241 }
6242 
6243 /*
6244  * nx_huge_pages needs to be resolved to true/false when kvm.ko is loaded, as
6245  * its default value of -1 is technically undefined behavior for a boolean.
6246  * Forward the module init call to SPTE code so that it too can handle module
6247  * params that need to be resolved/snapshot.
6248  */
kvm_mmu_x86_module_init(void)6249 void __init kvm_mmu_x86_module_init(void)
6250 {
6251 	if (nx_huge_pages == -1)
6252 		__set_nx_huge_pages(get_nx_auto_mode());
6253 
6254 	kvm_mmu_spte_module_init();
6255 }
6256 
6257 /*
6258  * The bulk of the MMU initialization is deferred until the vendor module is
6259  * loaded as many of the masks/values may be modified by VMX or SVM, i.e. need
6260  * to be reset when a potentially different vendor module is loaded.
6261  */
kvm_mmu_vendor_module_init(void)6262 int kvm_mmu_vendor_module_init(void)
6263 {
6264 	int ret = -ENOMEM;
6265 
6266 	/*
6267 	 * MMU roles use union aliasing which is, generally speaking, an
6268 	 * undefined behavior. However, we supposedly know how compilers behave
6269 	 * and the current status quo is unlikely to change. Guardians below are
6270 	 * supposed to let us know if the assumption becomes false.
6271 	 */
6272 	BUILD_BUG_ON(sizeof(union kvm_mmu_page_role) != sizeof(u32));
6273 	BUILD_BUG_ON(sizeof(union kvm_mmu_extended_role) != sizeof(u32));
6274 	BUILD_BUG_ON(sizeof(union kvm_cpu_role) != sizeof(u64));
6275 
6276 	kvm_mmu_reset_all_pte_masks();
6277 
6278 	pte_list_desc_cache = kmem_cache_create("pte_list_desc",
6279 					    sizeof(struct pte_list_desc),
6280 					    0, SLAB_ACCOUNT, NULL);
6281 	if (!pte_list_desc_cache)
6282 		goto out;
6283 
6284 	mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header",
6285 						  sizeof(struct kvm_mmu_page),
6286 						  0, SLAB_ACCOUNT, NULL);
6287 	if (!mmu_page_header_cache)
6288 		goto out;
6289 
6290 	if (percpu_counter_init(&kvm_total_used_mmu_pages, 0, GFP_KERNEL))
6291 		goto out;
6292 
6293 	ret = register_shrinker(&mmu_shrinker);
6294 	if (ret)
6295 		goto out;
6296 
6297 	return 0;
6298 
6299 out:
6300 	mmu_destroy_caches();
6301 	return ret;
6302 }
6303 
kvm_mmu_destroy(struct kvm_vcpu * vcpu)6304 void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
6305 {
6306 	kvm_mmu_unload(vcpu);
6307 	free_mmu_pages(&vcpu->arch.root_mmu);
6308 	free_mmu_pages(&vcpu->arch.guest_mmu);
6309 	mmu_free_memory_caches(vcpu);
6310 }
6311 
kvm_mmu_vendor_module_exit(void)6312 void kvm_mmu_vendor_module_exit(void)
6313 {
6314 	mmu_destroy_caches();
6315 	percpu_counter_destroy(&kvm_total_used_mmu_pages);
6316 	unregister_shrinker(&mmu_shrinker);
6317 }
6318 
6319 /*
6320  * Calculate the effective recovery period, accounting for '0' meaning "let KVM
6321  * select a halving time of 1 hour".  Returns true if recovery is enabled.
6322  */
calc_nx_huge_pages_recovery_period(uint * period)6323 static bool calc_nx_huge_pages_recovery_period(uint *period)
6324 {
6325 	/*
6326 	 * Use READ_ONCE to get the params, this may be called outside of the
6327 	 * param setters, e.g. by the kthread to compute its next timeout.
6328 	 */
6329 	bool enabled = READ_ONCE(nx_huge_pages);
6330 	uint ratio = READ_ONCE(nx_huge_pages_recovery_ratio);
6331 
6332 	if (!enabled || !ratio)
6333 		return false;
6334 
6335 	*period = READ_ONCE(nx_huge_pages_recovery_period_ms);
6336 	if (!*period) {
6337 		/* Make sure the period is not less than one second.  */
6338 		ratio = min(ratio, 3600u);
6339 		*period = 60 * 60 * 1000 / ratio;
6340 	}
6341 	return true;
6342 }
6343 
set_nx_huge_pages_recovery_param(const char * val,const struct kernel_param * kp)6344 static int set_nx_huge_pages_recovery_param(const char *val, const struct kernel_param *kp)
6345 {
6346 	bool was_recovery_enabled, is_recovery_enabled;
6347 	uint old_period, new_period;
6348 	int err;
6349 
6350 	was_recovery_enabled = calc_nx_huge_pages_recovery_period(&old_period);
6351 
6352 	err = param_set_uint(val, kp);
6353 	if (err)
6354 		return err;
6355 
6356 	is_recovery_enabled = calc_nx_huge_pages_recovery_period(&new_period);
6357 
6358 	if (is_recovery_enabled &&
6359 	    (!was_recovery_enabled || old_period > new_period)) {
6360 		struct kvm *kvm;
6361 
6362 		mutex_lock(&kvm_lock);
6363 
6364 		list_for_each_entry(kvm, &vm_list, vm_list)
6365 			wake_up_process(kvm->arch.nx_lpage_recovery_thread);
6366 
6367 		mutex_unlock(&kvm_lock);
6368 	}
6369 
6370 	return err;
6371 }
6372 
kvm_recover_nx_lpages(struct kvm * kvm)6373 static void kvm_recover_nx_lpages(struct kvm *kvm)
6374 {
6375 	unsigned long nx_lpage_splits = kvm->stat.nx_lpage_splits;
6376 	int rcu_idx;
6377 	struct kvm_mmu_page *sp;
6378 	unsigned int ratio;
6379 	LIST_HEAD(invalid_list);
6380 	bool flush = false;
6381 	ulong to_zap;
6382 
6383 	rcu_idx = srcu_read_lock(&kvm->srcu);
6384 	write_lock(&kvm->mmu_lock);
6385 
6386 	/*
6387 	 * Zapping TDP MMU shadow pages, including the remote TLB flush, must
6388 	 * be done under RCU protection, because the pages are freed via RCU
6389 	 * callback.
6390 	 */
6391 	rcu_read_lock();
6392 
6393 	ratio = READ_ONCE(nx_huge_pages_recovery_ratio);
6394 	to_zap = ratio ? DIV_ROUND_UP(nx_lpage_splits, ratio) : 0;
6395 	for ( ; to_zap; --to_zap) {
6396 		if (list_empty(&kvm->arch.lpage_disallowed_mmu_pages))
6397 			break;
6398 
6399 		/*
6400 		 * We use a separate list instead of just using active_mmu_pages
6401 		 * because the number of lpage_disallowed pages is expected to
6402 		 * be relatively small compared to the total.
6403 		 */
6404 		sp = list_first_entry(&kvm->arch.lpage_disallowed_mmu_pages,
6405 				      struct kvm_mmu_page,
6406 				      lpage_disallowed_link);
6407 		WARN_ON_ONCE(!sp->lpage_disallowed);
6408 		if (is_tdp_mmu_page(sp)) {
6409 			flush |= kvm_tdp_mmu_zap_sp(kvm, sp);
6410 		} else {
6411 			kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);
6412 			WARN_ON_ONCE(sp->lpage_disallowed);
6413 		}
6414 
6415 		if (need_resched() || rwlock_needbreak(&kvm->mmu_lock)) {
6416 			kvm_mmu_remote_flush_or_zap(kvm, &invalid_list, flush);
6417 			rcu_read_unlock();
6418 
6419 			cond_resched_rwlock_write(&kvm->mmu_lock);
6420 			flush = false;
6421 
6422 			rcu_read_lock();
6423 		}
6424 	}
6425 	kvm_mmu_remote_flush_or_zap(kvm, &invalid_list, flush);
6426 
6427 	rcu_read_unlock();
6428 
6429 	write_unlock(&kvm->mmu_lock);
6430 	srcu_read_unlock(&kvm->srcu, rcu_idx);
6431 }
6432 
get_nx_lpage_recovery_timeout(u64 start_time)6433 static long get_nx_lpage_recovery_timeout(u64 start_time)
6434 {
6435 	bool enabled;
6436 	uint period;
6437 
6438 	enabled = calc_nx_huge_pages_recovery_period(&period);
6439 
6440 	return enabled ? start_time + msecs_to_jiffies(period) - get_jiffies_64()
6441 		       : MAX_SCHEDULE_TIMEOUT;
6442 }
6443 
kvm_nx_lpage_recovery_worker(struct kvm * kvm,uintptr_t data)6444 static int kvm_nx_lpage_recovery_worker(struct kvm *kvm, uintptr_t data)
6445 {
6446 	u64 start_time;
6447 	long remaining_time;
6448 
6449 	while (true) {
6450 		start_time = get_jiffies_64();
6451 		remaining_time = get_nx_lpage_recovery_timeout(start_time);
6452 
6453 		set_current_state(TASK_INTERRUPTIBLE);
6454 		while (!kthread_should_stop() && remaining_time > 0) {
6455 			schedule_timeout(remaining_time);
6456 			remaining_time = get_nx_lpage_recovery_timeout(start_time);
6457 			set_current_state(TASK_INTERRUPTIBLE);
6458 		}
6459 
6460 		set_current_state(TASK_RUNNING);
6461 
6462 		if (kthread_should_stop())
6463 			return 0;
6464 
6465 		kvm_recover_nx_lpages(kvm);
6466 	}
6467 }
6468 
kvm_mmu_post_init_vm(struct kvm * kvm)6469 int kvm_mmu_post_init_vm(struct kvm *kvm)
6470 {
6471 	int err;
6472 
6473 	err = kvm_vm_create_worker_thread(kvm, kvm_nx_lpage_recovery_worker, 0,
6474 					  "kvm-nx-lpage-recovery",
6475 					  &kvm->arch.nx_lpage_recovery_thread);
6476 	if (!err)
6477 		kthread_unpark(kvm->arch.nx_lpage_recovery_thread);
6478 
6479 	return err;
6480 }
6481 
kvm_mmu_pre_destroy_vm(struct kvm * kvm)6482 void kvm_mmu_pre_destroy_vm(struct kvm *kvm)
6483 {
6484 	if (kvm->arch.nx_lpage_recovery_thread)
6485 		kthread_stop(kvm->arch.nx_lpage_recovery_thread);
6486 }
6487