1 /* SPDX-License-Identifier: GPL-2.0-only */
2 #ifndef __KVM_HOST_H
3 #define __KVM_HOST_H
4 
5 
6 #include <linux/types.h>
7 #include <linux/hardirq.h>
8 #include <linux/list.h>
9 #include <linux/mutex.h>
10 #include <linux/spinlock.h>
11 #include <linux/signal.h>
12 #include <linux/sched.h>
13 #include <linux/sched/stat.h>
14 #include <linux/bug.h>
15 #include <linux/minmax.h>
16 #include <linux/mm.h>
17 #include <linux/mmu_notifier.h>
18 #include <linux/preempt.h>
19 #include <linux/msi.h>
20 #include <linux/slab.h>
21 #include <linux/vmalloc.h>
22 #include <linux/rcupdate.h>
23 #include <linux/ratelimit.h>
24 #include <linux/err.h>
25 #include <linux/irqflags.h>
26 #include <linux/context_tracking.h>
27 #include <linux/irqbypass.h>
28 #include <linux/rcuwait.h>
29 #include <linux/refcount.h>
30 #include <linux/nospec.h>
31 #include <linux/notifier.h>
32 #include <linux/ftrace.h>
33 #include <linux/hashtable.h>
34 #include <linux/instrumentation.h>
35 #include <linux/interval_tree.h>
36 #include <linux/rbtree.h>
37 #include <linux/xarray.h>
38 #include <asm/signal.h>
39 
40 #include <linux/kvm.h>
41 #include <linux/kvm_para.h>
42 
43 #include <linux/kvm_types.h>
44 
45 #include <asm/kvm_host.h>
46 #include <linux/kvm_dirty_ring.h>
47 
48 #ifndef KVM_MAX_VCPU_IDS
49 #define KVM_MAX_VCPU_IDS KVM_MAX_VCPUS
50 #endif
51 
52 /*
53  * The bit 16 ~ bit 31 of kvm_userspace_memory_region::flags are internally
54  * used in kvm, other bits are visible for userspace which are defined in
55  * include/linux/kvm_h.
56  */
57 #define KVM_MEMSLOT_INVALID	(1UL << 16)
58 
59 /*
60  * Bit 63 of the memslot generation number is an "update in-progress flag",
61  * e.g. is temporarily set for the duration of kvm_swap_active_memslots().
62  * This flag effectively creates a unique generation number that is used to
63  * mark cached memslot data, e.g. MMIO accesses, as potentially being stale,
64  * i.e. may (or may not) have come from the previous memslots generation.
65  *
66  * This is necessary because the actual memslots update is not atomic with
67  * respect to the generation number update.  Updating the generation number
68  * first would allow a vCPU to cache a spte from the old memslots using the
69  * new generation number, and updating the generation number after switching
70  * to the new memslots would allow cache hits using the old generation number
71  * to reference the defunct memslots.
72  *
73  * This mechanism is used to prevent getting hits in KVM's caches while a
74  * memslot update is in-progress, and to prevent cache hits *after* updating
75  * the actual generation number against accesses that were inserted into the
76  * cache *before* the memslots were updated.
77  */
78 #define KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS	BIT_ULL(63)
79 
80 /* Two fragments for cross MMIO pages. */
81 #define KVM_MAX_MMIO_FRAGMENTS	2
82 
83 #ifndef KVM_ADDRESS_SPACE_NUM
84 #define KVM_ADDRESS_SPACE_NUM	1
85 #endif
86 
87 /*
88  * For the normal pfn, the highest 12 bits should be zero,
89  * so we can mask bit 62 ~ bit 52  to indicate the error pfn,
90  * mask bit 63 to indicate the noslot pfn.
91  */
92 #define KVM_PFN_ERR_MASK	(0x7ffULL << 52)
93 #define KVM_PFN_ERR_NOSLOT_MASK	(0xfffULL << 52)
94 #define KVM_PFN_NOSLOT		(0x1ULL << 63)
95 
96 #define KVM_PFN_ERR_FAULT	(KVM_PFN_ERR_MASK)
97 #define KVM_PFN_ERR_HWPOISON	(KVM_PFN_ERR_MASK + 1)
98 #define KVM_PFN_ERR_RO_FAULT	(KVM_PFN_ERR_MASK + 2)
99 #define KVM_PFN_ERR_SIGPENDING	(KVM_PFN_ERR_MASK + 3)
100 
101 /*
102  * error pfns indicate that the gfn is in slot but faild to
103  * translate it to pfn on host.
104  */
is_error_pfn(kvm_pfn_t pfn)105 static inline bool is_error_pfn(kvm_pfn_t pfn)
106 {
107 	return !!(pfn & KVM_PFN_ERR_MASK);
108 }
109 
110 /*
111  * KVM_PFN_ERR_SIGPENDING indicates that fetching the PFN was interrupted
112  * by a pending signal.  Note, the signal may or may not be fatal.
113  */
is_sigpending_pfn(kvm_pfn_t pfn)114 static inline bool is_sigpending_pfn(kvm_pfn_t pfn)
115 {
116 	return pfn == KVM_PFN_ERR_SIGPENDING;
117 }
118 
119 /*
120  * error_noslot pfns indicate that the gfn can not be
121  * translated to pfn - it is not in slot or failed to
122  * translate it to pfn.
123  */
is_error_noslot_pfn(kvm_pfn_t pfn)124 static inline bool is_error_noslot_pfn(kvm_pfn_t pfn)
125 {
126 	return !!(pfn & KVM_PFN_ERR_NOSLOT_MASK);
127 }
128 
129 /* noslot pfn indicates that the gfn is not in slot. */
is_noslot_pfn(kvm_pfn_t pfn)130 static inline bool is_noslot_pfn(kvm_pfn_t pfn)
131 {
132 	return pfn == KVM_PFN_NOSLOT;
133 }
134 
135 /*
136  * architectures with KVM_HVA_ERR_BAD other than PAGE_OFFSET (e.g. s390)
137  * provide own defines and kvm_is_error_hva
138  */
139 #ifndef KVM_HVA_ERR_BAD
140 
141 #define KVM_HVA_ERR_BAD		(PAGE_OFFSET)
142 #define KVM_HVA_ERR_RO_BAD	(PAGE_OFFSET + PAGE_SIZE)
143 
kvm_is_error_hva(unsigned long addr)144 static inline bool kvm_is_error_hva(unsigned long addr)
145 {
146 	return addr >= PAGE_OFFSET;
147 }
148 
149 #endif
150 
151 #define KVM_ERR_PTR_BAD_PAGE	(ERR_PTR(-ENOENT))
152 
is_error_page(struct page * page)153 static inline bool is_error_page(struct page *page)
154 {
155 	return IS_ERR(page);
156 }
157 
158 #define KVM_REQUEST_MASK           GENMASK(7,0)
159 #define KVM_REQUEST_NO_WAKEUP      BIT(8)
160 #define KVM_REQUEST_WAIT           BIT(9)
161 #define KVM_REQUEST_NO_ACTION      BIT(10)
162 /*
163  * Architecture-independent vcpu->requests bit members
164  * Bits 3-7 are reserved for more arch-independent bits.
165  */
166 #define KVM_REQ_TLB_FLUSH		(0 | KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
167 #define KVM_REQ_VM_DEAD			(1 | KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
168 #define KVM_REQ_UNBLOCK			2
169 #define KVM_REQ_DIRTY_RING_SOFT_FULL	3
170 #define KVM_REQUEST_ARCH_BASE		8
171 
172 /*
173  * KVM_REQ_OUTSIDE_GUEST_MODE exists is purely as way to force the vCPU to
174  * OUTSIDE_GUEST_MODE.  KVM_REQ_OUTSIDE_GUEST_MODE differs from a vCPU "kick"
175  * in that it ensures the vCPU has reached OUTSIDE_GUEST_MODE before continuing
176  * on.  A kick only guarantees that the vCPU is on its way out, e.g. a previous
177  * kick may have set vcpu->mode to EXITING_GUEST_MODE, and so there's no
178  * guarantee the vCPU received an IPI and has actually exited guest mode.
179  */
180 #define KVM_REQ_OUTSIDE_GUEST_MODE	(KVM_REQUEST_NO_ACTION | KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
181 
182 #define KVM_ARCH_REQ_FLAGS(nr, flags) ({ \
183 	BUILD_BUG_ON((unsigned)(nr) >= (sizeof_field(struct kvm_vcpu, requests) * 8) - KVM_REQUEST_ARCH_BASE); \
184 	(unsigned)(((nr) + KVM_REQUEST_ARCH_BASE) | (flags)); \
185 })
186 #define KVM_ARCH_REQ(nr)           KVM_ARCH_REQ_FLAGS(nr, 0)
187 
188 bool kvm_make_vcpus_request_mask(struct kvm *kvm, unsigned int req,
189 				 unsigned long *vcpu_bitmap);
190 bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req);
191 bool kvm_make_all_cpus_request_except(struct kvm *kvm, unsigned int req,
192 				      struct kvm_vcpu *except);
193 
194 #define KVM_USERSPACE_IRQ_SOURCE_ID		0
195 #define KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID	1
196 
197 extern struct mutex kvm_lock;
198 extern struct list_head vm_list;
199 
200 struct kvm_io_range {
201 	gpa_t addr;
202 	int len;
203 	struct kvm_io_device *dev;
204 };
205 
206 #define NR_IOBUS_DEVS 1000
207 
208 struct kvm_io_bus {
209 	int dev_count;
210 	int ioeventfd_count;
211 	struct kvm_io_range range[];
212 };
213 
214 enum kvm_bus {
215 	KVM_MMIO_BUS,
216 	KVM_PIO_BUS,
217 	KVM_VIRTIO_CCW_NOTIFY_BUS,
218 	KVM_FAST_MMIO_BUS,
219 	KVM_NR_BUSES
220 };
221 
222 int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
223 		     int len, const void *val);
224 int kvm_io_bus_write_cookie(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx,
225 			    gpa_t addr, int len, const void *val, long cookie);
226 int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
227 		    int len, void *val);
228 int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
229 			    int len, struct kvm_io_device *dev);
230 int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
231 			      struct kvm_io_device *dev);
232 struct kvm_io_device *kvm_io_bus_get_dev(struct kvm *kvm, enum kvm_bus bus_idx,
233 					 gpa_t addr);
234 
235 #ifdef CONFIG_KVM_ASYNC_PF
236 struct kvm_async_pf {
237 	struct work_struct work;
238 	struct list_head link;
239 	struct list_head queue;
240 	struct kvm_vcpu *vcpu;
241 	struct mm_struct *mm;
242 	gpa_t cr2_or_gpa;
243 	unsigned long addr;
244 	struct kvm_arch_async_pf arch;
245 	bool   wakeup_all;
246 	bool notpresent_injected;
247 };
248 
249 void kvm_clear_async_pf_completion_queue(struct kvm_vcpu *vcpu);
250 void kvm_check_async_pf_completion(struct kvm_vcpu *vcpu);
251 bool kvm_setup_async_pf(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
252 			unsigned long hva, struct kvm_arch_async_pf *arch);
253 int kvm_async_pf_wakeup_all(struct kvm_vcpu *vcpu);
254 #endif
255 
256 #ifdef KVM_ARCH_WANT_MMU_NOTIFIER
257 union kvm_mmu_notifier_arg {
258 	pte_t pte;
259 };
260 
261 struct kvm_gfn_range {
262 	struct kvm_memory_slot *slot;
263 	gfn_t start;
264 	gfn_t end;
265 	union kvm_mmu_notifier_arg arg;
266 	bool may_block;
267 };
268 bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range);
269 bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range);
270 bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range);
271 bool kvm_set_spte_gfn(struct kvm *kvm, struct kvm_gfn_range *range);
272 #endif
273 
274 enum {
275 	OUTSIDE_GUEST_MODE,
276 	IN_GUEST_MODE,
277 	EXITING_GUEST_MODE,
278 	READING_SHADOW_PAGE_TABLES,
279 };
280 
281 #define KVM_UNMAPPED_PAGE	((void *) 0x500 + POISON_POINTER_DELTA)
282 
283 struct kvm_host_map {
284 	/*
285 	 * Only valid if the 'pfn' is managed by the host kernel (i.e. There is
286 	 * a 'struct page' for it. When using mem= kernel parameter some memory
287 	 * can be used as guest memory but they are not managed by host
288 	 * kernel).
289 	 * If 'pfn' is not managed by the host kernel, this field is
290 	 * initialized to KVM_UNMAPPED_PAGE.
291 	 */
292 	struct page *page;
293 	void *hva;
294 	kvm_pfn_t pfn;
295 	kvm_pfn_t gfn;
296 };
297 
298 /*
299  * Used to check if the mapping is valid or not. Never use 'kvm_host_map'
300  * directly to check for that.
301  */
kvm_vcpu_mapped(struct kvm_host_map * map)302 static inline bool kvm_vcpu_mapped(struct kvm_host_map *map)
303 {
304 	return !!map->hva;
305 }
306 
kvm_vcpu_can_poll(ktime_t cur,ktime_t stop)307 static inline bool kvm_vcpu_can_poll(ktime_t cur, ktime_t stop)
308 {
309 	return single_task_running() && !need_resched() && ktime_before(cur, stop);
310 }
311 
312 /*
313  * Sometimes a large or cross-page mmio needs to be broken up into separate
314  * exits for userspace servicing.
315  */
316 struct kvm_mmio_fragment {
317 	gpa_t gpa;
318 	void *data;
319 	unsigned len;
320 };
321 
322 struct kvm_vcpu {
323 	struct kvm *kvm;
324 #ifdef CONFIG_PREEMPT_NOTIFIERS
325 	struct preempt_notifier preempt_notifier;
326 #endif
327 	int cpu;
328 	int vcpu_id; /* id given by userspace at creation */
329 	int vcpu_idx; /* index into kvm->vcpu_array */
330 	int ____srcu_idx; /* Don't use this directly.  You've been warned. */
331 #ifdef CONFIG_PROVE_RCU
332 	int srcu_depth;
333 #endif
334 	int mode;
335 	u64 requests;
336 	unsigned long guest_debug;
337 
338 	struct mutex mutex;
339 	struct kvm_run *run;
340 
341 #ifndef __KVM_HAVE_ARCH_WQP
342 	struct rcuwait wait;
343 #endif
344 	struct pid __rcu *pid;
345 	int sigset_active;
346 	sigset_t sigset;
347 	unsigned int halt_poll_ns;
348 	bool valid_wakeup;
349 
350 #ifdef CONFIG_HAS_IOMEM
351 	int mmio_needed;
352 	int mmio_read_completed;
353 	int mmio_is_write;
354 	int mmio_cur_fragment;
355 	int mmio_nr_fragments;
356 	struct kvm_mmio_fragment mmio_fragments[KVM_MAX_MMIO_FRAGMENTS];
357 #endif
358 
359 #ifdef CONFIG_KVM_ASYNC_PF
360 	struct {
361 		u32 queued;
362 		struct list_head queue;
363 		struct list_head done;
364 		spinlock_t lock;
365 	} async_pf;
366 #endif
367 
368 #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
369 	/*
370 	 * Cpu relax intercept or pause loop exit optimization
371 	 * in_spin_loop: set when a vcpu does a pause loop exit
372 	 *  or cpu relax intercepted.
373 	 * dy_eligible: indicates whether vcpu is eligible for directed yield.
374 	 */
375 	struct {
376 		bool in_spin_loop;
377 		bool dy_eligible;
378 	} spin_loop;
379 #endif
380 	bool preempted;
381 	bool ready;
382 	struct kvm_vcpu_arch arch;
383 	struct kvm_vcpu_stat stat;
384 	char stats_id[KVM_STATS_NAME_SIZE];
385 	struct kvm_dirty_ring dirty_ring;
386 
387 	/*
388 	 * The most recently used memslot by this vCPU and the slots generation
389 	 * for which it is valid.
390 	 * No wraparound protection is needed since generations won't overflow in
391 	 * thousands of years, even assuming 1M memslot operations per second.
392 	 */
393 	struct kvm_memory_slot *last_used_slot;
394 	u64 last_used_slot_gen;
395 };
396 
397 /*
398  * Start accounting time towards a guest.
399  * Must be called before entering guest context.
400  */
guest_timing_enter_irqoff(void)401 static __always_inline void guest_timing_enter_irqoff(void)
402 {
403 	/*
404 	 * This is running in ioctl context so its safe to assume that it's the
405 	 * stime pending cputime to flush.
406 	 */
407 	instrumentation_begin();
408 	vtime_account_guest_enter();
409 	instrumentation_end();
410 }
411 
412 /*
413  * Enter guest context and enter an RCU extended quiescent state.
414  *
415  * Between guest_context_enter_irqoff() and guest_context_exit_irqoff() it is
416  * unsafe to use any code which may directly or indirectly use RCU, tracing
417  * (including IRQ flag tracing), or lockdep. All code in this period must be
418  * non-instrumentable.
419  */
guest_context_enter_irqoff(void)420 static __always_inline void guest_context_enter_irqoff(void)
421 {
422 	/*
423 	 * KVM does not hold any references to rcu protected data when it
424 	 * switches CPU into a guest mode. In fact switching to a guest mode
425 	 * is very similar to exiting to userspace from rcu point of view. In
426 	 * addition CPU may stay in a guest mode for quite a long time (up to
427 	 * one time slice). Lets treat guest mode as quiescent state, just like
428 	 * we do with user-mode execution.
429 	 */
430 	if (!context_tracking_guest_enter()) {
431 		instrumentation_begin();
432 		rcu_virt_note_context_switch();
433 		instrumentation_end();
434 	}
435 }
436 
437 /*
438  * Deprecated. Architectures should move to guest_timing_enter_irqoff() and
439  * guest_state_enter_irqoff().
440  */
guest_enter_irqoff(void)441 static __always_inline void guest_enter_irqoff(void)
442 {
443 	guest_timing_enter_irqoff();
444 	guest_context_enter_irqoff();
445 }
446 
447 /**
448  * guest_state_enter_irqoff - Fixup state when entering a guest
449  *
450  * Entry to a guest will enable interrupts, but the kernel state is interrupts
451  * disabled when this is invoked. Also tell RCU about it.
452  *
453  * 1) Trace interrupts on state
454  * 2) Invoke context tracking if enabled to adjust RCU state
455  * 3) Tell lockdep that interrupts are enabled
456  *
457  * Invoked from architecture specific code before entering a guest.
458  * Must be called with interrupts disabled and the caller must be
459  * non-instrumentable.
460  * The caller has to invoke guest_timing_enter_irqoff() before this.
461  *
462  * Note: this is analogous to exit_to_user_mode().
463  */
guest_state_enter_irqoff(void)464 static __always_inline void guest_state_enter_irqoff(void)
465 {
466 	instrumentation_begin();
467 	trace_hardirqs_on_prepare();
468 	lockdep_hardirqs_on_prepare();
469 	instrumentation_end();
470 
471 	guest_context_enter_irqoff();
472 	lockdep_hardirqs_on(CALLER_ADDR0);
473 }
474 
475 /*
476  * Exit guest context and exit an RCU extended quiescent state.
477  *
478  * Between guest_context_enter_irqoff() and guest_context_exit_irqoff() it is
479  * unsafe to use any code which may directly or indirectly use RCU, tracing
480  * (including IRQ flag tracing), or lockdep. All code in this period must be
481  * non-instrumentable.
482  */
guest_context_exit_irqoff(void)483 static __always_inline void guest_context_exit_irqoff(void)
484 {
485 	context_tracking_guest_exit();
486 }
487 
488 /*
489  * Stop accounting time towards a guest.
490  * Must be called after exiting guest context.
491  */
guest_timing_exit_irqoff(void)492 static __always_inline void guest_timing_exit_irqoff(void)
493 {
494 	instrumentation_begin();
495 	/* Flush the guest cputime we spent on the guest */
496 	vtime_account_guest_exit();
497 	instrumentation_end();
498 }
499 
500 /*
501  * Deprecated. Architectures should move to guest_state_exit_irqoff() and
502  * guest_timing_exit_irqoff().
503  */
guest_exit_irqoff(void)504 static __always_inline void guest_exit_irqoff(void)
505 {
506 	guest_context_exit_irqoff();
507 	guest_timing_exit_irqoff();
508 }
509 
guest_exit(void)510 static inline void guest_exit(void)
511 {
512 	unsigned long flags;
513 
514 	local_irq_save(flags);
515 	guest_exit_irqoff();
516 	local_irq_restore(flags);
517 }
518 
519 /**
520  * guest_state_exit_irqoff - Establish state when returning from guest mode
521  *
522  * Entry from a guest disables interrupts, but guest mode is traced as
523  * interrupts enabled. Also with NO_HZ_FULL RCU might be idle.
524  *
525  * 1) Tell lockdep that interrupts are disabled
526  * 2) Invoke context tracking if enabled to reactivate RCU
527  * 3) Trace interrupts off state
528  *
529  * Invoked from architecture specific code after exiting a guest.
530  * Must be invoked with interrupts disabled and the caller must be
531  * non-instrumentable.
532  * The caller has to invoke guest_timing_exit_irqoff() after this.
533  *
534  * Note: this is analogous to enter_from_user_mode().
535  */
guest_state_exit_irqoff(void)536 static __always_inline void guest_state_exit_irqoff(void)
537 {
538 	lockdep_hardirqs_off(CALLER_ADDR0);
539 	guest_context_exit_irqoff();
540 
541 	instrumentation_begin();
542 	trace_hardirqs_off_finish();
543 	instrumentation_end();
544 }
545 
kvm_vcpu_exiting_guest_mode(struct kvm_vcpu * vcpu)546 static inline int kvm_vcpu_exiting_guest_mode(struct kvm_vcpu *vcpu)
547 {
548 	/*
549 	 * The memory barrier ensures a previous write to vcpu->requests cannot
550 	 * be reordered with the read of vcpu->mode.  It pairs with the general
551 	 * memory barrier following the write of vcpu->mode in VCPU RUN.
552 	 */
553 	smp_mb__before_atomic();
554 	return cmpxchg(&vcpu->mode, IN_GUEST_MODE, EXITING_GUEST_MODE);
555 }
556 
557 /*
558  * Some of the bitops functions do not support too long bitmaps.
559  * This number must be determined not to exceed such limits.
560  */
561 #define KVM_MEM_MAX_NR_PAGES ((1UL << 31) - 1)
562 
563 /*
564  * Since at idle each memslot belongs to two memslot sets it has to contain
565  * two embedded nodes for each data structure that it forms a part of.
566  *
567  * Two memslot sets (one active and one inactive) are necessary so the VM
568  * continues to run on one memslot set while the other is being modified.
569  *
570  * These two memslot sets normally point to the same set of memslots.
571  * They can, however, be desynchronized when performing a memslot management
572  * operation by replacing the memslot to be modified by its copy.
573  * After the operation is complete, both memslot sets once again point to
574  * the same, common set of memslot data.
575  *
576  * The memslots themselves are independent of each other so they can be
577  * individually added or deleted.
578  */
579 struct kvm_memory_slot {
580 	struct hlist_node id_node[2];
581 	struct interval_tree_node hva_node[2];
582 	struct rb_node gfn_node[2];
583 	gfn_t base_gfn;
584 	unsigned long npages;
585 	unsigned long *dirty_bitmap;
586 	struct kvm_arch_memory_slot arch;
587 	unsigned long userspace_addr;
588 	u32 flags;
589 	short id;
590 	u16 as_id;
591 };
592 
kvm_slot_dirty_track_enabled(const struct kvm_memory_slot * slot)593 static inline bool kvm_slot_dirty_track_enabled(const struct kvm_memory_slot *slot)
594 {
595 	return slot->flags & KVM_MEM_LOG_DIRTY_PAGES;
596 }
597 
kvm_dirty_bitmap_bytes(struct kvm_memory_slot * memslot)598 static inline unsigned long kvm_dirty_bitmap_bytes(struct kvm_memory_slot *memslot)
599 {
600 	return ALIGN(memslot->npages, BITS_PER_LONG) / 8;
601 }
602 
kvm_second_dirty_bitmap(struct kvm_memory_slot * memslot)603 static inline unsigned long *kvm_second_dirty_bitmap(struct kvm_memory_slot *memslot)
604 {
605 	unsigned long len = kvm_dirty_bitmap_bytes(memslot);
606 
607 	return memslot->dirty_bitmap + len / sizeof(*memslot->dirty_bitmap);
608 }
609 
610 #ifndef KVM_DIRTY_LOG_MANUAL_CAPS
611 #define KVM_DIRTY_LOG_MANUAL_CAPS KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE
612 #endif
613 
614 struct kvm_s390_adapter_int {
615 	u64 ind_addr;
616 	u64 summary_addr;
617 	u64 ind_offset;
618 	u32 summary_offset;
619 	u32 adapter_id;
620 };
621 
622 struct kvm_hv_sint {
623 	u32 vcpu;
624 	u32 sint;
625 };
626 
627 struct kvm_xen_evtchn {
628 	u32 port;
629 	u32 vcpu_id;
630 	int vcpu_idx;
631 	u32 priority;
632 };
633 
634 struct kvm_kernel_irq_routing_entry {
635 	u32 gsi;
636 	u32 type;
637 	int (*set)(struct kvm_kernel_irq_routing_entry *e,
638 		   struct kvm *kvm, int irq_source_id, int level,
639 		   bool line_status);
640 	union {
641 		struct {
642 			unsigned irqchip;
643 			unsigned pin;
644 		} irqchip;
645 		struct {
646 			u32 address_lo;
647 			u32 address_hi;
648 			u32 data;
649 			u32 flags;
650 			u32 devid;
651 		} msi;
652 		struct kvm_s390_adapter_int adapter;
653 		struct kvm_hv_sint hv_sint;
654 		struct kvm_xen_evtchn xen_evtchn;
655 	};
656 	struct hlist_node link;
657 };
658 
659 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
660 struct kvm_irq_routing_table {
661 	int chip[KVM_NR_IRQCHIPS][KVM_IRQCHIP_NUM_PINS];
662 	u32 nr_rt_entries;
663 	/*
664 	 * Array indexed by gsi. Each entry contains list of irq chips
665 	 * the gsi is connected to.
666 	 */
667 	struct hlist_head map[];
668 };
669 #endif
670 
671 bool kvm_arch_irqchip_in_kernel(struct kvm *kvm);
672 
673 #ifndef KVM_INTERNAL_MEM_SLOTS
674 #define KVM_INTERNAL_MEM_SLOTS 0
675 #endif
676 
677 #define KVM_MEM_SLOTS_NUM SHRT_MAX
678 #define KVM_USER_MEM_SLOTS (KVM_MEM_SLOTS_NUM - KVM_INTERNAL_MEM_SLOTS)
679 
680 #ifndef __KVM_VCPU_MULTIPLE_ADDRESS_SPACE
kvm_arch_vcpu_memslots_id(struct kvm_vcpu * vcpu)681 static inline int kvm_arch_vcpu_memslots_id(struct kvm_vcpu *vcpu)
682 {
683 	return 0;
684 }
685 #endif
686 
687 struct kvm_memslots {
688 	u64 generation;
689 	atomic_long_t last_used_slot;
690 	struct rb_root_cached hva_tree;
691 	struct rb_root gfn_tree;
692 	/*
693 	 * The mapping table from slot id to memslot.
694 	 *
695 	 * 7-bit bucket count matches the size of the old id to index array for
696 	 * 512 slots, while giving good performance with this slot count.
697 	 * Higher bucket counts bring only small performance improvements but
698 	 * always result in higher memory usage (even for lower memslot counts).
699 	 */
700 	DECLARE_HASHTABLE(id_hash, 7);
701 	int node_idx;
702 };
703 
704 struct kvm {
705 #ifdef KVM_HAVE_MMU_RWLOCK
706 	rwlock_t mmu_lock;
707 #else
708 	spinlock_t mmu_lock;
709 #endif /* KVM_HAVE_MMU_RWLOCK */
710 
711 	struct mutex slots_lock;
712 
713 	/*
714 	 * Protects the arch-specific fields of struct kvm_memory_slots in
715 	 * use by the VM. To be used under the slots_lock (above) or in a
716 	 * kvm->srcu critical section where acquiring the slots_lock would
717 	 * lead to deadlock with the synchronize_srcu in
718 	 * kvm_swap_active_memslots().
719 	 */
720 	struct mutex slots_arch_lock;
721 	struct mm_struct *mm; /* userspace tied to this vm */
722 	unsigned long nr_memslot_pages;
723 	/* The two memslot sets - active and inactive (per address space) */
724 	struct kvm_memslots __memslots[KVM_ADDRESS_SPACE_NUM][2];
725 	/* The current active memslot set for each address space */
726 	struct kvm_memslots __rcu *memslots[KVM_ADDRESS_SPACE_NUM];
727 	struct xarray vcpu_array;
728 	/*
729 	 * Protected by slots_lock, but can be read outside if an
730 	 * incorrect answer is acceptable.
731 	 */
732 	atomic_t nr_memslots_dirty_logging;
733 
734 	/* Used to wait for completion of MMU notifiers.  */
735 	spinlock_t mn_invalidate_lock;
736 	unsigned long mn_active_invalidate_count;
737 	struct rcuwait mn_memslots_update_rcuwait;
738 
739 	/* For management / invalidation of gfn_to_pfn_caches */
740 	spinlock_t gpc_lock;
741 	struct list_head gpc_list;
742 
743 	/*
744 	 * created_vcpus is protected by kvm->lock, and is incremented
745 	 * at the beginning of KVM_CREATE_VCPU.  online_vcpus is only
746 	 * incremented after storing the kvm_vcpu pointer in vcpus,
747 	 * and is accessed atomically.
748 	 */
749 	atomic_t online_vcpus;
750 	int max_vcpus;
751 	int created_vcpus;
752 	int last_boosted_vcpu;
753 	struct list_head vm_list;
754 	struct mutex lock;
755 	struct kvm_io_bus __rcu *buses[KVM_NR_BUSES];
756 #ifdef CONFIG_HAVE_KVM_EVENTFD
757 	struct {
758 		spinlock_t        lock;
759 		struct list_head  items;
760 		/* resampler_list update side is protected by resampler_lock. */
761 		struct list_head  resampler_list;
762 		struct mutex      resampler_lock;
763 	} irqfds;
764 	struct list_head ioeventfds;
765 #endif
766 	struct kvm_vm_stat stat;
767 	struct kvm_arch arch;
768 	refcount_t users_count;
769 #ifdef CONFIG_KVM_MMIO
770 	struct kvm_coalesced_mmio_ring *coalesced_mmio_ring;
771 	spinlock_t ring_lock;
772 	struct list_head coalesced_zones;
773 #endif
774 
775 	struct mutex irq_lock;
776 #ifdef CONFIG_HAVE_KVM_IRQCHIP
777 	/*
778 	 * Update side is protected by irq_lock.
779 	 */
780 	struct kvm_irq_routing_table __rcu *irq_routing;
781 #endif
782 #ifdef CONFIG_HAVE_KVM_IRQFD
783 	struct hlist_head irq_ack_notifier_list;
784 #endif
785 
786 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
787 	struct mmu_notifier mmu_notifier;
788 	unsigned long mmu_invalidate_seq;
789 	long mmu_invalidate_in_progress;
790 	unsigned long mmu_invalidate_range_start;
791 	unsigned long mmu_invalidate_range_end;
792 #endif
793 	struct list_head devices;
794 	u64 manual_dirty_log_protect;
795 	struct dentry *debugfs_dentry;
796 	struct kvm_stat_data **debugfs_stat_data;
797 	struct srcu_struct srcu;
798 	struct srcu_struct irq_srcu;
799 	pid_t userspace_pid;
800 	bool override_halt_poll_ns;
801 	unsigned int max_halt_poll_ns;
802 	u32 dirty_ring_size;
803 	bool dirty_ring_with_bitmap;
804 	bool vm_bugged;
805 	bool vm_dead;
806 
807 #ifdef CONFIG_HAVE_KVM_PM_NOTIFIER
808 	struct notifier_block pm_notifier;
809 #endif
810 	char stats_id[KVM_STATS_NAME_SIZE];
811 };
812 
813 #define kvm_err(fmt, ...) \
814 	pr_err("kvm [%i]: " fmt, task_pid_nr(current), ## __VA_ARGS__)
815 #define kvm_info(fmt, ...) \
816 	pr_info("kvm [%i]: " fmt, task_pid_nr(current), ## __VA_ARGS__)
817 #define kvm_debug(fmt, ...) \
818 	pr_debug("kvm [%i]: " fmt, task_pid_nr(current), ## __VA_ARGS__)
819 #define kvm_debug_ratelimited(fmt, ...) \
820 	pr_debug_ratelimited("kvm [%i]: " fmt, task_pid_nr(current), \
821 			     ## __VA_ARGS__)
822 #define kvm_pr_unimpl(fmt, ...) \
823 	pr_err_ratelimited("kvm [%i]: " fmt, \
824 			   task_tgid_nr(current), ## __VA_ARGS__)
825 
826 /* The guest did something we don't support. */
827 #define vcpu_unimpl(vcpu, fmt, ...)					\
828 	kvm_pr_unimpl("vcpu%i, guest rIP: 0x%lx " fmt,			\
829 			(vcpu)->vcpu_id, kvm_rip_read(vcpu), ## __VA_ARGS__)
830 
831 #define vcpu_debug(vcpu, fmt, ...)					\
832 	kvm_debug("vcpu%i " fmt, (vcpu)->vcpu_id, ## __VA_ARGS__)
833 #define vcpu_debug_ratelimited(vcpu, fmt, ...)				\
834 	kvm_debug_ratelimited("vcpu%i " fmt, (vcpu)->vcpu_id,           \
835 			      ## __VA_ARGS__)
836 #define vcpu_err(vcpu, fmt, ...)					\
837 	kvm_err("vcpu%i " fmt, (vcpu)->vcpu_id, ## __VA_ARGS__)
838 
kvm_vm_dead(struct kvm * kvm)839 static inline void kvm_vm_dead(struct kvm *kvm)
840 {
841 	kvm->vm_dead = true;
842 	kvm_make_all_cpus_request(kvm, KVM_REQ_VM_DEAD);
843 }
844 
kvm_vm_bugged(struct kvm * kvm)845 static inline void kvm_vm_bugged(struct kvm *kvm)
846 {
847 	kvm->vm_bugged = true;
848 	kvm_vm_dead(kvm);
849 }
850 
851 
852 #define KVM_BUG(cond, kvm, fmt...)				\
853 ({								\
854 	bool __ret = !!(cond);					\
855 								\
856 	if (WARN_ONCE(__ret && !(kvm)->vm_bugged, fmt))		\
857 		kvm_vm_bugged(kvm);				\
858 	unlikely(__ret);					\
859 })
860 
861 #define KVM_BUG_ON(cond, kvm)					\
862 ({								\
863 	bool __ret = !!(cond);					\
864 								\
865 	if (WARN_ON_ONCE(__ret && !(kvm)->vm_bugged))		\
866 		kvm_vm_bugged(kvm);				\
867 	unlikely(__ret);					\
868 })
869 
870 /*
871  * Note, "data corruption" refers to corruption of host kernel data structures,
872  * not guest data.  Guest data corruption, suspected or confirmed, that is tied
873  * and contained to a single VM should *never* BUG() and potentially panic the
874  * host, i.e. use this variant of KVM_BUG() if and only if a KVM data structure
875  * is corrupted and that corruption can have a cascading effect to other parts
876  * of the hosts and/or to other VMs.
877  */
878 #define KVM_BUG_ON_DATA_CORRUPTION(cond, kvm)			\
879 ({								\
880 	bool __ret = !!(cond);					\
881 								\
882 	if (IS_ENABLED(CONFIG_BUG_ON_DATA_CORRUPTION))		\
883 		BUG_ON(__ret);					\
884 	else if (WARN_ON_ONCE(__ret && !(kvm)->vm_bugged))	\
885 		kvm_vm_bugged(kvm);				\
886 	unlikely(__ret);					\
887 })
888 
kvm_vcpu_srcu_read_lock(struct kvm_vcpu * vcpu)889 static inline void kvm_vcpu_srcu_read_lock(struct kvm_vcpu *vcpu)
890 {
891 #ifdef CONFIG_PROVE_RCU
892 	WARN_ONCE(vcpu->srcu_depth++,
893 		  "KVM: Illegal vCPU srcu_idx LOCK, depth=%d", vcpu->srcu_depth - 1);
894 #endif
895 	vcpu->____srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
896 }
897 
kvm_vcpu_srcu_read_unlock(struct kvm_vcpu * vcpu)898 static inline void kvm_vcpu_srcu_read_unlock(struct kvm_vcpu *vcpu)
899 {
900 	srcu_read_unlock(&vcpu->kvm->srcu, vcpu->____srcu_idx);
901 
902 #ifdef CONFIG_PROVE_RCU
903 	WARN_ONCE(--vcpu->srcu_depth,
904 		  "KVM: Illegal vCPU srcu_idx UNLOCK, depth=%d", vcpu->srcu_depth);
905 #endif
906 }
907 
kvm_dirty_log_manual_protect_and_init_set(struct kvm * kvm)908 static inline bool kvm_dirty_log_manual_protect_and_init_set(struct kvm *kvm)
909 {
910 	return !!(kvm->manual_dirty_log_protect & KVM_DIRTY_LOG_INITIALLY_SET);
911 }
912 
kvm_get_bus(struct kvm * kvm,enum kvm_bus idx)913 static inline struct kvm_io_bus *kvm_get_bus(struct kvm *kvm, enum kvm_bus idx)
914 {
915 	return srcu_dereference_check(kvm->buses[idx], &kvm->srcu,
916 				      lockdep_is_held(&kvm->slots_lock) ||
917 				      !refcount_read(&kvm->users_count));
918 }
919 
kvm_get_vcpu(struct kvm * kvm,int i)920 static inline struct kvm_vcpu *kvm_get_vcpu(struct kvm *kvm, int i)
921 {
922 	int num_vcpus = atomic_read(&kvm->online_vcpus);
923 	i = array_index_nospec(i, num_vcpus);
924 
925 	/* Pairs with smp_wmb() in kvm_vm_ioctl_create_vcpu.  */
926 	smp_rmb();
927 	return xa_load(&kvm->vcpu_array, i);
928 }
929 
930 #define kvm_for_each_vcpu(idx, vcpup, kvm)		   \
931 	xa_for_each_range(&kvm->vcpu_array, idx, vcpup, 0, \
932 			  (atomic_read(&kvm->online_vcpus) - 1))
933 
kvm_get_vcpu_by_id(struct kvm * kvm,int id)934 static inline struct kvm_vcpu *kvm_get_vcpu_by_id(struct kvm *kvm, int id)
935 {
936 	struct kvm_vcpu *vcpu = NULL;
937 	unsigned long i;
938 
939 	if (id < 0)
940 		return NULL;
941 	if (id < KVM_MAX_VCPUS)
942 		vcpu = kvm_get_vcpu(kvm, id);
943 	if (vcpu && vcpu->vcpu_id == id)
944 		return vcpu;
945 	kvm_for_each_vcpu(i, vcpu, kvm)
946 		if (vcpu->vcpu_id == id)
947 			return vcpu;
948 	return NULL;
949 }
950 
951 void kvm_destroy_vcpus(struct kvm *kvm);
952 
953 void vcpu_load(struct kvm_vcpu *vcpu);
954 void vcpu_put(struct kvm_vcpu *vcpu);
955 
956 #ifdef __KVM_HAVE_IOAPIC
957 void kvm_arch_post_irq_ack_notifier_list_update(struct kvm *kvm);
958 void kvm_arch_post_irq_routing_update(struct kvm *kvm);
959 #else
kvm_arch_post_irq_ack_notifier_list_update(struct kvm * kvm)960 static inline void kvm_arch_post_irq_ack_notifier_list_update(struct kvm *kvm)
961 {
962 }
kvm_arch_post_irq_routing_update(struct kvm * kvm)963 static inline void kvm_arch_post_irq_routing_update(struct kvm *kvm)
964 {
965 }
966 #endif
967 
968 #ifdef CONFIG_HAVE_KVM_IRQFD
969 int kvm_irqfd_init(void);
970 void kvm_irqfd_exit(void);
971 #else
kvm_irqfd_init(void)972 static inline int kvm_irqfd_init(void)
973 {
974 	return 0;
975 }
976 
kvm_irqfd_exit(void)977 static inline void kvm_irqfd_exit(void)
978 {
979 }
980 #endif
981 int kvm_init(unsigned vcpu_size, unsigned vcpu_align, struct module *module);
982 void kvm_exit(void);
983 
984 void kvm_get_kvm(struct kvm *kvm);
985 bool kvm_get_kvm_safe(struct kvm *kvm);
986 void kvm_put_kvm(struct kvm *kvm);
987 bool file_is_kvm(struct file *file);
988 void kvm_put_kvm_no_destroy(struct kvm *kvm);
989 
__kvm_memslots(struct kvm * kvm,int as_id)990 static inline struct kvm_memslots *__kvm_memslots(struct kvm *kvm, int as_id)
991 {
992 	as_id = array_index_nospec(as_id, KVM_ADDRESS_SPACE_NUM);
993 	return srcu_dereference_check(kvm->memslots[as_id], &kvm->srcu,
994 			lockdep_is_held(&kvm->slots_lock) ||
995 			!refcount_read(&kvm->users_count));
996 }
997 
kvm_memslots(struct kvm * kvm)998 static inline struct kvm_memslots *kvm_memslots(struct kvm *kvm)
999 {
1000 	return __kvm_memslots(kvm, 0);
1001 }
1002 
kvm_vcpu_memslots(struct kvm_vcpu * vcpu)1003 static inline struct kvm_memslots *kvm_vcpu_memslots(struct kvm_vcpu *vcpu)
1004 {
1005 	int as_id = kvm_arch_vcpu_memslots_id(vcpu);
1006 
1007 	return __kvm_memslots(vcpu->kvm, as_id);
1008 }
1009 
kvm_memslots_empty(struct kvm_memslots * slots)1010 static inline bool kvm_memslots_empty(struct kvm_memslots *slots)
1011 {
1012 	return RB_EMPTY_ROOT(&slots->gfn_tree);
1013 }
1014 
1015 bool kvm_are_all_memslots_empty(struct kvm *kvm);
1016 
1017 #define kvm_for_each_memslot(memslot, bkt, slots)			      \
1018 	hash_for_each(slots->id_hash, bkt, memslot, id_node[slots->node_idx]) \
1019 		if (WARN_ON_ONCE(!memslot->npages)) {			      \
1020 		} else
1021 
1022 static inline
id_to_memslot(struct kvm_memslots * slots,int id)1023 struct kvm_memory_slot *id_to_memslot(struct kvm_memslots *slots, int id)
1024 {
1025 	struct kvm_memory_slot *slot;
1026 	int idx = slots->node_idx;
1027 
1028 	hash_for_each_possible(slots->id_hash, slot, id_node[idx], id) {
1029 		if (slot->id == id)
1030 			return slot;
1031 	}
1032 
1033 	return NULL;
1034 }
1035 
1036 /* Iterator used for walking memslots that overlap a gfn range. */
1037 struct kvm_memslot_iter {
1038 	struct kvm_memslots *slots;
1039 	struct rb_node *node;
1040 	struct kvm_memory_slot *slot;
1041 };
1042 
kvm_memslot_iter_next(struct kvm_memslot_iter * iter)1043 static inline void kvm_memslot_iter_next(struct kvm_memslot_iter *iter)
1044 {
1045 	iter->node = rb_next(iter->node);
1046 	if (!iter->node)
1047 		return;
1048 
1049 	iter->slot = container_of(iter->node, struct kvm_memory_slot, gfn_node[iter->slots->node_idx]);
1050 }
1051 
kvm_memslot_iter_start(struct kvm_memslot_iter * iter,struct kvm_memslots * slots,gfn_t start)1052 static inline void kvm_memslot_iter_start(struct kvm_memslot_iter *iter,
1053 					  struct kvm_memslots *slots,
1054 					  gfn_t start)
1055 {
1056 	int idx = slots->node_idx;
1057 	struct rb_node *tmp;
1058 	struct kvm_memory_slot *slot;
1059 
1060 	iter->slots = slots;
1061 
1062 	/*
1063 	 * Find the so called "upper bound" of a key - the first node that has
1064 	 * its key strictly greater than the searched one (the start gfn in our case).
1065 	 */
1066 	iter->node = NULL;
1067 	for (tmp = slots->gfn_tree.rb_node; tmp; ) {
1068 		slot = container_of(tmp, struct kvm_memory_slot, gfn_node[idx]);
1069 		if (start < slot->base_gfn) {
1070 			iter->node = tmp;
1071 			tmp = tmp->rb_left;
1072 		} else {
1073 			tmp = tmp->rb_right;
1074 		}
1075 	}
1076 
1077 	/*
1078 	 * Find the slot with the lowest gfn that can possibly intersect with
1079 	 * the range, so we'll ideally have slot start <= range start
1080 	 */
1081 	if (iter->node) {
1082 		/*
1083 		 * A NULL previous node means that the very first slot
1084 		 * already has a higher start gfn.
1085 		 * In this case slot start > range start.
1086 		 */
1087 		tmp = rb_prev(iter->node);
1088 		if (tmp)
1089 			iter->node = tmp;
1090 	} else {
1091 		/* a NULL node below means no slots */
1092 		iter->node = rb_last(&slots->gfn_tree);
1093 	}
1094 
1095 	if (iter->node) {
1096 		iter->slot = container_of(iter->node, struct kvm_memory_slot, gfn_node[idx]);
1097 
1098 		/*
1099 		 * It is possible in the slot start < range start case that the
1100 		 * found slot ends before or at range start (slot end <= range start)
1101 		 * and so it does not overlap the requested range.
1102 		 *
1103 		 * In such non-overlapping case the next slot (if it exists) will
1104 		 * already have slot start > range start, otherwise the logic above
1105 		 * would have found it instead of the current slot.
1106 		 */
1107 		if (iter->slot->base_gfn + iter->slot->npages <= start)
1108 			kvm_memslot_iter_next(iter);
1109 	}
1110 }
1111 
kvm_memslot_iter_is_valid(struct kvm_memslot_iter * iter,gfn_t end)1112 static inline bool kvm_memslot_iter_is_valid(struct kvm_memslot_iter *iter, gfn_t end)
1113 {
1114 	if (!iter->node)
1115 		return false;
1116 
1117 	/*
1118 	 * If this slot starts beyond or at the end of the range so does
1119 	 * every next one
1120 	 */
1121 	return iter->slot->base_gfn < end;
1122 }
1123 
1124 /* Iterate over each memslot at least partially intersecting [start, end) range */
1125 #define kvm_for_each_memslot_in_gfn_range(iter, slots, start, end)	\
1126 	for (kvm_memslot_iter_start(iter, slots, start);		\
1127 	     kvm_memslot_iter_is_valid(iter, end);			\
1128 	     kvm_memslot_iter_next(iter))
1129 
1130 /*
1131  * KVM_SET_USER_MEMORY_REGION ioctl allows the following operations:
1132  * - create a new memory slot
1133  * - delete an existing memory slot
1134  * - modify an existing memory slot
1135  *   -- move it in the guest physical memory space
1136  *   -- just change its flags
1137  *
1138  * Since flags can be changed by some of these operations, the following
1139  * differentiation is the best we can do for __kvm_set_memory_region():
1140  */
1141 enum kvm_mr_change {
1142 	KVM_MR_CREATE,
1143 	KVM_MR_DELETE,
1144 	KVM_MR_MOVE,
1145 	KVM_MR_FLAGS_ONLY,
1146 };
1147 
1148 int kvm_set_memory_region(struct kvm *kvm,
1149 			  const struct kvm_userspace_memory_region *mem);
1150 int __kvm_set_memory_region(struct kvm *kvm,
1151 			    const struct kvm_userspace_memory_region *mem);
1152 void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot);
1153 void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen);
1154 int kvm_arch_prepare_memory_region(struct kvm *kvm,
1155 				const struct kvm_memory_slot *old,
1156 				struct kvm_memory_slot *new,
1157 				enum kvm_mr_change change);
1158 void kvm_arch_commit_memory_region(struct kvm *kvm,
1159 				struct kvm_memory_slot *old,
1160 				const struct kvm_memory_slot *new,
1161 				enum kvm_mr_change change);
1162 /* flush all memory translations */
1163 void kvm_arch_flush_shadow_all(struct kvm *kvm);
1164 /* flush memory translations pointing to 'slot' */
1165 void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
1166 				   struct kvm_memory_slot *slot);
1167 
1168 int gfn_to_page_many_atomic(struct kvm_memory_slot *slot, gfn_t gfn,
1169 			    struct page **pages, int nr_pages);
1170 
1171 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn);
1172 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn);
1173 unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable);
1174 unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot, gfn_t gfn);
1175 unsigned long gfn_to_hva_memslot_prot(struct kvm_memory_slot *slot, gfn_t gfn,
1176 				      bool *writable);
1177 void kvm_release_page_clean(struct page *page);
1178 void kvm_release_page_dirty(struct page *page);
1179 
1180 kvm_pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn);
1181 kvm_pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault,
1182 		      bool *writable);
1183 kvm_pfn_t gfn_to_pfn_memslot(const struct kvm_memory_slot *slot, gfn_t gfn);
1184 kvm_pfn_t gfn_to_pfn_memslot_atomic(const struct kvm_memory_slot *slot, gfn_t gfn);
1185 kvm_pfn_t __gfn_to_pfn_memslot(const struct kvm_memory_slot *slot, gfn_t gfn,
1186 			       bool atomic, bool interruptible, bool *async,
1187 			       bool write_fault, bool *writable, hva_t *hva);
1188 
1189 void kvm_release_pfn_clean(kvm_pfn_t pfn);
1190 void kvm_release_pfn_dirty(kvm_pfn_t pfn);
1191 void kvm_set_pfn_dirty(kvm_pfn_t pfn);
1192 void kvm_set_pfn_accessed(kvm_pfn_t pfn);
1193 
1194 void kvm_release_pfn(kvm_pfn_t pfn, bool dirty);
1195 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
1196 			int len);
1197 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len);
1198 int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1199 			   void *data, unsigned long len);
1200 int kvm_read_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1201 				 void *data, unsigned int offset,
1202 				 unsigned long len);
1203 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,
1204 			 int offset, int len);
1205 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
1206 		    unsigned long len);
1207 int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1208 			   void *data, unsigned long len);
1209 int kvm_write_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1210 				  void *data, unsigned int offset,
1211 				  unsigned long len);
1212 int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1213 			      gpa_t gpa, unsigned long len);
1214 
1215 #define __kvm_get_guest(kvm, gfn, offset, v)				\
1216 ({									\
1217 	unsigned long __addr = gfn_to_hva(kvm, gfn);			\
1218 	typeof(v) __user *__uaddr = (typeof(__uaddr))(__addr + offset);	\
1219 	int __ret = -EFAULT;						\
1220 									\
1221 	if (!kvm_is_error_hva(__addr))					\
1222 		__ret = get_user(v, __uaddr);				\
1223 	__ret;								\
1224 })
1225 
1226 #define kvm_get_guest(kvm, gpa, v)					\
1227 ({									\
1228 	gpa_t __gpa = gpa;						\
1229 	struct kvm *__kvm = kvm;					\
1230 									\
1231 	__kvm_get_guest(__kvm, __gpa >> PAGE_SHIFT,			\
1232 			offset_in_page(__gpa), v);			\
1233 })
1234 
1235 #define __kvm_put_guest(kvm, gfn, offset, v)				\
1236 ({									\
1237 	unsigned long __addr = gfn_to_hva(kvm, gfn);			\
1238 	typeof(v) __user *__uaddr = (typeof(__uaddr))(__addr + offset);	\
1239 	int __ret = -EFAULT;						\
1240 									\
1241 	if (!kvm_is_error_hva(__addr))					\
1242 		__ret = put_user(v, __uaddr);				\
1243 	if (!__ret)							\
1244 		mark_page_dirty(kvm, gfn);				\
1245 	__ret;								\
1246 })
1247 
1248 #define kvm_put_guest(kvm, gpa, v)					\
1249 ({									\
1250 	gpa_t __gpa = gpa;						\
1251 	struct kvm *__kvm = kvm;					\
1252 									\
1253 	__kvm_put_guest(__kvm, __gpa >> PAGE_SHIFT,			\
1254 			offset_in_page(__gpa), v);			\
1255 })
1256 
1257 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len);
1258 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn);
1259 bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn);
1260 bool kvm_vcpu_is_visible_gfn(struct kvm_vcpu *vcpu, gfn_t gfn);
1261 unsigned long kvm_host_page_size(struct kvm_vcpu *vcpu, gfn_t gfn);
1262 void mark_page_dirty_in_slot(struct kvm *kvm, const struct kvm_memory_slot *memslot, gfn_t gfn);
1263 void mark_page_dirty(struct kvm *kvm, gfn_t gfn);
1264 
1265 struct kvm_memslots *kvm_vcpu_memslots(struct kvm_vcpu *vcpu);
1266 struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn);
1267 kvm_pfn_t kvm_vcpu_gfn_to_pfn_atomic(struct kvm_vcpu *vcpu, gfn_t gfn);
1268 kvm_pfn_t kvm_vcpu_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn);
1269 int kvm_vcpu_map(struct kvm_vcpu *vcpu, gpa_t gpa, struct kvm_host_map *map);
1270 void kvm_vcpu_unmap(struct kvm_vcpu *vcpu, struct kvm_host_map *map, bool dirty);
1271 unsigned long kvm_vcpu_gfn_to_hva(struct kvm_vcpu *vcpu, gfn_t gfn);
1272 unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *writable);
1273 int kvm_vcpu_read_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, void *data, int offset,
1274 			     int len);
1275 int kvm_vcpu_read_guest_atomic(struct kvm_vcpu *vcpu, gpa_t gpa, void *data,
1276 			       unsigned long len);
1277 int kvm_vcpu_read_guest(struct kvm_vcpu *vcpu, gpa_t gpa, void *data,
1278 			unsigned long len);
1279 int kvm_vcpu_write_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, const void *data,
1280 			      int offset, int len);
1281 int kvm_vcpu_write_guest(struct kvm_vcpu *vcpu, gpa_t gpa, const void *data,
1282 			 unsigned long len);
1283 void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn);
1284 
1285 /**
1286  * kvm_gpc_init - initialize gfn_to_pfn_cache.
1287  *
1288  * @gpc:	   struct gfn_to_pfn_cache object.
1289  * @kvm:	   pointer to kvm instance.
1290  * @vcpu:	   vCPU to be used for marking pages dirty and to be woken on
1291  *		   invalidation.
1292  * @usage:	   indicates if the resulting host physical PFN is used while
1293  *		   the @vcpu is IN_GUEST_MODE (in which case invalidation of
1294  *		   the cache from MMU notifiers---but not for KVM memslot
1295  *		   changes!---will also force @vcpu to exit the guest and
1296  *		   refresh the cache); and/or if the PFN used directly
1297  *		   by KVM (and thus needs a kernel virtual mapping).
1298  *
1299  * This sets up a gfn_to_pfn_cache by initializing locks and assigning the
1300  * immutable attributes.  Note, the cache must be zero-allocated (or zeroed by
1301  * the caller before init).
1302  */
1303 void kvm_gpc_init(struct gfn_to_pfn_cache *gpc, struct kvm *kvm,
1304 		  struct kvm_vcpu *vcpu, enum pfn_cache_usage usage);
1305 
1306 /**
1307  * kvm_gpc_activate - prepare a cached kernel mapping and HPA for a given guest
1308  *                    physical address.
1309  *
1310  * @gpc:	   struct gfn_to_pfn_cache object.
1311  * @gpa:	   guest physical address to map.
1312  * @len:	   sanity check; the range being access must fit a single page.
1313  *
1314  * @return:	   0 for success.
1315  *		   -EINVAL for a mapping which would cross a page boundary.
1316  *		   -EFAULT for an untranslatable guest physical address.
1317  *
1318  * This primes a gfn_to_pfn_cache and links it into the @gpc->kvm's list for
1319  * invalidations to be processed.  Callers are required to use kvm_gpc_check()
1320  * to ensure that the cache is valid before accessing the target page.
1321  */
1322 int kvm_gpc_activate(struct gfn_to_pfn_cache *gpc, gpa_t gpa, unsigned long len);
1323 
1324 /**
1325  * kvm_gpc_check - check validity of a gfn_to_pfn_cache.
1326  *
1327  * @gpc:	   struct gfn_to_pfn_cache object.
1328  * @len:	   sanity check; the range being access must fit a single page.
1329  *
1330  * @return:	   %true if the cache is still valid and the address matches.
1331  *		   %false if the cache is not valid.
1332  *
1333  * Callers outside IN_GUEST_MODE context should hold a read lock on @gpc->lock
1334  * while calling this function, and then continue to hold the lock until the
1335  * access is complete.
1336  *
1337  * Callers in IN_GUEST_MODE may do so without locking, although they should
1338  * still hold a read lock on kvm->scru for the memslot checks.
1339  */
1340 bool kvm_gpc_check(struct gfn_to_pfn_cache *gpc, unsigned long len);
1341 
1342 /**
1343  * kvm_gpc_refresh - update a previously initialized cache.
1344  *
1345  * @gpc:	   struct gfn_to_pfn_cache object.
1346  * @len:	   sanity check; the range being access must fit a single page.
1347  *
1348  * @return:	   0 for success.
1349  *		   -EINVAL for a mapping which would cross a page boundary.
1350  *		   -EFAULT for an untranslatable guest physical address.
1351  *
1352  * This will attempt to refresh a gfn_to_pfn_cache. Note that a successful
1353  * return from this function does not mean the page can be immediately
1354  * accessed because it may have raced with an invalidation. Callers must
1355  * still lock and check the cache status, as this function does not return
1356  * with the lock still held to permit access.
1357  */
1358 int kvm_gpc_refresh(struct gfn_to_pfn_cache *gpc, unsigned long len);
1359 
1360 /**
1361  * kvm_gpc_deactivate - deactivate and unlink a gfn_to_pfn_cache.
1362  *
1363  * @gpc:	   struct gfn_to_pfn_cache object.
1364  *
1365  * This removes a cache from the VM's list to be processed on MMU notifier
1366  * invocation.
1367  */
1368 void kvm_gpc_deactivate(struct gfn_to_pfn_cache *gpc);
1369 
1370 void kvm_sigset_activate(struct kvm_vcpu *vcpu);
1371 void kvm_sigset_deactivate(struct kvm_vcpu *vcpu);
1372 
1373 void kvm_vcpu_halt(struct kvm_vcpu *vcpu);
1374 bool kvm_vcpu_block(struct kvm_vcpu *vcpu);
1375 void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu);
1376 void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu);
1377 bool kvm_vcpu_wake_up(struct kvm_vcpu *vcpu);
1378 void kvm_vcpu_kick(struct kvm_vcpu *vcpu);
1379 int kvm_vcpu_yield_to(struct kvm_vcpu *target);
1380 void kvm_vcpu_on_spin(struct kvm_vcpu *vcpu, bool yield_to_kernel_mode);
1381 
1382 void kvm_flush_remote_tlbs(struct kvm *kvm);
1383 void kvm_flush_remote_tlbs_range(struct kvm *kvm, gfn_t gfn, u64 nr_pages);
1384 void kvm_flush_remote_tlbs_memslot(struct kvm *kvm,
1385 				   const struct kvm_memory_slot *memslot);
1386 
1387 #ifdef KVM_ARCH_NR_OBJS_PER_MEMORY_CACHE
1388 int kvm_mmu_topup_memory_cache(struct kvm_mmu_memory_cache *mc, int min);
1389 int __kvm_mmu_topup_memory_cache(struct kvm_mmu_memory_cache *mc, int capacity, int min);
1390 int kvm_mmu_memory_cache_nr_free_objects(struct kvm_mmu_memory_cache *mc);
1391 void kvm_mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc);
1392 void *kvm_mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc);
1393 #endif
1394 
1395 void kvm_mmu_invalidate_begin(struct kvm *kvm, unsigned long start,
1396 			      unsigned long end);
1397 void kvm_mmu_invalidate_end(struct kvm *kvm, unsigned long start,
1398 			    unsigned long end);
1399 
1400 long kvm_arch_dev_ioctl(struct file *filp,
1401 			unsigned int ioctl, unsigned long arg);
1402 long kvm_arch_vcpu_ioctl(struct file *filp,
1403 			 unsigned int ioctl, unsigned long arg);
1404 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf);
1405 
1406 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext);
1407 
1408 void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
1409 					struct kvm_memory_slot *slot,
1410 					gfn_t gfn_offset,
1411 					unsigned long mask);
1412 void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot);
1413 
1414 #ifndef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
1415 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log);
1416 int kvm_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log,
1417 		      int *is_dirty, struct kvm_memory_slot **memslot);
1418 #endif
1419 
1420 int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level,
1421 			bool line_status);
1422 int kvm_vm_ioctl_enable_cap(struct kvm *kvm,
1423 			    struct kvm_enable_cap *cap);
1424 int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg);
1425 long kvm_arch_vm_compat_ioctl(struct file *filp, unsigned int ioctl,
1426 			      unsigned long arg);
1427 
1428 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu);
1429 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu);
1430 
1431 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
1432 				    struct kvm_translation *tr);
1433 
1434 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs);
1435 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs);
1436 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
1437 				  struct kvm_sregs *sregs);
1438 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
1439 				  struct kvm_sregs *sregs);
1440 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
1441 				    struct kvm_mp_state *mp_state);
1442 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
1443 				    struct kvm_mp_state *mp_state);
1444 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
1445 					struct kvm_guest_debug *dbg);
1446 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu);
1447 
1448 void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu);
1449 
1450 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu);
1451 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu);
1452 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id);
1453 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu);
1454 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu);
1455 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu);
1456 
1457 #ifdef CONFIG_HAVE_KVM_PM_NOTIFIER
1458 int kvm_arch_pm_notifier(struct kvm *kvm, unsigned long state);
1459 #endif
1460 
1461 #ifdef __KVM_HAVE_ARCH_VCPU_DEBUGFS
1462 void kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu, struct dentry *debugfs_dentry);
1463 #else
kvm_create_vcpu_debugfs(struct kvm_vcpu * vcpu)1464 static inline void kvm_create_vcpu_debugfs(struct kvm_vcpu *vcpu) {}
1465 #endif
1466 
1467 #ifdef CONFIG_KVM_GENERIC_HARDWARE_ENABLING
1468 int kvm_arch_hardware_enable(void);
1469 void kvm_arch_hardware_disable(void);
1470 #endif
1471 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu);
1472 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu);
1473 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu);
1474 bool kvm_arch_dy_runnable(struct kvm_vcpu *vcpu);
1475 bool kvm_arch_dy_has_pending_interrupt(struct kvm_vcpu *vcpu);
1476 int kvm_arch_post_init_vm(struct kvm *kvm);
1477 void kvm_arch_pre_destroy_vm(struct kvm *kvm);
1478 int kvm_arch_create_vm_debugfs(struct kvm *kvm);
1479 
1480 #ifndef __KVM_HAVE_ARCH_VM_ALLOC
1481 /*
1482  * All architectures that want to use vzalloc currently also
1483  * need their own kvm_arch_alloc_vm implementation.
1484  */
kvm_arch_alloc_vm(void)1485 static inline struct kvm *kvm_arch_alloc_vm(void)
1486 {
1487 	return kzalloc(sizeof(struct kvm), GFP_KERNEL_ACCOUNT);
1488 }
1489 #endif
1490 
__kvm_arch_free_vm(struct kvm * kvm)1491 static inline void __kvm_arch_free_vm(struct kvm *kvm)
1492 {
1493 	kvfree(kvm);
1494 }
1495 
1496 #ifndef __KVM_HAVE_ARCH_VM_FREE
kvm_arch_free_vm(struct kvm * kvm)1497 static inline void kvm_arch_free_vm(struct kvm *kvm)
1498 {
1499 	__kvm_arch_free_vm(kvm);
1500 }
1501 #endif
1502 
1503 #ifndef __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS
kvm_arch_flush_remote_tlbs(struct kvm * kvm)1504 static inline int kvm_arch_flush_remote_tlbs(struct kvm *kvm)
1505 {
1506 	return -ENOTSUPP;
1507 }
1508 #else
1509 int kvm_arch_flush_remote_tlbs(struct kvm *kvm);
1510 #endif
1511 
1512 #ifndef __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS_RANGE
kvm_arch_flush_remote_tlbs_range(struct kvm * kvm,gfn_t gfn,u64 nr_pages)1513 static inline int kvm_arch_flush_remote_tlbs_range(struct kvm *kvm,
1514 						    gfn_t gfn, u64 nr_pages)
1515 {
1516 	return -EOPNOTSUPP;
1517 }
1518 #else
1519 int kvm_arch_flush_remote_tlbs_range(struct kvm *kvm, gfn_t gfn, u64 nr_pages);
1520 #endif
1521 
1522 #ifdef __KVM_HAVE_ARCH_NONCOHERENT_DMA
1523 void kvm_arch_register_noncoherent_dma(struct kvm *kvm);
1524 void kvm_arch_unregister_noncoherent_dma(struct kvm *kvm);
1525 bool kvm_arch_has_noncoherent_dma(struct kvm *kvm);
1526 #else
kvm_arch_register_noncoherent_dma(struct kvm * kvm)1527 static inline void kvm_arch_register_noncoherent_dma(struct kvm *kvm)
1528 {
1529 }
1530 
kvm_arch_unregister_noncoherent_dma(struct kvm * kvm)1531 static inline void kvm_arch_unregister_noncoherent_dma(struct kvm *kvm)
1532 {
1533 }
1534 
kvm_arch_has_noncoherent_dma(struct kvm * kvm)1535 static inline bool kvm_arch_has_noncoherent_dma(struct kvm *kvm)
1536 {
1537 	return false;
1538 }
1539 #endif
1540 #ifdef __KVM_HAVE_ARCH_ASSIGNED_DEVICE
1541 void kvm_arch_start_assignment(struct kvm *kvm);
1542 void kvm_arch_end_assignment(struct kvm *kvm);
1543 bool kvm_arch_has_assigned_device(struct kvm *kvm);
1544 #else
kvm_arch_start_assignment(struct kvm * kvm)1545 static inline void kvm_arch_start_assignment(struct kvm *kvm)
1546 {
1547 }
1548 
kvm_arch_end_assignment(struct kvm * kvm)1549 static inline void kvm_arch_end_assignment(struct kvm *kvm)
1550 {
1551 }
1552 
kvm_arch_has_assigned_device(struct kvm * kvm)1553 static __always_inline bool kvm_arch_has_assigned_device(struct kvm *kvm)
1554 {
1555 	return false;
1556 }
1557 #endif
1558 
kvm_arch_vcpu_get_wait(struct kvm_vcpu * vcpu)1559 static inline struct rcuwait *kvm_arch_vcpu_get_wait(struct kvm_vcpu *vcpu)
1560 {
1561 #ifdef __KVM_HAVE_ARCH_WQP
1562 	return vcpu->arch.waitp;
1563 #else
1564 	return &vcpu->wait;
1565 #endif
1566 }
1567 
1568 /*
1569  * Wake a vCPU if necessary, but don't do any stats/metadata updates.  Returns
1570  * true if the vCPU was blocking and was awakened, false otherwise.
1571  */
__kvm_vcpu_wake_up(struct kvm_vcpu * vcpu)1572 static inline bool __kvm_vcpu_wake_up(struct kvm_vcpu *vcpu)
1573 {
1574 	return !!rcuwait_wake_up(kvm_arch_vcpu_get_wait(vcpu));
1575 }
1576 
kvm_vcpu_is_blocking(struct kvm_vcpu * vcpu)1577 static inline bool kvm_vcpu_is_blocking(struct kvm_vcpu *vcpu)
1578 {
1579 	return rcuwait_active(kvm_arch_vcpu_get_wait(vcpu));
1580 }
1581 
1582 #ifdef __KVM_HAVE_ARCH_INTC_INITIALIZED
1583 /*
1584  * returns true if the virtual interrupt controller is initialized and
1585  * ready to accept virtual IRQ. On some architectures the virtual interrupt
1586  * controller is dynamically instantiated and this is not always true.
1587  */
1588 bool kvm_arch_intc_initialized(struct kvm *kvm);
1589 #else
kvm_arch_intc_initialized(struct kvm * kvm)1590 static inline bool kvm_arch_intc_initialized(struct kvm *kvm)
1591 {
1592 	return true;
1593 }
1594 #endif
1595 
1596 #ifdef CONFIG_GUEST_PERF_EVENTS
1597 unsigned long kvm_arch_vcpu_get_ip(struct kvm_vcpu *vcpu);
1598 
1599 void kvm_register_perf_callbacks(unsigned int (*pt_intr_handler)(void));
1600 void kvm_unregister_perf_callbacks(void);
1601 #else
kvm_register_perf_callbacks(void * ign)1602 static inline void kvm_register_perf_callbacks(void *ign) {}
kvm_unregister_perf_callbacks(void)1603 static inline void kvm_unregister_perf_callbacks(void) {}
1604 #endif /* CONFIG_GUEST_PERF_EVENTS */
1605 
1606 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type);
1607 void kvm_arch_destroy_vm(struct kvm *kvm);
1608 void kvm_arch_sync_events(struct kvm *kvm);
1609 
1610 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu);
1611 
1612 struct page *kvm_pfn_to_refcounted_page(kvm_pfn_t pfn);
1613 bool kvm_is_zone_device_page(struct page *page);
1614 
1615 struct kvm_irq_ack_notifier {
1616 	struct hlist_node link;
1617 	unsigned gsi;
1618 	void (*irq_acked)(struct kvm_irq_ack_notifier *kian);
1619 };
1620 
1621 int kvm_irq_map_gsi(struct kvm *kvm,
1622 		    struct kvm_kernel_irq_routing_entry *entries, int gsi);
1623 int kvm_irq_map_chip_pin(struct kvm *kvm, unsigned irqchip, unsigned pin);
1624 
1625 int kvm_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level,
1626 		bool line_status);
1627 int kvm_set_msi(struct kvm_kernel_irq_routing_entry *irq_entry, struct kvm *kvm,
1628 		int irq_source_id, int level, bool line_status);
1629 int kvm_arch_set_irq_inatomic(struct kvm_kernel_irq_routing_entry *e,
1630 			       struct kvm *kvm, int irq_source_id,
1631 			       int level, bool line_status);
1632 bool kvm_irq_has_notifier(struct kvm *kvm, unsigned irqchip, unsigned pin);
1633 void kvm_notify_acked_gsi(struct kvm *kvm, int gsi);
1634 void kvm_notify_acked_irq(struct kvm *kvm, unsigned irqchip, unsigned pin);
1635 void kvm_register_irq_ack_notifier(struct kvm *kvm,
1636 				   struct kvm_irq_ack_notifier *kian);
1637 void kvm_unregister_irq_ack_notifier(struct kvm *kvm,
1638 				   struct kvm_irq_ack_notifier *kian);
1639 int kvm_request_irq_source_id(struct kvm *kvm);
1640 void kvm_free_irq_source_id(struct kvm *kvm, int irq_source_id);
1641 bool kvm_arch_irqfd_allowed(struct kvm *kvm, struct kvm_irqfd *args);
1642 
1643 /*
1644  * Returns a pointer to the memslot if it contains gfn.
1645  * Otherwise returns NULL.
1646  */
1647 static inline struct kvm_memory_slot *
try_get_memslot(struct kvm_memory_slot * slot,gfn_t gfn)1648 try_get_memslot(struct kvm_memory_slot *slot, gfn_t gfn)
1649 {
1650 	if (!slot)
1651 		return NULL;
1652 
1653 	if (gfn >= slot->base_gfn && gfn < slot->base_gfn + slot->npages)
1654 		return slot;
1655 	else
1656 		return NULL;
1657 }
1658 
1659 /*
1660  * Returns a pointer to the memslot that contains gfn. Otherwise returns NULL.
1661  *
1662  * With "approx" set returns the memslot also when the address falls
1663  * in a hole. In that case one of the memslots bordering the hole is
1664  * returned.
1665  */
1666 static inline struct kvm_memory_slot *
search_memslots(struct kvm_memslots * slots,gfn_t gfn,bool approx)1667 search_memslots(struct kvm_memslots *slots, gfn_t gfn, bool approx)
1668 {
1669 	struct kvm_memory_slot *slot;
1670 	struct rb_node *node;
1671 	int idx = slots->node_idx;
1672 
1673 	slot = NULL;
1674 	for (node = slots->gfn_tree.rb_node; node; ) {
1675 		slot = container_of(node, struct kvm_memory_slot, gfn_node[idx]);
1676 		if (gfn >= slot->base_gfn) {
1677 			if (gfn < slot->base_gfn + slot->npages)
1678 				return slot;
1679 			node = node->rb_right;
1680 		} else
1681 			node = node->rb_left;
1682 	}
1683 
1684 	return approx ? slot : NULL;
1685 }
1686 
1687 static inline struct kvm_memory_slot *
____gfn_to_memslot(struct kvm_memslots * slots,gfn_t gfn,bool approx)1688 ____gfn_to_memslot(struct kvm_memslots *slots, gfn_t gfn, bool approx)
1689 {
1690 	struct kvm_memory_slot *slot;
1691 
1692 	slot = (struct kvm_memory_slot *)atomic_long_read(&slots->last_used_slot);
1693 	slot = try_get_memslot(slot, gfn);
1694 	if (slot)
1695 		return slot;
1696 
1697 	slot = search_memslots(slots, gfn, approx);
1698 	if (slot) {
1699 		atomic_long_set(&slots->last_used_slot, (unsigned long)slot);
1700 		return slot;
1701 	}
1702 
1703 	return NULL;
1704 }
1705 
1706 /*
1707  * __gfn_to_memslot() and its descendants are here to allow arch code to inline
1708  * the lookups in hot paths.  gfn_to_memslot() itself isn't here as an inline
1709  * because that would bloat other code too much.
1710  */
1711 static inline struct kvm_memory_slot *
__gfn_to_memslot(struct kvm_memslots * slots,gfn_t gfn)1712 __gfn_to_memslot(struct kvm_memslots *slots, gfn_t gfn)
1713 {
1714 	return ____gfn_to_memslot(slots, gfn, false);
1715 }
1716 
1717 static inline unsigned long
__gfn_to_hva_memslot(const struct kvm_memory_slot * slot,gfn_t gfn)1718 __gfn_to_hva_memslot(const struct kvm_memory_slot *slot, gfn_t gfn)
1719 {
1720 	/*
1721 	 * The index was checked originally in search_memslots.  To avoid
1722 	 * that a malicious guest builds a Spectre gadget out of e.g. page
1723 	 * table walks, do not let the processor speculate loads outside
1724 	 * the guest's registered memslots.
1725 	 */
1726 	unsigned long offset = gfn - slot->base_gfn;
1727 	offset = array_index_nospec(offset, slot->npages);
1728 	return slot->userspace_addr + offset * PAGE_SIZE;
1729 }
1730 
memslot_id(struct kvm * kvm,gfn_t gfn)1731 static inline int memslot_id(struct kvm *kvm, gfn_t gfn)
1732 {
1733 	return gfn_to_memslot(kvm, gfn)->id;
1734 }
1735 
1736 static inline gfn_t
hva_to_gfn_memslot(unsigned long hva,struct kvm_memory_slot * slot)1737 hva_to_gfn_memslot(unsigned long hva, struct kvm_memory_slot *slot)
1738 {
1739 	gfn_t gfn_offset = (hva - slot->userspace_addr) >> PAGE_SHIFT;
1740 
1741 	return slot->base_gfn + gfn_offset;
1742 }
1743 
gfn_to_gpa(gfn_t gfn)1744 static inline gpa_t gfn_to_gpa(gfn_t gfn)
1745 {
1746 	return (gpa_t)gfn << PAGE_SHIFT;
1747 }
1748 
gpa_to_gfn(gpa_t gpa)1749 static inline gfn_t gpa_to_gfn(gpa_t gpa)
1750 {
1751 	return (gfn_t)(gpa >> PAGE_SHIFT);
1752 }
1753 
pfn_to_hpa(kvm_pfn_t pfn)1754 static inline hpa_t pfn_to_hpa(kvm_pfn_t pfn)
1755 {
1756 	return (hpa_t)pfn << PAGE_SHIFT;
1757 }
1758 
kvm_is_error_gpa(struct kvm * kvm,gpa_t gpa)1759 static inline bool kvm_is_error_gpa(struct kvm *kvm, gpa_t gpa)
1760 {
1761 	unsigned long hva = gfn_to_hva(kvm, gpa_to_gfn(gpa));
1762 
1763 	return kvm_is_error_hva(hva);
1764 }
1765 
1766 enum kvm_stat_kind {
1767 	KVM_STAT_VM,
1768 	KVM_STAT_VCPU,
1769 };
1770 
1771 struct kvm_stat_data {
1772 	struct kvm *kvm;
1773 	const struct _kvm_stats_desc *desc;
1774 	enum kvm_stat_kind kind;
1775 };
1776 
1777 struct _kvm_stats_desc {
1778 	struct kvm_stats_desc desc;
1779 	char name[KVM_STATS_NAME_SIZE];
1780 };
1781 
1782 #define STATS_DESC_COMMON(type, unit, base, exp, sz, bsz)		       \
1783 	.flags = type | unit | base |					       \
1784 		 BUILD_BUG_ON_ZERO(type & ~KVM_STATS_TYPE_MASK) |	       \
1785 		 BUILD_BUG_ON_ZERO(unit & ~KVM_STATS_UNIT_MASK) |	       \
1786 		 BUILD_BUG_ON_ZERO(base & ~KVM_STATS_BASE_MASK),	       \
1787 	.exponent = exp,						       \
1788 	.size = sz,							       \
1789 	.bucket_size = bsz
1790 
1791 #define VM_GENERIC_STATS_DESC(stat, type, unit, base, exp, sz, bsz)	       \
1792 	{								       \
1793 		{							       \
1794 			STATS_DESC_COMMON(type, unit, base, exp, sz, bsz),     \
1795 			.offset = offsetof(struct kvm_vm_stat, generic.stat)   \
1796 		},							       \
1797 		.name = #stat,						       \
1798 	}
1799 #define VCPU_GENERIC_STATS_DESC(stat, type, unit, base, exp, sz, bsz)	       \
1800 	{								       \
1801 		{							       \
1802 			STATS_DESC_COMMON(type, unit, base, exp, sz, bsz),     \
1803 			.offset = offsetof(struct kvm_vcpu_stat, generic.stat) \
1804 		},							       \
1805 		.name = #stat,						       \
1806 	}
1807 #define VM_STATS_DESC(stat, type, unit, base, exp, sz, bsz)		       \
1808 	{								       \
1809 		{							       \
1810 			STATS_DESC_COMMON(type, unit, base, exp, sz, bsz),     \
1811 			.offset = offsetof(struct kvm_vm_stat, stat)	       \
1812 		},							       \
1813 		.name = #stat,						       \
1814 	}
1815 #define VCPU_STATS_DESC(stat, type, unit, base, exp, sz, bsz)		       \
1816 	{								       \
1817 		{							       \
1818 			STATS_DESC_COMMON(type, unit, base, exp, sz, bsz),     \
1819 			.offset = offsetof(struct kvm_vcpu_stat, stat)	       \
1820 		},							       \
1821 		.name = #stat,						       \
1822 	}
1823 /* SCOPE: VM, VM_GENERIC, VCPU, VCPU_GENERIC */
1824 #define STATS_DESC(SCOPE, stat, type, unit, base, exp, sz, bsz)		       \
1825 	SCOPE##_STATS_DESC(stat, type, unit, base, exp, sz, bsz)
1826 
1827 #define STATS_DESC_CUMULATIVE(SCOPE, name, unit, base, exponent)	       \
1828 	STATS_DESC(SCOPE, name, KVM_STATS_TYPE_CUMULATIVE,		       \
1829 		unit, base, exponent, 1, 0)
1830 #define STATS_DESC_INSTANT(SCOPE, name, unit, base, exponent)		       \
1831 	STATS_DESC(SCOPE, name, KVM_STATS_TYPE_INSTANT,			       \
1832 		unit, base, exponent, 1, 0)
1833 #define STATS_DESC_PEAK(SCOPE, name, unit, base, exponent)		       \
1834 	STATS_DESC(SCOPE, name, KVM_STATS_TYPE_PEAK,			       \
1835 		unit, base, exponent, 1, 0)
1836 #define STATS_DESC_LINEAR_HIST(SCOPE, name, unit, base, exponent, sz, bsz)     \
1837 	STATS_DESC(SCOPE, name, KVM_STATS_TYPE_LINEAR_HIST,		       \
1838 		unit, base, exponent, sz, bsz)
1839 #define STATS_DESC_LOG_HIST(SCOPE, name, unit, base, exponent, sz)	       \
1840 	STATS_DESC(SCOPE, name, KVM_STATS_TYPE_LOG_HIST,		       \
1841 		unit, base, exponent, sz, 0)
1842 
1843 /* Cumulative counter, read/write */
1844 #define STATS_DESC_COUNTER(SCOPE, name)					       \
1845 	STATS_DESC_CUMULATIVE(SCOPE, name, KVM_STATS_UNIT_NONE,		       \
1846 		KVM_STATS_BASE_POW10, 0)
1847 /* Instantaneous counter, read only */
1848 #define STATS_DESC_ICOUNTER(SCOPE, name)				       \
1849 	STATS_DESC_INSTANT(SCOPE, name, KVM_STATS_UNIT_NONE,		       \
1850 		KVM_STATS_BASE_POW10, 0)
1851 /* Peak counter, read/write */
1852 #define STATS_DESC_PCOUNTER(SCOPE, name)				       \
1853 	STATS_DESC_PEAK(SCOPE, name, KVM_STATS_UNIT_NONE,		       \
1854 		KVM_STATS_BASE_POW10, 0)
1855 
1856 /* Instantaneous boolean value, read only */
1857 #define STATS_DESC_IBOOLEAN(SCOPE, name)				       \
1858 	STATS_DESC_INSTANT(SCOPE, name, KVM_STATS_UNIT_BOOLEAN,		       \
1859 		KVM_STATS_BASE_POW10, 0)
1860 /* Peak (sticky) boolean value, read/write */
1861 #define STATS_DESC_PBOOLEAN(SCOPE, name)				       \
1862 	STATS_DESC_PEAK(SCOPE, name, KVM_STATS_UNIT_BOOLEAN,		       \
1863 		KVM_STATS_BASE_POW10, 0)
1864 
1865 /* Cumulative time in nanosecond */
1866 #define STATS_DESC_TIME_NSEC(SCOPE, name)				       \
1867 	STATS_DESC_CUMULATIVE(SCOPE, name, KVM_STATS_UNIT_SECONDS,	       \
1868 		KVM_STATS_BASE_POW10, -9)
1869 /* Linear histogram for time in nanosecond */
1870 #define STATS_DESC_LINHIST_TIME_NSEC(SCOPE, name, sz, bsz)		       \
1871 	STATS_DESC_LINEAR_HIST(SCOPE, name, KVM_STATS_UNIT_SECONDS,	       \
1872 		KVM_STATS_BASE_POW10, -9, sz, bsz)
1873 /* Logarithmic histogram for time in nanosecond */
1874 #define STATS_DESC_LOGHIST_TIME_NSEC(SCOPE, name, sz)			       \
1875 	STATS_DESC_LOG_HIST(SCOPE, name, KVM_STATS_UNIT_SECONDS,	       \
1876 		KVM_STATS_BASE_POW10, -9, sz)
1877 
1878 #define KVM_GENERIC_VM_STATS()						       \
1879 	STATS_DESC_COUNTER(VM_GENERIC, remote_tlb_flush),		       \
1880 	STATS_DESC_COUNTER(VM_GENERIC, remote_tlb_flush_requests)
1881 
1882 #define KVM_GENERIC_VCPU_STATS()					       \
1883 	STATS_DESC_COUNTER(VCPU_GENERIC, halt_successful_poll),		       \
1884 	STATS_DESC_COUNTER(VCPU_GENERIC, halt_attempted_poll),		       \
1885 	STATS_DESC_COUNTER(VCPU_GENERIC, halt_poll_invalid),		       \
1886 	STATS_DESC_COUNTER(VCPU_GENERIC, halt_wakeup),			       \
1887 	STATS_DESC_TIME_NSEC(VCPU_GENERIC, halt_poll_success_ns),	       \
1888 	STATS_DESC_TIME_NSEC(VCPU_GENERIC, halt_poll_fail_ns),		       \
1889 	STATS_DESC_TIME_NSEC(VCPU_GENERIC, halt_wait_ns),		       \
1890 	STATS_DESC_LOGHIST_TIME_NSEC(VCPU_GENERIC, halt_poll_success_hist,     \
1891 			HALT_POLL_HIST_COUNT),				       \
1892 	STATS_DESC_LOGHIST_TIME_NSEC(VCPU_GENERIC, halt_poll_fail_hist,	       \
1893 			HALT_POLL_HIST_COUNT),				       \
1894 	STATS_DESC_LOGHIST_TIME_NSEC(VCPU_GENERIC, halt_wait_hist,	       \
1895 			HALT_POLL_HIST_COUNT),				       \
1896 	STATS_DESC_IBOOLEAN(VCPU_GENERIC, blocking)
1897 
1898 extern struct dentry *kvm_debugfs_dir;
1899 
1900 ssize_t kvm_stats_read(char *id, const struct kvm_stats_header *header,
1901 		       const struct _kvm_stats_desc *desc,
1902 		       void *stats, size_t size_stats,
1903 		       char __user *user_buffer, size_t size, loff_t *offset);
1904 
1905 /**
1906  * kvm_stats_linear_hist_update() - Update bucket value for linear histogram
1907  * statistics data.
1908  *
1909  * @data: start address of the stats data
1910  * @size: the number of bucket of the stats data
1911  * @value: the new value used to update the linear histogram's bucket
1912  * @bucket_size: the size (width) of a bucket
1913  */
kvm_stats_linear_hist_update(u64 * data,size_t size,u64 value,size_t bucket_size)1914 static inline void kvm_stats_linear_hist_update(u64 *data, size_t size,
1915 						u64 value, size_t bucket_size)
1916 {
1917 	size_t index = div64_u64(value, bucket_size);
1918 
1919 	index = min(index, size - 1);
1920 	++data[index];
1921 }
1922 
1923 /**
1924  * kvm_stats_log_hist_update() - Update bucket value for logarithmic histogram
1925  * statistics data.
1926  *
1927  * @data: start address of the stats data
1928  * @size: the number of bucket of the stats data
1929  * @value: the new value used to update the logarithmic histogram's bucket
1930  */
kvm_stats_log_hist_update(u64 * data,size_t size,u64 value)1931 static inline void kvm_stats_log_hist_update(u64 *data, size_t size, u64 value)
1932 {
1933 	size_t index = fls64(value);
1934 
1935 	index = min(index, size - 1);
1936 	++data[index];
1937 }
1938 
1939 #define KVM_STATS_LINEAR_HIST_UPDATE(array, value, bsize)		       \
1940 	kvm_stats_linear_hist_update(array, ARRAY_SIZE(array), value, bsize)
1941 #define KVM_STATS_LOG_HIST_UPDATE(array, value)				       \
1942 	kvm_stats_log_hist_update(array, ARRAY_SIZE(array), value)
1943 
1944 
1945 extern const struct kvm_stats_header kvm_vm_stats_header;
1946 extern const struct _kvm_stats_desc kvm_vm_stats_desc[];
1947 extern const struct kvm_stats_header kvm_vcpu_stats_header;
1948 extern const struct _kvm_stats_desc kvm_vcpu_stats_desc[];
1949 
1950 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
mmu_invalidate_retry(struct kvm * kvm,unsigned long mmu_seq)1951 static inline int mmu_invalidate_retry(struct kvm *kvm, unsigned long mmu_seq)
1952 {
1953 	if (unlikely(kvm->mmu_invalidate_in_progress))
1954 		return 1;
1955 	/*
1956 	 * Ensure the read of mmu_invalidate_in_progress happens before
1957 	 * the read of mmu_invalidate_seq.  This interacts with the
1958 	 * smp_wmb() in mmu_notifier_invalidate_range_end to make sure
1959 	 * that the caller either sees the old (non-zero) value of
1960 	 * mmu_invalidate_in_progress or the new (incremented) value of
1961 	 * mmu_invalidate_seq.
1962 	 *
1963 	 * PowerPC Book3s HV KVM calls this under a per-page lock rather
1964 	 * than under kvm->mmu_lock, for scalability, so can't rely on
1965 	 * kvm->mmu_lock to keep things ordered.
1966 	 */
1967 	smp_rmb();
1968 	if (kvm->mmu_invalidate_seq != mmu_seq)
1969 		return 1;
1970 	return 0;
1971 }
1972 
mmu_invalidate_retry_hva(struct kvm * kvm,unsigned long mmu_seq,unsigned long hva)1973 static inline int mmu_invalidate_retry_hva(struct kvm *kvm,
1974 					   unsigned long mmu_seq,
1975 					   unsigned long hva)
1976 {
1977 	lockdep_assert_held(&kvm->mmu_lock);
1978 	/*
1979 	 * If mmu_invalidate_in_progress is non-zero, then the range maintained
1980 	 * by kvm_mmu_notifier_invalidate_range_start contains all addresses
1981 	 * that might be being invalidated. Note that it may include some false
1982 	 * positives, due to shortcuts when handing concurrent invalidations.
1983 	 */
1984 	if (unlikely(kvm->mmu_invalidate_in_progress) &&
1985 	    hva >= kvm->mmu_invalidate_range_start &&
1986 	    hva < kvm->mmu_invalidate_range_end)
1987 		return 1;
1988 	if (kvm->mmu_invalidate_seq != mmu_seq)
1989 		return 1;
1990 	return 0;
1991 }
1992 #endif
1993 
1994 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
1995 
1996 #define KVM_MAX_IRQ_ROUTES 4096 /* might need extension/rework in the future */
1997 
1998 bool kvm_arch_can_set_irq_routing(struct kvm *kvm);
1999 int kvm_set_irq_routing(struct kvm *kvm,
2000 			const struct kvm_irq_routing_entry *entries,
2001 			unsigned nr,
2002 			unsigned flags);
2003 int kvm_set_routing_entry(struct kvm *kvm,
2004 			  struct kvm_kernel_irq_routing_entry *e,
2005 			  const struct kvm_irq_routing_entry *ue);
2006 void kvm_free_irq_routing(struct kvm *kvm);
2007 
2008 #else
2009 
kvm_free_irq_routing(struct kvm * kvm)2010 static inline void kvm_free_irq_routing(struct kvm *kvm) {}
2011 
2012 #endif
2013 
2014 int kvm_send_userspace_msi(struct kvm *kvm, struct kvm_msi *msi);
2015 
2016 #ifdef CONFIG_HAVE_KVM_EVENTFD
2017 
2018 void kvm_eventfd_init(struct kvm *kvm);
2019 int kvm_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args);
2020 
2021 #ifdef CONFIG_HAVE_KVM_IRQFD
2022 int kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args);
2023 void kvm_irqfd_release(struct kvm *kvm);
2024 bool kvm_notify_irqfd_resampler(struct kvm *kvm,
2025 				unsigned int irqchip,
2026 				unsigned int pin);
2027 void kvm_irq_routing_update(struct kvm *);
2028 #else
kvm_irqfd(struct kvm * kvm,struct kvm_irqfd * args)2029 static inline int kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args)
2030 {
2031 	return -EINVAL;
2032 }
2033 
kvm_irqfd_release(struct kvm * kvm)2034 static inline void kvm_irqfd_release(struct kvm *kvm) {}
2035 
kvm_notify_irqfd_resampler(struct kvm * kvm,unsigned int irqchip,unsigned int pin)2036 static inline bool kvm_notify_irqfd_resampler(struct kvm *kvm,
2037 					      unsigned int irqchip,
2038 					      unsigned int pin)
2039 {
2040 	return false;
2041 }
2042 #endif
2043 
2044 #else
2045 
kvm_eventfd_init(struct kvm * kvm)2046 static inline void kvm_eventfd_init(struct kvm *kvm) {}
2047 
kvm_irqfd(struct kvm * kvm,struct kvm_irqfd * args)2048 static inline int kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args)
2049 {
2050 	return -EINVAL;
2051 }
2052 
kvm_irqfd_release(struct kvm * kvm)2053 static inline void kvm_irqfd_release(struct kvm *kvm) {}
2054 
2055 #ifdef CONFIG_HAVE_KVM_IRQCHIP
kvm_irq_routing_update(struct kvm * kvm)2056 static inline void kvm_irq_routing_update(struct kvm *kvm)
2057 {
2058 }
2059 #endif
2060 
kvm_ioeventfd(struct kvm * kvm,struct kvm_ioeventfd * args)2061 static inline int kvm_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args)
2062 {
2063 	return -ENOSYS;
2064 }
2065 
2066 #endif /* CONFIG_HAVE_KVM_EVENTFD */
2067 
2068 void kvm_arch_irq_routing_update(struct kvm *kvm);
2069 
__kvm_make_request(int req,struct kvm_vcpu * vcpu)2070 static inline void __kvm_make_request(int req, struct kvm_vcpu *vcpu)
2071 {
2072 	/*
2073 	 * Ensure the rest of the request is published to kvm_check_request's
2074 	 * caller.  Paired with the smp_mb__after_atomic in kvm_check_request.
2075 	 */
2076 	smp_wmb();
2077 	set_bit(req & KVM_REQUEST_MASK, (void *)&vcpu->requests);
2078 }
2079 
kvm_make_request(int req,struct kvm_vcpu * vcpu)2080 static __always_inline void kvm_make_request(int req, struct kvm_vcpu *vcpu)
2081 {
2082 	/*
2083 	 * Request that don't require vCPU action should never be logged in
2084 	 * vcpu->requests.  The vCPU won't clear the request, so it will stay
2085 	 * logged indefinitely and prevent the vCPU from entering the guest.
2086 	 */
2087 	BUILD_BUG_ON(!__builtin_constant_p(req) ||
2088 		     (req & KVM_REQUEST_NO_ACTION));
2089 
2090 	__kvm_make_request(req, vcpu);
2091 }
2092 
kvm_request_pending(struct kvm_vcpu * vcpu)2093 static inline bool kvm_request_pending(struct kvm_vcpu *vcpu)
2094 {
2095 	return READ_ONCE(vcpu->requests);
2096 }
2097 
kvm_test_request(int req,struct kvm_vcpu * vcpu)2098 static inline bool kvm_test_request(int req, struct kvm_vcpu *vcpu)
2099 {
2100 	return test_bit(req & KVM_REQUEST_MASK, (void *)&vcpu->requests);
2101 }
2102 
kvm_clear_request(int req,struct kvm_vcpu * vcpu)2103 static inline void kvm_clear_request(int req, struct kvm_vcpu *vcpu)
2104 {
2105 	clear_bit(req & KVM_REQUEST_MASK, (void *)&vcpu->requests);
2106 }
2107 
kvm_check_request(int req,struct kvm_vcpu * vcpu)2108 static inline bool kvm_check_request(int req, struct kvm_vcpu *vcpu)
2109 {
2110 	if (kvm_test_request(req, vcpu)) {
2111 		kvm_clear_request(req, vcpu);
2112 
2113 		/*
2114 		 * Ensure the rest of the request is visible to kvm_check_request's
2115 		 * caller.  Paired with the smp_wmb in kvm_make_request.
2116 		 */
2117 		smp_mb__after_atomic();
2118 		return true;
2119 	} else {
2120 		return false;
2121 	}
2122 }
2123 
2124 #ifdef CONFIG_KVM_GENERIC_HARDWARE_ENABLING
2125 extern bool kvm_rebooting;
2126 #endif
2127 
2128 extern unsigned int halt_poll_ns;
2129 extern unsigned int halt_poll_ns_grow;
2130 extern unsigned int halt_poll_ns_grow_start;
2131 extern unsigned int halt_poll_ns_shrink;
2132 
2133 struct kvm_device {
2134 	const struct kvm_device_ops *ops;
2135 	struct kvm *kvm;
2136 	void *private;
2137 	struct list_head vm_node;
2138 };
2139 
2140 /* create, destroy, and name are mandatory */
2141 struct kvm_device_ops {
2142 	const char *name;
2143 
2144 	/*
2145 	 * create is called holding kvm->lock and any operations not suitable
2146 	 * to do while holding the lock should be deferred to init (see
2147 	 * below).
2148 	 */
2149 	int (*create)(struct kvm_device *dev, u32 type);
2150 
2151 	/*
2152 	 * init is called after create if create is successful and is called
2153 	 * outside of holding kvm->lock.
2154 	 */
2155 	void (*init)(struct kvm_device *dev);
2156 
2157 	/*
2158 	 * Destroy is responsible for freeing dev.
2159 	 *
2160 	 * Destroy may be called before or after destructors are called
2161 	 * on emulated I/O regions, depending on whether a reference is
2162 	 * held by a vcpu or other kvm component that gets destroyed
2163 	 * after the emulated I/O.
2164 	 */
2165 	void (*destroy)(struct kvm_device *dev);
2166 
2167 	/*
2168 	 * Release is an alternative method to free the device. It is
2169 	 * called when the device file descriptor is closed. Once
2170 	 * release is called, the destroy method will not be called
2171 	 * anymore as the device is removed from the device list of
2172 	 * the VM. kvm->lock is held.
2173 	 */
2174 	void (*release)(struct kvm_device *dev);
2175 
2176 	int (*set_attr)(struct kvm_device *dev, struct kvm_device_attr *attr);
2177 	int (*get_attr)(struct kvm_device *dev, struct kvm_device_attr *attr);
2178 	int (*has_attr)(struct kvm_device *dev, struct kvm_device_attr *attr);
2179 	long (*ioctl)(struct kvm_device *dev, unsigned int ioctl,
2180 		      unsigned long arg);
2181 	int (*mmap)(struct kvm_device *dev, struct vm_area_struct *vma);
2182 };
2183 
2184 struct kvm_device *kvm_device_from_filp(struct file *filp);
2185 int kvm_register_device_ops(const struct kvm_device_ops *ops, u32 type);
2186 void kvm_unregister_device_ops(u32 type);
2187 
2188 extern struct kvm_device_ops kvm_mpic_ops;
2189 extern struct kvm_device_ops kvm_arm_vgic_v2_ops;
2190 extern struct kvm_device_ops kvm_arm_vgic_v3_ops;
2191 
2192 #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
2193 
kvm_vcpu_set_in_spin_loop(struct kvm_vcpu * vcpu,bool val)2194 static inline void kvm_vcpu_set_in_spin_loop(struct kvm_vcpu *vcpu, bool val)
2195 {
2196 	vcpu->spin_loop.in_spin_loop = val;
2197 }
kvm_vcpu_set_dy_eligible(struct kvm_vcpu * vcpu,bool val)2198 static inline void kvm_vcpu_set_dy_eligible(struct kvm_vcpu *vcpu, bool val)
2199 {
2200 	vcpu->spin_loop.dy_eligible = val;
2201 }
2202 
2203 #else /* !CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT */
2204 
kvm_vcpu_set_in_spin_loop(struct kvm_vcpu * vcpu,bool val)2205 static inline void kvm_vcpu_set_in_spin_loop(struct kvm_vcpu *vcpu, bool val)
2206 {
2207 }
2208 
kvm_vcpu_set_dy_eligible(struct kvm_vcpu * vcpu,bool val)2209 static inline void kvm_vcpu_set_dy_eligible(struct kvm_vcpu *vcpu, bool val)
2210 {
2211 }
2212 #endif /* CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT */
2213 
kvm_is_visible_memslot(struct kvm_memory_slot * memslot)2214 static inline bool kvm_is_visible_memslot(struct kvm_memory_slot *memslot)
2215 {
2216 	return (memslot && memslot->id < KVM_USER_MEM_SLOTS &&
2217 		!(memslot->flags & KVM_MEMSLOT_INVALID));
2218 }
2219 
2220 struct kvm_vcpu *kvm_get_running_vcpu(void);
2221 struct kvm_vcpu * __percpu *kvm_get_running_vcpus(void);
2222 
2223 #ifdef CONFIG_HAVE_KVM_IRQ_BYPASS
2224 bool kvm_arch_has_irq_bypass(void);
2225 int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *,
2226 			   struct irq_bypass_producer *);
2227 void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *,
2228 			   struct irq_bypass_producer *);
2229 void kvm_arch_irq_bypass_stop(struct irq_bypass_consumer *);
2230 void kvm_arch_irq_bypass_start(struct irq_bypass_consumer *);
2231 int kvm_arch_update_irqfd_routing(struct kvm *kvm, unsigned int host_irq,
2232 				  uint32_t guest_irq, bool set);
2233 bool kvm_arch_irqfd_route_changed(struct kvm_kernel_irq_routing_entry *,
2234 				  struct kvm_kernel_irq_routing_entry *);
2235 #endif /* CONFIG_HAVE_KVM_IRQ_BYPASS */
2236 
2237 #ifdef CONFIG_HAVE_KVM_INVALID_WAKEUPS
2238 /* If we wakeup during the poll time, was it a sucessful poll? */
vcpu_valid_wakeup(struct kvm_vcpu * vcpu)2239 static inline bool vcpu_valid_wakeup(struct kvm_vcpu *vcpu)
2240 {
2241 	return vcpu->valid_wakeup;
2242 }
2243 
2244 #else
vcpu_valid_wakeup(struct kvm_vcpu * vcpu)2245 static inline bool vcpu_valid_wakeup(struct kvm_vcpu *vcpu)
2246 {
2247 	return true;
2248 }
2249 #endif /* CONFIG_HAVE_KVM_INVALID_WAKEUPS */
2250 
2251 #ifdef CONFIG_HAVE_KVM_NO_POLL
2252 /* Callback that tells if we must not poll */
2253 bool kvm_arch_no_poll(struct kvm_vcpu *vcpu);
2254 #else
kvm_arch_no_poll(struct kvm_vcpu * vcpu)2255 static inline bool kvm_arch_no_poll(struct kvm_vcpu *vcpu)
2256 {
2257 	return false;
2258 }
2259 #endif /* CONFIG_HAVE_KVM_NO_POLL */
2260 
2261 #ifdef CONFIG_HAVE_KVM_VCPU_ASYNC_IOCTL
2262 long kvm_arch_vcpu_async_ioctl(struct file *filp,
2263 			       unsigned int ioctl, unsigned long arg);
2264 #else
kvm_arch_vcpu_async_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)2265 static inline long kvm_arch_vcpu_async_ioctl(struct file *filp,
2266 					     unsigned int ioctl,
2267 					     unsigned long arg)
2268 {
2269 	return -ENOIOCTLCMD;
2270 }
2271 #endif /* CONFIG_HAVE_KVM_VCPU_ASYNC_IOCTL */
2272 
2273 void kvm_arch_guest_memory_reclaimed(struct kvm *kvm);
2274 
2275 #ifdef CONFIG_HAVE_KVM_VCPU_RUN_PID_CHANGE
2276 int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu);
2277 #else
kvm_arch_vcpu_run_pid_change(struct kvm_vcpu * vcpu)2278 static inline int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu)
2279 {
2280 	return 0;
2281 }
2282 #endif /* CONFIG_HAVE_KVM_VCPU_RUN_PID_CHANGE */
2283 
2284 typedef int (*kvm_vm_thread_fn_t)(struct kvm *kvm, uintptr_t data);
2285 
2286 int kvm_vm_create_worker_thread(struct kvm *kvm, kvm_vm_thread_fn_t thread_fn,
2287 				uintptr_t data, const char *name,
2288 				struct task_struct **thread_ptr);
2289 
2290 #ifdef CONFIG_KVM_XFER_TO_GUEST_WORK
kvm_handle_signal_exit(struct kvm_vcpu * vcpu)2291 static inline void kvm_handle_signal_exit(struct kvm_vcpu *vcpu)
2292 {
2293 	vcpu->run->exit_reason = KVM_EXIT_INTR;
2294 	vcpu->stat.signal_exits++;
2295 }
2296 #endif /* CONFIG_KVM_XFER_TO_GUEST_WORK */
2297 
2298 /*
2299  * If more than one page is being (un)accounted, @virt must be the address of
2300  * the first page of a block of pages what were allocated together (i.e
2301  * accounted together).
2302  *
2303  * kvm_account_pgtable_pages() is thread-safe because mod_lruvec_page_state()
2304  * is thread-safe.
2305  */
kvm_account_pgtable_pages(void * virt,int nr)2306 static inline void kvm_account_pgtable_pages(void *virt, int nr)
2307 {
2308 	mod_lruvec_page_state(virt_to_page(virt), NR_SECONDARY_PAGETABLE, nr);
2309 }
2310 
2311 /*
2312  * This defines how many reserved entries we want to keep before we
2313  * kick the vcpu to the userspace to avoid dirty ring full.  This
2314  * value can be tuned to higher if e.g. PML is enabled on the host.
2315  */
2316 #define  KVM_DIRTY_RING_RSVD_ENTRIES  64
2317 
2318 /* Max number of entries allowed for each kvm dirty ring */
2319 #define  KVM_DIRTY_RING_MAX_ENTRIES  65536
2320 
2321 #endif
2322