1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3  * tools/testing/selftests/kvm/include/x86_64/processor.h
4  *
5  * Copyright (C) 2018, Google LLC.
6  */
7 
8 #ifndef SELFTEST_KVM_PROCESSOR_H
9 #define SELFTEST_KVM_PROCESSOR_H
10 
11 #include <assert.h>
12 #include <stdint.h>
13 #include <syscall.h>
14 
15 #include <asm/msr-index.h>
16 #include <asm/prctl.h>
17 
18 #include <linux/stringify.h>
19 
20 #include "../kvm_util.h"
21 
22 #define NMI_VECTOR		0x02
23 
24 #define X86_EFLAGS_FIXED	 (1u << 1)
25 
26 #define X86_CR4_VME		(1ul << 0)
27 #define X86_CR4_PVI		(1ul << 1)
28 #define X86_CR4_TSD		(1ul << 2)
29 #define X86_CR4_DE		(1ul << 3)
30 #define X86_CR4_PSE		(1ul << 4)
31 #define X86_CR4_PAE		(1ul << 5)
32 #define X86_CR4_MCE		(1ul << 6)
33 #define X86_CR4_PGE		(1ul << 7)
34 #define X86_CR4_PCE		(1ul << 8)
35 #define X86_CR4_OSFXSR		(1ul << 9)
36 #define X86_CR4_OSXMMEXCPT	(1ul << 10)
37 #define X86_CR4_UMIP		(1ul << 11)
38 #define X86_CR4_LA57		(1ul << 12)
39 #define X86_CR4_VMXE		(1ul << 13)
40 #define X86_CR4_SMXE		(1ul << 14)
41 #define X86_CR4_FSGSBASE	(1ul << 16)
42 #define X86_CR4_PCIDE		(1ul << 17)
43 #define X86_CR4_OSXSAVE		(1ul << 18)
44 #define X86_CR4_SMEP		(1ul << 20)
45 #define X86_CR4_SMAP		(1ul << 21)
46 #define X86_CR4_PKE		(1ul << 22)
47 
48 /* Note, these are ordered alphabetically to match kvm_cpuid_entry2.  Eww. */
49 enum cpuid_output_regs {
50 	KVM_CPUID_EAX,
51 	KVM_CPUID_EBX,
52 	KVM_CPUID_ECX,
53 	KVM_CPUID_EDX
54 };
55 
56 /*
57  * Pack the information into a 64-bit value so that each X86_FEATURE_XXX can be
58  * passed by value with no overhead.
59  */
60 struct kvm_x86_cpu_feature {
61 	u32	function;
62 	u16	index;
63 	u8	reg;
64 	u8	bit;
65 };
66 #define	KVM_X86_CPU_FEATURE(fn, idx, gpr, __bit)	\
67 ({							\
68 	struct kvm_x86_cpu_feature feature = {		\
69 		.function = fn,				\
70 		.index = idx,				\
71 		.reg = KVM_CPUID_##gpr,			\
72 		.bit = __bit,				\
73 	};						\
74 							\
75 	feature;					\
76 })
77 
78 /*
79  * Basic Leafs, a.k.a. Intel defined
80  */
81 #define	X86_FEATURE_MWAIT		KVM_X86_CPU_FEATURE(0x1, 0, ECX, 3)
82 #define	X86_FEATURE_VMX			KVM_X86_CPU_FEATURE(0x1, 0, ECX, 5)
83 #define	X86_FEATURE_SMX			KVM_X86_CPU_FEATURE(0x1, 0, ECX, 6)
84 #define	X86_FEATURE_PDCM		KVM_X86_CPU_FEATURE(0x1, 0, ECX, 15)
85 #define	X86_FEATURE_PCID		KVM_X86_CPU_FEATURE(0x1, 0, ECX, 17)
86 #define X86_FEATURE_X2APIC		KVM_X86_CPU_FEATURE(0x1, 0, ECX, 21)
87 #define	X86_FEATURE_MOVBE		KVM_X86_CPU_FEATURE(0x1, 0, ECX, 22)
88 #define	X86_FEATURE_TSC_DEADLINE_TIMER	KVM_X86_CPU_FEATURE(0x1, 0, ECX, 24)
89 #define	X86_FEATURE_XSAVE		KVM_X86_CPU_FEATURE(0x1, 0, ECX, 26)
90 #define	X86_FEATURE_OSXSAVE		KVM_X86_CPU_FEATURE(0x1, 0, ECX, 27)
91 #define	X86_FEATURE_RDRAND		KVM_X86_CPU_FEATURE(0x1, 0, ECX, 30)
92 #define	X86_FEATURE_MCE			KVM_X86_CPU_FEATURE(0x1, 0, EDX, 7)
93 #define	X86_FEATURE_APIC		KVM_X86_CPU_FEATURE(0x1, 0, EDX, 9)
94 #define	X86_FEATURE_CLFLUSH		KVM_X86_CPU_FEATURE(0x1, 0, EDX, 19)
95 #define	X86_FEATURE_XMM			KVM_X86_CPU_FEATURE(0x1, 0, EDX, 25)
96 #define	X86_FEATURE_XMM2		KVM_X86_CPU_FEATURE(0x1, 0, EDX, 26)
97 #define	X86_FEATURE_FSGSBASE		KVM_X86_CPU_FEATURE(0x7, 0, EBX, 0)
98 #define	X86_FEATURE_TSC_ADJUST		KVM_X86_CPU_FEATURE(0x7, 0, EBX, 1)
99 #define	X86_FEATURE_HLE			KVM_X86_CPU_FEATURE(0x7, 0, EBX, 4)
100 #define	X86_FEATURE_SMEP	        KVM_X86_CPU_FEATURE(0x7, 0, EBX, 7)
101 #define	X86_FEATURE_INVPCID		KVM_X86_CPU_FEATURE(0x7, 0, EBX, 10)
102 #define	X86_FEATURE_RTM			KVM_X86_CPU_FEATURE(0x7, 0, EBX, 11)
103 #define	X86_FEATURE_MPX			KVM_X86_CPU_FEATURE(0x7, 0, EBX, 14)
104 #define	X86_FEATURE_SMAP		KVM_X86_CPU_FEATURE(0x7, 0, EBX, 20)
105 #define	X86_FEATURE_PCOMMIT		KVM_X86_CPU_FEATURE(0x7, 0, EBX, 22)
106 #define	X86_FEATURE_CLFLUSHOPT		KVM_X86_CPU_FEATURE(0x7, 0, EBX, 23)
107 #define	X86_FEATURE_CLWB		KVM_X86_CPU_FEATURE(0x7, 0, EBX, 24)
108 #define	X86_FEATURE_UMIP		KVM_X86_CPU_FEATURE(0x7, 0, ECX, 2)
109 #define	X86_FEATURE_PKU			KVM_X86_CPU_FEATURE(0x7, 0, ECX, 3)
110 #define	X86_FEATURE_LA57		KVM_X86_CPU_FEATURE(0x7, 0, ECX, 16)
111 #define	X86_FEATURE_RDPID		KVM_X86_CPU_FEATURE(0x7, 0, ECX, 22)
112 #define	X86_FEATURE_SHSTK		KVM_X86_CPU_FEATURE(0x7, 0, ECX, 7)
113 #define	X86_FEATURE_IBT			KVM_X86_CPU_FEATURE(0x7, 0, EDX, 20)
114 #define	X86_FEATURE_AMX_TILE		KVM_X86_CPU_FEATURE(0x7, 0, EDX, 24)
115 #define	X86_FEATURE_SPEC_CTRL		KVM_X86_CPU_FEATURE(0x7, 0, EDX, 26)
116 #define	X86_FEATURE_ARCH_CAPABILITIES	KVM_X86_CPU_FEATURE(0x7, 0, EDX, 29)
117 #define	X86_FEATURE_PKS			KVM_X86_CPU_FEATURE(0x7, 0, ECX, 31)
118 #define	X86_FEATURE_XTILECFG		KVM_X86_CPU_FEATURE(0xD, 0, EAX, 17)
119 #define	X86_FEATURE_XTILEDATA		KVM_X86_CPU_FEATURE(0xD, 0, EAX, 18)
120 #define	X86_FEATURE_XSAVES		KVM_X86_CPU_FEATURE(0xD, 1, EAX, 3)
121 #define	X86_FEATURE_XFD			KVM_X86_CPU_FEATURE(0xD, 1, EAX, 4)
122 
123 /*
124  * Extended Leafs, a.k.a. AMD defined
125  */
126 #define	X86_FEATURE_SVM			KVM_X86_CPU_FEATURE(0x80000001, 0, ECX, 2)
127 #define	X86_FEATURE_NX			KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 20)
128 #define	X86_FEATURE_GBPAGES		KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 26)
129 #define	X86_FEATURE_RDTSCP		KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 27)
130 #define	X86_FEATURE_LM			KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 29)
131 #define	X86_FEATURE_RDPRU		KVM_X86_CPU_FEATURE(0x80000008, 0, EBX, 4)
132 #define	X86_FEATURE_AMD_IBPB		KVM_X86_CPU_FEATURE(0x80000008, 0, EBX, 12)
133 #define	X86_FEATURE_NPT			KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 0)
134 #define	X86_FEATURE_LBRV		KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 1)
135 #define	X86_FEATURE_NRIPS		KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 3)
136 #define X86_FEATURE_TSCRATEMSR          KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 4)
137 #define X86_FEATURE_PAUSEFILTER         KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 10)
138 #define X86_FEATURE_PFTHRESHOLD         KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 12)
139 #define	X86_FEATURE_VGIF		KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 16)
140 #define X86_FEATURE_SEV			KVM_X86_CPU_FEATURE(0x8000001F, 0, EAX, 1)
141 #define X86_FEATURE_SEV_ES		KVM_X86_CPU_FEATURE(0x8000001F, 0, EAX, 3)
142 
143 /*
144  * KVM defined paravirt features.
145  */
146 #define X86_FEATURE_KVM_CLOCKSOURCE	KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 0)
147 #define X86_FEATURE_KVM_NOP_IO_DELAY	KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 1)
148 #define X86_FEATURE_KVM_MMU_OP		KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 2)
149 #define X86_FEATURE_KVM_CLOCKSOURCE2	KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 3)
150 #define X86_FEATURE_KVM_ASYNC_PF	KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 4)
151 #define X86_FEATURE_KVM_STEAL_TIME	KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 5)
152 #define X86_FEATURE_KVM_PV_EOI		KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 6)
153 #define X86_FEATURE_KVM_PV_UNHALT	KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 7)
154 /* Bit 8 apparently isn't used?!?! */
155 #define X86_FEATURE_KVM_PV_TLB_FLUSH	KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 9)
156 #define X86_FEATURE_KVM_ASYNC_PF_VMEXIT	KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 10)
157 #define X86_FEATURE_KVM_PV_SEND_IPI	KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 11)
158 #define X86_FEATURE_KVM_POLL_CONTROL	KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 12)
159 #define X86_FEATURE_KVM_PV_SCHED_YIELD	KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 13)
160 #define X86_FEATURE_KVM_ASYNC_PF_INT	KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 14)
161 #define X86_FEATURE_KVM_MSI_EXT_DEST_ID	KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 15)
162 #define X86_FEATURE_KVM_HC_MAP_GPA_RANGE	KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 16)
163 #define X86_FEATURE_KVM_MIGRATION_CONTROL	KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 17)
164 
165 /* Page table bitfield declarations */
166 #define PTE_PRESENT_MASK        BIT_ULL(0)
167 #define PTE_WRITABLE_MASK       BIT_ULL(1)
168 #define PTE_USER_MASK           BIT_ULL(2)
169 #define PTE_ACCESSED_MASK       BIT_ULL(5)
170 #define PTE_DIRTY_MASK          BIT_ULL(6)
171 #define PTE_LARGE_MASK          BIT_ULL(7)
172 #define PTE_GLOBAL_MASK         BIT_ULL(8)
173 #define PTE_NX_MASK             BIT_ULL(63)
174 
175 #define PAGE_SHIFT		12
176 #define PAGE_SIZE		(1ULL << PAGE_SHIFT)
177 #define PAGE_MASK		(~(PAGE_SIZE-1))
178 
179 #define PHYSICAL_PAGE_MASK      GENMASK_ULL(51, 12)
180 #define PTE_GET_PFN(pte)        (((pte) & PHYSICAL_PAGE_MASK) >> PAGE_SHIFT)
181 
182 /* General Registers in 64-Bit Mode */
183 struct gpr64_regs {
184 	u64 rax;
185 	u64 rcx;
186 	u64 rdx;
187 	u64 rbx;
188 	u64 rsp;
189 	u64 rbp;
190 	u64 rsi;
191 	u64 rdi;
192 	u64 r8;
193 	u64 r9;
194 	u64 r10;
195 	u64 r11;
196 	u64 r12;
197 	u64 r13;
198 	u64 r14;
199 	u64 r15;
200 };
201 
202 struct desc64 {
203 	uint16_t limit0;
204 	uint16_t base0;
205 	unsigned base1:8, type:4, s:1, dpl:2, p:1;
206 	unsigned limit1:4, avl:1, l:1, db:1, g:1, base2:8;
207 	uint32_t base3;
208 	uint32_t zero1;
209 } __attribute__((packed));
210 
211 struct desc_ptr {
212 	uint16_t size;
213 	uint64_t address;
214 } __attribute__((packed));
215 
216 struct kvm_x86_state {
217 	struct kvm_xsave *xsave;
218 	struct kvm_vcpu_events events;
219 	struct kvm_mp_state mp_state;
220 	struct kvm_regs regs;
221 	struct kvm_xcrs xcrs;
222 	struct kvm_sregs sregs;
223 	struct kvm_debugregs debugregs;
224 	union {
225 		struct kvm_nested_state nested;
226 		char nested_[16384];
227 	};
228 	struct kvm_msrs msrs;
229 };
230 
get_desc64_base(const struct desc64 * desc)231 static inline uint64_t get_desc64_base(const struct desc64 *desc)
232 {
233 	return ((uint64_t)desc->base3 << 32) |
234 		(desc->base0 | ((desc->base1) << 16) | ((desc->base2) << 24));
235 }
236 
rdtsc(void)237 static inline uint64_t rdtsc(void)
238 {
239 	uint32_t eax, edx;
240 	uint64_t tsc_val;
241 	/*
242 	 * The lfence is to wait (on Intel CPUs) until all previous
243 	 * instructions have been executed. If software requires RDTSC to be
244 	 * executed prior to execution of any subsequent instruction, it can
245 	 * execute LFENCE immediately after RDTSC
246 	 */
247 	__asm__ __volatile__("lfence; rdtsc; lfence" : "=a"(eax), "=d"(edx));
248 	tsc_val = ((uint64_t)edx) << 32 | eax;
249 	return tsc_val;
250 }
251 
rdtscp(uint32_t * aux)252 static inline uint64_t rdtscp(uint32_t *aux)
253 {
254 	uint32_t eax, edx;
255 
256 	__asm__ __volatile__("rdtscp" : "=a"(eax), "=d"(edx), "=c"(*aux));
257 	return ((uint64_t)edx) << 32 | eax;
258 }
259 
rdmsr(uint32_t msr)260 static inline uint64_t rdmsr(uint32_t msr)
261 {
262 	uint32_t a, d;
263 
264 	__asm__ __volatile__("rdmsr" : "=a"(a), "=d"(d) : "c"(msr) : "memory");
265 
266 	return a | ((uint64_t) d << 32);
267 }
268 
wrmsr(uint32_t msr,uint64_t value)269 static inline void wrmsr(uint32_t msr, uint64_t value)
270 {
271 	uint32_t a = value;
272 	uint32_t d = value >> 32;
273 
274 	__asm__ __volatile__("wrmsr" :: "a"(a), "d"(d), "c"(msr) : "memory");
275 }
276 
277 
inw(uint16_t port)278 static inline uint16_t inw(uint16_t port)
279 {
280 	uint16_t tmp;
281 
282 	__asm__ __volatile__("in %%dx, %%ax"
283 		: /* output */ "=a" (tmp)
284 		: /* input */ "d" (port));
285 
286 	return tmp;
287 }
288 
get_es(void)289 static inline uint16_t get_es(void)
290 {
291 	uint16_t es;
292 
293 	__asm__ __volatile__("mov %%es, %[es]"
294 			     : /* output */ [es]"=rm"(es));
295 	return es;
296 }
297 
get_cs(void)298 static inline uint16_t get_cs(void)
299 {
300 	uint16_t cs;
301 
302 	__asm__ __volatile__("mov %%cs, %[cs]"
303 			     : /* output */ [cs]"=rm"(cs));
304 	return cs;
305 }
306 
get_ss(void)307 static inline uint16_t get_ss(void)
308 {
309 	uint16_t ss;
310 
311 	__asm__ __volatile__("mov %%ss, %[ss]"
312 			     : /* output */ [ss]"=rm"(ss));
313 	return ss;
314 }
315 
get_ds(void)316 static inline uint16_t get_ds(void)
317 {
318 	uint16_t ds;
319 
320 	__asm__ __volatile__("mov %%ds, %[ds]"
321 			     : /* output */ [ds]"=rm"(ds));
322 	return ds;
323 }
324 
get_fs(void)325 static inline uint16_t get_fs(void)
326 {
327 	uint16_t fs;
328 
329 	__asm__ __volatile__("mov %%fs, %[fs]"
330 			     : /* output */ [fs]"=rm"(fs));
331 	return fs;
332 }
333 
get_gs(void)334 static inline uint16_t get_gs(void)
335 {
336 	uint16_t gs;
337 
338 	__asm__ __volatile__("mov %%gs, %[gs]"
339 			     : /* output */ [gs]"=rm"(gs));
340 	return gs;
341 }
342 
get_tr(void)343 static inline uint16_t get_tr(void)
344 {
345 	uint16_t tr;
346 
347 	__asm__ __volatile__("str %[tr]"
348 			     : /* output */ [tr]"=rm"(tr));
349 	return tr;
350 }
351 
get_cr0(void)352 static inline uint64_t get_cr0(void)
353 {
354 	uint64_t cr0;
355 
356 	__asm__ __volatile__("mov %%cr0, %[cr0]"
357 			     : /* output */ [cr0]"=r"(cr0));
358 	return cr0;
359 }
360 
get_cr3(void)361 static inline uint64_t get_cr3(void)
362 {
363 	uint64_t cr3;
364 
365 	__asm__ __volatile__("mov %%cr3, %[cr3]"
366 			     : /* output */ [cr3]"=r"(cr3));
367 	return cr3;
368 }
369 
get_cr4(void)370 static inline uint64_t get_cr4(void)
371 {
372 	uint64_t cr4;
373 
374 	__asm__ __volatile__("mov %%cr4, %[cr4]"
375 			     : /* output */ [cr4]"=r"(cr4));
376 	return cr4;
377 }
378 
set_cr4(uint64_t val)379 static inline void set_cr4(uint64_t val)
380 {
381 	__asm__ __volatile__("mov %0, %%cr4" : : "r" (val) : "memory");
382 }
383 
get_gdt(void)384 static inline struct desc_ptr get_gdt(void)
385 {
386 	struct desc_ptr gdt;
387 	__asm__ __volatile__("sgdt %[gdt]"
388 			     : /* output */ [gdt]"=m"(gdt));
389 	return gdt;
390 }
391 
get_idt(void)392 static inline struct desc_ptr get_idt(void)
393 {
394 	struct desc_ptr idt;
395 	__asm__ __volatile__("sidt %[idt]"
396 			     : /* output */ [idt]"=m"(idt));
397 	return idt;
398 }
399 
outl(uint16_t port,uint32_t value)400 static inline void outl(uint16_t port, uint32_t value)
401 {
402 	__asm__ __volatile__("outl %%eax, %%dx" : : "d"(port), "a"(value));
403 }
404 
__cpuid(uint32_t function,uint32_t index,uint32_t * eax,uint32_t * ebx,uint32_t * ecx,uint32_t * edx)405 static inline void __cpuid(uint32_t function, uint32_t index,
406 			   uint32_t *eax, uint32_t *ebx,
407 			   uint32_t *ecx, uint32_t *edx)
408 {
409 	*eax = function;
410 	*ecx = index;
411 
412 	asm volatile("cpuid"
413 	    : "=a" (*eax),
414 	      "=b" (*ebx),
415 	      "=c" (*ecx),
416 	      "=d" (*edx)
417 	    : "0" (*eax), "2" (*ecx)
418 	    : "memory");
419 }
420 
cpuid(uint32_t function,uint32_t * eax,uint32_t * ebx,uint32_t * ecx,uint32_t * edx)421 static inline void cpuid(uint32_t function,
422 			 uint32_t *eax, uint32_t *ebx,
423 			 uint32_t *ecx, uint32_t *edx)
424 {
425 	return __cpuid(function, 0, eax, ebx, ecx, edx);
426 }
427 
this_cpu_has(struct kvm_x86_cpu_feature feature)428 static inline bool this_cpu_has(struct kvm_x86_cpu_feature feature)
429 {
430 	uint32_t gprs[4];
431 
432 	__cpuid(feature.function, feature.index,
433 		&gprs[KVM_CPUID_EAX], &gprs[KVM_CPUID_EBX],
434 		&gprs[KVM_CPUID_ECX], &gprs[KVM_CPUID_EDX]);
435 
436 	return gprs[feature.reg] & BIT(feature.bit);
437 }
438 
439 #define SET_XMM(__var, __xmm) \
440 	asm volatile("movq %0, %%"#__xmm : : "r"(__var) : #__xmm)
441 
set_xmm(int n,unsigned long val)442 static inline void set_xmm(int n, unsigned long val)
443 {
444 	switch (n) {
445 	case 0:
446 		SET_XMM(val, xmm0);
447 		break;
448 	case 1:
449 		SET_XMM(val, xmm1);
450 		break;
451 	case 2:
452 		SET_XMM(val, xmm2);
453 		break;
454 	case 3:
455 		SET_XMM(val, xmm3);
456 		break;
457 	case 4:
458 		SET_XMM(val, xmm4);
459 		break;
460 	case 5:
461 		SET_XMM(val, xmm5);
462 		break;
463 	case 6:
464 		SET_XMM(val, xmm6);
465 		break;
466 	case 7:
467 		SET_XMM(val, xmm7);
468 		break;
469 	}
470 }
471 
472 #define GET_XMM(__xmm)							\
473 ({									\
474 	unsigned long __val;						\
475 	asm volatile("movq %%"#__xmm", %0" : "=r"(__val));		\
476 	__val;								\
477 })
478 
get_xmm(int n)479 static inline unsigned long get_xmm(int n)
480 {
481 	assert(n >= 0 && n <= 7);
482 
483 	switch (n) {
484 	case 0:
485 		return GET_XMM(xmm0);
486 	case 1:
487 		return GET_XMM(xmm1);
488 	case 2:
489 		return GET_XMM(xmm2);
490 	case 3:
491 		return GET_XMM(xmm3);
492 	case 4:
493 		return GET_XMM(xmm4);
494 	case 5:
495 		return GET_XMM(xmm5);
496 	case 6:
497 		return GET_XMM(xmm6);
498 	case 7:
499 		return GET_XMM(xmm7);
500 	}
501 
502 	/* never reached */
503 	return 0;
504 }
505 
cpu_relax(void)506 static inline void cpu_relax(void)
507 {
508 	asm volatile("rep; nop" ::: "memory");
509 }
510 
511 #define vmmcall()		\
512 	__asm__ __volatile__(	\
513 		"vmmcall\n"	\
514 		)
515 
516 #define ud2()			\
517 	__asm__ __volatile__(	\
518 		"ud2\n"	\
519 		)
520 
521 #define hlt()			\
522 	__asm__ __volatile__(	\
523 		"hlt\n"	\
524 		)
525 
526 bool is_intel_cpu(void);
527 bool is_amd_cpu(void);
528 
x86_family(unsigned int eax)529 static inline unsigned int x86_family(unsigned int eax)
530 {
531 	unsigned int x86;
532 
533 	x86 = (eax >> 8) & 0xf;
534 
535 	if (x86 == 0xf)
536 		x86 += (eax >> 20) & 0xff;
537 
538 	return x86;
539 }
540 
x86_model(unsigned int eax)541 static inline unsigned int x86_model(unsigned int eax)
542 {
543 	return ((eax >> 12) & 0xf0) | ((eax >> 4) & 0x0f);
544 }
545 
546 struct kvm_x86_state *vcpu_save_state(struct kvm_vcpu *vcpu);
547 void vcpu_load_state(struct kvm_vcpu *vcpu, struct kvm_x86_state *state);
548 void kvm_x86_state_cleanup(struct kvm_x86_state *state);
549 
550 const struct kvm_msr_list *kvm_get_msr_index_list(void);
551 const struct kvm_msr_list *kvm_get_feature_msr_index_list(void);
552 bool kvm_msr_is_in_save_restore_list(uint32_t msr_index);
553 uint64_t kvm_get_feature_msr(uint64_t msr_index);
554 
vcpu_msrs_get(struct kvm_vcpu * vcpu,struct kvm_msrs * msrs)555 static inline void vcpu_msrs_get(struct kvm_vcpu *vcpu,
556 				 struct kvm_msrs *msrs)
557 {
558 	int r = __vcpu_ioctl(vcpu, KVM_GET_MSRS, msrs);
559 
560 	TEST_ASSERT(r == msrs->nmsrs,
561 		    "KVM_GET_MSRS failed, r: %i (failed on MSR %x)",
562 		    r, r < 0 || r >= msrs->nmsrs ? -1 : msrs->entries[r].index);
563 }
vcpu_msrs_set(struct kvm_vcpu * vcpu,struct kvm_msrs * msrs)564 static inline void vcpu_msrs_set(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs)
565 {
566 	int r = __vcpu_ioctl(vcpu, KVM_SET_MSRS, msrs);
567 
568 	TEST_ASSERT(r == msrs->nmsrs,
569 		    "KVM_GET_MSRS failed, r: %i (failed on MSR %x)",
570 		    r, r < 0 || r >= msrs->nmsrs ? -1 : msrs->entries[r].index);
571 }
vcpu_debugregs_get(struct kvm_vcpu * vcpu,struct kvm_debugregs * debugregs)572 static inline void vcpu_debugregs_get(struct kvm_vcpu *vcpu,
573 				      struct kvm_debugregs *debugregs)
574 {
575 	vcpu_ioctl(vcpu, KVM_GET_DEBUGREGS, debugregs);
576 }
vcpu_debugregs_set(struct kvm_vcpu * vcpu,struct kvm_debugregs * debugregs)577 static inline void vcpu_debugregs_set(struct kvm_vcpu *vcpu,
578 				      struct kvm_debugregs *debugregs)
579 {
580 	vcpu_ioctl(vcpu, KVM_SET_DEBUGREGS, debugregs);
581 }
vcpu_xsave_get(struct kvm_vcpu * vcpu,struct kvm_xsave * xsave)582 static inline void vcpu_xsave_get(struct kvm_vcpu *vcpu,
583 				  struct kvm_xsave *xsave)
584 {
585 	vcpu_ioctl(vcpu, KVM_GET_XSAVE, xsave);
586 }
vcpu_xsave2_get(struct kvm_vcpu * vcpu,struct kvm_xsave * xsave)587 static inline void vcpu_xsave2_get(struct kvm_vcpu *vcpu,
588 				   struct kvm_xsave *xsave)
589 {
590 	vcpu_ioctl(vcpu, KVM_GET_XSAVE2, xsave);
591 }
vcpu_xsave_set(struct kvm_vcpu * vcpu,struct kvm_xsave * xsave)592 static inline void vcpu_xsave_set(struct kvm_vcpu *vcpu,
593 				  struct kvm_xsave *xsave)
594 {
595 	vcpu_ioctl(vcpu, KVM_SET_XSAVE, xsave);
596 }
vcpu_xcrs_get(struct kvm_vcpu * vcpu,struct kvm_xcrs * xcrs)597 static inline void vcpu_xcrs_get(struct kvm_vcpu *vcpu,
598 				 struct kvm_xcrs *xcrs)
599 {
600 	vcpu_ioctl(vcpu, KVM_GET_XCRS, xcrs);
601 }
vcpu_xcrs_set(struct kvm_vcpu * vcpu,struct kvm_xcrs * xcrs)602 static inline void vcpu_xcrs_set(struct kvm_vcpu *vcpu, struct kvm_xcrs *xcrs)
603 {
604 	vcpu_ioctl(vcpu, KVM_SET_XCRS, xcrs);
605 }
606 
607 const struct kvm_cpuid2 *kvm_get_supported_cpuid(void);
608 const struct kvm_cpuid2 *kvm_get_supported_hv_cpuid(void);
609 const struct kvm_cpuid2 *vcpu_get_supported_hv_cpuid(struct kvm_vcpu *vcpu);
610 
611 bool kvm_cpuid_has(const struct kvm_cpuid2 *cpuid,
612 		   struct kvm_x86_cpu_feature feature);
613 
kvm_cpu_has(struct kvm_x86_cpu_feature feature)614 static inline bool kvm_cpu_has(struct kvm_x86_cpu_feature feature)
615 {
616 	return kvm_cpuid_has(kvm_get_supported_cpuid(), feature);
617 }
618 
kvm_cpuid2_size(int nr_entries)619 static inline size_t kvm_cpuid2_size(int nr_entries)
620 {
621 	return sizeof(struct kvm_cpuid2) +
622 	       sizeof(struct kvm_cpuid_entry2) * nr_entries;
623 }
624 
625 /*
626  * Allocate a "struct kvm_cpuid2* instance, with the 0-length arrary of
627  * entries sized to hold @nr_entries.  The caller is responsible for freeing
628  * the struct.
629  */
allocate_kvm_cpuid2(int nr_entries)630 static inline struct kvm_cpuid2 *allocate_kvm_cpuid2(int nr_entries)
631 {
632 	struct kvm_cpuid2 *cpuid;
633 
634 	cpuid = malloc(kvm_cpuid2_size(nr_entries));
635 	TEST_ASSERT(cpuid, "-ENOMEM when allocating kvm_cpuid2");
636 
637 	cpuid->nent = nr_entries;
638 
639 	return cpuid;
640 }
641 
642 const struct kvm_cpuid_entry2 *get_cpuid_entry(const struct kvm_cpuid2 *cpuid,
643 					       uint32_t function, uint32_t index);
644 void vcpu_init_cpuid(struct kvm_vcpu *vcpu, const struct kvm_cpuid2 *cpuid);
645 void vcpu_set_hv_cpuid(struct kvm_vcpu *vcpu);
646 
__vcpu_get_cpuid_entry(struct kvm_vcpu * vcpu,uint32_t function,uint32_t index)647 static inline struct kvm_cpuid_entry2 *__vcpu_get_cpuid_entry(struct kvm_vcpu *vcpu,
648 							      uint32_t function,
649 							      uint32_t index)
650 {
651 	return (struct kvm_cpuid_entry2 *)get_cpuid_entry(vcpu->cpuid,
652 							  function, index);
653 }
654 
vcpu_get_cpuid_entry(struct kvm_vcpu * vcpu,uint32_t function)655 static inline struct kvm_cpuid_entry2 *vcpu_get_cpuid_entry(struct kvm_vcpu *vcpu,
656 							    uint32_t function)
657 {
658 	return __vcpu_get_cpuid_entry(vcpu, function, 0);
659 }
660 
__vcpu_set_cpuid(struct kvm_vcpu * vcpu)661 static inline int __vcpu_set_cpuid(struct kvm_vcpu *vcpu)
662 {
663 	int r;
664 
665 	TEST_ASSERT(vcpu->cpuid, "Must do vcpu_init_cpuid() first");
666 	r = __vcpu_ioctl(vcpu, KVM_SET_CPUID2, vcpu->cpuid);
667 	if (r)
668 		return r;
669 
670 	/* On success, refresh the cache to pick up adjustments made by KVM. */
671 	vcpu_ioctl(vcpu, KVM_GET_CPUID2, vcpu->cpuid);
672 	return 0;
673 }
674 
vcpu_set_cpuid(struct kvm_vcpu * vcpu)675 static inline void vcpu_set_cpuid(struct kvm_vcpu *vcpu)
676 {
677 	TEST_ASSERT(vcpu->cpuid, "Must do vcpu_init_cpuid() first");
678 	vcpu_ioctl(vcpu, KVM_SET_CPUID2, vcpu->cpuid);
679 
680 	/* Refresh the cache to pick up adjustments made by KVM. */
681 	vcpu_ioctl(vcpu, KVM_GET_CPUID2, vcpu->cpuid);
682 }
683 
684 void vcpu_set_cpuid_maxphyaddr(struct kvm_vcpu *vcpu, uint8_t maxphyaddr);
685 
686 void vcpu_clear_cpuid_entry(struct kvm_vcpu *vcpu, uint32_t function);
687 void vcpu_set_or_clear_cpuid_feature(struct kvm_vcpu *vcpu,
688 				     struct kvm_x86_cpu_feature feature,
689 				     bool set);
690 
vcpu_set_cpuid_feature(struct kvm_vcpu * vcpu,struct kvm_x86_cpu_feature feature)691 static inline void vcpu_set_cpuid_feature(struct kvm_vcpu *vcpu,
692 					  struct kvm_x86_cpu_feature feature)
693 {
694 	vcpu_set_or_clear_cpuid_feature(vcpu, feature, true);
695 
696 }
697 
vcpu_clear_cpuid_feature(struct kvm_vcpu * vcpu,struct kvm_x86_cpu_feature feature)698 static inline void vcpu_clear_cpuid_feature(struct kvm_vcpu *vcpu,
699 					    struct kvm_x86_cpu_feature feature)
700 {
701 	vcpu_set_or_clear_cpuid_feature(vcpu, feature, false);
702 }
703 
__kvm_get_supported_cpuid_entry(uint32_t function,uint32_t index)704 static inline const struct kvm_cpuid_entry2 *__kvm_get_supported_cpuid_entry(uint32_t function,
705 									     uint32_t index)
706 {
707 	return get_cpuid_entry(kvm_get_supported_cpuid(), function, index);
708 }
709 
kvm_get_supported_cpuid_entry(uint32_t function)710 static inline const struct kvm_cpuid_entry2 *kvm_get_supported_cpuid_entry(uint32_t function)
711 {
712 	return __kvm_get_supported_cpuid_entry(function, 0);
713 }
714 
715 uint64_t vcpu_get_msr(struct kvm_vcpu *vcpu, uint64_t msr_index);
716 int _vcpu_set_msr(struct kvm_vcpu *vcpu, uint64_t msr_index, uint64_t msr_value);
717 
vcpu_set_msr(struct kvm_vcpu * vcpu,uint64_t msr_index,uint64_t msr_value)718 static inline void vcpu_set_msr(struct kvm_vcpu *vcpu, uint64_t msr_index,
719 				uint64_t msr_value)
720 {
721 	int r = _vcpu_set_msr(vcpu, msr_index, msr_value);
722 
723 	TEST_ASSERT(r == 1, KVM_IOCTL_ERROR(KVM_SET_MSRS, r));
724 }
725 
kvm_get_cpuid_max_basic(void)726 static inline uint32_t kvm_get_cpuid_max_basic(void)
727 {
728 	return kvm_get_supported_cpuid_entry(0)->eax;
729 }
730 
kvm_get_cpuid_max_extended(void)731 static inline uint32_t kvm_get_cpuid_max_extended(void)
732 {
733 	return kvm_get_supported_cpuid_entry(0x80000000)->eax;
734 }
735 
736 void kvm_get_cpu_address_width(unsigned int *pa_bits, unsigned int *va_bits);
737 bool vm_is_unrestricted_guest(struct kvm_vm *vm);
738 
739 struct ex_regs {
740 	uint64_t rax, rcx, rdx, rbx;
741 	uint64_t rbp, rsi, rdi;
742 	uint64_t r8, r9, r10, r11;
743 	uint64_t r12, r13, r14, r15;
744 	uint64_t vector;
745 	uint64_t error_code;
746 	uint64_t rip;
747 	uint64_t cs;
748 	uint64_t rflags;
749 };
750 
751 struct idt_entry {
752 	uint16_t offset0;
753 	uint16_t selector;
754 	uint16_t ist : 3;
755 	uint16_t : 5;
756 	uint16_t type : 4;
757 	uint16_t : 1;
758 	uint16_t dpl : 2;
759 	uint16_t p : 1;
760 	uint16_t offset1;
761 	uint32_t offset2; uint32_t reserved;
762 };
763 
764 void vm_init_descriptor_tables(struct kvm_vm *vm);
765 void vcpu_init_descriptor_tables(struct kvm_vcpu *vcpu);
766 void vm_install_exception_handler(struct kvm_vm *vm, int vector,
767 			void (*handler)(struct ex_regs *));
768 
769 /* If a toddler were to say "abracadabra". */
770 #define KVM_EXCEPTION_MAGIC 0xabacadabaULL
771 
772 /*
773  * KVM selftest exception fixup uses registers to coordinate with the exception
774  * handler, versus the kernel's in-memory tables and KVM-Unit-Tests's in-memory
775  * per-CPU data.  Using only registers avoids having to map memory into the
776  * guest, doesn't require a valid, stable GS.base, and reduces the risk of
777  * for recursive faults when accessing memory in the handler.  The downside to
778  * using registers is that it restricts what registers can be used by the actual
779  * instruction.  But, selftests are 64-bit only, making register* pressure a
780  * minor concern.  Use r9-r11 as they are volatile, i.e. don't need* to be saved
781  * by the callee, and except for r11 are not implicit parameters to any
782  * instructions.  Ideally, fixup would use r8-r10 and thus avoid implicit
783  * parameters entirely, but Hyper-V's hypercall ABI uses r8 and testing Hyper-V
784  * is higher priority than testing non-faulting SYSCALL/SYSRET.
785  *
786  * Note, the fixup handler deliberately does not handle #DE, i.e. the vector
787  * is guaranteed to be non-zero on fault.
788  *
789  * REGISTER INPUTS:
790  * r9  = MAGIC
791  * r10 = RIP
792  * r11 = new RIP on fault
793  *
794  * REGISTER OUTPUTS:
795  * r9  = exception vector (non-zero)
796  */
797 #define KVM_ASM_SAFE(insn)					\
798 	"mov $" __stringify(KVM_EXCEPTION_MAGIC) ", %%r9\n\t"	\
799 	"lea 1f(%%rip), %%r10\n\t"				\
800 	"lea 2f(%%rip), %%r11\n\t"				\
801 	"1: " insn "\n\t"					\
802 	"movb $0, %[vector]\n\t"				\
803 	"jmp 3f\n\t"						\
804 	"2:\n\t"						\
805 	"mov  %%r9b, %[vector]\n\t"				\
806 	"3:\n\t"
807 
808 #define KVM_ASM_SAFE_OUTPUTS(v)	[vector] "=qm"(v)
809 #define KVM_ASM_SAFE_CLOBBERS	"r9", "r10", "r11"
810 
811 #define kvm_asm_safe(insn, inputs...)			\
812 ({							\
813 	uint8_t vector;					\
814 							\
815 	asm volatile(KVM_ASM_SAFE(insn)			\
816 		     : KVM_ASM_SAFE_OUTPUTS(vector)	\
817 		     : inputs				\
818 		     : KVM_ASM_SAFE_CLOBBERS);		\
819 	vector;						\
820 })
821 
rdmsr_safe(uint32_t msr,uint64_t * val)822 static inline uint8_t rdmsr_safe(uint32_t msr, uint64_t *val)
823 {
824 	uint8_t vector;
825 	uint32_t a, d;
826 
827 	asm volatile(KVM_ASM_SAFE("rdmsr")
828 		     : "=a"(a), "=d"(d), KVM_ASM_SAFE_OUTPUTS(vector)
829 		     : "c"(msr)
830 		     : KVM_ASM_SAFE_CLOBBERS);
831 
832 	*val = (uint64_t)a | ((uint64_t)d << 32);
833 	return vector;
834 }
835 
wrmsr_safe(uint32_t msr,uint64_t val)836 static inline uint8_t wrmsr_safe(uint32_t msr, uint64_t val)
837 {
838 	return kvm_asm_safe("wrmsr", "a"(val & -1u), "d"(val >> 32), "c"(msr));
839 }
840 
841 bool kvm_is_tdp_enabled(void);
842 
843 uint64_t vm_get_page_table_entry(struct kvm_vm *vm, struct kvm_vcpu *vcpu,
844 				 uint64_t vaddr);
845 void vm_set_page_table_entry(struct kvm_vm *vm, struct kvm_vcpu *vcpu,
846 			     uint64_t vaddr, uint64_t pte);
847 
848 uint64_t kvm_hypercall(uint64_t nr, uint64_t a0, uint64_t a1, uint64_t a2,
849 		       uint64_t a3);
850 
851 void __vm_xsave_require_permission(int bit, const char *name);
852 
853 #define vm_xsave_require_permission(perm)	\
854 	__vm_xsave_require_permission(perm, #perm)
855 
856 enum pg_level {
857 	PG_LEVEL_NONE,
858 	PG_LEVEL_4K,
859 	PG_LEVEL_2M,
860 	PG_LEVEL_1G,
861 	PG_LEVEL_512G,
862 	PG_LEVEL_NUM
863 };
864 
865 #define PG_LEVEL_SHIFT(_level) ((_level - 1) * 9 + 12)
866 #define PG_LEVEL_SIZE(_level) (1ull << PG_LEVEL_SHIFT(_level))
867 
868 #define PG_SIZE_4K PG_LEVEL_SIZE(PG_LEVEL_4K)
869 #define PG_SIZE_2M PG_LEVEL_SIZE(PG_LEVEL_2M)
870 #define PG_SIZE_1G PG_LEVEL_SIZE(PG_LEVEL_1G)
871 
872 void __virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, int level);
873 void virt_map_level(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
874 		    uint64_t nr_bytes, int level);
875 
876 /*
877  * Basic CPU control in CR0
878  */
879 #define X86_CR0_PE          (1UL<<0) /* Protection Enable */
880 #define X86_CR0_MP          (1UL<<1) /* Monitor Coprocessor */
881 #define X86_CR0_EM          (1UL<<2) /* Emulation */
882 #define X86_CR0_TS          (1UL<<3) /* Task Switched */
883 #define X86_CR0_ET          (1UL<<4) /* Extension Type */
884 #define X86_CR0_NE          (1UL<<5) /* Numeric Error */
885 #define X86_CR0_WP          (1UL<<16) /* Write Protect */
886 #define X86_CR0_AM          (1UL<<18) /* Alignment Mask */
887 #define X86_CR0_NW          (1UL<<29) /* Not Write-through */
888 #define X86_CR0_CD          (1UL<<30) /* Cache Disable */
889 #define X86_CR0_PG          (1UL<<31) /* Paging */
890 
891 #define XSTATE_XTILE_CFG_BIT		17
892 #define XSTATE_XTILE_DATA_BIT		18
893 
894 #define XSTATE_XTILE_CFG_MASK		(1ULL << XSTATE_XTILE_CFG_BIT)
895 #define XSTATE_XTILE_DATA_MASK		(1ULL << XSTATE_XTILE_DATA_BIT)
896 #define XFEATURE_XTILE_MASK		(XSTATE_XTILE_CFG_MASK | \
897 					XSTATE_XTILE_DATA_MASK)
898 #endif /* SELFTEST_KVM_PROCESSOR_H */
899