1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * tools/testing/selftests/kvm/include/x86_64/processor.h
4 *
5 * Copyright (C) 2018, Google LLC.
6 */
7
8 #ifndef SELFTEST_KVM_PROCESSOR_H
9 #define SELFTEST_KVM_PROCESSOR_H
10
11 #include <assert.h>
12 #include <stdint.h>
13 #include <syscall.h>
14
15 #include <asm/msr-index.h>
16 #include <asm/prctl.h>
17
18 #include <linux/stringify.h>
19
20 #include "../kvm_util.h"
21
22 #define NMI_VECTOR 0x02
23
24 #define X86_EFLAGS_FIXED (1u << 1)
25
26 #define X86_CR4_VME (1ul << 0)
27 #define X86_CR4_PVI (1ul << 1)
28 #define X86_CR4_TSD (1ul << 2)
29 #define X86_CR4_DE (1ul << 3)
30 #define X86_CR4_PSE (1ul << 4)
31 #define X86_CR4_PAE (1ul << 5)
32 #define X86_CR4_MCE (1ul << 6)
33 #define X86_CR4_PGE (1ul << 7)
34 #define X86_CR4_PCE (1ul << 8)
35 #define X86_CR4_OSFXSR (1ul << 9)
36 #define X86_CR4_OSXMMEXCPT (1ul << 10)
37 #define X86_CR4_UMIP (1ul << 11)
38 #define X86_CR4_LA57 (1ul << 12)
39 #define X86_CR4_VMXE (1ul << 13)
40 #define X86_CR4_SMXE (1ul << 14)
41 #define X86_CR4_FSGSBASE (1ul << 16)
42 #define X86_CR4_PCIDE (1ul << 17)
43 #define X86_CR4_OSXSAVE (1ul << 18)
44 #define X86_CR4_SMEP (1ul << 20)
45 #define X86_CR4_SMAP (1ul << 21)
46 #define X86_CR4_PKE (1ul << 22)
47
48 /* Note, these are ordered alphabetically to match kvm_cpuid_entry2. Eww. */
49 enum cpuid_output_regs {
50 KVM_CPUID_EAX,
51 KVM_CPUID_EBX,
52 KVM_CPUID_ECX,
53 KVM_CPUID_EDX
54 };
55
56 /*
57 * Pack the information into a 64-bit value so that each X86_FEATURE_XXX can be
58 * passed by value with no overhead.
59 */
60 struct kvm_x86_cpu_feature {
61 u32 function;
62 u16 index;
63 u8 reg;
64 u8 bit;
65 };
66 #define KVM_X86_CPU_FEATURE(fn, idx, gpr, __bit) \
67 ({ \
68 struct kvm_x86_cpu_feature feature = { \
69 .function = fn, \
70 .index = idx, \
71 .reg = KVM_CPUID_##gpr, \
72 .bit = __bit, \
73 }; \
74 \
75 feature; \
76 })
77
78 /*
79 * Basic Leafs, a.k.a. Intel defined
80 */
81 #define X86_FEATURE_MWAIT KVM_X86_CPU_FEATURE(0x1, 0, ECX, 3)
82 #define X86_FEATURE_VMX KVM_X86_CPU_FEATURE(0x1, 0, ECX, 5)
83 #define X86_FEATURE_SMX KVM_X86_CPU_FEATURE(0x1, 0, ECX, 6)
84 #define X86_FEATURE_PDCM KVM_X86_CPU_FEATURE(0x1, 0, ECX, 15)
85 #define X86_FEATURE_PCID KVM_X86_CPU_FEATURE(0x1, 0, ECX, 17)
86 #define X86_FEATURE_X2APIC KVM_X86_CPU_FEATURE(0x1, 0, ECX, 21)
87 #define X86_FEATURE_MOVBE KVM_X86_CPU_FEATURE(0x1, 0, ECX, 22)
88 #define X86_FEATURE_TSC_DEADLINE_TIMER KVM_X86_CPU_FEATURE(0x1, 0, ECX, 24)
89 #define X86_FEATURE_XSAVE KVM_X86_CPU_FEATURE(0x1, 0, ECX, 26)
90 #define X86_FEATURE_OSXSAVE KVM_X86_CPU_FEATURE(0x1, 0, ECX, 27)
91 #define X86_FEATURE_RDRAND KVM_X86_CPU_FEATURE(0x1, 0, ECX, 30)
92 #define X86_FEATURE_MCE KVM_X86_CPU_FEATURE(0x1, 0, EDX, 7)
93 #define X86_FEATURE_APIC KVM_X86_CPU_FEATURE(0x1, 0, EDX, 9)
94 #define X86_FEATURE_CLFLUSH KVM_X86_CPU_FEATURE(0x1, 0, EDX, 19)
95 #define X86_FEATURE_XMM KVM_X86_CPU_FEATURE(0x1, 0, EDX, 25)
96 #define X86_FEATURE_XMM2 KVM_X86_CPU_FEATURE(0x1, 0, EDX, 26)
97 #define X86_FEATURE_FSGSBASE KVM_X86_CPU_FEATURE(0x7, 0, EBX, 0)
98 #define X86_FEATURE_TSC_ADJUST KVM_X86_CPU_FEATURE(0x7, 0, EBX, 1)
99 #define X86_FEATURE_HLE KVM_X86_CPU_FEATURE(0x7, 0, EBX, 4)
100 #define X86_FEATURE_SMEP KVM_X86_CPU_FEATURE(0x7, 0, EBX, 7)
101 #define X86_FEATURE_INVPCID KVM_X86_CPU_FEATURE(0x7, 0, EBX, 10)
102 #define X86_FEATURE_RTM KVM_X86_CPU_FEATURE(0x7, 0, EBX, 11)
103 #define X86_FEATURE_MPX KVM_X86_CPU_FEATURE(0x7, 0, EBX, 14)
104 #define X86_FEATURE_SMAP KVM_X86_CPU_FEATURE(0x7, 0, EBX, 20)
105 #define X86_FEATURE_PCOMMIT KVM_X86_CPU_FEATURE(0x7, 0, EBX, 22)
106 #define X86_FEATURE_CLFLUSHOPT KVM_X86_CPU_FEATURE(0x7, 0, EBX, 23)
107 #define X86_FEATURE_CLWB KVM_X86_CPU_FEATURE(0x7, 0, EBX, 24)
108 #define X86_FEATURE_UMIP KVM_X86_CPU_FEATURE(0x7, 0, ECX, 2)
109 #define X86_FEATURE_PKU KVM_X86_CPU_FEATURE(0x7, 0, ECX, 3)
110 #define X86_FEATURE_LA57 KVM_X86_CPU_FEATURE(0x7, 0, ECX, 16)
111 #define X86_FEATURE_RDPID KVM_X86_CPU_FEATURE(0x7, 0, ECX, 22)
112 #define X86_FEATURE_SHSTK KVM_X86_CPU_FEATURE(0x7, 0, ECX, 7)
113 #define X86_FEATURE_IBT KVM_X86_CPU_FEATURE(0x7, 0, EDX, 20)
114 #define X86_FEATURE_AMX_TILE KVM_X86_CPU_FEATURE(0x7, 0, EDX, 24)
115 #define X86_FEATURE_SPEC_CTRL KVM_X86_CPU_FEATURE(0x7, 0, EDX, 26)
116 #define X86_FEATURE_ARCH_CAPABILITIES KVM_X86_CPU_FEATURE(0x7, 0, EDX, 29)
117 #define X86_FEATURE_PKS KVM_X86_CPU_FEATURE(0x7, 0, ECX, 31)
118 #define X86_FEATURE_XTILECFG KVM_X86_CPU_FEATURE(0xD, 0, EAX, 17)
119 #define X86_FEATURE_XTILEDATA KVM_X86_CPU_FEATURE(0xD, 0, EAX, 18)
120 #define X86_FEATURE_XSAVES KVM_X86_CPU_FEATURE(0xD, 1, EAX, 3)
121 #define X86_FEATURE_XFD KVM_X86_CPU_FEATURE(0xD, 1, EAX, 4)
122
123 /*
124 * Extended Leafs, a.k.a. AMD defined
125 */
126 #define X86_FEATURE_SVM KVM_X86_CPU_FEATURE(0x80000001, 0, ECX, 2)
127 #define X86_FEATURE_NX KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 20)
128 #define X86_FEATURE_GBPAGES KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 26)
129 #define X86_FEATURE_RDTSCP KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 27)
130 #define X86_FEATURE_LM KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 29)
131 #define X86_FEATURE_RDPRU KVM_X86_CPU_FEATURE(0x80000008, 0, EBX, 4)
132 #define X86_FEATURE_AMD_IBPB KVM_X86_CPU_FEATURE(0x80000008, 0, EBX, 12)
133 #define X86_FEATURE_NPT KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 0)
134 #define X86_FEATURE_LBRV KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 1)
135 #define X86_FEATURE_NRIPS KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 3)
136 #define X86_FEATURE_TSCRATEMSR KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 4)
137 #define X86_FEATURE_PAUSEFILTER KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 10)
138 #define X86_FEATURE_PFTHRESHOLD KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 12)
139 #define X86_FEATURE_VGIF KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 16)
140 #define X86_FEATURE_SEV KVM_X86_CPU_FEATURE(0x8000001F, 0, EAX, 1)
141 #define X86_FEATURE_SEV_ES KVM_X86_CPU_FEATURE(0x8000001F, 0, EAX, 3)
142
143 /*
144 * KVM defined paravirt features.
145 */
146 #define X86_FEATURE_KVM_CLOCKSOURCE KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 0)
147 #define X86_FEATURE_KVM_NOP_IO_DELAY KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 1)
148 #define X86_FEATURE_KVM_MMU_OP KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 2)
149 #define X86_FEATURE_KVM_CLOCKSOURCE2 KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 3)
150 #define X86_FEATURE_KVM_ASYNC_PF KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 4)
151 #define X86_FEATURE_KVM_STEAL_TIME KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 5)
152 #define X86_FEATURE_KVM_PV_EOI KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 6)
153 #define X86_FEATURE_KVM_PV_UNHALT KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 7)
154 /* Bit 8 apparently isn't used?!?! */
155 #define X86_FEATURE_KVM_PV_TLB_FLUSH KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 9)
156 #define X86_FEATURE_KVM_ASYNC_PF_VMEXIT KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 10)
157 #define X86_FEATURE_KVM_PV_SEND_IPI KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 11)
158 #define X86_FEATURE_KVM_POLL_CONTROL KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 12)
159 #define X86_FEATURE_KVM_PV_SCHED_YIELD KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 13)
160 #define X86_FEATURE_KVM_ASYNC_PF_INT KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 14)
161 #define X86_FEATURE_KVM_MSI_EXT_DEST_ID KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 15)
162 #define X86_FEATURE_KVM_HC_MAP_GPA_RANGE KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 16)
163 #define X86_FEATURE_KVM_MIGRATION_CONTROL KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 17)
164
165 /* Page table bitfield declarations */
166 #define PTE_PRESENT_MASK BIT_ULL(0)
167 #define PTE_WRITABLE_MASK BIT_ULL(1)
168 #define PTE_USER_MASK BIT_ULL(2)
169 #define PTE_ACCESSED_MASK BIT_ULL(5)
170 #define PTE_DIRTY_MASK BIT_ULL(6)
171 #define PTE_LARGE_MASK BIT_ULL(7)
172 #define PTE_GLOBAL_MASK BIT_ULL(8)
173 #define PTE_NX_MASK BIT_ULL(63)
174
175 #define PAGE_SHIFT 12
176 #define PAGE_SIZE (1ULL << PAGE_SHIFT)
177 #define PAGE_MASK (~(PAGE_SIZE-1))
178
179 #define PHYSICAL_PAGE_MASK GENMASK_ULL(51, 12)
180 #define PTE_GET_PFN(pte) (((pte) & PHYSICAL_PAGE_MASK) >> PAGE_SHIFT)
181
182 /* General Registers in 64-Bit Mode */
183 struct gpr64_regs {
184 u64 rax;
185 u64 rcx;
186 u64 rdx;
187 u64 rbx;
188 u64 rsp;
189 u64 rbp;
190 u64 rsi;
191 u64 rdi;
192 u64 r8;
193 u64 r9;
194 u64 r10;
195 u64 r11;
196 u64 r12;
197 u64 r13;
198 u64 r14;
199 u64 r15;
200 };
201
202 struct desc64 {
203 uint16_t limit0;
204 uint16_t base0;
205 unsigned base1:8, type:4, s:1, dpl:2, p:1;
206 unsigned limit1:4, avl:1, l:1, db:1, g:1, base2:8;
207 uint32_t base3;
208 uint32_t zero1;
209 } __attribute__((packed));
210
211 struct desc_ptr {
212 uint16_t size;
213 uint64_t address;
214 } __attribute__((packed));
215
216 struct kvm_x86_state {
217 struct kvm_xsave *xsave;
218 struct kvm_vcpu_events events;
219 struct kvm_mp_state mp_state;
220 struct kvm_regs regs;
221 struct kvm_xcrs xcrs;
222 struct kvm_sregs sregs;
223 struct kvm_debugregs debugregs;
224 union {
225 struct kvm_nested_state nested;
226 char nested_[16384];
227 };
228 struct kvm_msrs msrs;
229 };
230
get_desc64_base(const struct desc64 * desc)231 static inline uint64_t get_desc64_base(const struct desc64 *desc)
232 {
233 return ((uint64_t)desc->base3 << 32) |
234 (desc->base0 | ((desc->base1) << 16) | ((desc->base2) << 24));
235 }
236
rdtsc(void)237 static inline uint64_t rdtsc(void)
238 {
239 uint32_t eax, edx;
240 uint64_t tsc_val;
241 /*
242 * The lfence is to wait (on Intel CPUs) until all previous
243 * instructions have been executed. If software requires RDTSC to be
244 * executed prior to execution of any subsequent instruction, it can
245 * execute LFENCE immediately after RDTSC
246 */
247 __asm__ __volatile__("lfence; rdtsc; lfence" : "=a"(eax), "=d"(edx));
248 tsc_val = ((uint64_t)edx) << 32 | eax;
249 return tsc_val;
250 }
251
rdtscp(uint32_t * aux)252 static inline uint64_t rdtscp(uint32_t *aux)
253 {
254 uint32_t eax, edx;
255
256 __asm__ __volatile__("rdtscp" : "=a"(eax), "=d"(edx), "=c"(*aux));
257 return ((uint64_t)edx) << 32 | eax;
258 }
259
rdmsr(uint32_t msr)260 static inline uint64_t rdmsr(uint32_t msr)
261 {
262 uint32_t a, d;
263
264 __asm__ __volatile__("rdmsr" : "=a"(a), "=d"(d) : "c"(msr) : "memory");
265
266 return a | ((uint64_t) d << 32);
267 }
268
wrmsr(uint32_t msr,uint64_t value)269 static inline void wrmsr(uint32_t msr, uint64_t value)
270 {
271 uint32_t a = value;
272 uint32_t d = value >> 32;
273
274 __asm__ __volatile__("wrmsr" :: "a"(a), "d"(d), "c"(msr) : "memory");
275 }
276
277
inw(uint16_t port)278 static inline uint16_t inw(uint16_t port)
279 {
280 uint16_t tmp;
281
282 __asm__ __volatile__("in %%dx, %%ax"
283 : /* output */ "=a" (tmp)
284 : /* input */ "d" (port));
285
286 return tmp;
287 }
288
get_es(void)289 static inline uint16_t get_es(void)
290 {
291 uint16_t es;
292
293 __asm__ __volatile__("mov %%es, %[es]"
294 : /* output */ [es]"=rm"(es));
295 return es;
296 }
297
get_cs(void)298 static inline uint16_t get_cs(void)
299 {
300 uint16_t cs;
301
302 __asm__ __volatile__("mov %%cs, %[cs]"
303 : /* output */ [cs]"=rm"(cs));
304 return cs;
305 }
306
get_ss(void)307 static inline uint16_t get_ss(void)
308 {
309 uint16_t ss;
310
311 __asm__ __volatile__("mov %%ss, %[ss]"
312 : /* output */ [ss]"=rm"(ss));
313 return ss;
314 }
315
get_ds(void)316 static inline uint16_t get_ds(void)
317 {
318 uint16_t ds;
319
320 __asm__ __volatile__("mov %%ds, %[ds]"
321 : /* output */ [ds]"=rm"(ds));
322 return ds;
323 }
324
get_fs(void)325 static inline uint16_t get_fs(void)
326 {
327 uint16_t fs;
328
329 __asm__ __volatile__("mov %%fs, %[fs]"
330 : /* output */ [fs]"=rm"(fs));
331 return fs;
332 }
333
get_gs(void)334 static inline uint16_t get_gs(void)
335 {
336 uint16_t gs;
337
338 __asm__ __volatile__("mov %%gs, %[gs]"
339 : /* output */ [gs]"=rm"(gs));
340 return gs;
341 }
342
get_tr(void)343 static inline uint16_t get_tr(void)
344 {
345 uint16_t tr;
346
347 __asm__ __volatile__("str %[tr]"
348 : /* output */ [tr]"=rm"(tr));
349 return tr;
350 }
351
get_cr0(void)352 static inline uint64_t get_cr0(void)
353 {
354 uint64_t cr0;
355
356 __asm__ __volatile__("mov %%cr0, %[cr0]"
357 : /* output */ [cr0]"=r"(cr0));
358 return cr0;
359 }
360
get_cr3(void)361 static inline uint64_t get_cr3(void)
362 {
363 uint64_t cr3;
364
365 __asm__ __volatile__("mov %%cr3, %[cr3]"
366 : /* output */ [cr3]"=r"(cr3));
367 return cr3;
368 }
369
get_cr4(void)370 static inline uint64_t get_cr4(void)
371 {
372 uint64_t cr4;
373
374 __asm__ __volatile__("mov %%cr4, %[cr4]"
375 : /* output */ [cr4]"=r"(cr4));
376 return cr4;
377 }
378
set_cr4(uint64_t val)379 static inline void set_cr4(uint64_t val)
380 {
381 __asm__ __volatile__("mov %0, %%cr4" : : "r" (val) : "memory");
382 }
383
get_gdt(void)384 static inline struct desc_ptr get_gdt(void)
385 {
386 struct desc_ptr gdt;
387 __asm__ __volatile__("sgdt %[gdt]"
388 : /* output */ [gdt]"=m"(gdt));
389 return gdt;
390 }
391
get_idt(void)392 static inline struct desc_ptr get_idt(void)
393 {
394 struct desc_ptr idt;
395 __asm__ __volatile__("sidt %[idt]"
396 : /* output */ [idt]"=m"(idt));
397 return idt;
398 }
399
outl(uint16_t port,uint32_t value)400 static inline void outl(uint16_t port, uint32_t value)
401 {
402 __asm__ __volatile__("outl %%eax, %%dx" : : "d"(port), "a"(value));
403 }
404
__cpuid(uint32_t function,uint32_t index,uint32_t * eax,uint32_t * ebx,uint32_t * ecx,uint32_t * edx)405 static inline void __cpuid(uint32_t function, uint32_t index,
406 uint32_t *eax, uint32_t *ebx,
407 uint32_t *ecx, uint32_t *edx)
408 {
409 *eax = function;
410 *ecx = index;
411
412 asm volatile("cpuid"
413 : "=a" (*eax),
414 "=b" (*ebx),
415 "=c" (*ecx),
416 "=d" (*edx)
417 : "0" (*eax), "2" (*ecx)
418 : "memory");
419 }
420
cpuid(uint32_t function,uint32_t * eax,uint32_t * ebx,uint32_t * ecx,uint32_t * edx)421 static inline void cpuid(uint32_t function,
422 uint32_t *eax, uint32_t *ebx,
423 uint32_t *ecx, uint32_t *edx)
424 {
425 return __cpuid(function, 0, eax, ebx, ecx, edx);
426 }
427
this_cpu_has(struct kvm_x86_cpu_feature feature)428 static inline bool this_cpu_has(struct kvm_x86_cpu_feature feature)
429 {
430 uint32_t gprs[4];
431
432 __cpuid(feature.function, feature.index,
433 &gprs[KVM_CPUID_EAX], &gprs[KVM_CPUID_EBX],
434 &gprs[KVM_CPUID_ECX], &gprs[KVM_CPUID_EDX]);
435
436 return gprs[feature.reg] & BIT(feature.bit);
437 }
438
439 #define SET_XMM(__var, __xmm) \
440 asm volatile("movq %0, %%"#__xmm : : "r"(__var) : #__xmm)
441
set_xmm(int n,unsigned long val)442 static inline void set_xmm(int n, unsigned long val)
443 {
444 switch (n) {
445 case 0:
446 SET_XMM(val, xmm0);
447 break;
448 case 1:
449 SET_XMM(val, xmm1);
450 break;
451 case 2:
452 SET_XMM(val, xmm2);
453 break;
454 case 3:
455 SET_XMM(val, xmm3);
456 break;
457 case 4:
458 SET_XMM(val, xmm4);
459 break;
460 case 5:
461 SET_XMM(val, xmm5);
462 break;
463 case 6:
464 SET_XMM(val, xmm6);
465 break;
466 case 7:
467 SET_XMM(val, xmm7);
468 break;
469 }
470 }
471
472 #define GET_XMM(__xmm) \
473 ({ \
474 unsigned long __val; \
475 asm volatile("movq %%"#__xmm", %0" : "=r"(__val)); \
476 __val; \
477 })
478
get_xmm(int n)479 static inline unsigned long get_xmm(int n)
480 {
481 assert(n >= 0 && n <= 7);
482
483 switch (n) {
484 case 0:
485 return GET_XMM(xmm0);
486 case 1:
487 return GET_XMM(xmm1);
488 case 2:
489 return GET_XMM(xmm2);
490 case 3:
491 return GET_XMM(xmm3);
492 case 4:
493 return GET_XMM(xmm4);
494 case 5:
495 return GET_XMM(xmm5);
496 case 6:
497 return GET_XMM(xmm6);
498 case 7:
499 return GET_XMM(xmm7);
500 }
501
502 /* never reached */
503 return 0;
504 }
505
cpu_relax(void)506 static inline void cpu_relax(void)
507 {
508 asm volatile("rep; nop" ::: "memory");
509 }
510
511 #define vmmcall() \
512 __asm__ __volatile__( \
513 "vmmcall\n" \
514 )
515
516 #define ud2() \
517 __asm__ __volatile__( \
518 "ud2\n" \
519 )
520
521 #define hlt() \
522 __asm__ __volatile__( \
523 "hlt\n" \
524 )
525
526 bool is_intel_cpu(void);
527 bool is_amd_cpu(void);
528
x86_family(unsigned int eax)529 static inline unsigned int x86_family(unsigned int eax)
530 {
531 unsigned int x86;
532
533 x86 = (eax >> 8) & 0xf;
534
535 if (x86 == 0xf)
536 x86 += (eax >> 20) & 0xff;
537
538 return x86;
539 }
540
x86_model(unsigned int eax)541 static inline unsigned int x86_model(unsigned int eax)
542 {
543 return ((eax >> 12) & 0xf0) | ((eax >> 4) & 0x0f);
544 }
545
546 struct kvm_x86_state *vcpu_save_state(struct kvm_vcpu *vcpu);
547 void vcpu_load_state(struct kvm_vcpu *vcpu, struct kvm_x86_state *state);
548 void kvm_x86_state_cleanup(struct kvm_x86_state *state);
549
550 const struct kvm_msr_list *kvm_get_msr_index_list(void);
551 const struct kvm_msr_list *kvm_get_feature_msr_index_list(void);
552 bool kvm_msr_is_in_save_restore_list(uint32_t msr_index);
553 uint64_t kvm_get_feature_msr(uint64_t msr_index);
554
vcpu_msrs_get(struct kvm_vcpu * vcpu,struct kvm_msrs * msrs)555 static inline void vcpu_msrs_get(struct kvm_vcpu *vcpu,
556 struct kvm_msrs *msrs)
557 {
558 int r = __vcpu_ioctl(vcpu, KVM_GET_MSRS, msrs);
559
560 TEST_ASSERT(r == msrs->nmsrs,
561 "KVM_GET_MSRS failed, r: %i (failed on MSR %x)",
562 r, r < 0 || r >= msrs->nmsrs ? -1 : msrs->entries[r].index);
563 }
vcpu_msrs_set(struct kvm_vcpu * vcpu,struct kvm_msrs * msrs)564 static inline void vcpu_msrs_set(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs)
565 {
566 int r = __vcpu_ioctl(vcpu, KVM_SET_MSRS, msrs);
567
568 TEST_ASSERT(r == msrs->nmsrs,
569 "KVM_GET_MSRS failed, r: %i (failed on MSR %x)",
570 r, r < 0 || r >= msrs->nmsrs ? -1 : msrs->entries[r].index);
571 }
vcpu_debugregs_get(struct kvm_vcpu * vcpu,struct kvm_debugregs * debugregs)572 static inline void vcpu_debugregs_get(struct kvm_vcpu *vcpu,
573 struct kvm_debugregs *debugregs)
574 {
575 vcpu_ioctl(vcpu, KVM_GET_DEBUGREGS, debugregs);
576 }
vcpu_debugregs_set(struct kvm_vcpu * vcpu,struct kvm_debugregs * debugregs)577 static inline void vcpu_debugregs_set(struct kvm_vcpu *vcpu,
578 struct kvm_debugregs *debugregs)
579 {
580 vcpu_ioctl(vcpu, KVM_SET_DEBUGREGS, debugregs);
581 }
vcpu_xsave_get(struct kvm_vcpu * vcpu,struct kvm_xsave * xsave)582 static inline void vcpu_xsave_get(struct kvm_vcpu *vcpu,
583 struct kvm_xsave *xsave)
584 {
585 vcpu_ioctl(vcpu, KVM_GET_XSAVE, xsave);
586 }
vcpu_xsave2_get(struct kvm_vcpu * vcpu,struct kvm_xsave * xsave)587 static inline void vcpu_xsave2_get(struct kvm_vcpu *vcpu,
588 struct kvm_xsave *xsave)
589 {
590 vcpu_ioctl(vcpu, KVM_GET_XSAVE2, xsave);
591 }
vcpu_xsave_set(struct kvm_vcpu * vcpu,struct kvm_xsave * xsave)592 static inline void vcpu_xsave_set(struct kvm_vcpu *vcpu,
593 struct kvm_xsave *xsave)
594 {
595 vcpu_ioctl(vcpu, KVM_SET_XSAVE, xsave);
596 }
vcpu_xcrs_get(struct kvm_vcpu * vcpu,struct kvm_xcrs * xcrs)597 static inline void vcpu_xcrs_get(struct kvm_vcpu *vcpu,
598 struct kvm_xcrs *xcrs)
599 {
600 vcpu_ioctl(vcpu, KVM_GET_XCRS, xcrs);
601 }
vcpu_xcrs_set(struct kvm_vcpu * vcpu,struct kvm_xcrs * xcrs)602 static inline void vcpu_xcrs_set(struct kvm_vcpu *vcpu, struct kvm_xcrs *xcrs)
603 {
604 vcpu_ioctl(vcpu, KVM_SET_XCRS, xcrs);
605 }
606
607 const struct kvm_cpuid2 *kvm_get_supported_cpuid(void);
608 const struct kvm_cpuid2 *kvm_get_supported_hv_cpuid(void);
609 const struct kvm_cpuid2 *vcpu_get_supported_hv_cpuid(struct kvm_vcpu *vcpu);
610
611 bool kvm_cpuid_has(const struct kvm_cpuid2 *cpuid,
612 struct kvm_x86_cpu_feature feature);
613
kvm_cpu_has(struct kvm_x86_cpu_feature feature)614 static inline bool kvm_cpu_has(struct kvm_x86_cpu_feature feature)
615 {
616 return kvm_cpuid_has(kvm_get_supported_cpuid(), feature);
617 }
618
kvm_cpuid2_size(int nr_entries)619 static inline size_t kvm_cpuid2_size(int nr_entries)
620 {
621 return sizeof(struct kvm_cpuid2) +
622 sizeof(struct kvm_cpuid_entry2) * nr_entries;
623 }
624
625 /*
626 * Allocate a "struct kvm_cpuid2* instance, with the 0-length arrary of
627 * entries sized to hold @nr_entries. The caller is responsible for freeing
628 * the struct.
629 */
allocate_kvm_cpuid2(int nr_entries)630 static inline struct kvm_cpuid2 *allocate_kvm_cpuid2(int nr_entries)
631 {
632 struct kvm_cpuid2 *cpuid;
633
634 cpuid = malloc(kvm_cpuid2_size(nr_entries));
635 TEST_ASSERT(cpuid, "-ENOMEM when allocating kvm_cpuid2");
636
637 cpuid->nent = nr_entries;
638
639 return cpuid;
640 }
641
642 const struct kvm_cpuid_entry2 *get_cpuid_entry(const struct kvm_cpuid2 *cpuid,
643 uint32_t function, uint32_t index);
644 void vcpu_init_cpuid(struct kvm_vcpu *vcpu, const struct kvm_cpuid2 *cpuid);
645 void vcpu_set_hv_cpuid(struct kvm_vcpu *vcpu);
646
__vcpu_get_cpuid_entry(struct kvm_vcpu * vcpu,uint32_t function,uint32_t index)647 static inline struct kvm_cpuid_entry2 *__vcpu_get_cpuid_entry(struct kvm_vcpu *vcpu,
648 uint32_t function,
649 uint32_t index)
650 {
651 return (struct kvm_cpuid_entry2 *)get_cpuid_entry(vcpu->cpuid,
652 function, index);
653 }
654
vcpu_get_cpuid_entry(struct kvm_vcpu * vcpu,uint32_t function)655 static inline struct kvm_cpuid_entry2 *vcpu_get_cpuid_entry(struct kvm_vcpu *vcpu,
656 uint32_t function)
657 {
658 return __vcpu_get_cpuid_entry(vcpu, function, 0);
659 }
660
__vcpu_set_cpuid(struct kvm_vcpu * vcpu)661 static inline int __vcpu_set_cpuid(struct kvm_vcpu *vcpu)
662 {
663 int r;
664
665 TEST_ASSERT(vcpu->cpuid, "Must do vcpu_init_cpuid() first");
666 r = __vcpu_ioctl(vcpu, KVM_SET_CPUID2, vcpu->cpuid);
667 if (r)
668 return r;
669
670 /* On success, refresh the cache to pick up adjustments made by KVM. */
671 vcpu_ioctl(vcpu, KVM_GET_CPUID2, vcpu->cpuid);
672 return 0;
673 }
674
vcpu_set_cpuid(struct kvm_vcpu * vcpu)675 static inline void vcpu_set_cpuid(struct kvm_vcpu *vcpu)
676 {
677 TEST_ASSERT(vcpu->cpuid, "Must do vcpu_init_cpuid() first");
678 vcpu_ioctl(vcpu, KVM_SET_CPUID2, vcpu->cpuid);
679
680 /* Refresh the cache to pick up adjustments made by KVM. */
681 vcpu_ioctl(vcpu, KVM_GET_CPUID2, vcpu->cpuid);
682 }
683
684 void vcpu_set_cpuid_maxphyaddr(struct kvm_vcpu *vcpu, uint8_t maxphyaddr);
685
686 void vcpu_clear_cpuid_entry(struct kvm_vcpu *vcpu, uint32_t function);
687 void vcpu_set_or_clear_cpuid_feature(struct kvm_vcpu *vcpu,
688 struct kvm_x86_cpu_feature feature,
689 bool set);
690
vcpu_set_cpuid_feature(struct kvm_vcpu * vcpu,struct kvm_x86_cpu_feature feature)691 static inline void vcpu_set_cpuid_feature(struct kvm_vcpu *vcpu,
692 struct kvm_x86_cpu_feature feature)
693 {
694 vcpu_set_or_clear_cpuid_feature(vcpu, feature, true);
695
696 }
697
vcpu_clear_cpuid_feature(struct kvm_vcpu * vcpu,struct kvm_x86_cpu_feature feature)698 static inline void vcpu_clear_cpuid_feature(struct kvm_vcpu *vcpu,
699 struct kvm_x86_cpu_feature feature)
700 {
701 vcpu_set_or_clear_cpuid_feature(vcpu, feature, false);
702 }
703
__kvm_get_supported_cpuid_entry(uint32_t function,uint32_t index)704 static inline const struct kvm_cpuid_entry2 *__kvm_get_supported_cpuid_entry(uint32_t function,
705 uint32_t index)
706 {
707 return get_cpuid_entry(kvm_get_supported_cpuid(), function, index);
708 }
709
kvm_get_supported_cpuid_entry(uint32_t function)710 static inline const struct kvm_cpuid_entry2 *kvm_get_supported_cpuid_entry(uint32_t function)
711 {
712 return __kvm_get_supported_cpuid_entry(function, 0);
713 }
714
715 uint64_t vcpu_get_msr(struct kvm_vcpu *vcpu, uint64_t msr_index);
716 int _vcpu_set_msr(struct kvm_vcpu *vcpu, uint64_t msr_index, uint64_t msr_value);
717
vcpu_set_msr(struct kvm_vcpu * vcpu,uint64_t msr_index,uint64_t msr_value)718 static inline void vcpu_set_msr(struct kvm_vcpu *vcpu, uint64_t msr_index,
719 uint64_t msr_value)
720 {
721 int r = _vcpu_set_msr(vcpu, msr_index, msr_value);
722
723 TEST_ASSERT(r == 1, KVM_IOCTL_ERROR(KVM_SET_MSRS, r));
724 }
725
kvm_get_cpuid_max_basic(void)726 static inline uint32_t kvm_get_cpuid_max_basic(void)
727 {
728 return kvm_get_supported_cpuid_entry(0)->eax;
729 }
730
kvm_get_cpuid_max_extended(void)731 static inline uint32_t kvm_get_cpuid_max_extended(void)
732 {
733 return kvm_get_supported_cpuid_entry(0x80000000)->eax;
734 }
735
736 void kvm_get_cpu_address_width(unsigned int *pa_bits, unsigned int *va_bits);
737 bool vm_is_unrestricted_guest(struct kvm_vm *vm);
738
739 struct ex_regs {
740 uint64_t rax, rcx, rdx, rbx;
741 uint64_t rbp, rsi, rdi;
742 uint64_t r8, r9, r10, r11;
743 uint64_t r12, r13, r14, r15;
744 uint64_t vector;
745 uint64_t error_code;
746 uint64_t rip;
747 uint64_t cs;
748 uint64_t rflags;
749 };
750
751 struct idt_entry {
752 uint16_t offset0;
753 uint16_t selector;
754 uint16_t ist : 3;
755 uint16_t : 5;
756 uint16_t type : 4;
757 uint16_t : 1;
758 uint16_t dpl : 2;
759 uint16_t p : 1;
760 uint16_t offset1;
761 uint32_t offset2; uint32_t reserved;
762 };
763
764 void vm_init_descriptor_tables(struct kvm_vm *vm);
765 void vcpu_init_descriptor_tables(struct kvm_vcpu *vcpu);
766 void vm_install_exception_handler(struct kvm_vm *vm, int vector,
767 void (*handler)(struct ex_regs *));
768
769 /* If a toddler were to say "abracadabra". */
770 #define KVM_EXCEPTION_MAGIC 0xabacadabaULL
771
772 /*
773 * KVM selftest exception fixup uses registers to coordinate with the exception
774 * handler, versus the kernel's in-memory tables and KVM-Unit-Tests's in-memory
775 * per-CPU data. Using only registers avoids having to map memory into the
776 * guest, doesn't require a valid, stable GS.base, and reduces the risk of
777 * for recursive faults when accessing memory in the handler. The downside to
778 * using registers is that it restricts what registers can be used by the actual
779 * instruction. But, selftests are 64-bit only, making register* pressure a
780 * minor concern. Use r9-r11 as they are volatile, i.e. don't need* to be saved
781 * by the callee, and except for r11 are not implicit parameters to any
782 * instructions. Ideally, fixup would use r8-r10 and thus avoid implicit
783 * parameters entirely, but Hyper-V's hypercall ABI uses r8 and testing Hyper-V
784 * is higher priority than testing non-faulting SYSCALL/SYSRET.
785 *
786 * Note, the fixup handler deliberately does not handle #DE, i.e. the vector
787 * is guaranteed to be non-zero on fault.
788 *
789 * REGISTER INPUTS:
790 * r9 = MAGIC
791 * r10 = RIP
792 * r11 = new RIP on fault
793 *
794 * REGISTER OUTPUTS:
795 * r9 = exception vector (non-zero)
796 */
797 #define KVM_ASM_SAFE(insn) \
798 "mov $" __stringify(KVM_EXCEPTION_MAGIC) ", %%r9\n\t" \
799 "lea 1f(%%rip), %%r10\n\t" \
800 "lea 2f(%%rip), %%r11\n\t" \
801 "1: " insn "\n\t" \
802 "movb $0, %[vector]\n\t" \
803 "jmp 3f\n\t" \
804 "2:\n\t" \
805 "mov %%r9b, %[vector]\n\t" \
806 "3:\n\t"
807
808 #define KVM_ASM_SAFE_OUTPUTS(v) [vector] "=qm"(v)
809 #define KVM_ASM_SAFE_CLOBBERS "r9", "r10", "r11"
810
811 #define kvm_asm_safe(insn, inputs...) \
812 ({ \
813 uint8_t vector; \
814 \
815 asm volatile(KVM_ASM_SAFE(insn) \
816 : KVM_ASM_SAFE_OUTPUTS(vector) \
817 : inputs \
818 : KVM_ASM_SAFE_CLOBBERS); \
819 vector; \
820 })
821
rdmsr_safe(uint32_t msr,uint64_t * val)822 static inline uint8_t rdmsr_safe(uint32_t msr, uint64_t *val)
823 {
824 uint8_t vector;
825 uint32_t a, d;
826
827 asm volatile(KVM_ASM_SAFE("rdmsr")
828 : "=a"(a), "=d"(d), KVM_ASM_SAFE_OUTPUTS(vector)
829 : "c"(msr)
830 : KVM_ASM_SAFE_CLOBBERS);
831
832 *val = (uint64_t)a | ((uint64_t)d << 32);
833 return vector;
834 }
835
wrmsr_safe(uint32_t msr,uint64_t val)836 static inline uint8_t wrmsr_safe(uint32_t msr, uint64_t val)
837 {
838 return kvm_asm_safe("wrmsr", "a"(val & -1u), "d"(val >> 32), "c"(msr));
839 }
840
841 bool kvm_is_tdp_enabled(void);
842
843 uint64_t vm_get_page_table_entry(struct kvm_vm *vm, struct kvm_vcpu *vcpu,
844 uint64_t vaddr);
845 void vm_set_page_table_entry(struct kvm_vm *vm, struct kvm_vcpu *vcpu,
846 uint64_t vaddr, uint64_t pte);
847
848 uint64_t kvm_hypercall(uint64_t nr, uint64_t a0, uint64_t a1, uint64_t a2,
849 uint64_t a3);
850
851 void __vm_xsave_require_permission(int bit, const char *name);
852
853 #define vm_xsave_require_permission(perm) \
854 __vm_xsave_require_permission(perm, #perm)
855
856 enum pg_level {
857 PG_LEVEL_NONE,
858 PG_LEVEL_4K,
859 PG_LEVEL_2M,
860 PG_LEVEL_1G,
861 PG_LEVEL_512G,
862 PG_LEVEL_NUM
863 };
864
865 #define PG_LEVEL_SHIFT(_level) ((_level - 1) * 9 + 12)
866 #define PG_LEVEL_SIZE(_level) (1ull << PG_LEVEL_SHIFT(_level))
867
868 #define PG_SIZE_4K PG_LEVEL_SIZE(PG_LEVEL_4K)
869 #define PG_SIZE_2M PG_LEVEL_SIZE(PG_LEVEL_2M)
870 #define PG_SIZE_1G PG_LEVEL_SIZE(PG_LEVEL_1G)
871
872 void __virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, int level);
873 void virt_map_level(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
874 uint64_t nr_bytes, int level);
875
876 /*
877 * Basic CPU control in CR0
878 */
879 #define X86_CR0_PE (1UL<<0) /* Protection Enable */
880 #define X86_CR0_MP (1UL<<1) /* Monitor Coprocessor */
881 #define X86_CR0_EM (1UL<<2) /* Emulation */
882 #define X86_CR0_TS (1UL<<3) /* Task Switched */
883 #define X86_CR0_ET (1UL<<4) /* Extension Type */
884 #define X86_CR0_NE (1UL<<5) /* Numeric Error */
885 #define X86_CR0_WP (1UL<<16) /* Write Protect */
886 #define X86_CR0_AM (1UL<<18) /* Alignment Mask */
887 #define X86_CR0_NW (1UL<<29) /* Not Write-through */
888 #define X86_CR0_CD (1UL<<30) /* Cache Disable */
889 #define X86_CR0_PG (1UL<<31) /* Paging */
890
891 #define XSTATE_XTILE_CFG_BIT 17
892 #define XSTATE_XTILE_DATA_BIT 18
893
894 #define XSTATE_XTILE_CFG_MASK (1ULL << XSTATE_XTILE_CFG_BIT)
895 #define XSTATE_XTILE_DATA_MASK (1ULL << XSTATE_XTILE_DATA_BIT)
896 #define XFEATURE_XTILE_MASK (XSTATE_XTILE_CFG_MASK | \
897 XSTATE_XTILE_DATA_MASK)
898 #endif /* SELFTEST_KVM_PROCESSOR_H */
899