1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Kernel-based Virtual Machine driver for Linux
4 *
5 * AMD SVM support
6 *
7 * Copyright (C) 2006 Qumranet, Inc.
8 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
9 *
10 * Authors:
11 * Yaniv Kamay <yaniv@qumranet.com>
12 * Avi Kivity <avi@qumranet.com>
13 */
14
15 #define pr_fmt(fmt) "SVM: " fmt
16
17 #include <linux/kvm_types.h>
18 #include <linux/hashtable.h>
19 #include <linux/amd-iommu.h>
20 #include <linux/kvm_host.h>
21
22 #include <asm/irq_remapping.h>
23
24 #include "trace.h"
25 #include "lapic.h"
26 #include "x86.h"
27 #include "irq.h"
28 #include "svm.h"
29
30 /* AVIC GATAG is encoded using VM and VCPU IDs */
31 #define AVIC_VCPU_ID_BITS 8
32 #define AVIC_VCPU_ID_MASK ((1 << AVIC_VCPU_ID_BITS) - 1)
33
34 #define AVIC_VM_ID_BITS 24
35 #define AVIC_VM_ID_NR (1 << AVIC_VM_ID_BITS)
36 #define AVIC_VM_ID_MASK ((1 << AVIC_VM_ID_BITS) - 1)
37
38 #define AVIC_GATAG(x, y) (((x & AVIC_VM_ID_MASK) << AVIC_VCPU_ID_BITS) | \
39 (y & AVIC_VCPU_ID_MASK))
40 #define AVIC_GATAG_TO_VMID(x) ((x >> AVIC_VCPU_ID_BITS) & AVIC_VM_ID_MASK)
41 #define AVIC_GATAG_TO_VCPUID(x) (x & AVIC_VCPU_ID_MASK)
42
43 /* Note:
44 * This hash table is used to map VM_ID to a struct kvm_svm,
45 * when handling AMD IOMMU GALOG notification to schedule in
46 * a particular vCPU.
47 */
48 #define SVM_VM_DATA_HASH_BITS 8
49 static DEFINE_HASHTABLE(svm_vm_data_hash, SVM_VM_DATA_HASH_BITS);
50 static u32 next_vm_id = 0;
51 static bool next_vm_id_wrapped = 0;
52 static DEFINE_SPINLOCK(svm_vm_data_hash_lock);
53
54 /*
55 * This is a wrapper of struct amd_iommu_ir_data.
56 */
57 struct amd_svm_iommu_ir {
58 struct list_head node; /* Used by SVM for per-vcpu ir_list */
59 void *data; /* Storing pointer to struct amd_ir_data */
60 };
61
62
63 /* Note:
64 * This function is called from IOMMU driver to notify
65 * SVM to schedule in a particular vCPU of a particular VM.
66 */
avic_ga_log_notifier(u32 ga_tag)67 int avic_ga_log_notifier(u32 ga_tag)
68 {
69 unsigned long flags;
70 struct kvm_svm *kvm_svm;
71 struct kvm_vcpu *vcpu = NULL;
72 u32 vm_id = AVIC_GATAG_TO_VMID(ga_tag);
73 u32 vcpu_id = AVIC_GATAG_TO_VCPUID(ga_tag);
74
75 pr_debug("SVM: %s: vm_id=%#x, vcpu_id=%#x\n", __func__, vm_id, vcpu_id);
76 trace_kvm_avic_ga_log(vm_id, vcpu_id);
77
78 spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
79 hash_for_each_possible(svm_vm_data_hash, kvm_svm, hnode, vm_id) {
80 if (kvm_svm->avic_vm_id != vm_id)
81 continue;
82 vcpu = kvm_get_vcpu_by_id(&kvm_svm->kvm, vcpu_id);
83 break;
84 }
85 spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
86
87 /* Note:
88 * At this point, the IOMMU should have already set the pending
89 * bit in the vAPIC backing page. So, we just need to schedule
90 * in the vcpu.
91 */
92 if (vcpu)
93 kvm_vcpu_wake_up(vcpu);
94
95 return 0;
96 }
97
avic_vm_destroy(struct kvm * kvm)98 void avic_vm_destroy(struct kvm *kvm)
99 {
100 unsigned long flags;
101 struct kvm_svm *kvm_svm = to_kvm_svm(kvm);
102
103 if (!enable_apicv)
104 return;
105
106 if (kvm_svm->avic_logical_id_table_page)
107 __free_page(kvm_svm->avic_logical_id_table_page);
108 if (kvm_svm->avic_physical_id_table_page)
109 __free_page(kvm_svm->avic_physical_id_table_page);
110
111 spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
112 hash_del(&kvm_svm->hnode);
113 spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
114 }
115
avic_vm_init(struct kvm * kvm)116 int avic_vm_init(struct kvm *kvm)
117 {
118 unsigned long flags;
119 int err = -ENOMEM;
120 struct kvm_svm *kvm_svm = to_kvm_svm(kvm);
121 struct kvm_svm *k2;
122 struct page *p_page;
123 struct page *l_page;
124 u32 vm_id;
125
126 if (!enable_apicv)
127 return 0;
128
129 /* Allocating physical APIC ID table (4KB) */
130 p_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
131 if (!p_page)
132 goto free_avic;
133
134 kvm_svm->avic_physical_id_table_page = p_page;
135
136 /* Allocating logical APIC ID table (4KB) */
137 l_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
138 if (!l_page)
139 goto free_avic;
140
141 kvm_svm->avic_logical_id_table_page = l_page;
142
143 spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
144 again:
145 vm_id = next_vm_id = (next_vm_id + 1) & AVIC_VM_ID_MASK;
146 if (vm_id == 0) { /* id is 1-based, zero is not okay */
147 next_vm_id_wrapped = 1;
148 goto again;
149 }
150 /* Is it still in use? Only possible if wrapped at least once */
151 if (next_vm_id_wrapped) {
152 hash_for_each_possible(svm_vm_data_hash, k2, hnode, vm_id) {
153 if (k2->avic_vm_id == vm_id)
154 goto again;
155 }
156 }
157 kvm_svm->avic_vm_id = vm_id;
158 hash_add(svm_vm_data_hash, &kvm_svm->hnode, kvm_svm->avic_vm_id);
159 spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
160
161 return 0;
162
163 free_avic:
164 avic_vm_destroy(kvm);
165 return err;
166 }
167
avic_init_vmcb(struct vcpu_svm * svm,struct vmcb * vmcb)168 void avic_init_vmcb(struct vcpu_svm *svm, struct vmcb *vmcb)
169 {
170 struct kvm_svm *kvm_svm = to_kvm_svm(svm->vcpu.kvm);
171 phys_addr_t bpa = __sme_set(page_to_phys(svm->avic_backing_page));
172 phys_addr_t lpa = __sme_set(page_to_phys(kvm_svm->avic_logical_id_table_page));
173 phys_addr_t ppa = __sme_set(page_to_phys(kvm_svm->avic_physical_id_table_page));
174
175 vmcb->control.avic_backing_page = bpa & AVIC_HPA_MASK;
176 vmcb->control.avic_logical_id = lpa & AVIC_HPA_MASK;
177 vmcb->control.avic_physical_id = ppa & AVIC_HPA_MASK;
178 vmcb->control.avic_physical_id |= AVIC_MAX_PHYSICAL_ID_COUNT;
179 vmcb->control.avic_vapic_bar = APIC_DEFAULT_PHYS_BASE & VMCB_AVIC_APIC_BAR_MASK;
180
181 if (kvm_apicv_activated(svm->vcpu.kvm))
182 vmcb->control.int_ctl |= AVIC_ENABLE_MASK;
183 else
184 vmcb->control.int_ctl &= ~AVIC_ENABLE_MASK;
185 }
186
avic_get_physical_id_entry(struct kvm_vcpu * vcpu,unsigned int index)187 static u64 *avic_get_physical_id_entry(struct kvm_vcpu *vcpu,
188 unsigned int index)
189 {
190 u64 *avic_physical_id_table;
191 struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm);
192
193 if (index >= AVIC_MAX_PHYSICAL_ID_COUNT)
194 return NULL;
195
196 avic_physical_id_table = page_address(kvm_svm->avic_physical_id_table_page);
197
198 return &avic_physical_id_table[index];
199 }
200
201 /*
202 * Note:
203 * AVIC hardware walks the nested page table to check permissions,
204 * but does not use the SPA address specified in the leaf page
205 * table entry since it uses address in the AVIC_BACKING_PAGE pointer
206 * field of the VMCB. Therefore, we set up the
207 * APIC_ACCESS_PAGE_PRIVATE_MEMSLOT (4KB) here.
208 */
avic_alloc_access_page(struct kvm * kvm)209 static int avic_alloc_access_page(struct kvm *kvm)
210 {
211 void __user *ret;
212 int r = 0;
213
214 mutex_lock(&kvm->slots_lock);
215
216 if (kvm->arch.apic_access_memslot_enabled)
217 goto out;
218
219 ret = __x86_set_memory_region(kvm,
220 APIC_ACCESS_PAGE_PRIVATE_MEMSLOT,
221 APIC_DEFAULT_PHYS_BASE,
222 PAGE_SIZE);
223 if (IS_ERR(ret)) {
224 r = PTR_ERR(ret);
225 goto out;
226 }
227
228 kvm->arch.apic_access_memslot_enabled = true;
229 out:
230 mutex_unlock(&kvm->slots_lock);
231 return r;
232 }
233
avic_init_backing_page(struct kvm_vcpu * vcpu)234 static int avic_init_backing_page(struct kvm_vcpu *vcpu)
235 {
236 u64 *entry, new_entry;
237 int id = vcpu->vcpu_id;
238 struct vcpu_svm *svm = to_svm(vcpu);
239
240 if (id >= AVIC_MAX_PHYSICAL_ID_COUNT)
241 return -EINVAL;
242
243 if (!vcpu->arch.apic->regs)
244 return -EINVAL;
245
246 if (kvm_apicv_activated(vcpu->kvm)) {
247 int ret;
248
249 ret = avic_alloc_access_page(vcpu->kvm);
250 if (ret)
251 return ret;
252 }
253
254 svm->avic_backing_page = virt_to_page(vcpu->arch.apic->regs);
255
256 /* Setting AVIC backing page address in the phy APIC ID table */
257 entry = avic_get_physical_id_entry(vcpu, id);
258 if (!entry)
259 return -EINVAL;
260
261 new_entry = __sme_set((page_to_phys(svm->avic_backing_page) &
262 AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK) |
263 AVIC_PHYSICAL_ID_ENTRY_VALID_MASK);
264 WRITE_ONCE(*entry, new_entry);
265
266 svm->avic_physical_id_cache = entry;
267
268 return 0;
269 }
270
avic_ring_doorbell(struct kvm_vcpu * vcpu)271 void avic_ring_doorbell(struct kvm_vcpu *vcpu)
272 {
273 /*
274 * Note, the vCPU could get migrated to a different pCPU at any point,
275 * which could result in signalling the wrong/previous pCPU. But if
276 * that happens the vCPU is guaranteed to do a VMRUN (after being
277 * migrated) and thus will process pending interrupts, i.e. a doorbell
278 * is not needed (and the spurious one is harmless).
279 */
280 int cpu = READ_ONCE(vcpu->cpu);
281
282 if (cpu != get_cpu())
283 wrmsrl(MSR_AMD64_SVM_AVIC_DOORBELL, kvm_cpu_get_apicid(cpu));
284 put_cpu();
285 }
286
287 /*
288 * A fast-path version of avic_kick_target_vcpus(), which attempts to match
289 * destination APIC ID to vCPU without looping through all vCPUs.
290 */
avic_kick_target_vcpus_fast(struct kvm * kvm,struct kvm_lapic * source,u32 icrl,u32 icrh,u32 index)291 static int avic_kick_target_vcpus_fast(struct kvm *kvm, struct kvm_lapic *source,
292 u32 icrl, u32 icrh, u32 index)
293 {
294 u32 l1_physical_id, dest;
295 struct kvm_vcpu *target_vcpu;
296 int dest_mode = icrl & APIC_DEST_MASK;
297 int shorthand = icrl & APIC_SHORT_MASK;
298 struct kvm_svm *kvm_svm = to_kvm_svm(kvm);
299
300 if (shorthand != APIC_DEST_NOSHORT)
301 return -EINVAL;
302
303 if (apic_x2apic_mode(source))
304 dest = icrh;
305 else
306 dest = GET_APIC_DEST_FIELD(icrh);
307
308 if (dest_mode == APIC_DEST_PHYSICAL) {
309 /* broadcast destination, use slow path */
310 if (apic_x2apic_mode(source) && dest == X2APIC_BROADCAST)
311 return -EINVAL;
312 if (!apic_x2apic_mode(source) && dest == APIC_BROADCAST)
313 return -EINVAL;
314
315 l1_physical_id = dest;
316
317 if (WARN_ON_ONCE(l1_physical_id != index))
318 return -EINVAL;
319
320 } else {
321 u32 bitmap, cluster;
322 int logid_index;
323
324 if (apic_x2apic_mode(source)) {
325 /* 16 bit dest mask, 16 bit cluster id */
326 bitmap = dest & 0xFFFF0000;
327 cluster = (dest >> 16) << 4;
328 } else if (kvm_lapic_get_reg(source, APIC_DFR) == APIC_DFR_FLAT) {
329 /* 8 bit dest mask*/
330 bitmap = dest;
331 cluster = 0;
332 } else {
333 /* 4 bit desk mask, 4 bit cluster id */
334 bitmap = dest & 0xF;
335 cluster = (dest >> 4) << 2;
336 }
337
338 if (unlikely(!bitmap))
339 /* guest bug: nobody to send the logical interrupt to */
340 return 0;
341
342 if (!is_power_of_2(bitmap))
343 /* multiple logical destinations, use slow path */
344 return -EINVAL;
345
346 logid_index = cluster + __ffs(bitmap);
347
348 if (apic_x2apic_mode(source)) {
349 l1_physical_id = logid_index;
350 } else {
351 u32 *avic_logical_id_table =
352 page_address(kvm_svm->avic_logical_id_table_page);
353
354 u32 logid_entry = avic_logical_id_table[logid_index];
355
356 if (WARN_ON_ONCE(index != logid_index))
357 return -EINVAL;
358
359 /* guest bug: non existing/reserved logical destination */
360 if (unlikely(!(logid_entry & AVIC_LOGICAL_ID_ENTRY_VALID_MASK)))
361 return 0;
362
363 l1_physical_id = logid_entry &
364 AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK;
365 }
366 }
367
368 target_vcpu = kvm_get_vcpu_by_id(kvm, l1_physical_id);
369 if (unlikely(!target_vcpu))
370 /* guest bug: non existing vCPU is a target of this IPI*/
371 return 0;
372
373 target_vcpu->arch.apic->irr_pending = true;
374 svm_complete_interrupt_delivery(target_vcpu,
375 icrl & APIC_MODE_MASK,
376 icrl & APIC_INT_LEVELTRIG,
377 icrl & APIC_VECTOR_MASK);
378 return 0;
379 }
380
avic_kick_target_vcpus(struct kvm * kvm,struct kvm_lapic * source,u32 icrl,u32 icrh,u32 index)381 static void avic_kick_target_vcpus(struct kvm *kvm, struct kvm_lapic *source,
382 u32 icrl, u32 icrh, u32 index)
383 {
384 unsigned long i;
385 struct kvm_vcpu *vcpu;
386
387 if (!avic_kick_target_vcpus_fast(kvm, source, icrl, icrh, index))
388 return;
389
390 trace_kvm_avic_kick_vcpu_slowpath(icrh, icrl, index);
391
392 /*
393 * Wake any target vCPUs that are blocking, i.e. waiting for a wake
394 * event. There's no need to signal doorbells, as hardware has handled
395 * vCPUs that were in guest at the time of the IPI, and vCPUs that have
396 * since entered the guest will have processed pending IRQs at VMRUN.
397 */
398 kvm_for_each_vcpu(i, vcpu, kvm) {
399 if (kvm_apic_match_dest(vcpu, source, icrl & APIC_SHORT_MASK,
400 GET_APIC_DEST_FIELD(icrh),
401 icrl & APIC_DEST_MASK)) {
402 vcpu->arch.apic->irr_pending = true;
403 svm_complete_interrupt_delivery(vcpu,
404 icrl & APIC_MODE_MASK,
405 icrl & APIC_INT_LEVELTRIG,
406 icrl & APIC_VECTOR_MASK);
407 }
408 }
409 }
410
avic_incomplete_ipi_interception(struct kvm_vcpu * vcpu)411 int avic_incomplete_ipi_interception(struct kvm_vcpu *vcpu)
412 {
413 struct vcpu_svm *svm = to_svm(vcpu);
414 u32 icrh = svm->vmcb->control.exit_info_1 >> 32;
415 u32 icrl = svm->vmcb->control.exit_info_1;
416 u32 id = svm->vmcb->control.exit_info_2 >> 32;
417 u32 index = svm->vmcb->control.exit_info_2 & 0x1FF;
418 struct kvm_lapic *apic = vcpu->arch.apic;
419
420 trace_kvm_avic_incomplete_ipi(vcpu->vcpu_id, icrh, icrl, id, index);
421
422 switch (id) {
423 case AVIC_IPI_FAILURE_INVALID_INT_TYPE:
424 /*
425 * Emulate IPIs that are not handled by AVIC hardware, which
426 * only virtualizes Fixed, Edge-Triggered INTRs. The exit is
427 * a trap, e.g. ICR holds the correct value and RIP has been
428 * advanced, KVM is responsible only for emulating the IPI.
429 * Sadly, hardware may sometimes leave the BUSY flag set, in
430 * which case KVM needs to emulate the ICR write as well in
431 * order to clear the BUSY flag.
432 */
433 if (icrl & APIC_ICR_BUSY)
434 kvm_apic_write_nodecode(vcpu, APIC_ICR);
435 else
436 kvm_apic_send_ipi(apic, icrl, icrh);
437 break;
438 case AVIC_IPI_FAILURE_TARGET_NOT_RUNNING:
439 /*
440 * At this point, we expect that the AVIC HW has already
441 * set the appropriate IRR bits on the valid target
442 * vcpus. So, we just need to kick the appropriate vcpu.
443 */
444 avic_kick_target_vcpus(vcpu->kvm, apic, icrl, icrh, index);
445 break;
446 case AVIC_IPI_FAILURE_INVALID_TARGET:
447 break;
448 case AVIC_IPI_FAILURE_INVALID_BACKING_PAGE:
449 WARN_ONCE(1, "Invalid backing page\n");
450 break;
451 default:
452 pr_err("Unknown IPI interception\n");
453 }
454
455 return 1;
456 }
457
avic_vcpu_get_apicv_inhibit_reasons(struct kvm_vcpu * vcpu)458 unsigned long avic_vcpu_get_apicv_inhibit_reasons(struct kvm_vcpu *vcpu)
459 {
460 if (is_guest_mode(vcpu))
461 return APICV_INHIBIT_REASON_NESTED;
462 return 0;
463 }
464
avic_get_logical_id_entry(struct kvm_vcpu * vcpu,u32 ldr,bool flat)465 static u32 *avic_get_logical_id_entry(struct kvm_vcpu *vcpu, u32 ldr, bool flat)
466 {
467 struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm);
468 int index;
469 u32 *logical_apic_id_table;
470 int dlid = GET_APIC_LOGICAL_ID(ldr);
471
472 if (!dlid)
473 return NULL;
474
475 if (flat) { /* flat */
476 index = ffs(dlid) - 1;
477 if (index > 7)
478 return NULL;
479 } else { /* cluster */
480 int cluster = (dlid & 0xf0) >> 4;
481 int apic = ffs(dlid & 0x0f) - 1;
482
483 if ((apic < 0) || (apic > 7) ||
484 (cluster >= 0xf))
485 return NULL;
486 index = (cluster << 2) + apic;
487 }
488
489 logical_apic_id_table = (u32 *) page_address(kvm_svm->avic_logical_id_table_page);
490
491 return &logical_apic_id_table[index];
492 }
493
avic_ldr_write(struct kvm_vcpu * vcpu,u8 g_physical_id,u32 ldr)494 static int avic_ldr_write(struct kvm_vcpu *vcpu, u8 g_physical_id, u32 ldr)
495 {
496 bool flat;
497 u32 *entry, new_entry;
498
499 flat = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR) == APIC_DFR_FLAT;
500 entry = avic_get_logical_id_entry(vcpu, ldr, flat);
501 if (!entry)
502 return -EINVAL;
503
504 new_entry = READ_ONCE(*entry);
505 new_entry &= ~AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK;
506 new_entry |= (g_physical_id & AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK);
507 new_entry |= AVIC_LOGICAL_ID_ENTRY_VALID_MASK;
508 WRITE_ONCE(*entry, new_entry);
509
510 return 0;
511 }
512
avic_invalidate_logical_id_entry(struct kvm_vcpu * vcpu)513 static void avic_invalidate_logical_id_entry(struct kvm_vcpu *vcpu)
514 {
515 struct vcpu_svm *svm = to_svm(vcpu);
516 bool flat = svm->dfr_reg == APIC_DFR_FLAT;
517 u32 *entry = avic_get_logical_id_entry(vcpu, svm->ldr_reg, flat);
518
519 if (entry)
520 clear_bit(AVIC_LOGICAL_ID_ENTRY_VALID_BIT, (unsigned long *)entry);
521 }
522
avic_handle_ldr_update(struct kvm_vcpu * vcpu)523 static int avic_handle_ldr_update(struct kvm_vcpu *vcpu)
524 {
525 int ret = 0;
526 struct vcpu_svm *svm = to_svm(vcpu);
527 u32 ldr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LDR);
528 u32 id = kvm_xapic_id(vcpu->arch.apic);
529
530 if (ldr == svm->ldr_reg)
531 return 0;
532
533 avic_invalidate_logical_id_entry(vcpu);
534
535 if (ldr)
536 ret = avic_ldr_write(vcpu, id, ldr);
537
538 if (!ret)
539 svm->ldr_reg = ldr;
540
541 return ret;
542 }
543
avic_handle_dfr_update(struct kvm_vcpu * vcpu)544 static void avic_handle_dfr_update(struct kvm_vcpu *vcpu)
545 {
546 struct vcpu_svm *svm = to_svm(vcpu);
547 u32 dfr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR);
548
549 if (svm->dfr_reg == dfr)
550 return;
551
552 avic_invalidate_logical_id_entry(vcpu);
553 svm->dfr_reg = dfr;
554 }
555
avic_unaccel_trap_write(struct kvm_vcpu * vcpu)556 static int avic_unaccel_trap_write(struct kvm_vcpu *vcpu)
557 {
558 u32 offset = to_svm(vcpu)->vmcb->control.exit_info_1 &
559 AVIC_UNACCEL_ACCESS_OFFSET_MASK;
560
561 switch (offset) {
562 case APIC_LDR:
563 if (avic_handle_ldr_update(vcpu))
564 return 0;
565 break;
566 case APIC_DFR:
567 avic_handle_dfr_update(vcpu);
568 break;
569 default:
570 break;
571 }
572
573 kvm_apic_write_nodecode(vcpu, offset);
574 return 1;
575 }
576
is_avic_unaccelerated_access_trap(u32 offset)577 static bool is_avic_unaccelerated_access_trap(u32 offset)
578 {
579 bool ret = false;
580
581 switch (offset) {
582 case APIC_ID:
583 case APIC_EOI:
584 case APIC_RRR:
585 case APIC_LDR:
586 case APIC_DFR:
587 case APIC_SPIV:
588 case APIC_ESR:
589 case APIC_ICR:
590 case APIC_LVTT:
591 case APIC_LVTTHMR:
592 case APIC_LVTPC:
593 case APIC_LVT0:
594 case APIC_LVT1:
595 case APIC_LVTERR:
596 case APIC_TMICT:
597 case APIC_TDCR:
598 ret = true;
599 break;
600 default:
601 break;
602 }
603 return ret;
604 }
605
avic_unaccelerated_access_interception(struct kvm_vcpu * vcpu)606 int avic_unaccelerated_access_interception(struct kvm_vcpu *vcpu)
607 {
608 struct vcpu_svm *svm = to_svm(vcpu);
609 int ret = 0;
610 u32 offset = svm->vmcb->control.exit_info_1 &
611 AVIC_UNACCEL_ACCESS_OFFSET_MASK;
612 u32 vector = svm->vmcb->control.exit_info_2 &
613 AVIC_UNACCEL_ACCESS_VECTOR_MASK;
614 bool write = (svm->vmcb->control.exit_info_1 >> 32) &
615 AVIC_UNACCEL_ACCESS_WRITE_MASK;
616 bool trap = is_avic_unaccelerated_access_trap(offset);
617
618 trace_kvm_avic_unaccelerated_access(vcpu->vcpu_id, offset,
619 trap, write, vector);
620 if (trap) {
621 /* Handling Trap */
622 WARN_ONCE(!write, "svm: Handling trap read.\n");
623 ret = avic_unaccel_trap_write(vcpu);
624 } else {
625 /* Handling Fault */
626 ret = kvm_emulate_instruction(vcpu, 0);
627 }
628
629 return ret;
630 }
631
avic_init_vcpu(struct vcpu_svm * svm)632 int avic_init_vcpu(struct vcpu_svm *svm)
633 {
634 int ret;
635 struct kvm_vcpu *vcpu = &svm->vcpu;
636
637 if (!enable_apicv || !irqchip_in_kernel(vcpu->kvm))
638 return 0;
639
640 ret = avic_init_backing_page(vcpu);
641 if (ret)
642 return ret;
643
644 INIT_LIST_HEAD(&svm->ir_list);
645 spin_lock_init(&svm->ir_list_lock);
646 svm->dfr_reg = APIC_DFR_FLAT;
647
648 return ret;
649 }
650
avic_apicv_post_state_restore(struct kvm_vcpu * vcpu)651 void avic_apicv_post_state_restore(struct kvm_vcpu *vcpu)
652 {
653 avic_handle_dfr_update(vcpu);
654 avic_handle_ldr_update(vcpu);
655 }
656
avic_set_pi_irte_mode(struct kvm_vcpu * vcpu,bool activate)657 static int avic_set_pi_irte_mode(struct kvm_vcpu *vcpu, bool activate)
658 {
659 int ret = 0;
660 unsigned long flags;
661 struct amd_svm_iommu_ir *ir;
662 struct vcpu_svm *svm = to_svm(vcpu);
663
664 if (!kvm_arch_has_assigned_device(vcpu->kvm))
665 return 0;
666
667 /*
668 * Here, we go through the per-vcpu ir_list to update all existing
669 * interrupt remapping table entry targeting this vcpu.
670 */
671 spin_lock_irqsave(&svm->ir_list_lock, flags);
672
673 if (list_empty(&svm->ir_list))
674 goto out;
675
676 list_for_each_entry(ir, &svm->ir_list, node) {
677 if (activate)
678 ret = amd_iommu_activate_guest_mode(ir->data);
679 else
680 ret = amd_iommu_deactivate_guest_mode(ir->data);
681 if (ret)
682 break;
683 }
684 out:
685 spin_unlock_irqrestore(&svm->ir_list_lock, flags);
686 return ret;
687 }
688
svm_ir_list_del(struct vcpu_svm * svm,struct amd_iommu_pi_data * pi)689 static void svm_ir_list_del(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi)
690 {
691 unsigned long flags;
692 struct amd_svm_iommu_ir *cur;
693
694 spin_lock_irqsave(&svm->ir_list_lock, flags);
695 list_for_each_entry(cur, &svm->ir_list, node) {
696 if (cur->data != pi->ir_data)
697 continue;
698 list_del(&cur->node);
699 kfree(cur);
700 break;
701 }
702 spin_unlock_irqrestore(&svm->ir_list_lock, flags);
703 }
704
svm_ir_list_add(struct vcpu_svm * svm,struct amd_iommu_pi_data * pi)705 static int svm_ir_list_add(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi)
706 {
707 int ret = 0;
708 unsigned long flags;
709 struct amd_svm_iommu_ir *ir;
710
711 /**
712 * In some cases, the existing irte is updated and re-set,
713 * so we need to check here if it's already been * added
714 * to the ir_list.
715 */
716 if (pi->ir_data && (pi->prev_ga_tag != 0)) {
717 struct kvm *kvm = svm->vcpu.kvm;
718 u32 vcpu_id = AVIC_GATAG_TO_VCPUID(pi->prev_ga_tag);
719 struct kvm_vcpu *prev_vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);
720 struct vcpu_svm *prev_svm;
721
722 if (!prev_vcpu) {
723 ret = -EINVAL;
724 goto out;
725 }
726
727 prev_svm = to_svm(prev_vcpu);
728 svm_ir_list_del(prev_svm, pi);
729 }
730
731 /**
732 * Allocating new amd_iommu_pi_data, which will get
733 * add to the per-vcpu ir_list.
734 */
735 ir = kzalloc(sizeof(struct amd_svm_iommu_ir), GFP_KERNEL_ACCOUNT);
736 if (!ir) {
737 ret = -ENOMEM;
738 goto out;
739 }
740 ir->data = pi->ir_data;
741
742 spin_lock_irqsave(&svm->ir_list_lock, flags);
743 list_add(&ir->node, &svm->ir_list);
744 spin_unlock_irqrestore(&svm->ir_list_lock, flags);
745 out:
746 return ret;
747 }
748
749 /*
750 * Note:
751 * The HW cannot support posting multicast/broadcast
752 * interrupts to a vCPU. So, we still use legacy interrupt
753 * remapping for these kind of interrupts.
754 *
755 * For lowest-priority interrupts, we only support
756 * those with single CPU as the destination, e.g. user
757 * configures the interrupts via /proc/irq or uses
758 * irqbalance to make the interrupts single-CPU.
759 */
760 static int
get_pi_vcpu_info(struct kvm * kvm,struct kvm_kernel_irq_routing_entry * e,struct vcpu_data * vcpu_info,struct vcpu_svm ** svm)761 get_pi_vcpu_info(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e,
762 struct vcpu_data *vcpu_info, struct vcpu_svm **svm)
763 {
764 struct kvm_lapic_irq irq;
765 struct kvm_vcpu *vcpu = NULL;
766
767 kvm_set_msi_irq(kvm, e, &irq);
768
769 if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu) ||
770 !kvm_irq_is_postable(&irq)) {
771 pr_debug("SVM: %s: use legacy intr remap mode for irq %u\n",
772 __func__, irq.vector);
773 return -1;
774 }
775
776 pr_debug("SVM: %s: use GA mode for irq %u\n", __func__,
777 irq.vector);
778 *svm = to_svm(vcpu);
779 vcpu_info->pi_desc_addr = __sme_set(page_to_phys((*svm)->avic_backing_page));
780 vcpu_info->vector = irq.vector;
781
782 return 0;
783 }
784
785 /*
786 * avic_pi_update_irte - set IRTE for Posted-Interrupts
787 *
788 * @kvm: kvm
789 * @host_irq: host irq of the interrupt
790 * @guest_irq: gsi of the interrupt
791 * @set: set or unset PI
792 * returns 0 on success, < 0 on failure
793 */
avic_pi_update_irte(struct kvm * kvm,unsigned int host_irq,uint32_t guest_irq,bool set)794 int avic_pi_update_irte(struct kvm *kvm, unsigned int host_irq,
795 uint32_t guest_irq, bool set)
796 {
797 struct kvm_kernel_irq_routing_entry *e;
798 struct kvm_irq_routing_table *irq_rt;
799 int idx, ret = 0;
800
801 if (!kvm_arch_has_assigned_device(kvm) ||
802 !irq_remapping_cap(IRQ_POSTING_CAP))
803 return 0;
804
805 pr_debug("SVM: %s: host_irq=%#x, guest_irq=%#x, set=%#x\n",
806 __func__, host_irq, guest_irq, set);
807
808 idx = srcu_read_lock(&kvm->irq_srcu);
809 irq_rt = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu);
810
811 if (guest_irq >= irq_rt->nr_rt_entries ||
812 hlist_empty(&irq_rt->map[guest_irq])) {
813 pr_warn_once("no route for guest_irq %u/%u (broken user space?)\n",
814 guest_irq, irq_rt->nr_rt_entries);
815 goto out;
816 }
817
818 hlist_for_each_entry(e, &irq_rt->map[guest_irq], link) {
819 struct vcpu_data vcpu_info;
820 struct vcpu_svm *svm = NULL;
821
822 if (e->type != KVM_IRQ_ROUTING_MSI)
823 continue;
824
825 /**
826 * Here, we setup with legacy mode in the following cases:
827 * 1. When cannot target interrupt to a specific vcpu.
828 * 2. Unsetting posted interrupt.
829 * 3. APIC virtualization is disabled for the vcpu.
830 * 4. IRQ has incompatible delivery mode (SMI, INIT, etc)
831 */
832 if (!get_pi_vcpu_info(kvm, e, &vcpu_info, &svm) && set &&
833 kvm_vcpu_apicv_active(&svm->vcpu)) {
834 struct amd_iommu_pi_data pi;
835
836 /* Try to enable guest_mode in IRTE */
837 pi.base = __sme_set(page_to_phys(svm->avic_backing_page) &
838 AVIC_HPA_MASK);
839 pi.ga_tag = AVIC_GATAG(to_kvm_svm(kvm)->avic_vm_id,
840 svm->vcpu.vcpu_id);
841 pi.is_guest_mode = true;
842 pi.vcpu_data = &vcpu_info;
843 ret = irq_set_vcpu_affinity(host_irq, &pi);
844
845 /**
846 * Here, we successfully setting up vcpu affinity in
847 * IOMMU guest mode. Now, we need to store the posted
848 * interrupt information in a per-vcpu ir_list so that
849 * we can reference to them directly when we update vcpu
850 * scheduling information in IOMMU irte.
851 */
852 if (!ret && pi.is_guest_mode)
853 svm_ir_list_add(svm, &pi);
854 } else {
855 /* Use legacy mode in IRTE */
856 struct amd_iommu_pi_data pi;
857
858 /**
859 * Here, pi is used to:
860 * - Tell IOMMU to use legacy mode for this interrupt.
861 * - Retrieve ga_tag of prior interrupt remapping data.
862 */
863 pi.prev_ga_tag = 0;
864 pi.is_guest_mode = false;
865 ret = irq_set_vcpu_affinity(host_irq, &pi);
866
867 /**
868 * Check if the posted interrupt was previously
869 * setup with the guest_mode by checking if the ga_tag
870 * was cached. If so, we need to clean up the per-vcpu
871 * ir_list.
872 */
873 if (!ret && pi.prev_ga_tag) {
874 int id = AVIC_GATAG_TO_VCPUID(pi.prev_ga_tag);
875 struct kvm_vcpu *vcpu;
876
877 vcpu = kvm_get_vcpu_by_id(kvm, id);
878 if (vcpu)
879 svm_ir_list_del(to_svm(vcpu), &pi);
880 }
881 }
882
883 if (!ret && svm) {
884 trace_kvm_pi_irte_update(host_irq, svm->vcpu.vcpu_id,
885 e->gsi, vcpu_info.vector,
886 vcpu_info.pi_desc_addr, set);
887 }
888
889 if (ret < 0) {
890 pr_err("%s: failed to update PI IRTE\n", __func__);
891 goto out;
892 }
893 }
894
895 ret = 0;
896 out:
897 srcu_read_unlock(&kvm->irq_srcu, idx);
898 return ret;
899 }
900
avic_check_apicv_inhibit_reasons(enum kvm_apicv_inhibit reason)901 bool avic_check_apicv_inhibit_reasons(enum kvm_apicv_inhibit reason)
902 {
903 ulong supported = BIT(APICV_INHIBIT_REASON_DISABLE) |
904 BIT(APICV_INHIBIT_REASON_ABSENT) |
905 BIT(APICV_INHIBIT_REASON_HYPERV) |
906 BIT(APICV_INHIBIT_REASON_NESTED) |
907 BIT(APICV_INHIBIT_REASON_IRQWIN) |
908 BIT(APICV_INHIBIT_REASON_PIT_REINJ) |
909 BIT(APICV_INHIBIT_REASON_X2APIC) |
910 BIT(APICV_INHIBIT_REASON_BLOCKIRQ) |
911 BIT(APICV_INHIBIT_REASON_SEV) |
912 BIT(APICV_INHIBIT_REASON_APIC_ID_MODIFIED) |
913 BIT(APICV_INHIBIT_REASON_APIC_BASE_MODIFIED);
914
915 return supported & BIT(reason);
916 }
917
918
919 static inline int
avic_update_iommu_vcpu_affinity(struct kvm_vcpu * vcpu,int cpu,bool r)920 avic_update_iommu_vcpu_affinity(struct kvm_vcpu *vcpu, int cpu, bool r)
921 {
922 int ret = 0;
923 unsigned long flags;
924 struct amd_svm_iommu_ir *ir;
925 struct vcpu_svm *svm = to_svm(vcpu);
926
927 if (!kvm_arch_has_assigned_device(vcpu->kvm))
928 return 0;
929
930 /*
931 * Here, we go through the per-vcpu ir_list to update all existing
932 * interrupt remapping table entry targeting this vcpu.
933 */
934 spin_lock_irqsave(&svm->ir_list_lock, flags);
935
936 if (list_empty(&svm->ir_list))
937 goto out;
938
939 list_for_each_entry(ir, &svm->ir_list, node) {
940 ret = amd_iommu_update_ga(cpu, r, ir->data);
941 if (ret)
942 break;
943 }
944 out:
945 spin_unlock_irqrestore(&svm->ir_list_lock, flags);
946 return ret;
947 }
948
avic_vcpu_load(struct kvm_vcpu * vcpu,int cpu)949 void avic_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
950 {
951 u64 entry;
952 int h_physical_id = kvm_cpu_get_apicid(cpu);
953 struct vcpu_svm *svm = to_svm(vcpu);
954
955 lockdep_assert_preemption_disabled();
956
957 if (WARN_ON(h_physical_id & ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK))
958 return;
959
960 /*
961 * No need to update anything if the vCPU is blocking, i.e. if the vCPU
962 * is being scheduled in after being preempted. The CPU entries in the
963 * Physical APIC table and IRTE are consumed iff IsRun{ning} is '1'.
964 * If the vCPU was migrated, its new CPU value will be stuffed when the
965 * vCPU unblocks.
966 */
967 if (kvm_vcpu_is_blocking(vcpu))
968 return;
969
970 entry = READ_ONCE(*(svm->avic_physical_id_cache));
971 WARN_ON(entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK);
972
973 entry &= ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK;
974 entry |= (h_physical_id & AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK);
975 entry |= AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;
976
977 WRITE_ONCE(*(svm->avic_physical_id_cache), entry);
978 avic_update_iommu_vcpu_affinity(vcpu, h_physical_id, true);
979 }
980
avic_vcpu_put(struct kvm_vcpu * vcpu)981 void avic_vcpu_put(struct kvm_vcpu *vcpu)
982 {
983 u64 entry;
984 struct vcpu_svm *svm = to_svm(vcpu);
985
986 lockdep_assert_preemption_disabled();
987
988 entry = READ_ONCE(*(svm->avic_physical_id_cache));
989
990 /* Nothing to do if IsRunning == '0' due to vCPU blocking. */
991 if (!(entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK))
992 return;
993
994 avic_update_iommu_vcpu_affinity(vcpu, -1, 0);
995
996 entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;
997 WRITE_ONCE(*(svm->avic_physical_id_cache), entry);
998 }
999
1000
avic_refresh_apicv_exec_ctrl(struct kvm_vcpu * vcpu)1001 void avic_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu)
1002 {
1003 struct vcpu_svm *svm = to_svm(vcpu);
1004 struct vmcb *vmcb = svm->vmcb01.ptr;
1005 bool activated = kvm_vcpu_apicv_active(vcpu);
1006
1007 if (!enable_apicv)
1008 return;
1009
1010 if (activated) {
1011 /**
1012 * During AVIC temporary deactivation, guest could update
1013 * APIC ID, DFR and LDR registers, which would not be trapped
1014 * by avic_unaccelerated_access_interception(). In this case,
1015 * we need to check and update the AVIC logical APIC ID table
1016 * accordingly before re-activating.
1017 */
1018 avic_apicv_post_state_restore(vcpu);
1019 vmcb->control.int_ctl |= AVIC_ENABLE_MASK;
1020 } else {
1021 vmcb->control.int_ctl &= ~AVIC_ENABLE_MASK;
1022 }
1023 vmcb_mark_dirty(vmcb, VMCB_AVIC);
1024
1025 if (activated)
1026 avic_vcpu_load(vcpu, vcpu->cpu);
1027 else
1028 avic_vcpu_put(vcpu);
1029
1030 avic_set_pi_irte_mode(vcpu, activated);
1031 }
1032
avic_vcpu_blocking(struct kvm_vcpu * vcpu)1033 void avic_vcpu_blocking(struct kvm_vcpu *vcpu)
1034 {
1035 if (!kvm_vcpu_apicv_active(vcpu))
1036 return;
1037
1038 /*
1039 * Unload the AVIC when the vCPU is about to block, _before_
1040 * the vCPU actually blocks.
1041 *
1042 * Any IRQs that arrive before IsRunning=0 will not cause an
1043 * incomplete IPI vmexit on the source, therefore vIRR will also
1044 * be checked by kvm_vcpu_check_block() before blocking. The
1045 * memory barrier implicit in set_current_state orders writing
1046 * IsRunning=0 before reading the vIRR. The processor needs a
1047 * matching memory barrier on interrupt delivery between writing
1048 * IRR and reading IsRunning; the lack of this barrier might be
1049 * the cause of errata #1235).
1050 */
1051 avic_vcpu_put(vcpu);
1052 }
1053
avic_vcpu_unblocking(struct kvm_vcpu * vcpu)1054 void avic_vcpu_unblocking(struct kvm_vcpu *vcpu)
1055 {
1056 if (!kvm_vcpu_apicv_active(vcpu))
1057 return;
1058
1059 avic_vcpu_load(vcpu, vcpu->cpu);
1060 }
1061