1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * KVM PMU support for Intel CPUs
4 *
5 * Copyright 2011 Red Hat, Inc. and/or its affiliates.
6 *
7 * Authors:
8 * Avi Kivity <avi@redhat.com>
9 * Gleb Natapov <gleb@redhat.com>
10 */
11 #include <linux/types.h>
12 #include <linux/kvm_host.h>
13 #include <linux/perf_event.h>
14 #include <asm/perf_event.h>
15 #include "x86.h"
16 #include "cpuid.h"
17 #include "lapic.h"
18 #include "nested.h"
19 #include "pmu.h"
20
21 #define MSR_PMC_FULL_WIDTH_BIT (MSR_IA32_PMC0 - MSR_IA32_PERFCTR0)
22
23 static struct kvm_event_hw_type_mapping intel_arch_events[] = {
24 [0] = { 0x3c, 0x00, PERF_COUNT_HW_CPU_CYCLES },
25 [1] = { 0xc0, 0x00, PERF_COUNT_HW_INSTRUCTIONS },
26 [2] = { 0x3c, 0x01, PERF_COUNT_HW_BUS_CYCLES },
27 [3] = { 0x2e, 0x4f, PERF_COUNT_HW_CACHE_REFERENCES },
28 [4] = { 0x2e, 0x41, PERF_COUNT_HW_CACHE_MISSES },
29 [5] = { 0xc4, 0x00, PERF_COUNT_HW_BRANCH_INSTRUCTIONS },
30 [6] = { 0xc5, 0x00, PERF_COUNT_HW_BRANCH_MISSES },
31 /* The above index must match CPUID 0x0A.EBX bit vector */
32 [7] = { 0x00, 0x03, PERF_COUNT_HW_REF_CPU_CYCLES },
33 };
34
35 /* mapping between fixed pmc index and intel_arch_events array */
36 static int fixed_pmc_events[] = {1, 0, 7};
37
reprogram_fixed_counters(struct kvm_pmu * pmu,u64 data)38 static void reprogram_fixed_counters(struct kvm_pmu *pmu, u64 data)
39 {
40 int i;
41
42 for (i = 0; i < pmu->nr_arch_fixed_counters; i++) {
43 u8 new_ctrl = fixed_ctrl_field(data, i);
44 u8 old_ctrl = fixed_ctrl_field(pmu->fixed_ctr_ctrl, i);
45 struct kvm_pmc *pmc;
46
47 pmc = get_fixed_pmc(pmu, MSR_CORE_PERF_FIXED_CTR0 + i);
48
49 if (old_ctrl == new_ctrl)
50 continue;
51
52 __set_bit(INTEL_PMC_IDX_FIXED + i, pmu->pmc_in_use);
53 reprogram_fixed_counter(pmc, new_ctrl, i);
54 }
55
56 pmu->fixed_ctr_ctrl = data;
57 }
58
59 /* function is called when global control register has been updated. */
global_ctrl_changed(struct kvm_pmu * pmu,u64 data)60 static void global_ctrl_changed(struct kvm_pmu *pmu, u64 data)
61 {
62 int bit;
63 u64 diff = pmu->global_ctrl ^ data;
64
65 pmu->global_ctrl = data;
66
67 for_each_set_bit(bit, (unsigned long *)&diff, X86_PMC_IDX_MAX)
68 reprogram_counter(pmu, bit);
69 }
70
intel_pmc_perf_hw_id(struct kvm_pmc * pmc)71 static unsigned int intel_pmc_perf_hw_id(struct kvm_pmc *pmc)
72 {
73 struct kvm_pmu *pmu = pmc_to_pmu(pmc);
74 u8 event_select = pmc->eventsel & ARCH_PERFMON_EVENTSEL_EVENT;
75 u8 unit_mask = (pmc->eventsel & ARCH_PERFMON_EVENTSEL_UMASK) >> 8;
76 int i;
77
78 for (i = 0; i < ARRAY_SIZE(intel_arch_events); i++) {
79 if (intel_arch_events[i].eventsel != event_select ||
80 intel_arch_events[i].unit_mask != unit_mask)
81 continue;
82
83 /* disable event that reported as not present by cpuid */
84 if ((i < 7) && !(pmu->available_event_types & (1 << i)))
85 return PERF_COUNT_HW_MAX + 1;
86
87 break;
88 }
89
90 if (i == ARRAY_SIZE(intel_arch_events))
91 return PERF_COUNT_HW_MAX;
92
93 return intel_arch_events[i].event_type;
94 }
95
96 /* check if a PMC is enabled by comparing it with globl_ctrl bits. */
intel_pmc_is_enabled(struct kvm_pmc * pmc)97 static bool intel_pmc_is_enabled(struct kvm_pmc *pmc)
98 {
99 struct kvm_pmu *pmu = pmc_to_pmu(pmc);
100
101 if (!intel_pmu_has_perf_global_ctrl(pmu))
102 return true;
103
104 return test_bit(pmc->idx, (unsigned long *)&pmu->global_ctrl);
105 }
106
intel_pmc_idx_to_pmc(struct kvm_pmu * pmu,int pmc_idx)107 static struct kvm_pmc *intel_pmc_idx_to_pmc(struct kvm_pmu *pmu, int pmc_idx)
108 {
109 if (pmc_idx < INTEL_PMC_IDX_FIXED)
110 return get_gp_pmc(pmu, MSR_P6_EVNTSEL0 + pmc_idx,
111 MSR_P6_EVNTSEL0);
112 else {
113 u32 idx = pmc_idx - INTEL_PMC_IDX_FIXED;
114
115 return get_fixed_pmc(pmu, idx + MSR_CORE_PERF_FIXED_CTR0);
116 }
117 }
118
intel_is_valid_rdpmc_ecx(struct kvm_vcpu * vcpu,unsigned int idx)119 static bool intel_is_valid_rdpmc_ecx(struct kvm_vcpu *vcpu, unsigned int idx)
120 {
121 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
122 bool fixed = idx & (1u << 30);
123
124 idx &= ~(3u << 30);
125
126 return fixed ? idx < pmu->nr_arch_fixed_counters
127 : idx < pmu->nr_arch_gp_counters;
128 }
129
intel_rdpmc_ecx_to_pmc(struct kvm_vcpu * vcpu,unsigned int idx,u64 * mask)130 static struct kvm_pmc *intel_rdpmc_ecx_to_pmc(struct kvm_vcpu *vcpu,
131 unsigned int idx, u64 *mask)
132 {
133 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
134 bool fixed = idx & (1u << 30);
135 struct kvm_pmc *counters;
136 unsigned int num_counters;
137
138 idx &= ~(3u << 30);
139 if (fixed) {
140 counters = pmu->fixed_counters;
141 num_counters = pmu->nr_arch_fixed_counters;
142 } else {
143 counters = pmu->gp_counters;
144 num_counters = pmu->nr_arch_gp_counters;
145 }
146 if (idx >= num_counters)
147 return NULL;
148 *mask &= pmu->counter_bitmask[fixed ? KVM_PMC_FIXED : KVM_PMC_GP];
149 return &counters[array_index_nospec(idx, num_counters)];
150 }
151
vcpu_get_perf_capabilities(struct kvm_vcpu * vcpu)152 static inline u64 vcpu_get_perf_capabilities(struct kvm_vcpu *vcpu)
153 {
154 if (!guest_cpuid_has(vcpu, X86_FEATURE_PDCM))
155 return 0;
156
157 return vcpu->arch.perf_capabilities;
158 }
159
fw_writes_is_enabled(struct kvm_vcpu * vcpu)160 static inline bool fw_writes_is_enabled(struct kvm_vcpu *vcpu)
161 {
162 return (vcpu_get_perf_capabilities(vcpu) & PMU_CAP_FW_WRITES) != 0;
163 }
164
get_fw_gp_pmc(struct kvm_pmu * pmu,u32 msr)165 static inline struct kvm_pmc *get_fw_gp_pmc(struct kvm_pmu *pmu, u32 msr)
166 {
167 if (!fw_writes_is_enabled(pmu_to_vcpu(pmu)))
168 return NULL;
169
170 return get_gp_pmc(pmu, msr, MSR_IA32_PMC0);
171 }
172
intel_pmu_lbr_is_compatible(struct kvm_vcpu * vcpu)173 bool intel_pmu_lbr_is_compatible(struct kvm_vcpu *vcpu)
174 {
175 /*
176 * As a first step, a guest could only enable LBR feature if its
177 * cpu model is the same as the host because the LBR registers
178 * would be pass-through to the guest and they're model specific.
179 */
180 return boot_cpu_data.x86_model == guest_cpuid_model(vcpu);
181 }
182
intel_pmu_lbr_is_enabled(struct kvm_vcpu * vcpu)183 bool intel_pmu_lbr_is_enabled(struct kvm_vcpu *vcpu)
184 {
185 struct x86_pmu_lbr *lbr = vcpu_to_lbr_records(vcpu);
186
187 return lbr->nr && (vcpu_get_perf_capabilities(vcpu) & PMU_CAP_LBR_FMT);
188 }
189
intel_pmu_is_valid_lbr_msr(struct kvm_vcpu * vcpu,u32 index)190 static bool intel_pmu_is_valid_lbr_msr(struct kvm_vcpu *vcpu, u32 index)
191 {
192 struct x86_pmu_lbr *records = vcpu_to_lbr_records(vcpu);
193 bool ret = false;
194
195 if (!intel_pmu_lbr_is_enabled(vcpu))
196 return ret;
197
198 ret = (index == MSR_LBR_SELECT) || (index == MSR_LBR_TOS) ||
199 (index >= records->from && index < records->from + records->nr) ||
200 (index >= records->to && index < records->to + records->nr);
201
202 if (!ret && records->info)
203 ret = (index >= records->info && index < records->info + records->nr);
204
205 return ret;
206 }
207
intel_is_valid_msr(struct kvm_vcpu * vcpu,u32 msr)208 static bool intel_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr)
209 {
210 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
211 int ret;
212
213 switch (msr) {
214 case MSR_CORE_PERF_FIXED_CTR_CTRL:
215 case MSR_CORE_PERF_GLOBAL_STATUS:
216 case MSR_CORE_PERF_GLOBAL_CTRL:
217 case MSR_CORE_PERF_GLOBAL_OVF_CTRL:
218 return intel_pmu_has_perf_global_ctrl(pmu);
219 break;
220 default:
221 ret = get_gp_pmc(pmu, msr, MSR_IA32_PERFCTR0) ||
222 get_gp_pmc(pmu, msr, MSR_P6_EVNTSEL0) ||
223 get_fixed_pmc(pmu, msr) || get_fw_gp_pmc(pmu, msr) ||
224 intel_pmu_is_valid_lbr_msr(vcpu, msr);
225 break;
226 }
227
228 return ret;
229 }
230
intel_msr_idx_to_pmc(struct kvm_vcpu * vcpu,u32 msr)231 static struct kvm_pmc *intel_msr_idx_to_pmc(struct kvm_vcpu *vcpu, u32 msr)
232 {
233 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
234 struct kvm_pmc *pmc;
235
236 pmc = get_fixed_pmc(pmu, msr);
237 pmc = pmc ? pmc : get_gp_pmc(pmu, msr, MSR_P6_EVNTSEL0);
238 pmc = pmc ? pmc : get_gp_pmc(pmu, msr, MSR_IA32_PERFCTR0);
239
240 return pmc;
241 }
242
intel_pmu_release_guest_lbr_event(struct kvm_vcpu * vcpu)243 static inline void intel_pmu_release_guest_lbr_event(struct kvm_vcpu *vcpu)
244 {
245 struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu);
246
247 if (lbr_desc->event) {
248 perf_event_release_kernel(lbr_desc->event);
249 lbr_desc->event = NULL;
250 vcpu_to_pmu(vcpu)->event_count--;
251 }
252 }
253
intel_pmu_create_guest_lbr_event(struct kvm_vcpu * vcpu)254 int intel_pmu_create_guest_lbr_event(struct kvm_vcpu *vcpu)
255 {
256 struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu);
257 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
258 struct perf_event *event;
259
260 /*
261 * The perf_event_attr is constructed in the minimum efficient way:
262 * - set 'pinned = true' to make it task pinned so that if another
263 * cpu pinned event reclaims LBR, the event->oncpu will be set to -1;
264 * - set '.exclude_host = true' to record guest branches behavior;
265 *
266 * - set '.config = INTEL_FIXED_VLBR_EVENT' to indicates host perf
267 * schedule the event without a real HW counter but a fake one;
268 * check is_guest_lbr_event() and __intel_get_event_constraints();
269 *
270 * - set 'sample_type = PERF_SAMPLE_BRANCH_STACK' and
271 * 'branch_sample_type = PERF_SAMPLE_BRANCH_CALL_STACK |
272 * PERF_SAMPLE_BRANCH_USER' to configure it as a LBR callstack
273 * event, which helps KVM to save/restore guest LBR records
274 * during host context switches and reduces quite a lot overhead,
275 * check branch_user_callstack() and intel_pmu_lbr_sched_task();
276 */
277 struct perf_event_attr attr = {
278 .type = PERF_TYPE_RAW,
279 .size = sizeof(attr),
280 .config = INTEL_FIXED_VLBR_EVENT,
281 .sample_type = PERF_SAMPLE_BRANCH_STACK,
282 .pinned = true,
283 .exclude_host = true,
284 .branch_sample_type = PERF_SAMPLE_BRANCH_CALL_STACK |
285 PERF_SAMPLE_BRANCH_USER,
286 };
287
288 if (unlikely(lbr_desc->event)) {
289 __set_bit(INTEL_PMC_IDX_FIXED_VLBR, pmu->pmc_in_use);
290 return 0;
291 }
292
293 event = perf_event_create_kernel_counter(&attr, -1,
294 current, NULL, NULL);
295 if (IS_ERR(event)) {
296 pr_debug_ratelimited("%s: failed %ld\n",
297 __func__, PTR_ERR(event));
298 return PTR_ERR(event);
299 }
300 lbr_desc->event = event;
301 pmu->event_count++;
302 __set_bit(INTEL_PMC_IDX_FIXED_VLBR, pmu->pmc_in_use);
303 return 0;
304 }
305
306 /*
307 * It's safe to access LBR msrs from guest when they have not
308 * been passthrough since the host would help restore or reset
309 * the LBR msrs records when the guest LBR event is scheduled in.
310 */
intel_pmu_handle_lbr_msrs_access(struct kvm_vcpu * vcpu,struct msr_data * msr_info,bool read)311 static bool intel_pmu_handle_lbr_msrs_access(struct kvm_vcpu *vcpu,
312 struct msr_data *msr_info, bool read)
313 {
314 struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu);
315 u32 index = msr_info->index;
316
317 if (!intel_pmu_is_valid_lbr_msr(vcpu, index))
318 return false;
319
320 if (!lbr_desc->event && intel_pmu_create_guest_lbr_event(vcpu) < 0)
321 goto dummy;
322
323 /*
324 * Disable irq to ensure the LBR feature doesn't get reclaimed by the
325 * host at the time the value is read from the msr, and this avoids the
326 * host LBR value to be leaked to the guest. If LBR has been reclaimed,
327 * return 0 on guest reads.
328 */
329 local_irq_disable();
330 if (lbr_desc->event->state == PERF_EVENT_STATE_ACTIVE) {
331 if (read)
332 rdmsrl(index, msr_info->data);
333 else
334 wrmsrl(index, msr_info->data);
335 __set_bit(INTEL_PMC_IDX_FIXED_VLBR, vcpu_to_pmu(vcpu)->pmc_in_use);
336 local_irq_enable();
337 return true;
338 }
339 clear_bit(INTEL_PMC_IDX_FIXED_VLBR, vcpu_to_pmu(vcpu)->pmc_in_use);
340 local_irq_enable();
341
342 dummy:
343 if (read)
344 msr_info->data = 0;
345 return true;
346 }
347
intel_pmu_get_msr(struct kvm_vcpu * vcpu,struct msr_data * msr_info)348 static int intel_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
349 {
350 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
351 struct kvm_pmc *pmc;
352 u32 msr = msr_info->index;
353
354 switch (msr) {
355 case MSR_CORE_PERF_FIXED_CTR_CTRL:
356 msr_info->data = pmu->fixed_ctr_ctrl;
357 return 0;
358 case MSR_CORE_PERF_GLOBAL_STATUS:
359 msr_info->data = pmu->global_status;
360 return 0;
361 case MSR_CORE_PERF_GLOBAL_CTRL:
362 msr_info->data = pmu->global_ctrl;
363 return 0;
364 case MSR_CORE_PERF_GLOBAL_OVF_CTRL:
365 msr_info->data = 0;
366 return 0;
367 default:
368 if ((pmc = get_gp_pmc(pmu, msr, MSR_IA32_PERFCTR0)) ||
369 (pmc = get_gp_pmc(pmu, msr, MSR_IA32_PMC0))) {
370 u64 val = pmc_read_counter(pmc);
371 msr_info->data =
372 val & pmu->counter_bitmask[KVM_PMC_GP];
373 return 0;
374 } else if ((pmc = get_fixed_pmc(pmu, msr))) {
375 u64 val = pmc_read_counter(pmc);
376 msr_info->data =
377 val & pmu->counter_bitmask[KVM_PMC_FIXED];
378 return 0;
379 } else if ((pmc = get_gp_pmc(pmu, msr, MSR_P6_EVNTSEL0))) {
380 msr_info->data = pmc->eventsel;
381 return 0;
382 } else if (intel_pmu_handle_lbr_msrs_access(vcpu, msr_info, true))
383 return 0;
384 }
385
386 return 1;
387 }
388
intel_pmu_set_msr(struct kvm_vcpu * vcpu,struct msr_data * msr_info)389 static int intel_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
390 {
391 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
392 struct kvm_pmc *pmc;
393 u32 msr = msr_info->index;
394 u64 data = msr_info->data;
395 u64 reserved_bits;
396
397 switch (msr) {
398 case MSR_CORE_PERF_FIXED_CTR_CTRL:
399 if (pmu->fixed_ctr_ctrl == data)
400 return 0;
401 if (!(data & pmu->fixed_ctr_ctrl_mask)) {
402 reprogram_fixed_counters(pmu, data);
403 return 0;
404 }
405 break;
406 case MSR_CORE_PERF_GLOBAL_STATUS:
407 if (msr_info->host_initiated) {
408 pmu->global_status = data;
409 return 0;
410 }
411 break; /* RO MSR */
412 case MSR_CORE_PERF_GLOBAL_CTRL:
413 if (pmu->global_ctrl == data)
414 return 0;
415 if (kvm_valid_perf_global_ctrl(pmu, data)) {
416 global_ctrl_changed(pmu, data);
417 return 0;
418 }
419 break;
420 case MSR_CORE_PERF_GLOBAL_OVF_CTRL:
421 if (!(data & pmu->global_ovf_ctrl_mask)) {
422 if (!msr_info->host_initiated)
423 pmu->global_status &= ~data;
424 return 0;
425 }
426 break;
427 default:
428 if ((pmc = get_gp_pmc(pmu, msr, MSR_IA32_PERFCTR0)) ||
429 (pmc = get_gp_pmc(pmu, msr, MSR_IA32_PMC0))) {
430 if ((msr & MSR_PMC_FULL_WIDTH_BIT) &&
431 (data & ~pmu->counter_bitmask[KVM_PMC_GP]))
432 return 1;
433 if (!msr_info->host_initiated &&
434 !(msr & MSR_PMC_FULL_WIDTH_BIT))
435 data = (s64)(s32)data;
436 pmc->counter += data - pmc_read_counter(pmc);
437 pmc_update_sample_period(pmc);
438 return 0;
439 } else if ((pmc = get_fixed_pmc(pmu, msr))) {
440 pmc->counter += data - pmc_read_counter(pmc);
441 pmc_update_sample_period(pmc);
442 return 0;
443 } else if ((pmc = get_gp_pmc(pmu, msr, MSR_P6_EVNTSEL0))) {
444 if (data == pmc->eventsel)
445 return 0;
446 reserved_bits = pmu->reserved_bits;
447 if ((pmc->idx == 2) &&
448 (pmu->raw_event_mask & HSW_IN_TX_CHECKPOINTED))
449 reserved_bits ^= HSW_IN_TX_CHECKPOINTED;
450 if (!(data & reserved_bits)) {
451 reprogram_gp_counter(pmc, data);
452 return 0;
453 }
454 } else if (intel_pmu_handle_lbr_msrs_access(vcpu, msr_info, false))
455 return 0;
456 }
457
458 return 1;
459 }
460
setup_fixed_pmc_eventsel(struct kvm_pmu * pmu)461 static void setup_fixed_pmc_eventsel(struct kvm_pmu *pmu)
462 {
463 size_t size = ARRAY_SIZE(fixed_pmc_events);
464 struct kvm_pmc *pmc;
465 u32 event;
466 int i;
467
468 for (i = 0; i < pmu->nr_arch_fixed_counters; i++) {
469 pmc = &pmu->fixed_counters[i];
470 event = fixed_pmc_events[array_index_nospec(i, size)];
471 pmc->eventsel = (intel_arch_events[event].unit_mask << 8) |
472 intel_arch_events[event].eventsel;
473 }
474 }
475
intel_pmu_refresh(struct kvm_vcpu * vcpu)476 static void intel_pmu_refresh(struct kvm_vcpu *vcpu)
477 {
478 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
479 struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu);
480
481 struct x86_pmu_capability x86_pmu;
482 struct kvm_cpuid_entry2 *entry;
483 union cpuid10_eax eax;
484 union cpuid10_edx edx;
485 int i;
486
487 pmu->nr_arch_gp_counters = 0;
488 pmu->nr_arch_fixed_counters = 0;
489 pmu->counter_bitmask[KVM_PMC_GP] = 0;
490 pmu->counter_bitmask[KVM_PMC_FIXED] = 0;
491 pmu->version = 0;
492 pmu->reserved_bits = 0xffffffff00200000ull;
493 pmu->raw_event_mask = X86_RAW_EVENT_MASK;
494 pmu->global_ctrl_mask = ~0ull;
495 pmu->global_ovf_ctrl_mask = ~0ull;
496 pmu->fixed_ctr_ctrl_mask = ~0ull;
497
498 entry = kvm_find_cpuid_entry(vcpu, 0xa, 0);
499 if (!entry || !vcpu->kvm->arch.enable_pmu)
500 return;
501 eax.full = entry->eax;
502 edx.full = entry->edx;
503
504 pmu->version = eax.split.version_id;
505 if (!pmu->version)
506 return;
507
508 perf_get_x86_pmu_capability(&x86_pmu);
509
510 pmu->nr_arch_gp_counters = min_t(int, eax.split.num_counters,
511 x86_pmu.num_counters_gp);
512 eax.split.bit_width = min_t(int, eax.split.bit_width, x86_pmu.bit_width_gp);
513 pmu->counter_bitmask[KVM_PMC_GP] = ((u64)1 << eax.split.bit_width) - 1;
514 eax.split.mask_length = min_t(int, eax.split.mask_length, x86_pmu.events_mask_len);
515 pmu->available_event_types = ~entry->ebx &
516 ((1ull << eax.split.mask_length) - 1);
517
518 if (pmu->version == 1) {
519 pmu->nr_arch_fixed_counters = 0;
520 } else {
521 pmu->nr_arch_fixed_counters =
522 min3(ARRAY_SIZE(fixed_pmc_events),
523 (size_t) edx.split.num_counters_fixed,
524 (size_t) x86_pmu.num_counters_fixed);
525 edx.split.bit_width_fixed = min_t(int,
526 edx.split.bit_width_fixed, x86_pmu.bit_width_fixed);
527 pmu->counter_bitmask[KVM_PMC_FIXED] =
528 ((u64)1 << edx.split.bit_width_fixed) - 1;
529 setup_fixed_pmc_eventsel(pmu);
530 }
531
532 for (i = 0; i < pmu->nr_arch_fixed_counters; i++)
533 pmu->fixed_ctr_ctrl_mask &= ~(0xbull << (i * 4));
534 pmu->global_ctrl = ((1ull << pmu->nr_arch_gp_counters) - 1) |
535 (((1ull << pmu->nr_arch_fixed_counters) - 1) << INTEL_PMC_IDX_FIXED);
536 pmu->global_ctrl_mask = ~pmu->global_ctrl;
537 pmu->global_ovf_ctrl_mask = pmu->global_ctrl_mask
538 & ~(MSR_CORE_PERF_GLOBAL_OVF_CTRL_OVF_BUF |
539 MSR_CORE_PERF_GLOBAL_OVF_CTRL_COND_CHGD);
540 if (vmx_pt_mode_is_host_guest())
541 pmu->global_ovf_ctrl_mask &=
542 ~MSR_CORE_PERF_GLOBAL_OVF_CTRL_TRACE_TOPA_PMI;
543
544 entry = kvm_find_cpuid_entry(vcpu, 7, 0);
545 if (entry &&
546 (boot_cpu_has(X86_FEATURE_HLE) || boot_cpu_has(X86_FEATURE_RTM)) &&
547 (entry->ebx & (X86_FEATURE_HLE|X86_FEATURE_RTM))) {
548 pmu->reserved_bits ^= HSW_IN_TX;
549 pmu->raw_event_mask |= (HSW_IN_TX|HSW_IN_TX_CHECKPOINTED);
550 }
551
552 bitmap_set(pmu->all_valid_pmc_idx,
553 0, pmu->nr_arch_gp_counters);
554 bitmap_set(pmu->all_valid_pmc_idx,
555 INTEL_PMC_MAX_GENERIC, pmu->nr_arch_fixed_counters);
556
557 nested_vmx_pmu_refresh(vcpu,
558 intel_is_valid_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL));
559
560 if (intel_pmu_lbr_is_compatible(vcpu))
561 x86_perf_get_lbr(&lbr_desc->records);
562 else
563 lbr_desc->records.nr = 0;
564
565 if (lbr_desc->records.nr)
566 bitmap_set(pmu->all_valid_pmc_idx, INTEL_PMC_IDX_FIXED_VLBR, 1);
567 }
568
intel_pmu_init(struct kvm_vcpu * vcpu)569 static void intel_pmu_init(struct kvm_vcpu *vcpu)
570 {
571 int i;
572 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
573 struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu);
574
575 for (i = 0; i < INTEL_PMC_MAX_GENERIC; i++) {
576 pmu->gp_counters[i].type = KVM_PMC_GP;
577 pmu->gp_counters[i].vcpu = vcpu;
578 pmu->gp_counters[i].idx = i;
579 pmu->gp_counters[i].current_config = 0;
580 }
581
582 for (i = 0; i < KVM_PMC_MAX_FIXED; i++) {
583 pmu->fixed_counters[i].type = KVM_PMC_FIXED;
584 pmu->fixed_counters[i].vcpu = vcpu;
585 pmu->fixed_counters[i].idx = i + INTEL_PMC_IDX_FIXED;
586 pmu->fixed_counters[i].current_config = 0;
587 }
588
589 vcpu->arch.perf_capabilities = vmx_get_perf_capabilities();
590 lbr_desc->records.nr = 0;
591 lbr_desc->event = NULL;
592 lbr_desc->msr_passthrough = false;
593 }
594
intel_pmu_reset(struct kvm_vcpu * vcpu)595 static void intel_pmu_reset(struct kvm_vcpu *vcpu)
596 {
597 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
598 struct kvm_pmc *pmc = NULL;
599 int i;
600
601 for (i = 0; i < INTEL_PMC_MAX_GENERIC; i++) {
602 pmc = &pmu->gp_counters[i];
603
604 pmc_stop_counter(pmc);
605 pmc->counter = pmc->eventsel = 0;
606 }
607
608 for (i = 0; i < KVM_PMC_MAX_FIXED; i++) {
609 pmc = &pmu->fixed_counters[i];
610
611 pmc_stop_counter(pmc);
612 pmc->counter = 0;
613 }
614
615 pmu->fixed_ctr_ctrl = pmu->global_ctrl = pmu->global_status = 0;
616
617 intel_pmu_release_guest_lbr_event(vcpu);
618 }
619
620 /*
621 * Emulate LBR_On_PMI behavior for 1 < pmu.version < 4.
622 *
623 * If Freeze_LBR_On_PMI = 1, the LBR is frozen on PMI and
624 * the KVM emulates to clear the LBR bit (bit 0) in IA32_DEBUGCTL.
625 *
626 * Guest needs to re-enable LBR to resume branches recording.
627 */
intel_pmu_legacy_freezing_lbrs_on_pmi(struct kvm_vcpu * vcpu)628 static void intel_pmu_legacy_freezing_lbrs_on_pmi(struct kvm_vcpu *vcpu)
629 {
630 u64 data = vmcs_read64(GUEST_IA32_DEBUGCTL);
631
632 if (data & DEBUGCTLMSR_FREEZE_LBRS_ON_PMI) {
633 data &= ~DEBUGCTLMSR_LBR;
634 vmcs_write64(GUEST_IA32_DEBUGCTL, data);
635 }
636 }
637
intel_pmu_deliver_pmi(struct kvm_vcpu * vcpu)638 static void intel_pmu_deliver_pmi(struct kvm_vcpu *vcpu)
639 {
640 u8 version = vcpu_to_pmu(vcpu)->version;
641
642 if (!intel_pmu_lbr_is_enabled(vcpu))
643 return;
644
645 if (version > 1 && version < 4)
646 intel_pmu_legacy_freezing_lbrs_on_pmi(vcpu);
647 }
648
vmx_update_intercept_for_lbr_msrs(struct kvm_vcpu * vcpu,bool set)649 static void vmx_update_intercept_for_lbr_msrs(struct kvm_vcpu *vcpu, bool set)
650 {
651 struct x86_pmu_lbr *lbr = vcpu_to_lbr_records(vcpu);
652 int i;
653
654 for (i = 0; i < lbr->nr; i++) {
655 vmx_set_intercept_for_msr(vcpu, lbr->from + i, MSR_TYPE_RW, set);
656 vmx_set_intercept_for_msr(vcpu, lbr->to + i, MSR_TYPE_RW, set);
657 if (lbr->info)
658 vmx_set_intercept_for_msr(vcpu, lbr->info + i, MSR_TYPE_RW, set);
659 }
660
661 vmx_set_intercept_for_msr(vcpu, MSR_LBR_SELECT, MSR_TYPE_RW, set);
662 vmx_set_intercept_for_msr(vcpu, MSR_LBR_TOS, MSR_TYPE_RW, set);
663 }
664
vmx_disable_lbr_msrs_passthrough(struct kvm_vcpu * vcpu)665 static inline void vmx_disable_lbr_msrs_passthrough(struct kvm_vcpu *vcpu)
666 {
667 struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu);
668
669 if (!lbr_desc->msr_passthrough)
670 return;
671
672 vmx_update_intercept_for_lbr_msrs(vcpu, true);
673 lbr_desc->msr_passthrough = false;
674 }
675
vmx_enable_lbr_msrs_passthrough(struct kvm_vcpu * vcpu)676 static inline void vmx_enable_lbr_msrs_passthrough(struct kvm_vcpu *vcpu)
677 {
678 struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu);
679
680 if (lbr_desc->msr_passthrough)
681 return;
682
683 vmx_update_intercept_for_lbr_msrs(vcpu, false);
684 lbr_desc->msr_passthrough = true;
685 }
686
687 /*
688 * Higher priority host perf events (e.g. cpu pinned) could reclaim the
689 * pmu resources (e.g. LBR) that were assigned to the guest. This is
690 * usually done via ipi calls (more details in perf_install_in_context).
691 *
692 * Before entering the non-root mode (with irq disabled here), double
693 * confirm that the pmu features enabled to the guest are not reclaimed
694 * by higher priority host events. Otherwise, disallow vcpu's access to
695 * the reclaimed features.
696 */
vmx_passthrough_lbr_msrs(struct kvm_vcpu * vcpu)697 void vmx_passthrough_lbr_msrs(struct kvm_vcpu *vcpu)
698 {
699 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
700 struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu);
701
702 if (!lbr_desc->event) {
703 vmx_disable_lbr_msrs_passthrough(vcpu);
704 if (vmcs_read64(GUEST_IA32_DEBUGCTL) & DEBUGCTLMSR_LBR)
705 goto warn;
706 if (test_bit(INTEL_PMC_IDX_FIXED_VLBR, pmu->pmc_in_use))
707 goto warn;
708 return;
709 }
710
711 if (lbr_desc->event->state < PERF_EVENT_STATE_ACTIVE) {
712 vmx_disable_lbr_msrs_passthrough(vcpu);
713 __clear_bit(INTEL_PMC_IDX_FIXED_VLBR, pmu->pmc_in_use);
714 goto warn;
715 } else
716 vmx_enable_lbr_msrs_passthrough(vcpu);
717
718 return;
719
720 warn:
721 pr_warn_ratelimited("kvm: vcpu-%d: fail to passthrough LBR.\n",
722 vcpu->vcpu_id);
723 }
724
intel_pmu_cleanup(struct kvm_vcpu * vcpu)725 static void intel_pmu_cleanup(struct kvm_vcpu *vcpu)
726 {
727 if (!(vmcs_read64(GUEST_IA32_DEBUGCTL) & DEBUGCTLMSR_LBR))
728 intel_pmu_release_guest_lbr_event(vcpu);
729 }
730
731 struct kvm_pmu_ops intel_pmu_ops __initdata = {
732 .pmc_perf_hw_id = intel_pmc_perf_hw_id,
733 .pmc_is_enabled = intel_pmc_is_enabled,
734 .pmc_idx_to_pmc = intel_pmc_idx_to_pmc,
735 .rdpmc_ecx_to_pmc = intel_rdpmc_ecx_to_pmc,
736 .msr_idx_to_pmc = intel_msr_idx_to_pmc,
737 .is_valid_rdpmc_ecx = intel_is_valid_rdpmc_ecx,
738 .is_valid_msr = intel_is_valid_msr,
739 .get_msr = intel_pmu_get_msr,
740 .set_msr = intel_pmu_set_msr,
741 .refresh = intel_pmu_refresh,
742 .init = intel_pmu_init,
743 .reset = intel_pmu_reset,
744 .deliver_pmi = intel_pmu_deliver_pmi,
745 .cleanup = intel_pmu_cleanup,
746 };
747