1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Kernel-based Virtual Machine -- Performance Monitoring Unit support
4  *
5  * Copyright 2015 Red Hat, Inc. and/or its affiliates.
6  *
7  * Authors:
8  *   Avi Kivity   <avi@redhat.com>
9  *   Gleb Natapov <gleb@redhat.com>
10  *   Wei Huang    <wei@redhat.com>
11  */
12 
13 #include <linux/types.h>
14 #include <linux/kvm_host.h>
15 #include <linux/perf_event.h>
16 #include <linux/bsearch.h>
17 #include <linux/sort.h>
18 #include <asm/perf_event.h>
19 #include <asm/cpu_device_id.h>
20 #include "x86.h"
21 #include "cpuid.h"
22 #include "lapic.h"
23 #include "pmu.h"
24 
25 /* This is enough to filter the vast majority of currently defined events. */
26 #define KVM_PMU_EVENT_FILTER_MAX_EVENTS 300
27 
28 struct x86_pmu_capability __read_mostly kvm_pmu_cap;
29 EXPORT_SYMBOL_GPL(kvm_pmu_cap);
30 
31 static const struct x86_cpu_id vmx_icl_pebs_cpu[] = {
32 	X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_D, NULL),
33 	X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_X, NULL),
34 	{}
35 };
36 
37 /* NOTE:
38  * - Each perf counter is defined as "struct kvm_pmc";
39  * - There are two types of perf counters: general purpose (gp) and fixed.
40  *   gp counters are stored in gp_counters[] and fixed counters are stored
41  *   in fixed_counters[] respectively. Both of them are part of "struct
42  *   kvm_pmu";
43  * - pmu.c understands the difference between gp counters and fixed counters.
44  *   However AMD doesn't support fixed-counters;
45  * - There are three types of index to access perf counters (PMC):
46  *     1. MSR (named msr): For example Intel has MSR_IA32_PERFCTRn and AMD
47  *        has MSR_K7_PERFCTRn and, for families 15H and later,
48  *        MSR_F15H_PERF_CTRn, where MSR_F15H_PERF_CTR[0-3] are
49  *        aliased to MSR_K7_PERFCTRn.
50  *     2. MSR Index (named idx): This normally is used by RDPMC instruction.
51  *        For instance AMD RDPMC instruction uses 0000_0003h in ECX to access
52  *        C001_0007h (MSR_K7_PERCTR3). Intel has a similar mechanism, except
53  *        that it also supports fixed counters. idx can be used to as index to
54  *        gp and fixed counters.
55  *     3. Global PMC Index (named pmc): pmc is an index specific to PMU
56  *        code. Each pmc, stored in kvm_pmc.idx field, is unique across
57  *        all perf counters (both gp and fixed). The mapping relationship
58  *        between pmc and perf counters is as the following:
59  *        * Intel: [0 .. KVM_INTEL_PMC_MAX_GENERIC-1] <=> gp counters
60  *                 [INTEL_PMC_IDX_FIXED .. INTEL_PMC_IDX_FIXED + 2] <=> fixed
61  *        * AMD:   [0 .. AMD64_NUM_COUNTERS-1] and, for families 15H
62  *          and later, [0 .. AMD64_NUM_COUNTERS_CORE-1] <=> gp counters
63  */
64 
65 static struct kvm_pmu_ops kvm_pmu_ops __read_mostly;
66 
67 #define KVM_X86_PMU_OP(func)					     \
68 	DEFINE_STATIC_CALL_NULL(kvm_x86_pmu_##func,			     \
69 				*(((struct kvm_pmu_ops *)0)->func));
70 #define KVM_X86_PMU_OP_OPTIONAL KVM_X86_PMU_OP
71 #include <asm/kvm-x86-pmu-ops.h>
72 
kvm_pmu_ops_update(const struct kvm_pmu_ops * pmu_ops)73 void kvm_pmu_ops_update(const struct kvm_pmu_ops *pmu_ops)
74 {
75 	memcpy(&kvm_pmu_ops, pmu_ops, sizeof(kvm_pmu_ops));
76 
77 #define __KVM_X86_PMU_OP(func) \
78 	static_call_update(kvm_x86_pmu_##func, kvm_pmu_ops.func);
79 #define KVM_X86_PMU_OP(func) \
80 	WARN_ON(!kvm_pmu_ops.func); __KVM_X86_PMU_OP(func)
81 #define KVM_X86_PMU_OP_OPTIONAL __KVM_X86_PMU_OP
82 #include <asm/kvm-x86-pmu-ops.h>
83 #undef __KVM_X86_PMU_OP
84 }
85 
pmc_is_enabled(struct kvm_pmc * pmc)86 static inline bool pmc_is_enabled(struct kvm_pmc *pmc)
87 {
88 	return static_call(kvm_x86_pmu_pmc_is_enabled)(pmc);
89 }
90 
kvm_pmi_trigger_fn(struct irq_work * irq_work)91 static void kvm_pmi_trigger_fn(struct irq_work *irq_work)
92 {
93 	struct kvm_pmu *pmu = container_of(irq_work, struct kvm_pmu, irq_work);
94 	struct kvm_vcpu *vcpu = pmu_to_vcpu(pmu);
95 
96 	kvm_pmu_deliver_pmi(vcpu);
97 }
98 
__kvm_perf_overflow(struct kvm_pmc * pmc,bool in_pmi)99 static inline void __kvm_perf_overflow(struct kvm_pmc *pmc, bool in_pmi)
100 {
101 	struct kvm_pmu *pmu = pmc_to_pmu(pmc);
102 	bool skip_pmi = false;
103 
104 	/* Ignore counters that have been reprogrammed already. */
105 	if (test_and_set_bit(pmc->idx, pmu->reprogram_pmi))
106 		return;
107 
108 	if (pmc->perf_event && pmc->perf_event->attr.precise_ip) {
109 		if (!in_pmi) {
110 			/*
111 			 * TODO: KVM is currently _choosing_ to not generate records
112 			 * for emulated instructions, avoiding BUFFER_OVF PMI when
113 			 * there are no records. Strictly speaking, it should be done
114 			 * as well in the right context to improve sampling accuracy.
115 			 */
116 			skip_pmi = true;
117 		} else {
118 			/* Indicate PEBS overflow PMI to guest. */
119 			skip_pmi = __test_and_set_bit(GLOBAL_STATUS_BUFFER_OVF_BIT,
120 						      (unsigned long *)&pmu->global_status);
121 		}
122 	} else {
123 		__set_bit(pmc->idx, (unsigned long *)&pmu->global_status);
124 	}
125 	kvm_make_request(KVM_REQ_PMU, pmc->vcpu);
126 
127 	if (!pmc->intr || skip_pmi)
128 		return;
129 
130 	/*
131 	 * Inject PMI. If vcpu was in a guest mode during NMI PMI
132 	 * can be ejected on a guest mode re-entry. Otherwise we can't
133 	 * be sure that vcpu wasn't executing hlt instruction at the
134 	 * time of vmexit and is not going to re-enter guest mode until
135 	 * woken up. So we should wake it, but this is impossible from
136 	 * NMI context. Do it from irq work instead.
137 	 */
138 	if (in_pmi && !kvm_handling_nmi_from_guest(pmc->vcpu))
139 		irq_work_queue(&pmc_to_pmu(pmc)->irq_work);
140 	else
141 		kvm_make_request(KVM_REQ_PMI, pmc->vcpu);
142 }
143 
kvm_perf_overflow(struct perf_event * perf_event,struct perf_sample_data * data,struct pt_regs * regs)144 static void kvm_perf_overflow(struct perf_event *perf_event,
145 			      struct perf_sample_data *data,
146 			      struct pt_regs *regs)
147 {
148 	struct kvm_pmc *pmc = perf_event->overflow_handler_context;
149 
150 	__kvm_perf_overflow(pmc, true);
151 }
152 
pmc_reprogram_counter(struct kvm_pmc * pmc,u32 type,u64 config,bool exclude_user,bool exclude_kernel,bool intr)153 static void pmc_reprogram_counter(struct kvm_pmc *pmc, u32 type,
154 				  u64 config, bool exclude_user,
155 				  bool exclude_kernel, bool intr)
156 {
157 	struct kvm_pmu *pmu = pmc_to_pmu(pmc);
158 	struct perf_event *event;
159 	struct perf_event_attr attr = {
160 		.type = type,
161 		.size = sizeof(attr),
162 		.pinned = true,
163 		.exclude_idle = true,
164 		.exclude_host = 1,
165 		.exclude_user = exclude_user,
166 		.exclude_kernel = exclude_kernel,
167 		.config = config,
168 	};
169 	bool pebs = test_bit(pmc->idx, (unsigned long *)&pmu->pebs_enable);
170 
171 	attr.sample_period = get_sample_period(pmc, pmc->counter);
172 
173 	if ((attr.config & HSW_IN_TX_CHECKPOINTED) &&
174 	    guest_cpuid_is_intel(pmc->vcpu)) {
175 		/*
176 		 * HSW_IN_TX_CHECKPOINTED is not supported with nonzero
177 		 * period. Just clear the sample period so at least
178 		 * allocating the counter doesn't fail.
179 		 */
180 		attr.sample_period = 0;
181 	}
182 	if (pebs) {
183 		/*
184 		 * The non-zero precision level of guest event makes the ordinary
185 		 * guest event becomes a guest PEBS event and triggers the host
186 		 * PEBS PMI handler to determine whether the PEBS overflow PMI
187 		 * comes from the host counters or the guest.
188 		 *
189 		 * For most PEBS hardware events, the difference in the software
190 		 * precision levels of guest and host PEBS events will not affect
191 		 * the accuracy of the PEBS profiling result, because the "event IP"
192 		 * in the PEBS record is calibrated on the guest side.
193 		 *
194 		 * On Icelake everything is fine. Other hardware (GLC+, TNT+) that
195 		 * could possibly care here is unsupported and needs changes.
196 		 */
197 		attr.precise_ip = 1;
198 		if (x86_match_cpu(vmx_icl_pebs_cpu) && pmc->idx == 32)
199 			attr.precise_ip = 3;
200 	}
201 
202 	event = perf_event_create_kernel_counter(&attr, -1, current,
203 						 kvm_perf_overflow, pmc);
204 	if (IS_ERR(event)) {
205 		pr_debug_ratelimited("kvm_pmu: event creation failed %ld for pmc->idx = %d\n",
206 			    PTR_ERR(event), pmc->idx);
207 		return;
208 	}
209 
210 	pmc->perf_event = event;
211 	pmc_to_pmu(pmc)->event_count++;
212 	clear_bit(pmc->idx, pmc_to_pmu(pmc)->reprogram_pmi);
213 	pmc->is_paused = false;
214 	pmc->intr = intr || pebs;
215 }
216 
pmc_pause_counter(struct kvm_pmc * pmc)217 static void pmc_pause_counter(struct kvm_pmc *pmc)
218 {
219 	u64 counter = pmc->counter;
220 
221 	if (!pmc->perf_event || pmc->is_paused)
222 		return;
223 
224 	/* update counter, reset event value to avoid redundant accumulation */
225 	counter += perf_event_pause(pmc->perf_event, true);
226 	pmc->counter = counter & pmc_bitmask(pmc);
227 	pmc->is_paused = true;
228 }
229 
pmc_resume_counter(struct kvm_pmc * pmc)230 static bool pmc_resume_counter(struct kvm_pmc *pmc)
231 {
232 	if (!pmc->perf_event)
233 		return false;
234 
235 	/* recalibrate sample period and check if it's accepted by perf core */
236 	if (perf_event_period(pmc->perf_event,
237 			      get_sample_period(pmc, pmc->counter)))
238 		return false;
239 
240 	if (test_bit(pmc->idx, (unsigned long *)&pmc_to_pmu(pmc)->pebs_enable) !=
241 	    (!!pmc->perf_event->attr.precise_ip))
242 		return false;
243 
244 	/* reuse perf_event to serve as pmc_reprogram_counter() does*/
245 	perf_event_enable(pmc->perf_event);
246 	pmc->is_paused = false;
247 
248 	clear_bit(pmc->idx, (unsigned long *)&pmc_to_pmu(pmc)->reprogram_pmi);
249 	return true;
250 }
251 
cmp_u64(const void * pa,const void * pb)252 static int cmp_u64(const void *pa, const void *pb)
253 {
254 	u64 a = *(u64 *)pa;
255 	u64 b = *(u64 *)pb;
256 
257 	return (a > b) - (a < b);
258 }
259 
check_pmu_event_filter(struct kvm_pmc * pmc)260 static bool check_pmu_event_filter(struct kvm_pmc *pmc)
261 {
262 	struct kvm_pmu_event_filter *filter;
263 	struct kvm *kvm = pmc->vcpu->kvm;
264 	bool allow_event = true;
265 	__u64 key;
266 	int idx;
267 
268 	if (!static_call(kvm_x86_pmu_hw_event_available)(pmc))
269 		return false;
270 
271 	filter = srcu_dereference(kvm->arch.pmu_event_filter, &kvm->srcu);
272 	if (!filter)
273 		goto out;
274 
275 	if (pmc_is_gp(pmc)) {
276 		key = pmc->eventsel & AMD64_RAW_EVENT_MASK_NB;
277 		if (bsearch(&key, filter->events, filter->nevents,
278 			    sizeof(__u64), cmp_u64))
279 			allow_event = filter->action == KVM_PMU_EVENT_ALLOW;
280 		else
281 			allow_event = filter->action == KVM_PMU_EVENT_DENY;
282 	} else {
283 		idx = pmc->idx - INTEL_PMC_IDX_FIXED;
284 		if (filter->action == KVM_PMU_EVENT_DENY &&
285 		    test_bit(idx, (ulong *)&filter->fixed_counter_bitmap))
286 			allow_event = false;
287 		if (filter->action == KVM_PMU_EVENT_ALLOW &&
288 		    !test_bit(idx, (ulong *)&filter->fixed_counter_bitmap))
289 			allow_event = false;
290 	}
291 
292 out:
293 	return allow_event;
294 }
295 
reprogram_counter(struct kvm_pmc * pmc)296 void reprogram_counter(struct kvm_pmc *pmc)
297 {
298 	struct kvm_pmu *pmu = pmc_to_pmu(pmc);
299 	u64 eventsel = pmc->eventsel;
300 	u64 new_config = eventsel;
301 	u8 fixed_ctr_ctrl;
302 
303 	pmc_pause_counter(pmc);
304 
305 	if (!pmc_speculative_in_use(pmc) || !pmc_is_enabled(pmc))
306 		return;
307 
308 	if (!check_pmu_event_filter(pmc))
309 		return;
310 
311 	if (eventsel & ARCH_PERFMON_EVENTSEL_PIN_CONTROL)
312 		printk_once("kvm pmu: pin control bit is ignored\n");
313 
314 	if (pmc_is_fixed(pmc)) {
315 		fixed_ctr_ctrl = fixed_ctrl_field(pmu->fixed_ctr_ctrl,
316 						  pmc->idx - INTEL_PMC_IDX_FIXED);
317 		if (fixed_ctr_ctrl & 0x1)
318 			eventsel |= ARCH_PERFMON_EVENTSEL_OS;
319 		if (fixed_ctr_ctrl & 0x2)
320 			eventsel |= ARCH_PERFMON_EVENTSEL_USR;
321 		if (fixed_ctr_ctrl & 0x8)
322 			eventsel |= ARCH_PERFMON_EVENTSEL_INT;
323 		new_config = (u64)fixed_ctr_ctrl;
324 	}
325 
326 	if (pmc->current_config == new_config && pmc_resume_counter(pmc))
327 		return;
328 
329 	pmc_release_perf_event(pmc);
330 
331 	pmc->current_config = new_config;
332 	pmc_reprogram_counter(pmc, PERF_TYPE_RAW,
333 			      (eventsel & pmu->raw_event_mask),
334 			      !(eventsel & ARCH_PERFMON_EVENTSEL_USR),
335 			      !(eventsel & ARCH_PERFMON_EVENTSEL_OS),
336 			      eventsel & ARCH_PERFMON_EVENTSEL_INT);
337 }
338 EXPORT_SYMBOL_GPL(reprogram_counter);
339 
kvm_pmu_handle_event(struct kvm_vcpu * vcpu)340 void kvm_pmu_handle_event(struct kvm_vcpu *vcpu)
341 {
342 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
343 	int bit;
344 
345 	for_each_set_bit(bit, pmu->reprogram_pmi, X86_PMC_IDX_MAX) {
346 		struct kvm_pmc *pmc = static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, bit);
347 
348 		if (unlikely(!pmc || !pmc->perf_event)) {
349 			clear_bit(bit, pmu->reprogram_pmi);
350 			continue;
351 		}
352 		reprogram_counter(pmc);
353 	}
354 
355 	/*
356 	 * Unused perf_events are only released if the corresponding MSRs
357 	 * weren't accessed during the last vCPU time slice. kvm_arch_sched_in
358 	 * triggers KVM_REQ_PMU if cleanup is needed.
359 	 */
360 	if (unlikely(pmu->need_cleanup))
361 		kvm_pmu_cleanup(vcpu);
362 }
363 
364 /* check if idx is a valid index to access PMU */
kvm_pmu_is_valid_rdpmc_ecx(struct kvm_vcpu * vcpu,unsigned int idx)365 bool kvm_pmu_is_valid_rdpmc_ecx(struct kvm_vcpu *vcpu, unsigned int idx)
366 {
367 	return static_call(kvm_x86_pmu_is_valid_rdpmc_ecx)(vcpu, idx);
368 }
369 
is_vmware_backdoor_pmc(u32 pmc_idx)370 bool is_vmware_backdoor_pmc(u32 pmc_idx)
371 {
372 	switch (pmc_idx) {
373 	case VMWARE_BACKDOOR_PMC_HOST_TSC:
374 	case VMWARE_BACKDOOR_PMC_REAL_TIME:
375 	case VMWARE_BACKDOOR_PMC_APPARENT_TIME:
376 		return true;
377 	}
378 	return false;
379 }
380 
kvm_pmu_rdpmc_vmware(struct kvm_vcpu * vcpu,unsigned idx,u64 * data)381 static int kvm_pmu_rdpmc_vmware(struct kvm_vcpu *vcpu, unsigned idx, u64 *data)
382 {
383 	u64 ctr_val;
384 
385 	switch (idx) {
386 	case VMWARE_BACKDOOR_PMC_HOST_TSC:
387 		ctr_val = rdtsc();
388 		break;
389 	case VMWARE_BACKDOOR_PMC_REAL_TIME:
390 		ctr_val = ktime_get_boottime_ns();
391 		break;
392 	case VMWARE_BACKDOOR_PMC_APPARENT_TIME:
393 		ctr_val = ktime_get_boottime_ns() +
394 			vcpu->kvm->arch.kvmclock_offset;
395 		break;
396 	default:
397 		return 1;
398 	}
399 
400 	*data = ctr_val;
401 	return 0;
402 }
403 
kvm_pmu_rdpmc(struct kvm_vcpu * vcpu,unsigned idx,u64 * data)404 int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data)
405 {
406 	bool fast_mode = idx & (1u << 31);
407 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
408 	struct kvm_pmc *pmc;
409 	u64 mask = fast_mode ? ~0u : ~0ull;
410 
411 	if (!pmu->version)
412 		return 1;
413 
414 	if (is_vmware_backdoor_pmc(idx))
415 		return kvm_pmu_rdpmc_vmware(vcpu, idx, data);
416 
417 	pmc = static_call(kvm_x86_pmu_rdpmc_ecx_to_pmc)(vcpu, idx, &mask);
418 	if (!pmc)
419 		return 1;
420 
421 	if (!(kvm_read_cr4(vcpu) & X86_CR4_PCE) &&
422 	    (static_call(kvm_x86_get_cpl)(vcpu) != 0) &&
423 	    (kvm_read_cr0(vcpu) & X86_CR0_PE))
424 		return 1;
425 
426 	*data = pmc_read_counter(pmc) & mask;
427 	return 0;
428 }
429 
kvm_pmu_deliver_pmi(struct kvm_vcpu * vcpu)430 void kvm_pmu_deliver_pmi(struct kvm_vcpu *vcpu)
431 {
432 	if (lapic_in_kernel(vcpu)) {
433 		static_call_cond(kvm_x86_pmu_deliver_pmi)(vcpu);
434 		kvm_apic_local_deliver(vcpu->arch.apic, APIC_LVTPC);
435 	}
436 }
437 
kvm_pmu_is_valid_msr(struct kvm_vcpu * vcpu,u32 msr)438 bool kvm_pmu_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr)
439 {
440 	return static_call(kvm_x86_pmu_msr_idx_to_pmc)(vcpu, msr) ||
441 		static_call(kvm_x86_pmu_is_valid_msr)(vcpu, msr);
442 }
443 
kvm_pmu_mark_pmc_in_use(struct kvm_vcpu * vcpu,u32 msr)444 static void kvm_pmu_mark_pmc_in_use(struct kvm_vcpu *vcpu, u32 msr)
445 {
446 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
447 	struct kvm_pmc *pmc = static_call(kvm_x86_pmu_msr_idx_to_pmc)(vcpu, msr);
448 
449 	if (pmc)
450 		__set_bit(pmc->idx, pmu->pmc_in_use);
451 }
452 
kvm_pmu_get_msr(struct kvm_vcpu * vcpu,struct msr_data * msr_info)453 int kvm_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
454 {
455 	return static_call(kvm_x86_pmu_get_msr)(vcpu, msr_info);
456 }
457 
kvm_pmu_set_msr(struct kvm_vcpu * vcpu,struct msr_data * msr_info)458 int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
459 {
460 	kvm_pmu_mark_pmc_in_use(vcpu, msr_info->index);
461 	return static_call(kvm_x86_pmu_set_msr)(vcpu, msr_info);
462 }
463 
464 /* refresh PMU settings. This function generally is called when underlying
465  * settings are changed (such as changes of PMU CPUID by guest VMs), which
466  * should rarely happen.
467  */
kvm_pmu_refresh(struct kvm_vcpu * vcpu)468 void kvm_pmu_refresh(struct kvm_vcpu *vcpu)
469 {
470 	static_call(kvm_x86_pmu_refresh)(vcpu);
471 }
472 
kvm_pmu_reset(struct kvm_vcpu * vcpu)473 void kvm_pmu_reset(struct kvm_vcpu *vcpu)
474 {
475 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
476 
477 	irq_work_sync(&pmu->irq_work);
478 	static_call(kvm_x86_pmu_reset)(vcpu);
479 }
480 
kvm_pmu_init(struct kvm_vcpu * vcpu)481 void kvm_pmu_init(struct kvm_vcpu *vcpu)
482 {
483 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
484 
485 	memset(pmu, 0, sizeof(*pmu));
486 	static_call(kvm_x86_pmu_init)(vcpu);
487 	init_irq_work(&pmu->irq_work, kvm_pmi_trigger_fn);
488 	pmu->event_count = 0;
489 	pmu->need_cleanup = false;
490 	kvm_pmu_refresh(vcpu);
491 }
492 
493 /* Release perf_events for vPMCs that have been unused for a full time slice.  */
kvm_pmu_cleanup(struct kvm_vcpu * vcpu)494 void kvm_pmu_cleanup(struct kvm_vcpu *vcpu)
495 {
496 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
497 	struct kvm_pmc *pmc = NULL;
498 	DECLARE_BITMAP(bitmask, X86_PMC_IDX_MAX);
499 	int i;
500 
501 	pmu->need_cleanup = false;
502 
503 	bitmap_andnot(bitmask, pmu->all_valid_pmc_idx,
504 		      pmu->pmc_in_use, X86_PMC_IDX_MAX);
505 
506 	for_each_set_bit(i, bitmask, X86_PMC_IDX_MAX) {
507 		pmc = static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, i);
508 
509 		if (pmc && pmc->perf_event && !pmc_speculative_in_use(pmc))
510 			pmc_stop_counter(pmc);
511 	}
512 
513 	static_call_cond(kvm_x86_pmu_cleanup)(vcpu);
514 
515 	bitmap_zero(pmu->pmc_in_use, X86_PMC_IDX_MAX);
516 }
517 
kvm_pmu_destroy(struct kvm_vcpu * vcpu)518 void kvm_pmu_destroy(struct kvm_vcpu *vcpu)
519 {
520 	kvm_pmu_reset(vcpu);
521 }
522 
kvm_pmu_incr_counter(struct kvm_pmc * pmc)523 static void kvm_pmu_incr_counter(struct kvm_pmc *pmc)
524 {
525 	u64 prev_count;
526 
527 	prev_count = pmc->counter;
528 	pmc->counter = (pmc->counter + 1) & pmc_bitmask(pmc);
529 
530 	reprogram_counter(pmc);
531 	if (pmc->counter < prev_count)
532 		__kvm_perf_overflow(pmc, false);
533 }
534 
eventsel_match_perf_hw_id(struct kvm_pmc * pmc,unsigned int perf_hw_id)535 static inline bool eventsel_match_perf_hw_id(struct kvm_pmc *pmc,
536 	unsigned int perf_hw_id)
537 {
538 	return !((pmc->eventsel ^ perf_get_hw_event_config(perf_hw_id)) &
539 		AMD64_RAW_EVENT_MASK_NB);
540 }
541 
cpl_is_matched(struct kvm_pmc * pmc)542 static inline bool cpl_is_matched(struct kvm_pmc *pmc)
543 {
544 	bool select_os, select_user;
545 	u64 config = pmc->current_config;
546 
547 	if (pmc_is_gp(pmc)) {
548 		select_os = config & ARCH_PERFMON_EVENTSEL_OS;
549 		select_user = config & ARCH_PERFMON_EVENTSEL_USR;
550 	} else {
551 		select_os = config & 0x1;
552 		select_user = config & 0x2;
553 	}
554 
555 	return (static_call(kvm_x86_get_cpl)(pmc->vcpu) == 0) ? select_os : select_user;
556 }
557 
kvm_pmu_trigger_event(struct kvm_vcpu * vcpu,u64 perf_hw_id)558 void kvm_pmu_trigger_event(struct kvm_vcpu *vcpu, u64 perf_hw_id)
559 {
560 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
561 	struct kvm_pmc *pmc;
562 	int i;
563 
564 	for_each_set_bit(i, pmu->all_valid_pmc_idx, X86_PMC_IDX_MAX) {
565 		pmc = static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, i);
566 
567 		if (!pmc || !pmc_is_enabled(pmc) || !pmc_speculative_in_use(pmc))
568 			continue;
569 
570 		/* Ignore checks for edge detect, pin control, invert and CMASK bits */
571 		if (eventsel_match_perf_hw_id(pmc, perf_hw_id) && cpl_is_matched(pmc))
572 			kvm_pmu_incr_counter(pmc);
573 	}
574 }
575 EXPORT_SYMBOL_GPL(kvm_pmu_trigger_event);
576 
kvm_vm_ioctl_set_pmu_event_filter(struct kvm * kvm,void __user * argp)577 int kvm_vm_ioctl_set_pmu_event_filter(struct kvm *kvm, void __user *argp)
578 {
579 	struct kvm_pmu_event_filter tmp, *filter;
580 	size_t size;
581 	int r;
582 
583 	if (copy_from_user(&tmp, argp, sizeof(tmp)))
584 		return -EFAULT;
585 
586 	if (tmp.action != KVM_PMU_EVENT_ALLOW &&
587 	    tmp.action != KVM_PMU_EVENT_DENY)
588 		return -EINVAL;
589 
590 	if (tmp.flags != 0)
591 		return -EINVAL;
592 
593 	if (tmp.nevents > KVM_PMU_EVENT_FILTER_MAX_EVENTS)
594 		return -E2BIG;
595 
596 	size = struct_size(filter, events, tmp.nevents);
597 	filter = kmalloc(size, GFP_KERNEL_ACCOUNT);
598 	if (!filter)
599 		return -ENOMEM;
600 
601 	r = -EFAULT;
602 	if (copy_from_user(filter, argp, size))
603 		goto cleanup;
604 
605 	/* Ensure nevents can't be changed between the user copies. */
606 	*filter = tmp;
607 
608 	/*
609 	 * Sort the in-kernel list so that we can search it with bsearch.
610 	 */
611 	sort(&filter->events, filter->nevents, sizeof(__u64), cmp_u64, NULL);
612 
613 	mutex_lock(&kvm->lock);
614 	filter = rcu_replace_pointer(kvm->arch.pmu_event_filter, filter,
615 				     mutex_is_locked(&kvm->lock));
616 	mutex_unlock(&kvm->lock);
617 
618 	synchronize_srcu_expedited(&kvm->srcu);
619 	r = 0;
620 cleanup:
621 	kfree(filter);
622 	return r;
623 }
624