1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Kernel-based Virtual Machine -- Performance Monitoring Unit support
4  *
5  * Copyright 2015 Red Hat, Inc. and/or its affiliates.
6  *
7  * Authors:
8  *   Avi Kivity   <avi@redhat.com>
9  *   Gleb Natapov <gleb@redhat.com>
10  *   Wei Huang    <wei@redhat.com>
11  */
12 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
13 
14 #include <linux/types.h>
15 #include <linux/kvm_host.h>
16 #include <linux/perf_event.h>
17 #include <linux/bsearch.h>
18 #include <linux/sort.h>
19 #include <asm/perf_event.h>
20 #include <asm/cpu_device_id.h>
21 #include "x86.h"
22 #include "cpuid.h"
23 #include "lapic.h"
24 #include "pmu.h"
25 
26 /* This is enough to filter the vast majority of currently defined events. */
27 #define KVM_PMU_EVENT_FILTER_MAX_EVENTS 300
28 
29 struct x86_pmu_capability __read_mostly kvm_pmu_cap;
30 EXPORT_SYMBOL_GPL(kvm_pmu_cap);
31 
32 /* Precise Distribution of Instructions Retired (PDIR) */
33 static const struct x86_cpu_id vmx_pebs_pdir_cpu[] = {
34 	X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_D, NULL),
35 	X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_X, NULL),
36 	/* Instruction-Accurate PDIR (PDIR++) */
37 	X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, NULL),
38 	{}
39 };
40 
41 /* Precise Distribution (PDist) */
42 static const struct x86_cpu_id vmx_pebs_pdist_cpu[] = {
43 	X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, NULL),
44 	{}
45 };
46 
47 /* NOTE:
48  * - Each perf counter is defined as "struct kvm_pmc";
49  * - There are two types of perf counters: general purpose (gp) and fixed.
50  *   gp counters are stored in gp_counters[] and fixed counters are stored
51  *   in fixed_counters[] respectively. Both of them are part of "struct
52  *   kvm_pmu";
53  * - pmu.c understands the difference between gp counters and fixed counters.
54  *   However AMD doesn't support fixed-counters;
55  * - There are three types of index to access perf counters (PMC):
56  *     1. MSR (named msr): For example Intel has MSR_IA32_PERFCTRn and AMD
57  *        has MSR_K7_PERFCTRn and, for families 15H and later,
58  *        MSR_F15H_PERF_CTRn, where MSR_F15H_PERF_CTR[0-3] are
59  *        aliased to MSR_K7_PERFCTRn.
60  *     2. MSR Index (named idx): This normally is used by RDPMC instruction.
61  *        For instance AMD RDPMC instruction uses 0000_0003h in ECX to access
62  *        C001_0007h (MSR_K7_PERCTR3). Intel has a similar mechanism, except
63  *        that it also supports fixed counters. idx can be used to as index to
64  *        gp and fixed counters.
65  *     3. Global PMC Index (named pmc): pmc is an index specific to PMU
66  *        code. Each pmc, stored in kvm_pmc.idx field, is unique across
67  *        all perf counters (both gp and fixed). The mapping relationship
68  *        between pmc and perf counters is as the following:
69  *        * Intel: [0 .. KVM_INTEL_PMC_MAX_GENERIC-1] <=> gp counters
70  *                 [INTEL_PMC_IDX_FIXED .. INTEL_PMC_IDX_FIXED + 2] <=> fixed
71  *        * AMD:   [0 .. AMD64_NUM_COUNTERS-1] and, for families 15H
72  *          and later, [0 .. AMD64_NUM_COUNTERS_CORE-1] <=> gp counters
73  */
74 
75 static struct kvm_pmu_ops kvm_pmu_ops __read_mostly;
76 
77 #define KVM_X86_PMU_OP(func)					     \
78 	DEFINE_STATIC_CALL_NULL(kvm_x86_pmu_##func,			     \
79 				*(((struct kvm_pmu_ops *)0)->func));
80 #define KVM_X86_PMU_OP_OPTIONAL KVM_X86_PMU_OP
81 #include <asm/kvm-x86-pmu-ops.h>
82 
kvm_pmu_ops_update(const struct kvm_pmu_ops * pmu_ops)83 void kvm_pmu_ops_update(const struct kvm_pmu_ops *pmu_ops)
84 {
85 	memcpy(&kvm_pmu_ops, pmu_ops, sizeof(kvm_pmu_ops));
86 
87 #define __KVM_X86_PMU_OP(func) \
88 	static_call_update(kvm_x86_pmu_##func, kvm_pmu_ops.func);
89 #define KVM_X86_PMU_OP(func) \
90 	WARN_ON(!kvm_pmu_ops.func); __KVM_X86_PMU_OP(func)
91 #define KVM_X86_PMU_OP_OPTIONAL __KVM_X86_PMU_OP
92 #include <asm/kvm-x86-pmu-ops.h>
93 #undef __KVM_X86_PMU_OP
94 }
95 
__kvm_perf_overflow(struct kvm_pmc * pmc,bool in_pmi)96 static inline void __kvm_perf_overflow(struct kvm_pmc *pmc, bool in_pmi)
97 {
98 	struct kvm_pmu *pmu = pmc_to_pmu(pmc);
99 	bool skip_pmi = false;
100 
101 	if (pmc->perf_event && pmc->perf_event->attr.precise_ip) {
102 		if (!in_pmi) {
103 			/*
104 			 * TODO: KVM is currently _choosing_ to not generate records
105 			 * for emulated instructions, avoiding BUFFER_OVF PMI when
106 			 * there are no records. Strictly speaking, it should be done
107 			 * as well in the right context to improve sampling accuracy.
108 			 */
109 			skip_pmi = true;
110 		} else {
111 			/* Indicate PEBS overflow PMI to guest. */
112 			skip_pmi = __test_and_set_bit(GLOBAL_STATUS_BUFFER_OVF_BIT,
113 						      (unsigned long *)&pmu->global_status);
114 		}
115 	} else {
116 		__set_bit(pmc->idx, (unsigned long *)&pmu->global_status);
117 	}
118 
119 	if (pmc->intr && !skip_pmi)
120 		kvm_make_request(KVM_REQ_PMI, pmc->vcpu);
121 }
122 
kvm_perf_overflow(struct perf_event * perf_event,struct perf_sample_data * data,struct pt_regs * regs)123 static void kvm_perf_overflow(struct perf_event *perf_event,
124 			      struct perf_sample_data *data,
125 			      struct pt_regs *regs)
126 {
127 	struct kvm_pmc *pmc = perf_event->overflow_handler_context;
128 
129 	/*
130 	 * Ignore overflow events for counters that are scheduled to be
131 	 * reprogrammed, e.g. if a PMI for the previous event races with KVM's
132 	 * handling of a related guest WRMSR.
133 	 */
134 	if (test_and_set_bit(pmc->idx, pmc_to_pmu(pmc)->reprogram_pmi))
135 		return;
136 
137 	__kvm_perf_overflow(pmc, true);
138 
139 	kvm_make_request(KVM_REQ_PMU, pmc->vcpu);
140 }
141 
pmc_get_pebs_precise_level(struct kvm_pmc * pmc)142 static u64 pmc_get_pebs_precise_level(struct kvm_pmc *pmc)
143 {
144 	/*
145 	 * For some model specific pebs counters with special capabilities
146 	 * (PDIR, PDIR++, PDIST), KVM needs to raise the event precise
147 	 * level to the maximum value (currently 3, backwards compatible)
148 	 * so that the perf subsystem would assign specific hardware counter
149 	 * with that capability for vPMC.
150 	 */
151 	if ((pmc->idx == 0 && x86_match_cpu(vmx_pebs_pdist_cpu)) ||
152 	    (pmc->idx == 32 && x86_match_cpu(vmx_pebs_pdir_cpu)))
153 		return 3;
154 
155 	/*
156 	 * The non-zero precision level of guest event makes the ordinary
157 	 * guest event becomes a guest PEBS event and triggers the host
158 	 * PEBS PMI handler to determine whether the PEBS overflow PMI
159 	 * comes from the host counters or the guest.
160 	 */
161 	return 1;
162 }
163 
pmc_reprogram_counter(struct kvm_pmc * pmc,u32 type,u64 config,bool exclude_user,bool exclude_kernel,bool intr)164 static int pmc_reprogram_counter(struct kvm_pmc *pmc, u32 type, u64 config,
165 				 bool exclude_user, bool exclude_kernel,
166 				 bool intr)
167 {
168 	struct kvm_pmu *pmu = pmc_to_pmu(pmc);
169 	struct perf_event *event;
170 	struct perf_event_attr attr = {
171 		.type = type,
172 		.size = sizeof(attr),
173 		.pinned = true,
174 		.exclude_idle = true,
175 		.exclude_host = 1,
176 		.exclude_user = exclude_user,
177 		.exclude_kernel = exclude_kernel,
178 		.config = config,
179 	};
180 	bool pebs = test_bit(pmc->idx, (unsigned long *)&pmu->pebs_enable);
181 
182 	attr.sample_period = get_sample_period(pmc, pmc->counter);
183 
184 	if ((attr.config & HSW_IN_TX_CHECKPOINTED) &&
185 	    guest_cpuid_is_intel(pmc->vcpu)) {
186 		/*
187 		 * HSW_IN_TX_CHECKPOINTED is not supported with nonzero
188 		 * period. Just clear the sample period so at least
189 		 * allocating the counter doesn't fail.
190 		 */
191 		attr.sample_period = 0;
192 	}
193 	if (pebs) {
194 		/*
195 		 * For most PEBS hardware events, the difference in the software
196 		 * precision levels of guest and host PEBS events will not affect
197 		 * the accuracy of the PEBS profiling result, because the "event IP"
198 		 * in the PEBS record is calibrated on the guest side.
199 		 */
200 		attr.precise_ip = pmc_get_pebs_precise_level(pmc);
201 	}
202 
203 	event = perf_event_create_kernel_counter(&attr, -1, current,
204 						 kvm_perf_overflow, pmc);
205 	if (IS_ERR(event)) {
206 		pr_debug_ratelimited("kvm_pmu: event creation failed %ld for pmc->idx = %d\n",
207 			    PTR_ERR(event), pmc->idx);
208 		return PTR_ERR(event);
209 	}
210 
211 	pmc->perf_event = event;
212 	pmc_to_pmu(pmc)->event_count++;
213 	pmc->is_paused = false;
214 	pmc->intr = intr || pebs;
215 	return 0;
216 }
217 
pmc_pause_counter(struct kvm_pmc * pmc)218 static void pmc_pause_counter(struct kvm_pmc *pmc)
219 {
220 	u64 counter = pmc->counter;
221 
222 	if (!pmc->perf_event || pmc->is_paused)
223 		return;
224 
225 	/* update counter, reset event value to avoid redundant accumulation */
226 	counter += perf_event_pause(pmc->perf_event, true);
227 	pmc->counter = counter & pmc_bitmask(pmc);
228 	pmc->is_paused = true;
229 }
230 
pmc_resume_counter(struct kvm_pmc * pmc)231 static bool pmc_resume_counter(struct kvm_pmc *pmc)
232 {
233 	if (!pmc->perf_event)
234 		return false;
235 
236 	/* recalibrate sample period and check if it's accepted by perf core */
237 	if (is_sampling_event(pmc->perf_event) &&
238 	    perf_event_period(pmc->perf_event,
239 			      get_sample_period(pmc, pmc->counter)))
240 		return false;
241 
242 	if (test_bit(pmc->idx, (unsigned long *)&pmc_to_pmu(pmc)->pebs_enable) !=
243 	    (!!pmc->perf_event->attr.precise_ip))
244 		return false;
245 
246 	/* reuse perf_event to serve as pmc_reprogram_counter() does*/
247 	perf_event_enable(pmc->perf_event);
248 	pmc->is_paused = false;
249 
250 	return true;
251 }
252 
pmc_release_perf_event(struct kvm_pmc * pmc)253 static void pmc_release_perf_event(struct kvm_pmc *pmc)
254 {
255 	if (pmc->perf_event) {
256 		perf_event_release_kernel(pmc->perf_event);
257 		pmc->perf_event = NULL;
258 		pmc->current_config = 0;
259 		pmc_to_pmu(pmc)->event_count--;
260 	}
261 }
262 
pmc_stop_counter(struct kvm_pmc * pmc)263 static void pmc_stop_counter(struct kvm_pmc *pmc)
264 {
265 	if (pmc->perf_event) {
266 		pmc->counter = pmc_read_counter(pmc);
267 		pmc_release_perf_event(pmc);
268 	}
269 }
270 
filter_cmp(const void * pa,const void * pb,u64 mask)271 static int filter_cmp(const void *pa, const void *pb, u64 mask)
272 {
273 	u64 a = *(u64 *)pa & mask;
274 	u64 b = *(u64 *)pb & mask;
275 
276 	return (a > b) - (a < b);
277 }
278 
279 
filter_sort_cmp(const void * pa,const void * pb)280 static int filter_sort_cmp(const void *pa, const void *pb)
281 {
282 	return filter_cmp(pa, pb, (KVM_PMU_MASKED_ENTRY_EVENT_SELECT |
283 				   KVM_PMU_MASKED_ENTRY_EXCLUDE));
284 }
285 
286 /*
287  * For the event filter, searching is done on the 'includes' list and
288  * 'excludes' list separately rather than on the 'events' list (which
289  * has both).  As a result the exclude bit can be ignored.
290  */
filter_event_cmp(const void * pa,const void * pb)291 static int filter_event_cmp(const void *pa, const void *pb)
292 {
293 	return filter_cmp(pa, pb, (KVM_PMU_MASKED_ENTRY_EVENT_SELECT));
294 }
295 
find_filter_index(u64 * events,u64 nevents,u64 key)296 static int find_filter_index(u64 *events, u64 nevents, u64 key)
297 {
298 	u64 *fe = bsearch(&key, events, nevents, sizeof(events[0]),
299 			  filter_event_cmp);
300 
301 	if (!fe)
302 		return -1;
303 
304 	return fe - events;
305 }
306 
is_filter_entry_match(u64 filter_event,u64 umask)307 static bool is_filter_entry_match(u64 filter_event, u64 umask)
308 {
309 	u64 mask = filter_event >> (KVM_PMU_MASKED_ENTRY_UMASK_MASK_SHIFT - 8);
310 	u64 match = filter_event & KVM_PMU_MASKED_ENTRY_UMASK_MATCH;
311 
312 	BUILD_BUG_ON((KVM_PMU_ENCODE_MASKED_ENTRY(0, 0xff, 0, false) >>
313 		     (KVM_PMU_MASKED_ENTRY_UMASK_MASK_SHIFT - 8)) !=
314 		     ARCH_PERFMON_EVENTSEL_UMASK);
315 
316 	return (umask & mask) == match;
317 }
318 
filter_contains_match(u64 * events,u64 nevents,u64 eventsel)319 static bool filter_contains_match(u64 *events, u64 nevents, u64 eventsel)
320 {
321 	u64 event_select = eventsel & kvm_pmu_ops.EVENTSEL_EVENT;
322 	u64 umask = eventsel & ARCH_PERFMON_EVENTSEL_UMASK;
323 	int i, index;
324 
325 	index = find_filter_index(events, nevents, event_select);
326 	if (index < 0)
327 		return false;
328 
329 	/*
330 	 * Entries are sorted by the event select.  Walk the list in both
331 	 * directions to process all entries with the targeted event select.
332 	 */
333 	for (i = index; i < nevents; i++) {
334 		if (filter_event_cmp(&events[i], &event_select))
335 			break;
336 
337 		if (is_filter_entry_match(events[i], umask))
338 			return true;
339 	}
340 
341 	for (i = index - 1; i >= 0; i--) {
342 		if (filter_event_cmp(&events[i], &event_select))
343 			break;
344 
345 		if (is_filter_entry_match(events[i], umask))
346 			return true;
347 	}
348 
349 	return false;
350 }
351 
is_gp_event_allowed(struct kvm_x86_pmu_event_filter * f,u64 eventsel)352 static bool is_gp_event_allowed(struct kvm_x86_pmu_event_filter *f,
353 				u64 eventsel)
354 {
355 	if (filter_contains_match(f->includes, f->nr_includes, eventsel) &&
356 	    !filter_contains_match(f->excludes, f->nr_excludes, eventsel))
357 		return f->action == KVM_PMU_EVENT_ALLOW;
358 
359 	return f->action == KVM_PMU_EVENT_DENY;
360 }
361 
is_fixed_event_allowed(struct kvm_x86_pmu_event_filter * filter,int idx)362 static bool is_fixed_event_allowed(struct kvm_x86_pmu_event_filter *filter,
363 				   int idx)
364 {
365 	int fixed_idx = idx - INTEL_PMC_IDX_FIXED;
366 
367 	if (filter->action == KVM_PMU_EVENT_DENY &&
368 	    test_bit(fixed_idx, (ulong *)&filter->fixed_counter_bitmap))
369 		return false;
370 	if (filter->action == KVM_PMU_EVENT_ALLOW &&
371 	    !test_bit(fixed_idx, (ulong *)&filter->fixed_counter_bitmap))
372 		return false;
373 
374 	return true;
375 }
376 
check_pmu_event_filter(struct kvm_pmc * pmc)377 static bool check_pmu_event_filter(struct kvm_pmc *pmc)
378 {
379 	struct kvm_x86_pmu_event_filter *filter;
380 	struct kvm *kvm = pmc->vcpu->kvm;
381 
382 	filter = srcu_dereference(kvm->arch.pmu_event_filter, &kvm->srcu);
383 	if (!filter)
384 		return true;
385 
386 	if (pmc_is_gp(pmc))
387 		return is_gp_event_allowed(filter, pmc->eventsel);
388 
389 	return is_fixed_event_allowed(filter, pmc->idx);
390 }
391 
pmc_event_is_allowed(struct kvm_pmc * pmc)392 static bool pmc_event_is_allowed(struct kvm_pmc *pmc)
393 {
394 	return pmc_is_globally_enabled(pmc) && pmc_speculative_in_use(pmc) &&
395 	       static_call(kvm_x86_pmu_hw_event_available)(pmc) &&
396 	       check_pmu_event_filter(pmc);
397 }
398 
reprogram_counter(struct kvm_pmc * pmc)399 static void reprogram_counter(struct kvm_pmc *pmc)
400 {
401 	struct kvm_pmu *pmu = pmc_to_pmu(pmc);
402 	u64 eventsel = pmc->eventsel;
403 	u64 new_config = eventsel;
404 	u8 fixed_ctr_ctrl;
405 
406 	pmc_pause_counter(pmc);
407 
408 	if (!pmc_event_is_allowed(pmc))
409 		goto reprogram_complete;
410 
411 	if (pmc->counter < pmc->prev_counter)
412 		__kvm_perf_overflow(pmc, false);
413 
414 	if (eventsel & ARCH_PERFMON_EVENTSEL_PIN_CONTROL)
415 		printk_once("kvm pmu: pin control bit is ignored\n");
416 
417 	if (pmc_is_fixed(pmc)) {
418 		fixed_ctr_ctrl = fixed_ctrl_field(pmu->fixed_ctr_ctrl,
419 						  pmc->idx - INTEL_PMC_IDX_FIXED);
420 		if (fixed_ctr_ctrl & 0x1)
421 			eventsel |= ARCH_PERFMON_EVENTSEL_OS;
422 		if (fixed_ctr_ctrl & 0x2)
423 			eventsel |= ARCH_PERFMON_EVENTSEL_USR;
424 		if (fixed_ctr_ctrl & 0x8)
425 			eventsel |= ARCH_PERFMON_EVENTSEL_INT;
426 		new_config = (u64)fixed_ctr_ctrl;
427 	}
428 
429 	if (pmc->current_config == new_config && pmc_resume_counter(pmc))
430 		goto reprogram_complete;
431 
432 	pmc_release_perf_event(pmc);
433 
434 	pmc->current_config = new_config;
435 
436 	/*
437 	 * If reprogramming fails, e.g. due to contention, leave the counter's
438 	 * regprogram bit set, i.e. opportunistically try again on the next PMU
439 	 * refresh.  Don't make a new request as doing so can stall the guest
440 	 * if reprogramming repeatedly fails.
441 	 */
442 	if (pmc_reprogram_counter(pmc, PERF_TYPE_RAW,
443 				  (eventsel & pmu->raw_event_mask),
444 				  !(eventsel & ARCH_PERFMON_EVENTSEL_USR),
445 				  !(eventsel & ARCH_PERFMON_EVENTSEL_OS),
446 				  eventsel & ARCH_PERFMON_EVENTSEL_INT))
447 		return;
448 
449 reprogram_complete:
450 	clear_bit(pmc->idx, (unsigned long *)&pmc_to_pmu(pmc)->reprogram_pmi);
451 	pmc->prev_counter = 0;
452 }
453 
kvm_pmu_handle_event(struct kvm_vcpu * vcpu)454 void kvm_pmu_handle_event(struct kvm_vcpu *vcpu)
455 {
456 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
457 	int bit;
458 
459 	for_each_set_bit(bit, pmu->reprogram_pmi, X86_PMC_IDX_MAX) {
460 		struct kvm_pmc *pmc = static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, bit);
461 
462 		if (unlikely(!pmc)) {
463 			clear_bit(bit, pmu->reprogram_pmi);
464 			continue;
465 		}
466 
467 		reprogram_counter(pmc);
468 	}
469 
470 	/*
471 	 * Unused perf_events are only released if the corresponding MSRs
472 	 * weren't accessed during the last vCPU time slice. kvm_arch_sched_in
473 	 * triggers KVM_REQ_PMU if cleanup is needed.
474 	 */
475 	if (unlikely(pmu->need_cleanup))
476 		kvm_pmu_cleanup(vcpu);
477 }
478 
479 /* check if idx is a valid index to access PMU */
kvm_pmu_is_valid_rdpmc_ecx(struct kvm_vcpu * vcpu,unsigned int idx)480 bool kvm_pmu_is_valid_rdpmc_ecx(struct kvm_vcpu *vcpu, unsigned int idx)
481 {
482 	return static_call(kvm_x86_pmu_is_valid_rdpmc_ecx)(vcpu, idx);
483 }
484 
is_vmware_backdoor_pmc(u32 pmc_idx)485 bool is_vmware_backdoor_pmc(u32 pmc_idx)
486 {
487 	switch (pmc_idx) {
488 	case VMWARE_BACKDOOR_PMC_HOST_TSC:
489 	case VMWARE_BACKDOOR_PMC_REAL_TIME:
490 	case VMWARE_BACKDOOR_PMC_APPARENT_TIME:
491 		return true;
492 	}
493 	return false;
494 }
495 
kvm_pmu_rdpmc_vmware(struct kvm_vcpu * vcpu,unsigned idx,u64 * data)496 static int kvm_pmu_rdpmc_vmware(struct kvm_vcpu *vcpu, unsigned idx, u64 *data)
497 {
498 	u64 ctr_val;
499 
500 	switch (idx) {
501 	case VMWARE_BACKDOOR_PMC_HOST_TSC:
502 		ctr_val = rdtsc();
503 		break;
504 	case VMWARE_BACKDOOR_PMC_REAL_TIME:
505 		ctr_val = ktime_get_boottime_ns();
506 		break;
507 	case VMWARE_BACKDOOR_PMC_APPARENT_TIME:
508 		ctr_val = ktime_get_boottime_ns() +
509 			vcpu->kvm->arch.kvmclock_offset;
510 		break;
511 	default:
512 		return 1;
513 	}
514 
515 	*data = ctr_val;
516 	return 0;
517 }
518 
kvm_pmu_rdpmc(struct kvm_vcpu * vcpu,unsigned idx,u64 * data)519 int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data)
520 {
521 	bool fast_mode = idx & (1u << 31);
522 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
523 	struct kvm_pmc *pmc;
524 	u64 mask = fast_mode ? ~0u : ~0ull;
525 
526 	if (!pmu->version)
527 		return 1;
528 
529 	if (is_vmware_backdoor_pmc(idx))
530 		return kvm_pmu_rdpmc_vmware(vcpu, idx, data);
531 
532 	pmc = static_call(kvm_x86_pmu_rdpmc_ecx_to_pmc)(vcpu, idx, &mask);
533 	if (!pmc)
534 		return 1;
535 
536 	if (!kvm_is_cr4_bit_set(vcpu, X86_CR4_PCE) &&
537 	    (static_call(kvm_x86_get_cpl)(vcpu) != 0) &&
538 	    kvm_is_cr0_bit_set(vcpu, X86_CR0_PE))
539 		return 1;
540 
541 	*data = pmc_read_counter(pmc) & mask;
542 	return 0;
543 }
544 
kvm_pmu_deliver_pmi(struct kvm_vcpu * vcpu)545 void kvm_pmu_deliver_pmi(struct kvm_vcpu *vcpu)
546 {
547 	if (lapic_in_kernel(vcpu)) {
548 		static_call_cond(kvm_x86_pmu_deliver_pmi)(vcpu);
549 		kvm_apic_local_deliver(vcpu->arch.apic, APIC_LVTPC);
550 	}
551 }
552 
kvm_pmu_is_valid_msr(struct kvm_vcpu * vcpu,u32 msr)553 bool kvm_pmu_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr)
554 {
555 	switch (msr) {
556 	case MSR_CORE_PERF_GLOBAL_STATUS:
557 	case MSR_CORE_PERF_GLOBAL_CTRL:
558 	case MSR_CORE_PERF_GLOBAL_OVF_CTRL:
559 		return kvm_pmu_has_perf_global_ctrl(vcpu_to_pmu(vcpu));
560 	default:
561 		break;
562 	}
563 	return static_call(kvm_x86_pmu_msr_idx_to_pmc)(vcpu, msr) ||
564 		static_call(kvm_x86_pmu_is_valid_msr)(vcpu, msr);
565 }
566 
kvm_pmu_mark_pmc_in_use(struct kvm_vcpu * vcpu,u32 msr)567 static void kvm_pmu_mark_pmc_in_use(struct kvm_vcpu *vcpu, u32 msr)
568 {
569 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
570 	struct kvm_pmc *pmc = static_call(kvm_x86_pmu_msr_idx_to_pmc)(vcpu, msr);
571 
572 	if (pmc)
573 		__set_bit(pmc->idx, pmu->pmc_in_use);
574 }
575 
kvm_pmu_get_msr(struct kvm_vcpu * vcpu,struct msr_data * msr_info)576 int kvm_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
577 {
578 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
579 	u32 msr = msr_info->index;
580 
581 	switch (msr) {
582 	case MSR_CORE_PERF_GLOBAL_STATUS:
583 	case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS:
584 		msr_info->data = pmu->global_status;
585 		break;
586 	case MSR_AMD64_PERF_CNTR_GLOBAL_CTL:
587 	case MSR_CORE_PERF_GLOBAL_CTRL:
588 		msr_info->data = pmu->global_ctrl;
589 		break;
590 	case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR:
591 	case MSR_CORE_PERF_GLOBAL_OVF_CTRL:
592 		msr_info->data = 0;
593 		break;
594 	default:
595 		return static_call(kvm_x86_pmu_get_msr)(vcpu, msr_info);
596 	}
597 
598 	return 0;
599 }
600 
kvm_pmu_set_msr(struct kvm_vcpu * vcpu,struct msr_data * msr_info)601 int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
602 {
603 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
604 	u32 msr = msr_info->index;
605 	u64 data = msr_info->data;
606 	u64 diff;
607 
608 	/*
609 	 * Note, AMD ignores writes to reserved bits and read-only PMU MSRs,
610 	 * whereas Intel generates #GP on attempts to write reserved/RO MSRs.
611 	 */
612 	switch (msr) {
613 	case MSR_CORE_PERF_GLOBAL_STATUS:
614 		if (!msr_info->host_initiated)
615 			return 1; /* RO MSR */
616 		fallthrough;
617 	case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS:
618 		/* Per PPR, Read-only MSR. Writes are ignored. */
619 		if (!msr_info->host_initiated)
620 			break;
621 
622 		if (data & pmu->global_status_mask)
623 			return 1;
624 
625 		pmu->global_status = data;
626 		break;
627 	case MSR_AMD64_PERF_CNTR_GLOBAL_CTL:
628 		data &= ~pmu->global_ctrl_mask;
629 		fallthrough;
630 	case MSR_CORE_PERF_GLOBAL_CTRL:
631 		if (!kvm_valid_perf_global_ctrl(pmu, data))
632 			return 1;
633 
634 		if (pmu->global_ctrl != data) {
635 			diff = pmu->global_ctrl ^ data;
636 			pmu->global_ctrl = data;
637 			reprogram_counters(pmu, diff);
638 		}
639 		break;
640 	case MSR_CORE_PERF_GLOBAL_OVF_CTRL:
641 		/*
642 		 * GLOBAL_OVF_CTRL, a.k.a. GLOBAL STATUS_RESET, clears bits in
643 		 * GLOBAL_STATUS, and so the set of reserved bits is the same.
644 		 */
645 		if (data & pmu->global_status_mask)
646 			return 1;
647 		fallthrough;
648 	case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR:
649 		if (!msr_info->host_initiated)
650 			pmu->global_status &= ~data;
651 		break;
652 	default:
653 		kvm_pmu_mark_pmc_in_use(vcpu, msr_info->index);
654 		return static_call(kvm_x86_pmu_set_msr)(vcpu, msr_info);
655 	}
656 
657 	return 0;
658 }
659 
kvm_pmu_reset(struct kvm_vcpu * vcpu)660 void kvm_pmu_reset(struct kvm_vcpu *vcpu)
661 {
662 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
663 	struct kvm_pmc *pmc;
664 	int i;
665 
666 	pmu->need_cleanup = false;
667 
668 	bitmap_zero(pmu->reprogram_pmi, X86_PMC_IDX_MAX);
669 
670 	for_each_set_bit(i, pmu->all_valid_pmc_idx, X86_PMC_IDX_MAX) {
671 		pmc = static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, i);
672 		if (!pmc)
673 			continue;
674 
675 		pmc_stop_counter(pmc);
676 		pmc->counter = 0;
677 
678 		if (pmc_is_gp(pmc))
679 			pmc->eventsel = 0;
680 	}
681 
682 	pmu->fixed_ctr_ctrl = pmu->global_ctrl = pmu->global_status = 0;
683 
684 	static_call_cond(kvm_x86_pmu_reset)(vcpu);
685 }
686 
687 
688 /*
689  * Refresh the PMU configuration for the vCPU, e.g. if userspace changes CPUID
690  * and/or PERF_CAPABILITIES.
691  */
kvm_pmu_refresh(struct kvm_vcpu * vcpu)692 void kvm_pmu_refresh(struct kvm_vcpu *vcpu)
693 {
694 	if (KVM_BUG_ON(kvm_vcpu_has_run(vcpu), vcpu->kvm))
695 		return;
696 
697 	/*
698 	 * Stop/release all existing counters/events before realizing the new
699 	 * vPMU model.
700 	 */
701 	kvm_pmu_reset(vcpu);
702 
703 	bitmap_zero(vcpu_to_pmu(vcpu)->all_valid_pmc_idx, X86_PMC_IDX_MAX);
704 	static_call(kvm_x86_pmu_refresh)(vcpu);
705 }
706 
kvm_pmu_init(struct kvm_vcpu * vcpu)707 void kvm_pmu_init(struct kvm_vcpu *vcpu)
708 {
709 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
710 
711 	memset(pmu, 0, sizeof(*pmu));
712 	static_call(kvm_x86_pmu_init)(vcpu);
713 	pmu->event_count = 0;
714 	pmu->need_cleanup = false;
715 	kvm_pmu_refresh(vcpu);
716 }
717 
718 /* Release perf_events for vPMCs that have been unused for a full time slice.  */
kvm_pmu_cleanup(struct kvm_vcpu * vcpu)719 void kvm_pmu_cleanup(struct kvm_vcpu *vcpu)
720 {
721 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
722 	struct kvm_pmc *pmc = NULL;
723 	DECLARE_BITMAP(bitmask, X86_PMC_IDX_MAX);
724 	int i;
725 
726 	pmu->need_cleanup = false;
727 
728 	bitmap_andnot(bitmask, pmu->all_valid_pmc_idx,
729 		      pmu->pmc_in_use, X86_PMC_IDX_MAX);
730 
731 	for_each_set_bit(i, bitmask, X86_PMC_IDX_MAX) {
732 		pmc = static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, i);
733 
734 		if (pmc && pmc->perf_event && !pmc_speculative_in_use(pmc))
735 			pmc_stop_counter(pmc);
736 	}
737 
738 	static_call_cond(kvm_x86_pmu_cleanup)(vcpu);
739 
740 	bitmap_zero(pmu->pmc_in_use, X86_PMC_IDX_MAX);
741 }
742 
kvm_pmu_destroy(struct kvm_vcpu * vcpu)743 void kvm_pmu_destroy(struct kvm_vcpu *vcpu)
744 {
745 	kvm_pmu_reset(vcpu);
746 }
747 
kvm_pmu_incr_counter(struct kvm_pmc * pmc)748 static void kvm_pmu_incr_counter(struct kvm_pmc *pmc)
749 {
750 	pmc->prev_counter = pmc->counter;
751 	pmc->counter = (pmc->counter + 1) & pmc_bitmask(pmc);
752 	kvm_pmu_request_counter_reprogram(pmc);
753 }
754 
eventsel_match_perf_hw_id(struct kvm_pmc * pmc,unsigned int perf_hw_id)755 static inline bool eventsel_match_perf_hw_id(struct kvm_pmc *pmc,
756 	unsigned int perf_hw_id)
757 {
758 	return !((pmc->eventsel ^ perf_get_hw_event_config(perf_hw_id)) &
759 		AMD64_RAW_EVENT_MASK_NB);
760 }
761 
cpl_is_matched(struct kvm_pmc * pmc)762 static inline bool cpl_is_matched(struct kvm_pmc *pmc)
763 {
764 	bool select_os, select_user;
765 	u64 config;
766 
767 	if (pmc_is_gp(pmc)) {
768 		config = pmc->eventsel;
769 		select_os = config & ARCH_PERFMON_EVENTSEL_OS;
770 		select_user = config & ARCH_PERFMON_EVENTSEL_USR;
771 	} else {
772 		config = fixed_ctrl_field(pmc_to_pmu(pmc)->fixed_ctr_ctrl,
773 					  pmc->idx - INTEL_PMC_IDX_FIXED);
774 		select_os = config & 0x1;
775 		select_user = config & 0x2;
776 	}
777 
778 	return (static_call(kvm_x86_get_cpl)(pmc->vcpu) == 0) ? select_os : select_user;
779 }
780 
kvm_pmu_trigger_event(struct kvm_vcpu * vcpu,u64 perf_hw_id)781 void kvm_pmu_trigger_event(struct kvm_vcpu *vcpu, u64 perf_hw_id)
782 {
783 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
784 	struct kvm_pmc *pmc;
785 	int i;
786 
787 	for_each_set_bit(i, pmu->all_valid_pmc_idx, X86_PMC_IDX_MAX) {
788 		pmc = static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, i);
789 
790 		if (!pmc || !pmc_event_is_allowed(pmc))
791 			continue;
792 
793 		/* Ignore checks for edge detect, pin control, invert and CMASK bits */
794 		if (eventsel_match_perf_hw_id(pmc, perf_hw_id) && cpl_is_matched(pmc))
795 			kvm_pmu_incr_counter(pmc);
796 	}
797 }
798 EXPORT_SYMBOL_GPL(kvm_pmu_trigger_event);
799 
is_masked_filter_valid(const struct kvm_x86_pmu_event_filter * filter)800 static bool is_masked_filter_valid(const struct kvm_x86_pmu_event_filter *filter)
801 {
802 	u64 mask = kvm_pmu_ops.EVENTSEL_EVENT |
803 		   KVM_PMU_MASKED_ENTRY_UMASK_MASK |
804 		   KVM_PMU_MASKED_ENTRY_UMASK_MATCH |
805 		   KVM_PMU_MASKED_ENTRY_EXCLUDE;
806 	int i;
807 
808 	for (i = 0; i < filter->nevents; i++) {
809 		if (filter->events[i] & ~mask)
810 			return false;
811 	}
812 
813 	return true;
814 }
815 
convert_to_masked_filter(struct kvm_x86_pmu_event_filter * filter)816 static void convert_to_masked_filter(struct kvm_x86_pmu_event_filter *filter)
817 {
818 	int i, j;
819 
820 	for (i = 0, j = 0; i < filter->nevents; i++) {
821 		/*
822 		 * Skip events that are impossible to match against a guest
823 		 * event.  When filtering, only the event select + unit mask
824 		 * of the guest event is used.  To maintain backwards
825 		 * compatibility, impossible filters can't be rejected :-(
826 		 */
827 		if (filter->events[i] & ~(kvm_pmu_ops.EVENTSEL_EVENT |
828 					  ARCH_PERFMON_EVENTSEL_UMASK))
829 			continue;
830 		/*
831 		 * Convert userspace events to a common in-kernel event so
832 		 * only one code path is needed to support both events.  For
833 		 * the in-kernel events use masked events because they are
834 		 * flexible enough to handle both cases.  To convert to masked
835 		 * events all that's needed is to add an "all ones" umask_mask,
836 		 * (unmasked filter events don't support EXCLUDE).
837 		 */
838 		filter->events[j++] = filter->events[i] |
839 				      (0xFFULL << KVM_PMU_MASKED_ENTRY_UMASK_MASK_SHIFT);
840 	}
841 
842 	filter->nevents = j;
843 }
844 
prepare_filter_lists(struct kvm_x86_pmu_event_filter * filter)845 static int prepare_filter_lists(struct kvm_x86_pmu_event_filter *filter)
846 {
847 	int i;
848 
849 	if (!(filter->flags & KVM_PMU_EVENT_FLAG_MASKED_EVENTS))
850 		convert_to_masked_filter(filter);
851 	else if (!is_masked_filter_valid(filter))
852 		return -EINVAL;
853 
854 	/*
855 	 * Sort entries by event select and includes vs. excludes so that all
856 	 * entries for a given event select can be processed efficiently during
857 	 * filtering.  The EXCLUDE flag uses a more significant bit than the
858 	 * event select, and so the sorted list is also effectively split into
859 	 * includes and excludes sub-lists.
860 	 */
861 	sort(&filter->events, filter->nevents, sizeof(filter->events[0]),
862 	     filter_sort_cmp, NULL);
863 
864 	i = filter->nevents;
865 	/* Find the first EXCLUDE event (only supported for masked events). */
866 	if (filter->flags & KVM_PMU_EVENT_FLAG_MASKED_EVENTS) {
867 		for (i = 0; i < filter->nevents; i++) {
868 			if (filter->events[i] & KVM_PMU_MASKED_ENTRY_EXCLUDE)
869 				break;
870 		}
871 	}
872 
873 	filter->nr_includes = i;
874 	filter->nr_excludes = filter->nevents - filter->nr_includes;
875 	filter->includes = filter->events;
876 	filter->excludes = filter->events + filter->nr_includes;
877 
878 	return 0;
879 }
880 
kvm_vm_ioctl_set_pmu_event_filter(struct kvm * kvm,void __user * argp)881 int kvm_vm_ioctl_set_pmu_event_filter(struct kvm *kvm, void __user *argp)
882 {
883 	struct kvm_pmu_event_filter __user *user_filter = argp;
884 	struct kvm_x86_pmu_event_filter *filter;
885 	struct kvm_pmu_event_filter tmp;
886 	struct kvm_vcpu *vcpu;
887 	unsigned long i;
888 	size_t size;
889 	int r;
890 
891 	if (copy_from_user(&tmp, user_filter, sizeof(tmp)))
892 		return -EFAULT;
893 
894 	if (tmp.action != KVM_PMU_EVENT_ALLOW &&
895 	    tmp.action != KVM_PMU_EVENT_DENY)
896 		return -EINVAL;
897 
898 	if (tmp.flags & ~KVM_PMU_EVENT_FLAGS_VALID_MASK)
899 		return -EINVAL;
900 
901 	if (tmp.nevents > KVM_PMU_EVENT_FILTER_MAX_EVENTS)
902 		return -E2BIG;
903 
904 	size = struct_size(filter, events, tmp.nevents);
905 	filter = kzalloc(size, GFP_KERNEL_ACCOUNT);
906 	if (!filter)
907 		return -ENOMEM;
908 
909 	filter->action = tmp.action;
910 	filter->nevents = tmp.nevents;
911 	filter->fixed_counter_bitmap = tmp.fixed_counter_bitmap;
912 	filter->flags = tmp.flags;
913 
914 	r = -EFAULT;
915 	if (copy_from_user(filter->events, user_filter->events,
916 			   sizeof(filter->events[0]) * filter->nevents))
917 		goto cleanup;
918 
919 	r = prepare_filter_lists(filter);
920 	if (r)
921 		goto cleanup;
922 
923 	mutex_lock(&kvm->lock);
924 	filter = rcu_replace_pointer(kvm->arch.pmu_event_filter, filter,
925 				     mutex_is_locked(&kvm->lock));
926 	mutex_unlock(&kvm->lock);
927 	synchronize_srcu_expedited(&kvm->srcu);
928 
929 	BUILD_BUG_ON(sizeof(((struct kvm_pmu *)0)->reprogram_pmi) >
930 		     sizeof(((struct kvm_pmu *)0)->__reprogram_pmi));
931 
932 	kvm_for_each_vcpu(i, vcpu, kvm)
933 		atomic64_set(&vcpu_to_pmu(vcpu)->__reprogram_pmi, -1ull);
934 
935 	kvm_make_all_cpus_request(kvm, KVM_REQ_PMU);
936 
937 	r = 0;
938 cleanup:
939 	kfree(filter);
940 	return r;
941 }
942