1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2012 ARM Ltd.
4  * Author: Marc Zyngier <marc.zyngier@arm.com>
5  */
6 
7 #include <linux/cpu.h>
8 #include <linux/kvm.h>
9 #include <linux/kvm_host.h>
10 #include <linux/interrupt.h>
11 #include <linux/irq.h>
12 #include <linux/irqdomain.h>
13 #include <linux/uaccess.h>
14 
15 #include <clocksource/arm_arch_timer.h>
16 #include <asm/arch_timer.h>
17 #include <asm/kvm_emulate.h>
18 #include <asm/kvm_hyp.h>
19 #include <asm/kvm_nested.h>
20 
21 #include <kvm/arm_vgic.h>
22 #include <kvm/arm_arch_timer.h>
23 
24 #include "trace.h"
25 
26 static struct timecounter *timecounter;
27 static unsigned int host_vtimer_irq;
28 static unsigned int host_ptimer_irq;
29 static u32 host_vtimer_irq_flags;
30 static u32 host_ptimer_irq_flags;
31 
32 static DEFINE_STATIC_KEY_FALSE(has_gic_active_state);
33 
34 static const u8 default_ppi[] = {
35 	[TIMER_PTIMER]  = 30,
36 	[TIMER_VTIMER]  = 27,
37 	[TIMER_HPTIMER] = 26,
38 	[TIMER_HVTIMER] = 28,
39 };
40 
41 static bool kvm_timer_irq_can_fire(struct arch_timer_context *timer_ctx);
42 static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level,
43 				 struct arch_timer_context *timer_ctx);
44 static bool kvm_timer_should_fire(struct arch_timer_context *timer_ctx);
45 static void kvm_arm_timer_write(struct kvm_vcpu *vcpu,
46 				struct arch_timer_context *timer,
47 				enum kvm_arch_timer_regs treg,
48 				u64 val);
49 static u64 kvm_arm_timer_read(struct kvm_vcpu *vcpu,
50 			      struct arch_timer_context *timer,
51 			      enum kvm_arch_timer_regs treg);
52 static bool kvm_arch_timer_get_input_level(int vintid);
53 
54 static struct irq_ops arch_timer_irq_ops = {
55 	.get_input_level = kvm_arch_timer_get_input_level,
56 };
57 
nr_timers(struct kvm_vcpu * vcpu)58 static int nr_timers(struct kvm_vcpu *vcpu)
59 {
60 	if (!vcpu_has_nv(vcpu))
61 		return NR_KVM_EL0_TIMERS;
62 
63 	return NR_KVM_TIMERS;
64 }
65 
timer_get_ctl(struct arch_timer_context * ctxt)66 u32 timer_get_ctl(struct arch_timer_context *ctxt)
67 {
68 	struct kvm_vcpu *vcpu = ctxt->vcpu;
69 
70 	switch(arch_timer_ctx_index(ctxt)) {
71 	case TIMER_VTIMER:
72 		return __vcpu_sys_reg(vcpu, CNTV_CTL_EL0);
73 	case TIMER_PTIMER:
74 		return __vcpu_sys_reg(vcpu, CNTP_CTL_EL0);
75 	case TIMER_HVTIMER:
76 		return __vcpu_sys_reg(vcpu, CNTHV_CTL_EL2);
77 	case TIMER_HPTIMER:
78 		return __vcpu_sys_reg(vcpu, CNTHP_CTL_EL2);
79 	default:
80 		WARN_ON(1);
81 		return 0;
82 	}
83 }
84 
timer_get_cval(struct arch_timer_context * ctxt)85 u64 timer_get_cval(struct arch_timer_context *ctxt)
86 {
87 	struct kvm_vcpu *vcpu = ctxt->vcpu;
88 
89 	switch(arch_timer_ctx_index(ctxt)) {
90 	case TIMER_VTIMER:
91 		return __vcpu_sys_reg(vcpu, CNTV_CVAL_EL0);
92 	case TIMER_PTIMER:
93 		return __vcpu_sys_reg(vcpu, CNTP_CVAL_EL0);
94 	case TIMER_HVTIMER:
95 		return __vcpu_sys_reg(vcpu, CNTHV_CVAL_EL2);
96 	case TIMER_HPTIMER:
97 		return __vcpu_sys_reg(vcpu, CNTHP_CVAL_EL2);
98 	default:
99 		WARN_ON(1);
100 		return 0;
101 	}
102 }
103 
timer_get_offset(struct arch_timer_context * ctxt)104 static u64 timer_get_offset(struct arch_timer_context *ctxt)
105 {
106 	u64 offset = 0;
107 
108 	if (!ctxt)
109 		return 0;
110 
111 	if (ctxt->offset.vm_offset)
112 		offset += *ctxt->offset.vm_offset;
113 	if (ctxt->offset.vcpu_offset)
114 		offset += *ctxt->offset.vcpu_offset;
115 
116 	return offset;
117 }
118 
timer_set_ctl(struct arch_timer_context * ctxt,u32 ctl)119 static void timer_set_ctl(struct arch_timer_context *ctxt, u32 ctl)
120 {
121 	struct kvm_vcpu *vcpu = ctxt->vcpu;
122 
123 	switch(arch_timer_ctx_index(ctxt)) {
124 	case TIMER_VTIMER:
125 		__vcpu_sys_reg(vcpu, CNTV_CTL_EL0) = ctl;
126 		break;
127 	case TIMER_PTIMER:
128 		__vcpu_sys_reg(vcpu, CNTP_CTL_EL0) = ctl;
129 		break;
130 	case TIMER_HVTIMER:
131 		__vcpu_sys_reg(vcpu, CNTHV_CTL_EL2) = ctl;
132 		break;
133 	case TIMER_HPTIMER:
134 		__vcpu_sys_reg(vcpu, CNTHP_CTL_EL2) = ctl;
135 		break;
136 	default:
137 		WARN_ON(1);
138 	}
139 }
140 
timer_set_cval(struct arch_timer_context * ctxt,u64 cval)141 static void timer_set_cval(struct arch_timer_context *ctxt, u64 cval)
142 {
143 	struct kvm_vcpu *vcpu = ctxt->vcpu;
144 
145 	switch(arch_timer_ctx_index(ctxt)) {
146 	case TIMER_VTIMER:
147 		__vcpu_sys_reg(vcpu, CNTV_CVAL_EL0) = cval;
148 		break;
149 	case TIMER_PTIMER:
150 		__vcpu_sys_reg(vcpu, CNTP_CVAL_EL0) = cval;
151 		break;
152 	case TIMER_HVTIMER:
153 		__vcpu_sys_reg(vcpu, CNTHV_CVAL_EL2) = cval;
154 		break;
155 	case TIMER_HPTIMER:
156 		__vcpu_sys_reg(vcpu, CNTHP_CVAL_EL2) = cval;
157 		break;
158 	default:
159 		WARN_ON(1);
160 	}
161 }
162 
timer_set_offset(struct arch_timer_context * ctxt,u64 offset)163 static void timer_set_offset(struct arch_timer_context *ctxt, u64 offset)
164 {
165 	if (!ctxt->offset.vm_offset) {
166 		WARN(offset, "timer %ld\n", arch_timer_ctx_index(ctxt));
167 		return;
168 	}
169 
170 	WRITE_ONCE(*ctxt->offset.vm_offset, offset);
171 }
172 
kvm_phys_timer_read(void)173 u64 kvm_phys_timer_read(void)
174 {
175 	return timecounter->cc->read(timecounter->cc);
176 }
177 
get_timer_map(struct kvm_vcpu * vcpu,struct timer_map * map)178 void get_timer_map(struct kvm_vcpu *vcpu, struct timer_map *map)
179 {
180 	if (vcpu_has_nv(vcpu)) {
181 		if (is_hyp_ctxt(vcpu)) {
182 			map->direct_vtimer = vcpu_hvtimer(vcpu);
183 			map->direct_ptimer = vcpu_hptimer(vcpu);
184 			map->emul_vtimer = vcpu_vtimer(vcpu);
185 			map->emul_ptimer = vcpu_ptimer(vcpu);
186 		} else {
187 			map->direct_vtimer = vcpu_vtimer(vcpu);
188 			map->direct_ptimer = vcpu_ptimer(vcpu);
189 			map->emul_vtimer = vcpu_hvtimer(vcpu);
190 			map->emul_ptimer = vcpu_hptimer(vcpu);
191 		}
192 	} else if (has_vhe()) {
193 		map->direct_vtimer = vcpu_vtimer(vcpu);
194 		map->direct_ptimer = vcpu_ptimer(vcpu);
195 		map->emul_vtimer = NULL;
196 		map->emul_ptimer = NULL;
197 	} else {
198 		map->direct_vtimer = vcpu_vtimer(vcpu);
199 		map->direct_ptimer = NULL;
200 		map->emul_vtimer = NULL;
201 		map->emul_ptimer = vcpu_ptimer(vcpu);
202 	}
203 
204 	trace_kvm_get_timer_map(vcpu->vcpu_id, map);
205 }
206 
userspace_irqchip(struct kvm * kvm)207 static inline bool userspace_irqchip(struct kvm *kvm)
208 {
209 	return static_branch_unlikely(&userspace_irqchip_in_use) &&
210 		unlikely(!irqchip_in_kernel(kvm));
211 }
212 
soft_timer_start(struct hrtimer * hrt,u64 ns)213 static void soft_timer_start(struct hrtimer *hrt, u64 ns)
214 {
215 	hrtimer_start(hrt, ktime_add_ns(ktime_get(), ns),
216 		      HRTIMER_MODE_ABS_HARD);
217 }
218 
soft_timer_cancel(struct hrtimer * hrt)219 static void soft_timer_cancel(struct hrtimer *hrt)
220 {
221 	hrtimer_cancel(hrt);
222 }
223 
kvm_arch_timer_handler(int irq,void * dev_id)224 static irqreturn_t kvm_arch_timer_handler(int irq, void *dev_id)
225 {
226 	struct kvm_vcpu *vcpu = *(struct kvm_vcpu **)dev_id;
227 	struct arch_timer_context *ctx;
228 	struct timer_map map;
229 
230 	/*
231 	 * We may see a timer interrupt after vcpu_put() has been called which
232 	 * sets the CPU's vcpu pointer to NULL, because even though the timer
233 	 * has been disabled in timer_save_state(), the hardware interrupt
234 	 * signal may not have been retired from the interrupt controller yet.
235 	 */
236 	if (!vcpu)
237 		return IRQ_HANDLED;
238 
239 	get_timer_map(vcpu, &map);
240 
241 	if (irq == host_vtimer_irq)
242 		ctx = map.direct_vtimer;
243 	else
244 		ctx = map.direct_ptimer;
245 
246 	if (kvm_timer_should_fire(ctx))
247 		kvm_timer_update_irq(vcpu, true, ctx);
248 
249 	if (userspace_irqchip(vcpu->kvm) &&
250 	    !static_branch_unlikely(&has_gic_active_state))
251 		disable_percpu_irq(host_vtimer_irq);
252 
253 	return IRQ_HANDLED;
254 }
255 
kvm_counter_compute_delta(struct arch_timer_context * timer_ctx,u64 val)256 static u64 kvm_counter_compute_delta(struct arch_timer_context *timer_ctx,
257 				     u64 val)
258 {
259 	u64 now = kvm_phys_timer_read() - timer_get_offset(timer_ctx);
260 
261 	if (now < val) {
262 		u64 ns;
263 
264 		ns = cyclecounter_cyc2ns(timecounter->cc,
265 					 val - now,
266 					 timecounter->mask,
267 					 &timer_ctx->ns_frac);
268 		return ns;
269 	}
270 
271 	return 0;
272 }
273 
kvm_timer_compute_delta(struct arch_timer_context * timer_ctx)274 static u64 kvm_timer_compute_delta(struct arch_timer_context *timer_ctx)
275 {
276 	return kvm_counter_compute_delta(timer_ctx, timer_get_cval(timer_ctx));
277 }
278 
kvm_timer_irq_can_fire(struct arch_timer_context * timer_ctx)279 static bool kvm_timer_irq_can_fire(struct arch_timer_context *timer_ctx)
280 {
281 	WARN_ON(timer_ctx && timer_ctx->loaded);
282 	return timer_ctx &&
283 		((timer_get_ctl(timer_ctx) &
284 		  (ARCH_TIMER_CTRL_IT_MASK | ARCH_TIMER_CTRL_ENABLE)) == ARCH_TIMER_CTRL_ENABLE);
285 }
286 
vcpu_has_wfit_active(struct kvm_vcpu * vcpu)287 static bool vcpu_has_wfit_active(struct kvm_vcpu *vcpu)
288 {
289 	return (cpus_have_final_cap(ARM64_HAS_WFXT) &&
290 		vcpu_get_flag(vcpu, IN_WFIT));
291 }
292 
wfit_delay_ns(struct kvm_vcpu * vcpu)293 static u64 wfit_delay_ns(struct kvm_vcpu *vcpu)
294 {
295 	u64 val = vcpu_get_reg(vcpu, kvm_vcpu_sys_get_rt(vcpu));
296 	struct arch_timer_context *ctx;
297 
298 	ctx = (vcpu_has_nv(vcpu) && is_hyp_ctxt(vcpu)) ? vcpu_hvtimer(vcpu)
299 						       : vcpu_vtimer(vcpu);
300 
301 	return kvm_counter_compute_delta(ctx, val);
302 }
303 
304 /*
305  * Returns the earliest expiration time in ns among guest timers.
306  * Note that it will return 0 if none of timers can fire.
307  */
kvm_timer_earliest_exp(struct kvm_vcpu * vcpu)308 static u64 kvm_timer_earliest_exp(struct kvm_vcpu *vcpu)
309 {
310 	u64 min_delta = ULLONG_MAX;
311 	int i;
312 
313 	for (i = 0; i < nr_timers(vcpu); i++) {
314 		struct arch_timer_context *ctx = &vcpu->arch.timer_cpu.timers[i];
315 
316 		WARN(ctx->loaded, "timer %d loaded\n", i);
317 		if (kvm_timer_irq_can_fire(ctx))
318 			min_delta = min(min_delta, kvm_timer_compute_delta(ctx));
319 	}
320 
321 	if (vcpu_has_wfit_active(vcpu))
322 		min_delta = min(min_delta, wfit_delay_ns(vcpu));
323 
324 	/* If none of timers can fire, then return 0 */
325 	if (min_delta == ULLONG_MAX)
326 		return 0;
327 
328 	return min_delta;
329 }
330 
kvm_bg_timer_expire(struct hrtimer * hrt)331 static enum hrtimer_restart kvm_bg_timer_expire(struct hrtimer *hrt)
332 {
333 	struct arch_timer_cpu *timer;
334 	struct kvm_vcpu *vcpu;
335 	u64 ns;
336 
337 	timer = container_of(hrt, struct arch_timer_cpu, bg_timer);
338 	vcpu = container_of(timer, struct kvm_vcpu, arch.timer_cpu);
339 
340 	/*
341 	 * Check that the timer has really expired from the guest's
342 	 * PoV (NTP on the host may have forced it to expire
343 	 * early). If we should have slept longer, restart it.
344 	 */
345 	ns = kvm_timer_earliest_exp(vcpu);
346 	if (unlikely(ns)) {
347 		hrtimer_forward_now(hrt, ns_to_ktime(ns));
348 		return HRTIMER_RESTART;
349 	}
350 
351 	kvm_vcpu_wake_up(vcpu);
352 	return HRTIMER_NORESTART;
353 }
354 
kvm_hrtimer_expire(struct hrtimer * hrt)355 static enum hrtimer_restart kvm_hrtimer_expire(struct hrtimer *hrt)
356 {
357 	struct arch_timer_context *ctx;
358 	struct kvm_vcpu *vcpu;
359 	u64 ns;
360 
361 	ctx = container_of(hrt, struct arch_timer_context, hrtimer);
362 	vcpu = ctx->vcpu;
363 
364 	trace_kvm_timer_hrtimer_expire(ctx);
365 
366 	/*
367 	 * Check that the timer has really expired from the guest's
368 	 * PoV (NTP on the host may have forced it to expire
369 	 * early). If not ready, schedule for a later time.
370 	 */
371 	ns = kvm_timer_compute_delta(ctx);
372 	if (unlikely(ns)) {
373 		hrtimer_forward_now(hrt, ns_to_ktime(ns));
374 		return HRTIMER_RESTART;
375 	}
376 
377 	kvm_timer_update_irq(vcpu, true, ctx);
378 	return HRTIMER_NORESTART;
379 }
380 
kvm_timer_should_fire(struct arch_timer_context * timer_ctx)381 static bool kvm_timer_should_fire(struct arch_timer_context *timer_ctx)
382 {
383 	enum kvm_arch_timers index;
384 	u64 cval, now;
385 
386 	if (!timer_ctx)
387 		return false;
388 
389 	index = arch_timer_ctx_index(timer_ctx);
390 
391 	if (timer_ctx->loaded) {
392 		u32 cnt_ctl = 0;
393 
394 		switch (index) {
395 		case TIMER_VTIMER:
396 		case TIMER_HVTIMER:
397 			cnt_ctl = read_sysreg_el0(SYS_CNTV_CTL);
398 			break;
399 		case TIMER_PTIMER:
400 		case TIMER_HPTIMER:
401 			cnt_ctl = read_sysreg_el0(SYS_CNTP_CTL);
402 			break;
403 		case NR_KVM_TIMERS:
404 			/* GCC is braindead */
405 			cnt_ctl = 0;
406 			break;
407 		}
408 
409 		return  (cnt_ctl & ARCH_TIMER_CTRL_ENABLE) &&
410 		        (cnt_ctl & ARCH_TIMER_CTRL_IT_STAT) &&
411 		       !(cnt_ctl & ARCH_TIMER_CTRL_IT_MASK);
412 	}
413 
414 	if (!kvm_timer_irq_can_fire(timer_ctx))
415 		return false;
416 
417 	cval = timer_get_cval(timer_ctx);
418 	now = kvm_phys_timer_read() - timer_get_offset(timer_ctx);
419 
420 	return cval <= now;
421 }
422 
kvm_cpu_has_pending_timer(struct kvm_vcpu * vcpu)423 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
424 {
425 	return vcpu_has_wfit_active(vcpu) && wfit_delay_ns(vcpu) == 0;
426 }
427 
428 /*
429  * Reflect the timer output level into the kvm_run structure
430  */
kvm_timer_update_run(struct kvm_vcpu * vcpu)431 void kvm_timer_update_run(struct kvm_vcpu *vcpu)
432 {
433 	struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
434 	struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
435 	struct kvm_sync_regs *regs = &vcpu->run->s.regs;
436 
437 	/* Populate the device bitmap with the timer states */
438 	regs->device_irq_level &= ~(KVM_ARM_DEV_EL1_VTIMER |
439 				    KVM_ARM_DEV_EL1_PTIMER);
440 	if (kvm_timer_should_fire(vtimer))
441 		regs->device_irq_level |= KVM_ARM_DEV_EL1_VTIMER;
442 	if (kvm_timer_should_fire(ptimer))
443 		regs->device_irq_level |= KVM_ARM_DEV_EL1_PTIMER;
444 }
445 
kvm_timer_update_irq(struct kvm_vcpu * vcpu,bool new_level,struct arch_timer_context * timer_ctx)446 static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level,
447 				 struct arch_timer_context *timer_ctx)
448 {
449 	int ret;
450 
451 	timer_ctx->irq.level = new_level;
452 	trace_kvm_timer_update_irq(vcpu->vcpu_id, timer_irq(timer_ctx),
453 				   timer_ctx->irq.level);
454 
455 	if (!userspace_irqchip(vcpu->kvm)) {
456 		ret = kvm_vgic_inject_irq(vcpu->kvm, vcpu->vcpu_id,
457 					  timer_irq(timer_ctx),
458 					  timer_ctx->irq.level,
459 					  timer_ctx);
460 		WARN_ON(ret);
461 	}
462 }
463 
464 /* Only called for a fully emulated timer */
timer_emulate(struct arch_timer_context * ctx)465 static void timer_emulate(struct arch_timer_context *ctx)
466 {
467 	bool should_fire = kvm_timer_should_fire(ctx);
468 
469 	trace_kvm_timer_emulate(ctx, should_fire);
470 
471 	if (should_fire != ctx->irq.level) {
472 		kvm_timer_update_irq(ctx->vcpu, should_fire, ctx);
473 		return;
474 	}
475 
476 	/*
477 	 * If the timer can fire now, we don't need to have a soft timer
478 	 * scheduled for the future.  If the timer cannot fire at all,
479 	 * then we also don't need a soft timer.
480 	 */
481 	if (should_fire || !kvm_timer_irq_can_fire(ctx))
482 		return;
483 
484 	soft_timer_start(&ctx->hrtimer, kvm_timer_compute_delta(ctx));
485 }
486 
set_cntvoff(u64 cntvoff)487 static void set_cntvoff(u64 cntvoff)
488 {
489 	kvm_call_hyp(__kvm_timer_set_cntvoff, cntvoff);
490 }
491 
set_cntpoff(u64 cntpoff)492 static void set_cntpoff(u64 cntpoff)
493 {
494 	if (has_cntpoff())
495 		write_sysreg_s(cntpoff, SYS_CNTPOFF_EL2);
496 }
497 
timer_save_state(struct arch_timer_context * ctx)498 static void timer_save_state(struct arch_timer_context *ctx)
499 {
500 	struct arch_timer_cpu *timer = vcpu_timer(ctx->vcpu);
501 	enum kvm_arch_timers index = arch_timer_ctx_index(ctx);
502 	unsigned long flags;
503 
504 	if (!timer->enabled)
505 		return;
506 
507 	local_irq_save(flags);
508 
509 	if (!ctx->loaded)
510 		goto out;
511 
512 	switch (index) {
513 		u64 cval;
514 
515 	case TIMER_VTIMER:
516 	case TIMER_HVTIMER:
517 		timer_set_ctl(ctx, read_sysreg_el0(SYS_CNTV_CTL));
518 		timer_set_cval(ctx, read_sysreg_el0(SYS_CNTV_CVAL));
519 
520 		/* Disable the timer */
521 		write_sysreg_el0(0, SYS_CNTV_CTL);
522 		isb();
523 
524 		/*
525 		 * The kernel may decide to run userspace after
526 		 * calling vcpu_put, so we reset cntvoff to 0 to
527 		 * ensure a consistent read between user accesses to
528 		 * the virtual counter and kernel access to the
529 		 * physical counter of non-VHE case.
530 		 *
531 		 * For VHE, the virtual counter uses a fixed virtual
532 		 * offset of zero, so no need to zero CNTVOFF_EL2
533 		 * register, but this is actually useful when switching
534 		 * between EL1/vEL2 with NV.
535 		 *
536 		 * Do it unconditionally, as this is either unavoidable
537 		 * or dirt cheap.
538 		 */
539 		set_cntvoff(0);
540 		break;
541 	case TIMER_PTIMER:
542 	case TIMER_HPTIMER:
543 		timer_set_ctl(ctx, read_sysreg_el0(SYS_CNTP_CTL));
544 		cval = read_sysreg_el0(SYS_CNTP_CVAL);
545 
546 		cval -= timer_get_offset(ctx);
547 
548 		timer_set_cval(ctx, cval);
549 
550 		/* Disable the timer */
551 		write_sysreg_el0(0, SYS_CNTP_CTL);
552 		isb();
553 
554 		set_cntpoff(0);
555 		break;
556 	case NR_KVM_TIMERS:
557 		BUG();
558 	}
559 
560 	trace_kvm_timer_save_state(ctx);
561 
562 	ctx->loaded = false;
563 out:
564 	local_irq_restore(flags);
565 }
566 
567 /*
568  * Schedule the background timer before calling kvm_vcpu_halt, so that this
569  * thread is removed from its waitqueue and made runnable when there's a timer
570  * interrupt to handle.
571  */
kvm_timer_blocking(struct kvm_vcpu * vcpu)572 static void kvm_timer_blocking(struct kvm_vcpu *vcpu)
573 {
574 	struct arch_timer_cpu *timer = vcpu_timer(vcpu);
575 	struct timer_map map;
576 
577 	get_timer_map(vcpu, &map);
578 
579 	/*
580 	 * If no timers are capable of raising interrupts (disabled or
581 	 * masked), then there's no more work for us to do.
582 	 */
583 	if (!kvm_timer_irq_can_fire(map.direct_vtimer) &&
584 	    !kvm_timer_irq_can_fire(map.direct_ptimer) &&
585 	    !kvm_timer_irq_can_fire(map.emul_vtimer) &&
586 	    !kvm_timer_irq_can_fire(map.emul_ptimer) &&
587 	    !vcpu_has_wfit_active(vcpu))
588 		return;
589 
590 	/*
591 	 * At least one guest time will expire. Schedule a background timer.
592 	 * Set the earliest expiration time among the guest timers.
593 	 */
594 	soft_timer_start(&timer->bg_timer, kvm_timer_earliest_exp(vcpu));
595 }
596 
kvm_timer_unblocking(struct kvm_vcpu * vcpu)597 static void kvm_timer_unblocking(struct kvm_vcpu *vcpu)
598 {
599 	struct arch_timer_cpu *timer = vcpu_timer(vcpu);
600 
601 	soft_timer_cancel(&timer->bg_timer);
602 }
603 
timer_restore_state(struct arch_timer_context * ctx)604 static void timer_restore_state(struct arch_timer_context *ctx)
605 {
606 	struct arch_timer_cpu *timer = vcpu_timer(ctx->vcpu);
607 	enum kvm_arch_timers index = arch_timer_ctx_index(ctx);
608 	unsigned long flags;
609 
610 	if (!timer->enabled)
611 		return;
612 
613 	local_irq_save(flags);
614 
615 	if (ctx->loaded)
616 		goto out;
617 
618 	switch (index) {
619 		u64 cval, offset;
620 
621 	case TIMER_VTIMER:
622 	case TIMER_HVTIMER:
623 		set_cntvoff(timer_get_offset(ctx));
624 		write_sysreg_el0(timer_get_cval(ctx), SYS_CNTV_CVAL);
625 		isb();
626 		write_sysreg_el0(timer_get_ctl(ctx), SYS_CNTV_CTL);
627 		break;
628 	case TIMER_PTIMER:
629 	case TIMER_HPTIMER:
630 		cval = timer_get_cval(ctx);
631 		offset = timer_get_offset(ctx);
632 		set_cntpoff(offset);
633 		cval += offset;
634 		write_sysreg_el0(cval, SYS_CNTP_CVAL);
635 		isb();
636 		write_sysreg_el0(timer_get_ctl(ctx), SYS_CNTP_CTL);
637 		break;
638 	case NR_KVM_TIMERS:
639 		BUG();
640 	}
641 
642 	trace_kvm_timer_restore_state(ctx);
643 
644 	ctx->loaded = true;
645 out:
646 	local_irq_restore(flags);
647 }
648 
set_timer_irq_phys_active(struct arch_timer_context * ctx,bool active)649 static inline void set_timer_irq_phys_active(struct arch_timer_context *ctx, bool active)
650 {
651 	int r;
652 	r = irq_set_irqchip_state(ctx->host_timer_irq, IRQCHIP_STATE_ACTIVE, active);
653 	WARN_ON(r);
654 }
655 
kvm_timer_vcpu_load_gic(struct arch_timer_context * ctx)656 static void kvm_timer_vcpu_load_gic(struct arch_timer_context *ctx)
657 {
658 	struct kvm_vcpu *vcpu = ctx->vcpu;
659 	bool phys_active = false;
660 
661 	/*
662 	 * Update the timer output so that it is likely to match the
663 	 * state we're about to restore. If the timer expires between
664 	 * this point and the register restoration, we'll take the
665 	 * interrupt anyway.
666 	 */
667 	kvm_timer_update_irq(ctx->vcpu, kvm_timer_should_fire(ctx), ctx);
668 
669 	if (irqchip_in_kernel(vcpu->kvm))
670 		phys_active = kvm_vgic_map_is_active(vcpu, timer_irq(ctx));
671 
672 	phys_active |= ctx->irq.level;
673 
674 	set_timer_irq_phys_active(ctx, phys_active);
675 }
676 
kvm_timer_vcpu_load_nogic(struct kvm_vcpu * vcpu)677 static void kvm_timer_vcpu_load_nogic(struct kvm_vcpu *vcpu)
678 {
679 	struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
680 
681 	/*
682 	 * Update the timer output so that it is likely to match the
683 	 * state we're about to restore. If the timer expires between
684 	 * this point and the register restoration, we'll take the
685 	 * interrupt anyway.
686 	 */
687 	kvm_timer_update_irq(vcpu, kvm_timer_should_fire(vtimer), vtimer);
688 
689 	/*
690 	 * When using a userspace irqchip with the architected timers and a
691 	 * host interrupt controller that doesn't support an active state, we
692 	 * must still prevent continuously exiting from the guest, and
693 	 * therefore mask the physical interrupt by disabling it on the host
694 	 * interrupt controller when the virtual level is high, such that the
695 	 * guest can make forward progress.  Once we detect the output level
696 	 * being de-asserted, we unmask the interrupt again so that we exit
697 	 * from the guest when the timer fires.
698 	 */
699 	if (vtimer->irq.level)
700 		disable_percpu_irq(host_vtimer_irq);
701 	else
702 		enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags);
703 }
704 
705 /* If _pred is true, set bit in _set, otherwise set it in _clr */
706 #define assign_clear_set_bit(_pred, _bit, _clr, _set)			\
707 	do {								\
708 		if (_pred)						\
709 			(_set) |= (_bit);				\
710 		else							\
711 			(_clr) |= (_bit);				\
712 	} while (0)
713 
kvm_timer_vcpu_load_nested_switch(struct kvm_vcpu * vcpu,struct timer_map * map)714 static void kvm_timer_vcpu_load_nested_switch(struct kvm_vcpu *vcpu,
715 					      struct timer_map *map)
716 {
717 	int hw, ret;
718 
719 	if (!irqchip_in_kernel(vcpu->kvm))
720 		return;
721 
722 	/*
723 	 * We only ever unmap the vtimer irq on a VHE system that runs nested
724 	 * virtualization, in which case we have both a valid emul_vtimer,
725 	 * emul_ptimer, direct_vtimer, and direct_ptimer.
726 	 *
727 	 * Since this is called from kvm_timer_vcpu_load(), a change between
728 	 * vEL2 and vEL1/0 will have just happened, and the timer_map will
729 	 * represent this, and therefore we switch the emul/direct mappings
730 	 * below.
731 	 */
732 	hw = kvm_vgic_get_map(vcpu, timer_irq(map->direct_vtimer));
733 	if (hw < 0) {
734 		kvm_vgic_unmap_phys_irq(vcpu, timer_irq(map->emul_vtimer));
735 		kvm_vgic_unmap_phys_irq(vcpu, timer_irq(map->emul_ptimer));
736 
737 		ret = kvm_vgic_map_phys_irq(vcpu,
738 					    map->direct_vtimer->host_timer_irq,
739 					    timer_irq(map->direct_vtimer),
740 					    &arch_timer_irq_ops);
741 		WARN_ON_ONCE(ret);
742 		ret = kvm_vgic_map_phys_irq(vcpu,
743 					    map->direct_ptimer->host_timer_irq,
744 					    timer_irq(map->direct_ptimer),
745 					    &arch_timer_irq_ops);
746 		WARN_ON_ONCE(ret);
747 
748 		/*
749 		 * The virtual offset behaviour is "interresting", as it
750 		 * always applies when HCR_EL2.E2H==0, but only when
751 		 * accessed from EL1 when HCR_EL2.E2H==1. So make sure we
752 		 * track E2H when putting the HV timer in "direct" mode.
753 		 */
754 		if (map->direct_vtimer == vcpu_hvtimer(vcpu)) {
755 			struct arch_timer_offset *offs = &map->direct_vtimer->offset;
756 
757 			if (vcpu_el2_e2h_is_set(vcpu))
758 				offs->vcpu_offset = NULL;
759 			else
760 				offs->vcpu_offset = &__vcpu_sys_reg(vcpu, CNTVOFF_EL2);
761 		}
762 	}
763 }
764 
timer_set_traps(struct kvm_vcpu * vcpu,struct timer_map * map)765 static void timer_set_traps(struct kvm_vcpu *vcpu, struct timer_map *map)
766 {
767 	bool tpt, tpc;
768 	u64 clr, set;
769 
770 	/*
771 	 * No trapping gets configured here with nVHE. See
772 	 * __timer_enable_traps(), which is where the stuff happens.
773 	 */
774 	if (!has_vhe())
775 		return;
776 
777 	/*
778 	 * Our default policy is not to trap anything. As we progress
779 	 * within this function, reality kicks in and we start adding
780 	 * traps based on emulation requirements.
781 	 */
782 	tpt = tpc = false;
783 
784 	/*
785 	 * We have two possibility to deal with a physical offset:
786 	 *
787 	 * - Either we have CNTPOFF (yay!) or the offset is 0:
788 	 *   we let the guest freely access the HW
789 	 *
790 	 * - or neither of these condition apply:
791 	 *   we trap accesses to the HW, but still use it
792 	 *   after correcting the physical offset
793 	 */
794 	if (!has_cntpoff() && timer_get_offset(map->direct_ptimer))
795 		tpt = tpc = true;
796 
797 	/*
798 	 * Apply the enable bits that the guest hypervisor has requested for
799 	 * its own guest. We can only add traps that wouldn't have been set
800 	 * above.
801 	 */
802 	if (vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu)) {
803 		u64 val = __vcpu_sys_reg(vcpu, CNTHCTL_EL2);
804 
805 		/* Use the VHE format for mental sanity */
806 		if (!vcpu_el2_e2h_is_set(vcpu))
807 			val = (val & (CNTHCTL_EL1PCEN | CNTHCTL_EL1PCTEN)) << 10;
808 
809 		tpt |= !(val & (CNTHCTL_EL1PCEN << 10));
810 		tpc |= !(val & (CNTHCTL_EL1PCTEN << 10));
811 	}
812 
813 	/*
814 	 * Now that we have collected our requirements, compute the
815 	 * trap and enable bits.
816 	 */
817 	set = 0;
818 	clr = 0;
819 
820 	assign_clear_set_bit(tpt, CNTHCTL_EL1PCEN << 10, set, clr);
821 	assign_clear_set_bit(tpc, CNTHCTL_EL1PCTEN << 10, set, clr);
822 
823 	/* This only happens on VHE, so use the CNTHCTL_EL2 accessor. */
824 	sysreg_clear_set(cnthctl_el2, clr, set);
825 }
826 
kvm_timer_vcpu_load(struct kvm_vcpu * vcpu)827 void kvm_timer_vcpu_load(struct kvm_vcpu *vcpu)
828 {
829 	struct arch_timer_cpu *timer = vcpu_timer(vcpu);
830 	struct timer_map map;
831 
832 	if (unlikely(!timer->enabled))
833 		return;
834 
835 	get_timer_map(vcpu, &map);
836 
837 	if (static_branch_likely(&has_gic_active_state)) {
838 		if (vcpu_has_nv(vcpu))
839 			kvm_timer_vcpu_load_nested_switch(vcpu, &map);
840 
841 		kvm_timer_vcpu_load_gic(map.direct_vtimer);
842 		if (map.direct_ptimer)
843 			kvm_timer_vcpu_load_gic(map.direct_ptimer);
844 	} else {
845 		kvm_timer_vcpu_load_nogic(vcpu);
846 	}
847 
848 	kvm_timer_unblocking(vcpu);
849 
850 	timer_restore_state(map.direct_vtimer);
851 	if (map.direct_ptimer)
852 		timer_restore_state(map.direct_ptimer);
853 	if (map.emul_vtimer)
854 		timer_emulate(map.emul_vtimer);
855 	if (map.emul_ptimer)
856 		timer_emulate(map.emul_ptimer);
857 
858 	timer_set_traps(vcpu, &map);
859 }
860 
kvm_timer_should_notify_user(struct kvm_vcpu * vcpu)861 bool kvm_timer_should_notify_user(struct kvm_vcpu *vcpu)
862 {
863 	struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
864 	struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
865 	struct kvm_sync_regs *sregs = &vcpu->run->s.regs;
866 	bool vlevel, plevel;
867 
868 	if (likely(irqchip_in_kernel(vcpu->kvm)))
869 		return false;
870 
871 	vlevel = sregs->device_irq_level & KVM_ARM_DEV_EL1_VTIMER;
872 	plevel = sregs->device_irq_level & KVM_ARM_DEV_EL1_PTIMER;
873 
874 	return kvm_timer_should_fire(vtimer) != vlevel ||
875 	       kvm_timer_should_fire(ptimer) != plevel;
876 }
877 
kvm_timer_vcpu_put(struct kvm_vcpu * vcpu)878 void kvm_timer_vcpu_put(struct kvm_vcpu *vcpu)
879 {
880 	struct arch_timer_cpu *timer = vcpu_timer(vcpu);
881 	struct timer_map map;
882 
883 	if (unlikely(!timer->enabled))
884 		return;
885 
886 	get_timer_map(vcpu, &map);
887 
888 	timer_save_state(map.direct_vtimer);
889 	if (map.direct_ptimer)
890 		timer_save_state(map.direct_ptimer);
891 
892 	/*
893 	 * Cancel soft timer emulation, because the only case where we
894 	 * need it after a vcpu_put is in the context of a sleeping VCPU, and
895 	 * in that case we already factor in the deadline for the physical
896 	 * timer when scheduling the bg_timer.
897 	 *
898 	 * In any case, we re-schedule the hrtimer for the physical timer when
899 	 * coming back to the VCPU thread in kvm_timer_vcpu_load().
900 	 */
901 	if (map.emul_vtimer)
902 		soft_timer_cancel(&map.emul_vtimer->hrtimer);
903 	if (map.emul_ptimer)
904 		soft_timer_cancel(&map.emul_ptimer->hrtimer);
905 
906 	if (kvm_vcpu_is_blocking(vcpu))
907 		kvm_timer_blocking(vcpu);
908 }
909 
910 /*
911  * With a userspace irqchip we have to check if the guest de-asserted the
912  * timer and if so, unmask the timer irq signal on the host interrupt
913  * controller to ensure that we see future timer signals.
914  */
unmask_vtimer_irq_user(struct kvm_vcpu * vcpu)915 static void unmask_vtimer_irq_user(struct kvm_vcpu *vcpu)
916 {
917 	struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
918 
919 	if (!kvm_timer_should_fire(vtimer)) {
920 		kvm_timer_update_irq(vcpu, false, vtimer);
921 		if (static_branch_likely(&has_gic_active_state))
922 			set_timer_irq_phys_active(vtimer, false);
923 		else
924 			enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags);
925 	}
926 }
927 
kvm_timer_sync_user(struct kvm_vcpu * vcpu)928 void kvm_timer_sync_user(struct kvm_vcpu *vcpu)
929 {
930 	struct arch_timer_cpu *timer = vcpu_timer(vcpu);
931 
932 	if (unlikely(!timer->enabled))
933 		return;
934 
935 	if (unlikely(!irqchip_in_kernel(vcpu->kvm)))
936 		unmask_vtimer_irq_user(vcpu);
937 }
938 
kvm_timer_vcpu_reset(struct kvm_vcpu * vcpu)939 int kvm_timer_vcpu_reset(struct kvm_vcpu *vcpu)
940 {
941 	struct arch_timer_cpu *timer = vcpu_timer(vcpu);
942 	struct timer_map map;
943 
944 	get_timer_map(vcpu, &map);
945 
946 	/*
947 	 * The bits in CNTV_CTL are architecturally reset to UNKNOWN for ARMv8
948 	 * and to 0 for ARMv7.  We provide an implementation that always
949 	 * resets the timer to be disabled and unmasked and is compliant with
950 	 * the ARMv7 architecture.
951 	 */
952 	for (int i = 0; i < nr_timers(vcpu); i++)
953 		timer_set_ctl(vcpu_get_timer(vcpu, i), 0);
954 
955 	/*
956 	 * A vcpu running at EL2 is in charge of the offset applied to
957 	 * the virtual timer, so use the physical VM offset, and point
958 	 * the vcpu offset to CNTVOFF_EL2.
959 	 */
960 	if (vcpu_has_nv(vcpu)) {
961 		struct arch_timer_offset *offs = &vcpu_vtimer(vcpu)->offset;
962 
963 		offs->vcpu_offset = &__vcpu_sys_reg(vcpu, CNTVOFF_EL2);
964 		offs->vm_offset = &vcpu->kvm->arch.timer_data.poffset;
965 	}
966 
967 	if (timer->enabled) {
968 		for (int i = 0; i < nr_timers(vcpu); i++)
969 			kvm_timer_update_irq(vcpu, false,
970 					     vcpu_get_timer(vcpu, i));
971 
972 		if (irqchip_in_kernel(vcpu->kvm)) {
973 			kvm_vgic_reset_mapped_irq(vcpu, timer_irq(map.direct_vtimer));
974 			if (map.direct_ptimer)
975 				kvm_vgic_reset_mapped_irq(vcpu, timer_irq(map.direct_ptimer));
976 		}
977 	}
978 
979 	if (map.emul_vtimer)
980 		soft_timer_cancel(&map.emul_vtimer->hrtimer);
981 	if (map.emul_ptimer)
982 		soft_timer_cancel(&map.emul_ptimer->hrtimer);
983 
984 	return 0;
985 }
986 
timer_context_init(struct kvm_vcpu * vcpu,int timerid)987 static void timer_context_init(struct kvm_vcpu *vcpu, int timerid)
988 {
989 	struct arch_timer_context *ctxt = vcpu_get_timer(vcpu, timerid);
990 	struct kvm *kvm = vcpu->kvm;
991 
992 	ctxt->vcpu = vcpu;
993 
994 	if (timerid == TIMER_VTIMER)
995 		ctxt->offset.vm_offset = &kvm->arch.timer_data.voffset;
996 	else
997 		ctxt->offset.vm_offset = &kvm->arch.timer_data.poffset;
998 
999 	hrtimer_init(&ctxt->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD);
1000 	ctxt->hrtimer.function = kvm_hrtimer_expire;
1001 
1002 	switch (timerid) {
1003 	case TIMER_PTIMER:
1004 	case TIMER_HPTIMER:
1005 		ctxt->host_timer_irq = host_ptimer_irq;
1006 		break;
1007 	case TIMER_VTIMER:
1008 	case TIMER_HVTIMER:
1009 		ctxt->host_timer_irq = host_vtimer_irq;
1010 		break;
1011 	}
1012 }
1013 
kvm_timer_vcpu_init(struct kvm_vcpu * vcpu)1014 void kvm_timer_vcpu_init(struct kvm_vcpu *vcpu)
1015 {
1016 	struct arch_timer_cpu *timer = vcpu_timer(vcpu);
1017 
1018 	for (int i = 0; i < NR_KVM_TIMERS; i++)
1019 		timer_context_init(vcpu, i);
1020 
1021 	/* Synchronize offsets across timers of a VM if not already provided */
1022 	if (!test_bit(KVM_ARCH_FLAG_VM_COUNTER_OFFSET, &vcpu->kvm->arch.flags)) {
1023 		timer_set_offset(vcpu_vtimer(vcpu), kvm_phys_timer_read());
1024 		timer_set_offset(vcpu_ptimer(vcpu), 0);
1025 	}
1026 
1027 	hrtimer_init(&timer->bg_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD);
1028 	timer->bg_timer.function = kvm_bg_timer_expire;
1029 }
1030 
kvm_timer_init_vm(struct kvm * kvm)1031 void kvm_timer_init_vm(struct kvm *kvm)
1032 {
1033 	for (int i = 0; i < NR_KVM_TIMERS; i++)
1034 		kvm->arch.timer_data.ppi[i] = default_ppi[i];
1035 }
1036 
kvm_timer_cpu_up(void)1037 void kvm_timer_cpu_up(void)
1038 {
1039 	enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags);
1040 	if (host_ptimer_irq)
1041 		enable_percpu_irq(host_ptimer_irq, host_ptimer_irq_flags);
1042 }
1043 
kvm_timer_cpu_down(void)1044 void kvm_timer_cpu_down(void)
1045 {
1046 	disable_percpu_irq(host_vtimer_irq);
1047 	if (host_ptimer_irq)
1048 		disable_percpu_irq(host_ptimer_irq);
1049 }
1050 
kvm_arm_timer_set_reg(struct kvm_vcpu * vcpu,u64 regid,u64 value)1051 int kvm_arm_timer_set_reg(struct kvm_vcpu *vcpu, u64 regid, u64 value)
1052 {
1053 	struct arch_timer_context *timer;
1054 
1055 	switch (regid) {
1056 	case KVM_REG_ARM_TIMER_CTL:
1057 		timer = vcpu_vtimer(vcpu);
1058 		kvm_arm_timer_write(vcpu, timer, TIMER_REG_CTL, value);
1059 		break;
1060 	case KVM_REG_ARM_TIMER_CNT:
1061 		if (!test_bit(KVM_ARCH_FLAG_VM_COUNTER_OFFSET,
1062 			      &vcpu->kvm->arch.flags)) {
1063 			timer = vcpu_vtimer(vcpu);
1064 			timer_set_offset(timer, kvm_phys_timer_read() - value);
1065 		}
1066 		break;
1067 	case KVM_REG_ARM_TIMER_CVAL:
1068 		timer = vcpu_vtimer(vcpu);
1069 		kvm_arm_timer_write(vcpu, timer, TIMER_REG_CVAL, value);
1070 		break;
1071 	case KVM_REG_ARM_PTIMER_CTL:
1072 		timer = vcpu_ptimer(vcpu);
1073 		kvm_arm_timer_write(vcpu, timer, TIMER_REG_CTL, value);
1074 		break;
1075 	case KVM_REG_ARM_PTIMER_CNT:
1076 		if (!test_bit(KVM_ARCH_FLAG_VM_COUNTER_OFFSET,
1077 			      &vcpu->kvm->arch.flags)) {
1078 			timer = vcpu_ptimer(vcpu);
1079 			timer_set_offset(timer, kvm_phys_timer_read() - value);
1080 		}
1081 		break;
1082 	case KVM_REG_ARM_PTIMER_CVAL:
1083 		timer = vcpu_ptimer(vcpu);
1084 		kvm_arm_timer_write(vcpu, timer, TIMER_REG_CVAL, value);
1085 		break;
1086 
1087 	default:
1088 		return -1;
1089 	}
1090 
1091 	return 0;
1092 }
1093 
read_timer_ctl(struct arch_timer_context * timer)1094 static u64 read_timer_ctl(struct arch_timer_context *timer)
1095 {
1096 	/*
1097 	 * Set ISTATUS bit if it's expired.
1098 	 * Note that according to ARMv8 ARM Issue A.k, ISTATUS bit is
1099 	 * UNKNOWN when ENABLE bit is 0, so we chose to set ISTATUS bit
1100 	 * regardless of ENABLE bit for our implementation convenience.
1101 	 */
1102 	u32 ctl = timer_get_ctl(timer);
1103 
1104 	if (!kvm_timer_compute_delta(timer))
1105 		ctl |= ARCH_TIMER_CTRL_IT_STAT;
1106 
1107 	return ctl;
1108 }
1109 
kvm_arm_timer_get_reg(struct kvm_vcpu * vcpu,u64 regid)1110 u64 kvm_arm_timer_get_reg(struct kvm_vcpu *vcpu, u64 regid)
1111 {
1112 	switch (regid) {
1113 	case KVM_REG_ARM_TIMER_CTL:
1114 		return kvm_arm_timer_read(vcpu,
1115 					  vcpu_vtimer(vcpu), TIMER_REG_CTL);
1116 	case KVM_REG_ARM_TIMER_CNT:
1117 		return kvm_arm_timer_read(vcpu,
1118 					  vcpu_vtimer(vcpu), TIMER_REG_CNT);
1119 	case KVM_REG_ARM_TIMER_CVAL:
1120 		return kvm_arm_timer_read(vcpu,
1121 					  vcpu_vtimer(vcpu), TIMER_REG_CVAL);
1122 	case KVM_REG_ARM_PTIMER_CTL:
1123 		return kvm_arm_timer_read(vcpu,
1124 					  vcpu_ptimer(vcpu), TIMER_REG_CTL);
1125 	case KVM_REG_ARM_PTIMER_CNT:
1126 		return kvm_arm_timer_read(vcpu,
1127 					  vcpu_ptimer(vcpu), TIMER_REG_CNT);
1128 	case KVM_REG_ARM_PTIMER_CVAL:
1129 		return kvm_arm_timer_read(vcpu,
1130 					  vcpu_ptimer(vcpu), TIMER_REG_CVAL);
1131 	}
1132 	return (u64)-1;
1133 }
1134 
kvm_arm_timer_read(struct kvm_vcpu * vcpu,struct arch_timer_context * timer,enum kvm_arch_timer_regs treg)1135 static u64 kvm_arm_timer_read(struct kvm_vcpu *vcpu,
1136 			      struct arch_timer_context *timer,
1137 			      enum kvm_arch_timer_regs treg)
1138 {
1139 	u64 val;
1140 
1141 	switch (treg) {
1142 	case TIMER_REG_TVAL:
1143 		val = timer_get_cval(timer) - kvm_phys_timer_read() + timer_get_offset(timer);
1144 		val = lower_32_bits(val);
1145 		break;
1146 
1147 	case TIMER_REG_CTL:
1148 		val = read_timer_ctl(timer);
1149 		break;
1150 
1151 	case TIMER_REG_CVAL:
1152 		val = timer_get_cval(timer);
1153 		break;
1154 
1155 	case TIMER_REG_CNT:
1156 		val = kvm_phys_timer_read() - timer_get_offset(timer);
1157 		break;
1158 
1159 	case TIMER_REG_VOFF:
1160 		val = *timer->offset.vcpu_offset;
1161 		break;
1162 
1163 	default:
1164 		BUG();
1165 	}
1166 
1167 	return val;
1168 }
1169 
kvm_arm_timer_read_sysreg(struct kvm_vcpu * vcpu,enum kvm_arch_timers tmr,enum kvm_arch_timer_regs treg)1170 u64 kvm_arm_timer_read_sysreg(struct kvm_vcpu *vcpu,
1171 			      enum kvm_arch_timers tmr,
1172 			      enum kvm_arch_timer_regs treg)
1173 {
1174 	struct arch_timer_context *timer;
1175 	struct timer_map map;
1176 	u64 val;
1177 
1178 	get_timer_map(vcpu, &map);
1179 	timer = vcpu_get_timer(vcpu, tmr);
1180 
1181 	if (timer == map.emul_vtimer || timer == map.emul_ptimer)
1182 		return kvm_arm_timer_read(vcpu, timer, treg);
1183 
1184 	preempt_disable();
1185 	timer_save_state(timer);
1186 
1187 	val = kvm_arm_timer_read(vcpu, timer, treg);
1188 
1189 	timer_restore_state(timer);
1190 	preempt_enable();
1191 
1192 	return val;
1193 }
1194 
kvm_arm_timer_write(struct kvm_vcpu * vcpu,struct arch_timer_context * timer,enum kvm_arch_timer_regs treg,u64 val)1195 static void kvm_arm_timer_write(struct kvm_vcpu *vcpu,
1196 				struct arch_timer_context *timer,
1197 				enum kvm_arch_timer_regs treg,
1198 				u64 val)
1199 {
1200 	switch (treg) {
1201 	case TIMER_REG_TVAL:
1202 		timer_set_cval(timer, kvm_phys_timer_read() - timer_get_offset(timer) + (s32)val);
1203 		break;
1204 
1205 	case TIMER_REG_CTL:
1206 		timer_set_ctl(timer, val & ~ARCH_TIMER_CTRL_IT_STAT);
1207 		break;
1208 
1209 	case TIMER_REG_CVAL:
1210 		timer_set_cval(timer, val);
1211 		break;
1212 
1213 	case TIMER_REG_VOFF:
1214 		*timer->offset.vcpu_offset = val;
1215 		break;
1216 
1217 	default:
1218 		BUG();
1219 	}
1220 }
1221 
kvm_arm_timer_write_sysreg(struct kvm_vcpu * vcpu,enum kvm_arch_timers tmr,enum kvm_arch_timer_regs treg,u64 val)1222 void kvm_arm_timer_write_sysreg(struct kvm_vcpu *vcpu,
1223 				enum kvm_arch_timers tmr,
1224 				enum kvm_arch_timer_regs treg,
1225 				u64 val)
1226 {
1227 	struct arch_timer_context *timer;
1228 	struct timer_map map;
1229 
1230 	get_timer_map(vcpu, &map);
1231 	timer = vcpu_get_timer(vcpu, tmr);
1232 	if (timer == map.emul_vtimer || timer == map.emul_ptimer) {
1233 		soft_timer_cancel(&timer->hrtimer);
1234 		kvm_arm_timer_write(vcpu, timer, treg, val);
1235 		timer_emulate(timer);
1236 	} else {
1237 		preempt_disable();
1238 		timer_save_state(timer);
1239 		kvm_arm_timer_write(vcpu, timer, treg, val);
1240 		timer_restore_state(timer);
1241 		preempt_enable();
1242 	}
1243 }
1244 
timer_irq_set_vcpu_affinity(struct irq_data * d,void * vcpu)1245 static int timer_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu)
1246 {
1247 	if (vcpu)
1248 		irqd_set_forwarded_to_vcpu(d);
1249 	else
1250 		irqd_clr_forwarded_to_vcpu(d);
1251 
1252 	return 0;
1253 }
1254 
timer_irq_set_irqchip_state(struct irq_data * d,enum irqchip_irq_state which,bool val)1255 static int timer_irq_set_irqchip_state(struct irq_data *d,
1256 				       enum irqchip_irq_state which, bool val)
1257 {
1258 	if (which != IRQCHIP_STATE_ACTIVE || !irqd_is_forwarded_to_vcpu(d))
1259 		return irq_chip_set_parent_state(d, which, val);
1260 
1261 	if (val)
1262 		irq_chip_mask_parent(d);
1263 	else
1264 		irq_chip_unmask_parent(d);
1265 
1266 	return 0;
1267 }
1268 
timer_irq_eoi(struct irq_data * d)1269 static void timer_irq_eoi(struct irq_data *d)
1270 {
1271 	if (!irqd_is_forwarded_to_vcpu(d))
1272 		irq_chip_eoi_parent(d);
1273 }
1274 
timer_irq_ack(struct irq_data * d)1275 static void timer_irq_ack(struct irq_data *d)
1276 {
1277 	d = d->parent_data;
1278 	if (d->chip->irq_ack)
1279 		d->chip->irq_ack(d);
1280 }
1281 
1282 static struct irq_chip timer_chip = {
1283 	.name			= "KVM",
1284 	.irq_ack		= timer_irq_ack,
1285 	.irq_mask		= irq_chip_mask_parent,
1286 	.irq_unmask		= irq_chip_unmask_parent,
1287 	.irq_eoi		= timer_irq_eoi,
1288 	.irq_set_type		= irq_chip_set_type_parent,
1289 	.irq_set_vcpu_affinity	= timer_irq_set_vcpu_affinity,
1290 	.irq_set_irqchip_state	= timer_irq_set_irqchip_state,
1291 };
1292 
timer_irq_domain_alloc(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs,void * arg)1293 static int timer_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
1294 				  unsigned int nr_irqs, void *arg)
1295 {
1296 	irq_hw_number_t hwirq = (uintptr_t)arg;
1297 
1298 	return irq_domain_set_hwirq_and_chip(domain, virq, hwirq,
1299 					     &timer_chip, NULL);
1300 }
1301 
timer_irq_domain_free(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs)1302 static void timer_irq_domain_free(struct irq_domain *domain, unsigned int virq,
1303 				  unsigned int nr_irqs)
1304 {
1305 }
1306 
1307 static const struct irq_domain_ops timer_domain_ops = {
1308 	.alloc	= timer_irq_domain_alloc,
1309 	.free	= timer_irq_domain_free,
1310 };
1311 
kvm_irq_fixup_flags(unsigned int virq,u32 * flags)1312 static void kvm_irq_fixup_flags(unsigned int virq, u32 *flags)
1313 {
1314 	*flags = irq_get_trigger_type(virq);
1315 	if (*flags != IRQF_TRIGGER_HIGH && *flags != IRQF_TRIGGER_LOW) {
1316 		kvm_err("Invalid trigger for timer IRQ%d, assuming level low\n",
1317 			virq);
1318 		*flags = IRQF_TRIGGER_LOW;
1319 	}
1320 }
1321 
kvm_irq_init(struct arch_timer_kvm_info * info)1322 static int kvm_irq_init(struct arch_timer_kvm_info *info)
1323 {
1324 	struct irq_domain *domain = NULL;
1325 
1326 	if (info->virtual_irq <= 0) {
1327 		kvm_err("kvm_arch_timer: invalid virtual timer IRQ: %d\n",
1328 			info->virtual_irq);
1329 		return -ENODEV;
1330 	}
1331 
1332 	host_vtimer_irq = info->virtual_irq;
1333 	kvm_irq_fixup_flags(host_vtimer_irq, &host_vtimer_irq_flags);
1334 
1335 	if (kvm_vgic_global_state.no_hw_deactivation) {
1336 		struct fwnode_handle *fwnode;
1337 		struct irq_data *data;
1338 
1339 		fwnode = irq_domain_alloc_named_fwnode("kvm-timer");
1340 		if (!fwnode)
1341 			return -ENOMEM;
1342 
1343 		/* Assume both vtimer and ptimer in the same parent */
1344 		data = irq_get_irq_data(host_vtimer_irq);
1345 		domain = irq_domain_create_hierarchy(data->domain, 0,
1346 						     NR_KVM_TIMERS, fwnode,
1347 						     &timer_domain_ops, NULL);
1348 		if (!domain) {
1349 			irq_domain_free_fwnode(fwnode);
1350 			return -ENOMEM;
1351 		}
1352 
1353 		arch_timer_irq_ops.flags |= VGIC_IRQ_SW_RESAMPLE;
1354 		WARN_ON(irq_domain_push_irq(domain, host_vtimer_irq,
1355 					    (void *)TIMER_VTIMER));
1356 	}
1357 
1358 	if (info->physical_irq > 0) {
1359 		host_ptimer_irq = info->physical_irq;
1360 		kvm_irq_fixup_flags(host_ptimer_irq, &host_ptimer_irq_flags);
1361 
1362 		if (domain)
1363 			WARN_ON(irq_domain_push_irq(domain, host_ptimer_irq,
1364 						    (void *)TIMER_PTIMER));
1365 	}
1366 
1367 	return 0;
1368 }
1369 
kvm_timer_hyp_init(bool has_gic)1370 int __init kvm_timer_hyp_init(bool has_gic)
1371 {
1372 	struct arch_timer_kvm_info *info;
1373 	int err;
1374 
1375 	info = arch_timer_get_kvm_info();
1376 	timecounter = &info->timecounter;
1377 
1378 	if (!timecounter->cc) {
1379 		kvm_err("kvm_arch_timer: uninitialized timecounter\n");
1380 		return -ENODEV;
1381 	}
1382 
1383 	err = kvm_irq_init(info);
1384 	if (err)
1385 		return err;
1386 
1387 	/* First, do the virtual EL1 timer irq */
1388 
1389 	err = request_percpu_irq(host_vtimer_irq, kvm_arch_timer_handler,
1390 				 "kvm guest vtimer", kvm_get_running_vcpus());
1391 	if (err) {
1392 		kvm_err("kvm_arch_timer: can't request vtimer interrupt %d (%d)\n",
1393 			host_vtimer_irq, err);
1394 		return err;
1395 	}
1396 
1397 	if (has_gic) {
1398 		err = irq_set_vcpu_affinity(host_vtimer_irq,
1399 					    kvm_get_running_vcpus());
1400 		if (err) {
1401 			kvm_err("kvm_arch_timer: error setting vcpu affinity\n");
1402 			goto out_free_vtimer_irq;
1403 		}
1404 
1405 		static_branch_enable(&has_gic_active_state);
1406 	}
1407 
1408 	kvm_debug("virtual timer IRQ%d\n", host_vtimer_irq);
1409 
1410 	/* Now let's do the physical EL1 timer irq */
1411 
1412 	if (info->physical_irq > 0) {
1413 		err = request_percpu_irq(host_ptimer_irq, kvm_arch_timer_handler,
1414 					 "kvm guest ptimer", kvm_get_running_vcpus());
1415 		if (err) {
1416 			kvm_err("kvm_arch_timer: can't request ptimer interrupt %d (%d)\n",
1417 				host_ptimer_irq, err);
1418 			goto out_free_vtimer_irq;
1419 		}
1420 
1421 		if (has_gic) {
1422 			err = irq_set_vcpu_affinity(host_ptimer_irq,
1423 						    kvm_get_running_vcpus());
1424 			if (err) {
1425 				kvm_err("kvm_arch_timer: error setting vcpu affinity\n");
1426 				goto out_free_ptimer_irq;
1427 			}
1428 		}
1429 
1430 		kvm_debug("physical timer IRQ%d\n", host_ptimer_irq);
1431 	} else if (has_vhe()) {
1432 		kvm_err("kvm_arch_timer: invalid physical timer IRQ: %d\n",
1433 			info->physical_irq);
1434 		err = -ENODEV;
1435 		goto out_free_vtimer_irq;
1436 	}
1437 
1438 	return 0;
1439 
1440 out_free_ptimer_irq:
1441 	if (info->physical_irq > 0)
1442 		free_percpu_irq(host_ptimer_irq, kvm_get_running_vcpus());
1443 out_free_vtimer_irq:
1444 	free_percpu_irq(host_vtimer_irq, kvm_get_running_vcpus());
1445 	return err;
1446 }
1447 
kvm_timer_vcpu_terminate(struct kvm_vcpu * vcpu)1448 void kvm_timer_vcpu_terminate(struct kvm_vcpu *vcpu)
1449 {
1450 	struct arch_timer_cpu *timer = vcpu_timer(vcpu);
1451 
1452 	soft_timer_cancel(&timer->bg_timer);
1453 }
1454 
timer_irqs_are_valid(struct kvm_vcpu * vcpu)1455 static bool timer_irqs_are_valid(struct kvm_vcpu *vcpu)
1456 {
1457 	u32 ppis = 0;
1458 	bool valid;
1459 
1460 	mutex_lock(&vcpu->kvm->arch.config_lock);
1461 
1462 	for (int i = 0; i < nr_timers(vcpu); i++) {
1463 		struct arch_timer_context *ctx;
1464 		int irq;
1465 
1466 		ctx = vcpu_get_timer(vcpu, i);
1467 		irq = timer_irq(ctx);
1468 		if (kvm_vgic_set_owner(vcpu, irq, ctx))
1469 			break;
1470 
1471 		/*
1472 		 * We know by construction that we only have PPIs, so
1473 		 * all values are less than 32.
1474 		 */
1475 		ppis |= BIT(irq);
1476 	}
1477 
1478 	valid = hweight32(ppis) == nr_timers(vcpu);
1479 
1480 	if (valid)
1481 		set_bit(KVM_ARCH_FLAG_TIMER_PPIS_IMMUTABLE, &vcpu->kvm->arch.flags);
1482 
1483 	mutex_unlock(&vcpu->kvm->arch.config_lock);
1484 
1485 	return valid;
1486 }
1487 
kvm_arch_timer_get_input_level(int vintid)1488 static bool kvm_arch_timer_get_input_level(int vintid)
1489 {
1490 	struct kvm_vcpu *vcpu = kvm_get_running_vcpu();
1491 
1492 	if (WARN(!vcpu, "No vcpu context!\n"))
1493 		return false;
1494 
1495 	for (int i = 0; i < nr_timers(vcpu); i++) {
1496 		struct arch_timer_context *ctx;
1497 
1498 		ctx = vcpu_get_timer(vcpu, i);
1499 		if (timer_irq(ctx) == vintid)
1500 			return kvm_timer_should_fire(ctx);
1501 	}
1502 
1503 	/* A timer IRQ has fired, but no matching timer was found? */
1504 	WARN_RATELIMIT(1, "timer INTID%d unknown\n", vintid);
1505 
1506 	return false;
1507 }
1508 
kvm_timer_enable(struct kvm_vcpu * vcpu)1509 int kvm_timer_enable(struct kvm_vcpu *vcpu)
1510 {
1511 	struct arch_timer_cpu *timer = vcpu_timer(vcpu);
1512 	struct timer_map map;
1513 	int ret;
1514 
1515 	if (timer->enabled)
1516 		return 0;
1517 
1518 	/* Without a VGIC we do not map virtual IRQs to physical IRQs */
1519 	if (!irqchip_in_kernel(vcpu->kvm))
1520 		goto no_vgic;
1521 
1522 	/*
1523 	 * At this stage, we have the guarantee that the vgic is both
1524 	 * available and initialized.
1525 	 */
1526 	if (!timer_irqs_are_valid(vcpu)) {
1527 		kvm_debug("incorrectly configured timer irqs\n");
1528 		return -EINVAL;
1529 	}
1530 
1531 	get_timer_map(vcpu, &map);
1532 
1533 	ret = kvm_vgic_map_phys_irq(vcpu,
1534 				    map.direct_vtimer->host_timer_irq,
1535 				    timer_irq(map.direct_vtimer),
1536 				    &arch_timer_irq_ops);
1537 	if (ret)
1538 		return ret;
1539 
1540 	if (map.direct_ptimer) {
1541 		ret = kvm_vgic_map_phys_irq(vcpu,
1542 					    map.direct_ptimer->host_timer_irq,
1543 					    timer_irq(map.direct_ptimer),
1544 					    &arch_timer_irq_ops);
1545 	}
1546 
1547 	if (ret)
1548 		return ret;
1549 
1550 no_vgic:
1551 	timer->enabled = 1;
1552 	return 0;
1553 }
1554 
1555 /* If we have CNTPOFF, permanently set ECV to enable it */
kvm_timer_init_vhe(void)1556 void kvm_timer_init_vhe(void)
1557 {
1558 	if (cpus_have_final_cap(ARM64_HAS_ECV_CNTPOFF))
1559 		sysreg_clear_set(cnthctl_el2, 0, CNTHCTL_ECV);
1560 }
1561 
kvm_arm_timer_set_attr(struct kvm_vcpu * vcpu,struct kvm_device_attr * attr)1562 int kvm_arm_timer_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
1563 {
1564 	int __user *uaddr = (int __user *)(long)attr->addr;
1565 	int irq, idx, ret = 0;
1566 
1567 	if (!irqchip_in_kernel(vcpu->kvm))
1568 		return -EINVAL;
1569 
1570 	if (get_user(irq, uaddr))
1571 		return -EFAULT;
1572 
1573 	if (!(irq_is_ppi(irq)))
1574 		return -EINVAL;
1575 
1576 	mutex_lock(&vcpu->kvm->arch.config_lock);
1577 
1578 	if (test_bit(KVM_ARCH_FLAG_TIMER_PPIS_IMMUTABLE,
1579 		     &vcpu->kvm->arch.flags)) {
1580 		ret = -EBUSY;
1581 		goto out;
1582 	}
1583 
1584 	switch (attr->attr) {
1585 	case KVM_ARM_VCPU_TIMER_IRQ_VTIMER:
1586 		idx = TIMER_VTIMER;
1587 		break;
1588 	case KVM_ARM_VCPU_TIMER_IRQ_PTIMER:
1589 		idx = TIMER_PTIMER;
1590 		break;
1591 	case KVM_ARM_VCPU_TIMER_IRQ_HVTIMER:
1592 		idx = TIMER_HVTIMER;
1593 		break;
1594 	case KVM_ARM_VCPU_TIMER_IRQ_HPTIMER:
1595 		idx = TIMER_HPTIMER;
1596 		break;
1597 	default:
1598 		ret = -ENXIO;
1599 		goto out;
1600 	}
1601 
1602 	/*
1603 	 * We cannot validate the IRQ unicity before we run, so take it at
1604 	 * face value. The verdict will be given on first vcpu run, for each
1605 	 * vcpu. Yes this is late. Blame it on the stupid API.
1606 	 */
1607 	vcpu->kvm->arch.timer_data.ppi[idx] = irq;
1608 
1609 out:
1610 	mutex_unlock(&vcpu->kvm->arch.config_lock);
1611 	return ret;
1612 }
1613 
kvm_arm_timer_get_attr(struct kvm_vcpu * vcpu,struct kvm_device_attr * attr)1614 int kvm_arm_timer_get_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
1615 {
1616 	int __user *uaddr = (int __user *)(long)attr->addr;
1617 	struct arch_timer_context *timer;
1618 	int irq;
1619 
1620 	switch (attr->attr) {
1621 	case KVM_ARM_VCPU_TIMER_IRQ_VTIMER:
1622 		timer = vcpu_vtimer(vcpu);
1623 		break;
1624 	case KVM_ARM_VCPU_TIMER_IRQ_PTIMER:
1625 		timer = vcpu_ptimer(vcpu);
1626 		break;
1627 	case KVM_ARM_VCPU_TIMER_IRQ_HVTIMER:
1628 		timer = vcpu_hvtimer(vcpu);
1629 		break;
1630 	case KVM_ARM_VCPU_TIMER_IRQ_HPTIMER:
1631 		timer = vcpu_hptimer(vcpu);
1632 		break;
1633 	default:
1634 		return -ENXIO;
1635 	}
1636 
1637 	irq = timer_irq(timer);
1638 	return put_user(irq, uaddr);
1639 }
1640 
kvm_arm_timer_has_attr(struct kvm_vcpu * vcpu,struct kvm_device_attr * attr)1641 int kvm_arm_timer_has_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
1642 {
1643 	switch (attr->attr) {
1644 	case KVM_ARM_VCPU_TIMER_IRQ_VTIMER:
1645 	case KVM_ARM_VCPU_TIMER_IRQ_PTIMER:
1646 	case KVM_ARM_VCPU_TIMER_IRQ_HVTIMER:
1647 	case KVM_ARM_VCPU_TIMER_IRQ_HPTIMER:
1648 		return 0;
1649 	}
1650 
1651 	return -ENXIO;
1652 }
1653 
kvm_vm_ioctl_set_counter_offset(struct kvm * kvm,struct kvm_arm_counter_offset * offset)1654 int kvm_vm_ioctl_set_counter_offset(struct kvm *kvm,
1655 				    struct kvm_arm_counter_offset *offset)
1656 {
1657 	int ret = 0;
1658 
1659 	if (offset->reserved)
1660 		return -EINVAL;
1661 
1662 	mutex_lock(&kvm->lock);
1663 
1664 	if (lock_all_vcpus(kvm)) {
1665 		set_bit(KVM_ARCH_FLAG_VM_COUNTER_OFFSET, &kvm->arch.flags);
1666 
1667 		/*
1668 		 * If userspace decides to set the offset using this
1669 		 * API rather than merely restoring the counter
1670 		 * values, the offset applies to both the virtual and
1671 		 * physical views.
1672 		 */
1673 		kvm->arch.timer_data.voffset = offset->counter_offset;
1674 		kvm->arch.timer_data.poffset = offset->counter_offset;
1675 
1676 		unlock_all_vcpus(kvm);
1677 	} else {
1678 		ret = -EBUSY;
1679 	}
1680 
1681 	mutex_unlock(&kvm->lock);
1682 
1683 	return ret;
1684 }
1685