1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2019 Western Digital Corporation or its affiliates.
4  *
5  * Authors:
6  *     Atish Patra <atish.patra@wdc.com>
7  */
8 
9 #include <linux/errno.h>
10 #include <linux/err.h>
11 #include <linux/kvm_host.h>
12 #include <linux/uaccess.h>
13 #include <clocksource/timer-riscv.h>
14 #include <asm/csr.h>
15 #include <asm/delay.h>
16 #include <asm/kvm_vcpu_timer.h>
17 
kvm_riscv_current_cycles(struct kvm_guest_timer * gt)18 static u64 kvm_riscv_current_cycles(struct kvm_guest_timer *gt)
19 {
20 	return get_cycles64() + gt->time_delta;
21 }
22 
kvm_riscv_delta_cycles2ns(u64 cycles,struct kvm_guest_timer * gt,struct kvm_vcpu_timer * t)23 static u64 kvm_riscv_delta_cycles2ns(u64 cycles,
24 				     struct kvm_guest_timer *gt,
25 				     struct kvm_vcpu_timer *t)
26 {
27 	unsigned long flags;
28 	u64 cycles_now, cycles_delta, delta_ns;
29 
30 	local_irq_save(flags);
31 	cycles_now = kvm_riscv_current_cycles(gt);
32 	if (cycles_now < cycles)
33 		cycles_delta = cycles - cycles_now;
34 	else
35 		cycles_delta = 0;
36 	delta_ns = (cycles_delta * gt->nsec_mult) >> gt->nsec_shift;
37 	local_irq_restore(flags);
38 
39 	return delta_ns;
40 }
41 
kvm_riscv_vcpu_hrtimer_expired(struct hrtimer * h)42 static enum hrtimer_restart kvm_riscv_vcpu_hrtimer_expired(struct hrtimer *h)
43 {
44 	u64 delta_ns;
45 	struct kvm_vcpu_timer *t = container_of(h, struct kvm_vcpu_timer, hrt);
46 	struct kvm_vcpu *vcpu = container_of(t, struct kvm_vcpu, arch.timer);
47 	struct kvm_guest_timer *gt = &vcpu->kvm->arch.timer;
48 
49 	if (kvm_riscv_current_cycles(gt) < t->next_cycles) {
50 		delta_ns = kvm_riscv_delta_cycles2ns(t->next_cycles, gt, t);
51 		hrtimer_forward_now(&t->hrt, ktime_set(0, delta_ns));
52 		return HRTIMER_RESTART;
53 	}
54 
55 	t->next_set = false;
56 	kvm_riscv_vcpu_set_interrupt(vcpu, IRQ_VS_TIMER);
57 
58 	return HRTIMER_NORESTART;
59 }
60 
kvm_riscv_vcpu_timer_cancel(struct kvm_vcpu_timer * t)61 static int kvm_riscv_vcpu_timer_cancel(struct kvm_vcpu_timer *t)
62 {
63 	if (!t->init_done || !t->next_set)
64 		return -EINVAL;
65 
66 	hrtimer_cancel(&t->hrt);
67 	t->next_set = false;
68 
69 	return 0;
70 }
71 
kvm_riscv_vcpu_update_vstimecmp(struct kvm_vcpu * vcpu,u64 ncycles)72 static int kvm_riscv_vcpu_update_vstimecmp(struct kvm_vcpu *vcpu, u64 ncycles)
73 {
74 #if defined(CONFIG_32BIT)
75 		csr_write(CSR_VSTIMECMP, ncycles & 0xFFFFFFFF);
76 		csr_write(CSR_VSTIMECMPH, ncycles >> 32);
77 #else
78 		csr_write(CSR_VSTIMECMP, ncycles);
79 #endif
80 		return 0;
81 }
82 
kvm_riscv_vcpu_update_hrtimer(struct kvm_vcpu * vcpu,u64 ncycles)83 static int kvm_riscv_vcpu_update_hrtimer(struct kvm_vcpu *vcpu, u64 ncycles)
84 {
85 	struct kvm_vcpu_timer *t = &vcpu->arch.timer;
86 	struct kvm_guest_timer *gt = &vcpu->kvm->arch.timer;
87 	u64 delta_ns;
88 
89 	if (!t->init_done)
90 		return -EINVAL;
91 
92 	kvm_riscv_vcpu_unset_interrupt(vcpu, IRQ_VS_TIMER);
93 
94 	delta_ns = kvm_riscv_delta_cycles2ns(ncycles, gt, t);
95 	t->next_cycles = ncycles;
96 	hrtimer_start(&t->hrt, ktime_set(0, delta_ns), HRTIMER_MODE_REL);
97 	t->next_set = true;
98 
99 	return 0;
100 }
101 
kvm_riscv_vcpu_timer_next_event(struct kvm_vcpu * vcpu,u64 ncycles)102 int kvm_riscv_vcpu_timer_next_event(struct kvm_vcpu *vcpu, u64 ncycles)
103 {
104 	struct kvm_vcpu_timer *t = &vcpu->arch.timer;
105 
106 	return t->timer_next_event(vcpu, ncycles);
107 }
108 
kvm_riscv_vcpu_vstimer_expired(struct hrtimer * h)109 static enum hrtimer_restart kvm_riscv_vcpu_vstimer_expired(struct hrtimer *h)
110 {
111 	u64 delta_ns;
112 	struct kvm_vcpu_timer *t = container_of(h, struct kvm_vcpu_timer, hrt);
113 	struct kvm_vcpu *vcpu = container_of(t, struct kvm_vcpu, arch.timer);
114 	struct kvm_guest_timer *gt = &vcpu->kvm->arch.timer;
115 
116 	if (kvm_riscv_current_cycles(gt) < t->next_cycles) {
117 		delta_ns = kvm_riscv_delta_cycles2ns(t->next_cycles, gt, t);
118 		hrtimer_forward_now(&t->hrt, ktime_set(0, delta_ns));
119 		return HRTIMER_RESTART;
120 	}
121 
122 	t->next_set = false;
123 	kvm_vcpu_kick(vcpu);
124 
125 	return HRTIMER_NORESTART;
126 }
127 
kvm_riscv_vcpu_timer_pending(struct kvm_vcpu * vcpu)128 bool kvm_riscv_vcpu_timer_pending(struct kvm_vcpu *vcpu)
129 {
130 	struct kvm_vcpu_timer *t = &vcpu->arch.timer;
131 	struct kvm_guest_timer *gt = &vcpu->kvm->arch.timer;
132 
133 	if (!kvm_riscv_delta_cycles2ns(t->next_cycles, gt, t) ||
134 	    kvm_riscv_vcpu_has_interrupts(vcpu, 1UL << IRQ_VS_TIMER))
135 		return true;
136 	else
137 		return false;
138 }
139 
kvm_riscv_vcpu_timer_blocking(struct kvm_vcpu * vcpu)140 static void kvm_riscv_vcpu_timer_blocking(struct kvm_vcpu *vcpu)
141 {
142 	struct kvm_vcpu_timer *t = &vcpu->arch.timer;
143 	struct kvm_guest_timer *gt = &vcpu->kvm->arch.timer;
144 	u64 delta_ns;
145 
146 	if (!t->init_done)
147 		return;
148 
149 	delta_ns = kvm_riscv_delta_cycles2ns(t->next_cycles, gt, t);
150 	hrtimer_start(&t->hrt, ktime_set(0, delta_ns), HRTIMER_MODE_REL);
151 	t->next_set = true;
152 }
153 
kvm_riscv_vcpu_timer_unblocking(struct kvm_vcpu * vcpu)154 static void kvm_riscv_vcpu_timer_unblocking(struct kvm_vcpu *vcpu)
155 {
156 	kvm_riscv_vcpu_timer_cancel(&vcpu->arch.timer);
157 }
158 
kvm_riscv_vcpu_get_reg_timer(struct kvm_vcpu * vcpu,const struct kvm_one_reg * reg)159 int kvm_riscv_vcpu_get_reg_timer(struct kvm_vcpu *vcpu,
160 				 const struct kvm_one_reg *reg)
161 {
162 	struct kvm_vcpu_timer *t = &vcpu->arch.timer;
163 	struct kvm_guest_timer *gt = &vcpu->kvm->arch.timer;
164 	u64 __user *uaddr = (u64 __user *)(unsigned long)reg->addr;
165 	unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK |
166 					    KVM_REG_SIZE_MASK |
167 					    KVM_REG_RISCV_TIMER);
168 	u64 reg_val;
169 
170 	if (KVM_REG_SIZE(reg->id) != sizeof(u64))
171 		return -EINVAL;
172 	if (reg_num >= sizeof(struct kvm_riscv_timer) / sizeof(u64))
173 		return -ENOENT;
174 
175 	switch (reg_num) {
176 	case KVM_REG_RISCV_TIMER_REG(frequency):
177 		reg_val = riscv_timebase;
178 		break;
179 	case KVM_REG_RISCV_TIMER_REG(time):
180 		reg_val = kvm_riscv_current_cycles(gt);
181 		break;
182 	case KVM_REG_RISCV_TIMER_REG(compare):
183 		reg_val = t->next_cycles;
184 		break;
185 	case KVM_REG_RISCV_TIMER_REG(state):
186 		reg_val = (t->next_set) ? KVM_RISCV_TIMER_STATE_ON :
187 					  KVM_RISCV_TIMER_STATE_OFF;
188 		break;
189 	default:
190 		return -ENOENT;
191 	}
192 
193 	if (copy_to_user(uaddr, &reg_val, KVM_REG_SIZE(reg->id)))
194 		return -EFAULT;
195 
196 	return 0;
197 }
198 
kvm_riscv_vcpu_set_reg_timer(struct kvm_vcpu * vcpu,const struct kvm_one_reg * reg)199 int kvm_riscv_vcpu_set_reg_timer(struct kvm_vcpu *vcpu,
200 				 const struct kvm_one_reg *reg)
201 {
202 	struct kvm_vcpu_timer *t = &vcpu->arch.timer;
203 	struct kvm_guest_timer *gt = &vcpu->kvm->arch.timer;
204 	u64 __user *uaddr = (u64 __user *)(unsigned long)reg->addr;
205 	unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK |
206 					    KVM_REG_SIZE_MASK |
207 					    KVM_REG_RISCV_TIMER);
208 	u64 reg_val;
209 	int ret = 0;
210 
211 	if (KVM_REG_SIZE(reg->id) != sizeof(u64))
212 		return -EINVAL;
213 	if (reg_num >= sizeof(struct kvm_riscv_timer) / sizeof(u64))
214 		return -ENOENT;
215 
216 	if (copy_from_user(&reg_val, uaddr, KVM_REG_SIZE(reg->id)))
217 		return -EFAULT;
218 
219 	switch (reg_num) {
220 	case KVM_REG_RISCV_TIMER_REG(frequency):
221 		if (reg_val != riscv_timebase)
222 			return -EINVAL;
223 		break;
224 	case KVM_REG_RISCV_TIMER_REG(time):
225 		gt->time_delta = reg_val - get_cycles64();
226 		break;
227 	case KVM_REG_RISCV_TIMER_REG(compare):
228 		t->next_cycles = reg_val;
229 		break;
230 	case KVM_REG_RISCV_TIMER_REG(state):
231 		if (reg_val == KVM_RISCV_TIMER_STATE_ON)
232 			ret = kvm_riscv_vcpu_timer_next_event(vcpu, reg_val);
233 		else
234 			ret = kvm_riscv_vcpu_timer_cancel(t);
235 		break;
236 	default:
237 		ret = -ENOENT;
238 		break;
239 	}
240 
241 	return ret;
242 }
243 
kvm_riscv_vcpu_timer_init(struct kvm_vcpu * vcpu)244 int kvm_riscv_vcpu_timer_init(struct kvm_vcpu *vcpu)
245 {
246 	struct kvm_vcpu_timer *t = &vcpu->arch.timer;
247 
248 	if (t->init_done)
249 		return -EINVAL;
250 
251 	hrtimer_init(&t->hrt, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
252 	t->init_done = true;
253 	t->next_set = false;
254 
255 	/* Enable sstc for every vcpu if available in hardware */
256 	if (riscv_isa_extension_available(NULL, SSTC)) {
257 		t->sstc_enabled = true;
258 		t->hrt.function = kvm_riscv_vcpu_vstimer_expired;
259 		t->timer_next_event = kvm_riscv_vcpu_update_vstimecmp;
260 	} else {
261 		t->sstc_enabled = false;
262 		t->hrt.function = kvm_riscv_vcpu_hrtimer_expired;
263 		t->timer_next_event = kvm_riscv_vcpu_update_hrtimer;
264 	}
265 
266 	return 0;
267 }
268 
kvm_riscv_vcpu_timer_deinit(struct kvm_vcpu * vcpu)269 int kvm_riscv_vcpu_timer_deinit(struct kvm_vcpu *vcpu)
270 {
271 	int ret;
272 
273 	ret = kvm_riscv_vcpu_timer_cancel(&vcpu->arch.timer);
274 	vcpu->arch.timer.init_done = false;
275 
276 	return ret;
277 }
278 
kvm_riscv_vcpu_timer_reset(struct kvm_vcpu * vcpu)279 int kvm_riscv_vcpu_timer_reset(struct kvm_vcpu *vcpu)
280 {
281 	struct kvm_vcpu_timer *t = &vcpu->arch.timer;
282 
283 	t->next_cycles = -1ULL;
284 	return kvm_riscv_vcpu_timer_cancel(&vcpu->arch.timer);
285 }
286 
kvm_riscv_vcpu_update_timedelta(struct kvm_vcpu * vcpu)287 static void kvm_riscv_vcpu_update_timedelta(struct kvm_vcpu *vcpu)
288 {
289 	struct kvm_guest_timer *gt = &vcpu->kvm->arch.timer;
290 
291 #if defined(CONFIG_32BIT)
292 	csr_write(CSR_HTIMEDELTA, (u32)(gt->time_delta));
293 	csr_write(CSR_HTIMEDELTAH, (u32)(gt->time_delta >> 32));
294 #else
295 	csr_write(CSR_HTIMEDELTA, gt->time_delta);
296 #endif
297 }
298 
kvm_riscv_vcpu_timer_restore(struct kvm_vcpu * vcpu)299 void kvm_riscv_vcpu_timer_restore(struct kvm_vcpu *vcpu)
300 {
301 	struct kvm_vcpu_timer *t = &vcpu->arch.timer;
302 
303 	kvm_riscv_vcpu_update_timedelta(vcpu);
304 
305 	if (!t->sstc_enabled)
306 		return;
307 
308 #if defined(CONFIG_32BIT)
309 	csr_write(CSR_VSTIMECMP, (u32)t->next_cycles);
310 	csr_write(CSR_VSTIMECMPH, (u32)(t->next_cycles >> 32));
311 #else
312 	csr_write(CSR_VSTIMECMP, t->next_cycles);
313 #endif
314 
315 	/* timer should be enabled for the remaining operations */
316 	if (unlikely(!t->init_done))
317 		return;
318 
319 	kvm_riscv_vcpu_timer_unblocking(vcpu);
320 }
321 
kvm_riscv_vcpu_timer_sync(struct kvm_vcpu * vcpu)322 void kvm_riscv_vcpu_timer_sync(struct kvm_vcpu *vcpu)
323 {
324 	struct kvm_vcpu_timer *t = &vcpu->arch.timer;
325 
326 	if (!t->sstc_enabled)
327 		return;
328 
329 #if defined(CONFIG_32BIT)
330 	t->next_cycles = csr_read(CSR_VSTIMECMP);
331 	t->next_cycles |= (u64)csr_read(CSR_VSTIMECMPH) << 32;
332 #else
333 	t->next_cycles = csr_read(CSR_VSTIMECMP);
334 #endif
335 }
336 
kvm_riscv_vcpu_timer_save(struct kvm_vcpu * vcpu)337 void kvm_riscv_vcpu_timer_save(struct kvm_vcpu *vcpu)
338 {
339 	struct kvm_vcpu_timer *t = &vcpu->arch.timer;
340 
341 	if (!t->sstc_enabled)
342 		return;
343 
344 	/*
345 	 * The vstimecmp CSRs are saved by kvm_riscv_vcpu_timer_sync()
346 	 * upon every VM exit so no need to save here.
347 	 */
348 
349 	/* timer should be enabled for the remaining operations */
350 	if (unlikely(!t->init_done))
351 		return;
352 
353 	if (kvm_vcpu_is_blocking(vcpu))
354 		kvm_riscv_vcpu_timer_blocking(vcpu);
355 }
356 
kvm_riscv_guest_timer_init(struct kvm * kvm)357 void kvm_riscv_guest_timer_init(struct kvm *kvm)
358 {
359 	struct kvm_guest_timer *gt = &kvm->arch.timer;
360 
361 	riscv_cs_get_mult_shift(&gt->nsec_mult, &gt->nsec_shift);
362 	gt->time_delta = -get_cycles64();
363 }
364