1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (c) 2022 Ventana Micro Systems Inc.
4 */
5
6 #include <linux/bitmap.h>
7 #include <linux/cpumask.h>
8 #include <linux/errno.h>
9 #include <linux/err.h>
10 #include <linux/module.h>
11 #include <linux/smp.h>
12 #include <linux/kvm_host.h>
13 #include <asm/cacheflush.h>
14 #include <asm/csr.h>
15
16 /*
17 * Instruction encoding of hfence.gvma is:
18 * HFENCE.GVMA rs1, rs2
19 * HFENCE.GVMA zero, rs2
20 * HFENCE.GVMA rs1
21 * HFENCE.GVMA
22 *
23 * rs1!=zero and rs2!=zero ==> HFENCE.GVMA rs1, rs2
24 * rs1==zero and rs2!=zero ==> HFENCE.GVMA zero, rs2
25 * rs1!=zero and rs2==zero ==> HFENCE.GVMA rs1
26 * rs1==zero and rs2==zero ==> HFENCE.GVMA
27 *
28 * Instruction encoding of HFENCE.GVMA is:
29 * 0110001 rs2(5) rs1(5) 000 00000 1110011
30 */
31
kvm_riscv_local_hfence_gvma_vmid_gpa(unsigned long vmid,gpa_t gpa,gpa_t gpsz,unsigned long order)32 void kvm_riscv_local_hfence_gvma_vmid_gpa(unsigned long vmid,
33 gpa_t gpa, gpa_t gpsz,
34 unsigned long order)
35 {
36 gpa_t pos;
37
38 if (PTRS_PER_PTE < (gpsz >> order)) {
39 kvm_riscv_local_hfence_gvma_vmid_all(vmid);
40 return;
41 }
42
43 for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order)) {
44 /*
45 * rs1 = a0 (GPA >> 2)
46 * rs2 = a1 (VMID)
47 * HFENCE.GVMA a0, a1
48 * 0110001 01011 01010 000 00000 1110011
49 */
50 asm volatile ("srli a0, %0, 2\n"
51 "add a1, %1, zero\n"
52 ".word 0x62b50073\n"
53 :: "r" (pos), "r" (vmid)
54 : "a0", "a1", "memory");
55 }
56 }
57
kvm_riscv_local_hfence_gvma_vmid_all(unsigned long vmid)58 void kvm_riscv_local_hfence_gvma_vmid_all(unsigned long vmid)
59 {
60 /*
61 * rs1 = zero
62 * rs2 = a0 (VMID)
63 * HFENCE.GVMA zero, a0
64 * 0110001 01010 00000 000 00000 1110011
65 */
66 asm volatile ("add a0, %0, zero\n"
67 ".word 0x62a00073\n"
68 :: "r" (vmid) : "a0", "memory");
69 }
70
kvm_riscv_local_hfence_gvma_gpa(gpa_t gpa,gpa_t gpsz,unsigned long order)71 void kvm_riscv_local_hfence_gvma_gpa(gpa_t gpa, gpa_t gpsz,
72 unsigned long order)
73 {
74 gpa_t pos;
75
76 if (PTRS_PER_PTE < (gpsz >> order)) {
77 kvm_riscv_local_hfence_gvma_all();
78 return;
79 }
80
81 for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order)) {
82 /*
83 * rs1 = a0 (GPA >> 2)
84 * rs2 = zero
85 * HFENCE.GVMA a0
86 * 0110001 00000 01010 000 00000 1110011
87 */
88 asm volatile ("srli a0, %0, 2\n"
89 ".word 0x62050073\n"
90 :: "r" (pos) : "a0", "memory");
91 }
92 }
93
kvm_riscv_local_hfence_gvma_all(void)94 void kvm_riscv_local_hfence_gvma_all(void)
95 {
96 /*
97 * rs1 = zero
98 * rs2 = zero
99 * HFENCE.GVMA
100 * 0110001 00000 00000 000 00000 1110011
101 */
102 asm volatile (".word 0x62000073" ::: "memory");
103 }
104
105 /*
106 * Instruction encoding of hfence.gvma is:
107 * HFENCE.VVMA rs1, rs2
108 * HFENCE.VVMA zero, rs2
109 * HFENCE.VVMA rs1
110 * HFENCE.VVMA
111 *
112 * rs1!=zero and rs2!=zero ==> HFENCE.VVMA rs1, rs2
113 * rs1==zero and rs2!=zero ==> HFENCE.VVMA zero, rs2
114 * rs1!=zero and rs2==zero ==> HFENCE.VVMA rs1
115 * rs1==zero and rs2==zero ==> HFENCE.VVMA
116 *
117 * Instruction encoding of HFENCE.VVMA is:
118 * 0010001 rs2(5) rs1(5) 000 00000 1110011
119 */
120
kvm_riscv_local_hfence_vvma_asid_gva(unsigned long vmid,unsigned long asid,unsigned long gva,unsigned long gvsz,unsigned long order)121 void kvm_riscv_local_hfence_vvma_asid_gva(unsigned long vmid,
122 unsigned long asid,
123 unsigned long gva,
124 unsigned long gvsz,
125 unsigned long order)
126 {
127 unsigned long pos, hgatp;
128
129 if (PTRS_PER_PTE < (gvsz >> order)) {
130 kvm_riscv_local_hfence_vvma_asid_all(vmid, asid);
131 return;
132 }
133
134 hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT);
135
136 for (pos = gva; pos < (gva + gvsz); pos += BIT(order)) {
137 /*
138 * rs1 = a0 (GVA)
139 * rs2 = a1 (ASID)
140 * HFENCE.VVMA a0, a1
141 * 0010001 01011 01010 000 00000 1110011
142 */
143 asm volatile ("add a0, %0, zero\n"
144 "add a1, %1, zero\n"
145 ".word 0x22b50073\n"
146 :: "r" (pos), "r" (asid)
147 : "a0", "a1", "memory");
148 }
149
150 csr_write(CSR_HGATP, hgatp);
151 }
152
kvm_riscv_local_hfence_vvma_asid_all(unsigned long vmid,unsigned long asid)153 void kvm_riscv_local_hfence_vvma_asid_all(unsigned long vmid,
154 unsigned long asid)
155 {
156 unsigned long hgatp;
157
158 hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT);
159
160 /*
161 * rs1 = zero
162 * rs2 = a0 (ASID)
163 * HFENCE.VVMA zero, a0
164 * 0010001 01010 00000 000 00000 1110011
165 */
166 asm volatile ("add a0, %0, zero\n"
167 ".word 0x22a00073\n"
168 :: "r" (asid) : "a0", "memory");
169
170 csr_write(CSR_HGATP, hgatp);
171 }
172
kvm_riscv_local_hfence_vvma_gva(unsigned long vmid,unsigned long gva,unsigned long gvsz,unsigned long order)173 void kvm_riscv_local_hfence_vvma_gva(unsigned long vmid,
174 unsigned long gva, unsigned long gvsz,
175 unsigned long order)
176 {
177 unsigned long pos, hgatp;
178
179 if (PTRS_PER_PTE < (gvsz >> order)) {
180 kvm_riscv_local_hfence_vvma_all(vmid);
181 return;
182 }
183
184 hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT);
185
186 for (pos = gva; pos < (gva + gvsz); pos += BIT(order)) {
187 /*
188 * rs1 = a0 (GVA)
189 * rs2 = zero
190 * HFENCE.VVMA a0
191 * 0010001 00000 01010 000 00000 1110011
192 */
193 asm volatile ("add a0, %0, zero\n"
194 ".word 0x22050073\n"
195 :: "r" (pos) : "a0", "memory");
196 }
197
198 csr_write(CSR_HGATP, hgatp);
199 }
200
kvm_riscv_local_hfence_vvma_all(unsigned long vmid)201 void kvm_riscv_local_hfence_vvma_all(unsigned long vmid)
202 {
203 unsigned long hgatp;
204
205 hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT);
206
207 /*
208 * rs1 = zero
209 * rs2 = zero
210 * HFENCE.VVMA
211 * 0010001 00000 00000 000 00000 1110011
212 */
213 asm volatile (".word 0x22000073" ::: "memory");
214
215 csr_write(CSR_HGATP, hgatp);
216 }
217
kvm_riscv_local_tlb_sanitize(struct kvm_vcpu * vcpu)218 void kvm_riscv_local_tlb_sanitize(struct kvm_vcpu *vcpu)
219 {
220 unsigned long vmid;
221
222 if (!kvm_riscv_gstage_vmid_bits() ||
223 vcpu->arch.last_exit_cpu == vcpu->cpu)
224 return;
225
226 /*
227 * On RISC-V platforms with hardware VMID support, we share same
228 * VMID for all VCPUs of a particular Guest/VM. This means we might
229 * have stale G-stage TLB entries on the current Host CPU due to
230 * some other VCPU of the same Guest which ran previously on the
231 * current Host CPU.
232 *
233 * To cleanup stale TLB entries, we simply flush all G-stage TLB
234 * entries by VMID whenever underlying Host CPU changes for a VCPU.
235 */
236
237 vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid);
238 kvm_riscv_local_hfence_gvma_vmid_all(vmid);
239 }
240
kvm_riscv_fence_i_process(struct kvm_vcpu * vcpu)241 void kvm_riscv_fence_i_process(struct kvm_vcpu *vcpu)
242 {
243 local_flush_icache_all();
244 }
245
kvm_riscv_hfence_gvma_vmid_all_process(struct kvm_vcpu * vcpu)246 void kvm_riscv_hfence_gvma_vmid_all_process(struct kvm_vcpu *vcpu)
247 {
248 struct kvm_vmid *vmid;
249
250 vmid = &vcpu->kvm->arch.vmid;
251 kvm_riscv_local_hfence_gvma_vmid_all(READ_ONCE(vmid->vmid));
252 }
253
kvm_riscv_hfence_vvma_all_process(struct kvm_vcpu * vcpu)254 void kvm_riscv_hfence_vvma_all_process(struct kvm_vcpu *vcpu)
255 {
256 struct kvm_vmid *vmid;
257
258 vmid = &vcpu->kvm->arch.vmid;
259 kvm_riscv_local_hfence_vvma_all(READ_ONCE(vmid->vmid));
260 }
261
vcpu_hfence_dequeue(struct kvm_vcpu * vcpu,struct kvm_riscv_hfence * out_data)262 static bool vcpu_hfence_dequeue(struct kvm_vcpu *vcpu,
263 struct kvm_riscv_hfence *out_data)
264 {
265 bool ret = false;
266 struct kvm_vcpu_arch *varch = &vcpu->arch;
267
268 spin_lock(&varch->hfence_lock);
269
270 if (varch->hfence_queue[varch->hfence_head].type) {
271 memcpy(out_data, &varch->hfence_queue[varch->hfence_head],
272 sizeof(*out_data));
273 varch->hfence_queue[varch->hfence_head].type = 0;
274
275 varch->hfence_head++;
276 if (varch->hfence_head == KVM_RISCV_VCPU_MAX_HFENCE)
277 varch->hfence_head = 0;
278
279 ret = true;
280 }
281
282 spin_unlock(&varch->hfence_lock);
283
284 return ret;
285 }
286
vcpu_hfence_enqueue(struct kvm_vcpu * vcpu,const struct kvm_riscv_hfence * data)287 static bool vcpu_hfence_enqueue(struct kvm_vcpu *vcpu,
288 const struct kvm_riscv_hfence *data)
289 {
290 bool ret = false;
291 struct kvm_vcpu_arch *varch = &vcpu->arch;
292
293 spin_lock(&varch->hfence_lock);
294
295 if (!varch->hfence_queue[varch->hfence_tail].type) {
296 memcpy(&varch->hfence_queue[varch->hfence_tail],
297 data, sizeof(*data));
298
299 varch->hfence_tail++;
300 if (varch->hfence_tail == KVM_RISCV_VCPU_MAX_HFENCE)
301 varch->hfence_tail = 0;
302
303 ret = true;
304 }
305
306 spin_unlock(&varch->hfence_lock);
307
308 return ret;
309 }
310
kvm_riscv_hfence_process(struct kvm_vcpu * vcpu)311 void kvm_riscv_hfence_process(struct kvm_vcpu *vcpu)
312 {
313 struct kvm_riscv_hfence d = { 0 };
314 struct kvm_vmid *v = &vcpu->kvm->arch.vmid;
315
316 while (vcpu_hfence_dequeue(vcpu, &d)) {
317 switch (d.type) {
318 case KVM_RISCV_HFENCE_UNKNOWN:
319 break;
320 case KVM_RISCV_HFENCE_GVMA_VMID_GPA:
321 kvm_riscv_local_hfence_gvma_vmid_gpa(
322 READ_ONCE(v->vmid),
323 d.addr, d.size, d.order);
324 break;
325 case KVM_RISCV_HFENCE_VVMA_ASID_GVA:
326 kvm_riscv_local_hfence_vvma_asid_gva(
327 READ_ONCE(v->vmid), d.asid,
328 d.addr, d.size, d.order);
329 break;
330 case KVM_RISCV_HFENCE_VVMA_ASID_ALL:
331 kvm_riscv_local_hfence_vvma_asid_all(
332 READ_ONCE(v->vmid), d.asid);
333 break;
334 case KVM_RISCV_HFENCE_VVMA_GVA:
335 kvm_riscv_local_hfence_vvma_gva(
336 READ_ONCE(v->vmid),
337 d.addr, d.size, d.order);
338 break;
339 default:
340 break;
341 }
342 }
343 }
344
make_xfence_request(struct kvm * kvm,unsigned long hbase,unsigned long hmask,unsigned int req,unsigned int fallback_req,const struct kvm_riscv_hfence * data)345 static void make_xfence_request(struct kvm *kvm,
346 unsigned long hbase, unsigned long hmask,
347 unsigned int req, unsigned int fallback_req,
348 const struct kvm_riscv_hfence *data)
349 {
350 unsigned long i;
351 struct kvm_vcpu *vcpu;
352 unsigned int actual_req = req;
353 DECLARE_BITMAP(vcpu_mask, KVM_MAX_VCPUS);
354
355 bitmap_clear(vcpu_mask, 0, KVM_MAX_VCPUS);
356 kvm_for_each_vcpu(i, vcpu, kvm) {
357 if (hbase != -1UL) {
358 if (vcpu->vcpu_id < hbase)
359 continue;
360 if (!(hmask & (1UL << (vcpu->vcpu_id - hbase))))
361 continue;
362 }
363
364 bitmap_set(vcpu_mask, i, 1);
365
366 if (!data || !data->type)
367 continue;
368
369 /*
370 * Enqueue hfence data to VCPU hfence queue. If we don't
371 * have space in the VCPU hfence queue then fallback to
372 * a more conservative hfence request.
373 */
374 if (!vcpu_hfence_enqueue(vcpu, data))
375 actual_req = fallback_req;
376 }
377
378 kvm_make_vcpus_request_mask(kvm, actual_req, vcpu_mask);
379 }
380
kvm_riscv_fence_i(struct kvm * kvm,unsigned long hbase,unsigned long hmask)381 void kvm_riscv_fence_i(struct kvm *kvm,
382 unsigned long hbase, unsigned long hmask)
383 {
384 make_xfence_request(kvm, hbase, hmask, KVM_REQ_FENCE_I,
385 KVM_REQ_FENCE_I, NULL);
386 }
387
kvm_riscv_hfence_gvma_vmid_gpa(struct kvm * kvm,unsigned long hbase,unsigned long hmask,gpa_t gpa,gpa_t gpsz,unsigned long order)388 void kvm_riscv_hfence_gvma_vmid_gpa(struct kvm *kvm,
389 unsigned long hbase, unsigned long hmask,
390 gpa_t gpa, gpa_t gpsz,
391 unsigned long order)
392 {
393 struct kvm_riscv_hfence data;
394
395 data.type = KVM_RISCV_HFENCE_GVMA_VMID_GPA;
396 data.asid = 0;
397 data.addr = gpa;
398 data.size = gpsz;
399 data.order = order;
400 make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE,
401 KVM_REQ_HFENCE_GVMA_VMID_ALL, &data);
402 }
403
kvm_riscv_hfence_gvma_vmid_all(struct kvm * kvm,unsigned long hbase,unsigned long hmask)404 void kvm_riscv_hfence_gvma_vmid_all(struct kvm *kvm,
405 unsigned long hbase, unsigned long hmask)
406 {
407 make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE_GVMA_VMID_ALL,
408 KVM_REQ_HFENCE_GVMA_VMID_ALL, NULL);
409 }
410
kvm_riscv_hfence_vvma_asid_gva(struct kvm * kvm,unsigned long hbase,unsigned long hmask,unsigned long gva,unsigned long gvsz,unsigned long order,unsigned long asid)411 void kvm_riscv_hfence_vvma_asid_gva(struct kvm *kvm,
412 unsigned long hbase, unsigned long hmask,
413 unsigned long gva, unsigned long gvsz,
414 unsigned long order, unsigned long asid)
415 {
416 struct kvm_riscv_hfence data;
417
418 data.type = KVM_RISCV_HFENCE_VVMA_ASID_GVA;
419 data.asid = asid;
420 data.addr = gva;
421 data.size = gvsz;
422 data.order = order;
423 make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE,
424 KVM_REQ_HFENCE_VVMA_ALL, &data);
425 }
426
kvm_riscv_hfence_vvma_asid_all(struct kvm * kvm,unsigned long hbase,unsigned long hmask,unsigned long asid)427 void kvm_riscv_hfence_vvma_asid_all(struct kvm *kvm,
428 unsigned long hbase, unsigned long hmask,
429 unsigned long asid)
430 {
431 struct kvm_riscv_hfence data;
432
433 data.type = KVM_RISCV_HFENCE_VVMA_ASID_ALL;
434 data.asid = asid;
435 data.addr = data.size = data.order = 0;
436 make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE,
437 KVM_REQ_HFENCE_VVMA_ALL, &data);
438 }
439
kvm_riscv_hfence_vvma_gva(struct kvm * kvm,unsigned long hbase,unsigned long hmask,unsigned long gva,unsigned long gvsz,unsigned long order)440 void kvm_riscv_hfence_vvma_gva(struct kvm *kvm,
441 unsigned long hbase, unsigned long hmask,
442 unsigned long gva, unsigned long gvsz,
443 unsigned long order)
444 {
445 struct kvm_riscv_hfence data;
446
447 data.type = KVM_RISCV_HFENCE_VVMA_GVA;
448 data.asid = 0;
449 data.addr = gva;
450 data.size = gvsz;
451 data.order = order;
452 make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE,
453 KVM_REQ_HFENCE_VVMA_ALL, &data);
454 }
455
kvm_riscv_hfence_vvma_all(struct kvm * kvm,unsigned long hbase,unsigned long hmask)456 void kvm_riscv_hfence_vvma_all(struct kvm *kvm,
457 unsigned long hbase, unsigned long hmask)
458 {
459 make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE_VVMA_ALL,
460 KVM_REQ_HFENCE_VVMA_ALL, NULL);
461 }
462