1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (c) 2022 Ventana Micro Systems Inc.
4  */
5 
6 #include <linux/bitmap.h>
7 #include <linux/cpumask.h>
8 #include <linux/errno.h>
9 #include <linux/err.h>
10 #include <linux/module.h>
11 #include <linux/smp.h>
12 #include <linux/kvm_host.h>
13 #include <asm/cacheflush.h>
14 #include <asm/csr.h>
15 
16 /*
17  * Instruction encoding of hfence.gvma is:
18  * HFENCE.GVMA rs1, rs2
19  * HFENCE.GVMA zero, rs2
20  * HFENCE.GVMA rs1
21  * HFENCE.GVMA
22  *
23  * rs1!=zero and rs2!=zero ==> HFENCE.GVMA rs1, rs2
24  * rs1==zero and rs2!=zero ==> HFENCE.GVMA zero, rs2
25  * rs1!=zero and rs2==zero ==> HFENCE.GVMA rs1
26  * rs1==zero and rs2==zero ==> HFENCE.GVMA
27  *
28  * Instruction encoding of HFENCE.GVMA is:
29  * 0110001 rs2(5) rs1(5) 000 00000 1110011
30  */
31 
kvm_riscv_local_hfence_gvma_vmid_gpa(unsigned long vmid,gpa_t gpa,gpa_t gpsz,unsigned long order)32 void kvm_riscv_local_hfence_gvma_vmid_gpa(unsigned long vmid,
33 					  gpa_t gpa, gpa_t gpsz,
34 					  unsigned long order)
35 {
36 	gpa_t pos;
37 
38 	if (PTRS_PER_PTE < (gpsz >> order)) {
39 		kvm_riscv_local_hfence_gvma_vmid_all(vmid);
40 		return;
41 	}
42 
43 	for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order)) {
44 		/*
45 		 * rs1 = a0 (GPA >> 2)
46 		 * rs2 = a1 (VMID)
47 		 * HFENCE.GVMA a0, a1
48 		 * 0110001 01011 01010 000 00000 1110011
49 		 */
50 		asm volatile ("srli a0, %0, 2\n"
51 			      "add a1, %1, zero\n"
52 			      ".word 0x62b50073\n"
53 			      :: "r" (pos), "r" (vmid)
54 			      : "a0", "a1", "memory");
55 	}
56 }
57 
kvm_riscv_local_hfence_gvma_vmid_all(unsigned long vmid)58 void kvm_riscv_local_hfence_gvma_vmid_all(unsigned long vmid)
59 {
60 	/*
61 	 * rs1 = zero
62 	 * rs2 = a0 (VMID)
63 	 * HFENCE.GVMA zero, a0
64 	 * 0110001 01010 00000 000 00000 1110011
65 	 */
66 	asm volatile ("add a0, %0, zero\n"
67 		      ".word 0x62a00073\n"
68 		      :: "r" (vmid) : "a0", "memory");
69 }
70 
kvm_riscv_local_hfence_gvma_gpa(gpa_t gpa,gpa_t gpsz,unsigned long order)71 void kvm_riscv_local_hfence_gvma_gpa(gpa_t gpa, gpa_t gpsz,
72 				     unsigned long order)
73 {
74 	gpa_t pos;
75 
76 	if (PTRS_PER_PTE < (gpsz >> order)) {
77 		kvm_riscv_local_hfence_gvma_all();
78 		return;
79 	}
80 
81 	for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order)) {
82 		/*
83 		 * rs1 = a0 (GPA >> 2)
84 		 * rs2 = zero
85 		 * HFENCE.GVMA a0
86 		 * 0110001 00000 01010 000 00000 1110011
87 		 */
88 		asm volatile ("srli a0, %0, 2\n"
89 			      ".word 0x62050073\n"
90 			      :: "r" (pos) : "a0", "memory");
91 	}
92 }
93 
kvm_riscv_local_hfence_gvma_all(void)94 void kvm_riscv_local_hfence_gvma_all(void)
95 {
96 	/*
97 	 * rs1 = zero
98 	 * rs2 = zero
99 	 * HFENCE.GVMA
100 	 * 0110001 00000 00000 000 00000 1110011
101 	 */
102 	asm volatile (".word 0x62000073" ::: "memory");
103 }
104 
105 /*
106  * Instruction encoding of hfence.gvma is:
107  * HFENCE.VVMA rs1, rs2
108  * HFENCE.VVMA zero, rs2
109  * HFENCE.VVMA rs1
110  * HFENCE.VVMA
111  *
112  * rs1!=zero and rs2!=zero ==> HFENCE.VVMA rs1, rs2
113  * rs1==zero and rs2!=zero ==> HFENCE.VVMA zero, rs2
114  * rs1!=zero and rs2==zero ==> HFENCE.VVMA rs1
115  * rs1==zero and rs2==zero ==> HFENCE.VVMA
116  *
117  * Instruction encoding of HFENCE.VVMA is:
118  * 0010001 rs2(5) rs1(5) 000 00000 1110011
119  */
120 
kvm_riscv_local_hfence_vvma_asid_gva(unsigned long vmid,unsigned long asid,unsigned long gva,unsigned long gvsz,unsigned long order)121 void kvm_riscv_local_hfence_vvma_asid_gva(unsigned long vmid,
122 					  unsigned long asid,
123 					  unsigned long gva,
124 					  unsigned long gvsz,
125 					  unsigned long order)
126 {
127 	unsigned long pos, hgatp;
128 
129 	if (PTRS_PER_PTE < (gvsz >> order)) {
130 		kvm_riscv_local_hfence_vvma_asid_all(vmid, asid);
131 		return;
132 	}
133 
134 	hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT);
135 
136 	for (pos = gva; pos < (gva + gvsz); pos += BIT(order)) {
137 		/*
138 		 * rs1 = a0 (GVA)
139 		 * rs2 = a1 (ASID)
140 		 * HFENCE.VVMA a0, a1
141 		 * 0010001 01011 01010 000 00000 1110011
142 		 */
143 		asm volatile ("add a0, %0, zero\n"
144 			      "add a1, %1, zero\n"
145 			      ".word 0x22b50073\n"
146 			      :: "r" (pos), "r" (asid)
147 			      : "a0", "a1", "memory");
148 	}
149 
150 	csr_write(CSR_HGATP, hgatp);
151 }
152 
kvm_riscv_local_hfence_vvma_asid_all(unsigned long vmid,unsigned long asid)153 void kvm_riscv_local_hfence_vvma_asid_all(unsigned long vmid,
154 					  unsigned long asid)
155 {
156 	unsigned long hgatp;
157 
158 	hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT);
159 
160 	/*
161 	 * rs1 = zero
162 	 * rs2 = a0 (ASID)
163 	 * HFENCE.VVMA zero, a0
164 	 * 0010001 01010 00000 000 00000 1110011
165 	 */
166 	asm volatile ("add a0, %0, zero\n"
167 		      ".word 0x22a00073\n"
168 		      :: "r" (asid) : "a0", "memory");
169 
170 	csr_write(CSR_HGATP, hgatp);
171 }
172 
kvm_riscv_local_hfence_vvma_gva(unsigned long vmid,unsigned long gva,unsigned long gvsz,unsigned long order)173 void kvm_riscv_local_hfence_vvma_gva(unsigned long vmid,
174 				     unsigned long gva, unsigned long gvsz,
175 				     unsigned long order)
176 {
177 	unsigned long pos, hgatp;
178 
179 	if (PTRS_PER_PTE < (gvsz >> order)) {
180 		kvm_riscv_local_hfence_vvma_all(vmid);
181 		return;
182 	}
183 
184 	hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT);
185 
186 	for (pos = gva; pos < (gva + gvsz); pos += BIT(order)) {
187 		/*
188 		 * rs1 = a0 (GVA)
189 		 * rs2 = zero
190 		 * HFENCE.VVMA a0
191 		 * 0010001 00000 01010 000 00000 1110011
192 		 */
193 		asm volatile ("add a0, %0, zero\n"
194 			      ".word 0x22050073\n"
195 			      :: "r" (pos) : "a0", "memory");
196 	}
197 
198 	csr_write(CSR_HGATP, hgatp);
199 }
200 
kvm_riscv_local_hfence_vvma_all(unsigned long vmid)201 void kvm_riscv_local_hfence_vvma_all(unsigned long vmid)
202 {
203 	unsigned long hgatp;
204 
205 	hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT);
206 
207 	/*
208 	 * rs1 = zero
209 	 * rs2 = zero
210 	 * HFENCE.VVMA
211 	 * 0010001 00000 00000 000 00000 1110011
212 	 */
213 	asm volatile (".word 0x22000073" ::: "memory");
214 
215 	csr_write(CSR_HGATP, hgatp);
216 }
217 
kvm_riscv_local_tlb_sanitize(struct kvm_vcpu * vcpu)218 void kvm_riscv_local_tlb_sanitize(struct kvm_vcpu *vcpu)
219 {
220 	unsigned long vmid;
221 
222 	if (!kvm_riscv_gstage_vmid_bits() ||
223 	    vcpu->arch.last_exit_cpu == vcpu->cpu)
224 		return;
225 
226 	/*
227 	 * On RISC-V platforms with hardware VMID support, we share same
228 	 * VMID for all VCPUs of a particular Guest/VM. This means we might
229 	 * have stale G-stage TLB entries on the current Host CPU due to
230 	 * some other VCPU of the same Guest which ran previously on the
231 	 * current Host CPU.
232 	 *
233 	 * To cleanup stale TLB entries, we simply flush all G-stage TLB
234 	 * entries by VMID whenever underlying Host CPU changes for a VCPU.
235 	 */
236 
237 	vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid);
238 	kvm_riscv_local_hfence_gvma_vmid_all(vmid);
239 }
240 
kvm_riscv_fence_i_process(struct kvm_vcpu * vcpu)241 void kvm_riscv_fence_i_process(struct kvm_vcpu *vcpu)
242 {
243 	local_flush_icache_all();
244 }
245 
kvm_riscv_hfence_gvma_vmid_all_process(struct kvm_vcpu * vcpu)246 void kvm_riscv_hfence_gvma_vmid_all_process(struct kvm_vcpu *vcpu)
247 {
248 	struct kvm_vmid *vmid;
249 
250 	vmid = &vcpu->kvm->arch.vmid;
251 	kvm_riscv_local_hfence_gvma_vmid_all(READ_ONCE(vmid->vmid));
252 }
253 
kvm_riscv_hfence_vvma_all_process(struct kvm_vcpu * vcpu)254 void kvm_riscv_hfence_vvma_all_process(struct kvm_vcpu *vcpu)
255 {
256 	struct kvm_vmid *vmid;
257 
258 	vmid = &vcpu->kvm->arch.vmid;
259 	kvm_riscv_local_hfence_vvma_all(READ_ONCE(vmid->vmid));
260 }
261 
vcpu_hfence_dequeue(struct kvm_vcpu * vcpu,struct kvm_riscv_hfence * out_data)262 static bool vcpu_hfence_dequeue(struct kvm_vcpu *vcpu,
263 				struct kvm_riscv_hfence *out_data)
264 {
265 	bool ret = false;
266 	struct kvm_vcpu_arch *varch = &vcpu->arch;
267 
268 	spin_lock(&varch->hfence_lock);
269 
270 	if (varch->hfence_queue[varch->hfence_head].type) {
271 		memcpy(out_data, &varch->hfence_queue[varch->hfence_head],
272 		       sizeof(*out_data));
273 		varch->hfence_queue[varch->hfence_head].type = 0;
274 
275 		varch->hfence_head++;
276 		if (varch->hfence_head == KVM_RISCV_VCPU_MAX_HFENCE)
277 			varch->hfence_head = 0;
278 
279 		ret = true;
280 	}
281 
282 	spin_unlock(&varch->hfence_lock);
283 
284 	return ret;
285 }
286 
vcpu_hfence_enqueue(struct kvm_vcpu * vcpu,const struct kvm_riscv_hfence * data)287 static bool vcpu_hfence_enqueue(struct kvm_vcpu *vcpu,
288 				const struct kvm_riscv_hfence *data)
289 {
290 	bool ret = false;
291 	struct kvm_vcpu_arch *varch = &vcpu->arch;
292 
293 	spin_lock(&varch->hfence_lock);
294 
295 	if (!varch->hfence_queue[varch->hfence_tail].type) {
296 		memcpy(&varch->hfence_queue[varch->hfence_tail],
297 		       data, sizeof(*data));
298 
299 		varch->hfence_tail++;
300 		if (varch->hfence_tail == KVM_RISCV_VCPU_MAX_HFENCE)
301 			varch->hfence_tail = 0;
302 
303 		ret = true;
304 	}
305 
306 	spin_unlock(&varch->hfence_lock);
307 
308 	return ret;
309 }
310 
kvm_riscv_hfence_process(struct kvm_vcpu * vcpu)311 void kvm_riscv_hfence_process(struct kvm_vcpu *vcpu)
312 {
313 	struct kvm_riscv_hfence d = { 0 };
314 	struct kvm_vmid *v = &vcpu->kvm->arch.vmid;
315 
316 	while (vcpu_hfence_dequeue(vcpu, &d)) {
317 		switch (d.type) {
318 		case KVM_RISCV_HFENCE_UNKNOWN:
319 			break;
320 		case KVM_RISCV_HFENCE_GVMA_VMID_GPA:
321 			kvm_riscv_local_hfence_gvma_vmid_gpa(
322 						READ_ONCE(v->vmid),
323 						d.addr, d.size, d.order);
324 			break;
325 		case KVM_RISCV_HFENCE_VVMA_ASID_GVA:
326 			kvm_riscv_local_hfence_vvma_asid_gva(
327 						READ_ONCE(v->vmid), d.asid,
328 						d.addr, d.size, d.order);
329 			break;
330 		case KVM_RISCV_HFENCE_VVMA_ASID_ALL:
331 			kvm_riscv_local_hfence_vvma_asid_all(
332 						READ_ONCE(v->vmid), d.asid);
333 			break;
334 		case KVM_RISCV_HFENCE_VVMA_GVA:
335 			kvm_riscv_local_hfence_vvma_gva(
336 						READ_ONCE(v->vmid),
337 						d.addr, d.size, d.order);
338 			break;
339 		default:
340 			break;
341 		}
342 	}
343 }
344 
make_xfence_request(struct kvm * kvm,unsigned long hbase,unsigned long hmask,unsigned int req,unsigned int fallback_req,const struct kvm_riscv_hfence * data)345 static void make_xfence_request(struct kvm *kvm,
346 				unsigned long hbase, unsigned long hmask,
347 				unsigned int req, unsigned int fallback_req,
348 				const struct kvm_riscv_hfence *data)
349 {
350 	unsigned long i;
351 	struct kvm_vcpu *vcpu;
352 	unsigned int actual_req = req;
353 	DECLARE_BITMAP(vcpu_mask, KVM_MAX_VCPUS);
354 
355 	bitmap_clear(vcpu_mask, 0, KVM_MAX_VCPUS);
356 	kvm_for_each_vcpu(i, vcpu, kvm) {
357 		if (hbase != -1UL) {
358 			if (vcpu->vcpu_id < hbase)
359 				continue;
360 			if (!(hmask & (1UL << (vcpu->vcpu_id - hbase))))
361 				continue;
362 		}
363 
364 		bitmap_set(vcpu_mask, i, 1);
365 
366 		if (!data || !data->type)
367 			continue;
368 
369 		/*
370 		 * Enqueue hfence data to VCPU hfence queue. If we don't
371 		 * have space in the VCPU hfence queue then fallback to
372 		 * a more conservative hfence request.
373 		 */
374 		if (!vcpu_hfence_enqueue(vcpu, data))
375 			actual_req = fallback_req;
376 	}
377 
378 	kvm_make_vcpus_request_mask(kvm, actual_req, vcpu_mask);
379 }
380 
kvm_riscv_fence_i(struct kvm * kvm,unsigned long hbase,unsigned long hmask)381 void kvm_riscv_fence_i(struct kvm *kvm,
382 		       unsigned long hbase, unsigned long hmask)
383 {
384 	make_xfence_request(kvm, hbase, hmask, KVM_REQ_FENCE_I,
385 			    KVM_REQ_FENCE_I, NULL);
386 }
387 
kvm_riscv_hfence_gvma_vmid_gpa(struct kvm * kvm,unsigned long hbase,unsigned long hmask,gpa_t gpa,gpa_t gpsz,unsigned long order)388 void kvm_riscv_hfence_gvma_vmid_gpa(struct kvm *kvm,
389 				    unsigned long hbase, unsigned long hmask,
390 				    gpa_t gpa, gpa_t gpsz,
391 				    unsigned long order)
392 {
393 	struct kvm_riscv_hfence data;
394 
395 	data.type = KVM_RISCV_HFENCE_GVMA_VMID_GPA;
396 	data.asid = 0;
397 	data.addr = gpa;
398 	data.size = gpsz;
399 	data.order = order;
400 	make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE,
401 			    KVM_REQ_HFENCE_GVMA_VMID_ALL, &data);
402 }
403 
kvm_riscv_hfence_gvma_vmid_all(struct kvm * kvm,unsigned long hbase,unsigned long hmask)404 void kvm_riscv_hfence_gvma_vmid_all(struct kvm *kvm,
405 				    unsigned long hbase, unsigned long hmask)
406 {
407 	make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE_GVMA_VMID_ALL,
408 			    KVM_REQ_HFENCE_GVMA_VMID_ALL, NULL);
409 }
410 
kvm_riscv_hfence_vvma_asid_gva(struct kvm * kvm,unsigned long hbase,unsigned long hmask,unsigned long gva,unsigned long gvsz,unsigned long order,unsigned long asid)411 void kvm_riscv_hfence_vvma_asid_gva(struct kvm *kvm,
412 				    unsigned long hbase, unsigned long hmask,
413 				    unsigned long gva, unsigned long gvsz,
414 				    unsigned long order, unsigned long asid)
415 {
416 	struct kvm_riscv_hfence data;
417 
418 	data.type = KVM_RISCV_HFENCE_VVMA_ASID_GVA;
419 	data.asid = asid;
420 	data.addr = gva;
421 	data.size = gvsz;
422 	data.order = order;
423 	make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE,
424 			    KVM_REQ_HFENCE_VVMA_ALL, &data);
425 }
426 
kvm_riscv_hfence_vvma_asid_all(struct kvm * kvm,unsigned long hbase,unsigned long hmask,unsigned long asid)427 void kvm_riscv_hfence_vvma_asid_all(struct kvm *kvm,
428 				    unsigned long hbase, unsigned long hmask,
429 				    unsigned long asid)
430 {
431 	struct kvm_riscv_hfence data;
432 
433 	data.type = KVM_RISCV_HFENCE_VVMA_ASID_ALL;
434 	data.asid = asid;
435 	data.addr = data.size = data.order = 0;
436 	make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE,
437 			    KVM_REQ_HFENCE_VVMA_ALL, &data);
438 }
439 
kvm_riscv_hfence_vvma_gva(struct kvm * kvm,unsigned long hbase,unsigned long hmask,unsigned long gva,unsigned long gvsz,unsigned long order)440 void kvm_riscv_hfence_vvma_gva(struct kvm *kvm,
441 			       unsigned long hbase, unsigned long hmask,
442 			       unsigned long gva, unsigned long gvsz,
443 			       unsigned long order)
444 {
445 	struct kvm_riscv_hfence data;
446 
447 	data.type = KVM_RISCV_HFENCE_VVMA_GVA;
448 	data.asid = 0;
449 	data.addr = gva;
450 	data.size = gvsz;
451 	data.order = order;
452 	make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE,
453 			    KVM_REQ_HFENCE_VVMA_ALL, &data);
454 }
455 
kvm_riscv_hfence_vvma_all(struct kvm * kvm,unsigned long hbase,unsigned long hmask)456 void kvm_riscv_hfence_vvma_all(struct kvm *kvm,
457 			       unsigned long hbase, unsigned long hmask)
458 {
459 	make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE_VVMA_ALL,
460 			    KVM_REQ_HFENCE_VVMA_ALL, NULL);
461 }
462