1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2012 - Virtual Open Systems and Columbia University
4  * Author: Christoffer Dall <c.dall@virtualopensystems.com>
5  */
6 
7 #include <linux/mman.h>
8 #include <linux/kvm_host.h>
9 #include <linux/io.h>
10 #include <linux/hugetlb.h>
11 #include <linux/sched/signal.h>
12 #include <trace/events/kvm.h>
13 #include <asm/pgalloc.h>
14 #include <asm/cacheflush.h>
15 #include <asm/kvm_arm.h>
16 #include <asm/kvm_mmu.h>
17 #include <asm/kvm_pgtable.h>
18 #include <asm/kvm_ras.h>
19 #include <asm/kvm_asm.h>
20 #include <asm/kvm_emulate.h>
21 #include <asm/virt.h>
22 
23 #include "trace.h"
24 
25 static struct kvm_pgtable *hyp_pgtable;
26 static DEFINE_MUTEX(kvm_hyp_pgd_mutex);
27 
28 static unsigned long __ro_after_init hyp_idmap_start;
29 static unsigned long __ro_after_init hyp_idmap_end;
30 static phys_addr_t __ro_after_init hyp_idmap_vector;
31 
32 static unsigned long __ro_after_init io_map_base;
33 
__stage2_range_addr_end(phys_addr_t addr,phys_addr_t end,phys_addr_t size)34 static phys_addr_t __stage2_range_addr_end(phys_addr_t addr, phys_addr_t end,
35 					   phys_addr_t size)
36 {
37 	phys_addr_t boundary = ALIGN_DOWN(addr + size, size);
38 
39 	return (boundary - 1 < end - 1) ? boundary : end;
40 }
41 
stage2_range_addr_end(phys_addr_t addr,phys_addr_t end)42 static phys_addr_t stage2_range_addr_end(phys_addr_t addr, phys_addr_t end)
43 {
44 	phys_addr_t size = kvm_granule_size(KVM_PGTABLE_MIN_BLOCK_LEVEL);
45 
46 	return __stage2_range_addr_end(addr, end, size);
47 }
48 
49 /*
50  * Release kvm_mmu_lock periodically if the memory region is large. Otherwise,
51  * we may see kernel panics with CONFIG_DETECT_HUNG_TASK,
52  * CONFIG_LOCKUP_DETECTOR, CONFIG_LOCKDEP. Additionally, holding the lock too
53  * long will also starve other vCPUs. We have to also make sure that the page
54  * tables are not freed while we released the lock.
55  */
stage2_apply_range(struct kvm_s2_mmu * mmu,phys_addr_t addr,phys_addr_t end,int (* fn)(struct kvm_pgtable *,u64,u64),bool resched)56 static int stage2_apply_range(struct kvm_s2_mmu *mmu, phys_addr_t addr,
57 			      phys_addr_t end,
58 			      int (*fn)(struct kvm_pgtable *, u64, u64),
59 			      bool resched)
60 {
61 	struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);
62 	int ret;
63 	u64 next;
64 
65 	do {
66 		struct kvm_pgtable *pgt = mmu->pgt;
67 		if (!pgt)
68 			return -EINVAL;
69 
70 		next = stage2_range_addr_end(addr, end);
71 		ret = fn(pgt, addr, next - addr);
72 		if (ret)
73 			break;
74 
75 		if (resched && next != end)
76 			cond_resched_rwlock_write(&kvm->mmu_lock);
77 	} while (addr = next, addr != end);
78 
79 	return ret;
80 }
81 
82 #define stage2_apply_range_resched(mmu, addr, end, fn)			\
83 	stage2_apply_range(mmu, addr, end, fn, true)
84 
85 /*
86  * Get the maximum number of page-tables pages needed to split a range
87  * of blocks into PAGE_SIZE PTEs. It assumes the range is already
88  * mapped at level 2, or at level 1 if allowed.
89  */
kvm_mmu_split_nr_page_tables(u64 range)90 static int kvm_mmu_split_nr_page_tables(u64 range)
91 {
92 	int n = 0;
93 
94 	if (KVM_PGTABLE_MIN_BLOCK_LEVEL < 2)
95 		n += DIV_ROUND_UP(range, PUD_SIZE);
96 	n += DIV_ROUND_UP(range, PMD_SIZE);
97 	return n;
98 }
99 
need_split_memcache_topup_or_resched(struct kvm * kvm)100 static bool need_split_memcache_topup_or_resched(struct kvm *kvm)
101 {
102 	struct kvm_mmu_memory_cache *cache;
103 	u64 chunk_size, min;
104 
105 	if (need_resched() || rwlock_needbreak(&kvm->mmu_lock))
106 		return true;
107 
108 	chunk_size = kvm->arch.mmu.split_page_chunk_size;
109 	min = kvm_mmu_split_nr_page_tables(chunk_size);
110 	cache = &kvm->arch.mmu.split_page_cache;
111 	return kvm_mmu_memory_cache_nr_free_objects(cache) < min;
112 }
113 
kvm_mmu_split_huge_pages(struct kvm * kvm,phys_addr_t addr,phys_addr_t end)114 static int kvm_mmu_split_huge_pages(struct kvm *kvm, phys_addr_t addr,
115 				    phys_addr_t end)
116 {
117 	struct kvm_mmu_memory_cache *cache;
118 	struct kvm_pgtable *pgt;
119 	int ret, cache_capacity;
120 	u64 next, chunk_size;
121 
122 	lockdep_assert_held_write(&kvm->mmu_lock);
123 
124 	chunk_size = kvm->arch.mmu.split_page_chunk_size;
125 	cache_capacity = kvm_mmu_split_nr_page_tables(chunk_size);
126 
127 	if (chunk_size == 0)
128 		return 0;
129 
130 	cache = &kvm->arch.mmu.split_page_cache;
131 
132 	do {
133 		if (need_split_memcache_topup_or_resched(kvm)) {
134 			write_unlock(&kvm->mmu_lock);
135 			cond_resched();
136 			/* Eager page splitting is best-effort. */
137 			ret = __kvm_mmu_topup_memory_cache(cache,
138 							   cache_capacity,
139 							   cache_capacity);
140 			write_lock(&kvm->mmu_lock);
141 			if (ret)
142 				break;
143 		}
144 
145 		pgt = kvm->arch.mmu.pgt;
146 		if (!pgt)
147 			return -EINVAL;
148 
149 		next = __stage2_range_addr_end(addr, end, chunk_size);
150 		ret = kvm_pgtable_stage2_split(pgt, addr, next - addr, cache);
151 		if (ret)
152 			break;
153 	} while (addr = next, addr != end);
154 
155 	return ret;
156 }
157 
memslot_is_logging(struct kvm_memory_slot * memslot)158 static bool memslot_is_logging(struct kvm_memory_slot *memslot)
159 {
160 	return memslot->dirty_bitmap && !(memslot->flags & KVM_MEM_READONLY);
161 }
162 
163 /**
164  * kvm_arch_flush_remote_tlbs() - flush all VM TLB entries for v7/8
165  * @kvm:	pointer to kvm structure.
166  *
167  * Interface to HYP function to flush all VM TLB entries
168  */
kvm_arch_flush_remote_tlbs(struct kvm * kvm)169 int kvm_arch_flush_remote_tlbs(struct kvm *kvm)
170 {
171 	kvm_call_hyp(__kvm_tlb_flush_vmid, &kvm->arch.mmu);
172 	return 0;
173 }
174 
kvm_arch_flush_remote_tlbs_range(struct kvm * kvm,gfn_t gfn,u64 nr_pages)175 int kvm_arch_flush_remote_tlbs_range(struct kvm *kvm,
176 				      gfn_t gfn, u64 nr_pages)
177 {
178 	kvm_tlb_flush_vmid_range(&kvm->arch.mmu,
179 				gfn << PAGE_SHIFT, nr_pages << PAGE_SHIFT);
180 	return 0;
181 }
182 
kvm_is_device_pfn(unsigned long pfn)183 static bool kvm_is_device_pfn(unsigned long pfn)
184 {
185 	return !pfn_is_map_memory(pfn);
186 }
187 
stage2_memcache_zalloc_page(void * arg)188 static void *stage2_memcache_zalloc_page(void *arg)
189 {
190 	struct kvm_mmu_memory_cache *mc = arg;
191 	void *virt;
192 
193 	/* Allocated with __GFP_ZERO, so no need to zero */
194 	virt = kvm_mmu_memory_cache_alloc(mc);
195 	if (virt)
196 		kvm_account_pgtable_pages(virt, 1);
197 	return virt;
198 }
199 
kvm_host_zalloc_pages_exact(size_t size)200 static void *kvm_host_zalloc_pages_exact(size_t size)
201 {
202 	return alloc_pages_exact(size, GFP_KERNEL_ACCOUNT | __GFP_ZERO);
203 }
204 
kvm_s2_zalloc_pages_exact(size_t size)205 static void *kvm_s2_zalloc_pages_exact(size_t size)
206 {
207 	void *virt = kvm_host_zalloc_pages_exact(size);
208 
209 	if (virt)
210 		kvm_account_pgtable_pages(virt, (size >> PAGE_SHIFT));
211 	return virt;
212 }
213 
kvm_s2_free_pages_exact(void * virt,size_t size)214 static void kvm_s2_free_pages_exact(void *virt, size_t size)
215 {
216 	kvm_account_pgtable_pages(virt, -(size >> PAGE_SHIFT));
217 	free_pages_exact(virt, size);
218 }
219 
220 static struct kvm_pgtable_mm_ops kvm_s2_mm_ops;
221 
stage2_free_unlinked_table_rcu_cb(struct rcu_head * head)222 static void stage2_free_unlinked_table_rcu_cb(struct rcu_head *head)
223 {
224 	struct page *page = container_of(head, struct page, rcu_head);
225 	void *pgtable = page_to_virt(page);
226 	u32 level = page_private(page);
227 
228 	kvm_pgtable_stage2_free_unlinked(&kvm_s2_mm_ops, pgtable, level);
229 }
230 
stage2_free_unlinked_table(void * addr,u32 level)231 static void stage2_free_unlinked_table(void *addr, u32 level)
232 {
233 	struct page *page = virt_to_page(addr);
234 
235 	set_page_private(page, (unsigned long)level);
236 	call_rcu(&page->rcu_head, stage2_free_unlinked_table_rcu_cb);
237 }
238 
kvm_host_get_page(void * addr)239 static void kvm_host_get_page(void *addr)
240 {
241 	get_page(virt_to_page(addr));
242 }
243 
kvm_host_put_page(void * addr)244 static void kvm_host_put_page(void *addr)
245 {
246 	put_page(virt_to_page(addr));
247 }
248 
kvm_s2_put_page(void * addr)249 static void kvm_s2_put_page(void *addr)
250 {
251 	struct page *p = virt_to_page(addr);
252 	/* Dropping last refcount, the page will be freed */
253 	if (page_count(p) == 1)
254 		kvm_account_pgtable_pages(addr, -1);
255 	put_page(p);
256 }
257 
kvm_host_page_count(void * addr)258 static int kvm_host_page_count(void *addr)
259 {
260 	return page_count(virt_to_page(addr));
261 }
262 
kvm_host_pa(void * addr)263 static phys_addr_t kvm_host_pa(void *addr)
264 {
265 	return __pa(addr);
266 }
267 
kvm_host_va(phys_addr_t phys)268 static void *kvm_host_va(phys_addr_t phys)
269 {
270 	return __va(phys);
271 }
272 
clean_dcache_guest_page(void * va,size_t size)273 static void clean_dcache_guest_page(void *va, size_t size)
274 {
275 	__clean_dcache_guest_page(va, size);
276 }
277 
invalidate_icache_guest_page(void * va,size_t size)278 static void invalidate_icache_guest_page(void *va, size_t size)
279 {
280 	__invalidate_icache_guest_page(va, size);
281 }
282 
283 /*
284  * Unmapping vs dcache management:
285  *
286  * If a guest maps certain memory pages as uncached, all writes will
287  * bypass the data cache and go directly to RAM.  However, the CPUs
288  * can still speculate reads (not writes) and fill cache lines with
289  * data.
290  *
291  * Those cache lines will be *clean* cache lines though, so a
292  * clean+invalidate operation is equivalent to an invalidate
293  * operation, because no cache lines are marked dirty.
294  *
295  * Those clean cache lines could be filled prior to an uncached write
296  * by the guest, and the cache coherent IO subsystem would therefore
297  * end up writing old data to disk.
298  *
299  * This is why right after unmapping a page/section and invalidating
300  * the corresponding TLBs, we flush to make sure the IO subsystem will
301  * never hit in the cache.
302  *
303  * This is all avoided on systems that have ARM64_HAS_STAGE2_FWB, as
304  * we then fully enforce cacheability of RAM, no matter what the guest
305  * does.
306  */
307 /**
308  * unmap_stage2_range -- Clear stage2 page table entries to unmap a range
309  * @mmu:   The KVM stage-2 MMU pointer
310  * @start: The intermediate physical base address of the range to unmap
311  * @size:  The size of the area to unmap
312  * @may_block: Whether or not we are permitted to block
313  *
314  * Clear a range of stage-2 mappings, lowering the various ref-counts.  Must
315  * be called while holding mmu_lock (unless for freeing the stage2 pgd before
316  * destroying the VM), otherwise another faulting VCPU may come in and mess
317  * with things behind our backs.
318  */
__unmap_stage2_range(struct kvm_s2_mmu * mmu,phys_addr_t start,u64 size,bool may_block)319 static void __unmap_stage2_range(struct kvm_s2_mmu *mmu, phys_addr_t start, u64 size,
320 				 bool may_block)
321 {
322 	struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);
323 	phys_addr_t end = start + size;
324 
325 	lockdep_assert_held_write(&kvm->mmu_lock);
326 	WARN_ON(size & ~PAGE_MASK);
327 	WARN_ON(stage2_apply_range(mmu, start, end, kvm_pgtable_stage2_unmap,
328 				   may_block));
329 }
330 
unmap_stage2_range(struct kvm_s2_mmu * mmu,phys_addr_t start,u64 size)331 static void unmap_stage2_range(struct kvm_s2_mmu *mmu, phys_addr_t start, u64 size)
332 {
333 	__unmap_stage2_range(mmu, start, size, true);
334 }
335 
stage2_flush_memslot(struct kvm * kvm,struct kvm_memory_slot * memslot)336 static void stage2_flush_memslot(struct kvm *kvm,
337 				 struct kvm_memory_slot *memslot)
338 {
339 	phys_addr_t addr = memslot->base_gfn << PAGE_SHIFT;
340 	phys_addr_t end = addr + PAGE_SIZE * memslot->npages;
341 
342 	stage2_apply_range_resched(&kvm->arch.mmu, addr, end, kvm_pgtable_stage2_flush);
343 }
344 
345 /**
346  * stage2_flush_vm - Invalidate cache for pages mapped in stage 2
347  * @kvm: The struct kvm pointer
348  *
349  * Go through the stage 2 page tables and invalidate any cache lines
350  * backing memory already mapped to the VM.
351  */
stage2_flush_vm(struct kvm * kvm)352 static void stage2_flush_vm(struct kvm *kvm)
353 {
354 	struct kvm_memslots *slots;
355 	struct kvm_memory_slot *memslot;
356 	int idx, bkt;
357 
358 	idx = srcu_read_lock(&kvm->srcu);
359 	write_lock(&kvm->mmu_lock);
360 
361 	slots = kvm_memslots(kvm);
362 	kvm_for_each_memslot(memslot, bkt, slots)
363 		stage2_flush_memslot(kvm, memslot);
364 
365 	write_unlock(&kvm->mmu_lock);
366 	srcu_read_unlock(&kvm->srcu, idx);
367 }
368 
369 /**
370  * free_hyp_pgds - free Hyp-mode page tables
371  */
free_hyp_pgds(void)372 void __init free_hyp_pgds(void)
373 {
374 	mutex_lock(&kvm_hyp_pgd_mutex);
375 	if (hyp_pgtable) {
376 		kvm_pgtable_hyp_destroy(hyp_pgtable);
377 		kfree(hyp_pgtable);
378 		hyp_pgtable = NULL;
379 	}
380 	mutex_unlock(&kvm_hyp_pgd_mutex);
381 }
382 
kvm_host_owns_hyp_mappings(void)383 static bool kvm_host_owns_hyp_mappings(void)
384 {
385 	if (is_kernel_in_hyp_mode())
386 		return false;
387 
388 	if (static_branch_likely(&kvm_protected_mode_initialized))
389 		return false;
390 
391 	/*
392 	 * This can happen at boot time when __create_hyp_mappings() is called
393 	 * after the hyp protection has been enabled, but the static key has
394 	 * not been flipped yet.
395 	 */
396 	if (!hyp_pgtable && is_protected_kvm_enabled())
397 		return false;
398 
399 	WARN_ON(!hyp_pgtable);
400 
401 	return true;
402 }
403 
__create_hyp_mappings(unsigned long start,unsigned long size,unsigned long phys,enum kvm_pgtable_prot prot)404 int __create_hyp_mappings(unsigned long start, unsigned long size,
405 			  unsigned long phys, enum kvm_pgtable_prot prot)
406 {
407 	int err;
408 
409 	if (WARN_ON(!kvm_host_owns_hyp_mappings()))
410 		return -EINVAL;
411 
412 	mutex_lock(&kvm_hyp_pgd_mutex);
413 	err = kvm_pgtable_hyp_map(hyp_pgtable, start, size, phys, prot);
414 	mutex_unlock(&kvm_hyp_pgd_mutex);
415 
416 	return err;
417 }
418 
kvm_kaddr_to_phys(void * kaddr)419 static phys_addr_t kvm_kaddr_to_phys(void *kaddr)
420 {
421 	if (!is_vmalloc_addr(kaddr)) {
422 		BUG_ON(!virt_addr_valid(kaddr));
423 		return __pa(kaddr);
424 	} else {
425 		return page_to_phys(vmalloc_to_page(kaddr)) +
426 		       offset_in_page(kaddr);
427 	}
428 }
429 
430 struct hyp_shared_pfn {
431 	u64 pfn;
432 	int count;
433 	struct rb_node node;
434 };
435 
436 static DEFINE_MUTEX(hyp_shared_pfns_lock);
437 static struct rb_root hyp_shared_pfns = RB_ROOT;
438 
find_shared_pfn(u64 pfn,struct rb_node *** node,struct rb_node ** parent)439 static struct hyp_shared_pfn *find_shared_pfn(u64 pfn, struct rb_node ***node,
440 					      struct rb_node **parent)
441 {
442 	struct hyp_shared_pfn *this;
443 
444 	*node = &hyp_shared_pfns.rb_node;
445 	*parent = NULL;
446 	while (**node) {
447 		this = container_of(**node, struct hyp_shared_pfn, node);
448 		*parent = **node;
449 		if (this->pfn < pfn)
450 			*node = &((**node)->rb_left);
451 		else if (this->pfn > pfn)
452 			*node = &((**node)->rb_right);
453 		else
454 			return this;
455 	}
456 
457 	return NULL;
458 }
459 
share_pfn_hyp(u64 pfn)460 static int share_pfn_hyp(u64 pfn)
461 {
462 	struct rb_node **node, *parent;
463 	struct hyp_shared_pfn *this;
464 	int ret = 0;
465 
466 	mutex_lock(&hyp_shared_pfns_lock);
467 	this = find_shared_pfn(pfn, &node, &parent);
468 	if (this) {
469 		this->count++;
470 		goto unlock;
471 	}
472 
473 	this = kzalloc(sizeof(*this), GFP_KERNEL);
474 	if (!this) {
475 		ret = -ENOMEM;
476 		goto unlock;
477 	}
478 
479 	this->pfn = pfn;
480 	this->count = 1;
481 	rb_link_node(&this->node, parent, node);
482 	rb_insert_color(&this->node, &hyp_shared_pfns);
483 	ret = kvm_call_hyp_nvhe(__pkvm_host_share_hyp, pfn, 1);
484 unlock:
485 	mutex_unlock(&hyp_shared_pfns_lock);
486 
487 	return ret;
488 }
489 
unshare_pfn_hyp(u64 pfn)490 static int unshare_pfn_hyp(u64 pfn)
491 {
492 	struct rb_node **node, *parent;
493 	struct hyp_shared_pfn *this;
494 	int ret = 0;
495 
496 	mutex_lock(&hyp_shared_pfns_lock);
497 	this = find_shared_pfn(pfn, &node, &parent);
498 	if (WARN_ON(!this)) {
499 		ret = -ENOENT;
500 		goto unlock;
501 	}
502 
503 	this->count--;
504 	if (this->count)
505 		goto unlock;
506 
507 	rb_erase(&this->node, &hyp_shared_pfns);
508 	kfree(this);
509 	ret = kvm_call_hyp_nvhe(__pkvm_host_unshare_hyp, pfn, 1);
510 unlock:
511 	mutex_unlock(&hyp_shared_pfns_lock);
512 
513 	return ret;
514 }
515 
kvm_share_hyp(void * from,void * to)516 int kvm_share_hyp(void *from, void *to)
517 {
518 	phys_addr_t start, end, cur;
519 	u64 pfn;
520 	int ret;
521 
522 	if (is_kernel_in_hyp_mode())
523 		return 0;
524 
525 	/*
526 	 * The share hcall maps things in the 'fixed-offset' region of the hyp
527 	 * VA space, so we can only share physically contiguous data-structures
528 	 * for now.
529 	 */
530 	if (is_vmalloc_or_module_addr(from) || is_vmalloc_or_module_addr(to))
531 		return -EINVAL;
532 
533 	if (kvm_host_owns_hyp_mappings())
534 		return create_hyp_mappings(from, to, PAGE_HYP);
535 
536 	start = ALIGN_DOWN(__pa(from), PAGE_SIZE);
537 	end = PAGE_ALIGN(__pa(to));
538 	for (cur = start; cur < end; cur += PAGE_SIZE) {
539 		pfn = __phys_to_pfn(cur);
540 		ret = share_pfn_hyp(pfn);
541 		if (ret)
542 			return ret;
543 	}
544 
545 	return 0;
546 }
547 
kvm_unshare_hyp(void * from,void * to)548 void kvm_unshare_hyp(void *from, void *to)
549 {
550 	phys_addr_t start, end, cur;
551 	u64 pfn;
552 
553 	if (is_kernel_in_hyp_mode() || kvm_host_owns_hyp_mappings() || !from)
554 		return;
555 
556 	start = ALIGN_DOWN(__pa(from), PAGE_SIZE);
557 	end = PAGE_ALIGN(__pa(to));
558 	for (cur = start; cur < end; cur += PAGE_SIZE) {
559 		pfn = __phys_to_pfn(cur);
560 		WARN_ON(unshare_pfn_hyp(pfn));
561 	}
562 }
563 
564 /**
565  * create_hyp_mappings - duplicate a kernel virtual address range in Hyp mode
566  * @from:	The virtual kernel start address of the range
567  * @to:		The virtual kernel end address of the range (exclusive)
568  * @prot:	The protection to be applied to this range
569  *
570  * The same virtual address as the kernel virtual address is also used
571  * in Hyp-mode mapping (modulo HYP_PAGE_OFFSET) to the same underlying
572  * physical pages.
573  */
create_hyp_mappings(void * from,void * to,enum kvm_pgtable_prot prot)574 int create_hyp_mappings(void *from, void *to, enum kvm_pgtable_prot prot)
575 {
576 	phys_addr_t phys_addr;
577 	unsigned long virt_addr;
578 	unsigned long start = kern_hyp_va((unsigned long)from);
579 	unsigned long end = kern_hyp_va((unsigned long)to);
580 
581 	if (is_kernel_in_hyp_mode())
582 		return 0;
583 
584 	if (!kvm_host_owns_hyp_mappings())
585 		return -EPERM;
586 
587 	start = start & PAGE_MASK;
588 	end = PAGE_ALIGN(end);
589 
590 	for (virt_addr = start; virt_addr < end; virt_addr += PAGE_SIZE) {
591 		int err;
592 
593 		phys_addr = kvm_kaddr_to_phys(from + virt_addr - start);
594 		err = __create_hyp_mappings(virt_addr, PAGE_SIZE, phys_addr,
595 					    prot);
596 		if (err)
597 			return err;
598 	}
599 
600 	return 0;
601 }
602 
__hyp_alloc_private_va_range(unsigned long base)603 static int __hyp_alloc_private_va_range(unsigned long base)
604 {
605 	lockdep_assert_held(&kvm_hyp_pgd_mutex);
606 
607 	if (!PAGE_ALIGNED(base))
608 		return -EINVAL;
609 
610 	/*
611 	 * Verify that BIT(VA_BITS - 1) hasn't been flipped by
612 	 * allocating the new area, as it would indicate we've
613 	 * overflowed the idmap/IO address range.
614 	 */
615 	if ((base ^ io_map_base) & BIT(VA_BITS - 1))
616 		return -ENOMEM;
617 
618 	io_map_base = base;
619 
620 	return 0;
621 }
622 
623 /**
624  * hyp_alloc_private_va_range - Allocates a private VA range.
625  * @size:	The size of the VA range to reserve.
626  * @haddr:	The hypervisor virtual start address of the allocation.
627  *
628  * The private virtual address (VA) range is allocated below io_map_base
629  * and aligned based on the order of @size.
630  *
631  * Return: 0 on success or negative error code on failure.
632  */
hyp_alloc_private_va_range(size_t size,unsigned long * haddr)633 int hyp_alloc_private_va_range(size_t size, unsigned long *haddr)
634 {
635 	unsigned long base;
636 	int ret = 0;
637 
638 	mutex_lock(&kvm_hyp_pgd_mutex);
639 
640 	/*
641 	 * This assumes that we have enough space below the idmap
642 	 * page to allocate our VAs. If not, the check in
643 	 * __hyp_alloc_private_va_range() will kick. A potential
644 	 * alternative would be to detect that overflow and switch
645 	 * to an allocation above the idmap.
646 	 *
647 	 * The allocated size is always a multiple of PAGE_SIZE.
648 	 */
649 	size = PAGE_ALIGN(size);
650 	base = io_map_base - size;
651 	ret = __hyp_alloc_private_va_range(base);
652 
653 	mutex_unlock(&kvm_hyp_pgd_mutex);
654 
655 	if (!ret)
656 		*haddr = base;
657 
658 	return ret;
659 }
660 
__create_hyp_private_mapping(phys_addr_t phys_addr,size_t size,unsigned long * haddr,enum kvm_pgtable_prot prot)661 static int __create_hyp_private_mapping(phys_addr_t phys_addr, size_t size,
662 					unsigned long *haddr,
663 					enum kvm_pgtable_prot prot)
664 {
665 	unsigned long addr;
666 	int ret = 0;
667 
668 	if (!kvm_host_owns_hyp_mappings()) {
669 		addr = kvm_call_hyp_nvhe(__pkvm_create_private_mapping,
670 					 phys_addr, size, prot);
671 		if (IS_ERR_VALUE(addr))
672 			return addr;
673 		*haddr = addr;
674 
675 		return 0;
676 	}
677 
678 	size = PAGE_ALIGN(size + offset_in_page(phys_addr));
679 	ret = hyp_alloc_private_va_range(size, &addr);
680 	if (ret)
681 		return ret;
682 
683 	ret = __create_hyp_mappings(addr, size, phys_addr, prot);
684 	if (ret)
685 		return ret;
686 
687 	*haddr = addr + offset_in_page(phys_addr);
688 	return ret;
689 }
690 
create_hyp_stack(phys_addr_t phys_addr,unsigned long * haddr)691 int create_hyp_stack(phys_addr_t phys_addr, unsigned long *haddr)
692 {
693 	unsigned long base;
694 	size_t size;
695 	int ret;
696 
697 	mutex_lock(&kvm_hyp_pgd_mutex);
698 	/*
699 	 * Efficient stack verification using the PAGE_SHIFT bit implies
700 	 * an alignment of our allocation on the order of the size.
701 	 */
702 	size = PAGE_SIZE * 2;
703 	base = ALIGN_DOWN(io_map_base - size, size);
704 
705 	ret = __hyp_alloc_private_va_range(base);
706 
707 	mutex_unlock(&kvm_hyp_pgd_mutex);
708 
709 	if (ret) {
710 		kvm_err("Cannot allocate hyp stack guard page\n");
711 		return ret;
712 	}
713 
714 	/*
715 	 * Since the stack grows downwards, map the stack to the page
716 	 * at the higher address and leave the lower guard page
717 	 * unbacked.
718 	 *
719 	 * Any valid stack address now has the PAGE_SHIFT bit as 1
720 	 * and addresses corresponding to the guard page have the
721 	 * PAGE_SHIFT bit as 0 - this is used for overflow detection.
722 	 */
723 	ret = __create_hyp_mappings(base + PAGE_SIZE, PAGE_SIZE, phys_addr,
724 				    PAGE_HYP);
725 	if (ret)
726 		kvm_err("Cannot map hyp stack\n");
727 
728 	*haddr = base + size;
729 
730 	return ret;
731 }
732 
733 /**
734  * create_hyp_io_mappings - Map IO into both kernel and HYP
735  * @phys_addr:	The physical start address which gets mapped
736  * @size:	Size of the region being mapped
737  * @kaddr:	Kernel VA for this mapping
738  * @haddr:	HYP VA for this mapping
739  */
create_hyp_io_mappings(phys_addr_t phys_addr,size_t size,void __iomem ** kaddr,void __iomem ** haddr)740 int create_hyp_io_mappings(phys_addr_t phys_addr, size_t size,
741 			   void __iomem **kaddr,
742 			   void __iomem **haddr)
743 {
744 	unsigned long addr;
745 	int ret;
746 
747 	if (is_protected_kvm_enabled())
748 		return -EPERM;
749 
750 	*kaddr = ioremap(phys_addr, size);
751 	if (!*kaddr)
752 		return -ENOMEM;
753 
754 	if (is_kernel_in_hyp_mode()) {
755 		*haddr = *kaddr;
756 		return 0;
757 	}
758 
759 	ret = __create_hyp_private_mapping(phys_addr, size,
760 					   &addr, PAGE_HYP_DEVICE);
761 	if (ret) {
762 		iounmap(*kaddr);
763 		*kaddr = NULL;
764 		*haddr = NULL;
765 		return ret;
766 	}
767 
768 	*haddr = (void __iomem *)addr;
769 	return 0;
770 }
771 
772 /**
773  * create_hyp_exec_mappings - Map an executable range into HYP
774  * @phys_addr:	The physical start address which gets mapped
775  * @size:	Size of the region being mapped
776  * @haddr:	HYP VA for this mapping
777  */
create_hyp_exec_mappings(phys_addr_t phys_addr,size_t size,void ** haddr)778 int create_hyp_exec_mappings(phys_addr_t phys_addr, size_t size,
779 			     void **haddr)
780 {
781 	unsigned long addr;
782 	int ret;
783 
784 	BUG_ON(is_kernel_in_hyp_mode());
785 
786 	ret = __create_hyp_private_mapping(phys_addr, size,
787 					   &addr, PAGE_HYP_EXEC);
788 	if (ret) {
789 		*haddr = NULL;
790 		return ret;
791 	}
792 
793 	*haddr = (void *)addr;
794 	return 0;
795 }
796 
797 static struct kvm_pgtable_mm_ops kvm_user_mm_ops = {
798 	/* We shouldn't need any other callback to walk the PT */
799 	.phys_to_virt		= kvm_host_va,
800 };
801 
get_user_mapping_size(struct kvm * kvm,u64 addr)802 static int get_user_mapping_size(struct kvm *kvm, u64 addr)
803 {
804 	struct kvm_pgtable pgt = {
805 		.pgd		= (kvm_pteref_t)kvm->mm->pgd,
806 		.ia_bits	= vabits_actual,
807 		.start_level	= (KVM_PGTABLE_MAX_LEVELS -
808 				   CONFIG_PGTABLE_LEVELS),
809 		.mm_ops		= &kvm_user_mm_ops,
810 	};
811 	unsigned long flags;
812 	kvm_pte_t pte = 0;	/* Keep GCC quiet... */
813 	u32 level = ~0;
814 	int ret;
815 
816 	/*
817 	 * Disable IRQs so that we hazard against a concurrent
818 	 * teardown of the userspace page tables (which relies on
819 	 * IPI-ing threads).
820 	 */
821 	local_irq_save(flags);
822 	ret = kvm_pgtable_get_leaf(&pgt, addr, &pte, &level);
823 	local_irq_restore(flags);
824 
825 	if (ret)
826 		return ret;
827 
828 	/*
829 	 * Not seeing an error, but not updating level? Something went
830 	 * deeply wrong...
831 	 */
832 	if (WARN_ON(level >= KVM_PGTABLE_MAX_LEVELS))
833 		return -EFAULT;
834 
835 	/* Oops, the userspace PTs are gone... Replay the fault */
836 	if (!kvm_pte_valid(pte))
837 		return -EAGAIN;
838 
839 	return BIT(ARM64_HW_PGTABLE_LEVEL_SHIFT(level));
840 }
841 
842 static struct kvm_pgtable_mm_ops kvm_s2_mm_ops = {
843 	.zalloc_page		= stage2_memcache_zalloc_page,
844 	.zalloc_pages_exact	= kvm_s2_zalloc_pages_exact,
845 	.free_pages_exact	= kvm_s2_free_pages_exact,
846 	.free_unlinked_table	= stage2_free_unlinked_table,
847 	.get_page		= kvm_host_get_page,
848 	.put_page		= kvm_s2_put_page,
849 	.page_count		= kvm_host_page_count,
850 	.phys_to_virt		= kvm_host_va,
851 	.virt_to_phys		= kvm_host_pa,
852 	.dcache_clean_inval_poc	= clean_dcache_guest_page,
853 	.icache_inval_pou	= invalidate_icache_guest_page,
854 };
855 
856 /**
857  * kvm_init_stage2_mmu - Initialise a S2 MMU structure
858  * @kvm:	The pointer to the KVM structure
859  * @mmu:	The pointer to the s2 MMU structure
860  * @type:	The machine type of the virtual machine
861  *
862  * Allocates only the stage-2 HW PGD level table(s).
863  * Note we don't need locking here as this is only called when the VM is
864  * created, which can only be done once.
865  */
kvm_init_stage2_mmu(struct kvm * kvm,struct kvm_s2_mmu * mmu,unsigned long type)866 int kvm_init_stage2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu, unsigned long type)
867 {
868 	u32 kvm_ipa_limit = get_kvm_ipa_limit();
869 	int cpu, err;
870 	struct kvm_pgtable *pgt;
871 	u64 mmfr0, mmfr1;
872 	u32 phys_shift;
873 
874 	if (type & ~KVM_VM_TYPE_ARM_IPA_SIZE_MASK)
875 		return -EINVAL;
876 
877 	phys_shift = KVM_VM_TYPE_ARM_IPA_SIZE(type);
878 	if (is_protected_kvm_enabled()) {
879 		phys_shift = kvm_ipa_limit;
880 	} else if (phys_shift) {
881 		if (phys_shift > kvm_ipa_limit ||
882 		    phys_shift < ARM64_MIN_PARANGE_BITS)
883 			return -EINVAL;
884 	} else {
885 		phys_shift = KVM_PHYS_SHIFT;
886 		if (phys_shift > kvm_ipa_limit) {
887 			pr_warn_once("%s using unsupported default IPA limit, upgrade your VMM\n",
888 				     current->comm);
889 			return -EINVAL;
890 		}
891 	}
892 
893 	mmfr0 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
894 	mmfr1 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);
895 	kvm->arch.vtcr = kvm_get_vtcr(mmfr0, mmfr1, phys_shift);
896 
897 	if (mmu->pgt != NULL) {
898 		kvm_err("kvm_arch already initialized?\n");
899 		return -EINVAL;
900 	}
901 
902 	pgt = kzalloc(sizeof(*pgt), GFP_KERNEL_ACCOUNT);
903 	if (!pgt)
904 		return -ENOMEM;
905 
906 	mmu->arch = &kvm->arch;
907 	err = kvm_pgtable_stage2_init(pgt, mmu, &kvm_s2_mm_ops);
908 	if (err)
909 		goto out_free_pgtable;
910 
911 	mmu->last_vcpu_ran = alloc_percpu(typeof(*mmu->last_vcpu_ran));
912 	if (!mmu->last_vcpu_ran) {
913 		err = -ENOMEM;
914 		goto out_destroy_pgtable;
915 	}
916 
917 	for_each_possible_cpu(cpu)
918 		*per_cpu_ptr(mmu->last_vcpu_ran, cpu) = -1;
919 
920 	 /* The eager page splitting is disabled by default */
921 	mmu->split_page_chunk_size = KVM_ARM_EAGER_SPLIT_CHUNK_SIZE_DEFAULT;
922 	mmu->split_page_cache.gfp_zero = __GFP_ZERO;
923 
924 	mmu->pgt = pgt;
925 	mmu->pgd_phys = __pa(pgt->pgd);
926 	return 0;
927 
928 out_destroy_pgtable:
929 	kvm_pgtable_stage2_destroy(pgt);
930 out_free_pgtable:
931 	kfree(pgt);
932 	return err;
933 }
934 
kvm_uninit_stage2_mmu(struct kvm * kvm)935 void kvm_uninit_stage2_mmu(struct kvm *kvm)
936 {
937 	kvm_free_stage2_pgd(&kvm->arch.mmu);
938 	kvm_mmu_free_memory_cache(&kvm->arch.mmu.split_page_cache);
939 }
940 
stage2_unmap_memslot(struct kvm * kvm,struct kvm_memory_slot * memslot)941 static void stage2_unmap_memslot(struct kvm *kvm,
942 				 struct kvm_memory_slot *memslot)
943 {
944 	hva_t hva = memslot->userspace_addr;
945 	phys_addr_t addr = memslot->base_gfn << PAGE_SHIFT;
946 	phys_addr_t size = PAGE_SIZE * memslot->npages;
947 	hva_t reg_end = hva + size;
948 
949 	/*
950 	 * A memory region could potentially cover multiple VMAs, and any holes
951 	 * between them, so iterate over all of them to find out if we should
952 	 * unmap any of them.
953 	 *
954 	 *     +--------------------------------------------+
955 	 * +---------------+----------------+   +----------------+
956 	 * |   : VMA 1     |      VMA 2     |   |    VMA 3  :    |
957 	 * +---------------+----------------+   +----------------+
958 	 *     |               memory region                |
959 	 *     +--------------------------------------------+
960 	 */
961 	do {
962 		struct vm_area_struct *vma;
963 		hva_t vm_start, vm_end;
964 
965 		vma = find_vma_intersection(current->mm, hva, reg_end);
966 		if (!vma)
967 			break;
968 
969 		/*
970 		 * Take the intersection of this VMA with the memory region
971 		 */
972 		vm_start = max(hva, vma->vm_start);
973 		vm_end = min(reg_end, vma->vm_end);
974 
975 		if (!(vma->vm_flags & VM_PFNMAP)) {
976 			gpa_t gpa = addr + (vm_start - memslot->userspace_addr);
977 			unmap_stage2_range(&kvm->arch.mmu, gpa, vm_end - vm_start);
978 		}
979 		hva = vm_end;
980 	} while (hva < reg_end);
981 }
982 
983 /**
984  * stage2_unmap_vm - Unmap Stage-2 RAM mappings
985  * @kvm: The struct kvm pointer
986  *
987  * Go through the memregions and unmap any regular RAM
988  * backing memory already mapped to the VM.
989  */
stage2_unmap_vm(struct kvm * kvm)990 void stage2_unmap_vm(struct kvm *kvm)
991 {
992 	struct kvm_memslots *slots;
993 	struct kvm_memory_slot *memslot;
994 	int idx, bkt;
995 
996 	idx = srcu_read_lock(&kvm->srcu);
997 	mmap_read_lock(current->mm);
998 	write_lock(&kvm->mmu_lock);
999 
1000 	slots = kvm_memslots(kvm);
1001 	kvm_for_each_memslot(memslot, bkt, slots)
1002 		stage2_unmap_memslot(kvm, memslot);
1003 
1004 	write_unlock(&kvm->mmu_lock);
1005 	mmap_read_unlock(current->mm);
1006 	srcu_read_unlock(&kvm->srcu, idx);
1007 }
1008 
kvm_free_stage2_pgd(struct kvm_s2_mmu * mmu)1009 void kvm_free_stage2_pgd(struct kvm_s2_mmu *mmu)
1010 {
1011 	struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);
1012 	struct kvm_pgtable *pgt = NULL;
1013 
1014 	write_lock(&kvm->mmu_lock);
1015 	pgt = mmu->pgt;
1016 	if (pgt) {
1017 		mmu->pgd_phys = 0;
1018 		mmu->pgt = NULL;
1019 		free_percpu(mmu->last_vcpu_ran);
1020 	}
1021 	write_unlock(&kvm->mmu_lock);
1022 
1023 	if (pgt) {
1024 		kvm_pgtable_stage2_destroy(pgt);
1025 		kfree(pgt);
1026 	}
1027 }
1028 
hyp_mc_free_fn(void * addr,void * unused)1029 static void hyp_mc_free_fn(void *addr, void *unused)
1030 {
1031 	free_page((unsigned long)addr);
1032 }
1033 
hyp_mc_alloc_fn(void * unused)1034 static void *hyp_mc_alloc_fn(void *unused)
1035 {
1036 	return (void *)__get_free_page(GFP_KERNEL_ACCOUNT);
1037 }
1038 
free_hyp_memcache(struct kvm_hyp_memcache * mc)1039 void free_hyp_memcache(struct kvm_hyp_memcache *mc)
1040 {
1041 	if (is_protected_kvm_enabled())
1042 		__free_hyp_memcache(mc, hyp_mc_free_fn,
1043 				    kvm_host_va, NULL);
1044 }
1045 
topup_hyp_memcache(struct kvm_hyp_memcache * mc,unsigned long min_pages)1046 int topup_hyp_memcache(struct kvm_hyp_memcache *mc, unsigned long min_pages)
1047 {
1048 	if (!is_protected_kvm_enabled())
1049 		return 0;
1050 
1051 	return __topup_hyp_memcache(mc, min_pages, hyp_mc_alloc_fn,
1052 				    kvm_host_pa, NULL);
1053 }
1054 
1055 /**
1056  * kvm_phys_addr_ioremap - map a device range to guest IPA
1057  *
1058  * @kvm:	The KVM pointer
1059  * @guest_ipa:	The IPA at which to insert the mapping
1060  * @pa:		The physical address of the device
1061  * @size:	The size of the mapping
1062  * @writable:   Whether or not to create a writable mapping
1063  */
kvm_phys_addr_ioremap(struct kvm * kvm,phys_addr_t guest_ipa,phys_addr_t pa,unsigned long size,bool writable)1064 int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa,
1065 			  phys_addr_t pa, unsigned long size, bool writable)
1066 {
1067 	phys_addr_t addr;
1068 	int ret = 0;
1069 	struct kvm_mmu_memory_cache cache = { .gfp_zero = __GFP_ZERO };
1070 	struct kvm_pgtable *pgt = kvm->arch.mmu.pgt;
1071 	enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_DEVICE |
1072 				     KVM_PGTABLE_PROT_R |
1073 				     (writable ? KVM_PGTABLE_PROT_W : 0);
1074 
1075 	if (is_protected_kvm_enabled())
1076 		return -EPERM;
1077 
1078 	size += offset_in_page(guest_ipa);
1079 	guest_ipa &= PAGE_MASK;
1080 
1081 	for (addr = guest_ipa; addr < guest_ipa + size; addr += PAGE_SIZE) {
1082 		ret = kvm_mmu_topup_memory_cache(&cache,
1083 						 kvm_mmu_cache_min_pages(kvm));
1084 		if (ret)
1085 			break;
1086 
1087 		write_lock(&kvm->mmu_lock);
1088 		ret = kvm_pgtable_stage2_map(pgt, addr, PAGE_SIZE, pa, prot,
1089 					     &cache, 0);
1090 		write_unlock(&kvm->mmu_lock);
1091 		if (ret)
1092 			break;
1093 
1094 		pa += PAGE_SIZE;
1095 	}
1096 
1097 	kvm_mmu_free_memory_cache(&cache);
1098 	return ret;
1099 }
1100 
1101 /**
1102  * stage2_wp_range() - write protect stage2 memory region range
1103  * @mmu:        The KVM stage-2 MMU pointer
1104  * @addr:	Start address of range
1105  * @end:	End address of range
1106  */
stage2_wp_range(struct kvm_s2_mmu * mmu,phys_addr_t addr,phys_addr_t end)1107 static void stage2_wp_range(struct kvm_s2_mmu *mmu, phys_addr_t addr, phys_addr_t end)
1108 {
1109 	stage2_apply_range_resched(mmu, addr, end, kvm_pgtable_stage2_wrprotect);
1110 }
1111 
1112 /**
1113  * kvm_mmu_wp_memory_region() - write protect stage 2 entries for memory slot
1114  * @kvm:	The KVM pointer
1115  * @slot:	The memory slot to write protect
1116  *
1117  * Called to start logging dirty pages after memory region
1118  * KVM_MEM_LOG_DIRTY_PAGES operation is called. After this function returns
1119  * all present PUD, PMD and PTEs are write protected in the memory region.
1120  * Afterwards read of dirty page log can be called.
1121  *
1122  * Acquires kvm_mmu_lock. Called with kvm->slots_lock mutex acquired,
1123  * serializing operations for VM memory regions.
1124  */
kvm_mmu_wp_memory_region(struct kvm * kvm,int slot)1125 static void kvm_mmu_wp_memory_region(struct kvm *kvm, int slot)
1126 {
1127 	struct kvm_memslots *slots = kvm_memslots(kvm);
1128 	struct kvm_memory_slot *memslot = id_to_memslot(slots, slot);
1129 	phys_addr_t start, end;
1130 
1131 	if (WARN_ON_ONCE(!memslot))
1132 		return;
1133 
1134 	start = memslot->base_gfn << PAGE_SHIFT;
1135 	end = (memslot->base_gfn + memslot->npages) << PAGE_SHIFT;
1136 
1137 	write_lock(&kvm->mmu_lock);
1138 	stage2_wp_range(&kvm->arch.mmu, start, end);
1139 	write_unlock(&kvm->mmu_lock);
1140 	kvm_flush_remote_tlbs_memslot(kvm, memslot);
1141 }
1142 
1143 /**
1144  * kvm_mmu_split_memory_region() - split the stage 2 blocks into PAGE_SIZE
1145  *				   pages for memory slot
1146  * @kvm:	The KVM pointer
1147  * @slot:	The memory slot to split
1148  *
1149  * Acquires kvm->mmu_lock. Called with kvm->slots_lock mutex acquired,
1150  * serializing operations for VM memory regions.
1151  */
kvm_mmu_split_memory_region(struct kvm * kvm,int slot)1152 static void kvm_mmu_split_memory_region(struct kvm *kvm, int slot)
1153 {
1154 	struct kvm_memslots *slots;
1155 	struct kvm_memory_slot *memslot;
1156 	phys_addr_t start, end;
1157 
1158 	lockdep_assert_held(&kvm->slots_lock);
1159 
1160 	slots = kvm_memslots(kvm);
1161 	memslot = id_to_memslot(slots, slot);
1162 
1163 	start = memslot->base_gfn << PAGE_SHIFT;
1164 	end = (memslot->base_gfn + memslot->npages) << PAGE_SHIFT;
1165 
1166 	write_lock(&kvm->mmu_lock);
1167 	kvm_mmu_split_huge_pages(kvm, start, end);
1168 	write_unlock(&kvm->mmu_lock);
1169 }
1170 
1171 /*
1172  * kvm_arch_mmu_enable_log_dirty_pt_masked() - enable dirty logging for selected pages.
1173  * @kvm:	The KVM pointer
1174  * @slot:	The memory slot associated with mask
1175  * @gfn_offset:	The gfn offset in memory slot
1176  * @mask:	The mask of pages at offset 'gfn_offset' in this memory
1177  *		slot to enable dirty logging on
1178  *
1179  * Writes protect selected pages to enable dirty logging, and then
1180  * splits them to PAGE_SIZE. Caller must acquire kvm->mmu_lock.
1181  */
kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm * kvm,struct kvm_memory_slot * slot,gfn_t gfn_offset,unsigned long mask)1182 void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
1183 		struct kvm_memory_slot *slot,
1184 		gfn_t gfn_offset, unsigned long mask)
1185 {
1186 	phys_addr_t base_gfn = slot->base_gfn + gfn_offset;
1187 	phys_addr_t start = (base_gfn +  __ffs(mask)) << PAGE_SHIFT;
1188 	phys_addr_t end = (base_gfn + __fls(mask) + 1) << PAGE_SHIFT;
1189 
1190 	lockdep_assert_held_write(&kvm->mmu_lock);
1191 
1192 	stage2_wp_range(&kvm->arch.mmu, start, end);
1193 
1194 	/*
1195 	 * Eager-splitting is done when manual-protect is set.  We
1196 	 * also check for initially-all-set because we can avoid
1197 	 * eager-splitting if initially-all-set is false.
1198 	 * Initially-all-set equal false implies that huge-pages were
1199 	 * already split when enabling dirty logging: no need to do it
1200 	 * again.
1201 	 */
1202 	if (kvm_dirty_log_manual_protect_and_init_set(kvm))
1203 		kvm_mmu_split_huge_pages(kvm, start, end);
1204 }
1205 
kvm_send_hwpoison_signal(unsigned long address,short lsb)1206 static void kvm_send_hwpoison_signal(unsigned long address, short lsb)
1207 {
1208 	send_sig_mceerr(BUS_MCEERR_AR, (void __user *)address, lsb, current);
1209 }
1210 
fault_supports_stage2_huge_mapping(struct kvm_memory_slot * memslot,unsigned long hva,unsigned long map_size)1211 static bool fault_supports_stage2_huge_mapping(struct kvm_memory_slot *memslot,
1212 					       unsigned long hva,
1213 					       unsigned long map_size)
1214 {
1215 	gpa_t gpa_start;
1216 	hva_t uaddr_start, uaddr_end;
1217 	size_t size;
1218 
1219 	/* The memslot and the VMA are guaranteed to be aligned to PAGE_SIZE */
1220 	if (map_size == PAGE_SIZE)
1221 		return true;
1222 
1223 	size = memslot->npages * PAGE_SIZE;
1224 
1225 	gpa_start = memslot->base_gfn << PAGE_SHIFT;
1226 
1227 	uaddr_start = memslot->userspace_addr;
1228 	uaddr_end = uaddr_start + size;
1229 
1230 	/*
1231 	 * Pages belonging to memslots that don't have the same alignment
1232 	 * within a PMD/PUD for userspace and IPA cannot be mapped with stage-2
1233 	 * PMD/PUD entries, because we'll end up mapping the wrong pages.
1234 	 *
1235 	 * Consider a layout like the following:
1236 	 *
1237 	 *    memslot->userspace_addr:
1238 	 *    +-----+--------------------+--------------------+---+
1239 	 *    |abcde|fgh  Stage-1 block  |    Stage-1 block tv|xyz|
1240 	 *    +-----+--------------------+--------------------+---+
1241 	 *
1242 	 *    memslot->base_gfn << PAGE_SHIFT:
1243 	 *      +---+--------------------+--------------------+-----+
1244 	 *      |abc|def  Stage-2 block  |    Stage-2 block   |tvxyz|
1245 	 *      +---+--------------------+--------------------+-----+
1246 	 *
1247 	 * If we create those stage-2 blocks, we'll end up with this incorrect
1248 	 * mapping:
1249 	 *   d -> f
1250 	 *   e -> g
1251 	 *   f -> h
1252 	 */
1253 	if ((gpa_start & (map_size - 1)) != (uaddr_start & (map_size - 1)))
1254 		return false;
1255 
1256 	/*
1257 	 * Next, let's make sure we're not trying to map anything not covered
1258 	 * by the memslot. This means we have to prohibit block size mappings
1259 	 * for the beginning and end of a non-block aligned and non-block sized
1260 	 * memory slot (illustrated by the head and tail parts of the
1261 	 * userspace view above containing pages 'abcde' and 'xyz',
1262 	 * respectively).
1263 	 *
1264 	 * Note that it doesn't matter if we do the check using the
1265 	 * userspace_addr or the base_gfn, as both are equally aligned (per
1266 	 * the check above) and equally sized.
1267 	 */
1268 	return (hva & ~(map_size - 1)) >= uaddr_start &&
1269 	       (hva & ~(map_size - 1)) + map_size <= uaddr_end;
1270 }
1271 
1272 /*
1273  * Check if the given hva is backed by a transparent huge page (THP) and
1274  * whether it can be mapped using block mapping in stage2. If so, adjust
1275  * the stage2 PFN and IPA accordingly. Only PMD_SIZE THPs are currently
1276  * supported. This will need to be updated to support other THP sizes.
1277  *
1278  * Returns the size of the mapping.
1279  */
1280 static long
transparent_hugepage_adjust(struct kvm * kvm,struct kvm_memory_slot * memslot,unsigned long hva,kvm_pfn_t * pfnp,phys_addr_t * ipap)1281 transparent_hugepage_adjust(struct kvm *kvm, struct kvm_memory_slot *memslot,
1282 			    unsigned long hva, kvm_pfn_t *pfnp,
1283 			    phys_addr_t *ipap)
1284 {
1285 	kvm_pfn_t pfn = *pfnp;
1286 
1287 	/*
1288 	 * Make sure the adjustment is done only for THP pages. Also make
1289 	 * sure that the HVA and IPA are sufficiently aligned and that the
1290 	 * block map is contained within the memslot.
1291 	 */
1292 	if (fault_supports_stage2_huge_mapping(memslot, hva, PMD_SIZE)) {
1293 		int sz = get_user_mapping_size(kvm, hva);
1294 
1295 		if (sz < 0)
1296 			return sz;
1297 
1298 		if (sz < PMD_SIZE)
1299 			return PAGE_SIZE;
1300 
1301 		/*
1302 		 * The address we faulted on is backed by a transparent huge
1303 		 * page.  However, because we map the compound huge page and
1304 		 * not the individual tail page, we need to transfer the
1305 		 * refcount to the head page.  We have to be careful that the
1306 		 * THP doesn't start to split while we are adjusting the
1307 		 * refcounts.
1308 		 *
1309 		 * We are sure this doesn't happen, because mmu_invalidate_retry
1310 		 * was successful and we are holding the mmu_lock, so if this
1311 		 * THP is trying to split, it will be blocked in the mmu
1312 		 * notifier before touching any of the pages, specifically
1313 		 * before being able to call __split_huge_page_refcount().
1314 		 *
1315 		 * We can therefore safely transfer the refcount from PG_tail
1316 		 * to PG_head and switch the pfn from a tail page to the head
1317 		 * page accordingly.
1318 		 */
1319 		*ipap &= PMD_MASK;
1320 		kvm_release_pfn_clean(pfn);
1321 		pfn &= ~(PTRS_PER_PMD - 1);
1322 		get_page(pfn_to_page(pfn));
1323 		*pfnp = pfn;
1324 
1325 		return PMD_SIZE;
1326 	}
1327 
1328 	/* Use page mapping if we cannot use block mapping. */
1329 	return PAGE_SIZE;
1330 }
1331 
get_vma_page_shift(struct vm_area_struct * vma,unsigned long hva)1332 static int get_vma_page_shift(struct vm_area_struct *vma, unsigned long hva)
1333 {
1334 	unsigned long pa;
1335 
1336 	if (is_vm_hugetlb_page(vma) && !(vma->vm_flags & VM_PFNMAP))
1337 		return huge_page_shift(hstate_vma(vma));
1338 
1339 	if (!(vma->vm_flags & VM_PFNMAP))
1340 		return PAGE_SHIFT;
1341 
1342 	VM_BUG_ON(is_vm_hugetlb_page(vma));
1343 
1344 	pa = (vma->vm_pgoff << PAGE_SHIFT) + (hva - vma->vm_start);
1345 
1346 #ifndef __PAGETABLE_PMD_FOLDED
1347 	if ((hva & (PUD_SIZE - 1)) == (pa & (PUD_SIZE - 1)) &&
1348 	    ALIGN_DOWN(hva, PUD_SIZE) >= vma->vm_start &&
1349 	    ALIGN(hva, PUD_SIZE) <= vma->vm_end)
1350 		return PUD_SHIFT;
1351 #endif
1352 
1353 	if ((hva & (PMD_SIZE - 1)) == (pa & (PMD_SIZE - 1)) &&
1354 	    ALIGN_DOWN(hva, PMD_SIZE) >= vma->vm_start &&
1355 	    ALIGN(hva, PMD_SIZE) <= vma->vm_end)
1356 		return PMD_SHIFT;
1357 
1358 	return PAGE_SHIFT;
1359 }
1360 
1361 /*
1362  * The page will be mapped in stage 2 as Normal Cacheable, so the VM will be
1363  * able to see the page's tags and therefore they must be initialised first. If
1364  * PG_mte_tagged is set, tags have already been initialised.
1365  *
1366  * The race in the test/set of the PG_mte_tagged flag is handled by:
1367  * - preventing VM_SHARED mappings in a memslot with MTE preventing two VMs
1368  *   racing to santise the same page
1369  * - mmap_lock protects between a VM faulting a page in and the VMM performing
1370  *   an mprotect() to add VM_MTE
1371  */
sanitise_mte_tags(struct kvm * kvm,kvm_pfn_t pfn,unsigned long size)1372 static void sanitise_mte_tags(struct kvm *kvm, kvm_pfn_t pfn,
1373 			      unsigned long size)
1374 {
1375 	unsigned long i, nr_pages = size >> PAGE_SHIFT;
1376 	struct page *page = pfn_to_page(pfn);
1377 
1378 	if (!kvm_has_mte(kvm))
1379 		return;
1380 
1381 	for (i = 0; i < nr_pages; i++, page++) {
1382 		if (try_page_mte_tagging(page)) {
1383 			mte_clear_page_tags(page_address(page));
1384 			set_page_mte_tagged(page);
1385 		}
1386 	}
1387 }
1388 
kvm_vma_mte_allowed(struct vm_area_struct * vma)1389 static bool kvm_vma_mte_allowed(struct vm_area_struct *vma)
1390 {
1391 	return vma->vm_flags & VM_MTE_ALLOWED;
1392 }
1393 
user_mem_abort(struct kvm_vcpu * vcpu,phys_addr_t fault_ipa,struct kvm_memory_slot * memslot,unsigned long hva,unsigned long fault_status)1394 static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
1395 			  struct kvm_memory_slot *memslot, unsigned long hva,
1396 			  unsigned long fault_status)
1397 {
1398 	int ret = 0;
1399 	bool write_fault, writable, force_pte = false;
1400 	bool exec_fault, mte_allowed;
1401 	bool device = false;
1402 	unsigned long mmu_seq;
1403 	struct kvm *kvm = vcpu->kvm;
1404 	struct kvm_mmu_memory_cache *memcache = &vcpu->arch.mmu_page_cache;
1405 	struct vm_area_struct *vma;
1406 	short vma_shift;
1407 	gfn_t gfn;
1408 	kvm_pfn_t pfn;
1409 	bool logging_active = memslot_is_logging(memslot);
1410 	unsigned long fault_level = kvm_vcpu_trap_get_fault_level(vcpu);
1411 	long vma_pagesize, fault_granule;
1412 	enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_R;
1413 	struct kvm_pgtable *pgt;
1414 
1415 	fault_granule = 1UL << ARM64_HW_PGTABLE_LEVEL_SHIFT(fault_level);
1416 	write_fault = kvm_is_write_fault(vcpu);
1417 	exec_fault = kvm_vcpu_trap_is_exec_fault(vcpu);
1418 	VM_BUG_ON(write_fault && exec_fault);
1419 
1420 	if (fault_status == ESR_ELx_FSC_PERM && !write_fault && !exec_fault) {
1421 		kvm_err("Unexpected L2 read permission error\n");
1422 		return -EFAULT;
1423 	}
1424 
1425 	/*
1426 	 * Permission faults just need to update the existing leaf entry,
1427 	 * and so normally don't require allocations from the memcache. The
1428 	 * only exception to this is when dirty logging is enabled at runtime
1429 	 * and a write fault needs to collapse a block entry into a table.
1430 	 */
1431 	if (fault_status != ESR_ELx_FSC_PERM ||
1432 	    (logging_active && write_fault)) {
1433 		ret = kvm_mmu_topup_memory_cache(memcache,
1434 						 kvm_mmu_cache_min_pages(kvm));
1435 		if (ret)
1436 			return ret;
1437 	}
1438 
1439 	/*
1440 	 * Let's check if we will get back a huge page backed by hugetlbfs, or
1441 	 * get block mapping for device MMIO region.
1442 	 */
1443 	mmap_read_lock(current->mm);
1444 	vma = vma_lookup(current->mm, hva);
1445 	if (unlikely(!vma)) {
1446 		kvm_err("Failed to find VMA for hva 0x%lx\n", hva);
1447 		mmap_read_unlock(current->mm);
1448 		return -EFAULT;
1449 	}
1450 
1451 	/*
1452 	 * logging_active is guaranteed to never be true for VM_PFNMAP
1453 	 * memslots.
1454 	 */
1455 	if (logging_active) {
1456 		force_pte = true;
1457 		vma_shift = PAGE_SHIFT;
1458 	} else {
1459 		vma_shift = get_vma_page_shift(vma, hva);
1460 	}
1461 
1462 	switch (vma_shift) {
1463 #ifndef __PAGETABLE_PMD_FOLDED
1464 	case PUD_SHIFT:
1465 		if (fault_supports_stage2_huge_mapping(memslot, hva, PUD_SIZE))
1466 			break;
1467 		fallthrough;
1468 #endif
1469 	case CONT_PMD_SHIFT:
1470 		vma_shift = PMD_SHIFT;
1471 		fallthrough;
1472 	case PMD_SHIFT:
1473 		if (fault_supports_stage2_huge_mapping(memslot, hva, PMD_SIZE))
1474 			break;
1475 		fallthrough;
1476 	case CONT_PTE_SHIFT:
1477 		vma_shift = PAGE_SHIFT;
1478 		force_pte = true;
1479 		fallthrough;
1480 	case PAGE_SHIFT:
1481 		break;
1482 	default:
1483 		WARN_ONCE(1, "Unknown vma_shift %d", vma_shift);
1484 	}
1485 
1486 	vma_pagesize = 1UL << vma_shift;
1487 	if (vma_pagesize == PMD_SIZE || vma_pagesize == PUD_SIZE)
1488 		fault_ipa &= ~(vma_pagesize - 1);
1489 
1490 	gfn = fault_ipa >> PAGE_SHIFT;
1491 	mte_allowed = kvm_vma_mte_allowed(vma);
1492 
1493 	/* Don't use the VMA after the unlock -- it may have vanished */
1494 	vma = NULL;
1495 
1496 	/*
1497 	 * Read mmu_invalidate_seq so that KVM can detect if the results of
1498 	 * vma_lookup() or __gfn_to_pfn_memslot() become stale prior to
1499 	 * acquiring kvm->mmu_lock.
1500 	 *
1501 	 * Rely on mmap_read_unlock() for an implicit smp_rmb(), which pairs
1502 	 * with the smp_wmb() in kvm_mmu_invalidate_end().
1503 	 */
1504 	mmu_seq = vcpu->kvm->mmu_invalidate_seq;
1505 	mmap_read_unlock(current->mm);
1506 
1507 	pfn = __gfn_to_pfn_memslot(memslot, gfn, false, false, NULL,
1508 				   write_fault, &writable, NULL);
1509 	if (pfn == KVM_PFN_ERR_HWPOISON) {
1510 		kvm_send_hwpoison_signal(hva, vma_shift);
1511 		return 0;
1512 	}
1513 	if (is_error_noslot_pfn(pfn))
1514 		return -EFAULT;
1515 
1516 	if (kvm_is_device_pfn(pfn)) {
1517 		/*
1518 		 * If the page was identified as device early by looking at
1519 		 * the VMA flags, vma_pagesize is already representing the
1520 		 * largest quantity we can map.  If instead it was mapped
1521 		 * via gfn_to_pfn_prot(), vma_pagesize is set to PAGE_SIZE
1522 		 * and must not be upgraded.
1523 		 *
1524 		 * In both cases, we don't let transparent_hugepage_adjust()
1525 		 * change things at the last minute.
1526 		 */
1527 		device = true;
1528 	} else if (logging_active && !write_fault) {
1529 		/*
1530 		 * Only actually map the page as writable if this was a write
1531 		 * fault.
1532 		 */
1533 		writable = false;
1534 	}
1535 
1536 	if (exec_fault && device)
1537 		return -ENOEXEC;
1538 
1539 	read_lock(&kvm->mmu_lock);
1540 	pgt = vcpu->arch.hw_mmu->pgt;
1541 	if (mmu_invalidate_retry(kvm, mmu_seq))
1542 		goto out_unlock;
1543 
1544 	/*
1545 	 * If we are not forced to use page mapping, check if we are
1546 	 * backed by a THP and thus use block mapping if possible.
1547 	 */
1548 	if (vma_pagesize == PAGE_SIZE && !(force_pte || device)) {
1549 		if (fault_status ==  ESR_ELx_FSC_PERM &&
1550 		    fault_granule > PAGE_SIZE)
1551 			vma_pagesize = fault_granule;
1552 		else
1553 			vma_pagesize = transparent_hugepage_adjust(kvm, memslot,
1554 								   hva, &pfn,
1555 								   &fault_ipa);
1556 
1557 		if (vma_pagesize < 0) {
1558 			ret = vma_pagesize;
1559 			goto out_unlock;
1560 		}
1561 	}
1562 
1563 	if (fault_status != ESR_ELx_FSC_PERM && !device && kvm_has_mte(kvm)) {
1564 		/* Check the VMM hasn't introduced a new disallowed VMA */
1565 		if (mte_allowed) {
1566 			sanitise_mte_tags(kvm, pfn, vma_pagesize);
1567 		} else {
1568 			ret = -EFAULT;
1569 			goto out_unlock;
1570 		}
1571 	}
1572 
1573 	if (writable)
1574 		prot |= KVM_PGTABLE_PROT_W;
1575 
1576 	if (exec_fault)
1577 		prot |= KVM_PGTABLE_PROT_X;
1578 
1579 	if (device)
1580 		prot |= KVM_PGTABLE_PROT_DEVICE;
1581 	else if (cpus_have_const_cap(ARM64_HAS_CACHE_DIC))
1582 		prot |= KVM_PGTABLE_PROT_X;
1583 
1584 	/*
1585 	 * Under the premise of getting a FSC_PERM fault, we just need to relax
1586 	 * permissions only if vma_pagesize equals fault_granule. Otherwise,
1587 	 * kvm_pgtable_stage2_map() should be called to change block size.
1588 	 */
1589 	if (fault_status == ESR_ELx_FSC_PERM && vma_pagesize == fault_granule)
1590 		ret = kvm_pgtable_stage2_relax_perms(pgt, fault_ipa, prot);
1591 	else
1592 		ret = kvm_pgtable_stage2_map(pgt, fault_ipa, vma_pagesize,
1593 					     __pfn_to_phys(pfn), prot,
1594 					     memcache,
1595 					     KVM_PGTABLE_WALK_HANDLE_FAULT |
1596 					     KVM_PGTABLE_WALK_SHARED);
1597 
1598 	/* Mark the page dirty only if the fault is handled successfully */
1599 	if (writable && !ret) {
1600 		kvm_set_pfn_dirty(pfn);
1601 		mark_page_dirty_in_slot(kvm, memslot, gfn);
1602 	}
1603 
1604 out_unlock:
1605 	read_unlock(&kvm->mmu_lock);
1606 	kvm_release_pfn_clean(pfn);
1607 	return ret != -EAGAIN ? ret : 0;
1608 }
1609 
1610 /* Resolve the access fault by making the page young again. */
handle_access_fault(struct kvm_vcpu * vcpu,phys_addr_t fault_ipa)1611 static void handle_access_fault(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa)
1612 {
1613 	kvm_pte_t pte;
1614 	struct kvm_s2_mmu *mmu;
1615 
1616 	trace_kvm_access_fault(fault_ipa);
1617 
1618 	read_lock(&vcpu->kvm->mmu_lock);
1619 	mmu = vcpu->arch.hw_mmu;
1620 	pte = kvm_pgtable_stage2_mkyoung(mmu->pgt, fault_ipa);
1621 	read_unlock(&vcpu->kvm->mmu_lock);
1622 
1623 	if (kvm_pte_valid(pte))
1624 		kvm_set_pfn_accessed(kvm_pte_to_pfn(pte));
1625 }
1626 
1627 /**
1628  * kvm_handle_guest_abort - handles all 2nd stage aborts
1629  * @vcpu:	the VCPU pointer
1630  *
1631  * Any abort that gets to the host is almost guaranteed to be caused by a
1632  * missing second stage translation table entry, which can mean that either the
1633  * guest simply needs more memory and we must allocate an appropriate page or it
1634  * can mean that the guest tried to access I/O memory, which is emulated by user
1635  * space. The distinction is based on the IPA causing the fault and whether this
1636  * memory region has been registered as standard RAM by user space.
1637  */
kvm_handle_guest_abort(struct kvm_vcpu * vcpu)1638 int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
1639 {
1640 	unsigned long fault_status;
1641 	phys_addr_t fault_ipa;
1642 	struct kvm_memory_slot *memslot;
1643 	unsigned long hva;
1644 	bool is_iabt, write_fault, writable;
1645 	gfn_t gfn;
1646 	int ret, idx;
1647 
1648 	fault_status = kvm_vcpu_trap_get_fault_type(vcpu);
1649 
1650 	fault_ipa = kvm_vcpu_get_fault_ipa(vcpu);
1651 	is_iabt = kvm_vcpu_trap_is_iabt(vcpu);
1652 
1653 	if (fault_status == ESR_ELx_FSC_FAULT) {
1654 		/* Beyond sanitised PARange (which is the IPA limit) */
1655 		if (fault_ipa >= BIT_ULL(get_kvm_ipa_limit())) {
1656 			kvm_inject_size_fault(vcpu);
1657 			return 1;
1658 		}
1659 
1660 		/* Falls between the IPA range and the PARange? */
1661 		if (fault_ipa >= BIT_ULL(vcpu->arch.hw_mmu->pgt->ia_bits)) {
1662 			fault_ipa |= kvm_vcpu_get_hfar(vcpu) & GENMASK(11, 0);
1663 
1664 			if (is_iabt)
1665 				kvm_inject_pabt(vcpu, fault_ipa);
1666 			else
1667 				kvm_inject_dabt(vcpu, fault_ipa);
1668 			return 1;
1669 		}
1670 	}
1671 
1672 	/* Synchronous External Abort? */
1673 	if (kvm_vcpu_abt_issea(vcpu)) {
1674 		/*
1675 		 * For RAS the host kernel may handle this abort.
1676 		 * There is no need to pass the error into the guest.
1677 		 */
1678 		if (kvm_handle_guest_sea(fault_ipa, kvm_vcpu_get_esr(vcpu)))
1679 			kvm_inject_vabt(vcpu);
1680 
1681 		return 1;
1682 	}
1683 
1684 	trace_kvm_guest_fault(*vcpu_pc(vcpu), kvm_vcpu_get_esr(vcpu),
1685 			      kvm_vcpu_get_hfar(vcpu), fault_ipa);
1686 
1687 	/* Check the stage-2 fault is trans. fault or write fault */
1688 	if (fault_status != ESR_ELx_FSC_FAULT &&
1689 	    fault_status != ESR_ELx_FSC_PERM &&
1690 	    fault_status != ESR_ELx_FSC_ACCESS) {
1691 		kvm_err("Unsupported FSC: EC=%#x xFSC=%#lx ESR_EL2=%#lx\n",
1692 			kvm_vcpu_trap_get_class(vcpu),
1693 			(unsigned long)kvm_vcpu_trap_get_fault(vcpu),
1694 			(unsigned long)kvm_vcpu_get_esr(vcpu));
1695 		return -EFAULT;
1696 	}
1697 
1698 	idx = srcu_read_lock(&vcpu->kvm->srcu);
1699 
1700 	gfn = fault_ipa >> PAGE_SHIFT;
1701 	memslot = gfn_to_memslot(vcpu->kvm, gfn);
1702 	hva = gfn_to_hva_memslot_prot(memslot, gfn, &writable);
1703 	write_fault = kvm_is_write_fault(vcpu);
1704 	if (kvm_is_error_hva(hva) || (write_fault && !writable)) {
1705 		/*
1706 		 * The guest has put either its instructions or its page-tables
1707 		 * somewhere it shouldn't have. Userspace won't be able to do
1708 		 * anything about this (there's no syndrome for a start), so
1709 		 * re-inject the abort back into the guest.
1710 		 */
1711 		if (is_iabt) {
1712 			ret = -ENOEXEC;
1713 			goto out;
1714 		}
1715 
1716 		if (kvm_vcpu_abt_iss1tw(vcpu)) {
1717 			kvm_inject_dabt(vcpu, kvm_vcpu_get_hfar(vcpu));
1718 			ret = 1;
1719 			goto out_unlock;
1720 		}
1721 
1722 		/*
1723 		 * Check for a cache maintenance operation. Since we
1724 		 * ended-up here, we know it is outside of any memory
1725 		 * slot. But we can't find out if that is for a device,
1726 		 * or if the guest is just being stupid. The only thing
1727 		 * we know for sure is that this range cannot be cached.
1728 		 *
1729 		 * So let's assume that the guest is just being
1730 		 * cautious, and skip the instruction.
1731 		 */
1732 		if (kvm_is_error_hva(hva) && kvm_vcpu_dabt_is_cm(vcpu)) {
1733 			kvm_incr_pc(vcpu);
1734 			ret = 1;
1735 			goto out_unlock;
1736 		}
1737 
1738 		/*
1739 		 * The IPA is reported as [MAX:12], so we need to
1740 		 * complement it with the bottom 12 bits from the
1741 		 * faulting VA. This is always 12 bits, irrespective
1742 		 * of the page size.
1743 		 */
1744 		fault_ipa |= kvm_vcpu_get_hfar(vcpu) & ((1 << 12) - 1);
1745 		ret = io_mem_abort(vcpu, fault_ipa);
1746 		goto out_unlock;
1747 	}
1748 
1749 	/* Userspace should not be able to register out-of-bounds IPAs */
1750 	VM_BUG_ON(fault_ipa >= kvm_phys_size(vcpu->kvm));
1751 
1752 	if (fault_status == ESR_ELx_FSC_ACCESS) {
1753 		handle_access_fault(vcpu, fault_ipa);
1754 		ret = 1;
1755 		goto out_unlock;
1756 	}
1757 
1758 	ret = user_mem_abort(vcpu, fault_ipa, memslot, hva, fault_status);
1759 	if (ret == 0)
1760 		ret = 1;
1761 out:
1762 	if (ret == -ENOEXEC) {
1763 		kvm_inject_pabt(vcpu, kvm_vcpu_get_hfar(vcpu));
1764 		ret = 1;
1765 	}
1766 out_unlock:
1767 	srcu_read_unlock(&vcpu->kvm->srcu, idx);
1768 	return ret;
1769 }
1770 
kvm_unmap_gfn_range(struct kvm * kvm,struct kvm_gfn_range * range)1771 bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range)
1772 {
1773 	if (!kvm->arch.mmu.pgt)
1774 		return false;
1775 
1776 	__unmap_stage2_range(&kvm->arch.mmu, range->start << PAGE_SHIFT,
1777 			     (range->end - range->start) << PAGE_SHIFT,
1778 			     range->may_block);
1779 
1780 	return false;
1781 }
1782 
kvm_set_spte_gfn(struct kvm * kvm,struct kvm_gfn_range * range)1783 bool kvm_set_spte_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
1784 {
1785 	kvm_pfn_t pfn = pte_pfn(range->arg.pte);
1786 
1787 	if (!kvm->arch.mmu.pgt)
1788 		return false;
1789 
1790 	WARN_ON(range->end - range->start != 1);
1791 
1792 	/*
1793 	 * If the page isn't tagged, defer to user_mem_abort() for sanitising
1794 	 * the MTE tags. The S2 pte should have been unmapped by
1795 	 * mmu_notifier_invalidate_range_end().
1796 	 */
1797 	if (kvm_has_mte(kvm) && !page_mte_tagged(pfn_to_page(pfn)))
1798 		return false;
1799 
1800 	/*
1801 	 * We've moved a page around, probably through CoW, so let's treat
1802 	 * it just like a translation fault and the map handler will clean
1803 	 * the cache to the PoC.
1804 	 *
1805 	 * The MMU notifiers will have unmapped a huge PMD before calling
1806 	 * ->change_pte() (which in turn calls kvm_set_spte_gfn()) and
1807 	 * therefore we never need to clear out a huge PMD through this
1808 	 * calling path and a memcache is not required.
1809 	 */
1810 	kvm_pgtable_stage2_map(kvm->arch.mmu.pgt, range->start << PAGE_SHIFT,
1811 			       PAGE_SIZE, __pfn_to_phys(pfn),
1812 			       KVM_PGTABLE_PROT_R, NULL, 0);
1813 
1814 	return false;
1815 }
1816 
kvm_age_gfn(struct kvm * kvm,struct kvm_gfn_range * range)1817 bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
1818 {
1819 	u64 size = (range->end - range->start) << PAGE_SHIFT;
1820 
1821 	if (!kvm->arch.mmu.pgt)
1822 		return false;
1823 
1824 	return kvm_pgtable_stage2_test_clear_young(kvm->arch.mmu.pgt,
1825 						   range->start << PAGE_SHIFT,
1826 						   size, true);
1827 }
1828 
kvm_test_age_gfn(struct kvm * kvm,struct kvm_gfn_range * range)1829 bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
1830 {
1831 	u64 size = (range->end - range->start) << PAGE_SHIFT;
1832 
1833 	if (!kvm->arch.mmu.pgt)
1834 		return false;
1835 
1836 	return kvm_pgtable_stage2_test_clear_young(kvm->arch.mmu.pgt,
1837 						   range->start << PAGE_SHIFT,
1838 						   size, false);
1839 }
1840 
kvm_mmu_get_httbr(void)1841 phys_addr_t kvm_mmu_get_httbr(void)
1842 {
1843 	return __pa(hyp_pgtable->pgd);
1844 }
1845 
kvm_get_idmap_vector(void)1846 phys_addr_t kvm_get_idmap_vector(void)
1847 {
1848 	return hyp_idmap_vector;
1849 }
1850 
kvm_map_idmap_text(void)1851 static int kvm_map_idmap_text(void)
1852 {
1853 	unsigned long size = hyp_idmap_end - hyp_idmap_start;
1854 	int err = __create_hyp_mappings(hyp_idmap_start, size, hyp_idmap_start,
1855 					PAGE_HYP_EXEC);
1856 	if (err)
1857 		kvm_err("Failed to idmap %lx-%lx\n",
1858 			hyp_idmap_start, hyp_idmap_end);
1859 
1860 	return err;
1861 }
1862 
kvm_hyp_zalloc_page(void * arg)1863 static void *kvm_hyp_zalloc_page(void *arg)
1864 {
1865 	return (void *)get_zeroed_page(GFP_KERNEL);
1866 }
1867 
1868 static struct kvm_pgtable_mm_ops kvm_hyp_mm_ops = {
1869 	.zalloc_page		= kvm_hyp_zalloc_page,
1870 	.get_page		= kvm_host_get_page,
1871 	.put_page		= kvm_host_put_page,
1872 	.phys_to_virt		= kvm_host_va,
1873 	.virt_to_phys		= kvm_host_pa,
1874 };
1875 
kvm_mmu_init(u32 * hyp_va_bits)1876 int __init kvm_mmu_init(u32 *hyp_va_bits)
1877 {
1878 	int err;
1879 	u32 idmap_bits;
1880 	u32 kernel_bits;
1881 
1882 	hyp_idmap_start = __pa_symbol(__hyp_idmap_text_start);
1883 	hyp_idmap_start = ALIGN_DOWN(hyp_idmap_start, PAGE_SIZE);
1884 	hyp_idmap_end = __pa_symbol(__hyp_idmap_text_end);
1885 	hyp_idmap_end = ALIGN(hyp_idmap_end, PAGE_SIZE);
1886 	hyp_idmap_vector = __pa_symbol(__kvm_hyp_init);
1887 
1888 	/*
1889 	 * We rely on the linker script to ensure at build time that the HYP
1890 	 * init code does not cross a page boundary.
1891 	 */
1892 	BUG_ON((hyp_idmap_start ^ (hyp_idmap_end - 1)) & PAGE_MASK);
1893 
1894 	/*
1895 	 * The ID map may be configured to use an extended virtual address
1896 	 * range. This is only the case if system RAM is out of range for the
1897 	 * currently configured page size and VA_BITS_MIN, in which case we will
1898 	 * also need the extended virtual range for the HYP ID map, or we won't
1899 	 * be able to enable the EL2 MMU.
1900 	 *
1901 	 * However, in some cases the ID map may be configured for fewer than
1902 	 * the number of VA bits used by the regular kernel stage 1. This
1903 	 * happens when VA_BITS=52 and the kernel image is placed in PA space
1904 	 * below 48 bits.
1905 	 *
1906 	 * At EL2, there is only one TTBR register, and we can't switch between
1907 	 * translation tables *and* update TCR_EL2.T0SZ at the same time. Bottom
1908 	 * line: we need to use the extended range with *both* our translation
1909 	 * tables.
1910 	 *
1911 	 * So use the maximum of the idmap VA bits and the regular kernel stage
1912 	 * 1 VA bits to assure that the hypervisor can both ID map its code page
1913 	 * and map any kernel memory.
1914 	 */
1915 	idmap_bits = 64 - ((idmap_t0sz & TCR_T0SZ_MASK) >> TCR_T0SZ_OFFSET);
1916 	kernel_bits = vabits_actual;
1917 	*hyp_va_bits = max(idmap_bits, kernel_bits);
1918 
1919 	kvm_debug("Using %u-bit virtual addresses at EL2\n", *hyp_va_bits);
1920 	kvm_debug("IDMAP page: %lx\n", hyp_idmap_start);
1921 	kvm_debug("HYP VA range: %lx:%lx\n",
1922 		  kern_hyp_va(PAGE_OFFSET),
1923 		  kern_hyp_va((unsigned long)high_memory - 1));
1924 
1925 	if (hyp_idmap_start >= kern_hyp_va(PAGE_OFFSET) &&
1926 	    hyp_idmap_start <  kern_hyp_va((unsigned long)high_memory - 1) &&
1927 	    hyp_idmap_start != (unsigned long)__hyp_idmap_text_start) {
1928 		/*
1929 		 * The idmap page is intersecting with the VA space,
1930 		 * it is not safe to continue further.
1931 		 */
1932 		kvm_err("IDMAP intersecting with HYP VA, unable to continue\n");
1933 		err = -EINVAL;
1934 		goto out;
1935 	}
1936 
1937 	hyp_pgtable = kzalloc(sizeof(*hyp_pgtable), GFP_KERNEL);
1938 	if (!hyp_pgtable) {
1939 		kvm_err("Hyp mode page-table not allocated\n");
1940 		err = -ENOMEM;
1941 		goto out;
1942 	}
1943 
1944 	err = kvm_pgtable_hyp_init(hyp_pgtable, *hyp_va_bits, &kvm_hyp_mm_ops);
1945 	if (err)
1946 		goto out_free_pgtable;
1947 
1948 	err = kvm_map_idmap_text();
1949 	if (err)
1950 		goto out_destroy_pgtable;
1951 
1952 	io_map_base = hyp_idmap_start;
1953 	return 0;
1954 
1955 out_destroy_pgtable:
1956 	kvm_pgtable_hyp_destroy(hyp_pgtable);
1957 out_free_pgtable:
1958 	kfree(hyp_pgtable);
1959 	hyp_pgtable = NULL;
1960 out:
1961 	return err;
1962 }
1963 
kvm_arch_commit_memory_region(struct kvm * kvm,struct kvm_memory_slot * old,const struct kvm_memory_slot * new,enum kvm_mr_change change)1964 void kvm_arch_commit_memory_region(struct kvm *kvm,
1965 				   struct kvm_memory_slot *old,
1966 				   const struct kvm_memory_slot *new,
1967 				   enum kvm_mr_change change)
1968 {
1969 	bool log_dirty_pages = new && new->flags & KVM_MEM_LOG_DIRTY_PAGES;
1970 
1971 	/*
1972 	 * At this point memslot has been committed and there is an
1973 	 * allocated dirty_bitmap[], dirty pages will be tracked while the
1974 	 * memory slot is write protected.
1975 	 */
1976 	if (log_dirty_pages) {
1977 
1978 		if (change == KVM_MR_DELETE)
1979 			return;
1980 
1981 		/*
1982 		 * Huge and normal pages are write-protected and split
1983 		 * on either of these two cases:
1984 		 *
1985 		 * 1. with initial-all-set: gradually with CLEAR ioctls,
1986 		 */
1987 		if (kvm_dirty_log_manual_protect_and_init_set(kvm))
1988 			return;
1989 		/*
1990 		 * or
1991 		 * 2. without initial-all-set: all in one shot when
1992 		 *    enabling dirty logging.
1993 		 */
1994 		kvm_mmu_wp_memory_region(kvm, new->id);
1995 		kvm_mmu_split_memory_region(kvm, new->id);
1996 	} else {
1997 		/*
1998 		 * Free any leftovers from the eager page splitting cache. Do
1999 		 * this when deleting, moving, disabling dirty logging, or
2000 		 * creating the memslot (a nop). Doing it for deletes makes
2001 		 * sure we don't leak memory, and there's no need to keep the
2002 		 * cache around for any of the other cases.
2003 		 */
2004 		kvm_mmu_free_memory_cache(&kvm->arch.mmu.split_page_cache);
2005 	}
2006 }
2007 
kvm_arch_prepare_memory_region(struct kvm * kvm,const struct kvm_memory_slot * old,struct kvm_memory_slot * new,enum kvm_mr_change change)2008 int kvm_arch_prepare_memory_region(struct kvm *kvm,
2009 				   const struct kvm_memory_slot *old,
2010 				   struct kvm_memory_slot *new,
2011 				   enum kvm_mr_change change)
2012 {
2013 	hva_t hva, reg_end;
2014 	int ret = 0;
2015 
2016 	if (change != KVM_MR_CREATE && change != KVM_MR_MOVE &&
2017 			change != KVM_MR_FLAGS_ONLY)
2018 		return 0;
2019 
2020 	/*
2021 	 * Prevent userspace from creating a memory region outside of the IPA
2022 	 * space addressable by the KVM guest IPA space.
2023 	 */
2024 	if ((new->base_gfn + new->npages) > (kvm_phys_size(kvm) >> PAGE_SHIFT))
2025 		return -EFAULT;
2026 
2027 	hva = new->userspace_addr;
2028 	reg_end = hva + (new->npages << PAGE_SHIFT);
2029 
2030 	mmap_read_lock(current->mm);
2031 	/*
2032 	 * A memory region could potentially cover multiple VMAs, and any holes
2033 	 * between them, so iterate over all of them.
2034 	 *
2035 	 *     +--------------------------------------------+
2036 	 * +---------------+----------------+   +----------------+
2037 	 * |   : VMA 1     |      VMA 2     |   |    VMA 3  :    |
2038 	 * +---------------+----------------+   +----------------+
2039 	 *     |               memory region                |
2040 	 *     +--------------------------------------------+
2041 	 */
2042 	do {
2043 		struct vm_area_struct *vma;
2044 
2045 		vma = find_vma_intersection(current->mm, hva, reg_end);
2046 		if (!vma)
2047 			break;
2048 
2049 		if (kvm_has_mte(kvm) && !kvm_vma_mte_allowed(vma)) {
2050 			ret = -EINVAL;
2051 			break;
2052 		}
2053 
2054 		if (vma->vm_flags & VM_PFNMAP) {
2055 			/* IO region dirty page logging not allowed */
2056 			if (new->flags & KVM_MEM_LOG_DIRTY_PAGES) {
2057 				ret = -EINVAL;
2058 				break;
2059 			}
2060 		}
2061 		hva = min(reg_end, vma->vm_end);
2062 	} while (hva < reg_end);
2063 
2064 	mmap_read_unlock(current->mm);
2065 	return ret;
2066 }
2067 
kvm_arch_free_memslot(struct kvm * kvm,struct kvm_memory_slot * slot)2068 void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot)
2069 {
2070 }
2071 
kvm_arch_memslots_updated(struct kvm * kvm,u64 gen)2072 void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen)
2073 {
2074 }
2075 
kvm_arch_flush_shadow_all(struct kvm * kvm)2076 void kvm_arch_flush_shadow_all(struct kvm *kvm)
2077 {
2078 	kvm_uninit_stage2_mmu(kvm);
2079 }
2080 
kvm_arch_flush_shadow_memslot(struct kvm * kvm,struct kvm_memory_slot * slot)2081 void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
2082 				   struct kvm_memory_slot *slot)
2083 {
2084 	gpa_t gpa = slot->base_gfn << PAGE_SHIFT;
2085 	phys_addr_t size = slot->npages << PAGE_SHIFT;
2086 
2087 	write_lock(&kvm->mmu_lock);
2088 	unmap_stage2_range(&kvm->arch.mmu, gpa, size);
2089 	write_unlock(&kvm->mmu_lock);
2090 }
2091 
2092 /*
2093  * See note at ARMv7 ARM B1.14.4 (TL;DR: S/W ops are not easily virtualized).
2094  *
2095  * Main problems:
2096  * - S/W ops are local to a CPU (not broadcast)
2097  * - We have line migration behind our back (speculation)
2098  * - System caches don't support S/W at all (damn!)
2099  *
2100  * In the face of the above, the best we can do is to try and convert
2101  * S/W ops to VA ops. Because the guest is not allowed to infer the
2102  * S/W to PA mapping, it can only use S/W to nuke the whole cache,
2103  * which is a rather good thing for us.
2104  *
2105  * Also, it is only used when turning caches on/off ("The expected
2106  * usage of the cache maintenance instructions that operate by set/way
2107  * is associated with the cache maintenance instructions associated
2108  * with the powerdown and powerup of caches, if this is required by
2109  * the implementation.").
2110  *
2111  * We use the following policy:
2112  *
2113  * - If we trap a S/W operation, we enable VM trapping to detect
2114  *   caches being turned on/off, and do a full clean.
2115  *
2116  * - We flush the caches on both caches being turned on and off.
2117  *
2118  * - Once the caches are enabled, we stop trapping VM ops.
2119  */
kvm_set_way_flush(struct kvm_vcpu * vcpu)2120 void kvm_set_way_flush(struct kvm_vcpu *vcpu)
2121 {
2122 	unsigned long hcr = *vcpu_hcr(vcpu);
2123 
2124 	/*
2125 	 * If this is the first time we do a S/W operation
2126 	 * (i.e. HCR_TVM not set) flush the whole memory, and set the
2127 	 * VM trapping.
2128 	 *
2129 	 * Otherwise, rely on the VM trapping to wait for the MMU +
2130 	 * Caches to be turned off. At that point, we'll be able to
2131 	 * clean the caches again.
2132 	 */
2133 	if (!(hcr & HCR_TVM)) {
2134 		trace_kvm_set_way_flush(*vcpu_pc(vcpu),
2135 					vcpu_has_cache_enabled(vcpu));
2136 		stage2_flush_vm(vcpu->kvm);
2137 		*vcpu_hcr(vcpu) = hcr | HCR_TVM;
2138 	}
2139 }
2140 
kvm_toggle_cache(struct kvm_vcpu * vcpu,bool was_enabled)2141 void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled)
2142 {
2143 	bool now_enabled = vcpu_has_cache_enabled(vcpu);
2144 
2145 	/*
2146 	 * If switching the MMU+caches on, need to invalidate the caches.
2147 	 * If switching it off, need to clean the caches.
2148 	 * Clean + invalidate does the trick always.
2149 	 */
2150 	if (now_enabled != was_enabled)
2151 		stage2_flush_vm(vcpu->kvm);
2152 
2153 	/* Caches are now on, stop trapping VM ops (until a S/W op) */
2154 	if (now_enabled)
2155 		*vcpu_hcr(vcpu) &= ~HCR_TVM;
2156 
2157 	trace_kvm_toggle_cache(*vcpu_pc(vcpu), was_enabled, now_enabled);
2158 }
2159