138 static kvm_pte_t *kvm_pte_follow(kvm_pte_t pte, struct kvm_pgtable_mm_ops *mm_ops)  in kvm_pte_follow()  argument
140 	return mm_ops->phys_to_virt(kvm_pte_to_phys(pte));  in kvm_pte_follow()
148 static kvm_pte_t kvm_init_table_pte(kvm_pte_t *childp, struct kvm_pgtable_mm_ops *mm_ops)  in kvm_init_table_pte()  argument
150 	kvm_pte_t pte = kvm_phys_to_pte(mm_ops->virt_to_phys(childp));  in kvm_init_table_pte()
206 			      struct kvm_pgtable_mm_ops *mm_ops, kvm_pteref_t pgtable, u32 level);
209 				      struct kvm_pgtable_mm_ops *mm_ops,  in __kvm_pgtable_visit()  argument
218 		.mm_ops	= mm_ops,  in __kvm_pgtable_visit()
259 	childp = (kvm_pteref_t)kvm_pte_follow(ctx.old, mm_ops);  in __kvm_pgtable_visit()
260 	ret = __kvm_pgtable_walk(data, mm_ops, childp, level + 1);  in __kvm_pgtable_visit()
275 			      struct kvm_pgtable_mm_ops *mm_ops, kvm_pteref_t pgtable, u32 level)  in __kvm_pgtable_walk()  argument
289 		ret = __kvm_pgtable_visit(data, mm_ops, pteref, level);  in __kvm_pgtable_walk()
312 		ret = __kvm_pgtable_walk(data, pgt->mm_ops, pteref, pgt->start_level);  in _kvm_pgtable_walk()
452 		ctx->mm_ops->get_page(ctx->ptep);  in hyp_map_walker_try_leaf()
465 	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;  in hyp_map_walker()  local
473 	childp = (kvm_pte_t *)mm_ops->zalloc_page(NULL);  in hyp_map_walker()
477 	new = kvm_init_table_pte(childp, mm_ops);  in hyp_map_walker()
478 	mm_ops->get_page(ctx->ptep);  in hyp_map_walker()
513 	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;  in hyp_unmap_walker()  local
519 		childp = kvm_pte_follow(ctx->old, mm_ops);  in hyp_unmap_walker()
521 		if (mm_ops->page_count(childp) != 1)  in hyp_unmap_walker()
539 	mm_ops->put_page(ctx->ptep);  in hyp_unmap_walker()
542 		mm_ops->put_page(childp);  in hyp_unmap_walker()
556 	if (!pgt->mm_ops->page_count)  in kvm_pgtable_hyp_unmap()
564 			 struct kvm_pgtable_mm_ops *mm_ops)  in kvm_pgtable_hyp_init()  argument
568 	pgt->pgd = (kvm_pteref_t)mm_ops->zalloc_page(NULL);  in kvm_pgtable_hyp_init()
574 	pgt->mm_ops		= mm_ops;  in kvm_pgtable_hyp_init()
584 	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;  in hyp_free_walker()  local
589 	mm_ops->put_page(ctx->ptep);  in hyp_free_walker()
592 		mm_ops->put_page(kvm_pte_follow(ctx->old, mm_ops));  in hyp_free_walker()
605 	pgt->mm_ops->put_page(kvm_dereference_pteref(&walker, pgt->pgd));  in kvm_pgtable_hyp_destroy()
789 	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;  in stage2_try_break_pte()  local
817 		mm_ops->put_page(ctx->ptep);  in stage2_try_break_pte()
824 	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;  in stage2_make_pte()  local
829 		mm_ops->get_page(ctx->ptep);  in stage2_make_pte()
849 				struct kvm_pgtable_mm_ops *mm_ops)  in stage2_unmap_put_pte()  argument
866 	mm_ops->put_page(ctx->ptep);  in stage2_unmap_put_pte()
918 	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;  in stage2_map_walker_try_leaf()  local
941 	if (!kvm_pgtable_walk_skip_cmo(ctx) && mm_ops->dcache_clean_inval_poc &&  in stage2_map_walker_try_leaf()
943 		mm_ops->dcache_clean_inval_poc(kvm_pte_follow(new, mm_ops),  in stage2_map_walker_try_leaf()
946 	if (!kvm_pgtable_walk_skip_cmo(ctx) && mm_ops->icache_inval_pou &&  in stage2_map_walker_try_leaf()
948 		mm_ops->icache_inval_pou(kvm_pte_follow(new, mm_ops), granule);  in stage2_map_walker_try_leaf()
958 	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;  in stage2_map_walk_table_pre()  local
959 	kvm_pte_t *childp = kvm_pte_follow(ctx->old, mm_ops);  in stage2_map_walk_table_pre()
969 	mm_ops->free_unlinked_table(childp, ctx->level);  in stage2_map_walk_table_pre()
976 	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;  in stage2_map_walk_leaf()  local
990 	childp = mm_ops->zalloc_page(data->memcache);  in stage2_map_walk_leaf()
995 		mm_ops->put_page(childp);  in stage2_map_walk_leaf()
1004 	new = kvm_init_table_pte(childp, mm_ops);  in stage2_map_walk_leaf()
1095 	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;  in stage2_unmap_walker()  local
1102 			mm_ops->put_page(ctx->ptep);  in stage2_unmap_walker()
1108 		childp = kvm_pte_follow(ctx->old, mm_ops);  in stage2_unmap_walker()
1110 		if (mm_ops->page_count(childp) != 1)  in stage2_unmap_walker()
1121 	stage2_unmap_put_pte(ctx, mmu, mm_ops);  in stage2_unmap_walker()
1123 	if (need_flush && mm_ops->dcache_clean_inval_poc)  in stage2_unmap_walker()
1124 		mm_ops->dcache_clean_inval_poc(kvm_pte_follow(ctx->old, mm_ops),  in stage2_unmap_walker()
1128 		mm_ops->put_page(childp);  in stage2_unmap_walker()
1162 	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;  in stage2_attr_walker()  local
1182 		if (mm_ops->icache_inval_pou &&  in stage2_attr_walker()
1184 			mm_ops->icache_inval_pou(kvm_pte_follow(pte, mm_ops),  in stage2_attr_walker()
1326 	struct kvm_pgtable_mm_ops *mm_ops = pgt->mm_ops;  in stage2_flush_walker()  local
1331 	if (mm_ops->dcache_clean_inval_poc)  in stage2_flush_walker()
1332 		mm_ops->dcache_clean_inval_poc(kvm_pte_follow(ctx->old, mm_ops),  in stage2_flush_walker()
1379 	struct kvm_pgtable_mm_ops *mm_ops = pgt->mm_ops;  in kvm_pgtable_stage2_create_unlinked()  local
1390 	pgtable = mm_ops->zalloc_page(mc);  in kvm_pgtable_stage2_create_unlinked()
1394 	ret = __kvm_pgtable_walk(&data, mm_ops, (kvm_pteref_t)pgtable,  in kvm_pgtable_stage2_create_unlinked()
1397 		kvm_pgtable_stage2_free_unlinked(mm_ops, pgtable, level);  in kvm_pgtable_stage2_create_unlinked()
1398 		mm_ops->put_page(pgtable);  in kvm_pgtable_stage2_create_unlinked()
1429 	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;  in stage2_split_walker()  local
1480 		kvm_pgtable_stage2_free_unlinked(mm_ops, childp, level);  in stage2_split_walker()
1481 		mm_ops->put_page(childp);  in stage2_split_walker()
1490 	new = kvm_init_table_pte(childp, mm_ops);  in stage2_split_walker()
1509 			      struct kvm_pgtable_mm_ops *mm_ops,  in __kvm_pgtable_stage2_init()  argument
1520 	pgt->pgd = (kvm_pteref_t)mm_ops->zalloc_pages_exact(pgd_sz);  in __kvm_pgtable_stage2_init()
1526 	pgt->mm_ops		= mm_ops;  in __kvm_pgtable_stage2_init()
1548 	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;  in stage2_free_walker()  local
1553 	mm_ops->put_page(ctx->ptep);  in stage2_free_walker()
1556 		mm_ops->put_page(kvm_pte_follow(ctx->old, mm_ops));  in stage2_free_walker()
1572 	pgt->mm_ops->free_pages_exact(kvm_dereference_pteref(&walker, pgt->pgd), pgd_sz);  in kvm_pgtable_stage2_destroy()
1576 void kvm_pgtable_stage2_free_unlinked(struct kvm_pgtable_mm_ops *mm_ops, void *pgtable, u32 level)  in kvm_pgtable_stage2_free_unlinked()  argument
1596 	WARN_ON(__kvm_pgtable_walk(&data, mm_ops, ptep, level + 1));  in kvm_pgtable_stage2_free_unlinked()
1598 	WARN_ON(mm_ops->page_count(pgtable) != 1);  in kvm_pgtable_stage2_free_unlinked()
1599 	mm_ops->put_page(pgtable);  in kvm_pgtable_stage2_free_unlinked()