305 			    unsigned long init, unsigned long gaddr)  in gmap_alloc_table()  argument
321 		page->index = gaddr;  in gmap_alloc_table()
378 static int __gmap_unmap_by_gaddr(struct gmap *gmap, unsigned long gaddr)  in __gmap_unmap_by_gaddr()  argument
383 						   gaddr >> PMD_SHIFT);  in __gmap_unmap_by_gaddr()
473 unsigned long __gmap_translate(struct gmap *gmap, unsigned long gaddr)  in __gmap_translate()  argument
478 		radix_tree_lookup(&gmap->guest_to_host, gaddr >> PMD_SHIFT);  in __gmap_translate()
480 	return vmaddr ? (vmaddr | (gaddr & ~PMD_MASK)) : -EFAULT;  in __gmap_translate()
493 unsigned long gmap_translate(struct gmap *gmap, unsigned long gaddr)  in gmap_translate()  argument
498 	rc = __gmap_translate(gmap, gaddr);  in gmap_translate()
526 			   unsigned long gaddr);
539 int __gmap_link(struct gmap *gmap, unsigned long gaddr, unsigned long vmaddr)  in __gmap_link()  argument
555 		table += (gaddr & _REGION1_INDEX) >> _REGION1_SHIFT;  in __gmap_link()
558 				     gaddr & _REGION1_MASK))  in __gmap_link()
563 		table += (gaddr & _REGION2_INDEX) >> _REGION2_SHIFT;  in __gmap_link()
566 				     gaddr & _REGION2_MASK))  in __gmap_link()
571 		table += (gaddr & _REGION3_INDEX) >> _REGION3_SHIFT;  in __gmap_link()
574 				     gaddr & _REGION3_MASK))  in __gmap_link()
578 	table += (gaddr & _SEGMENT_INDEX) >> _SEGMENT_SHIFT;  in __gmap_link()
618 		gmap_pmdp_xchg(gmap, (pmd_t *)table, __pmd(unprot), gaddr);  in __gmap_link()
635 int gmap_fault(struct gmap *gmap, unsigned long gaddr,  in gmap_fault()  argument
646 	vmaddr = __gmap_translate(gmap, gaddr);  in gmap_fault()
663 	rc = __gmap_link(gmap, gaddr, vmaddr);  in gmap_fault()
673 void __gmap_zap(struct gmap *gmap, unsigned long gaddr)  in __gmap_zap()  argument
682 						   gaddr >> PMD_SHIFT);  in __gmap_zap()
684 		vmaddr |= gaddr & ~PMD_MASK;  in __gmap_zap()
702 	unsigned long gaddr, vmaddr, size;  in gmap_discard()  local
706 	for (gaddr = from; gaddr < to;  in gmap_discard()
707 	     gaddr = (gaddr + PMD_SIZE) & PMD_MASK) {  in gmap_discard()
711 					  gaddr >> PMD_SHIFT);  in gmap_discard()
714 		vmaddr |= gaddr & ~PMD_MASK;  in gmap_discard()
725 		size = min(to - gaddr, PMD_SIZE - (gaddr & ~PMD_MASK));  in gmap_discard()
794 					     unsigned long gaddr, int level)  in gmap_table_walk()  argument
806 	    gaddr & (-1UL << (31 + (asce_type >> 2) * 11)))  in gmap_table_walk()
811 		table += (gaddr & _REGION1_INDEX) >> _REGION1_SHIFT;  in gmap_table_walk()
819 		table += (gaddr & _REGION2_INDEX) >> _REGION2_SHIFT;  in gmap_table_walk()
827 		table += (gaddr & _REGION3_INDEX) >> _REGION3_SHIFT;  in gmap_table_walk()
835 		table += (gaddr & _SEGMENT_INDEX) >> _SEGMENT_SHIFT;  in gmap_table_walk()
841 		table += (gaddr & _PAGE_INDEX) >> _PAGE_SHIFT;  in gmap_table_walk()
855 static pte_t *gmap_pte_op_walk(struct gmap *gmap, unsigned long gaddr,  in gmap_pte_op_walk()  argument
862 	table = gmap_table_walk(gmap, gaddr, 1); /* get segment pointer */  in gmap_pte_op_walk()
865 	return pte_alloc_map_lock(gmap->mm, (pmd_t *) table, gaddr, ptl);  in gmap_pte_op_walk()
879 static int gmap_pte_op_fixup(struct gmap *gmap, unsigned long gaddr,  in gmap_pte_op_fixup()  argument
894 	return __gmap_link(gmap, gaddr, vmaddr);  in gmap_pte_op_fixup()
915 static inline pmd_t *gmap_pmd_op_walk(struct gmap *gmap, unsigned long gaddr)  in gmap_pmd_op_walk()  argument
920 	pmdp = (pmd_t *) gmap_table_walk(gmap, gaddr, 1);  in gmap_pmd_op_walk()
965 static int gmap_protect_pmd(struct gmap *gmap, unsigned long gaddr,  in gmap_protect_pmd()  argument
978 		gmap_pmdp_xchg(gmap, pmdp, new, gaddr);  in gmap_protect_pmd()
984 		gmap_pmdp_xchg(gmap, pmdp, new, gaddr);  in gmap_protect_pmd()
1010 static int gmap_protect_pte(struct gmap *gmap, unsigned long gaddr,  in gmap_protect_pte()  argument
1021 	ptep = pte_alloc_map_lock(gmap->mm, pmdp, gaddr, &ptl);  in gmap_protect_pte()
1028 	rc = ptep_force_prot(gmap->mm, gaddr, ptep, prot, pbits);  in gmap_protect_pte()
1046 static int gmap_protect_range(struct gmap *gmap, unsigned long gaddr,  in gmap_protect_range()  argument
1056 		pmdp = gmap_pmd_op_walk(gmap, gaddr);  in gmap_protect_range()
1059 				rc = gmap_protect_pte(gmap, gaddr, pmdp, prot,  in gmap_protect_range()
1063 					gaddr += PAGE_SIZE;  in gmap_protect_range()
1066 				rc = gmap_protect_pmd(gmap, gaddr, pmdp, prot,  in gmap_protect_range()
1069 					dist = HPAGE_SIZE - (gaddr & ~HPAGE_MASK);  in gmap_protect_range()
1071 					gaddr = (gaddr & HPAGE_MASK) + HPAGE_SIZE;  in gmap_protect_range()
1081 			vmaddr = __gmap_translate(gmap, gaddr);  in gmap_protect_range()
1084 			rc = gmap_pte_op_fixup(gmap, gaddr, vmaddr, prot);  in gmap_protect_range()
1106 int gmap_mprotect_notify(struct gmap *gmap, unsigned long gaddr,  in gmap_mprotect_notify()  argument
1111 	if ((gaddr & ~PAGE_MASK) || (len & ~PAGE_MASK) || gmap_is_shadow(gmap))  in gmap_mprotect_notify()
1116 	rc = gmap_protect_range(gmap, gaddr, len, prot, GMAP_NOTIFY_MPROT);  in gmap_mprotect_notify()
1135 int gmap_read_table(struct gmap *gmap, unsigned long gaddr, unsigned long *val)  in gmap_read_table()  argument
1147 		ptep = gmap_pte_op_walk(gmap, gaddr, &ptl);  in gmap_read_table()
1152 				address += gaddr & ~PAGE_MASK;  in gmap_read_table()
1162 		vmaddr = __gmap_translate(gmap, gaddr);  in gmap_read_table()
1167 		rc = gmap_pte_op_fixup(gmap, gaddr, vmaddr, PROT_READ);  in gmap_read_table()
2187 			       unsigned long gaddr)  in gmap_shadow_notify()  argument
2202 	if (!(sg->orig_asce & _ASCE_REAL_SPACE) && gaddr >= start &&  in gmap_shadow_notify()
2203 	    gaddr < end) {  in gmap_shadow_notify()
2251 	unsigned long offset, gaddr = 0;  in ptep_notify()  local
2263 			gaddr = __gmap_segment_gaddr(table) + offset;  in ptep_notify()
2272 				gmap_shadow_notify(sg, vmaddr, gaddr);  in ptep_notify()
2276 			gmap_call_notifier(gmap, gaddr, gaddr + PAGE_SIZE - 1);  in ptep_notify()
2283 			     unsigned long gaddr)  in pmdp_notify_gmap()  argument
2286 	gmap_call_notifier(gmap, gaddr, gaddr + HPAGE_SIZE - 1);  in pmdp_notify_gmap()
2300 			   unsigned long gaddr)  in gmap_pmdp_xchg()  argument
2302 	gaddr &= HPAGE_MASK;  in gmap_pmdp_xchg()
2303 	pmdp_notify_gmap(gmap, pmdp, gaddr);  in gmap_pmdp_xchg()
2306 		__pmdp_idte(gaddr, (pmd_t *)pmdp, IDTE_GUEST_ASCE, gmap->asce,  in gmap_pmdp_xchg()
2309 		__pmdp_idte(gaddr, (pmd_t *)pmdp, 0, 0, IDTE_GLOBAL);  in gmap_pmdp_xchg()
2320 	unsigned long gaddr;  in gmap_pmdp_clear()  local
2328 			gaddr = __gmap_segment_gaddr((unsigned long *)pmdp);  in gmap_pmdp_clear()
2329 			pmdp_notify_gmap(gmap, pmdp, gaddr);  in gmap_pmdp_clear()
2371 	unsigned long *entry, gaddr;  in gmap_pmdp_idte_local()  local
2382 			gaddr = __gmap_segment_gaddr(entry);  in gmap_pmdp_idte_local()
2383 			pmdp_notify_gmap(gmap, pmdp, gaddr);  in gmap_pmdp_idte_local()
2387 				__pmdp_idte(gaddr, pmdp, IDTE_GUEST_ASCE,  in gmap_pmdp_idte_local()
2390 				__pmdp_idte(gaddr, pmdp, 0, 0, IDTE_LOCAL);  in gmap_pmdp_idte_local()
2406 	unsigned long *entry, gaddr;  in gmap_pmdp_idte_global()  local
2417 			gaddr = __gmap_segment_gaddr(entry);  in gmap_pmdp_idte_global()
2418 			pmdp_notify_gmap(gmap, pmdp, gaddr);  in gmap_pmdp_idte_global()
2422 				__pmdp_idte(gaddr, pmdp, IDTE_GUEST_ASCE,  in gmap_pmdp_idte_global()
2425 				__pmdp_idte(gaddr, pmdp, 0, 0, IDTE_GLOBAL);  in gmap_pmdp_idte_global()
2446 					  unsigned long gaddr)  in gmap_test_and_clear_dirty_pmd()  argument
2458 	gmap_protect_pmd(gmap, gaddr, pmdp, PROT_READ, 0);  in gmap_test_and_clear_dirty_pmd()
2473 			     unsigned long gaddr, unsigned long vmaddr)  in gmap_sync_dirty_log_pmd()  argument
2480 	pmdp = gmap_pmd_op_walk(gmap, gaddr);  in gmap_sync_dirty_log_pmd()
2485 		if (gmap_test_and_clear_dirty_pmd(gmap, pmdp, gaddr))  in gmap_sync_dirty_log_pmd()