126 static inline unsigned p2m_top_index(unsigned long pfn)  in p2m_top_index()  argument
128 	BUG_ON(pfn >= MAX_P2M_PFN);  in p2m_top_index()
129 	return pfn / (P2M_MID_PER_PAGE * P2M_PER_PAGE);  in p2m_top_index()
132 static inline unsigned p2m_mid_index(unsigned long pfn)  in p2m_mid_index()  argument
134 	return (pfn / P2M_PER_PAGE) % P2M_MID_PER_PAGE;  in p2m_mid_index()
137 static inline unsigned p2m_index(unsigned long pfn)  in p2m_index()  argument
139 	return pfn % P2M_PER_PAGE;  in p2m_index()
174 static void p2m_init_identity(unsigned long *p2m, unsigned long pfn)  in p2m_init_identity()  argument
179 		p2m[i] = IDENTITY_FRAME(pfn + i);  in p2m_init_identity()
219 	unsigned long pfn, mfn;  in xen_build_mfn_list_list()  local
242 	for (pfn = 0; pfn < xen_max_p2m_pfn && pfn < MAX_P2M_PFN;  in xen_build_mfn_list_list()
243 	     pfn += P2M_PER_PAGE) {  in xen_build_mfn_list_list()
244 		topidx = p2m_top_index(pfn);  in xen_build_mfn_list_list()
245 		mididx = p2m_mid_index(pfn);  in xen_build_mfn_list_list()
248 		ptep = lookup_address((unsigned long)(xen_p2m_addr + pfn),  in xen_build_mfn_list_list()
262 			pfn += (P2M_MID_PER_PAGE - 1) * P2M_PER_PAGE;  in xen_build_mfn_list_list()
297 	unsigned long pfn;  in xen_build_dynamic_phys_to_machine()  local
302 	for (pfn = xen_start_info->nr_pages; pfn < xen_p2m_size; pfn++)  in xen_build_dynamic_phys_to_machine()
303 		xen_p2m_addr[pfn] = INVALID_P2M_ENTRY;  in xen_build_dynamic_phys_to_machine()
313 static int xen_p2m_elem_type(unsigned long pfn)  in xen_p2m_elem_type()  argument
317 	if (pfn >= xen_p2m_size)  in xen_p2m_elem_type()
320 	mfn = xen_p2m_addr[pfn];  in xen_p2m_elem_type()
334 	unsigned long pfn;  in xen_rebuild_p2m_list()  local
356 	for (pfn = 0; pfn < xen_max_p2m_pfn; pfn += chunk) {  in xen_rebuild_p2m_list()
367 		chunk = (pfn & (P2M_PER_PAGE * P2M_MID_PER_PAGE - 1)) ?  in xen_rebuild_p2m_list()
370 		type = xen_p2m_elem_type(pfn);  in xen_rebuild_p2m_list()
374 				if (xen_p2m_elem_type(pfn + i) != type)  in xen_rebuild_p2m_list()
383 			copy_page(mfns, xen_p2m_addr + pfn);  in xen_rebuild_p2m_list()
384 			ptep = populate_extra_pte((unsigned long)(p2m + pfn));  in xen_rebuild_p2m_list()
394 			ptep = populate_extra_pte((unsigned long)(p2m + pfn));  in xen_rebuild_p2m_list()
405 				(unsigned long)(p2m + pfn) + i * PMD_SIZE);  in xen_rebuild_p2m_list()
435 unsigned long get_phys_to_machine(unsigned long pfn)  in get_phys_to_machine()  argument
440 	if (unlikely(pfn >= xen_p2m_size)) {  in get_phys_to_machine()
441 		if (pfn < xen_max_p2m_pfn)  in get_phys_to_machine()
442 			return xen_chk_extra_mem(pfn);  in get_phys_to_machine()
444 		return IDENTITY_FRAME(pfn);  in get_phys_to_machine()
447 	ptep = lookup_address((unsigned long)(xen_p2m_addr + pfn), &level);  in get_phys_to_machine()
456 		return IDENTITY_FRAME(pfn);  in get_phys_to_machine()
458 	return xen_p2m_addr[pfn];  in get_phys_to_machine()
531 int xen_alloc_p2m_entry(unsigned long pfn)  in xen_alloc_p2m_entry()  argument
538 	unsigned long addr = (unsigned long)(xen_p2m_addr + pfn);  in xen_alloc_p2m_entry()
552 	if (p2m_top_mfn && pfn < MAX_P2M_PFN) {  in xen_alloc_p2m_entry()
553 		topidx = p2m_top_index(pfn);  in xen_alloc_p2m_entry()
598 			p2m_init_identity(p2m, pfn & ~(P2M_PER_PAGE - 1));  in xen_alloc_p2m_entry()
610 				mid_mfn[p2m_mid_index(pfn)] = virt_to_mfn(p2m);  in xen_alloc_p2m_entry()
621 	if (pfn >= xen_p2m_last_pfn) {  in xen_alloc_p2m_entry()
622 		xen_p2m_last_pfn = ALIGN(pfn + 1, P2M_PER_PAGE);  in xen_alloc_p2m_entry()
633 	unsigned long pfn;  in set_phys_range_identity()  local
644 	for (pfn = pfn_s; pfn < pfn_e; pfn++)  in set_phys_range_identity()
645 		xen_p2m_addr[pfn] = IDENTITY_FRAME(pfn);  in set_phys_range_identity()
647 	return pfn - pfn_s;  in set_phys_range_identity()
650 bool __set_phys_to_machine(unsigned long pfn, unsigned long mfn)  in __set_phys_to_machine()  argument
656 	if (unlikely(pfn >= xen_p2m_size))  in __set_phys_to_machine()
663 	if (likely(!xen_safe_write_ulong(xen_p2m_addr + pfn, mfn)))  in __set_phys_to_machine()
666 	ptep = lookup_address((unsigned long)(xen_p2m_addr + pfn), &level);  in __set_phys_to_machine()
673 		return mfn == IDENTITY_FRAME(pfn);  in __set_phys_to_machine()
678 bool set_phys_to_machine(unsigned long pfn, unsigned long mfn)  in set_phys_to_machine()  argument
680 	if (unlikely(!__set_phys_to_machine(pfn, mfn))) {  in set_phys_to_machine()
683 		ret = xen_alloc_p2m_entry(pfn);  in set_phys_to_machine()
687 		return __set_phys_to_machine(pfn, mfn);  in set_phys_to_machine()
711 		unsigned long mfn, pfn;  in set_foreign_p2m_mapping()  local
727 		pfn = page_to_pfn(pages[i]);  in set_foreign_p2m_mapping()
729 		WARN(pfn_to_mfn(pfn) != INVALID_P2M_ENTRY, "page must be ballooned");  in set_foreign_p2m_mapping()
731 		if (likely(set_phys_to_machine(pfn, FOREIGN_FRAME(mfn))))  in set_foreign_p2m_mapping()
788 		unsigned long pfn = page_to_pfn(pages[i]);  in clear_foreign_p2m_mapping()  local
791 			set_phys_to_machine(pfn, INVALID_P2M_ENTRY);  in clear_foreign_p2m_mapping()
812 	unsigned long pfn, first_pfn;  in p2m_dump_show()  local
818 	for (pfn = 0; pfn < xen_p2m_size; pfn++) {  in p2m_dump_show()
819 		type = xen_p2m_elem_type(pfn);  in p2m_dump_show()
821 			seq_printf(m, " [0x%lx->0x%lx] %s\n", first_pfn, pfn,  in p2m_dump_show()
824 			first_pfn = pfn;  in p2m_dump_show()
827 	seq_printf(m, " [0x%lx->0x%lx] %s\n", first_pfn, pfn,  in p2m_dump_show()