126 static inline unsigned p2m_top_index(unsigned long pfn)  in p2m_top_index()  argument
128 	BUG_ON(pfn >= MAX_P2M_PFN);  in p2m_top_index()
129 	return pfn / (P2M_MID_PER_PAGE * P2M_PER_PAGE);  in p2m_top_index()
132 static inline unsigned p2m_mid_index(unsigned long pfn)  in p2m_mid_index()  argument
134 	return (pfn / P2M_PER_PAGE) % P2M_MID_PER_PAGE;  in p2m_mid_index()
169 static void p2m_init_identity(unsigned long *p2m, unsigned long pfn)  in p2m_init_identity()  argument
174 		p2m[i] = IDENTITY_FRAME(pfn + i);  in p2m_init_identity()
214 	unsigned long pfn, mfn;  in xen_build_mfn_list_list()  local
237 	for (pfn = 0; pfn < xen_max_p2m_pfn && pfn < MAX_P2M_PFN;  in xen_build_mfn_list_list()
238 	     pfn += P2M_PER_PAGE) {  in xen_build_mfn_list_list()
239 		topidx = p2m_top_index(pfn);  in xen_build_mfn_list_list()
240 		mididx = p2m_mid_index(pfn);  in xen_build_mfn_list_list()
243 		ptep = lookup_address((unsigned long)(xen_p2m_addr + pfn),  in xen_build_mfn_list_list()
257 			pfn += (P2M_MID_PER_PAGE - 1) * P2M_PER_PAGE;  in xen_build_mfn_list_list()
292 	unsigned long pfn;  in xen_build_dynamic_phys_to_machine()  local
297 	for (pfn = xen_start_info->nr_pages; pfn < xen_p2m_size; pfn++)  in xen_build_dynamic_phys_to_machine()
298 		xen_p2m_addr[pfn] = INVALID_P2M_ENTRY;  in xen_build_dynamic_phys_to_machine()
308 static int xen_p2m_elem_type(unsigned long pfn)  in xen_p2m_elem_type()  argument
312 	if (pfn >= xen_p2m_size)  in xen_p2m_elem_type()
315 	mfn = xen_p2m_addr[pfn];  in xen_p2m_elem_type()
329 	unsigned long pfn;  in xen_rebuild_p2m_list()  local
351 	for (pfn = 0; pfn < xen_max_p2m_pfn; pfn += chunk) {  in xen_rebuild_p2m_list()
362 		chunk = (pfn & (P2M_PER_PAGE * P2M_MID_PER_PAGE - 1)) ?  in xen_rebuild_p2m_list()
365 		type = xen_p2m_elem_type(pfn);  in xen_rebuild_p2m_list()
369 				if (xen_p2m_elem_type(pfn + i) != type)  in xen_rebuild_p2m_list()
378 			copy_page(mfns, xen_p2m_addr + pfn);  in xen_rebuild_p2m_list()
379 			ptep = populate_extra_pte((unsigned long)(p2m + pfn));  in xen_rebuild_p2m_list()
389 			ptep = populate_extra_pte((unsigned long)(p2m + pfn));  in xen_rebuild_p2m_list()
400 				(unsigned long)(p2m + pfn) + i * PMD_SIZE);  in xen_rebuild_p2m_list()
430 unsigned long get_phys_to_machine(unsigned long pfn)  in get_phys_to_machine()  argument
435 	if (unlikely(pfn >= xen_p2m_size)) {  in get_phys_to_machine()
436 		if (pfn < xen_max_p2m_pfn)  in get_phys_to_machine()
437 			return xen_chk_extra_mem(pfn);  in get_phys_to_machine()
439 		return IDENTITY_FRAME(pfn);  in get_phys_to_machine()
442 	ptep = lookup_address((unsigned long)(xen_p2m_addr + pfn), &level);  in get_phys_to_machine()
451 		return IDENTITY_FRAME(pfn);  in get_phys_to_machine()
453 	return xen_p2m_addr[pfn];  in get_phys_to_machine()
526 int xen_alloc_p2m_entry(unsigned long pfn)  in xen_alloc_p2m_entry()  argument
533 	unsigned long addr = (unsigned long)(xen_p2m_addr + pfn);  in xen_alloc_p2m_entry()
547 	if (p2m_top_mfn && pfn < MAX_P2M_PFN) {  in xen_alloc_p2m_entry()
548 		topidx = p2m_top_index(pfn);  in xen_alloc_p2m_entry()
593 			p2m_init_identity(p2m, pfn & ~(P2M_PER_PAGE - 1));  in xen_alloc_p2m_entry()
605 				mid_mfn[p2m_mid_index(pfn)] = virt_to_mfn(p2m);  in xen_alloc_p2m_entry()
616 	if (pfn >= xen_p2m_last_pfn) {  in xen_alloc_p2m_entry()
617 		xen_p2m_last_pfn = ALIGN(pfn + 1, P2M_PER_PAGE);  in xen_alloc_p2m_entry()
628 	unsigned long pfn;  in set_phys_range_identity()  local
639 	for (pfn = pfn_s; pfn < pfn_e; pfn++)  in set_phys_range_identity()
640 		xen_p2m_addr[pfn] = IDENTITY_FRAME(pfn);  in set_phys_range_identity()
642 	return pfn - pfn_s;  in set_phys_range_identity()
645 bool __set_phys_to_machine(unsigned long pfn, unsigned long mfn)  in __set_phys_to_machine()  argument
651 	if (unlikely(pfn >= xen_p2m_size))  in __set_phys_to_machine()
658 	if (likely(!xen_safe_write_ulong(xen_p2m_addr + pfn, mfn)))  in __set_phys_to_machine()
661 	ptep = lookup_address((unsigned long)(xen_p2m_addr + pfn), &level);  in __set_phys_to_machine()
668 		return mfn == IDENTITY_FRAME(pfn);  in __set_phys_to_machine()
673 bool set_phys_to_machine(unsigned long pfn, unsigned long mfn)  in set_phys_to_machine()  argument
675 	if (unlikely(!__set_phys_to_machine(pfn, mfn))) {  in set_phys_to_machine()
678 		ret = xen_alloc_p2m_entry(pfn);  in set_phys_to_machine()
682 		return __set_phys_to_machine(pfn, mfn);  in set_phys_to_machine()
706 		unsigned long mfn, pfn;  in set_foreign_p2m_mapping()  local
722 		pfn = page_to_pfn(pages[i]);  in set_foreign_p2m_mapping()
724 		WARN(pfn_to_mfn(pfn) != INVALID_P2M_ENTRY, "page must be ballooned");  in set_foreign_p2m_mapping()
726 		if (likely(set_phys_to_machine(pfn, FOREIGN_FRAME(mfn))))  in set_foreign_p2m_mapping()
783 		unsigned long pfn = page_to_pfn(pages[i]);  in clear_foreign_p2m_mapping()  local
786 			set_phys_to_machine(pfn, INVALID_P2M_ENTRY);  in clear_foreign_p2m_mapping()
807 	unsigned long pfn, first_pfn;  in p2m_dump_show()  local
813 	for (pfn = 0; pfn < xen_p2m_size; pfn++) {  in p2m_dump_show()
814 		type = xen_p2m_elem_type(pfn);  in p2m_dump_show()
816 			seq_printf(m, " [0x%lx->0x%lx] %s\n", first_pfn, pfn,  in p2m_dump_show()
819 			first_pfn = pfn;  in p2m_dump_show()
822 	seq_printf(m, " [0x%lx->0x%lx] %s\n", first_pfn, pfn,  in p2m_dump_show()