63 static int ioremap_pte_range(pmd_t *pmd, unsigned long addr,  in ioremap_pte_range()  argument
70 	pte = pte_alloc_kernel(pmd, addr);  in ioremap_pte_range()
75 		set_pte_at(&init_mm, addr, pte, pfn_pte(pfn, prot));  in ioremap_pte_range()
77 	} while (pte++, addr += PAGE_SIZE, addr != end);  in ioremap_pte_range()
81 static int ioremap_try_huge_pmd(pmd_t *pmd, unsigned long addr,  in ioremap_try_huge_pmd()  argument
88 	if ((end - addr) != PMD_SIZE)  in ioremap_try_huge_pmd()
91 	if (!IS_ALIGNED(addr, PMD_SIZE))  in ioremap_try_huge_pmd()
97 	if (pmd_present(*pmd) && !pmd_free_pte_page(pmd, addr))  in ioremap_try_huge_pmd()
103 static inline int ioremap_pmd_range(pud_t *pud, unsigned long addr,  in ioremap_pmd_range()  argument
109 	pmd = pmd_alloc(&init_mm, pud, addr);  in ioremap_pmd_range()
113 		next = pmd_addr_end(addr, end);  in ioremap_pmd_range()
115 		if (ioremap_try_huge_pmd(pmd, addr, next, phys_addr, prot))  in ioremap_pmd_range()
118 		if (ioremap_pte_range(pmd, addr, next, phys_addr, prot))  in ioremap_pmd_range()
120 	} while (pmd++, phys_addr += (next - addr), addr = next, addr != end);  in ioremap_pmd_range()
124 static int ioremap_try_huge_pud(pud_t *pud, unsigned long addr,  in ioremap_try_huge_pud()  argument
131 	if ((end - addr) != PUD_SIZE)  in ioremap_try_huge_pud()
134 	if (!IS_ALIGNED(addr, PUD_SIZE))  in ioremap_try_huge_pud()
140 	if (pud_present(*pud) && !pud_free_pmd_page(pud, addr))  in ioremap_try_huge_pud()
146 static inline int ioremap_pud_range(p4d_t *p4d, unsigned long addr,  in ioremap_pud_range()  argument
152 	pud = pud_alloc(&init_mm, p4d, addr);  in ioremap_pud_range()
156 		next = pud_addr_end(addr, end);  in ioremap_pud_range()
158 		if (ioremap_try_huge_pud(pud, addr, next, phys_addr, prot))  in ioremap_pud_range()
161 		if (ioremap_pmd_range(pud, addr, next, phys_addr, prot))  in ioremap_pud_range()
163 	} while (pud++, phys_addr += (next - addr), addr = next, addr != end);  in ioremap_pud_range()
167 static int ioremap_try_huge_p4d(p4d_t *p4d, unsigned long addr,  in ioremap_try_huge_p4d()  argument
174 	if ((end - addr) != P4D_SIZE)  in ioremap_try_huge_p4d()
177 	if (!IS_ALIGNED(addr, P4D_SIZE))  in ioremap_try_huge_p4d()
183 	if (p4d_present(*p4d) && !p4d_free_pud_page(p4d, addr))  in ioremap_try_huge_p4d()
189 static inline int ioremap_p4d_range(pgd_t *pgd, unsigned long addr,  in ioremap_p4d_range()  argument
195 	p4d = p4d_alloc(&init_mm, pgd, addr);  in ioremap_p4d_range()
199 		next = p4d_addr_end(addr, end);  in ioremap_p4d_range()
201 		if (ioremap_try_huge_p4d(p4d, addr, next, phys_addr, prot))  in ioremap_p4d_range()
204 		if (ioremap_pud_range(p4d, addr, next, phys_addr, prot))  in ioremap_p4d_range()
206 	} while (p4d++, phys_addr += (next - addr), addr = next, addr != end);  in ioremap_p4d_range()
210 int ioremap_page_range(unsigned long addr,  in ioremap_page_range()  argument
219 	BUG_ON(addr >= end);  in ioremap_page_range()
221 	start = addr;  in ioremap_page_range()
222 	pgd = pgd_offset_k(addr);  in ioremap_page_range()
224 		next = pgd_addr_end(addr, end);  in ioremap_page_range()
225 		err = ioremap_p4d_range(pgd, addr, next, phys_addr, prot);  in ioremap_page_range()
228 	} while (pgd++, phys_addr += (next - addr), addr = next, addr != end);  in ioremap_page_range()