27 							unsigned long addr,  in hugetlb_get_unmapped_area_bottomup()  argument
45 	addr = vm_unmapped_area(&info);  in hugetlb_get_unmapped_area_bottomup()
47 	if ((addr & ~PAGE_MASK) && task_size > VA_EXCLUDE_END) {  in hugetlb_get_unmapped_area_bottomup()
48 		VM_BUG_ON(addr != -ENOMEM);  in hugetlb_get_unmapped_area_bottomup()
51 		addr = vm_unmapped_area(&info);  in hugetlb_get_unmapped_area_bottomup()
54 	return addr;  in hugetlb_get_unmapped_area_bottomup()
65 	unsigned long addr = addr0;  in hugetlb_get_unmapped_area_topdown()  local
77 	addr = vm_unmapped_area(&info);  in hugetlb_get_unmapped_area_topdown()
85 	if (addr & ~PAGE_MASK) {  in hugetlb_get_unmapped_area_topdown()
86 		VM_BUG_ON(addr != -ENOMEM);  in hugetlb_get_unmapped_area_topdown()
90 		addr = vm_unmapped_area(&info);  in hugetlb_get_unmapped_area_topdown()
93 	return addr;  in hugetlb_get_unmapped_area_topdown()
97 hugetlb_get_unmapped_area(struct file *file, unsigned long addr,  in hugetlb_get_unmapped_area()  argument
114 		if (prepare_hugepage_range(file, addr, len))  in hugetlb_get_unmapped_area()
116 		return addr;  in hugetlb_get_unmapped_area()
119 	if (addr) {  in hugetlb_get_unmapped_area()
120 		addr = ALIGN(addr, huge_page_size(h));  in hugetlb_get_unmapped_area()
121 		vma = find_vma(mm, addr);  in hugetlb_get_unmapped_area()
122 		if (task_size - len >= addr &&  in hugetlb_get_unmapped_area()
123 		    (!vma || addr + len <= vm_start_gap(vma)))  in hugetlb_get_unmapped_area()
124 			return addr;  in hugetlb_get_unmapped_area()
127 		return hugetlb_get_unmapped_area_bottomup(file, addr, len,  in hugetlb_get_unmapped_area()
130 		return hugetlb_get_unmapped_area_topdown(file, addr, len,  in hugetlb_get_unmapped_area()
276 			unsigned long addr, unsigned long sz)  in huge_pte_alloc()  argument
283 	pgd = pgd_offset(mm, addr);  in huge_pte_alloc()
284 	p4d = p4d_offset(pgd, addr);  in huge_pte_alloc()
285 	pud = pud_alloc(mm, p4d, addr);  in huge_pte_alloc()
290 	pmd = pmd_alloc(mm, pud, addr);  in huge_pte_alloc()
295 	return pte_alloc_map(mm, pmd, addr);  in huge_pte_alloc()
299 		       unsigned long addr, unsigned long sz)  in huge_pte_offset()  argument
306 	pgd = pgd_offset(mm, addr);  in huge_pte_offset()
309 	p4d = p4d_offset(pgd, addr);  in huge_pte_offset()
312 	pud = pud_offset(p4d, addr);  in huge_pte_offset()
317 	pmd = pmd_offset(pud, addr);  in huge_pte_offset()
322 	return pte_offset_map(pmd, addr);  in huge_pte_offset()
325 void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,  in set_huge_pte_at()  argument
347 	addr &= ~(size - 1);  in set_huge_pte_at()
354 	maybe_tlb_batch_add(mm, addr, ptep, orig, 0, orig_shift);  in set_huge_pte_at()
357 		maybe_tlb_batch_add(mm, addr + REAL_HPAGE_SIZE, ptep, orig, 0,  in set_huge_pte_at()
361 pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,  in huge_ptep_get_and_clear()  argument
385 	addr &= ~(size - 1);  in huge_ptep_get_and_clear()
389 	maybe_tlb_batch_add(mm, addr, ptep, entry, 0, orig_shift);  in huge_ptep_get_and_clear()
392 		maybe_tlb_batch_add(mm, addr + REAL_HPAGE_SIZE, ptep, entry, 0,  in huge_ptep_get_and_clear()
411 			   unsigned long addr)  in hugetlb_free_pte_range()  argument
416 	pte_free_tlb(tlb, token, addr);  in hugetlb_free_pte_range()
421 				   unsigned long addr, unsigned long end,  in hugetlb_free_pmd_range()  argument
428 	start = addr;  in hugetlb_free_pmd_range()
429 	pmd = pmd_offset(pud, addr);  in hugetlb_free_pmd_range()
431 		next = pmd_addr_end(addr, end);  in hugetlb_free_pmd_range()
437 			hugetlb_free_pte_range(tlb, pmd, addr);  in hugetlb_free_pmd_range()
438 	} while (pmd++, addr = next, addr != end);  in hugetlb_free_pmd_range()
458 				   unsigned long addr, unsigned long end,  in hugetlb_free_pud_range()  argument
465 	start = addr;  in hugetlb_free_pud_range()
466 	pud = pud_offset(p4d, addr);  in hugetlb_free_pud_range()
468 		next = pud_addr_end(addr, end);  in hugetlb_free_pud_range()
474 			hugetlb_free_pmd_range(tlb, pud, addr, next, floor,  in hugetlb_free_pud_range()
476 	} while (pud++, addr = next, addr != end);  in hugetlb_free_pud_range()
496 			    unsigned long addr, unsigned long end,  in hugetlb_free_pgd_range()  argument
503 	addr &= PMD_MASK;  in hugetlb_free_pgd_range()
504 	if (addr < floor) {  in hugetlb_free_pgd_range()
505 		addr += PMD_SIZE;  in hugetlb_free_pgd_range()
506 		if (!addr)  in hugetlb_free_pgd_range()
516 	if (addr > end - 1)  in hugetlb_free_pgd_range()
519 	pgd = pgd_offset(tlb->mm, addr);  in hugetlb_free_pgd_range()
520 	p4d = p4d_offset(pgd, addr);  in hugetlb_free_pgd_range()
522 		next = p4d_addr_end(addr, end);  in hugetlb_free_pgd_range()
525 		hugetlb_free_pud_range(tlb, p4d, addr, next, floor, ceiling);  in hugetlb_free_pgd_range()
526 	} while (p4d++, addr = next, addr != end);  in hugetlb_free_pgd_range()