90 static int slice_area_is_free(struct mm_struct *mm, unsigned long addr,  in slice_area_is_free()  argument
95 	if ((mm->context.slb_addr_limit - len) < addr)  in slice_area_is_free()
97 	vma = find_vma(mm, addr);  in slice_area_is_free()
101 static int slice_low_has_vma(struct mm_struct *mm, unsigned long slice)  in slice_low_has_vma()  argument
103 	return !slice_area_is_free(mm, slice << SLICE_LOW_SHIFT,  in slice_low_has_vma()
107 static int slice_high_has_vma(struct mm_struct *mm, unsigned long slice)  in slice_high_has_vma()  argument
120 	return !slice_area_is_free(mm, start, end - start);  in slice_high_has_vma()
123 static void slice_mask_for_free(struct mm_struct *mm, struct slice_mask *ret,  in slice_mask_for_free()  argument
133 		if (!slice_low_has_vma(mm, i))  in slice_mask_for_free()
140 		if (!slice_high_has_vma(mm, i))  in slice_mask_for_free()
145 static struct slice_mask *slice_mask_for_size(struct mm_struct *mm, int psize)  in slice_mask_for_size()  argument
149 		return &mm->context.mask_64k;  in slice_mask_for_size()
152 		return &mm->context.mask_4k;  in slice_mask_for_size()
155 		return &mm->context.mask_16m;  in slice_mask_for_size()
157 		return &mm->context.mask_16g;  in slice_mask_for_size()
162 static struct slice_mask *slice_mask_for_size(struct mm_struct *mm, int psize)  in slice_mask_for_size()  argument
165 		return &mm->context.mask_base_psize;  in slice_mask_for_size()
168 		return &mm->context.mask_512k;  in slice_mask_for_size()
170 		return &mm->context.mask_8m;  in slice_mask_for_size()
178 static bool slice_check_range_fits(struct mm_struct *mm,  in slice_check_range_fits()  argument
213 	struct mm_struct *mm = parm;  in slice_flush_segments()  local
216 	if (mm != current->active_mm)  in slice_flush_segments()
227 static void slice_convert(struct mm_struct *mm,  in slice_convert()  argument
237 	slice_dbg("slice_convert(mm=%p, psize=%d)\n", mm, psize);  in slice_convert()
240 	psize_mask = slice_mask_for_size(mm, psize);  in slice_convert()
247 	lpsizes = mm->context.low_slices_psize;  in slice_convert()
257 		old_mask = slice_mask_for_size(mm, old_psize);  in slice_convert()
266 	hpsizes = mm->context.high_slices_psize;  in slice_convert()
267 	for (i = 0; i < GET_HIGH_SLICE_INDEX(mm->context.slb_addr_limit); i++) {  in slice_convert()
276 		old_mask = slice_mask_for_size(mm, old_psize);  in slice_convert()
286 		  (unsigned long)mm->context.low_slices_psize,  in slice_convert()
287 		  (unsigned long)mm->context.high_slices_psize);  in slice_convert()
291 	copro_flush_all_slbs(mm);  in slice_convert()
318 static unsigned long slice_find_area_bottomup(struct mm_struct *mm,  in slice_find_area_bottomup()  argument
364 static unsigned long slice_find_area_topdown(struct mm_struct *mm,  in slice_find_area_topdown()  argument
378 	addr = mm->mmap_base;  in slice_find_area_topdown()
386 		addr += mm->context.slb_addr_limit - DEFAULT_MAP_WINDOW;  in slice_find_area_topdown()
419 	return slice_find_area_bottomup(mm, len, available, psize, high_limit);  in slice_find_area_topdown()
423 static unsigned long slice_find_area(struct mm_struct *mm, unsigned long len,  in slice_find_area()  argument
428 		return slice_find_area_topdown(mm, len, mask, psize, high_limit);  in slice_find_area()
430 		return slice_find_area_bottomup(mm, len, mask, psize, high_limit);  in slice_find_area()
479 	struct mm_struct *mm = current->mm;  in slice_get_unmapped_area()  local
498 	if (high_limit > mm->context.slb_addr_limit) {  in slice_get_unmapped_area()
504 		mm->context.slb_addr_limit = high_limit;  in slice_get_unmapped_area()
506 		on_each_cpu(slice_flush_segments, mm, 1);  in slice_get_unmapped_area()
510 	BUG_ON(mm->task_size == 0);  in slice_get_unmapped_area()
511 	BUG_ON(mm->context.slb_addr_limit == 0);  in slice_get_unmapped_area()
514 	slice_dbg("slice_get_unmapped_area(mm=%p, psize=%d...\n", mm, psize);  in slice_get_unmapped_area()
524 		    !slice_area_is_free(mm, addr, len))  in slice_get_unmapped_area()
531 	maskp = slice_mask_for_size(mm, psize);  in slice_get_unmapped_area()
558 		compat_maskp = slice_mask_for_size(mm, MMU_PAGE_4K);  in slice_get_unmapped_area()
576 		if (slice_check_range_fits(mm, &good_mask, addr, len)) {  in slice_get_unmapped_area()
585 		newaddr = slice_find_area(mm, len, &good_mask,  in slice_get_unmapped_area()
599 	slice_mask_for_free(mm, &potential_mask, high_limit);  in slice_get_unmapped_area()
604 		if (slice_check_range_fits(mm, &potential_mask, addr, len)) {  in slice_get_unmapped_area()
621 		newaddr = slice_find_area(mm, len, &good_mask,  in slice_get_unmapped_area()
632 	newaddr = slice_find_area(mm, len, &potential_mask,  in slice_get_unmapped_area()
639 		newaddr = slice_find_area(mm, len, &potential_mask,  in slice_get_unmapped_area()
656 	if (need_extra_context(mm, newaddr)) {  in slice_get_unmapped_area()
657 		if (alloc_extended_context(mm, newaddr) < 0)  in slice_get_unmapped_area()
667 		slice_convert(mm, &potential_mask, psize);  in slice_get_unmapped_area()
669 			on_each_cpu(slice_flush_segments, mm, 1);  in slice_get_unmapped_area()
674 	if (need_extra_context(mm, newaddr)) {  in slice_get_unmapped_area()
675 		if (alloc_extended_context(mm, newaddr) < 0)  in slice_get_unmapped_area()
689 				       current->mm->context.user_psize, 0);  in arch_get_unmapped_area()
699 				       current->mm->context.user_psize, 1);  in arch_get_unmapped_area_topdown()
702 unsigned int get_slice_psize(struct mm_struct *mm, unsigned long addr)  in get_slice_psize()  argument
710 		psizes = mm->context.low_slices_psize;  in get_slice_psize()
713 		psizes = mm->context.high_slices_psize;  in get_slice_psize()
721 void slice_init_new_context_exec(struct mm_struct *mm)  in slice_init_new_context_exec()  argument
727 	slice_dbg("slice_init_new_context_exec(mm=%p)\n", mm);  in slice_init_new_context_exec()
735 	mm->context.slb_addr_limit = DEFAULT_MAP_WINDOW_USER64;  in slice_init_new_context_exec()
737 	mm->context.slb_addr_limit = DEFAULT_MAP_WINDOW;  in slice_init_new_context_exec()
740 	mm->context.user_psize = psize;  in slice_init_new_context_exec()
745 	lpsizes = mm->context.low_slices_psize;  in slice_init_new_context_exec()
748 	hpsizes = mm->context.high_slices_psize;  in slice_init_new_context_exec()
754 	mask = slice_mask_for_size(mm, psize);  in slice_init_new_context_exec()
760 void slice_set_range_psize(struct mm_struct *mm, unsigned long start,  in slice_set_range_psize()  argument
768 	slice_convert(mm, &mask, psize);  in slice_set_range_psize()
791 int slice_is_hugepage_only_range(struct mm_struct *mm, unsigned long addr,  in slice_is_hugepage_only_range()  argument
795 	unsigned int psize = mm->context.user_psize;  in slice_is_hugepage_only_range()
799 	maskp = slice_mask_for_size(mm, psize);  in slice_is_hugepage_only_range()
806 		compat_maskp = slice_mask_for_size(mm, MMU_PAGE_4K);  in slice_is_hugepage_only_range()
808 		return !slice_check_range_fits(mm, &available, addr, len);  in slice_is_hugepage_only_range()
812 	return !slice_check_range_fits(mm, maskp, addr, len);  in slice_is_hugepage_only_range()