86 static int slice_area_is_free(struct mm_struct *mm, unsigned long addr,  in slice_area_is_free()  argument
91 	if ((mm_ctx_slb_addr_limit(&mm->context) - len) < addr)  in slice_area_is_free()
93 	vma = find_vma(mm, addr);  in slice_area_is_free()
97 static int slice_low_has_vma(struct mm_struct *mm, unsigned long slice)  in slice_low_has_vma()  argument
99 	return !slice_area_is_free(mm, slice << SLICE_LOW_SHIFT,  in slice_low_has_vma()
103 static int slice_high_has_vma(struct mm_struct *mm, unsigned long slice)  in slice_high_has_vma()  argument
114 	return !slice_area_is_free(mm, start, end - start);  in slice_high_has_vma()
117 static void slice_mask_for_free(struct mm_struct *mm, struct slice_mask *ret,  in slice_mask_for_free()  argument
127 		if (!slice_low_has_vma(mm, i))  in slice_mask_for_free()
134 		if (!slice_high_has_vma(mm, i))  in slice_mask_for_free()
138 static bool slice_check_range_fits(struct mm_struct *mm,  in slice_check_range_fits()  argument
173 	struct mm_struct *mm = parm;  in slice_flush_segments()  local
176 	if (mm != current->active_mm)  in slice_flush_segments()
187 static void slice_convert(struct mm_struct *mm,  in slice_convert()  argument
197 	slice_dbg("slice_convert(mm=%p, psize=%d)\n", mm, psize);  in slice_convert()
200 	psize_mask = slice_mask_for_size(&mm->context, psize);  in slice_convert()
207 	lpsizes = mm_ctx_low_slices(&mm->context);  in slice_convert()
217 		old_mask = slice_mask_for_size(&mm->context, old_psize);  in slice_convert()
226 	hpsizes = mm_ctx_high_slices(&mm->context);  in slice_convert()
227 	for (i = 0; i < GET_HIGH_SLICE_INDEX(mm_ctx_slb_addr_limit(&mm->context)); i++) {  in slice_convert()
236 		old_mask = slice_mask_for_size(&mm->context, old_psize);  in slice_convert()
246 		  (unsigned long)mm_ctx_low_slices(&mm->context),  in slice_convert()
247 		  (unsigned long)mm_ctx_high_slices(&mm->context));  in slice_convert()
251 	copro_flush_all_slbs(mm);  in slice_convert()
278 static unsigned long slice_find_area_bottomup(struct mm_struct *mm,  in slice_find_area_bottomup()  argument
322 static unsigned long slice_find_area_topdown(struct mm_struct *mm,  in slice_find_area_topdown()  argument
343 		addr += mm_ctx_slb_addr_limit(&mm->context) - DEFAULT_MAP_WINDOW;  in slice_find_area_topdown()
376 	return slice_find_area_bottomup(mm, TASK_UNMAPPED_BASE, len, available, psize, high_limit);  in slice_find_area_topdown()
380 static unsigned long slice_find_area(struct mm_struct *mm, unsigned long len,  in slice_find_area()  argument
385 		return slice_find_area_topdown(mm, mm->mmap_base, len, mask, psize, high_limit);  in slice_find_area()
387 		return slice_find_area_bottomup(mm, mm->mmap_base, len, mask, psize, high_limit);  in slice_find_area()
436 	struct mm_struct *mm = current->mm;  in slice_get_unmapped_area()  local
455 	if (high_limit > mm_ctx_slb_addr_limit(&mm->context)) {  in slice_get_unmapped_area()
461 		mm_ctx_set_slb_addr_limit(&mm->context, high_limit);  in slice_get_unmapped_area()
463 		on_each_cpu(slice_flush_segments, mm, 1);  in slice_get_unmapped_area()
467 	BUG_ON(mm->task_size == 0);  in slice_get_unmapped_area()
468 	BUG_ON(mm_ctx_slb_addr_limit(&mm->context) == 0);  in slice_get_unmapped_area()
471 	slice_dbg("slice_get_unmapped_area(mm=%p, psize=%d...\n", mm, psize);  in slice_get_unmapped_area()
481 		    !slice_area_is_free(mm, addr, len))  in slice_get_unmapped_area()
488 	maskp = slice_mask_for_size(&mm->context, psize);  in slice_get_unmapped_area()
515 		compat_maskp = slice_mask_for_size(&mm->context, MMU_PAGE_4K);  in slice_get_unmapped_area()
533 		if (slice_check_range_fits(mm, &good_mask, addr, len)) {  in slice_get_unmapped_area()
542 		newaddr = slice_find_area(mm, len, &good_mask,  in slice_get_unmapped_area()
556 	slice_mask_for_free(mm, &potential_mask, high_limit);  in slice_get_unmapped_area()
561 		if (slice_check_range_fits(mm, &potential_mask, addr, len)) {  in slice_get_unmapped_area()
578 		newaddr = slice_find_area(mm, len, &good_mask,  in slice_get_unmapped_area()
589 	newaddr = slice_find_area(mm, len, &potential_mask,  in slice_get_unmapped_area()
596 		newaddr = slice_find_area(mm, len, &potential_mask,  in slice_get_unmapped_area()
612 	if (need_extra_context(mm, newaddr)) {  in slice_get_unmapped_area()
613 		if (alloc_extended_context(mm, newaddr) < 0)  in slice_get_unmapped_area()
623 		slice_convert(mm, &potential_mask, psize);  in slice_get_unmapped_area()
625 			on_each_cpu(slice_flush_segments, mm, 1);  in slice_get_unmapped_area()
630 	if (need_extra_context(mm, newaddr)) {  in slice_get_unmapped_area()
631 		if (alloc_extended_context(mm, newaddr) < 0)  in slice_get_unmapped_area()
648 				       mm_ctx_user_psize(&current->mm->context), 0);  in arch_get_unmapped_area()
661 				       mm_ctx_user_psize(&current->mm->context), 1);  in arch_get_unmapped_area_topdown()
664 unsigned int notrace get_slice_psize(struct mm_struct *mm, unsigned long addr)  in get_slice_psize()  argument
672 		psizes = mm_ctx_low_slices(&mm->context);  in get_slice_psize()
675 		psizes = mm_ctx_high_slices(&mm->context);  in get_slice_psize()
683 void slice_init_new_context_exec(struct mm_struct *mm)  in slice_init_new_context_exec()  argument
689 	slice_dbg("slice_init_new_context_exec(mm=%p)\n", mm);  in slice_init_new_context_exec()
696 	mm_ctx_set_slb_addr_limit(&mm->context, SLB_ADDR_LIMIT_DEFAULT);  in slice_init_new_context_exec()
697 	mm_ctx_set_user_psize(&mm->context, psize);  in slice_init_new_context_exec()
702 	lpsizes = mm_ctx_low_slices(&mm->context);  in slice_init_new_context_exec()
705 	hpsizes = mm_ctx_high_slices(&mm->context);  in slice_init_new_context_exec()
711 	mask = slice_mask_for_size(&mm->context, psize);  in slice_init_new_context_exec()
719 	struct mm_struct *mm = current->mm;  in slice_setup_new_exec()  local
721 	slice_dbg("slice_setup_new_exec(mm=%p)\n", mm);  in slice_setup_new_exec()
726 	mm_ctx_set_slb_addr_limit(&mm->context, DEFAULT_MAP_WINDOW);  in slice_setup_new_exec()
729 void slice_set_range_psize(struct mm_struct *mm, unsigned long start,  in slice_set_range_psize()  argument
737 	slice_convert(mm, &mask, psize);  in slice_set_range_psize()
760 int slice_is_hugepage_only_range(struct mm_struct *mm, unsigned long addr,  in slice_is_hugepage_only_range()  argument
764 	unsigned int psize = mm_ctx_user_psize(&mm->context);  in slice_is_hugepage_only_range()
768 	maskp = slice_mask_for_size(&mm->context, psize);  in slice_is_hugepage_only_range()
775 		compat_maskp = slice_mask_for_size(&mm->context, MMU_PAGE_4K);  in slice_is_hugepage_only_range()
777 		return !slice_check_range_fits(mm, &available, addr, len);  in slice_is_hugepage_only_range()
780 	return !slice_check_range_fits(mm, maskp, addr, len);  in slice_is_hugepage_only_range()