262 get_file_region_entry_from_cache(struct resv_map *resv, long from, long to)  in get_file_region_entry_from_cache()  argument
266 	VM_BUG_ON(resv->region_cache_count <= 0);  in get_file_region_entry_from_cache()
268 	resv->region_cache_count--;  in get_file_region_entry_from_cache()
269 	nrg = list_first_entry(&resv->region_cache, struct file_region, link);  in get_file_region_entry_from_cache()
292 						struct resv_map *resv,  in record_hugetlb_cgroup_uncharge_info()  argument
311 		if (!resv->pages_per_hpage)  in record_hugetlb_cgroup_uncharge_info()
312 			resv->pages_per_hpage = pages_per_huge_page(h);  in record_hugetlb_cgroup_uncharge_info()
316 		VM_BUG_ON(resv->pages_per_hpage != pages_per_huge_page(h));  in record_hugetlb_cgroup_uncharge_info()
344 static void coalesce_file_region(struct resv_map *resv, struct file_region *rg)  in coalesce_file_region()  argument
349 	if (&prg->link != &resv->regions && prg->to == rg->from &&  in coalesce_file_region()
361 	if (&nrg->link != &resv->regions && nrg->from == rg->to &&  in coalesce_file_region()
397 static long add_reservation_in_range(struct resv_map *resv, long f, long t,  in add_reservation_in_range()  argument
402 	struct list_head *head = &resv->regions;  in add_reservation_in_range()
437 			add += hugetlb_resv_map_add(resv, iter->link.prev,  in add_reservation_in_range()
451 		add += hugetlb_resv_map_add(resv, rg, last_accounted_offset,  in add_reservation_in_range()
459 static int allocate_file_region_entries(struct resv_map *resv,  in allocate_file_region_entries()  argument
461 	__must_hold(&resv->lock)  in allocate_file_region_entries()
478 	while (resv->region_cache_count <  in allocate_file_region_entries()
479 	       (resv->adds_in_progress + regions_needed)) {  in allocate_file_region_entries()
480 		to_allocate = resv->adds_in_progress + regions_needed -  in allocate_file_region_entries()
481 			      resv->region_cache_count;  in allocate_file_region_entries()
487 		VM_BUG_ON(resv->region_cache_count < resv->adds_in_progress);  in allocate_file_region_entries()
489 		spin_unlock(&resv->lock);  in allocate_file_region_entries()
497 		spin_lock(&resv->lock);  in allocate_file_region_entries()
499 		list_splice(&allocated_regions, &resv->region_cache);  in allocate_file_region_entries()
500 		resv->region_cache_count += to_allocate;  in allocate_file_region_entries()
530 static long region_add(struct resv_map *resv, long f, long t,  in region_add()  argument
536 	spin_lock(&resv->lock);  in region_add()
540 	add_reservation_in_range(resv, f, t, NULL, NULL,  in region_add()
553 	    resv->region_cache_count <  in region_add()
554 		    resv->adds_in_progress +  in region_add()
562 			    resv, actual_regions_needed - in_regions_needed)) {  in region_add()
569 	add = add_reservation_in_range(resv, f, t, h_cg, h, NULL);  in region_add()
571 	resv->adds_in_progress -= in_regions_needed;  in region_add()
573 	spin_unlock(&resv->lock);  in region_add()
597 static long region_chg(struct resv_map *resv, long f, long t,  in region_chg()  argument
602 	spin_lock(&resv->lock);  in region_chg()
605 	chg = add_reservation_in_range(resv, f, t, NULL, NULL,  in region_chg()
611 	if (allocate_file_region_entries(resv, *out_regions_needed))  in region_chg()
614 	resv->adds_in_progress += *out_regions_needed;  in region_chg()
616 	spin_unlock(&resv->lock);  in region_chg()
633 static void region_abort(struct resv_map *resv, long f, long t,  in region_abort()  argument
636 	spin_lock(&resv->lock);  in region_abort()
637 	VM_BUG_ON(!resv->region_cache_count);  in region_abort()
638 	resv->adds_in_progress -= regions_needed;  in region_abort()
639 	spin_unlock(&resv->lock);  in region_abort()
656 static long region_del(struct resv_map *resv, long f, long t)  in region_del()  argument
658 	struct list_head *head = &resv->regions;  in region_del()
664 	spin_lock(&resv->lock);  in region_del()
685 			    resv->region_cache_count > resv->adds_in_progress) {  in region_del()
686 				nrg = list_first_entry(&resv->region_cache,  in region_del()
690 				resv->region_cache_count--;  in region_del()
694 				spin_unlock(&resv->lock);  in region_del()
703 				resv, rg, t - f, false);  in region_del()
723 			hugetlb_cgroup_uncharge_file_region(resv, rg,  in region_del()
731 			hugetlb_cgroup_uncharge_file_region(resv, rg,  in region_del()
737 			hugetlb_cgroup_uncharge_file_region(resv, rg,  in region_del()
745 	spin_unlock(&resv->lock);  in region_del()
783 static long region_count(struct resv_map *resv, long f, long t)  in region_count()  argument
785 	struct list_head *head = &resv->regions;  in region_count()
789 	spin_lock(&resv->lock);  in region_count()
805 	spin_unlock(&resv->lock);  in region_count()
2540 	struct resv_map *resv;  in __vma_reservation_common()  local
2545 	resv = vma_resv_map(vma);  in __vma_reservation_common()
2546 	if (!resv)  in __vma_reservation_common()
2552 		ret = region_chg(resv, idx, idx + 1, &dummy_out_regions_needed);  in __vma_reservation_common()
2560 		ret = region_add(resv, idx, idx + 1, 1, NULL, NULL);  in __vma_reservation_common()
2565 		region_abort(resv, idx, idx + 1, 1);  in __vma_reservation_common()
2570 			ret = region_add(resv, idx, idx + 1, 1, NULL, NULL);  in __vma_reservation_common()
2574 			region_abort(resv, idx, idx + 1, 1);  in __vma_reservation_common()
2575 			ret = region_del(resv, idx, idx + 1);  in __vma_reservation_common()
2580 			region_abort(resv, idx, idx + 1, 1);  in __vma_reservation_common()
2581 			ret = region_del(resv, idx, idx + 1);  in __vma_reservation_common()
2583 			ret = region_add(resv, idx, idx + 1, 1, NULL, NULL);  in __vma_reservation_common()
4611 	struct resv_map *resv = vma_resv_map(vma);  in hugetlb_vm_op_open()  local
4622 	if (resv && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {  in hugetlb_vm_op_open()
4623 		resv_map_dup_hugetlb_cgroup_uncharge_info(resv);  in hugetlb_vm_op_open()
4624 		kref_get(&resv->refs);  in hugetlb_vm_op_open()
4650 	struct resv_map *resv;  in hugetlb_vm_op_close()  local
4657 	resv = vma_resv_map(vma);  in hugetlb_vm_op_close()
4658 	if (!resv || !is_vma_resv_set(vma, HPAGE_RESV_OWNER))  in hugetlb_vm_op_close()
4664 	reserve = (end - start) - region_count(resv, start, end);  in hugetlb_vm_op_close()
4665 	hugetlb_cgroup_uncharge_counter(resv, start, end);  in hugetlb_vm_op_close()
4675 	kref_put(&resv->refs, resv_map_release);  in hugetlb_vm_op_close()