Lines Matching +full:charge +full:- +full:current +full:- +full:limit +full:- +full:mapping
1 // SPDX-License-Identifier: GPL-2.0-only
87 bool free = (spool->count == 0) && (spool->used_hpages == 0); in unlock_or_release_subpool()
89 spin_unlock(&spool->lock); in unlock_or_release_subpool()
95 if (spool->min_hpages != -1) in unlock_or_release_subpool()
96 hugetlb_acct_memory(spool->hstate, in unlock_or_release_subpool()
97 -spool->min_hpages); in unlock_or_release_subpool()
111 spin_lock_init(&spool->lock); in hugepage_new_subpool()
112 spool->count = 1; in hugepage_new_subpool()
113 spool->max_hpages = max_hpages; in hugepage_new_subpool()
114 spool->hstate = h; in hugepage_new_subpool()
115 spool->min_hpages = min_hpages; in hugepage_new_subpool()
117 if (min_hpages != -1 && hugetlb_acct_memory(h, min_hpages)) { in hugepage_new_subpool()
121 spool->rsv_hpages = min_hpages; in hugepage_new_subpool()
128 spin_lock(&spool->lock); in hugepage_put_subpool()
129 BUG_ON(!spool->count); in hugepage_put_subpool()
130 spool->count--; in hugepage_put_subpool()
136 * Return -ENOMEM if there are not enough resources to satisfy the
150 spin_lock(&spool->lock); in hugepage_subpool_get_pages()
152 if (spool->max_hpages != -1) { /* maximum size accounting */ in hugepage_subpool_get_pages()
153 if ((spool->used_hpages + delta) <= spool->max_hpages) in hugepage_subpool_get_pages()
154 spool->used_hpages += delta; in hugepage_subpool_get_pages()
156 ret = -ENOMEM; in hugepage_subpool_get_pages()
162 if (spool->min_hpages != -1 && spool->rsv_hpages) { in hugepage_subpool_get_pages()
163 if (delta > spool->rsv_hpages) { in hugepage_subpool_get_pages()
168 ret = delta - spool->rsv_hpages; in hugepage_subpool_get_pages()
169 spool->rsv_hpages = 0; in hugepage_subpool_get_pages()
172 spool->rsv_hpages -= delta; in hugepage_subpool_get_pages()
177 spin_unlock(&spool->lock); in hugepage_subpool_get_pages()
195 spin_lock(&spool->lock); in hugepage_subpool_put_pages()
197 if (spool->max_hpages != -1) /* maximum size accounting */ in hugepage_subpool_put_pages()
198 spool->used_hpages -= delta; in hugepage_subpool_put_pages()
201 if (spool->min_hpages != -1 && spool->used_hpages < spool->min_hpages) { in hugepage_subpool_put_pages()
202 if (spool->rsv_hpages + delta <= spool->min_hpages) in hugepage_subpool_put_pages()
205 ret = spool->rsv_hpages + delta - spool->min_hpages; in hugepage_subpool_put_pages()
207 spool->rsv_hpages += delta; in hugepage_subpool_put_pages()
208 if (spool->rsv_hpages > spool->min_hpages) in hugepage_subpool_put_pages()
209 spool->rsv_hpages = spool->min_hpages; in hugepage_subpool_put_pages()
223 return HUGETLBFS_SB(inode->i_sb)->spool; in subpool_inode()
228 return subpool_inode(file_inode(vma->vm_file)); in subpool_vma()
239 VM_BUG_ON(resv->region_cache_count <= 0); in get_file_region_entry_from_cache()
241 resv->region_cache_count--; in get_file_region_entry_from_cache()
242 nrg = list_first_entry(&resv->region_cache, struct file_region, link); in get_file_region_entry_from_cache()
243 list_del(&nrg->link); in get_file_region_entry_from_cache()
245 nrg->from = from; in get_file_region_entry_from_cache()
246 nrg->to = to; in get_file_region_entry_from_cache()
255 nrg->reservation_counter = rg->reservation_counter; in copy_hugetlb_cgroup_uncharge_info()
256 nrg->css = rg->css; in copy_hugetlb_cgroup_uncharge_info()
257 if (rg->css) in copy_hugetlb_cgroup_uncharge_info()
258 css_get(rg->css); in copy_hugetlb_cgroup_uncharge_info()
270 nrg->reservation_counter = in record_hugetlb_cgroup_uncharge_info()
271 &h_cg->rsvd_hugepage[hstate_index(h)]; in record_hugetlb_cgroup_uncharge_info()
272 nrg->css = &h_cg->css; in record_hugetlb_cgroup_uncharge_info()
273 if (!resv->pages_per_hpage) in record_hugetlb_cgroup_uncharge_info()
274 resv->pages_per_hpage = pages_per_huge_page(h); in record_hugetlb_cgroup_uncharge_info()
278 VM_BUG_ON(resv->pages_per_hpage != pages_per_huge_page(h)); in record_hugetlb_cgroup_uncharge_info()
280 nrg->reservation_counter = NULL; in record_hugetlb_cgroup_uncharge_info()
281 nrg->css = NULL; in record_hugetlb_cgroup_uncharge_info()
291 rg->reservation_counter == org->reservation_counter && in has_same_uncharge_info()
292 rg->css == org->css; in has_same_uncharge_info()
304 if (&prg->link != &resv->regions && prg->to == rg->from && in coalesce_file_region()
306 prg->to = rg->to; in coalesce_file_region()
308 list_del(&rg->link); in coalesce_file_region()
315 if (&nrg->link != &resv->regions && nrg->from == rg->to && in coalesce_file_region()
317 nrg->from = rg->from; in coalesce_file_region()
319 list_del(&rg->link); in coalesce_file_region()
325 * Must be called with resv->lock held.
337 struct list_head *head = &resv->regions; in add_reservation_in_range()
345 * [last_accounted_offset, rg->from), at every iteration, with some in add_reservation_in_range()
350 if (rg->from < f) { in add_reservation_in_range()
354 if (rg->to > last_accounted_offset) in add_reservation_in_range()
355 last_accounted_offset = rg->to; in add_reservation_in_range()
362 if (rg->from > t) in add_reservation_in_range()
365 /* Add an entry for last_accounted_offset -> rg->from, and in add_reservation_in_range()
368 if (rg->from > last_accounted_offset) { in add_reservation_in_range()
369 add += rg->from - last_accounted_offset; in add_reservation_in_range()
372 resv, last_accounted_offset, rg->from); in add_reservation_in_range()
375 list_add(&nrg->link, rg->link.prev); in add_reservation_in_range()
381 last_accounted_offset = rg->to; in add_reservation_in_range()
388 add += t - last_accounted_offset; in add_reservation_in_range()
393 list_add(&nrg->link, rg->link.prev); in add_reservation_in_range()
403 /* Must be called with resv->lock acquired. Will drop lock to allocate entries.
407 __must_hold(&resv->lock) in allocate_file_region_entries()
426 while (resv->region_cache_count < in allocate_file_region_entries()
427 (resv->adds_in_progress + regions_needed)) { in allocate_file_region_entries()
428 to_allocate = resv->adds_in_progress + regions_needed - in allocate_file_region_entries()
429 resv->region_cache_count; in allocate_file_region_entries()
435 VM_BUG_ON(resv->region_cache_count < resv->adds_in_progress); in allocate_file_region_entries()
437 spin_unlock(&resv->lock); in allocate_file_region_entries()
442 list_add(&trg->link, &allocated_regions); in allocate_file_region_entries()
445 spin_lock(&resv->lock); in allocate_file_region_entries()
447 list_splice(&allocated_regions, &resv->region_cache); in allocate_file_region_entries()
448 resv->region_cache_count += to_allocate; in allocate_file_region_entries()
455 list_del(&rg->link); in allocate_file_region_entries()
458 return -ENOMEM; in allocate_file_region_entries()
473 * this operation and we were not able to allocate, it returns -ENOMEM.
484 spin_lock(&resv->lock); in region_add()
501 resv->region_cache_count < in region_add()
502 resv->adds_in_progress + in region_add()
503 (actual_regions_needed - in_regions_needed)) { in region_add()
507 VM_BUG_ON(t - f <= 1); in region_add()
510 resv, actual_regions_needed - in_regions_needed)) { in region_add()
511 return -ENOMEM; in region_add()
519 resv->adds_in_progress -= in_regions_needed; in region_add()
521 spin_unlock(&resv->lock); in region_add()
538 * resv->adds_in_progress. This value needs to be provided to a follow up call
543 * zero. -ENOMEM is returned if a new file_region structure or cache entry
551 spin_lock(&resv->lock); in region_chg()
561 return -ENOMEM; in region_chg()
563 resv->adds_in_progress += *out_regions_needed; in region_chg()
565 spin_unlock(&resv->lock); in region_chg()
585 spin_lock(&resv->lock); in region_abort()
586 VM_BUG_ON(!resv->region_cache_count); in region_abort()
587 resv->adds_in_progress -= regions_needed; in region_abort()
588 spin_unlock(&resv->lock); in region_abort()
600 * be allocated. If the allocation fails, -ENOMEM will be returned.
602 * a region and possibly return -ENOMEM. Callers specifying
603 * t == LONG_MAX do not need to check for -ENOMEM error.
607 struct list_head *head = &resv->regions; in region_del()
613 spin_lock(&resv->lock); in region_del()
622 if (rg->to <= f && (rg->to != rg->from || rg->to != f)) in region_del()
625 if (rg->from >= t) in region_del()
628 if (f > rg->from && t < rg->to) { /* Must split region */ in region_del()
634 resv->region_cache_count > resv->adds_in_progress) { in region_del()
635 nrg = list_first_entry(&resv->region_cache, in region_del()
638 list_del(&nrg->link); in region_del()
639 resv->region_cache_count--; in region_del()
643 spin_unlock(&resv->lock); in region_del()
646 return -ENOMEM; in region_del()
650 del += t - f; in region_del()
652 resv, rg, t - f); in region_del()
655 nrg->from = t; in region_del()
656 nrg->to = rg->to; in region_del()
660 INIT_LIST_HEAD(&nrg->link); in region_del()
663 rg->to = f; in region_del()
665 list_add(&nrg->link, &rg->link); in region_del()
670 if (f <= rg->from && t >= rg->to) { /* Remove entire region */ in region_del()
671 del += rg->to - rg->from; in region_del()
673 rg->to - rg->from); in region_del()
674 list_del(&rg->link); in region_del()
679 if (f <= rg->from) { /* Trim beginning of region */ in region_del()
681 t - rg->from); in region_del()
683 del += t - rg->from; in region_del()
684 rg->from = t; in region_del()
687 rg->to - f); in region_del()
689 del += rg->to - f; in region_del()
690 rg->to = f; in region_del()
694 spin_unlock(&resv->lock); in region_del()
727 struct list_head *head = &resv->regions; in region_count()
731 spin_lock(&resv->lock); in region_count()
737 if (rg->to <= f) in region_count()
739 if (rg->from >= t) in region_count()
742 seg_from = max(rg->from, f); in region_count()
743 seg_to = min(rg->to, t); in region_count()
745 chg += seg_to - seg_from; in region_count()
747 spin_unlock(&resv->lock); in region_count()
754 * the mapping, in pagecache page units; huge pages here.
759 return ((address - vma->vm_start) >> huge_page_shift(h)) + in vma_hugecache_offset()
760 (vma->vm_pgoff >> huge_page_order(h)); in vma_hugecache_offset()
776 if (vma->vm_ops && vma->vm_ops->pagesize) in vma_kernel_pagesize()
777 return vma->vm_ops->pagesize(vma); in vma_kernel_pagesize()
785 * architectures where it differs, an architecture-specific 'strong'
804 * faults in a MAP_PRIVATE mapping. Only the process that called mmap()
812 * The private mapping reservation is represented in a subtly different
813 * manner to a shared mapping. A shared mapping has a region map associated
816 * after the page is instantiated. A private mapping has a region map
823 return (unsigned long)vma->vm_private_data; in get_vma_private_data()
829 vma->vm_private_data = (void *)value; in set_vma_private_data()
839 resv_map->reservation_counter = NULL; in resv_map_set_hugetlb_cgroup_uncharge_info()
840 resv_map->pages_per_hpage = 0; in resv_map_set_hugetlb_cgroup_uncharge_info()
841 resv_map->css = NULL; in resv_map_set_hugetlb_cgroup_uncharge_info()
843 resv_map->reservation_counter = in resv_map_set_hugetlb_cgroup_uncharge_info()
844 &h_cg->rsvd_hugepage[hstate_index(h)]; in resv_map_set_hugetlb_cgroup_uncharge_info()
845 resv_map->pages_per_hpage = pages_per_huge_page(h); in resv_map_set_hugetlb_cgroup_uncharge_info()
846 resv_map->css = &h_cg->css; in resv_map_set_hugetlb_cgroup_uncharge_info()
862 kref_init(&resv_map->refs); in resv_map_alloc()
863 spin_lock_init(&resv_map->lock); in resv_map_alloc()
864 INIT_LIST_HEAD(&resv_map->regions); in resv_map_alloc()
866 resv_map->adds_in_progress = 0; in resv_map_alloc()
870 * re-initialized to the proper values, to indicate that hugetlb cgroup in resv_map_alloc()
871 * reservations are to be un-charged from here. in resv_map_alloc()
875 INIT_LIST_HEAD(&resv_map->region_cache); in resv_map_alloc()
876 list_add(&rg->link, &resv_map->region_cache); in resv_map_alloc()
877 resv_map->region_cache_count = 1; in resv_map_alloc()
885 struct list_head *head = &resv_map->region_cache; in resv_map_release()
893 list_del(&rg->link); in resv_map_release()
897 VM_BUG_ON(resv_map->adds_in_progress); in resv_map_release()
909 * The VERY common case is inode->mapping == &inode->i_data but, in inode_resv_map()
912 return (struct resv_map *)(&inode->i_data)->private_data; in inode_resv_map()
918 if (vma->vm_flags & VM_MAYSHARE) { in vma_resv_map()
919 struct address_space *mapping = vma->vm_file->f_mapping; in vma_resv_map() local
920 struct inode *inode = mapping->host; in vma_resv_map()
933 VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma); in set_vma_resv_map()
942 VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma); in set_vma_resv_flags()
958 if (!(vma->vm_flags & VM_MAYSHARE)) in reset_vma_resv_huge_pages()
959 vma->vm_private_data = (void *)0; in reset_vma_resv_huge_pages()
965 if (vma->vm_flags & VM_NORESERVE) { in vma_has_reserves()
973 * properly, so add work-around here. in vma_has_reserves()
975 if (vma->vm_flags & VM_MAYSHARE && chg == 0) in vma_has_reserves()
982 if (vma->vm_flags & VM_MAYSHARE) { in vma_has_reserves()
1003 * could have been performed on the private mapping. in vma_has_reserves()
1006 * Very Subtle - The value of chg comes from a previous in vma_has_reserves()
1028 list_move(&page->lru, &h->hugepage_freelists[nid]); in enqueue_huge_page()
1029 h->free_huge_pages++; in enqueue_huge_page()
1030 h->free_huge_pages_node[nid]++; in enqueue_huge_page()
1036 bool nocma = !!(current->flags & PF_MEMALLOC_NOCMA); in dequeue_huge_page_node_exact()
1038 list_for_each_entry(page, &h->hugepage_freelists[nid], lru) { in dequeue_huge_page_node_exact()
1045 list_move(&page->lru, &h->hugepage_activelist); in dequeue_huge_page_node_exact()
1047 h->free_huge_pages--; in dequeue_huge_page_node_exact()
1048 h->free_huge_pages_node[nid]--; in dequeue_huge_page_node_exact()
1108 h->free_huge_pages - h->resv_huge_pages == 0) in dequeue_huge_page_vma()
1112 if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0) in dequeue_huge_page_vma()
1120 h->resv_huge_pages--; in dequeue_huge_page_vma()
1133 * nodes_allowed previously, so h->next_node_to_{alloc|free} might
1165 nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed); in hstate_next_node_to_alloc()
1166 h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed); in hstate_next_node_to_alloc()
1172 * helper for free_pool_huge_page() - return the previously saved
1183 nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed); in hstate_next_node_to_free()
1184 h->next_nid_to_free = next_node_allowed(nid, nodes_allowed); in hstate_next_node_to_free()
1193 nr_nodes--)
1199 nr_nodes--)
1302 h->nr_huge_pages--; in update_and_free_page()
1303 h->nr_huge_pages_node[page_to_nid(page)]--; in update_and_free_page()
1340 * Test to determine whether the hugepage is "active/in-use" (i.e. being linked
1341 * to hstate->hugepage_activelist.)
1373 return (unsigned long)page[2].mapping == -1U; in PageHugeTemporary()
1378 page[2].mapping = (void *)-1U; in SetPageHugeTemporary()
1383 page[2].mapping = NULL; in ClearPageHugeTemporary()
1402 page->mapping = NULL; in __free_huge_page()
1432 h->resv_huge_pages++; in __free_huge_page()
1435 list_del(&page->lru); in __free_huge_page()
1438 } else if (h->surplus_huge_pages_node[nid]) { in __free_huge_page()
1440 list_del(&page->lru); in __free_huge_page()
1442 h->surplus_huge_pages--; in __free_huge_page()
1443 h->surplus_huge_pages_node[nid]--; in __free_huge_page()
1452 * As free_huge_page() can be called from a non-task context, we have
1457 * be freed and frees them one-by-one. As the page->mapping pointer is
1472 struct page, mapping); in free_hpage_workfn()
1473 node = node->next; in free_hpage_workfn()
1482 * Defer freeing if in non-task context to avoid hugetlb_lock deadlock. in free_huge_page()
1490 if (llist_add((struct llist_node *)&page->mapping, in free_huge_page()
1501 INIT_LIST_HEAD(&page->lru); in prep_new_huge_page()
1506 h->nr_huge_pages++; in prep_new_huge_page()
1507 h->nr_huge_pages_node[nid]++; in prep_new_huge_page()
1524 * boot, it's safer to be consistent with the not-gigantic in prep_compound_gigantic_page()
1538 atomic_set(compound_mapcount_ptr(page), -1); in prep_compound_gigantic_page()
1572 * Find and lock address space (mapping) in write mode.
1580 struct address_space *mapping = page_mapping(hpage); in hugetlb_page_mapping_lock_write() local
1582 if (!mapping) in hugetlb_page_mapping_lock_write()
1583 return mapping; in hugetlb_page_mapping_lock_write()
1585 if (i_mmap_trylock_write(mapping)) in hugetlb_page_mapping_lock_write()
1586 return mapping; in hugetlb_page_mapping_lock_write()
1601 compound_idx = page_to_pfn(page) - page_to_pfn(page_head); in __basepage_index()
1603 compound_idx = page - page_head; in __basepage_index()
1723 if ((!acct_surplus || h->surplus_huge_pages_node[node]) && in free_pool_huge_page()
1724 !list_empty(&h->hugepage_freelists[node])) { in free_pool_huge_page()
1726 list_entry(h->hugepage_freelists[node].next, in free_pool_huge_page()
1728 list_del(&page->lru); in free_pool_huge_page()
1729 h->free_huge_pages--; in free_pool_huge_page()
1730 h->free_huge_pages_node[node]--; in free_pool_huge_page()
1732 h->surplus_huge_pages--; in free_pool_huge_page()
1733 h->surplus_huge_pages_node[node]--; in free_pool_huge_page()
1746 * nothing for in-use hugepages and non-hugepages.
1749 * -EBUSY: failed to dissolved free hugepages or the hugepage is in-use
1756 int rc = -EBUSY; in dissolve_free_huge_page()
1772 if (h->free_huge_pages - h->resv_huge_pages == 0) in dissolve_free_huge_page()
1782 list_del(&head->lru); in dissolve_free_huge_page()
1783 h->free_huge_pages--; in dissolve_free_huge_page()
1784 h->free_huge_pages_node[nid]--; in dissolve_free_huge_page()
1785 h->max_huge_pages--; in dissolve_free_huge_page()
1833 if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) in alloc_surplus_huge_page()
1849 if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) { in alloc_surplus_huge_page()
1855 h->surplus_huge_pages++; in alloc_surplus_huge_page()
1856 h->surplus_huge_pages_node[page_to_nid(page)]++; in alloc_surplus_huge_page()
1911 if (h->free_huge_pages - h->resv_huge_pages > 0) { in alloc_huge_page_nodemask()
1956 needed = (h->resv_huge_pages + delta) - h->free_huge_pages; in gather_surplus_pages()
1958 h->resv_huge_pages += delta; in gather_surplus_pages()
1965 ret = -ENOMEM; in gather_surplus_pages()
1975 list_add(&page->lru, &surplus_list); in gather_surplus_pages()
1985 needed = (h->resv_huge_pages + delta) - in gather_surplus_pages()
1986 (h->free_huge_pages + allocated); in gather_surplus_pages()
2006 h->resv_huge_pages += delta; in gather_surplus_pages()
2011 if ((--needed) < 0) in gather_surplus_pages()
2015 * no users -- drop the buddy allocator's reference. in gather_surplus_pages()
2057 * by pre-allocated pages. Only free surplus pages. in return_unused_surplus_pages()
2059 nr_pages = min(unused_resv_pages, h->surplus_huge_pages); in return_unused_surplus_pages()
2067 * on-line nodes with memory and will handle the hstate accounting. in return_unused_surplus_pages()
2073 while (nr_pages--) { in return_unused_surplus_pages()
2074 h->resv_huge_pages--; in return_unused_surplus_pages()
2075 unused_resv_pages--; in return_unused_surplus_pages()
2083 h->resv_huge_pages -= unused_resv_pages; in return_unused_surplus_pages()
2150 if (vma->vm_flags & VM_MAYSHARE) { in __vma_reservation_common()
2163 if (vma->vm_flags & VM_MAYSHARE) in __vma_reservation_common()
2168 * However, a file associated with mapping could have been in __vma_reservation_common()
2171 * Subtle - The reserve map for private mappings has the in __vma_reservation_common()
2276 return ERR_PTR(-ENOMEM); in alloc_huge_page()
2279 * Processes that did not create the mapping will have no in alloc_huge_page()
2281 * that the allocation will not exceed the subpool limit. in alloc_huge_page()
2283 * checked against any subpool limit. in alloc_huge_page()
2289 return ERR_PTR(-ENOSPC); in alloc_huge_page()
2304 /* If this allocation is not consuming a reservation, charge it now. in alloc_huge_page()
2332 h->resv_huge_pages--; in alloc_huge_page()
2335 list_add(&page->lru, &h->hugepage_activelist); in alloc_huge_page()
2365 hugetlb_acct_memory(h, -rsv_adjust); in alloc_huge_page()
2382 return ERR_PTR(-ENOSPC); in alloc_huge_page()
2413 INIT_LIST_HEAD(&m->list); in __alloc_bootmem_huge_page()
2414 list_add(&m->list, &huge_boot_pages); in __alloc_bootmem_huge_page()
2415 m->hstate = h; in __alloc_bootmem_huge_page()
2422 if (unlikely(order > (MAX_ORDER - 1))) in prep_compound_huge_page()
2435 struct hstate *h = m->hstate; in gather_bootmem_prealloc()
2438 prep_compound_huge_page(page, h->order); in gather_bootmem_prealloc()
2447 * side-effects, like CommitLimit going negative. in gather_bootmem_prealloc()
2450 adjust_managed_page_count(page, 1 << h->order); in gather_bootmem_prealloc()
2462 * Bit mask controlling how hard we retry per-node allocations. in hugetlb_hstate_alloc_pages()
2474 /* bit mask controlling how hard we retry per-node allocations */ in hugetlb_hstate_alloc_pages()
2478 for (i = 0; i < h->max_huge_pages; ++i) { in hugetlb_hstate_alloc_pages()
2492 if (i < h->max_huge_pages) { in hugetlb_hstate_alloc_pages()
2497 h->max_huge_pages, buf, i); in hugetlb_hstate_alloc_pages()
2498 h->max_huge_pages = i; in hugetlb_hstate_alloc_pages()
2527 pr_info("HugeTLB registered %s page size, pre-allocated %ld pages\n", in report_hugepages()
2528 buf, h->free_huge_pages); in report_hugepages()
2543 struct list_head *freel = &h->hugepage_freelists[i]; in try_to_free_low()
2545 if (count >= h->nr_huge_pages) in try_to_free_low()
2549 list_del(&page->lru); in try_to_free_low()
2551 h->free_huge_pages--; in try_to_free_low()
2552 h->free_huge_pages_node[page_to_nid(page)]--; in try_to_free_low()
2564 * Increment or decrement surplus_huge_pages. Keep node-specific counters
2565 * balanced by operating on them in a round-robin fashion.
2573 VM_BUG_ON(delta != -1 && delta != 1); in adjust_pool_surplus()
2577 if (h->surplus_huge_pages_node[node]) in adjust_pool_surplus()
2582 if (h->surplus_huge_pages_node[node] < in adjust_pool_surplus()
2583 h->nr_huge_pages_node[node]) in adjust_pool_surplus()
2590 h->surplus_huge_pages += delta; in adjust_pool_surplus()
2591 h->surplus_huge_pages_node[node] += delta; in adjust_pool_surplus()
2595 #define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages)
2603 * Bit mask controlling how hard we retry per-node allocations. in set_max_huge_pages()
2610 return -ENOMEM; in set_max_huge_pages()
2623 count += h->nr_huge_pages - h->nr_huge_pages_node[nid]; in set_max_huge_pages()
2645 return -EINVAL; in set_max_huge_pages()
2661 while (h->surplus_huge_pages && count > persistent_huge_pages(h)) { in set_max_huge_pages()
2662 if (!adjust_pool_surplus(h, nodes_allowed, -1)) in set_max_huge_pages()
2683 /* Bail for signals. Probably ctrl-c from user */ in set_max_huge_pages()
2684 if (signal_pending(current)) in set_max_huge_pages()
2703 min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages; in set_max_huge_pages()
2716 h->max_huge_pages = persistent_huge_pages(h); in set_max_huge_pages()
2759 nr_huge_pages = h->nr_huge_pages; in nr_hugepages_show_common()
2761 nr_huge_pages = h->nr_huge_pages_node[nid]; in nr_hugepages_show_common()
2774 return -EINVAL; in __nr_hugepages_store_common()
2832 * hstate attribute for optionally mempolicy-based constraint on persistent
2854 return sprintf(buf, "%lu\n", h->nr_overcommit_huge_pages); in nr_overcommit_hugepages_show()
2865 return -EINVAL; in nr_overcommit_hugepages_store()
2872 h->nr_overcommit_huge_pages = input; in nr_overcommit_hugepages_store()
2888 free_huge_pages = h->free_huge_pages; in free_hugepages_show()
2890 free_huge_pages = h->free_huge_pages_node[nid]; in free_hugepages_show()
2900 return sprintf(buf, "%lu\n", h->resv_huge_pages); in resv_hugepages_show()
2913 surplus_huge_pages = h->surplus_huge_pages; in surplus_hugepages_show()
2915 surplus_huge_pages = h->surplus_huge_pages_node[nid]; in surplus_hugepages_show()
2944 hstate_kobjs[hi] = kobject_create_and_add(h->name, parent); in hugetlb_sysfs_add_hstate()
2946 return -ENOMEM; in hugetlb_sysfs_add_hstate()
2968 pr_err("HugeTLB: Unable to add hstate %s", h->name); in hugetlb_sysfs_init()
2975 * node_hstate/s - associate per node hstate attributes, via their kobjects,
3002 * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj.
3003 * Returns node id via non-NULL nidp.
3013 if (nhs->hstate_kobjs[i] == kobj) { in kobj_to_node_hstate()
3026 * No-op if no hstate attributes attached.
3031 struct node_hstate *nhs = &node_hstates[node->dev.id]; in hugetlb_unregister_node()
3033 if (!nhs->hugepages_kobj) in hugetlb_unregister_node()
3038 if (nhs->hstate_kobjs[idx]) { in hugetlb_unregister_node()
3039 kobject_put(nhs->hstate_kobjs[idx]); in hugetlb_unregister_node()
3040 nhs->hstate_kobjs[idx] = NULL; in hugetlb_unregister_node()
3044 kobject_put(nhs->hugepages_kobj); in hugetlb_unregister_node()
3045 nhs->hugepages_kobj = NULL; in hugetlb_unregister_node()
3051 * No-op if attributes already registered.
3056 struct node_hstate *nhs = &node_hstates[node->dev.id]; in hugetlb_register_node()
3059 if (nhs->hugepages_kobj) in hugetlb_register_node()
3062 nhs->hugepages_kobj = kobject_create_and_add("hugepages", in hugetlb_register_node()
3063 &node->dev.kobj); in hugetlb_register_node()
3064 if (!nhs->hugepages_kobj) in hugetlb_register_node()
3068 err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj, in hugetlb_register_node()
3069 nhs->hstate_kobjs, in hugetlb_register_node()
3073 h->name, node->dev.id); in hugetlb_register_node()
3082 * devices of nodes that have memory. All on-line nodes should have
3091 if (node->dev.id == nid) in hugetlb_register_all_nodes()
3108 *nidp = -1; in kobj_to_node_hstate()
3122 pr_warn("HugeTLB: huge pages not supported, ignoring associated command-line parameters\n"); in hugetlb_init()
3199 h->order = order; in hugetlb_add_hstate()
3200 h->mask = ~((1ULL << (order + PAGE_SHIFT)) - 1); in hugetlb_add_hstate()
3201 h->nr_huge_pages = 0; in hugetlb_add_hstate()
3202 h->free_huge_pages = 0; in hugetlb_add_hstate()
3204 INIT_LIST_HEAD(&h->hugepage_freelists[i]); in hugetlb_add_hstate()
3205 INIT_LIST_HEAD(&h->hugepage_activelist); in hugetlb_add_hstate()
3206 h->next_nid_to_alloc = first_memory_node; in hugetlb_add_hstate()
3207 h->next_nid_to_free = first_memory_node; in hugetlb_add_hstate()
3208 snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB", in hugetlb_add_hstate()
3241 mhp = &parsed_hstate->max_huge_pages; in hugepages_setup()
3256 if (hugetlb_max_hstate && parsed_hstate->order >= MAX_ORDER) in hugepages_setup()
3310 hugetlb_add_hstate(ilog2(size) - PAGE_SHIFT); in hugepagesz_setup()
3337 hugetlb_add_hstate(ilog2(size) - PAGE_SHIFT); in default_hugepagesz_setup()
3365 unsigned int *array = h->free_huge_pages_node; in allowed_mems_nr()
3401 unsigned long tmp = h->max_huge_pages; in hugetlb_sysctl_handler_common()
3405 return -EOPNOTSUPP; in hugetlb_sysctl_handler_common()
3444 return -EOPNOTSUPP; in hugetlb_overcommit_handler()
3446 tmp = h->nr_overcommit_huge_pages; in hugetlb_overcommit_handler()
3449 return -EINVAL; in hugetlb_overcommit_handler()
3458 h->nr_overcommit_huge_pages = tmp; in hugetlb_overcommit_handler()
3476 unsigned long count = h->nr_huge_pages; in hugetlb_report_meminfo()
3488 h->free_huge_pages, in hugetlb_report_meminfo()
3489 h->resv_huge_pages, in hugetlb_report_meminfo()
3490 h->surplus_huge_pages, in hugetlb_report_meminfo()
3508 nid, h->nr_huge_pages_node[nid], in hugetlb_report_node_meminfo()
3509 nid, h->free_huge_pages_node[nid], in hugetlb_report_node_meminfo()
3510 nid, h->surplus_huge_pages_node[nid]); in hugetlb_report_node_meminfo()
3525 h->nr_huge_pages_node[nid], in hugetlb_show_meminfo()
3526 h->free_huge_pages_node[nid], in hugetlb_show_meminfo()
3527 h->surplus_huge_pages_node[nid], in hugetlb_show_meminfo()
3528 1UL << (huge_page_order(h) + PAGE_SHIFT - 10)); in hugetlb_show_meminfo()
3534 atomic_long_read(&mm->hugetlb_usage) << (PAGE_SHIFT - 10)); in hugetlb_report_usage()
3544 nr_total_pages += h->nr_huge_pages * pages_per_huge_page(h); in hugetlb_total_pages()
3550 int ret = -ENOMEM; in hugetlb_acct_memory()
3558 * current cpuset. Application can still potentially OOM'ed by kernel in hugetlb_acct_memory()
3564 * The change of semantics for shared hugetlb mapping with cpuset is in hugetlb_acct_memory()
3566 * we fall back to check against current free page availability as in hugetlb_acct_memory()
3573 * the memory policy of the current task. Similar to the description in hugetlb_acct_memory()
3588 return_unused_surplus_pages(h, (unsigned long) -delta); in hugetlb_acct_memory()
3608 kref_get(&resv->refs); in hugetlb_vm_op_open()
3622 start = vma_hugecache_offset(h, vma, vma->vm_start); in hugetlb_vm_op_close()
3623 end = vma_hugecache_offset(h, vma, vma->vm_end); in hugetlb_vm_op_close()
3625 reserve = (end - start) - region_count(resv, start, end); in hugetlb_vm_op_close()
3633 hugetlb_acct_memory(h, -gbl_reserve); in hugetlb_vm_op_close()
3636 kref_put(&resv->refs, resv_map_release); in hugetlb_vm_op_close()
3642 return -EINVAL; in hugetlb_vm_op_split()
3655 * handle_mm_fault() to try to instantiate regular-sized pages in the
3687 vma->vm_page_prot))); in make_huge_pte()
3690 vma->vm_page_prot)); in make_huge_pte()
3744 struct address_space *mapping = vma->vm_file->f_mapping; in copy_hugetlb_page_range() local
3748 cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE; in copy_hugetlb_page_range()
3752 vma->vm_start, in copy_hugetlb_page_range()
3753 vma->vm_end); in copy_hugetlb_page_range()
3762 i_mmap_lock_read(mapping); in copy_hugetlb_page_range()
3765 for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) { in copy_hugetlb_page_range()
3772 ret = -ENOMEM; in copy_hugetlb_page_range()
3841 i_mmap_unlock_read(mapping); in copy_hugetlb_page_range()
3850 struct mm_struct *mm = vma->vm_mm; in __unmap_hugepage_range()
3965 vma->vm_flags &= ~VM_MAYSHARE; in __unmap_hugepage_range_final()
3985 mm = vma->vm_mm; in unmap_hugepage_range()
4003 struct address_space *mapping; in unmap_ref_private() local
4011 pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + in unmap_ref_private()
4012 vma->vm_pgoff; in unmap_ref_private()
4013 mapping = vma->vm_file->f_mapping; in unmap_ref_private()
4016 * Take the mapping lock for the duration of the table walk. As in unmap_ref_private()
4017 * this mapping should be shared between all the VMAs, in unmap_ref_private()
4020 i_mmap_lock_write(mapping); in unmap_ref_private()
4021 vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) { in unmap_ref_private()
4022 /* Do not unmap the current VMA */ in unmap_ref_private()
4031 if (iter_vma->vm_flags & VM_MAYSHARE) in unmap_ref_private()
4037 * areas. This is because a future no-page fault on this VMA in unmap_ref_private()
4045 i_mmap_unlock_write(mapping); in unmap_ref_private()
4070 /* If no-one else is actually using this page, avoid the copy in hugetlb_cow()
4079 * If the process that created a MAP_PRIVATE mapping is about to in hugetlb_cow()
4083 * consumed or not. If reserves were used, a partial faulted mapping in hugetlb_cow()
4102 * If a process owning a MAP_PRIVATE mapping fails to COW, in hugetlb_cow()
4119 * race occurs while re-acquiring page table in hugetlb_cow()
4182 struct address_space *mapping; in hugetlbfs_pagecache_page() local
4185 mapping = vma->vm_file->f_mapping; in hugetlbfs_pagecache_page()
4188 return find_lock_page(mapping, idx); in hugetlbfs_pagecache_page()
4198 struct address_space *mapping; in hugetlbfs_pagecache_present() local
4202 mapping = vma->vm_file->f_mapping; in hugetlbfs_pagecache_present()
4205 page = find_get_page(mapping, idx); in hugetlbfs_pagecache_present()
4211 int huge_add_to_page_cache(struct page *page, struct address_space *mapping, in huge_add_to_page_cache() argument
4214 struct inode *inode = mapping->host; in huge_add_to_page_cache()
4216 int err = add_to_page_cache(page, mapping, idx, GFP_KERNEL); in huge_add_to_page_cache()
4224 * by non-hugetlbfs specific code paths. in huge_add_to_page_cache()
4228 spin_lock(&inode->i_lock); in huge_add_to_page_cache()
4229 inode->i_blocks += blocks_per_huge_page(h); in huge_add_to_page_cache()
4230 spin_unlock(&inode->i_lock); in huge_add_to_page_cache()
4236 struct address_space *mapping, pgoff_t idx, in hugetlb_no_page() argument
4256 current->pid); in hugetlb_no_page()
4265 size = i_size_read(mapping->host) >> huge_page_shift(h); in hugetlb_no_page()
4270 page = find_lock_page(mapping, idx); in hugetlb_no_page()
4295 hash = hugetlb_fault_mutex_hash(mapping, idx); in hugetlb_no_page()
4297 i_mmap_unlock_read(mapping); in hugetlb_no_page()
4299 i_mmap_lock_read(mapping); in hugetlb_no_page()
4332 if (vma->vm_flags & VM_MAYSHARE) { in hugetlb_no_page()
4333 int err = huge_add_to_page_cache(page, mapping, idx); in hugetlb_no_page()
4336 if (err == -EEXIST) in hugetlb_no_page()
4362 * If we are going to COW a private mapping later, we examine the in hugetlb_no_page()
4367 if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) { in hugetlb_no_page()
4386 new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE) in hugetlb_no_page()
4387 && (vma->vm_flags & VM_SHARED))); in hugetlb_no_page()
4391 if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) { in hugetlb_no_page()
4420 u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx) in hugetlb_fault_mutex_hash() argument
4425 key[0] = (unsigned long) mapping; in hugetlb_fault_mutex_hash()
4430 return hash & (num_fault_mutexes - 1); in hugetlb_fault_mutex_hash()
4437 u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx) in hugetlb_fault_mutex_hash() argument
4454 struct address_space *mapping; in hugetlb_fault() local
4485 mapping = vma->vm_file->f_mapping; in hugetlb_fault()
4486 i_mmap_lock_read(mapping); in hugetlb_fault()
4489 i_mmap_unlock_read(mapping); in hugetlb_fault()
4499 hash = hugetlb_fault_mutex_hash(mapping, idx); in hugetlb_fault()
4504 ret = hugetlb_no_page(mm, vma, mapping, idx, address, ptep, flags); in hugetlb_fault()
4521 * If we are going to COW the mapping later, we examine the pending in hugetlb_fault()
4536 if (!(vma->vm_flags & VM_MAYSHARE)) in hugetlb_fault()
4586 i_mmap_unlock_read(mapping); in hugetlb_fault()
4590 * the page is not used after unlocked before returning from the current in hugetlb_fault()
4610 struct address_space *mapping; in hugetlb_mcopy_atomic_pte() local
4613 int vm_shared = dst_vma->vm_flags & VM_SHARED; in hugetlb_mcopy_atomic_pte()
4621 ret = -ENOMEM; in hugetlb_mcopy_atomic_pte()
4632 ret = -ENOENT; in hugetlb_mcopy_atomic_pte()
4649 mapping = dst_vma->vm_file->f_mapping; in hugetlb_mcopy_atomic_pte()
4656 size = i_size_read(mapping->host) >> huge_page_shift(h); in hugetlb_mcopy_atomic_pte()
4657 ret = -EFAULT; in hugetlb_mcopy_atomic_pte()
4667 ret = huge_add_to_page_cache(page, mapping, idx); in hugetlb_mcopy_atomic_pte()
4684 size = i_size_read(mapping->host) >> huge_page_shift(h); in hugetlb_mcopy_atomic_pte()
4685 ret = -EFAULT; in hugetlb_mcopy_atomic_pte()
4689 ret = -EEXIST; in hugetlb_mcopy_atomic_pte()
4700 _dst_pte = make_huge_pte(dst_vma, page, dst_vma->vm_flags & VM_WRITE); in hugetlb_mcopy_atomic_pte()
4701 if (dst_vma->vm_flags & VM_WRITE) in hugetlb_mcopy_atomic_pte()
4708 dst_vma->vm_flags & VM_WRITE); in hugetlb_mcopy_atomic_pte()
4711 /* No need to invalidate - it was non-present before */ in hugetlb_mcopy_atomic_pte()
4739 int err = -EFAULT; in follow_hugetlb_page()
4741 while (vaddr < vma->vm_end && remainder) { in follow_hugetlb_page()
4751 if (fatal_signal_pending(current)) { in follow_hugetlb_page()
4813 * FAULT_FLAG_TRIED can co-exist in follow_hugetlb_page()
4850 (vaddr + huge_page_size(h) < vma->vm_end) && in follow_hugetlb_page()
4853 remainder -= pages_per_huge_page(h); in follow_hugetlb_page()
4875 err = -ENOMEM; in follow_hugetlb_page()
4885 --remainder; in follow_hugetlb_page()
4887 if (vaddr < vma->vm_end && remainder && in follow_hugetlb_page()
4919 struct mm_struct *mm = vma->vm_mm; in hugetlb_change_protection()
4941 i_mmap_lock_write(vma->vm_file->f_mapping); in hugetlb_change_protection()
5002 i_mmap_unlock_write(vma->vm_file->f_mapping); in hugetlb_change_protection()
5005 return pages << h->order; in hugetlb_change_protection()
5013 long ret, chg, add = -1; in hugetlb_reserve_pages()
5023 return -EINVAL; in hugetlb_reserve_pages()
5037 * to reserve the full area even if read-only as mprotect() may be in hugetlb_reserve_pages()
5038 * called to make the mapping read-write. Assume !vma is a shm mapping in hugetlb_reserve_pages()
5040 if (!vma || vma->vm_flags & VM_MAYSHARE) { in hugetlb_reserve_pages()
5051 /* Private mapping. */ in hugetlb_reserve_pages()
5054 return -ENOMEM; in hugetlb_reserve_pages()
5056 chg = to - from; in hugetlb_reserve_pages()
5071 ret = -ENOMEM; in hugetlb_reserve_pages()
5075 if (vma && !(vma->vm_flags & VM_MAYSHARE) && h_cg) { in hugetlb_reserve_pages()
5083 * There must be enough pages in the subpool for the mapping. If in hugetlb_reserve_pages()
5089 ret = -ENOSPC; in hugetlb_reserve_pages()
5107 * the reservation was consumed. Private mappings are per-VMA and in hugetlb_reserve_pages()
5113 if (!vma || vma->vm_flags & VM_MAYSHARE) { in hugetlb_reserve_pages()
5117 hugetlb_acct_memory(h, -gbl_reserve); in hugetlb_reserve_pages()
5131 (chg - add) * pages_per_huge_page(h), h_cg); in hugetlb_reserve_pages()
5134 chg - add); in hugetlb_reserve_pages()
5135 hugetlb_acct_memory(h, -rsv_adjust); in hugetlb_reserve_pages()
5146 if (!vma || vma->vm_flags & VM_MAYSHARE) in hugetlb_reserve_pages()
5153 kref_put(&resv_map->refs, resv_map_release); in hugetlb_reserve_pages()
5181 spin_lock(&inode->i_lock); in hugetlb_unreserve_pages()
5182 inode->i_blocks -= (blocks_per_huge_page(h) * freed); in hugetlb_unreserve_pages()
5183 spin_unlock(&inode->i_lock); in hugetlb_unreserve_pages()
5189 gbl_reserve = hugepage_subpool_put_pages(spool, (chg - freed)); in hugetlb_unreserve_pages()
5190 hugetlb_acct_memory(h, -gbl_reserve); in hugetlb_unreserve_pages()
5200 unsigned long saddr = ((idx - svma->vm_pgoff) << PAGE_SHIFT) + in page_table_shareable()
5201 svma->vm_start; in page_table_shareable()
5206 unsigned long vm_flags = vma->vm_flags & VM_LOCKED_CLEAR_MASK; in page_table_shareable()
5207 unsigned long svm_flags = svma->vm_flags & VM_LOCKED_CLEAR_MASK; in page_table_shareable()
5215 sbase < svma->vm_start || svma->vm_end < s_end) in page_table_shareable()
5229 if (vma->vm_flags & VM_MAYSHARE && range_in_vma(vma, base, end)) in vma_shareable()
5244 if (!(vma->vm_flags & VM_MAYSHARE)) in adjust_range_if_pmd_sharing_possible()
5255 *start = max(vma->vm_start, a_start); in adjust_range_if_pmd_sharing_possible()
5256 *end = min(vma->vm_end, a_end); in adjust_range_if_pmd_sharing_possible()
5279 struct address_space *mapping = vma->vm_file->f_mapping; in huge_pmd_share() local
5280 pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) + in huge_pmd_share()
5281 vma->vm_pgoff; in huge_pmd_share()
5291 i_mmap_assert_locked(mapping); in huge_pmd_share()
5292 vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) { in huge_pmd_share()
5298 spte = huge_pte_offset(svma->vm_mm, saddr, in huge_pmd_share()
5327 * Hugetlb pte page is ref counted at the time of mapping. If pte is shared
5343 i_mmap_assert_write_locked(vma->vm_file->f_mapping); in huge_pmd_unshare()
5351 *addr = ALIGN(*addr, HPAGE_SIZE * PTRS_PER_PTE) - HPAGE_SIZE; in huge_pmd_unshare()
5405 * huge_pte_offset() - Walk the page table to resolve the hugepage
5430 /* must be pud huge, non-present or none */ in huge_pte_offset()
5437 /* must be pmd huge, non-present or none */ in huge_pte_offset()
5451 return ERR_PTR(-EINVAL); in follow_huge_addr()
5545 list_move_tail(&page->lru, list); in isolate_huge_page()
5556 list_move_tail(&page->lru, &(page_hstate(page))->hugepage_activelist); in putback_active_hugepage()
5574 * Also note that we have to transfer the per-node surplus state in move_hugetlb_state()
5576 * the per-node's. in move_hugetlb_state()
5586 if (h->surplus_huge_pages_node[old_nid]) { in move_hugetlb_state()
5587 h->surplus_huge_pages_node[old_nid]--; in move_hugetlb_state()
5588 h->surplus_huge_pages_node[new_nid]++; in move_hugetlb_state()
5634 size = min(per_node, hugetlb_cma_size - reserved); in hugetlb_cma_reserve()
5661 pr_warn("hugetlb_cma: the option isn't supported by current arch\n"); in hugetlb_cma_check()