Lines Matching refs:rmap_item
385 struct ksm_rmap_item *rmap_item; in alloc_rmap_item() local
387 rmap_item = kmem_cache_zalloc(rmap_item_cache, GFP_KERNEL | in alloc_rmap_item()
389 if (rmap_item) in alloc_rmap_item()
391 return rmap_item; in alloc_rmap_item()
394 static inline void free_rmap_item(struct ksm_rmap_item *rmap_item) in free_rmap_item() argument
397 rmap_item->mm->ksm_rmap_items--; in free_rmap_item()
398 rmap_item->mm = NULL; /* debug safety */ in free_rmap_item()
399 kmem_cache_free(rmap_item_cache, rmap_item); in free_rmap_item()
567 static void break_cow(struct ksm_rmap_item *rmap_item) in break_cow() argument
569 struct mm_struct *mm = rmap_item->mm; in break_cow()
570 unsigned long addr = rmap_item->address; in break_cow()
577 put_anon_vma(rmap_item->anon_vma); in break_cow()
586 static struct page *get_mergeable_page(struct ksm_rmap_item *rmap_item) in get_mergeable_page() argument
588 struct mm_struct *mm = rmap_item->mm; in get_mergeable_page()
589 unsigned long addr = rmap_item->address; in get_mergeable_page()
670 struct ksm_rmap_item *rmap_item; in remove_node_from_stable_tree() local
675 hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) { in remove_node_from_stable_tree()
676 if (rmap_item->hlist.next) { in remove_node_from_stable_tree()
678 trace_ksm_remove_rmap_item(stable_node->kpfn, rmap_item, rmap_item->mm); in remove_node_from_stable_tree()
683 rmap_item->mm->ksm_merging_pages--; in remove_node_from_stable_tree()
687 put_anon_vma(rmap_item->anon_vma); in remove_node_from_stable_tree()
688 rmap_item->address &= PAGE_MASK; in remove_node_from_stable_tree()
814 static void remove_rmap_item_from_tree(struct ksm_rmap_item *rmap_item) in remove_rmap_item_from_tree() argument
816 if (rmap_item->address & STABLE_FLAG) { in remove_rmap_item_from_tree()
820 stable_node = rmap_item->head; in remove_rmap_item_from_tree()
825 hlist_del(&rmap_item->hlist); in remove_rmap_item_from_tree()
834 rmap_item->mm->ksm_merging_pages--; in remove_rmap_item_from_tree()
839 put_anon_vma(rmap_item->anon_vma); in remove_rmap_item_from_tree()
840 rmap_item->head = NULL; in remove_rmap_item_from_tree()
841 rmap_item->address &= PAGE_MASK; in remove_rmap_item_from_tree()
843 } else if (rmap_item->address & UNSTABLE_FLAG) { in remove_rmap_item_from_tree()
852 age = (unsigned char)(ksm_scan.seqnr - rmap_item->address); in remove_rmap_item_from_tree()
855 rb_erase(&rmap_item->node, in remove_rmap_item_from_tree()
856 root_unstable_tree + NUMA(rmap_item->nid)); in remove_rmap_item_from_tree()
858 rmap_item->address &= PAGE_MASK; in remove_rmap_item_from_tree()
867 struct ksm_rmap_item *rmap_item = *rmap_list; in remove_trailing_rmap_items() local
868 *rmap_list = rmap_item->rmap_list; in remove_trailing_rmap_items()
869 remove_rmap_item_from_tree(rmap_item); in remove_trailing_rmap_items()
870 free_rmap_item(rmap_item); in remove_trailing_rmap_items()
1354 static int try_to_merge_with_ksm_page(struct ksm_rmap_item *rmap_item, in try_to_merge_with_ksm_page() argument
1357 struct mm_struct *mm = rmap_item->mm; in try_to_merge_with_ksm_page()
1362 vma = find_mergeable_vma(mm, rmap_item->address); in try_to_merge_with_ksm_page()
1371 remove_rmap_item_from_tree(rmap_item); in try_to_merge_with_ksm_page()
1374 rmap_item->anon_vma = vma->anon_vma; in try_to_merge_with_ksm_page()
1379 rmap_item, mm, err); in try_to_merge_with_ksm_page()
1393 static struct page *try_to_merge_two_pages(struct ksm_rmap_item *rmap_item, in try_to_merge_two_pages() argument
1400 err = try_to_merge_with_ksm_page(rmap_item, page, NULL); in try_to_merge_two_pages()
1409 break_cow(rmap_item); in try_to_merge_two_pages()
2001 struct ksm_rmap_item *unstable_tree_search_insert(struct ksm_rmap_item *rmap_item, in unstable_tree_search_insert() argument
2057 rmap_item->address |= UNSTABLE_FLAG; in unstable_tree_search_insert()
2058 rmap_item->address |= (ksm_scan.seqnr & SEQNR_MASK); in unstable_tree_search_insert()
2059 DO_NUMA(rmap_item->nid = nid); in unstable_tree_search_insert()
2060 rb_link_node(&rmap_item->node, parent, new); in unstable_tree_search_insert()
2061 rb_insert_color(&rmap_item->node, root); in unstable_tree_search_insert()
2072 static void stable_tree_append(struct ksm_rmap_item *rmap_item, in stable_tree_append() argument
2094 rmap_item->head = stable_node; in stable_tree_append()
2095 rmap_item->address |= STABLE_FLAG; in stable_tree_append()
2096 hlist_add_head(&rmap_item->hlist, &stable_node->hlist); in stable_tree_append()
2098 if (rmap_item->hlist.next) in stable_tree_append()
2103 rmap_item->mm->ksm_merging_pages++; in stable_tree_append()
2115 static void cmp_and_merge_page(struct page *page, struct ksm_rmap_item *rmap_item) in cmp_and_merge_page() argument
2117 struct mm_struct *mm = rmap_item->mm; in cmp_and_merge_page()
2136 rmap_item->head == stable_node) in cmp_and_merge_page()
2148 if (kpage == page && rmap_item->head == stable_node) { in cmp_and_merge_page()
2153 remove_rmap_item_from_tree(rmap_item); in cmp_and_merge_page()
2159 err = try_to_merge_with_ksm_page(rmap_item, page, kpage); in cmp_and_merge_page()
2166 stable_tree_append(rmap_item, page_stable_node(kpage), in cmp_and_merge_page()
2181 if (rmap_item->oldchecksum != checksum) { in cmp_and_merge_page()
2182 rmap_item->oldchecksum = checksum; in cmp_and_merge_page()
2194 vma = find_mergeable_vma(mm, rmap_item->address); in cmp_and_merge_page()
2197 ZERO_PAGE(rmap_item->address)); in cmp_and_merge_page()
2199 page_to_pfn(ZERO_PAGE(rmap_item->address)), in cmp_and_merge_page()
2200 rmap_item, mm, err); in cmp_and_merge_page()
2217 unstable_tree_search_insert(rmap_item, page, &tree_page); in cmp_and_merge_page()
2221 kpage = try_to_merge_two_pages(rmap_item, page, in cmp_and_merge_page()
2246 stable_tree_append(rmap_item, stable_node, in cmp_and_merge_page()
2259 break_cow(rmap_item); in cmp_and_merge_page()
2283 struct ksm_rmap_item *rmap_item; in get_next_rmap_item() local
2286 rmap_item = *rmap_list; in get_next_rmap_item()
2287 if ((rmap_item->address & PAGE_MASK) == addr) in get_next_rmap_item()
2288 return rmap_item; in get_next_rmap_item()
2289 if (rmap_item->address > addr) in get_next_rmap_item()
2291 *rmap_list = rmap_item->rmap_list; in get_next_rmap_item()
2292 remove_rmap_item_from_tree(rmap_item); in get_next_rmap_item()
2293 free_rmap_item(rmap_item); in get_next_rmap_item()
2296 rmap_item = alloc_rmap_item(); in get_next_rmap_item()
2297 if (rmap_item) { in get_next_rmap_item()
2299 rmap_item->mm = mm_slot->slot.mm; in get_next_rmap_item()
2300 rmap_item->mm->ksm_rmap_items++; in get_next_rmap_item()
2301 rmap_item->address = addr; in get_next_rmap_item()
2302 rmap_item->rmap_list = *rmap_list; in get_next_rmap_item()
2303 *rmap_list = rmap_item; in get_next_rmap_item()
2305 return rmap_item; in get_next_rmap_item()
2314 struct ksm_rmap_item *rmap_item; in scan_get_next_rmap_item() local
2407 rmap_item = get_next_rmap_item(mm_slot, in scan_get_next_rmap_item()
2409 if (rmap_item) { in scan_get_next_rmap_item()
2411 &rmap_item->rmap_list; in scan_get_next_rmap_item()
2416 return rmap_item; in scan_get_next_rmap_item()
2487 struct ksm_rmap_item *rmap_item; in ksm_do_scan() local
2493 rmap_item = scan_get_next_rmap_item(&page); in ksm_do_scan()
2494 if (!rmap_item) in ksm_do_scan()
2496 cmp_and_merge_page(page, rmap_item); in ksm_do_scan()
2841 struct ksm_rmap_item *rmap_item; in rmap_walk_ksm() local
2856 hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) { in rmap_walk_ksm()
2857 struct anon_vma *anon_vma = rmap_item->anon_vma; in rmap_walk_ksm()
2877 addr = rmap_item->address & PAGE_MASK; in rmap_walk_ksm()
2887 if ((rmap_item->mm == vma->vm_mm) == search_new_forks) in rmap_walk_ksm()
2916 struct ksm_rmap_item *rmap_item; in collect_procs_ksm() local
2924 hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) { in collect_procs_ksm()
2925 struct anon_vma *av = rmap_item->anon_vma; in collect_procs_ksm()
2941 addr = rmap_item->address & PAGE_MASK; in collect_procs_ksm()