123 	struct rmap_item *rmap_list;
139 	struct rmap_item **rmap_list;
195 struct rmap_item {  struct
196 	struct rmap_item *rmap_list;  argument
308 	rmap_item_cache = KSM_KMEM_CACHE(rmap_item, 0);  in ksm_slab_init()
377 static inline struct rmap_item *alloc_rmap_item(void)  in alloc_rmap_item()
379 	struct rmap_item *rmap_item;  in alloc_rmap_item()  local
381 	rmap_item = kmem_cache_zalloc(rmap_item_cache, GFP_KERNEL |  in alloc_rmap_item()
383 	if (rmap_item)  in alloc_rmap_item()
385 	return rmap_item;  in alloc_rmap_item()
388 static inline void free_rmap_item(struct rmap_item *rmap_item)  in free_rmap_item()  argument
391 	rmap_item->mm = NULL;	/* debug safety */  in free_rmap_item()
392 	kmem_cache_free(rmap_item_cache, rmap_item);  in free_rmap_item()
533 static void break_cow(struct rmap_item *rmap_item)  in break_cow()  argument
535 	struct mm_struct *mm = rmap_item->mm;  in break_cow()
536 	unsigned long addr = rmap_item->address;  in break_cow()
543 	put_anon_vma(rmap_item->anon_vma);  in break_cow()
552 static struct page *get_mergeable_page(struct rmap_item *rmap_item)  in get_mergeable_page()  argument
554 	struct mm_struct *mm = rmap_item->mm;  in get_mergeable_page()
555 	unsigned long addr = rmap_item->address;  in get_mergeable_page()
633 	struct rmap_item *rmap_item;  in remove_node_from_stable_tree()  local
638 	hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) {  in remove_node_from_stable_tree()
639 		if (rmap_item->hlist.next)  in remove_node_from_stable_tree()
645 		put_anon_vma(rmap_item->anon_vma);  in remove_node_from_stable_tree()
646 		rmap_item->address &= PAGE_MASK;  in remove_node_from_stable_tree()
773 static void remove_rmap_item_from_tree(struct rmap_item *rmap_item)  in remove_rmap_item_from_tree()  argument
775 	if (rmap_item->address & STABLE_FLAG) {  in remove_rmap_item_from_tree()
779 		stable_node = rmap_item->head;  in remove_rmap_item_from_tree()
784 		hlist_del(&rmap_item->hlist);  in remove_rmap_item_from_tree()
795 		put_anon_vma(rmap_item->anon_vma);  in remove_rmap_item_from_tree()
796 		rmap_item->address &= PAGE_MASK;  in remove_rmap_item_from_tree()
798 	} else if (rmap_item->address & UNSTABLE_FLAG) {  in remove_rmap_item_from_tree()
807 		age = (unsigned char)(ksm_scan.seqnr - rmap_item->address);  in remove_rmap_item_from_tree()
810 			rb_erase(&rmap_item->node,  in remove_rmap_item_from_tree()
811 				 root_unstable_tree + NUMA(rmap_item->nid));  in remove_rmap_item_from_tree()
813 		rmap_item->address &= PAGE_MASK;  in remove_rmap_item_from_tree()
820 				       struct rmap_item **rmap_list)  in remove_trailing_rmap_items()
823 		struct rmap_item *rmap_item = *rmap_list;  in remove_trailing_rmap_items()  local
824 		*rmap_list = rmap_item->rmap_list;  in remove_trailing_rmap_items()
825 		remove_rmap_item_from_tree(rmap_item);  in remove_trailing_rmap_items()
826 		free_rmap_item(rmap_item);  in remove_trailing_rmap_items()
1276 static int try_to_merge_with_ksm_page(struct rmap_item *rmap_item,  in try_to_merge_with_ksm_page()  argument
1279 	struct mm_struct *mm = rmap_item->mm;  in try_to_merge_with_ksm_page()
1284 	vma = find_mergeable_vma(mm, rmap_item->address);  in try_to_merge_with_ksm_page()
1293 	remove_rmap_item_from_tree(rmap_item);  in try_to_merge_with_ksm_page()
1296 	rmap_item->anon_vma = vma->anon_vma;  in try_to_merge_with_ksm_page()
1313 static struct page *try_to_merge_two_pages(struct rmap_item *rmap_item,  in try_to_merge_two_pages()  argument
1315 					   struct rmap_item *tree_rmap_item,  in try_to_merge_two_pages()
1320 	err = try_to_merge_with_ksm_page(rmap_item, page, NULL);  in try_to_merge_two_pages()
1329 			break_cow(rmap_item);  in try_to_merge_two_pages()
1923 struct rmap_item *unstable_tree_search_insert(struct rmap_item *rmap_item,  in unstable_tree_search_insert()  argument
1937 		struct rmap_item *tree_rmap_item;  in unstable_tree_search_insert()
1942 		tree_rmap_item = rb_entry(*new, struct rmap_item, node);  in unstable_tree_search_insert()
1979 	rmap_item->address |= UNSTABLE_FLAG;  in unstable_tree_search_insert()
1980 	rmap_item->address |= (ksm_scan.seqnr & SEQNR_MASK);  in unstable_tree_search_insert()
1981 	DO_NUMA(rmap_item->nid = nid);  in unstable_tree_search_insert()
1982 	rb_link_node(&rmap_item->node, parent, new);  in unstable_tree_search_insert()
1983 	rb_insert_color(&rmap_item->node, root);  in unstable_tree_search_insert()
1994 static void stable_tree_append(struct rmap_item *rmap_item,  in stable_tree_append()  argument
2016 	rmap_item->head = stable_node;  in stable_tree_append()
2017 	rmap_item->address |= STABLE_FLAG;  in stable_tree_append()
2018 	hlist_add_head(&rmap_item->hlist, &stable_node->hlist);  in stable_tree_append()
2020 	if (rmap_item->hlist.next)  in stable_tree_append()
2035 static void cmp_and_merge_page(struct page *page, struct rmap_item *rmap_item)  in cmp_and_merge_page()  argument
2037 	struct mm_struct *mm = rmap_item->mm;  in cmp_and_merge_page()
2038 	struct rmap_item *tree_rmap_item;  in cmp_and_merge_page()
2056 		    rmap_item->head == stable_node)  in cmp_and_merge_page()
2068 	if (kpage == page && rmap_item->head == stable_node) {  in cmp_and_merge_page()
2073 	remove_rmap_item_from_tree(rmap_item);  in cmp_and_merge_page()
2079 		err = try_to_merge_with_ksm_page(rmap_item, page, kpage);  in cmp_and_merge_page()
2086 			stable_tree_append(rmap_item, page_stable_node(kpage),  in cmp_and_merge_page()
2101 	if (rmap_item->oldchecksum != checksum) {  in cmp_and_merge_page()
2102 		rmap_item->oldchecksum = checksum;  in cmp_and_merge_page()
2114 		vma = find_mergeable_vma(mm, rmap_item->address);  in cmp_and_merge_page()
2116 					    ZERO_PAGE(rmap_item->address));  in cmp_and_merge_page()
2126 		unstable_tree_search_insert(rmap_item, page, &tree_page);  in cmp_and_merge_page()
2130 		kpage = try_to_merge_two_pages(rmap_item, page,  in cmp_and_merge_page()
2155 				stable_tree_append(rmap_item, stable_node,  in cmp_and_merge_page()
2168 				break_cow(rmap_item);  in cmp_and_merge_page()
2188 static struct rmap_item *get_next_rmap_item(struct mm_slot *mm_slot,  in get_next_rmap_item()
2189 					    struct rmap_item **rmap_list,  in get_next_rmap_item()
2192 	struct rmap_item *rmap_item;  in get_next_rmap_item()  local
2195 		rmap_item = *rmap_list;  in get_next_rmap_item()
2196 		if ((rmap_item->address & PAGE_MASK) == addr)  in get_next_rmap_item()
2197 			return rmap_item;  in get_next_rmap_item()
2198 		if (rmap_item->address > addr)  in get_next_rmap_item()
2200 		*rmap_list = rmap_item->rmap_list;  in get_next_rmap_item()
2201 		remove_rmap_item_from_tree(rmap_item);  in get_next_rmap_item()
2202 		free_rmap_item(rmap_item);  in get_next_rmap_item()
2205 	rmap_item = alloc_rmap_item();  in get_next_rmap_item()
2206 	if (rmap_item) {  in get_next_rmap_item()
2208 		rmap_item->mm = mm_slot->mm;  in get_next_rmap_item()
2209 		rmap_item->address = addr;  in get_next_rmap_item()
2210 		rmap_item->rmap_list = *rmap_list;  in get_next_rmap_item()
2211 		*rmap_list = rmap_item;  in get_next_rmap_item()
2213 	return rmap_item;  in get_next_rmap_item()
2216 static struct rmap_item *scan_get_next_rmap_item(struct page **page)  in scan_get_next_rmap_item()
2221 	struct rmap_item *rmap_item;  in scan_get_next_rmap_item()  local
2306 				rmap_item = get_next_rmap_item(slot,  in scan_get_next_rmap_item()
2308 				if (rmap_item) {  in scan_get_next_rmap_item()
2310 							&rmap_item->rmap_list;  in scan_get_next_rmap_item()
2315 				return rmap_item;  in scan_get_next_rmap_item()
2381 	struct rmap_item *rmap_item;  in ksm_do_scan()  local
2386 		rmap_item = scan_get_next_rmap_item(&page);  in ksm_do_scan()
2387 		if (!rmap_item)  in ksm_do_scan()
2389 		cmp_and_merge_page(page, rmap_item);  in ksm_do_scan()
2593 	struct rmap_item *rmap_item;  in rmap_walk_ksm()  local
2608 	hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) {  in rmap_walk_ksm()
2609 		struct anon_vma *anon_vma = rmap_item->anon_vma;  in rmap_walk_ksm()
2623 			addr = rmap_item->address & ~KSM_FLAG_MASK;  in rmap_walk_ksm()
2633 			if ((rmap_item->mm == vma->vm_mm) == search_new_forks)  in rmap_walk_ksm()