124 	struct rmap_item *rmap_list;
140 	struct rmap_item **rmap_list;
196 struct rmap_item {  struct
197 	struct rmap_item *rmap_list;  argument
308 	rmap_item_cache = KSM_KMEM_CACHE(rmap_item, 0);  in ksm_slab_init()
377 static inline struct rmap_item *alloc_rmap_item(void)  in alloc_rmap_item()
379 	struct rmap_item *rmap_item;  in alloc_rmap_item()  local
381 	rmap_item = kmem_cache_zalloc(rmap_item_cache, GFP_KERNEL |  in alloc_rmap_item()
383 	if (rmap_item)  in alloc_rmap_item()
385 	return rmap_item;  in alloc_rmap_item()
388 static inline void free_rmap_item(struct rmap_item *rmap_item)  in free_rmap_item()  argument
391 	rmap_item->mm = NULL;	/* debug safety */  in free_rmap_item()
392 	kmem_cache_free(rmap_item_cache, rmap_item);  in free_rmap_item()
533 static void break_cow(struct rmap_item *rmap_item)  in break_cow()  argument
535 	struct mm_struct *mm = rmap_item->mm;  in break_cow()
536 	unsigned long addr = rmap_item->address;  in break_cow()
543 	put_anon_vma(rmap_item->anon_vma);  in break_cow()
552 static struct page *get_mergeable_page(struct rmap_item *rmap_item)  in get_mergeable_page()  argument
554 	struct mm_struct *mm = rmap_item->mm;  in get_mergeable_page()
555 	unsigned long addr = rmap_item->address;  in get_mergeable_page()
633 	struct rmap_item *rmap_item;  in remove_node_from_stable_tree()  local
638 	hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) {  in remove_node_from_stable_tree()
639 		if (rmap_item->hlist.next)  in remove_node_from_stable_tree()
645 		put_anon_vma(rmap_item->anon_vma);  in remove_node_from_stable_tree()
646 		rmap_item->address &= PAGE_MASK;  in remove_node_from_stable_tree()
758 static void remove_rmap_item_from_tree(struct rmap_item *rmap_item)  in remove_rmap_item_from_tree()  argument
760 	if (rmap_item->address & STABLE_FLAG) {  in remove_rmap_item_from_tree()
764 		stable_node = rmap_item->head;  in remove_rmap_item_from_tree()
769 		hlist_del(&rmap_item->hlist);  in remove_rmap_item_from_tree()
780 		put_anon_vma(rmap_item->anon_vma);  in remove_rmap_item_from_tree()
781 		rmap_item->address &= PAGE_MASK;  in remove_rmap_item_from_tree()
783 	} else if (rmap_item->address & UNSTABLE_FLAG) {  in remove_rmap_item_from_tree()
792 		age = (unsigned char)(ksm_scan.seqnr - rmap_item->address);  in remove_rmap_item_from_tree()
795 			rb_erase(&rmap_item->node,  in remove_rmap_item_from_tree()
796 				 root_unstable_tree + NUMA(rmap_item->nid));  in remove_rmap_item_from_tree()
798 		rmap_item->address &= PAGE_MASK;  in remove_rmap_item_from_tree()
805 				       struct rmap_item **rmap_list)  in remove_trailing_rmap_items()
808 		struct rmap_item *rmap_item = *rmap_list;  in remove_trailing_rmap_items()  local
809 		*rmap_list = rmap_item->rmap_list;  in remove_trailing_rmap_items()
810 		remove_rmap_item_from_tree(rmap_item);  in remove_trailing_rmap_items()
811 		free_rmap_item(rmap_item);  in remove_trailing_rmap_items()
1280 static int try_to_merge_with_ksm_page(struct rmap_item *rmap_item,  in try_to_merge_with_ksm_page()  argument
1283 	struct mm_struct *mm = rmap_item->mm;  in try_to_merge_with_ksm_page()
1288 	vma = find_mergeable_vma(mm, rmap_item->address);  in try_to_merge_with_ksm_page()
1297 	remove_rmap_item_from_tree(rmap_item);  in try_to_merge_with_ksm_page()
1300 	rmap_item->anon_vma = vma->anon_vma;  in try_to_merge_with_ksm_page()
1317 static struct page *try_to_merge_two_pages(struct rmap_item *rmap_item,  in try_to_merge_two_pages()  argument
1319 					   struct rmap_item *tree_rmap_item,  in try_to_merge_two_pages()
1324 	err = try_to_merge_with_ksm_page(rmap_item, page, NULL);  in try_to_merge_two_pages()
1333 			break_cow(rmap_item);  in try_to_merge_two_pages()
1920 struct rmap_item *unstable_tree_search_insert(struct rmap_item *rmap_item,  in unstable_tree_search_insert()  argument
1934 		struct rmap_item *tree_rmap_item;  in unstable_tree_search_insert()
1939 		tree_rmap_item = rb_entry(*new, struct rmap_item, node);  in unstable_tree_search_insert()
1976 	rmap_item->address |= UNSTABLE_FLAG;  in unstable_tree_search_insert()
1977 	rmap_item->address |= (ksm_scan.seqnr & SEQNR_MASK);  in unstable_tree_search_insert()
1978 	DO_NUMA(rmap_item->nid = nid);  in unstable_tree_search_insert()
1979 	rb_link_node(&rmap_item->node, parent, new);  in unstable_tree_search_insert()
1980 	rb_insert_color(&rmap_item->node, root);  in unstable_tree_search_insert()
1991 static void stable_tree_append(struct rmap_item *rmap_item,  in stable_tree_append()  argument
2013 	rmap_item->head = stable_node;  in stable_tree_append()
2014 	rmap_item->address |= STABLE_FLAG;  in stable_tree_append()
2015 	hlist_add_head(&rmap_item->hlist, &stable_node->hlist);  in stable_tree_append()
2017 	if (rmap_item->hlist.next)  in stable_tree_append()
2032 static void cmp_and_merge_page(struct page *page, struct rmap_item *rmap_item)  in cmp_and_merge_page()  argument
2034 	struct mm_struct *mm = rmap_item->mm;  in cmp_and_merge_page()
2035 	struct rmap_item *tree_rmap_item;  in cmp_and_merge_page()
2053 		    rmap_item->head == stable_node)  in cmp_and_merge_page()
2065 	if (kpage == page && rmap_item->head == stable_node) {  in cmp_and_merge_page()
2070 	remove_rmap_item_from_tree(rmap_item);  in cmp_and_merge_page()
2073 		err = try_to_merge_with_ksm_page(rmap_item, page, kpage);  in cmp_and_merge_page()
2080 			stable_tree_append(rmap_item, page_stable_node(kpage),  in cmp_and_merge_page()
2095 	if (rmap_item->oldchecksum != checksum) {  in cmp_and_merge_page()
2096 		rmap_item->oldchecksum = checksum;  in cmp_and_merge_page()
2108 		vma = find_mergeable_vma(mm, rmap_item->address);  in cmp_and_merge_page()
2110 					    ZERO_PAGE(rmap_item->address));  in cmp_and_merge_page()
2120 		unstable_tree_search_insert(rmap_item, page, &tree_page);  in cmp_and_merge_page()
2124 		kpage = try_to_merge_two_pages(rmap_item, page,  in cmp_and_merge_page()
2149 				stable_tree_append(rmap_item, stable_node,  in cmp_and_merge_page()
2162 				break_cow(rmap_item);  in cmp_and_merge_page()
2182 static struct rmap_item *get_next_rmap_item(struct mm_slot *mm_slot,  in get_next_rmap_item()
2183 					    struct rmap_item **rmap_list,  in get_next_rmap_item()
2186 	struct rmap_item *rmap_item;  in get_next_rmap_item()  local
2189 		rmap_item = *rmap_list;  in get_next_rmap_item()
2190 		if ((rmap_item->address & PAGE_MASK) == addr)  in get_next_rmap_item()
2191 			return rmap_item;  in get_next_rmap_item()
2192 		if (rmap_item->address > addr)  in get_next_rmap_item()
2194 		*rmap_list = rmap_item->rmap_list;  in get_next_rmap_item()
2195 		remove_rmap_item_from_tree(rmap_item);  in get_next_rmap_item()
2196 		free_rmap_item(rmap_item);  in get_next_rmap_item()
2199 	rmap_item = alloc_rmap_item();  in get_next_rmap_item()
2200 	if (rmap_item) {  in get_next_rmap_item()
2202 		rmap_item->mm = mm_slot->mm;  in get_next_rmap_item()
2203 		rmap_item->address = addr;  in get_next_rmap_item()
2204 		rmap_item->rmap_list = *rmap_list;  in get_next_rmap_item()
2205 		*rmap_list = rmap_item;  in get_next_rmap_item()
2207 	return rmap_item;  in get_next_rmap_item()
2210 static struct rmap_item *scan_get_next_rmap_item(struct page **page)  in scan_get_next_rmap_item()
2215 	struct rmap_item *rmap_item;  in scan_get_next_rmap_item()  local
2299 				rmap_item = get_next_rmap_item(slot,  in scan_get_next_rmap_item()
2301 				if (rmap_item) {  in scan_get_next_rmap_item()
2303 							&rmap_item->rmap_list;  in scan_get_next_rmap_item()
2308 				return rmap_item;  in scan_get_next_rmap_item()
2374 	struct rmap_item *rmap_item;  in ksm_do_scan()  local
2379 		rmap_item = scan_get_next_rmap_item(&page);  in ksm_do_scan()
2380 		if (!rmap_item)  in ksm_do_scan()
2382 		cmp_and_merge_page(page, rmap_item);  in ksm_do_scan()
2582 	struct rmap_item *rmap_item;  in rmap_walk_ksm()  local
2597 	hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) {  in rmap_walk_ksm()
2598 		struct anon_vma *anon_vma = rmap_item->anon_vma;  in rmap_walk_ksm()
2612 			addr = rmap_item->address & ~KSM_FLAG_MASK;  in rmap_walk_ksm()
2622 			if ((rmap_item->mm == vma->vm_mm) == search_new_forks)  in rmap_walk_ksm()