78 static inline struct anon_vma *anon_vma_alloc(void)  in anon_vma_alloc()
80 	struct anon_vma *anon_vma;  in anon_vma_alloc()  local
82 	anon_vma = kmem_cache_alloc(anon_vma_cachep, GFP_KERNEL);  in anon_vma_alloc()
83 	if (anon_vma) {  in anon_vma_alloc()
84 		atomic_set(&anon_vma->refcount, 1);  in anon_vma_alloc()
85 		anon_vma->degree = 1;	/* Reference for first vma */  in anon_vma_alloc()
86 		anon_vma->parent = anon_vma;  in anon_vma_alloc()
91 		anon_vma->root = anon_vma;  in anon_vma_alloc()
94 	return anon_vma;  in anon_vma_alloc()
97 static inline void anon_vma_free(struct anon_vma *anon_vma)  in anon_vma_free()  argument
99 	VM_BUG_ON(atomic_read(&anon_vma->refcount));  in anon_vma_free()
119 	if (rwsem_is_locked(&anon_vma->root->rwsem)) {  in anon_vma_free()
120 		anon_vma_lock_write(anon_vma);  in anon_vma_free()
121 		anon_vma_unlock_write(anon_vma);  in anon_vma_free()
124 	kmem_cache_free(anon_vma_cachep, anon_vma);  in anon_vma_free()
139 				struct anon_vma *anon_vma)  in anon_vma_chain_link()  argument
142 	avc->anon_vma = anon_vma;  in anon_vma_chain_link()
144 	anon_vma_interval_tree_insert(avc, &anon_vma->rb_root);  in anon_vma_chain_link()
178 	struct anon_vma *anon_vma, *allocated;  in __anon_vma_prepare()  local
187 	anon_vma = find_mergeable_anon_vma(vma);  in __anon_vma_prepare()
189 	if (!anon_vma) {  in __anon_vma_prepare()
190 		anon_vma = anon_vma_alloc();  in __anon_vma_prepare()
191 		if (unlikely(!anon_vma))  in __anon_vma_prepare()
193 		allocated = anon_vma;  in __anon_vma_prepare()
196 	anon_vma_lock_write(anon_vma);  in __anon_vma_prepare()
199 	if (likely(!vma->anon_vma)) {  in __anon_vma_prepare()
200 		vma->anon_vma = anon_vma;  in __anon_vma_prepare()
201 		anon_vma_chain_link(vma, avc, anon_vma);  in __anon_vma_prepare()
203 		anon_vma->degree++;  in __anon_vma_prepare()
208 	anon_vma_unlock_write(anon_vma);  in __anon_vma_prepare()
231 static inline struct anon_vma *lock_anon_vma_root(struct anon_vma *root, struct anon_vma *anon_vma)  in lock_anon_vma_root()  argument
233 	struct anon_vma *new_root = anon_vma->root;  in lock_anon_vma_root()
243 static inline void unlock_anon_vma_root(struct anon_vma *root)  in unlock_anon_vma_root()
264 	struct anon_vma *root = NULL;  in anon_vma_clone()
267 		struct anon_vma *anon_vma;  in anon_vma_clone()  local
277 		anon_vma = pavc->anon_vma;  in anon_vma_clone()
278 		root = lock_anon_vma_root(root, anon_vma);  in anon_vma_clone()
279 		anon_vma_chain_link(dst, avc, anon_vma);  in anon_vma_clone()
289 		if (!dst->anon_vma && anon_vma != src->anon_vma &&  in anon_vma_clone()
290 				anon_vma->degree < 2)  in anon_vma_clone()
291 			dst->anon_vma = anon_vma;  in anon_vma_clone()
293 	if (dst->anon_vma)  in anon_vma_clone()
294 		dst->anon_vma->degree++;  in anon_vma_clone()
305 	dst->anon_vma = NULL;  in anon_vma_clone()
318 	struct anon_vma *anon_vma;  in anon_vma_fork()  local
322 	if (!pvma->anon_vma)  in anon_vma_fork()
326 	vma->anon_vma = NULL;  in anon_vma_fork()
337 	if (vma->anon_vma)  in anon_vma_fork()
341 	anon_vma = anon_vma_alloc();  in anon_vma_fork()
342 	if (!anon_vma)  in anon_vma_fork()
352 	anon_vma->root = pvma->anon_vma->root;  in anon_vma_fork()
353 	anon_vma->parent = pvma->anon_vma;  in anon_vma_fork()
359 	get_anon_vma(anon_vma->root);  in anon_vma_fork()
361 	vma->anon_vma = anon_vma;  in anon_vma_fork()
362 	anon_vma_lock_write(anon_vma);  in anon_vma_fork()
363 	anon_vma_chain_link(vma, avc, anon_vma);  in anon_vma_fork()
364 	anon_vma->parent->degree++;  in anon_vma_fork()
365 	anon_vma_unlock_write(anon_vma);  in anon_vma_fork()
370 	put_anon_vma(anon_vma);  in anon_vma_fork()
379 	struct anon_vma *root = NULL;  in unlink_anon_vmas()
386 		struct anon_vma *anon_vma = avc->anon_vma;  in unlink_anon_vmas()  local
388 		root = lock_anon_vma_root(root, anon_vma);  in unlink_anon_vmas()
389 		anon_vma_interval_tree_remove(avc, &anon_vma->rb_root);  in unlink_anon_vmas()
395 		if (RB_EMPTY_ROOT(&anon_vma->rb_root.rb_root)) {  in unlink_anon_vmas()
396 			anon_vma->parent->degree--;  in unlink_anon_vmas()
403 	if (vma->anon_vma)  in unlink_anon_vmas()
404 		vma->anon_vma->degree--;  in unlink_anon_vmas()
413 		struct anon_vma *anon_vma = avc->anon_vma;  in unlink_anon_vmas()  local
415 		VM_WARN_ON(anon_vma->degree);  in unlink_anon_vmas()
416 		put_anon_vma(anon_vma);  in unlink_anon_vmas()
425 	struct anon_vma *anon_vma = data;  in anon_vma_ctor()  local
427 	init_rwsem(&anon_vma->rwsem);  in anon_vma_ctor()
428 	atomic_set(&anon_vma->refcount, 0);  in anon_vma_ctor()
429 	anon_vma->rb_root = RB_ROOT_CACHED;  in anon_vma_ctor()
434 	anon_vma_cachep = kmem_cache_create("anon_vma", sizeof(struct anon_vma),  in anon_vma_init()
464 struct anon_vma *page_get_anon_vma(struct page *page)  in page_get_anon_vma()
466 	struct anon_vma *anon_vma = NULL;  in page_get_anon_vma()  local
476 	anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON);  in page_get_anon_vma()
477 	if (!atomic_inc_not_zero(&anon_vma->refcount)) {  in page_get_anon_vma()
478 		anon_vma = NULL;  in page_get_anon_vma()
491 		put_anon_vma(anon_vma);  in page_get_anon_vma()
497 	return anon_vma;  in page_get_anon_vma()
507 struct anon_vma *page_lock_anon_vma_read(struct page *page)  in page_lock_anon_vma_read()
509 	struct anon_vma *anon_vma = NULL;  in page_lock_anon_vma_read()  local
510 	struct anon_vma *root_anon_vma;  in page_lock_anon_vma_read()
520 	anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON);  in page_lock_anon_vma_read()
521 	root_anon_vma = READ_ONCE(anon_vma->root);  in page_lock_anon_vma_read()
530 			anon_vma = NULL;  in page_lock_anon_vma_read()
536 	if (!atomic_inc_not_zero(&anon_vma->refcount)) {  in page_lock_anon_vma_read()
537 		anon_vma = NULL;  in page_lock_anon_vma_read()
543 		put_anon_vma(anon_vma);  in page_lock_anon_vma_read()
549 	anon_vma_lock_read(anon_vma);  in page_lock_anon_vma_read()
551 	if (atomic_dec_and_test(&anon_vma->refcount)) {  in page_lock_anon_vma_read()
557 		anon_vma_unlock_read(anon_vma);  in page_lock_anon_vma_read()
558 		__put_anon_vma(anon_vma);  in page_lock_anon_vma_read()
559 		anon_vma = NULL;  in page_lock_anon_vma_read()
562 	return anon_vma;  in page_lock_anon_vma_read()
566 	return anon_vma;  in page_lock_anon_vma_read()
569 void page_unlock_anon_vma_read(struct anon_vma *anon_vma)  in page_unlock_anon_vma_read()  argument
571 	anon_vma_unlock_read(anon_vma);  in page_unlock_anon_vma_read()
691 		struct anon_vma *page__anon_vma = page_anon_vma(page);  in page_address_in_vma()
696 		if (!vma->anon_vma || !page__anon_vma ||  in page_address_in_vma()
697 		    vma->anon_vma->root != page__anon_vma->root)  in page_address_in_vma()
1002 	struct anon_vma *anon_vma = vma->anon_vma;  in page_move_anon_rmap()  local
1007 	VM_BUG_ON_VMA(!anon_vma, vma);  in page_move_anon_rmap()
1009 	anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;  in page_move_anon_rmap()
1015 	WRITE_ONCE(page->mapping, (struct address_space *) anon_vma);  in page_move_anon_rmap()
1028 	struct anon_vma *anon_vma = vma->anon_vma;  in __page_set_anon_rmap()  local
1030 	BUG_ON(!anon_vma);  in __page_set_anon_rmap()
1041 		anon_vma = anon_vma->root;  in __page_set_anon_rmap()
1043 	anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;  in __page_set_anon_rmap()
1044 	page->mapping = (struct address_space *) anon_vma;  in __page_set_anon_rmap()
1070 	BUG_ON(page_anon_vma(page)->root != vma->anon_vma->root);  in __page_check_anon_rmap()
1758 void __put_anon_vma(struct anon_vma *anon_vma)  in __put_anon_vma()  argument
1760 	struct anon_vma *root = anon_vma->root;  in __put_anon_vma()
1762 	anon_vma_free(anon_vma);  in __put_anon_vma()
1763 	if (root != anon_vma && atomic_dec_and_test(&root->refcount))  in __put_anon_vma()
1767 static struct anon_vma *rmap_walk_anon_lock(struct page *page,  in rmap_walk_anon_lock()
1770 	struct anon_vma *anon_vma;  in rmap_walk_anon_lock()  local
1781 	anon_vma = page_anon_vma(page);  in rmap_walk_anon_lock()
1782 	if (!anon_vma)  in rmap_walk_anon_lock()
1785 	anon_vma_lock_read(anon_vma);  in rmap_walk_anon_lock()
1786 	return anon_vma;  in rmap_walk_anon_lock()
1806 	struct anon_vma *anon_vma;  in rmap_walk_anon()  local
1811 		anon_vma = page_anon_vma(page);  in rmap_walk_anon()
1813 		VM_BUG_ON_PAGE(!anon_vma, page);  in rmap_walk_anon()
1815 		anon_vma = rmap_walk_anon_lock(page, rwc);  in rmap_walk_anon()
1817 	if (!anon_vma)  in rmap_walk_anon()
1822 	anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root,  in rmap_walk_anon()
1839 		anon_vma_unlock_read(anon_vma);  in rmap_walk_anon()
1927 	struct anon_vma *anon_vma = vma->anon_vma;  in __hugepage_set_anon_rmap()  local
1929 	BUG_ON(!anon_vma);  in __hugepage_set_anon_rmap()
1934 		anon_vma = anon_vma->root;  in __hugepage_set_anon_rmap()
1936 	anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;  in __hugepage_set_anon_rmap()
1937 	page->mapping = (struct address_space *) anon_vma;  in __hugepage_set_anon_rmap()
1944 	struct anon_vma *anon_vma = vma->anon_vma;  in hugepage_add_anon_rmap()  local
1948 	BUG_ON(!anon_vma);  in hugepage_add_anon_rmap()