90 static inline struct anon_vma *anon_vma_alloc(void)  in anon_vma_alloc()
92 	struct anon_vma *anon_vma;  in anon_vma_alloc()  local
94 	anon_vma = kmem_cache_alloc(anon_vma_cachep, GFP_KERNEL);  in anon_vma_alloc()
95 	if (anon_vma) {  in anon_vma_alloc()
96 		atomic_set(&anon_vma->refcount, 1);  in anon_vma_alloc()
97 		anon_vma->num_children = 0;  in anon_vma_alloc()
98 		anon_vma->num_active_vmas = 0;  in anon_vma_alloc()
99 		anon_vma->parent = anon_vma;  in anon_vma_alloc()
104 		anon_vma->root = anon_vma;  in anon_vma_alloc()
107 	return anon_vma;  in anon_vma_alloc()
110 static inline void anon_vma_free(struct anon_vma *anon_vma)  in anon_vma_free()  argument
112 	VM_BUG_ON(atomic_read(&anon_vma->refcount));  in anon_vma_free()
132 	if (rwsem_is_locked(&anon_vma->root->rwsem)) {  in anon_vma_free()
133 		anon_vma_lock_write(anon_vma);  in anon_vma_free()
134 		anon_vma_unlock_write(anon_vma);  in anon_vma_free()
137 	kmem_cache_free(anon_vma_cachep, anon_vma);  in anon_vma_free()
152 				struct anon_vma *anon_vma)  in anon_vma_chain_link()  argument
155 	avc->anon_vma = anon_vma;  in anon_vma_chain_link()
157 	anon_vma_interval_tree_insert(avc, &anon_vma->rb_root);  in anon_vma_chain_link()
191 	struct anon_vma *anon_vma, *allocated;  in __anon_vma_prepare()  local
200 	anon_vma = find_mergeable_anon_vma(vma);  in __anon_vma_prepare()
202 	if (!anon_vma) {  in __anon_vma_prepare()
203 		anon_vma = anon_vma_alloc();  in __anon_vma_prepare()
204 		if (unlikely(!anon_vma))  in __anon_vma_prepare()
206 		anon_vma->num_children++; /* self-parent link for new root */  in __anon_vma_prepare()
207 		allocated = anon_vma;  in __anon_vma_prepare()
210 	anon_vma_lock_write(anon_vma);  in __anon_vma_prepare()
213 	if (likely(!vma->anon_vma)) {  in __anon_vma_prepare()
214 		vma->anon_vma = anon_vma;  in __anon_vma_prepare()
215 		anon_vma_chain_link(vma, avc, anon_vma);  in __anon_vma_prepare()
216 		anon_vma->num_active_vmas++;  in __anon_vma_prepare()
221 	anon_vma_unlock_write(anon_vma);  in __anon_vma_prepare()
244 static inline struct anon_vma *lock_anon_vma_root(struct anon_vma *root, struct anon_vma *anon_vma)  in lock_anon_vma_root()  argument
246 	struct anon_vma *new_root = anon_vma->root;  in lock_anon_vma_root()
256 static inline void unlock_anon_vma_root(struct anon_vma *root)  in unlock_anon_vma_root()
284 	struct anon_vma *root = NULL;  in anon_vma_clone()
287 		struct anon_vma *anon_vma;  in anon_vma_clone()  local
297 		anon_vma = pavc->anon_vma;  in anon_vma_clone()
298 		root = lock_anon_vma_root(root, anon_vma);  in anon_vma_clone()
299 		anon_vma_chain_link(dst, avc, anon_vma);  in anon_vma_clone()
308 		if (!dst->anon_vma && src->anon_vma &&  in anon_vma_clone()
309 		    anon_vma->num_children < 2 &&  in anon_vma_clone()
310 		    anon_vma->num_active_vmas == 0)  in anon_vma_clone()
311 			dst->anon_vma = anon_vma;  in anon_vma_clone()
313 	if (dst->anon_vma)  in anon_vma_clone()
314 		dst->anon_vma->num_active_vmas++;  in anon_vma_clone()
325 	dst->anon_vma = NULL;  in anon_vma_clone()
338 	struct anon_vma *anon_vma;  in anon_vma_fork()  local
342 	if (!pvma->anon_vma)  in anon_vma_fork()
346 	vma->anon_vma = NULL;  in anon_vma_fork()
357 	if (vma->anon_vma)  in anon_vma_fork()
361 	anon_vma = anon_vma_alloc();  in anon_vma_fork()
362 	if (!anon_vma)  in anon_vma_fork()
364 	anon_vma->num_active_vmas++;  in anon_vma_fork()
373 	anon_vma->root = pvma->anon_vma->root;  in anon_vma_fork()
374 	anon_vma->parent = pvma->anon_vma;  in anon_vma_fork()
380 	get_anon_vma(anon_vma->root);  in anon_vma_fork()
382 	vma->anon_vma = anon_vma;  in anon_vma_fork()
383 	anon_vma_lock_write(anon_vma);  in anon_vma_fork()
384 	anon_vma_chain_link(vma, avc, anon_vma);  in anon_vma_fork()
385 	anon_vma->parent->num_children++;  in anon_vma_fork()
386 	anon_vma_unlock_write(anon_vma);  in anon_vma_fork()
391 	put_anon_vma(anon_vma);  in anon_vma_fork()
400 	struct anon_vma *root = NULL;  in unlink_anon_vmas()
407 		struct anon_vma *anon_vma = avc->anon_vma;  in unlink_anon_vmas()  local
409 		root = lock_anon_vma_root(root, anon_vma);  in unlink_anon_vmas()
410 		anon_vma_interval_tree_remove(avc, &anon_vma->rb_root);  in unlink_anon_vmas()
416 		if (RB_EMPTY_ROOT(&anon_vma->rb_root.rb_root)) {  in unlink_anon_vmas()
417 			anon_vma->parent->num_children--;  in unlink_anon_vmas()
424 	if (vma->anon_vma) {  in unlink_anon_vmas()
425 		vma->anon_vma->num_active_vmas--;  in unlink_anon_vmas()
431 		vma->anon_vma = NULL;  in unlink_anon_vmas()
441 		struct anon_vma *anon_vma = avc->anon_vma;  in unlink_anon_vmas()  local
443 		VM_WARN_ON(anon_vma->num_children);  in unlink_anon_vmas()
444 		VM_WARN_ON(anon_vma->num_active_vmas);  in unlink_anon_vmas()
445 		put_anon_vma(anon_vma);  in unlink_anon_vmas()
454 	struct anon_vma *anon_vma = data;  in anon_vma_ctor()  local
456 	init_rwsem(&anon_vma->rwsem);  in anon_vma_ctor()
457 	atomic_set(&anon_vma->refcount, 0);  in anon_vma_ctor()
458 	anon_vma->rb_root = RB_ROOT_CACHED;  in anon_vma_ctor()
463 	anon_vma_cachep = kmem_cache_create("anon_vma", sizeof(struct anon_vma),  in anon_vma_init()
494 struct anon_vma *folio_get_anon_vma(struct folio *folio)  in folio_get_anon_vma()
496 	struct anon_vma *anon_vma = NULL;  in folio_get_anon_vma()  local
506 	anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON);  in folio_get_anon_vma()
507 	if (!atomic_inc_not_zero(&anon_vma->refcount)) {  in folio_get_anon_vma()
508 		anon_vma = NULL;  in folio_get_anon_vma()
521 		put_anon_vma(anon_vma);  in folio_get_anon_vma()
527 	return anon_vma;  in folio_get_anon_vma()
538 struct anon_vma *folio_lock_anon_vma_read(struct folio *folio,  in folio_lock_anon_vma_read()
541 	struct anon_vma *anon_vma = NULL;  in folio_lock_anon_vma_read()  local
542 	struct anon_vma *root_anon_vma;  in folio_lock_anon_vma_read()
552 	anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON);  in folio_lock_anon_vma_read()
553 	root_anon_vma = READ_ONCE(anon_vma->root);  in folio_lock_anon_vma_read()
562 			anon_vma = NULL;  in folio_lock_anon_vma_read()
568 		anon_vma = NULL;  in folio_lock_anon_vma_read()
574 	if (!atomic_inc_not_zero(&anon_vma->refcount)) {  in folio_lock_anon_vma_read()
575 		anon_vma = NULL;  in folio_lock_anon_vma_read()
581 		put_anon_vma(anon_vma);  in folio_lock_anon_vma_read()
587 	anon_vma_lock_read(anon_vma);  in folio_lock_anon_vma_read()
589 	if (atomic_dec_and_test(&anon_vma->refcount)) {  in folio_lock_anon_vma_read()
595 		anon_vma_unlock_read(anon_vma);  in folio_lock_anon_vma_read()
596 		__put_anon_vma(anon_vma);  in folio_lock_anon_vma_read()
597 		anon_vma = NULL;  in folio_lock_anon_vma_read()
600 	return anon_vma;  in folio_lock_anon_vma_read()
604 	return anon_vma;  in folio_lock_anon_vma_read()
749 		struct anon_vma *page__anon_vma = folio_anon_vma(folio);  in page_address_in_vma()
754 		if (!vma->anon_vma || !page__anon_vma ||  in page_address_in_vma()
755 		    vma->anon_vma->root != page__anon_vma->root)  in page_address_in_vma()
1108 	void *anon_vma = vma->anon_vma;  in page_move_anon_rmap()  local
1112 	VM_BUG_ON_VMA(!anon_vma, vma);  in page_move_anon_rmap()
1114 	anon_vma += PAGE_MAPPING_ANON;  in page_move_anon_rmap()
1120 	WRITE_ONCE(folio->mapping, anon_vma);  in page_move_anon_rmap()
1135 	struct anon_vma *anon_vma = vma->anon_vma;  in __page_set_anon_rmap()  local
1137 	BUG_ON(!anon_vma);  in __page_set_anon_rmap()
1148 		anon_vma = anon_vma->root;  in __page_set_anon_rmap()
1156 	anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;  in __page_set_anon_rmap()
1157 	WRITE_ONCE(folio->mapping, (struct address_space *) anon_vma);  in __page_set_anon_rmap()
1185 	VM_BUG_ON_FOLIO(folio_anon_vma(folio)->root != vma->anon_vma->root,  in __page_check_anon_rmap()
2361 void __put_anon_vma(struct anon_vma *anon_vma)  in __put_anon_vma()  argument
2363 	struct anon_vma *root = anon_vma->root;  in __put_anon_vma()
2365 	anon_vma_free(anon_vma);  in __put_anon_vma()
2366 	if (root != anon_vma && atomic_dec_and_test(&root->refcount))  in __put_anon_vma()
2370 static struct anon_vma *rmap_walk_anon_lock(struct folio *folio,  in rmap_walk_anon_lock()
2373 	struct anon_vma *anon_vma;  in rmap_walk_anon_lock()  local
2384 	anon_vma = folio_anon_vma(folio);  in rmap_walk_anon_lock()
2385 	if (!anon_vma)  in rmap_walk_anon_lock()
2388 	if (anon_vma_trylock_read(anon_vma))  in rmap_walk_anon_lock()
2392 		anon_vma = NULL;  in rmap_walk_anon_lock()
2397 	anon_vma_lock_read(anon_vma);  in rmap_walk_anon_lock()
2399 	return anon_vma;  in rmap_walk_anon_lock()
2415 	struct anon_vma *anon_vma;  in rmap_walk_anon()  local
2420 		anon_vma = folio_anon_vma(folio);  in rmap_walk_anon()
2422 		VM_BUG_ON_FOLIO(!anon_vma, folio);  in rmap_walk_anon()
2424 		anon_vma = rmap_walk_anon_lock(folio, rwc);  in rmap_walk_anon()
2426 	if (!anon_vma)  in rmap_walk_anon()
2431 	anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root,  in rmap_walk_anon()
2449 		anon_vma_unlock_read(anon_vma);  in rmap_walk_anon()
2547 	struct anon_vma *anon_vma = vma->anon_vma;  in hugepage_add_anon_rmap()  local
2551 	BUG_ON(!anon_vma);  in hugepage_add_anon_rmap()