189 		interval_sub->mm->notifier_subscriptions;  in mmu_interval_read_begin()
262 			     struct mm_struct *mm)  in mn_itree_release()  argument
267 		.mm = mm,  in mn_itree_release()
300 			     struct mm_struct *mm)  in mn_hlist_release()  argument
319 			subscription->ops->release(subscription, mm);  in mn_hlist_release()
348 void __mmu_notifier_release(struct mm_struct *mm)  in __mmu_notifier_release()  argument
351 		mm->notifier_subscriptions;  in __mmu_notifier_release()
354 		mn_itree_release(subscriptions, mm);  in __mmu_notifier_release()
357 		mn_hlist_release(subscriptions, mm);  in __mmu_notifier_release()
365 int __mmu_notifier_clear_flush_young(struct mm_struct *mm,  in __mmu_notifier_clear_flush_young()  argument
374 				 &mm->notifier_subscriptions->list, hlist,  in __mmu_notifier_clear_flush_young()
378 				subscription, mm, start, end);  in __mmu_notifier_clear_flush_young()
385 int __mmu_notifier_clear_young(struct mm_struct *mm,  in __mmu_notifier_clear_young()  argument
394 				 &mm->notifier_subscriptions->list, hlist,  in __mmu_notifier_clear_young()
398 								mm, start, end);  in __mmu_notifier_clear_young()
405 int __mmu_notifier_test_young(struct mm_struct *mm,  in __mmu_notifier_test_young()  argument
413 				 &mm->notifier_subscriptions->list, hlist,  in __mmu_notifier_test_young()
416 			young = subscription->ops->test_young(subscription, mm,  in __mmu_notifier_test_young()
427 void __mmu_notifier_change_pte(struct mm_struct *mm, unsigned long address,  in __mmu_notifier_change_pte()  argument
435 				 &mm->notifier_subscriptions->list, hlist,  in __mmu_notifier_change_pte()
438 			subscription->ops->change_pte(subscription, mm, address,  in __mmu_notifier_change_pte()
516 		range->mm->notifier_subscriptions;  in __mmu_notifier_invalidate_range_start()
554 							    range->mm,  in mn_hlist_invalidate_end()
573 		range->mm->notifier_subscriptions;  in __mmu_notifier_invalidate_range_end()
584 void __mmu_notifier_invalidate_range(struct mm_struct *mm,  in __mmu_notifier_invalidate_range()  argument
592 				 &mm->notifier_subscriptions->list, hlist,  in __mmu_notifier_invalidate_range()
595 			subscription->ops->invalidate_range(subscription, mm,  in __mmu_notifier_invalidate_range()
607 			    struct mm_struct *mm)  in __mmu_notifier_register()  argument
612 	mmap_assert_write_locked(mm);  in __mmu_notifier_register()
613 	BUG_ON(atomic_read(&mm->mm_users) <= 0);  in __mmu_notifier_register()
622 	if (!mm->notifier_subscriptions) {  in __mmu_notifier_register()
641 	ret = mm_take_all_locks(mm);  in __mmu_notifier_register()
662 		smp_store_release(&mm->notifier_subscriptions, subscriptions);  in __mmu_notifier_register()
666 		mmgrab(mm);  in __mmu_notifier_register()
667 		subscription->mm = mm;  in __mmu_notifier_register()
670 		spin_lock(&mm->notifier_subscriptions->lock);  in __mmu_notifier_register()
672 				   &mm->notifier_subscriptions->list);  in __mmu_notifier_register()
673 		spin_unlock(&mm->notifier_subscriptions->lock);  in __mmu_notifier_register()
675 		mm->notifier_subscriptions->has_itree = true;  in __mmu_notifier_register()
677 	mm_drop_all_locks(mm);  in __mmu_notifier_register()
678 	BUG_ON(atomic_read(&mm->mm_users) <= 0);  in __mmu_notifier_register()
707 			  struct mm_struct *mm)  in mmu_notifier_register()  argument
711 	mmap_write_lock(mm);  in mmu_notifier_register()
712 	ret = __mmu_notifier_register(subscription, mm);  in mmu_notifier_register()
713 	mmap_write_unlock(mm);  in mmu_notifier_register()
719 find_get_mmu_notifier(struct mm_struct *mm, const struct mmu_notifier_ops *ops)  in find_get_mmu_notifier()  argument
723 	spin_lock(&mm->notifier_subscriptions->lock);  in find_get_mmu_notifier()
725 				 &mm->notifier_subscriptions->list, hlist,  in find_get_mmu_notifier()
726 				 lockdep_is_held(&mm->notifier_subscriptions->lock)) {  in find_get_mmu_notifier()
734 		spin_unlock(&mm->notifier_subscriptions->lock);  in find_get_mmu_notifier()
737 	spin_unlock(&mm->notifier_subscriptions->lock);  in find_get_mmu_notifier()
759 					     struct mm_struct *mm)  in mmu_notifier_get_locked()  argument
764 	mmap_assert_write_locked(mm);  in mmu_notifier_get_locked()
766 	if (mm->notifier_subscriptions) {  in mmu_notifier_get_locked()
767 		subscription = find_get_mmu_notifier(mm, ops);  in mmu_notifier_get_locked()
772 	subscription = ops->alloc_notifier(mm);  in mmu_notifier_get_locked()
776 	ret = __mmu_notifier_register(subscription, mm);  in mmu_notifier_get_locked()
787 void __mmu_notifier_subscriptions_destroy(struct mm_struct *mm)  in __mmu_notifier_subscriptions_destroy()  argument
789 	BUG_ON(!hlist_empty(&mm->notifier_subscriptions->list));  in __mmu_notifier_subscriptions_destroy()
790 	kfree(mm->notifier_subscriptions);  in __mmu_notifier_subscriptions_destroy()
791 	mm->notifier_subscriptions = LIST_POISON1; /* debug */  in __mmu_notifier_subscriptions_destroy()
805 			     struct mm_struct *mm)  in mmu_notifier_unregister()  argument
807 	BUG_ON(atomic_read(&mm->mm_count) <= 0);  in mmu_notifier_unregister()
822 			subscription->ops->release(subscription, mm);  in mmu_notifier_unregister()
825 		spin_lock(&mm->notifier_subscriptions->lock);  in mmu_notifier_unregister()
831 		spin_unlock(&mm->notifier_subscriptions->lock);  in mmu_notifier_unregister()
840 	BUG_ON(atomic_read(&mm->mm_count) <= 0);  in mmu_notifier_unregister()
842 	mmdrop(mm);  in mmu_notifier_unregister()
850 	struct mm_struct *mm = subscription->mm;  in mmu_notifier_free_rcu()  local
854 	mmdrop(mm);  in mmu_notifier_free_rcu()
881 	struct mm_struct *mm = subscription->mm;  in mmu_notifier_put()  local
883 	spin_lock(&mm->notifier_subscriptions->lock);  in mmu_notifier_put()
887 	spin_unlock(&mm->notifier_subscriptions->lock);  in mmu_notifier_put()
893 	spin_unlock(&mm->notifier_subscriptions->lock);  in mmu_notifier_put()
898 	struct mmu_interval_notifier *interval_sub, struct mm_struct *mm,  in __mmu_interval_notifier_insert()  argument
902 	interval_sub->mm = mm;  in __mmu_interval_notifier_insert()
916 	if (WARN_ON(atomic_read(&mm->mm_users) <= 0))  in __mmu_interval_notifier_insert()
920 	mmgrab(mm);  in __mmu_interval_notifier_insert()
980 				 struct mm_struct *mm, unsigned long start,  in mmu_interval_notifier_insert()  argument
987 	might_lock(&mm->mmap_lock);  in mmu_interval_notifier_insert()
989 	subscriptions = smp_load_acquire(&mm->notifier_subscriptions);  in mmu_interval_notifier_insert()
991 		ret = mmu_notifier_register(NULL, mm);  in mmu_interval_notifier_insert()
994 		subscriptions = mm->notifier_subscriptions;  in mmu_interval_notifier_insert()
996 	return __mmu_interval_notifier_insert(interval_sub, mm, subscriptions,  in mmu_interval_notifier_insert()
1002 	struct mmu_interval_notifier *interval_sub, struct mm_struct *mm,  in mmu_interval_notifier_insert_locked()  argument
1007 		mm->notifier_subscriptions;  in mmu_interval_notifier_insert_locked()
1010 	mmap_assert_write_locked(mm);  in mmu_interval_notifier_insert_locked()
1013 		ret = __mmu_notifier_register(NULL, mm);  in mmu_interval_notifier_insert_locked()
1016 		subscriptions = mm->notifier_subscriptions;  in mmu_interval_notifier_insert_locked()
1018 	return __mmu_interval_notifier_insert(interval_sub, mm, subscriptions,  in mmu_interval_notifier_insert_locked()
1035 	struct mm_struct *mm = interval_sub->mm;  in mmu_interval_notifier_remove()  local
1037 		mm->notifier_subscriptions;  in mmu_interval_notifier_remove()
1073 	mmdrop(mm);  in mmu_interval_notifier_remove()