169  * @interval_sub: The interval subscription
173  * subscription. If the mm invokes invalidation during the critical section
194 	 * If the subscription has a different seq value under the user_lock  in mmu_interval_read_begin()
197 	 * If the subscription currently has the same seq value as the  in mmu_interval_read_begin()
302 	struct mmu_notifier *subscription;  in mn_hlist_release()  local
310 	hlist_for_each_entry_rcu(subscription, &subscriptions->list, hlist,  in mn_hlist_release()
318 		if (subscription->ops->release)  in mn_hlist_release()
319 			subscription->ops->release(subscription, mm);  in mn_hlist_release()
323 		subscription = hlist_entry(subscriptions->list.first,  in mn_hlist_release()
331 		hlist_del_init_rcu(&subscription->hlist);  in mn_hlist_release()
369 	struct mmu_notifier *subscription;  in __mmu_notifier_clear_flush_young()  local
373 	hlist_for_each_entry_rcu(subscription,  in __mmu_notifier_clear_flush_young()
376 		if (subscription->ops->clear_flush_young)  in __mmu_notifier_clear_flush_young()
377 			young |= subscription->ops->clear_flush_young(  in __mmu_notifier_clear_flush_young()
378 				subscription, mm, start, end);  in __mmu_notifier_clear_flush_young()
389 	struct mmu_notifier *subscription;  in __mmu_notifier_clear_young()  local
393 	hlist_for_each_entry_rcu(subscription,  in __mmu_notifier_clear_young()
396 		if (subscription->ops->clear_young)  in __mmu_notifier_clear_young()
397 			young |= subscription->ops->clear_young(subscription,  in __mmu_notifier_clear_young()
408 	struct mmu_notifier *subscription;  in __mmu_notifier_test_young()  local
412 	hlist_for_each_entry_rcu(subscription,  in __mmu_notifier_test_young()
415 		if (subscription->ops->test_young) {  in __mmu_notifier_test_young()
416 			young = subscription->ops->test_young(subscription, mm,  in __mmu_notifier_test_young()
430 	struct mmu_notifier *subscription;  in __mmu_notifier_change_pte()  local
434 	hlist_for_each_entry_rcu(subscription,  in __mmu_notifier_change_pte()
437 		if (subscription->ops->change_pte)  in __mmu_notifier_change_pte()
438 			subscription->ops->change_pte(subscription, mm, address,  in __mmu_notifier_change_pte()
479 	struct mmu_notifier *subscription;  in mn_hlist_invalidate_range_start()  local
484 	hlist_for_each_entry_rcu(subscription, &subscriptions->list, hlist,  in mn_hlist_invalidate_range_start()
486 		const struct mmu_notifier_ops *ops = subscription->ops;  in mn_hlist_invalidate_range_start()
493 			_ret = ops->invalidate_range_start(subscription, range);  in mn_hlist_invalidate_range_start()
522 		hlist_for_each_entry_rcu(subscription, &subscriptions->list,  in mn_hlist_invalidate_range_start()
524 			if (!subscription->ops->invalidate_range_end)  in mn_hlist_invalidate_range_start()
527 			subscription->ops->invalidate_range_end(subscription,  in mn_hlist_invalidate_range_start()
556 	struct mmu_notifier *subscription;  in mn_hlist_invalidate_end()  local
560 	hlist_for_each_entry_rcu(subscription, &subscriptions->list, hlist,  in mn_hlist_invalidate_end()
575 		if (!only_end && subscription->ops->invalidate_range)  in mn_hlist_invalidate_end()
576 			subscription->ops->invalidate_range(subscription,  in mn_hlist_invalidate_end()
580 		if (subscription->ops->invalidate_range_end) {  in mn_hlist_invalidate_end()
583 			subscription->ops->invalidate_range_end(subscription,  in mn_hlist_invalidate_end()
610 	struct mmu_notifier *subscription;  in __mmu_notifier_invalidate_range()  local
614 	hlist_for_each_entry_rcu(subscription,  in __mmu_notifier_invalidate_range()
617 		if (subscription->ops->invalidate_range)  in __mmu_notifier_invalidate_range()
618 			subscription->ops->invalidate_range(subscription, mm,  in __mmu_notifier_invalidate_range()
629 int __mmu_notifier_register(struct mmu_notifier *subscription,  in __mmu_notifier_register()  argument
680 	if (subscription) {  in __mmu_notifier_register()
683 		subscription->mm = mm;  in __mmu_notifier_register()
684 		subscription->users = 1;  in __mmu_notifier_register()
687 		hlist_add_head_rcu(&subscription->hlist,  in __mmu_notifier_register()
705  * @subscription: The notifier to attach
719  * While the caller has a mmu_notifier get the subscription->mm pointer will remain
722 int mmu_notifier_register(struct mmu_notifier *subscription,  in mmu_notifier_register()  argument
728 	ret = __mmu_notifier_register(subscription, mm);  in mmu_notifier_register()
737 	struct mmu_notifier *subscription;  in find_get_mmu_notifier()  local
740 	hlist_for_each_entry_rcu(subscription,  in find_get_mmu_notifier()
743 		if (subscription->ops != ops)  in find_get_mmu_notifier()
746 		if (likely(subscription->users != UINT_MAX))  in find_get_mmu_notifier()
747 			subscription->users++;  in find_get_mmu_notifier()
749 			subscription = ERR_PTR(-EOVERFLOW);  in find_get_mmu_notifier()
751 		return subscription;  in find_get_mmu_notifier()
777 	struct mmu_notifier *subscription;  in mmu_notifier_get_locked()  local
783 		subscription = find_get_mmu_notifier(mm, ops);  in mmu_notifier_get_locked()
784 		if (subscription)  in mmu_notifier_get_locked()
785 			return subscription;  in mmu_notifier_get_locked()
788 	subscription = ops->alloc_notifier(mm);  in mmu_notifier_get_locked()
789 	if (IS_ERR(subscription))  in mmu_notifier_get_locked()
790 		return subscription;  in mmu_notifier_get_locked()
791 	subscription->ops = ops;  in mmu_notifier_get_locked()
792 	ret = __mmu_notifier_register(subscription, mm);  in mmu_notifier_get_locked()
795 	return subscription;  in mmu_notifier_get_locked()
797 	subscription->ops->free_notifier(subscription);  in mmu_notifier_get_locked()
820 void mmu_notifier_unregister(struct mmu_notifier *subscription,  in mmu_notifier_unregister()  argument
825 	if (!hlist_unhashed(&subscription->hlist)) {  in mmu_notifier_unregister()
837 		if (subscription->ops->release)  in mmu_notifier_unregister()
838 			subscription->ops->release(subscription, mm);  in mmu_notifier_unregister()
846 		hlist_del_init_rcu(&subscription->hlist);  in mmu_notifier_unregister()
864 	struct mmu_notifier *subscription =  in mmu_notifier_free_rcu()  local
866 	struct mm_struct *mm = subscription->mm;  in mmu_notifier_free_rcu()
868 	subscription->ops->free_notifier(subscription);  in mmu_notifier_free_rcu()
875  * @subscription: The notifier to act on
895 void mmu_notifier_put(struct mmu_notifier *subscription)  in mmu_notifier_put()  argument
897 	struct mm_struct *mm = subscription->mm;  in mmu_notifier_put()
900 	if (WARN_ON(!subscription->users) || --subscription->users)  in mmu_notifier_put()
902 	hlist_del_init_rcu(&subscription->hlist);  in mmu_notifier_put()
905 	call_srcu(&srcu, &subscription->rcu, mmu_notifier_free_rcu);  in mmu_notifier_put()
965 		 * The starting seq for a subscription not under invalidation  in __mmu_interval_notifier_insert()
981  * @interval_sub: Interval subscription to register
1053  * @interval_sub: Interval subscription to unregister