302 	struct mmu_notifier *subscription;  in mn_hlist_release()  local
310 	hlist_for_each_entry_rcu(subscription, &subscriptions->list, hlist,  in mn_hlist_release()
318 		if (subscription->ops->release)  in mn_hlist_release()
319 			subscription->ops->release(subscription, mm);  in mn_hlist_release()
323 		subscription = hlist_entry(subscriptions->list.first,  in mn_hlist_release()
331 		hlist_del_init_rcu(&subscription->hlist);  in mn_hlist_release()
369 	struct mmu_notifier *subscription;  in __mmu_notifier_clear_flush_young()  local
373 	hlist_for_each_entry_rcu(subscription,  in __mmu_notifier_clear_flush_young()
376 		if (subscription->ops->clear_flush_young)  in __mmu_notifier_clear_flush_young()
377 			young |= subscription->ops->clear_flush_young(  in __mmu_notifier_clear_flush_young()
378 				subscription, mm, start, end);  in __mmu_notifier_clear_flush_young()
389 	struct mmu_notifier *subscription;  in __mmu_notifier_clear_young()  local
393 	hlist_for_each_entry_rcu(subscription,  in __mmu_notifier_clear_young()
396 		if (subscription->ops->clear_young)  in __mmu_notifier_clear_young()
397 			young |= subscription->ops->clear_young(subscription,  in __mmu_notifier_clear_young()
408 	struct mmu_notifier *subscription;  in __mmu_notifier_test_young()  local
412 	hlist_for_each_entry_rcu(subscription,  in __mmu_notifier_test_young()
415 		if (subscription->ops->test_young) {  in __mmu_notifier_test_young()
416 			young = subscription->ops->test_young(subscription, mm,  in __mmu_notifier_test_young()
430 	struct mmu_notifier *subscription;  in __mmu_notifier_change_pte()  local
434 	hlist_for_each_entry_rcu(subscription,  in __mmu_notifier_change_pte()
437 		if (subscription->ops->change_pte)  in __mmu_notifier_change_pte()
438 			subscription->ops->change_pte(subscription, mm, address,  in __mmu_notifier_change_pte()
479 	struct mmu_notifier *subscription;  in mn_hlist_invalidate_range_start()  local
484 	hlist_for_each_entry_rcu(subscription, &subscriptions->list, hlist,  in mn_hlist_invalidate_range_start()
486 		const struct mmu_notifier_ops *ops = subscription->ops;  in mn_hlist_invalidate_range_start()
493 			_ret = ops->invalidate_range_start(subscription, range);  in mn_hlist_invalidate_range_start()
533 	struct mmu_notifier *subscription;  in mn_hlist_invalidate_end()  local
537 	hlist_for_each_entry_rcu(subscription, &subscriptions->list, hlist,  in mn_hlist_invalidate_end()
552 		if (!only_end && subscription->ops->invalidate_range)  in mn_hlist_invalidate_end()
553 			subscription->ops->invalidate_range(subscription,  in mn_hlist_invalidate_end()
557 		if (subscription->ops->invalidate_range_end) {  in mn_hlist_invalidate_end()
560 			subscription->ops->invalidate_range_end(subscription,  in mn_hlist_invalidate_end()
587 	struct mmu_notifier *subscription;  in __mmu_notifier_invalidate_range()  local
591 	hlist_for_each_entry_rcu(subscription,  in __mmu_notifier_invalidate_range()
594 		if (subscription->ops->invalidate_range)  in __mmu_notifier_invalidate_range()
595 			subscription->ops->invalidate_range(subscription, mm,  in __mmu_notifier_invalidate_range()
606 int __mmu_notifier_register(struct mmu_notifier *subscription,  in __mmu_notifier_register()  argument
664 	if (subscription) {  in __mmu_notifier_register()
667 		subscription->mm = mm;  in __mmu_notifier_register()
668 		subscription->users = 1;  in __mmu_notifier_register()
671 		hlist_add_head_rcu(&subscription->hlist,  in __mmu_notifier_register()
706 int mmu_notifier_register(struct mmu_notifier *subscription,  in mmu_notifier_register()  argument
712 	ret = __mmu_notifier_register(subscription, mm);  in mmu_notifier_register()
721 	struct mmu_notifier *subscription;  in find_get_mmu_notifier()  local
724 	hlist_for_each_entry_rcu(subscription,  in find_get_mmu_notifier()
727 		if (subscription->ops != ops)  in find_get_mmu_notifier()
730 		if (likely(subscription->users != UINT_MAX))  in find_get_mmu_notifier()
731 			subscription->users++;  in find_get_mmu_notifier()
733 			subscription = ERR_PTR(-EOVERFLOW);  in find_get_mmu_notifier()
735 		return subscription;  in find_get_mmu_notifier()
761 	struct mmu_notifier *subscription;  in mmu_notifier_get_locked()  local
767 		subscription = find_get_mmu_notifier(mm, ops);  in mmu_notifier_get_locked()
768 		if (subscription)  in mmu_notifier_get_locked()
769 			return subscription;  in mmu_notifier_get_locked()
772 	subscription = ops->alloc_notifier(mm);  in mmu_notifier_get_locked()
773 	if (IS_ERR(subscription))  in mmu_notifier_get_locked()
774 		return subscription;  in mmu_notifier_get_locked()
775 	subscription->ops = ops;  in mmu_notifier_get_locked()
776 	ret = __mmu_notifier_register(subscription, mm);  in mmu_notifier_get_locked()
779 	return subscription;  in mmu_notifier_get_locked()
781 	subscription->ops->free_notifier(subscription);  in mmu_notifier_get_locked()
804 void mmu_notifier_unregister(struct mmu_notifier *subscription,  in mmu_notifier_unregister()  argument
809 	if (!hlist_unhashed(&subscription->hlist)) {  in mmu_notifier_unregister()
821 		if (subscription->ops->release)  in mmu_notifier_unregister()
822 			subscription->ops->release(subscription, mm);  in mmu_notifier_unregister()
830 		hlist_del_init_rcu(&subscription->hlist);  in mmu_notifier_unregister()
848 	struct mmu_notifier *subscription =  in mmu_notifier_free_rcu()  local
850 	struct mm_struct *mm = subscription->mm;  in mmu_notifier_free_rcu()
852 	subscription->ops->free_notifier(subscription);  in mmu_notifier_free_rcu()
879 void mmu_notifier_put(struct mmu_notifier *subscription)  in mmu_notifier_put()  argument
881 	struct mm_struct *mm = subscription->mm;  in mmu_notifier_put()
884 	if (WARN_ON(!subscription->users) || --subscription->users)  in mmu_notifier_put()
886 	hlist_del_init_rcu(&subscription->hlist);  in mmu_notifier_put()
889 	call_srcu(&srcu, &subscription->rcu, mmu_notifier_free_rcu);  in mmu_notifier_put()