59 	struct mmu_notifier *mn;  in __mmu_notifier_release()  local
67 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist)  in __mmu_notifier_release()
74 		if (mn->ops->release)  in __mmu_notifier_release()
75 			mn->ops->release(mn, mm);  in __mmu_notifier_release()
79 		mn = hlist_entry(mm->mmu_notifier_mm->list.first,  in __mmu_notifier_release()
88 		hlist_del_init_rcu(&mn->hlist);  in __mmu_notifier_release()
114 	struct mmu_notifier *mn;  in __mmu_notifier_clear_flush_young()  local
118 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {  in __mmu_notifier_clear_flush_young()
119 		if (mn->ops->clear_flush_young)  in __mmu_notifier_clear_flush_young()
120 			young |= mn->ops->clear_flush_young(mn, mm, start, end);  in __mmu_notifier_clear_flush_young()
131 	struct mmu_notifier *mn;  in __mmu_notifier_clear_young()  local
135 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {  in __mmu_notifier_clear_young()
136 		if (mn->ops->clear_young)  in __mmu_notifier_clear_young()
137 			young |= mn->ops->clear_young(mn, mm, start, end);  in __mmu_notifier_clear_young()
147 	struct mmu_notifier *mn;  in __mmu_notifier_test_young()  local
151 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {  in __mmu_notifier_test_young()
152 		if (mn->ops->test_young) {  in __mmu_notifier_test_young()
153 			young = mn->ops->test_young(mn, mm, address);  in __mmu_notifier_test_young()
166 	struct mmu_notifier *mn;  in __mmu_notifier_change_pte()  local
170 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {  in __mmu_notifier_change_pte()
171 		if (mn->ops->change_pte)  in __mmu_notifier_change_pte()
172 			mn->ops->change_pte(mn, mm, address, pte);  in __mmu_notifier_change_pte()
181 	struct mmu_notifier *mn;  in __mmu_notifier_invalidate_range_start()  local
186 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {  in __mmu_notifier_invalidate_range_start()
187 		if (mn->ops->invalidate_range_start) {  in __mmu_notifier_invalidate_range_start()
188 			int _ret = mn->ops->invalidate_range_start(mn, mm, start, end, blockable);  in __mmu_notifier_invalidate_range_start()
191 						mn->ops->invalidate_range_start, _ret,  in __mmu_notifier_invalidate_range_start()
208 	struct mmu_notifier *mn;  in __mmu_notifier_invalidate_range_end()  local
212 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {  in __mmu_notifier_invalidate_range_end()
226 		if (!only_end && mn->ops->invalidate_range)  in __mmu_notifier_invalidate_range_end()
227 			mn->ops->invalidate_range(mn, mm, start, end);  in __mmu_notifier_invalidate_range_end()
228 		if (mn->ops->invalidate_range_end)  in __mmu_notifier_invalidate_range_end()
229 			mn->ops->invalidate_range_end(mn, mm, start, end);  in __mmu_notifier_invalidate_range_end()
238 	struct mmu_notifier *mn;  in __mmu_notifier_invalidate_range()  local
242 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {  in __mmu_notifier_invalidate_range()
243 		if (mn->ops->invalidate_range)  in __mmu_notifier_invalidate_range()
244 			mn->ops->invalidate_range(mn, mm, start, end);  in __mmu_notifier_invalidate_range()
256 	struct mmu_notifier *mn;  in mm_has_blockable_invalidate_notifiers()  local
266 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {  in mm_has_blockable_invalidate_notifiers()
267 		if (!mn->ops->invalidate_range &&  in mm_has_blockable_invalidate_notifiers()
268 		    !mn->ops->invalidate_range_start &&  in mm_has_blockable_invalidate_notifiers()
269 		    !mn->ops->invalidate_range_end)  in mm_has_blockable_invalidate_notifiers()
272 		if (!(mn->ops->flags & MMU_INVALIDATE_DOES_NOT_BLOCK)) {  in mm_has_blockable_invalidate_notifiers()
281 static int do_mmu_notifier_register(struct mmu_notifier *mn,  in do_mmu_notifier_register()  argument
319 	hlist_add_head(&mn->hlist, &mm->mmu_notifier_mm->list);  in do_mmu_notifier_register()
345 int mmu_notifier_register(struct mmu_notifier *mn, struct mm_struct *mm)  in mmu_notifier_register()  argument
347 	return do_mmu_notifier_register(mn, mm, 1);  in mmu_notifier_register()
355 int __mmu_notifier_register(struct mmu_notifier *mn, struct mm_struct *mm)  in __mmu_notifier_register()  argument
357 	return do_mmu_notifier_register(mn, mm, 0);  in __mmu_notifier_register()
379 void mmu_notifier_unregister(struct mmu_notifier *mn, struct mm_struct *mm)  in mmu_notifier_unregister()  argument
383 	if (!hlist_unhashed(&mn->hlist)) {  in mmu_notifier_unregister()
395 		if (mn->ops->release)  in mmu_notifier_unregister()
396 			mn->ops->release(mn, mm);  in mmu_notifier_unregister()
404 		hlist_del_init_rcu(&mn->hlist);  in mmu_notifier_unregister()
423 void mmu_notifier_unregister_no_release(struct mmu_notifier *mn,  in mmu_notifier_unregister_no_release()  argument
431 	hlist_del_init_rcu(&mn->hlist);  in mmu_notifier_unregister_no_release()