57 void __mmu_notifier_release(struct mm_struct *mm)  in __mmu_notifier_release()  argument
67 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist)  in __mmu_notifier_release()
75 			mn->ops->release(mn, mm);  in __mmu_notifier_release()
77 	spin_lock(&mm->mmu_notifier_mm->lock);  in __mmu_notifier_release()
78 	while (unlikely(!hlist_empty(&mm->mmu_notifier_mm->list))) {  in __mmu_notifier_release()
79 		mn = hlist_entry(mm->mmu_notifier_mm->list.first,  in __mmu_notifier_release()
90 	spin_unlock(&mm->mmu_notifier_mm->lock);  in __mmu_notifier_release()
110 int __mmu_notifier_clear_flush_young(struct mm_struct *mm,  in __mmu_notifier_clear_flush_young()  argument
118 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {  in __mmu_notifier_clear_flush_young()
120 			young |= mn->ops->clear_flush_young(mn, mm, start, end);  in __mmu_notifier_clear_flush_young()
127 int __mmu_notifier_clear_young(struct mm_struct *mm,  in __mmu_notifier_clear_young()  argument
135 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {  in __mmu_notifier_clear_young()
137 			young |= mn->ops->clear_young(mn, mm, start, end);  in __mmu_notifier_clear_young()
144 int __mmu_notifier_test_young(struct mm_struct *mm,  in __mmu_notifier_test_young()  argument
151 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {  in __mmu_notifier_test_young()
153 			young = mn->ops->test_young(mn, mm, address);  in __mmu_notifier_test_young()
163 void __mmu_notifier_change_pte(struct mm_struct *mm, unsigned long address,  in __mmu_notifier_change_pte()  argument
170 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {  in __mmu_notifier_change_pte()
172 			mn->ops->change_pte(mn, mm, address, pte);  in __mmu_notifier_change_pte()
177 int __mmu_notifier_invalidate_range_start(struct mm_struct *mm,  in __mmu_notifier_invalidate_range_start()  argument
186 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {  in __mmu_notifier_invalidate_range_start()
188 			int _ret = mn->ops->invalidate_range_start(mn, mm, start, end, blockable);  in __mmu_notifier_invalidate_range_start()
203 void __mmu_notifier_invalidate_range_end(struct mm_struct *mm,  in __mmu_notifier_invalidate_range_end()  argument
212 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {  in __mmu_notifier_invalidate_range_end()
227 			mn->ops->invalidate_range(mn, mm, start, end);  in __mmu_notifier_invalidate_range_end()
229 			mn->ops->invalidate_range_end(mn, mm, start, end);  in __mmu_notifier_invalidate_range_end()
235 void __mmu_notifier_invalidate_range(struct mm_struct *mm,  in __mmu_notifier_invalidate_range()  argument
242 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {  in __mmu_notifier_invalidate_range()
244 			mn->ops->invalidate_range(mn, mm, start, end);  in __mmu_notifier_invalidate_range()
254 bool mm_has_blockable_invalidate_notifiers(struct mm_struct *mm)  in mm_has_blockable_invalidate_notifiers()  argument
260 	WARN_ON_ONCE(!rwsem_is_locked(&mm->mmap_sem));  in mm_has_blockable_invalidate_notifiers()
262 	if (!mm_has_notifiers(mm))  in mm_has_blockable_invalidate_notifiers()
266 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {  in mm_has_blockable_invalidate_notifiers()
282 				    struct mm_struct *mm,  in do_mmu_notifier_register()  argument
288 	BUG_ON(atomic_read(&mm->mm_users) <= 0);  in do_mmu_notifier_register()
296 		down_write(&mm->mmap_sem);  in do_mmu_notifier_register()
297 	ret = mm_take_all_locks(mm);  in do_mmu_notifier_register()
301 	if (!mm_has_notifiers(mm)) {  in do_mmu_notifier_register()
305 		mm->mmu_notifier_mm = mmu_notifier_mm;  in do_mmu_notifier_register()
308 	mmgrab(mm);  in do_mmu_notifier_register()
318 	spin_lock(&mm->mmu_notifier_mm->lock);  in do_mmu_notifier_register()
319 	hlist_add_head(&mn->hlist, &mm->mmu_notifier_mm->list);  in do_mmu_notifier_register()
320 	spin_unlock(&mm->mmu_notifier_mm->lock);  in do_mmu_notifier_register()
322 	mm_drop_all_locks(mm);  in do_mmu_notifier_register()
325 		up_write(&mm->mmap_sem);  in do_mmu_notifier_register()
328 	BUG_ON(atomic_read(&mm->mm_users) <= 0);  in do_mmu_notifier_register()
345 int mmu_notifier_register(struct mmu_notifier *mn, struct mm_struct *mm)  in mmu_notifier_register()  argument
347 	return do_mmu_notifier_register(mn, mm, 1);  in mmu_notifier_register()
355 int __mmu_notifier_register(struct mmu_notifier *mn, struct mm_struct *mm)  in __mmu_notifier_register()  argument
357 	return do_mmu_notifier_register(mn, mm, 0);  in __mmu_notifier_register()
362 void __mmu_notifier_mm_destroy(struct mm_struct *mm)  in __mmu_notifier_mm_destroy()  argument
364 	BUG_ON(!hlist_empty(&mm->mmu_notifier_mm->list));  in __mmu_notifier_mm_destroy()
365 	kfree(mm->mmu_notifier_mm);  in __mmu_notifier_mm_destroy()
366 	mm->mmu_notifier_mm = LIST_POISON1; /* debug */  in __mmu_notifier_mm_destroy()
379 void mmu_notifier_unregister(struct mmu_notifier *mn, struct mm_struct *mm)  in mmu_notifier_unregister()  argument
381 	BUG_ON(atomic_read(&mm->mm_count) <= 0);  in mmu_notifier_unregister()
396 			mn->ops->release(mn, mm);  in mmu_notifier_unregister()
399 		spin_lock(&mm->mmu_notifier_mm->lock);  in mmu_notifier_unregister()
405 		spin_unlock(&mm->mmu_notifier_mm->lock);  in mmu_notifier_unregister()
414 	BUG_ON(atomic_read(&mm->mm_count) <= 0);  in mmu_notifier_unregister()
416 	mmdrop(mm);  in mmu_notifier_unregister()
424 					struct mm_struct *mm)  in mmu_notifier_unregister_no_release()  argument
426 	spin_lock(&mm->mmu_notifier_mm->lock);  in mmu_notifier_unregister_no_release()
432 	spin_unlock(&mm->mmu_notifier_mm->lock);  in mmu_notifier_unregister_no_release()
434 	BUG_ON(atomic_read(&mm->mm_count) <= 0);  in mmu_notifier_unregister_no_release()
435 	mmdrop(mm);  in mmu_notifier_unregister_no_release()