42 void __mmu_notifier_release(struct mm_struct *mm)  in __mmu_notifier_release()  argument
52 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist)  in __mmu_notifier_release()
60 			mn->ops->release(mn, mm);  in __mmu_notifier_release()
62 	spin_lock(&mm->mmu_notifier_mm->lock);  in __mmu_notifier_release()
63 	while (unlikely(!hlist_empty(&mm->mmu_notifier_mm->list))) {  in __mmu_notifier_release()
64 		mn = hlist_entry(mm->mmu_notifier_mm->list.first,  in __mmu_notifier_release()
75 	spin_unlock(&mm->mmu_notifier_mm->lock);  in __mmu_notifier_release()
95 int __mmu_notifier_clear_flush_young(struct mm_struct *mm,  in __mmu_notifier_clear_flush_young()  argument
103 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {  in __mmu_notifier_clear_flush_young()
105 			young |= mn->ops->clear_flush_young(mn, mm, start, end);  in __mmu_notifier_clear_flush_young()
112 int __mmu_notifier_clear_young(struct mm_struct *mm,  in __mmu_notifier_clear_young()  argument
120 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {  in __mmu_notifier_clear_young()
122 			young |= mn->ops->clear_young(mn, mm, start, end);  in __mmu_notifier_clear_young()
129 int __mmu_notifier_test_young(struct mm_struct *mm,  in __mmu_notifier_test_young()  argument
136 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {  in __mmu_notifier_test_young()
138 			young = mn->ops->test_young(mn, mm, address);  in __mmu_notifier_test_young()
148 void __mmu_notifier_change_pte(struct mm_struct *mm, unsigned long address,  in __mmu_notifier_change_pte()  argument
155 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {  in __mmu_notifier_change_pte()
157 			mn->ops->change_pte(mn, mm, address, pte);  in __mmu_notifier_change_pte()
169 	hlist_for_each_entry_rcu(mn, &range->mm->mmu_notifier_mm->list, hlist) {  in __mmu_notifier_invalidate_range_start()
201 	hlist_for_each_entry_rcu(mn, &range->mm->mmu_notifier_mm->list, hlist) {  in __mmu_notifier_invalidate_range_end()
216 			mn->ops->invalidate_range(mn, range->mm,  in __mmu_notifier_invalidate_range_end()
231 void __mmu_notifier_invalidate_range(struct mm_struct *mm,  in __mmu_notifier_invalidate_range()  argument
238 	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {  in __mmu_notifier_invalidate_range()
240 			mn->ops->invalidate_range(mn, mm, start, end);  in __mmu_notifier_invalidate_range()
249 int __mmu_notifier_register(struct mmu_notifier *mn, struct mm_struct *mm)  in __mmu_notifier_register()  argument
254 	lockdep_assert_held_write(&mm->mmap_sem);  in __mmu_notifier_register()
255 	BUG_ON(atomic_read(&mm->mm_users) <= 0);  in __mmu_notifier_register()
264 	mn->mm = mm;  in __mmu_notifier_register()
267 	if (!mm->mmu_notifier_mm) {  in __mmu_notifier_register()
282 	ret = mm_take_all_locks(mm);  in __mmu_notifier_register()
287 	mmgrab(mm);  in __mmu_notifier_register()
298 		mm->mmu_notifier_mm = mmu_notifier_mm;  in __mmu_notifier_register()
300 	spin_lock(&mm->mmu_notifier_mm->lock);  in __mmu_notifier_register()
301 	hlist_add_head_rcu(&mn->hlist, &mm->mmu_notifier_mm->list);  in __mmu_notifier_register()
302 	spin_unlock(&mm->mmu_notifier_mm->lock);  in __mmu_notifier_register()
304 	mm_drop_all_locks(mm);  in __mmu_notifier_register()
305 	BUG_ON(atomic_read(&mm->mm_users) <= 0);  in __mmu_notifier_register()
333 int mmu_notifier_register(struct mmu_notifier *mn, struct mm_struct *mm)  in mmu_notifier_register()  argument
337 	down_write(&mm->mmap_sem);  in mmu_notifier_register()
338 	ret = __mmu_notifier_register(mn, mm);  in mmu_notifier_register()
339 	up_write(&mm->mmap_sem);  in mmu_notifier_register()
345 find_get_mmu_notifier(struct mm_struct *mm, const struct mmu_notifier_ops *ops)  in find_get_mmu_notifier()  argument
349 	spin_lock(&mm->mmu_notifier_mm->lock);  in find_get_mmu_notifier()
350 	hlist_for_each_entry_rcu (mn, &mm->mmu_notifier_mm->list, hlist) {  in find_get_mmu_notifier()
358 		spin_unlock(&mm->mmu_notifier_mm->lock);  in find_get_mmu_notifier()
361 	spin_unlock(&mm->mmu_notifier_mm->lock);  in find_get_mmu_notifier()
383 					     struct mm_struct *mm)  in mmu_notifier_get_locked()  argument
388 	lockdep_assert_held_write(&mm->mmap_sem);  in mmu_notifier_get_locked()
390 	if (mm->mmu_notifier_mm) {  in mmu_notifier_get_locked()
391 		mn = find_get_mmu_notifier(mm, ops);  in mmu_notifier_get_locked()
396 	mn = ops->alloc_notifier(mm);  in mmu_notifier_get_locked()
400 	ret = __mmu_notifier_register(mn, mm);  in mmu_notifier_get_locked()
411 void __mmu_notifier_mm_destroy(struct mm_struct *mm)  in __mmu_notifier_mm_destroy()  argument
413 	BUG_ON(!hlist_empty(&mm->mmu_notifier_mm->list));  in __mmu_notifier_mm_destroy()
414 	kfree(mm->mmu_notifier_mm);  in __mmu_notifier_mm_destroy()
415 	mm->mmu_notifier_mm = LIST_POISON1; /* debug */  in __mmu_notifier_mm_destroy()
428 void mmu_notifier_unregister(struct mmu_notifier *mn, struct mm_struct *mm)  in mmu_notifier_unregister()  argument
430 	BUG_ON(atomic_read(&mm->mm_count) <= 0);  in mmu_notifier_unregister()
445 			mn->ops->release(mn, mm);  in mmu_notifier_unregister()
448 		spin_lock(&mm->mmu_notifier_mm->lock);  in mmu_notifier_unregister()
454 		spin_unlock(&mm->mmu_notifier_mm->lock);  in mmu_notifier_unregister()
463 	BUG_ON(atomic_read(&mm->mm_count) <= 0);  in mmu_notifier_unregister()
465 	mmdrop(mm);  in mmu_notifier_unregister()
472 	struct mm_struct *mm = mn->mm;  in mmu_notifier_free_rcu()  local
476 	mmdrop(mm);  in mmu_notifier_free_rcu()
503 	struct mm_struct *mm = mn->mm;  in mmu_notifier_put()  local
505 	spin_lock(&mm->mmu_notifier_mm->lock);  in mmu_notifier_put()
509 	spin_unlock(&mm->mmu_notifier_mm->lock);  in mmu_notifier_put()
515 	spin_unlock(&mm->mmu_notifier_mm->lock);  in mmu_notifier_put()