87 mn_itree_is_invalidating(struct mmu_notifier_subscriptions *subscriptions)  in mn_itree_is_invalidating()  argument
89 	lockdep_assert_held(&subscriptions->lock);  in mn_itree_is_invalidating()
90 	return subscriptions->invalidate_seq & 1;  in mn_itree_is_invalidating()
94 mn_itree_inv_start_range(struct mmu_notifier_subscriptions *subscriptions,  in mn_itree_inv_start_range()  argument
101 	spin_lock(&subscriptions->lock);  in mn_itree_inv_start_range()
102 	subscriptions->active_invalidate_ranges++;  in mn_itree_inv_start_range()
103 	node = interval_tree_iter_first(&subscriptions->itree, range->start,  in mn_itree_inv_start_range()
106 		subscriptions->invalidate_seq |= 1;  in mn_itree_inv_start_range()
111 	*seq = subscriptions->invalidate_seq;  in mn_itree_inv_start_range()
112 	spin_unlock(&subscriptions->lock);  in mn_itree_inv_start_range()
129 static void mn_itree_inv_end(struct mmu_notifier_subscriptions *subscriptions)  in mn_itree_inv_end()  argument
134 	spin_lock(&subscriptions->lock);  in mn_itree_inv_end()
135 	if (--subscriptions->active_invalidate_ranges ||  in mn_itree_inv_end()
136 	    !mn_itree_is_invalidating(subscriptions)) {  in mn_itree_inv_end()
137 		spin_unlock(&subscriptions->lock);  in mn_itree_inv_end()
142 	subscriptions->invalidate_seq++;  in mn_itree_inv_end()
151 				  &subscriptions->deferred_list,  in mn_itree_inv_end()
155 					     &subscriptions->itree);  in mn_itree_inv_end()
158 					     &subscriptions->itree);  in mn_itree_inv_end()
161 	spin_unlock(&subscriptions->lock);  in mn_itree_inv_end()
163 	wake_up_all(&subscriptions->wq);  in mn_itree_inv_end()
188 	struct mmu_notifier_subscriptions *subscriptions =  in mmu_interval_read_begin()  local
232 	spin_lock(&subscriptions->lock);  in mmu_interval_read_begin()
235 	is_invalidating = seq == subscriptions->invalidate_seq;  in mmu_interval_read_begin()
236 	spin_unlock(&subscriptions->lock);  in mmu_interval_read_begin()
248 		wait_event(subscriptions->wq,  in mmu_interval_read_begin()
249 			   READ_ONCE(subscriptions->invalidate_seq) != seq);  in mmu_interval_read_begin()
261 static void mn_itree_release(struct mmu_notifier_subscriptions *subscriptions,  in mn_itree_release()  argument
276 		     mn_itree_inv_start_range(subscriptions, &range, &cur_seq);  in mn_itree_release()
284 	mn_itree_inv_end(subscriptions);  in mn_itree_release()
299 static void mn_hlist_release(struct mmu_notifier_subscriptions *subscriptions,  in mn_hlist_release()  argument
310 	hlist_for_each_entry_rcu(subscription, &subscriptions->list, hlist,  in mn_hlist_release()
321 	spin_lock(&subscriptions->lock);  in mn_hlist_release()
322 	while (unlikely(!hlist_empty(&subscriptions->list))) {  in mn_hlist_release()
323 		subscription = hlist_entry(subscriptions->list.first,  in mn_hlist_release()
333 	spin_unlock(&subscriptions->lock);  in mn_hlist_release()
350 	struct mmu_notifier_subscriptions *subscriptions =  in __mmu_notifier_release()  local
353 	if (subscriptions->has_itree)  in __mmu_notifier_release()
354 		mn_itree_release(subscriptions, mm);  in __mmu_notifier_release()
356 	if (!hlist_empty(&subscriptions->list))  in __mmu_notifier_release()
357 		mn_hlist_release(subscriptions, mm);  in __mmu_notifier_release()
444 static int mn_itree_invalidate(struct mmu_notifier_subscriptions *subscriptions,  in mn_itree_invalidate()  argument
451 		     mn_itree_inv_start_range(subscriptions, range, &cur_seq);  in mn_itree_invalidate()
471 	mn_itree_inv_end(subscriptions);  in mn_itree_invalidate()
476 	struct mmu_notifier_subscriptions *subscriptions,  in mn_hlist_invalidate_range_start()  argument
484 	hlist_for_each_entry_rcu(subscription, &subscriptions->list, hlist,  in mn_hlist_invalidate_range_start()
515 	struct mmu_notifier_subscriptions *subscriptions =  in __mmu_notifier_invalidate_range_start()  local
519 	if (subscriptions->has_itree) {  in __mmu_notifier_invalidate_range_start()
520 		ret = mn_itree_invalidate(subscriptions, range);  in __mmu_notifier_invalidate_range_start()
524 	if (!hlist_empty(&subscriptions->list))  in __mmu_notifier_invalidate_range_start()
525 		return mn_hlist_invalidate_range_start(subscriptions, range);  in __mmu_notifier_invalidate_range_start()
530 mn_hlist_invalidate_end(struct mmu_notifier_subscriptions *subscriptions,  in mn_hlist_invalidate_end()  argument
537 	hlist_for_each_entry_rcu(subscription, &subscriptions->list, hlist,  in mn_hlist_invalidate_end()
572 	struct mmu_notifier_subscriptions *subscriptions =  in __mmu_notifier_invalidate_range_end()  local
576 	if (subscriptions->has_itree)  in __mmu_notifier_invalidate_range_end()
577 		mn_itree_inv_end(subscriptions);  in __mmu_notifier_invalidate_range_end()
579 	if (!hlist_empty(&subscriptions->list))  in __mmu_notifier_invalidate_range_end()
580 		mn_hlist_invalidate_end(subscriptions, range, only_end);  in __mmu_notifier_invalidate_range_end()
609 	struct mmu_notifier_subscriptions *subscriptions = NULL;  in __mmu_notifier_register()  local
628 		subscriptions = kzalloc(  in __mmu_notifier_register()
630 		if (!subscriptions)  in __mmu_notifier_register()
633 		INIT_HLIST_HEAD(&subscriptions->list);  in __mmu_notifier_register()
634 		spin_lock_init(&subscriptions->lock);  in __mmu_notifier_register()
635 		subscriptions->invalidate_seq = 2;  in __mmu_notifier_register()
636 		subscriptions->itree = RB_ROOT_CACHED;  in __mmu_notifier_register()
637 		init_waitqueue_head(&subscriptions->wq);  in __mmu_notifier_register()
638 		INIT_HLIST_HEAD(&subscriptions->deferred_list);  in __mmu_notifier_register()
661 	if (subscriptions)  in __mmu_notifier_register()
662 		smp_store_release(&mm->notifier_subscriptions, subscriptions);  in __mmu_notifier_register()
682 	kfree(subscriptions);  in __mmu_notifier_register()
899 	struct mmu_notifier_subscriptions *subscriptions, unsigned long start,  in __mmu_interval_notifier_insert()  argument
935 	spin_lock(&subscriptions->lock);  in __mmu_interval_notifier_insert()
936 	if (subscriptions->active_invalidate_ranges) {  in __mmu_interval_notifier_insert()
937 		if (mn_itree_is_invalidating(subscriptions))  in __mmu_interval_notifier_insert()
939 				       &subscriptions->deferred_list);  in __mmu_interval_notifier_insert()
941 			subscriptions->invalidate_seq |= 1;  in __mmu_interval_notifier_insert()
943 					     &subscriptions->itree);  in __mmu_interval_notifier_insert()
945 		interval_sub->invalidate_seq = subscriptions->invalidate_seq;  in __mmu_interval_notifier_insert()
947 		WARN_ON(mn_itree_is_invalidating(subscriptions));  in __mmu_interval_notifier_insert()
955 			subscriptions->invalidate_seq - 1;  in __mmu_interval_notifier_insert()
957 				     &subscriptions->itree);  in __mmu_interval_notifier_insert()
959 	spin_unlock(&subscriptions->lock);  in __mmu_interval_notifier_insert()
984 	struct mmu_notifier_subscriptions *subscriptions;  in mmu_interval_notifier_insert()  local
989 	subscriptions = smp_load_acquire(&mm->notifier_subscriptions);  in mmu_interval_notifier_insert()
990 	if (!subscriptions || !subscriptions->has_itree) {  in mmu_interval_notifier_insert()
994 		subscriptions = mm->notifier_subscriptions;  in mmu_interval_notifier_insert()
996 	return __mmu_interval_notifier_insert(interval_sub, mm, subscriptions,  in mmu_interval_notifier_insert()
1006 	struct mmu_notifier_subscriptions *subscriptions =  in mmu_interval_notifier_insert_locked()  local
1012 	if (!subscriptions || !subscriptions->has_itree) {  in mmu_interval_notifier_insert_locked()
1016 		subscriptions = mm->notifier_subscriptions;  in mmu_interval_notifier_insert_locked()
1018 	return __mmu_interval_notifier_insert(interval_sub, mm, subscriptions,  in mmu_interval_notifier_insert_locked()
1036 	struct mmu_notifier_subscriptions *subscriptions =  in mmu_interval_notifier_remove()  local
1042 	spin_lock(&subscriptions->lock);  in mmu_interval_notifier_remove()
1043 	if (mn_itree_is_invalidating(subscriptions)) {  in mmu_interval_notifier_remove()
1052 				       &subscriptions->deferred_list);  in mmu_interval_notifier_remove()
1053 			seq = subscriptions->invalidate_seq;  in mmu_interval_notifier_remove()
1058 				     &subscriptions->itree);  in mmu_interval_notifier_remove()
1060 	spin_unlock(&subscriptions->lock);  in mmu_interval_notifier_remove()
1069 		wait_event(subscriptions->wq,  in mmu_interval_notifier_remove()
1070 			   READ_ONCE(subscriptions->invalidate_seq) != seq);  in mmu_interval_notifier_remove()