118 static void ib_ucontext_notifier_start_account(struct ib_ucontext *context)  in ib_ucontext_notifier_start_account()  argument
120 	atomic_inc(&context->notifier_count);  in ib_ucontext_notifier_start_account()
127 static void ib_ucontext_notifier_end_account(struct ib_ucontext *context)  in ib_ucontext_notifier_end_account()  argument
129 	int zero_notifiers = atomic_dec_and_test(&context->notifier_count);  in ib_ucontext_notifier_end_account()
132 	    !list_empty(&context->no_private_counters)) {  in ib_ucontext_notifier_end_account()
139 		down_write(&context->umem_rwsem);  in ib_ucontext_notifier_end_account()
143 		if (!atomic_read(&context->notifier_count)) {  in ib_ucontext_notifier_end_account()
145 						 &context->no_private_counters,  in ib_ucontext_notifier_end_account()
155 		up_write(&context->umem_rwsem);  in ib_ucontext_notifier_end_account()
171 	item->context->invalidate_range(item, ib_umem_start(item),  in ib_umem_notifier_release_trampoline()
179 	struct ib_ucontext *context = container_of(mn, struct ib_ucontext, mn);  in ib_umem_notifier_release()  local
181 	if (!context->invalidate_range)  in ib_umem_notifier_release()
184 	ib_ucontext_notifier_start_account(context);  in ib_umem_notifier_release()
185 	down_read(&context->umem_rwsem);  in ib_umem_notifier_release()
186 	rbt_ib_umem_for_each_in_range(&context->umem_tree, 0,  in ib_umem_notifier_release()
191 	up_read(&context->umem_rwsem);  in ib_umem_notifier_release()
198 	item->context->invalidate_range(item, start, start + PAGE_SIZE);  in invalidate_page_trampoline()
207 	item->context->invalidate_range(item, start, end);  in invalidate_range_start_trampoline()
217 	struct ib_ucontext *context = container_of(mn, struct ib_ucontext, mn);  in ib_umem_notifier_invalidate_range_start()  local
220 	if (!context->invalidate_range)  in ib_umem_notifier_invalidate_range_start()
224 		down_read(&context->umem_rwsem);  in ib_umem_notifier_invalidate_range_start()
225 	else if (!down_read_trylock(&context->umem_rwsem))  in ib_umem_notifier_invalidate_range_start()
228 	ib_ucontext_notifier_start_account(context);  in ib_umem_notifier_invalidate_range_start()
229 	ret = rbt_ib_umem_for_each_in_range(&context->umem_tree, start,  in ib_umem_notifier_invalidate_range_start()
233 	up_read(&context->umem_rwsem);  in ib_umem_notifier_invalidate_range_start()
250 	struct ib_ucontext *context = container_of(mn, struct ib_ucontext, mn);  in ib_umem_notifier_invalidate_range_end()  local
252 	if (!context->invalidate_range)  in ib_umem_notifier_invalidate_range_end()
260 	down_read(&context->umem_rwsem);  in ib_umem_notifier_invalidate_range_end()
261 	rbt_ib_umem_for_each_in_range(&context->umem_tree, start,  in ib_umem_notifier_invalidate_range_end()
264 	up_read(&context->umem_rwsem);  in ib_umem_notifier_invalidate_range_end()
265 	ib_ucontext_notifier_end_account(context);  in ib_umem_notifier_invalidate_range_end()
274 struct ib_umem *ib_alloc_odp_umem(struct ib_ucontext *context,  in ib_alloc_odp_umem()  argument
287 	umem->context    = context;  in ib_alloc_odp_umem()
317 	down_write(&context->umem_rwsem);  in ib_alloc_odp_umem()
318 	context->odp_mrs_count++;  in ib_alloc_odp_umem()
319 	rbt_ib_umem_insert(&odp_data->interval_tree, &context->umem_tree);  in ib_alloc_odp_umem()
320 	if (likely(!atomic_read(&context->notifier_count)))  in ib_alloc_odp_umem()
324 			 &context->no_private_counters);  in ib_alloc_odp_umem()
325 	up_write(&context->umem_rwsem);  in ib_alloc_odp_umem()
341 int ib_umem_odp_get(struct ib_ucontext *context, struct ib_umem *umem,  in ib_umem_odp_get()  argument
374 	if (context->tgid != our_pid) {  in ib_umem_odp_get()
413 	down_write(&context->umem_rwsem);  in ib_umem_odp_get()
414 	context->odp_mrs_count++;  in ib_umem_odp_get()
417 				   &context->umem_tree);  in ib_umem_odp_get()
418 	if (likely(!atomic_read(&context->notifier_count)) ||  in ib_umem_odp_get()
419 	    context->odp_mrs_count == 1)  in ib_umem_odp_get()
423 			 &context->no_private_counters);  in ib_umem_odp_get()
424 	downgrade_write(&context->umem_rwsem);  in ib_umem_odp_get()
426 	if (context->odp_mrs_count == 1) {  in ib_umem_odp_get()
431 		atomic_set(&context->notifier_count, 0);  in ib_umem_odp_get()
432 		INIT_HLIST_NODE(&context->mn.hlist);  in ib_umem_odp_get()
433 		context->mn.ops = &ib_umem_notifiers;  in ib_umem_odp_get()
439 		ret_val = mmu_notifier_register(&context->mn, mm);  in ib_umem_odp_get()
448 	up_read(&context->umem_rwsem);  in ib_umem_odp_get()
460 	up_read(&context->umem_rwsem);  in ib_umem_odp_get()
473 	struct ib_ucontext *context = umem->context;  in ib_umem_odp_release()  local
484 	down_write(&context->umem_rwsem);  in ib_umem_odp_release()
487 				   &context->umem_tree);  in ib_umem_odp_release()
488 	context->odp_mrs_count--;  in ib_umem_odp_release()
501 	downgrade_write(&context->umem_rwsem);  in ib_umem_odp_release()
502 	if (!context->odp_mrs_count) {  in ib_umem_odp_release()
506 		owning_process = get_pid_task(context->tgid,  in ib_umem_odp_release()
522 		mmu_notifier_unregister(&context->mn, owning_mm);  in ib_umem_odp_release()
530 	up_read(&context->umem_rwsem);  in ib_umem_odp_release()
563 	struct ib_device *dev = umem->context->device;  in ib_umem_odp_map_dma_single_page()
603 	if (umem->context->invalidate_range || !stored_page)  in ib_umem_odp_map_dma_single_page()
606 	if (remove_existing_mapping && umem->context->invalidate_range) {  in ib_umem_odp_map_dma_single_page()
672 	owning_process = get_pid_task(umem->context->tgid, PIDTYPE_PID);  in ib_umem_odp_map_dma_pages()
764 	struct ib_device *dev = umem->context->device;  in ib_umem_odp_unmap_dma_pages()
799 			if (!umem->context->invalidate_range)  in ib_umem_odp_unmap_dma_pages()