612 static int __drm_gpuva_insert(struct drm_gpuva_manager *mgr,
626 drm_gpuva_in_mm_range(struct drm_gpuva_manager *mgr, u64 addr, u64 range)  in drm_gpuva_in_mm_range()  argument
629 	u64 mm_start = mgr->mm_start;  in drm_gpuva_in_mm_range()
630 	u64 mm_end = mm_start + mgr->mm_range;  in drm_gpuva_in_mm_range()
636 drm_gpuva_in_kernel_node(struct drm_gpuva_manager *mgr, u64 addr, u64 range)  in drm_gpuva_in_kernel_node()  argument
639 	u64 kstart = mgr->kernel_alloc_node.va.addr;  in drm_gpuva_in_kernel_node()
640 	u64 krange = mgr->kernel_alloc_node.va.range;  in drm_gpuva_in_kernel_node()
647 drm_gpuva_range_valid(struct drm_gpuva_manager *mgr,  in drm_gpuva_range_valid()  argument
651 	       drm_gpuva_in_mm_range(mgr, addr, range) &&  in drm_gpuva_range_valid()
652 	       !drm_gpuva_in_kernel_node(mgr, addr, range);  in drm_gpuva_range_valid()
671 drm_gpuva_manager_init(struct drm_gpuva_manager *mgr,  in drm_gpuva_manager_init()  argument
677 	mgr->rb.tree = RB_ROOT_CACHED;  in drm_gpuva_manager_init()
678 	INIT_LIST_HEAD(&mgr->rb.list);  in drm_gpuva_manager_init()
681 	mgr->mm_start = start_offset;  in drm_gpuva_manager_init()
682 	mgr->mm_range = range;  in drm_gpuva_manager_init()
684 	mgr->name = name ? name : "unknown";  in drm_gpuva_manager_init()
685 	mgr->ops = ops;  in drm_gpuva_manager_init()
687 	memset(&mgr->kernel_alloc_node, 0, sizeof(struct drm_gpuva));  in drm_gpuva_manager_init()
690 		mgr->kernel_alloc_node.va.addr = reserve_offset;  in drm_gpuva_manager_init()
691 		mgr->kernel_alloc_node.va.range = reserve_range;  in drm_gpuva_manager_init()
695 			__drm_gpuva_insert(mgr, &mgr->kernel_alloc_node);  in drm_gpuva_manager_init()
708 drm_gpuva_manager_destroy(struct drm_gpuva_manager *mgr)  in drm_gpuva_manager_destroy()  argument
710 	mgr->name = NULL;  in drm_gpuva_manager_destroy()
712 	if (mgr->kernel_alloc_node.va.range)  in drm_gpuva_manager_destroy()
713 		__drm_gpuva_remove(&mgr->kernel_alloc_node);  in drm_gpuva_manager_destroy()
715 	WARN(!RB_EMPTY_ROOT(&mgr->rb.tree.rb_root),  in drm_gpuva_manager_destroy()
721 __drm_gpuva_insert(struct drm_gpuva_manager *mgr,  in __drm_gpuva_insert()  argument
727 	if (drm_gpuva_it_iter_first(&mgr->rb.tree,  in __drm_gpuva_insert()
732 	va->mgr = mgr;  in __drm_gpuva_insert()
734 	drm_gpuva_it_insert(va, &mgr->rb.tree);  in __drm_gpuva_insert()
740 		head = &mgr->rb.list;  in __drm_gpuva_insert()
762 drm_gpuva_insert(struct drm_gpuva_manager *mgr,  in drm_gpuva_insert()  argument
768 	if (unlikely(!drm_gpuva_range_valid(mgr, addr, range)))  in drm_gpuva_insert()
771 	return __drm_gpuva_insert(mgr, va);  in drm_gpuva_insert()
778 	drm_gpuva_it_remove(va, &va->mgr->rb.tree);  in __drm_gpuva_remove()
795 	struct drm_gpuva_manager *mgr = va->mgr;  in drm_gpuva_remove()  local
797 	if (unlikely(va == &mgr->kernel_alloc_node)) {  in drm_gpuva_remove()
863 drm_gpuva_find_first(struct drm_gpuva_manager *mgr,  in drm_gpuva_find_first()  argument
868 	return drm_gpuva_it_iter_first(&mgr->rb.tree, addr, last);  in drm_gpuva_find_first()
881 drm_gpuva_find(struct drm_gpuva_manager *mgr,  in drm_gpuva_find()  argument
886 	va = drm_gpuva_find_first(mgr, addr, range);  in drm_gpuva_find()
914 drm_gpuva_find_prev(struct drm_gpuva_manager *mgr, u64 start)  in drm_gpuva_find_prev()  argument
916 	if (!drm_gpuva_range_valid(mgr, start - 1, 1))  in drm_gpuva_find_prev()
919 	return drm_gpuva_it_iter_first(&mgr->rb.tree, start - 1, start);  in drm_gpuva_find_prev()
936 drm_gpuva_find_next(struct drm_gpuva_manager *mgr, u64 end)  in drm_gpuva_find_next()  argument
938 	if (!drm_gpuva_range_valid(mgr, end, 1))  in drm_gpuva_find_next()
941 	return drm_gpuva_it_iter_first(&mgr->rb.tree, end, end + 1);  in drm_gpuva_find_next()
955 drm_gpuva_interval_empty(struct drm_gpuva_manager *mgr, u64 addr, u64 range)  in drm_gpuva_interval_empty()  argument
957 	return !drm_gpuva_find_first(mgr, addr, range);  in drm_gpuva_interval_empty()
971 drm_gpuva_map(struct drm_gpuva_manager *mgr,  in drm_gpuva_map()  argument
976 	drm_gpuva_insert(mgr, va);  in drm_gpuva_map()
996 	struct drm_gpuva_manager *mgr = curr->mgr;  in drm_gpuva_remap()  local
1002 		drm_gpuva_insert(mgr, prev);  in drm_gpuva_remap()
1007 		drm_gpuva_insert(mgr, next);  in drm_gpuva_remap()
1074 __drm_gpuva_sm_map(struct drm_gpuva_manager *mgr,  in __drm_gpuva_sm_map()  argument
1083 	if (unlikely(!drm_gpuva_range_valid(mgr, req_addr, req_range)))  in __drm_gpuva_sm_map()
1086 	drm_gpuva_for_each_va_range_safe(va, next, mgr, req_addr, req_end) {  in __drm_gpuva_sm_map()
1216 __drm_gpuva_sm_unmap(struct drm_gpuva_manager *mgr,  in __drm_gpuva_sm_unmap()  argument
1224 	if (unlikely(!drm_gpuva_range_valid(mgr, req_addr, req_range)))  in __drm_gpuva_sm_unmap()
1227 	drm_gpuva_for_each_va_range_safe(va, next, mgr, req_addr, req_end) {  in __drm_gpuva_sm_unmap()
1306 drm_gpuva_sm_map(struct drm_gpuva_manager *mgr, void *priv,  in drm_gpuva_sm_map()  argument
1310 	const struct drm_gpuva_fn_ops *ops = mgr->ops;  in drm_gpuva_sm_map()
1317 	return __drm_gpuva_sm_map(mgr, ops, priv,  in drm_gpuva_sm_map()
1351 drm_gpuva_sm_unmap(struct drm_gpuva_manager *mgr, void *priv,  in drm_gpuva_sm_unmap()  argument
1354 	const struct drm_gpuva_fn_ops *ops = mgr->ops;  in drm_gpuva_sm_unmap()
1360 	return __drm_gpuva_sm_unmap(mgr, ops, priv,  in drm_gpuva_sm_unmap()
1366 gpuva_op_alloc(struct drm_gpuva_manager *mgr)  in gpuva_op_alloc()  argument
1368 	const struct drm_gpuva_fn_ops *fn = mgr->ops;  in gpuva_op_alloc()
1383 gpuva_op_free(struct drm_gpuva_manager *mgr,  in gpuva_op_free()  argument
1386 	const struct drm_gpuva_fn_ops *fn = mgr->ops;  in gpuva_op_free()
1399 		struct drm_gpuva_manager *mgr;  in drm_gpuva_sm_step()  member
1402 	struct drm_gpuva_manager *mgr = args->mgr;  in drm_gpuva_sm_step()  local
1406 	op = gpuva_op_alloc(mgr);  in drm_gpuva_sm_step()
1445 	gpuva_op_free(mgr, op);  in drm_gpuva_sm_step()
1489 drm_gpuva_sm_map_ops_create(struct drm_gpuva_manager *mgr,  in drm_gpuva_sm_map_ops_create()  argument
1495 		struct drm_gpuva_manager *mgr;  in drm_gpuva_sm_map_ops_create()  member
1506 	args.mgr = mgr;  in drm_gpuva_sm_map_ops_create()
1509 	ret = __drm_gpuva_sm_map(mgr, &gpuva_list_ops, &args,  in drm_gpuva_sm_map_ops_create()
1518 	drm_gpuva_ops_free(mgr, ops);  in drm_gpuva_sm_map_ops_create()
1553 drm_gpuva_sm_unmap_ops_create(struct drm_gpuva_manager *mgr,  in drm_gpuva_sm_unmap_ops_create()  argument
1558 		struct drm_gpuva_manager *mgr;  in drm_gpuva_sm_unmap_ops_create()  member
1569 	args.mgr = mgr;  in drm_gpuva_sm_unmap_ops_create()
1572 	ret = __drm_gpuva_sm_unmap(mgr, &gpuva_list_ops, &args,  in drm_gpuva_sm_unmap_ops_create()
1580 	drm_gpuva_ops_free(mgr, ops);  in drm_gpuva_sm_unmap_ops_create()
1604 drm_gpuva_prefetch_ops_create(struct drm_gpuva_manager *mgr,  in drm_gpuva_prefetch_ops_create()  argument
1619 	drm_gpuva_for_each_va_range(va, mgr, addr, end) {  in drm_gpuva_prefetch_ops_create()
1620 		op = gpuva_op_alloc(mgr);  in drm_gpuva_prefetch_ops_create()
1634 	drm_gpuva_ops_free(mgr, ops);  in drm_gpuva_prefetch_ops_create()
1659 drm_gpuva_gem_unmap_ops_create(struct drm_gpuva_manager *mgr,  in drm_gpuva_gem_unmap_ops_create()  argument
1676 		op = gpuva_op_alloc(mgr);  in drm_gpuva_gem_unmap_ops_create()
1690 	drm_gpuva_ops_free(mgr, ops);  in drm_gpuva_gem_unmap_ops_create()
1704 drm_gpuva_ops_free(struct drm_gpuva_manager *mgr,  in drm_gpuva_ops_free()  argument
1718 		gpuva_op_free(mgr, op);  in drm_gpuva_ops_free()