392 			       struct amdgpu_vm_pt_cursor *cursor)  in amdgpu_vm_pt_start()  argument
394 	cursor->pfn = start;  in amdgpu_vm_pt_start()
395 	cursor->parent = NULL;  in amdgpu_vm_pt_start()
396 	cursor->entry = &vm->root;  in amdgpu_vm_pt_start()
397 	cursor->level = adev->vm_manager.root_level;  in amdgpu_vm_pt_start()
411 				    struct amdgpu_vm_pt_cursor *cursor)  in amdgpu_vm_pt_descendant()  argument
415 	if (!cursor->entry->entries)  in amdgpu_vm_pt_descendant()
418 	BUG_ON(!cursor->entry->base.bo);  in amdgpu_vm_pt_descendant()
419 	mask = amdgpu_vm_entries_mask(adev, cursor->level);  in amdgpu_vm_pt_descendant()
420 	shift = amdgpu_vm_level_shift(adev, cursor->level);  in amdgpu_vm_pt_descendant()
422 	++cursor->level;  in amdgpu_vm_pt_descendant()
423 	idx = (cursor->pfn >> shift) & mask;  in amdgpu_vm_pt_descendant()
424 	cursor->parent = cursor->entry;  in amdgpu_vm_pt_descendant()
425 	cursor->entry = &cursor->entry->entries[idx];  in amdgpu_vm_pt_descendant()
440 				 struct amdgpu_vm_pt_cursor *cursor)  in amdgpu_vm_pt_sibling()  argument
445 	if (!cursor->parent)  in amdgpu_vm_pt_sibling()
449 	shift = amdgpu_vm_level_shift(adev, cursor->level - 1);  in amdgpu_vm_pt_sibling()
450 	num_entries = amdgpu_vm_num_entries(adev, cursor->level - 1);  in amdgpu_vm_pt_sibling()
452 	if (cursor->entry == &cursor->parent->entries[num_entries - 1])  in amdgpu_vm_pt_sibling()
455 	cursor->pfn += 1ULL << shift;  in amdgpu_vm_pt_sibling()
456 	cursor->pfn &= ~((1ULL << shift) - 1);  in amdgpu_vm_pt_sibling()
457 	++cursor->entry;  in amdgpu_vm_pt_sibling()
470 static bool amdgpu_vm_pt_ancestor(struct amdgpu_vm_pt_cursor *cursor)  in amdgpu_vm_pt_ancestor()  argument
472 	if (!cursor->parent)  in amdgpu_vm_pt_ancestor()
475 	--cursor->level;  in amdgpu_vm_pt_ancestor()
476 	cursor->entry = cursor->parent;  in amdgpu_vm_pt_ancestor()
477 	cursor->parent = amdgpu_vm_pt_parent(cursor->parent);  in amdgpu_vm_pt_ancestor()
490 			      struct amdgpu_vm_pt_cursor *cursor)  in amdgpu_vm_pt_next()  argument
493 	if (amdgpu_vm_pt_descendant(adev, cursor))  in amdgpu_vm_pt_next()
497 	while (!amdgpu_vm_pt_sibling(adev, cursor)) {  in amdgpu_vm_pt_next()
499 		if (!amdgpu_vm_pt_ancestor(cursor)) {  in amdgpu_vm_pt_next()
500 			cursor->pfn = ~0ll;  in amdgpu_vm_pt_next()
519 				   struct amdgpu_vm_pt_cursor *cursor)  in amdgpu_vm_pt_first_dfs()  argument
522 		*cursor = *start;  in amdgpu_vm_pt_first_dfs()
524 		amdgpu_vm_pt_start(adev, vm, 0, cursor);  in amdgpu_vm_pt_first_dfs()
525 	while (amdgpu_vm_pt_descendant(adev, cursor));  in amdgpu_vm_pt_first_dfs()
552 				  struct amdgpu_vm_pt_cursor *cursor)  in amdgpu_vm_pt_next_dfs()  argument
554 	if (!cursor->entry)  in amdgpu_vm_pt_next_dfs()
557 	if (!cursor->parent)  in amdgpu_vm_pt_next_dfs()
558 		cursor->entry = NULL;  in amdgpu_vm_pt_next_dfs()
559 	else if (amdgpu_vm_pt_sibling(adev, cursor))  in amdgpu_vm_pt_next_dfs()
560 		while (amdgpu_vm_pt_descendant(adev, cursor));  in amdgpu_vm_pt_next_dfs()
562 		amdgpu_vm_pt_ancestor(cursor);  in amdgpu_vm_pt_next_dfs()
568 #define for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry)		\  argument
569 	for (amdgpu_vm_pt_first_dfs((adev), (vm), (start), &(cursor)),		\
570 	     (entry) = (cursor).entry, amdgpu_vm_pt_next_dfs((adev), &(cursor));\
572 	     (entry) = (cursor).entry, amdgpu_vm_pt_next_dfs((adev), &(cursor)))
896 			       struct amdgpu_vm_pt_cursor *cursor,  in amdgpu_vm_alloc_pts()  argument
899 	struct amdgpu_vm_pt *entry = cursor->entry;  in amdgpu_vm_alloc_pts()
904 	if (cursor->level < AMDGPU_VM_PTB && !entry->entries) {  in amdgpu_vm_alloc_pts()
907 		num_entries = amdgpu_vm_num_entries(adev, cursor->level);  in amdgpu_vm_alloc_pts()
918 	amdgpu_vm_bo_param(adev, vm, cursor->level, immediate, &bp);  in amdgpu_vm_alloc_pts()
927 	pt->parent = amdgpu_bo_ref(cursor->parent->base.bo);  in amdgpu_vm_alloc_pts()
972 	struct amdgpu_vm_pt_cursor cursor;  in amdgpu_vm_free_pts()  local
977 	for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry)  in amdgpu_vm_free_pts()
1268 	struct amdgpu_vm_pt_cursor cursor;  in amdgpu_vm_invalidate_pds()  local
1271 	for_each_amdgpu_vm_pt_dfs_safe(adev, vm, NULL, cursor, entry)  in amdgpu_vm_invalidate_pds()
1436 	struct amdgpu_vm_pt_cursor cursor;  in amdgpu_vm_update_ptes()  local
1445 	amdgpu_vm_pt_start(adev, params->vm, start, &cursor);  in amdgpu_vm_update_ptes()
1446 	while (cursor.pfn < end) {  in amdgpu_vm_update_ptes()
1456 						&cursor, params->immediate);  in amdgpu_vm_update_ptes()
1461 		shift = amdgpu_vm_level_shift(adev, cursor.level);  in amdgpu_vm_update_ptes()
1462 		parent_shift = amdgpu_vm_level_shift(adev, cursor.level - 1);  in amdgpu_vm_update_ptes()
1465 			if (amdgpu_vm_pt_descendant(adev, &cursor))  in amdgpu_vm_update_ptes()
1470 			if (cursor.level != AMDGPU_VM_PTB) {  in amdgpu_vm_update_ptes()
1471 				if (!amdgpu_vm_pt_descendant(adev, &cursor))  in amdgpu_vm_update_ptes()
1480 			if (amdgpu_vm_pt_descendant(adev, &cursor))  in amdgpu_vm_update_ptes()
1486 			if (!amdgpu_vm_pt_ancestor(&cursor))  in amdgpu_vm_update_ptes()
1491 		pt = cursor.entry->base.bo;  in amdgpu_vm_update_ptes()
1500 			if (!amdgpu_vm_pt_ancestor(&cursor))  in amdgpu_vm_update_ptes()
1503 			pt = cursor.entry->base.bo;  in amdgpu_vm_update_ptes()
1511 		mask = amdgpu_vm_entries_mask(adev, cursor.level);  in amdgpu_vm_update_ptes()
1512 		pe_start = ((cursor.pfn >> shift) & mask) * 8;  in amdgpu_vm_update_ptes()
1514 		entry_end += cursor.pfn & ~(entry_end - 1);  in amdgpu_vm_update_ptes()
1532 			amdgpu_vm_update_flags(params, pt, cursor.level,  in amdgpu_vm_update_ptes()
1549 		if (amdgpu_vm_pt_descendant(adev, &cursor)) {  in amdgpu_vm_update_ptes()
1556 			while (cursor.pfn < frag_start) {  in amdgpu_vm_update_ptes()
1557 				amdgpu_vm_free_pts(adev, params->vm, &cursor);  in amdgpu_vm_update_ptes()
1558 				amdgpu_vm_pt_next(adev, &cursor);  in amdgpu_vm_update_ptes()
1563 			amdgpu_vm_pt_next(adev, &cursor);  in amdgpu_vm_update_ptes()