1935 rmap_walk_init_level(struct slot_rmap_walk_iterator *iterator, int level)  in rmap_walk_init_level()  argument
1937 	iterator->level = level;  in rmap_walk_init_level()
1938 	iterator->gfn = iterator->start_gfn;  in rmap_walk_init_level()
1939 	iterator->rmap = __gfn_to_rmap(iterator->gfn, level, iterator->slot);  in rmap_walk_init_level()
1940 	iterator->end_rmap = __gfn_to_rmap(iterator->end_gfn, level,  in rmap_walk_init_level()
1941 					   iterator->slot);  in rmap_walk_init_level()
1945 slot_rmap_walk_init(struct slot_rmap_walk_iterator *iterator,  in slot_rmap_walk_init()  argument
1949 	iterator->slot = slot;  in slot_rmap_walk_init()
1950 	iterator->start_level = start_level;  in slot_rmap_walk_init()
1951 	iterator->end_level = end_level;  in slot_rmap_walk_init()
1952 	iterator->start_gfn = start_gfn;  in slot_rmap_walk_init()
1953 	iterator->end_gfn = end_gfn;  in slot_rmap_walk_init()
1955 	rmap_walk_init_level(iterator, iterator->start_level);  in slot_rmap_walk_init()
1958 static bool slot_rmap_walk_okay(struct slot_rmap_walk_iterator *iterator)  in slot_rmap_walk_okay()  argument
1960 	return !!iterator->rmap;  in slot_rmap_walk_okay()
1963 static void slot_rmap_walk_next(struct slot_rmap_walk_iterator *iterator)  in slot_rmap_walk_next()  argument
1965 	if (++iterator->rmap <= iterator->end_rmap) {  in slot_rmap_walk_next()
1966 		iterator->gfn += (1UL << KVM_HPAGE_GFN_SHIFT(iterator->level));  in slot_rmap_walk_next()
1970 	if (++iterator->level > iterator->end_level) {  in slot_rmap_walk_next()
1971 		iterator->rmap = NULL;  in slot_rmap_walk_next()
1975 	rmap_walk_init_level(iterator, iterator->level);  in slot_rmap_walk_next()
1998 	struct slot_rmap_walk_iterator iterator;  in kvm_handle_hva_range()  local
2023 						 &iterator)  in kvm_handle_hva_range()
2024 				ret |= handler(kvm, iterator.rmap, memslot,  in kvm_handle_hva_range()
2025 					       iterator.gfn, iterator.level, data);  in kvm_handle_hva_range()
2635 static void shadow_walk_init_using_root(struct kvm_shadow_walk_iterator *iterator,  in shadow_walk_init_using_root()  argument
2639 	iterator->addr = addr;  in shadow_walk_init_using_root()
2640 	iterator->shadow_addr = root;  in shadow_walk_init_using_root()
2641 	iterator->level = vcpu->arch.mmu->shadow_root_level;  in shadow_walk_init_using_root()
2643 	if (iterator->level == PT64_ROOT_4LEVEL &&  in shadow_walk_init_using_root()
2646 		--iterator->level;  in shadow_walk_init_using_root()
2648 	if (iterator->level == PT32E_ROOT_LEVEL) {  in shadow_walk_init_using_root()
2655 		iterator->shadow_addr  in shadow_walk_init_using_root()
2657 		iterator->shadow_addr &= PT64_BASE_ADDR_MASK;  in shadow_walk_init_using_root()
2658 		--iterator->level;  in shadow_walk_init_using_root()
2659 		if (!iterator->shadow_addr)  in shadow_walk_init_using_root()
2660 			iterator->level = 0;  in shadow_walk_init_using_root()
2664 static void shadow_walk_init(struct kvm_shadow_walk_iterator *iterator,  in shadow_walk_init()  argument
2667 	shadow_walk_init_using_root(iterator, vcpu, vcpu->arch.mmu->root_hpa,  in shadow_walk_init()
2671 static bool shadow_walk_okay(struct kvm_shadow_walk_iterator *iterator)  in shadow_walk_okay()  argument
2673 	if (iterator->level < PT_PAGE_TABLE_LEVEL)  in shadow_walk_okay()
2676 	iterator->index = SHADOW_PT_INDEX(iterator->addr, iterator->level);  in shadow_walk_okay()
2677 	iterator->sptep	= ((u64 *)__va(iterator->shadow_addr)) + iterator->index;  in shadow_walk_okay()
2681 static void __shadow_walk_next(struct kvm_shadow_walk_iterator *iterator,  in __shadow_walk_next()  argument
2684 	if (is_last_spte(spte, iterator->level)) {  in __shadow_walk_next()
2685 		iterator->level = 0;  in __shadow_walk_next()
2689 	iterator->shadow_addr = spte & PT64_BASE_ADDR_MASK;  in __shadow_walk_next()
2690 	--iterator->level;  in __shadow_walk_next()
2693 static void shadow_walk_next(struct kvm_shadow_walk_iterator *iterator)  in shadow_walk_next()  argument
2695 	__shadow_walk_next(iterator, *iterator->sptep);  in shadow_walk_next()
3534 	struct kvm_shadow_walk_iterator iterator;  in fast_page_fault()  local
3551 		for_each_shadow_entry_lockless(vcpu, gva, iterator, spte)  in fast_page_fault()
3553 			    iterator.level < level)  in fast_page_fault()
3556 		sp = page_header(__pa(iterator.sptep));  in fast_page_fault()
3616 							iterator.sptep, spte,  in fast_page_fault()
3629 	trace_fast_page_fault(vcpu, gva, error_code, iterator.sptep,  in fast_page_fault()
4039 	struct kvm_shadow_walk_iterator iterator;  in walk_shadow_page_get_mmio_spte()  local
4049 	for (shadow_walk_init(&iterator, vcpu, addr),  in walk_shadow_page_get_mmio_spte()
4050 		 leaf = root = iterator.level;  in walk_shadow_page_get_mmio_spte()
4051 	     shadow_walk_okay(&iterator);  in walk_shadow_page_get_mmio_spte()
4052 	     __shadow_walk_next(&iterator, spte)) {  in walk_shadow_page_get_mmio_spte()
4053 		spte = mmu_spte_get_lockless(iterator.sptep);  in walk_shadow_page_get_mmio_spte()
4062 						    iterator.level);  in walk_shadow_page_get_mmio_spte()
4137 	struct kvm_shadow_walk_iterator iterator;  in shadow_page_table_clear_flood()  local
4144 	for_each_shadow_entry_lockless(vcpu, addr, iterator, spte) {  in shadow_page_table_clear_flood()
4145 		clear_sp_write_flooding_count(iterator.sptep);  in shadow_page_table_clear_flood()
5679 	struct slot_rmap_walk_iterator iterator;  in slot_handle_level_range()  local
5683 			end_gfn, &iterator) {  in slot_handle_level_range()
5684 		if (iterator.rmap)  in slot_handle_level_range()
5685 			flush |= fn(kvm, iterator.rmap);  in slot_handle_level_range()
5691 						iterator.gfn - start_gfn + 1);  in slot_handle_level_range()