1764 rmap_walk_init_level(struct slot_rmap_walk_iterator *iterator, int level)  in rmap_walk_init_level()  argument
1766 	iterator->level = level;  in rmap_walk_init_level()
1767 	iterator->gfn = iterator->start_gfn;  in rmap_walk_init_level()
1768 	iterator->rmap = __gfn_to_rmap(iterator->gfn, level, iterator->slot);  in rmap_walk_init_level()
1769 	iterator->end_rmap = __gfn_to_rmap(iterator->end_gfn, level,  in rmap_walk_init_level()
1770 					   iterator->slot);  in rmap_walk_init_level()
1774 slot_rmap_walk_init(struct slot_rmap_walk_iterator *iterator,  in slot_rmap_walk_init()  argument
1778 	iterator->slot = slot;  in slot_rmap_walk_init()
1779 	iterator->start_level = start_level;  in slot_rmap_walk_init()
1780 	iterator->end_level = end_level;  in slot_rmap_walk_init()
1781 	iterator->start_gfn = start_gfn;  in slot_rmap_walk_init()
1782 	iterator->end_gfn = end_gfn;  in slot_rmap_walk_init()
1784 	rmap_walk_init_level(iterator, iterator->start_level);  in slot_rmap_walk_init()
1787 static bool slot_rmap_walk_okay(struct slot_rmap_walk_iterator *iterator)  in slot_rmap_walk_okay()  argument
1789 	return !!iterator->rmap;  in slot_rmap_walk_okay()
1792 static void slot_rmap_walk_next(struct slot_rmap_walk_iterator *iterator)  in slot_rmap_walk_next()  argument
1794 	if (++iterator->rmap <= iterator->end_rmap) {  in slot_rmap_walk_next()
1795 		iterator->gfn += (1UL << KVM_HPAGE_GFN_SHIFT(iterator->level));  in slot_rmap_walk_next()
1799 	if (++iterator->level > iterator->end_level) {  in slot_rmap_walk_next()
1800 		iterator->rmap = NULL;  in slot_rmap_walk_next()
1804 	rmap_walk_init_level(iterator, iterator->level);  in slot_rmap_walk_next()
1827 	struct slot_rmap_walk_iterator iterator;  in kvm_handle_hva_range()  local
1852 						 &iterator)  in kvm_handle_hva_range()
1853 				ret |= handler(kvm, iterator.rmap, memslot,  in kvm_handle_hva_range()
1854 					       iterator.gfn, iterator.level, data);  in kvm_handle_hva_range()
2454 static void shadow_walk_init_using_root(struct kvm_shadow_walk_iterator *iterator,  in shadow_walk_init_using_root()  argument
2458 	iterator->addr = addr;  in shadow_walk_init_using_root()
2459 	iterator->shadow_addr = root;  in shadow_walk_init_using_root()
2460 	iterator->level = vcpu->arch.mmu.shadow_root_level;  in shadow_walk_init_using_root()
2462 	if (iterator->level == PT64_ROOT_4LEVEL &&  in shadow_walk_init_using_root()
2465 		--iterator->level;  in shadow_walk_init_using_root()
2467 	if (iterator->level == PT32E_ROOT_LEVEL) {  in shadow_walk_init_using_root()
2474 		iterator->shadow_addr  in shadow_walk_init_using_root()
2476 		iterator->shadow_addr &= PT64_BASE_ADDR_MASK;  in shadow_walk_init_using_root()
2477 		--iterator->level;  in shadow_walk_init_using_root()
2478 		if (!iterator->shadow_addr)  in shadow_walk_init_using_root()
2479 			iterator->level = 0;  in shadow_walk_init_using_root()
2483 static void shadow_walk_init(struct kvm_shadow_walk_iterator *iterator,  in shadow_walk_init()  argument
2486 	shadow_walk_init_using_root(iterator, vcpu, vcpu->arch.mmu.root_hpa,  in shadow_walk_init()
2490 static bool shadow_walk_okay(struct kvm_shadow_walk_iterator *iterator)  in shadow_walk_okay()  argument
2492 	if (iterator->level < PT_PAGE_TABLE_LEVEL)  in shadow_walk_okay()
2495 	iterator->index = SHADOW_PT_INDEX(iterator->addr, iterator->level);  in shadow_walk_okay()
2496 	iterator->sptep	= ((u64 *)__va(iterator->shadow_addr)) + iterator->index;  in shadow_walk_okay()
2500 static void __shadow_walk_next(struct kvm_shadow_walk_iterator *iterator,  in __shadow_walk_next()  argument
2503 	if (is_last_spte(spte, iterator->level)) {  in __shadow_walk_next()
2504 		iterator->level = 0;  in __shadow_walk_next()
2508 	iterator->shadow_addr = spte & PT64_BASE_ADDR_MASK;  in __shadow_walk_next()
2509 	--iterator->level;  in __shadow_walk_next()
2512 static void shadow_walk_next(struct kvm_shadow_walk_iterator *iterator)  in shadow_walk_next()  argument
2514 	__shadow_walk_next(iterator, *iterator->sptep);  in shadow_walk_next()
3093 	struct kvm_shadow_walk_iterator iterator;  in __direct_map()  local
3101 	for_each_shadow_entry(vcpu, (u64)gfn << PAGE_SHIFT, iterator) {  in __direct_map()
3102 		if (iterator.level == level) {  in __direct_map()
3103 			emulate = mmu_set_spte(vcpu, iterator.sptep, ACC_ALL,  in __direct_map()
3106 			direct_pte_prefetch(vcpu, iterator.sptep);  in __direct_map()
3111 		drop_large_spte(vcpu, iterator.sptep);  in __direct_map()
3112 		if (!is_shadow_present_pte(*iterator.sptep)) {  in __direct_map()
3113 			u64 base_addr = iterator.addr;  in __direct_map()
3115 			base_addr &= PT64_LVL_ADDR_MASK(iterator.level);  in __direct_map()
3117 			sp = kvm_mmu_get_page(vcpu, pseudo_gfn, iterator.addr,  in __direct_map()
3118 					      iterator.level - 1, 1, ACC_ALL);  in __direct_map()
3120 			link_shadow_page(vcpu, iterator.sptep, sp);  in __direct_map()
3307 	struct kvm_shadow_walk_iterator iterator;  in fast_page_fault()  local
3324 		for_each_shadow_entry_lockless(vcpu, gva, iterator, spte)  in fast_page_fault()
3326 			    iterator.level < level)  in fast_page_fault()
3329 		sp = page_header(__pa(iterator.sptep));  in fast_page_fault()
3389 							iterator.sptep, spte,  in fast_page_fault()
3402 	trace_fast_page_fault(vcpu, gva, error_code, iterator.sptep,  in fast_page_fault()
3806 	struct kvm_shadow_walk_iterator iterator;  in walk_shadow_page_get_mmio_spte()  local
3816 	for (shadow_walk_init(&iterator, vcpu, addr),  in walk_shadow_page_get_mmio_spte()
3817 		 leaf = root = iterator.level;  in walk_shadow_page_get_mmio_spte()
3818 	     shadow_walk_okay(&iterator);  in walk_shadow_page_get_mmio_spte()
3819 	     __shadow_walk_next(&iterator, spte)) {  in walk_shadow_page_get_mmio_spte()
3820 		spte = mmu_spte_get_lockless(iterator.sptep);  in walk_shadow_page_get_mmio_spte()
3829 						    iterator.level);  in walk_shadow_page_get_mmio_spte()
3904 	struct kvm_shadow_walk_iterator iterator;  in shadow_page_table_clear_flood()  local
3911 	for_each_shadow_entry_lockless(vcpu, addr, iterator, spte) {  in shadow_page_table_clear_flood()
3912 		clear_sp_write_flooding_count(iterator.sptep);  in shadow_page_table_clear_flood()
5476 	struct slot_rmap_walk_iterator iterator;  in slot_handle_level_range()  local
5480 			end_gfn, &iterator) {  in slot_handle_level_range()
5481 		if (iterator.rmap)  in slot_handle_level_range()
5482 			flush |= fn(kvm, iterator.rmap);  in slot_handle_level_range()