1 // SPDX-License-Identifier: GPL-2.0
13 	iter->sptep = iter->pt_path[iter->level - 1] +  in tdp_iter_refresh_sptep()
14 		SHADOW_PT_INDEX(iter->gfn << PAGE_SHIFT, iter->level);  in tdp_iter_refresh_sptep()
15 	iter->old_spte = READ_ONCE(*rcu_dereference(iter->sptep));  in tdp_iter_refresh_sptep()
20 	return gfn & -KVM_PAGES_PER_HPAGE(level);  in round_gfn_for_level()
29 	iter->yielded_gfn = iter->next_last_level_gfn;  in tdp_iter_restart()
30 	iter->level = iter->root_level;  in tdp_iter_restart()
32 	iter->gfn = round_gfn_for_level(iter->next_last_level_gfn, iter->level);  in tdp_iter_restart()
35 	iter->valid = true;  in tdp_iter_restart()
39  * Sets a TDP iterator to walk a pre-order traversal of the paging structure
48 	iter->next_last_level_gfn = next_last_level_gfn;  in tdp_iter_start()
49 	iter->root_level = root_level;  in tdp_iter_start()
50 	iter->min_level = min_level;  in tdp_iter_start()
51 	iter->pt_path[iter->root_level - 1] = (tdp_ptep_t)root_pt;  in tdp_iter_start()
52 	iter->as_id = kvm_mmu_page_as_id(sptep_to_sp(root_pt));  in tdp_iter_start()
66 	 * last-level entry.  in spte_to_child_pt()
75  * Steps down one level in the paging structure towards the goal GFN. Returns
76  * true if the iterator was able to step down a level, false otherwise.
82 	if (iter->level == iter->min_level)  in try_step_down()
86 	 * Reread the SPTE before stepping down to avoid traversing into page  in try_step_down()
89 	iter->old_spte = READ_ONCE(*rcu_dereference(iter->sptep));  in try_step_down()
91 	child_pt = spte_to_child_pt(iter->old_spte, iter->level);  in try_step_down()
95 	iter->level--;  in try_step_down()
96 	iter->pt_path[iter->level - 1] = child_pt;  in try_step_down()
97 	iter->gfn = round_gfn_for_level(iter->next_last_level_gfn, iter->level);  in try_step_down()
106  * page table, or it could be non-present. Returns true if the iterator was
107  * able to step to the next entry in the page table, false if the iterator was
116 	if (SHADOW_PT_INDEX(iter->gfn << PAGE_SHIFT, iter->level) ==  in try_step_side()
117             (PT64_ENT_PER_PAGE - 1))  in try_step_side()
120 	iter->gfn += KVM_PAGES_PER_HPAGE(iter->level);  in try_step_side()
121 	iter->next_last_level_gfn = iter->gfn;  in try_step_side()
122 	iter->sptep++;  in try_step_side()
123 	iter->old_spte = READ_ONCE(*rcu_dereference(iter->sptep));  in try_step_side()
131  * successful step up, false if already in the root page.
135 	if (iter->level == iter->root_level)  in try_step_up()
138 	iter->level++;  in try_step_up()
139 	iter->gfn = round_gfn_for_level(iter->gfn, iter->level);  in try_step_up()
146  * Step to the next SPTE in a pre-order traversal of the paging structure.
147  * To get to the next SPTE, the iterator either steps down towards the goal
148  * GFN, if at a present, non-last-level SPTE, or over to a SPTE mapping a
152  * 1. If the current SPTE is a non-last-level SPTE, step down into the page
154  * 2. If the iterator cannot step down, it will try to step to the next SPTE
156  * 3. If the iterator cannot step to the next entry in the current page, it will
157  *    try to step up to the parent paging structure page. In this case, that
158  *    SPTE will have already been visited, and so the iterator must also step
170 	iter->valid = false;  in tdp_iter_next()