343 				u64 old_spte, u64 new_spte, int level,
346 static void handle_changed_spte_acc_track(u64 old_spte, u64 new_spte, int level)  in handle_changed_spte_acc_track()  argument
352 	    (!is_shadow_present_pte(new_spte) || !is_accessed_spte(new_spte) ||  in handle_changed_spte_acc_track()
353 	     spte_to_pfn(old_spte) != spte_to_pfn(new_spte)))  in handle_changed_spte_acc_track()
358 					  u64 old_spte, u64 new_spte, int level)  in handle_changed_spte_dirty_log()  argument
366 	pfn_changed = spte_to_pfn(old_spte) != spte_to_pfn(new_spte);  in handle_changed_spte_dirty_log()
369 	    is_writable_pte(new_spte)) {  in handle_changed_spte_dirty_log()
527 				  u64 old_spte, u64 new_spte, int level,  in __handle_changed_spte()  argument
531 	bool is_present = is_shadow_present_pte(new_spte);  in __handle_changed_spte()
533 	bool is_leaf = is_present && is_last_spte(new_spte, level);  in __handle_changed_spte()
534 	bool pfn_changed = spte_to_pfn(old_spte) != spte_to_pfn(new_spte);  in __handle_changed_spte()
554 		       as_id, gfn, old_spte, new_spte, level);  in __handle_changed_spte()
563 	if (old_spte == new_spte)  in __handle_changed_spte()
566 	trace_kvm_tdp_mmu_spte_changed(as_id, gfn, level, old_spte, new_spte);  in __handle_changed_spte()
569 		check_spte_writable_invariants(new_spte);  in __handle_changed_spte()
584 			    !is_mmio_spte(new_spte) &&  in __handle_changed_spte()
585 			    !is_removed_spte(new_spte)))  in __handle_changed_spte()
592 			       as_id, gfn, old_spte, new_spte, level);  in __handle_changed_spte()
600 	    (!is_present || !is_dirty_spte(new_spte) || pfn_changed))  in __handle_changed_spte()
615 				u64 old_spte, u64 new_spte, int level,  in handle_changed_spte()  argument
618 	__handle_changed_spte(kvm, as_id, gfn, old_spte, new_spte, level,  in handle_changed_spte()
620 	handle_changed_spte_acc_track(old_spte, new_spte, level);  in handle_changed_spte()
622 				      new_spte, level);  in handle_changed_spte()
644 					  u64 new_spte)  in tdp_mmu_set_spte_atomic()  argument
662 	if (!try_cmpxchg64(sptep, &iter->old_spte, new_spte))  in tdp_mmu_set_spte_atomic()
666 			      new_spte, iter->level, true);  in tdp_mmu_set_spte_atomic()
667 	handle_changed_spte_acc_track(iter->old_spte, new_spte, iter->level);  in tdp_mmu_set_spte_atomic()
727 			      u64 old_spte, u64 new_spte, gfn_t gfn, int level,  in __tdp_mmu_set_spte()  argument
739 	WARN_ON(is_removed_spte(old_spte) || is_removed_spte(new_spte));  in __tdp_mmu_set_spte()
741 	old_spte = kvm_tdp_mmu_write_spte(sptep, old_spte, new_spte, level);  in __tdp_mmu_set_spte()
743 	__handle_changed_spte(kvm, as_id, gfn, old_spte, new_spte, level, false);  in __tdp_mmu_set_spte()
746 		handle_changed_spte_acc_track(old_spte, new_spte, level);  in __tdp_mmu_set_spte()
749 					      new_spte, level);  in __tdp_mmu_set_spte()
754 				     u64 new_spte, bool record_acc_track,  in _tdp_mmu_set_spte()  argument
760 					    iter->old_spte, new_spte,  in _tdp_mmu_set_spte()
766 				    u64 new_spte)  in tdp_mmu_set_spte()  argument
768 	_tdp_mmu_set_spte(kvm, iter, new_spte, true, true);  in tdp_mmu_set_spte()
773 						 u64 new_spte)  in tdp_mmu_set_spte_no_acc_track()  argument
775 	_tdp_mmu_set_spte(kvm, iter, new_spte, false, true);  in tdp_mmu_set_spte_no_acc_track()
780 						 u64 new_spte)  in tdp_mmu_set_spte_no_dirty_log()  argument
782 	_tdp_mmu_set_spte(kvm, iter, new_spte, true, false);  in tdp_mmu_set_spte_no_dirty_log()
1067 	u64 new_spte;  in tdp_mmu_map_handle_target_level()  local
1073 		new_spte = make_mmio_spte(vcpu, iter->gfn, ACC_ALL);  in tdp_mmu_map_handle_target_level()
1077 					 fault->map_writable, &new_spte);  in tdp_mmu_map_handle_target_level()
1079 	if (new_spte == iter->old_spte)  in tdp_mmu_map_handle_target_level()
1081 	else if (tdp_mmu_set_spte_atomic(vcpu->kvm, iter, new_spte))  in tdp_mmu_map_handle_target_level()
1099 	if (unlikely(is_mmio_spte(new_spte))) {  in tdp_mmu_map_handle_target_level()
1102 				     new_spte);  in tdp_mmu_map_handle_target_level()
1271 	u64 new_spte = 0;  in age_gfn_range()  local
1277 	new_spte = iter->old_spte;  in age_gfn_range()
1279 	if (spte_ad_enabled(new_spte)) {  in age_gfn_range()
1280 		new_spte &= ~shadow_accessed_mask;  in age_gfn_range()
1286 		if (is_writable_pte(new_spte))  in age_gfn_range()
1287 			kvm_set_pfn_dirty(spte_to_pfn(new_spte));  in age_gfn_range()
1289 		new_spte = mark_spte_for_access_track(new_spte);  in age_gfn_range()
1292 	tdp_mmu_set_spte_no_acc_track(kvm, iter, new_spte);  in age_gfn_range()
1316 	u64 new_spte;  in set_spte_gfn()  local
1334 		new_spte = kvm_mmu_changed_pte_notifier_make_spte(iter->old_spte,  in set_spte_gfn()
1337 		tdp_mmu_set_spte(kvm, iter, new_spte);  in set_spte_gfn()
1368 	u64 new_spte;  in wrprot_gfn_range()  local
1385 		new_spte = iter.old_spte & ~PT_WRITABLE_MASK;  in wrprot_gfn_range()
1387 		if (tdp_mmu_set_spte_atomic(kvm, &iter, new_spte))  in wrprot_gfn_range()
1612 	u64 new_spte;  in clear_dirty_gfn_range()  local
1627 				new_spte = iter.old_spte & ~PT_WRITABLE_MASK;  in clear_dirty_gfn_range()
1632 				new_spte = iter.old_spte & ~shadow_dirty_mask;  in clear_dirty_gfn_range()
1637 		if (tdp_mmu_set_spte_atomic(kvm, &iter, new_spte))  in clear_dirty_gfn_range()
1680 	u64 new_spte;  in clear_dirty_pt_masked()  local
1697 				new_spte = iter.old_spte & ~PT_WRITABLE_MASK;  in clear_dirty_pt_masked()
1702 				new_spte = iter.old_spte & ~shadow_dirty_mask;  in clear_dirty_pt_masked()
1707 		tdp_mmu_set_spte_no_dirty_log(kvm, &iter, new_spte);  in clear_dirty_pt_masked()
1807 	u64 new_spte;  in write_protect_gfn()  local
1819 		new_spte = iter.old_spte &  in write_protect_gfn()
1822 		if (new_spte == iter.old_spte)  in write_protect_gfn()
1825 		tdp_mmu_set_spte(kvm, &iter, new_spte);  in write_protect_gfn()