23 bool kvm_page_track_write_tracking_enabled(struct kvm *kvm)  in kvm_page_track_write_tracking_enabled()  argument
26 	       !tdp_enabled || kvm_shadow_root_allocated(kvm);  in kvm_page_track_write_tracking_enabled()
47 int kvm_page_track_create_memslot(struct kvm *kvm,  in kvm_page_track_create_memslot()  argument
51 	if (!kvm_page_track_write_tracking_enabled(kvm))  in kvm_page_track_create_memslot()
77 void __kvm_write_track_add_gfn(struct kvm *kvm, struct kvm_memory_slot *slot,  in __kvm_write_track_add_gfn()  argument
80 	lockdep_assert_held_write(&kvm->mmu_lock);  in __kvm_write_track_add_gfn()
82 	lockdep_assert_once(lockdep_is_held(&kvm->slots_lock) ||  in __kvm_write_track_add_gfn()
83 			    srcu_read_lock_held(&kvm->srcu));  in __kvm_write_track_add_gfn()
85 	if (KVM_BUG_ON(!kvm_page_track_write_tracking_enabled(kvm), kvm))  in __kvm_write_track_add_gfn()
96 	if (kvm_mmu_slot_gfn_write_protect(kvm, slot, gfn, PG_LEVEL_4K))  in __kvm_write_track_add_gfn()
97 		kvm_flush_remote_tlbs(kvm);  in __kvm_write_track_add_gfn()
100 void __kvm_write_track_remove_gfn(struct kvm *kvm,  in __kvm_write_track_remove_gfn()  argument
103 	lockdep_assert_held_write(&kvm->mmu_lock);  in __kvm_write_track_remove_gfn()
105 	lockdep_assert_once(lockdep_is_held(&kvm->slots_lock) ||  in __kvm_write_track_remove_gfn()
106 			    srcu_read_lock_held(&kvm->srcu));  in __kvm_write_track_remove_gfn()
108 	if (KVM_BUG_ON(!kvm_page_track_write_tracking_enabled(kvm), kvm))  in __kvm_write_track_remove_gfn()
123 bool kvm_gfn_is_write_tracked(struct kvm *kvm,  in kvm_gfn_is_write_tracked()  argument
131 	if (!kvm_page_track_write_tracking_enabled(kvm))  in kvm_gfn_is_write_tracked()
139 void kvm_page_track_cleanup(struct kvm *kvm)  in kvm_page_track_cleanup()  argument
143 	head = &kvm->arch.track_notifier_head;  in kvm_page_track_cleanup()
147 int kvm_page_track_init(struct kvm *kvm)  in kvm_page_track_init()  argument
151 	head = &kvm->arch.track_notifier_head;  in kvm_page_track_init()
160 int kvm_page_track_register_notifier(struct kvm *kvm,  in kvm_page_track_register_notifier()  argument
165 	if (!kvm || kvm->mm != current->mm)  in kvm_page_track_register_notifier()
168 	kvm_get_kvm(kvm);  in kvm_page_track_register_notifier()
170 	head = &kvm->arch.track_notifier_head;  in kvm_page_track_register_notifier()
172 	write_lock(&kvm->mmu_lock);  in kvm_page_track_register_notifier()
174 	write_unlock(&kvm->mmu_lock);  in kvm_page_track_register_notifier()
183 void kvm_page_track_unregister_notifier(struct kvm *kvm,  in kvm_page_track_unregister_notifier()  argument
188 	head = &kvm->arch.track_notifier_head;  in kvm_page_track_unregister_notifier()
190 	write_lock(&kvm->mmu_lock);  in kvm_page_track_unregister_notifier()
192 	write_unlock(&kvm->mmu_lock);  in kvm_page_track_unregister_notifier()
195 	kvm_put_kvm(kvm);  in kvm_page_track_unregister_notifier()
206 void __kvm_page_track_write(struct kvm *kvm, gpa_t gpa, const u8 *new, int bytes)  in __kvm_page_track_write()  argument
212 	head = &kvm->arch.track_notifier_head;  in __kvm_page_track_write()
229 void kvm_page_track_delete_slot(struct kvm *kvm, struct kvm_memory_slot *slot)  in kvm_page_track_delete_slot()  argument
235 	head = &kvm->arch.track_notifier_head;  in kvm_page_track_delete_slot()
255 int kvm_write_track_add_gfn(struct kvm *kvm, gfn_t gfn)  in kvm_write_track_add_gfn()  argument
260 	idx = srcu_read_lock(&kvm->srcu);  in kvm_write_track_add_gfn()
262 	slot = gfn_to_memslot(kvm, gfn);  in kvm_write_track_add_gfn()
264 		srcu_read_unlock(&kvm->srcu, idx);  in kvm_write_track_add_gfn()
268 	write_lock(&kvm->mmu_lock);  in kvm_write_track_add_gfn()
269 	__kvm_write_track_add_gfn(kvm, slot, gfn);  in kvm_write_track_add_gfn()
270 	write_unlock(&kvm->mmu_lock);  in kvm_write_track_add_gfn()
272 	srcu_read_unlock(&kvm->srcu, idx);  in kvm_write_track_add_gfn()
285 int kvm_write_track_remove_gfn(struct kvm *kvm, gfn_t gfn)  in kvm_write_track_remove_gfn()  argument
290 	idx = srcu_read_lock(&kvm->srcu);  in kvm_write_track_remove_gfn()
292 	slot = gfn_to_memslot(kvm, gfn);  in kvm_write_track_remove_gfn()
294 		srcu_read_unlock(&kvm->srcu, idx);  in kvm_write_track_remove_gfn()
298 	write_lock(&kvm->mmu_lock);  in kvm_write_track_remove_gfn()
299 	__kvm_write_track_remove_gfn(kvm, slot, gfn);  in kvm_write_track_remove_gfn()
300 	write_unlock(&kvm->mmu_lock);  in kvm_write_track_remove_gfn()
302 	srcu_read_unlock(&kvm->srcu, idx);  in kvm_write_track_remove_gfn()