147 static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot, gfn_t gfn);
1374 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)  in gfn_to_memslot()  argument
1376 	return __gfn_to_memslot(kvm_memslots(kvm), gfn);  in gfn_to_memslot()
1380 struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn)  in kvm_vcpu_gfn_to_memslot()  argument
1382 	return __gfn_to_memslot(kvm_vcpu_memslots(vcpu), gfn);  in kvm_vcpu_gfn_to_memslot()
1385 bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)  in kvm_is_visible_gfn()  argument
1387 	struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn);  in kvm_is_visible_gfn()
1397 unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn)  in kvm_host_page_size()  argument
1404 	addr = gfn_to_hva(kvm, gfn);  in kvm_host_page_size()
1426 static unsigned long __gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,  in __gfn_to_hva_many()  argument
1436 		*nr_pages = slot->npages - (gfn - slot->base_gfn);  in __gfn_to_hva_many()
1438 	return __gfn_to_hva_memslot(slot, gfn);  in __gfn_to_hva_many()
1441 static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,  in gfn_to_hva_many()  argument
1444 	return __gfn_to_hva_many(slot, gfn, nr_pages, true);  in gfn_to_hva_many()
1448 					gfn_t gfn)  in gfn_to_hva_memslot()  argument
1450 	return gfn_to_hva_many(slot, gfn, NULL);  in gfn_to_hva_memslot()
1454 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)  in gfn_to_hva()  argument
1456 	return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL);  in gfn_to_hva()
1460 unsigned long kvm_vcpu_gfn_to_hva(struct kvm_vcpu *vcpu, gfn_t gfn)  in kvm_vcpu_gfn_to_hva()  argument
1462 	return gfn_to_hva_many(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn, NULL);  in kvm_vcpu_gfn_to_hva()
1475 				      gfn_t gfn, bool *writable)  in gfn_to_hva_memslot_prot()  argument
1477 	unsigned long hva = __gfn_to_hva_many(slot, gfn, NULL, false);  in gfn_to_hva_memslot_prot()
1485 unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable)  in gfn_to_hva_prot()  argument
1487 	struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);  in gfn_to_hva_prot()
1489 	return gfn_to_hva_memslot_prot(slot, gfn, writable);  in gfn_to_hva_prot()
1492 unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *writable)  in kvm_vcpu_gfn_to_hva_prot()  argument
1494 	struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);  in kvm_vcpu_gfn_to_hva_prot()
1496 	return gfn_to_hva_memslot_prot(slot, gfn, writable);  in kvm_vcpu_gfn_to_hva_prot()
1699 kvm_pfn_t __gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn,  in __gfn_to_pfn_memslot()  argument
1703 	unsigned long addr = __gfn_to_hva_many(slot, gfn, NULL, write_fault);  in __gfn_to_pfn_memslot()
1728 kvm_pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault,  in gfn_to_pfn_prot()  argument
1731 	return __gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn, false, NULL,  in gfn_to_pfn_prot()
1736 kvm_pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn)  in gfn_to_pfn_memslot()  argument
1738 	return __gfn_to_pfn_memslot(slot, gfn, false, NULL, true, NULL);  in gfn_to_pfn_memslot()
1742 kvm_pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn)  in gfn_to_pfn_memslot_atomic()  argument
1744 	return __gfn_to_pfn_memslot(slot, gfn, true, NULL, true, NULL);  in gfn_to_pfn_memslot_atomic()
1748 kvm_pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn)  in gfn_to_pfn_atomic()  argument
1750 	return gfn_to_pfn_memslot_atomic(gfn_to_memslot(kvm, gfn), gfn);  in gfn_to_pfn_atomic()
1754 kvm_pfn_t kvm_vcpu_gfn_to_pfn_atomic(struct kvm_vcpu *vcpu, gfn_t gfn)  in kvm_vcpu_gfn_to_pfn_atomic()  argument
1756 	return gfn_to_pfn_memslot_atomic(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn);  in kvm_vcpu_gfn_to_pfn_atomic()
1760 kvm_pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn)  in gfn_to_pfn()  argument
1762 	return gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn);  in gfn_to_pfn()
1766 kvm_pfn_t kvm_vcpu_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn)  in kvm_vcpu_gfn_to_pfn()  argument
1768 	return gfn_to_pfn_memslot(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn);  in kvm_vcpu_gfn_to_pfn()
1772 int gfn_to_page_many_atomic(struct kvm_memory_slot *slot, gfn_t gfn,  in gfn_to_page_many_atomic()  argument
1778 	addr = gfn_to_hva_many(slot, gfn, &entry);  in gfn_to_page_many_atomic()
1802 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)  in gfn_to_page()  argument
1806 	pfn = gfn_to_pfn(kvm, gfn);  in gfn_to_page()
1812 static int __kvm_map_gfn(struct kvm_memory_slot *slot, gfn_t gfn,  in __kvm_map_gfn()  argument
1822 	pfn = gfn_to_pfn_memslot(slot, gfn);  in __kvm_map_gfn()
1841 	map->gfn = gfn;  in __kvm_map_gfn()
1846 int kvm_vcpu_map(struct kvm_vcpu *vcpu, gfn_t gfn, struct kvm_host_map *map)  in kvm_vcpu_map()  argument
1848 	return __kvm_map_gfn(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn, map);  in kvm_vcpu_map()
1869 		kvm_vcpu_mark_page_dirty(vcpu, map->gfn);  in kvm_vcpu_unmap()
1880 struct page *kvm_vcpu_gfn_to_page(struct kvm_vcpu *vcpu, gfn_t gfn)  in kvm_vcpu_gfn_to_page()  argument
1884 	pfn = kvm_vcpu_gfn_to_pfn(vcpu, gfn);  in kvm_vcpu_gfn_to_page()
1952 static int __kvm_read_guest_page(struct kvm_memory_slot *slot, gfn_t gfn,  in __kvm_read_guest_page()  argument
1958 	addr = gfn_to_hva_memslot_prot(slot, gfn, NULL);  in __kvm_read_guest_page()
1967 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,  in kvm_read_guest_page()  argument
1970 	struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);  in kvm_read_guest_page()
1972 	return __kvm_read_guest_page(slot, gfn, data, offset, len);  in kvm_read_guest_page()
1976 int kvm_vcpu_read_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, void *data,  in kvm_vcpu_read_guest_page()  argument
1979 	struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);  in kvm_vcpu_read_guest_page()
1981 	return __kvm_read_guest_page(slot, gfn, data, offset, len);  in kvm_vcpu_read_guest_page()
1987 	gfn_t gfn = gpa >> PAGE_SHIFT;  in kvm_read_guest()  local
1993 		ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);  in kvm_read_guest()
1999 		++gfn;  in kvm_read_guest()
2007 	gfn_t gfn = gpa >> PAGE_SHIFT;  in kvm_vcpu_read_guest()  local
2013 		ret = kvm_vcpu_read_guest_page(vcpu, gfn, data, offset, seg);  in kvm_vcpu_read_guest()
2019 		++gfn;  in kvm_vcpu_read_guest()
2025 static int __kvm_read_guest_atomic(struct kvm_memory_slot *slot, gfn_t gfn,  in __kvm_read_guest_atomic()  argument
2031 	addr = gfn_to_hva_memslot_prot(slot, gfn, NULL);  in __kvm_read_guest_atomic()
2045 	gfn_t gfn = gpa >> PAGE_SHIFT;  in kvm_read_guest_atomic()  local
2046 	struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);  in kvm_read_guest_atomic()
2049 	return __kvm_read_guest_atomic(slot, gfn, data, offset, len);  in kvm_read_guest_atomic()
2056 	gfn_t gfn = gpa >> PAGE_SHIFT;  in kvm_vcpu_read_guest_atomic()  local
2057 	struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);  in kvm_vcpu_read_guest_atomic()
2060 	return __kvm_read_guest_atomic(slot, gfn, data, offset, len);  in kvm_vcpu_read_guest_atomic()
2064 static int __kvm_write_guest_page(struct kvm_memory_slot *memslot, gfn_t gfn,  in __kvm_write_guest_page()  argument
2070 	addr = gfn_to_hva_memslot(memslot, gfn);  in __kvm_write_guest_page()
2076 	mark_page_dirty_in_slot(memslot, gfn);  in __kvm_write_guest_page()
2080 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn,  in kvm_write_guest_page()  argument
2083 	struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);  in kvm_write_guest_page()
2085 	return __kvm_write_guest_page(slot, gfn, data, offset, len);  in kvm_write_guest_page()
2089 int kvm_vcpu_write_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn,  in kvm_vcpu_write_guest_page()  argument
2092 	struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);  in kvm_vcpu_write_guest_page()
2094 	return __kvm_write_guest_page(slot, gfn, data, offset, len);  in kvm_vcpu_write_guest_page()
2101 	gfn_t gfn = gpa >> PAGE_SHIFT;  in kvm_write_guest()  local
2107 		ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);  in kvm_write_guest()
2113 		++gfn;  in kvm_write_guest()
2122 	gfn_t gfn = gpa >> PAGE_SHIFT;  in kvm_vcpu_write_guest()  local
2128 		ret = kvm_vcpu_write_guest_page(vcpu, gfn, data, offset, seg);  in kvm_vcpu_write_guest()
2134 		++gfn;  in kvm_vcpu_write_guest()
2246 int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)  in kvm_clear_guest_page()  argument
2250 	return kvm_write_guest_page(kvm, gfn, zero_page, offset, len);  in kvm_clear_guest_page()
2256 	gfn_t gfn = gpa >> PAGE_SHIFT;  in kvm_clear_guest()  local
2262 		ret = kvm_clear_guest_page(kvm, gfn, offset, seg);  in kvm_clear_guest()
2267 		++gfn;  in kvm_clear_guest()
2274 				    gfn_t gfn)  in mark_page_dirty_in_slot()  argument
2277 		unsigned long rel_gfn = gfn - memslot->base_gfn;  in mark_page_dirty_in_slot()
2283 void mark_page_dirty(struct kvm *kvm, gfn_t gfn)  in mark_page_dirty()  argument
2287 	memslot = gfn_to_memslot(kvm, gfn);  in mark_page_dirty()
2288 	mark_page_dirty_in_slot(memslot, gfn);  in mark_page_dirty()
2292 void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn)  in kvm_vcpu_mark_page_dirty()  argument
2296 	memslot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);  in kvm_vcpu_mark_page_dirty()
2297 	mark_page_dirty_in_slot(memslot, gfn);  in kvm_vcpu_mark_page_dirty()