1 // SPDX-License-Identifier: GPL-2.0-only
2
3 #ifndef KVM_X86_MMU_SPTE_H
4 #define KVM_X86_MMU_SPTE_H
5
6 #include "mmu_internal.h"
7
8 #define PT_FIRST_AVAIL_BITS_SHIFT 10
9 #define PT64_SECOND_AVAIL_BITS_SHIFT 54
10
11 /*
12 * The mask used to denote special SPTEs, which can be either MMIO SPTEs or
13 * Access Tracking SPTEs.
14 */
15 #define SPTE_SPECIAL_MASK (3ULL << 52)
16 #define SPTE_AD_ENABLED_MASK (0ULL << 52)
17 #define SPTE_AD_DISABLED_MASK (1ULL << 52)
18 #define SPTE_AD_WRPROT_ONLY_MASK (2ULL << 52)
19 #define SPTE_MMIO_MASK (3ULL << 52)
20
21 #ifdef CONFIG_DYNAMIC_PHYSICAL_MASK
22 #define PT64_BASE_ADDR_MASK (physical_mask & ~(u64)(PAGE_SIZE-1))
23 #else
24 #define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
25 #endif
26 #define PT64_LVL_ADDR_MASK(level) \
27 (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
28 * PT64_LEVEL_BITS))) - 1))
29 #define PT64_LVL_OFFSET_MASK(level) \
30 (PT64_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
31 * PT64_LEVEL_BITS))) - 1))
32
33 #define PT64_PERM_MASK (PT_PRESENT_MASK | PT_WRITABLE_MASK | shadow_user_mask \
34 | shadow_x_mask | shadow_nx_mask | shadow_me_mask)
35
36 #define ACC_EXEC_MASK 1
37 #define ACC_WRITE_MASK PT_WRITABLE_MASK
38 #define ACC_USER_MASK PT_USER_MASK
39 #define ACC_ALL (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK)
40
41 /* The mask for the R/X bits in EPT PTEs */
42 #define PT64_EPT_READABLE_MASK 0x1ull
43 #define PT64_EPT_EXECUTABLE_MASK 0x4ull
44
45 #define PT64_LEVEL_BITS 9
46
47 #define PT64_LEVEL_SHIFT(level) \
48 (PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS)
49
50 #define PT64_INDEX(address, level)\
51 (((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))
52 #define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level)
53
54
55 #define SPTE_HOST_WRITEABLE (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
56 #define SPTE_MMU_WRITEABLE (1ULL << (PT_FIRST_AVAIL_BITS_SHIFT + 1))
57
58 /*
59 * Due to limited space in PTEs, the MMIO generation is a 18 bit subset of
60 * the memslots generation and is derived as follows:
61 *
62 * Bits 0-8 of the MMIO generation are propagated to spte bits 3-11
63 * Bits 9-17 of the MMIO generation are propagated to spte bits 54-62
64 *
65 * The KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS flag is intentionally not included in
66 * the MMIO generation number, as doing so would require stealing a bit from
67 * the "real" generation number and thus effectively halve the maximum number
68 * of MMIO generations that can be handled before encountering a wrap (which
69 * requires a full MMU zap). The flag is instead explicitly queried when
70 * checking for MMIO spte cache hits.
71 */
72
73 #define MMIO_SPTE_GEN_LOW_START 3
74 #define MMIO_SPTE_GEN_LOW_END 11
75
76 #define MMIO_SPTE_GEN_HIGH_START PT64_SECOND_AVAIL_BITS_SHIFT
77 #define MMIO_SPTE_GEN_HIGH_END 62
78
79 #define MMIO_SPTE_GEN_LOW_MASK GENMASK_ULL(MMIO_SPTE_GEN_LOW_END, \
80 MMIO_SPTE_GEN_LOW_START)
81 #define MMIO_SPTE_GEN_HIGH_MASK GENMASK_ULL(MMIO_SPTE_GEN_HIGH_END, \
82 MMIO_SPTE_GEN_HIGH_START)
83
84 #define MMIO_SPTE_GEN_LOW_BITS (MMIO_SPTE_GEN_LOW_END - MMIO_SPTE_GEN_LOW_START + 1)
85 #define MMIO_SPTE_GEN_HIGH_BITS (MMIO_SPTE_GEN_HIGH_END - MMIO_SPTE_GEN_HIGH_START + 1)
86
87 /* remember to adjust the comment above as well if you change these */
88 static_assert(MMIO_SPTE_GEN_LOW_BITS == 9 && MMIO_SPTE_GEN_HIGH_BITS == 9);
89
90 #define MMIO_SPTE_GEN_LOW_SHIFT (MMIO_SPTE_GEN_LOW_START - 0)
91 #define MMIO_SPTE_GEN_HIGH_SHIFT (MMIO_SPTE_GEN_HIGH_START - MMIO_SPTE_GEN_LOW_BITS)
92
93 #define MMIO_SPTE_GEN_MASK GENMASK_ULL(MMIO_SPTE_GEN_LOW_BITS + MMIO_SPTE_GEN_HIGH_BITS - 1, 0)
94
95 extern u64 __read_mostly shadow_nx_mask;
96 extern u64 __read_mostly shadow_x_mask; /* mutual exclusive with nx_mask */
97 extern u64 __read_mostly shadow_user_mask;
98 extern u64 __read_mostly shadow_accessed_mask;
99 extern u64 __read_mostly shadow_dirty_mask;
100 extern u64 __read_mostly shadow_mmio_value;
101 extern u64 __read_mostly shadow_mmio_access_mask;
102 extern u64 __read_mostly shadow_present_mask;
103 extern u64 __read_mostly shadow_me_mask;
104
105 /*
106 * SPTEs used by MMUs without A/D bits are marked with SPTE_AD_DISABLED_MASK;
107 * shadow_acc_track_mask is the set of bits to be cleared in non-accessed
108 * pages.
109 */
110 extern u64 __read_mostly shadow_acc_track_mask;
111
112 /*
113 * This mask must be set on all non-zero Non-Present or Reserved SPTEs in order
114 * to guard against L1TF attacks.
115 */
116 extern u64 __read_mostly shadow_nonpresent_or_rsvd_mask;
117
118 /*
119 * The number of high-order 1 bits to use in the mask above.
120 */
121 #define SHADOW_NONPRESENT_OR_RSVD_MASK_LEN 5
122
123 /*
124 * The mask/shift to use for saving the original R/X bits when marking the PTE
125 * as not-present for access tracking purposes. We do not save the W bit as the
126 * PTEs being access tracked also need to be dirty tracked, so the W bit will be
127 * restored only when a write is attempted to the page.
128 */
129 #define SHADOW_ACC_TRACK_SAVED_BITS_MASK (PT64_EPT_READABLE_MASK | \
130 PT64_EPT_EXECUTABLE_MASK)
131 #define SHADOW_ACC_TRACK_SAVED_BITS_SHIFT PT64_SECOND_AVAIL_BITS_SHIFT
132
133 /*
134 * In some cases, we need to preserve the GFN of a non-present or reserved
135 * SPTE when we usurp the upper five bits of the physical address space to
136 * defend against L1TF, e.g. for MMIO SPTEs. To preserve the GFN, we'll
137 * shift bits of the GFN that overlap with shadow_nonpresent_or_rsvd_mask
138 * left into the reserved bits, i.e. the GFN in the SPTE will be split into
139 * high and low parts. This mask covers the lower bits of the GFN.
140 */
141 extern u64 __read_mostly shadow_nonpresent_or_rsvd_lower_gfn_mask;
142
143 /*
144 * The number of non-reserved physical address bits irrespective of features
145 * that repurpose legal bits, e.g. MKTME.
146 */
147 extern u8 __read_mostly shadow_phys_bits;
148
is_mmio_spte(u64 spte)149 static inline bool is_mmio_spte(u64 spte)
150 {
151 return (spte & SPTE_SPECIAL_MASK) == SPTE_MMIO_MASK;
152 }
153
sp_ad_disabled(struct kvm_mmu_page * sp)154 static inline bool sp_ad_disabled(struct kvm_mmu_page *sp)
155 {
156 return sp->role.ad_disabled;
157 }
158
spte_ad_enabled(u64 spte)159 static inline bool spte_ad_enabled(u64 spte)
160 {
161 MMU_WARN_ON(is_mmio_spte(spte));
162 return (spte & SPTE_SPECIAL_MASK) != SPTE_AD_DISABLED_MASK;
163 }
164
spte_ad_need_write_protect(u64 spte)165 static inline bool spte_ad_need_write_protect(u64 spte)
166 {
167 MMU_WARN_ON(is_mmio_spte(spte));
168 return (spte & SPTE_SPECIAL_MASK) != SPTE_AD_ENABLED_MASK;
169 }
170
spte_shadow_accessed_mask(u64 spte)171 static inline u64 spte_shadow_accessed_mask(u64 spte)
172 {
173 MMU_WARN_ON(is_mmio_spte(spte));
174 return spte_ad_enabled(spte) ? shadow_accessed_mask : 0;
175 }
176
spte_shadow_dirty_mask(u64 spte)177 static inline u64 spte_shadow_dirty_mask(u64 spte)
178 {
179 MMU_WARN_ON(is_mmio_spte(spte));
180 return spte_ad_enabled(spte) ? shadow_dirty_mask : 0;
181 }
182
is_access_track_spte(u64 spte)183 static inline bool is_access_track_spte(u64 spte)
184 {
185 return !spte_ad_enabled(spte) && (spte & shadow_acc_track_mask) == 0;
186 }
187
is_shadow_present_pte(u64 pte)188 static inline int is_shadow_present_pte(u64 pte)
189 {
190 return (pte != 0) && !is_mmio_spte(pte);
191 }
192
is_large_pte(u64 pte)193 static inline int is_large_pte(u64 pte)
194 {
195 return pte & PT_PAGE_SIZE_MASK;
196 }
197
is_last_spte(u64 pte,int level)198 static inline int is_last_spte(u64 pte, int level)
199 {
200 if (level == PG_LEVEL_4K)
201 return 1;
202 if (is_large_pte(pte))
203 return 1;
204 return 0;
205 }
206
is_executable_pte(u64 spte)207 static inline bool is_executable_pte(u64 spte)
208 {
209 return (spte & (shadow_x_mask | shadow_nx_mask)) == shadow_x_mask;
210 }
211
spte_to_pfn(u64 pte)212 static inline kvm_pfn_t spte_to_pfn(u64 pte)
213 {
214 return (pte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT;
215 }
216
is_accessed_spte(u64 spte)217 static inline bool is_accessed_spte(u64 spte)
218 {
219 u64 accessed_mask = spte_shadow_accessed_mask(spte);
220
221 return accessed_mask ? spte & accessed_mask
222 : !is_access_track_spte(spte);
223 }
224
is_dirty_spte(u64 spte)225 static inline bool is_dirty_spte(u64 spte)
226 {
227 u64 dirty_mask = spte_shadow_dirty_mask(spte);
228
229 return dirty_mask ? spte & dirty_mask : spte & PT_WRITABLE_MASK;
230 }
231
spte_can_locklessly_be_made_writable(u64 spte)232 static inline bool spte_can_locklessly_be_made_writable(u64 spte)
233 {
234 return (spte & (SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE)) ==
235 (SPTE_HOST_WRITEABLE | SPTE_MMU_WRITEABLE);
236 }
237
get_mmio_spte_generation(u64 spte)238 static inline u64 get_mmio_spte_generation(u64 spte)
239 {
240 u64 gen;
241
242 gen = (spte & MMIO_SPTE_GEN_LOW_MASK) >> MMIO_SPTE_GEN_LOW_SHIFT;
243 gen |= (spte & MMIO_SPTE_GEN_HIGH_MASK) >> MMIO_SPTE_GEN_HIGH_SHIFT;
244 return gen;
245 }
246
247 /* Bits which may be returned by set_spte() */
248 #define SET_SPTE_WRITE_PROTECTED_PT BIT(0)
249 #define SET_SPTE_NEED_REMOTE_TLB_FLUSH BIT(1)
250 #define SET_SPTE_SPURIOUS BIT(2)
251
252 int make_spte(struct kvm_vcpu *vcpu, unsigned int pte_access, int level,
253 gfn_t gfn, kvm_pfn_t pfn, u64 old_spte, bool speculative,
254 bool can_unsync, bool host_writable, bool ad_disabled,
255 u64 *new_spte);
256 u64 make_nonleaf_spte(u64 *child_pt, bool ad_disabled);
257 u64 make_mmio_spte(struct kvm_vcpu *vcpu, u64 gfn, unsigned int access);
258 u64 mark_spte_for_access_track(u64 spte);
259 u64 kvm_mmu_changed_pte_notifier_make_spte(u64 old_spte, kvm_pfn_t new_pfn);
260
261 void kvm_mmu_reset_all_pte_masks(void);
262
263 #endif
264
