1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * amx tests
4 *
5 * Copyright (C) 2021, Intel, Inc.
6 *
7 * Tests for amx #NM exception and save/restore.
8 */
9
10 #define _GNU_SOURCE /* for program_invocation_short_name */
11 #include <fcntl.h>
12 #include <stdio.h>
13 #include <stdlib.h>
14 #include <string.h>
15 #include <sys/ioctl.h>
16 #include <sys/syscall.h>
17
18 #include "test_util.h"
19
20 #include "kvm_util.h"
21 #include "processor.h"
22 #include "vmx.h"
23
24 #ifndef __x86_64__
25 # error This test is 64-bit only
26 #endif
27
28 #define NUM_TILES 8
29 #define TILE_SIZE 1024
30 #define XSAVE_SIZE ((NUM_TILES * TILE_SIZE) + PAGE_SIZE)
31
32 /* Tile configuration associated: */
33 #define MAX_TILES 16
34 #define RESERVED_BYTES 14
35
36 #define XFEATURE_XTILECFG 17
37 #define XFEATURE_XTILEDATA 18
38 #define XFEATURE_MASK_XTILECFG (1 << XFEATURE_XTILECFG)
39 #define XFEATURE_MASK_XTILEDATA (1 << XFEATURE_XTILEDATA)
40 #define XFEATURE_MASK_XTILE (XFEATURE_MASK_XTILECFG | XFEATURE_MASK_XTILEDATA)
41
42 #define TILE_CPUID 0x1d
43 #define XSTATE_CPUID 0xd
44 #define TILE_PALETTE_CPUID_SUBLEAVE 0x1
45 #define XSTATE_USER_STATE_SUBLEAVE 0x0
46
47 #define XSAVE_HDR_OFFSET 512
48
49 struct xsave_data {
50 u8 area[XSAVE_SIZE];
51 } __aligned(64);
52
53 struct tile_config {
54 u8 palette_id;
55 u8 start_row;
56 u8 reserved[RESERVED_BYTES];
57 u16 colsb[MAX_TILES];
58 u8 rows[MAX_TILES];
59 };
60
61 struct tile_data {
62 u8 data[NUM_TILES * TILE_SIZE];
63 };
64
65 struct xtile_info {
66 u16 bytes_per_tile;
67 u16 bytes_per_row;
68 u16 max_names;
69 u16 max_rows;
70 u32 xsave_offset;
71 u32 xsave_size;
72 };
73
74 static struct xtile_info xtile;
75
__xgetbv(u32 index)76 static inline u64 __xgetbv(u32 index)
77 {
78 u32 eax, edx;
79
80 asm volatile("xgetbv;"
81 : "=a" (eax), "=d" (edx)
82 : "c" (index));
83 return eax + ((u64)edx << 32);
84 }
85
__xsetbv(u32 index,u64 value)86 static inline void __xsetbv(u32 index, u64 value)
87 {
88 u32 eax = value;
89 u32 edx = value >> 32;
90
91 asm volatile("xsetbv" :: "a" (eax), "d" (edx), "c" (index));
92 }
93
__ldtilecfg(void * cfg)94 static inline void __ldtilecfg(void *cfg)
95 {
96 asm volatile(".byte 0xc4,0xe2,0x78,0x49,0x00"
97 : : "a"(cfg));
98 }
99
__tileloadd(void * tile)100 static inline void __tileloadd(void *tile)
101 {
102 asm volatile(".byte 0xc4,0xe2,0x7b,0x4b,0x04,0x10"
103 : : "a"(tile), "d"(0));
104 }
105
__tilerelease(void)106 static inline void __tilerelease(void)
107 {
108 asm volatile(".byte 0xc4, 0xe2, 0x78, 0x49, 0xc0" ::);
109 }
110
__xsavec(struct xsave_data * data,uint64_t rfbm)111 static inline void __xsavec(struct xsave_data *data, uint64_t rfbm)
112 {
113 uint32_t rfbm_lo = rfbm;
114 uint32_t rfbm_hi = rfbm >> 32;
115
116 asm volatile("xsavec (%%rdi)"
117 : : "D" (data), "a" (rfbm_lo), "d" (rfbm_hi)
118 : "memory");
119 }
120
check_cpuid_xsave(void)121 static inline void check_cpuid_xsave(void)
122 {
123 GUEST_ASSERT(this_cpu_has(X86_FEATURE_XSAVE));
124 GUEST_ASSERT(this_cpu_has(X86_FEATURE_OSXSAVE));
125 }
126
check_xsave_supports_xtile(void)127 static bool check_xsave_supports_xtile(void)
128 {
129 return __xgetbv(0) & XFEATURE_MASK_XTILE;
130 }
131
enum_xtile_config(void)132 static bool enum_xtile_config(void)
133 {
134 u32 eax, ebx, ecx, edx;
135
136 __cpuid(TILE_CPUID, TILE_PALETTE_CPUID_SUBLEAVE, &eax, &ebx, &ecx, &edx);
137 if (!eax || !ebx || !ecx)
138 return false;
139
140 xtile.max_names = ebx >> 16;
141 if (xtile.max_names < NUM_TILES)
142 return false;
143
144 xtile.bytes_per_tile = eax >> 16;
145 if (xtile.bytes_per_tile < TILE_SIZE)
146 return false;
147
148 xtile.bytes_per_row = ebx;
149 xtile.max_rows = ecx;
150
151 return true;
152 }
153
enum_xsave_tile(void)154 static bool enum_xsave_tile(void)
155 {
156 u32 eax, ebx, ecx, edx;
157
158 __cpuid(XSTATE_CPUID, XFEATURE_XTILEDATA, &eax, &ebx, &ecx, &edx);
159 if (!eax || !ebx)
160 return false;
161
162 xtile.xsave_offset = ebx;
163 xtile.xsave_size = eax;
164
165 return true;
166 }
167
check_xsave_size(void)168 static bool check_xsave_size(void)
169 {
170 u32 eax, ebx, ecx, edx;
171 bool valid = false;
172
173 __cpuid(XSTATE_CPUID, XSTATE_USER_STATE_SUBLEAVE, &eax, &ebx, &ecx, &edx);
174 if (ebx && ebx <= XSAVE_SIZE)
175 valid = true;
176
177 return valid;
178 }
179
check_xtile_info(void)180 static bool check_xtile_info(void)
181 {
182 bool ret = false;
183
184 if (!check_xsave_size())
185 return ret;
186
187 if (!enum_xsave_tile())
188 return ret;
189
190 if (!enum_xtile_config())
191 return ret;
192
193 if (sizeof(struct tile_data) >= xtile.xsave_size)
194 ret = true;
195
196 return ret;
197 }
198
set_tilecfg(struct tile_config * cfg)199 static void set_tilecfg(struct tile_config *cfg)
200 {
201 int i;
202
203 /* Only palette id 1 */
204 cfg->palette_id = 1;
205 for (i = 0; i < xtile.max_names; i++) {
206 cfg->colsb[i] = xtile.bytes_per_row;
207 cfg->rows[i] = xtile.max_rows;
208 }
209 }
210
set_xstatebv(void * data,uint64_t bv)211 static void set_xstatebv(void *data, uint64_t bv)
212 {
213 *(uint64_t *)(data + XSAVE_HDR_OFFSET) = bv;
214 }
215
get_xstatebv(void * data)216 static u64 get_xstatebv(void *data)
217 {
218 return *(u64 *)(data + XSAVE_HDR_OFFSET);
219 }
220
init_regs(void)221 static void init_regs(void)
222 {
223 uint64_t cr4, xcr0;
224
225 /* turn on CR4.OSXSAVE */
226 cr4 = get_cr4();
227 cr4 |= X86_CR4_OSXSAVE;
228 set_cr4(cr4);
229
230 xcr0 = __xgetbv(0);
231 xcr0 |= XFEATURE_MASK_XTILE;
232 __xsetbv(0x0, xcr0);
233 }
234
guest_code(struct tile_config * amx_cfg,struct tile_data * tiledata,struct xsave_data * xsave_data)235 static void __attribute__((__flatten__)) guest_code(struct tile_config *amx_cfg,
236 struct tile_data *tiledata,
237 struct xsave_data *xsave_data)
238 {
239 init_regs();
240 check_cpuid_xsave();
241 GUEST_ASSERT(check_xsave_supports_xtile());
242 GUEST_ASSERT(check_xtile_info());
243
244 /* check xtile configs */
245 GUEST_ASSERT(xtile.xsave_offset == 2816);
246 GUEST_ASSERT(xtile.xsave_size == 8192);
247 GUEST_ASSERT(xtile.max_names == 8);
248 GUEST_ASSERT(xtile.bytes_per_tile == 1024);
249 GUEST_ASSERT(xtile.bytes_per_row == 64);
250 GUEST_ASSERT(xtile.max_rows == 16);
251 GUEST_SYNC(1);
252
253 /* xfd=0, enable amx */
254 wrmsr(MSR_IA32_XFD, 0);
255 GUEST_SYNC(2);
256 GUEST_ASSERT(rdmsr(MSR_IA32_XFD) == 0);
257 set_tilecfg(amx_cfg);
258 __ldtilecfg(amx_cfg);
259 GUEST_SYNC(3);
260 /* Check save/restore when trap to userspace */
261 __tileloadd(tiledata);
262 GUEST_SYNC(4);
263 __tilerelease();
264 GUEST_SYNC(5);
265 /* bit 18 not in the XCOMP_BV after xsavec() */
266 set_xstatebv(xsave_data, XFEATURE_MASK_XTILEDATA);
267 __xsavec(xsave_data, XFEATURE_MASK_XTILEDATA);
268 GUEST_ASSERT((get_xstatebv(xsave_data) & XFEATURE_MASK_XTILEDATA) == 0);
269
270 /* xfd=0x40000, disable amx tiledata */
271 wrmsr(MSR_IA32_XFD, XFEATURE_MASK_XTILEDATA);
272 GUEST_SYNC(6);
273 GUEST_ASSERT(rdmsr(MSR_IA32_XFD) == XFEATURE_MASK_XTILEDATA);
274 set_tilecfg(amx_cfg);
275 __ldtilecfg(amx_cfg);
276 /* Trigger #NM exception */
277 __tileloadd(tiledata);
278 GUEST_SYNC(10);
279
280 GUEST_DONE();
281 }
282
guest_nm_handler(struct ex_regs * regs)283 void guest_nm_handler(struct ex_regs *regs)
284 {
285 /* Check if #NM is triggered by XFEATURE_MASK_XTILEDATA */
286 GUEST_SYNC(7);
287 GUEST_ASSERT(rdmsr(MSR_IA32_XFD_ERR) == XFEATURE_MASK_XTILEDATA);
288 GUEST_SYNC(8);
289 GUEST_ASSERT(rdmsr(MSR_IA32_XFD_ERR) == XFEATURE_MASK_XTILEDATA);
290 /* Clear xfd_err */
291 wrmsr(MSR_IA32_XFD_ERR, 0);
292 /* xfd=0, enable amx */
293 wrmsr(MSR_IA32_XFD, 0);
294 GUEST_SYNC(9);
295 }
296
main(int argc,char * argv[])297 int main(int argc, char *argv[])
298 {
299 struct kvm_regs regs1, regs2;
300 struct kvm_vcpu *vcpu;
301 struct kvm_vm *vm;
302 struct kvm_run *run;
303 struct kvm_x86_state *state;
304 int xsave_restore_size;
305 vm_vaddr_t amx_cfg, tiledata, xsavedata;
306 struct ucall uc;
307 u32 amx_offset;
308 int stage, ret;
309
310 vm_xsave_require_permission(XSTATE_XTILE_DATA_BIT);
311
312 /* Create VM */
313 vm = vm_create_with_one_vcpu(&vcpu, guest_code);
314
315 TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_XSAVE));
316 TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_AMX_TILE));
317 TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_XTILECFG));
318 TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_XTILEDATA));
319
320 /* Get xsave/restore max size */
321 xsave_restore_size = kvm_get_supported_cpuid_entry(0xd)->ecx;
322
323 run = vcpu->run;
324 vcpu_regs_get(vcpu, ®s1);
325
326 /* Register #NM handler */
327 vm_init_descriptor_tables(vm);
328 vcpu_init_descriptor_tables(vcpu);
329 vm_install_exception_handler(vm, NM_VECTOR, guest_nm_handler);
330
331 /* amx cfg for guest_code */
332 amx_cfg = vm_vaddr_alloc_page(vm);
333 memset(addr_gva2hva(vm, amx_cfg), 0x0, getpagesize());
334
335 /* amx tiledata for guest_code */
336 tiledata = vm_vaddr_alloc_pages(vm, 2);
337 memset(addr_gva2hva(vm, tiledata), rand() | 1, 2 * getpagesize());
338
339 /* xsave data for guest_code */
340 xsavedata = vm_vaddr_alloc_pages(vm, 3);
341 memset(addr_gva2hva(vm, xsavedata), 0, 3 * getpagesize());
342 vcpu_args_set(vcpu, 3, amx_cfg, tiledata, xsavedata);
343
344 for (stage = 1; ; stage++) {
345 vcpu_run(vcpu);
346 TEST_ASSERT(run->exit_reason == KVM_EXIT_IO,
347 "Stage %d: unexpected exit reason: %u (%s),\n",
348 stage, run->exit_reason,
349 exit_reason_str(run->exit_reason));
350
351 switch (get_ucall(vcpu, &uc)) {
352 case UCALL_ABORT:
353 REPORT_GUEST_ASSERT(uc);
354 /* NOT REACHED */
355 case UCALL_SYNC:
356 switch (uc.args[1]) {
357 case 1:
358 case 2:
359 case 3:
360 case 5:
361 case 6:
362 case 7:
363 case 8:
364 fprintf(stderr, "GUEST_SYNC(%ld)\n", uc.args[1]);
365 break;
366 case 4:
367 case 10:
368 fprintf(stderr,
369 "GUEST_SYNC(%ld), check save/restore status\n", uc.args[1]);
370
371 /* Compacted mode, get amx offset by xsave area
372 * size subtract 8K amx size.
373 */
374 amx_offset = xsave_restore_size - NUM_TILES*TILE_SIZE;
375 state = vcpu_save_state(vcpu);
376 void *amx_start = (void *)state->xsave + amx_offset;
377 void *tiles_data = (void *)addr_gva2hva(vm, tiledata);
378 /* Only check TMM0 register, 1 tile */
379 ret = memcmp(amx_start, tiles_data, TILE_SIZE);
380 TEST_ASSERT(ret == 0, "memcmp failed, ret=%d\n", ret);
381 kvm_x86_state_cleanup(state);
382 break;
383 case 9:
384 fprintf(stderr,
385 "GUEST_SYNC(%ld), #NM exception and enable amx\n", uc.args[1]);
386 break;
387 }
388 break;
389 case UCALL_DONE:
390 fprintf(stderr, "UCALL_DONE\n");
391 goto done;
392 default:
393 TEST_FAIL("Unknown ucall %lu", uc.cmd);
394 }
395
396 state = vcpu_save_state(vcpu);
397 memset(®s1, 0, sizeof(regs1));
398 vcpu_regs_get(vcpu, ®s1);
399
400 kvm_vm_release(vm);
401
402 /* Restore state in a new VM. */
403 vcpu = vm_recreate_with_one_vcpu(vm);
404 vcpu_load_state(vcpu, state);
405 run = vcpu->run;
406 kvm_x86_state_cleanup(state);
407
408 memset(®s2, 0, sizeof(regs2));
409 vcpu_regs_get(vcpu, ®s2);
410 TEST_ASSERT(!memcmp(®s1, ®s2, sizeof(regs2)),
411 "Unexpected register values after vcpu_load_state; rdi: %lx rsi: %lx",
412 (ulong) regs2.rdi, (ulong) regs2.rsi);
413 }
414 done:
415 kvm_vm_free(vm);
416 }
417
