1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2021 Western Digital Corporation or its affiliates.
4 * Copyright (C) 2022 Ventana Micro Systems Inc.
5 *
6 * Authors:
7 * Anup Patel <apatel@ventanamicro.com>
8 */
9
10 #include <linux/atomic.h>
11 #include <linux/bitmap.h>
12 #include <linux/kvm_host.h>
13 #include <linux/math.h>
14 #include <linux/spinlock.h>
15 #include <linux/swab.h>
16 #include <kvm/iodev.h>
17 #include <asm/csr.h>
18 #include <asm/kvm_aia_imsic.h>
19
20 #define IMSIC_MAX_EIX (IMSIC_MAX_ID / BITS_PER_TYPE(u64))
21
22 struct imsic_mrif_eix {
23 unsigned long eip[BITS_PER_TYPE(u64) / BITS_PER_LONG];
24 unsigned long eie[BITS_PER_TYPE(u64) / BITS_PER_LONG];
25 };
26
27 struct imsic_mrif {
28 struct imsic_mrif_eix eix[IMSIC_MAX_EIX];
29 unsigned long eithreshold;
30 unsigned long eidelivery;
31 };
32
33 struct imsic {
34 struct kvm_io_device iodev;
35
36 u32 nr_msis;
37 u32 nr_eix;
38 u32 nr_hw_eix;
39
40 /*
41 * At any point in time, the register state is in
42 * one of the following places:
43 *
44 * 1) Hardware: IMSIC VS-file (vsfile_cpu >= 0)
45 * 2) Software: IMSIC SW-file (vsfile_cpu < 0)
46 */
47
48 /* IMSIC VS-file */
49 rwlock_t vsfile_lock;
50 int vsfile_cpu;
51 int vsfile_hgei;
52 void __iomem *vsfile_va;
53 phys_addr_t vsfile_pa;
54
55 /* IMSIC SW-file */
56 struct imsic_mrif *swfile;
57 phys_addr_t swfile_pa;
58 };
59
60 #define imsic_vs_csr_read(__c) \
61 ({ \
62 unsigned long __r; \
63 csr_write(CSR_VSISELECT, __c); \
64 __r = csr_read(CSR_VSIREG); \
65 __r; \
66 })
67
68 #define imsic_read_switchcase(__ireg) \
69 case __ireg: \
70 return imsic_vs_csr_read(__ireg);
71 #define imsic_read_switchcase_2(__ireg) \
72 imsic_read_switchcase(__ireg + 0) \
73 imsic_read_switchcase(__ireg + 1)
74 #define imsic_read_switchcase_4(__ireg) \
75 imsic_read_switchcase_2(__ireg + 0) \
76 imsic_read_switchcase_2(__ireg + 2)
77 #define imsic_read_switchcase_8(__ireg) \
78 imsic_read_switchcase_4(__ireg + 0) \
79 imsic_read_switchcase_4(__ireg + 4)
80 #define imsic_read_switchcase_16(__ireg) \
81 imsic_read_switchcase_8(__ireg + 0) \
82 imsic_read_switchcase_8(__ireg + 8)
83 #define imsic_read_switchcase_32(__ireg) \
84 imsic_read_switchcase_16(__ireg + 0) \
85 imsic_read_switchcase_16(__ireg + 16)
86 #define imsic_read_switchcase_64(__ireg) \
87 imsic_read_switchcase_32(__ireg + 0) \
88 imsic_read_switchcase_32(__ireg + 32)
89
imsic_eix_read(int ireg)90 static unsigned long imsic_eix_read(int ireg)
91 {
92 switch (ireg) {
93 imsic_read_switchcase_64(IMSIC_EIP0)
94 imsic_read_switchcase_64(IMSIC_EIE0)
95 }
96
97 return 0;
98 }
99
100 #define imsic_vs_csr_swap(__c, __v) \
101 ({ \
102 unsigned long __r; \
103 csr_write(CSR_VSISELECT, __c); \
104 __r = csr_swap(CSR_VSIREG, __v); \
105 __r; \
106 })
107
108 #define imsic_swap_switchcase(__ireg, __v) \
109 case __ireg: \
110 return imsic_vs_csr_swap(__ireg, __v);
111 #define imsic_swap_switchcase_2(__ireg, __v) \
112 imsic_swap_switchcase(__ireg + 0, __v) \
113 imsic_swap_switchcase(__ireg + 1, __v)
114 #define imsic_swap_switchcase_4(__ireg, __v) \
115 imsic_swap_switchcase_2(__ireg + 0, __v) \
116 imsic_swap_switchcase_2(__ireg + 2, __v)
117 #define imsic_swap_switchcase_8(__ireg, __v) \
118 imsic_swap_switchcase_4(__ireg + 0, __v) \
119 imsic_swap_switchcase_4(__ireg + 4, __v)
120 #define imsic_swap_switchcase_16(__ireg, __v) \
121 imsic_swap_switchcase_8(__ireg + 0, __v) \
122 imsic_swap_switchcase_8(__ireg + 8, __v)
123 #define imsic_swap_switchcase_32(__ireg, __v) \
124 imsic_swap_switchcase_16(__ireg + 0, __v) \
125 imsic_swap_switchcase_16(__ireg + 16, __v)
126 #define imsic_swap_switchcase_64(__ireg, __v) \
127 imsic_swap_switchcase_32(__ireg + 0, __v) \
128 imsic_swap_switchcase_32(__ireg + 32, __v)
129
imsic_eix_swap(int ireg,unsigned long val)130 static unsigned long imsic_eix_swap(int ireg, unsigned long val)
131 {
132 switch (ireg) {
133 imsic_swap_switchcase_64(IMSIC_EIP0, val)
134 imsic_swap_switchcase_64(IMSIC_EIE0, val)
135 }
136
137 return 0;
138 }
139
140 #define imsic_vs_csr_write(__c, __v) \
141 do { \
142 csr_write(CSR_VSISELECT, __c); \
143 csr_write(CSR_VSIREG, __v); \
144 } while (0)
145
146 #define imsic_write_switchcase(__ireg, __v) \
147 case __ireg: \
148 imsic_vs_csr_write(__ireg, __v); \
149 break;
150 #define imsic_write_switchcase_2(__ireg, __v) \
151 imsic_write_switchcase(__ireg + 0, __v) \
152 imsic_write_switchcase(__ireg + 1, __v)
153 #define imsic_write_switchcase_4(__ireg, __v) \
154 imsic_write_switchcase_2(__ireg + 0, __v) \
155 imsic_write_switchcase_2(__ireg + 2, __v)
156 #define imsic_write_switchcase_8(__ireg, __v) \
157 imsic_write_switchcase_4(__ireg + 0, __v) \
158 imsic_write_switchcase_4(__ireg + 4, __v)
159 #define imsic_write_switchcase_16(__ireg, __v) \
160 imsic_write_switchcase_8(__ireg + 0, __v) \
161 imsic_write_switchcase_8(__ireg + 8, __v)
162 #define imsic_write_switchcase_32(__ireg, __v) \
163 imsic_write_switchcase_16(__ireg + 0, __v) \
164 imsic_write_switchcase_16(__ireg + 16, __v)
165 #define imsic_write_switchcase_64(__ireg, __v) \
166 imsic_write_switchcase_32(__ireg + 0, __v) \
167 imsic_write_switchcase_32(__ireg + 32, __v)
168
imsic_eix_write(int ireg,unsigned long val)169 static void imsic_eix_write(int ireg, unsigned long val)
170 {
171 switch (ireg) {
172 imsic_write_switchcase_64(IMSIC_EIP0, val)
173 imsic_write_switchcase_64(IMSIC_EIE0, val)
174 }
175 }
176
177 #define imsic_vs_csr_set(__c, __v) \
178 do { \
179 csr_write(CSR_VSISELECT, __c); \
180 csr_set(CSR_VSIREG, __v); \
181 } while (0)
182
183 #define imsic_set_switchcase(__ireg, __v) \
184 case __ireg: \
185 imsic_vs_csr_set(__ireg, __v); \
186 break;
187 #define imsic_set_switchcase_2(__ireg, __v) \
188 imsic_set_switchcase(__ireg + 0, __v) \
189 imsic_set_switchcase(__ireg + 1, __v)
190 #define imsic_set_switchcase_4(__ireg, __v) \
191 imsic_set_switchcase_2(__ireg + 0, __v) \
192 imsic_set_switchcase_2(__ireg + 2, __v)
193 #define imsic_set_switchcase_8(__ireg, __v) \
194 imsic_set_switchcase_4(__ireg + 0, __v) \
195 imsic_set_switchcase_4(__ireg + 4, __v)
196 #define imsic_set_switchcase_16(__ireg, __v) \
197 imsic_set_switchcase_8(__ireg + 0, __v) \
198 imsic_set_switchcase_8(__ireg + 8, __v)
199 #define imsic_set_switchcase_32(__ireg, __v) \
200 imsic_set_switchcase_16(__ireg + 0, __v) \
201 imsic_set_switchcase_16(__ireg + 16, __v)
202 #define imsic_set_switchcase_64(__ireg, __v) \
203 imsic_set_switchcase_32(__ireg + 0, __v) \
204 imsic_set_switchcase_32(__ireg + 32, __v)
205
imsic_eix_set(int ireg,unsigned long val)206 static void imsic_eix_set(int ireg, unsigned long val)
207 {
208 switch (ireg) {
209 imsic_set_switchcase_64(IMSIC_EIP0, val)
210 imsic_set_switchcase_64(IMSIC_EIE0, val)
211 }
212 }
213
imsic_mrif_atomic_rmw(struct imsic_mrif * mrif,unsigned long * ptr,unsigned long new_val,unsigned long wr_mask)214 static unsigned long imsic_mrif_atomic_rmw(struct imsic_mrif *mrif,
215 unsigned long *ptr,
216 unsigned long new_val,
217 unsigned long wr_mask)
218 {
219 unsigned long old_val = 0, tmp = 0;
220
221 __asm__ __volatile__ (
222 "0: lr.w.aq %1, %0\n"
223 " and %2, %1, %3\n"
224 " or %2, %2, %4\n"
225 " sc.w.rl %2, %2, %0\n"
226 " bnez %2, 0b"
227 : "+A" (*ptr), "+r" (old_val), "+r" (tmp)
228 : "r" (~wr_mask), "r" (new_val & wr_mask)
229 : "memory");
230
231 return old_val;
232 }
233
imsic_mrif_atomic_or(struct imsic_mrif * mrif,unsigned long * ptr,unsigned long val)234 static unsigned long imsic_mrif_atomic_or(struct imsic_mrif *mrif,
235 unsigned long *ptr,
236 unsigned long val)
237 {
238 return atomic_long_fetch_or(val, (atomic_long_t *)ptr);
239 }
240
241 #define imsic_mrif_atomic_write(__mrif, __ptr, __new_val) \
242 imsic_mrif_atomic_rmw(__mrif, __ptr, __new_val, -1UL)
243 #define imsic_mrif_atomic_read(__mrif, __ptr) \
244 imsic_mrif_atomic_or(__mrif, __ptr, 0)
245
imsic_mrif_topei(struct imsic_mrif * mrif,u32 nr_eix,u32 nr_msis)246 static u32 imsic_mrif_topei(struct imsic_mrif *mrif, u32 nr_eix, u32 nr_msis)
247 {
248 struct imsic_mrif_eix *eix;
249 u32 i, imin, imax, ei, max_msi;
250 unsigned long eipend[BITS_PER_TYPE(u64) / BITS_PER_LONG];
251 unsigned long eithreshold = imsic_mrif_atomic_read(mrif,
252 &mrif->eithreshold);
253
254 max_msi = (eithreshold && (eithreshold <= nr_msis)) ?
255 eithreshold : nr_msis;
256 for (ei = 0; ei < nr_eix; ei++) {
257 eix = &mrif->eix[ei];
258 eipend[0] = imsic_mrif_atomic_read(mrif, &eix->eie[0]) &
259 imsic_mrif_atomic_read(mrif, &eix->eip[0]);
260 #ifdef CONFIG_32BIT
261 eipend[1] = imsic_mrif_atomic_read(mrif, &eix->eie[1]) &
262 imsic_mrif_atomic_read(mrif, &eix->eip[1]);
263 if (!eipend[0] && !eipend[1])
264 #else
265 if (!eipend[0])
266 #endif
267 continue;
268
269 imin = ei * BITS_PER_TYPE(u64);
270 imax = ((imin + BITS_PER_TYPE(u64)) < max_msi) ?
271 imin + BITS_PER_TYPE(u64) : max_msi;
272 for (i = (!imin) ? 1 : imin; i < imax; i++) {
273 if (test_bit(i - imin, eipend))
274 return (i << TOPEI_ID_SHIFT) | i;
275 }
276 }
277
278 return 0;
279 }
280
imsic_mrif_isel_check(u32 nr_eix,unsigned long isel)281 static int imsic_mrif_isel_check(u32 nr_eix, unsigned long isel)
282 {
283 u32 num = 0;
284
285 switch (isel) {
286 case IMSIC_EIDELIVERY:
287 case IMSIC_EITHRESHOLD:
288 break;
289 case IMSIC_EIP0 ... IMSIC_EIP63:
290 num = isel - IMSIC_EIP0;
291 break;
292 case IMSIC_EIE0 ... IMSIC_EIE63:
293 num = isel - IMSIC_EIE0;
294 break;
295 default:
296 return -ENOENT;
297 }
298 #ifndef CONFIG_32BIT
299 if (num & 0x1)
300 return -EINVAL;
301 #endif
302 if ((num / 2) >= nr_eix)
303 return -EINVAL;
304
305 return 0;
306 }
307
imsic_mrif_rmw(struct imsic_mrif * mrif,u32 nr_eix,unsigned long isel,unsigned long * val,unsigned long new_val,unsigned long wr_mask)308 static int imsic_mrif_rmw(struct imsic_mrif *mrif, u32 nr_eix,
309 unsigned long isel, unsigned long *val,
310 unsigned long new_val, unsigned long wr_mask)
311 {
312 bool pend;
313 struct imsic_mrif_eix *eix;
314 unsigned long *ei, num, old_val = 0;
315
316 switch (isel) {
317 case IMSIC_EIDELIVERY:
318 old_val = imsic_mrif_atomic_rmw(mrif, &mrif->eidelivery,
319 new_val, wr_mask & 0x1);
320 break;
321 case IMSIC_EITHRESHOLD:
322 old_val = imsic_mrif_atomic_rmw(mrif, &mrif->eithreshold,
323 new_val, wr_mask & (IMSIC_MAX_ID - 1));
324 break;
325 case IMSIC_EIP0 ... IMSIC_EIP63:
326 case IMSIC_EIE0 ... IMSIC_EIE63:
327 if (isel >= IMSIC_EIP0 && isel <= IMSIC_EIP63) {
328 pend = true;
329 num = isel - IMSIC_EIP0;
330 } else {
331 pend = false;
332 num = isel - IMSIC_EIE0;
333 }
334
335 if ((num / 2) >= nr_eix)
336 return -EINVAL;
337 eix = &mrif->eix[num / 2];
338
339 #ifndef CONFIG_32BIT
340 if (num & 0x1)
341 return -EINVAL;
342 ei = (pend) ? &eix->eip[0] : &eix->eie[0];
343 #else
344 ei = (pend) ? &eix->eip[num & 0x1] : &eix->eie[num & 0x1];
345 #endif
346
347 /* Bit0 of EIP0 or EIE0 is read-only */
348 if (!num)
349 wr_mask &= ~BIT(0);
350
351 old_val = imsic_mrif_atomic_rmw(mrif, ei, new_val, wr_mask);
352 break;
353 default:
354 return -ENOENT;
355 }
356
357 if (val)
358 *val = old_val;
359
360 return 0;
361 }
362
363 struct imsic_vsfile_read_data {
364 int hgei;
365 u32 nr_eix;
366 bool clear;
367 struct imsic_mrif *mrif;
368 };
369
imsic_vsfile_local_read(void * data)370 static void imsic_vsfile_local_read(void *data)
371 {
372 u32 i;
373 struct imsic_mrif_eix *eix;
374 struct imsic_vsfile_read_data *idata = data;
375 struct imsic_mrif *mrif = idata->mrif;
376 unsigned long new_hstatus, old_hstatus, old_vsiselect;
377
378 old_vsiselect = csr_read(CSR_VSISELECT);
379 old_hstatus = csr_read(CSR_HSTATUS);
380 new_hstatus = old_hstatus & ~HSTATUS_VGEIN;
381 new_hstatus |= ((unsigned long)idata->hgei) << HSTATUS_VGEIN_SHIFT;
382 csr_write(CSR_HSTATUS, new_hstatus);
383
384 /*
385 * We don't use imsic_mrif_atomic_xyz() functions to store
386 * values in MRIF because imsic_vsfile_read() is always called
387 * with pointer to temporary MRIF on stack.
388 */
389
390 if (idata->clear) {
391 mrif->eidelivery = imsic_vs_csr_swap(IMSIC_EIDELIVERY, 0);
392 mrif->eithreshold = imsic_vs_csr_swap(IMSIC_EITHRESHOLD, 0);
393 for (i = 0; i < idata->nr_eix; i++) {
394 eix = &mrif->eix[i];
395 eix->eip[0] = imsic_eix_swap(IMSIC_EIP0 + i * 2, 0);
396 eix->eie[0] = imsic_eix_swap(IMSIC_EIE0 + i * 2, 0);
397 #ifdef CONFIG_32BIT
398 eix->eip[1] = imsic_eix_swap(IMSIC_EIP0 + i * 2 + 1, 0);
399 eix->eie[1] = imsic_eix_swap(IMSIC_EIE0 + i * 2 + 1, 0);
400 #endif
401 }
402 } else {
403 mrif->eidelivery = imsic_vs_csr_read(IMSIC_EIDELIVERY);
404 mrif->eithreshold = imsic_vs_csr_read(IMSIC_EITHRESHOLD);
405 for (i = 0; i < idata->nr_eix; i++) {
406 eix = &mrif->eix[i];
407 eix->eip[0] = imsic_eix_read(IMSIC_EIP0 + i * 2);
408 eix->eie[0] = imsic_eix_read(IMSIC_EIE0 + i * 2);
409 #ifdef CONFIG_32BIT
410 eix->eip[1] = imsic_eix_read(IMSIC_EIP0 + i * 2 + 1);
411 eix->eie[1] = imsic_eix_read(IMSIC_EIE0 + i * 2 + 1);
412 #endif
413 }
414 }
415
416 csr_write(CSR_HSTATUS, old_hstatus);
417 csr_write(CSR_VSISELECT, old_vsiselect);
418 }
419
imsic_vsfile_read(int vsfile_hgei,int vsfile_cpu,u32 nr_eix,bool clear,struct imsic_mrif * mrif)420 static void imsic_vsfile_read(int vsfile_hgei, int vsfile_cpu, u32 nr_eix,
421 bool clear, struct imsic_mrif *mrif)
422 {
423 struct imsic_vsfile_read_data idata;
424
425 /* We can only read clear if we have a IMSIC VS-file */
426 if (vsfile_cpu < 0 || vsfile_hgei <= 0)
427 return;
428
429 /* We can only read clear on local CPU */
430 idata.hgei = vsfile_hgei;
431 idata.nr_eix = nr_eix;
432 idata.clear = clear;
433 idata.mrif = mrif;
434 on_each_cpu_mask(cpumask_of(vsfile_cpu),
435 imsic_vsfile_local_read, &idata, 1);
436 }
437
438 struct imsic_vsfile_rw_data {
439 int hgei;
440 int isel;
441 bool write;
442 unsigned long val;
443 };
444
imsic_vsfile_local_rw(void * data)445 static void imsic_vsfile_local_rw(void *data)
446 {
447 struct imsic_vsfile_rw_data *idata = data;
448 unsigned long new_hstatus, old_hstatus, old_vsiselect;
449
450 old_vsiselect = csr_read(CSR_VSISELECT);
451 old_hstatus = csr_read(CSR_HSTATUS);
452 new_hstatus = old_hstatus & ~HSTATUS_VGEIN;
453 new_hstatus |= ((unsigned long)idata->hgei) << HSTATUS_VGEIN_SHIFT;
454 csr_write(CSR_HSTATUS, new_hstatus);
455
456 switch (idata->isel) {
457 case IMSIC_EIDELIVERY:
458 if (idata->write)
459 imsic_vs_csr_write(IMSIC_EIDELIVERY, idata->val);
460 else
461 idata->val = imsic_vs_csr_read(IMSIC_EIDELIVERY);
462 break;
463 case IMSIC_EITHRESHOLD:
464 if (idata->write)
465 imsic_vs_csr_write(IMSIC_EITHRESHOLD, idata->val);
466 else
467 idata->val = imsic_vs_csr_read(IMSIC_EITHRESHOLD);
468 break;
469 case IMSIC_EIP0 ... IMSIC_EIP63:
470 case IMSIC_EIE0 ... IMSIC_EIE63:
471 #ifndef CONFIG_32BIT
472 if (idata->isel & 0x1)
473 break;
474 #endif
475 if (idata->write)
476 imsic_eix_write(idata->isel, idata->val);
477 else
478 idata->val = imsic_eix_read(idata->isel);
479 break;
480 default:
481 break;
482 }
483
484 csr_write(CSR_HSTATUS, old_hstatus);
485 csr_write(CSR_VSISELECT, old_vsiselect);
486 }
487
imsic_vsfile_rw(int vsfile_hgei,int vsfile_cpu,u32 nr_eix,unsigned long isel,bool write,unsigned long * val)488 static int imsic_vsfile_rw(int vsfile_hgei, int vsfile_cpu, u32 nr_eix,
489 unsigned long isel, bool write,
490 unsigned long *val)
491 {
492 int rc;
493 struct imsic_vsfile_rw_data rdata;
494
495 /* We can only access register if we have a IMSIC VS-file */
496 if (vsfile_cpu < 0 || vsfile_hgei <= 0)
497 return -EINVAL;
498
499 /* Check IMSIC register iselect */
500 rc = imsic_mrif_isel_check(nr_eix, isel);
501 if (rc)
502 return rc;
503
504 /* We can only access register on local CPU */
505 rdata.hgei = vsfile_hgei;
506 rdata.isel = isel;
507 rdata.write = write;
508 rdata.val = (write) ? *val : 0;
509 on_each_cpu_mask(cpumask_of(vsfile_cpu),
510 imsic_vsfile_local_rw, &rdata, 1);
511
512 if (!write)
513 *val = rdata.val;
514
515 return 0;
516 }
517
imsic_vsfile_local_clear(int vsfile_hgei,u32 nr_eix)518 static void imsic_vsfile_local_clear(int vsfile_hgei, u32 nr_eix)
519 {
520 u32 i;
521 unsigned long new_hstatus, old_hstatus, old_vsiselect;
522
523 /* We can only zero-out if we have a IMSIC VS-file */
524 if (vsfile_hgei <= 0)
525 return;
526
527 old_vsiselect = csr_read(CSR_VSISELECT);
528 old_hstatus = csr_read(CSR_HSTATUS);
529 new_hstatus = old_hstatus & ~HSTATUS_VGEIN;
530 new_hstatus |= ((unsigned long)vsfile_hgei) << HSTATUS_VGEIN_SHIFT;
531 csr_write(CSR_HSTATUS, new_hstatus);
532
533 imsic_vs_csr_write(IMSIC_EIDELIVERY, 0);
534 imsic_vs_csr_write(IMSIC_EITHRESHOLD, 0);
535 for (i = 0; i < nr_eix; i++) {
536 imsic_eix_write(IMSIC_EIP0 + i * 2, 0);
537 imsic_eix_write(IMSIC_EIE0 + i * 2, 0);
538 #ifdef CONFIG_32BIT
539 imsic_eix_write(IMSIC_EIP0 + i * 2 + 1, 0);
540 imsic_eix_write(IMSIC_EIE0 + i * 2 + 1, 0);
541 #endif
542 }
543
544 csr_write(CSR_HSTATUS, old_hstatus);
545 csr_write(CSR_VSISELECT, old_vsiselect);
546 }
547
imsic_vsfile_local_update(int vsfile_hgei,u32 nr_eix,struct imsic_mrif * mrif)548 static void imsic_vsfile_local_update(int vsfile_hgei, u32 nr_eix,
549 struct imsic_mrif *mrif)
550 {
551 u32 i;
552 struct imsic_mrif_eix *eix;
553 unsigned long new_hstatus, old_hstatus, old_vsiselect;
554
555 /* We can only update if we have a HW IMSIC context */
556 if (vsfile_hgei <= 0)
557 return;
558
559 /*
560 * We don't use imsic_mrif_atomic_xyz() functions to read values
561 * from MRIF in this function because it is always called with
562 * pointer to temporary MRIF on stack.
563 */
564
565 old_vsiselect = csr_read(CSR_VSISELECT);
566 old_hstatus = csr_read(CSR_HSTATUS);
567 new_hstatus = old_hstatus & ~HSTATUS_VGEIN;
568 new_hstatus |= ((unsigned long)vsfile_hgei) << HSTATUS_VGEIN_SHIFT;
569 csr_write(CSR_HSTATUS, new_hstatus);
570
571 for (i = 0; i < nr_eix; i++) {
572 eix = &mrif->eix[i];
573 imsic_eix_set(IMSIC_EIP0 + i * 2, eix->eip[0]);
574 imsic_eix_set(IMSIC_EIE0 + i * 2, eix->eie[0]);
575 #ifdef CONFIG_32BIT
576 imsic_eix_set(IMSIC_EIP0 + i * 2 + 1, eix->eip[1]);
577 imsic_eix_set(IMSIC_EIE0 + i * 2 + 1, eix->eie[1]);
578 #endif
579 }
580 imsic_vs_csr_write(IMSIC_EITHRESHOLD, mrif->eithreshold);
581 imsic_vs_csr_write(IMSIC_EIDELIVERY, mrif->eidelivery);
582
583 csr_write(CSR_HSTATUS, old_hstatus);
584 csr_write(CSR_VSISELECT, old_vsiselect);
585 }
586
imsic_vsfile_cleanup(struct imsic * imsic)587 static void imsic_vsfile_cleanup(struct imsic *imsic)
588 {
589 int old_vsfile_hgei, old_vsfile_cpu;
590 unsigned long flags;
591
592 /*
593 * We don't use imsic_mrif_atomic_xyz() functions to clear the
594 * SW-file in this function because it is always called when the
595 * VCPU is being destroyed.
596 */
597
598 write_lock_irqsave(&imsic->vsfile_lock, flags);
599 old_vsfile_hgei = imsic->vsfile_hgei;
600 old_vsfile_cpu = imsic->vsfile_cpu;
601 imsic->vsfile_cpu = imsic->vsfile_hgei = -1;
602 imsic->vsfile_va = NULL;
603 imsic->vsfile_pa = 0;
604 write_unlock_irqrestore(&imsic->vsfile_lock, flags);
605
606 memset(imsic->swfile, 0, sizeof(*imsic->swfile));
607
608 if (old_vsfile_cpu >= 0)
609 kvm_riscv_aia_free_hgei(old_vsfile_cpu, old_vsfile_hgei);
610 }
611
imsic_swfile_extirq_update(struct kvm_vcpu * vcpu)612 static void imsic_swfile_extirq_update(struct kvm_vcpu *vcpu)
613 {
614 struct imsic *imsic = vcpu->arch.aia_context.imsic_state;
615 struct imsic_mrif *mrif = imsic->swfile;
616
617 if (imsic_mrif_atomic_read(mrif, &mrif->eidelivery) &&
618 imsic_mrif_topei(mrif, imsic->nr_eix, imsic->nr_msis))
619 kvm_riscv_vcpu_set_interrupt(vcpu, IRQ_VS_EXT);
620 else
621 kvm_riscv_vcpu_unset_interrupt(vcpu, IRQ_VS_EXT);
622 }
623
imsic_swfile_read(struct kvm_vcpu * vcpu,bool clear,struct imsic_mrif * mrif)624 static void imsic_swfile_read(struct kvm_vcpu *vcpu, bool clear,
625 struct imsic_mrif *mrif)
626 {
627 struct imsic *imsic = vcpu->arch.aia_context.imsic_state;
628
629 /*
630 * We don't use imsic_mrif_atomic_xyz() functions to read and
631 * write SW-file and MRIF in this function because it is always
632 * called when VCPU is not using SW-file and the MRIF points to
633 * a temporary MRIF on stack.
634 */
635
636 memcpy(mrif, imsic->swfile, sizeof(*mrif));
637 if (clear) {
638 memset(imsic->swfile, 0, sizeof(*imsic->swfile));
639 kvm_riscv_vcpu_unset_interrupt(vcpu, IRQ_VS_EXT);
640 }
641 }
642
imsic_swfile_update(struct kvm_vcpu * vcpu,struct imsic_mrif * mrif)643 static void imsic_swfile_update(struct kvm_vcpu *vcpu,
644 struct imsic_mrif *mrif)
645 {
646 u32 i;
647 struct imsic_mrif_eix *seix, *eix;
648 struct imsic *imsic = vcpu->arch.aia_context.imsic_state;
649 struct imsic_mrif *smrif = imsic->swfile;
650
651 imsic_mrif_atomic_write(smrif, &smrif->eidelivery, mrif->eidelivery);
652 imsic_mrif_atomic_write(smrif, &smrif->eithreshold, mrif->eithreshold);
653 for (i = 0; i < imsic->nr_eix; i++) {
654 seix = &smrif->eix[i];
655 eix = &mrif->eix[i];
656 imsic_mrif_atomic_or(smrif, &seix->eip[0], eix->eip[0]);
657 imsic_mrif_atomic_or(smrif, &seix->eie[0], eix->eie[0]);
658 #ifdef CONFIG_32BIT
659 imsic_mrif_atomic_or(smrif, &seix->eip[1], eix->eip[1]);
660 imsic_mrif_atomic_or(smrif, &seix->eie[1], eix->eie[1]);
661 #endif
662 }
663
664 imsic_swfile_extirq_update(vcpu);
665 }
666
kvm_riscv_vcpu_aia_imsic_release(struct kvm_vcpu * vcpu)667 void kvm_riscv_vcpu_aia_imsic_release(struct kvm_vcpu *vcpu)
668 {
669 unsigned long flags;
670 struct imsic_mrif tmrif;
671 int old_vsfile_hgei, old_vsfile_cpu;
672 struct imsic *imsic = vcpu->arch.aia_context.imsic_state;
673
674 /* Read and clear IMSIC VS-file details */
675 write_lock_irqsave(&imsic->vsfile_lock, flags);
676 old_vsfile_hgei = imsic->vsfile_hgei;
677 old_vsfile_cpu = imsic->vsfile_cpu;
678 imsic->vsfile_cpu = imsic->vsfile_hgei = -1;
679 imsic->vsfile_va = NULL;
680 imsic->vsfile_pa = 0;
681 write_unlock_irqrestore(&imsic->vsfile_lock, flags);
682
683 /* Do nothing, if no IMSIC VS-file to release */
684 if (old_vsfile_cpu < 0)
685 return;
686
687 /*
688 * At this point, all interrupt producers are still using
689 * the old IMSIC VS-file so we first re-direct all interrupt
690 * producers.
691 */
692
693 /* Purge the G-stage mapping */
694 kvm_riscv_gstage_iounmap(vcpu->kvm,
695 vcpu->arch.aia_context.imsic_addr,
696 IMSIC_MMIO_PAGE_SZ);
697
698 /* TODO: Purge the IOMMU mapping ??? */
699
700 /*
701 * At this point, all interrupt producers have been re-directed
702 * to somewhere else so we move register state from the old IMSIC
703 * VS-file to the IMSIC SW-file.
704 */
705
706 /* Read and clear register state from old IMSIC VS-file */
707 memset(&tmrif, 0, sizeof(tmrif));
708 imsic_vsfile_read(old_vsfile_hgei, old_vsfile_cpu, imsic->nr_hw_eix,
709 true, &tmrif);
710
711 /* Update register state in IMSIC SW-file */
712 imsic_swfile_update(vcpu, &tmrif);
713
714 /* Free-up old IMSIC VS-file */
715 kvm_riscv_aia_free_hgei(old_vsfile_cpu, old_vsfile_hgei);
716 }
717
kvm_riscv_vcpu_aia_imsic_update(struct kvm_vcpu * vcpu)718 int kvm_riscv_vcpu_aia_imsic_update(struct kvm_vcpu *vcpu)
719 {
720 unsigned long flags;
721 phys_addr_t new_vsfile_pa;
722 struct imsic_mrif tmrif;
723 void __iomem *new_vsfile_va;
724 struct kvm *kvm = vcpu->kvm;
725 struct kvm_run *run = vcpu->run;
726 struct kvm_vcpu_aia *vaia = &vcpu->arch.aia_context;
727 struct imsic *imsic = vaia->imsic_state;
728 int ret = 0, new_vsfile_hgei = -1, old_vsfile_hgei, old_vsfile_cpu;
729
730 /* Do nothing for emulation mode */
731 if (kvm->arch.aia.mode == KVM_DEV_RISCV_AIA_MODE_EMUL)
732 return 1;
733
734 /* Read old IMSIC VS-file details */
735 read_lock_irqsave(&imsic->vsfile_lock, flags);
736 old_vsfile_hgei = imsic->vsfile_hgei;
737 old_vsfile_cpu = imsic->vsfile_cpu;
738 read_unlock_irqrestore(&imsic->vsfile_lock, flags);
739
740 /* Do nothing if we are continuing on same CPU */
741 if (old_vsfile_cpu == vcpu->cpu)
742 return 1;
743
744 /* Allocate new IMSIC VS-file */
745 ret = kvm_riscv_aia_alloc_hgei(vcpu->cpu, vcpu,
746 &new_vsfile_va, &new_vsfile_pa);
747 if (ret <= 0) {
748 /* For HW acceleration mode, we can't continue */
749 if (kvm->arch.aia.mode == KVM_DEV_RISCV_AIA_MODE_HWACCEL) {
750 run->fail_entry.hardware_entry_failure_reason =
751 CSR_HSTATUS;
752 run->fail_entry.cpu = vcpu->cpu;
753 run->exit_reason = KVM_EXIT_FAIL_ENTRY;
754 return 0;
755 }
756
757 /* Release old IMSIC VS-file */
758 if (old_vsfile_cpu >= 0)
759 kvm_riscv_vcpu_aia_imsic_release(vcpu);
760
761 /* For automatic mode, we continue */
762 goto done;
763 }
764 new_vsfile_hgei = ret;
765
766 /*
767 * At this point, all interrupt producers are still using
768 * to the old IMSIC VS-file so we first move all interrupt
769 * producers to the new IMSIC VS-file.
770 */
771
772 /* Zero-out new IMSIC VS-file */
773 imsic_vsfile_local_clear(new_vsfile_hgei, imsic->nr_hw_eix);
774
775 /* Update G-stage mapping for the new IMSIC VS-file */
776 ret = kvm_riscv_gstage_ioremap(kvm, vcpu->arch.aia_context.imsic_addr,
777 new_vsfile_pa, IMSIC_MMIO_PAGE_SZ,
778 true, true);
779 if (ret)
780 goto fail_free_vsfile_hgei;
781
782 /* TODO: Update the IOMMU mapping ??? */
783
784 /* Update new IMSIC VS-file details in IMSIC context */
785 write_lock_irqsave(&imsic->vsfile_lock, flags);
786 imsic->vsfile_hgei = new_vsfile_hgei;
787 imsic->vsfile_cpu = vcpu->cpu;
788 imsic->vsfile_va = new_vsfile_va;
789 imsic->vsfile_pa = new_vsfile_pa;
790 write_unlock_irqrestore(&imsic->vsfile_lock, flags);
791
792 /*
793 * At this point, all interrupt producers have been moved
794 * to the new IMSIC VS-file so we move register state from
795 * the old IMSIC VS/SW-file to the new IMSIC VS-file.
796 */
797
798 memset(&tmrif, 0, sizeof(tmrif));
799 if (old_vsfile_cpu >= 0) {
800 /* Read and clear register state from old IMSIC VS-file */
801 imsic_vsfile_read(old_vsfile_hgei, old_vsfile_cpu,
802 imsic->nr_hw_eix, true, &tmrif);
803
804 /* Free-up old IMSIC VS-file */
805 kvm_riscv_aia_free_hgei(old_vsfile_cpu, old_vsfile_hgei);
806 } else {
807 /* Read and clear register state from IMSIC SW-file */
808 imsic_swfile_read(vcpu, true, &tmrif);
809 }
810
811 /* Restore register state in the new IMSIC VS-file */
812 imsic_vsfile_local_update(new_vsfile_hgei, imsic->nr_hw_eix, &tmrif);
813
814 done:
815 /* Set VCPU HSTATUS.VGEIN to new IMSIC VS-file */
816 vcpu->arch.guest_context.hstatus &= ~HSTATUS_VGEIN;
817 if (new_vsfile_hgei > 0)
818 vcpu->arch.guest_context.hstatus |=
819 ((unsigned long)new_vsfile_hgei) << HSTATUS_VGEIN_SHIFT;
820
821 /* Continue run-loop */
822 return 1;
823
824 fail_free_vsfile_hgei:
825 kvm_riscv_aia_free_hgei(vcpu->cpu, new_vsfile_hgei);
826 return ret;
827 }
828
kvm_riscv_vcpu_aia_imsic_rmw(struct kvm_vcpu * vcpu,unsigned long isel,unsigned long * val,unsigned long new_val,unsigned long wr_mask)829 int kvm_riscv_vcpu_aia_imsic_rmw(struct kvm_vcpu *vcpu, unsigned long isel,
830 unsigned long *val, unsigned long new_val,
831 unsigned long wr_mask)
832 {
833 u32 topei;
834 struct imsic_mrif_eix *eix;
835 int r, rc = KVM_INSN_CONTINUE_NEXT_SEPC;
836 struct imsic *imsic = vcpu->arch.aia_context.imsic_state;
837
838 if (isel == KVM_RISCV_AIA_IMSIC_TOPEI) {
839 /* Read pending and enabled interrupt with highest priority */
840 topei = imsic_mrif_topei(imsic->swfile, imsic->nr_eix,
841 imsic->nr_msis);
842 if (val)
843 *val = topei;
844
845 /* Writes ignore value and clear top pending interrupt */
846 if (topei && wr_mask) {
847 topei >>= TOPEI_ID_SHIFT;
848 if (topei) {
849 eix = &imsic->swfile->eix[topei /
850 BITS_PER_TYPE(u64)];
851 clear_bit(topei & (BITS_PER_TYPE(u64) - 1),
852 eix->eip);
853 }
854 }
855 } else {
856 r = imsic_mrif_rmw(imsic->swfile, imsic->nr_eix, isel,
857 val, new_val, wr_mask);
858 /* Forward unknown IMSIC register to user-space */
859 if (r)
860 rc = (r == -ENOENT) ? 0 : KVM_INSN_ILLEGAL_TRAP;
861 }
862
863 if (wr_mask)
864 imsic_swfile_extirq_update(vcpu);
865
866 return rc;
867 }
868
kvm_riscv_aia_imsic_rw_attr(struct kvm * kvm,unsigned long type,bool write,unsigned long * val)869 int kvm_riscv_aia_imsic_rw_attr(struct kvm *kvm, unsigned long type,
870 bool write, unsigned long *val)
871 {
872 u32 isel, vcpu_id;
873 unsigned long flags;
874 struct imsic *imsic;
875 struct kvm_vcpu *vcpu;
876 int rc, vsfile_hgei, vsfile_cpu;
877
878 if (!kvm_riscv_aia_initialized(kvm))
879 return -ENODEV;
880
881 vcpu_id = KVM_DEV_RISCV_AIA_IMSIC_GET_VCPU(type);
882 vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);
883 if (!vcpu)
884 return -ENODEV;
885
886 isel = KVM_DEV_RISCV_AIA_IMSIC_GET_ISEL(type);
887 imsic = vcpu->arch.aia_context.imsic_state;
888
889 read_lock_irqsave(&imsic->vsfile_lock, flags);
890
891 rc = 0;
892 vsfile_hgei = imsic->vsfile_hgei;
893 vsfile_cpu = imsic->vsfile_cpu;
894 if (vsfile_cpu < 0) {
895 if (write) {
896 rc = imsic_mrif_rmw(imsic->swfile, imsic->nr_eix,
897 isel, NULL, *val, -1UL);
898 imsic_swfile_extirq_update(vcpu);
899 } else
900 rc = imsic_mrif_rmw(imsic->swfile, imsic->nr_eix,
901 isel, val, 0, 0);
902 }
903
904 read_unlock_irqrestore(&imsic->vsfile_lock, flags);
905
906 if (!rc && vsfile_cpu >= 0)
907 rc = imsic_vsfile_rw(vsfile_hgei, vsfile_cpu, imsic->nr_eix,
908 isel, write, val);
909
910 return rc;
911 }
912
kvm_riscv_aia_imsic_has_attr(struct kvm * kvm,unsigned long type)913 int kvm_riscv_aia_imsic_has_attr(struct kvm *kvm, unsigned long type)
914 {
915 u32 isel, vcpu_id;
916 struct imsic *imsic;
917 struct kvm_vcpu *vcpu;
918
919 if (!kvm_riscv_aia_initialized(kvm))
920 return -ENODEV;
921
922 vcpu_id = KVM_DEV_RISCV_AIA_IMSIC_GET_VCPU(type);
923 vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);
924 if (!vcpu)
925 return -ENODEV;
926
927 isel = KVM_DEV_RISCV_AIA_IMSIC_GET_ISEL(type);
928 imsic = vcpu->arch.aia_context.imsic_state;
929 return imsic_mrif_isel_check(imsic->nr_eix, isel);
930 }
931
kvm_riscv_vcpu_aia_imsic_reset(struct kvm_vcpu * vcpu)932 void kvm_riscv_vcpu_aia_imsic_reset(struct kvm_vcpu *vcpu)
933 {
934 struct imsic *imsic = vcpu->arch.aia_context.imsic_state;
935
936 if (!imsic)
937 return;
938
939 kvm_riscv_vcpu_aia_imsic_release(vcpu);
940
941 memset(imsic->swfile, 0, sizeof(*imsic->swfile));
942 }
943
kvm_riscv_vcpu_aia_imsic_inject(struct kvm_vcpu * vcpu,u32 guest_index,u32 offset,u32 iid)944 int kvm_riscv_vcpu_aia_imsic_inject(struct kvm_vcpu *vcpu,
945 u32 guest_index, u32 offset, u32 iid)
946 {
947 unsigned long flags;
948 struct imsic_mrif_eix *eix;
949 struct imsic *imsic = vcpu->arch.aia_context.imsic_state;
950
951 /* We only emulate one IMSIC MMIO page for each Guest VCPU */
952 if (!imsic || !iid || guest_index ||
953 (offset != IMSIC_MMIO_SETIPNUM_LE &&
954 offset != IMSIC_MMIO_SETIPNUM_BE))
955 return -ENODEV;
956
957 iid = (offset == IMSIC_MMIO_SETIPNUM_BE) ? __swab32(iid) : iid;
958 if (imsic->nr_msis <= iid)
959 return -EINVAL;
960
961 read_lock_irqsave(&imsic->vsfile_lock, flags);
962
963 if (imsic->vsfile_cpu >= 0) {
964 writel(iid, imsic->vsfile_va + IMSIC_MMIO_SETIPNUM_LE);
965 kvm_vcpu_kick(vcpu);
966 } else {
967 eix = &imsic->swfile->eix[iid / BITS_PER_TYPE(u64)];
968 set_bit(iid & (BITS_PER_TYPE(u64) - 1), eix->eip);
969 imsic_swfile_extirq_update(vcpu);
970 }
971
972 read_unlock_irqrestore(&imsic->vsfile_lock, flags);
973
974 return 0;
975 }
976
imsic_mmio_read(struct kvm_vcpu * vcpu,struct kvm_io_device * dev,gpa_t addr,int len,void * val)977 static int imsic_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
978 gpa_t addr, int len, void *val)
979 {
980 if (len != 4 || (addr & 0x3) != 0)
981 return -EOPNOTSUPP;
982
983 *((u32 *)val) = 0;
984
985 return 0;
986 }
987
imsic_mmio_write(struct kvm_vcpu * vcpu,struct kvm_io_device * dev,gpa_t addr,int len,const void * val)988 static int imsic_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
989 gpa_t addr, int len, const void *val)
990 {
991 struct kvm_msi msi = { 0 };
992
993 if (len != 4 || (addr & 0x3) != 0)
994 return -EOPNOTSUPP;
995
996 msi.address_hi = addr >> 32;
997 msi.address_lo = (u32)addr;
998 msi.data = *((const u32 *)val);
999 kvm_riscv_aia_inject_msi(vcpu->kvm, &msi);
1000
1001 return 0;
1002 };
1003
1004 static struct kvm_io_device_ops imsic_iodoev_ops = {
1005 .read = imsic_mmio_read,
1006 .write = imsic_mmio_write,
1007 };
1008
kvm_riscv_vcpu_aia_imsic_init(struct kvm_vcpu * vcpu)1009 int kvm_riscv_vcpu_aia_imsic_init(struct kvm_vcpu *vcpu)
1010 {
1011 int ret = 0;
1012 struct imsic *imsic;
1013 struct page *swfile_page;
1014 struct kvm *kvm = vcpu->kvm;
1015
1016 /* Fail if we have zero IDs */
1017 if (!kvm->arch.aia.nr_ids)
1018 return -EINVAL;
1019
1020 /* Allocate IMSIC context */
1021 imsic = kzalloc(sizeof(*imsic), GFP_KERNEL);
1022 if (!imsic)
1023 return -ENOMEM;
1024 vcpu->arch.aia_context.imsic_state = imsic;
1025
1026 /* Setup IMSIC context */
1027 imsic->nr_msis = kvm->arch.aia.nr_ids + 1;
1028 rwlock_init(&imsic->vsfile_lock);
1029 imsic->nr_eix = BITS_TO_U64(imsic->nr_msis);
1030 imsic->nr_hw_eix = BITS_TO_U64(kvm_riscv_aia_max_ids);
1031 imsic->vsfile_hgei = imsic->vsfile_cpu = -1;
1032
1033 /* Setup IMSIC SW-file */
1034 swfile_page = alloc_pages(GFP_KERNEL | __GFP_ZERO,
1035 get_order(sizeof(*imsic->swfile)));
1036 if (!swfile_page) {
1037 ret = -ENOMEM;
1038 goto fail_free_imsic;
1039 }
1040 imsic->swfile = page_to_virt(swfile_page);
1041 imsic->swfile_pa = page_to_phys(swfile_page);
1042
1043 /* Setup IO device */
1044 kvm_iodevice_init(&imsic->iodev, &imsic_iodoev_ops);
1045 mutex_lock(&kvm->slots_lock);
1046 ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS,
1047 vcpu->arch.aia_context.imsic_addr,
1048 KVM_DEV_RISCV_IMSIC_SIZE,
1049 &imsic->iodev);
1050 mutex_unlock(&kvm->slots_lock);
1051 if (ret)
1052 goto fail_free_swfile;
1053
1054 return 0;
1055
1056 fail_free_swfile:
1057 free_pages((unsigned long)imsic->swfile,
1058 get_order(sizeof(*imsic->swfile)));
1059 fail_free_imsic:
1060 vcpu->arch.aia_context.imsic_state = NULL;
1061 kfree(imsic);
1062 return ret;
1063 }
1064
kvm_riscv_vcpu_aia_imsic_cleanup(struct kvm_vcpu * vcpu)1065 void kvm_riscv_vcpu_aia_imsic_cleanup(struct kvm_vcpu *vcpu)
1066 {
1067 struct kvm *kvm = vcpu->kvm;
1068 struct imsic *imsic = vcpu->arch.aia_context.imsic_state;
1069
1070 if (!imsic)
1071 return;
1072
1073 imsic_vsfile_cleanup(imsic);
1074
1075 mutex_lock(&kvm->slots_lock);
1076 kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS, &imsic->iodev);
1077 mutex_unlock(&kvm->slots_lock);
1078
1079 free_pages((unsigned long)imsic->swfile,
1080 get_order(sizeof(*imsic->swfile)));
1081
1082 vcpu->arch.aia_context.imsic_state = NULL;
1083 kfree(imsic);
1084 }
1085
