1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright 2017 Benjamin Herrenschmidt, IBM Corporation
4 */
5
6 /* File to be included by other .c files */
7
8 #define XGLUE(a,b) a##b
9 #define GLUE(a,b) XGLUE(a,b)
10
11 /* Dummy interrupt used when taking interrupts out of a queue in H_CPPR */
12 #define XICS_DUMMY 1
13
GLUE(X_PFX,ack_pending)14 static void GLUE(X_PFX,ack_pending)(struct kvmppc_xive_vcpu *xc)
15 {
16 u8 cppr;
17 u16 ack;
18
19 /*
20 * Ensure any previous store to CPPR is ordered vs.
21 * the subsequent loads from PIPR or ACK.
22 */
23 eieio();
24
25 /* Perform the acknowledge OS to register cycle. */
26 ack = be16_to_cpu(__x_readw(__x_tima + TM_SPC_ACK_OS_REG));
27
28 /* Synchronize subsequent queue accesses */
29 mb();
30
31 /* XXX Check grouping level */
32
33 /* Anything ? */
34 if (!((ack >> 8) & TM_QW1_NSR_EO))
35 return;
36
37 /* Grab CPPR of the most favored pending interrupt */
38 cppr = ack & 0xff;
39 if (cppr < 8)
40 xc->pending |= 1 << cppr;
41
42 #ifdef XIVE_RUNTIME_CHECKS
43 /* Check consistency */
44 if (cppr >= xc->hw_cppr)
45 pr_warn("KVM-XIVE: CPU %d odd ack CPPR, got %d at %d\n",
46 smp_processor_id(), cppr, xc->hw_cppr);
47 #endif
48
49 /*
50 * Update our image of the HW CPPR. We don't yet modify
51 * xc->cppr, this will be done as we scan for interrupts
52 * in the queues.
53 */
54 xc->hw_cppr = cppr;
55 }
56
GLUE(X_PFX,esb_load)57 static u8 GLUE(X_PFX,esb_load)(struct xive_irq_data *xd, u32 offset)
58 {
59 u64 val;
60
61 if (offset == XIVE_ESB_SET_PQ_10 && xd->flags & XIVE_IRQ_FLAG_STORE_EOI)
62 offset |= XIVE_ESB_LD_ST_MO;
63
64 if (xd->flags & XIVE_IRQ_FLAG_SHIFT_BUG)
65 offset |= offset << 4;
66
67 val =__x_readq(__x_eoi_page(xd) + offset);
68 #ifdef __LITTLE_ENDIAN__
69 val >>= 64-8;
70 #endif
71 return (u8)val;
72 }
73
74
GLUE(X_PFX,source_eoi)75 static void GLUE(X_PFX,source_eoi)(u32 hw_irq, struct xive_irq_data *xd)
76 {
77 /* If the XIVE supports the new "store EOI facility, use it */
78 if (xd->flags & XIVE_IRQ_FLAG_STORE_EOI)
79 __x_writeq(0, __x_eoi_page(xd) + XIVE_ESB_STORE_EOI);
80 else if (hw_irq && xd->flags & XIVE_IRQ_FLAG_EOI_FW)
81 opal_int_eoi(hw_irq);
82 else if (xd->flags & XIVE_IRQ_FLAG_LSI) {
83 /*
84 * For LSIs the HW EOI cycle is used rather than PQ bits,
85 * as they are automatically re-triggred in HW when still
86 * pending.
87 */
88 __x_readq(__x_eoi_page(xd) + XIVE_ESB_LOAD_EOI);
89 } else {
90 uint64_t eoi_val;
91
92 /*
93 * Otherwise for EOI, we use the special MMIO that does
94 * a clear of both P and Q and returns the old Q,
95 * except for LSIs where we use the "EOI cycle" special
96 * load.
97 *
98 * This allows us to then do a re-trigger if Q was set
99 * rather than synthetizing an interrupt in software
100 */
101 eoi_val = GLUE(X_PFX,esb_load)(xd, XIVE_ESB_SET_PQ_00);
102
103 /* Re-trigger if needed */
104 if ((eoi_val & 1) && __x_trig_page(xd))
105 __x_writeq(0, __x_trig_page(xd));
106 }
107 }
108
109 enum {
110 scan_fetch,
111 scan_poll,
112 scan_eoi,
113 };
114
GLUE(X_PFX,scan_interrupts)115 static u32 GLUE(X_PFX,scan_interrupts)(struct kvmppc_xive_vcpu *xc,
116 u8 pending, int scan_type)
117 {
118 u32 hirq = 0;
119 u8 prio = 0xff;
120
121 /* Find highest pending priority */
122 while ((xc->mfrr != 0xff || pending != 0) && hirq == 0) {
123 struct xive_q *q;
124 u32 idx, toggle;
125 __be32 *qpage;
126
127 /*
128 * If pending is 0 this will return 0xff which is what
129 * we want
130 */
131 prio = ffs(pending) - 1;
132
133 /* Don't scan past the guest cppr */
134 if (prio >= xc->cppr || prio > 7) {
135 if (xc->mfrr < xc->cppr) {
136 prio = xc->mfrr;
137 hirq = XICS_IPI;
138 }
139 break;
140 }
141
142 /* Grab queue and pointers */
143 q = &xc->queues[prio];
144 idx = q->idx;
145 toggle = q->toggle;
146
147 /*
148 * Snapshot the queue page. The test further down for EOI
149 * must use the same "copy" that was used by __xive_read_eq
150 * since qpage can be set concurrently and we don't want
151 * to miss an EOI.
152 */
153 qpage = READ_ONCE(q->qpage);
154
155 skip_ipi:
156 /*
157 * Try to fetch from the queue. Will return 0 for a
158 * non-queueing priority (ie, qpage = 0).
159 */
160 hirq = __xive_read_eq(qpage, q->msk, &idx, &toggle);
161
162 /*
163 * If this was a signal for an MFFR change done by
164 * H_IPI we skip it. Additionally, if we were fetching
165 * we EOI it now, thus re-enabling reception of a new
166 * such signal.
167 *
168 * We also need to do that if prio is 0 and we had no
169 * page for the queue. In this case, we have non-queued
170 * IPI that needs to be EOId.
171 *
172 * This is safe because if we have another pending MFRR
173 * change that wasn't observed above, the Q bit will have
174 * been set and another occurrence of the IPI will trigger.
175 */
176 if (hirq == XICS_IPI || (prio == 0 && !qpage)) {
177 if (scan_type == scan_fetch) {
178 GLUE(X_PFX,source_eoi)(xc->vp_ipi,
179 &xc->vp_ipi_data);
180 q->idx = idx;
181 q->toggle = toggle;
182 }
183 /* Loop back on same queue with updated idx/toggle */
184 #ifdef XIVE_RUNTIME_CHECKS
185 WARN_ON(hirq && hirq != XICS_IPI);
186 #endif
187 if (hirq)
188 goto skip_ipi;
189 }
190
191 /* If it's the dummy interrupt, continue searching */
192 if (hirq == XICS_DUMMY)
193 goto skip_ipi;
194
195 /* Clear the pending bit if the queue is now empty */
196 if (!hirq) {
197 pending &= ~(1 << prio);
198
199 /*
200 * Check if the queue count needs adjusting due to
201 * interrupts being moved away.
202 */
203 if (atomic_read(&q->pending_count)) {
204 int p = atomic_xchg(&q->pending_count, 0);
205 if (p) {
206 #ifdef XIVE_RUNTIME_CHECKS
207 WARN_ON(p > atomic_read(&q->count));
208 #endif
209 atomic_sub(p, &q->count);
210 }
211 }
212 }
213
214 /*
215 * If the most favoured prio we found pending is less
216 * favored (or equal) than a pending IPI, we return
217 * the IPI instead.
218 */
219 if (prio >= xc->mfrr && xc->mfrr < xc->cppr) {
220 prio = xc->mfrr;
221 hirq = XICS_IPI;
222 break;
223 }
224
225 /* If fetching, update queue pointers */
226 if (scan_type == scan_fetch) {
227 q->idx = idx;
228 q->toggle = toggle;
229 }
230 }
231
232 /* If we are just taking a "peek", do nothing else */
233 if (scan_type == scan_poll)
234 return hirq;
235
236 /* Update the pending bits */
237 xc->pending = pending;
238
239 /*
240 * If this is an EOI that's it, no CPPR adjustment done here,
241 * all we needed was cleanup the stale pending bits and check
242 * if there's anything left.
243 */
244 if (scan_type == scan_eoi)
245 return hirq;
246
247 /*
248 * If we found an interrupt, adjust what the guest CPPR should
249 * be as if we had just fetched that interrupt from HW.
250 *
251 * Note: This can only make xc->cppr smaller as the previous
252 * loop will only exit with hirq != 0 if prio is lower than
253 * the current xc->cppr. Thus we don't need to re-check xc->mfrr
254 * for pending IPIs.
255 */
256 if (hirq)
257 xc->cppr = prio;
258 /*
259 * If it was an IPI the HW CPPR might have been lowered too much
260 * as the HW interrupt we use for IPIs is routed to priority 0.
261 *
262 * We re-sync it here.
263 */
264 if (xc->cppr != xc->hw_cppr) {
265 xc->hw_cppr = xc->cppr;
266 __x_writeb(xc->cppr, __x_tima + TM_QW1_OS + TM_CPPR);
267 }
268
269 return hirq;
270 }
271
GLUE(X_PFX,h_xirr)272 X_STATIC unsigned long GLUE(X_PFX,h_xirr)(struct kvm_vcpu *vcpu)
273 {
274 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
275 u8 old_cppr;
276 u32 hirq;
277
278 pr_devel("H_XIRR\n");
279
280 xc->GLUE(X_STAT_PFX,h_xirr)++;
281
282 /* First collect pending bits from HW */
283 GLUE(X_PFX,ack_pending)(xc);
284
285 pr_devel(" new pending=0x%02x hw_cppr=%d cppr=%d\n",
286 xc->pending, xc->hw_cppr, xc->cppr);
287
288 /* Grab previous CPPR and reverse map it */
289 old_cppr = xive_prio_to_guest(xc->cppr);
290
291 /* Scan for actual interrupts */
292 hirq = GLUE(X_PFX,scan_interrupts)(xc, xc->pending, scan_fetch);
293
294 pr_devel(" got hirq=0x%x hw_cppr=%d cppr=%d\n",
295 hirq, xc->hw_cppr, xc->cppr);
296
297 #ifdef XIVE_RUNTIME_CHECKS
298 /* That should never hit */
299 if (hirq & 0xff000000)
300 pr_warn("XIVE: Weird guest interrupt number 0x%08x\n", hirq);
301 #endif
302
303 /*
304 * XXX We could check if the interrupt is masked here and
305 * filter it. If we chose to do so, we would need to do:
306 *
307 * if (masked) {
308 * lock();
309 * if (masked) {
310 * old_Q = true;
311 * hirq = 0;
312 * }
313 * unlock();
314 * }
315 */
316
317 /* Return interrupt and old CPPR in GPR4 */
318 vcpu->arch.regs.gpr[4] = hirq | (old_cppr << 24);
319
320 return H_SUCCESS;
321 }
322
GLUE(X_PFX,h_ipoll)323 X_STATIC unsigned long GLUE(X_PFX,h_ipoll)(struct kvm_vcpu *vcpu, unsigned long server)
324 {
325 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
326 u8 pending = xc->pending;
327 u32 hirq;
328
329 pr_devel("H_IPOLL(server=%ld)\n", server);
330
331 xc->GLUE(X_STAT_PFX,h_ipoll)++;
332
333 /* Grab the target VCPU if not the current one */
334 if (xc->server_num != server) {
335 vcpu = kvmppc_xive_find_server(vcpu->kvm, server);
336 if (!vcpu)
337 return H_PARAMETER;
338 xc = vcpu->arch.xive_vcpu;
339
340 /* Scan all priorities */
341 pending = 0xff;
342 } else {
343 /* Grab pending interrupt if any */
344 __be64 qw1 = __x_readq(__x_tima + TM_QW1_OS);
345 u8 pipr = be64_to_cpu(qw1) & 0xff;
346 if (pipr < 8)
347 pending |= 1 << pipr;
348 }
349
350 hirq = GLUE(X_PFX,scan_interrupts)(xc, pending, scan_poll);
351
352 /* Return interrupt and old CPPR in GPR4 */
353 vcpu->arch.regs.gpr[4] = hirq | (xc->cppr << 24);
354
355 return H_SUCCESS;
356 }
357
GLUE(X_PFX,push_pending_to_hw)358 static void GLUE(X_PFX,push_pending_to_hw)(struct kvmppc_xive_vcpu *xc)
359 {
360 u8 pending, prio;
361
362 pending = xc->pending;
363 if (xc->mfrr != 0xff) {
364 if (xc->mfrr < 8)
365 pending |= 1 << xc->mfrr;
366 else
367 pending |= 0x80;
368 }
369 if (!pending)
370 return;
371 prio = ffs(pending) - 1;
372
373 __x_writeb(prio, __x_tima + TM_SPC_SET_OS_PENDING);
374 }
375
GLUE(X_PFX,scan_for_rerouted_irqs)376 static void GLUE(X_PFX,scan_for_rerouted_irqs)(struct kvmppc_xive *xive,
377 struct kvmppc_xive_vcpu *xc)
378 {
379 unsigned int prio;
380
381 /* For each priority that is now masked */
382 for (prio = xc->cppr; prio < KVMPPC_XIVE_Q_COUNT; prio++) {
383 struct xive_q *q = &xc->queues[prio];
384 struct kvmppc_xive_irq_state *state;
385 struct kvmppc_xive_src_block *sb;
386 u32 idx, toggle, entry, irq, hw_num;
387 struct xive_irq_data *xd;
388 __be32 *qpage;
389 u16 src;
390
391 idx = q->idx;
392 toggle = q->toggle;
393 qpage = READ_ONCE(q->qpage);
394 if (!qpage)
395 continue;
396
397 /* For each interrupt in the queue */
398 for (;;) {
399 entry = be32_to_cpup(qpage + idx);
400
401 /* No more ? */
402 if ((entry >> 31) == toggle)
403 break;
404 irq = entry & 0x7fffffff;
405
406 /* Skip dummies and IPIs */
407 if (irq == XICS_DUMMY || irq == XICS_IPI)
408 goto next;
409 sb = kvmppc_xive_find_source(xive, irq, &src);
410 if (!sb)
411 goto next;
412 state = &sb->irq_state[src];
413
414 /* Has it been rerouted ? */
415 if (xc->server_num == state->act_server)
416 goto next;
417
418 /*
419 * Allright, it *has* been re-routed, kill it from
420 * the queue.
421 */
422 qpage[idx] = cpu_to_be32((entry & 0x80000000) | XICS_DUMMY);
423
424 /* Find the HW interrupt */
425 kvmppc_xive_select_irq(state, &hw_num, &xd);
426
427 /* If it's not an LSI, set PQ to 11 the EOI will force a resend */
428 if (!(xd->flags & XIVE_IRQ_FLAG_LSI))
429 GLUE(X_PFX,esb_load)(xd, XIVE_ESB_SET_PQ_11);
430
431 /* EOI the source */
432 GLUE(X_PFX,source_eoi)(hw_num, xd);
433
434 next:
435 idx = (idx + 1) & q->msk;
436 if (idx == 0)
437 toggle ^= 1;
438 }
439 }
440 }
441
GLUE(X_PFX,h_cppr)442 X_STATIC int GLUE(X_PFX,h_cppr)(struct kvm_vcpu *vcpu, unsigned long cppr)
443 {
444 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
445 struct kvmppc_xive *xive = vcpu->kvm->arch.xive;
446 u8 old_cppr;
447
448 pr_devel("H_CPPR(cppr=%ld)\n", cppr);
449
450 xc->GLUE(X_STAT_PFX,h_cppr)++;
451
452 /* Map CPPR */
453 cppr = xive_prio_from_guest(cppr);
454
455 /* Remember old and update SW state */
456 old_cppr = xc->cppr;
457 xc->cppr = cppr;
458
459 /*
460 * Order the above update of xc->cppr with the subsequent
461 * read of xc->mfrr inside push_pending_to_hw()
462 */
463 smp_mb();
464
465 if (cppr > old_cppr) {
466 /*
467 * We are masking less, we need to look for pending things
468 * to deliver and set VP pending bits accordingly to trigger
469 * a new interrupt otherwise we might miss MFRR changes for
470 * which we have optimized out sending an IPI signal.
471 */
472 GLUE(X_PFX,push_pending_to_hw)(xc);
473 } else {
474 /*
475 * We are masking more, we need to check the queue for any
476 * interrupt that has been routed to another CPU, take
477 * it out (replace it with the dummy) and retrigger it.
478 *
479 * This is necessary since those interrupts may otherwise
480 * never be processed, at least not until this CPU restores
481 * its CPPR.
482 *
483 * This is in theory racy vs. HW adding new interrupts to
484 * the queue. In practice this works because the interesting
485 * cases are when the guest has done a set_xive() to move the
486 * interrupt away, which flushes the xive, followed by the
487 * target CPU doing a H_CPPR. So any new interrupt coming into
488 * the queue must still be routed to us and isn't a source
489 * of concern.
490 */
491 GLUE(X_PFX,scan_for_rerouted_irqs)(xive, xc);
492 }
493
494 /* Apply new CPPR */
495 xc->hw_cppr = cppr;
496 __x_writeb(cppr, __x_tima + TM_QW1_OS + TM_CPPR);
497
498 return H_SUCCESS;
499 }
500
GLUE(X_PFX,h_eoi)501 X_STATIC int GLUE(X_PFX,h_eoi)(struct kvm_vcpu *vcpu, unsigned long xirr)
502 {
503 struct kvmppc_xive *xive = vcpu->kvm->arch.xive;
504 struct kvmppc_xive_src_block *sb;
505 struct kvmppc_xive_irq_state *state;
506 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
507 struct xive_irq_data *xd;
508 u8 new_cppr = xirr >> 24;
509 u32 irq = xirr & 0x00ffffff, hw_num;
510 u16 src;
511 int rc = 0;
512
513 pr_devel("H_EOI(xirr=%08lx)\n", xirr);
514
515 xc->GLUE(X_STAT_PFX,h_eoi)++;
516
517 xc->cppr = xive_prio_from_guest(new_cppr);
518
519 /*
520 * IPIs are synthetized from MFRR and thus don't need
521 * any special EOI handling. The underlying interrupt
522 * used to signal MFRR changes is EOId when fetched from
523 * the queue.
524 */
525 if (irq == XICS_IPI || irq == 0) {
526 /*
527 * This barrier orders the setting of xc->cppr vs.
528 * subsquent test of xc->mfrr done inside
529 * scan_interrupts and push_pending_to_hw
530 */
531 smp_mb();
532 goto bail;
533 }
534
535 /* Find interrupt source */
536 sb = kvmppc_xive_find_source(xive, irq, &src);
537 if (!sb) {
538 pr_devel(" source not found !\n");
539 rc = H_PARAMETER;
540 /* Same as above */
541 smp_mb();
542 goto bail;
543 }
544 state = &sb->irq_state[src];
545 kvmppc_xive_select_irq(state, &hw_num, &xd);
546
547 state->in_eoi = true;
548
549 /*
550 * This barrier orders both setting of in_eoi above vs,
551 * subsequent test of guest_priority, and the setting
552 * of xc->cppr vs. subsquent test of xc->mfrr done inside
553 * scan_interrupts and push_pending_to_hw
554 */
555 smp_mb();
556
557 again:
558 if (state->guest_priority == MASKED) {
559 arch_spin_lock(&sb->lock);
560 if (state->guest_priority != MASKED) {
561 arch_spin_unlock(&sb->lock);
562 goto again;
563 }
564 pr_devel(" EOI on saved P...\n");
565
566 /* Clear old_p, that will cause unmask to perform an EOI */
567 state->old_p = false;
568
569 arch_spin_unlock(&sb->lock);
570 } else {
571 pr_devel(" EOI on source...\n");
572
573 /* Perform EOI on the source */
574 GLUE(X_PFX,source_eoi)(hw_num, xd);
575
576 /* If it's an emulated LSI, check level and resend */
577 if (state->lsi && state->asserted)
578 __x_writeq(0, __x_trig_page(xd));
579
580 }
581
582 /*
583 * This barrier orders the above guest_priority check
584 * and spin_lock/unlock with clearing in_eoi below.
585 *
586 * It also has to be a full mb() as it must ensure
587 * the MMIOs done in source_eoi() are completed before
588 * state->in_eoi is visible.
589 */
590 mb();
591 state->in_eoi = false;
592 bail:
593
594 /* Re-evaluate pending IRQs and update HW */
595 GLUE(X_PFX,scan_interrupts)(xc, xc->pending, scan_eoi);
596 GLUE(X_PFX,push_pending_to_hw)(xc);
597 pr_devel(" after scan pending=%02x\n", xc->pending);
598
599 /* Apply new CPPR */
600 xc->hw_cppr = xc->cppr;
601 __x_writeb(xc->cppr, __x_tima + TM_QW1_OS + TM_CPPR);
602
603 return rc;
604 }
605
GLUE(X_PFX,h_ipi)606 X_STATIC int GLUE(X_PFX,h_ipi)(struct kvm_vcpu *vcpu, unsigned long server,
607 unsigned long mfrr)
608 {
609 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
610
611 pr_devel("H_IPI(server=%08lx,mfrr=%ld)\n", server, mfrr);
612
613 xc->GLUE(X_STAT_PFX,h_ipi)++;
614
615 /* Find target */
616 vcpu = kvmppc_xive_find_server(vcpu->kvm, server);
617 if (!vcpu)
618 return H_PARAMETER;
619 xc = vcpu->arch.xive_vcpu;
620
621 /* Locklessly write over MFRR */
622 xc->mfrr = mfrr;
623
624 /*
625 * The load of xc->cppr below and the subsequent MMIO store
626 * to the IPI must happen after the above mfrr update is
627 * globally visible so that:
628 *
629 * - Synchronize with another CPU doing an H_EOI or a H_CPPR
630 * updating xc->cppr then reading xc->mfrr.
631 *
632 * - The target of the IPI sees the xc->mfrr update
633 */
634 mb();
635
636 /* Shoot the IPI if most favored than target cppr */
637 if (mfrr < xc->cppr)
638 __x_writeq(0, __x_trig_page(&xc->vp_ipi_data));
639
640 return H_SUCCESS;
641 }
642
