1 /*
2 * Copyright 2017 Benjamin Herrenschmidt, IBM Corporation
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License, version 2, as
6 * published by the Free Software Foundation.
7 */
8
9 /* File to be included by other .c files */
10
11 #define XGLUE(a,b) a##b
12 #define GLUE(a,b) XGLUE(a,b)
13
14 /* Dummy interrupt used when taking interrupts out of a queue in H_CPPR */
15 #define XICS_DUMMY 1
16
GLUE(X_PFX,ack_pending)17 static void GLUE(X_PFX,ack_pending)(struct kvmppc_xive_vcpu *xc)
18 {
19 u8 cppr;
20 u16 ack;
21
22 /*
23 * Ensure any previous store to CPPR is ordered vs.
24 * the subsequent loads from PIPR or ACK.
25 */
26 eieio();
27
28 /* Perform the acknowledge OS to register cycle. */
29 ack = be16_to_cpu(__x_readw(__x_tima + TM_SPC_ACK_OS_REG));
30
31 /* Synchronize subsequent queue accesses */
32 mb();
33
34 /* XXX Check grouping level */
35
36 /* Anything ? */
37 if (!((ack >> 8) & TM_QW1_NSR_EO))
38 return;
39
40 /* Grab CPPR of the most favored pending interrupt */
41 cppr = ack & 0xff;
42 if (cppr < 8)
43 xc->pending |= 1 << cppr;
44
45 #ifdef XIVE_RUNTIME_CHECKS
46 /* Check consistency */
47 if (cppr >= xc->hw_cppr)
48 pr_warn("KVM-XIVE: CPU %d odd ack CPPR, got %d at %d\n",
49 smp_processor_id(), cppr, xc->hw_cppr);
50 #endif
51
52 /*
53 * Update our image of the HW CPPR. We don't yet modify
54 * xc->cppr, this will be done as we scan for interrupts
55 * in the queues.
56 */
57 xc->hw_cppr = cppr;
58 }
59
GLUE(X_PFX,esb_load)60 static u8 GLUE(X_PFX,esb_load)(struct xive_irq_data *xd, u32 offset)
61 {
62 u64 val;
63
64 if (xd->flags & XIVE_IRQ_FLAG_SHIFT_BUG)
65 offset |= offset << 4;
66
67 val =__x_readq(__x_eoi_page(xd) + offset);
68 #ifdef __LITTLE_ENDIAN__
69 val >>= 64-8;
70 #endif
71 return (u8)val;
72 }
73
74
GLUE(X_PFX,source_eoi)75 static void GLUE(X_PFX,source_eoi)(u32 hw_irq, struct xive_irq_data *xd)
76 {
77 /* If the XIVE supports the new "store EOI facility, use it */
78 if (xd->flags & XIVE_IRQ_FLAG_STORE_EOI)
79 __x_writeq(0, __x_eoi_page(xd) + XIVE_ESB_STORE_EOI);
80 else if (hw_irq && xd->flags & XIVE_IRQ_FLAG_EOI_FW)
81 opal_int_eoi(hw_irq);
82 else if (xd->flags & XIVE_IRQ_FLAG_LSI) {
83 /*
84 * For LSIs the HW EOI cycle is used rather than PQ bits,
85 * as they are automatically re-triggred in HW when still
86 * pending.
87 */
88 __x_readq(__x_eoi_page(xd) + XIVE_ESB_LOAD_EOI);
89 } else {
90 uint64_t eoi_val;
91
92 /*
93 * Otherwise for EOI, we use the special MMIO that does
94 * a clear of both P and Q and returns the old Q,
95 * except for LSIs where we use the "EOI cycle" special
96 * load.
97 *
98 * This allows us to then do a re-trigger if Q was set
99 * rather than synthetizing an interrupt in software
100 */
101 eoi_val = GLUE(X_PFX,esb_load)(xd, XIVE_ESB_SET_PQ_00);
102
103 /* Re-trigger if needed */
104 if ((eoi_val & 1) && __x_trig_page(xd))
105 __x_writeq(0, __x_trig_page(xd));
106 }
107 }
108
109 enum {
110 scan_fetch,
111 scan_poll,
112 scan_eoi,
113 };
114
GLUE(X_PFX,scan_interrupts)115 static u32 GLUE(X_PFX,scan_interrupts)(struct kvmppc_xive_vcpu *xc,
116 u8 pending, int scan_type)
117 {
118 u32 hirq = 0;
119 u8 prio = 0xff;
120
121 /* Find highest pending priority */
122 while ((xc->mfrr != 0xff || pending != 0) && hirq == 0) {
123 struct xive_q *q;
124 u32 idx, toggle;
125 __be32 *qpage;
126
127 /*
128 * If pending is 0 this will return 0xff which is what
129 * we want
130 */
131 prio = ffs(pending) - 1;
132
133 /*
134 * If the most favoured prio we found pending is less
135 * favored (or equal) than a pending IPI, we return
136 * the IPI instead.
137 *
138 * Note: If pending was 0 and mfrr is 0xff, we will
139 * not spurriously take an IPI because mfrr cannot
140 * then be smaller than cppr.
141 */
142 if (prio >= xc->mfrr && xc->mfrr < xc->cppr) {
143 prio = xc->mfrr;
144 hirq = XICS_IPI;
145 break;
146 }
147
148 /* Don't scan past the guest cppr */
149 if (prio >= xc->cppr || prio > 7)
150 break;
151
152 /* Grab queue and pointers */
153 q = &xc->queues[prio];
154 idx = q->idx;
155 toggle = q->toggle;
156
157 /*
158 * Snapshot the queue page. The test further down for EOI
159 * must use the same "copy" that was used by __xive_read_eq
160 * since qpage can be set concurrently and we don't want
161 * to miss an EOI.
162 */
163 qpage = READ_ONCE(q->qpage);
164
165 skip_ipi:
166 /*
167 * Try to fetch from the queue. Will return 0 for a
168 * non-queueing priority (ie, qpage = 0).
169 */
170 hirq = __xive_read_eq(qpage, q->msk, &idx, &toggle);
171
172 /*
173 * If this was a signal for an MFFR change done by
174 * H_IPI we skip it. Additionally, if we were fetching
175 * we EOI it now, thus re-enabling reception of a new
176 * such signal.
177 *
178 * We also need to do that if prio is 0 and we had no
179 * page for the queue. In this case, we have non-queued
180 * IPI that needs to be EOId.
181 *
182 * This is safe because if we have another pending MFRR
183 * change that wasn't observed above, the Q bit will have
184 * been set and another occurrence of the IPI will trigger.
185 */
186 if (hirq == XICS_IPI || (prio == 0 && !qpage)) {
187 if (scan_type == scan_fetch)
188 GLUE(X_PFX,source_eoi)(xc->vp_ipi,
189 &xc->vp_ipi_data);
190 /* Loop back on same queue with updated idx/toggle */
191 #ifdef XIVE_RUNTIME_CHECKS
192 WARN_ON(hirq && hirq != XICS_IPI);
193 #endif
194 if (hirq)
195 goto skip_ipi;
196 }
197
198 /* If it's the dummy interrupt, continue searching */
199 if (hirq == XICS_DUMMY)
200 goto skip_ipi;
201
202 /* If fetching, update queue pointers */
203 if (scan_type == scan_fetch) {
204 q->idx = idx;
205 q->toggle = toggle;
206 }
207
208 /* Something found, stop searching */
209 if (hirq)
210 break;
211
212 /* Clear the pending bit on the now empty queue */
213 pending &= ~(1 << prio);
214
215 /*
216 * Check if the queue count needs adjusting due to
217 * interrupts being moved away.
218 */
219 if (atomic_read(&q->pending_count)) {
220 int p = atomic_xchg(&q->pending_count, 0);
221 if (p) {
222 #ifdef XIVE_RUNTIME_CHECKS
223 WARN_ON(p > atomic_read(&q->count));
224 #endif
225 atomic_sub(p, &q->count);
226 }
227 }
228 }
229
230 /* If we are just taking a "peek", do nothing else */
231 if (scan_type == scan_poll)
232 return hirq;
233
234 /* Update the pending bits */
235 xc->pending = pending;
236
237 /*
238 * If this is an EOI that's it, no CPPR adjustment done here,
239 * all we needed was cleanup the stale pending bits and check
240 * if there's anything left.
241 */
242 if (scan_type == scan_eoi)
243 return hirq;
244
245 /*
246 * If we found an interrupt, adjust what the guest CPPR should
247 * be as if we had just fetched that interrupt from HW.
248 *
249 * Note: This can only make xc->cppr smaller as the previous
250 * loop will only exit with hirq != 0 if prio is lower than
251 * the current xc->cppr. Thus we don't need to re-check xc->mfrr
252 * for pending IPIs.
253 */
254 if (hirq)
255 xc->cppr = prio;
256 /*
257 * If it was an IPI the HW CPPR might have been lowered too much
258 * as the HW interrupt we use for IPIs is routed to priority 0.
259 *
260 * We re-sync it here.
261 */
262 if (xc->cppr != xc->hw_cppr) {
263 xc->hw_cppr = xc->cppr;
264 __x_writeb(xc->cppr, __x_tima + TM_QW1_OS + TM_CPPR);
265 }
266
267 return hirq;
268 }
269
GLUE(X_PFX,h_xirr)270 X_STATIC unsigned long GLUE(X_PFX,h_xirr)(struct kvm_vcpu *vcpu)
271 {
272 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
273 u8 old_cppr;
274 u32 hirq;
275
276 pr_devel("H_XIRR\n");
277
278 xc->GLUE(X_STAT_PFX,h_xirr)++;
279
280 /* First collect pending bits from HW */
281 GLUE(X_PFX,ack_pending)(xc);
282
283 /*
284 * Cleanup the old-style bits if needed (they may have been
285 * set by pull or an escalation interrupts).
286 */
287 if (test_bit(BOOK3S_IRQPRIO_EXTERNAL, &vcpu->arch.pending_exceptions))
288 clear_bit(BOOK3S_IRQPRIO_EXTERNAL_LEVEL,
289 &vcpu->arch.pending_exceptions);
290
291 pr_devel(" new pending=0x%02x hw_cppr=%d cppr=%d\n",
292 xc->pending, xc->hw_cppr, xc->cppr);
293
294 /* Grab previous CPPR and reverse map it */
295 old_cppr = xive_prio_to_guest(xc->cppr);
296
297 /* Scan for actual interrupts */
298 hirq = GLUE(X_PFX,scan_interrupts)(xc, xc->pending, scan_fetch);
299
300 pr_devel(" got hirq=0x%x hw_cppr=%d cppr=%d\n",
301 hirq, xc->hw_cppr, xc->cppr);
302
303 #ifdef XIVE_RUNTIME_CHECKS
304 /* That should never hit */
305 if (hirq & 0xff000000)
306 pr_warn("XIVE: Weird guest interrupt number 0x%08x\n", hirq);
307 #endif
308
309 /*
310 * XXX We could check if the interrupt is masked here and
311 * filter it. If we chose to do so, we would need to do:
312 *
313 * if (masked) {
314 * lock();
315 * if (masked) {
316 * old_Q = true;
317 * hirq = 0;
318 * }
319 * unlock();
320 * }
321 */
322
323 /* Return interrupt and old CPPR in GPR4 */
324 vcpu->arch.regs.gpr[4] = hirq | (old_cppr << 24);
325
326 return H_SUCCESS;
327 }
328
GLUE(X_PFX,h_ipoll)329 X_STATIC unsigned long GLUE(X_PFX,h_ipoll)(struct kvm_vcpu *vcpu, unsigned long server)
330 {
331 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
332 u8 pending = xc->pending;
333 u32 hirq;
334
335 pr_devel("H_IPOLL(server=%ld)\n", server);
336
337 xc->GLUE(X_STAT_PFX,h_ipoll)++;
338
339 /* Grab the target VCPU if not the current one */
340 if (xc->server_num != server) {
341 vcpu = kvmppc_xive_find_server(vcpu->kvm, server);
342 if (!vcpu)
343 return H_PARAMETER;
344 xc = vcpu->arch.xive_vcpu;
345
346 /* Scan all priorities */
347 pending = 0xff;
348 } else {
349 /* Grab pending interrupt if any */
350 __be64 qw1 = __x_readq(__x_tima + TM_QW1_OS);
351 u8 pipr = be64_to_cpu(qw1) & 0xff;
352 if (pipr < 8)
353 pending |= 1 << pipr;
354 }
355
356 hirq = GLUE(X_PFX,scan_interrupts)(xc, pending, scan_poll);
357
358 /* Return interrupt and old CPPR in GPR4 */
359 vcpu->arch.regs.gpr[4] = hirq | (xc->cppr << 24);
360
361 return H_SUCCESS;
362 }
363
GLUE(X_PFX,push_pending_to_hw)364 static void GLUE(X_PFX,push_pending_to_hw)(struct kvmppc_xive_vcpu *xc)
365 {
366 u8 pending, prio;
367
368 pending = xc->pending;
369 if (xc->mfrr != 0xff) {
370 if (xc->mfrr < 8)
371 pending |= 1 << xc->mfrr;
372 else
373 pending |= 0x80;
374 }
375 if (!pending)
376 return;
377 prio = ffs(pending) - 1;
378
379 __x_writeb(prio, __x_tima + TM_SPC_SET_OS_PENDING);
380 }
381
GLUE(X_PFX,scan_for_rerouted_irqs)382 static void GLUE(X_PFX,scan_for_rerouted_irqs)(struct kvmppc_xive *xive,
383 struct kvmppc_xive_vcpu *xc)
384 {
385 unsigned int prio;
386
387 /* For each priority that is now masked */
388 for (prio = xc->cppr; prio < KVMPPC_XIVE_Q_COUNT; prio++) {
389 struct xive_q *q = &xc->queues[prio];
390 struct kvmppc_xive_irq_state *state;
391 struct kvmppc_xive_src_block *sb;
392 u32 idx, toggle, entry, irq, hw_num;
393 struct xive_irq_data *xd;
394 __be32 *qpage;
395 u16 src;
396
397 idx = q->idx;
398 toggle = q->toggle;
399 qpage = READ_ONCE(q->qpage);
400 if (!qpage)
401 continue;
402
403 /* For each interrupt in the queue */
404 for (;;) {
405 entry = be32_to_cpup(qpage + idx);
406
407 /* No more ? */
408 if ((entry >> 31) == toggle)
409 break;
410 irq = entry & 0x7fffffff;
411
412 /* Skip dummies and IPIs */
413 if (irq == XICS_DUMMY || irq == XICS_IPI)
414 goto next;
415 sb = kvmppc_xive_find_source(xive, irq, &src);
416 if (!sb)
417 goto next;
418 state = &sb->irq_state[src];
419
420 /* Has it been rerouted ? */
421 if (xc->server_num == state->act_server)
422 goto next;
423
424 /*
425 * Allright, it *has* been re-routed, kill it from
426 * the queue.
427 */
428 qpage[idx] = cpu_to_be32((entry & 0x80000000) | XICS_DUMMY);
429
430 /* Find the HW interrupt */
431 kvmppc_xive_select_irq(state, &hw_num, &xd);
432
433 /* If it's not an LSI, set PQ to 11 the EOI will force a resend */
434 if (!(xd->flags & XIVE_IRQ_FLAG_LSI))
435 GLUE(X_PFX,esb_load)(xd, XIVE_ESB_SET_PQ_11);
436
437 /* EOI the source */
438 GLUE(X_PFX,source_eoi)(hw_num, xd);
439
440 next:
441 idx = (idx + 1) & q->msk;
442 if (idx == 0)
443 toggle ^= 1;
444 }
445 }
446 }
447
GLUE(X_PFX,h_cppr)448 X_STATIC int GLUE(X_PFX,h_cppr)(struct kvm_vcpu *vcpu, unsigned long cppr)
449 {
450 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
451 struct kvmppc_xive *xive = vcpu->kvm->arch.xive;
452 u8 old_cppr;
453
454 pr_devel("H_CPPR(cppr=%ld)\n", cppr);
455
456 xc->GLUE(X_STAT_PFX,h_cppr)++;
457
458 /* Map CPPR */
459 cppr = xive_prio_from_guest(cppr);
460
461 /* Remember old and update SW state */
462 old_cppr = xc->cppr;
463 xc->cppr = cppr;
464
465 /*
466 * Order the above update of xc->cppr with the subsequent
467 * read of xc->mfrr inside push_pending_to_hw()
468 */
469 smp_mb();
470
471 if (cppr > old_cppr) {
472 /*
473 * We are masking less, we need to look for pending things
474 * to deliver and set VP pending bits accordingly to trigger
475 * a new interrupt otherwise we might miss MFRR changes for
476 * which we have optimized out sending an IPI signal.
477 */
478 GLUE(X_PFX,push_pending_to_hw)(xc);
479 } else {
480 /*
481 * We are masking more, we need to check the queue for any
482 * interrupt that has been routed to another CPU, take
483 * it out (replace it with the dummy) and retrigger it.
484 *
485 * This is necessary since those interrupts may otherwise
486 * never be processed, at least not until this CPU restores
487 * its CPPR.
488 *
489 * This is in theory racy vs. HW adding new interrupts to
490 * the queue. In practice this works because the interesting
491 * cases are when the guest has done a set_xive() to move the
492 * interrupt away, which flushes the xive, followed by the
493 * target CPU doing a H_CPPR. So any new interrupt coming into
494 * the queue must still be routed to us and isn't a source
495 * of concern.
496 */
497 GLUE(X_PFX,scan_for_rerouted_irqs)(xive, xc);
498 }
499
500 /* Apply new CPPR */
501 xc->hw_cppr = cppr;
502 __x_writeb(cppr, __x_tima + TM_QW1_OS + TM_CPPR);
503
504 return H_SUCCESS;
505 }
506
GLUE(X_PFX,h_eoi)507 X_STATIC int GLUE(X_PFX,h_eoi)(struct kvm_vcpu *vcpu, unsigned long xirr)
508 {
509 struct kvmppc_xive *xive = vcpu->kvm->arch.xive;
510 struct kvmppc_xive_src_block *sb;
511 struct kvmppc_xive_irq_state *state;
512 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
513 struct xive_irq_data *xd;
514 u8 new_cppr = xirr >> 24;
515 u32 irq = xirr & 0x00ffffff, hw_num;
516 u16 src;
517 int rc = 0;
518
519 pr_devel("H_EOI(xirr=%08lx)\n", xirr);
520
521 xc->GLUE(X_STAT_PFX,h_eoi)++;
522
523 xc->cppr = xive_prio_from_guest(new_cppr);
524
525 /*
526 * IPIs are synthetized from MFRR and thus don't need
527 * any special EOI handling. The underlying interrupt
528 * used to signal MFRR changes is EOId when fetched from
529 * the queue.
530 */
531 if (irq == XICS_IPI || irq == 0) {
532 /*
533 * This barrier orders the setting of xc->cppr vs.
534 * subsquent test of xc->mfrr done inside
535 * scan_interrupts and push_pending_to_hw
536 */
537 smp_mb();
538 goto bail;
539 }
540
541 /* Find interrupt source */
542 sb = kvmppc_xive_find_source(xive, irq, &src);
543 if (!sb) {
544 pr_devel(" source not found !\n");
545 rc = H_PARAMETER;
546 /* Same as above */
547 smp_mb();
548 goto bail;
549 }
550 state = &sb->irq_state[src];
551 kvmppc_xive_select_irq(state, &hw_num, &xd);
552
553 state->in_eoi = true;
554
555 /*
556 * This barrier orders both setting of in_eoi above vs,
557 * subsequent test of guest_priority, and the setting
558 * of xc->cppr vs. subsquent test of xc->mfrr done inside
559 * scan_interrupts and push_pending_to_hw
560 */
561 smp_mb();
562
563 again:
564 if (state->guest_priority == MASKED) {
565 arch_spin_lock(&sb->lock);
566 if (state->guest_priority != MASKED) {
567 arch_spin_unlock(&sb->lock);
568 goto again;
569 }
570 pr_devel(" EOI on saved P...\n");
571
572 /* Clear old_p, that will cause unmask to perform an EOI */
573 state->old_p = false;
574
575 arch_spin_unlock(&sb->lock);
576 } else {
577 pr_devel(" EOI on source...\n");
578
579 /* Perform EOI on the source */
580 GLUE(X_PFX,source_eoi)(hw_num, xd);
581
582 /* If it's an emulated LSI, check level and resend */
583 if (state->lsi && state->asserted)
584 __x_writeq(0, __x_trig_page(xd));
585
586 }
587
588 /*
589 * This barrier orders the above guest_priority check
590 * and spin_lock/unlock with clearing in_eoi below.
591 *
592 * It also has to be a full mb() as it must ensure
593 * the MMIOs done in source_eoi() are completed before
594 * state->in_eoi is visible.
595 */
596 mb();
597 state->in_eoi = false;
598 bail:
599
600 /* Re-evaluate pending IRQs and update HW */
601 GLUE(X_PFX,scan_interrupts)(xc, xc->pending, scan_eoi);
602 GLUE(X_PFX,push_pending_to_hw)(xc);
603 pr_devel(" after scan pending=%02x\n", xc->pending);
604
605 /* Apply new CPPR */
606 xc->hw_cppr = xc->cppr;
607 __x_writeb(xc->cppr, __x_tima + TM_QW1_OS + TM_CPPR);
608
609 return rc;
610 }
611
GLUE(X_PFX,h_ipi)612 X_STATIC int GLUE(X_PFX,h_ipi)(struct kvm_vcpu *vcpu, unsigned long server,
613 unsigned long mfrr)
614 {
615 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
616
617 pr_devel("H_IPI(server=%08lx,mfrr=%ld)\n", server, mfrr);
618
619 xc->GLUE(X_STAT_PFX,h_ipi)++;
620
621 /* Find target */
622 vcpu = kvmppc_xive_find_server(vcpu->kvm, server);
623 if (!vcpu)
624 return H_PARAMETER;
625 xc = vcpu->arch.xive_vcpu;
626
627 /* Locklessly write over MFRR */
628 xc->mfrr = mfrr;
629
630 /*
631 * The load of xc->cppr below and the subsequent MMIO store
632 * to the IPI must happen after the above mfrr update is
633 * globally visible so that:
634 *
635 * - Synchronize with another CPU doing an H_EOI or a H_CPPR
636 * updating xc->cppr then reading xc->mfrr.
637 *
638 * - The target of the IPI sees the xc->mfrr update
639 */
640 mb();
641
642 /* Shoot the IPI if most favored than target cppr */
643 if (mfrr < xc->cppr)
644 __x_writeq(0, __x_trig_page(&xc->vp_ipi_data));
645
646 return H_SUCCESS;
647 }
648
