1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright 2017 Benjamin Herrenschmidt, IBM Corporation.
4 */
5
6 #define pr_fmt(fmt) "xive-kvm: " fmt
7
8 #include <linux/kernel.h>
9 #include <linux/kvm_host.h>
10 #include <linux/err.h>
11 #include <linux/gfp.h>
12 #include <linux/spinlock.h>
13 #include <linux/delay.h>
14 #include <linux/percpu.h>
15 #include <linux/cpumask.h>
16 #include <linux/uaccess.h>
17 #include <asm/kvm_book3s.h>
18 #include <asm/kvm_ppc.h>
19 #include <asm/hvcall.h>
20 #include <asm/xics.h>
21 #include <asm/xive.h>
22 #include <asm/xive-regs.h>
23 #include <asm/debug.h>
24 #include <asm/debugfs.h>
25 #include <asm/time.h>
26 #include <asm/opal.h>
27
28 #include <linux/debugfs.h>
29 #include <linux/seq_file.h>
30
31 #include "book3s_xive.h"
32
33
34 /*
35 * Virtual mode variants of the hcalls for use on radix/radix
36 * with AIL. They require the VCPU's VP to be "pushed"
37 *
38 * We still instantiate them here because we use some of the
39 * generated utility functions as well in this file.
40 */
41 #define XIVE_RUNTIME_CHECKS
42 #define X_PFX xive_vm_
43 #define X_STATIC static
44 #define X_STAT_PFX stat_vm_
45 #define __x_tima xive_tima
46 #define __x_eoi_page(xd) ((void __iomem *)((xd)->eoi_mmio))
47 #define __x_trig_page(xd) ((void __iomem *)((xd)->trig_mmio))
48 #define __x_writeb __raw_writeb
49 #define __x_readw __raw_readw
50 #define __x_readq __raw_readq
51 #define __x_writeq __raw_writeq
52
53 #include "book3s_xive_template.c"
54
55 /*
56 * We leave a gap of a couple of interrupts in the queue to
57 * account for the IPI and additional safety guard.
58 */
59 #define XIVE_Q_GAP 2
60
61 /*
62 * Push a vcpu's context to the XIVE on guest entry.
63 * This assumes we are in virtual mode (MMU on)
64 */
kvmppc_xive_push_vcpu(struct kvm_vcpu * vcpu)65 void kvmppc_xive_push_vcpu(struct kvm_vcpu *vcpu)
66 {
67 void __iomem *tima = local_paca->kvm_hstate.xive_tima_virt;
68 u64 pq;
69
70 /*
71 * Nothing to do if the platform doesn't have a XIVE
72 * or this vCPU doesn't have its own XIVE context
73 * (e.g. because it's not using an in-kernel interrupt controller).
74 */
75 if (!tima || !vcpu->arch.xive_cam_word)
76 return;
77
78 eieio();
79 __raw_writeq(vcpu->arch.xive_saved_state.w01, tima + TM_QW1_OS);
80 __raw_writel(vcpu->arch.xive_cam_word, tima + TM_QW1_OS + TM_WORD2);
81 vcpu->arch.xive_pushed = 1;
82 eieio();
83
84 /*
85 * We clear the irq_pending flag. There is a small chance of a
86 * race vs. the escalation interrupt happening on another
87 * processor setting it again, but the only consequence is to
88 * cause a spurious wakeup on the next H_CEDE, which is not an
89 * issue.
90 */
91 vcpu->arch.irq_pending = 0;
92
93 /*
94 * In single escalation mode, if the escalation interrupt is
95 * on, we mask it.
96 */
97 if (vcpu->arch.xive_esc_on) {
98 pq = __raw_readq((void __iomem *)(vcpu->arch.xive_esc_vaddr +
99 XIVE_ESB_SET_PQ_01));
100 mb();
101
102 /*
103 * We have a possible subtle race here: The escalation
104 * interrupt might have fired and be on its way to the
105 * host queue while we mask it, and if we unmask it
106 * early enough (re-cede right away), there is a
107 * theorical possibility that it fires again, thus
108 * landing in the target queue more than once which is
109 * a big no-no.
110 *
111 * Fortunately, solving this is rather easy. If the
112 * above load setting PQ to 01 returns a previous
113 * value where P is set, then we know the escalation
114 * interrupt is somewhere on its way to the host. In
115 * that case we simply don't clear the xive_esc_on
116 * flag below. It will be eventually cleared by the
117 * handler for the escalation interrupt.
118 *
119 * Then, when doing a cede, we check that flag again
120 * before re-enabling the escalation interrupt, and if
121 * set, we abort the cede.
122 */
123 if (!(pq & XIVE_ESB_VAL_P))
124 /* Now P is 0, we can clear the flag */
125 vcpu->arch.xive_esc_on = 0;
126 }
127 }
128 EXPORT_SYMBOL_GPL(kvmppc_xive_push_vcpu);
129
130 /*
131 * This is a simple trigger for a generic XIVE IRQ. This must
132 * only be called for interrupts that support a trigger page
133 */
xive_irq_trigger(struct xive_irq_data * xd)134 static bool xive_irq_trigger(struct xive_irq_data *xd)
135 {
136 /* This should be only for MSIs */
137 if (WARN_ON(xd->flags & XIVE_IRQ_FLAG_LSI))
138 return false;
139
140 /* Those interrupts should always have a trigger page */
141 if (WARN_ON(!xd->trig_mmio))
142 return false;
143
144 out_be64(xd->trig_mmio, 0);
145
146 return true;
147 }
148
xive_esc_irq(int irq,void * data)149 static irqreturn_t xive_esc_irq(int irq, void *data)
150 {
151 struct kvm_vcpu *vcpu = data;
152
153 vcpu->arch.irq_pending = 1;
154 smp_mb();
155 if (vcpu->arch.ceded)
156 kvmppc_fast_vcpu_kick(vcpu);
157
158 /* Since we have the no-EOI flag, the interrupt is effectively
159 * disabled now. Clearing xive_esc_on means we won't bother
160 * doing so on the next entry.
161 *
162 * This also allows the entry code to know that if a PQ combination
163 * of 10 is observed while xive_esc_on is true, it means the queue
164 * contains an unprocessed escalation interrupt. We don't make use of
165 * that knowledge today but might (see comment in book3s_hv_rmhandler.S)
166 */
167 vcpu->arch.xive_esc_on = false;
168
169 /* This orders xive_esc_on = false vs. subsequent stale_p = true */
170 smp_wmb(); /* goes with smp_mb() in cleanup_single_escalation */
171
172 return IRQ_HANDLED;
173 }
174
kvmppc_xive_attach_escalation(struct kvm_vcpu * vcpu,u8 prio,bool single_escalation)175 int kvmppc_xive_attach_escalation(struct kvm_vcpu *vcpu, u8 prio,
176 bool single_escalation)
177 {
178 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
179 struct xive_q *q = &xc->queues[prio];
180 char *name = NULL;
181 int rc;
182
183 /* Already there ? */
184 if (xc->esc_virq[prio])
185 return 0;
186
187 /* Hook up the escalation interrupt */
188 xc->esc_virq[prio] = irq_create_mapping(NULL, q->esc_irq);
189 if (!xc->esc_virq[prio]) {
190 pr_err("Failed to map escalation interrupt for queue %d of VCPU %d\n",
191 prio, xc->server_num);
192 return -EIO;
193 }
194
195 if (single_escalation)
196 name = kasprintf(GFP_KERNEL, "kvm-%d-%d",
197 vcpu->kvm->arch.lpid, xc->server_num);
198 else
199 name = kasprintf(GFP_KERNEL, "kvm-%d-%d-%d",
200 vcpu->kvm->arch.lpid, xc->server_num, prio);
201 if (!name) {
202 pr_err("Failed to allocate escalation irq name for queue %d of VCPU %d\n",
203 prio, xc->server_num);
204 rc = -ENOMEM;
205 goto error;
206 }
207
208 pr_devel("Escalation %s irq %d (prio %d)\n", name, xc->esc_virq[prio], prio);
209
210 rc = request_irq(xc->esc_virq[prio], xive_esc_irq,
211 IRQF_NO_THREAD, name, vcpu);
212 if (rc) {
213 pr_err("Failed to request escalation interrupt for queue %d of VCPU %d\n",
214 prio, xc->server_num);
215 goto error;
216 }
217 xc->esc_virq_names[prio] = name;
218
219 /* In single escalation mode, we grab the ESB MMIO of the
220 * interrupt and mask it. Also populate the VCPU v/raddr
221 * of the ESB page for use by asm entry/exit code. Finally
222 * set the XIVE_IRQ_NO_EOI flag which will prevent the
223 * core code from performing an EOI on the escalation
224 * interrupt, thus leaving it effectively masked after
225 * it fires once.
226 */
227 if (single_escalation) {
228 struct irq_data *d = irq_get_irq_data(xc->esc_virq[prio]);
229 struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);
230
231 xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_01);
232 vcpu->arch.xive_esc_raddr = xd->eoi_page;
233 vcpu->arch.xive_esc_vaddr = (__force u64)xd->eoi_mmio;
234 xd->flags |= XIVE_IRQ_NO_EOI;
235 }
236
237 return 0;
238 error:
239 irq_dispose_mapping(xc->esc_virq[prio]);
240 xc->esc_virq[prio] = 0;
241 kfree(name);
242 return rc;
243 }
244
xive_provision_queue(struct kvm_vcpu * vcpu,u8 prio)245 static int xive_provision_queue(struct kvm_vcpu *vcpu, u8 prio)
246 {
247 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
248 struct kvmppc_xive *xive = xc->xive;
249 struct xive_q *q = &xc->queues[prio];
250 void *qpage;
251 int rc;
252
253 if (WARN_ON(q->qpage))
254 return 0;
255
256 /* Allocate the queue and retrieve infos on current node for now */
257 qpage = (__be32 *)__get_free_pages(GFP_KERNEL, xive->q_page_order);
258 if (!qpage) {
259 pr_err("Failed to allocate queue %d for VCPU %d\n",
260 prio, xc->server_num);
261 return -ENOMEM;
262 }
263 memset(qpage, 0, 1 << xive->q_order);
264
265 /*
266 * Reconfigure the queue. This will set q->qpage only once the
267 * queue is fully configured. This is a requirement for prio 0
268 * as we will stop doing EOIs for every IPI as soon as we observe
269 * qpage being non-NULL, and instead will only EOI when we receive
270 * corresponding queue 0 entries
271 */
272 rc = xive_native_configure_queue(xc->vp_id, q, prio, qpage,
273 xive->q_order, true);
274 if (rc)
275 pr_err("Failed to configure queue %d for VCPU %d\n",
276 prio, xc->server_num);
277 return rc;
278 }
279
280 /* Called with xive->lock held */
xive_check_provisioning(struct kvm * kvm,u8 prio)281 static int xive_check_provisioning(struct kvm *kvm, u8 prio)
282 {
283 struct kvmppc_xive *xive = kvm->arch.xive;
284 struct kvm_vcpu *vcpu;
285 int i, rc;
286
287 lockdep_assert_held(&xive->lock);
288
289 /* Already provisioned ? */
290 if (xive->qmap & (1 << prio))
291 return 0;
292
293 pr_devel("Provisioning prio... %d\n", prio);
294
295 /* Provision each VCPU and enable escalations if needed */
296 kvm_for_each_vcpu(i, vcpu, kvm) {
297 if (!vcpu->arch.xive_vcpu)
298 continue;
299 rc = xive_provision_queue(vcpu, prio);
300 if (rc == 0 && !xive->single_escalation)
301 kvmppc_xive_attach_escalation(vcpu, prio,
302 xive->single_escalation);
303 if (rc)
304 return rc;
305 }
306
307 /* Order previous stores and mark it as provisioned */
308 mb();
309 xive->qmap |= (1 << prio);
310 return 0;
311 }
312
xive_inc_q_pending(struct kvm * kvm,u32 server,u8 prio)313 static void xive_inc_q_pending(struct kvm *kvm, u32 server, u8 prio)
314 {
315 struct kvm_vcpu *vcpu;
316 struct kvmppc_xive_vcpu *xc;
317 struct xive_q *q;
318
319 /* Locate target server */
320 vcpu = kvmppc_xive_find_server(kvm, server);
321 if (!vcpu) {
322 pr_warn("%s: Can't find server %d\n", __func__, server);
323 return;
324 }
325 xc = vcpu->arch.xive_vcpu;
326 if (WARN_ON(!xc))
327 return;
328
329 q = &xc->queues[prio];
330 atomic_inc(&q->pending_count);
331 }
332
xive_try_pick_queue(struct kvm_vcpu * vcpu,u8 prio)333 static int xive_try_pick_queue(struct kvm_vcpu *vcpu, u8 prio)
334 {
335 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
336 struct xive_q *q;
337 u32 max;
338
339 if (WARN_ON(!xc))
340 return -ENXIO;
341 if (!xc->valid)
342 return -ENXIO;
343
344 q = &xc->queues[prio];
345 if (WARN_ON(!q->qpage))
346 return -ENXIO;
347
348 /* Calculate max number of interrupts in that queue. */
349 max = (q->msk + 1) - XIVE_Q_GAP;
350 return atomic_add_unless(&q->count, 1, max) ? 0 : -EBUSY;
351 }
352
kvmppc_xive_select_target(struct kvm * kvm,u32 * server,u8 prio)353 int kvmppc_xive_select_target(struct kvm *kvm, u32 *server, u8 prio)
354 {
355 struct kvm_vcpu *vcpu;
356 int i, rc;
357
358 /* Locate target server */
359 vcpu = kvmppc_xive_find_server(kvm, *server);
360 if (!vcpu) {
361 pr_devel("Can't find server %d\n", *server);
362 return -EINVAL;
363 }
364
365 pr_devel("Finding irq target on 0x%x/%d...\n", *server, prio);
366
367 /* Try pick it */
368 rc = xive_try_pick_queue(vcpu, prio);
369 if (rc == 0)
370 return rc;
371
372 pr_devel(" .. failed, looking up candidate...\n");
373
374 /* Failed, pick another VCPU */
375 kvm_for_each_vcpu(i, vcpu, kvm) {
376 if (!vcpu->arch.xive_vcpu)
377 continue;
378 rc = xive_try_pick_queue(vcpu, prio);
379 if (rc == 0) {
380 *server = vcpu->arch.xive_vcpu->server_num;
381 pr_devel(" found on 0x%x/%d\n", *server, prio);
382 return rc;
383 }
384 }
385 pr_devel(" no available target !\n");
386
387 /* No available target ! */
388 return -EBUSY;
389 }
390
xive_lock_and_mask(struct kvmppc_xive * xive,struct kvmppc_xive_src_block * sb,struct kvmppc_xive_irq_state * state)391 static u8 xive_lock_and_mask(struct kvmppc_xive *xive,
392 struct kvmppc_xive_src_block *sb,
393 struct kvmppc_xive_irq_state *state)
394 {
395 struct xive_irq_data *xd;
396 u32 hw_num;
397 u8 old_prio;
398 u64 val;
399
400 /*
401 * Take the lock, set masked, try again if racing
402 * with H_EOI
403 */
404 for (;;) {
405 arch_spin_lock(&sb->lock);
406 old_prio = state->guest_priority;
407 state->guest_priority = MASKED;
408 mb();
409 if (!state->in_eoi)
410 break;
411 state->guest_priority = old_prio;
412 arch_spin_unlock(&sb->lock);
413 }
414
415 /* No change ? Bail */
416 if (old_prio == MASKED)
417 return old_prio;
418
419 /* Get the right irq */
420 kvmppc_xive_select_irq(state, &hw_num, &xd);
421
422 /*
423 * If the interrupt is marked as needing masking via
424 * firmware, we do it here. Firmware masking however
425 * is "lossy", it won't return the old p and q bits
426 * and won't set the interrupt to a state where it will
427 * record queued ones. If this is an issue we should do
428 * lazy masking instead.
429 *
430 * For now, we work around this in unmask by forcing
431 * an interrupt whenever we unmask a non-LSI via FW
432 * (if ever).
433 */
434 if (xd->flags & OPAL_XIVE_IRQ_MASK_VIA_FW) {
435 xive_native_configure_irq(hw_num,
436 kvmppc_xive_vp(xive, state->act_server),
437 MASKED, state->number);
438 /* set old_p so we can track if an H_EOI was done */
439 state->old_p = true;
440 state->old_q = false;
441 } else {
442 /* Set PQ to 10, return old P and old Q and remember them */
443 val = xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_10);
444 state->old_p = !!(val & 2);
445 state->old_q = !!(val & 1);
446
447 /*
448 * Synchronize hardware to sensure the queues are updated
449 * when masking
450 */
451 xive_native_sync_source(hw_num);
452 }
453
454 return old_prio;
455 }
456
xive_lock_for_unmask(struct kvmppc_xive_src_block * sb,struct kvmppc_xive_irq_state * state)457 static void xive_lock_for_unmask(struct kvmppc_xive_src_block *sb,
458 struct kvmppc_xive_irq_state *state)
459 {
460 /*
461 * Take the lock try again if racing with H_EOI
462 */
463 for (;;) {
464 arch_spin_lock(&sb->lock);
465 if (!state->in_eoi)
466 break;
467 arch_spin_unlock(&sb->lock);
468 }
469 }
470
xive_finish_unmask(struct kvmppc_xive * xive,struct kvmppc_xive_src_block * sb,struct kvmppc_xive_irq_state * state,u8 prio)471 static void xive_finish_unmask(struct kvmppc_xive *xive,
472 struct kvmppc_xive_src_block *sb,
473 struct kvmppc_xive_irq_state *state,
474 u8 prio)
475 {
476 struct xive_irq_data *xd;
477 u32 hw_num;
478
479 /* If we aren't changing a thing, move on */
480 if (state->guest_priority != MASKED)
481 goto bail;
482
483 /* Get the right irq */
484 kvmppc_xive_select_irq(state, &hw_num, &xd);
485
486 /*
487 * See comment in xive_lock_and_mask() concerning masking
488 * via firmware.
489 */
490 if (xd->flags & OPAL_XIVE_IRQ_MASK_VIA_FW) {
491 xive_native_configure_irq(hw_num,
492 kvmppc_xive_vp(xive, state->act_server),
493 state->act_priority, state->number);
494 /* If an EOI is needed, do it here */
495 if (!state->old_p)
496 xive_vm_source_eoi(hw_num, xd);
497 /* If this is not an LSI, force a trigger */
498 if (!(xd->flags & OPAL_XIVE_IRQ_LSI))
499 xive_irq_trigger(xd);
500 goto bail;
501 }
502
503 /* Old Q set, set PQ to 11 */
504 if (state->old_q)
505 xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_11);
506
507 /*
508 * If not old P, then perform an "effective" EOI,
509 * on the source. This will handle the cases where
510 * FW EOI is needed.
511 */
512 if (!state->old_p)
513 xive_vm_source_eoi(hw_num, xd);
514
515 /* Synchronize ordering and mark unmasked */
516 mb();
517 bail:
518 state->guest_priority = prio;
519 }
520
521 /*
522 * Target an interrupt to a given server/prio, this will fallback
523 * to another server if necessary and perform the HW targetting
524 * updates as needed
525 *
526 * NOTE: Must be called with the state lock held
527 */
xive_target_interrupt(struct kvm * kvm,struct kvmppc_xive_irq_state * state,u32 server,u8 prio)528 static int xive_target_interrupt(struct kvm *kvm,
529 struct kvmppc_xive_irq_state *state,
530 u32 server, u8 prio)
531 {
532 struct kvmppc_xive *xive = kvm->arch.xive;
533 u32 hw_num;
534 int rc;
535
536 /*
537 * This will return a tentative server and actual
538 * priority. The count for that new target will have
539 * already been incremented.
540 */
541 rc = kvmppc_xive_select_target(kvm, &server, prio);
542
543 /*
544 * We failed to find a target ? Not much we can do
545 * at least until we support the GIQ.
546 */
547 if (rc)
548 return rc;
549
550 /*
551 * Increment the old queue pending count if there
552 * was one so that the old queue count gets adjusted later
553 * when observed to be empty.
554 */
555 if (state->act_priority != MASKED)
556 xive_inc_q_pending(kvm,
557 state->act_server,
558 state->act_priority);
559 /*
560 * Update state and HW
561 */
562 state->act_priority = prio;
563 state->act_server = server;
564
565 /* Get the right irq */
566 kvmppc_xive_select_irq(state, &hw_num, NULL);
567
568 return xive_native_configure_irq(hw_num,
569 kvmppc_xive_vp(xive, server),
570 prio, state->number);
571 }
572
573 /*
574 * Targetting rules: In order to avoid losing track of
575 * pending interrupts accross mask and unmask, which would
576 * allow queue overflows, we implement the following rules:
577 *
578 * - Unless it was never enabled (or we run out of capacity)
579 * an interrupt is always targetted at a valid server/queue
580 * pair even when "masked" by the guest. This pair tends to
581 * be the last one used but it can be changed under some
582 * circumstances. That allows us to separate targetting
583 * from masking, we only handle accounting during (re)targetting,
584 * this also allows us to let an interrupt drain into its target
585 * queue after masking, avoiding complex schemes to remove
586 * interrupts out of remote processor queues.
587 *
588 * - When masking, we set PQ to 10 and save the previous value
589 * of P and Q.
590 *
591 * - When unmasking, if saved Q was set, we set PQ to 11
592 * otherwise we leave PQ to the HW state which will be either
593 * 10 if nothing happened or 11 if the interrupt fired while
594 * masked. Effectively we are OR'ing the previous Q into the
595 * HW Q.
596 *
597 * Then if saved P is clear, we do an effective EOI (Q->P->Trigger)
598 * which will unmask the interrupt and shoot a new one if Q was
599 * set.
600 *
601 * Otherwise (saved P is set) we leave PQ unchanged (so 10 or 11,
602 * effectively meaning an H_EOI from the guest is still expected
603 * for that interrupt).
604 *
605 * - If H_EOI occurs while masked, we clear the saved P.
606 *
607 * - When changing target, we account on the new target and
608 * increment a separate "pending" counter on the old one.
609 * This pending counter will be used to decrement the old
610 * target's count when its queue has been observed empty.
611 */
612
kvmppc_xive_set_xive(struct kvm * kvm,u32 irq,u32 server,u32 priority)613 int kvmppc_xive_set_xive(struct kvm *kvm, u32 irq, u32 server,
614 u32 priority)
615 {
616 struct kvmppc_xive *xive = kvm->arch.xive;
617 struct kvmppc_xive_src_block *sb;
618 struct kvmppc_xive_irq_state *state;
619 u8 new_act_prio;
620 int rc = 0;
621 u16 idx;
622
623 if (!xive)
624 return -ENODEV;
625
626 pr_devel("set_xive ! irq 0x%x server 0x%x prio %d\n",
627 irq, server, priority);
628
629 /* First, check provisioning of queues */
630 if (priority != MASKED) {
631 mutex_lock(&xive->lock);
632 rc = xive_check_provisioning(xive->kvm,
633 xive_prio_from_guest(priority));
634 mutex_unlock(&xive->lock);
635 }
636 if (rc) {
637 pr_devel(" provisioning failure %d !\n", rc);
638 return rc;
639 }
640
641 sb = kvmppc_xive_find_source(xive, irq, &idx);
642 if (!sb)
643 return -EINVAL;
644 state = &sb->irq_state[idx];
645
646 /*
647 * We first handle masking/unmasking since the locking
648 * might need to be retried due to EOIs, we'll handle
649 * targetting changes later. These functions will return
650 * with the SB lock held.
651 *
652 * xive_lock_and_mask() will also set state->guest_priority
653 * but won't otherwise change other fields of the state.
654 *
655 * xive_lock_for_unmask will not actually unmask, this will
656 * be done later by xive_finish_unmask() once the targetting
657 * has been done, so we don't try to unmask an interrupt
658 * that hasn't yet been targetted.
659 */
660 if (priority == MASKED)
661 xive_lock_and_mask(xive, sb, state);
662 else
663 xive_lock_for_unmask(sb, state);
664
665
666 /*
667 * Then we handle targetting.
668 *
669 * First calculate a new "actual priority"
670 */
671 new_act_prio = state->act_priority;
672 if (priority != MASKED)
673 new_act_prio = xive_prio_from_guest(priority);
674
675 pr_devel(" new_act_prio=%x act_server=%x act_prio=%x\n",
676 new_act_prio, state->act_server, state->act_priority);
677
678 /*
679 * Then check if we actually need to change anything,
680 *
681 * The condition for re-targetting the interrupt is that
682 * we have a valid new priority (new_act_prio is not 0xff)
683 * and either the server or the priority changed.
684 *
685 * Note: If act_priority was ff and the new priority is
686 * also ff, we don't do anything and leave the interrupt
687 * untargetted. An attempt of doing an int_on on an
688 * untargetted interrupt will fail. If that is a problem
689 * we could initialize interrupts with valid default
690 */
691
692 if (new_act_prio != MASKED &&
693 (state->act_server != server ||
694 state->act_priority != new_act_prio))
695 rc = xive_target_interrupt(kvm, state, server, new_act_prio);
696
697 /*
698 * Perform the final unmasking of the interrupt source
699 * if necessary
700 */
701 if (priority != MASKED)
702 xive_finish_unmask(xive, sb, state, priority);
703
704 /*
705 * Finally Update saved_priority to match. Only int_on/off
706 * set this field to a different value.
707 */
708 state->saved_priority = priority;
709
710 arch_spin_unlock(&sb->lock);
711 return rc;
712 }
713
kvmppc_xive_get_xive(struct kvm * kvm,u32 irq,u32 * server,u32 * priority)714 int kvmppc_xive_get_xive(struct kvm *kvm, u32 irq, u32 *server,
715 u32 *priority)
716 {
717 struct kvmppc_xive *xive = kvm->arch.xive;
718 struct kvmppc_xive_src_block *sb;
719 struct kvmppc_xive_irq_state *state;
720 u16 idx;
721
722 if (!xive)
723 return -ENODEV;
724
725 sb = kvmppc_xive_find_source(xive, irq, &idx);
726 if (!sb)
727 return -EINVAL;
728 state = &sb->irq_state[idx];
729 arch_spin_lock(&sb->lock);
730 *server = state->act_server;
731 *priority = state->guest_priority;
732 arch_spin_unlock(&sb->lock);
733
734 return 0;
735 }
736
kvmppc_xive_int_on(struct kvm * kvm,u32 irq)737 int kvmppc_xive_int_on(struct kvm *kvm, u32 irq)
738 {
739 struct kvmppc_xive *xive = kvm->arch.xive;
740 struct kvmppc_xive_src_block *sb;
741 struct kvmppc_xive_irq_state *state;
742 u16 idx;
743
744 if (!xive)
745 return -ENODEV;
746
747 sb = kvmppc_xive_find_source(xive, irq, &idx);
748 if (!sb)
749 return -EINVAL;
750 state = &sb->irq_state[idx];
751
752 pr_devel("int_on(irq=0x%x)\n", irq);
753
754 /*
755 * Check if interrupt was not targetted
756 */
757 if (state->act_priority == MASKED) {
758 pr_devel("int_on on untargetted interrupt\n");
759 return -EINVAL;
760 }
761
762 /* If saved_priority is 0xff, do nothing */
763 if (state->saved_priority == MASKED)
764 return 0;
765
766 /*
767 * Lock and unmask it.
768 */
769 xive_lock_for_unmask(sb, state);
770 xive_finish_unmask(xive, sb, state, state->saved_priority);
771 arch_spin_unlock(&sb->lock);
772
773 return 0;
774 }
775
kvmppc_xive_int_off(struct kvm * kvm,u32 irq)776 int kvmppc_xive_int_off(struct kvm *kvm, u32 irq)
777 {
778 struct kvmppc_xive *xive = kvm->arch.xive;
779 struct kvmppc_xive_src_block *sb;
780 struct kvmppc_xive_irq_state *state;
781 u16 idx;
782
783 if (!xive)
784 return -ENODEV;
785
786 sb = kvmppc_xive_find_source(xive, irq, &idx);
787 if (!sb)
788 return -EINVAL;
789 state = &sb->irq_state[idx];
790
791 pr_devel("int_off(irq=0x%x)\n", irq);
792
793 /*
794 * Lock and mask
795 */
796 state->saved_priority = xive_lock_and_mask(xive, sb, state);
797 arch_spin_unlock(&sb->lock);
798
799 return 0;
800 }
801
xive_restore_pending_irq(struct kvmppc_xive * xive,u32 irq)802 static bool xive_restore_pending_irq(struct kvmppc_xive *xive, u32 irq)
803 {
804 struct kvmppc_xive_src_block *sb;
805 struct kvmppc_xive_irq_state *state;
806 u16 idx;
807
808 sb = kvmppc_xive_find_source(xive, irq, &idx);
809 if (!sb)
810 return false;
811 state = &sb->irq_state[idx];
812 if (!state->valid)
813 return false;
814
815 /*
816 * Trigger the IPI. This assumes we never restore a pass-through
817 * interrupt which should be safe enough
818 */
819 xive_irq_trigger(&state->ipi_data);
820
821 return true;
822 }
823
kvmppc_xive_get_icp(struct kvm_vcpu * vcpu)824 u64 kvmppc_xive_get_icp(struct kvm_vcpu *vcpu)
825 {
826 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
827
828 if (!xc)
829 return 0;
830
831 /* Return the per-cpu state for state saving/migration */
832 return (u64)xc->cppr << KVM_REG_PPC_ICP_CPPR_SHIFT |
833 (u64)xc->mfrr << KVM_REG_PPC_ICP_MFRR_SHIFT |
834 (u64)0xff << KVM_REG_PPC_ICP_PPRI_SHIFT;
835 }
836
kvmppc_xive_set_icp(struct kvm_vcpu * vcpu,u64 icpval)837 int kvmppc_xive_set_icp(struct kvm_vcpu *vcpu, u64 icpval)
838 {
839 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
840 struct kvmppc_xive *xive = vcpu->kvm->arch.xive;
841 u8 cppr, mfrr;
842 u32 xisr;
843
844 if (!xc || !xive)
845 return -ENOENT;
846
847 /* Grab individual state fields. We don't use pending_pri */
848 cppr = icpval >> KVM_REG_PPC_ICP_CPPR_SHIFT;
849 xisr = (icpval >> KVM_REG_PPC_ICP_XISR_SHIFT) &
850 KVM_REG_PPC_ICP_XISR_MASK;
851 mfrr = icpval >> KVM_REG_PPC_ICP_MFRR_SHIFT;
852
853 pr_devel("set_icp vcpu %d cppr=0x%x mfrr=0x%x xisr=0x%x\n",
854 xc->server_num, cppr, mfrr, xisr);
855
856 /*
857 * We can't update the state of a "pushed" VCPU, but that
858 * shouldn't happen because the vcpu->mutex makes running a
859 * vcpu mutually exclusive with doing one_reg get/set on it.
860 */
861 if (WARN_ON(vcpu->arch.xive_pushed))
862 return -EIO;
863
864 /* Update VCPU HW saved state */
865 vcpu->arch.xive_saved_state.cppr = cppr;
866 xc->hw_cppr = xc->cppr = cppr;
867
868 /*
869 * Update MFRR state. If it's not 0xff, we mark the VCPU as
870 * having a pending MFRR change, which will re-evaluate the
871 * target. The VCPU will thus potentially get a spurious
872 * interrupt but that's not a big deal.
873 */
874 xc->mfrr = mfrr;
875 if (mfrr < cppr)
876 xive_irq_trigger(&xc->vp_ipi_data);
877
878 /*
879 * Now saved XIRR is "interesting". It means there's something in
880 * the legacy "1 element" queue... for an IPI we simply ignore it,
881 * as the MFRR restore will handle that. For anything else we need
882 * to force a resend of the source.
883 * However the source may not have been setup yet. If that's the
884 * case, we keep that info and increment a counter in the xive to
885 * tell subsequent xive_set_source() to go look.
886 */
887 if (xisr > XICS_IPI && !xive_restore_pending_irq(xive, xisr)) {
888 xc->delayed_irq = xisr;
889 xive->delayed_irqs++;
890 pr_devel(" xisr restore delayed\n");
891 }
892
893 return 0;
894 }
895
kvmppc_xive_set_mapped(struct kvm * kvm,unsigned long guest_irq,struct irq_desc * host_desc)896 int kvmppc_xive_set_mapped(struct kvm *kvm, unsigned long guest_irq,
897 struct irq_desc *host_desc)
898 {
899 struct kvmppc_xive *xive = kvm->arch.xive;
900 struct kvmppc_xive_src_block *sb;
901 struct kvmppc_xive_irq_state *state;
902 struct irq_data *host_data = irq_desc_get_irq_data(host_desc);
903 unsigned int host_irq = irq_desc_get_irq(host_desc);
904 unsigned int hw_irq = (unsigned int)irqd_to_hwirq(host_data);
905 u16 idx;
906 u8 prio;
907 int rc;
908
909 if (!xive)
910 return -ENODEV;
911
912 pr_devel("set_mapped girq 0x%lx host HW irq 0x%x...\n",guest_irq, hw_irq);
913
914 sb = kvmppc_xive_find_source(xive, guest_irq, &idx);
915 if (!sb)
916 return -EINVAL;
917 state = &sb->irq_state[idx];
918
919 /*
920 * Mark the passed-through interrupt as going to a VCPU,
921 * this will prevent further EOIs and similar operations
922 * from the XIVE code. It will also mask the interrupt
923 * to either PQ=10 or 11 state, the latter if the interrupt
924 * is pending. This will allow us to unmask or retrigger it
925 * after routing it to the guest with a simple EOI.
926 *
927 * The "state" argument is a "token", all it needs is to be
928 * non-NULL to switch to passed-through or NULL for the
929 * other way around. We may not yet have an actual VCPU
930 * target here and we don't really care.
931 */
932 rc = irq_set_vcpu_affinity(host_irq, state);
933 if (rc) {
934 pr_err("Failed to set VCPU affinity for irq %d\n", host_irq);
935 return rc;
936 }
937
938 /*
939 * Mask and read state of IPI. We need to know if its P bit
940 * is set as that means it's potentially already using a
941 * queue entry in the target
942 */
943 prio = xive_lock_and_mask(xive, sb, state);
944 pr_devel(" old IPI prio %02x P:%d Q:%d\n", prio,
945 state->old_p, state->old_q);
946
947 /* Turn the IPI hard off */
948 xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_01);
949
950 /*
951 * Reset ESB guest mapping. Needed when ESB pages are exposed
952 * to the guest in XIVE native mode
953 */
954 if (xive->ops && xive->ops->reset_mapped)
955 xive->ops->reset_mapped(kvm, guest_irq);
956
957 /* Grab info about irq */
958 state->pt_number = hw_irq;
959 state->pt_data = irq_data_get_irq_handler_data(host_data);
960
961 /*
962 * Configure the IRQ to match the existing configuration of
963 * the IPI if it was already targetted. Otherwise this will
964 * mask the interrupt in a lossy way (act_priority is 0xff)
965 * which is fine for a never started interrupt.
966 */
967 xive_native_configure_irq(hw_irq,
968 kvmppc_xive_vp(xive, state->act_server),
969 state->act_priority, state->number);
970
971 /*
972 * We do an EOI to enable the interrupt (and retrigger if needed)
973 * if the guest has the interrupt unmasked and the P bit was *not*
974 * set in the IPI. If it was set, we know a slot may still be in
975 * use in the target queue thus we have to wait for a guest
976 * originated EOI
977 */
978 if (prio != MASKED && !state->old_p)
979 xive_vm_source_eoi(hw_irq, state->pt_data);
980
981 /* Clear old_p/old_q as they are no longer relevant */
982 state->old_p = state->old_q = false;
983
984 /* Restore guest prio (unlocks EOI) */
985 mb();
986 state->guest_priority = prio;
987 arch_spin_unlock(&sb->lock);
988
989 return 0;
990 }
991 EXPORT_SYMBOL_GPL(kvmppc_xive_set_mapped);
992
kvmppc_xive_clr_mapped(struct kvm * kvm,unsigned long guest_irq,struct irq_desc * host_desc)993 int kvmppc_xive_clr_mapped(struct kvm *kvm, unsigned long guest_irq,
994 struct irq_desc *host_desc)
995 {
996 struct kvmppc_xive *xive = kvm->arch.xive;
997 struct kvmppc_xive_src_block *sb;
998 struct kvmppc_xive_irq_state *state;
999 unsigned int host_irq = irq_desc_get_irq(host_desc);
1000 u16 idx;
1001 u8 prio;
1002 int rc;
1003
1004 if (!xive)
1005 return -ENODEV;
1006
1007 pr_devel("clr_mapped girq 0x%lx...\n", guest_irq);
1008
1009 sb = kvmppc_xive_find_source(xive, guest_irq, &idx);
1010 if (!sb)
1011 return -EINVAL;
1012 state = &sb->irq_state[idx];
1013
1014 /*
1015 * Mask and read state of IRQ. We need to know if its P bit
1016 * is set as that means it's potentially already using a
1017 * queue entry in the target
1018 */
1019 prio = xive_lock_and_mask(xive, sb, state);
1020 pr_devel(" old IRQ prio %02x P:%d Q:%d\n", prio,
1021 state->old_p, state->old_q);
1022
1023 /*
1024 * If old_p is set, the interrupt is pending, we switch it to
1025 * PQ=11. This will force a resend in the host so the interrupt
1026 * isn't lost to whatver host driver may pick it up
1027 */
1028 if (state->old_p)
1029 xive_vm_esb_load(state->pt_data, XIVE_ESB_SET_PQ_11);
1030
1031 /* Release the passed-through interrupt to the host */
1032 rc = irq_set_vcpu_affinity(host_irq, NULL);
1033 if (rc) {
1034 pr_err("Failed to clr VCPU affinity for irq %d\n", host_irq);
1035 return rc;
1036 }
1037
1038 /* Forget about the IRQ */
1039 state->pt_number = 0;
1040 state->pt_data = NULL;
1041
1042 /*
1043 * Reset ESB guest mapping. Needed when ESB pages are exposed
1044 * to the guest in XIVE native mode
1045 */
1046 if (xive->ops && xive->ops->reset_mapped) {
1047 xive->ops->reset_mapped(kvm, guest_irq);
1048 }
1049
1050 /* Reconfigure the IPI */
1051 xive_native_configure_irq(state->ipi_number,
1052 kvmppc_xive_vp(xive, state->act_server),
1053 state->act_priority, state->number);
1054
1055 /*
1056 * If old_p is set (we have a queue entry potentially
1057 * occupied) or the interrupt is masked, we set the IPI
1058 * to PQ=10 state. Otherwise we just re-enable it (PQ=00).
1059 */
1060 if (prio == MASKED || state->old_p)
1061 xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_10);
1062 else
1063 xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_00);
1064
1065 /* Restore guest prio (unlocks EOI) */
1066 mb();
1067 state->guest_priority = prio;
1068 arch_spin_unlock(&sb->lock);
1069
1070 return 0;
1071 }
1072 EXPORT_SYMBOL_GPL(kvmppc_xive_clr_mapped);
1073
kvmppc_xive_disable_vcpu_interrupts(struct kvm_vcpu * vcpu)1074 void kvmppc_xive_disable_vcpu_interrupts(struct kvm_vcpu *vcpu)
1075 {
1076 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
1077 struct kvm *kvm = vcpu->kvm;
1078 struct kvmppc_xive *xive = kvm->arch.xive;
1079 int i, j;
1080
1081 for (i = 0; i <= xive->max_sbid; i++) {
1082 struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
1083
1084 if (!sb)
1085 continue;
1086 for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++) {
1087 struct kvmppc_xive_irq_state *state = &sb->irq_state[j];
1088
1089 if (!state->valid)
1090 continue;
1091 if (state->act_priority == MASKED)
1092 continue;
1093 if (state->act_server != xc->server_num)
1094 continue;
1095
1096 /* Clean it up */
1097 arch_spin_lock(&sb->lock);
1098 state->act_priority = MASKED;
1099 xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_01);
1100 xive_native_configure_irq(state->ipi_number, 0, MASKED, 0);
1101 if (state->pt_number) {
1102 xive_vm_esb_load(state->pt_data, XIVE_ESB_SET_PQ_01);
1103 xive_native_configure_irq(state->pt_number, 0, MASKED, 0);
1104 }
1105 arch_spin_unlock(&sb->lock);
1106 }
1107 }
1108
1109 /* Disable vcpu's escalation interrupt */
1110 if (vcpu->arch.xive_esc_on) {
1111 __raw_readq((void __iomem *)(vcpu->arch.xive_esc_vaddr +
1112 XIVE_ESB_SET_PQ_01));
1113 vcpu->arch.xive_esc_on = false;
1114 }
1115
1116 /*
1117 * Clear pointers to escalation interrupt ESB.
1118 * This is safe because the vcpu->mutex is held, preventing
1119 * any other CPU from concurrently executing a KVM_RUN ioctl.
1120 */
1121 vcpu->arch.xive_esc_vaddr = 0;
1122 vcpu->arch.xive_esc_raddr = 0;
1123 }
1124
1125 /*
1126 * In single escalation mode, the escalation interrupt is marked so
1127 * that EOI doesn't re-enable it, but just sets the stale_p flag to
1128 * indicate that the P bit has already been dealt with. However, the
1129 * assembly code that enters the guest sets PQ to 00 without clearing
1130 * stale_p (because it has no easy way to address it). Hence we have
1131 * to adjust stale_p before shutting down the interrupt.
1132 */
xive_cleanup_single_escalation(struct kvm_vcpu * vcpu,struct kvmppc_xive_vcpu * xc,int irq)1133 void xive_cleanup_single_escalation(struct kvm_vcpu *vcpu,
1134 struct kvmppc_xive_vcpu *xc, int irq)
1135 {
1136 struct irq_data *d = irq_get_irq_data(irq);
1137 struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);
1138
1139 /*
1140 * This slightly odd sequence gives the right result
1141 * (i.e. stale_p set if xive_esc_on is false) even if
1142 * we race with xive_esc_irq() and xive_irq_eoi().
1143 */
1144 xd->stale_p = false;
1145 smp_mb(); /* paired with smb_wmb in xive_esc_irq */
1146 if (!vcpu->arch.xive_esc_on)
1147 xd->stale_p = true;
1148 }
1149
kvmppc_xive_cleanup_vcpu(struct kvm_vcpu * vcpu)1150 void kvmppc_xive_cleanup_vcpu(struct kvm_vcpu *vcpu)
1151 {
1152 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
1153 struct kvmppc_xive *xive = vcpu->kvm->arch.xive;
1154 int i;
1155
1156 if (!kvmppc_xics_enabled(vcpu))
1157 return;
1158
1159 if (!xc)
1160 return;
1161
1162 pr_devel("cleanup_vcpu(cpu=%d)\n", xc->server_num);
1163
1164 /* Ensure no interrupt is still routed to that VP */
1165 xc->valid = false;
1166 kvmppc_xive_disable_vcpu_interrupts(vcpu);
1167
1168 /* Mask the VP IPI */
1169 xive_vm_esb_load(&xc->vp_ipi_data, XIVE_ESB_SET_PQ_01);
1170
1171 /* Free escalations */
1172 for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
1173 if (xc->esc_virq[i]) {
1174 if (xc->xive->single_escalation)
1175 xive_cleanup_single_escalation(vcpu, xc,
1176 xc->esc_virq[i]);
1177 free_irq(xc->esc_virq[i], vcpu);
1178 irq_dispose_mapping(xc->esc_virq[i]);
1179 kfree(xc->esc_virq_names[i]);
1180 }
1181 }
1182
1183 /* Disable the VP */
1184 xive_native_disable_vp(xc->vp_id);
1185
1186 /* Clear the cam word so guest entry won't try to push context */
1187 vcpu->arch.xive_cam_word = 0;
1188
1189 /* Free the queues */
1190 for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
1191 struct xive_q *q = &xc->queues[i];
1192
1193 xive_native_disable_queue(xc->vp_id, q, i);
1194 if (q->qpage) {
1195 free_pages((unsigned long)q->qpage,
1196 xive->q_page_order);
1197 q->qpage = NULL;
1198 }
1199 }
1200
1201 /* Free the IPI */
1202 if (xc->vp_ipi) {
1203 xive_cleanup_irq_data(&xc->vp_ipi_data);
1204 xive_native_free_irq(xc->vp_ipi);
1205 }
1206 /* Free the VP */
1207 kfree(xc);
1208
1209 /* Cleanup the vcpu */
1210 vcpu->arch.irq_type = KVMPPC_IRQ_DEFAULT;
1211 vcpu->arch.xive_vcpu = NULL;
1212 }
1213
kvmppc_xive_vcpu_id_valid(struct kvmppc_xive * xive,u32 cpu)1214 static bool kvmppc_xive_vcpu_id_valid(struct kvmppc_xive *xive, u32 cpu)
1215 {
1216 /* We have a block of xive->nr_servers VPs. We just need to check
1217 * packed vCPU ids are below that.
1218 */
1219 return kvmppc_pack_vcpu_id(xive->kvm, cpu) < xive->nr_servers;
1220 }
1221
kvmppc_xive_compute_vp_id(struct kvmppc_xive * xive,u32 cpu,u32 * vp)1222 int kvmppc_xive_compute_vp_id(struct kvmppc_xive *xive, u32 cpu, u32 *vp)
1223 {
1224 u32 vp_id;
1225
1226 if (!kvmppc_xive_vcpu_id_valid(xive, cpu)) {
1227 pr_devel("Out of bounds !\n");
1228 return -EINVAL;
1229 }
1230
1231 if (xive->vp_base == XIVE_INVALID_VP) {
1232 xive->vp_base = xive_native_alloc_vp_block(xive->nr_servers);
1233 pr_devel("VP_Base=%x nr_servers=%d\n", xive->vp_base, xive->nr_servers);
1234
1235 if (xive->vp_base == XIVE_INVALID_VP)
1236 return -ENOSPC;
1237 }
1238
1239 vp_id = kvmppc_xive_vp(xive, cpu);
1240 if (kvmppc_xive_vp_in_use(xive->kvm, vp_id)) {
1241 pr_devel("Duplicate !\n");
1242 return -EEXIST;
1243 }
1244
1245 *vp = vp_id;
1246
1247 return 0;
1248 }
1249
kvmppc_xive_connect_vcpu(struct kvm_device * dev,struct kvm_vcpu * vcpu,u32 cpu)1250 int kvmppc_xive_connect_vcpu(struct kvm_device *dev,
1251 struct kvm_vcpu *vcpu, u32 cpu)
1252 {
1253 struct kvmppc_xive *xive = dev->private;
1254 struct kvmppc_xive_vcpu *xc;
1255 int i, r = -EBUSY;
1256 u32 vp_id;
1257
1258 pr_devel("connect_vcpu(cpu=%d)\n", cpu);
1259
1260 if (dev->ops != &kvm_xive_ops) {
1261 pr_devel("Wrong ops !\n");
1262 return -EPERM;
1263 }
1264 if (xive->kvm != vcpu->kvm)
1265 return -EPERM;
1266 if (vcpu->arch.irq_type != KVMPPC_IRQ_DEFAULT)
1267 return -EBUSY;
1268
1269 /* We need to synchronize with queue provisioning */
1270 mutex_lock(&xive->lock);
1271
1272 r = kvmppc_xive_compute_vp_id(xive, cpu, &vp_id);
1273 if (r)
1274 goto bail;
1275
1276 xc = kzalloc(sizeof(*xc), GFP_KERNEL);
1277 if (!xc) {
1278 r = -ENOMEM;
1279 goto bail;
1280 }
1281
1282 vcpu->arch.xive_vcpu = xc;
1283 xc->xive = xive;
1284 xc->vcpu = vcpu;
1285 xc->server_num = cpu;
1286 xc->vp_id = vp_id;
1287 xc->mfrr = 0xff;
1288 xc->valid = true;
1289
1290 r = xive_native_get_vp_info(xc->vp_id, &xc->vp_cam, &xc->vp_chip_id);
1291 if (r)
1292 goto bail;
1293
1294 /* Configure VCPU fields for use by assembly push/pull */
1295 vcpu->arch.xive_saved_state.w01 = cpu_to_be64(0xff000000);
1296 vcpu->arch.xive_cam_word = cpu_to_be32(xc->vp_cam | TM_QW1W2_VO);
1297
1298 /* Allocate IPI */
1299 xc->vp_ipi = xive_native_alloc_irq();
1300 if (!xc->vp_ipi) {
1301 pr_err("Failed to allocate xive irq for VCPU IPI\n");
1302 r = -EIO;
1303 goto bail;
1304 }
1305 pr_devel(" IPI=0x%x\n", xc->vp_ipi);
1306
1307 r = xive_native_populate_irq_data(xc->vp_ipi, &xc->vp_ipi_data);
1308 if (r)
1309 goto bail;
1310
1311 /*
1312 * Enable the VP first as the single escalation mode will
1313 * affect escalation interrupts numbering
1314 */
1315 r = xive_native_enable_vp(xc->vp_id, xive->single_escalation);
1316 if (r) {
1317 pr_err("Failed to enable VP in OPAL, err %d\n", r);
1318 goto bail;
1319 }
1320
1321 /*
1322 * Initialize queues. Initially we set them all for no queueing
1323 * and we enable escalation for queue 0 only which we'll use for
1324 * our mfrr change notifications. If the VCPU is hot-plugged, we
1325 * do handle provisioning however based on the existing "map"
1326 * of enabled queues.
1327 */
1328 for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
1329 struct xive_q *q = &xc->queues[i];
1330
1331 /* Single escalation, no queue 7 */
1332 if (i == 7 && xive->single_escalation)
1333 break;
1334
1335 /* Is queue already enabled ? Provision it */
1336 if (xive->qmap & (1 << i)) {
1337 r = xive_provision_queue(vcpu, i);
1338 if (r == 0 && !xive->single_escalation)
1339 kvmppc_xive_attach_escalation(
1340 vcpu, i, xive->single_escalation);
1341 if (r)
1342 goto bail;
1343 } else {
1344 r = xive_native_configure_queue(xc->vp_id,
1345 q, i, NULL, 0, true);
1346 if (r) {
1347 pr_err("Failed to configure queue %d for VCPU %d\n",
1348 i, cpu);
1349 goto bail;
1350 }
1351 }
1352 }
1353
1354 /* If not done above, attach priority 0 escalation */
1355 r = kvmppc_xive_attach_escalation(vcpu, 0, xive->single_escalation);
1356 if (r)
1357 goto bail;
1358
1359 /* Route the IPI */
1360 r = xive_native_configure_irq(xc->vp_ipi, xc->vp_id, 0, XICS_IPI);
1361 if (!r)
1362 xive_vm_esb_load(&xc->vp_ipi_data, XIVE_ESB_SET_PQ_00);
1363
1364 bail:
1365 mutex_unlock(&xive->lock);
1366 if (r) {
1367 kvmppc_xive_cleanup_vcpu(vcpu);
1368 return r;
1369 }
1370
1371 vcpu->arch.irq_type = KVMPPC_IRQ_XICS;
1372 return 0;
1373 }
1374
1375 /*
1376 * Scanning of queues before/after migration save
1377 */
xive_pre_save_set_queued(struct kvmppc_xive * xive,u32 irq)1378 static void xive_pre_save_set_queued(struct kvmppc_xive *xive, u32 irq)
1379 {
1380 struct kvmppc_xive_src_block *sb;
1381 struct kvmppc_xive_irq_state *state;
1382 u16 idx;
1383
1384 sb = kvmppc_xive_find_source(xive, irq, &idx);
1385 if (!sb)
1386 return;
1387
1388 state = &sb->irq_state[idx];
1389
1390 /* Some sanity checking */
1391 if (!state->valid) {
1392 pr_err("invalid irq 0x%x in cpu queue!\n", irq);
1393 return;
1394 }
1395
1396 /*
1397 * If the interrupt is in a queue it should have P set.
1398 * We warn so that gets reported. A backtrace isn't useful
1399 * so no need to use a WARN_ON.
1400 */
1401 if (!state->saved_p)
1402 pr_err("Interrupt 0x%x is marked in a queue but P not set !\n", irq);
1403
1404 /* Set flag */
1405 state->in_queue = true;
1406 }
1407
xive_pre_save_mask_irq(struct kvmppc_xive * xive,struct kvmppc_xive_src_block * sb,u32 irq)1408 static void xive_pre_save_mask_irq(struct kvmppc_xive *xive,
1409 struct kvmppc_xive_src_block *sb,
1410 u32 irq)
1411 {
1412 struct kvmppc_xive_irq_state *state = &sb->irq_state[irq];
1413
1414 if (!state->valid)
1415 return;
1416
1417 /* Mask and save state, this will also sync HW queues */
1418 state->saved_scan_prio = xive_lock_and_mask(xive, sb, state);
1419
1420 /* Transfer P and Q */
1421 state->saved_p = state->old_p;
1422 state->saved_q = state->old_q;
1423
1424 /* Unlock */
1425 arch_spin_unlock(&sb->lock);
1426 }
1427
xive_pre_save_unmask_irq(struct kvmppc_xive * xive,struct kvmppc_xive_src_block * sb,u32 irq)1428 static void xive_pre_save_unmask_irq(struct kvmppc_xive *xive,
1429 struct kvmppc_xive_src_block *sb,
1430 u32 irq)
1431 {
1432 struct kvmppc_xive_irq_state *state = &sb->irq_state[irq];
1433
1434 if (!state->valid)
1435 return;
1436
1437 /*
1438 * Lock / exclude EOI (not technically necessary if the
1439 * guest isn't running concurrently. If this becomes a
1440 * performance issue we can probably remove the lock.
1441 */
1442 xive_lock_for_unmask(sb, state);
1443
1444 /* Restore mask/prio if it wasn't masked */
1445 if (state->saved_scan_prio != MASKED)
1446 xive_finish_unmask(xive, sb, state, state->saved_scan_prio);
1447
1448 /* Unlock */
1449 arch_spin_unlock(&sb->lock);
1450 }
1451
xive_pre_save_queue(struct kvmppc_xive * xive,struct xive_q * q)1452 static void xive_pre_save_queue(struct kvmppc_xive *xive, struct xive_q *q)
1453 {
1454 u32 idx = q->idx;
1455 u32 toggle = q->toggle;
1456 u32 irq;
1457
1458 do {
1459 irq = __xive_read_eq(q->qpage, q->msk, &idx, &toggle);
1460 if (irq > XICS_IPI)
1461 xive_pre_save_set_queued(xive, irq);
1462 } while(irq);
1463 }
1464
xive_pre_save_scan(struct kvmppc_xive * xive)1465 static void xive_pre_save_scan(struct kvmppc_xive *xive)
1466 {
1467 struct kvm_vcpu *vcpu = NULL;
1468 int i, j;
1469
1470 /*
1471 * See comment in xive_get_source() about how this
1472 * work. Collect a stable state for all interrupts
1473 */
1474 for (i = 0; i <= xive->max_sbid; i++) {
1475 struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
1476 if (!sb)
1477 continue;
1478 for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++)
1479 xive_pre_save_mask_irq(xive, sb, j);
1480 }
1481
1482 /* Then scan the queues and update the "in_queue" flag */
1483 kvm_for_each_vcpu(i, vcpu, xive->kvm) {
1484 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
1485 if (!xc)
1486 continue;
1487 for (j = 0; j < KVMPPC_XIVE_Q_COUNT; j++) {
1488 if (xc->queues[j].qpage)
1489 xive_pre_save_queue(xive, &xc->queues[j]);
1490 }
1491 }
1492
1493 /* Finally restore interrupt states */
1494 for (i = 0; i <= xive->max_sbid; i++) {
1495 struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
1496 if (!sb)
1497 continue;
1498 for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++)
1499 xive_pre_save_unmask_irq(xive, sb, j);
1500 }
1501 }
1502
xive_post_save_scan(struct kvmppc_xive * xive)1503 static void xive_post_save_scan(struct kvmppc_xive *xive)
1504 {
1505 u32 i, j;
1506
1507 /* Clear all the in_queue flags */
1508 for (i = 0; i <= xive->max_sbid; i++) {
1509 struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
1510 if (!sb)
1511 continue;
1512 for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++)
1513 sb->irq_state[j].in_queue = false;
1514 }
1515
1516 /* Next get_source() will do a new scan */
1517 xive->saved_src_count = 0;
1518 }
1519
1520 /*
1521 * This returns the source configuration and state to user space.
1522 */
xive_get_source(struct kvmppc_xive * xive,long irq,u64 addr)1523 static int xive_get_source(struct kvmppc_xive *xive, long irq, u64 addr)
1524 {
1525 struct kvmppc_xive_src_block *sb;
1526 struct kvmppc_xive_irq_state *state;
1527 u64 __user *ubufp = (u64 __user *) addr;
1528 u64 val, prio;
1529 u16 idx;
1530
1531 sb = kvmppc_xive_find_source(xive, irq, &idx);
1532 if (!sb)
1533 return -ENOENT;
1534
1535 state = &sb->irq_state[idx];
1536
1537 if (!state->valid)
1538 return -ENOENT;
1539
1540 pr_devel("get_source(%ld)...\n", irq);
1541
1542 /*
1543 * So to properly save the state into something that looks like a
1544 * XICS migration stream we cannot treat interrupts individually.
1545 *
1546 * We need, instead, mask them all (& save their previous PQ state)
1547 * to get a stable state in the HW, then sync them to ensure that
1548 * any interrupt that had already fired hits its queue, and finally
1549 * scan all the queues to collect which interrupts are still present
1550 * in the queues, so we can set the "pending" flag on them and
1551 * they can be resent on restore.
1552 *
1553 * So we do it all when the "first" interrupt gets saved, all the
1554 * state is collected at that point, the rest of xive_get_source()
1555 * will merely collect and convert that state to the expected
1556 * userspace bit mask.
1557 */
1558 if (xive->saved_src_count == 0)
1559 xive_pre_save_scan(xive);
1560 xive->saved_src_count++;
1561
1562 /* Convert saved state into something compatible with xics */
1563 val = state->act_server;
1564 prio = state->saved_scan_prio;
1565
1566 if (prio == MASKED) {
1567 val |= KVM_XICS_MASKED;
1568 prio = state->saved_priority;
1569 }
1570 val |= prio << KVM_XICS_PRIORITY_SHIFT;
1571 if (state->lsi) {
1572 val |= KVM_XICS_LEVEL_SENSITIVE;
1573 if (state->saved_p)
1574 val |= KVM_XICS_PENDING;
1575 } else {
1576 if (state->saved_p)
1577 val |= KVM_XICS_PRESENTED;
1578
1579 if (state->saved_q)
1580 val |= KVM_XICS_QUEUED;
1581
1582 /*
1583 * We mark it pending (which will attempt a re-delivery)
1584 * if we are in a queue *or* we were masked and had
1585 * Q set which is equivalent to the XICS "masked pending"
1586 * state
1587 */
1588 if (state->in_queue || (prio == MASKED && state->saved_q))
1589 val |= KVM_XICS_PENDING;
1590 }
1591
1592 /*
1593 * If that was the last interrupt saved, reset the
1594 * in_queue flags
1595 */
1596 if (xive->saved_src_count == xive->src_count)
1597 xive_post_save_scan(xive);
1598
1599 /* Copy the result to userspace */
1600 if (put_user(val, ubufp))
1601 return -EFAULT;
1602
1603 return 0;
1604 }
1605
kvmppc_xive_create_src_block(struct kvmppc_xive * xive,int irq)1606 struct kvmppc_xive_src_block *kvmppc_xive_create_src_block(
1607 struct kvmppc_xive *xive, int irq)
1608 {
1609 struct kvmppc_xive_src_block *sb;
1610 int i, bid;
1611
1612 bid = irq >> KVMPPC_XICS_ICS_SHIFT;
1613
1614 mutex_lock(&xive->lock);
1615
1616 /* block already exists - somebody else got here first */
1617 if (xive->src_blocks[bid])
1618 goto out;
1619
1620 /* Create the ICS */
1621 sb = kzalloc(sizeof(*sb), GFP_KERNEL);
1622 if (!sb)
1623 goto out;
1624
1625 sb->id = bid;
1626
1627 for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) {
1628 sb->irq_state[i].number = (bid << KVMPPC_XICS_ICS_SHIFT) | i;
1629 sb->irq_state[i].eisn = 0;
1630 sb->irq_state[i].guest_priority = MASKED;
1631 sb->irq_state[i].saved_priority = MASKED;
1632 sb->irq_state[i].act_priority = MASKED;
1633 }
1634 smp_wmb();
1635 xive->src_blocks[bid] = sb;
1636
1637 if (bid > xive->max_sbid)
1638 xive->max_sbid = bid;
1639
1640 out:
1641 mutex_unlock(&xive->lock);
1642 return xive->src_blocks[bid];
1643 }
1644
xive_check_delayed_irq(struct kvmppc_xive * xive,u32 irq)1645 static bool xive_check_delayed_irq(struct kvmppc_xive *xive, u32 irq)
1646 {
1647 struct kvm *kvm = xive->kvm;
1648 struct kvm_vcpu *vcpu = NULL;
1649 int i;
1650
1651 kvm_for_each_vcpu(i, vcpu, kvm) {
1652 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
1653
1654 if (!xc)
1655 continue;
1656
1657 if (xc->delayed_irq == irq) {
1658 xc->delayed_irq = 0;
1659 xive->delayed_irqs--;
1660 return true;
1661 }
1662 }
1663 return false;
1664 }
1665
xive_set_source(struct kvmppc_xive * xive,long irq,u64 addr)1666 static int xive_set_source(struct kvmppc_xive *xive, long irq, u64 addr)
1667 {
1668 struct kvmppc_xive_src_block *sb;
1669 struct kvmppc_xive_irq_state *state;
1670 u64 __user *ubufp = (u64 __user *) addr;
1671 u16 idx;
1672 u64 val;
1673 u8 act_prio, guest_prio;
1674 u32 server;
1675 int rc = 0;
1676
1677 if (irq < KVMPPC_XICS_FIRST_IRQ || irq >= KVMPPC_XICS_NR_IRQS)
1678 return -ENOENT;
1679
1680 pr_devel("set_source(irq=0x%lx)\n", irq);
1681
1682 /* Find the source */
1683 sb = kvmppc_xive_find_source(xive, irq, &idx);
1684 if (!sb) {
1685 pr_devel("No source, creating source block...\n");
1686 sb = kvmppc_xive_create_src_block(xive, irq);
1687 if (!sb) {
1688 pr_devel("Failed to create block...\n");
1689 return -ENOMEM;
1690 }
1691 }
1692 state = &sb->irq_state[idx];
1693
1694 /* Read user passed data */
1695 if (get_user(val, ubufp)) {
1696 pr_devel("fault getting user info !\n");
1697 return -EFAULT;
1698 }
1699
1700 server = val & KVM_XICS_DESTINATION_MASK;
1701 guest_prio = val >> KVM_XICS_PRIORITY_SHIFT;
1702
1703 pr_devel(" val=0x016%llx (server=0x%x, guest_prio=%d)\n",
1704 val, server, guest_prio);
1705
1706 /*
1707 * If the source doesn't already have an IPI, allocate
1708 * one and get the corresponding data
1709 */
1710 if (!state->ipi_number) {
1711 state->ipi_number = xive_native_alloc_irq();
1712 if (state->ipi_number == 0) {
1713 pr_devel("Failed to allocate IPI !\n");
1714 return -ENOMEM;
1715 }
1716 xive_native_populate_irq_data(state->ipi_number, &state->ipi_data);
1717 pr_devel(" src_ipi=0x%x\n", state->ipi_number);
1718 }
1719
1720 /*
1721 * We use lock_and_mask() to set us in the right masked
1722 * state. We will override that state from the saved state
1723 * further down, but this will handle the cases of interrupts
1724 * that need FW masking. We set the initial guest_priority to
1725 * 0 before calling it to ensure it actually performs the masking.
1726 */
1727 state->guest_priority = 0;
1728 xive_lock_and_mask(xive, sb, state);
1729
1730 /*
1731 * Now, we select a target if we have one. If we don't we
1732 * leave the interrupt untargetted. It means that an interrupt
1733 * can become "untargetted" accross migration if it was masked
1734 * by set_xive() but there is little we can do about it.
1735 */
1736
1737 /* First convert prio and mark interrupt as untargetted */
1738 act_prio = xive_prio_from_guest(guest_prio);
1739 state->act_priority = MASKED;
1740
1741 /*
1742 * We need to drop the lock due to the mutex below. Hopefully
1743 * nothing is touching that interrupt yet since it hasn't been
1744 * advertized to a running guest yet
1745 */
1746 arch_spin_unlock(&sb->lock);
1747
1748 /* If we have a priority target the interrupt */
1749 if (act_prio != MASKED) {
1750 /* First, check provisioning of queues */
1751 mutex_lock(&xive->lock);
1752 rc = xive_check_provisioning(xive->kvm, act_prio);
1753 mutex_unlock(&xive->lock);
1754
1755 /* Target interrupt */
1756 if (rc == 0)
1757 rc = xive_target_interrupt(xive->kvm, state,
1758 server, act_prio);
1759 /*
1760 * If provisioning or targetting failed, leave it
1761 * alone and masked. It will remain disabled until
1762 * the guest re-targets it.
1763 */
1764 }
1765
1766 /*
1767 * Find out if this was a delayed irq stashed in an ICP,
1768 * in which case, treat it as pending
1769 */
1770 if (xive->delayed_irqs && xive_check_delayed_irq(xive, irq)) {
1771 val |= KVM_XICS_PENDING;
1772 pr_devel(" Found delayed ! forcing PENDING !\n");
1773 }
1774
1775 /* Cleanup the SW state */
1776 state->old_p = false;
1777 state->old_q = false;
1778 state->lsi = false;
1779 state->asserted = false;
1780
1781 /* Restore LSI state */
1782 if (val & KVM_XICS_LEVEL_SENSITIVE) {
1783 state->lsi = true;
1784 if (val & KVM_XICS_PENDING)
1785 state->asserted = true;
1786 pr_devel(" LSI ! Asserted=%d\n", state->asserted);
1787 }
1788
1789 /*
1790 * Restore P and Q. If the interrupt was pending, we
1791 * force Q and !P, which will trigger a resend.
1792 *
1793 * That means that a guest that had both an interrupt
1794 * pending (queued) and Q set will restore with only
1795 * one instance of that interrupt instead of 2, but that
1796 * is perfectly fine as coalescing interrupts that haven't
1797 * been presented yet is always allowed.
1798 */
1799 if (val & KVM_XICS_PRESENTED && !(val & KVM_XICS_PENDING))
1800 state->old_p = true;
1801 if (val & KVM_XICS_QUEUED || val & KVM_XICS_PENDING)
1802 state->old_q = true;
1803
1804 pr_devel(" P=%d, Q=%d\n", state->old_p, state->old_q);
1805
1806 /*
1807 * If the interrupt was unmasked, update guest priority and
1808 * perform the appropriate state transition and do a
1809 * re-trigger if necessary.
1810 */
1811 if (val & KVM_XICS_MASKED) {
1812 pr_devel(" masked, saving prio\n");
1813 state->guest_priority = MASKED;
1814 state->saved_priority = guest_prio;
1815 } else {
1816 pr_devel(" unmasked, restoring to prio %d\n", guest_prio);
1817 xive_finish_unmask(xive, sb, state, guest_prio);
1818 state->saved_priority = guest_prio;
1819 }
1820
1821 /* Increment the number of valid sources and mark this one valid */
1822 if (!state->valid)
1823 xive->src_count++;
1824 state->valid = true;
1825
1826 return 0;
1827 }
1828
kvmppc_xive_set_irq(struct kvm * kvm,int irq_source_id,u32 irq,int level,bool line_status)1829 int kvmppc_xive_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level,
1830 bool line_status)
1831 {
1832 struct kvmppc_xive *xive = kvm->arch.xive;
1833 struct kvmppc_xive_src_block *sb;
1834 struct kvmppc_xive_irq_state *state;
1835 u16 idx;
1836
1837 if (!xive)
1838 return -ENODEV;
1839
1840 sb = kvmppc_xive_find_source(xive, irq, &idx);
1841 if (!sb)
1842 return -EINVAL;
1843
1844 /* Perform locklessly .... (we need to do some RCUisms here...) */
1845 state = &sb->irq_state[idx];
1846 if (!state->valid)
1847 return -EINVAL;
1848
1849 /* We don't allow a trigger on a passed-through interrupt */
1850 if (state->pt_number)
1851 return -EINVAL;
1852
1853 if ((level == 1 && state->lsi) || level == KVM_INTERRUPT_SET_LEVEL)
1854 state->asserted = 1;
1855 else if (level == 0 || level == KVM_INTERRUPT_UNSET) {
1856 state->asserted = 0;
1857 return 0;
1858 }
1859
1860 /* Trigger the IPI */
1861 xive_irq_trigger(&state->ipi_data);
1862
1863 return 0;
1864 }
1865
kvmppc_xive_set_nr_servers(struct kvmppc_xive * xive,u64 addr)1866 int kvmppc_xive_set_nr_servers(struct kvmppc_xive *xive, u64 addr)
1867 {
1868 u32 __user *ubufp = (u32 __user *) addr;
1869 u32 nr_servers;
1870 int rc = 0;
1871
1872 if (get_user(nr_servers, ubufp))
1873 return -EFAULT;
1874
1875 pr_devel("%s nr_servers=%u\n", __func__, nr_servers);
1876
1877 if (!nr_servers || nr_servers > KVM_MAX_VCPU_ID)
1878 return -EINVAL;
1879
1880 mutex_lock(&xive->lock);
1881 if (xive->vp_base != XIVE_INVALID_VP)
1882 /* The VP block is allocated once and freed when the device
1883 * is released. Better not allow to change its size since its
1884 * used by connect_vcpu to validate vCPU ids are valid (eg,
1885 * setting it back to a higher value could allow connect_vcpu
1886 * to come up with a VP id that goes beyond the VP block, which
1887 * is likely to cause a crash in OPAL).
1888 */
1889 rc = -EBUSY;
1890 else if (nr_servers > KVM_MAX_VCPUS)
1891 /* We don't need more servers. Higher vCPU ids get packed
1892 * down below KVM_MAX_VCPUS by kvmppc_pack_vcpu_id().
1893 */
1894 xive->nr_servers = KVM_MAX_VCPUS;
1895 else
1896 xive->nr_servers = nr_servers;
1897
1898 mutex_unlock(&xive->lock);
1899
1900 return rc;
1901 }
1902
xive_set_attr(struct kvm_device * dev,struct kvm_device_attr * attr)1903 static int xive_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
1904 {
1905 struct kvmppc_xive *xive = dev->private;
1906
1907 /* We honor the existing XICS ioctl */
1908 switch (attr->group) {
1909 case KVM_DEV_XICS_GRP_SOURCES:
1910 return xive_set_source(xive, attr->attr, attr->addr);
1911 case KVM_DEV_XICS_GRP_CTRL:
1912 switch (attr->attr) {
1913 case KVM_DEV_XICS_NR_SERVERS:
1914 return kvmppc_xive_set_nr_servers(xive, attr->addr);
1915 }
1916 }
1917 return -ENXIO;
1918 }
1919
xive_get_attr(struct kvm_device * dev,struct kvm_device_attr * attr)1920 static int xive_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
1921 {
1922 struct kvmppc_xive *xive = dev->private;
1923
1924 /* We honor the existing XICS ioctl */
1925 switch (attr->group) {
1926 case KVM_DEV_XICS_GRP_SOURCES:
1927 return xive_get_source(xive, attr->attr, attr->addr);
1928 }
1929 return -ENXIO;
1930 }
1931
xive_has_attr(struct kvm_device * dev,struct kvm_device_attr * attr)1932 static int xive_has_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
1933 {
1934 /* We honor the same limits as XICS, at least for now */
1935 switch (attr->group) {
1936 case KVM_DEV_XICS_GRP_SOURCES:
1937 if (attr->attr >= KVMPPC_XICS_FIRST_IRQ &&
1938 attr->attr < KVMPPC_XICS_NR_IRQS)
1939 return 0;
1940 break;
1941 case KVM_DEV_XICS_GRP_CTRL:
1942 switch (attr->attr) {
1943 case KVM_DEV_XICS_NR_SERVERS:
1944 return 0;
1945 }
1946 }
1947 return -ENXIO;
1948 }
1949
kvmppc_xive_cleanup_irq(u32 hw_num,struct xive_irq_data * xd)1950 static void kvmppc_xive_cleanup_irq(u32 hw_num, struct xive_irq_data *xd)
1951 {
1952 xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_01);
1953 xive_native_configure_irq(hw_num, 0, MASKED, 0);
1954 }
1955
kvmppc_xive_free_sources(struct kvmppc_xive_src_block * sb)1956 void kvmppc_xive_free_sources(struct kvmppc_xive_src_block *sb)
1957 {
1958 int i;
1959
1960 for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) {
1961 struct kvmppc_xive_irq_state *state = &sb->irq_state[i];
1962
1963 if (!state->valid)
1964 continue;
1965
1966 kvmppc_xive_cleanup_irq(state->ipi_number, &state->ipi_data);
1967 xive_cleanup_irq_data(&state->ipi_data);
1968 xive_native_free_irq(state->ipi_number);
1969
1970 /* Pass-through, cleanup too but keep IRQ hw data */
1971 if (state->pt_number)
1972 kvmppc_xive_cleanup_irq(state->pt_number, state->pt_data);
1973
1974 state->valid = false;
1975 }
1976 }
1977
1978 /*
1979 * Called when device fd is closed. kvm->lock is held.
1980 */
kvmppc_xive_release(struct kvm_device * dev)1981 static void kvmppc_xive_release(struct kvm_device *dev)
1982 {
1983 struct kvmppc_xive *xive = dev->private;
1984 struct kvm *kvm = xive->kvm;
1985 struct kvm_vcpu *vcpu;
1986 int i;
1987
1988 pr_devel("Releasing xive device\n");
1989
1990 /*
1991 * Since this is the device release function, we know that
1992 * userspace does not have any open fd referring to the
1993 * device. Therefore there can not be any of the device
1994 * attribute set/get functions being executed concurrently,
1995 * and similarly, the connect_vcpu and set/clr_mapped
1996 * functions also cannot be being executed.
1997 */
1998
1999 debugfs_remove(xive->dentry);
2000
2001 /*
2002 * We should clean up the vCPU interrupt presenters first.
2003 */
2004 kvm_for_each_vcpu(i, vcpu, kvm) {
2005 /*
2006 * Take vcpu->mutex to ensure that no one_reg get/set ioctl
2007 * (i.e. kvmppc_xive_[gs]et_icp) can be done concurrently.
2008 * Holding the vcpu->mutex also means that the vcpu cannot
2009 * be executing the KVM_RUN ioctl, and therefore it cannot
2010 * be executing the XIVE push or pull code or accessing
2011 * the XIVE MMIO regions.
2012 */
2013 mutex_lock(&vcpu->mutex);
2014 kvmppc_xive_cleanup_vcpu(vcpu);
2015 mutex_unlock(&vcpu->mutex);
2016 }
2017
2018 /*
2019 * Now that we have cleared vcpu->arch.xive_vcpu, vcpu->arch.irq_type
2020 * and vcpu->arch.xive_esc_[vr]addr on each vcpu, we are safe
2021 * against xive code getting called during vcpu execution or
2022 * set/get one_reg operations.
2023 */
2024 kvm->arch.xive = NULL;
2025
2026 /* Mask and free interrupts */
2027 for (i = 0; i <= xive->max_sbid; i++) {
2028 if (xive->src_blocks[i])
2029 kvmppc_xive_free_sources(xive->src_blocks[i]);
2030 kfree(xive->src_blocks[i]);
2031 xive->src_blocks[i] = NULL;
2032 }
2033
2034 if (xive->vp_base != XIVE_INVALID_VP)
2035 xive_native_free_vp_block(xive->vp_base);
2036
2037 /*
2038 * A reference of the kvmppc_xive pointer is now kept under
2039 * the xive_devices struct of the machine for reuse. It is
2040 * freed when the VM is destroyed for now until we fix all the
2041 * execution paths.
2042 */
2043
2044 kfree(dev);
2045 }
2046
2047 /*
2048 * When the guest chooses the interrupt mode (XICS legacy or XIVE
2049 * native), the VM will switch of KVM device. The previous device will
2050 * be "released" before the new one is created.
2051 *
2052 * Until we are sure all execution paths are well protected, provide a
2053 * fail safe (transitional) method for device destruction, in which
2054 * the XIVE device pointer is recycled and not directly freed.
2055 */
kvmppc_xive_get_device(struct kvm * kvm,u32 type)2056 struct kvmppc_xive *kvmppc_xive_get_device(struct kvm *kvm, u32 type)
2057 {
2058 struct kvmppc_xive **kvm_xive_device = type == KVM_DEV_TYPE_XIVE ?
2059 &kvm->arch.xive_devices.native :
2060 &kvm->arch.xive_devices.xics_on_xive;
2061 struct kvmppc_xive *xive = *kvm_xive_device;
2062
2063 if (!xive) {
2064 xive = kzalloc(sizeof(*xive), GFP_KERNEL);
2065 *kvm_xive_device = xive;
2066 } else {
2067 memset(xive, 0, sizeof(*xive));
2068 }
2069
2070 return xive;
2071 }
2072
2073 /*
2074 * Create a XICS device with XIVE backend. kvm->lock is held.
2075 */
kvmppc_xive_create(struct kvm_device * dev,u32 type)2076 static int kvmppc_xive_create(struct kvm_device *dev, u32 type)
2077 {
2078 struct kvmppc_xive *xive;
2079 struct kvm *kvm = dev->kvm;
2080
2081 pr_devel("Creating xive for partition\n");
2082
2083 /* Already there ? */
2084 if (kvm->arch.xive)
2085 return -EEXIST;
2086
2087 xive = kvmppc_xive_get_device(kvm, type);
2088 if (!xive)
2089 return -ENOMEM;
2090
2091 dev->private = xive;
2092 xive->dev = dev;
2093 xive->kvm = kvm;
2094 mutex_init(&xive->lock);
2095
2096 /* We use the default queue size set by the host */
2097 xive->q_order = xive_native_default_eq_shift();
2098 if (xive->q_order < PAGE_SHIFT)
2099 xive->q_page_order = 0;
2100 else
2101 xive->q_page_order = xive->q_order - PAGE_SHIFT;
2102
2103 /* VP allocation is delayed to the first call to connect_vcpu */
2104 xive->vp_base = XIVE_INVALID_VP;
2105 /* KVM_MAX_VCPUS limits the number of VMs to roughly 64 per sockets
2106 * on a POWER9 system.
2107 */
2108 xive->nr_servers = KVM_MAX_VCPUS;
2109
2110 xive->single_escalation = xive_native_has_single_escalation();
2111
2112 kvm->arch.xive = xive;
2113 return 0;
2114 }
2115
kvmppc_xive_debug_show_queues(struct seq_file * m,struct kvm_vcpu * vcpu)2116 int kvmppc_xive_debug_show_queues(struct seq_file *m, struct kvm_vcpu *vcpu)
2117 {
2118 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
2119 unsigned int i;
2120
2121 for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
2122 struct xive_q *q = &xc->queues[i];
2123 u32 i0, i1, idx;
2124
2125 if (!q->qpage && !xc->esc_virq[i])
2126 continue;
2127
2128 seq_printf(m, " [q%d]: ", i);
2129
2130 if (q->qpage) {
2131 idx = q->idx;
2132 i0 = be32_to_cpup(q->qpage + idx);
2133 idx = (idx + 1) & q->msk;
2134 i1 = be32_to_cpup(q->qpage + idx);
2135 seq_printf(m, "T=%d %08x %08x...\n", q->toggle,
2136 i0, i1);
2137 }
2138 if (xc->esc_virq[i]) {
2139 struct irq_data *d = irq_get_irq_data(xc->esc_virq[i]);
2140 struct xive_irq_data *xd =
2141 irq_data_get_irq_handler_data(d);
2142 u64 pq = xive_vm_esb_load(xd, XIVE_ESB_GET);
2143
2144 seq_printf(m, "E:%c%c I(%d:%llx:%llx)",
2145 (pq & XIVE_ESB_VAL_P) ? 'P' : 'p',
2146 (pq & XIVE_ESB_VAL_Q) ? 'Q' : 'q',
2147 xc->esc_virq[i], pq, xd->eoi_page);
2148 seq_puts(m, "\n");
2149 }
2150 }
2151 return 0;
2152 }
2153
xive_debug_show(struct seq_file * m,void * private)2154 static int xive_debug_show(struct seq_file *m, void *private)
2155 {
2156 struct kvmppc_xive *xive = m->private;
2157 struct kvm *kvm = xive->kvm;
2158 struct kvm_vcpu *vcpu;
2159 u64 t_rm_h_xirr = 0;
2160 u64 t_rm_h_ipoll = 0;
2161 u64 t_rm_h_cppr = 0;
2162 u64 t_rm_h_eoi = 0;
2163 u64 t_rm_h_ipi = 0;
2164 u64 t_vm_h_xirr = 0;
2165 u64 t_vm_h_ipoll = 0;
2166 u64 t_vm_h_cppr = 0;
2167 u64 t_vm_h_eoi = 0;
2168 u64 t_vm_h_ipi = 0;
2169 unsigned int i;
2170
2171 if (!kvm)
2172 return 0;
2173
2174 seq_printf(m, "=========\nVCPU state\n=========\n");
2175
2176 kvm_for_each_vcpu(i, vcpu, kvm) {
2177 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
2178
2179 if (!xc)
2180 continue;
2181
2182 seq_printf(m, "cpu server %#x VP:%#x CPPR:%#x HWCPPR:%#x"
2183 " MFRR:%#x PEND:%#x h_xirr: R=%lld V=%lld\n",
2184 xc->server_num, xc->vp_id, xc->cppr, xc->hw_cppr,
2185 xc->mfrr, xc->pending,
2186 xc->stat_rm_h_xirr, xc->stat_vm_h_xirr);
2187
2188 kvmppc_xive_debug_show_queues(m, vcpu);
2189
2190 t_rm_h_xirr += xc->stat_rm_h_xirr;
2191 t_rm_h_ipoll += xc->stat_rm_h_ipoll;
2192 t_rm_h_cppr += xc->stat_rm_h_cppr;
2193 t_rm_h_eoi += xc->stat_rm_h_eoi;
2194 t_rm_h_ipi += xc->stat_rm_h_ipi;
2195 t_vm_h_xirr += xc->stat_vm_h_xirr;
2196 t_vm_h_ipoll += xc->stat_vm_h_ipoll;
2197 t_vm_h_cppr += xc->stat_vm_h_cppr;
2198 t_vm_h_eoi += xc->stat_vm_h_eoi;
2199 t_vm_h_ipi += xc->stat_vm_h_ipi;
2200 }
2201
2202 seq_printf(m, "Hcalls totals\n");
2203 seq_printf(m, " H_XIRR R=%10lld V=%10lld\n", t_rm_h_xirr, t_vm_h_xirr);
2204 seq_printf(m, " H_IPOLL R=%10lld V=%10lld\n", t_rm_h_ipoll, t_vm_h_ipoll);
2205 seq_printf(m, " H_CPPR R=%10lld V=%10lld\n", t_rm_h_cppr, t_vm_h_cppr);
2206 seq_printf(m, " H_EOI R=%10lld V=%10lld\n", t_rm_h_eoi, t_vm_h_eoi);
2207 seq_printf(m, " H_IPI R=%10lld V=%10lld\n", t_rm_h_ipi, t_vm_h_ipi);
2208
2209 return 0;
2210 }
2211
2212 DEFINE_SHOW_ATTRIBUTE(xive_debug);
2213
xive_debugfs_init(struct kvmppc_xive * xive)2214 static void xive_debugfs_init(struct kvmppc_xive *xive)
2215 {
2216 char *name;
2217
2218 name = kasprintf(GFP_KERNEL, "kvm-xive-%p", xive);
2219 if (!name) {
2220 pr_err("%s: no memory for name\n", __func__);
2221 return;
2222 }
2223
2224 xive->dentry = debugfs_create_file(name, S_IRUGO, powerpc_debugfs_root,
2225 xive, &xive_debug_fops);
2226
2227 pr_debug("%s: created %s\n", __func__, name);
2228 kfree(name);
2229 }
2230
kvmppc_xive_init(struct kvm_device * dev)2231 static void kvmppc_xive_init(struct kvm_device *dev)
2232 {
2233 struct kvmppc_xive *xive = (struct kvmppc_xive *)dev->private;
2234
2235 /* Register some debug interfaces */
2236 xive_debugfs_init(xive);
2237 }
2238
2239 struct kvm_device_ops kvm_xive_ops = {
2240 .name = "kvm-xive",
2241 .create = kvmppc_xive_create,
2242 .init = kvmppc_xive_init,
2243 .release = kvmppc_xive_release,
2244 .set_attr = xive_set_attr,
2245 .get_attr = xive_get_attr,
2246 .has_attr = xive_has_attr,
2247 };
2248
kvmppc_xive_init_module(void)2249 void kvmppc_xive_init_module(void)
2250 {
2251 __xive_vm_h_xirr = xive_vm_h_xirr;
2252 __xive_vm_h_ipoll = xive_vm_h_ipoll;
2253 __xive_vm_h_ipi = xive_vm_h_ipi;
2254 __xive_vm_h_cppr = xive_vm_h_cppr;
2255 __xive_vm_h_eoi = xive_vm_h_eoi;
2256 }
2257
kvmppc_xive_exit_module(void)2258 void kvmppc_xive_exit_module(void)
2259 {
2260 __xive_vm_h_xirr = NULL;
2261 __xive_vm_h_ipoll = NULL;
2262 __xive_vm_h_ipi = NULL;
2263 __xive_vm_h_cppr = NULL;
2264 __xive_vm_h_eoi = NULL;
2265 }
2266
