1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * VFIO based Physical Subchannel device driver
4 *
5 * Copyright IBM Corp. 2017
6 * Copyright Red Hat, Inc. 2019
7 *
8 * Author(s): Dong Jia Shi <bjsdjshi@linux.vnet.ibm.com>
9 * Xiao Feng Ren <renxiaof@linux.vnet.ibm.com>
10 * Cornelia Huck <cohuck@redhat.com>
11 */
12
13 #include <linux/module.h>
14 #include <linux/init.h>
15 #include <linux/slab.h>
16 #include <linux/mdev.h>
17
18 #include <asm/isc.h>
19
20 #include "chp.h"
21 #include "ioasm.h"
22 #include "css.h"
23 #include "vfio_ccw_private.h"
24
25 struct workqueue_struct *vfio_ccw_work_q;
26 static struct kmem_cache *vfio_ccw_io_region;
27 static struct kmem_cache *vfio_ccw_cmd_region;
28 static struct kmem_cache *vfio_ccw_schib_region;
29 static struct kmem_cache *vfio_ccw_crw_region;
30
31 debug_info_t *vfio_ccw_debug_msg_id;
32 debug_info_t *vfio_ccw_debug_trace_id;
33
34 /*
35 * Helpers
36 */
vfio_ccw_sch_quiesce(struct subchannel * sch)37 int vfio_ccw_sch_quiesce(struct subchannel *sch)
38 {
39 struct vfio_ccw_private *private = dev_get_drvdata(&sch->dev);
40 DECLARE_COMPLETION_ONSTACK(completion);
41 int iretry, ret = 0;
42
43 iretry = 255;
44 do {
45
46 ret = cio_cancel_halt_clear(sch, &iretry);
47
48 if (ret == -EIO) {
49 pr_err("vfio_ccw: could not quiesce subchannel 0.%x.%04x!\n",
50 sch->schid.ssid, sch->schid.sch_no);
51 break;
52 }
53
54 /*
55 * Flush all I/O and wait for
56 * cancel/halt/clear completion.
57 */
58 private->completion = &completion;
59 spin_unlock_irq(sch->lock);
60
61 if (ret == -EBUSY)
62 wait_for_completion_timeout(&completion, 3*HZ);
63
64 private->completion = NULL;
65 flush_workqueue(vfio_ccw_work_q);
66 spin_lock_irq(sch->lock);
67 ret = cio_disable_subchannel(sch);
68 } while (ret == -EBUSY);
69
70 return ret;
71 }
72
vfio_ccw_sch_io_todo(struct work_struct * work)73 static void vfio_ccw_sch_io_todo(struct work_struct *work)
74 {
75 struct vfio_ccw_private *private;
76 struct irb *irb;
77 bool is_final;
78 bool cp_is_finished = false;
79
80 private = container_of(work, struct vfio_ccw_private, io_work);
81 irb = &private->irb;
82
83 is_final = !(scsw_actl(&irb->scsw) &
84 (SCSW_ACTL_DEVACT | SCSW_ACTL_SCHACT));
85 if (scsw_is_solicited(&irb->scsw)) {
86 cp_update_scsw(&private->cp, &irb->scsw);
87 if (is_final && private->state == VFIO_CCW_STATE_CP_PENDING) {
88 cp_free(&private->cp);
89 cp_is_finished = true;
90 }
91 }
92 mutex_lock(&private->io_mutex);
93 memcpy(private->io_region->irb_area, irb, sizeof(*irb));
94 mutex_unlock(&private->io_mutex);
95
96 /*
97 * Reset to IDLE only if processing of a channel program
98 * has finished. Do not overwrite a possible processing
99 * state if the interrupt was unsolicited, or if the final
100 * interrupt was for HSCH or CSCH.
101 */
102 if (cp_is_finished)
103 private->state = VFIO_CCW_STATE_IDLE;
104
105 if (private->io_trigger)
106 eventfd_signal(private->io_trigger, 1);
107 }
108
vfio_ccw_crw_todo(struct work_struct * work)109 static void vfio_ccw_crw_todo(struct work_struct *work)
110 {
111 struct vfio_ccw_private *private;
112
113 private = container_of(work, struct vfio_ccw_private, crw_work);
114
115 if (!list_empty(&private->crw) && private->crw_trigger)
116 eventfd_signal(private->crw_trigger, 1);
117 }
118
119 /*
120 * Css driver callbacks
121 */
vfio_ccw_sch_irq(struct subchannel * sch)122 static void vfio_ccw_sch_irq(struct subchannel *sch)
123 {
124 struct vfio_ccw_private *private = dev_get_drvdata(&sch->dev);
125
126 inc_irq_stat(IRQIO_CIO);
127 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_INTERRUPT);
128 }
129
vfio_ccw_alloc_private(struct subchannel * sch)130 static struct vfio_ccw_private *vfio_ccw_alloc_private(struct subchannel *sch)
131 {
132 struct vfio_ccw_private *private;
133
134 private = kzalloc(sizeof(*private), GFP_KERNEL);
135 if (!private)
136 return ERR_PTR(-ENOMEM);
137
138 private->sch = sch;
139 mutex_init(&private->io_mutex);
140 private->state = VFIO_CCW_STATE_STANDBY;
141 INIT_LIST_HEAD(&private->crw);
142 INIT_WORK(&private->io_work, vfio_ccw_sch_io_todo);
143 INIT_WORK(&private->crw_work, vfio_ccw_crw_todo);
144
145 private->cp.guest_cp = kcalloc(CCWCHAIN_LEN_MAX, sizeof(struct ccw1),
146 GFP_KERNEL);
147 if (!private->cp.guest_cp)
148 goto out_free_private;
149
150 private->io_region = kmem_cache_zalloc(vfio_ccw_io_region,
151 GFP_KERNEL | GFP_DMA);
152 if (!private->io_region)
153 goto out_free_cp;
154
155 private->cmd_region = kmem_cache_zalloc(vfio_ccw_cmd_region,
156 GFP_KERNEL | GFP_DMA);
157 if (!private->cmd_region)
158 goto out_free_io;
159
160 private->schib_region = kmem_cache_zalloc(vfio_ccw_schib_region,
161 GFP_KERNEL | GFP_DMA);
162
163 if (!private->schib_region)
164 goto out_free_cmd;
165
166 private->crw_region = kmem_cache_zalloc(vfio_ccw_crw_region,
167 GFP_KERNEL | GFP_DMA);
168
169 if (!private->crw_region)
170 goto out_free_schib;
171 return private;
172
173 out_free_schib:
174 kmem_cache_free(vfio_ccw_schib_region, private->schib_region);
175 out_free_cmd:
176 kmem_cache_free(vfio_ccw_cmd_region, private->cmd_region);
177 out_free_io:
178 kmem_cache_free(vfio_ccw_io_region, private->io_region);
179 out_free_cp:
180 kfree(private->cp.guest_cp);
181 out_free_private:
182 mutex_destroy(&private->io_mutex);
183 kfree(private);
184 return ERR_PTR(-ENOMEM);
185 }
186
vfio_ccw_free_private(struct vfio_ccw_private * private)187 static void vfio_ccw_free_private(struct vfio_ccw_private *private)
188 {
189 struct vfio_ccw_crw *crw, *temp;
190
191 list_for_each_entry_safe(crw, temp, &private->crw, next) {
192 list_del(&crw->next);
193 kfree(crw);
194 }
195
196 kmem_cache_free(vfio_ccw_crw_region, private->crw_region);
197 kmem_cache_free(vfio_ccw_schib_region, private->schib_region);
198 kmem_cache_free(vfio_ccw_cmd_region, private->cmd_region);
199 kmem_cache_free(vfio_ccw_io_region, private->io_region);
200 kfree(private->cp.guest_cp);
201 mutex_destroy(&private->io_mutex);
202 kfree(private);
203 }
vfio_ccw_sch_probe(struct subchannel * sch)204 static int vfio_ccw_sch_probe(struct subchannel *sch)
205 {
206 struct pmcw *pmcw = &sch->schib.pmcw;
207 struct vfio_ccw_private *private;
208 int ret = -ENOMEM;
209
210 if (pmcw->qf) {
211 dev_warn(&sch->dev, "vfio: ccw: does not support QDIO: %s\n",
212 dev_name(&sch->dev));
213 return -ENODEV;
214 }
215
216 private = vfio_ccw_alloc_private(sch);
217 if (IS_ERR(private))
218 return PTR_ERR(private);
219
220 dev_set_drvdata(&sch->dev, private);
221
222 private->mdev_type.sysfs_name = "io";
223 private->mdev_type.pretty_name = "I/O subchannel (Non-QDIO)";
224 private->mdev_types[0] = &private->mdev_type;
225 ret = mdev_register_parent(&private->parent, &sch->dev,
226 &vfio_ccw_mdev_driver,
227 private->mdev_types, 1);
228 if (ret)
229 goto out_free;
230
231 VFIO_CCW_MSG_EVENT(4, "bound to subchannel %x.%x.%04x\n",
232 sch->schid.cssid, sch->schid.ssid,
233 sch->schid.sch_no);
234 return 0;
235
236 out_free:
237 dev_set_drvdata(&sch->dev, NULL);
238 vfio_ccw_free_private(private);
239 return ret;
240 }
241
vfio_ccw_sch_remove(struct subchannel * sch)242 static void vfio_ccw_sch_remove(struct subchannel *sch)
243 {
244 struct vfio_ccw_private *private = dev_get_drvdata(&sch->dev);
245
246 mdev_unregister_parent(&private->parent);
247
248 dev_set_drvdata(&sch->dev, NULL);
249
250 vfio_ccw_free_private(private);
251
252 VFIO_CCW_MSG_EVENT(4, "unbound from subchannel %x.%x.%04x\n",
253 sch->schid.cssid, sch->schid.ssid,
254 sch->schid.sch_no);
255 }
256
vfio_ccw_sch_shutdown(struct subchannel * sch)257 static void vfio_ccw_sch_shutdown(struct subchannel *sch)
258 {
259 struct vfio_ccw_private *private = dev_get_drvdata(&sch->dev);
260
261 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_CLOSE);
262 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_NOT_OPER);
263 }
264
265 /**
266 * vfio_ccw_sch_event - process subchannel event
267 * @sch: subchannel
268 * @process: non-zero if function is called in process context
269 *
270 * An unspecified event occurred for this subchannel. Adjust data according
271 * to the current operational state of the subchannel. Return zero when the
272 * event has been handled sufficiently or -EAGAIN when this function should
273 * be called again in process context.
274 */
vfio_ccw_sch_event(struct subchannel * sch,int process)275 static int vfio_ccw_sch_event(struct subchannel *sch, int process)
276 {
277 struct vfio_ccw_private *private = dev_get_drvdata(&sch->dev);
278 unsigned long flags;
279 int rc = -EAGAIN;
280
281 spin_lock_irqsave(sch->lock, flags);
282 if (!device_is_registered(&sch->dev))
283 goto out_unlock;
284
285 if (work_pending(&sch->todo_work))
286 goto out_unlock;
287
288 rc = 0;
289
290 if (cio_update_schib(sch))
291 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_NOT_OPER);
292
293 out_unlock:
294 spin_unlock_irqrestore(sch->lock, flags);
295
296 return rc;
297 }
298
vfio_ccw_queue_crw(struct vfio_ccw_private * private,unsigned int rsc,unsigned int erc,unsigned int rsid)299 static void vfio_ccw_queue_crw(struct vfio_ccw_private *private,
300 unsigned int rsc,
301 unsigned int erc,
302 unsigned int rsid)
303 {
304 struct vfio_ccw_crw *crw;
305
306 /*
307 * If unable to allocate a CRW, just drop the event and
308 * carry on. The guest will either see a later one or
309 * learn when it issues its own store subchannel.
310 */
311 crw = kzalloc(sizeof(*crw), GFP_ATOMIC);
312 if (!crw)
313 return;
314
315 /*
316 * Build the CRW based on the inputs given to us.
317 */
318 crw->crw.rsc = rsc;
319 crw->crw.erc = erc;
320 crw->crw.rsid = rsid;
321
322 list_add_tail(&crw->next, &private->crw);
323 queue_work(vfio_ccw_work_q, &private->crw_work);
324 }
325
vfio_ccw_chp_event(struct subchannel * sch,struct chp_link * link,int event)326 static int vfio_ccw_chp_event(struct subchannel *sch,
327 struct chp_link *link, int event)
328 {
329 struct vfio_ccw_private *private = dev_get_drvdata(&sch->dev);
330 int mask = chp_ssd_get_mask(&sch->ssd_info, link);
331 int retry = 255;
332
333 if (!private || !mask)
334 return 0;
335
336 trace_vfio_ccw_chp_event(private->sch->schid, mask, event);
337 VFIO_CCW_MSG_EVENT(2, "sch %x.%x.%04x: mask=0x%x event=%d\n",
338 sch->schid.cssid,
339 sch->schid.ssid, sch->schid.sch_no,
340 mask, event);
341
342 if (cio_update_schib(sch))
343 return -ENODEV;
344
345 switch (event) {
346 case CHP_VARY_OFF:
347 /* Path logically turned off */
348 sch->opm &= ~mask;
349 sch->lpm &= ~mask;
350 if (sch->schib.pmcw.lpum & mask)
351 cio_cancel_halt_clear(sch, &retry);
352 break;
353 case CHP_OFFLINE:
354 /* Path is gone */
355 if (sch->schib.pmcw.lpum & mask)
356 cio_cancel_halt_clear(sch, &retry);
357 vfio_ccw_queue_crw(private, CRW_RSC_CPATH, CRW_ERC_PERRN,
358 link->chpid.id);
359 break;
360 case CHP_VARY_ON:
361 /* Path logically turned on */
362 sch->opm |= mask;
363 sch->lpm |= mask;
364 break;
365 case CHP_ONLINE:
366 /* Path became available */
367 sch->lpm |= mask & sch->opm;
368 vfio_ccw_queue_crw(private, CRW_RSC_CPATH, CRW_ERC_INIT,
369 link->chpid.id);
370 break;
371 }
372
373 return 0;
374 }
375
376 static struct css_device_id vfio_ccw_sch_ids[] = {
377 { .match_flags = 0x1, .type = SUBCHANNEL_TYPE_IO, },
378 { /* end of list */ },
379 };
380 MODULE_DEVICE_TABLE(css, vfio_ccw_sch_ids);
381
382 static struct css_driver vfio_ccw_sch_driver = {
383 .drv = {
384 .name = "vfio_ccw",
385 .owner = THIS_MODULE,
386 },
387 .subchannel_type = vfio_ccw_sch_ids,
388 .irq = vfio_ccw_sch_irq,
389 .probe = vfio_ccw_sch_probe,
390 .remove = vfio_ccw_sch_remove,
391 .shutdown = vfio_ccw_sch_shutdown,
392 .sch_event = vfio_ccw_sch_event,
393 .chp_event = vfio_ccw_chp_event,
394 };
395
vfio_ccw_debug_init(void)396 static int __init vfio_ccw_debug_init(void)
397 {
398 vfio_ccw_debug_msg_id = debug_register("vfio_ccw_msg", 16, 1,
399 11 * sizeof(long));
400 if (!vfio_ccw_debug_msg_id)
401 goto out_unregister;
402 debug_register_view(vfio_ccw_debug_msg_id, &debug_sprintf_view);
403 debug_set_level(vfio_ccw_debug_msg_id, 2);
404 vfio_ccw_debug_trace_id = debug_register("vfio_ccw_trace", 16, 1, 16);
405 if (!vfio_ccw_debug_trace_id)
406 goto out_unregister;
407 debug_register_view(vfio_ccw_debug_trace_id, &debug_hex_ascii_view);
408 debug_set_level(vfio_ccw_debug_trace_id, 2);
409 return 0;
410
411 out_unregister:
412 debug_unregister(vfio_ccw_debug_msg_id);
413 debug_unregister(vfio_ccw_debug_trace_id);
414 return -1;
415 }
416
vfio_ccw_debug_exit(void)417 static void vfio_ccw_debug_exit(void)
418 {
419 debug_unregister(vfio_ccw_debug_msg_id);
420 debug_unregister(vfio_ccw_debug_trace_id);
421 }
422
vfio_ccw_destroy_regions(void)423 static void vfio_ccw_destroy_regions(void)
424 {
425 kmem_cache_destroy(vfio_ccw_crw_region);
426 kmem_cache_destroy(vfio_ccw_schib_region);
427 kmem_cache_destroy(vfio_ccw_cmd_region);
428 kmem_cache_destroy(vfio_ccw_io_region);
429 }
430
vfio_ccw_sch_init(void)431 static int __init vfio_ccw_sch_init(void)
432 {
433 int ret;
434
435 ret = vfio_ccw_debug_init();
436 if (ret)
437 return ret;
438
439 vfio_ccw_work_q = create_singlethread_workqueue("vfio-ccw");
440 if (!vfio_ccw_work_q) {
441 ret = -ENOMEM;
442 goto out_regions;
443 }
444
445 vfio_ccw_io_region = kmem_cache_create_usercopy("vfio_ccw_io_region",
446 sizeof(struct ccw_io_region), 0,
447 SLAB_ACCOUNT, 0,
448 sizeof(struct ccw_io_region), NULL);
449 if (!vfio_ccw_io_region) {
450 ret = -ENOMEM;
451 goto out_regions;
452 }
453
454 vfio_ccw_cmd_region = kmem_cache_create_usercopy("vfio_ccw_cmd_region",
455 sizeof(struct ccw_cmd_region), 0,
456 SLAB_ACCOUNT, 0,
457 sizeof(struct ccw_cmd_region), NULL);
458 if (!vfio_ccw_cmd_region) {
459 ret = -ENOMEM;
460 goto out_regions;
461 }
462
463 vfio_ccw_schib_region = kmem_cache_create_usercopy("vfio_ccw_schib_region",
464 sizeof(struct ccw_schib_region), 0,
465 SLAB_ACCOUNT, 0,
466 sizeof(struct ccw_schib_region), NULL);
467
468 if (!vfio_ccw_schib_region) {
469 ret = -ENOMEM;
470 goto out_regions;
471 }
472
473 vfio_ccw_crw_region = kmem_cache_create_usercopy("vfio_ccw_crw_region",
474 sizeof(struct ccw_crw_region), 0,
475 SLAB_ACCOUNT, 0,
476 sizeof(struct ccw_crw_region), NULL);
477
478 if (!vfio_ccw_crw_region) {
479 ret = -ENOMEM;
480 goto out_regions;
481 }
482
483 ret = mdev_register_driver(&vfio_ccw_mdev_driver);
484 if (ret)
485 goto out_regions;
486
487 isc_register(VFIO_CCW_ISC);
488 ret = css_driver_register(&vfio_ccw_sch_driver);
489 if (ret) {
490 isc_unregister(VFIO_CCW_ISC);
491 goto out_driver;
492 }
493
494 return ret;
495
496 out_driver:
497 mdev_unregister_driver(&vfio_ccw_mdev_driver);
498 out_regions:
499 vfio_ccw_destroy_regions();
500 destroy_workqueue(vfio_ccw_work_q);
501 vfio_ccw_debug_exit();
502 return ret;
503 }
504
vfio_ccw_sch_exit(void)505 static void __exit vfio_ccw_sch_exit(void)
506 {
507 css_driver_unregister(&vfio_ccw_sch_driver);
508 mdev_unregister_driver(&vfio_ccw_mdev_driver);
509 isc_unregister(VFIO_CCW_ISC);
510 vfio_ccw_destroy_regions();
511 destroy_workqueue(vfio_ccw_work_q);
512 vfio_ccw_debug_exit();
513 }
514 module_init(vfio_ccw_sch_init);
515 module_exit(vfio_ccw_sch_exit);
516
517 MODULE_LICENSE("GPL v2");
518
