1 // SPDX-License-Identifier: GPL-2.0-only
3  * vDPA bridge driver for modern virtio-pci device
13 #include <linux/pci.h>
15 #include <linux/virtio.h>
53 	return &vp_vdpa->mdev;  in vdpa_to_mdev()
81 	struct virtio_pci_modern_device *mdev = &vp_vdpa->mdev;  in vp_vdpa_free_irq()
82 	struct pci_dev *pdev = mdev->pci_dev;  in vp_vdpa_free_irq()
85 	for (i = 0; i < vp_vdpa->queues; i++) {  in vp_vdpa_free_irq()
86 		if (vp_vdpa->vring[i].irq != VIRTIO_MSI_NO_VECTOR) {  in vp_vdpa_free_irq()
88 			devm_free_irq(&pdev->dev, vp_vdpa->vring[i].irq,  in vp_vdpa_free_irq()
89 				      &vp_vdpa->vring[i]);  in vp_vdpa_free_irq()
90 			vp_vdpa->vring[i].irq = VIRTIO_MSI_NO_VECTOR;  in vp_vdpa_free_irq()
94 	if (vp_vdpa->config_irq != VIRTIO_MSI_NO_VECTOR) {  in vp_vdpa_free_irq()
96 		devm_free_irq(&pdev->dev, vp_vdpa->config_irq, vp_vdpa);  in vp_vdpa_free_irq()
97 		vp_vdpa->config_irq = VIRTIO_MSI_NO_VECTOR;  in vp_vdpa_free_irq()
100 	if (vp_vdpa->vectors) {  in vp_vdpa_free_irq()
102 		vp_vdpa->vectors = 0;  in vp_vdpa_free_irq()
110 	if (vring->cb.callback)  in vp_vdpa_vq_handler()
111 		return vring->cb.callback(vring->cb.private);  in vp_vdpa_vq_handler()
120 	if (vp_vdpa->config_cb.callback)  in vp_vdpa_config_handler()
121 		return vp_vdpa->config_cb.callback(vp_vdpa->config_cb.private);  in vp_vdpa_config_handler()
128 	struct virtio_pci_modern_device *mdev = &vp_vdpa->mdev;  in vp_vdpa_request_irq()
129 	struct pci_dev *pdev = mdev->pci_dev;  in vp_vdpa_request_irq()
131 	int queues = vp_vdpa->queues;  in vp_vdpa_request_irq()
136 		dev_err(&pdev->dev,  in vp_vdpa_request_irq()
142 	vp_vdpa->vectors = vectors;  in vp_vdpa_request_irq()
145 		snprintf(vp_vdpa->vring[i].msix_name, VP_VDPA_NAME_SIZE,  in vp_vdpa_request_irq()
146 			"vp-vdpa[%s]-%d\n", pci_name(pdev), i);  in vp_vdpa_request_irq()
148 		ret = devm_request_irq(&pdev->dev, irq,  in vp_vdpa_request_irq()
150 				       0, vp_vdpa->vring[i].msix_name,  in vp_vdpa_request_irq()
151 				       &vp_vdpa->vring[i]);  in vp_vdpa_request_irq()
153 			dev_err(&pdev->dev,  in vp_vdpa_request_irq()
158 		vp_vdpa->vring[i].irq = irq;  in vp_vdpa_request_irq()
161 	snprintf(vp_vdpa->msix_name, VP_VDPA_NAME_SIZE, "vp-vdpa[%s]-config\n",  in vp_vdpa_request_irq()
164 	ret = devm_request_irq(&pdev->dev, irq,	vp_vdpa_config_handler, 0,  in vp_vdpa_request_irq()
165 			       vp_vdpa->msix_name, vp_vdpa);  in vp_vdpa_request_irq()
167 		dev_err(&pdev->dev,  in vp_vdpa_request_irq()
172 	vp_vdpa->config_irq = irq;  in vp_vdpa_request_irq()
183 	struct virtio_pci_modern_device *mdev = &vp_vdpa->mdev;  in vp_vdpa_set_status()
197 	struct virtio_pci_modern_device *mdev = &vp_vdpa->mdev;  in vp_vdpa_reset()
216 	/* Note that this is not supported by virtio specification, so  in vp_vdpa_get_vq_state()
217 	 * we return -EOPNOTSUPP here. This means we can't support live  in vp_vdpa_get_vq_state()
220 	return -EOPNOTSUPP;  in vp_vdpa_get_vq_state()
226 	const struct vdpa_vq_state_split *split = &state->split;  in vp_vdpa_set_vq_state_split()
228 	if (split->avail_index == 0)  in vp_vdpa_set_vq_state_split()
231 	return -EOPNOTSUPP;  in vp_vdpa_set_vq_state_split()
237 	const struct vdpa_vq_state_packed *packed = &state->packed;  in vp_vdpa_set_vq_state_packed()
239 	if (packed->last_avail_counter == 1 &&  in vp_vdpa_set_vq_state_packed()
240 	    packed->last_avail_idx == 0 &&  in vp_vdpa_set_vq_state_packed()
241 	    packed->last_used_counter == 1 &&  in vp_vdpa_set_vq_state_packed()
242 	    packed->last_used_idx == 0)  in vp_vdpa_set_vq_state_packed()
245 	return -EOPNOTSUPP;  in vp_vdpa_set_vq_state_packed()
253 	/* Note that this is not supported by virtio specification.  in vp_vdpa_set_vq_state()
266 	return -EOPNOTSUPP;  in vp_vdpa_set_vq_state()
274 	vp_vdpa->vring[qid].cb = *cb;  in vp_vdpa_set_vq_cb()
316 	vp_iowrite16(qid, vp_vdpa->vring[qid].notify);  in vp_vdpa_kick_vq()
330 	return mdev->id.device;  in vp_vdpa_get_device_id()
337 	return mdev->id.vendor;  in vp_vdpa_get_vendor_id()
349 	return mdev->device_len;  in vp_vdpa_get_config_size()
357 	struct virtio_pci_modern_device *mdev = &vp_vdpa->mdev;  in vp_vdpa_get_config()
363 		old = vp_ioread8(&mdev->common->config_generation);  in vp_vdpa_get_config()
366 			*p++ = vp_ioread8(mdev->device + offset + i);  in vp_vdpa_get_config()
368 		new = vp_ioread8(&mdev->common->config_generation);  in vp_vdpa_get_config()
377 	struct virtio_pci_modern_device *mdev = &vp_vdpa->mdev;  in vp_vdpa_set_config()
382 		vp_iowrite8(*p++, mdev->device + offset + i);  in vp_vdpa_set_config()
390 	vp_vdpa->config_cb = *cb;  in vp_vdpa_set_config_cb()
397 	struct virtio_pci_modern_device *mdev = &vp_vdpa->mdev;  in vp_vdpa_get_vq_notification()
400 	notify.addr = vp_vdpa->vring[qid].notify_pa;  in vp_vdpa_get_vq_notification()
401 	notify.size = mdev->notify_offset_multiplier;  in vp_vdpa_get_vq_notification()
440 	struct device *dev = &pdev->dev;  in vp_vdpa_probe()
455 	mdev = &vp_vdpa->mdev;  in vp_vdpa_probe()
456 	mdev->pci_dev = pdev;  in vp_vdpa_probe()
460 		dev_err(&pdev->dev, "Failed to probe modern PCI device\n");  in vp_vdpa_probe()
467 	vp_vdpa->vdpa.dma_dev = &pdev->dev;  in vp_vdpa_probe()
468 	vp_vdpa->queues = vp_modern_get_num_queues(mdev);  in vp_vdpa_probe()
472 		dev_err(&pdev->dev,  in vp_vdpa_probe()
477 	vp_vdpa->vring = devm_kcalloc(&pdev->dev, vp_vdpa->queues,  in vp_vdpa_probe()
478 				      sizeof(*vp_vdpa->vring),  in vp_vdpa_probe()
480 	if (!vp_vdpa->vring) {  in vp_vdpa_probe()
481 		ret = -ENOMEM;  in vp_vdpa_probe()
482 		dev_err(&pdev->dev, "Fail to allocate virtqueues\n");  in vp_vdpa_probe()
486 	for (i = 0; i < vp_vdpa->queues; i++) {  in vp_vdpa_probe()
487 		vp_vdpa->vring[i].irq = VIRTIO_MSI_NO_VECTOR;  in vp_vdpa_probe()
488 		vp_vdpa->vring[i].notify =  in vp_vdpa_probe()
490 						&vp_vdpa->vring[i].notify_pa);  in vp_vdpa_probe()
491 		if (!vp_vdpa->vring[i].notify) {  in vp_vdpa_probe()
492 			ret = -EINVAL;  in vp_vdpa_probe()
493 			dev_warn(&pdev->dev, "Fail to map vq notify %d\n", i);  in vp_vdpa_probe()
497 	vp_vdpa->config_irq = VIRTIO_MSI_NO_VECTOR;  in vp_vdpa_probe()
499 	ret = vdpa_register_device(&vp_vdpa->vdpa, vp_vdpa->queues);  in vp_vdpa_probe()
501 		dev_err(&pdev->dev, "Failed to register to vdpa bus\n");  in vp_vdpa_probe()
508 	put_device(&vp_vdpa->vdpa.dev);  in vp_vdpa_probe()
516 	vdpa_unregister_device(&vp_vdpa->vdpa);  in vp_vdpa_remove()
517 	vp_modern_remove(&vp_vdpa->mdev);  in vp_vdpa_remove()
521 	.name		= "vp-vdpa",
530 MODULE_DESCRIPTION("vp-vdpa");