109 static inline unsigned int index_from_irqs(struct vmd_dev *vmd,  in index_from_irqs()  argument
112 	return irqs - vmd->irqs;  in index_from_irqs()
127 	struct vmd_dev *vmd = irq_data_get_irq_handler_data(data);  in vmd_compose_msi_msg()  local
131 			  MSI_ADDR_DEST_ID(index_from_irqs(vmd, irq));  in vmd_compose_msi_msg()
195 static struct vmd_irq_list *vmd_next_irq(struct vmd_dev *vmd, struct msi_desc *desc)  in vmd_next_irq()  argument
200 	if (vmd->msix_count == 1)  in vmd_next_irq()
201 		return &vmd->irqs[0];  in vmd_next_irq()
211 		return &vmd->irqs[0];  in vmd_next_irq()
215 	for (i = 1; i < vmd->msix_count; i++)  in vmd_next_irq()
216 		if (vmd->irqs[i].count < vmd->irqs[best].count)  in vmd_next_irq()
218 	vmd->irqs[best].count++;  in vmd_next_irq()
221 	return &vmd->irqs[best];  in vmd_next_irq()
229 	struct vmd_dev *vmd = vmd_from_bus(msi_desc_to_pci_dev(desc)->bus);  in vmd_msi_init()  local
237 	vmdirq->irq = vmd_next_irq(vmd, desc);  in vmd_msi_init()
239 	index = index_from_irqs(vmd, vmdirq->irq);  in vmd_msi_init()
240 	vector = pci_irq_vector(vmd->dev, index);  in vmd_msi_init()
243 			    handle_untracked_irq, vmd, NULL);  in vmd_msi_init()
267 	struct vmd_dev *vmd = vmd_from_bus(pdev->bus);  in vmd_msi_prepare()  local
269 	if (nvec > vmd->msix_count)  in vmd_msi_prepare()
270 		return vmd->msix_count;  in vmd_msi_prepare()
305 	struct vmd_dev *vmd = vmd_from_bus(pdev->bus);  in to_vmd_dev()  local
307 	return &vmd->dev->dev;  in to_vmd_dev()
414 static void vmd_teardown_dma_ops(struct vmd_dev *vmd)  in vmd_teardown_dma_ops()  argument
416 	struct dma_domain *domain = &vmd->dma_domain;  in vmd_teardown_dma_ops()
418 	if (get_dma_ops(&vmd->dev->dev))  in vmd_teardown_dma_ops()
428 static void vmd_setup_dma_ops(struct vmd_dev *vmd)  in vmd_setup_dma_ops()  argument
430 	const struct dma_map_ops *source = get_dma_ops(&vmd->dev->dev);  in vmd_setup_dma_ops()
431 	struct dma_map_ops *dest = &vmd->dma_ops;  in vmd_setup_dma_ops()
432 	struct dma_domain *domain = &vmd->dma_domain;  in vmd_setup_dma_ops()
434 	domain->domain_nr = vmd->sysdata.domain;  in vmd_setup_dma_ops()
460 static void vmd_teardown_dma_ops(struct vmd_dev *vmd) {}  in vmd_teardown_dma_ops()  argument
461 static void vmd_setup_dma_ops(struct vmd_dev *vmd) {}  in vmd_setup_dma_ops()  argument
464 static char __iomem *vmd_cfg_addr(struct vmd_dev *vmd, struct pci_bus *bus,  in vmd_cfg_addr()  argument
467 	char __iomem *addr = vmd->cfgbar +  in vmd_cfg_addr()
470 	if ((addr - vmd->cfgbar) + len >=  in vmd_cfg_addr()
471 	    resource_size(&vmd->dev->resource[VMD_CFGBAR]))  in vmd_cfg_addr()
484 	struct vmd_dev *vmd = vmd_from_bus(bus);  in vmd_pci_read()  local
485 	char __iomem *addr = vmd_cfg_addr(vmd, bus, devfn, reg, len);  in vmd_pci_read()
492 	spin_lock_irqsave(&vmd->cfg_lock, flags);  in vmd_pci_read()
507 	spin_unlock_irqrestore(&vmd->cfg_lock, flags);  in vmd_pci_read()
519 	struct vmd_dev *vmd = vmd_from_bus(bus);  in vmd_pci_write()  local
520 	char __iomem *addr = vmd_cfg_addr(vmd, bus, devfn, reg, len);  in vmd_pci_write()
527 	spin_lock_irqsave(&vmd->cfg_lock, flags);  in vmd_pci_write()
545 	spin_unlock_irqrestore(&vmd->cfg_lock, flags);  in vmd_pci_write()
554 static void vmd_attach_resources(struct vmd_dev *vmd)  in vmd_attach_resources()  argument
556 	vmd->dev->resource[VMD_MEMBAR1].child = &vmd->resources[1];  in vmd_attach_resources()
557 	vmd->dev->resource[VMD_MEMBAR2].child = &vmd->resources[2];  in vmd_attach_resources()
560 static void vmd_detach_resources(struct vmd_dev *vmd)  in vmd_detach_resources()  argument
562 	vmd->dev->resource[VMD_MEMBAR1].child = NULL;  in vmd_detach_resources()
563 	vmd->dev->resource[VMD_MEMBAR2].child = NULL;  in vmd_detach_resources()
582 static int vmd_enable_domain(struct vmd_dev *vmd, unsigned long features)  in vmd_enable_domain()  argument
584 	struct pci_sysdata *sd = &vmd->sysdata;  in vmd_enable_domain()
604 		ret = pci_read_config_dword(vmd->dev, PCI_REG_VMLOCK, &vmlock);  in vmd_enable_domain()
611 			membar2 = pci_iomap(vmd->dev, VMD_MEMBAR2, 0);  in vmd_enable_domain()
614 			offset[0] = vmd->dev->resource[VMD_MEMBAR1].start -  in vmd_enable_domain()
616 			offset[1] = vmd->dev->resource[VMD_MEMBAR2].start -  in vmd_enable_domain()
618 			pci_iounmap(vmd->dev, membar2);  in vmd_enable_domain()
629 		pci_read_config_dword(vmd->dev, PCI_REG_VMCAP, &vmcap);  in vmd_enable_domain()
630 		pci_read_config_dword(vmd->dev, PCI_REG_VMCONFIG, &vmconfig);  in vmd_enable_domain()
636 	res = &vmd->dev->resource[VMD_CFGBAR];  in vmd_enable_domain()
637 	vmd->resources[0] = (struct resource) {  in vmd_enable_domain()
661 	res = &vmd->dev->resource[VMD_MEMBAR1];  in vmd_enable_domain()
666 	vmd->resources[1] = (struct resource) {  in vmd_enable_domain()
674 	res = &vmd->dev->resource[VMD_MEMBAR2];  in vmd_enable_domain()
679 	vmd->resources[2] = (struct resource) {  in vmd_enable_domain()
692 	sd->node = pcibus_to_node(vmd->dev->bus);  in vmd_enable_domain()
694 	fn = irq_domain_alloc_named_id_fwnode("VMD-MSI", vmd->sysdata.domain);  in vmd_enable_domain()
698 	vmd->irq_domain = pci_msi_create_irq_domain(fn, &vmd_msi_domain_info,  in vmd_enable_domain()
701 	if (!vmd->irq_domain)  in vmd_enable_domain()
704 	pci_add_resource(&resources, &vmd->resources[0]);  in vmd_enable_domain()
705 	pci_add_resource_offset(&resources, &vmd->resources[1], offset[0]);  in vmd_enable_domain()
706 	pci_add_resource_offset(&resources, &vmd->resources[2], offset[1]);  in vmd_enable_domain()
708 	vmd->bus = pci_create_root_bus(&vmd->dev->dev, busn_start, &vmd_ops,  in vmd_enable_domain()
710 	if (!vmd->bus) {  in vmd_enable_domain()
712 		irq_domain_remove(vmd->irq_domain);  in vmd_enable_domain()
716 	vmd_attach_resources(vmd);  in vmd_enable_domain()
717 	vmd_setup_dma_ops(vmd);  in vmd_enable_domain()
718 	dev_set_msi_domain(&vmd->bus->dev, vmd->irq_domain);  in vmd_enable_domain()
719 	pci_rescan_bus(vmd->bus);  in vmd_enable_domain()
721 	WARN(sysfs_create_link(&vmd->dev->dev.kobj, &vmd->bus->dev.kobj,  in vmd_enable_domain()
742 	struct vmd_dev *vmd;  in vmd_probe()  local
748 	vmd = devm_kzalloc(&dev->dev, sizeof(*vmd), GFP_KERNEL);  in vmd_probe()
749 	if (!vmd)  in vmd_probe()
752 	vmd->dev = dev;  in vmd_probe()
757 	vmd->cfgbar = pcim_iomap(dev, VMD_CFGBAR, 0);  in vmd_probe()
758 	if (!vmd->cfgbar)  in vmd_probe()
766 	vmd->msix_count = pci_msix_vec_count(dev);  in vmd_probe()
767 	if (vmd->msix_count < 0)  in vmd_probe()
770 	vmd->msix_count = pci_alloc_irq_vectors(dev, 1, vmd->msix_count,  in vmd_probe()
772 	if (vmd->msix_count < 0)  in vmd_probe()
773 		return vmd->msix_count;  in vmd_probe()
775 	vmd->irqs = devm_kcalloc(&dev->dev, vmd->msix_count, sizeof(*vmd->irqs),  in vmd_probe()
777 	if (!vmd->irqs)  in vmd_probe()
780 	for (i = 0; i < vmd->msix_count; i++) {  in vmd_probe()
781 		err = init_srcu_struct(&vmd->irqs[i].srcu);  in vmd_probe()
785 		INIT_LIST_HEAD(&vmd->irqs[i].irq_list);  in vmd_probe()
788 				       "vmd", &vmd->irqs[i]);  in vmd_probe()
793 	spin_lock_init(&vmd->cfg_lock);  in vmd_probe()
794 	pci_set_drvdata(dev, vmd);  in vmd_probe()
795 	err = vmd_enable_domain(vmd, (unsigned long) id->driver_data);  in vmd_probe()
799 	dev_info(&vmd->dev->dev, "Bound to PCI domain %04x\n",  in vmd_probe()
800 		 vmd->sysdata.domain);  in vmd_probe()
804 static void vmd_cleanup_srcu(struct vmd_dev *vmd)  in vmd_cleanup_srcu()  argument
808 	for (i = 0; i < vmd->msix_count; i++)  in vmd_cleanup_srcu()
809 		cleanup_srcu_struct(&vmd->irqs[i].srcu);  in vmd_cleanup_srcu()
814 	struct vmd_dev *vmd = pci_get_drvdata(dev);  in vmd_remove()  local
816 	vmd_detach_resources(vmd);  in vmd_remove()
817 	sysfs_remove_link(&vmd->dev->dev.kobj, "domain");  in vmd_remove()
818 	pci_stop_root_bus(vmd->bus);  in vmd_remove()
819 	pci_remove_root_bus(vmd->bus);  in vmd_remove()
820 	vmd_cleanup_srcu(vmd);  in vmd_remove()
821 	vmd_teardown_dma_ops(vmd);  in vmd_remove()
822 	irq_domain_remove(vmd->irq_domain);  in vmd_remove()
829 	struct vmd_dev *vmd = pci_get_drvdata(pdev);  in vmd_suspend()  local
832 	for (i = 0; i < vmd->msix_count; i++)  in vmd_suspend()
833                 devm_free_irq(dev, pci_irq_vector(pdev, i), &vmd->irqs[i]);  in vmd_suspend()
842 	struct vmd_dev *vmd = pci_get_drvdata(pdev);  in vmd_resume()  local
845 	for (i = 0; i < vmd->msix_count; i++) {  in vmd_resume()
848 				       "vmd", &vmd->irqs[i]);  in vmd_resume()