114 static inline unsigned int index_from_irqs(struct vmd_dev *vmd,  in index_from_irqs()  argument
117 	return irqs - vmd->irqs;  in index_from_irqs()
132 	struct vmd_dev *vmd = irq_data_get_irq_handler_data(data);  in vmd_compose_msi_msg()  local
136 			  MSI_ADDR_DEST_ID(index_from_irqs(vmd, irq));  in vmd_compose_msi_msg()
200 static struct vmd_irq_list *vmd_next_irq(struct vmd_dev *vmd, struct msi_desc *desc)  in vmd_next_irq()  argument
205 	if (vmd->msix_count == 1)  in vmd_next_irq()
206 		return &vmd->irqs[0];  in vmd_next_irq()
216 		return &vmd->irqs[0];  in vmd_next_irq()
220 	for (i = 1; i < vmd->msix_count; i++)  in vmd_next_irq()
221 		if (vmd->irqs[i].count < vmd->irqs[best].count)  in vmd_next_irq()
223 	vmd->irqs[best].count++;  in vmd_next_irq()
226 	return &vmd->irqs[best];  in vmd_next_irq()
234 	struct vmd_dev *vmd = vmd_from_bus(msi_desc_to_pci_dev(desc)->bus);  in vmd_msi_init()  local
242 	vmdirq->irq = vmd_next_irq(vmd, desc);  in vmd_msi_init()
244 	index = index_from_irqs(vmd, vmdirq->irq);  in vmd_msi_init()
245 	vector = pci_irq_vector(vmd->dev, index);  in vmd_msi_init()
248 			    handle_untracked_irq, vmd, NULL);  in vmd_msi_init()
272 	struct vmd_dev *vmd = vmd_from_bus(pdev->bus);  in vmd_msi_prepare()  local
274 	if (nvec > vmd->msix_count)  in vmd_msi_prepare()
275 		return vmd->msix_count;  in vmd_msi_prepare()
301 static int vmd_create_irq_domain(struct vmd_dev *vmd)  in vmd_create_irq_domain()  argument
305 	fn = irq_domain_alloc_named_id_fwnode("VMD-MSI", vmd->sysdata.domain);  in vmd_create_irq_domain()
309 	vmd->irq_domain = pci_msi_create_irq_domain(fn, &vmd_msi_domain_info, NULL);  in vmd_create_irq_domain()
310 	if (!vmd->irq_domain) {  in vmd_create_irq_domain()
318 static void vmd_remove_irq_domain(struct vmd_dev *vmd)  in vmd_remove_irq_domain()  argument
320 	if (vmd->irq_domain) {  in vmd_remove_irq_domain()
321 		struct fwnode_handle *fn = vmd->irq_domain->fwnode;  in vmd_remove_irq_domain()
323 		irq_domain_remove(vmd->irq_domain);  in vmd_remove_irq_domain()
328 static char __iomem *vmd_cfg_addr(struct vmd_dev *vmd, struct pci_bus *bus,  in vmd_cfg_addr()  argument
331 	char __iomem *addr = vmd->cfgbar +  in vmd_cfg_addr()
332 			     ((bus->number - vmd->busn_start) << 20) +  in vmd_cfg_addr()
335 	if ((addr - vmd->cfgbar) + len >=  in vmd_cfg_addr()
336 	    resource_size(&vmd->dev->resource[VMD_CFGBAR]))  in vmd_cfg_addr()
349 	struct vmd_dev *vmd = vmd_from_bus(bus);  in vmd_pci_read()  local
350 	char __iomem *addr = vmd_cfg_addr(vmd, bus, devfn, reg, len);  in vmd_pci_read()
357 	spin_lock_irqsave(&vmd->cfg_lock, flags);  in vmd_pci_read()
372 	spin_unlock_irqrestore(&vmd->cfg_lock, flags);  in vmd_pci_read()
384 	struct vmd_dev *vmd = vmd_from_bus(bus);  in vmd_pci_write()  local
385 	char __iomem *addr = vmd_cfg_addr(vmd, bus, devfn, reg, len);  in vmd_pci_write()
392 	spin_lock_irqsave(&vmd->cfg_lock, flags);  in vmd_pci_write()
410 	spin_unlock_irqrestore(&vmd->cfg_lock, flags);  in vmd_pci_write()
419 static void vmd_attach_resources(struct vmd_dev *vmd)  in vmd_attach_resources()  argument
421 	vmd->dev->resource[VMD_MEMBAR1].child = &vmd->resources[1];  in vmd_attach_resources()
422 	vmd->dev->resource[VMD_MEMBAR2].child = &vmd->resources[2];  in vmd_attach_resources()
425 static void vmd_detach_resources(struct vmd_dev *vmd)  in vmd_detach_resources()  argument
427 	vmd->dev->resource[VMD_MEMBAR1].child = NULL;  in vmd_detach_resources()
428 	vmd->dev->resource[VMD_MEMBAR2].child = NULL;  in vmd_detach_resources()
447 static int vmd_get_phys_offsets(struct vmd_dev *vmd, bool native_hint,  in vmd_get_phys_offsets()  argument
451 	struct pci_dev *dev = vmd->dev;  in vmd_get_phys_offsets()
501 static int vmd_get_bus_number_start(struct vmd_dev *vmd)  in vmd_get_bus_number_start()  argument
503 	struct pci_dev *dev = vmd->dev;  in vmd_get_bus_number_start()
512 			vmd->busn_start = 0;  in vmd_get_bus_number_start()
515 			vmd->busn_start = 128;  in vmd_get_bus_number_start()
518 			vmd->busn_start = 224;  in vmd_get_bus_number_start()
544 static int vmd_alloc_irqs(struct vmd_dev *vmd)  in vmd_alloc_irqs()  argument
546 	struct pci_dev *dev = vmd->dev;  in vmd_alloc_irqs()
549 	vmd->msix_count = pci_msix_vec_count(dev);  in vmd_alloc_irqs()
550 	if (vmd->msix_count < 0)  in vmd_alloc_irqs()
553 	vmd->msix_count = pci_alloc_irq_vectors(dev, 1, vmd->msix_count,  in vmd_alloc_irqs()
555 	if (vmd->msix_count < 0)  in vmd_alloc_irqs()
556 		return vmd->msix_count;  in vmd_alloc_irqs()
558 	vmd->irqs = devm_kcalloc(&dev->dev, vmd->msix_count, sizeof(*vmd->irqs),  in vmd_alloc_irqs()
560 	if (!vmd->irqs)  in vmd_alloc_irqs()
563 	for (i = 0; i < vmd->msix_count; i++) {  in vmd_alloc_irqs()
564 		err = init_srcu_struct(&vmd->irqs[i].srcu);  in vmd_alloc_irqs()
568 		INIT_LIST_HEAD(&vmd->irqs[i].irq_list);  in vmd_alloc_irqs()
571 				       "vmd", &vmd->irqs[i]);  in vmd_alloc_irqs()
579 static int vmd_enable_domain(struct vmd_dev *vmd, unsigned long features)  in vmd_enable_domain()  argument
581 	struct pci_sysdata *sd = &vmd->sysdata;  in vmd_enable_domain()
599 		ret = vmd_get_phys_offsets(vmd, true, &offset[0], &offset[1]);  in vmd_enable_domain()
603 		ret = vmd_get_phys_offsets(vmd, false, &offset[0], &offset[1]);  in vmd_enable_domain()
613 		ret = vmd_get_bus_number_start(vmd);  in vmd_enable_domain()
618 	res = &vmd->dev->resource[VMD_CFGBAR];  in vmd_enable_domain()
619 	vmd->resources[0] = (struct resource) {  in vmd_enable_domain()
621 		.start = vmd->busn_start,  in vmd_enable_domain()
622 		.end   = vmd->busn_start + (resource_size(res) >> 20) - 1,  in vmd_enable_domain()
643 	res = &vmd->dev->resource[VMD_MEMBAR1];  in vmd_enable_domain()
648 	vmd->resources[1] = (struct resource) {  in vmd_enable_domain()
656 	res = &vmd->dev->resource[VMD_MEMBAR2];  in vmd_enable_domain()
661 	vmd->resources[2] = (struct resource) {  in vmd_enable_domain()
669 	sd->vmd_dev = vmd->dev;  in vmd_enable_domain()
674 	sd->node = pcibus_to_node(vmd->dev->bus);  in vmd_enable_domain()
676 	ret = vmd_create_irq_domain(vmd);  in vmd_enable_domain()
684 	irq_domain_update_bus_token(vmd->irq_domain, DOMAIN_BUS_VMD_MSI);  in vmd_enable_domain()
686 	pci_add_resource(&resources, &vmd->resources[0]);  in vmd_enable_domain()
687 	pci_add_resource_offset(&resources, &vmd->resources[1], offset[0]);  in vmd_enable_domain()
688 	pci_add_resource_offset(&resources, &vmd->resources[2], offset[1]);  in vmd_enable_domain()
690 	vmd->bus = pci_create_root_bus(&vmd->dev->dev, vmd->busn_start,  in vmd_enable_domain()
692 	if (!vmd->bus) {  in vmd_enable_domain()
694 		vmd_remove_irq_domain(vmd);  in vmd_enable_domain()
698 	vmd_attach_resources(vmd);  in vmd_enable_domain()
699 	if (vmd->irq_domain)  in vmd_enable_domain()
700 		dev_set_msi_domain(&vmd->bus->dev, vmd->irq_domain);  in vmd_enable_domain()
702 	pci_scan_child_bus(vmd->bus);  in vmd_enable_domain()
703 	pci_assign_unassigned_bus_resources(vmd->bus);  in vmd_enable_domain()
710 	list_for_each_entry(child, &vmd->bus->children, node)  in vmd_enable_domain()
713 	pci_bus_add_devices(vmd->bus);  in vmd_enable_domain()
715 	WARN(sysfs_create_link(&vmd->dev->dev.kobj, &vmd->bus->dev.kobj,  in vmd_enable_domain()
722 	struct vmd_dev *vmd;  in vmd_probe()  local
728 	vmd = devm_kzalloc(&dev->dev, sizeof(*vmd), GFP_KERNEL);  in vmd_probe()
729 	if (!vmd)  in vmd_probe()
732 	vmd->dev = dev;  in vmd_probe()
737 	vmd->cfgbar = pcim_iomap(dev, VMD_CFGBAR, 0);  in vmd_probe()
738 	if (!vmd->cfgbar)  in vmd_probe()
746 	err = vmd_alloc_irqs(vmd);  in vmd_probe()
750 	spin_lock_init(&vmd->cfg_lock);  in vmd_probe()
751 	pci_set_drvdata(dev, vmd);  in vmd_probe()
752 	err = vmd_enable_domain(vmd, (unsigned long) id->driver_data);  in vmd_probe()
756 	dev_info(&vmd->dev->dev, "Bound to PCI domain %04x\n",  in vmd_probe()
757 		 vmd->sysdata.domain);  in vmd_probe()
761 static void vmd_cleanup_srcu(struct vmd_dev *vmd)  in vmd_cleanup_srcu()  argument
765 	for (i = 0; i < vmd->msix_count; i++)  in vmd_cleanup_srcu()
766 		cleanup_srcu_struct(&vmd->irqs[i].srcu);  in vmd_cleanup_srcu()
771 	struct vmd_dev *vmd = pci_get_drvdata(dev);  in vmd_remove()  local
773 	sysfs_remove_link(&vmd->dev->dev.kobj, "domain");  in vmd_remove()
774 	pci_stop_root_bus(vmd->bus);  in vmd_remove()
775 	pci_remove_root_bus(vmd->bus);  in vmd_remove()
776 	vmd_cleanup_srcu(vmd);  in vmd_remove()
777 	vmd_detach_resources(vmd);  in vmd_remove()
778 	vmd_remove_irq_domain(vmd);  in vmd_remove()
785 	struct vmd_dev *vmd = pci_get_drvdata(pdev);  in vmd_suspend()  local
788 	for (i = 0; i < vmd->msix_count; i++)  in vmd_suspend()
789 		devm_free_irq(dev, pci_irq_vector(pdev, i), &vmd->irqs[i]);  in vmd_suspend()
797 	struct vmd_dev *vmd = pci_get_drvdata(pdev);  in vmd_resume()  local
800 	for (i = 0; i < vmd->msix_count; i++) {  in vmd_resume()
803 				       "vmd", &vmd->irqs[i]);  in vmd_resume()