1 // SPDX-License-Identifier: GPL-2.0
25  * msi_alloc_desc - Allocate an initialized msi_desc
28  * @affinity:	Optional pointer to an affinity mask array size of @nvec
30  * If @affinity is not %NULL then an affinity array[@nvec] is allocated
31  * and the affinity masks and flags from @affinity are copied.
36 					const struct irq_affinity_desc *affinity)  in msi_alloc_desc()  argument
43 	desc->dev = dev;  in msi_alloc_desc()
44 	desc->nvec_used = nvec;  in msi_alloc_desc()
45 	if (affinity) {  in msi_alloc_desc()
46 		desc->affinity = kmemdup(affinity, nvec * sizeof(*desc->affinity), GFP_KERNEL);  in msi_alloc_desc()
47 		if (!desc->affinity) {  in msi_alloc_desc()
57 	kfree(desc->affinity);  in msi_free_desc()
65 	desc->msi_index = index;  in msi_insert_desc()
66 	ret = xa_insert(&md->__store, index, desc, GFP_KERNEL);  in msi_insert_desc()
73  * msi_add_msi_desc - Allocate and initialize a MSI descriptor
83 	lockdep_assert_held(&dev->msi.data->mutex);  in msi_add_msi_desc()
85 	desc = msi_alloc_desc(dev, init_desc->nvec_used, init_desc->affinity);  in msi_add_msi_desc()
87 		return -ENOMEM;  in msi_add_msi_desc()
90 	desc->pci = init_desc->pci;  in msi_add_msi_desc()
91 	return msi_insert_desc(dev->msi.data, desc, init_desc->msi_index);  in msi_add_msi_desc()
95  * msi_add_simple_msi_descs - Allocate and initialize MSI descriptors
104 	unsigned int idx, last = index + ndesc - 1;  in msi_add_simple_msi_descs()
108 	lockdep_assert_held(&dev->msi.data->mutex);  in msi_add_simple_msi_descs()
114 		ret = msi_insert_desc(dev->msi.data, desc, idx);  in msi_add_simple_msi_descs()
121 	ret = -ENOMEM;  in msi_add_simple_msi_descs()
133 		return !desc->irq;  in msi_desc_match()
135 		return !!desc->irq;  in msi_desc_match()
142  * msi_free_msi_descs_range - Free MSI descriptors of a device
151 	struct xarray *xa = &dev->msi.data->__store;  in msi_free_msi_descs_range()
155 	lockdep_assert_held(&dev->msi.data->mutex);  in msi_free_msi_descs_range()
167 	*msg = entry->msg;  in __get_cached_msi_msg()
182 	WARN_ON_ONCE(!xa_empty(&md->__store));  in msi_device_data_release()
183 	xa_destroy(&md->__store);  in msi_device_data_release()
184 	dev->msi.data = NULL;  in msi_device_data_release()
188  * msi_setup_device_data - Setup MSI device data
202 	if (dev->msi.data)  in msi_setup_device_data()
207 		return -ENOMEM;  in msi_setup_device_data()
215 	xa_init(&md->__store);  in msi_setup_device_data()
216 	mutex_init(&md->mutex);  in msi_setup_device_data()
217 	dev->msi.data = md;  in msi_setup_device_data()
223  * msi_lock_descs - Lock the MSI descriptor storage of a device
228 	mutex_lock(&dev->msi.data->mutex);  in msi_lock_descs()
233  * msi_unlock_descs - Unlock the MSI descriptor storage of a device
239 	dev->msi.data->__iter_idx = MSI_MAX_INDEX;  in msi_unlock_descs()
240 	mutex_unlock(&dev->msi.data->mutex);  in msi_unlock_descs()
248 	xa_for_each_start(&md->__store, md->__iter_idx, desc, md->__iter_idx) {  in msi_find_desc()
252 	md->__iter_idx = MSI_MAX_INDEX;  in msi_find_desc()
257  * msi_first_desc - Get the first MSI descriptor of a device
269 	struct msi_device_data *md = dev->msi.data;  in msi_first_desc()
274 	lockdep_assert_held(&md->mutex);  in msi_first_desc()
276 	md->__iter_idx = 0;  in msi_first_desc()
282  * msi_next_desc - Get the next MSI descriptor of a device
295 	struct msi_device_data *md = dev->msi.data;  in msi_next_desc()
300 	lockdep_assert_held(&md->mutex);  in msi_next_desc()
302 	if (md->__iter_idx >= (unsigned long)MSI_MAX_INDEX)  in msi_next_desc()
305 	md->__iter_idx++;  in msi_next_desc()
311  * msi_get_virq - Return Linux interrupt number of a MSI interrupt
313  * @index:	MSI interrupt index to look for (0-based)
323 	if (!dev->msi.data)  in msi_get_virq()
326 	pcimsi = dev_is_pci(dev) ? to_pci_dev(dev)->msi_enabled : false;  in msi_get_virq()
329 	desc = xa_load(&dev->msi.data->__store, pcimsi ? 0 : index);  in msi_get_virq()
330 	if (desc && desc->irq) {  in msi_get_virq()
332 		 * PCI-MSI has only one descriptor for multiple interrupts.  in msi_get_virq()
333 		 * PCI-MSIX and platform MSI use a descriptor per  in msi_get_virq()
337 			if (index < desc->nvec_used)  in msi_get_virq()
338 				ret = desc->irq + index;  in msi_get_virq()
340 			ret = desc->irq;  in msi_get_virq()
367 	bool is_msix = dev_is_pci(dev) ? to_pci_dev(dev)->msix_enabled : false;  in msi_mode_show()
374 	struct device_attribute *attrs = desc->sysfs_attrs;  in msi_sysfs_remove_desc()
380 	desc->sysfs_attrs = NULL;  in msi_sysfs_remove_desc()
381 	for (i = 0; i < desc->nvec_used; i++) {  in msi_sysfs_remove_desc()
383 			sysfs_remove_file_from_group(&dev->kobj, &attrs[i].attr, msi_irqs_group.name);  in msi_sysfs_remove_desc()
394 	attrs = kcalloc(desc->nvec_used, sizeof(*attrs), GFP_KERNEL);  in msi_sysfs_populate_desc()
396 		return -ENOMEM;  in msi_sysfs_populate_desc()
398 	desc->sysfs_attrs = attrs;  in msi_sysfs_populate_desc()
399 	for (i = 0; i < desc->nvec_used; i++) {  in msi_sysfs_populate_desc()
401 		attrs[i].attr.name = kasprintf(GFP_KERNEL, "%d", desc->irq + i);  in msi_sysfs_populate_desc()
403 			ret = -ENOMEM;  in msi_sysfs_populate_desc()
410 		ret = sysfs_add_file_to_group(&dev->kobj, &attrs[i].attr, msi_irqs_group.name);  in msi_sysfs_populate_desc()
425  * msi_device_populate_sysfs - Populate msi_irqs sysfs entries for a device
434 		if (desc->sysfs_attrs)  in msi_device_populate_sysfs()
444  * msi_device_destroy_sysfs - Destroy msi_irqs sysfs entries for a device
466 	data->chip->irq_write_msi_msg(data, msg);  in irq_chip_write_msi_msg()
471 	struct msi_domain_info *info = domain->host_data;  in msi_check_level()
475 	 * not advertized that it is level-capable, signal the breakage.  in msi_check_level()
477 	WARN_ON(!((info->flags & MSI_FLAG_LEVEL_CAPABLE) &&  in msi_check_level()
478 		  (info->chip->flags & IRQCHIP_SUPPORTS_LEVEL_MSI)) &&  in msi_check_level()
483  * msi_domain_set_affinity - Generic affinity setter function for MSI domains
485  * @mask:	The affinity mask to set
496 	struct irq_data *parent = irq_data->parent_data;  in msi_domain_set_affinity()
500 	ret = parent->chip->irq_set_affinity(parent, mask, force);  in msi_domain_set_affinity()
503 		msi_check_level(irq_data->domain, msg);  in msi_domain_set_affinity()
516 	msi_check_level(irq_data->domain, msg);  in msi_domain_activate()
533 	struct msi_domain_info *info = domain->host_data;  in msi_domain_alloc()
534 	struct msi_domain_ops *ops = info->ops;  in msi_domain_alloc()
535 	irq_hw_number_t hwirq = ops->get_hwirq(info, arg);  in msi_domain_alloc()
539 		return -EEXIST;  in msi_domain_alloc()
541 	if (domain->parent) {  in msi_domain_alloc()
548 		ret = ops->msi_init(domain, info, virq + i, hwirq + i, arg);  in msi_domain_alloc()
550 			if (ops->msi_free) {  in msi_domain_alloc()
551 				for (i--; i > 0; i--)  in msi_domain_alloc()
552 					ops->msi_free(domain, info, virq + i);  in msi_domain_alloc()
565 	struct msi_domain_info *info = domain->host_data;  in msi_domain_free()
568 	if (info->ops->msi_free) {  in msi_domain_free()
570 			info->ops->msi_free(domain, info, virq + i);  in msi_domain_free()
585 	return arg->hwirq;  in msi_domain_ops_get_hwirq()
598 	arg->desc = desc;  in msi_domain_ops_set_desc()
606 	irq_domain_set_hwirq_and_chip(domain, virq, hwirq, info->chip,  in msi_domain_ops_init()
607 				      info->chip_data);  in msi_domain_ops_init()
608 	if (info->handler && info->handler_name) {  in msi_domain_ops_init()
609 		__irq_set_handler(virq, info->handler, 0, info->handler_name);  in msi_domain_ops_init()
610 		if (info->handler_data)  in msi_domain_ops_init()
611 			irq_set_handler_data(virq, info->handler_data);  in msi_domain_ops_init()
635 	struct msi_domain_ops *ops = info->ops;  in msi_domain_update_dom_ops()
638 		info->ops = &msi_domain_ops_default;  in msi_domain_update_dom_ops()
642 	if (ops->domain_alloc_irqs == NULL)  in msi_domain_update_dom_ops()
643 		ops->domain_alloc_irqs = msi_domain_ops_default.domain_alloc_irqs;  in msi_domain_update_dom_ops()
644 	if (ops->domain_free_irqs == NULL)  in msi_domain_update_dom_ops()
645 		ops->domain_free_irqs = msi_domain_ops_default.domain_free_irqs;  in msi_domain_update_dom_ops()
647 	if (!(info->flags & MSI_FLAG_USE_DEF_DOM_OPS))  in msi_domain_update_dom_ops()
650 	if (ops->get_hwirq == NULL)  in msi_domain_update_dom_ops()
651 		ops->get_hwirq = msi_domain_ops_default.get_hwirq;  in msi_domain_update_dom_ops()
652 	if (ops->msi_init == NULL)  in msi_domain_update_dom_ops()
653 		ops->msi_init = msi_domain_ops_default.msi_init;  in msi_domain_update_dom_ops()
654 	if (ops->msi_check == NULL)  in msi_domain_update_dom_ops()
655 		ops->msi_check = msi_domain_ops_default.msi_check;  in msi_domain_update_dom_ops()
656 	if (ops->msi_prepare == NULL)  in msi_domain_update_dom_ops()
657 		ops->msi_prepare = msi_domain_ops_default.msi_prepare;  in msi_domain_update_dom_ops()
658 	if (ops->set_desc == NULL)  in msi_domain_update_dom_ops()
659 		ops->set_desc = msi_domain_ops_default.set_desc;  in msi_domain_update_dom_ops()
664 	struct irq_chip *chip = info->chip;  in msi_domain_update_chip_ops()
666 	BUG_ON(!chip || !chip->irq_mask || !chip->irq_unmask);  in msi_domain_update_chip_ops()
667 	if (!chip->irq_set_affinity)  in msi_domain_update_chip_ops()
668 		chip->irq_set_affinity = msi_domain_set_affinity;  in msi_domain_update_chip_ops()
672  * msi_create_irq_domain - Create an MSI interrupt domain
686 	if (info->flags & MSI_FLAG_USE_DEF_CHIP_OPS)  in msi_create_irq_domain()
692 	if (domain && !domain->name && info->chip)  in msi_create_irq_domain()
693 		domain->name = info->chip->name;  in msi_create_irq_domain()
701 	struct msi_domain_info *info = domain->host_data;  in msi_domain_prepare_irqs()
702 	struct msi_domain_ops *ops = info->ops;  in msi_domain_prepare_irqs()
705 	ret = ops->msi_check(domain, info, dev);  in msi_domain_prepare_irqs()
707 		ret = ops->msi_prepare(domain, dev, nvec, arg);  in msi_domain_prepare_irqs()
715 	struct msi_domain_info *info = domain->host_data;  in msi_domain_populate_irqs()
716 	struct msi_domain_ops *ops = info->ops;  in msi_domain_populate_irqs()
726 		desc = xa_load(&dev->msi.data->__store, virq);  in msi_domain_populate_irqs()
727 		desc->irq = virq;  in msi_domain_populate_irqs()
729 		ops->set_desc(arg, desc);  in msi_domain_populate_irqs()
740 	for (--virq; virq >= virq_base; virq--)  in msi_domain_populate_irqs()
742 	msi_free_msi_descs_range(dev, MSI_DESC_ALL, virq_base, virq_base + nvec - 1);  in msi_domain_populate_irqs()
765 	switch(domain->bus_token) {  in msi_check_reservation_mode()
773 	if (!(info->flags & MSI_FLAG_MUST_REACTIVATE))  in msi_check_reservation_mode()
784 	return desc->pci.msi_attrib.is_msix || desc->pci.msi_attrib.can_mask;  in msi_check_reservation_mode()
790 	switch(domain->bus_token) {  in msi_handle_pci_fail()
797 		return -ENOSPC;  in msi_handle_pci_fail()
801 	if (desc->nvec_used > 1)  in msi_handle_pci_fail()
805 	return allocated ? allocated : -ENOSPC;  in msi_handle_pci_fail()
827 		 * different way by using a catch-all vector.  in msi_init_virq()
856 	struct msi_domain_info *info = domain->host_data;  in __msi_domain_alloc_irqs()
857 	struct msi_domain_ops *ops = info->ops;  in __msi_domain_alloc_irqs()
873 	if (info->flags & MSI_FLAG_ACTIVATE_EARLY)  in __msi_domain_alloc_irqs()
883 		 * MSI affinity setting requires a special quirk (X86) when  in __msi_domain_alloc_irqs()
886 		if (domain->flags & IRQ_DOMAIN_MSI_NOMASK_QUIRK)  in __msi_domain_alloc_irqs()
891 		ops->set_desc(&arg, desc);  in __msi_domain_alloc_irqs()
893 		virq = __irq_domain_alloc_irqs(domain, -1, desc->nvec_used,  in __msi_domain_alloc_irqs()
895 					       desc->affinity);  in __msi_domain_alloc_irqs()
899 		for (i = 0; i < desc->nvec_used; i++) {  in __msi_domain_alloc_irqs()
906 		if (info->flags & MSI_FLAG_DEV_SYSFS) {  in __msi_domain_alloc_irqs()
920 	if (!(info->flags & MSI_FLAG_ALLOC_SIMPLE_MSI_DESCS))  in msi_domain_add_simple_msi_descs()
927  * msi_domain_alloc_irqs_descs_locked - Allocate interrupts from a MSI interrupt domain
942 	struct msi_domain_info *info = domain->host_data;  in msi_domain_alloc_irqs_descs_locked()
943 	struct msi_domain_ops *ops = info->ops;  in msi_domain_alloc_irqs_descs_locked()
946 	lockdep_assert_held(&dev->msi.data->mutex);  in msi_domain_alloc_irqs_descs_locked()
952 	ret = ops->domain_alloc_irqs(domain, dev, nvec);  in msi_domain_alloc_irqs_descs_locked()
959  * msi_domain_alloc_irqs - Allocate interrupts from a MSI interrupt domain
979 	struct msi_domain_info *info = domain->host_data;  in __msi_domain_free_irqs()
987 		for (i = 0; i < desc->nvec_used; i++) {  in __msi_domain_free_irqs()
988 			irqd = irq_domain_get_irq_data(domain, desc->irq + i);  in __msi_domain_free_irqs()
993 		irq_domain_free_irqs(desc->irq, desc->nvec_used);  in __msi_domain_free_irqs()
994 		if (info->flags & MSI_FLAG_DEV_SYSFS)  in __msi_domain_free_irqs()
996 		desc->irq = 0;  in __msi_domain_free_irqs()
1003 	if (info->flags & MSI_FLAG_FREE_MSI_DESCS)  in msi_domain_free_msi_descs()
1008 …* msi_domain_free_irqs_descs_locked - Free interrupts from a MSI interrupt @domain associated to @…
1019 	struct msi_domain_info *info = domain->host_data;  in msi_domain_free_irqs_descs_locked()
1020 	struct msi_domain_ops *ops = info->ops;  in msi_domain_free_irqs_descs_locked()
1022 	lockdep_assert_held(&dev->msi.data->mutex);  in msi_domain_free_irqs_descs_locked()
1024 	ops->domain_free_irqs(domain, dev);  in msi_domain_free_irqs_descs_locked()
1029  * msi_domain_free_irqs - Free interrupts from a MSI interrupt @domain associated to @dev
1042  * msi_get_domain_info - Get the MSI interrupt domain info for @domain
1045  * Return: the pointer to the msi_domain_info stored in @domain->host_data.
1049 	return (struct msi_domain_info *)domain->host_data;  in msi_get_domain_info()