1 // SPDX-License-Identifier: GPL-2.0
9  * PCI compatible and non PCI compatible devices.
17 #include <linux/pci.h>
22  * alloc_msi_entry - Allocate an initialized msi_desc
41 	INIT_LIST_HEAD(&desc->list);  in alloc_msi_entry()
42 	desc->dev = dev;  in alloc_msi_entry()
43 	desc->nvec_used = nvec;  in alloc_msi_entry()
45 		desc->affinity = kmemdup(affinity,  in alloc_msi_entry()
46 			nvec * sizeof(*desc->affinity), GFP_KERNEL);  in alloc_msi_entry()
47 		if (!desc->affinity) {  in alloc_msi_entry()
58 	kfree(entry->affinity);  in free_msi_entry()
64 	*msg = entry->msg;  in __get_cached_msi_msg()
83 	retval = kstrtoul(attr->attr.name, 10, &irq);  in msi_mode_show()
89 		return -ENODEV;  in msi_mode_show()
92 		is_msix = entry->msi_attrib.is_msix;  in msi_mode_show()
98  * msi_populate_sysfs - Populate msi_irqs sysfs entries for devices
99  * @dev:	The device(PCI, platform etc) who will get sysfs entries
112 	int ret = -ENOMEM;  in msi_populate_sysfs()
119 		num_msi += entry->nvec_used;  in msi_populate_sysfs()
126 		return ERR_PTR(-ENOMEM);  in msi_populate_sysfs()
129 		for (i = 0; i < entry->nvec_used; i++) {  in msi_populate_sysfs()
133 			msi_attrs[count] = &msi_dev_attr->attr;  in msi_populate_sysfs()
135 			sysfs_attr_init(&msi_dev_attr->attr);  in msi_populate_sysfs()
136 			msi_dev_attr->attr.name = kasprintf(GFP_KERNEL, "%d",  in msi_populate_sysfs()
137 							    entry->irq + i);  in msi_populate_sysfs()
138 			if (!msi_dev_attr->attr.name)  in msi_populate_sysfs()
140 			msi_dev_attr->attr.mode = 0444;  in msi_populate_sysfs()
141 			msi_dev_attr->show = msi_mode_show;  in msi_populate_sysfs()
149 	msi_irq_group->name = "msi_irqs";  in msi_populate_sysfs()
150 	msi_irq_group->attrs = msi_attrs;  in msi_populate_sysfs()
157 	ret = sysfs_create_groups(&dev->kobj, msi_irq_groups);  in msi_populate_sysfs()
172 		kfree(msi_attr->name);  in msi_populate_sysfs()
182  * msi_destroy_sysfs - Destroy msi_irqs sysfs entries for devices
183  * @dev:		The device(PCI, platform etc) who will remove sysfs entries
193 		sysfs_remove_groups(&dev->kobj, msi_irq_groups);  in msi_destroy_sysfs()
194 		msi_attrs = msi_irq_groups[0]->attrs;  in msi_destroy_sysfs()
198 			kfree(dev_attr->attr.name);  in msi_destroy_sysfs()
212 	data->chip->irq_write_msi_msg(data, msg);  in irq_chip_write_msi_msg()
215 static void msi_check_level(struct irq_domain *domain, struct msi_msg *msg)  in msi_check_level()  argument
217 	struct msi_domain_info *info = domain->host_data;  in msi_check_level()
221 	 * not advertized that it is level-capable, signal the breakage.  in msi_check_level()
223 	WARN_ON(!((info->flags & MSI_FLAG_LEVEL_CAPABLE) &&  in msi_check_level()
224 		  (info->chip->flags & IRQCHIP_SUPPORTS_LEVEL_MSI)) &&  in msi_check_level()
229  * msi_domain_set_affinity - Generic affinity setter function for MSI domains
242 	struct irq_data *parent = irq_data->parent_data;  in msi_domain_set_affinity()
246 	ret = parent->chip->irq_set_affinity(parent, mask, force);  in msi_domain_set_affinity()
249 		msi_check_level(irq_data->domain, msg);  in msi_domain_set_affinity()
256 static int msi_domain_activate(struct irq_domain *domain,  in msi_domain_activate()  argument
262 	msi_check_level(irq_data->domain, msg);  in msi_domain_activate()
267 static void msi_domain_deactivate(struct irq_domain *domain,  in msi_domain_deactivate()  argument
276 static int msi_domain_alloc(struct irq_domain *domain, unsigned int virq,  in msi_domain_alloc()  argument
279 	struct msi_domain_info *info = domain->host_data;  in msi_domain_alloc()
280 	struct msi_domain_ops *ops = info->ops;  in msi_domain_alloc()
281 	irq_hw_number_t hwirq = ops->get_hwirq(info, arg);  in msi_domain_alloc()
284 	if (irq_find_mapping(domain, hwirq) > 0)  in msi_domain_alloc()
285 		return -EEXIST;  in msi_domain_alloc()
287 	if (domain->parent) {  in msi_domain_alloc()
288 		ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, arg);  in msi_domain_alloc()
294 		ret = ops->msi_init(domain, info, virq + i, hwirq + i, arg);  in msi_domain_alloc()
296 			if (ops->msi_free) {  in msi_domain_alloc()
297 				for (i--; i > 0; i--)  in msi_domain_alloc()
298 					ops->msi_free(domain, info, virq + i);  in msi_domain_alloc()
300 			irq_domain_free_irqs_top(domain, virq, nr_irqs);  in msi_domain_alloc()
308 static void msi_domain_free(struct irq_domain *domain, unsigned int virq,  in msi_domain_free()  argument
311 	struct msi_domain_info *info = domain->host_data;  in msi_domain_free()
314 	if (info->ops->msi_free) {  in msi_domain_free()
316 			info->ops->msi_free(domain, info, virq + i);  in msi_domain_free()
318 	irq_domain_free_irqs_top(domain, virq, nr_irqs);  in msi_domain_free()
331 	return arg->hwirq;  in msi_domain_ops_get_hwirq()
334 static int msi_domain_ops_prepare(struct irq_domain *domain, struct device *dev,  in msi_domain_ops_prepare()  argument
344 	arg->desc = desc;  in msi_domain_ops_set_desc()
347 static int msi_domain_ops_init(struct irq_domain *domain,  in msi_domain_ops_init()  argument
352 	irq_domain_set_hwirq_and_chip(domain, virq, hwirq, info->chip,  in msi_domain_ops_init()
353 				      info->chip_data);  in msi_domain_ops_init()
354 	if (info->handler && info->handler_name) {  in msi_domain_ops_init()
355 		__irq_set_handler(virq, info->handler, 0, info->handler_name);  in msi_domain_ops_init()
356 		if (info->handler_data)  in msi_domain_ops_init()
357 			irq_set_handler_data(virq, info->handler_data);  in msi_domain_ops_init()
362 static int msi_domain_ops_check(struct irq_domain *domain,  in msi_domain_ops_check()  argument
381 	struct msi_domain_ops *ops = info->ops;  in msi_domain_update_dom_ops()
384 		info->ops = &msi_domain_ops_default;  in msi_domain_update_dom_ops()
388 	if (ops->domain_alloc_irqs == NULL)  in msi_domain_update_dom_ops()
389 		ops->domain_alloc_irqs = msi_domain_ops_default.domain_alloc_irqs;  in msi_domain_update_dom_ops()
390 	if (ops->domain_free_irqs == NULL)  in msi_domain_update_dom_ops()
391 		ops->domain_free_irqs = msi_domain_ops_default.domain_free_irqs;  in msi_domain_update_dom_ops()
393 	if (!(info->flags & MSI_FLAG_USE_DEF_DOM_OPS))  in msi_domain_update_dom_ops()
396 	if (ops->get_hwirq == NULL)  in msi_domain_update_dom_ops()
397 		ops->get_hwirq = msi_domain_ops_default.get_hwirq;  in msi_domain_update_dom_ops()
398 	if (ops->msi_init == NULL)  in msi_domain_update_dom_ops()
399 		ops->msi_init = msi_domain_ops_default.msi_init;  in msi_domain_update_dom_ops()
400 	if (ops->msi_check == NULL)  in msi_domain_update_dom_ops()
401 		ops->msi_check = msi_domain_ops_default.msi_check;  in msi_domain_update_dom_ops()
402 	if (ops->msi_prepare == NULL)  in msi_domain_update_dom_ops()
403 		ops->msi_prepare = msi_domain_ops_default.msi_prepare;  in msi_domain_update_dom_ops()
404 	if (ops->set_desc == NULL)  in msi_domain_update_dom_ops()
405 		ops->set_desc = msi_domain_ops_default.set_desc;  in msi_domain_update_dom_ops()
410 	struct irq_chip *chip = info->chip;  in msi_domain_update_chip_ops()
412 	BUG_ON(!chip || !chip->irq_mask || !chip->irq_unmask);  in msi_domain_update_chip_ops()
413 	if (!chip->irq_set_affinity)  in msi_domain_update_chip_ops()
414 		chip->irq_set_affinity = msi_domain_set_affinity;  in msi_domain_update_chip_ops()
418  * msi_create_irq_domain - Create an MSI interrupt domain
420  * @info:	MSI domain info
421  * @parent:	Parent irq domain
429 	struct irq_domain *domain;  in msi_create_irq_domain()  local
432 	if (info->flags & MSI_FLAG_USE_DEF_CHIP_OPS)  in msi_create_irq_domain()
435 	domain = irq_domain_create_hierarchy(parent, IRQ_DOMAIN_FLAG_MSI, 0,  in msi_create_irq_domain()
438 	if (domain && !domain->name && info->chip)  in msi_create_irq_domain()
439 		domain->name = info->chip->name;  in msi_create_irq_domain()
441 	return domain;  in msi_create_irq_domain()
444 int msi_domain_prepare_irqs(struct irq_domain *domain, struct device *dev,  in msi_domain_prepare_irqs()  argument
447 	struct msi_domain_info *info = domain->host_data;  in msi_domain_prepare_irqs()
448 	struct msi_domain_ops *ops = info->ops;  in msi_domain_prepare_irqs()
451 	ret = ops->msi_check(domain, info, dev);  in msi_domain_prepare_irqs()
453 		ret = ops->msi_prepare(domain, dev, nvec, arg);  in msi_domain_prepare_irqs()
458 int msi_domain_populate_irqs(struct irq_domain *domain, struct device *dev,  in msi_domain_populate_irqs()  argument
461 	struct msi_domain_info *info = domain->host_data;  in msi_domain_populate_irqs()
462 	struct msi_domain_ops *ops = info->ops;  in msi_domain_populate_irqs()
467 		/* Don't even try the multi-MSI brain damage. */  in msi_domain_populate_irqs()
468 		if (WARN_ON(!desc->irq || desc->nvec_used != 1)) {  in msi_domain_populate_irqs()
469 			ret = -EINVAL;  in msi_domain_populate_irqs()
473 		if (!(desc->irq >= virq && desc->irq < (virq + nvec)))  in msi_domain_populate_irqs()
476 		ops->set_desc(arg, desc);  in msi_domain_populate_irqs()
477 		/* Assumes the domain mutex is held! */  in msi_domain_populate_irqs()
478 		ret = irq_domain_alloc_irqs_hierarchy(domain, desc->irq, 1,  in msi_domain_populate_irqs()
483 		irq_set_msi_desc_off(desc->irq, 0, desc);  in msi_domain_populate_irqs()
489 			if (!(desc->irq >= virq && desc->irq < (virq + nvec)))  in msi_domain_populate_irqs()
492 			irq_domain_free_irqs_common(domain, desc->irq, 1);  in msi_domain_populate_irqs()
502  * dummy vector to the device. If the PCI/MSI device does not support
508  * used. For now reservation mode is restricted to PCI/MSI.
510 static bool msi_check_reservation_mode(struct irq_domain *domain,  in msi_check_reservation_mode()  argument
516 	switch(domain->bus_token) {  in msi_check_reservation_mode()
524 	if (!(info->flags & MSI_FLAG_MUST_REACTIVATE))  in msi_check_reservation_mode()
535 	return desc->msi_attrib.is_msix || desc->msi_attrib.maskbit;  in msi_check_reservation_mode()
538 int __msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev,  in __msi_domain_alloc_irqs()  argument
541 	struct msi_domain_info *info = domain->host_data;  in __msi_domain_alloc_irqs()
542 	struct msi_domain_ops *ops = info->ops;  in __msi_domain_alloc_irqs()
549 	ret = msi_domain_prepare_irqs(domain, dev, nvec, &arg);  in __msi_domain_alloc_irqs()
554 		ops->set_desc(&arg, desc);  in __msi_domain_alloc_irqs()
556 		virq = __irq_domain_alloc_irqs(domain, -1, desc->nvec_used,  in __msi_domain_alloc_irqs()
558 					       desc->affinity);  in __msi_domain_alloc_irqs()
560 			ret = -ENOSPC;  in __msi_domain_alloc_irqs()
561 			if (ops->handle_error)  in __msi_domain_alloc_irqs()
562 				ret = ops->handle_error(domain, desc, ret);  in __msi_domain_alloc_irqs()
563 			if (ops->msi_finish)  in __msi_domain_alloc_irqs()
564 				ops->msi_finish(&arg, ret);  in __msi_domain_alloc_irqs()
568 		for (i = 0; i < desc->nvec_used; i++) {  in __msi_domain_alloc_irqs()
574 	if (ops->msi_finish)  in __msi_domain_alloc_irqs()
575 		ops->msi_finish(&arg, 0);  in __msi_domain_alloc_irqs()
577 	can_reserve = msi_check_reservation_mode(domain, info, dev);  in __msi_domain_alloc_irqs()
580 	 * This flag is set by the PCI layer as we need to activate  in __msi_domain_alloc_irqs()
581 	 * the MSI entries before the PCI layer enables MSI in the  in __msi_domain_alloc_irqs()
584 	if (!(info->flags & MSI_FLAG_ACTIVATE_EARLY))  in __msi_domain_alloc_irqs()
588 		if (desc->irq == i) {  in __msi_domain_alloc_irqs()
589 			virq = desc->irq;  in __msi_domain_alloc_irqs()
590 			dev_dbg(dev, "irq [%d-%d] for MSI\n",  in __msi_domain_alloc_irqs()
591 				virq, virq + desc->nvec_used - 1);  in __msi_domain_alloc_irqs()
594 		irq_data = irq_domain_get_irq_data(domain, i);  in __msi_domain_alloc_irqs()
597 			if (domain->flags & IRQ_DOMAIN_MSI_NOMASK_QUIRK)  in __msi_domain_alloc_irqs()
612 			irq_data = irq_domain_get_irq_data(domain, i);  in __msi_domain_alloc_irqs()
619 	msi_domain_free_irqs(domain, dev);  in __msi_domain_alloc_irqs()
624  * msi_domain_alloc_irqs - Allocate interrupts from a MSI interrupt domain
625  * @domain:	The domain to allocate from
632 int msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev,  in msi_domain_alloc_irqs()  argument
635 	struct msi_domain_info *info = domain->host_data;  in msi_domain_alloc_irqs()
636 	struct msi_domain_ops *ops = info->ops;  in msi_domain_alloc_irqs()
638 	return ops->domain_alloc_irqs(domain, dev, nvec);  in msi_domain_alloc_irqs()
641 void __msi_domain_free_irqs(struct irq_domain *domain, struct device *dev)  in __msi_domain_free_irqs()  argument
648 		irq_data = irq_domain_get_irq_data(domain, i);  in __msi_domain_free_irqs()
659 		if (desc->irq) {  in __msi_domain_free_irqs()
660 			irq_domain_free_irqs(desc->irq, desc->nvec_used);  in __msi_domain_free_irqs()
661 			desc->irq = 0;  in __msi_domain_free_irqs()
667  * msi_domain_free_irqs - Free interrupts from a MSI interrupt @domain associated to @dev
668  * @domain:	The domain to managing the interrupts
672 void msi_domain_free_irqs(struct irq_domain *domain, struct device *dev)  in msi_domain_free_irqs()  argument
674 	struct msi_domain_info *info = domain->host_data;  in msi_domain_free_irqs()
675 	struct msi_domain_ops *ops = info->ops;  in msi_domain_free_irqs()
677 	return ops->domain_free_irqs(domain, dev);  in msi_domain_free_irqs()
681  * msi_get_domain_info - Get the MSI interrupt domain info for @domain
682  * @domain:	The interrupt domain to retrieve data from
684  * Return: the pointer to the msi_domain_info stored in @domain->host_data.
686 struct msi_domain_info *msi_get_domain_info(struct irq_domain *domain)  in msi_get_domain_info()  argument
688 	return (struct msi_domain_info *)domain->host_data;  in msi_get_domain_info()