1 // SPDX-License-Identifier: GPL-2.0
14 #include <linux/pci-epc.h>
15 #include <linux/pci-epf.h>
16 #include <linux/pci-ep-cfs.h>
35  * pci_epc_put() - release the PCI endpoint controller
45 	module_put(epc->ops->owner);  in pci_epc_put()
46 	put_device(&epc->dev);  in pci_epc_put()
51  * pci_epc_get() - get the PCI endpoint controller
59 	int ret = -EINVAL;  in pci_epc_get()
70 		if (!try_module_get(epc->ops->owner)) {  in pci_epc_get()
71 			ret = -EINVAL;  in pci_epc_get()
76 		get_device(&epc->dev);  in pci_epc_get()
87  * pci_epc_get_first_free_bar() - helper to get first unreserved BAR
101  * pci_epc_get_next_free_bar() - helper to get unreserved BAR starting from @bar
116 	/* If 'bar - 1' is a 64-bit BAR, move to the next BAR */  in pci_epc_get_next_free_bar()
117 	if ((epc_features->bar_fixed_64bit << 1) & 1 << bar)  in pci_epc_get_next_free_bar()
120 	/* Find if the reserved BAR is also a 64-bit BAR */  in pci_epc_get_next_free_bar()
121 	free_bar = epc_features->reserved_bar & epc_features->bar_fixed_64bit;  in pci_epc_get_next_free_bar()
123 	/* Set the adjacent bit if the reserved BAR is also a 64-bit BAR */  in pci_epc_get_next_free_bar()
125 	free_bar |= epc_features->reserved_bar;  in pci_epc_get_next_free_bar()
136  * pci_epc_get_features() - get the features supported by EPC
152 	if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions)  in pci_epc_get_features()
155 	if (vfunc_no > 0 && (!epc->max_vfs || vfunc_no > epc->max_vfs[func_no]))  in pci_epc_get_features()
158 	if (!epc->ops->get_features)  in pci_epc_get_features()
161 	mutex_lock(&epc->lock);  in pci_epc_get_features()
162 	epc_features = epc->ops->get_features(epc, func_no, vfunc_no);  in pci_epc_get_features()
163 	mutex_unlock(&epc->lock);  in pci_epc_get_features()
170  * pci_epc_stop() - stop the PCI link
177 	if (IS_ERR(epc) || !epc->ops->stop)  in pci_epc_stop()
180 	mutex_lock(&epc->lock);  in pci_epc_stop()
181 	epc->ops->stop(epc);  in pci_epc_stop()
182 	mutex_unlock(&epc->lock);  in pci_epc_stop()
187  * pci_epc_start() - start the PCI link
197 		return -EINVAL;  in pci_epc_start()
199 	if (!epc->ops->start)  in pci_epc_start()
202 	mutex_lock(&epc->lock);  in pci_epc_start()
203 	ret = epc->ops->start(epc);  in pci_epc_start()
204 	mutex_unlock(&epc->lock);  in pci_epc_start()
211  * pci_epc_raise_irq() - interrupt the host system
215  * @type: specify the type of interrupt; legacy, MSI or MSI-X
216  * @interrupt_num: the MSI or MSI-X interrupt number
218  * Invoke to raise an legacy, MSI or MSI-X interrupt
225 	if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions)  in pci_epc_raise_irq()
226 		return -EINVAL;  in pci_epc_raise_irq()
228 	if (vfunc_no > 0 && (!epc->max_vfs || vfunc_no > epc->max_vfs[func_no]))  in pci_epc_raise_irq()
229 		return -EINVAL;  in pci_epc_raise_irq()
231 	if (!epc->ops->raise_irq)  in pci_epc_raise_irq()
234 	mutex_lock(&epc->lock);  in pci_epc_raise_irq()
235 	ret = epc->ops->raise_irq(epc, func_no, vfunc_no, type, interrupt_num);  in pci_epc_raise_irq()
236 	mutex_unlock(&epc->lock);  in pci_epc_raise_irq()
243  * pci_epc_map_msi_irq() - Map physical address to MSI address and return
244  *                         MSI data
245  * @epc: the EPC device which has the MSI capability
249  * @interrupt_num: the MSI interrupt number
251  * @msi_data: the data that should be written in order to raise MSI interrupt
253  * @msi_addr_offset: Offset of MSI address from the aligned outbound address
254  *                   to which the MSI address is mapped
256  * Invoke to map physical address to MSI address and return MSI data. The
270 		return -EINVAL;  in pci_epc_map_msi_irq()
272 	if (vfunc_no > 0 && (!epc->max_vfs || vfunc_no > epc->max_vfs[func_no]))  in pci_epc_map_msi_irq()
273 		return -EINVAL;  in pci_epc_map_msi_irq()
275 	if (!epc->ops->map_msi_irq)  in pci_epc_map_msi_irq()
276 		return -EINVAL;  in pci_epc_map_msi_irq()
278 	mutex_lock(&epc->lock);  in pci_epc_map_msi_irq()
279 	ret = epc->ops->map_msi_irq(epc, func_no, vfunc_no, phys_addr,  in pci_epc_map_msi_irq()
282 	mutex_unlock(&epc->lock);  in pci_epc_map_msi_irq()
289  * pci_epc_get_msi() - get the number of MSI interrupt numbers allocated
290  * @epc: the EPC device to which MSI interrupts was requested
294  * Invoke to get the number of MSI interrupts allocated by the RC
300 	if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions)  in pci_epc_get_msi()
303 	if (vfunc_no > 0 && (!epc->max_vfs || vfunc_no > epc->max_vfs[func_no]))  in pci_epc_get_msi()
306 	if (!epc->ops->get_msi)  in pci_epc_get_msi()
309 	mutex_lock(&epc->lock);  in pci_epc_get_msi()
310 	interrupt = epc->ops->get_msi(epc, func_no, vfunc_no);  in pci_epc_get_msi()
311 	mutex_unlock(&epc->lock);  in pci_epc_get_msi()
323  * pci_epc_set_msi() - set the number of MSI interrupt numbers required
324  * @epc: the EPC device on which MSI has to be configured
327  * @interrupts: number of MSI interrupts required by the EPF
329  * Invoke to set the required number of MSI interrupts.
336 	if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions ||  in pci_epc_set_msi()
338 		return -EINVAL;  in pci_epc_set_msi()
340 	if (vfunc_no > 0 && (!epc->max_vfs || vfunc_no > epc->max_vfs[func_no]))  in pci_epc_set_msi()
341 		return -EINVAL;  in pci_epc_set_msi()
343 	if (!epc->ops->set_msi)  in pci_epc_set_msi()
348 	mutex_lock(&epc->lock);  in pci_epc_set_msi()
349 	ret = epc->ops->set_msi(epc, func_no, vfunc_no, encode_int);  in pci_epc_set_msi()
350 	mutex_unlock(&epc->lock);  in pci_epc_set_msi()
357  * pci_epc_get_msix() - get the number of MSI-X interrupt numbers allocated
358  * @epc: the EPC device to which MSI-X interrupts was requested
362  * Invoke to get the number of MSI-X interrupts allocated by the RC
368 	if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions)  in pci_epc_get_msix()
371 	if (vfunc_no > 0 && (!epc->max_vfs || vfunc_no > epc->max_vfs[func_no]))  in pci_epc_get_msix()
374 	if (!epc->ops->get_msix)  in pci_epc_get_msix()
377 	mutex_lock(&epc->lock);  in pci_epc_get_msix()
378 	interrupt = epc->ops->get_msix(epc, func_no, vfunc_no);  in pci_epc_get_msix()
379 	mutex_unlock(&epc->lock);  in pci_epc_get_msix()
389  * pci_epc_set_msix() - set the number of MSI-X interrupt numbers required
390  * @epc: the EPC device on which MSI-X has to be configured
393  * @interrupts: number of MSI-X interrupts required by the EPF
394  * @bir: BAR where the MSI-X table resides
395  * @offset: Offset pointing to the start of MSI-X table
397  * Invoke to set the required number of MSI-X interrupts.
404 	if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions ||  in pci_epc_set_msix()
406 		return -EINVAL;  in pci_epc_set_msix()
408 	if (vfunc_no > 0 && (!epc->max_vfs || vfunc_no > epc->max_vfs[func_no]))  in pci_epc_set_msix()
409 		return -EINVAL;  in pci_epc_set_msix()
411 	if (!epc->ops->set_msix)  in pci_epc_set_msix()
414 	mutex_lock(&epc->lock);  in pci_epc_set_msix()
415 	ret = epc->ops->set_msix(epc, func_no, vfunc_no, interrupts - 1, bir,  in pci_epc_set_msix()
417 	mutex_unlock(&epc->lock);  in pci_epc_set_msix()
424  * pci_epc_unmap_addr() - unmap CPU address from PCI address
435 	if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions)  in pci_epc_unmap_addr()
438 	if (vfunc_no > 0 && (!epc->max_vfs || vfunc_no > epc->max_vfs[func_no]))  in pci_epc_unmap_addr()
441 	if (!epc->ops->unmap_addr)  in pci_epc_unmap_addr()
444 	mutex_lock(&epc->lock);  in pci_epc_unmap_addr()
445 	epc->ops->unmap_addr(epc, func_no, vfunc_no, phys_addr);  in pci_epc_unmap_addr()
446 	mutex_unlock(&epc->lock);  in pci_epc_unmap_addr()
451  * pci_epc_map_addr() - map CPU address to PCI address
466 	if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions)  in pci_epc_map_addr()
467 		return -EINVAL;  in pci_epc_map_addr()
469 	if (vfunc_no > 0 && (!epc->max_vfs || vfunc_no > epc->max_vfs[func_no]))  in pci_epc_map_addr()
470 		return -EINVAL;  in pci_epc_map_addr()
472 	if (!epc->ops->map_addr)  in pci_epc_map_addr()
475 	mutex_lock(&epc->lock);  in pci_epc_map_addr()
476 	ret = epc->ops->map_addr(epc, func_no, vfunc_no, phys_addr, pci_addr,  in pci_epc_map_addr()
478 	mutex_unlock(&epc->lock);  in pci_epc_map_addr()
485  * pci_epc_clear_bar() - reset the BAR
496 	if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions ||  in pci_epc_clear_bar()
497 	    (epf_bar->barno == BAR_5 &&  in pci_epc_clear_bar()
498 	     epf_bar->flags & PCI_BASE_ADDRESS_MEM_TYPE_64))  in pci_epc_clear_bar()
501 	if (vfunc_no > 0 && (!epc->max_vfs || vfunc_no > epc->max_vfs[func_no]))  in pci_epc_clear_bar()
504 	if (!epc->ops->clear_bar)  in pci_epc_clear_bar()
507 	mutex_lock(&epc->lock);  in pci_epc_clear_bar()
508 	epc->ops->clear_bar(epc, func_no, vfunc_no, epf_bar);  in pci_epc_clear_bar()
509 	mutex_unlock(&epc->lock);  in pci_epc_clear_bar()
514  * pci_epc_set_bar() - configure BAR in order for host to assign PCI addr space
526 	int flags = epf_bar->flags;  in pci_epc_set_bar()
528 	if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions ||  in pci_epc_set_bar()
529 	    (epf_bar->barno == BAR_5 &&  in pci_epc_set_bar()
533 	    (upper_32_bits(epf_bar->size) &&  in pci_epc_set_bar()
535 		return -EINVAL;  in pci_epc_set_bar()
537 	if (vfunc_no > 0 && (!epc->max_vfs || vfunc_no > epc->max_vfs[func_no]))  in pci_epc_set_bar()
538 		return -EINVAL;  in pci_epc_set_bar()
540 	if (!epc->ops->set_bar)  in pci_epc_set_bar()
543 	mutex_lock(&epc->lock);  in pci_epc_set_bar()
544 	ret = epc->ops->set_bar(epc, func_no, vfunc_no, epf_bar);  in pci_epc_set_bar()
545 	mutex_unlock(&epc->lock);  in pci_epc_set_bar()
552  * pci_epc_write_header() - write standard configuration header
568 	if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions)  in pci_epc_write_header()
569 		return -EINVAL;  in pci_epc_write_header()
571 	if (vfunc_no > 0 && (!epc->max_vfs || vfunc_no > epc->max_vfs[func_no]))  in pci_epc_write_header()
572 		return -EINVAL;  in pci_epc_write_header()
576 		return -EINVAL;  in pci_epc_write_header()
578 	if (!epc->ops->write_header)  in pci_epc_write_header()
581 	mutex_lock(&epc->lock);  in pci_epc_write_header()
582 	ret = epc->ops->write_header(epc, func_no, vfunc_no, header);  in pci_epc_write_header()
583 	mutex_unlock(&epc->lock);  in pci_epc_write_header()
590  * pci_epc_add_epf() - bind PCI endpoint function to an endpoint controller
607 	if (IS_ERR_OR_NULL(epc) || epf->is_vf)  in pci_epc_add_epf()
608 		return -EINVAL;  in pci_epc_add_epf()
610 	if (type == PRIMARY_INTERFACE && epf->epc)  in pci_epc_add_epf()
611 		return -EBUSY;  in pci_epc_add_epf()
613 	if (type == SECONDARY_INTERFACE && epf->sec_epc)  in pci_epc_add_epf()
614 		return -EBUSY;  in pci_epc_add_epf()
616 	mutex_lock(&epc->lock);  in pci_epc_add_epf()
617 	func_no = find_first_zero_bit(&epc->function_num_map,  in pci_epc_add_epf()
620 		ret = -EINVAL;  in pci_epc_add_epf()
624 	if (func_no > epc->max_functions - 1) {  in pci_epc_add_epf()
625 		dev_err(&epc->dev, "Exceeding max supported Function Number\n");  in pci_epc_add_epf()
626 		ret = -EINVAL;  in pci_epc_add_epf()
630 	set_bit(func_no, &epc->function_num_map);  in pci_epc_add_epf()
632 		epf->func_no = func_no;  in pci_epc_add_epf()
633 		epf->epc = epc;  in pci_epc_add_epf()
634 		list = &epf->list;  in pci_epc_add_epf()
636 		epf->sec_epc_func_no = func_no;  in pci_epc_add_epf()
637 		epf->sec_epc = epc;  in pci_epc_add_epf()
638 		list = &epf->sec_epc_list;  in pci_epc_add_epf()
641 	list_add_tail(list, &epc->pci_epf);  in pci_epc_add_epf()
643 	mutex_unlock(&epc->lock);  in pci_epc_add_epf()
650  * pci_epc_remove_epf() - remove PCI endpoint function from endpoint controller
668 		func_no = epf->func_no;  in pci_epc_remove_epf()
669 		list = &epf->list;  in pci_epc_remove_epf()
671 		func_no = epf->sec_epc_func_no;  in pci_epc_remove_epf()
672 		list = &epf->sec_epc_list;  in pci_epc_remove_epf()
675 	mutex_lock(&epc->lock);  in pci_epc_remove_epf()
676 	clear_bit(func_no, &epc->function_num_map);  in pci_epc_remove_epf()
678 	epf->epc = NULL;  in pci_epc_remove_epf()
679 	mutex_unlock(&epc->lock);  in pci_epc_remove_epf()
684  * pci_epc_linkup() - Notify the EPF device that EPC device has established a
696 	atomic_notifier_call_chain(&epc->notifier, LINK_UP, NULL);  in pci_epc_linkup()
701  * pci_epc_init_notify() - Notify the EPF device that EPC device's core
713 	atomic_notifier_call_chain(&epc->notifier, CORE_INIT, NULL);  in pci_epc_init_notify()
718  * pci_epc_destroy() - destroy the EPC device
725 	pci_ep_cfs_remove_epc_group(epc->group);  in pci_epc_destroy()
726 	device_unregister(&epc->dev);  in pci_epc_destroy()
732  * devm_pci_epc_destroy() - destroy the EPC device
750  * __pci_epc_create() - create a new endpoint controller (EPC) device
765 		ret = -EINVAL;  in __pci_epc_create()
771 		ret = -ENOMEM;  in __pci_epc_create()
775 	mutex_init(&epc->lock);  in __pci_epc_create()
776 	INIT_LIST_HEAD(&epc->pci_epf);  in __pci_epc_create()
777 	ATOMIC_INIT_NOTIFIER_HEAD(&epc->notifier);  in __pci_epc_create()
779 	device_initialize(&epc->dev);  in __pci_epc_create()
780 	epc->dev.class = pci_epc_class;  in __pci_epc_create()
781 	epc->dev.parent = dev;  in __pci_epc_create()
782 	epc->ops = ops;  in __pci_epc_create()
784 	ret = dev_set_name(&epc->dev, "%s", dev_name(dev));  in __pci_epc_create()
788 	ret = device_add(&epc->dev);  in __pci_epc_create()
792 	epc->group = pci_ep_cfs_add_epc_group(dev_name(dev));  in __pci_epc_create()
797 	put_device(&epc->dev);  in __pci_epc_create()
806  * __devm_pci_epc_create() - create a new endpoint controller (EPC) device
824 		return ERR_PTR(-ENOMEM);  in __devm_pci_epc_create()
842 		pr_err("failed to create pci epc class --> %ld\n",  in pci_epc_init()