1 // SPDX-License-Identifier: GPL-2.0
3 // Cadence PCIe endpoint controller driver.
4 // Author: Cyrille Pitchen <cyrille.pitchen@free-electrons.com>
9 #include <linux/pci-epc.h>
13 #include "pcie-cadence.h"
19 static u8 cdns_pcie_get_fn_from_vfn(struct cdns_pcie *pcie, u8 fn, u8 vfn)  in cdns_pcie_get_fn_from_vfn()  argument
27 	first_vf_offset = cdns_pcie_ep_fn_readw(pcie, fn, cap + PCI_SRIOV_VF_OFFSET);  in cdns_pcie_get_fn_from_vfn()
28 	stride = cdns_pcie_ep_fn_readw(pcie, fn, cap +  PCI_SRIOV_VF_STRIDE);  in cdns_pcie_get_fn_from_vfn()
29 	fn = fn + first_vf_offset + ((vfn - 1) * stride);  in cdns_pcie_get_fn_from_vfn()
39 	struct cdns_pcie *pcie = &ep->pcie;  in cdns_pcie_ep_write_header()  local
43 		dev_err(&epc->dev, "Only Virtual Function #1 has deviceID\n");  in cdns_pcie_ep_write_header()
44 		return -EINVAL;  in cdns_pcie_ep_write_header()
47 		cdns_pcie_ep_fn_writew(pcie, fn, reg, hdr->deviceid);  in cdns_pcie_ep_write_header()
51 	cdns_pcie_ep_fn_writew(pcie, fn, PCI_DEVICE_ID, hdr->deviceid);  in cdns_pcie_ep_write_header()
52 	cdns_pcie_ep_fn_writeb(pcie, fn, PCI_REVISION_ID, hdr->revid);  in cdns_pcie_ep_write_header()
53 	cdns_pcie_ep_fn_writeb(pcie, fn, PCI_CLASS_PROG, hdr->progif_code);  in cdns_pcie_ep_write_header()
54 	cdns_pcie_ep_fn_writew(pcie, fn, PCI_CLASS_DEVICE,  in cdns_pcie_ep_write_header()
55 			       hdr->subclass_code | hdr->baseclass_code << 8);  in cdns_pcie_ep_write_header()
56 	cdns_pcie_ep_fn_writeb(pcie, fn, PCI_CACHE_LINE_SIZE,  in cdns_pcie_ep_write_header()
57 			       hdr->cache_line_size);  in cdns_pcie_ep_write_header()
58 	cdns_pcie_ep_fn_writew(pcie, fn, PCI_SUBSYSTEM_ID, hdr->subsys_id);  in cdns_pcie_ep_write_header()
59 	cdns_pcie_ep_fn_writeb(pcie, fn, PCI_INTERRUPT_PIN, hdr->interrupt_pin);  in cdns_pcie_ep_write_header()
67 		u32 id = CDNS_PCIE_LM_ID_VENDOR(hdr->vendorid) |  in cdns_pcie_ep_write_header()
68 			 CDNS_PCIE_LM_ID_SUBSYS(hdr->subsys_vendor_id);  in cdns_pcie_ep_write_header()
70 		cdns_pcie_writel(pcie, CDNS_PCIE_LM_ID, id);  in cdns_pcie_ep_write_header()
80 	struct cdns_pcie_epf *epf = &ep->epf[fn];  in cdns_pcie_ep_set_bar()
81 	struct cdns_pcie *pcie = &ep->pcie;  in cdns_pcie_ep_set_bar()  local
82 	dma_addr_t bar_phys = epf_bar->phys_addr;  in cdns_pcie_ep_set_bar()
83 	enum pci_barno bar = epf_bar->barno;  in cdns_pcie_ep_set_bar()
84 	int flags = epf_bar->flags;  in cdns_pcie_ep_set_bar()
89 	sz = max_t(size_t, epf_bar->size, CDNS_PCIE_EP_MIN_APERTURE);  in cdns_pcie_ep_set_bar()
94 	sz = 1ULL << fls64(sz - 1);  in cdns_pcie_ep_set_bar()
95 	aperture = ilog2(sz) - 7; /* 128B -> 0, 256B -> 1, 512B -> 2, ... */  in cdns_pcie_ep_set_bar()
104 			return -EINVAL;  in cdns_pcie_ep_set_bar()
107 			epf_bar->flags |= PCI_BASE_ADDRESS_MEM_TYPE_64;  in cdns_pcie_ep_set_bar()
126 	b = (bar < BAR_4) ? bar : bar - BAR_4;  in cdns_pcie_ep_set_bar()
129 		cfg = cdns_pcie_readl(pcie, reg);  in cdns_pcie_ep_set_bar()
134 		cdns_pcie_writel(pcie, reg, cfg);  in cdns_pcie_ep_set_bar()
137 	fn = cdns_pcie_get_fn_from_vfn(pcie, fn, vfn);  in cdns_pcie_ep_set_bar()
138 	cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_EP_FUNC_BAR_ADDR0(fn, bar),  in cdns_pcie_ep_set_bar()
140 	cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_EP_FUNC_BAR_ADDR1(fn, bar),  in cdns_pcie_ep_set_bar()
144 		epf = &epf->epf[vfn - 1];  in cdns_pcie_ep_set_bar()
145 	epf->epf_bar[bar] = epf_bar;  in cdns_pcie_ep_set_bar()
154 	struct cdns_pcie_epf *epf = &ep->epf[fn];  in cdns_pcie_ep_clear_bar()
155 	struct cdns_pcie *pcie = &ep->pcie;  in cdns_pcie_ep_clear_bar()  local
156 	enum pci_barno bar = epf_bar->barno;  in cdns_pcie_ep_clear_bar()
163 	b = (bar < BAR_4) ? bar : bar - BAR_4;  in cdns_pcie_ep_clear_bar()
167 		cfg = cdns_pcie_readl(pcie, reg);  in cdns_pcie_ep_clear_bar()
171 		cdns_pcie_writel(pcie, reg, cfg);  in cdns_pcie_ep_clear_bar()
174 	fn = cdns_pcie_get_fn_from_vfn(pcie, fn, vfn);  in cdns_pcie_ep_clear_bar()
175 	cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_EP_FUNC_BAR_ADDR0(fn, bar), 0);  in cdns_pcie_ep_clear_bar()
176 	cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_EP_FUNC_BAR_ADDR1(fn, bar), 0);  in cdns_pcie_ep_clear_bar()
179 		epf = &epf->epf[vfn - 1];  in cdns_pcie_ep_clear_bar()
180 	epf->epf_bar[bar] = NULL;  in cdns_pcie_ep_clear_bar()
187 	struct cdns_pcie *pcie = &ep->pcie;  in cdns_pcie_ep_map_addr()  local
190 	r = find_first_zero_bit(&ep->ob_region_map, BITS_PER_LONG);  in cdns_pcie_ep_map_addr()
191 	if (r >= ep->max_regions - 1) {  in cdns_pcie_ep_map_addr()
192 		dev_err(&epc->dev, "no free outbound region\n");  in cdns_pcie_ep_map_addr()
193 		return -EINVAL;  in cdns_pcie_ep_map_addr()
196 	fn = cdns_pcie_get_fn_from_vfn(pcie, fn, vfn);  in cdns_pcie_ep_map_addr()
197 	cdns_pcie_set_outbound_region(pcie, 0, fn, r, false, addr, pci_addr, size);  in cdns_pcie_ep_map_addr()
199 	set_bit(r, &ep->ob_region_map);  in cdns_pcie_ep_map_addr()
200 	ep->ob_addr[r] = addr;  in cdns_pcie_ep_map_addr()
209 	struct cdns_pcie *pcie = &ep->pcie;  in cdns_pcie_ep_unmap_addr()  local
212 	for (r = 0; r < ep->max_regions - 1; r++)  in cdns_pcie_ep_unmap_addr()
213 		if (ep->ob_addr[r] == addr)  in cdns_pcie_ep_unmap_addr()
216 	if (r == ep->max_regions - 1)  in cdns_pcie_ep_unmap_addr()
219 	cdns_pcie_reset_outbound_region(pcie, r);  in cdns_pcie_ep_unmap_addr()
221 	ep->ob_addr[r] = 0;  in cdns_pcie_ep_unmap_addr()
222 	clear_bit(r, &ep->ob_region_map);  in cdns_pcie_ep_unmap_addr()
228 	struct cdns_pcie *pcie = &ep->pcie;  in cdns_pcie_ep_set_msi()  local
232 	fn = cdns_pcie_get_fn_from_vfn(pcie, fn, vfn);  in cdns_pcie_ep_set_msi()
238 	flags = cdns_pcie_ep_fn_readw(pcie, fn, cap + PCI_MSI_FLAGS);  in cdns_pcie_ep_set_msi()
242 	cdns_pcie_ep_fn_writew(pcie, fn, cap + PCI_MSI_FLAGS, flags);  in cdns_pcie_ep_set_msi()
250 	struct cdns_pcie *pcie = &ep->pcie;  in cdns_pcie_ep_get_msi()  local
254 	fn = cdns_pcie_get_fn_from_vfn(pcie, fn, vfn);  in cdns_pcie_ep_get_msi()
257 	flags = cdns_pcie_ep_fn_readw(pcie, fn, cap + PCI_MSI_FLAGS);  in cdns_pcie_ep_get_msi()
259 		return -EINVAL;  in cdns_pcie_ep_get_msi()
273 	struct cdns_pcie *pcie = &ep->pcie;  in cdns_pcie_ep_get_msix()  local
277 	func_no = cdns_pcie_get_fn_from_vfn(pcie, func_no, vfunc_no);  in cdns_pcie_ep_get_msix()
280 	val = cdns_pcie_ep_fn_readw(pcie, func_no, reg);  in cdns_pcie_ep_get_msix()
282 		return -EINVAL;  in cdns_pcie_ep_get_msix()
294 	struct cdns_pcie *pcie = &ep->pcie;  in cdns_pcie_ep_set_msix()  local
298 	fn = cdns_pcie_get_fn_from_vfn(pcie, fn, vfn);  in cdns_pcie_ep_set_msix()
301 	val = cdns_pcie_ep_fn_readw(pcie, fn, reg);  in cdns_pcie_ep_set_msix()
304 	cdns_pcie_ep_fn_writew(pcie, fn, reg, val);  in cdns_pcie_ep_set_msix()
309 	cdns_pcie_ep_fn_writel(pcie, fn, reg, val);  in cdns_pcie_ep_set_msix()
314 	cdns_pcie_ep_fn_writel(pcie, fn, reg, val);  in cdns_pcie_ep_set_msix()
322 	struct cdns_pcie *pcie = &ep->pcie;  in cdns_pcie_ep_assert_intx()  local
331 	if (unlikely(ep->irq_pci_addr != CDNS_PCIE_EP_IRQ_PCI_ADDR_LEGACY ||  in cdns_pcie_ep_assert_intx()
332 		     ep->irq_pci_fn != fn)) {  in cdns_pcie_ep_assert_intx()
334 		cdns_pcie_set_outbound_region_for_normal_msg(pcie, 0, fn, 0,  in cdns_pcie_ep_assert_intx()
335 							     ep->irq_phys_addr);  in cdns_pcie_ep_assert_intx()
336 		ep->irq_pci_addr = CDNS_PCIE_EP_IRQ_PCI_ADDR_LEGACY;  in cdns_pcie_ep_assert_intx()
337 		ep->irq_pci_fn = fn;  in cdns_pcie_ep_assert_intx()
341 		ep->irq_pending |= BIT(intx);  in cdns_pcie_ep_assert_intx()
344 		ep->irq_pending &= ~BIT(intx);  in cdns_pcie_ep_assert_intx()
348 	spin_lock_irqsave(&ep->lock, flags);  in cdns_pcie_ep_assert_intx()
349 	status = cdns_pcie_ep_fn_readw(pcie, fn, PCI_STATUS);  in cdns_pcie_ep_assert_intx()
350 	if (((status & PCI_STATUS_INTERRUPT) != 0) ^ (ep->irq_pending != 0)) {  in cdns_pcie_ep_assert_intx()
352 		cdns_pcie_ep_fn_writew(pcie, fn, PCI_STATUS, status);  in cdns_pcie_ep_assert_intx()
354 	spin_unlock_irqrestore(&ep->lock, flags);  in cdns_pcie_ep_assert_intx()
359 	writel(0, ep->irq_cpu_addr + offset);  in cdns_pcie_ep_assert_intx()
367 	cmd = cdns_pcie_ep_fn_readw(&ep->pcie, fn, PCI_COMMAND);  in cdns_pcie_ep_send_legacy_irq()
369 		return -EINVAL;  in cdns_pcie_ep_send_legacy_irq()
383 	struct cdns_pcie *pcie = &ep->pcie;  in cdns_pcie_ep_send_msi_irq()  local
389 	fn = cdns_pcie_get_fn_from_vfn(pcie, fn, vfn);  in cdns_pcie_ep_send_msi_irq()
391 	/* Check whether the MSI feature has been enabled by the PCI host. */  in cdns_pcie_ep_send_msi_irq()
392 	flags = cdns_pcie_ep_fn_readw(pcie, fn, cap + PCI_MSI_FLAGS);  in cdns_pcie_ep_send_msi_irq()
394 		return -EINVAL;  in cdns_pcie_ep_send_msi_irq()
400 		return -EINVAL;  in cdns_pcie_ep_send_msi_irq()
403 	data_mask = msi_count - 1;  in cdns_pcie_ep_send_msi_irq()
404 	data = cdns_pcie_ep_fn_readw(pcie, fn, cap + PCI_MSI_DATA_64);  in cdns_pcie_ep_send_msi_irq()
405 	data = (data & ~data_mask) | ((interrupt_num - 1) & data_mask);  in cdns_pcie_ep_send_msi_irq()
408 	pci_addr = cdns_pcie_ep_fn_readl(pcie, fn, cap + PCI_MSI_ADDRESS_HI);  in cdns_pcie_ep_send_msi_irq()
410 	pci_addr |= cdns_pcie_ep_fn_readl(pcie, fn, cap + PCI_MSI_ADDRESS_LO);  in cdns_pcie_ep_send_msi_irq()
414 	if (unlikely(ep->irq_pci_addr != (pci_addr & ~pci_addr_mask) ||  in cdns_pcie_ep_send_msi_irq()
415 		     ep->irq_pci_fn != fn)) {  in cdns_pcie_ep_send_msi_irq()
417 		cdns_pcie_set_outbound_region(pcie, 0, fn, 0,  in cdns_pcie_ep_send_msi_irq()
419 					      ep->irq_phys_addr,  in cdns_pcie_ep_send_msi_irq()
422 		ep->irq_pci_addr = (pci_addr & ~pci_addr_mask);  in cdns_pcie_ep_send_msi_irq()
423 		ep->irq_pci_fn = fn;  in cdns_pcie_ep_send_msi_irq()
425 	writel(data, ep->irq_cpu_addr + (pci_addr & pci_addr_mask));  in cdns_pcie_ep_send_msi_irq()
437 	struct cdns_pcie *pcie = &ep->pcie;  in cdns_pcie_ep_map_msi_irq()  local
444 	fn = cdns_pcie_get_fn_from_vfn(pcie, fn, vfn);  in cdns_pcie_ep_map_msi_irq()
446 	/* Check whether the MSI feature has been enabled by the PCI host. */  in cdns_pcie_ep_map_msi_irq()
447 	flags = cdns_pcie_ep_fn_readw(pcie, fn, cap + PCI_MSI_FLAGS);  in cdns_pcie_ep_map_msi_irq()
449 		return -EINVAL;  in cdns_pcie_ep_map_msi_irq()
455 		return -EINVAL;  in cdns_pcie_ep_map_msi_irq()
458 	data_mask = msi_count - 1;  in cdns_pcie_ep_map_msi_irq()
459 	data = cdns_pcie_ep_fn_readw(pcie, fn, cap + PCI_MSI_DATA_64);  in cdns_pcie_ep_map_msi_irq()
463 	pci_addr = cdns_pcie_ep_fn_readl(pcie, fn, cap + PCI_MSI_ADDRESS_HI);  in cdns_pcie_ep_map_msi_irq()
465 	pci_addr |= cdns_pcie_ep_fn_readl(pcie, fn, cap + PCI_MSI_ADDRESS_LO);  in cdns_pcie_ep_map_msi_irq()
488 	struct cdns_pcie *pcie = &ep->pcie;  in cdns_pcie_ep_send_msix_irq()  local
496 	epf = &ep->epf[fn];  in cdns_pcie_ep_send_msix_irq()
498 		epf = &epf->epf[vfn - 1];  in cdns_pcie_ep_send_msix_irq()
500 	fn = cdns_pcie_get_fn_from_vfn(pcie, fn, vfn);  in cdns_pcie_ep_send_msix_irq()
502 	/* Check whether the MSI-X feature has been enabled by the PCI host. */  in cdns_pcie_ep_send_msix_irq()
503 	flags = cdns_pcie_ep_fn_readw(pcie, fn, cap + PCI_MSIX_FLAGS);  in cdns_pcie_ep_send_msix_irq()
505 		return -EINVAL;  in cdns_pcie_ep_send_msix_irq()
508 	tbl_offset = cdns_pcie_ep_fn_readl(pcie, fn, reg);  in cdns_pcie_ep_send_msix_irq()
512 	msix_tbl = epf->epf_bar[bir]->addr + tbl_offset;  in cdns_pcie_ep_send_msix_irq()
513 	msg_addr = msix_tbl[(interrupt_num - 1)].msg_addr;  in cdns_pcie_ep_send_msix_irq()
514 	msg_data = msix_tbl[(interrupt_num - 1)].msg_data;  in cdns_pcie_ep_send_msix_irq()
517 	if (ep->irq_pci_addr != (msg_addr & ~pci_addr_mask) ||  in cdns_pcie_ep_send_msix_irq()
518 	    ep->irq_pci_fn != fn) {  in cdns_pcie_ep_send_msix_irq()
520 		cdns_pcie_set_outbound_region(pcie, 0, fn, 0,  in cdns_pcie_ep_send_msix_irq()
522 					      ep->irq_phys_addr,  in cdns_pcie_ep_send_msix_irq()
525 		ep->irq_pci_addr = (msg_addr & ~pci_addr_mask);  in cdns_pcie_ep_send_msix_irq()
526 		ep->irq_pci_fn = fn;  in cdns_pcie_ep_send_msix_irq()
528 	writel(msg_data, ep->irq_cpu_addr + (msg_addr & pci_addr_mask));  in cdns_pcie_ep_send_msix_irq()
538 	struct cdns_pcie *pcie = &ep->pcie;  in cdns_pcie_ep_raise_irq()  local
539 	struct device *dev = pcie->dev;  in cdns_pcie_ep_raise_irq()
545 			return -EINVAL;  in cdns_pcie_ep_raise_irq()
559 	return -EINVAL;  in cdns_pcie_ep_raise_irq()
565 	struct cdns_pcie *pcie = &ep->pcie;  in cdns_pcie_ep_start()  local
566 	struct device *dev = pcie->dev;  in cdns_pcie_ep_start()
567 	int max_epfs = sizeof(epc->function_num_map) * 8;  in cdns_pcie_ep_start()
574 	cdns_pcie_writel(pcie, CDNS_PCIE_LM_EP_FUNC_CFG, epc->function_num_map);  in cdns_pcie_ep_start()
576 	if (ep->quirk_disable_flr) {  in cdns_pcie_ep_start()
578 			if (!(epc->function_num_map & BIT(epf)))  in cdns_pcie_ep_start()
581 			value = cdns_pcie_ep_fn_readl(pcie, epf,  in cdns_pcie_ep_start()
585 			cdns_pcie_ep_fn_writel(pcie, epf,  in cdns_pcie_ep_start()
591 	ret = cdns_pcie_start_link(pcie);  in cdns_pcie_ep_start()
642 	struct device *dev = ep->pcie.dev;  in cdns_pcie_ep_setup()
644 	struct device_node *np = dev->of_node;  in cdns_pcie_ep_setup()
645 	struct cdns_pcie *pcie = &ep->pcie;  in cdns_pcie_ep_setup()  local
652 	pcie->is_rc = false;  in cdns_pcie_ep_setup()
654 	pcie->reg_base = devm_platform_ioremap_resource_byname(pdev, "reg");  in cdns_pcie_ep_setup()
655 	if (IS_ERR(pcie->reg_base)) {  in cdns_pcie_ep_setup()
657 		return PTR_ERR(pcie->reg_base);  in cdns_pcie_ep_setup()
663 		return -EINVAL;  in cdns_pcie_ep_setup()
665 	pcie->mem_res = res;  in cdns_pcie_ep_setup()
667 	ep->max_regions = CDNS_PCIE_MAX_OB;  in cdns_pcie_ep_setup()
668 	of_property_read_u32(np, "cdns,max-outbound-regions", &ep->max_regions);  in cdns_pcie_ep_setup()
670 	ep->ob_addr = devm_kcalloc(dev,  in cdns_pcie_ep_setup()
671 				   ep->max_regions, sizeof(*ep->ob_addr),  in cdns_pcie_ep_setup()
673 	if (!ep->ob_addr)  in cdns_pcie_ep_setup()
674 		return -ENOMEM;  in cdns_pcie_ep_setup()
677 	cdns_pcie_writel(pcie, CDNS_PCIE_LM_EP_FUNC_CFG, BIT(0));  in cdns_pcie_ep_setup()
687 	if (of_property_read_u8(np, "max-functions", &epc->max_functions) < 0)  in cdns_pcie_ep_setup()
688 		epc->max_functions = 1;  in cdns_pcie_ep_setup()
690 	ep->epf = devm_kcalloc(dev, epc->max_functions, sizeof(*ep->epf),  in cdns_pcie_ep_setup()
692 	if (!ep->epf)  in cdns_pcie_ep_setup()
693 		return -ENOMEM;  in cdns_pcie_ep_setup()
695 	epc->max_vfs = devm_kcalloc(dev, epc->max_functions,  in cdns_pcie_ep_setup()
696 				    sizeof(*epc->max_vfs), GFP_KERNEL);  in cdns_pcie_ep_setup()
697 	if (!epc->max_vfs)  in cdns_pcie_ep_setup()
698 		return -ENOMEM;  in cdns_pcie_ep_setup()
700 	ret = of_property_read_u8_array(np, "max-virtual-functions",  in cdns_pcie_ep_setup()
701 					epc->max_vfs, epc->max_functions);  in cdns_pcie_ep_setup()
703 		for (i = 0; i < epc->max_functions; i++) {  in cdns_pcie_ep_setup()
704 			epf = &ep->epf[i];  in cdns_pcie_ep_setup()
705 			if (epc->max_vfs[i] == 0)  in cdns_pcie_ep_setup()
707 			epf->epf = devm_kcalloc(dev, epc->max_vfs[i],  in cdns_pcie_ep_setup()
708 						sizeof(*ep->epf), GFP_KERNEL);  in cdns_pcie_ep_setup()
709 			if (!epf->epf)  in cdns_pcie_ep_setup()
710 				return -ENOMEM;  in cdns_pcie_ep_setup()
714 	ret = pci_epc_mem_init(epc, pcie->mem_res->start,  in cdns_pcie_ep_setup()
715 			       resource_size(pcie->mem_res), PAGE_SIZE);  in cdns_pcie_ep_setup()
721 	ep->irq_cpu_addr = pci_epc_mem_alloc_addr(epc, &ep->irq_phys_addr,  in cdns_pcie_ep_setup()
723 	if (!ep->irq_cpu_addr) {  in cdns_pcie_ep_setup()
725 		ret = -ENOMEM;  in cdns_pcie_ep_setup()
728 	ep->irq_pci_addr = CDNS_PCIE_EP_IRQ_PCI_ADDR_NONE;  in cdns_pcie_ep_setup()
730 	set_bit(0, &ep->ob_region_map);  in cdns_pcie_ep_setup()
732 	if (ep->quirk_detect_quiet_flag)  in cdns_pcie_ep_setup()
733 		cdns_pcie_detect_quiet_min_delay_set(&ep->pcie);  in cdns_pcie_ep_setup()
735 	spin_lock_init(&ep->lock);  in cdns_pcie_ep_setup()