24 #include "pcie-iproc.h"
99  * iProc PCIe outbound mapping controller specific parameters
146  * iProc PCIe inbound mapping type
160  * iProc PCIe inbound mapping controller specific parameters
232  * iProc PCIe host registers
309 /* iProc PCIe PAXB BCMA registers */
320 /* iProc PCIe PAXB registers */
336 /* iProc PCIe PAXB v2 registers */
365 /* iProc PCIe PAXC v1 registers */
374 /* iProc PCIe PAXC v2 registers */
401 	struct iproc_pcie *pcie = bus->sysdata;  in iproc_data()  local
402 	return pcie;  in iproc_data()
410 static inline u16 iproc_pcie_reg_offset(struct iproc_pcie *pcie,  in iproc_pcie_reg_offset()  argument
413 	return pcie->reg_offsets[reg];  in iproc_pcie_reg_offset()
416 static inline u32 iproc_pcie_read_reg(struct iproc_pcie *pcie,  in iproc_pcie_read_reg()  argument
419 	u16 offset = iproc_pcie_reg_offset(pcie, reg);  in iproc_pcie_read_reg()
424 	return readl(pcie->base + offset);  in iproc_pcie_read_reg()
427 static inline void iproc_pcie_write_reg(struct iproc_pcie *pcie,  in iproc_pcie_write_reg()  argument
430 	u16 offset = iproc_pcie_reg_offset(pcie, reg);  in iproc_pcie_write_reg()
435 	writel(val, pcie->base + offset);  in iproc_pcie_write_reg()
447 	struct iproc_pcie *pcie = iproc_data(bus);  in iproc_pcie_apb_err_disable()  local
450 	if (bus->number && pcie->has_apb_err_disable) {  in iproc_pcie_apb_err_disable()
451 		val = iproc_pcie_read_reg(pcie, IPROC_PCIE_APB_ERR_EN);  in iproc_pcie_apb_err_disable()
456 		iproc_pcie_write_reg(pcie, IPROC_PCIE_APB_ERR_EN, val);  in iproc_pcie_apb_err_disable()
460 static void __iomem *iproc_pcie_map_ep_cfg_reg(struct iproc_pcie *pcie,  in iproc_pcie_map_ep_cfg_reg()  argument
476 	iproc_pcie_write_reg(pcie, IPROC_PCIE_CFG_ADDR, val);  in iproc_pcie_map_ep_cfg_reg()
477 	offset = iproc_pcie_reg_offset(pcie, IPROC_PCIE_CFG_DATA);  in iproc_pcie_map_ep_cfg_reg()
482 	return (pcie->base + offset);  in iproc_pcie_map_ep_cfg_reg()
485 static unsigned int iproc_pcie_cfg_retry(struct iproc_pcie *pcie,  in iproc_pcie_cfg_retry()  argument
493 	 * As per PCIe spec r3.1, sec 2.3.2, CRS Software Visibility only  in iproc_pcie_cfg_retry()
516 		status = iproc_pcie_read_reg(pcie, IPROC_PCIE_CFG_RD_STATUS);  in iproc_pcie_cfg_retry()
530 static void iproc_pcie_fix_cap(struct iproc_pcie *pcie, int where, u32 *val)  in iproc_pcie_fix_cap()  argument
544 				pcie->fix_paxc_cap = true;  in iproc_pcie_fix_cap()
548 		if (pcie->fix_paxc_cap) {  in iproc_pcie_fix_cap()
549 			/* advertise PM, force next capability to PCIe */  in iproc_pcie_fix_cap()
556 		if (pcie->fix_paxc_cap) {  in iproc_pcie_fix_cap()
576 	struct iproc_pcie *pcie = iproc_data(bus);  in iproc_pcie_config_read()  local
588 			iproc_pcie_fix_cap(pcie, where, val);  in iproc_pcie_config_read()
593 	cfg_data_p = iproc_pcie_map_ep_cfg_reg(pcie, busno, slot, fn, where);  in iproc_pcie_config_read()
598 	data = iproc_pcie_cfg_retry(pcie, cfg_data_p);  in iproc_pcie_config_read()
617 	if (pcie->rej_unconfig_pf &&  in iproc_pcie_config_read()
630 static void __iomem *iproc_pcie_map_cfg_bus(struct iproc_pcie *pcie,  in iproc_pcie_map_cfg_bus()  argument
643 		iproc_pcie_write_reg(pcie, IPROC_PCIE_CFG_IND_ADDR,  in iproc_pcie_map_cfg_bus()
645 		offset = iproc_pcie_reg_offset(pcie, IPROC_PCIE_CFG_IND_DATA);  in iproc_pcie_map_cfg_bus()
649 			return (pcie->base + offset);  in iproc_pcie_map_cfg_bus()
652 	return iproc_pcie_map_ep_cfg_reg(pcie, busno, slot, fn, where);  in iproc_pcie_map_cfg_bus()
663 static int iproc_pci_raw_config_read32(struct iproc_pcie *pcie,  in iproc_pci_raw_config_read32()  argument
669 	addr = iproc_pcie_map_cfg_bus(pcie, 0, devfn, where & ~0x3);  in iproc_pci_raw_config_read32()
683 static int iproc_pci_raw_config_write32(struct iproc_pcie *pcie,  in iproc_pci_raw_config_write32()  argument
690 	addr = iproc_pcie_map_cfg_bus(pcie, 0, devfn, where & ~0x3);  in iproc_pci_raw_config_write32()
711 	struct iproc_pcie *pcie = iproc_data(bus);  in iproc_pcie_config_read32()  local
714 	if (pcie->iproc_cfg_read)  in iproc_pcie_config_read32()
741 static void iproc_pcie_perst_ctrl(struct iproc_pcie *pcie, bool assert)  in iproc_pcie_perst_ctrl()  argument
750 	if (pcie->ep_is_internal)  in iproc_pcie_perst_ctrl()
754 		val = iproc_pcie_read_reg(pcie, IPROC_PCIE_CLK_CTRL);  in iproc_pcie_perst_ctrl()
757 		iproc_pcie_write_reg(pcie, IPROC_PCIE_CLK_CTRL, val);  in iproc_pcie_perst_ctrl()
760 		val = iproc_pcie_read_reg(pcie, IPROC_PCIE_CLK_CTRL);  in iproc_pcie_perst_ctrl()
762 		iproc_pcie_write_reg(pcie, IPROC_PCIE_CLK_CTRL, val);  in iproc_pcie_perst_ctrl()
767 int iproc_pcie_shutdown(struct iproc_pcie *pcie)  in iproc_pcie_shutdown()  argument
769 	iproc_pcie_perst_ctrl(pcie, true);  in iproc_pcie_shutdown()
776 static int iproc_pcie_check_link(struct iproc_pcie *pcie)  in iproc_pcie_check_link()  argument
778 	struct device *dev = pcie->dev;  in iproc_pcie_check_link()
786 	if (pcie->ep_is_internal)  in iproc_pcie_check_link()
789 	val = iproc_pcie_read_reg(pcie, IPROC_PCIE_LINK_STATUS);  in iproc_pcie_check_link()
796 	iproc_pci_raw_config_read32(pcie, 0, PCI_HEADER_TYPE, 1, &hdr_type);  in iproc_pcie_check_link()
806 	iproc_pci_raw_config_read32(pcie, 0, PCI_BRIDGE_CTRL_REG_OFFSET,  in iproc_pcie_check_link()
810 	iproc_pci_raw_config_write32(pcie, 0, PCI_BRIDGE_CTRL_REG_OFFSET,  in iproc_pcie_check_link()
814 	iproc_pci_raw_config_read32(pcie, 0, IPROC_PCI_EXP_CAP + PCI_EXP_LNKSTA,  in iproc_pcie_check_link()
824 		iproc_pci_raw_config_read32(pcie, 0,  in iproc_pcie_check_link()
831 			iproc_pci_raw_config_write32(pcie, 0,  in iproc_pcie_check_link()
836 			iproc_pci_raw_config_read32(pcie, 0,  in iproc_pcie_check_link()
849 static void iproc_pcie_enable(struct iproc_pcie *pcie)  in iproc_pcie_enable()  argument
851 	iproc_pcie_write_reg(pcie, IPROC_PCIE_INTX_EN, SYS_RC_INTX_MASK);  in iproc_pcie_enable()
854 static inline bool iproc_pcie_ob_is_valid(struct iproc_pcie *pcie,  in iproc_pcie_ob_is_valid()  argument
859 	val = iproc_pcie_read_reg(pcie, MAP_REG(IPROC_PCIE_OARR0, window_idx));  in iproc_pcie_ob_is_valid()
864 static inline int iproc_pcie_ob_write(struct iproc_pcie *pcie, int window_idx,  in iproc_pcie_ob_write()  argument
867 	struct device *dev = pcie->dev;  in iproc_pcie_ob_write()
874 	oarr_offset = iproc_pcie_reg_offset(pcie, MAP_REG(IPROC_PCIE_OARR0,  in iproc_pcie_ob_write()
876 	omap_offset = iproc_pcie_reg_offset(pcie, MAP_REG(IPROC_PCIE_OMAP0,  in iproc_pcie_ob_write()
887 	       OARR_VALID, pcie->base + oarr_offset);  in iproc_pcie_ob_write()
888 	writel(upper_32_bits(axi_addr), pcie->base + oarr_offset + 4);  in iproc_pcie_ob_write()
891 	writel(lower_32_bits(pci_addr), pcie->base + omap_offset);  in iproc_pcie_ob_write()
892 	writel(upper_32_bits(pci_addr), pcie->base + omap_offset + 4);  in iproc_pcie_ob_write()
897 		readl(pcie->base + oarr_offset),  in iproc_pcie_ob_write()
898 		readl(pcie->base + oarr_offset + 4));  in iproc_pcie_ob_write()
900 		readl(pcie->base + omap_offset),  in iproc_pcie_ob_write()
901 		readl(pcie->base + omap_offset + 4));  in iproc_pcie_ob_write()
917 static int iproc_pcie_setup_ob(struct iproc_pcie *pcie, u64 axi_addr,  in iproc_pcie_setup_ob()  argument
920 	struct iproc_pcie_ob *ob = &pcie->ob;  in iproc_pcie_setup_ob()
921 	struct device *dev = pcie->dev;  in iproc_pcie_setup_ob()
932 	 * PCIe core before programming the OARR  in iproc_pcie_setup_ob()
939 			&pcie->ob_map[window_idx];  in iproc_pcie_setup_ob()
945 		if (iproc_pcie_ob_is_valid(pcie, window_idx))  in iproc_pcie_setup_ob()
990 			ret = iproc_pcie_ob_write(pcie, window_idx, size_idx,  in iproc_pcie_setup_ob()
1019 static int iproc_pcie_map_ranges(struct iproc_pcie *pcie,  in iproc_pcie_map_ranges()  argument
1022 	struct device *dev = pcie->dev;  in iproc_pcie_map_ranges()
1035 			ret = iproc_pcie_setup_ob(pcie, res->start,  in iproc_pcie_map_ranges()
1050 static inline bool iproc_pcie_ib_is_in_use(struct iproc_pcie *pcie,  in iproc_pcie_ib_is_in_use()  argument
1053 	const struct iproc_pcie_ib_map *ib_map = &pcie->ib_map[region_idx];  in iproc_pcie_ib_is_in_use()
1056 	val = iproc_pcie_read_reg(pcie, MAP_REG(IPROC_PCIE_IARR0, region_idx));  in iproc_pcie_ib_is_in_use()
1067 static int iproc_pcie_ib_write(struct iproc_pcie *pcie, int region_idx,  in iproc_pcie_ib_write()  argument
1071 	struct device *dev = pcie->dev;  in iproc_pcie_ib_write()
1072 	const struct iproc_pcie_ib_map *ib_map = &pcie->ib_map[region_idx];  in iproc_pcie_ib_write()
1077 	iarr_offset = iproc_pcie_reg_offset(pcie,  in iproc_pcie_ib_write()
1079 	imap_offset = iproc_pcie_reg_offset(pcie,  in iproc_pcie_ib_write()
1093 	       pcie->base + iarr_offset);  in iproc_pcie_ib_write()
1094 	writel(upper_32_bits(pci_addr), pcie->base + iarr_offset + 4);  in iproc_pcie_ib_write()
1097 		readl(pcie->base + iarr_offset),  in iproc_pcie_ib_write()
1098 		readl(pcie->base + iarr_offset + 4));  in iproc_pcie_ib_write()
1106 		val = readl(pcie->base + imap_offset);  in iproc_pcie_ib_write()
1108 		writel(val, pcie->base + imap_offset);  in iproc_pcie_ib_write()
1110 		       pcie->base + imap_offset + ib_map->imap_addr_offset);  in iproc_pcie_ib_write()
1113 			window_idx, readl(pcie->base + imap_offset),  in iproc_pcie_ib_write()
1114 			readl(pcie->base + imap_offset +  in iproc_pcie_ib_write()
1124 static int iproc_pcie_setup_ib(struct iproc_pcie *pcie,  in iproc_pcie_setup_ib()  argument
1128 	struct device *dev = pcie->dev;  in iproc_pcie_setup_ib()
1129 	struct iproc_pcie_ib *ib = &pcie->ib;  in iproc_pcie_setup_ib()
1139 			&pcie->ib_map[region_idx];  in iproc_pcie_setup_ib()
1145 		if (iproc_pcie_ib_is_in_use(pcie, region_idx) ||  in iproc_pcie_setup_ib()
1166 			ret = iproc_pcie_ib_write(pcie, region_idx, size_idx,  in iproc_pcie_setup_ib()
1186 static int iproc_pcie_map_dma_ranges(struct iproc_pcie *pcie)  in iproc_pcie_map_dma_ranges()  argument
1188 	struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);  in iproc_pcie_map_dma_ranges()
1194 		ret = iproc_pcie_setup_ib(pcie, entry, IPROC_PCIE_IB_MAP_MEM);  in iproc_pcie_map_dma_ranges()
1202 static void iproc_pcie_invalidate_mapping(struct iproc_pcie *pcie)  in iproc_pcie_invalidate_mapping()  argument
1204 	struct iproc_pcie_ib *ib = &pcie->ib;  in iproc_pcie_invalidate_mapping()
1205 	struct iproc_pcie_ob *ob = &pcie->ob;  in iproc_pcie_invalidate_mapping()
1208 	if (pcie->ep_is_internal)  in iproc_pcie_invalidate_mapping()
1211 	if (pcie->need_ob_cfg) {  in iproc_pcie_invalidate_mapping()
1214 			iproc_pcie_write_reg(pcie,  in iproc_pcie_invalidate_mapping()
1219 	if (pcie->need_ib_cfg) {  in iproc_pcie_invalidate_mapping()
1222 			iproc_pcie_write_reg(pcie,  in iproc_pcie_invalidate_mapping()
1228 static int iproce_pcie_get_msi(struct iproc_pcie *pcie,  in iproce_pcie_get_msi()  argument
1232 	struct device *dev = pcie->dev;  in iproce_pcie_get_msi()
1256 static int iproc_pcie_paxb_v2_msi_steer(struct iproc_pcie *pcie, u64 msi_addr)  in iproc_pcie_paxb_v2_msi_steer()  argument
1268 	ret = iproc_pcie_setup_ib(pcie, &entry, IPROC_PCIE_IB_MAP_IO);  in iproc_pcie_paxb_v2_msi_steer()
1272 static void iproc_pcie_paxc_v2_msi_steer(struct iproc_pcie *pcie, u64 msi_addr,  in iproc_pcie_paxc_v2_msi_steer()  argument
1282 		val = iproc_pcie_read_reg(pcie, IPROC_PCIE_MSI_EN_CFG);  in iproc_pcie_paxc_v2_msi_steer()
1284 		iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_EN_CFG, val);  in iproc_pcie_paxc_v2_msi_steer()
1294 	iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_BASE_ADDR,  in iproc_pcie_paxc_v2_msi_steer()
1298 	iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_WINDOW_SIZE, 0);  in iproc_pcie_paxc_v2_msi_steer()
1301 	val = iproc_pcie_read_reg(pcie, IPROC_PCIE_MSI_GIC_MODE);  in iproc_pcie_paxc_v2_msi_steer()
1303 	iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_GIC_MODE, val);  in iproc_pcie_paxc_v2_msi_steer()
1310 	iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_ADDR_HI,  in iproc_pcie_paxc_v2_msi_steer()
1312 	iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_ADDR_LO,  in iproc_pcie_paxc_v2_msi_steer()
1316 	val = iproc_pcie_read_reg(pcie, IPROC_PCIE_MSI_EN_CFG);  in iproc_pcie_paxc_v2_msi_steer()
1318 	iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_EN_CFG, val);  in iproc_pcie_paxc_v2_msi_steer()
1321 static int iproc_pcie_msi_steer(struct iproc_pcie *pcie,  in iproc_pcie_msi_steer()  argument
1324 	struct device *dev = pcie->dev;  in iproc_pcie_msi_steer()
1328 	ret = iproce_pcie_get_msi(pcie, msi_node, &msi_addr);  in iproc_pcie_msi_steer()
1334 	switch (pcie->type) {  in iproc_pcie_msi_steer()
1336 		ret = iproc_pcie_paxb_v2_msi_steer(pcie, msi_addr);  in iproc_pcie_msi_steer()
1341 		iproc_pcie_paxc_v2_msi_steer(pcie, msi_addr, true);  in iproc_pcie_msi_steer()
1350 static int iproc_pcie_msi_enable(struct iproc_pcie *pcie)  in iproc_pcie_msi_enable()  argument
1360 	msi_node = of_parse_phandle(pcie->dev->of_node, "msi-parent", 0);  in iproc_pcie_msi_enable()
1366 		msi_map = of_get_property(pcie->dev->of_node, "msi-map", &len);  in iproc_pcie_msi_enable()
1377 	 * Certain revisions of the iProc PCIe controller require additional  in iproc_pcie_msi_enable()
1381 	if (pcie->need_msi_steer) {  in iproc_pcie_msi_enable()
1382 		ret = iproc_pcie_msi_steer(pcie, msi_node);  in iproc_pcie_msi_enable()
1391 	ret = iproc_msi_init(pcie, msi_node);  in iproc_pcie_msi_enable()
1398 static void iproc_pcie_msi_disable(struct iproc_pcie *pcie)  in iproc_pcie_msi_disable()  argument
1400 	iproc_msi_exit(pcie);  in iproc_pcie_msi_disable()
1403 static int iproc_pcie_rev_init(struct iproc_pcie *pcie)  in iproc_pcie_rev_init()  argument
1405 	struct device *dev = pcie->dev;  in iproc_pcie_rev_init()
1409 	switch (pcie->type) {  in iproc_pcie_rev_init()
1415 		pcie->has_apb_err_disable = true;  in iproc_pcie_rev_init()
1416 		if (pcie->need_ob_cfg) {  in iproc_pcie_rev_init()
1417 			pcie->ob_map = paxb_ob_map;  in iproc_pcie_rev_init()
1418 			pcie->ob.nr_windows = ARRAY_SIZE(paxb_ob_map);  in iproc_pcie_rev_init()
1423 		pcie->iproc_cfg_read = true;  in iproc_pcie_rev_init()
1424 		pcie->has_apb_err_disable = true;  in iproc_pcie_rev_init()
1425 		if (pcie->need_ob_cfg) {  in iproc_pcie_rev_init()
1426 			pcie->ob_map = paxb_v2_ob_map;  in iproc_pcie_rev_init()
1427 			pcie->ob.nr_windows = ARRAY_SIZE(paxb_v2_ob_map);  in iproc_pcie_rev_init()
1429 		pcie->ib.nr_regions = ARRAY_SIZE(paxb_v2_ib_map);  in iproc_pcie_rev_init()
1430 		pcie->ib_map = paxb_v2_ib_map;  in iproc_pcie_rev_init()
1431 		pcie->need_msi_steer = true;  in iproc_pcie_rev_init()
1437 		pcie->ep_is_internal = true;  in iproc_pcie_rev_init()
1438 		pcie->iproc_cfg_read = true;  in iproc_pcie_rev_init()
1439 		pcie->rej_unconfig_pf = true;  in iproc_pcie_rev_init()
1443 		pcie->ep_is_internal = true;  in iproc_pcie_rev_init()
1444 		pcie->iproc_cfg_read = true;  in iproc_pcie_rev_init()
1445 		pcie->rej_unconfig_pf = true;  in iproc_pcie_rev_init()
1446 		pcie->need_msi_steer = true;  in iproc_pcie_rev_init()
1449 		dev_err(dev, "incompatible iProc PCIe interface\n");  in iproc_pcie_rev_init()
1453 	pcie->reg_offsets = devm_kcalloc(dev, IPROC_PCIE_MAX_NUM_REG,  in iproc_pcie_rev_init()
1454 					 sizeof(*pcie->reg_offsets),  in iproc_pcie_rev_init()
1456 	if (!pcie->reg_offsets)  in iproc_pcie_rev_init()
1460 	pcie->reg_offsets[0] = (pcie->type == IPROC_PCIE_PAXC_V2) ?  in iproc_pcie_rev_init()
1463 		pcie->reg_offsets[reg_idx] = regs[reg_idx] ?  in iproc_pcie_rev_init()
1469 int iproc_pcie_setup(struct iproc_pcie *pcie, struct list_head *res)  in iproc_pcie_setup()  argument
1473 	struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);  in iproc_pcie_setup()
1475 	dev = pcie->dev;  in iproc_pcie_setup()
1477 	ret = iproc_pcie_rev_init(pcie);  in iproc_pcie_setup()
1483 	ret = phy_init(pcie->phy);  in iproc_pcie_setup()
1485 		dev_err(dev, "unable to initialize PCIe PHY\n");  in iproc_pcie_setup()
1489 	ret = phy_power_on(pcie->phy);  in iproc_pcie_setup()
1491 		dev_err(dev, "unable to power on PCIe PHY\n");  in iproc_pcie_setup()
1495 	iproc_pcie_perst_ctrl(pcie, true);  in iproc_pcie_setup()
1496 	iproc_pcie_perst_ctrl(pcie, false);  in iproc_pcie_setup()
1498 	iproc_pcie_invalidate_mapping(pcie);  in iproc_pcie_setup()
1500 	if (pcie->need_ob_cfg) {  in iproc_pcie_setup()
1501 		ret = iproc_pcie_map_ranges(pcie, res);  in iproc_pcie_setup()
1508 	if (pcie->need_ib_cfg) {  in iproc_pcie_setup()
1509 		ret = iproc_pcie_map_dma_ranges(pcie);  in iproc_pcie_setup()
1514 	ret = iproc_pcie_check_link(pcie);  in iproc_pcie_setup()
1516 		dev_err(dev, "no PCIe EP device detected\n");  in iproc_pcie_setup()
1520 	iproc_pcie_enable(pcie);  in iproc_pcie_setup()
1523 		if (iproc_pcie_msi_enable(pcie))  in iproc_pcie_setup()
1527 	host->sysdata = pcie;  in iproc_pcie_setup()
1528 	host->map_irq = pcie->map_irq;  in iproc_pcie_setup()
1539 	phy_power_off(pcie->phy);  in iproc_pcie_setup()
1541 	phy_exit(pcie->phy);  in iproc_pcie_setup()
1546 int iproc_pcie_remove(struct iproc_pcie *pcie)  in iproc_pcie_remove()  argument
1548 	struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);  in iproc_pcie_remove()
1553 	iproc_pcie_msi_disable(pcie);  in iproc_pcie_remove()
1555 	phy_power_off(pcie->phy);  in iproc_pcie_remove()
1556 	phy_exit(pcie->phy);  in iproc_pcie_remove()
1568 	struct iproc_pcie *pcie = iproc_data(pdev->bus);  in quirk_paxc_disable_msi_parsing()  local
1571 		iproc_pcie_paxc_v2_msi_steer(pcie, 0, false);  in quirk_paxc_disable_msi_parsing()
1605 MODULE_DESCRIPTION("Broadcom iPROC PCIe common driver");