25 #include "pcie-iproc.h"
92  * struct iproc_pcie_ob_map - iProc PCIe outbound mapping controller-specific
139  * enum iproc_pcie_ib_map_type - iProc PCIe inbound mapping type
151  * struct iproc_pcie_ib_map - iProc PCIe inbound mapping controller-specific
230  * iProc PCIe host registers
307 /* iProc PCIe PAXB BCMA registers */
318 /* iProc PCIe PAXB registers */
334 /* iProc PCIe PAXB v2 registers */
365 /* iProc PCIe PAXC v1 registers */
374 /* iProc PCIe PAXC v2 registers */
401 	struct iproc_pcie *pcie = bus->sysdata;  in iproc_data()  local
402 	return pcie;  in iproc_data()
410 static inline u16 iproc_pcie_reg_offset(struct iproc_pcie *pcie,  in iproc_pcie_reg_offset()  argument
413 	return pcie->reg_offsets[reg];  in iproc_pcie_reg_offset()
416 static inline u32 iproc_pcie_read_reg(struct iproc_pcie *pcie,  in iproc_pcie_read_reg()  argument
419 	u16 offset = iproc_pcie_reg_offset(pcie, reg);  in iproc_pcie_read_reg()
424 	return readl(pcie->base + offset);  in iproc_pcie_read_reg()
427 static inline void iproc_pcie_write_reg(struct iproc_pcie *pcie,  in iproc_pcie_write_reg()  argument
430 	u16 offset = iproc_pcie_reg_offset(pcie, reg);  in iproc_pcie_write_reg()
435 	writel(val, pcie->base + offset);  in iproc_pcie_write_reg()
447 	struct iproc_pcie *pcie = iproc_data(bus);  in iproc_pcie_apb_err_disable()  local
450 	if (bus->number && pcie->has_apb_err_disable) {  in iproc_pcie_apb_err_disable()
451 		val = iproc_pcie_read_reg(pcie, IPROC_PCIE_APB_ERR_EN);  in iproc_pcie_apb_err_disable()
456 		iproc_pcie_write_reg(pcie, IPROC_PCIE_APB_ERR_EN, val);  in iproc_pcie_apb_err_disable()
460 static void __iomem *iproc_pcie_map_ep_cfg_reg(struct iproc_pcie *pcie,  in iproc_pcie_map_ep_cfg_reg()  argument
472 	iproc_pcie_write_reg(pcie, IPROC_PCIE_CFG_ADDR, val);  in iproc_pcie_map_ep_cfg_reg()
473 	offset = iproc_pcie_reg_offset(pcie, IPROC_PCIE_CFG_DATA);  in iproc_pcie_map_ep_cfg_reg()
478 	return (pcie->base + offset);  in iproc_pcie_map_ep_cfg_reg()
481 static unsigned int iproc_pcie_cfg_retry(struct iproc_pcie *pcie,  in iproc_pcie_cfg_retry()  argument
489 	 * As per PCIe spec r3.1, sec 2.3.2, CRS Software Visibility only  in iproc_pcie_cfg_retry()
512 		status = iproc_pcie_read_reg(pcie, IPROC_PCIE_CFG_RD_STATUS);  in iproc_pcie_cfg_retry()
526 static void iproc_pcie_fix_cap(struct iproc_pcie *pcie, int where, u32 *val)  in iproc_pcie_fix_cap()  argument
540 				pcie->fix_paxc_cap = true;  in iproc_pcie_fix_cap()
544 		if (pcie->fix_paxc_cap) {  in iproc_pcie_fix_cap()
545 			/* advertise PM, force next capability to PCIe */  in iproc_pcie_fix_cap()
552 		if (pcie->fix_paxc_cap) {  in iproc_pcie_fix_cap()
572 	struct iproc_pcie *pcie = iproc_data(bus);  in iproc_pcie_config_read()  local
582 			iproc_pcie_fix_cap(pcie, where, val);  in iproc_pcie_config_read()
587 	cfg_data_p = iproc_pcie_map_ep_cfg_reg(pcie, busno, devfn, where);  in iproc_pcie_config_read()
592 	data = iproc_pcie_cfg_retry(pcie, cfg_data_p);  in iproc_pcie_config_read()
611 	if (pcie->rej_unconfig_pf &&  in iproc_pcie_config_read()
624 static void __iomem *iproc_pcie_map_cfg_bus(struct iproc_pcie *pcie,  in iproc_pcie_map_cfg_bus()  argument
635 		iproc_pcie_write_reg(pcie, IPROC_PCIE_CFG_IND_ADDR,  in iproc_pcie_map_cfg_bus()
637 		offset = iproc_pcie_reg_offset(pcie, IPROC_PCIE_CFG_IND_DATA);  in iproc_pcie_map_cfg_bus()
641 			return (pcie->base + offset);  in iproc_pcie_map_cfg_bus()
644 	return iproc_pcie_map_ep_cfg_reg(pcie, busno, devfn, where);  in iproc_pcie_map_cfg_bus()
655 static int iproc_pci_raw_config_read32(struct iproc_pcie *pcie,  in iproc_pci_raw_config_read32()  argument
661 	addr = iproc_pcie_map_cfg_bus(pcie, 0, devfn, where & ~0x3);  in iproc_pci_raw_config_read32()
675 static int iproc_pci_raw_config_write32(struct iproc_pcie *pcie,  in iproc_pci_raw_config_write32()  argument
682 	addr = iproc_pcie_map_cfg_bus(pcie, 0, devfn, where & ~0x3);  in iproc_pci_raw_config_write32()
703 	struct iproc_pcie *pcie = iproc_data(bus);  in iproc_pcie_config_read32()  local
706 	if (pcie->iproc_cfg_read)  in iproc_pcie_config_read32()
733 static void iproc_pcie_perst_ctrl(struct iproc_pcie *pcie, bool assert)  in iproc_pcie_perst_ctrl()  argument
742 	if (pcie->ep_is_internal)  in iproc_pcie_perst_ctrl()
746 		val = iproc_pcie_read_reg(pcie, IPROC_PCIE_CLK_CTRL);  in iproc_pcie_perst_ctrl()
749 		iproc_pcie_write_reg(pcie, IPROC_PCIE_CLK_CTRL, val);  in iproc_pcie_perst_ctrl()
752 		val = iproc_pcie_read_reg(pcie, IPROC_PCIE_CLK_CTRL);  in iproc_pcie_perst_ctrl()
754 		iproc_pcie_write_reg(pcie, IPROC_PCIE_CLK_CTRL, val);  in iproc_pcie_perst_ctrl()
759 int iproc_pcie_shutdown(struct iproc_pcie *pcie)  in iproc_pcie_shutdown()  argument
761 	iproc_pcie_perst_ctrl(pcie, true);  in iproc_pcie_shutdown()
768 static int iproc_pcie_check_link(struct iproc_pcie *pcie)  in iproc_pcie_check_link()  argument
770 	struct device *dev = pcie->dev;  in iproc_pcie_check_link()
778 	if (pcie->ep_is_internal)  in iproc_pcie_check_link()
781 	val = iproc_pcie_read_reg(pcie, IPROC_PCIE_LINK_STATUS);  in iproc_pcie_check_link()
788 	iproc_pci_raw_config_read32(pcie, 0, PCI_HEADER_TYPE, 1, &hdr_type);  in iproc_pcie_check_link()
798 	iproc_pci_raw_config_read32(pcie, 0, PCI_BRIDGE_CTRL_REG_OFFSET,  in iproc_pcie_check_link()
802 	iproc_pci_raw_config_write32(pcie, 0, PCI_BRIDGE_CTRL_REG_OFFSET,  in iproc_pcie_check_link()
806 	iproc_pci_raw_config_read32(pcie, 0, IPROC_PCI_EXP_CAP + PCI_EXP_LNKSTA,  in iproc_pcie_check_link()
816 		iproc_pci_raw_config_read32(pcie, 0,  in iproc_pcie_check_link()
823 			iproc_pci_raw_config_write32(pcie, 0,  in iproc_pcie_check_link()
828 			iproc_pci_raw_config_read32(pcie, 0,  in iproc_pcie_check_link()
841 static void iproc_pcie_enable(struct iproc_pcie *pcie)  in iproc_pcie_enable()  argument
843 	iproc_pcie_write_reg(pcie, IPROC_PCIE_INTX_EN, SYS_RC_INTX_MASK);  in iproc_pcie_enable()
846 static inline bool iproc_pcie_ob_is_valid(struct iproc_pcie *pcie,  in iproc_pcie_ob_is_valid()  argument
851 	val = iproc_pcie_read_reg(pcie, MAP_REG(IPROC_PCIE_OARR0, window_idx));  in iproc_pcie_ob_is_valid()
856 static inline int iproc_pcie_ob_write(struct iproc_pcie *pcie, int window_idx,  in iproc_pcie_ob_write()  argument
859 	struct device *dev = pcie->dev;  in iproc_pcie_ob_write()
866 	oarr_offset = iproc_pcie_reg_offset(pcie, MAP_REG(IPROC_PCIE_OARR0,  in iproc_pcie_ob_write()
868 	omap_offset = iproc_pcie_reg_offset(pcie, MAP_REG(IPROC_PCIE_OMAP0,  in iproc_pcie_ob_write()
879 	       OARR_VALID, pcie->base + oarr_offset);  in iproc_pcie_ob_write()
880 	writel(upper_32_bits(axi_addr), pcie->base + oarr_offset + 4);  in iproc_pcie_ob_write()
883 	writel(lower_32_bits(pci_addr), pcie->base + omap_offset);  in iproc_pcie_ob_write()
884 	writel(upper_32_bits(pci_addr), pcie->base + omap_offset + 4);  in iproc_pcie_ob_write()
889 		readl(pcie->base + oarr_offset),  in iproc_pcie_ob_write()
890 		readl(pcie->base + oarr_offset + 4));  in iproc_pcie_ob_write()
892 		readl(pcie->base + omap_offset),  in iproc_pcie_ob_write()
893 		readl(pcie->base + omap_offset + 4));  in iproc_pcie_ob_write()
909 static int iproc_pcie_setup_ob(struct iproc_pcie *pcie, u64 axi_addr,  in iproc_pcie_setup_ob()  argument
912 	struct iproc_pcie_ob *ob = &pcie->ob;  in iproc_pcie_setup_ob()
913 	struct device *dev = pcie->dev;  in iproc_pcie_setup_ob()
924 	 * PCIe core before programming the OARR  in iproc_pcie_setup_ob()
931 			&pcie->ob_map[window_idx];  in iproc_pcie_setup_ob()
937 		if (iproc_pcie_ob_is_valid(pcie, window_idx))  in iproc_pcie_setup_ob()
982 			ret = iproc_pcie_ob_write(pcie, window_idx, size_idx,  in iproc_pcie_setup_ob()
1011 static int iproc_pcie_map_ranges(struct iproc_pcie *pcie,  in iproc_pcie_map_ranges()  argument
1014 	struct device *dev = pcie->dev;  in iproc_pcie_map_ranges()
1027 			ret = iproc_pcie_setup_ob(pcie, res->start,  in iproc_pcie_map_ranges()
1042 static inline bool iproc_pcie_ib_is_in_use(struct iproc_pcie *pcie,  in iproc_pcie_ib_is_in_use()  argument
1045 	const struct iproc_pcie_ib_map *ib_map = &pcie->ib_map[region_idx];  in iproc_pcie_ib_is_in_use()
1048 	val = iproc_pcie_read_reg(pcie, MAP_REG(IPROC_PCIE_IARR0, region_idx));  in iproc_pcie_ib_is_in_use()
1059 static int iproc_pcie_ib_write(struct iproc_pcie *pcie, int region_idx,  in iproc_pcie_ib_write()  argument
1063 	struct device *dev = pcie->dev;  in iproc_pcie_ib_write()
1064 	const struct iproc_pcie_ib_map *ib_map = &pcie->ib_map[region_idx];  in iproc_pcie_ib_write()
1069 	iarr_offset = iproc_pcie_reg_offset(pcie,  in iproc_pcie_ib_write()
1071 	imap_offset = iproc_pcie_reg_offset(pcie,  in iproc_pcie_ib_write()
1085 	       pcie->base + iarr_offset);  in iproc_pcie_ib_write()
1086 	writel(upper_32_bits(pci_addr), pcie->base + iarr_offset + 4);  in iproc_pcie_ib_write()
1089 		readl(pcie->base + iarr_offset),  in iproc_pcie_ib_write()
1090 		readl(pcie->base + iarr_offset + 4));  in iproc_pcie_ib_write()
1098 		val = readl(pcie->base + imap_offset);  in iproc_pcie_ib_write()
1100 		writel(val, pcie->base + imap_offset);  in iproc_pcie_ib_write()
1102 		       pcie->base + imap_offset + ib_map->imap_addr_offset);  in iproc_pcie_ib_write()
1105 			window_idx, readl(pcie->base + imap_offset),  in iproc_pcie_ib_write()
1106 			readl(pcie->base + imap_offset +  in iproc_pcie_ib_write()
1116 static int iproc_pcie_setup_ib(struct iproc_pcie *pcie,  in iproc_pcie_setup_ib()  argument
1120 	struct device *dev = pcie->dev;  in iproc_pcie_setup_ib()
1121 	struct iproc_pcie_ib *ib = &pcie->ib;  in iproc_pcie_setup_ib()
1131 			&pcie->ib_map[region_idx];  in iproc_pcie_setup_ib()
1137 		if (iproc_pcie_ib_is_in_use(pcie, region_idx) ||  in iproc_pcie_setup_ib()
1158 			ret = iproc_pcie_ib_write(pcie, region_idx, size_idx,  in iproc_pcie_setup_ib()
1178 static int iproc_pcie_map_dma_ranges(struct iproc_pcie *pcie)  in iproc_pcie_map_dma_ranges()  argument
1180 	struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);  in iproc_pcie_map_dma_ranges()
1186 		ret = iproc_pcie_setup_ib(pcie, entry, IPROC_PCIE_IB_MAP_MEM);  in iproc_pcie_map_dma_ranges()
1194 static void iproc_pcie_invalidate_mapping(struct iproc_pcie *pcie)  in iproc_pcie_invalidate_mapping()  argument
1196 	struct iproc_pcie_ib *ib = &pcie->ib;  in iproc_pcie_invalidate_mapping()
1197 	struct iproc_pcie_ob *ob = &pcie->ob;  in iproc_pcie_invalidate_mapping()
1200 	if (pcie->ep_is_internal)  in iproc_pcie_invalidate_mapping()
1203 	if (pcie->need_ob_cfg) {  in iproc_pcie_invalidate_mapping()
1206 			iproc_pcie_write_reg(pcie,  in iproc_pcie_invalidate_mapping()
1211 	if (pcie->need_ib_cfg) {  in iproc_pcie_invalidate_mapping()
1214 			iproc_pcie_write_reg(pcie,  in iproc_pcie_invalidate_mapping()
1220 static int iproce_pcie_get_msi(struct iproc_pcie *pcie,  in iproce_pcie_get_msi()  argument
1224 	struct device *dev = pcie->dev;  in iproce_pcie_get_msi()
1248 static int iproc_pcie_paxb_v2_msi_steer(struct iproc_pcie *pcie, u64 msi_addr)  in iproc_pcie_paxb_v2_msi_steer()  argument
1260 	ret = iproc_pcie_setup_ib(pcie, &entry, IPROC_PCIE_IB_MAP_IO);  in iproc_pcie_paxb_v2_msi_steer()
1264 static void iproc_pcie_paxc_v2_msi_steer(struct iproc_pcie *pcie, u64 msi_addr,  in iproc_pcie_paxc_v2_msi_steer()  argument
1274 		val = iproc_pcie_read_reg(pcie, IPROC_PCIE_MSI_EN_CFG);  in iproc_pcie_paxc_v2_msi_steer()
1276 		iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_EN_CFG, val);  in iproc_pcie_paxc_v2_msi_steer()
1286 	iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_BASE_ADDR,  in iproc_pcie_paxc_v2_msi_steer()
1290 	iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_WINDOW_SIZE, 0);  in iproc_pcie_paxc_v2_msi_steer()
1293 	val = iproc_pcie_read_reg(pcie, IPROC_PCIE_MSI_GIC_MODE);  in iproc_pcie_paxc_v2_msi_steer()
1295 	iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_GIC_MODE, val);  in iproc_pcie_paxc_v2_msi_steer()
1302 	iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_ADDR_HI,  in iproc_pcie_paxc_v2_msi_steer()
1304 	iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_ADDR_LO,  in iproc_pcie_paxc_v2_msi_steer()
1308 	val = iproc_pcie_read_reg(pcie, IPROC_PCIE_MSI_EN_CFG);  in iproc_pcie_paxc_v2_msi_steer()
1310 	iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_EN_CFG, val);  in iproc_pcie_paxc_v2_msi_steer()
1313 static int iproc_pcie_msi_steer(struct iproc_pcie *pcie,  in iproc_pcie_msi_steer()  argument
1316 	struct device *dev = pcie->dev;  in iproc_pcie_msi_steer()
1320 	ret = iproce_pcie_get_msi(pcie, msi_node, &msi_addr);  in iproc_pcie_msi_steer()
1326 	switch (pcie->type) {  in iproc_pcie_msi_steer()
1328 		ret = iproc_pcie_paxb_v2_msi_steer(pcie, msi_addr);  in iproc_pcie_msi_steer()
1333 		iproc_pcie_paxc_v2_msi_steer(pcie, msi_addr, true);  in iproc_pcie_msi_steer()
1342 static int iproc_pcie_msi_enable(struct iproc_pcie *pcie)  in iproc_pcie_msi_enable()  argument
1352 	msi_node = of_parse_phandle(pcie->dev->of_node, "msi-parent", 0);  in iproc_pcie_msi_enable()
1358 		msi_map = of_get_property(pcie->dev->of_node, "msi-map", &len);  in iproc_pcie_msi_enable()
1369 	 * Certain revisions of the iProc PCIe controller require additional  in iproc_pcie_msi_enable()
1373 	if (pcie->need_msi_steer) {  in iproc_pcie_msi_enable()
1374 		ret = iproc_pcie_msi_steer(pcie, msi_node);  in iproc_pcie_msi_enable()
1383 	ret = iproc_msi_init(pcie, msi_node);  in iproc_pcie_msi_enable()
1390 static void iproc_pcie_msi_disable(struct iproc_pcie *pcie)  in iproc_pcie_msi_disable()  argument
1392 	iproc_msi_exit(pcie);  in iproc_pcie_msi_disable()
1395 static int iproc_pcie_rev_init(struct iproc_pcie *pcie)  in iproc_pcie_rev_init()  argument
1397 	struct device *dev = pcie->dev;  in iproc_pcie_rev_init()
1401 	switch (pcie->type) {  in iproc_pcie_rev_init()
1407 		pcie->has_apb_err_disable = true;  in iproc_pcie_rev_init()
1408 		if (pcie->need_ob_cfg) {  in iproc_pcie_rev_init()
1409 			pcie->ob_map = paxb_ob_map;  in iproc_pcie_rev_init()
1410 			pcie->ob.nr_windows = ARRAY_SIZE(paxb_ob_map);  in iproc_pcie_rev_init()
1415 		pcie->iproc_cfg_read = true;  in iproc_pcie_rev_init()
1416 		pcie->has_apb_err_disable = true;  in iproc_pcie_rev_init()
1417 		if (pcie->need_ob_cfg) {  in iproc_pcie_rev_init()
1418 			pcie->ob_map = paxb_v2_ob_map;  in iproc_pcie_rev_init()
1419 			pcie->ob.nr_windows = ARRAY_SIZE(paxb_v2_ob_map);  in iproc_pcie_rev_init()
1421 		pcie->ib.nr_regions = ARRAY_SIZE(paxb_v2_ib_map);  in iproc_pcie_rev_init()
1422 		pcie->ib_map = paxb_v2_ib_map;  in iproc_pcie_rev_init()
1423 		pcie->need_msi_steer = true;  in iproc_pcie_rev_init()
1429 		pcie->ep_is_internal = true;  in iproc_pcie_rev_init()
1430 		pcie->iproc_cfg_read = true;  in iproc_pcie_rev_init()
1431 		pcie->rej_unconfig_pf = true;  in iproc_pcie_rev_init()
1435 		pcie->ep_is_internal = true;  in iproc_pcie_rev_init()
1436 		pcie->iproc_cfg_read = true;  in iproc_pcie_rev_init()
1437 		pcie->rej_unconfig_pf = true;  in iproc_pcie_rev_init()
1438 		pcie->need_msi_steer = true;  in iproc_pcie_rev_init()
1441 		dev_err(dev, "incompatible iProc PCIe interface\n");  in iproc_pcie_rev_init()
1445 	pcie->reg_offsets = devm_kcalloc(dev, IPROC_PCIE_MAX_NUM_REG,  in iproc_pcie_rev_init()
1446 					 sizeof(*pcie->reg_offsets),  in iproc_pcie_rev_init()
1448 	if (!pcie->reg_offsets)  in iproc_pcie_rev_init()
1452 	pcie->reg_offsets[0] = (pcie->type == IPROC_PCIE_PAXC_V2) ?  in iproc_pcie_rev_init()
1455 		pcie->reg_offsets[reg_idx] = regs[reg_idx] ?  in iproc_pcie_rev_init()
1461 int iproc_pcie_setup(struct iproc_pcie *pcie, struct list_head *res)  in iproc_pcie_setup()  argument
1466 	struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);  in iproc_pcie_setup()
1468 	dev = pcie->dev;  in iproc_pcie_setup()
1470 	ret = iproc_pcie_rev_init(pcie);  in iproc_pcie_setup()
1476 	ret = phy_init(pcie->phy);  in iproc_pcie_setup()
1478 		dev_err(dev, "unable to initialize PCIe PHY\n");  in iproc_pcie_setup()
1482 	ret = phy_power_on(pcie->phy);  in iproc_pcie_setup()
1484 		dev_err(dev, "unable to power on PCIe PHY\n");  in iproc_pcie_setup()
1488 	iproc_pcie_perst_ctrl(pcie, true);  in iproc_pcie_setup()
1489 	iproc_pcie_perst_ctrl(pcie, false);  in iproc_pcie_setup()
1491 	iproc_pcie_invalidate_mapping(pcie);  in iproc_pcie_setup()
1493 	if (pcie->need_ob_cfg) {  in iproc_pcie_setup()
1494 		ret = iproc_pcie_map_ranges(pcie, res);  in iproc_pcie_setup()
1501 	if (pcie->need_ib_cfg) {  in iproc_pcie_setup()
1502 		ret = iproc_pcie_map_dma_ranges(pcie);  in iproc_pcie_setup()
1507 	ret = iproc_pcie_check_link(pcie);  in iproc_pcie_setup()
1509 		dev_err(dev, "no PCIe EP device detected\n");  in iproc_pcie_setup()
1513 	iproc_pcie_enable(pcie);  in iproc_pcie_setup()
1516 		if (iproc_pcie_msi_enable(pcie))  in iproc_pcie_setup()
1520 	host->sysdata = pcie;  in iproc_pcie_setup()
1521 	host->map_irq = pcie->map_irq;  in iproc_pcie_setup()
1537 	phy_power_off(pcie->phy);  in iproc_pcie_setup()
1539 	phy_exit(pcie->phy);  in iproc_pcie_setup()
1544 int iproc_pcie_remove(struct iproc_pcie *pcie)  in iproc_pcie_remove()  argument
1546 	struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);  in iproc_pcie_remove()
1551 	iproc_pcie_msi_disable(pcie);  in iproc_pcie_remove()
1553 	phy_power_off(pcie->phy);  in iproc_pcie_remove()
1554 	phy_exit(pcie->phy);  in iproc_pcie_remove()
1566 	struct iproc_pcie *pcie = iproc_data(pdev->bus);  in quirk_paxc_disable_msi_parsing()  local
1569 		iproc_pcie_paxc_v2_msi_steer(pcie, 0, false);  in quirk_paxc_disable_msi_parsing()
1603 MODULE_DESCRIPTION("Broadcom iPROC PCIe common driver");