3  * MediaTek PCIe host controller driver.
116  * struct mtk_gen3_pcie - PCIe port information
117  * @dev: pointer to PCIe device
123  * @clks: PCIe clocks
124  * @num_clks: PCIe clocks count for this port
125  * @irq: PCIe controller interrupt number
199 	struct mtk_gen3_pcie *pcie = bus->sysdata;  in mtk_pcie_config_tlp_header()  local
208 	writel_relaxed(val, pcie->base + PCIE_CFGNUM_REG);  in mtk_pcie_config_tlp_header()
214 	struct mtk_gen3_pcie *pcie = bus->sysdata;  in mtk_pcie_map_bus()  local
216 	return pcie->base + PCIE_CFG_OFFSET_ADDR + where;  in mtk_pcie_map_bus()
244 static int mtk_pcie_set_trans_table(struct mtk_gen3_pcie *pcie,  in mtk_pcie_set_trans_table()  argument
254 		dev_err(pcie->dev, "not enough translate table for addr: %#llx, limited to [%d]\n",  in mtk_pcie_set_trans_table()
259 	table = pcie->base + PCIE_TRANS_TABLE_BASE_REG +  in mtk_pcie_set_trans_table()
281 static void mtk_pcie_enable_msi(struct mtk_gen3_pcie *pcie)  in mtk_pcie_enable_msi()  argument
287 		struct mtk_msi_set *msi_set = &pcie->msi_sets[i];  in mtk_pcie_enable_msi()
289 		msi_set->base = pcie->base + PCIE_MSI_SET_BASE_REG +  in mtk_pcie_enable_msi()
291 		msi_set->msg_addr = pcie->reg_base + PCIE_MSI_SET_BASE_REG +  in mtk_pcie_enable_msi()
297 			       pcie->base + PCIE_MSI_SET_ADDR_HI_BASE +  in mtk_pcie_enable_msi()
301 	val = readl_relaxed(pcie->base + PCIE_MSI_SET_ENABLE_REG);  in mtk_pcie_enable_msi()
303 	writel_relaxed(val, pcie->base + PCIE_MSI_SET_ENABLE_REG);  in mtk_pcie_enable_msi()
305 	val = readl_relaxed(pcie->base + PCIE_INT_ENABLE_REG);  in mtk_pcie_enable_msi()
307 	writel_relaxed(val, pcie->base + PCIE_INT_ENABLE_REG);  in mtk_pcie_enable_msi()
310 static int mtk_pcie_startup_port(struct mtk_gen3_pcie *pcie)  in mtk_pcie_startup_port()  argument
313 	struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);  in mtk_pcie_startup_port()
319 	val = readl_relaxed(pcie->base + PCIE_SETTING_REG);  in mtk_pcie_startup_port()
321 	writel_relaxed(val, pcie->base + PCIE_SETTING_REG);  in mtk_pcie_startup_port()
324 	val = readl_relaxed(pcie->base + PCIE_PCI_IDS_1);  in mtk_pcie_startup_port()
327 	writel_relaxed(val, pcie->base + PCIE_PCI_IDS_1);  in mtk_pcie_startup_port()
330 	val = readl_relaxed(pcie->base + PCIE_INT_ENABLE_REG);  in mtk_pcie_startup_port()
332 	writel_relaxed(val, pcie->base + PCIE_INT_ENABLE_REG);  in mtk_pcie_startup_port()
335 	val = readl_relaxed(pcie->base + PCIE_MISC_CTRL_REG);  in mtk_pcie_startup_port()
337 	writel_relaxed(val, pcie->base + PCIE_MISC_CTRL_REG);  in mtk_pcie_startup_port()
340 	val = readl_relaxed(pcie->base + PCIE_RST_CTRL_REG);  in mtk_pcie_startup_port()
342 	writel_relaxed(val, pcie->base + PCIE_RST_CTRL_REG);  in mtk_pcie_startup_port()
345 	 * Described in PCIe CEM specification sections 2.2 (PERST# Signal)  in mtk_pcie_startup_port()
354 	writel_relaxed(val, pcie->base + PCIE_RST_CTRL_REG);  in mtk_pcie_startup_port()
357 	err = readl_poll_timeout(pcie->base + PCIE_LINK_STATUS_REG, val,  in mtk_pcie_startup_port()
364 		val = readl_relaxed(pcie->base + PCIE_LTSSM_STATUS_REG);  in mtk_pcie_startup_port()
368 		dev_err(pcie->dev,  in mtk_pcie_startup_port()
369 			"PCIe link down, current LTSSM state: %s (%#x)\n",  in mtk_pcie_startup_port()
374 	mtk_pcie_enable_msi(pcie);  in mtk_pcie_startup_port()
376 	/* Set PCIe translation windows */  in mtk_pcie_startup_port()
397 		err = mtk_pcie_set_trans_table(pcie, cpu_addr, pci_addr, size,  in mtk_pcie_startup_port()
402 		dev_dbg(pcie->dev, "set %s trans window[%d]: cpu_addr = %#llx, pci_addr = %#llx, size = %#llx\n",  in mtk_pcie_startup_port()
446 	struct mtk_gen3_pcie *pcie = data->domain->host_data;  in mtk_compose_msi_msg()  local
454 	dev_dbg(pcie->dev, "msi#%#lx address_hi %#x address_lo %#x data %d\n",  in mtk_compose_msi_msg()
471 	struct mtk_gen3_pcie *pcie = data->domain->host_data;  in mtk_msi_bottom_irq_mask()  local
477 	raw_spin_lock_irqsave(&pcie->irq_lock, flags);  in mtk_msi_bottom_irq_mask()
481 	raw_spin_unlock_irqrestore(&pcie->irq_lock, flags);  in mtk_msi_bottom_irq_mask()
487 	struct mtk_gen3_pcie *pcie = data->domain->host_data;  in mtk_msi_bottom_irq_unmask()  local
493 	raw_spin_lock_irqsave(&pcie->irq_lock, flags);  in mtk_msi_bottom_irq_unmask()
497 	raw_spin_unlock_irqrestore(&pcie->irq_lock, flags);  in mtk_msi_bottom_irq_unmask()
513 	struct mtk_gen3_pcie *pcie = domain->host_data;  in mtk_msi_bottom_domain_alloc()  local
517 	mutex_lock(&pcie->lock);  in mtk_msi_bottom_domain_alloc()
519 	hwirq = bitmap_find_free_region(pcie->msi_irq_in_use, PCIE_MSI_IRQS_NUM,  in mtk_msi_bottom_domain_alloc()
522 	mutex_unlock(&pcie->lock);  in mtk_msi_bottom_domain_alloc()
528 	msi_set = &pcie->msi_sets[set_idx];  in mtk_msi_bottom_domain_alloc()
541 	struct mtk_gen3_pcie *pcie = domain->host_data;  in mtk_msi_bottom_domain_free()  local
544 	mutex_lock(&pcie->lock);  in mtk_msi_bottom_domain_free()
546 	bitmap_release_region(pcie->msi_irq_in_use, data->hwirq,  in mtk_msi_bottom_domain_free()
549 	mutex_unlock(&pcie->lock);  in mtk_msi_bottom_domain_free()
561 	struct mtk_gen3_pcie *pcie = irq_data_get_irq_chip_data(data);  in mtk_intx_mask()  local
565 	raw_spin_lock_irqsave(&pcie->irq_lock, flags);  in mtk_intx_mask()
566 	val = readl_relaxed(pcie->base + PCIE_INT_ENABLE_REG);  in mtk_intx_mask()
568 	writel_relaxed(val, pcie->base + PCIE_INT_ENABLE_REG);  in mtk_intx_mask()
569 	raw_spin_unlock_irqrestore(&pcie->irq_lock, flags);  in mtk_intx_mask()
574 	struct mtk_gen3_pcie *pcie = irq_data_get_irq_chip_data(data);  in mtk_intx_unmask()  local
578 	raw_spin_lock_irqsave(&pcie->irq_lock, flags);  in mtk_intx_unmask()
579 	val = readl_relaxed(pcie->base + PCIE_INT_ENABLE_REG);  in mtk_intx_unmask()
581 	writel_relaxed(val, pcie->base + PCIE_INT_ENABLE_REG);  in mtk_intx_unmask()
582 	raw_spin_unlock_irqrestore(&pcie->irq_lock, flags);  in mtk_intx_unmask()
595 	struct mtk_gen3_pcie *pcie = irq_data_get_irq_chip_data(data);  in mtk_intx_eoi()  local
599 	writel_relaxed(BIT(hwirq), pcie->base + PCIE_INT_STATUS_REG);  in mtk_intx_eoi()
623 static int mtk_pcie_init_irq_domains(struct mtk_gen3_pcie *pcie)  in mtk_pcie_init_irq_domains()  argument
625 	struct device *dev = pcie->dev;  in mtk_pcie_init_irq_domains()
629 	raw_spin_lock_init(&pcie->irq_lock);  in mtk_pcie_init_irq_domains()
638 	pcie->intx_domain = irq_domain_add_linear(intc_node, PCI_NUM_INTX,  in mtk_pcie_init_irq_domains()
639 						  &intx_domain_ops, pcie);  in mtk_pcie_init_irq_domains()
640 	if (!pcie->intx_domain) {  in mtk_pcie_init_irq_domains()
647 	mutex_init(&pcie->lock);  in mtk_pcie_init_irq_domains()
649 	pcie->msi_bottom_domain = irq_domain_add_linear(node, PCIE_MSI_IRQS_NUM,  in mtk_pcie_init_irq_domains()
650 				  &mtk_msi_bottom_domain_ops, pcie);  in mtk_pcie_init_irq_domains()
651 	if (!pcie->msi_bottom_domain) {  in mtk_pcie_init_irq_domains()
657 	pcie->msi_domain = pci_msi_create_irq_domain(dev->fwnode,  in mtk_pcie_init_irq_domains()
659 						     pcie->msi_bottom_domain);  in mtk_pcie_init_irq_domains()
660 	if (!pcie->msi_domain) {  in mtk_pcie_init_irq_domains()
670 	irq_domain_remove(pcie->msi_bottom_domain);  in mtk_pcie_init_irq_domains()
672 	irq_domain_remove(pcie->intx_domain);  in mtk_pcie_init_irq_domains()
678 static void mtk_pcie_irq_teardown(struct mtk_gen3_pcie *pcie)  in mtk_pcie_irq_teardown()  argument
680 	irq_set_chained_handler_and_data(pcie->irq, NULL, NULL);  in mtk_pcie_irq_teardown()
682 	if (pcie->intx_domain)  in mtk_pcie_irq_teardown()
683 		irq_domain_remove(pcie->intx_domain);  in mtk_pcie_irq_teardown()
685 	if (pcie->msi_domain)  in mtk_pcie_irq_teardown()
686 		irq_domain_remove(pcie->msi_domain);  in mtk_pcie_irq_teardown()
688 	if (pcie->msi_bottom_domain)  in mtk_pcie_irq_teardown()
689 		irq_domain_remove(pcie->msi_bottom_domain);  in mtk_pcie_irq_teardown()
691 	irq_dispose_mapping(pcie->irq);  in mtk_pcie_irq_teardown()
694 static void mtk_pcie_msi_handler(struct mtk_gen3_pcie *pcie, int set_idx)  in mtk_pcie_msi_handler()  argument
696 	struct mtk_msi_set *msi_set = &pcie->msi_sets[set_idx];  in mtk_pcie_msi_handler()
711 			generic_handle_domain_irq(pcie->msi_bottom_domain, hwirq);  in mtk_pcie_msi_handler()
718 	struct mtk_gen3_pcie *pcie = irq_desc_get_handler_data(desc);  in mtk_pcie_irq_handler()  local
725 	status = readl_relaxed(pcie->base + PCIE_INT_STATUS_REG);  in mtk_pcie_irq_handler()
728 		generic_handle_domain_irq(pcie->intx_domain,  in mtk_pcie_irq_handler()
734 		mtk_pcie_msi_handler(pcie, irq_bit - PCIE_MSI_SHIFT);  in mtk_pcie_irq_handler()
736 		writel_relaxed(BIT(irq_bit), pcie->base + PCIE_INT_STATUS_REG);  in mtk_pcie_irq_handler()
742 static int mtk_pcie_setup_irq(struct mtk_gen3_pcie *pcie)  in mtk_pcie_setup_irq()  argument
744 	struct device *dev = pcie->dev;  in mtk_pcie_setup_irq()
748 	err = mtk_pcie_init_irq_domains(pcie);  in mtk_pcie_setup_irq()
752 	pcie->irq = platform_get_irq(pdev, 0);  in mtk_pcie_setup_irq()
753 	if (pcie->irq < 0)  in mtk_pcie_setup_irq()
754 		return pcie->irq;  in mtk_pcie_setup_irq()
756 	irq_set_chained_handler_and_data(pcie->irq, mtk_pcie_irq_handler, pcie);  in mtk_pcie_setup_irq()
761 static int mtk_pcie_parse_port(struct mtk_gen3_pcie *pcie)  in mtk_pcie_parse_port()  argument
763 	struct device *dev = pcie->dev;  in mtk_pcie_parse_port()
768 	regs = platform_get_resource_byname(pdev, IORESOURCE_MEM, "pcie-mac");  in mtk_pcie_parse_port()
771 	pcie->base = devm_ioremap_resource(dev, regs);  in mtk_pcie_parse_port()
772 	if (IS_ERR(pcie->base)) {  in mtk_pcie_parse_port()
774 		return PTR_ERR(pcie->base);  in mtk_pcie_parse_port()
777 	pcie->reg_base = regs->start;  in mtk_pcie_parse_port()
779 	pcie->phy_reset = devm_reset_control_get_optional_exclusive(dev, "phy");  in mtk_pcie_parse_port()
780 	if (IS_ERR(pcie->phy_reset)) {  in mtk_pcie_parse_port()
781 		ret = PTR_ERR(pcie->phy_reset);  in mtk_pcie_parse_port()
788 	pcie->mac_reset = devm_reset_control_get_optional_exclusive(dev, "mac");  in mtk_pcie_parse_port()
789 	if (IS_ERR(pcie->mac_reset)) {  in mtk_pcie_parse_port()
790 		ret = PTR_ERR(pcie->mac_reset);  in mtk_pcie_parse_port()
797 	pcie->phy = devm_phy_optional_get(dev, "pcie-phy");  in mtk_pcie_parse_port()
798 	if (IS_ERR(pcie->phy)) {  in mtk_pcie_parse_port()
799 		ret = PTR_ERR(pcie->phy);  in mtk_pcie_parse_port()
806 	pcie->num_clks = devm_clk_bulk_get_all(dev, &pcie->clks);  in mtk_pcie_parse_port()
807 	if (pcie->num_clks < 0) {  in mtk_pcie_parse_port()
809 		return pcie->num_clks;  in mtk_pcie_parse_port()
815 static int mtk_pcie_power_up(struct mtk_gen3_pcie *pcie)  in mtk_pcie_power_up()  argument
817 	struct device *dev = pcie->dev;  in mtk_pcie_power_up()
821 	reset_control_deassert(pcie->phy_reset);  in mtk_pcie_power_up()
823 	err = phy_init(pcie->phy);  in mtk_pcie_power_up()
829 	err = phy_power_on(pcie->phy);  in mtk_pcie_power_up()
836 	reset_control_deassert(pcie->mac_reset);  in mtk_pcie_power_up()
841 	err = clk_bulk_prepare_enable(pcie->num_clks, pcie->clks);  in mtk_pcie_power_up()
852 	reset_control_assert(pcie->mac_reset);  in mtk_pcie_power_up()
853 	phy_power_off(pcie->phy);  in mtk_pcie_power_up()
855 	phy_exit(pcie->phy);  in mtk_pcie_power_up()
857 	reset_control_assert(pcie->phy_reset);  in mtk_pcie_power_up()
862 static void mtk_pcie_power_down(struct mtk_gen3_pcie *pcie)  in mtk_pcie_power_down()  argument
864 	clk_bulk_disable_unprepare(pcie->num_clks, pcie->clks);  in mtk_pcie_power_down()
866 	pm_runtime_put_sync(pcie->dev);  in mtk_pcie_power_down()
867 	pm_runtime_disable(pcie->dev);  in mtk_pcie_power_down()
868 	reset_control_assert(pcie->mac_reset);  in mtk_pcie_power_down()
870 	phy_power_off(pcie->phy);  in mtk_pcie_power_down()
871 	phy_exit(pcie->phy);  in mtk_pcie_power_down()
872 	reset_control_assert(pcie->phy_reset);  in mtk_pcie_power_down()
875 static int mtk_pcie_setup(struct mtk_gen3_pcie *pcie)  in mtk_pcie_setup()  argument
879 	err = mtk_pcie_parse_port(pcie);  in mtk_pcie_setup()
887 	reset_control_assert(pcie->phy_reset);  in mtk_pcie_setup()
888 	reset_control_assert(pcie->mac_reset);  in mtk_pcie_setup()
892 	err = mtk_pcie_power_up(pcie);  in mtk_pcie_setup()
897 	err = mtk_pcie_startup_port(pcie);  in mtk_pcie_setup()
901 	err = mtk_pcie_setup_irq(pcie);  in mtk_pcie_setup()
908 	mtk_pcie_power_down(pcie);  in mtk_pcie_setup()
916 	struct mtk_gen3_pcie *pcie;  in mtk_pcie_probe()  local
920 	host = devm_pci_alloc_host_bridge(dev, sizeof(*pcie));  in mtk_pcie_probe()
924 	pcie = pci_host_bridge_priv(host);  in mtk_pcie_probe()
926 	pcie->dev = dev;  in mtk_pcie_probe()
927 	platform_set_drvdata(pdev, pcie);  in mtk_pcie_probe()
929 	err = mtk_pcie_setup(pcie);  in mtk_pcie_probe()
934 	host->sysdata = pcie;  in mtk_pcie_probe()
938 		mtk_pcie_irq_teardown(pcie);  in mtk_pcie_probe()
939 		mtk_pcie_power_down(pcie);  in mtk_pcie_probe()
948 	struct mtk_gen3_pcie *pcie = platform_get_drvdata(pdev);  in mtk_pcie_remove()  local
949 	struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);  in mtk_pcie_remove()
956 	mtk_pcie_irq_teardown(pcie);  in mtk_pcie_remove()
957 	mtk_pcie_power_down(pcie);  in mtk_pcie_remove()
962 static void mtk_pcie_irq_save(struct mtk_gen3_pcie *pcie)  in mtk_pcie_irq_save()  argument
966 	raw_spin_lock(&pcie->irq_lock);  in mtk_pcie_irq_save()
968 	pcie->saved_irq_state = readl_relaxed(pcie->base + PCIE_INT_ENABLE_REG);  in mtk_pcie_irq_save()
971 		struct mtk_msi_set *msi_set = &pcie->msi_sets[i];  in mtk_pcie_irq_save()
977 	raw_spin_unlock(&pcie->irq_lock);  in mtk_pcie_irq_save()
980 static void mtk_pcie_irq_restore(struct mtk_gen3_pcie *pcie)  in mtk_pcie_irq_restore()  argument
984 	raw_spin_lock(&pcie->irq_lock);  in mtk_pcie_irq_restore()
986 	writel_relaxed(pcie->saved_irq_state, pcie->base + PCIE_INT_ENABLE_REG);  in mtk_pcie_irq_restore()
989 		struct mtk_msi_set *msi_set = &pcie->msi_sets[i];  in mtk_pcie_irq_restore()
995 	raw_spin_unlock(&pcie->irq_lock);  in mtk_pcie_irq_restore()
998 static int mtk_pcie_turn_off_link(struct mtk_gen3_pcie *pcie)  in mtk_pcie_turn_off_link()  argument
1002 	val = readl_relaxed(pcie->base + PCIE_ICMD_PM_REG);  in mtk_pcie_turn_off_link()
1004 	writel_relaxed(val, pcie->base + PCIE_ICMD_PM_REG);  in mtk_pcie_turn_off_link()
1007 	return readl_poll_timeout(pcie->base + PCIE_LTSSM_STATUS_REG, val,  in mtk_pcie_turn_off_link()
1015 	struct mtk_gen3_pcie *pcie = dev_get_drvdata(dev);  in mtk_pcie_suspend_noirq()  local
1020 	err = mtk_pcie_turn_off_link(pcie);  in mtk_pcie_suspend_noirq()
1022 		dev_err(pcie->dev, "cannot enter L2 state\n");  in mtk_pcie_suspend_noirq()
1027 	val = readl_relaxed(pcie->base + PCIE_RST_CTRL_REG);  in mtk_pcie_suspend_noirq()
1029 	writel_relaxed(val, pcie->base + PCIE_RST_CTRL_REG);  in mtk_pcie_suspend_noirq()
1031 	dev_dbg(pcie->dev, "entered L2 states successfully");  in mtk_pcie_suspend_noirq()
1033 	mtk_pcie_irq_save(pcie);  in mtk_pcie_suspend_noirq()
1034 	mtk_pcie_power_down(pcie);  in mtk_pcie_suspend_noirq()
1041 	struct mtk_gen3_pcie *pcie = dev_get_drvdata(dev);  in mtk_pcie_resume_noirq()  local
1044 	err = mtk_pcie_power_up(pcie);  in mtk_pcie_resume_noirq()
1048 	err = mtk_pcie_startup_port(pcie);  in mtk_pcie_resume_noirq()
1050 		mtk_pcie_power_down(pcie);  in mtk_pcie_resume_noirq()
1054 	mtk_pcie_irq_restore(pcie);  in mtk_pcie_resume_noirq()
1065 	{ .compatible = "mediatek,mt8192-pcie" },
1074 		.name = "mtk-pcie-gen3",